tests/testthat/test-StartAutopilot.R

In datarobot: 'DataRobot' Predictive Modeling API

library(testthat)

# TODO: Add test that we fail appropriately when the status response includes a status
#       field that indicates failure.
# Note: That will require slightly reorganizing the tests, since at the moment
#       `withSetTargetMocks` expects the same assertions for each test.
withSetTargetMocks <- function(..., multiseriesMock = FALSE, crossSeriesGroupByMock = FALSE) {
  # Set up stubs for PATCH and GET. We expect PATCH is called once (to request setting the target),
  # and GET is called three times:
  #  (1) get status of the async request (not done yet)
  #  (2) get status of the async request (now we're done)
  #  (3) get updated project data (we don't use this for anything, but it's part of the async
  #      workflow, so it's expected to happen)
  #
  # Note: This does not include the GET request to confirm that the project status is 'aim'
  #       (ready for target-setting), since that is mocked separately via GetProjectStatus

  patchStub <- stub(httr::PATCH)
  getStub <- stub(httr::GET)

  aimUrl <- datarobot:::UrlJoin(projectUrl, "aim")
  statusUrl <- datarobot:::UrlJoin(fakeEndpoint, "status", "some-status")

  patchStub$onCall(1)$expects(url = aimUrl)
  patchStub$onCall(1)$returns(httr:::response(url = aimUrl,
                                              status_code = 202L,
                                              headers = list(location = statusUrl),
                                              content = raw(0)))

  getStub$onCall(1)$expects(url = statusUrl)
  getStub$onCall(1)$returns(httr:::response(url = statusUrl,
                                            status_code = 200L,
                                            content = charToRaw('{"status": "someStatus"}')))

  getStub$onCall(2)$expects(url = statusUrl)
  getStub$onCall(2)$returns(httr:::response(url = statusUrl,
                                            status_code = 303L,
                                            headers = list(location = projectUrl),
                                            content = raw(0)))

  if (isTRUE(multiseriesMock)) {
    getMultiseriesJson <- fileToChar("responses/GetMultiseries.json")
    getMultiseriesUrl <- UrlJoin(projectUrl, "multiseriesProperties")
    multiseriesRequestResponse <- httr:::response(url = getMultiseriesUrl,
                                                  status_code = 303L,
                                                  content = charToRaw(getMultiseriesJson))
    getStub$onCall(3)$returns(multiseriesRequestResponse)
    getStub$onCall(4)$expects(url = statusUrl)
    getStub$onCall(4)$returns(httr:::response(url = statusUrl,
                                              status_code = 303L,
                                              headers = list(location = projectUrl),
                                              content = raw(0)))
    if (isTRUE(crossSeriesGroupByMock)) {
      getCrossSeriesJson <- fileToChar("responses/GetCrossSeries.json")
      getCrossSeriesUrl <- UrlJoin(projectUrl, "crossSeriesProperties")
      crossSeriesRequestResponse <- httr:::response(url = getCrossSeriesUrl,
                                                    status_code = 303L,
                                                    content = charToRaw(getCrossSeriesJson))
      getStub$onCall(5)$returns(crossSeriesRequestResponse)
      getStub$onCall(6)$expects(url = statusUrl)
      getStub$onCall(6)$returns(httr:::response(url = statusUrl,
                                                status_code = 303L,
                                                headers = list(location = projectUrl),
                                                content = raw(0)))
      getStub$onCall(7)$expects(url = projectUrl)
      getStub$onCall(7)$returns(httr:::response(url = projectUrl,
                                                status_code = 200L,
                                                content = raw(0)))
    } else {
      getStub$onCall(5)$expects(url = projectUrl)
      getStub$onCall(5)$returns(httr:::response(url = projectUrl,
                                                status_code = 200L,
                                                content = raw(0)))
    }
  }
  else {
    getStub$onCall(3)$expects(url = projectUrl)
    getStub$onCall(3)$returns(httr:::response(url = projectUrl,
                                              status_code = 200L,
                                              content = raw(0)))
  }

  postStub <- stub(httr::POST)
  postMultiseriesModelUrl <- UrlJoin(projectUrl, "multiseriesProperties")
  multiseriesRequestResponse <- httr:::response(url = postMultiseriesModelUrl,
                                                status_code = 303L,
                                                headers = list(location = statusUrl),
                                                content = raw(0))
  postStub$onCall(1)$returns(multiseriesRequestResponse)
  if (isTRUE(crossSeriesGroupByMock)) {
    crossSeriesRequestResponse <- httr:::response(url = getCrossSeriesUrl,
                                                  status_code = 303L,
                                                  headers = list(location = statusUrl),
                                                  content = charToRaw(getCrossSeriesJson))
    postStub$onCall(2)$returns(crossSeriesRequestResponse)
  }

  with_mock("httr::PATCH" = patchStub$f,
            "httr::POST" = postStub$f,
            "httr::GET" = getStub$f,
            # Mock patch to be able to record the input so we can test against it
            "datarobot:::DataRobotPATCH" = function(routeString,
                                                    addUrl = TRUE,
                                                    body = NULL,
                                                    returnRawResponse = FALSE, ...) {
              bodyForInspect <<- body
              datarobot:::MakeDataRobotRequest(httr::PATCH, routeString,
                                               addUrl = addUrl,
                                               returnRawResponse = returnRawResponse,
                                               body = body, ...)
            },
            "datarobot:::Endpoint" = function() fakeEndpoint,
            "datarobot:::Token" = function() fakeToken,
            GetProjectStatus = function(...) list(stage = ProjectStage$AIM),
            ...) # Tests get injected here.

  expect_equal(patchStub$calledTimes(), 1)

  if (isTRUE(crossSeriesGroupByMock)) {
    expect_equal(postStub$calledTimes(), 2)
    expect_equal(getStub$calledTimes(), 7)
  } else if (isTRUE(multiseriesMock)) {
    expect_equal(postStub$calledTimes(), 1)
    expect_equal(getStub$calledTimes(), 5)
  } else {
    expect_equal(getStub$calledTimes(), 3)
  }
}

test_that("Required parameters are present", {
  # TODO: Fix this.
  # (These don't use mocks and don't look for specific errors, so are likely to spuriously passs.)
  expect_error(SetTarget())
  expect_error(SetTarget(target = fakeTarget))
  expect_error(SetTarget(fakeProject, target = NULL))
  expect_error(SetTarget(fakeProject, target = fakeTarget, majorityDownsamplingRate = 0.9))
})


test_that("Use projectId only", {
  withSetTargetMocks({
    expect_message(SetTarget(project = fakeProjectId, target = fakeTarget),
    "Autopilot started")
    expect_equal(as.character(bodyForInspect$target), fakeTarget)
  })
})

test_that("Use full project list only", {
  withSetTargetMocks({
    expect_message(SetTarget(project = fakeProject, target = fakeTarget),
    "Autopilot started")
    expect_equal(as.character(bodyForInspect$target), fakeTarget)
  })
})

test_that("Use non-null metric", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, metric = "testMetric"),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$metric), "testMetric")
  })
})

test_that("Use non-null weights", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, weights = "testWeights"),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$weights), "testWeights")
  })
})

test_that("Use non-null featurelistId", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, featurelistId = fakeFeaturelistId),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$featurelistId), fakeFeaturelistId)
  })
})

test_that("Use non-null mode", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = "testMode"),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$mode), "testMode")
  })
})

test_that("Use non-null seed", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, seed = 19),
      "Autopilot started")
    expect_equal(as.numeric(bodyForInspect$seed), 19)
  })
})

test_that("Use non-null positiveClass", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, positiveClass = "testClass"),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$positiveClass), "testClass")
  })
})

test_that("Use non-null blueprintThreshold", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, blueprintThreshold = 8),
      "Autopilot started")
    expect_equal(as.numeric(bodyForInspect$blueprintThreshold), 8)
  })
})

test_that("Use non-null responseCap", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, responseCap = 0.8),
      "Autopilot started")
    expect_equal(as.numeric(bodyForInspect$responseCap), 0.8)
  })
})

test_that("Use non-null downsampling", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget,
                smartDownsampled = TRUE, majorityDownsamplingRate = 0.9),
      "Autopilot started")
    expect_true(as.logical(bodyForInspect$smartDownsampled))
    expect_equal(as.numeric(bodyForInspect$majorityDownsamplingRate), 0.9)
  })
})

test_that("Use non-null accuracyOptimizedBlueprints", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget,
                accuracyOptimizedBlueprints = TRUE),
      "Autopilot started")
    expect_true(as.logical(bodyForInspect$accuracyOptimizedMb))
  })
})

test_that("Use non-null offset and exposure", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget,
                offset = c("offset_var1", "offset_var2"),
                exposure = "exposure_var"),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$offset), c("offset_var1", "offset_var2"))
    expect_equal(as.character(bodyForInspect$exposure), "exposure_var")
  })
})

test_that("Use non-null eventsCount", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget,
                eventsCount = "events_count_column"),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$eventsCount), "events_count_column")
  })
})

test_that("Use monotonic constraints", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget,
                monotonicIncreasingFeaturelistId = "monotonic-flist-up",
                monotonicDecreasingFeaturelistId = "monotonic-flist-down",
                onlyIncludeMonotonicBlueprints = TRUE),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$monotonicIncreasingFeaturelistId),
                 "monotonic-flist-up")
    expect_equal(as.character(bodyForInspect$monotonicDecreasingFeaturelistId),
                 "monotonic-flist-down")
    expect_true(as.logical(bodyForInspect$onlyIncludeMonotonicBlueprints))
  })
})

test_that("Use actual feature lists for monotonic constraints", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget,
                monotonicIncreasingFeaturelistId = fakeFeaturelist,
                monotonicDecreasingFeaturelistId = fakeFeaturelist,
                onlyIncludeMonotonicBlueprints = TRUE),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$monotonicIncreasingFeaturelistId),
                 fakeFeaturelistId)
    expect_equal(as.character(bodyForInspect$monotonicDecreasingFeaturelistId),
                 fakeFeaturelistId)
    expect_true(as.logical(bodyForInspect$onlyIncludeMonotonicBlueprints))
  })
})

test_that("Use valid targetTypes", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, targetType = TargetType$Multiclass),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$targetType), TargetType$Multiclass)
  })
})

test_that("Fail on invalid targetTypes", {
  expect_error(withSetTargetMocks(
    SetTarget(project = fakeProject, target = fakeTarget, targetType = "MAGIC")),
    paste0("Invalid ", sQuote("TargetType"), ". Must be in ", sQuote("Binary"), ", ",
          sQuote("Multiclass"), ", ", sQuote("Regression"), " but got ", sQuote("MAGIC"),
          " instead."))
})

test_that("Use full autopilot mode", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$FullAuto),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$mode), AutopilotMode$FullAuto)
  })
})

test_that("Use manual mode", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$Manual),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$mode), AutopilotMode$Manual)
  })
})

test_that("Use quickrun mode", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$Quick),
      "Autopilot started"
    )
    # Prior to DSX-2417 using Quick would actually use FullAuto but with quickrun == TRUE
    # now we just use Quick mode
    expect_equal(as.character(bodyForInspect$mode), AutopilotMode$Quick)
    expect_null(bodyForInspect$quickrun)
  })
})


partition <- CreateDatetimePartitionSpecification(fakeDateColumn,
                                                  autopilotDataSelectionMethod = NULL,
                                                  validationDuration = NULL,
                                                  holdoutStartDate = NULL,
                                                  holdoutDuration = NULL,
                                                  gapDuration = NULL,
                                                  numberOfBacktests = NULL,
                                                  backtests = NULL)
test_that("Datetime partition with empty backtests", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, partition = partition),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$cvMethod), "datetime")
    expect_equal(as.character(bodyForInspect$datetimePartitionColumn), fakeDateColumn)
    expect_false(as.logical(bodyForInspect$useTimeSeries))
    expect_false(as.logical(bodyForInspect$defaultToKnownInAdvance))
  })
})


partition <- CreateDatetimePartitionSpecification(fakeDateColumn,
                                      featureSettings = list(featureName = "Product_offers",
                                                             knownInAdvance = TRUE))
test_that("Datetime partition with feature settings", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, partition = partition),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$cvMethod), "datetime")
    expect_equal(as.character(bodyForInspect$datetimePartitionColumn), fakeDateColumn)
    expect_false(as.logical(bodyForInspect$useTimeSeries))
    expect_false(as.logical(bodyForInspect$defaultToKnownInAdvance))
    expect_type(bodyForInspect$featureSettings, "list")
    expect_equal(bodyForInspect$featureSettings[[1]]$featureName, "Product_offers")
    expect_true(bodyForInspect$featureSettings[[1]]$knownInAdvance)
  })
})


partition <- CreateDatetimePartitionSpecification(fakeDateColumn,
                                      featureSettings = list(featureName = "Sales",
                                                             doNotDerive = TRUE))
test_that("Datetime partition with doNotDerive feature setting", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, partition = partition),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$cvMethod), "datetime")
    expect_equal(as.character(bodyForInspect$datetimePartitionColumn), fakeDateColumn)
    expect_false(as.logical(bodyForInspect$useTimeSeries))
    expect_false(as.logical(bodyForInspect$defaultToKnownInAdvance))
    expect_type(bodyForInspect$featureSettings, "list")
    expect_equal(bodyForInspect$featureSettings[[1]]$featureName, "Sales")
    expect_true(bodyForInspect$featureSettings[[1]]$doNotDerive)
  })
})


partition <- CreateDatetimePartitionSpecification(fakeDateColumn,
                                                  treatAsExponential = TreatAsExponential$Always,
                                                  differencingMethod = DifferencingMethod$Seasonal,
                                                  periodicities = list(list("timeSteps" = 10,
                                                                            "timeUnit" = "HOUR"),
                                                                       list("timeSteps" = 600,
                                                                            "timeUnit" = "MINUTE"),
                                                                       list("timeSteps" = 7,
                                                                            "timeUnit" = "DAY")))
test_that("Datetime partition with exponential, differencing, and periodicities", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$Quick,
                partition = partition),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$cvMethod), "datetime")
    expect_equal(as.character(bodyForInspect$datetimePartitionColumn), fakeDateColumn)
    expect_false(as.logical(bodyForInspect$useTimeSeries))
    expect_false(as.logical(bodyForInspect$defaultToKnownInAdvance))
    expect_equal(as.character(bodyForInspect$treatAsExponential), TreatAsExponential$Always)
    expect_equal(as.character(bodyForInspect$differencingMethod), DifferencingMethod$Seasonal)
    expect_equal(bodyForInspect$periodicities[[1]]$timeSteps, 10)
    expect_equal(bodyForInspect$periodicities[[1]]$timeUnit, "HOUR")
    expect_equal(bodyForInspect$periodicities[[2]]$timeSteps, 600)
    expect_equal(bodyForInspect$periodicities[[2]]$timeUnit, "MINUTE")
  })
})


partition <- CreateDatetimePartitionSpecification(fakeDateColumn,
                                                  windowsBasisUnit = "ROW",
                                                  periodicities = list(list("timeSteps" = 10,
                                                                            "timeUnit" = "ROW")))
test_that("Datetime partition with windowsBasisUnit", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$Quick,
                partition = partition),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$cvMethod), "datetime")
    expect_equal(as.character(bodyForInspect$datetimePartitionColumn), fakeDateColumn)
    expect_false(as.logical(bodyForInspect$useTimeSeries))
    expect_false(as.logical(bodyForInspect$defaultToKnownInAdvance))
    expect_equal(as.character(bodyForInspect$windowsBasisUnit), "ROW")
    expect_equal(bodyForInspect$periodicities[[1]]$timeSteps, 10)
    expect_equal(bodyForInspect$periodicities[[1]]$timeUnit, "ROW")
  })
})

test_that("Datetime partition with invalid partition", {
  with_mock("datarobot:::Endpoint" = function() fakeEndpoint,
            "datarobot:::Token" = function() fakeToken,
            GetProjectStatus = function(...) list(stage = ProjectStage$AIM), {
    expect_error(SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$Quick,
                partition = list(fakeDateColumn)),
      "must use a valid partition object")
  })
})


partition <- CreateDatetimePartitionSpecification(fakeDateColumn, useTimeSeries = TRUE)
test_that("True time series partition", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$Quick,
                partition = partition),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$cvMethod), "datetime")
    expect_equal(as.character(bodyForInspect$datetimePartitionColumn), fakeDateColumn)
    expect_true(as.logical(bodyForInspect$useTimeSeries))
    expect_false(as.logical(bodyForInspect$defaultToKnownInAdvance))
  })
})


partition <- CreateDatetimePartitionSpecification(fakeDateColumn, useTimeSeries = TRUE,
                                                  multiseriesIdColumns = fakeMultiIdColumn)
test_that("Multiseries partition", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$Quick,
                partition = partition),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$cvMethod), "datetime")
    expect_equal(as.character(bodyForInspect$datetimePartitionColumn), fakeDateColumn)
    expect_equal(bodyForInspect$multiseriesIdColumns[[1]], fakeMultiIdColumn)
    expect_true(as.logical(bodyForInspect$useTimeSeries))
    expect_false(as.logical(bodyForInspect$defaultToKnownInAdvance))
  }, multiseriesMock = TRUE)
})


partition <- CreateDatetimePartitionSpecification(fakeDateColumn, useTimeSeries = TRUE,
                                                  multiseriesIdColumns = fakeMultiIdColumn,
                                                  useCrossSeries = TRUE,
                                                  aggregationType = "total")
test_that("Cross series partition", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$Quick,
                partition = partition),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$cvMethod), "datetime")
    expect_equal(as.character(bodyForInspect$datetimePartitionColumn), fakeDateColumn)
    expect_equal(bodyForInspect$multiseriesIdColumns[[1]], fakeMultiIdColumn)
    expect_true(as.logical(bodyForInspect$useTimeSeries))
    expect_false(as.logical(bodyForInspect$defaultToKnownInAdvance))
    expect_true(as.logical(bodyForInspect$useCrossSeriesFeatures))
    expect_equal(as.character(bodyForInspect$aggregationType), "total")
  }, multiseriesMock = TRUE)
})


partition <- CreateDatetimePartitionSpecification(fakeDateColumn,
                                                  useTimeSeries = TRUE,
                                                  calendar = fakeCalendar)
test_that("Time series partition with calendar", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$Quick,
                partition = partition),
      "Autopilot started")
    expect_equal(as.character(bodyForInspect$cvMethod), "datetime")
    expect_equal(as.character(bodyForInspect$datetimePartitionColumn), fakeDateColumn)
    expect_true(as.logical(bodyForInspect$useTimeSeries))
    expect_false(as.logical(bodyForInspect$defaultToKnownInAdvance))
  })
})


partition <- CreateDatetimePartitionSpecification(fakeDateColumn, useTimeSeries = TRUE,
                                                  multiseriesIdColumns = fakeMultiIdColumn,
                                                  useCrossSeries = TRUE,
                                                  crossSeriesGroupByColumns = fakeCrossIdColumn)
test_that("Cross series partition with crossSeriesGroupByColumns", {
  withSetTargetMocks({
    expect_message(
      SetTarget(project = fakeProject, target = fakeTarget, mode = AutopilotMode$Quick,
                partition = partition),
      "Autopilot started")
    expect_equal(bodyForInspect$multiseriesIdColumns[[1]], fakeMultiIdColumn)
    expect_true(as.logical(bodyForInspect$useTimeSeries))
    expect_false(as.logical(bodyForInspect$defaultToKnownInAdvance))
    expect_true(as.logical(bodyForInspect$useCrossSeriesFeatures))
    expect_equal(as.character(bodyForInspect$crossSeriesGroupByColumns), fakeCrossIdColumn)
  }, multiseriesMock = TRUE, crossSeriesGroupByMock = TRUE)
})