tests/testthat/test_base_generateHyperParsEffect.R

context("hyperparameterValidation")

test_that("generate data", {
  # generate data with nested no trafo
  ps = makeParamSet(makeNumericParam("C", lower = -5, upper = 5,
    trafo = function(x) 2^x)
  )
  ctrl = makeTuneControlRandom(maxit = 5L)
  rdesc = makeResampleDesc("Holdout")
  lrn = makeTuneWrapper("classif.ksvm", control = ctrl,
    resampling = rdesc, par.set = ps,
    show.info = FALSE)
  res = resample(lrn, task = pid.task, resampling = cv2,
    extract = getTuneResult)
  orig = getNestedTuneResultsOptPathDf(res)
  new = generateHyperParsEffectData(res, include.diagnostics = TRUE)
  expect_equivalent(new$data, orig)

  # generate data, no include diag, trafo
  rdesc = makeResampleDesc("Holdout")
  res = tuneParams("classif.ksvm", task = pid.task, resampling = rdesc,
    par.set = ps, control = ctrl, measures = acc)
  orig = as.data.frame(trafoOptPath(res$opt.path))
  orig = dropNamed(orig, c("eol", "error.message"))
  orig = plyr::rename(orig, c(dob = "iteration"))
  new = generateHyperParsEffectData(res, trafo = TRUE)
  expect_equivalent(new$data, orig)
})

test_that("1 numeric hyperparam", {
  # generate data
  ps = makeParamSet(makeDiscreteParam("C", values = 2^ (-2:2)))
  ctrl = makeTuneControlGrid()
  rdesc = makeResampleDesc("Holdout")
  res = tuneParams("classif.ksvm", task = pid.task, resampling = rdesc,
    par.set = ps, control = ctrl, measures = acc)
  orig = as.data.frame(res$opt.path)
  orig$C = as.numeric(as.character(orig$C))
  new = generateHyperParsEffectData(res, include.diagnostics = TRUE)
  expect_equivalent(new$data, orig)

  # make sure plot is created and can be saved
  plt = plotHyperParsEffect(new, x = "iteration", y = "acc.test.mean",
    plot.type = "line")
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)

  # make sure plot has expected attributes
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    c("GeomPoint", "GeomLine"))
  expect_equal(plt$labels$x, "iteration")
  expect_equal(plt$labels$y, "acc.test.mean")

  # FIXME: make sure plot looks as expected
})

test_that("1 discrete hyperparam", {
  # generate data
  ps = makeParamSet(makeDiscreteParam("kernel", values = c("vanilladot",
    "polydot", "rbfdot"))
  )
  ctrl = makeTuneControlGrid()
  rdesc = makeResampleDesc("Holdout")
  res = tuneParams("classif.ksvm", task = pid.task, resampling = rdesc,
    par.set = ps, control = ctrl, measures = acc)
  orig = as.data.frame(res$opt.path)
  new = generateHyperParsEffectData(res, include.diagnostics = TRUE)
  expect_equivalent(new$data, orig)

  # make sure plot is created and can be saved
  plt = plotHyperParsEffect(new, x = "kernel", y = "acc.test.mean")
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)

  # make sure plot has expected attributes
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    "GeomPoint")
  expect_equal(plt$labels$x, "kernel")
  expect_equal(plt$labels$y, "acc.test.mean")

  # FIXME: make sure plot looks as expected
})

test_that("1 numeric hyperparam with optimizer failure", {
  # generate data
  ps = makeParamSet(makeDiscreteParam("C", values = c(-1, 0.5, 1.5))
  )
  ctrl = makeTuneControlGrid()
  rdesc = makeResampleDesc("Holdout")
  res = tuneParams("classif.ksvm", task = pid.task, resampling = rdesc,
    par.set = ps, control = ctrl, measures = acc)
  orig = as.data.frame(res$opt.path)
  orig$C = as.numeric(as.character(orig$C))
  new = generateHyperParsEffectData(res, include.diagnostics = TRUE)
  expect_equivalent(new$data, orig)

  # make sure plot is created and can be saved
  plt = plotHyperParsEffect(new, x = "C", y = "acc.test.mean")
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)

  # make sure plot has expected attributes
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    "GeomPoint")
  expect_equal(plt$labels$x, "C")
  expect_equal(plt$labels$y, "acc.test.mean")

  # FIXME: make sure plot looks as expected
})

test_that("1 numeric hyperparam with nested cv", {
  # generate data
  ps = makeParamSet(makeNumericParam("C", lower = 0.01, upper = 2)
  )
  ctrl = makeTuneControlRandom(maxit = 5L)
  rdesc = makeResampleDesc("Holdout")
  lrn = makeTuneWrapper("classif.ksvm", control = ctrl,
    resampling = rdesc, par.set = ps,
    show.info = FALSE)
  res = resample(lrn, task = pid.task, resampling = cv2,
    extract = getTuneResult)
  orig = getNestedTuneResultsOptPathDf(res)
  new = generateHyperParsEffectData(res, include.diagnostics = TRUE)
  expect_equivalent(new$data, orig)

  # make sure plot is created and can be saved
  plt = plotHyperParsEffect(new, x = "C", y = "mmce.test.mean")
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)

  # make sure plot has expected attributes
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    "GeomPoint")
  expect_equal(plt$labels$x, "C")
  expect_equal(plt$labels$y, "mmce.test.mean")

  # FIXME: make sure plot looks as expected
})

test_that("2 hyperparams", {
  # generate data
  ps = makeParamSet(
    makeNumericParam("C", lower = -5, upper = 5, trafo = function(x) 2^x),
    makeNumericParam("sigma", lower = -5, upper = 5, trafo = function(x) 2^x))
  ctrl = makeTuneControlRandom(maxit = 5L)
  rdesc = makeResampleDesc("Holdout")
  learn = makeLearner("classif.ksvm", par.vals = list(kernel = "rbfdot"))
  res = tuneParams(learn, task = pid.task, control = ctrl, measures = acc,
    resampling = rdesc, par.set = ps, show.info = FALSE)
  data = generateHyperParsEffectData(res)


  # test line creation
  plt = plotHyperParsEffect(data, x = "iteration", y = "acc.test.mean",
    plot.type = "line")
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    c("GeomLine", "GeomPoint"))
  expect_equal(plt$labels$x, "iteration")
  expect_equal(plt$labels$y, "acc.test.mean")

  # test heatcontour creation with interpolation
  plt = plotHyperParsEffect(data, x = "C", y = "sigma", z = "acc.test.mean",
    plot.type = "heatmap", interpolate = "regr.earth",
    show.experiments = TRUE)
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    c("GeomPoint", "GeomRaster"))
  expect_equal(plt$labels$x, "C")
  expect_equal(plt$labels$y, "sigma")
  expect_equal(plt$labels$fill, "acc.test.mean")
  expect_equal(plt$labels$shape, "learner_status")

  # learner crash
  ps = makeParamSet(
    makeDiscreteParam("C", values = c(-1, 0.5, 1.5)),
    makeDiscreteParam("sigma", values = c(-1, 0.5, 1.5)))
  ctrl = makeTuneControlGrid()
  rdesc = makeResampleDesc("Holdout")
  learn = makeLearner("classif.ksvm", par.vals = list(kernel = "rbfdot"))
  res = tuneParams(learn, task = pid.task, control = ctrl, measures = acc,
    resampling = rdesc, par.set = ps, show.info = FALSE)
  data = generateHyperParsEffectData(res)
  plt = plotHyperParsEffect(data, x = "C", y = "sigma", z = "acc.test.mean",
    plot.type = "heatmap", interpolate = "regr.earth")
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    c("GeomPoint", "GeomRaster"))
  expect_equal(plt$labels$x, "C")
  expect_equal(plt$labels$y, "sigma")
  expect_equal(plt$labels$fill, "acc.test.mean")
  expect_equal(plt$labels$shape, "learner_status")

  # FIXME: make sure plots looks as expected
})

test_that("2 hyperparams nested", {
  # generate data
  ps = makeParamSet(
    makeNumericParam("C", lower = -5, upper = 5, trafo = function(x) 2^x),
    makeNumericParam("sigma", lower = -5, upper = 5, trafo = function(x) 2^x))
  ctrl = makeTuneControlRandom(maxit = 5L)
  rdesc = makeResampleDesc("Holdout")
  learn = makeLearner("classif.ksvm", par.vals = list(kernel = "rbfdot"))
  lrn = makeTuneWrapper(learn, control = ctrl,
    measures = list(acc, mmce), resampling = rdesc,
    par.set = ps, show.info = FALSE)
  res = resample(lrn, task = pid.task, resampling = cv2,
    extract = getTuneResult)
  data = generateHyperParsEffectData(res)

  # contour plot
  plt = plotHyperParsEffect(data, x = "C", y = "sigma", z = "acc.test.mean",
    plot.type = "contour", interpolate = "regr.earth",
    show.interpolated = TRUE)
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    c("GeomPoint", "GeomRaster", "GeomContour"))
  expect_equal(plt$labels$x, "C")
  expect_equal(plt$labels$y, "sigma")
  expect_equal(plt$labels$fill, "acc.test.mean")
  expect_equal(plt$labels$shape, "learner_status")

  # learner crashes
  ps = makeParamSet(
    makeDiscreteParam("C", values = c(-1, 0.5, 1.5)),
    makeDiscreteParam("sigma", values = c(-1, 0.5, 1.5)))
  lrn = makeTuneWrapper(learn, control = ctrl,
    measures = list(acc, mmce), resampling = rdesc,
    par.set = ps, show.info = FALSE)
  res = resample(lrn, task = pid.task, resampling = cv2,
    extract = getTuneResult)
  data = generateHyperParsEffectData(res)
  plt = plotHyperParsEffect(data, x = "C", y = "sigma", z = "acc.test.mean",
    plot.type = "heatmap", interpolate = "regr.earth",
    show.experiments = TRUE)
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    c("GeomPoint", "GeomRaster"))
  expect_equal(plt$labels$x, "C")
  expect_equal(plt$labels$y, "sigma")
  expect_equal(plt$labels$fill, "acc.test.mean")
  expect_equal(plt$labels$shape, "learner_status")
})

test_that("2+ hyperparams", {
  # generate data
  ps = makeParamSet(
    makeNumericParam("C", lower = -5, upper = 5, trafo = function(x) 2^x),
    makeNumericParam("sigma", lower = -5, upper = 5, trafo = function(x) 2^x),
    makeDiscreteParam("degree", values = 2:5))
  ctrl = makeTuneControlRandom(maxit = 5L)
  rdesc = makeResampleDesc("Holdout", predict = "both")
  learn = makeLearner("classif.ksvm", par.vals = list(kernel = "besseldot"))
  res = tuneParams(learn, task = pid.task, control = ctrl,
    measures = list(acc, setAggregation(acc, train.mean)), resampling = rdesc,
    par.set = ps, show.info = FALSE)
  data = generateHyperParsEffectData(res, partial.dep = TRUE)


  # test single hyperparam creation
  plt = plotHyperParsEffect(data, x = "C", y = "acc.test.mean",
    plot.type = "line", partial.dep.learn = "regr.randomForest")
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    c("GeomLine", "GeomPoint"))
  expect_equal(plt$labels$x, "C")
  expect_equal(plt$labels$y, "acc.test.mean")

  # test bivariate
  plt = plotHyperParsEffect(data, x = "C", y = "sigma", z = "acc.test.mean",
    plot.type = "heatmap", partial.dep.learn = "regr.randomForest")
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)
  expect_equal(class(plt$layers[[1]]$geom)[1], "GeomTile")
  expect_equal(plt$labels$x, "C")
  expect_equal(plt$labels$y, "sigma")
  expect_equal(plt$labels$fill, "acc.test.mean")


  # simple example with nested cv
  ps = makeParamSet(
    makeNumericParam("C", lower = -5, upper = 5, trafo = function(x) 2^x),
    makeNumericParam("sigma", lower = -5, upper = 5, trafo = function(x) 2^x),
    makeDiscreteParam("degree", values = 2:5))
  ctrl = makeTuneControlRandom(maxit = 5L)
  rdesc = makeResampleDesc("Holdout", predict = "both")
  learn = makeLearner("classif.ksvm", par.vals = list(kernel = "besseldot"))
  lrn = makeTuneWrapper(learn, control = ctrl,
    measures = list(acc, mmce), resampling = rdesc, par.set = ps,
    show.info = FALSE)
  res = resample(lrn, task = pid.task, resampling = cv2, extract = getTuneResult)
  data = generateHyperParsEffectData(res, partial.dep = TRUE)
  plt = plotHyperParsEffect(data, x = "C", y = "acc.test.mean",
    plot.type = "line", partial.dep.learn = "regr.randomForest")
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    c("GeomLine", "GeomPoint"))
  expect_equal(plt$labels$x, "C")
  expect_equal(plt$labels$y, "acc.test.mean")

  # learner crash with imputation works
  ps = makeParamSet(
    makeDiscreteParam("C", values = c(-1, 0.5, 1.5)),
    makeDiscreteParam("sigma", values = c(-1, 0.5, 1.5)))
  ctrl = makeTuneControlGrid()
  rdesc = makeResampleDesc("Holdout")
  learn = makeLearner("classif.ksvm", par.vals = list(kernel = "besseldot"))
  res = tuneParams(learn, task = pid.task, control = ctrl, measures = acc,
    resampling = rdesc, par.set = ps, show.info = FALSE)
  data = generateHyperParsEffectData(res, partial.dep = TRUE)
  plt = plotHyperParsEffect(data, x = "C", y = "acc.test.mean",
    plot.type = "line", partial.dep.learn = "regr.randomForest")
  print(plt)
  dir = tempdir()
  path = stri_paste(dir, "/test.svg")
  ggsave(path)
  expect_set_equal(sapply(plt$layers, function(x) class(x$geom)[1]),
    c("GeomLine", "GeomPoint"))
  expect_equal(plt$labels$x, "C")
  expect_equal(plt$labels$y, "acc.test.mean")

  # FIXME: make sure plots looks as expected
})
riebetob/mlr documentation built on May 20, 2019, 5:58 p.m.