In slds-lmu/paper_2021_multi_fidelity_surrogates: Surrogate benchmarks for HPO problems

knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>"
)

Overview

reticulate::use_condaenv('mlr3keras', required = TRUE)
devtools::load_all()

Current performances

perfs = rbindlist(map(cfgs()$keys(), function(k) {
  sdir = cfgs(k, workdir=workdir)$subdir
  if (!is.null(sdir)) {
     file = paste0(sdir, "surrogate_test_metrics.csv")
     fread(file)[, problem := k]
  }
}), fill=TRUE)
perfs[, grp := NULL]
knitr::kable(perfs[, c(6,1,2:5)])

Measuring Surrogate Performance

cfg = cfgs("nb301", workdir=workdir)
obj = cfg$get_objective()
data = cfg$data
idx = 1:1000000
ptest = obj$eval_dt(data$xtest[idx])
dt = rbind(
  compute_metrics(
    response = cfg$data$ytest[idx,],
    prediction = as.matrix(ptest)
  ),
  compute_metrics(
    response = cfg$data$ytest[idx,],
    prediction = as.matrix(ptest),
    stratify = cfg$data$xtest[idx, cfg$task_col]
  )
)
dt

Measuring the "interestingness" of the landscape

• Variation in the function? Evaluate at random points and measure variance. We likely to rescale to get comparable varianes?

cfg = cfgs("rbv2_super", workdir=workdir)
obj = cfg$get_objective()
idx = 1:1000
ptest = obj$eval_dt(cfg$data$xtest[idx])
compute_metrics(
  response = cfg$data$ytest[idx,],
  prediction = as.matrix(ptest),
  stratify = cfg$data$xtest[idx, cfg$task_col]
)

list.files(workdir)

slds-lmu/paper_2021_multi_fidelity_surrogates documentation built on Feb. 20, 2022, 11:53 a.m.