tests/testthat/test-gafs.R

test_that("gafsControl errors working", {
  skip_on_cran()
  expect_snapshot(gafsControl(method = "larry"), error = TRUE)

  expect_snapshot(
    gafsControl(metric = c("larry", "harry", "moe")),
    error = TRUE
  )

  expect_snapshot(
    gafsControl(maximize = c("larry", "harry", "moe")),
    error = TRUE
  )
})

test_that("high level tests", {
  skip_on_cran()
  expect_silent(pop <- gafs_initial(vars = 10, popSize = 10))
  expect_silent(gafs_lrSelection(population = pop, fitness = 1:10))
  expect_silent(gafs_spCrossover(
    population = pop,
    fitness = 1:10,
    parents = 1:2
  ))

  train_data <- twoClassSim(10, noiseVars = 1)
  test_data <- twoClassSim(10, noiseVars = 2)

  expect_silent(
    ctrl <- gafsControl(functions = rfGA, method = "cv", number = 3)
  )

  ## Too slow
  # expect_silent(
  #   rf_search <- gafs(x = train_data[, -ncol(train_data)],
  #                      y = train_data$Class,
  #                      iters = 2,
  #                      gafsControl = ctrl)
  #   )
})

# ------------------------------------------------------------------------------
# the genetic operators

test_that("the selection operators pick a new population of the same size", {
  pop <- withr::with_seed(6104, gafs_initial(vars = 8, popSize = 6))
  fitness <- c(1, 5, 3, 9, 7, 2)

  # each operator returns a population and the fitness that came with it
  for (op in list(
    gafs_lrSelection,
    gafs_nlrSelection,
    gafs_rwSelection,
    gafs_tourSelection
  )) {
    out <- withr::with_seed(2951, op(population = pop, fitness = fitness))
    expect_named(out, c("population", "fitness"))
    expect_identical(dim(out$population), dim(pop))
    # every selected row is one of the originals, with its own fitness
    expect_in(out$fitness, fitness)
    expect_identical(nrow(out$population), length(out$fitness))
  }
})

test_that("tournament selection favours the fittest of each sample", {
  pop <- withr::with_seed(3327, gafs_initial(vars = 8, popSize = 4))
  # with k equal to the population size, every tournament sees everyone, so the
  # best individual wins every time
  out <- gafs_tourSelection(pop, fitness = c(1, 2, 3, 10), k = 4)
  expect_all_equal(out$fitness, 10)
})

test_that("the crossover operators return two children", {
  pop <- withr::with_seed(8817, gafs_initial(vars = 8, popSize = 4))
  fitness <- c(1, 5, 3, 9)

  sp <- withr::with_seed(4402, gafs_spCrossover(pop, fitness, parents = 1:2))
  expect_named(sp, c("children", "fitness"))
  expect_identical(dim(sp$children), c(2L, 8L))
  # the children are made of their parents' values
  expect_in(as.vector(sp$children), c(0, 1))

  un <- withr::with_seed(4402, gafs_uCrossover(pop, parents = 1:2))
  expect_identical(dim(un$children), c(2L, 8L))
  # the uniform operator swaps whole positions, so the pair together still
  # holds what the parents held at each position
  expect_setequal(
    as.vector(un$children[, 1]),
    as.vector(pop[1:2, 1])
  )
})

test_that("single-point crossover copies the parents at the extremes", {
  pop <- withr::with_seed(1288, gafs_initial(vars = 4, popSize = 2))
  fitness <- c(2, 8)

  # The crossover point is drawn from 0:n. At 0 the parents swap wholesale and
  # take their fitness with them; at n they are passed through unchanged. These
  # seeds are the ones that draw those two values for a four-variable problem.
  # the children are built in a double matrix, so the values are compared
  # rather than the storage type
  swapped <- withr::with_seed(1, gafs_spCrossover(pop, fitness, parents = 1:2))
  expect_equal(swapped$children, unname(pop[2:1, ]))
  expect_identical(swapped$fitness, fitness[2:1])

  passed <- withr::with_seed(2, gafs_spCrossover(pop, fitness, parents = 1:2))
  expect_equal(passed$children, unname(pop[1:2, ]))
  expect_identical(passed$fitness, fitness)
})

test_that("random mutation flips exactly one position", {
  pop <- withr::with_seed(9975, gafs_initial(vars = 10, popSize = 3))
  mutated <- withr::with_seed(5525, gafs_raMutation(pop, parent = 2))

  expect_length(mutated, 10)
  expect_identical(sum(mutated != pop[2, ]), 1L)
  expect_in(mutated, c(0, 1))
})

# ------------------------------------------------------------------------------
# the function list

test_that("ga_func_check reports the functions a search needs", {
  # caretGA is complete, so it passes
  expect_invisible(caret:::ga_func_check(caretGA))

  # dropping a required element is reported by name
  expect_snapshot(
    caret:::ga_func_check(caretGA[c("fit", "pred")]),
    error = TRUE
  )
})

test_that("ga_func_check checks each function's arguments", {
  wrong_args <- caretGA
  wrong_args$mutation <- function(x, y) x

  expect_snapshot(caret:::ga_func_check(wrong_args), error = TRUE)
})

# ------------------------------------------------------------------------------
# methods on a fitted search (fixtures live in helper-feature-selection.R)

test_that("print.gafs describes the search", {
  skip_on_cran()

  ga <- gafs_fixture()
  expect_snapshot(print(ga), transform = mask_decimals)
})

test_that("print.gafs names the classes and a varying mutation rate", {
  skip_on_cran()
  skip_if_not_installed("MASS")

  dat <- fs_data(classification = TRUE)
  ctrl <- gafsControl(functions = caretGA, method = "cv", number = 3)
  set.seed(9214)
  ga <- gafs(
    x = dat[, 1:4],
    y = dat$y,
    gafsControl = ctrl,
    popSize = 4,
    iters = 2,
    # a probability that changes with the generation prints as "variable"
    pmutation = function(generation) 0.5 / generation,
    differences = TRUE,
    method = "lda",
    trControl = trainControl(method = "cv", number = 3)
  )
  expect_snapshot(print(ga), transform = mask_decimals)
})

test_that("print.gafs says when the generation was chosen by hand", {
  skip_on_cran()

  ga <- gafs_fixture()
  new_iter <- ifelse(ga$optIter == 1, 2, 1)
  dat <- fs_data()
  manual <- update(ga, iter = new_iter, x = dat[, 1:4], y = dat$y)

  expect_false(manual$auto)
  expect_match(
    paste(capture.output(print(manual)), collapse = " "),
    "Best iteration chosen manually"
  )
})

test_that("varImp.gafs ranks the variables by their effect on performance", {
  skip_on_cran()

  # a longer search, so every variable has been in and out of the population
  # often enough for the differences to be estimated
  ga <- gafs_fixture(popSize = 8, iters = 8)
  vi <- varImp(ga)
  expect_s3_class(vi, "data.frame")
  expect_named(vi, "RMSE")
  # RMSE is minimized, so the importances are negated and sorted downwards. A
  # variable that never moved in or out often enough has no estimate.
  present <- vi[[1]][!is.na(vi[[1]])]
  expect_gt(length(present), 0)
  expect_identical(present, sort(present, decreasing = TRUE))
})

test_that("varImp.gafs needs the differences to have been computed", {
  skip_on_cran()

  ga <- gafs_fixture(differences = FALSE)
  expect_snapshot(varImp(ga), error = TRUE)
})

test_that("plot.gafs draws the search history", {
  skip_on_cran()

  ga <- gafs_fixture()

  gg <- plot(ga)
  expect_s3_class(gg, "ggplot")
  built <- ggplot2::ggplot_build(gg)
  # the GA counts generations rather than iterations
  expect_identical(built$plot$labels$x, "Generation")
  expect_contains(names(built$data[[1]]), "colour")

  expect_s3_class(plot(ga, estimate = "internal"), "ggplot")
  expect_s3_class(ggplot2::ggplot(ga), "ggplot")

  dat <- plot(ga, output = "data")
  expect_contains(names(dat), c("Iter", "Resample", "Estimate"))

  drawn <- plot(ga, output = "lattice")
  expect_s3_class(drawn, "trellis")
  draw_trellis(drawn)
  draw_trellis(plot(ga, estimate = "internal", output = "lattice"))
})

test_that("plot.gafs checks the metric it was asked for", {
  skip_on_cran()

  ga <- gafs_fixture()
  expect_snapshot(plot(ga, metric = "Bogus"), error = TRUE)
  expect_snapshot(
    plot(ga, metric = "Bogus", estimate = "internal"),
    error = TRUE
  )
})

# ------------------------------------------------------------------------------
# control validation, suggestions and elitism

test_that("gafs needs a seed per resample plus one", {
  skip_on_cran()

  dat <- fs_data()
  ctrl <- gafsControl(
    functions = caretGA,
    method = "cv",
    number = 3,
    seeds = 1:2
  )
  expect_snapshot(
    gafs(
      x = dat[, 1:4],
      y = dat$y,
      gafsControl = ctrl,
      popSize = 4,
      iters = 2,
      method = "lm",
      trControl = trainControl(method = "cv", number = 3)
    ),
    error = TRUE
  )
})

test_that("gafs checks a suggested starting population", {
  skip_on_cran()

  dat <- fs_data()
  ctrl <- gafsControl(functions = caretGA, method = "cv", number = 3)

  # One column per predictor is expected; this has too few. Not snapshotted: the
  # error is raised inside the resampling loop, so foreach re-raises it with the
  # loop body as its call, and covr rewrites that when it instruments the
  # package.
  expect_error(
    gafs(
      x = dat[, 1:4],
      y = dat$y,
      gafsControl = ctrl,
      popSize = 4,
      iters = 2,
      suggestions = matrix(1, nrow = 2, ncol = 2),
      method = "lm",
      trControl = trainControl(method = "cv", number = 3)
    ),
    "do not match number of variables"
  )
})

test_that("gafs accepts a suggested starting population", {
  skip_on_cran()

  dat <- fs_data()
  ctrl <- gafsControl(functions = caretGA, method = "cv", number = 3)

  set.seed(1058)
  fit <- gafs(
    x = dat[, 1:4],
    y = dat$y,
    gafsControl = ctrl,
    popSize = 4,
    iters = 2,
    # two of the four individuals are given, the rest are drawn
    suggestions = matrix(c(1, 0, 1, 0, 0, 1, 0, 1), nrow = 2, byrow = TRUE),
    differences = FALSE,
    method = "lm",
    trControl = trainControl(method = "cv", number = 3)
  )
  expect_s3_class(fit, "gafs")
})

test_that("gafs names an unnamed external fitness result", {
  skip_on_cran()

  dat <- fs_data()
  unnamed <- caretGA
  unnamed$fitness_extern <- function(data, lev = NULL, model = NULL) {
    unname(defaultSummary(data, lev, model))
  }
  ctrl <- gafsControl(functions = unnamed, method = "cv", number = 3)

  set.seed(3364)
  # two warnings: the unnamed result, then the metric that is therefore missing
  expect_snapshot(
    fit <- gafs(
      x = dat[, 1:4],
      y = dat$y,
      gafsControl = ctrl,
      popSize = 4,
      iters = 2,
      differences = FALSE,
      method = "lm",
      trControl = trainControl(method = "cv", number = 3)
    )
  )
  expect_in("external1", names(fit$external))
})

test_that("gafs falls back when the external metric is not computed", {
  skip_on_cran()

  dat <- fs_data()
  ctrl <- gafsControl(
    functions = caretGA,
    method = "cv",
    number = 3,
    metric = c(internal = "RMSE", external = "Bogus")
  )

  set.seed(9903)
  expect_snapshot_warning(
    fit <- gafs(
      x = dat[, 1:4],
      y = dat$y,
      gafsControl = ctrl,
      popSize = 4,
      iters = 2,
      differences = FALSE,
      method = "lm",
      trControl = trainControl(method = "cv", number = 3)
    )
  )
  expect_identical(unname(fit$control$metric["external"]), "RMSE")
})

test_that("gafs can hold out data and keep an elite", {
  skip_on_cran()

  dat <- fs_data(n = 80)
  ctrl <- gafsControl(
    functions = caretGA,
    method = "cv",
    number = 3,
    holdout = 0.25
  )

  set.seed(7551)
  fit <- gafs(
    x = dat[, 1:4],
    y = dat$y,
    gafsControl = ctrl,
    popSize = 6,
    iters = 3,
    # the best individuals are carried into the next generation unchanged
    elite = 2,
    differences = FALSE,
    method = "lm",
    trControl = trainControl(method = "cv", number = 3)
  )
  expect_identical(fit$control$holdout, 0.25)
  expect_identical(fit$ga_param$elite, 2)
})

test_that("gafs reports each generation when asked", {
  skip_on_cran()

  dat <- fs_data()
  ctrl <- gafsControl(
    functions = caretGA,
    method = "cv",
    number = 2,
    verbose = TRUE
  )

  # as for safs, the lines carry resampled performance values, so the text is
  # matched rather than snapshotted
  set.seed(2242)
  progress <- capture.output(
    fit <- gafs(
      x = dat[, 1:4],
      y = dat$y,
      gafsControl = ctrl,
      popSize = 4,
      iters = 3,
      differences = FALSE,
      method = "lm",
      trControl = trainControl(method = "cv", number = 3)
    )
  )
  joined <- paste(progress, collapse = " ")
  expect_match(joined, "Fold1")
  expect_match(joined, "->")
  expect_s3_class(fit, "gafs")
})

# ------------------------------------------------------------------------------
# control validation, unnamed folds and the recipe interface's checks

test_that("gafsControl wants named metric and maximize vectors", {
  # both need an internal and an external element, by name
  expect_snapshot(gafsControl(metric = c("RMSE", "Rsquared")), error = TRUE)
  expect_snapshot(
    gafsControl(maximize = c(internal = TRUE, wrong = FALSE)),
    error = TRUE
  )
})

test_that("gafs names the resamples it was handed", {
  skip_on_cran()

  dat <- fs_data()
  # an unnamed index gets numbered names
  folds <- unname(createFolds(dat$y, k = 3, returnTrain = TRUE))
  ctrl <- gafsControl(functions = caretGA, method = "cv", index = folds)

  set.seed(2242)
  fit <- gafs(
    x = dat[, 1:4],
    y = dat$y,
    gafsControl = ctrl,
    popSize = 4,
    iters = 2,
    differences = FALSE,
    method = "lm",
    trControl = trainControl(method = "cv", number = 3)
  )
  expect_setequal(unique(fit$external$Resample), names(fit$control$index))
  expect_match(names(fit$control$index)[1], "^Resample")
})

test_that("gafs names the resamples for a recipe fit", {
  skip_on_cran()

  dat <- fs_data()
  rec <- recipes::recipe(y ~ ., data = dat)
  folds <- unname(createFolds(dat$y, k = 3, returnTrain = TRUE))

  set.seed(2242)
  fit <- gafs(
    rec,
    data = dat,
    gafsControl = gafsControl(
      functions = caretGA,
      method = "cv",
      index = folds
    ),
    popSize = 4,
    iters = 2,
    differences = FALSE,
    method = "lm",
    trControl = trainControl(method = "cv", number = 3)
  )
  expect_match(names(fit$control$index)[1], "^Resample")
})

test_that("the gafs recipe path checks its external fitness function", {
  skip_on_cran()

  dat <- fs_data()
  dat$extra <- seq_len(nrow(dat))
  rec <- recipes::recipe(y ~ ., data = dat)
  rec <- recipes::update_role(rec, extra, new_role = "performance var")

  # an unnamed result is named by position, and a metric the function does not
  # compute falls back to the first one it does
  unnamed <- caretGA
  unnamed$fitness_extern <- function(data, lev = NULL, model = NULL) {
    unname(c(
      defaultSummary(data, lev, model),
      Extra = as.numeric("extra" %in% names(data))
    ))
  }

  set.seed(3364)
  expect_snapshot(
    fit <- gafs(
      rec,
      data = dat,
      gafsControl = gafsControl(functions = unnamed, method = "cv", number = 3),
      popSize = 4,
      iters = 2,
      differences = FALSE,
      method = "lm",
      trControl = trainControl(method = "cv", number = 3)
    )
  )
  expect_in("external1", names(fit$external))
})

test_that("the gafs recipe path computes differences with a holdout", {
  skip_on_cran()

  dat <- fs_data(n = 80)
  rec <- recipes::recipe(y ~ ., data = dat)

  set.seed(3388)
  fit <- gafs(
    rec,
    data = dat,
    gafsControl = gafsControl(
      functions = caretGA,
      method = "cv",
      number = 3,
      holdout = 0.25
    ),
    popSize = 8,
    iters = 8,
    differences = TRUE,
    method = "lm",
    trControl = trainControl(method = "cv", number = 3)
  )
  expect_identical(fit$control$holdout, 0.25)
  expect_s3_class(fit$differences, "data.frame")
})

Try the caret package in your browser

Any scripts or data that you put into this service are public.

caret documentation built on Nov. 15, 2026, 9:06 a.m.