tests/testthat/test-resamples.R

test_that("resample calculations", {
  skip_on_cran()
  skip_if_not_installed("MASS")
  set.seed(4793)
  tr_dat <- SLC14_1(200)

  ctrl <- trainControl(method = "cv", number = 3)

  set.seed(5423)
  lm_fit <- train(y ~ ., data = tr_dat, method = "lm", trControl = ctrl)
  set.seed(5423)
  # the snapshot captures the first convergence warning; rlm warns once per
  # non-converged fold, so silence the repeats
  suppressWarnings(
    expect_snapshot_warning(
      rlm_fit <- train(y ~ ., data = tr_dat, method = "rlm", trControl = ctrl)
    )
  )

  rs <- resamples(list(lm = lm_fit, rlm = rlm_fit))
  rs_d <- diff(rs, metric = "RMSE")
  t_test <- t.test(rs$values$`lm~RMSE`, rs$values$`rlm~RMSE`, paired = TRUE)
  expect_equal(rs_d$statistics$RMSE$lm.diff.rlm$conf.int, t_test$conf.int)

  lm_t_test <- t.test(rs$values$`lm~RMSE`)
  rlm_t_test <- t.test(rs$values$`rlm~RMSE`)

  rs_plot_dat <- ggplot(rs, metric = "RMSE")$data
  expect_equal(lm_t_test$estimate, rs_plot_dat$Estimate[1], ignore_attr = TRUE)
  expect_equal(rlm_t_test$estimate, rs_plot_dat$Estimate[2], ignore_attr = TRUE)
  expect_equal(
    lm_t_test$conf.int[1],
    rs_plot_dat$LowerLimit[1],
    ignore_attr = TRUE
  )
  expect_equal(
    rlm_t_test$conf.int[1],
    rs_plot_dat$LowerLimit[2],
    ignore_attr = TRUE
  )
  expect_equal(
    lm_t_test$conf.int[2],
    rs_plot_dat$UpperLimit[1],
    ignore_attr = TRUE
  )
  expect_equal(
    rlm_t_test$conf.int[2],
    rs_plot_dat$UpperLimit[2],
    ignore_attr = TRUE
  )

  rs_const <- rs
  rs_const$values$`lm~RMSE` <- 3

  rs_plot_const_dat <- ggplot(rs_const, metric = "RMSE")$data
  expect_equal(rs_plot_const_dat$Estimate[1], 3.0, ignore_attr = TRUE)
  expect_equal(rs_plot_const_dat$LowerLimit[1], NA_real_, ignore_attr = TRUE)
  expect_equal(rs_plot_const_dat$UpperLimit[1], NA_real_, ignore_attr = TRUE)
})

# ------------------------------------------------------------------------------

test_that("test group-k-fold", {
  skip_on_cran()
  get_data <- function(n = 500) {
    prevalence <- seq(0.1, 0.9, length.out = 26)
    dat <- sample(letters, size = n, replace = TRUE, prob = sample(prevalence))
    data.frame(grp = dat, stringsAsFactors = TRUE)
  }

  check_rs <- function(ind, dat) {
    in_samp <- unique(dat$grp[ind])
    out_samp <- unique(dat$grp[-ind])
    length(intersect(out_samp, in_samp)) > 0
  }

  set.seed(5683286)
  running_sum <- 0
  for (k in (1:5) * 100) {
    for (j in 1:20) {
      dat <- get_data(n = k)
      lvls <- length(unique(as.character(dat$grp)))
      for (i in 1:20) {
        inds <- groupKFold(dat$grp, k = i)
        running_sum <- running_sum +
          sum(vapply(inds, check_rs, logical(1), dat))
      }
    }
  }
  expect_true(running_sum == 0)
})

# ------------------------------------------------------------------------------
# Methods that act on a resamples object; `rs_fixture` lives in
# helper-resamples.R

test_that("as.data.frame.resamples pulls out one metric in long-ish form", {
  df <- as.data.frame(rs_fixture, metric = "RMSE")
  # one column per model, plus the Resample labels tacked on the end
  expect_named(df, c("A", "B", "C", "Resample"))
  expect_identical(df$A, c(1, 2, 3, 4, 5))
  expect_identical(df$Resample, paste0("Fold", 1:5))
})

test_that("as.matrix.resamples returns a resample-by-model matrix", {
  m <- as.matrix(rs_fixture, metric = "RMSE")
  expect_identical(dim(m), c(5L, 3L))
  expect_identical(colnames(m), c("A", "B", "C"))
  expect_identical(rownames(m), paste0("Fold", 1:5))
  expect_identical(unname(m[, "C"]), c(7, 6, 5, 4, 3))
  # asking for a metric that isn't there should be an error, not empty output
  expect_snapshot(as.matrix(rs_fixture, metric = "nope"), error = TRUE)
})

test_that("sort.resamples orders models by mean performance", {
  # A has the lowest mean RMSE, C the highest
  expect_identical(sort(rs_fixture, metric = "RMSE"), c("A", "B", "C"))
  expect_identical(
    sort(rs_fixture, metric = "RMSE", decreasing = TRUE),
    c("C", "B", "A")
  )
})

test_that("modelCor correlates models across resamples", {
  cm <- modelCor(rs_fixture, metric = "RMSE")
  expect_identical(diag(cm), c(A = 1, B = 1, C = 1))
  # C was built as a mirror image of A, so they sit at -1 (up to floating point)
  expect_equal(cm["A", "C"], -1)
  # A and B rise together
  expect_gt(cm["A", "B"], 0)
})

test_that("summary.resamples reports the usual five-number summary per model", {
  sm <- summary(rs_fixture)
  expect_s3_class(sm, "summary.resamples")
  expect_named(sm$statistics, c("RMSE", "Rsquared"))
  # the row means match what we put in (A averages 3, C averages 5)
  expect_identical(sm$statistics$RMSE["A", "Mean"], 3)
  expect_identical(sm$statistics$RMSE["C", "Mean"], 5)
})

test_that("diff.resamples computes pairwise differences and tests", {
  d <- diff(rs_fixture, metric = "RMSE")
  expect_s3_class(d, "diff.resamples")
  expect_identical(colnames(d$difs$RMSE), c("A.diff.B", "A.diff.C", "B.diff.C"))
  # A - B averages -1, A - C averages -2 (A is the better, lower-RMSE model)
  expect_identical(mean(d$difs$RMSE[, "A.diff.B"]), -1)
  expect_identical(mean(d$difs$RMSE[, "A.diff.C"]), -2)
  # each comparison carries a t-test, and the CI is Bonferroni-widened
  expect_s3_class(d$statistics$RMSE[["A.diff.B"]], "htest")
  expect_gt(d$confLevel, 0.95)
})

test_that("compare_models runs a paired test between two fitted models", {
  skip_on_cran()

  set.seed(1)
  dat <- data.frame(x1 = rnorm(80), x2 = rnorm(80))
  dat$y <- 2 * dat$x1 + rnorm(80)
  ctrl <- trainControl(method = "cv", number = 5)

  # same seed before each fit so the two models share the same resamples
  set.seed(2)
  m1 <- train(y ~ ., data = dat, method = "lm", trControl = ctrl)
  set.seed(2)
  m2 <- train(y ~ ., data = dat, method = "knn", trControl = ctrl)

  cmp <- compare_models(m1, m2)
  expect_s3_class(cmp, "htest")
  expect_false(is.null(cmp$p.value))
})

# ------------------------------------------------------------------------------
# PCA, clustering and diff summaries (computations)

test_that("prcomp.resamples runs a PCA on the resampled results", {
  pc <- prcomp(rs_fixture)
  expect_s3_class(pc, "prcomp.resamples")
  expect_identical(pc$metric, "RMSE")
})

test_that("cluster.resamples clusters the models", {
  cl <- cluster(rs_fixture)
  expect_s3_class(cl, "cluster.resamples")
})

test_that("summary.diff.resamples summarises the pairwise differences", {
  sm <- summary(diff(rs_fixture))
  expect_s3_class(sm, "summary.diff.resamples")
  expect_named(sm$table, c("RMSE", "Rsquared"))
})

# ------------------------------------------------------------------------------
# print methods (deterministic fixture -> snapshot)

test_that("resamples print methods render", {
  expect_snapshot(print(rs_fixture))
  expect_snapshot(print(summary(rs_fixture)))
  expect_snapshot(print(diff(rs_fixture)))
  expect_snapshot(print(summary(diff(rs_fixture))))
})

# ------------------------------------------------------------------------------
# plot methods (smoke tests: they should build a trellis object)

test_that("resamples plot methods return trellis objects", {
  expect_s3_class(draw_trellis(xyplot(rs_fixture)), "trellis")
  expect_s3_class(draw_trellis(dotplot(rs_fixture)), "trellis")
  expect_s3_class(draw_trellis(bwplot(rs_fixture)), "trellis")
  expect_s3_class(draw_trellis(densityplot(rs_fixture)), "trellis")
  expect_s3_class(draw_trellis(splom(rs_fixture)), "trellis")
  expect_s3_class(draw_trellis(parallelplot(rs_fixture)), "trellis")
})

test_that("plot.prcomp.resamples draws each type of PCA plot", {
  pc <- prcomp(rs_fixture)
  for (what in c("scree", "cumulative", "loadings", "components")) {
    expect_s3_class(draw_trellis(plot(pc, what = what)), "trellis")
  }
})

test_that("diff.resamples plot methods return trellis objects", {
  d <- diff(rs_fixture)
  expect_s3_class(draw_trellis(densityplot(d)), "trellis")
  expect_s3_class(draw_trellis(bwplot(d)), "trellis")
  expect_s3_class(draw_trellis(dotplot(d)), "trellis")
})

test_that("xyplot.resamples draws each comparison it offers", {
  # a scatter of two models, and the Bland-Altman view of their difference
  expect_s3_class(
    draw_trellis(xyplot(rs_fixture, what = "BlandAltman")),
    "trellis"
  )

  # the three timing views plot performance against how long each model took
  for (w in c("tTime", "mTime", "pTime")) {
    drawn <- draw_trellis(xyplot(rs_fixture, what = w))
    expect_s3_class(drawn, "trellis")
  }

  # the times can be shown in other units
  for (u in c("sec", "min", "hour")) {
    expect_s3_class(
      draw_trellis(xyplot(rs_fixture, what = "tTime", units = u)),
      "trellis"
    )
  }
})

test_that("xyplot.resamples checks what it was asked to draw", {
  expect_snapshot(xyplot(rs_fixture, units = "fortnight"), error = TRUE)
  expect_snapshot(xyplot(rs_fixture, what = "bogus"), error = TRUE)
  expect_snapshot(
    xyplot(rs_fixture, metric = c("RMSE", "Rsquared")),
    error = TRUE
  )
  # Bland-Altman compares exactly two models
  expect_snapshot(
    xyplot(rs_fixture, what = "BlandAltman", models = c("A", "B", "C")),
    error = TRUE
  )
})

test_that("splom.resamples draws each of its variants", {
  # the default matrix of scatterplots, and the two panel layouts
  expect_s3_class(
    draw_trellis(splom(rs_fixture, variables = "models")),
    "trellis"
  )
  expect_s3_class(
    draw_trellis(splom(rs_fixture, variables = "metrics")),
    "trellis"
  )
  expect_snapshot(splom(rs_fixture, variables = "bogus"), error = TRUE)
})

test_that("levelplot.diff.resamples draws p-values and differences", {
  d <- diff(rs_fixture)
  for (w in c("pvalues", "differences")) {
    expect_s3_class(draw_trellis(levelplot(d, what = w)), "trellis")
  }
  expect_snapshot(levelplot(d, metric = c("RMSE", "Rsquared")), error = TRUE)
})

test_that("dotplot.resamples and its diff method draw with options", {
  # a confidence level and a metric subset
  expect_s3_class(
    draw_trellis(dotplot(rs_fixture, metric = "RMSE", conf.level = 0.8)),
    "trellis"
  )
  d <- diff(rs_fixture)
  expect_s3_class(draw_trellis(dotplot(d, metric = "RMSE")), "trellis")
})

test_that("print.prcomp.resamples reports the rotation", {
  pc <- prcomp(rs_fixture)

  # The rotation is not snapshotted: the sign of an eigenvector is arbitrary, so
  # the values (and the sign of the near-zero third component in particular)
  # differ between BLAS implementations. The headings and labels are stable.
  out <- capture.output(print(pc))
  expect_true(any(grepl("Metric: RMSE", out)))
  expect_true(any(grepl("Std. Dev.", out, fixed = TRUE)))
  expect_true(any(grepl("Cum. Percent Var.", out, fixed = TRUE)))
  expect_true(any(grepl("Rotation:", out, fixed = TRUE)))
  # one column per component, and one row per row of the rotation matrix
  expect_true(any(grepl("PC1 *PC2 *PC3", out)))
  expect_all_true(vapply(
    rownames(pc$rotation),
    function(r) any(grepl(r, out, fixed = TRUE)),
    logical(1)
  ))
})

test_that("plot.prcomp.resamples ignores a plot type it does not know", {
  pc <- prcomp(rs_fixture)
  # there is no validation, so an unknown `what` simply draws nothing
  expect_null(plot(pc, what = "bogus"))
})

# ------------------------------------------------------------------------------
# resamples() on real fits: what it checks and what it warns about

test_that("resamples refuses models that cannot be compared", {
  skip_on_cran()

  reg <- engine_regression(60)
  ctrl <- trainControl(method = "cv", number = 3)
  set.seed(1121)
  a <- train(y ~ ., data = reg, method = "lm", trControl = ctrl)
  set.seed(1121)
  b <- train(
    y ~ .,
    data = reg,
    method = "knn",
    tuneGrid = data.frame(k = 5),
    trControl = ctrl
  )

  # the same folds, so these two are comparable
  expect_s3_class(resamples(list(a = a, b = b)), "resamples")

  # a different seed means different folds
  set.seed(8802)
  other_folds <- train(y ~ ., data = reg, method = "lm", trControl = ctrl)
  expect_snapshot(resamples(list(a = a, b = other_folds)), error = TRUE)

  # a different number of resamples
  set.seed(1121)
  fewer <- train(
    y ~ .,
    data = reg,
    method = "lm",
    trControl = trainControl(method = "cv", number = 2)
  )
  expect_snapshot(resamples(list(a = a, b = fewer)), error = TRUE)

  # leave-one-out gives a single estimate, so there is nothing to compare
  set.seed(1121)
  loo <- train(
    y ~ .,
    data = reg[1:16, ],
    method = "lm",
    trControl = trainControl(method = "LOOCV")
  )
  expect_snapshot(resamples(list(a = loo)), error = TRUE)
})

test_that("resamples warns when a model kept every candidate", {
  skip_on_cran()

  dat <- engine_three_class()
  folds <- createFolds(dat$Species, k = 3, returnTrain = TRUE)
  ctrl <- trainControl(method = "cv", index = folds, returnResamp = "all")

  set.seed(3390)
  a <- train(
    Species ~ .,
    data = dat,
    method = "knn",
    tuneGrid = data.frame(k = c(3, 5)),
    trControl = ctrl
  )
  set.seed(3390)
  b <- train(Species ~ ., data = dat, method = "lda", trControl = ctrl)

  # with returnResamp = "all" the resamples cover every candidate, so caret says
  # it is using the chosen one -- once per model
  expect_snapshot(rs <- resamples(list(a = a, b = b)))
  expect_s3_class(rs, "resamples")
})

test_that("resamples reports models with different metrics", {
  skip_on_cran()

  dat <- engine_three_class()
  folds <- createFolds(dat$Species, k = 3, returnTrain = TRUE)

  set.seed(3390)
  a <- train(
    Species ~ .,
    data = dat,
    method = "knn",
    tuneGrid = data.frame(k = 5),
    trControl = trainControl(method = "cv", index = folds)
  )
  set.seed(3390)
  b <- train(
    Species ~ .,
    data = dat,
    method = "knn",
    tuneGrid = data.frame(k = 5),
    trControl = trainControl(
      method = "cv",
      index = folds,
      summaryFunction = function(data, lev = NULL, model = NULL) {
        c(Custom = mean(data$obs == data$pred))
      }
    ),
    metric = "Custom"
  )

  # only the metrics both models computed can be compared
  expect_snapshot_warning(rs <- resamples(list(a = a, b = b)))
  expect_length(rs$metrics, 0)
})

test_that("resamples collects the timings a model recorded", {
  skip_on_cran()

  reg <- engine_regression(60)
  folds <- createFolds(reg$y, k = 3, returnTrain = TRUE)
  set.seed(6210)
  timed <- train(
    y ~ .,
    data = reg,
    method = "lm",
    trControl = trainControl(method = "cv", index = folds, timingSamps = 5)
  )
  set.seed(6210)
  plain <- train(
    y ~ .,
    data = reg,
    method = "knn",
    tuneGrid = data.frame(k = 5),
    trControl = trainControl(method = "cv", index = folds)
  )

  rs <- resamples(list(timed = timed, plain = plain))
  # the prediction time is only recorded for the model that asked for it
  expect_false(is.na(rs$timings["timed", "Prediction"]))
  expect_true(is.na(rs$timings["plain", "Prediction"]))
})

# ------------------------------------------------------------------------------
# the argument checks on the resamples methods

test_that("the resamples methods each want a single metric", {
  both <- c("RMSE", "Rsquared")

  expect_snapshot(as.matrix(rs_fixture, metric = "Bogus"), error = TRUE)
  expect_snapshot(prcomp(rs_fixture, metric = both), error = TRUE)
  expect_snapshot(cluster(rs_fixture, metric = both), error = TRUE)
  expect_snapshot(parallelplot(rs_fixture, metric = both), error = TRUE)
  expect_snapshot(splom(rs_fixture, metric = both), error = TRUE)

  # cluster() only knows how to handle a resamples object
  expect_snapshot(cluster(1:10), error = TRUE)

  # the PCA plot draws one thing at a time
  pc <- prcomp(rs_fixture)
  expect_snapshot(plot(pc, what = c("scree", "loadings")), error = TRUE)
})

test_that("splom.resamples needs two metrics for a metric panel", {
  expect_snapshot(
    splom(rs_fixture, variables = "metrics", metric = "RMSE"),
    error = TRUE
  )
})

test_that("the distribution plots can show a single metric", {
  # with one metric there is nothing to condition on, so the formula has no
  # conditioning term
  for (f in list(densityplot, bwplot, dotplot)) {
    drawn <- draw_trellis(f(rs_fixture, metric = "RMSE"))
    expect_s3_class(drawn, "trellis")
    expect_identical(dim(drawn), 1L)
  }

  d <- diff(rs_fixture)
  for (f in list(densityplot, bwplot)) {
    drawn <- draw_trellis(f(d, metric = "RMSE"))
    expect_identical(dim(drawn), 1L)
  }

  # dotplot.diff.resamples takes the first metric when given several, and says so
  expect_snapshot_warning(drawn <- dotplot(d, metric = c("RMSE", "Rsquared")))
  expect_s3_class(draw_trellis(drawn), "trellis")
})

test_that("ggplot.resamples facets only when there is more than one metric", {
  # the default is the first metric only, so there is nothing to facet by
  plain <- ggplot2::ggplot_build(ggplot2::ggplot(rs_fixture))
  expect_identical(nrow(plain$layout$layout), 1L)

  # asking for both gives a panel each
  faceted <- ggplot2::ggplot_build(
    ggplot2::ggplot(rs_fixture, metric = c("RMSE", "Rsquared"))
  )
  expect_identical(nrow(faceted$layout$layout), 2L)
})

test_that("plot.prcomp.resamples draws the remaining plot types", {
  pc <- prcomp(rs_fixture)

  # `dims` is how many components to show: two gives a scatter of PC2 on PC1,
  # more than two a scatterplot matrix
  expect_s3_class(
    draw_trellis(plot(pc, what = "components", dims = 2)),
    "trellis"
  )
  expect_s3_class(
    draw_trellis(plot(pc, what = "components", dims = 3)),
    "trellis"
  )
})

test_that("print.prcomp.resamples honours the digits argument", {
  pc <- prcomp(rs_fixture)

  # As above, the rotation itself is never snapshotted: two of the three models
  # in the fixture are perfectly (negatively) correlated, so the third component
  # is numerical noise whose values differ between BLAS implementations. Only
  # the number of digits asked for is checked here.
  out <- capture.output(print(pc, digits = 2))
  expect_true(any(grepl("Rotation:", out, fixed = TRUE)))
  loadings <- out[grepl("^Resample", out)]
  expect_length(loadings, nrow(pc$rotation))
  # two significant digits, so no loading is printed with more than four
  # characters after its decimal point
  expect_all_true(!grepl("[0-9]\\.[0-9]{5,}", loadings))
})

# ------------------------------------------------------------------------------
# resamples() on the other object types

test_that("resamples names models that were given without one", {
  skip_on_cran()

  reg <- engine_regression(60)
  folds <- createFolds(reg$y, k = 3, returnTrain = TRUE)
  ctrl <- trainControl(method = "cv", index = folds)
  set.seed(1121)
  a <- train(y ~ ., data = reg, method = "lm", trControl = ctrl)
  set.seed(1121)
  b <- train(
    y ~ .,
    data = reg,
    method = "knn",
    tuneGrid = data.frame(k = 5),
    trControl = ctrl
  )

  # an empty name is filled in with a numbered one
  rs <- resamples(list(a = a, b))
  expect_identical(rs$models, c("a", "Model2"))
})

test_that("resamples warns for rfe and sbf models that kept every subset", {
  skip_on_cran()
  skip_if_not_installed("MASS")

  cls <- engine_two_class(80)
  x <- cls[, names(cls) != "Class"]
  folds <- createFolds(cls$Class, k = 3, returnTrain = TRUE)

  set.seed(3390)
  rf <- rfe(
    x,
    cls$Class,
    sizes = c(2, 4),
    rfeControl = rfeControl(
      functions = ldaFuncs,
      method = "cv",
      index = folds,
      returnResamp = "all"
    )
  )
  set.seed(3390)
  sf <- suppressWarnings(sbf(
    x,
    cls$Class,
    sbfControl = sbfControl(
      functions = ldaSBF,
      method = "cv",
      index = folds,
      returnResamp = "all"
    )
  ))

  # both say they are using the chosen subset rather than all of them
  expect_snapshot(rs <- resamples(list(rfe = rf, sbf = sf)))
  expect_s3_class(rs, "resamples")
})

Try the caret package in your browser

Any scripts or data that you put into this service are public.

caret documentation built on Nov. 15, 2026, 9:06 a.m.