tests/testthat/test-resampleSummary.R

# Tests for resampleSummary(), which turns a matrix of per-resample predictions
# into mean/sd performance metrics (and, optionally, the stacked obs/pred data).

test_that("resampleSummary summarises regression resamples", {
  obs <- c(1, 2, 3, 4, 5)
  resampled <- data.frame(
    R1 = c(1.1, 2.1, 2.9, 4.2, 5.1),
    R2 = c(0.9, 2.0, 3.1, 3.8, 5.0)
  )

  out <- resampleSummary(obs, resampled)
  expect_named(out, c("metrics", "data"))
  # regression metrics: RMSE / Rsquared / MAE, as mean then sd (six values)
  expect_length(out$metrics, 6L)

  # the mean RMSE equals the mean of the per-resample RMSEs
  rmses <- vapply(
    resampled,
    function(p) postResample(p, obs)["RMSE"],
    numeric(1)
  )
  expect_equal(unname(out$metrics["RMSE"][1]), mean(rmses))

  # kept data is the predictions stacked with their resample label
  expect_s3_class(out$data, "data.frame")
  expect_named(out$data, c("obs", "pred", "group"))
  expect_identical(nrow(out$data), 10L)
})

test_that("resampleSummary can drop the stacked data", {
  obs <- c(1, 2, 3, 4, 5)
  resampled <- data.frame(R1 = c(1, 2, 3, 4, 5), R2 = c(1, 2, 3, 4, 6))
  out <- resampleSummary(obs, resampled, keepData = FALSE)
  expect_null(out$data)
})

test_that("resampleSummary handles the leave-one-out case", {
  # in LOOCV each resample holds out a single observation, so every column has
  # one prediction; the metrics are computed once and their sd is 0
  obs <- c(1, 2, 3, 4, 5)
  m <- matrix(NA_real_, 5, 5)
  diag(m) <- c(1.1, 2.1, 2.9, 4.2, 5.1)

  out <- resampleSummary(obs, as.data.frame(m))
  expect_length(out$metrics, 6L)
  # the second half (the standard deviations) is all zero
  expect_all_equal(out$metrics[4:6], 0)
  expect_identical(nrow(out$data), 5L)
})

test_that("resampleSummary summarises classification resamples", {
  obs <- factor(c("a", "b", "a", "b", "a"), levels = c("a", "b"))
  resampled <- data.frame(
    R1 = factor(c("a", "b", "a", "a", "a"), levels = c("a", "b")),
    R2 = factor(c("a", "b", "b", "b", "a"), levels = c("a", "b"))
  )

  out <- resampleSummary(obs, resampled)
  # classification metrics are Accuracy / Kappa, as mean then sd
  expect_in(c("Accuracy", "Kappa"), names(out$metrics))
  # the stacked predictions stay a factor with the original levels
  expect_s3_class(out$data$pred, "factor")
  expect_identical(levels(out$data$pred), c("a", "b"))
})

test_that("resampleSummary stacks complete factor resamples", {
  obs <- factor(c("a", "b", "a", "b"), levels = c("a", "b"))
  resampled <- data.frame(
    R1 = factor(c("a", "b", "a", "a"), levels = c("a", "b")),
    R2 = factor(c("a", "b", "b", "b"), levels = c("a", "b"))
  )
  out <- resampleSummary(obs, resampled)
  expect_s3_class(out$data$pred, "factor")
  expect_identical(levels(out$data$pred), levels(obs))
})

test_that("resampleSummary handles leave-one-out factor resamples", {
  # in the LOO layout each column holds a single held-out prediction
  lv <- c("a", "b")
  obs <- factor(c("a", "b", "a", "b"), levels = lv)
  resampled <- data.frame(
    R1 = factor(c("a", NA, NA, NA), levels = lv),
    R2 = factor(c(NA, "b", NA, NA), levels = lv),
    R3 = factor(c(NA, NA, "b", NA), levels = lv),
    R4 = factor(c(NA, NA, NA, "b"), levels = lv)
  )
  out <- resampleSummary(obs, resampled)
  expect_s3_class(out$data$pred, "factor")
  expect_identical(levels(out$data$pred), lv)
  # a single stacked resample has no spread, so the sds are zeroed
  expect_all_equal(unname(out$metrics[3:4]), 0)
})

Try the caret package in your browser

Any scripts or data that you put into this service are public.

caret documentation built on Nov. 15, 2026, 9:06 a.m.