tests/testthat/test-confusionMatrix.R

test_that("Confusion matrix works", {
  skip_on_cran()
  set.seed(442)
  train <- twoClassSim(
    n = 1000,
    intercept = -8,
    linearVars = 3,
    noiseVars = 10,
    corrVars = 4,
    corrValue = 0.6
  )

  ctrl <- trainControl(method = "cv", classProbs = TRUE)

  fullModel <- train(
    Class ~ .,
    data = train,
    method = "knn",
    preProc = c("center", "scale"),
    tuneLength = 4,
    trControl = ctrl
  )
  dat <- train$Class
  ref <- predict(fullModel, newdata = train)
  dat2 <- as.character(dat)
  ref2 <- as.character(ref)
  dat2 <- factor(dat2, levels = c("Class2", "Class1"))
  ref2 <- factor(ref2, levels = c("Class1", "Class2"))
  dat3 <- rep("Class1", length(ref2))
  dat3 <- factor(dat3)
  dat4 <- factor(dat3, levels = c("Class1", "Class4"))
  dat5 <- as.character(dat3)
  dat5[200] <- "Class4"
  dat5 <- factor(dat5, levels = c("Class1", "Class4"))
  cm1 <- confusionMatrix(dat, ref)
  cm2 <- suppressWarnings(confusionMatrix(dat2, ref2))
  cm3 <- suppressWarnings(confusionMatrix(dat3, ref2))
  cm4 <- suppressWarnings(confusionMatrix(dat4, ref2))
  expect_s3_class(cm1, "confusionMatrix")
  expect_s3_class(cm2, "confusionMatrix")
  expect_s3_class(cm3, "confusionMatrix")
  expect_s3_class(cm4, "confusionMatrix")
  expect_snapshot_warning(confusionMatrix(dat2, ref2))
  expect_snapshot_warning(confusionMatrix(dat3, ref2))
  expect_snapshot(confusionMatrix(dat5, ref2), error = TRUE)
  expect_snapshot(confusionMatrix(dat5, ref), error = TRUE)
  expect_identical(cm1$overall, cm2$overall)
  expect_identical(cm4$overall, cm3$overall)
  expect_true(identical(cm1, cm2))
  expect_true(identical(cm3, cm4))

  mat1 <- as.matrix(cm1$table)
  cm11 <- confusionMatrix(mat1)
  expect_equal(cm1, cm11)
})

# ------------------------------------------------------------------------------
# confusionMatrix.table / .matrix on fixed tables (cm_tab2, cm_tab3 live in
# helper-confusionMatrix.R)

test_that("confusionMatrix.table computes two-class statistics", {
  cm <- confusionMatrix(cm_tab2)
  expect_s3_class(cm, "confusionMatrix")
  # first level is the default positive class
  expect_identical(cm$positive, "yes")
  expect_identical(cm$mode, "sens_spec")
  # computed ratios: expect_equal (tolerance) rather than expect_identical, so
  # the test survives future changes to how the statistic is derived
  expect_equal(cm$byClass["Sensitivity"], c(Sensitivity = 0.8))
  expect_equal(cm$byClass["Specificity"], c(Specificity = 0.9))
  expect_identical(
    names(cm$overall),
    c(
      "Accuracy",
      "Kappa",
      "AccuracyLower",
      "AccuracyUpper",
      "AccuracyNull",
      "AccuracyPValue",
      "McnemarPValue"
    )
  )
})

test_that("confusionMatrix.table honours the positive and prevalence args", {
  # choose the other class as positive: sensitivity/specificity swap
  cm <- confusionMatrix(cm_tab2, positive = "no")
  expect_identical(cm$positive, "no")
  # computed ratio, so keep the tolerant comparison
  expect_equal(cm$byClass["Sensitivity"], c(Sensitivity = 0.9))

  # a supplied prevalence overrides the value taken from the table (a pass
  # through, so it round-trips exactly)
  cm_prev <- confusionMatrix(cm_tab2, prevalence = 0.25)
  expect_identical(cm_prev$byClass["Prevalence"], c(Prevalence = 0.25))
})

test_that("confusionMatrix.table handles the multiclass case", {
  cm <- confusionMatrix(cm_tab3)
  # per-class stats come back as a matrix, one row per class
  expect_true(is.matrix(cm$byClass))
  expect_identical(nrow(cm$byClass), 3L)
  expect_identical(rownames(cm$byClass), paste("Class:", LETTERS[1:3]))

  # a named per-class prevalence vector is accepted and used
  cm_prev <- confusionMatrix(cm_tab3, prevalence = c(A = 0.3, B = 0.3, C = 0.4))
  expect_identical(cm_prev$byClass["Class: A", "Prevalence"], 0.3)
})

test_that("confusionMatrix.matrix matches confusionMatrix.table", {
  from_matrix <- confusionMatrix(unclass(cm_tab2))
  from_table <- confusionMatrix(cm_tab2)
  expect_identical(from_matrix$table, from_table$table)
  expect_identical(from_matrix$byClass, from_table$byClass)
})

test_that("confusionMatrix.table rejects bad input", {
  expect_snapshot(confusionMatrix(cm_tab2, mode = "nope"), error = TRUE)
  expect_snapshot(
    confusionMatrix(as.table(matrix(1:6, nrow = 2))),
    error = TRUE
  )
  # a one-level table has too few classes
  one <- as.table(matrix(
    5,
    dimnames = list(Prediction = "a", Reference = "a")
  ))
  expect_snapshot(confusionMatrix(one), error = TRUE)
  # positive must be a string
  expect_snapshot(confusionMatrix(cm_tab2, positive = 1), error = TRUE)
  # row and column classes must match
  mismatched <- as.table(matrix(
    c(8, 2, 1, 9),
    nrow = 2,
    dimnames = list(c("a", "b"), c("a", "c"))
  ))
  expect_snapshot(confusionMatrix(mismatched), error = TRUE)
})

test_that("confusionMatrix.table rejects mis-sized prevalence vectors", {
  # two classes need a single prevalence
  expect_snapshot(
    confusionMatrix(cm_tab2, prevalence = c(0.2, 0.3)),
    error = TRUE
  )
  # multiclass needs one prevalence per class...
  expect_snapshot(
    confusionMatrix(cm_tab3, prevalence = c(0.5, 0.5)),
    error = TRUE
  )
  # ...and it must be named
  expect_snapshot(
    confusionMatrix(cm_tab3, prevalence = c(0.3, 0.3, 0.4)),
    error = TRUE
  )
})

# ------------------------------------------------------------------------------
# as.matrix / as.table methods

test_that("as.matrix.confusionMatrix returns each requested piece", {
  cm <- confusionMatrix(cm_tab2)
  expect_identical(dim(as.matrix(cm, what = "xtabs")), c(2L, 2L))
  expect_identical(nrow(as.matrix(cm, what = "overall")), 7L)
  expect_identical(nrow(as.matrix(cm, what = "classes")), 11L)
  # the multiclass "classes" matrix is transposed to one column per class
  cm3 <- confusionMatrix(cm_tab3)
  expect_identical(colnames(as.matrix(cm3, what = "classes")), LETTERS[1:3])
  expect_snapshot(as.matrix(cm, what = "nope"), error = TRUE)
})

test_that("as.table.confusionMatrix returns the underlying table", {
  cm <- confusionMatrix(cm_tab2)
  expect_identical(as.table(cm), cm$table)
})

# ------------------------------------------------------------------------------
# print methods (deterministic tables -> safe to snapshot)

test_that("print.confusionMatrix renders each mode", {
  expect_snapshot(print(confusionMatrix(cm_tab2, mode = "sens_spec")))
  expect_snapshot(print(confusionMatrix(cm_tab2, mode = "prec_recall")))
  expect_snapshot(print(confusionMatrix(cm_tab2, mode = "everything")))
  expect_snapshot(print(confusionMatrix(cm_tab3)))
  # the table can be printed on its own, without the statistics block
  expect_snapshot(print(confusionMatrix(cm_tab2), printStats = FALSE))
})

# ------------------------------------------------------------------------------
# confusionMatrix.default input validation

test_that("confusionMatrix.default validates its arguments", {
  a <- factor(c("a", "b", "a", "b"))
  b <- factor(c("a", "a", "b", "b"))
  expect_snapshot(confusionMatrix(a, b, mode = "nope"), error = TRUE)
  # non-factor input
  expect_snapshot(confusionMatrix(1:4, b), error = TRUE)
  # positive must be a string
  expect_snapshot(confusionMatrix(a, b, positive = 1), error = TRUE)
  # data has a level the reference does not
  more <- factor(c("a", "b", "c", "b"))
  expect_snapshot(confusionMatrix(more, b), error = TRUE)
  # data and reference share no levels
  x <- factor(c("x", "y"), levels = c("x", "y"))
  y <- factor(c("a", "b"), levels = c("a", "b"))
  expect_snapshot(confusionMatrix(x, y), error = TRUE)
})

# ------------------------------------------------------------------------------
# resampName text for each resampling scheme (pure helper, fake control objects)

test_that("resampName describes each resampling method", {
  mk <- function(method, ...) {
    list(
      control = c(
        list(method = method, index = list(1:3, 1:3, 1:3)),
        list(...)
      )
    )
  }
  rn <- function(...) caret:::resampName(...)

  # no control element -> empty string
  expect_identical(caret:::resampName(list()), "")

  expect_identical(rn(mk("cv", number = 10)), "Cross-Validated (10 fold)")
  expect_identical(
    rn(mk("repeatedcv", number = 10, repeats = 5)),
    "Cross-Validated (10 fold, repeated 5 times)"
  )
  expect_identical(rn(mk("boot")), "Bootstrapped (3 reps)")
  expect_identical(rn(mk("boot632")), "Bootstrapped (3 reps)")
  expect_identical(
    rn(mk("LGOCV", p = 0.75)),
    "Repeated Train/Test Splits Estimated (3 reps, 75%)"
  )
  expect_identical(rn(mk("LOOCV")), "Leave-One-Out Cross-Validation")
  expect_identical(rn(mk("none")), "None")
  expect_identical(rn(mk("apparent")), "Apparent")
  expect_identical(rn(mk("custom")), "Custom Resampling (3 reps)")
  expect_identical(
    rn(mk("timeslice", horizon = 5, fixedWindow = TRUE)),
    "Rolling Forecasting Origin Resampling (5 held-out with a fixed window)"
  )
  expect_identical(rn(mk("oob")), "Out of Bag Resampling")
  expect_identical(
    rn(mk("adaptive_cv", number = 10, repeats = 5)),
    "Adaptively Cross-Validated (10 fold, repeated 5 times)"
  )

  # numbers = FALSE gives the short form
  expect_identical(
    rn(mk("cv", number = 10), numbers = FALSE),
    "(Cross-Validation)"
  )
})

# ------------------------------------------------------------------------------
# resampled confusion matrices from train / rfe / sbf objects

test_that("confusionMatrix.train summarises the resampled matrix", {
  skip_on_cran()

  set.seed(1)
  dat <- twoClassSim(200)
  fit <- train(
    Class ~ .,
    data = dat,
    method = "knn",
    tuneLength = 2,
    trControl = trainControl(method = "cv", number = 3, classProbs = TRUE)
  )

  cm <- confusionMatrix(fit)
  expect_s3_class(cm, "confusionMatrix.train")
  expect_identical(cm$norm, "overall")
  # percentages, so the whole table sums to ~100 (floating point -> expect_equal)
  expect_equal(sum(cm$table), 100)

  # the other normalisations run and print without error
  expect_identical(confusionMatrix(fit, norm = "none")$norm, "none")
  expect_identical(confusionMatrix(fit, norm = "average")$norm, "average")
  expect_output(print(cm), "Confusion Matrix")
})

test_that("confusionMatrix errors on regression and non-resampled fits", {
  skip_on_cran()

  set.seed(1)
  reg <- SLC14_1(100)
  reg_fit <- train(
    y ~ .,
    data = reg,
    method = "lm",
    trControl = trainControl(method = "cv", number = 3)
  )
  expect_snapshot(confusionMatrix(reg_fit), error = TRUE)

  set.seed(1)
  dat <- twoClassSim(120)
  loo_fit <- train(
    Class ~ .,
    data = dat,
    method = "knn",
    tuneLength = 1,
    trControl = trainControl(method = "LOOCV")
  )
  expect_snapshot(confusionMatrix(loo_fit), error = TRUE)
})

test_that("confusionMatrix works for rfe and sbf objects", {
  skip_on_cran()
  skip_if_not_installed("MASS")

  set.seed(1)
  dat <- twoClassSim(200)

  set.seed(1)
  rf <- rfe(
    dat[, 1:6],
    dat$Class,
    sizes = c(2, 4),
    rfeControl = rfeControl(functions = lrFuncs, method = "cv", number = 3)
  )
  expect_s3_class(confusionMatrix(rf), "confusionMatrix.rfe")

  set.seed(1)
  sf <- sbf(
    dat[, 1:6],
    dat$Class,
    sbfControl = sbfControl(
      functions = ldaSBF,
      method = "cv",
      number = 3,
      saveDetails = TRUE
    )
  )
  expect_s3_class(confusionMatrix(sf), "confusionMatrix.sbf")
})

test_that("confusionMatrix needs at least two factor levels", {
  one <- factor(rep("a", 4), levels = "a")
  expect_snapshot(confusionMatrix(one, one), error = TRUE)
})

test_that("confusionMatrix.matrix rejects non-square matrices", {
  expect_snapshot(confusionMatrix(matrix(1:6, nrow = 2)), error = TRUE)
})

test_that("confusionMatrix.table rejects tables with other than 2 dimensions", {
  expect_snapshot(
    confusionMatrix(as.table(array(1:8, dim = c(2, 2, 2)))),
    error = TRUE
  )
})

test_that("confusionMatrix requires named prevalences for 3+ classes", {
  expect_snapshot(
    confusionMatrix(cm_tab3, prevalence = c(0.2, 0.3, 0.5)),
    error = TRUE
  )
})

test_that("confusionMatrix reports missing test statistics for empty tables", {
  # binomial tests cannot run on an all-zero table, so their statistics are NA
  zt <- as.table(matrix(0L, 2, 2, dimnames = list(c("a", "b"), c("a", "b"))))
  cm <- confusionMatrix(zt)
  expect_true(is.na(cm$overall["AccuracyLower"]))
  expect_true(is.na(cm$overall["AccuracyPValue"]))
})

test_that("print.confusionMatrix defaults and validates the mode", {
  cm <- confusionMatrix(cm_tab2)
  # a NULL mode falls back to sens_spec
  expect_snapshot(print(cm, mode = NULL))
  expect_snapshot(print(cm, mode = "nope"), error = TRUE)
})

test_that("print.confusionMatrix filters multiclass statistics by mode", {
  # a perfect table keeps the printed statistics exact
  cm <- confusionMatrix(table(iris$Species, iris$Species), mode = "everything")
  expect_snapshot(print(cm, mode = "prec_recall"))
})

test_that("confusionMatrix.train works without stored resampling indices", {
  skip_on_cran()

  set.seed(1)
  fit <- train(
    Species ~ .,
    data = iris,
    method = "knn",
    tuneGrid = data.frame(k = 5),
    trControl = trainControl(method = "cv", number = 3)
  )
  bare <- fit
  bare$control$index <- NULL
  cm <- confusionMatrix(bare)
  expect_s3_class(cm, "confusionMatrix.train")
})

test_that("print.confusionMatrix.train shows counts for a single resample", {
  skip_on_cran()

  set.seed(1)
  fit <- train(
    Species ~ .,
    data = iris,
    method = "knn",
    tuneGrid = data.frame(k = 5),
    trControl = trainControl(method = "LGOCV", number = 1, p = 0.75)
  )
  cm <- confusionMatrix(fit, norm = "none")
  expect_snapshot(print(cm))
})

test_that("confusionMatrix.train can rebuild resampled tables from predictions", {
  skip_on_cran()

  set.seed(1)
  fit <- train(
    Species ~ .,
    data = iris,
    method = "knn",
    tuneGrid = data.frame(k = 5),
    trControl = trainControl(method = "cv", number = 3, savePredictions = "all")
  )
  # without the pre-computed tables, the saved predictions are used
  bare <- fit
  bare$resampledCM <- NULL
  cm <- confusionMatrix(bare)
  expect_s3_class(cm, "confusionMatrix.train")
  expect_equal(sum(cm$table), 100)
})

test_that("confusion matrices are refused for regression fits", {
  fake <- structure(list(modelType = "Regression"), class = "train")
  expect_snapshot(caret:::train_resampledCM(fake), error = TRUE)
})

test_that("the resampled-table helpers explain what 50+ class objects need", {
  fake_train <- structure(
    list(
      modelType = "Classification",
      levels = paste0("class", 1:51),
      resampledCM = NULL,
      pred = NULL
    ),
    class = "train"
  )
  expect_snapshot(caret:::train_resampledCM(fake_train), error = TRUE)

  fake_rfe <- structure(
    list(
      obsLevels = paste0("class", 1:51),
      resample = data.frame(Variables = 1),
      pred = NULL
    ),
    class = "rfe"
  )
  expect_snapshot(caret:::rfe_resampledCM(fake_rfe), error = TRUE)

  fake_sbf <- structure(
    list(obsLevels = c("a", "b"), pred = NULL),
    class = "sbf"
  )
  expect_snapshot(caret:::sbf_resampledCM(fake_sbf), error = TRUE)
})

test_that("confusionMatrix.rfe can rebuild resampled tables from predictions", {
  skip_on_cran()

  set.seed(1)
  dat <- twoClassSim(120)
  set.seed(1)
  rf <- rfe(
    dat[, 1:8],
    dat$Class,
    sizes = c(2, 4),
    rfeControl = rfeControl(
      functions = lrFuncs,
      method = "cv",
      number = 3,
      saveDetails = TRUE
    )
  )
  # drop the pre-computed per-resample cells so the saved predictions are used
  bare <- rf
  bare$resample <- bare$resample[, !grepl("\\.cell", names(bare$resample))]
  cm <- confusionMatrix(bare)
  expect_s3_class(cm, "confusionMatrix.rfe")
})

Try the caret package in your browser

Any scripts or data that you put into this service are public.

caret documentation built on Nov. 15, 2026, 9:06 a.m.