Nothing
test_that("resample calculations", {
skip_on_cran()
skip_if_not_installed("MASS")
set.seed(4793)
tr_dat <- SLC14_1(200)
ctrl <- trainControl(method = "cv", number = 3)
set.seed(5423)
lm_fit <- train(y ~ ., data = tr_dat, method = "lm", trControl = ctrl)
set.seed(5423)
# the snapshot captures the first convergence warning; rlm warns once per
# non-converged fold, so silence the repeats
suppressWarnings(
expect_snapshot_warning(
rlm_fit <- train(y ~ ., data = tr_dat, method = "rlm", trControl = ctrl)
)
)
rs <- resamples(list(lm = lm_fit, rlm = rlm_fit))
rs_d <- diff(rs, metric = "RMSE")
t_test <- t.test(rs$values$`lm~RMSE`, rs$values$`rlm~RMSE`, paired = TRUE)
expect_equal(rs_d$statistics$RMSE$lm.diff.rlm$conf.int, t_test$conf.int)
lm_t_test <- t.test(rs$values$`lm~RMSE`)
rlm_t_test <- t.test(rs$values$`rlm~RMSE`)
rs_plot_dat <- ggplot(rs, metric = "RMSE")$data
expect_equal(lm_t_test$estimate, rs_plot_dat$Estimate[1], ignore_attr = TRUE)
expect_equal(rlm_t_test$estimate, rs_plot_dat$Estimate[2], ignore_attr = TRUE)
expect_equal(
lm_t_test$conf.int[1],
rs_plot_dat$LowerLimit[1],
ignore_attr = TRUE
)
expect_equal(
rlm_t_test$conf.int[1],
rs_plot_dat$LowerLimit[2],
ignore_attr = TRUE
)
expect_equal(
lm_t_test$conf.int[2],
rs_plot_dat$UpperLimit[1],
ignore_attr = TRUE
)
expect_equal(
rlm_t_test$conf.int[2],
rs_plot_dat$UpperLimit[2],
ignore_attr = TRUE
)
rs_const <- rs
rs_const$values$`lm~RMSE` <- 3
rs_plot_const_dat <- ggplot(rs_const, metric = "RMSE")$data
expect_equal(rs_plot_const_dat$Estimate[1], 3.0, ignore_attr = TRUE)
expect_equal(rs_plot_const_dat$LowerLimit[1], NA_real_, ignore_attr = TRUE)
expect_equal(rs_plot_const_dat$UpperLimit[1], NA_real_, ignore_attr = TRUE)
})
# ------------------------------------------------------------------------------
test_that("test group-k-fold", {
skip_on_cran()
get_data <- function(n = 500) {
prevalence <- seq(0.1, 0.9, length.out = 26)
dat <- sample(letters, size = n, replace = TRUE, prob = sample(prevalence))
data.frame(grp = dat, stringsAsFactors = TRUE)
}
check_rs <- function(ind, dat) {
in_samp <- unique(dat$grp[ind])
out_samp <- unique(dat$grp[-ind])
length(intersect(out_samp, in_samp)) > 0
}
set.seed(5683286)
running_sum <- 0
for (k in (1:5) * 100) {
for (j in 1:20) {
dat <- get_data(n = k)
lvls <- length(unique(as.character(dat$grp)))
for (i in 1:20) {
inds <- groupKFold(dat$grp, k = i)
running_sum <- running_sum +
sum(vapply(inds, check_rs, logical(1), dat))
}
}
}
expect_true(running_sum == 0)
})
# ------------------------------------------------------------------------------
# Methods that act on a resamples object; `rs_fixture` lives in
# helper-resamples.R
test_that("as.data.frame.resamples pulls out one metric in long-ish form", {
df <- as.data.frame(rs_fixture, metric = "RMSE")
# one column per model, plus the Resample labels tacked on the end
expect_named(df, c("A", "B", "C", "Resample"))
expect_identical(df$A, c(1, 2, 3, 4, 5))
expect_identical(df$Resample, paste0("Fold", 1:5))
})
test_that("as.matrix.resamples returns a resample-by-model matrix", {
m <- as.matrix(rs_fixture, metric = "RMSE")
expect_identical(dim(m), c(5L, 3L))
expect_identical(colnames(m), c("A", "B", "C"))
expect_identical(rownames(m), paste0("Fold", 1:5))
expect_identical(unname(m[, "C"]), c(7, 6, 5, 4, 3))
# asking for a metric that isn't there should be an error, not empty output
expect_snapshot(as.matrix(rs_fixture, metric = "nope"), error = TRUE)
})
test_that("sort.resamples orders models by mean performance", {
# A has the lowest mean RMSE, C the highest
expect_identical(sort(rs_fixture, metric = "RMSE"), c("A", "B", "C"))
expect_identical(
sort(rs_fixture, metric = "RMSE", decreasing = TRUE),
c("C", "B", "A")
)
})
test_that("modelCor correlates models across resamples", {
cm <- modelCor(rs_fixture, metric = "RMSE")
expect_identical(diag(cm), c(A = 1, B = 1, C = 1))
# C was built as a mirror image of A, so they sit at -1 (up to floating point)
expect_equal(cm["A", "C"], -1)
# A and B rise together
expect_gt(cm["A", "B"], 0)
})
test_that("summary.resamples reports the usual five-number summary per model", {
sm <- summary(rs_fixture)
expect_s3_class(sm, "summary.resamples")
expect_named(sm$statistics, c("RMSE", "Rsquared"))
# the row means match what we put in (A averages 3, C averages 5)
expect_identical(sm$statistics$RMSE["A", "Mean"], 3)
expect_identical(sm$statistics$RMSE["C", "Mean"], 5)
})
test_that("diff.resamples computes pairwise differences and tests", {
d <- diff(rs_fixture, metric = "RMSE")
expect_s3_class(d, "diff.resamples")
expect_identical(colnames(d$difs$RMSE), c("A.diff.B", "A.diff.C", "B.diff.C"))
# A - B averages -1, A - C averages -2 (A is the better, lower-RMSE model)
expect_identical(mean(d$difs$RMSE[, "A.diff.B"]), -1)
expect_identical(mean(d$difs$RMSE[, "A.diff.C"]), -2)
# each comparison carries a t-test, and the CI is Bonferroni-widened
expect_s3_class(d$statistics$RMSE[["A.diff.B"]], "htest")
expect_gt(d$confLevel, 0.95)
})
test_that("compare_models runs a paired test between two fitted models", {
skip_on_cran()
set.seed(1)
dat <- data.frame(x1 = rnorm(80), x2 = rnorm(80))
dat$y <- 2 * dat$x1 + rnorm(80)
ctrl <- trainControl(method = "cv", number = 5)
# same seed before each fit so the two models share the same resamples
set.seed(2)
m1 <- train(y ~ ., data = dat, method = "lm", trControl = ctrl)
set.seed(2)
m2 <- train(y ~ ., data = dat, method = "knn", trControl = ctrl)
cmp <- compare_models(m1, m2)
expect_s3_class(cmp, "htest")
expect_false(is.null(cmp$p.value))
})
# ------------------------------------------------------------------------------
# PCA, clustering and diff summaries (computations)
test_that("prcomp.resamples runs a PCA on the resampled results", {
pc <- prcomp(rs_fixture)
expect_s3_class(pc, "prcomp.resamples")
expect_identical(pc$metric, "RMSE")
})
test_that("cluster.resamples clusters the models", {
cl <- cluster(rs_fixture)
expect_s3_class(cl, "cluster.resamples")
})
test_that("summary.diff.resamples summarises the pairwise differences", {
sm <- summary(diff(rs_fixture))
expect_s3_class(sm, "summary.diff.resamples")
expect_named(sm$table, c("RMSE", "Rsquared"))
})
# ------------------------------------------------------------------------------
# print methods (deterministic fixture -> snapshot)
test_that("resamples print methods render", {
expect_snapshot(print(rs_fixture))
expect_snapshot(print(summary(rs_fixture)))
expect_snapshot(print(diff(rs_fixture)))
expect_snapshot(print(summary(diff(rs_fixture))))
})
# ------------------------------------------------------------------------------
# plot methods (smoke tests: they should build a trellis object)
test_that("resamples plot methods return trellis objects", {
expect_s3_class(draw_trellis(xyplot(rs_fixture)), "trellis")
expect_s3_class(draw_trellis(dotplot(rs_fixture)), "trellis")
expect_s3_class(draw_trellis(bwplot(rs_fixture)), "trellis")
expect_s3_class(draw_trellis(densityplot(rs_fixture)), "trellis")
expect_s3_class(draw_trellis(splom(rs_fixture)), "trellis")
expect_s3_class(draw_trellis(parallelplot(rs_fixture)), "trellis")
})
test_that("plot.prcomp.resamples draws each type of PCA plot", {
pc <- prcomp(rs_fixture)
for (what in c("scree", "cumulative", "loadings", "components")) {
expect_s3_class(draw_trellis(plot(pc, what = what)), "trellis")
}
})
test_that("diff.resamples plot methods return trellis objects", {
d <- diff(rs_fixture)
expect_s3_class(draw_trellis(densityplot(d)), "trellis")
expect_s3_class(draw_trellis(bwplot(d)), "trellis")
expect_s3_class(draw_trellis(dotplot(d)), "trellis")
})
test_that("xyplot.resamples draws each comparison it offers", {
# a scatter of two models, and the Bland-Altman view of their difference
expect_s3_class(
draw_trellis(xyplot(rs_fixture, what = "BlandAltman")),
"trellis"
)
# the three timing views plot performance against how long each model took
for (w in c("tTime", "mTime", "pTime")) {
drawn <- draw_trellis(xyplot(rs_fixture, what = w))
expect_s3_class(drawn, "trellis")
}
# the times can be shown in other units
for (u in c("sec", "min", "hour")) {
expect_s3_class(
draw_trellis(xyplot(rs_fixture, what = "tTime", units = u)),
"trellis"
)
}
})
test_that("xyplot.resamples checks what it was asked to draw", {
expect_snapshot(xyplot(rs_fixture, units = "fortnight"), error = TRUE)
expect_snapshot(xyplot(rs_fixture, what = "bogus"), error = TRUE)
expect_snapshot(
xyplot(rs_fixture, metric = c("RMSE", "Rsquared")),
error = TRUE
)
# Bland-Altman compares exactly two models
expect_snapshot(
xyplot(rs_fixture, what = "BlandAltman", models = c("A", "B", "C")),
error = TRUE
)
})
test_that("splom.resamples draws each of its variants", {
# the default matrix of scatterplots, and the two panel layouts
expect_s3_class(
draw_trellis(splom(rs_fixture, variables = "models")),
"trellis"
)
expect_s3_class(
draw_trellis(splom(rs_fixture, variables = "metrics")),
"trellis"
)
expect_snapshot(splom(rs_fixture, variables = "bogus"), error = TRUE)
})
test_that("levelplot.diff.resamples draws p-values and differences", {
d <- diff(rs_fixture)
for (w in c("pvalues", "differences")) {
expect_s3_class(draw_trellis(levelplot(d, what = w)), "trellis")
}
expect_snapshot(levelplot(d, metric = c("RMSE", "Rsquared")), error = TRUE)
})
test_that("dotplot.resamples and its diff method draw with options", {
# a confidence level and a metric subset
expect_s3_class(
draw_trellis(dotplot(rs_fixture, metric = "RMSE", conf.level = 0.8)),
"trellis"
)
d <- diff(rs_fixture)
expect_s3_class(draw_trellis(dotplot(d, metric = "RMSE")), "trellis")
})
test_that("print.prcomp.resamples reports the rotation", {
pc <- prcomp(rs_fixture)
# The rotation is not snapshotted: the sign of an eigenvector is arbitrary, so
# the values (and the sign of the near-zero third component in particular)
# differ between BLAS implementations. The headings and labels are stable.
out <- capture.output(print(pc))
expect_true(any(grepl("Metric: RMSE", out)))
expect_true(any(grepl("Std. Dev.", out, fixed = TRUE)))
expect_true(any(grepl("Cum. Percent Var.", out, fixed = TRUE)))
expect_true(any(grepl("Rotation:", out, fixed = TRUE)))
# one column per component, and one row per row of the rotation matrix
expect_true(any(grepl("PC1 *PC2 *PC3", out)))
expect_all_true(vapply(
rownames(pc$rotation),
function(r) any(grepl(r, out, fixed = TRUE)),
logical(1)
))
})
test_that("plot.prcomp.resamples ignores a plot type it does not know", {
pc <- prcomp(rs_fixture)
# there is no validation, so an unknown `what` simply draws nothing
expect_null(plot(pc, what = "bogus"))
})
# ------------------------------------------------------------------------------
# resamples() on real fits: what it checks and what it warns about
test_that("resamples refuses models that cannot be compared", {
skip_on_cran()
reg <- engine_regression(60)
ctrl <- trainControl(method = "cv", number = 3)
set.seed(1121)
a <- train(y ~ ., data = reg, method = "lm", trControl = ctrl)
set.seed(1121)
b <- train(
y ~ .,
data = reg,
method = "knn",
tuneGrid = data.frame(k = 5),
trControl = ctrl
)
# the same folds, so these two are comparable
expect_s3_class(resamples(list(a = a, b = b)), "resamples")
# a different seed means different folds
set.seed(8802)
other_folds <- train(y ~ ., data = reg, method = "lm", trControl = ctrl)
expect_snapshot(resamples(list(a = a, b = other_folds)), error = TRUE)
# a different number of resamples
set.seed(1121)
fewer <- train(
y ~ .,
data = reg,
method = "lm",
trControl = trainControl(method = "cv", number = 2)
)
expect_snapshot(resamples(list(a = a, b = fewer)), error = TRUE)
# leave-one-out gives a single estimate, so there is nothing to compare
set.seed(1121)
loo <- train(
y ~ .,
data = reg[1:16, ],
method = "lm",
trControl = trainControl(method = "LOOCV")
)
expect_snapshot(resamples(list(a = loo)), error = TRUE)
})
test_that("resamples warns when a model kept every candidate", {
skip_on_cran()
dat <- engine_three_class()
folds <- createFolds(dat$Species, k = 3, returnTrain = TRUE)
ctrl <- trainControl(method = "cv", index = folds, returnResamp = "all")
set.seed(3390)
a <- train(
Species ~ .,
data = dat,
method = "knn",
tuneGrid = data.frame(k = c(3, 5)),
trControl = ctrl
)
set.seed(3390)
b <- train(Species ~ ., data = dat, method = "lda", trControl = ctrl)
# with returnResamp = "all" the resamples cover every candidate, so caret says
# it is using the chosen one -- once per model
expect_snapshot(rs <- resamples(list(a = a, b = b)))
expect_s3_class(rs, "resamples")
})
test_that("resamples reports models with different metrics", {
skip_on_cran()
dat <- engine_three_class()
folds <- createFolds(dat$Species, k = 3, returnTrain = TRUE)
set.seed(3390)
a <- train(
Species ~ .,
data = dat,
method = "knn",
tuneGrid = data.frame(k = 5),
trControl = trainControl(method = "cv", index = folds)
)
set.seed(3390)
b <- train(
Species ~ .,
data = dat,
method = "knn",
tuneGrid = data.frame(k = 5),
trControl = trainControl(
method = "cv",
index = folds,
summaryFunction = function(data, lev = NULL, model = NULL) {
c(Custom = mean(data$obs == data$pred))
}
),
metric = "Custom"
)
# only the metrics both models computed can be compared
expect_snapshot_warning(rs <- resamples(list(a = a, b = b)))
expect_length(rs$metrics, 0)
})
test_that("resamples collects the timings a model recorded", {
skip_on_cran()
reg <- engine_regression(60)
folds <- createFolds(reg$y, k = 3, returnTrain = TRUE)
set.seed(6210)
timed <- train(
y ~ .,
data = reg,
method = "lm",
trControl = trainControl(method = "cv", index = folds, timingSamps = 5)
)
set.seed(6210)
plain <- train(
y ~ .,
data = reg,
method = "knn",
tuneGrid = data.frame(k = 5),
trControl = trainControl(method = "cv", index = folds)
)
rs <- resamples(list(timed = timed, plain = plain))
# the prediction time is only recorded for the model that asked for it
expect_false(is.na(rs$timings["timed", "Prediction"]))
expect_true(is.na(rs$timings["plain", "Prediction"]))
})
# ------------------------------------------------------------------------------
# the argument checks on the resamples methods
test_that("the resamples methods each want a single metric", {
both <- c("RMSE", "Rsquared")
expect_snapshot(as.matrix(rs_fixture, metric = "Bogus"), error = TRUE)
expect_snapshot(prcomp(rs_fixture, metric = both), error = TRUE)
expect_snapshot(cluster(rs_fixture, metric = both), error = TRUE)
expect_snapshot(parallelplot(rs_fixture, metric = both), error = TRUE)
expect_snapshot(splom(rs_fixture, metric = both), error = TRUE)
# cluster() only knows how to handle a resamples object
expect_snapshot(cluster(1:10), error = TRUE)
# the PCA plot draws one thing at a time
pc <- prcomp(rs_fixture)
expect_snapshot(plot(pc, what = c("scree", "loadings")), error = TRUE)
})
test_that("splom.resamples needs two metrics for a metric panel", {
expect_snapshot(
splom(rs_fixture, variables = "metrics", metric = "RMSE"),
error = TRUE
)
})
test_that("the distribution plots can show a single metric", {
# with one metric there is nothing to condition on, so the formula has no
# conditioning term
for (f in list(densityplot, bwplot, dotplot)) {
drawn <- draw_trellis(f(rs_fixture, metric = "RMSE"))
expect_s3_class(drawn, "trellis")
expect_identical(dim(drawn), 1L)
}
d <- diff(rs_fixture)
for (f in list(densityplot, bwplot)) {
drawn <- draw_trellis(f(d, metric = "RMSE"))
expect_identical(dim(drawn), 1L)
}
# dotplot.diff.resamples takes the first metric when given several, and says so
expect_snapshot_warning(drawn <- dotplot(d, metric = c("RMSE", "Rsquared")))
expect_s3_class(draw_trellis(drawn), "trellis")
})
test_that("ggplot.resamples facets only when there is more than one metric", {
# the default is the first metric only, so there is nothing to facet by
plain <- ggplot2::ggplot_build(ggplot2::ggplot(rs_fixture))
expect_identical(nrow(plain$layout$layout), 1L)
# asking for both gives a panel each
faceted <- ggplot2::ggplot_build(
ggplot2::ggplot(rs_fixture, metric = c("RMSE", "Rsquared"))
)
expect_identical(nrow(faceted$layout$layout), 2L)
})
test_that("plot.prcomp.resamples draws the remaining plot types", {
pc <- prcomp(rs_fixture)
# `dims` is how many components to show: two gives a scatter of PC2 on PC1,
# more than two a scatterplot matrix
expect_s3_class(
draw_trellis(plot(pc, what = "components", dims = 2)),
"trellis"
)
expect_s3_class(
draw_trellis(plot(pc, what = "components", dims = 3)),
"trellis"
)
})
test_that("print.prcomp.resamples honours the digits argument", {
pc <- prcomp(rs_fixture)
# As above, the rotation itself is never snapshotted: two of the three models
# in the fixture are perfectly (negatively) correlated, so the third component
# is numerical noise whose values differ between BLAS implementations. Only
# the number of digits asked for is checked here.
out <- capture.output(print(pc, digits = 2))
expect_true(any(grepl("Rotation:", out, fixed = TRUE)))
loadings <- out[grepl("^Resample", out)]
expect_length(loadings, nrow(pc$rotation))
# two significant digits, so no loading is printed with more than four
# characters after its decimal point
expect_all_true(!grepl("[0-9]\\.[0-9]{5,}", loadings))
})
# ------------------------------------------------------------------------------
# resamples() on the other object types
test_that("resamples names models that were given without one", {
skip_on_cran()
reg <- engine_regression(60)
folds <- createFolds(reg$y, k = 3, returnTrain = TRUE)
ctrl <- trainControl(method = "cv", index = folds)
set.seed(1121)
a <- train(y ~ ., data = reg, method = "lm", trControl = ctrl)
set.seed(1121)
b <- train(
y ~ .,
data = reg,
method = "knn",
tuneGrid = data.frame(k = 5),
trControl = ctrl
)
# an empty name is filled in with a numbered one
rs <- resamples(list(a = a, b))
expect_identical(rs$models, c("a", "Model2"))
})
test_that("resamples warns for rfe and sbf models that kept every subset", {
skip_on_cran()
skip_if_not_installed("MASS")
cls <- engine_two_class(80)
x <- cls[, names(cls) != "Class"]
folds <- createFolds(cls$Class, k = 3, returnTrain = TRUE)
set.seed(3390)
rf <- rfe(
x,
cls$Class,
sizes = c(2, 4),
rfeControl = rfeControl(
functions = ldaFuncs,
method = "cv",
index = folds,
returnResamp = "all"
)
)
set.seed(3390)
sf <- suppressWarnings(sbf(
x,
cls$Class,
sbfControl = sbfControl(
functions = ldaSBF,
method = "cv",
index = folds,
returnResamp = "all"
)
))
# both say they are using the chosen subset rather than all of them
expect_snapshot(rs <- resamples(list(rfe = rf, sbf = sf)))
expect_s3_class(rs, "resamples")
})
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.