tests/testthat/test-curve_engine.R

# Synthetic learning curve: replicate value = ceiling - scale / sqrt(n) plus
# training noise that shrinks with n and test-set noise. Fast: no model fits.
synthetic_curve <- function(
  ceiling = 0.95,
  scale = 2,
  train_sd = 0.05,
  test_sd = 0.01
) {
  data_function <- function(n) data.frame(n = n, z = stats::rnorm(1))
  model_function <- function(d) d
  attr(model_function, "model") <- "glm"
  metric_function <- function(test_data, fit, model) {
    ceiling -
      scale / sqrt(fit$n) +
      train_sd * sqrt(100 / fit$n) * fit$z +
      test_sd * test_data$z
  }
  attr(metric_function, "metric") <- "auc"
  list(
    data_function = data_function,
    model_function = model_function,
    metric_function = metric_function
  )
}

# True answer for the 20th percentile.
synthetic_answer <- function(
  target,
  ceiling = 0.95,
  scale = 2,
  train_sd = 0.05,
  test_sd = 0.01
) {
  f <- function(n) {
    ceiling -
      scale / sqrt(n) +
      stats::qnorm(0.2) * sqrt(train_sd^2 * 100 / n + test_sd^2) -
      target
  }
  stats::uniroot(f, c(5, 1e8))$root
}

run_curve <- function(sc, target, seed = 1, ...) {
  set.seed(seed)
  suppressWarnings(suppressMessages(simulate_custom(
    sc$data_function,
    sc$model_function,
    sc$metric_function,
    target_performance = target,
    mean_or_assurance = "assurance",
    test_n = 1,
    n_reps_total = 400,
    n_reps_per = 20,
    progress = FALSE,
    verify_reps = 50,
    method = "curve",
    ...
  )))
}

test_that("fit_learning_curve recovers a known curve", {
  n <- c(50, 100, 200, 400, 800, 1600)
  est <- 0.9 - 3 * n^(-0.7)
  f <- fit_learning_curve(n, est, rep(1e6, length(n)))
  expect_equal(f$a, 0.9, tolerance = 0.01)
  expect_equal(f$c, 0.7, tolerance = 0.05)
  expect_equal(curve_crossing(f, 0.85), (3 / 0.05)^(1 / 0.7), tolerance = 0.05)
  expect_identical(curve_crossing(f, 0.95), Inf)
  expect_null(fit_learning_curve(c(10, 10), c(0.5, 0.6), c(100, 100)))
  # The ceiling is capped at the metric's maximum.
  capped <- fit_learning_curve(
    n,
    -0.002 + 0.01 * log(n) / 10,
    rep(1, length(n)),
    a_max = 0
  )
  expect_lte(capped$a, 0)
})

test_that("the curve engine finds the answer from a start that is far too low or too high", {
  sc <- synthetic_curve()
  truth <- synthetic_answer(0.85)
  for (mult in c(1 / 16, 16)) {
    res <- run_curve(sc, 0.85, min_n_floor = 10, start_n = round(truth * mult))
    # "not_verified" is possible here by chance: the check flags about 2-5%
    # of answers that sit at or just below the target.
    expect_true(res$status %in% c("ok", "not_verified"))
    expect_lt(abs(log(res$min_n / truth)), log(1.25))
  }
})

test_that("a slowly converging but reachable target is not declared unreachable", {
  # The case that made the adaptive stage's plateau rule fire falsely: strict
  # target, large replicate noise, slow convergence.
  sc <- synthetic_curve(
    ceiling = 1,
    scale = 1.5,
    train_sd = 0.15,
    test_sd = 0.02
  )
  truth <- synthetic_answer(
    0.95,
    ceiling = 1,
    scale = 1.5,
    train_sd = 0.15,
    test_sd = 0.02
  )
  statuses <- vapply(1:5, function(s) run_curve(sc, 0.95, seed = s)$status, "")
  expect_false(any(statuses %in% c("unreachable", "not_bracketed")))
})

test_that("an unreachable target is reported, not answered", {
  sc <- synthetic_curve(ceiling = 0.85)
  res <- run_curve(sc, 0.9)
  expect_true(res$status %in% c("unreachable", "not_bracketed"))
  expect_true(is.na(res$min_n))
})

test_that("the curve engine respects user bounds and is reproducible", {
  sc <- synthetic_curve()
  a <- run_curve(
    sc,
    0.85,
    seed = 3,
    min_sample_size = 300,
    max_sample_size = 900
  )
  expect_lte(max(as.numeric(names(a$summaries$mean_performance))), 900)
  expect_gte(min(as.numeric(names(a$summaries$mean_performance))), 300)
  expect_lte(a$min_n, 900)
  b <- run_curve(
    sc,
    0.85,
    seed = 3,
    min_sample_size = 300,
    max_sample_size = 900
  )
  expect_identical(a$min_n, b$min_n)
})

# CSSE-scale curve (as used for ML models): slope ~ N(1 + bias, s / sqrt(n)),
# CSSE = -(1 - slope)^2. With bias 0 the 20th percentile reaches -0.01 at
# n = (qnorm(0.9) * s / 0.1)^2, about 1,200 (|1 - slope| is two-sided); with bias 0.12 its ceiling is
# -0.0144, so -0.01 is unreachable.
csse_curve <- function(bias = 0, s = 2.7) {
  data_function <- function(n) data.frame(n = n, z = stats::rnorm(1))
  model_function <- function(d) d
  attr(model_function, "model") <- "ridge"
  metric_function <- function(test_data, fit, model) {
    -(1 - (1 + bias + s / sqrt(fit$n) * fit$z + 0.003 * test_data$z))^2
  }
  attr(metric_function, "metric") <- "csse"
  list(
    data_function = data_function,
    model_function = model_function,
    metric_function = metric_function
  )
}

test_that("a CSSE-scale target is found from far below or above", {
  sc <- csse_curve()
  truth <- (stats::qnorm(0.9) * 2.7 / 0.1)^2
  for (mult in c(1 / 8, 8)) {
    res <- run_curve(sc, -0.01, seed = 2, start_n = round(truth * mult))
    expect_true(res$status %in% c("ok", "not_verified"))
    expect_lt(abs(log(res$min_n / truth)), log(1.3))
  }
})

test_that("an unreachable CSSE target stops without running to huge n", {
  sc <- csse_curve(bias = 0.12)
  res <- run_curve(sc, -0.01, seed = 2, start_n = 500)
  expect_true(res$status %in% c("unreachable", "not_bracketed"))
  expect_lte(res$diagnostics$bounds[2], 1e6)
})

test_that("an answer beyond the user's upper bound is reported, not clamped", {
  sc <- synthetic_curve()
  truth <- synthetic_answer(0.85)
  res <- run_curve(
    sc,
    0.85,
    seed = 1,
    min_sample_size = 100,
    max_sample_size = round(truth / 2)
  )
  expect_identical(res$status, "not_bracketed")
  expect_true(is.na(res$min_n))
})

test_that("a target just above the ceiling stops by max_n", {
  # Ceiling -(0.105)^2 = -0.011 against a target of -0.01. From the data such
  # a curve cannot be told apart from one that creeps up to the target far
  # out (ridge p5 reaches -0.01 only near n = 40,000), so the search may go
  # as far as max_n; it must then stop with a status rather than an answer.
  sc <- csse_curve(bias = 0.105)
  res <- run_curve(sc, -0.01, seed = 1, start_n = 500, max_n = 2e5)
  expect_true(res$status %in% c("unreachable", "not_bracketed"))
  expect_lte(res$diagnostics$bounds[2], 2e5)
})

test_that("a near-flat curve that meets the target only far out stops, not chases", {
  # Ceiling just above -0.01: the target is met only near n = 280,000.
  sc <- csse_curve(bias = 0.095)
  for (seed in 1:3) {
    res <- run_curve(sc, -0.01, seed = seed, start_n = 500)
    expect_identical(res$status, "not_bracketed")
    expect_identical(res$diagnostics$adaptive_stop_reason, "near_ceiling")
    expect_lte(res$diagnostics$bounds[2], 2e5)
  }
})

test_that("a CSSE curve whose ceiling is the target, or just short of it, stops early", {
  # bias 0.1: the criterion approaches the target from below and never meets
  # it; bias 0.103: the ceiling is 0.003 short in |slope - 1|. Either way the
  # search must stop with a status, well before max_n, not chase the target.
  for (bias in c(0.1, 0.103)) {
    for (seed in 1:3) {
      res <- run_curve(
        csse_curve(bias = bias),
        -0.01,
        seed = seed,
        start_n = 500
      )
      expect_true(res$status %in% c("unreachable", "not_bracketed"))
      expect_lte(res$diagnostics$bounds[2], 1e5)
    }
  }
})

test_that("the ridge p5 replicates (seed 1101) give no answer, and stop cheaply", {
  # Replicate values stored from core-bin-ridge-p5 on ed6f1be, seed 1101, where
  # the criterion sits within Monte Carlo error of the target from n = 28,000
  # on. That run returned 76,091 after simulating up to 125,810. Values are
  # drawn from the stored sample size nearest on the log scale. Before the
  # near-ceiling stop, seeds 1 and 4 here returned answers (20,538 and, after
  # simulating up to 150,528, 68,953); the others end below the target either
  # way.
  fx <- readRDS(test_path("fixtures", "ridge_p5_s1101_csse.rds"))
  data_function <- function(n) data.frame(n = n)
  model_function <- function(d) d
  attr(model_function, "model") <- "ridge"
  metric_function <- function(test_data, fit, model) {
    i <- which.min(abs(log(fx$n / fit$n)))
    sample(fx$values[[i]], 1L)
  }
  attr(metric_function, "metric") <- "csse"
  for (seed in 1:6) {
    set.seed(seed)
    res <- suppressWarnings(suppressMessages(simulate_custom(
      data_function,
      model_function,
      metric_function,
      target_performance = fx$target,
      mean_or_assurance = "assurance",
      test_n = 1,
      n_reps_total = 1000,
      n_reps_per = 20,
      progress = FALSE,
      verify_reps = 50,
      method = "curve",
      start_n = 588
    )))
    expect_identical(res$status, "not_bracketed")
    expect_lte(res$diagnostics$bounds[2], 1e5)
    if (identical(res$diagnostics$adaptive_stop_reason, "near_ceiling")) {
      # The message says what the last doubling showed, on the slope scale.
      expect_match(
        res$status_message,
        "From n = [0-9,]+ to n = [0-9,]+, the calibration slope.s distance from 1 went from [0-9.]+ to [0-9.]+"
      )
    }
  }
})

test_that("a pilot point that meets the target by chance is confirmed first", {
  # The first 20 replicates at n0 (well below the answer) meet the target by
  # construction; later ones follow the curve. The pilot must top n0 up and
  # carry on, not treat it as bracketing the target.
  sc <- synthetic_curve()
  truth <- synthetic_answer(0.85)
  n0 <- round(truth / 4)
  for (seed in 1:2) {
    calls_at_n0 <- 0L
    lucky <- sc$metric_function
    sc$metric_function <- function(test_data, fit, model) {
      if (round(fit$n) == n0) {
        calls_at_n0 <<- calls_at_n0 + 1L
        if (calls_at_n0 <= 20L) {
          return(0.99)
        }
      }
      lucky(test_data, fit, model)
    }
    attr(sc$metric_function, "metric") <- "auc"
    res <- run_curve(sc, 0.85, seed = seed, start_n = n0)
    expect_gte(calls_at_n0, 80L)
    expect_true(res$status %in% c("ok", "not_verified"))
    expect_lt(abs(log(res$min_n / truth)), log(1.3))
    sc$metric_function <- lucky
  }
})

test_that("a fitted shape at the end of the c grid is recorded", {
  grid <- curve_c_grid
  expect_identical(curve_shape_at_bound(list(c = grid[1])), "lower")
  expect_identical(curve_shape_at_bound(list(c = grid[length(grid)])), "upper")
  expect_identical(curve_shape_at_bound(list(c = grid[10])), NA_character_)
  expect_identical(curve_shape_at_bound(NULL), NA_character_)
  res <- run_curve(synthetic_curve(), 0.85, seed = 1)
  expect_true(is.character(res$diagnostics$shape_at_bound))
  expect_length(res$diagnostics$shape_at_bound, 1L)
})

test_that("failed replicates are told apart from legitimate fallback-valued ones", {
  data_function <- function(n) data.frame(n = n, z = stats::rnorm(1))
  model_function <- function(d) d
  attr(model_function, "model") <- "glm"
  metric_function <- function(test_data, fit, model) {
    if (fit$z < -1.5) {
      stop("did not converge")
    }
    0.95 - 2 / sqrt(fit$n) + 0.02 * fit$z
  }
  attr(metric_function, "metric") <- "auc"
  ev <- new_evaluator(
    data_function,
    model_function,
    metric_function,
    1,
    0.5,
    streams = new_simulation_streams(1L)
  )
  v <- ev$batch(100, 50, "search")
  expect_identical(sum(attr(v, "failed")), sum(v == 0.5))
  expect_gt(sum(attr(v, "failed")), 0)
})

test_that("a target near a slowly approached ceiling is not declared unreachable early", {
  # AUC-like curve with ceiling 0.80 and target 0.799 (reachable at ~20,000):
  # with few points the fitted ceiling collapsed onto the largest value seen.
  data_function <- function(n) data.frame(n = n, z = stats::rnorm(1))
  model_function <- function(d) d
  attr(model_function, "model") <- "glm"
  metric_function <- function(test_data, fit, model) {
    0.80 - 2 / fit$n^0.75 + 0.03 * sqrt(100 / fit$n) * fit$z
  }
  attr(metric_function, "metric") <- "auc"
  sc <- list(
    data_function = data_function,
    model_function = model_function,
    metric_function = metric_function
  )
  statuses <- vapply(
    1:3,
    function(s) run_curve(sc, 0.799, seed = s, start_n = 200)$status,
    ""
  )
  expect_false(any(statuses == "unreachable"))
})

test_that("a target near the ceiling is flagged; an ordinary one is not", {
  data_function <- function(n) data.frame(n = n, z = stats::rnorm(1))
  model_function <- function(d) d
  attr(model_function, "model") <- "glm"
  metric_function <- function(test_data, fit, model) {
    0.80 - 2 / fit$n^0.75 + 0.03 * sqrt(100 / fit$n) * fit$z
  }
  attr(metric_function, "metric") <- "auc"
  near <- run_curve(
    list(
      data_function = data_function,
      model_function = model_function,
      metric_function = metric_function
    ),
    0.799,
    seed = 2,
    start_n = 2000
  )
  # Target 0.001 below the ceiling: the answer is flagged as near the
  # ceiling or as poorly determined (its interval spans more than 2x).
  if (identical(near$status, "ok")) {
    expect_true(
      near$diagnostics$near_ceiling || near$diagnostics$poorly_determined
    )
  }
  plain <- run_curve(synthetic_curve(), 0.85, seed = 1)
  expect_false(plain$diagnostics$near_ceiling)
})

test_that("max_n defaults to 200,000 for random forests and xgboost", {
  expect_identical(default_max_n("rf"), 2e5)
  expect_identical(default_max_n("xgboost"), 2e5)
  expect_identical(default_max_n("glm"), 1e6)
  expect_identical(default_max_n(NULL), 1e6)
})

test_that("the quantile SE factor is estimated from the data", {
  normal <- run_curve(synthetic_curve(), 0.85, seed = 1)
  skewed <- run_curve(csse_curve(), -0.01, seed = 1, start_n = 300)
  # About 1.43 for a normal 20th percentile; larger for skewed CSSE values.
  expect_lt(abs(normal$diagnostics$curve$se_factor - 1.43), 0.3)
  expect_gt(
    skewed$diagnostics$curve$se_factor,
    normal$diagnostics$curve$se_factor
  )
})

test_that("an ordinary answer passes the shape-free cross-check", {
  res <- run_curve(synthetic_curve(), 0.85, seed = 2)
  expect_true(is.finite(res$diagnostics$crosscheck_n))
  expect_false(res$diagnostics$crosscheck_disagrees)
  expect_false(res$diagnostics$poorly_determined)
  expect_gt(res$diagnostics$gain_per_doubling, 0)
})

test_that("weighted_isotonic and isotonic_crossing behave", {
  expect_equal(
    weighted_isotonic(c(1, 3, 2, 4), c(1, 3, 1, 1)),
    c(1, 2.75, 2.75, 4)
  )
  expect_equal(
    isotonic_crossing(c(100, 200, 400), c(0.7, 0.8, 0.9), c(1, 1, 1), 0.85),
    exp(mean(log(c(200, 400))))
  )
  expect_identical(
    isotonic_crossing(c(100, 200), c(0.5, 0.6), c(1, 1), 0.9),
    Inf
  )
})

test_that("curve results plot, including stopped searches", {
  ok <- run_curve(synthetic_curve(), 0.85, seed = 1)
  d <- plot(ok, plot = FALSE)
  expect_s3_class(d$observed_data, "data.frame")
  expect_true(all(c("n", "y", "lo", "hi", "reps") %in% names(d$observed_data)))
  expect_s3_class(d$fitted_curve, "data.frame")
  pdf(NULL)
  on.exit(dev.off(), add = TRUE)
  expect_s3_class(plot(ok), "ggplot")
  stopped <- run_curve(csse_curve(bias = 0.12), -0.01, seed = 2, start_n = 500)
  expect_true(stopped$status %in% c("unreachable", "not_bracketed"))
  expect_s3_class(plot(stopped), "ggplot")
})

test_that("messages for CSSE targets use the calibration slope scale", {
  stopped <- run_curve(csse_curve(bias = 0.12), -0.01, seed = 2, start_n = 500)
  expect_match(stopped$status_message, "calibration slope within")
  expect_false(grepl("-0.01", stopped$status_message, fixed = TRUE))
})

test_that("results are identical on one core and several", {
  skip_on_os("windows")
  sc <- synthetic_curve()
  one <- run_curve(sc, 0.85, seed = 4)
  two <- run_curve(sc, 0.85, seed = 4, cores = 2)
  expect_identical(one$min_n, two$min_n)
  expect_identical(one$verification$performance, two$verification$performance)
})

test_that("the live plot redraws after each batch and does not change results", {
  draws <- 0L
  local_mocked_bindings(plot_learning_curve = function(x, ...) {
    draws <<- draws + 1L
    expect_true(length(x$data) >= 1L)
    invisible(NULL)
  })
  quiet <- run_curve(synthetic_curve(), 0.85, seed = 5)
  old <- options(pmsims.live_plot_force = TRUE)
  on.exit(options(old), add = TRUE)
  live <- run_curve(synthetic_curve(), 0.85, seed = 5, live_plot = TRUE)
  expect_gt(draws, 5L)
  expect_identical(live$min_n, quiet$min_n)
})

test_that("the live plot draws for real on a graphics device", {
  old <- options(pmsims.live_plot_force = TRUE)
  on.exit(options(old), add = TRUE)
  pdf(NULL)
  on.exit(dev.off(), add = TRUE)
  expect_no_error(run_curve(
    csse_curve(),
    -0.01,
    seed = 3,
    start_n = 300,
    live_plot = TRUE
  ))
})

test_that("too many failed replicates stop the search with a clear status", {
  # About 40% of replicates fail at every n: each point is left out of the fit
  # (20% or more failed), but the evaluator's own check needs half.
  sc <- synthetic_curve()
  metric <- sc$metric_function
  sc$metric_function <- function(test_data, fit, model) {
    if (fit$z < stats::qnorm(0.4)) {
      stop("did not converge")
    }
    metric(test_data, fit, model)
  }
  attr(sc$metric_function, "metric") <- "auc"
  res <- run_curve(sc, 0.85)
  expect_identical(res$status, "replicates_failed")
  expect_match(res$status_message, "did not converge.*20% or more failed")
  expect_lt(res$diagnostics$total_replicates, 100)
})

test_that("a search limit is simulated, not snapped onto a nearby point", {
  # From 64, halving reaches 16; the next request (8) is clamped to the floor
  # of 15, within 7% of 16, and must still be simulated at 15.
  low <- run_curve(synthetic_curve(), 0.2, min_n_floor = 15, start_n = 64)
  expect_identical(low$diagnostics$bounds[1], 15)
  expect_identical(low$diagnostics$adaptive_stop_reason, "lower_bound")
  # Doubling from 130 reaches 1,040; the next request is clamped to max_n.
  high <- run_curve(
    synthetic_curve(ceiling = 0.85),
    0.9,
    start_n = 130,
    max_n = 1100
  )
  expect_identical(high$diagnostics$bounds[2], 1100)
  expect_identical(high$diagnostics$adaptive_stop_reason, "max_n_reached")
})

test_that("with a single sample size, a met target is returned and verified", {
  res <- run_curve(
    synthetic_curve(),
    0.75,
    min_sample_size = 500,
    max_sample_size = 500
  )
  expect_identical(res$min_n, 500)
  expect_identical(res$status, "ok")
  expect_false(is.null(res$verification))
})

test_that("a curve that is flat above the target gives no spurious cross-check", {
  # Decreasing values: the fitted curve is flat (b = 0) and crosses the target
  # at n = 0, so the answer is the smallest sample size allowed.
  sc <- synthetic_curve(scale = -2)
  res <- run_curve(sc, 0.7, min_n_floor = 10, start_n = 40)
  expect_identical(res$min_n, 10)
  expect_false(res$diagnostics$crosscheck_disagrees)
  expect_true(is.finite(res$diagnostics$gain_per_doubling))
})

test_that("the live plot shows CSSE on the slope scale only as plot() does", {
  states <- list()
  local_mocked_bindings(plot_learning_curve = function(x, ...) {
    states[[length(states) + 1L]] <<- x
    invisible(NULL)
  })
  old <- options(pmsims.live_plot_force = TRUE)
  on.exit(options(old), add = TRUE)
  # A calibration slope target above 1, searched on the CSSE scale.
  sc <- csse_curve(bias = 0.2)
  plan <- plan_internal_csse("calibration_slope", "ridge", 1.1)
  sc$metric_function <- describe_as_calibration_slope(sc$metric_function, plan)
  run_curve(
    sc,
    plan$target_performance,
    seed = 1,
    start_n = 300,
    live_plot = TRUE
  )
  last <- states[[length(states)]]
  expect_true(last$internal_csse)
  expect_identical(last$metric, "calibration_slope")
  expect_equal(last$target_performance, 1.1)
  # A CSSE target asked for directly stays on the CSSE scale.
  states <- list()
  run_curve(csse_curve(), -0.01, seed = 1, start_n = 300, live_plot = TRUE)
  last <- states[[length(states)]]
  expect_false(last$internal_csse)
  expect_identical(last$metric, "csse")
  expect_identical(last$target_performance, -0.01)
})

test_that("the curve plot's x-axis has labels across narrow ranges", {
  # The ggplot2 default gives no label above 1,000 for this range.
  expect_identical(log_axis_breaks(c(654, 1884)), c(800, 1000, 1200, 1500))
  expect_identical(
    log_axis_breaks(c(800, 2e5)),
    c(1000, 3000, 10000, 30000, 1e5)
  )
  for (r in list(c(60, 140), c(255, 1469), c(39, 1368), c(10, 1e6))) {
    b <- log_axis_breaks(r)
    expect_gte(length(b), 3L)
    expect_true(all(b >= r[1] & b <= r[2]))
  }
})

Try the pmsims package in your browser

Any scripts or data that you put into this service are public.

pmsims documentation built on Nov. 15, 2026, 5:07 p.m.