tests/testthat/test-simulate_wrappers.R

mock_simulate_custom <- function(...) {
  args <- list(...)

  list(
    outcome = attr(args$data_function, "outcome"),
    min_n = 80,
    perf_n = args$target_performance,
    target_performance = args$target_performance,
    summaries = list(mean_performance = c(`80` = args$target_performance)),
    results = matrix(
      args$target_performance,
      nrow = 1,
      dimnames = list("80", NULL)
    ),
    mlpwr_ds = NULL
  )
}

test_that("simulate_binary returns a pmsims object", {
  local_mocked_bindings(
    binary_tuning = function(...) c(mu_lp = 0, sigma_sq = 1, beta_signal = 0.3),
    simulate_custom = mock_simulate_custom,
    .package = "pmsims"
  )

  result <- simulate_binary(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    outcome_prevalence = 0.2,
    maximum_achievable_cstatistic = 0.75,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  expect_s3_class(result, "pmsims")
  expect_equal(result$outcome, "binary")
  expect_true(is.numeric(result$min_n))
  expect_gt(result$min_n, 0)
  expect_equal(result$target_performance, 0.9)
  expect_identical(result$signal_parameters, 10)
  expect_null(result$parameters)
  expect_identical(result$metric, "calibration_slope")
  expect_identical(result$metric_2, "auc")
  expect_identical(result$outcome_prevalence, 0.2)
  expect_identical(result$maximum_achievable_cstatistic, 0.75)
  expect_null(result$prevalence)
  expect_null(result$cstatistic)
})

test_that("simulate_continuous returns a pmsims object", {
  local_mocked_bindings(
    continuous_tuning = function(...) 0.25,
    simulate_custom = mock_simulate_custom,
    .package = "pmsims"
  )

  result <- simulate_continuous(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    maximum_achievable_rsquared = 0.5,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  expect_s3_class(result, "pmsims")
  expect_equal(result$outcome, "continuous")
  expect_true(is.numeric(result$min_n))
  expect_gt(result$min_n, 0)
  expect_equal(result$target_performance, 0.9)
  expect_identical(result$signal_parameters, 10)
  expect_null(result$parameters)
  expect_identical(result$maximum_achievable_rsquared, 0.5)
  expect_null(result$r2)
})

test_that("simulate_survival returns a pmsims object", {
  local_mocked_bindings(
    survival_tuning = function(...) {
      c(lambda_opt = 0.1, sigma_sq = 0.2, beta_signal = 0.3)
    },
    simulate_custom = mock_simulate_custom,
    .package = "pmsims"
  )

  result <- simulate_survival(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    maximum_achievable_cindex = 0.75,
    baseline_hazard = 0.01,
    censoring_rate = 0.3,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  expect_s3_class(result, "pmsims")
  expect_equal(result$outcome, "survival")
  expect_true(is.numeric(result$min_n))
  expect_gt(result$min_n, 0)
  expect_equal(result$target_performance, 0.9)
  expect_identical(result$signal_parameters, 10)
  expect_null(result$parameters)
  expect_identical(result$maximum_achievable_cindex, 0.75)
  expect_null(result$cstatistic)
})

test_that("wrapper dots are forwarded to simulate_custom", {
  binary_args <- NULL
  continuous_args <- NULL
  survival_args <- NULL

  local_mocked_bindings(
    binary_tuning = function(...) c(mu_lp = 0, sigma_sq = 1, beta_signal = 0.3),
    continuous_tuning = function(...) 0.25,
    survival_tuning = function(...) {
      c(lambda_opt = 0.1, sigma_sq = 0.2, beta_signal = 0.3)
    },
    simulate_custom = function(...) {
      args <- list(...)

      outcome <- attr(args$data_function, "outcome")
      if (identical(outcome, "binary")) {
        binary_args <<- args
      } else if (identical(outcome, "continuous")) {
        continuous_args <<- args
      } else if (identical(outcome, "survival")) {
        survival_args <<- args
      }

      mock_simulate_custom(...)
    },
    .package = "pmsims"
  )

  simulate_binary(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    outcome_prevalence = 0.2,
    maximum_achievable_cstatistic = 0.75,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance",
    progress = FALSE,
    min_sample_size = 50
  )

  simulate_continuous(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    maximum_achievable_rsquared = 0.5,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance",
    progress = FALSE,
    min_sample_size = 60
  )

  simulate_survival(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    maximum_achievable_cindex = 0.75,
    baseline_hazard = 0.01,
    censoring_rate = 0.3,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance",
    progress = FALSE,
    min_sample_size = 70
  )

  expect_false(binary_args$progress)
  expect_false(continuous_args$progress)
  expect_false(survival_args$progress)

  expect_identical(binary_args$min_sample_size, 50)
  expect_identical(continuous_args$min_sample_size, 60)
  expect_identical(survival_args$min_sample_size, 70)
})

test_that("wrappers store binary predictor prevalence on output", {
  local_mocked_bindings(
    binary_tuning = function(...) c(mu_lp = 0, sigma_sq = 1, beta_signal = 0.3),
    continuous_tuning = function(...) 0.25,
    survival_tuning = function(...) {
      c(lambda_opt = 0.1, sigma_sq = 0.2, beta_signal = 0.3)
    },
    simulate_custom = mock_simulate_custom,
    .package = "pmsims"
  )

  binary_result <- simulate_binary(
    signal_parameters = 10,
    noise_parameters = 0,
    data_control = list(
      predictor_distribution = "binary",
      binary_predictor_prevalence = 0.3
    ),
    outcome_prevalence = 0.2,
    maximum_achievable_cstatistic = 0.75,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  continuous_result <- simulate_continuous(
    signal_parameters = 10,
    noise_parameters = 0,
    data_control = list(
      predictor_distribution = "binary",
      binary_predictor_prevalence = 0.4
    ),
    maximum_achievable_rsquared = 0.5,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  survival_result <- simulate_survival(
    signal_parameters = 10,
    noise_parameters = 0,
    data_control = list(
      predictor_distribution = "binary",
      binary_predictor_prevalence = 0.25
    ),
    maximum_achievable_cindex = 0.75,
    baseline_hazard = 0.01,
    censoring_rate = 0.3,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  expect_identical(binary_result$binary_predictor_prevalence, 0.3)
  expect_identical(continuous_result$binary_predictor_prevalence, 0.4)
  expect_identical(survival_result$binary_predictor_prevalence, 0.25)
})

test_that("wrapper calibration slope bounds are enforced", {
  expect_error(
    simulate_binary(
      signal_parameters = 10,
      noise_parameters = 0,
      predictor_type = "continuous",
      outcome_prevalence = 0.2,
      maximum_achievable_cstatistic = 0.75,
      metric = "calibration_slope",
      target_performance = 0.7,
      n_reps_total = 40,
      mean_or_assurance = "assurance"
    ),
    "Requested target calibration slope is too low; check and try again.",
    fixed = TRUE
  )

  expect_error(
    simulate_continuous(
      signal_parameters = 10,
      noise_parameters = 0,
      predictor_type = "continuous",
      maximum_achievable_rsquared = 0.5,
      metric = "calibration_slope",
      target_performance = 1.3,
      n_reps_total = 40,
      mean_or_assurance = "assurance"
    ),
    "Requested target calibration slope is too high; check and try again.",
    fixed = TRUE
  )

  expect_error(
    simulate_survival(
      signal_parameters = 10,
      noise_parameters = 0,
      predictor_type = "continuous",
      maximum_achievable_cindex = 0.75,
      baseline_hazard = 0.01,
      censoring_rate = 0.3,
      metric = "calibration_slope",
      target_performance = 0.7,
      n_reps_total = 40,
      mean_or_assurance = "assurance"
    ),
    "Requested target calibration slope is too low; check and try again.",
    fixed = TRUE
  )
})

test_that("simulate_binary requires achievable AUC targets", {
  expect_error(
    simulate_binary(
      signal_parameters = 10,
      noise_parameters = 0,
      predictor_type = "continuous",
      outcome_prevalence = 0.2,
      maximum_achievable_cstatistic = 0.80,
      metric = "auc",
      target_performance = 0.9,
      n_reps_total = 40,
      mean_or_assurance = "assurance"
    ),
    "Requested target AUC must be less than the maximum achievable AUC because both are specified on the same metric scale.",
    fixed = TRUE
  )
})

test_that("simulate_binary errors when maximum achievable AUC equals target", {
  expect_error(
    simulate_binary(
      signal_parameters = 10,
      noise_parameters = 0,
      predictor_type = "continuous",
      outcome_prevalence = 0.2,
      maximum_achievable_cstatistic = 0.8,
      metric = "auc",
      target_performance = 0.8,
      n_reps_total = 40,
      mean_or_assurance = "assurance"
    ),
    "Requested target AUC must be less than the maximum achievable AUC because both are specified on the same metric scale.",
    fixed = TRUE
  )
})

test_that("machine learning wrappers search on the CSSE scale internally", {
  skip_if_not_installed("glmnet")

  captured <- NULL

  local_mocked_bindings(
    binary_tuning = function(...) c(mu_lp = 0, sigma_sq = 1, beta_signal = 0.3),
    simulate_custom = function(...) {
      captured <<- list(...)
      mock_simulate_custom(...)
    },
    .package = "pmsims"
  )

  result <- simulate_binary(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    outcome_prevalence = 0.2,
    maximum_achievable_cstatistic = 0.75,
    model = "ridge",
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  # The search ran against CSSE with a converted target ...
  expect_identical(attr(captured$metric_function, "metric"), "csse")
  expect_equal(captured$target_performance, -0.01)

  # ... but the user sees the calibration slope throughout.
  expect_identical(result$metric, "calibration_slope")
  expect_equal(result$target_performance, 0.9)
  expect_equal(result$perf_n, 0.9)
  expect_true(result$internal_csse)
  expect_equal(result$csse_direction, "below")
  expect_identical(result$metric_2, "auc")
})

test_that("non-machine-learning wrappers use the calibration slope directly", {
  captured <- NULL

  local_mocked_bindings(
    binary_tuning = function(...) c(mu_lp = 0, sigma_sq = 1, beta_signal = 0.3),
    simulate_custom = function(...) {
      captured <<- list(...)
      mock_simulate_custom(...)
    },
    .package = "pmsims"
  )

  result <- simulate_binary(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    outcome_prevalence = 0.2,
    maximum_achievable_cstatistic = 0.75,
    model = "glm",
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  expect_identical(
    attr(captured$metric_function, "metric"),
    "calibration_slope"
  )
  expect_equal(captured$target_performance, 0.9)
  expect_null(result$internal_csse)
})

test_that("an explicit CSSE request is passed through unchanged", {
  skip_if_not_installed("glmnet")

  captured <- NULL

  local_mocked_bindings(
    binary_tuning = function(...) c(mu_lp = 0, sigma_sq = 1, beta_signal = 0.3),
    simulate_custom = function(...) {
      captured <<- list(...)
      mock_simulate_custom(...)
    },
    .package = "pmsims"
  )

  result <- simulate_binary(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    outcome_prevalence = 0.2,
    maximum_achievable_cstatistic = 0.75,
    model = "ridge",
    metric = "csse",
    target_performance = -0.01,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  # The user's target is used as supplied, with no adjustment.
  expect_identical(attr(captured$metric_function, "metric"), "csse")
  expect_equal(captured$target_performance, -0.01)
  expect_identical(result$metric, "csse")
  expect_equal(result$target_performance, -0.01)
  expect_equal(result$perf_n, -0.01)
  expect_null(result$internal_csse)
})

test_that("simulate_continuous converts the calibration slope for ML models", {
  skip_if_not_installed("glmnet")

  captured <- NULL

  local_mocked_bindings(
    continuous_tuning = function(...) 0.25,
    simulate_custom = function(...) {
      captured <<- list(...)
      mock_simulate_custom(...)
    },
    .package = "pmsims"
  )

  result <- simulate_continuous(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    maximum_achievable_rsquared = 0.5,
    model = "ridge",
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  expect_identical(attr(captured$metric_function, "metric"), "csse")
  expect_equal(captured$target_performance, -0.01)
  expect_identical(result$metric, "calibration_slope")
  expect_equal(result$target_performance, 0.9)
  expect_equal(result$perf_n, 0.9)
  expect_true(result$internal_csse)
  expect_identical(result$metric_2, "r2")
})

test_that("simulate_survival converts the calibration slope for ML models", {
  skip_if_not_installed("glmnet")

  captured <- NULL

  local_mocked_bindings(
    survival_tuning = function(...) {
      c(lambda_opt = 0.1, sigma_sq = 0.2, beta_signal = 0.3)
    },
    simulate_custom = function(...) {
      captured <<- list(...)
      mock_simulate_custom(...)
    },
    .package = "pmsims"
  )

  # The unmocked second-metric evaluation fits a Cox ridge, which emits glmnet
  # tie-handling deprecation notices unrelated to this test.
  result <- suppressWarnings(simulate_survival(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    maximum_achievable_cindex = 0.7,
    baseline_hazard = 0.01,
    censoring_rate = 0.3,
    model = "ridge",
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  ))

  expect_identical(attr(captured$metric_function, "metric"), "csse")
  expect_equal(captured$target_performance, -0.01)
  expect_identical(result$metric, "calibration_slope")
  expect_equal(result$target_performance, 0.9)
  expect_equal(result$perf_n, 0.9)
  expect_true(result$internal_csse)
  expect_identical(result$metric_2, "cindex")
})

test_that("a calibration slope target above 1 is restored above 1", {
  skip_if_not_installed("glmnet")

  local_mocked_bindings(
    binary_tuning = function(...) c(mu_lp = 0, sigma_sq = 1, beta_signal = 0.3),
    simulate_custom = mock_simulate_custom,
    .package = "pmsims"
  )

  result <- simulate_binary(
    signal_parameters = 10,
    noise_parameters = 0,
    predictor_type = "continuous",
    outcome_prevalence = 0.2,
    maximum_achievable_cstatistic = 0.75,
    model = "ridge",
    metric = "calibration_slope",
    target_performance = 1.1,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  )

  expect_equal(result$csse_direction, "above")
  expect_equal(result$target_performance, 1.1)
  expect_equal(result$perf_n, 1.1)
})

test_that("resolve_data_control reports the configuration the generator uses", {
  # Requested and effective values coincide when nothing is substituted.
  plain <- resolve_data_control(list(predictor_distribution = "uniform"), 1)
  expect_identical(plain$effective_distribution, "uniform")
  expect_identical(plain$effective_nonlinear_strength, 0)

  # Complexity 4 swaps a left-at-default "normal" for the Friedman-canonical
  # "uniform", so the requested value would misdescribe the simulated data.
  friedman <- resolve_data_control(NULL, 4)
  expect_identical(friedman$predictor_distribution, "normal")
  expect_identical(friedman$effective_distribution, "uniform")

  # An explicit choice at complexity 4 is left alone.
  explicit <- resolve_data_control(list(predictor_distribution = "t"), 4)
  expect_identical(explicit$effective_distribution, "t")

  # An unset nonlinear_strength picks up the per-complexity default.
  expect_null(resolve_data_control(NULL, 2)$nonlinear_strength)
  expect_identical(
    resolve_data_control(NULL, 2)$effective_nonlinear_strength,
    0.2
  )
  expect_identical(
    resolve_data_control(NULL, 3)$effective_nonlinear_strength,
    0.3
  )

  # An explicit nonlinear_strength is passed through.
  explicit_ns <- resolve_data_control(list(nonlinear_strength = 0.45), 2)
  expect_identical(explicit_ns$effective_nonlinear_strength, 0.45)
})

test_that("wrappers record the effective data-generating configuration", {
  local_mocked_bindings(
    binary_tuning = function(...) c(mu_lp = 0, sigma_sq = 1, beta_signal = 0.3),
    simulate_custom = mock_simulate_custom,
    .package = "pmsims"
  )

  # The mocked tuner returns a fixed beta_signal rather than one tuned for the
  # Friedman signal, which makes the incidental metric_2 glm fit separate. That
  # is an artefact of the mock, not of the configuration being asserted here.
  result <- suppressWarnings(simulate_binary(
    signal_parameters = 5,
    noise_parameters = 0,
    complexity = 4,
    outcome_prevalence = 0.2,
    maximum_achievable_cstatistic = 0.75,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  ))

  expect_identical(result$complexity, 4)
  # Not the requested "normal": complexity 4 draws uniform predictors.
  expect_identical(result$predictor_distribution, "uniform")
  expect_identical(result$correlation, 0.3)

  quadratic <- suppressWarnings(simulate_binary(
    signal_parameters = 10,
    noise_parameters = 0,
    complexity = 2,
    outcome_prevalence = 0.2,
    maximum_achievable_cstatistic = 0.75,
    metric = "calibration_slope",
    target_performance = 0.9,
    n_reps_total = 40,
    mean_or_assurance = "assurance"
  ))

  # Resolved from the complexity-level default rather than left NULL.
  expect_identical(quadratic$nonlinear_strength, 0.2)
})

friedman_signal_error <- paste(
  "Friedman (complexity = 4) requires exactly 5 signal predictors.",
  "Please set the number of signal predictors to 5 and try again."
)

# Each wrapper with its required arguments filled in. Tuners and the engine are
# mocked, so the tests below only exercise the input checks.
friedman_wrapper_calls <- list(
  binary = function(...) {
    simulate_binary(
      ...,
      outcome_prevalence = 0.2,
      maximum_achievable_cstatistic = 0.75,
      target_performance = 0.9,
      n_reps_total = 40
    )
  },
  continuous = function(...) {
    simulate_continuous(
      ...,
      maximum_achievable_rsquared = 0.5,
      target_performance = 0.9,
      n_reps_total = 40
    )
  },
  survival = function(...) {
    simulate_survival(
      ...,
      maximum_achievable_cindex = 0.75,
      baseline_hazard = 0.01,
      censoring_rate = 0.3,
      target_performance = 0.9,
      n_reps_total = 40
    )
  }
)

local_friedman_mocks <- function(env = parent.frame()) {
  local_mocked_bindings(
    binary_tuning = function(...) c(mu_lp = 0, sigma_sq = 1, beta_signal = 0.3),
    continuous_tuning = function(...) 0.25,
    survival_tuning = function(...) {
      c(lambda_opt = 0.1, sigma_sq = 0.2, beta_signal = 0.3)
    },
    simulate_custom = mock_simulate_custom,
    .package = "pmsims",
    .env = env
  )
}

test_that("validate_complexity requires exactly 5 signal predictors at C4", {
  expect_true(validate_complexity(4, 5))
  expect_true(validate_complexity(4, 5L))

  rejected <- list(0, 1, 4, 6, 10, 4.5, NA, NA_real_, NULL, c(5, 5), "5")
  for (signal_parameters in rejected) {
    expect_error(
      validate_complexity(4, signal_parameters),
      friedman_signal_error,
      fixed = TRUE
    )
  }
})

test_that("validate_complexity does not restrict signal predictors at C1-C3", {
  for (complexity in 1:3) {
    for (signal_parameters in c(1, 3, 5, 6, 10)) {
      expect_true(validate_complexity(complexity, signal_parameters))
    }
  }
})

test_that("wrappers reject C4 without 5 signal predictors before any tuning", {
  # A tuner or engine call would surface as a different error message.
  stop_if_called <- function(...) stop("reached the simulation code")
  local_mocked_bindings(
    binary_tuning = stop_if_called,
    continuous_tuning = stop_if_called,
    survival_tuning = stop_if_called,
    simulate_custom = stop_if_called,
    .package = "pmsims"
  )

  for (wrapper in friedman_wrapper_calls) {
    for (signal_parameters in c(4, 6, 10)) {
      expect_error(
        wrapper(signal_parameters = signal_parameters, complexity = 4),
        friedman_signal_error,
        fixed = TRUE
      )
    }
  }
})

test_that("wrappers accept C4 with 5 signal predictors and any number of noise", {
  local_friedman_mocks()

  for (wrapper in friedman_wrapper_calls) {
    for (noise_parameters in c(0, 3, 100)) {
      # As above, the mocked tuners can make the incidental metric_2 fit warn.
      result <- suppressWarnings(wrapper(
        signal_parameters = 5,
        noise_parameters = noise_parameters,
        complexity = 4
      ))
      expect_s3_class(result, "pmsims")
      expect_identical(result$signal_parameters, 5)
      expect_identical(result$noise_parameters, noise_parameters)
    }
  }
})

test_that("wrappers leave signal predictors unrestricted at C1-C3", {
  local_friedman_mocks()

  for (wrapper in friedman_wrapper_calls) {
    for (complexity in 1:3) {
      for (signal_parameters in c(3, 10)) {
        result <- suppressWarnings(wrapper(
          signal_parameters = signal_parameters,
          complexity = complexity
        ))
        expect_identical(result$complexity, complexity)
        expect_identical(result$signal_parameters, signal_parameters)
      }
    }
  }
})

test_that("resolve_n_reps_total raises the mlpwr budget at small p only", {
  boost <- function(...) {
    args <- utils::modifyList(
      list(
        n_reps_total = 1000,
        signal_parameters = 5,
        model = "glm",
        method = "mlpwr"
      ),
      list(...),
      keep.null = TRUE
    )
    suppressMessages(do.call(resolve_n_reps_total, args))
  }

  expect_message(
    resolve_n_reps_total(1000, 5, "glm", method = "mlpwr"),
    "raised from 1000 to 2000"
  )
  expect_identical(boost(), 2000)

  # Not for the default curve engine, ML models, larger p or when switched off.
  expect_identical(boost(method = NULL), 1000)
  expect_identical(boost(method = "curve"), 1000)
  expect_identical(boost(model = "lasso"), 1000)
  expect_identical(boost(signal_parameters = 11), 1000)
  expect_identical(boost(auto = FALSE), 1000)

  # Only ever increases the budget.
  expect_identical(boost(n_reps_total = 5000), 5000)
})

Try the pmsims package in your browser

Any scripts or data that you put into this service are public.

pmsims documentation built on Nov. 15, 2026, 5:07 p.m.