R/data_generators.R

Defines functions nonlinear_reference nonlinear_raw generate_linear_predictor generate_predictors draw_correlated_predictors normal_to_family resolve_family draw_predictors resolve_nonlinear_strength update_arguments generate_survival_data generate_binary_data generate_continuous_data default_data_generators

Documented in default_data_generators generate_binary_data generate_continuous_data generate_linear_predictor generate_predictors generate_survival_data

# =============================================================================
# Data-generating functions for simulation studies
#
# Supports continuous, binary, and survival outcomes across four complexity
# levels. All generators share two internal workhorses:
#   - generate_predictors()       : builds the (possibly correlated) X matrix
#   - generate_linear_predictor() : constructs lp from X
#
# =============================================================================
# INTERFACE SUMMARY
# =============================================================================
#
# Core parameters (all apply uniformly across every complexity setting):
#
#   n_signal_parameters : number of signal predictors
#   noise_parameters    : number of noise predictors
#   beta_signal         : base effect size for signal predictors (overall scale)
#   predictor_type      : "continuous" (default) or "binary".
#                         When "binary", ALL predictors are Bernoulli and
#                         binary_prevalence must be supplied; distribution is
#                         ignored. When "continuous", distribution governs the
#                         family.
#   binary_prevalence   : Bernoulli probability for ALL predictors.
#                         Required (and used) when predictor_type = "binary".
#                         Ignored when predictor_type = "continuous".
#                         Default = 0 (no binary predictors).
#   correlation         : common pairwise correlation (default = 0.3; 0 = none)
#   distribution        : single global distribution family for ALL continuous
#                         predictors (default = "normal"; ignored when
#                         predictor_type = "binary"). Supported families:
#                           "normal", "uniform", "exponential",
#                           "lognormal", "t", "laplace".
#                         For complexity 4 the Friedman canonical default is
#                         "uniform", applied automatically when distribution is
#                         left at "normal".
#
# Complexity and nonlinear strength:
#
#   complexity          : integer 1-4, controls the functional form of lp
#                           1 = Linear
#                           2 = Quadratic
#                           3 = Quadratic + Interaction
#                           4 = Friedman (1991)
#
#   nonlinear_strength  : for C2/C3 only, the FRACTION of signal variance carried
#                         by the nonlinear (linearly-inaccessible) component, in
#                         [0, 1). Defaults per complexity:
#                           C1 -> 0.0   (pure linear)
#                           C2 -> 0.2   (20% nonlinear)
#                           C3 -> 0.3   (30% nonlinear)
#                           C4 -> 0.0   (canonical Friedman; argument ignored)
#                         Supplied as a variance fraction f and converted
#                         internally to the nonlinear-to-linear SD ratio
#                           kappa = sqrt(f / (1 - f))
#                         so the nonlinear variance fraction in the population
#                         is f, up to the error in the reference estimates
#                         (see generate_linear_predictor()). Larger f hides
#                         more of the signal from a linear-in-X model, making
#                         C2/C3 harder for AUC / C-index and requiring more
#                         sample size. C1 and C4 ignore this argument.
#
# Supported distribution families (continuous predictors):
#   "normal"      : mean = 0, sd = 1
#   "uniform"     : min = 0, max = 1
#   "exponential" : rate = 1
#   "lognormal"   : meanlog = 0, sdlog = 1
#   "t"           : df = 5
#   "laplace"     : location = 0, scale = 1
#
# Outcome-specific arguments:
#   continuous : intercept (default 0)
#   binary     : mu_lp (log-odds intercept, default 0), baseline_prob (doc only)
#   survival   : baseline_hazard, censoring_rate, intercept (default 0)
# =============================================================================

# =============================================================================
# Complexity-level defaults
# =============================================================================

# Default distribution per complexity level (applies when distribution = "normal"
# and binary_prevalence = 0)
COMPLEXITY_DIST_DEFAULTS <- c(
  "1" = "normal",
  "2" = "normal",
  "3" = "normal",
  "4" = "uniform" # Friedman canonical
)

# Nonlinear strength per complexity for C2/C3: the FRACTION of signal variance
# carried by the nonlinear (linearly-inaccessible) component. 0 = pure linear.
# Internally converted to the nonlinear-to-linear SD ratio kappa = sqrt(f/(1-f))
# so the nonlinear variance fraction in the population is f, up to the error in
# the reference estimates. Larger f hides more signal from a linear-in-X model
# => harder for AUC / C-index and more sample size required. C1 and C4 ignore
# this (C1 is pure linear; C4 uses the canonical Friedman form).
COMPLEXITY_NONLINEAR_STRENGTH_DEFAULTS <- c(
  "1" = 0.0,
  "2" = 0.2,
  "3" = 0.3,
  "4" = 0.0
)

# =============================================================================
# Entry point
# =============================================================================

#' Create default data generating functions
#'
#' @param opts A list with two elements:
#'   \describe{
#'     \item{\code{type}}{Outcome type: \code{"continuous"}, \code{"binary"},
#'       or \code{"survival"}.}
#'     \item{\code{args}}{Named list of arguments to pre-set on the
#'       corresponding generator function.}
#'   }
#' @return A partially-applied generator function whose formals have been set
#'   to the values in \code{opts$args}.
#' @keywords internal
default_data_generators <- function(opts) {
  type <- opts$type
  f <- switch(
    type,
    continuous = generate_continuous_data,
    binary = generate_binary_data,
    survival = generate_survival_data,
    stop(sprintf(
      'opts$type must be "continuous", "binary", or "survival". Got: "%s".',
      type
    ))
  )
  return(update_arguments(f, opts))
}

# =============================================================================
# Outcome-level generators
# =============================================================================

#' Simulate continuous outcome data
#'
#' @param n                  Sample size.
#' @param n_signal_parameters Number of signal predictors. These occupy the
#'   first \code{n_signal_parameters} columns (\code{x1} ... \code{x_S}).
#' @param noise_parameters    Number of noise predictors (zero coefficient).
#' @param beta_signal        Base effect size / overall scale of the signal.
#' @param complexity         Integer 1-4 specifying the functional form of the
#'   linear predictor:
#'   \enumerate{
#'     \item \strong{Linear} — \eqn{lp = \alpha + \beta\sum_j x_j}.
#'     \item \strong{Quadratic} — linear + a quadratic nonlinear component.
#'     \item \strong{Quadratic + Interaction} — linear + quadratic and pairwise
#'       interaction nonlinear component.
#'     \item \strong{Friedman} — canonical Friedman (1991) benchmark.
#'   }
#' @param nonlinear_strength Fraction of signal variance carried by the
#'   nonlinear component (C2/C3 only), in [0, 1). When \code{NULL} (default), the
#'   complexity-level default is used: C1 = 0, C2 = 0.2, C3 = 0.3, C4 = 0.
#'   Ignored for C1 (pure linear) and C4 (canonical Friedman).
#' @param predictor_type     Type of predictors: \code{"continuous"} (default)
#'   or \code{"binary"}. When \code{"binary"}, all predictors are drawn as
#'   Bernoulli(\code{binary_prevalence}); \code{binary_prevalence} must be in
#'   (0, 1] and \code{distribution} is ignored.
#' @param binary_prevalence  Scalar in (0, 1]. Bernoulli probability applied to
#'   all predictors when \code{predictor_type = "binary"}. Default = 0.
#' @param correlation        Scalar in \eqn{[-1, 1]}. Common pairwise correlation
#'   applied via a Gaussian copula: \code{correlation} is the correlation of
#'   the underlying normal variables, which equals the Pearson correlation of
#'   the predictors only when they are normal. Default = 0.3. Set to 0 for
#'   independence.
#' @param distribution       Distribution family for \emph{all} continuous
#'   predictors. Default = \code{"normal"}. For complexity 4, if left at
#'   \code{"normal"} the framework uses \code{"uniform"} (Friedman canonical).
#' @param intercept          Scalar intercept added to the linear predictor.
#'   Default = 0.
#'
#' @return A data frame with columns \code{y}, \code{x1}, \code{x2}, ...
#'
#' @references
#' Friedman, J. H. (1991). Multivariate adaptive regression splines.
#'   \emph{The Annals of Statistics}, 19(1), 1-67. \doi{10.1214/aos/1176347963}
#'
#' @keywords internal
generate_continuous_data <- function(
  n,
  n_signal_parameters,
  noise_parameters,
  beta_signal,
  complexity = 1,
  nonlinear_strength = NULL,
  predictor_type = "continuous",
  binary_prevalence = 0,
  correlation = 0.3,
  distribution = "normal",
  intercept = 0
) {
  X <- generate_predictors(
    n,
    n_signal_parameters,
    noise_parameters,
    complexity,
    predictor_type,
    binary_prevalence,
    correlation,
    distribution
  )
  lp <- generate_linear_predictor(
    X,
    n_signal_parameters,
    noise_parameters,
    intercept,
    beta_signal,
    complexity,
    nonlinear_strength,
    predictor_type = predictor_type,
    binary_prevalence = binary_prevalence,
    correlation = correlation,
    distribution = distribution
  )
  y <- stats::rnorm(n, lp, 1)
  return(as.data.frame(cbind(y, X)))
}

#' Simulate binary outcome data
#'
#' @inheritParams generate_continuous_data
#' @param mu_lp         Intercept on the log-odds scale. Default = 0.
#' @param baseline_prob Nominal baseline event probability (documentation only;
#'   the realised probability is determined by \code{mu_lp}).
#'
#' @return A data frame with columns \code{y} (0/1), \code{x1}, \code{x2}, ...
#' @keywords internal
generate_binary_data <- function(
  n,
  n_signal_parameters,
  noise_parameters,
  beta_signal,
  complexity = 1,
  nonlinear_strength = NULL,
  predictor_type = "continuous",
  binary_prevalence = 0,
  correlation = 0.3,
  distribution = "normal",
  mu_lp = 0,
  baseline_prob = 0.5
) {
  X <- generate_predictors(
    n,
    n_signal_parameters,
    noise_parameters,
    complexity,
    predictor_type,
    binary_prevalence,
    correlation,
    distribution
  )
  lp <- generate_linear_predictor(
    X,
    n_signal_parameters,
    noise_parameters,
    intercept = mu_lp,
    beta_signal,
    complexity,
    nonlinear_strength,
    predictor_type = predictor_type,
    binary_prevalence = binary_prevalence,
    correlation = correlation,
    distribution = distribution
  )
  y <- stats::rbinom(n, 1, stats::plogis(lp))
  return(as.data.frame(cbind(y, X)))
}

#' Simulate survival outcome data
#'
#' @inheritParams generate_continuous_data
#' @param baseline_hazard Baseline hazard rate (exponential survival model).
#' @param censoring_rate  Administrative censoring proportion in (0, 1).
#'
#' @return A data frame with columns \code{time}, \code{event} (0 = censored,
#'   1 = event), \code{x1}, \code{x2}, ...
#' @keywords internal
generate_survival_data <- function(
  n,
  n_signal_parameters,
  noise_parameters,
  beta_signal,
  baseline_hazard,
  censoring_rate,
  complexity = 1,
  nonlinear_strength = NULL,
  predictor_type = "continuous",
  binary_prevalence = 0,
  correlation = 0.3,
  distribution = "normal",
  intercept = 0
) {
  X <- generate_predictors(
    n,
    n_signal_parameters,
    noise_parameters,
    complexity,
    predictor_type,
    binary_prevalence,
    correlation,
    distribution
  )
  lp <- generate_linear_predictor(
    X,
    n_signal_parameters,
    noise_parameters,
    intercept,
    beta_signal,
    complexity,
    nonlinear_strength,
    predictor_type = predictor_type,
    binary_prevalence = binary_prevalence,
    correlation = correlation,
    distribution = distribution
  )

  event_time <- stats::rexp(n, rate = baseline_hazard * exp(lp))
  T_observe <- stats::quantile(event_time, 1 - censoring_rate)
  censor_time <- rep(T_observe, n)
  event <- as.numeric(event_time <= censor_time)
  survival_time <- pmin(event_time, censor_time)

  return(data.frame(time = survival_time, event = event, X))
}

# =============================================================================
# Internal helpers
# =============================================================================

# -----------------------------------------------------------------------------
# update_arguments
# Bakes opts$args values into the formals of fn.
# -----------------------------------------------------------------------------
update_arguments <- function(fn, opts) {
  for (key in names(opts$args)) {
    if (key %in% names(formals(fn))) {
      formals(fn)[[key]] <- opts$args[[key]]
    }
  }
  attr(fn, "outcome") <- opts$type
  return(fn)
}

# -----------------------------------------------------------------------------
# resolve_nonlinear_strength
#
# Returns the nonlinear-strength fraction to use, applying the complexity-level
# default when the user has not supplied an explicit value (NULL), and
# validating that it lies in [0, 1).
# -----------------------------------------------------------------------------
resolve_nonlinear_strength <- function(nonlinear_strength, complexity) {
  if (is.null(nonlinear_strength)) {
    nonlinear_strength <-
      COMPLEXITY_NONLINEAR_STRENGTH_DEFAULTS[[as.character(complexity)]]
  }
  if (
    !is.numeric(nonlinear_strength) ||
      length(nonlinear_strength) != 1 ||
      !is.finite(nonlinear_strength) ||
      nonlinear_strength < 0 ||
      nonlinear_strength >= 1
  ) {
    stop(
      "nonlinear_strength must be a single number in [0, 1). Got: ",
      format(nonlinear_strength)
    )
  }
  return(nonlinear_strength)
}

# -----------------------------------------------------------------------------
# draw_predictors
#
# Draws an n x p matrix of i.i.d. observations from the chosen distribution
# family using its canonical default parameters. All p predictors share the
# same family (the distribution argument is global).
#
# Supported families and their canonical defaults:
#   normal      : mean = 0, sd = 1
#   uniform     : min = 0, max = 1
#   exponential : rate = 1
#   lognormal   : meanlog = 0, sdlog = 1
#   t           : df = 5
#   laplace     : location = 0, scale = 1  (via quantile transform)
#   binary      : prop = binary_prevalence  (passed in as argument)
# -----------------------------------------------------------------------------
draw_predictors <- function(n, p, family, binary_prevalence = 0) {
  vals <- switch(
    family,

    normal = stats::rnorm(n * p),

    uniform = stats::runif(n * p),

    exponential = stats::rexp(n * p),

    lognormal = stats::rlnorm(n * p),

    t = stats::rt(n * p, df = 5),

    laplace = {
      u <- stats::runif(n * p, -0.5, 0.5)
      -sign(u) * log(1 - 2 * abs(u)) # standard Laplace (loc=0, scale=1)
    },

    binary = stats::rbinom(n * p, 1, binary_prevalence),

    stop(sprintf(
      paste0(
        'Unknown distribution "%s". ',
        'Supported: "normal", "uniform", "exponential", ',
        '"lognormal", "t", "laplace".'
      ),
      family
    ))
  )

  matrix(vals, nrow = n, ncol = p)
}

# -----------------------------------------------------------------------------
# resolve_family
#
# Determines the actual distribution family to draw from. Resolution order:
#   1. predictor_type == "binary"  -> "binary"  (primary switch; overrides all)
#   2. complexity == 4 AND distribution == "normal"
#                                  -> "uniform" (C4 Friedman canonical default)
#   3. Otherwise                   -> distribution as supplied
# -----------------------------------------------------------------------------
resolve_family <- function(
  complexity,
  predictor_type,
  distribution,
  binary_prevalence
) {
  if (predictor_type == "binary") {
    return("binary")
  }
  if (complexity == 4 && distribution == "normal") {
    return("uniform")
  }
  return(distribution)
}

# -----------------------------------------------------------------------------
# normal_to_family
#
# Transforms a matrix of standard normal values Z, column by column, to the
# chosen family using the quantile transform F^{-1}(pnorm(Z)). The families and
# their parameters match draw_predictors(). Where the family allows, the
# transform is computed from the normal tail probabilities directly so that it
# stays finite and precise for extreme Z (pnorm(z) rounds to 1 for z > 8.3).
# -----------------------------------------------------------------------------
normal_to_family <- function(Z, family, binary_prevalence = 0) {
  vals <- switch(
    family,

    normal = Z,

    uniform = stats::pnorm(Z),

    # -log(1 - pnorm(Z)), via the upper tail.
    exponential = -stats::pnorm(Z, lower.tail = FALSE, log.p = TRUE),

    lognormal = exp(Z),

    # Symmetric, so work from the lower tail of -|Z| and restore the sign.
    t = -sign(Z) * stats::qt(stats::pnorm(-abs(Z)), df = 5),

    # Standard Laplace quantile, again from the lower tail of -|Z|.
    laplace = -sign(Z) *
      (log(2) + stats::pnorm(-abs(Z), log.p = TRUE)),

    # 1 when pnorm(Z) > 1 - prevalence, i.e. Z > qnorm(1 - prevalence).
    binary = (Z > stats::qnorm(1 - binary_prevalence)) * 1,

    stop(sprintf(
      paste0(
        'Unknown distribution "%s". ',
        'Supported: "normal", "uniform", "exponential", ',
        '"lognormal", "t", "laplace".'
      ),
      family
    ))
  )

  matrix(vals, nrow = nrow(Z), ncol = ncol(Z))
}

# -----------------------------------------------------------------------------
# draw_correlated_predictors
#
# Draws an n x p matrix from the chosen family with a common pairwise
# correlation rho, via a Gaussian copula: equicorrelated standard normals
# (Z %*% chol(R)) are transformed to the family with normal_to_family(). The
# marginal distribution of every column is exactly the family's.
#
# rho is the correlation of the underlying normals. For normal predictors it is
# also the Pearson correlation of the result. For other families the Pearson
# correlation is lower (e.g. about 0.18 for binary predictors with prevalence
# 0.3 when rho = 0.3), while the rank dependence is the same for every family.
# -----------------------------------------------------------------------------
draw_correlated_predictors <- function(
  n,
  p,
  family,
  rho,
  binary_prevalence = 0
) {
  min_rho <- if (p > 1) -1 / (p - 1) else -1
  if (rho < min_rho) {
    warning(sprintf(
      paste0(
        "correlation = %.3f may produce a non-PSD equicorrelation matrix ",
        "for p = %d predictors (minimum valid rho = %.4f)."
      ),
      rho,
      p,
      min_rho
    ))
  }

  cor_mat <- matrix(rho, p, p)
  diag(cor_mat) <- 1

  eigs <- eigen(cor_mat, symmetric = TRUE, only.values = TRUE)$values
  if (any(eigs < -1e-8)) {
    stop(sprintf(
      "Equicorrelation matrix with rho = %.3f is not PSD for p = %d.",
      rho,
      p
    ))
  }

  # Rows of Z %*% U, with U = chol(R) upper triangular, have covariance
  # t(U) %*% U = R. For an equicorrelation matrix each row of U is constant to
  # the right of the diagonal (U[i, j] = U[i, p] for j > i), so column j of
  # Z %*% U is a running sum over the earlier columns of Z plus U[j, j] times
  # its own: O(n p) work instead of the matrix product's O(n p^2), which for
  # 80 predictors was most of the time spent drawing a 30,000-row test set.
  # The terms are added in the same order as in the matrix product, but
  # chol() can return rows that differ in the last bit, so the result is
  # equal to the matrix product to rounding error (bit-identical when the
  # rows are exactly constant, as for up to about 10 predictors).
  U <- chol(cor_mat)
  Z <- matrix(stats::rnorm(n * p), nrow = n, ncol = p)
  running <- numeric(n)
  for (j in seq_len(p)) {
    z_j <- Z[, j]
    Z[, j] <- running + U[j, j] * z_j
    running <- running + U[j, p] * z_j # not needed after the last column
  }
  normal_to_family(Z, family, binary_prevalence)
}

# =============================================================================
# generate_predictors
# =============================================================================

#' Generate the n x p predictor matrix
#'
#' Draws all predictors from a single global distribution family. With a
#' non-zero \code{correlation}, they are drawn through a Gaussian copula with a
#' common pairwise correlation.
#'
#' @param n                Sample size.
#' @param n_signal_parameters Number of signal predictors.
#' @param noise_parameters  Number of noise predictors.
#' @param complexity       Integer 1-4 (used to resolve the C4 distribution
#'   default).
#' @param predictor_type   \code{"continuous"} (default) or \code{"binary"}.
#' @param binary_prevalence Bernoulli probability; used when
#'   \code{predictor_type = "binary"}.
#' @param correlation      Scalar common pairwise correlation; 0 = independent.
#' @param distribution     Global continuous distribution family; used when
#'   \code{predictor_type = "continuous"}.
#'
#' @return Named n x p numeric matrix (column names: x1, x2, ...).
#' @keywords internal
generate_predictors <- function(
  n,
  n_signal_parameters,
  noise_parameters,
  complexity = 1,
  predictor_type = "continuous",
  binary_prevalence = 0,
  correlation = 0.3,
  distribution = "normal"
) {
  p <- n_signal_parameters + noise_parameters
  col_names <- paste0("x", seq_len(p))

  # ---- validate inputs -------------------------------------------------------
  if (!is.numeric(n_signal_parameters) || n_signal_parameters < 1) {
    stop("n_signal_parameters must be a positive integer.")
  }
  if (!is.numeric(noise_parameters) || noise_parameters < 0) {
    stop("noise_parameters must be a non-negative integer.")
  }
  if (!predictor_type %in% c("continuous", "binary")) {
    stop('predictor_type must be "continuous" or "binary".')
  }
  if (
    !is.numeric(correlation) ||
      length(correlation) != 1 ||
      correlation < -1 ||
      correlation > 1
  ) {
    stop("correlation must be a single numeric value in [-1, 1].")
  }
  if (predictor_type == "binary") {
    if (binary_prevalence <= 0 || binary_prevalence > 1) {
      stop(
        "binary_prevalence must be in (0, 1] when predictor_type = \"binary\"."
      )
    }
  }

  # ---- determine distribution family -----------------------------------------
  family <- resolve_family(
    complexity,
    predictor_type,
    distribution,
    binary_prevalence
  )

  # ---- draw all predictors from the global family ----------------------------
  # With a non-zero correlation, draw through the Gaussian copula instead.
  X <- if (correlation == 0) {
    draw_predictors(n, p, family, binary_prevalence)
  } else {
    draw_correlated_predictors(n, p, family, correlation, binary_prevalence)
  }

  colnames(X) <- col_names
  return(X)
}

# =============================================================================
# generate_linear_predictor
# =============================================================================

#' Construct the linear predictor
#'
#' @param X                  n x p predictor matrix (colnames: x1, x2, ...).
#' @param n_signal_parameters Signal predictor count.
#' @param noise_parameters    Noise predictor count.
#' @param intercept          Scalar intercept (log-odds intercept for binary,
#'   log-hazard intercept for survival, mean intercept for continuous).
#' @param beta_signal        Base effect size (overall scale). The tuning
#'   routines scale this to hit the target oracle metric.
#' @param complexity         Integer 1-4.
#' @param nonlinear_strength Fraction of signal variance carried by the
#'   nonlinear component (C2/C3 only), in [0, 1). \code{NULL} uses the
#'   complexity-level default in \code{COMPLEXITY_NONLINEAR_STRENGTH_DEFAULTS}.
#' @param predictor_type,binary_prevalence,correlation,distribution The
#'   settings \code{X} was drawn with (see \code{\link{generate_predictors}}).
#'   Required for complexity 2 and 3, where they select the reference
#'   estimates for the nonlinear component; they must match how \code{X} was
#'   drawn, or the component is no longer orthogonal to the linear terms.
#'
#' @details
#' \strong{Why variance fraction rather than an R^2 split.}
#' For a continuous outcome whose target is R^2, splitting the signal by
#' latent-scale variance is exact: R^2 is a variance ratio, so a correctly
#' specified linear model recovers exactly the linear share. For binary (AUC)
#' and survival (Harrell's C) outcomes that logic is only approximate, because
#' AUC and C are rank/link-mediated. What carries over is the key property: the
#' nonlinear component is built to be \emph{inaccessible to any linear-in-X
#' model}, so the discrimination it carries genuinely requires a nonlinear
#' learner and more sample size.
#'
#' \strong{Construction.} The linear predictor is
#' \deqn{lp = \alpha + \beta\,\Bigl(\,\underbrace{\textstyle\sum_j x_j}_{L}
#'        \; + \; \kappa\,\mathrm{sd}(L)\, \underbrace{\tilde N}_{N\,\mathrm{std}} \Bigr)}
#' where \eqn{L = \sum_j x_j} is the linear score (identical to complexity 1)
#' and \eqn{\tilde N} is the nonlinear aggregate (\eqn{\sum_j x_j^2} for C2;
#' plus \eqn{\sum_{j<k} x_j x_k} for C3) \emph{residualised against the full
#' design} \eqn{[1, x_1, \ldots, x_S]} and standardised to unit SD. The
#' projection coefficients and both SDs are estimated once per predictor
#' configuration from a 500,000-row reference draw (see
#' \code{nonlinear_reference()}), so every row is generated by the same model
#' whatever the number of rows drawn. Because \eqn{L} and \eqn{\tilde N} are
#' orthogonal in the population (up to the error in these estimates), the
#' nonlinear variance fraction is
#' \deqn{f = \frac{\kappa^2}{1 + \kappa^2},\qquad\text{equivalently}\qquad
#'       \kappa = \sqrt{\frac{f}{1 - f}}.}
#' The user supplies \code{nonlinear_strength} = \eqn{f} (e.g. 0.2 = 20% of the
#' signal variance is nonlinear) and \eqn{\kappa} is computed internally. The
#' population fraction is \eqn{f} up to the error in the reference draw:
#' within 0.002 for normal, uniform, Laplace and exponential predictors, but
#' only within about 0.02 for t and lognormal predictors, whose squares are
#' heavy-tailed.
#'
#' Because the bracket is fixed given X, \code{lp} is linear in
#' \code{beta_signal}, so the tuning routines (which scale \code{beta_signal} to
#' hit the target oracle AUC / C-index) work unchanged.
#'
#' \strong{Calibrating difficulty.} \eqn{f} sets the difficulty order but not, by
#' itself, a precise AUC/C-index gap: the metric reduction a given \eqn{f}
#' produces also depends on the tuned scale, the prevalence/censoring, and the
#' predictor family. To target a specific linear-accessible metric, calibrate
#' \eqn{f} per complexity by simulation (raise it until a fitted GLM/Cox on the
#' linear terms reaches the desired value while \code{beta_signal} keeps the
#' oracle metric on target).
#'
#' \strong{Complexity 1 — Linear:} \eqn{lp = \alpha + \beta \sum_j x_j}.
#'
#' \strong{Complexity 4 — Friedman (1991)} (canonical, \code{nonlinear_strength}
#' ignored): \eqn{lp = \alpha + \beta[10\sin(\pi x_1 x_2) + 20(x_3-0.5)^2
#' + 10x_4 + 5x_5]}. Friedman contains linear-accessible terms
#' (\eqn{10x_4 + 5x_5}); to place C4 on the same controlled ladder as C2/C3,
#' residualise the Friedman vector against Xs and apply the same kappa split.
#'
#' @references
#' Friedman, J. H. (1991). Multivariate adaptive regression splines.
#'   \emph{The Annals of Statistics}, 19(1), 1-67.
#'
#' @return Numeric vector of length n.
#' @keywords internal
generate_linear_predictor <- function(
  X,
  n_signal_parameters,
  noise_parameters,
  intercept,
  beta_signal,
  complexity,
  nonlinear_strength = NULL,
  predictor_type,
  binary_prevalence,
  correlation,
  distribution
) {
  n <- nrow(X)
  lp <- rep(intercept, n)

  if (n_signal_parameters == 0) {
    return(lp)
  }

  eff_beta <- beta_signal # overall signal scale (no strength weight)

  Xs <- X[, seq_len(n_signal_parameters), drop = FALSE]
  S <- n_signal_parameters
  lin <- rowSums(Xs)

  # ---- Complexity 1: pure linear (the f -> 0 limit) -------------------------
  if (complexity == 1) {
    return(lp + eff_beta * lin)
  }

  # ---- Complexity 4: canonical Friedman (1991); nonlinear_strength ignored --
  if (complexity == 4) {
    if (n_signal_parameters < 5) {
      warning(sprintf(
        "Complexity 4 requires >=5 signal predictors; only %d supplied.",
        n_signal_parameters
      ))
    }
    xcol <- function(k) if (k <= S) Xs[, k] else rep(0, n)
    x1 <- xcol(1)
    x2 <- xcol(2)
    x3 <- xcol(3)
    x4 <- xcol(4)
    x5 <- xcol(5)
    fr <- 10 * sin(pi * x1 * x2) + 20 * (x3 - 0.5)^2 + 10 * x4 + 5 * x5
    return(lp + eff_beta * fr)
  }

  # ---- Complexity 2 / 3: linear + inaccessible nonlinear component ----------
  # Resolve the nonlinear variance fraction f and convert to the SD ratio kappa.
  f <- resolve_nonlinear_strength(nonlinear_strength, complexity)
  kappa <- sqrt(f / (1 - f)) # f = kappa^2 / (1 + kappa^2)

  if (kappa == 0) {
    # f = 0 -> reduces to pure linear
    return(lp + eff_beta * lin)
  }

  if (!complexity %in% c(2, 3)) {
    stop("complexity must be 1, 2, 3 or 4.")
  }
  if (complexity == 3 && S < 2) {
    warning("Complexity 3 needs >= 2 signal predictors. Falling back to C2.")
  }
  settings <- c(
    "predictor_type",
    "binary_prevalence",
    "correlation",
    "distribution"
  )
  not_given <- settings[c(
    missing(predictor_type),
    missing(binary_prevalence),
    missing(correlation),
    missing(distribution)
  )]
  if (length(not_given) > 0) {
    stop(
      "Complexity 2 and 3 need the settings X was drawn with; missing: ",
      paste(not_given, collapse = ", "),
      "."
    )
  }

  # Residualise the nonlinear aggregate against [1, x1..xS] and scale it with
  # reference estimates (projection coefficients, SD of the residual and SD
  # of the linear part) for this predictor configuration, so that the model
  # for a row does not depend on how many rows are drawn with it.
  ref <- nonlinear_reference(
    n_signal_parameters = S,
    complexity = complexity,
    predictor_type = predictor_type,
    binary_prevalence = binary_prevalence,
    correlation = correlation,
    distribution = distribution
  )
  N_ortho <- nonlinear_raw(Xs, complexity) -
    drop(cbind(1, Xs) %*% ref$gamma)
  N_std <- N_ortho / ref$sd_N

  # lp = intercept + eff_beta * ( L + kappa * SD(L) * N_std )
  # Linear part identical to C1; nonlinear variance fraction is f in the
  # population, up to the error in the reference estimates. Linearity in
  # beta_signal lets oracle-metric tuning scale it.
  lp + eff_beta * (lin + kappa * ref$sd_lin * N_std)
}


# -----------------------------------------------------------------------------
# Nonlinear aggregate for complexity 2 (sum of squares) and 3 (squares plus
# all pairwise interactions) of the signal predictors.
# -----------------------------------------------------------------------------
nonlinear_raw <- function(Xs, complexity) {
  S <- ncol(Xs)
  if (complexity == 2 || S < 2) {
    return(rowSums(Xs^2))
  }
  inter <- rowSums(vapply(
    seq_len(S - 1),
    function(j) Xs[, j] * rowSums(Xs[, (j + 1):S, drop = FALSE]),
    numeric(nrow(Xs))
  ))
  rowSums(Xs^2) + inter
}

# -----------------------------------------------------------------------------
# Reference estimates for the C2/C3 nonlinear component, for one predictor
# configuration: the coefficients projecting the nonlinear aggregate on
# [1, x1..xS], the SD of the projection residual, and the SD of the linear
# part sum(x). Estimated once from a reference draw of `n_ref` rows with a
# fixed seed (the caller's random-number stream is left untouched) and cached
# for the session, so they are identical in every call.
# -----------------------------------------------------------------------------
.nonlinear_cache <- new.env(parent = emptyenv())

nonlinear_reference <- function(
  n_signal_parameters,
  complexity,
  predictor_type,
  binary_prevalence,
  correlation,
  distribution,
  n_ref = 500000L
) {
  key <- paste(
    n_signal_parameters,
    complexity,
    predictor_type,
    binary_prevalence,
    correlation,
    distribution,
    n_ref,
    sep = "|"
  )
  hit <- .nonlinear_cache[[key]]
  if (!is.null(hit)) {
    return(hit)
  }
  ref <- with_rng_restored({
    set.seed(20250601L, kind = "Mersenne-Twister", normal.kind = "Inversion")
    Xr <- generate_predictors(
      n = n_ref,
      n_signal_parameters = n_signal_parameters,
      noise_parameters = 0,
      complexity = complexity,
      predictor_type = predictor_type,
      binary_prevalence = binary_prevalence,
      correlation = correlation,
      distribution = distribution
    )
    Nraw <- nonlinear_raw(Xr, complexity)
    design <- cbind(1, Xr)
    fit <- stats::lm.fit(design, Nraw)
    gamma <- fit$coefficients
    gamma[is.na(gamma)] <- 0
    list(
      gamma = unname(gamma),
      sd_N = stats::sd(fit$residuals),
      sd_lin = stats::sd(rowSums(Xr)),
      sd_raw = stats::sd(Nraw)
    )
  })
  # Degenerate when (almost) all of the aggregate is linear in x.
  if (!is.finite(ref$sd_N) || ref$sd_N <= 1e-8 * ref$sd_raw) {
    stop(
      "Nonlinear component is degenerate -- e.g. binary predictors under ",
      "complexity 2/3, where x^2 = x collapses the quadratic term. Use ",
      "continuous predictors for C2/C3 (or complexity 1 for binary ",
      "predictors)."
    )
  }
  assign(key, ref, envir = .nonlinear_cache)
  ref
}

Try the pmsims package in your browser

Any scripts or data that you put into this service are public.

pmsims documentation built on Nov. 15, 2026, 5:07 p.m.