Nothing
default_metric_generator <- function(metric, data_function) {
outcome <- attr(data_function, "outcome")
if (outcome == "binary") {
if (metric == "auc") {
metric_function <- binary_auc_metric
} else if (metric == "calibration_slope") {
metric_function <- binary_calib_slope
} else if (metric == "calibration_in_the_large") {
metric_function <- binary_calib_itl
} else if (metric == "brier_score") {
metric_function <- binary_brier_score
} else if (metric == "brier_score_scaled") {
metric_function <- binary_brier_score_scaled
} else if (metric == "csse") {
metric_function <- binary_csse
} else {
stop(paste(
"Default metric",
metric,
"for",
outcome,
"outcomes does not exist."
))
}
}
if (outcome == "survival") {
if (metric == "cindex") {
metric_function <- survival_cindex
} else if (metric == "auc") {
metric_function <- survival_auc
} else if (metric == "calibration_slope") {
metric_function <- survival_calib_slope
} else if (metric == "calibration_slope_free") {
metric_function <- survival_calib_slope_free
} else if (metric == "csse") {
metric_function <- survival_csse
} else if (metric == "ibs") {
# Reserved for future Integrated Brier Score support.
metric_function <- NULL
} else {
stop(paste(
"Default metric",
metric,
"for",
outcome,
"outcomes does not exist."
))
}
}
if (outcome == "continuous") {
if (metric == "r2") {
metric_function <- continuous_r2
} else if (metric == "calibration_slope") {
metric_function <- continuous_calib_slope
} else if (metric == "calibration_in_the_large") {
metric_function <- continuous_calib_itl
} else if (metric == "csse") {
metric_function <- continuous_csse
} else {
stop(paste(
"Default metric",
metric,
"for",
outcome,
"outcomes does not exist."
))
}
}
attr(metric_function, "metric") <- metric
return(metric_function)
}
#' Predict from supported pmsims model objects
#'
#' @param x Predictor data supplied as a data frame or matrix.
#' @param y Optional outcome data retained for API compatibility.
#' @param fit Fitted model object returned by one of the supported model
#' generators.
#' @param model Character string identifying the model family.
#' @param type Prediction scale requested from the fitted model.
#'
#' @return Numeric predictions, or a survival-probability matrix when the
#' requested model and `type` support it.
#' @keywords internal
#' @noRd
predict_custom <- function(x, y = NULL, fit, model, type = "response") {
# Ensure x is data.frame or matrix for predict functions. The matrix gets
# no row names: none of the predictions use them, and for a reordered test
# set they would be 30,000 new strings.
if (is.data.frame(x)) {
x_df <- x
x_mat <- as.matrix(x, rownames.force = FALSE)
} else {
x_df <- as.data.frame(x)
x_mat <- as.matrix(x)
}
# GLM (base R)
if (model %in% c("lm", "glm")) {
if (inherits(fit, "glm") && type %in% c("link", "response")) {
eta <- plain_linear_predictor(fit, cbind(1, x_mat), intercept = TRUE)
if (!is.null(eta)) {
return(if (type == "link") eta else fit$family$linkinv(eta))
}
}
return(stats::predict(fit, newdata = x_df, type = type))
}
# LASSO / RIDGE (glmnet::cv.glmnet)
if (model %in% c("lasso", "ridge")) {
# glmnet prediction requires a numeric matrix.
require_optional_packages("glmnet", "lasso predictions")
s_val <- "lambda.min"
# Choose glmnet type mapping
glmnet_type <- switch(
type,
response = "response",
link = "link",
lp = "link",
stop("Type '", type, "' not supported for lasso.")
)
preds <- as.numeric(stats::predict(
fit,
newx = x_mat,
s = s_val,
type = glmnet_type
))
# For binary responses, type = "response" returns probabilities.
# For Cox models, type = "link" returns the linear predictor.
return(preds)
}
# Random forest via ranger or randomForestSRC (rfsrc)
if (model %in% c("rf", "ranger", "rfsrc")) {
is_ranger <- inherits(fit, "ranger")
is_rfsrc <- inherits(fit, "rfsrc")
# Unknown object: fall back to a generic predict and extract sensibly
if (!is_ranger && !is_rfsrc) {
pr <- try(stats::predict(fit, newdata = x_df), silent = TRUE)
if (!inherits(pr, "try-error")) {
if (is.list(pr) && !is.null(pr$predictions)) {
return(as.numeric(pr$predictions))
}
return(as.numeric(pr))
}
stop(
"rf: model object is neither 'ranger' nor 'rfsrc', and generic predict() failed."
)
}
# ----------------------------------------------------------------- ranger
if (is_ranger) {
require_optional_packages("ranger", "random forest (ranger) predictions")
# Survival risk score (see below) without the full n x death-times
# prediction matrices: the sum of each terminal node's cumulative
# hazard, averaged over trees. Equal to rowSums(predict()$chf) up to
# the order of summation.
if (identical(fit$treetype, "Survival") && type %in% c("lp", "link")) {
chf_sum <- rsf_tree_average(fit, x_df, sum)
if (!is.null(chf_sum)) {
eps <- rf_prob_eps(fit) # 1 / (2 x num.trees)
return(log(pmax(chf_sum, eps)))
}
}
pr <- stats::predict(fit, data = x_df, num.threads = pmsims_threads())
# Survival forest
if (identical(fit$treetype, "Survival")) {
if (type == "survival") {
# n x length(unique.death.times) matrix of survival probabilities
return(pr$survival)
}
if (type %in% c("lp", "link")) {
# Risk score from cumulative hazard summed over the time grid.
# log() puts it on a Cox-lp-like scale: log H(t) = log H0(t) + eta.
eps <- rf_prob_eps(fit) # 1 / (2 x num.trees)
chf_sum <- pmax(rowSums(pr$chf), eps)
#chf_sum <- pmax(rowSums(pr$chf), .Machine$double.eps)
return(log(chf_sum))
}
stop(
"rf (ranger survival): type '",
type,
"' not supported (use 'survival' or 'lp')."
)
}
preds <- pr$predictions
# Classification probabilities => matrix (cols per class)
if (is.matrix(preds) && ncol(preds) >= 2) {
# eps is the finest resolution the ensemble can express; derive it from
# the fitted forest rather than hard-coding 300 trees.
eps <- rf_prob_eps(fit)
p <- pmin(pmax(as.numeric(preds[, ncol(preds)]), eps), 1 - eps)
eta <- stats::qlogis(p)
# Out-of-bag recalibration (see the recalibration section below). The
# map is monotone, so AUC / C-index are unchanged; it corrects only the
# spread of the predicted probabilities.
rc <- rf_recal_binary(fit)
if (!is.null(rc)) {
eta <- rc$a + rc$b * eta
}
if (type == "link") {
return(eta)
}
if (type == "response") {
return(stats::plogis(eta))
}
}
# Regression / single numeric prediction
if (is.numeric(preds) && is.vector(preds)) {
return(as.numeric(preds))
}
stop("rf (ranger): unsupported prediction structure.")
}
# ------------------------------------------------------- randomForestSRC
if (is_rfsrc) {
require_optional_packages(
"randomForestSRC",
"random survival forest (rfsrc) predictions"
)
pr <- stats::predict(fit, newdata = x_df)
# Survival forest
if (identical(fit$family, "surv")) {
if (type == "survival") {
# n x length(time.interest) matrix of survival probabilities
return(pr$survival)
}
if (type %in% c("lp", "link")) {
# Ensemble mortality is rfsrc's native risk score (higher = higher risk)
risk <- pmax(as.numeric(pr$predicted), .Machine$double.eps)
return(log(risk))
}
stop(
"rf (rfsrc survival): type '",
type,
"' not supported (use 'survival' or 'lp')."
)
}
preds <- pr$predicted
# Classification probabilities => matrix (cols per class)
if (is.matrix(preds) && ncol(preds) >= 2) {
if (type == "response") {
return(as.numeric(preds[, ncol(preds)]))
} # P(positive class)
if (type == "link") {
p <- pmin(
pmax(as.numeric(preds[, ncol(preds)]), .Machine$double.eps),
1 - .Machine$double.eps
)
return(stats::qlogis(p))
}
}
# Regression / single numeric prediction
if (is.numeric(preds) && is.null(dim(preds))) {
return(as.numeric(preds))
}
stop("rf (rfsrc): unsupported prediction structure.")
}
}
# xgboost
if (model == "xgboost" || inherits(fit, "xgb.Booster")) {
require_optional_packages("xgboost", "xgboost predictions")
# xgboost predict expects a matrix or xgb.DMatrix
dmat <- xgboost::xgb.DMatrix(data = x_mat, nthread = pmsims_threads())
preds <- stats::predict(fit, dmat)
# For binary: preds are probabilities (objective = binary:logistic)
if (type == "response") {
return(as.numeric(preds))
}
# For link: return logit
if (type == "link") {
p <- pmin(
pmax(as.numeric(preds), .Machine$double.eps),
1 - .Machine$double.eps
)
return(stats::qlogis(p))
}
# For linear predictor / risk score.
# NOTE: with objective = "survival:cox", xgboost returns predictions on the
# HAZARD-RATIO scale, i.e. preds = exp(lp). The log-hazard linear predictor,
# used for the predicted survival in the calibration slope, is log(preds).
# Ranking metrics (C-index, AUC) are unaffected because log is monotone.
if (type == "lp") {
return(log(pmax(as.numeric(preds), .Machine$double.eps)))
}
# For survival probabilities, not directly available from xgboost cox objective
if (type == "survival") {
stop(
"xgboost: direct survival probability matrix is not available from xgboost predictions. Consider using type = 'lp' and mapping to survival via a baseline if needed."
)
}
stop("xgboost: unsupported 'type' requested.")
}
# Cox models or other types that might use survival predictions
if (model == "coxph") {
if (type %in% c("lp", "link") && inherits(fit, "coxph")) {
# predict.coxph(): centred at the training means.
lp <- plain_linear_predictor(
fit,
x_mat - rep(fit$means, each = nrow(x_mat)),
intercept = FALSE
)
if (!is.null(lp)) {
return(lp)
}
}
fit_for_prediction <- fit
formula_env <- new.env(
parent = environment(stats::formula(fit_for_prediction))
)
formula_env$Surv <- survival::Surv
environment(fit_for_prediction$formula) <- formula_env
attr(fit_for_prediction$terms, ".Environment") <- formula_env
if (type %in% c("lp", "link")) {
return(stats::predict(fit_for_prediction, newdata = x_df, type = "lp"))
} else if (type == "survival") {
return(stats::predict(
fit_for_prediction,
newdata = x_df,
type = "survival"
))
} else {
stop("coxph predict_custom: only 'lp' or 'survival' supported.")
}
}
stop("predict_custom: unknown model type '", model, "'.")
}
# Linear predictor of a glm/lm/coxph fit of the form `y ~ .` on numeric
# predictors, computed as predict() does (the design matrix times the
# coefficients) but without building a model frame for the 30,000 test rows.
# `X` is the design matrix: the predictors, with a leading column of ones when
# `intercept` is TRUE. Returns NULL, so that the caller uses predict(), unless
# the coefficients are exactly one per column of X, in order and all
# estimated, with no offset (term or argument) or other special terms; prior
# weights do not affect predictions. Unlike predict() the result carries no
# names.
plain_linear_predictor <- function(fit, X, intercept) {
cf <- stats::coef(fit)
tt <- stats::terms(fit)
expected <- if (intercept) c("(Intercept)", colnames(X)[-1]) else colnames(X)
plain <- is.numeric(X) &&
identical(names(cf), expected) &&
all(is.finite(cf)) &&
is.null(attr(tt, "offset")) &&
is.null(fit$call$offset) &&
!length(unlist(attr(tt, "specials")))
if (!plain) {
return(NULL)
}
drop(X %*% cf)
}
# Binary metrics
binary_auc_metric <- function(data, fit, model) {
y <- data[, "y"]
x <- data[, names(data) != "y", drop = FALSE]
y_hat <- as.numeric(predict_custom(x, y, fit, model, type = "response"))
auc <- rank_auc(y, y_hat)
if (is.null(auc)) {
auc <- pROC::auc(
y,
y_hat,
levels = c(0, 1),
direction = "<",
quiet = TRUE
)[1]
}
auc
}
# pROC::auc(y, score) for a 0/1 outcome, from the Mann-Whitney rank formula:
# the probability that a random case scores above a random control, ties
# counting one half (average ranks), which is the area under the empirical
# ROC curve. The direction is fixed (pROC's direction = "<", as in the pROC
# fallback above), so a score that ranks the outcome the wrong way round gives
# an AUC below 0.5. Returns NULL, so that the caller uses pROC, unless the
# outcome is numeric 0/1 (no missing values) with both classes present and all
# scores are finite.
rank_auc <- function(y, score) {
if (!is.numeric(y) || !all(y %in% c(0, 1)) || !all(is.finite(score))) {
return(NULL)
}
case <- y == 1
n_case <- sum(case)
n_control <- length(y) - n_case
if (n_case == 0L || n_control == 0L) {
return(NULL)
}
r <- rank(score)
# as.numeric(): the product of the class counts overflows an integer above
# 2^31 (e.g. a 100,000-row test set at prevalence 0.4).
(sum(r[case]) - n_case * (n_case + 1) / 2) / (as.numeric(n_case) * n_control)
}
binary_calib_slope <- function(data, fit, model) {
y <- data[, "y"]
x <- data[, names(data) != "y", drop = FALSE]
y_link <- predict_custom(x, y, fit, model, type = "link")
binary_calibration_slope(y, y_link)
}
binary_csse <- function(data, fit, model) {
y <- data[, "y"]
x <- data[, names(data) != "y", drop = FALSE]
y_link <- predict_custom(x, y, fit, model, type = "link")
-(1 - binary_calibration_slope(y, y_link))^2
}
binary_calib_itl <- function(data, fit, model) {
y <- data[, "y"]
x <- data[, names(data) != "y", drop = FALSE]
y_link <- predict_custom(x, y, fit, model, type = "link")
itl <- calibration_glm(y, offset = y_link)
if (!is.null(itl)) {
return(abs(itl))
}
slope_itl <- try(
stats::glm(y ~ 1, offset = y_link, data = data, family = stats::binomial()),
silent = TRUE
)
if (inherits(slope_itl, "try-error")) {
return(NaN)
} else {
return(abs(as.numeric(stats::coef(slope_itl)[1])))
}
}
binary_brier_score <- function(data, fit, model) {
y <- data[, "y"]
x <- data[, names(data) != "y", drop = FALSE]
y_hat <- predict_custom(x, y, fit, model, type = "response")
bs <- mean((y - y_hat)^2)
return(bs)
}
binary_brier_score_scaled <- function(data, fit, model) {
y <- data[, "y"]
x <- data[, names(data) != "y", drop = FALSE]
y_hat <- predict_custom(x, y, fit, model, type = "response")
return(1 - mean((y - y_hat)^2) / mean((y - mean(y))^2))
}
# Continuous metrics
continuous_r2 <- function(data, fit, model) {
y <- data[, "y"]
x <- data[, names(data) != "y", drop = FALSE]
y_hat <- predict_custom(x, y, fit, model, type = "response")
n <- length(y)
mse <- sum((y_hat - y)^2) / n
mst <- stats::var(y) * (n + 1) / n
r2 <- 1 - (mse / mst)
return(r2)
}
continuous_calib_slope <- function(data, fit, model) {
y <- data[, "y"]
x <- data[, names(data) != "y", drop = FALSE]
y_hat <- predict_custom(x, y, fit, model, type = "response")
continuous_calibration_slope(y, y_hat)
}
continuous_csse <- function(data, fit, model) {
y <- data[, "y"]
x <- data[, names(data) != "y", drop = FALSE]
y_hat <- predict_custom(x, y, fit, model, type = "response")
-(1 - continuous_calibration_slope(y, y_hat))^2
}
continuous_calib_itl <- function(data, fit, model) {
y <- data[, "y"]
x <- data[, names(data) != "y", drop = FALSE]
y_hat <- predict_custom(x, y, fit, model, type = "response")
# The intercept of lm(y ~ 1, offset = y_hat) is the mean residual.
if (all(is.finite(y)) && all(is.finite(y_hat))) {
return(mean(y - y_hat))
}
slope <- try(stats::lm(y ~ 1, offset = y_hat), silent = TRUE)
if (inherits(slope, "try-error")) {
return(NaN)
} else {
return(as.numeric(stats::coef(slope)[1]))
}
}
# Survival metrics
survival_cindex <- function(data, fit, model) {
y_surv <- survival::Surv(data$time, data$event)
x <- data[, !(names(data) %in% c("time", "event", "id")), drop = FALSE]
# request linear predictor / risk score
y_hat <- try(predict_custom(x, NULL, fit, model, type = "lp"), silent = TRUE)
if (inherits(y_hat, "try-error")) {
return(NaN)
}
# The concordance only: its standard error (an influence matrix over all
# test rows) is not used.
cf <- try(
survival::concordancefit(
y_surv,
-1 * as.numeric(y_hat),
std.err = FALSE
),
silent = TRUE
)
if (inherits(cf, "try-error") || is.null(cf)) {
return(NaN)
}
return(cf$concordance)
}
# Calibration slope for survival models, used by metric = "calibration_slope"
# and (through survival_csse()) by metric = "csse".
# coxph / lasso / ridge : the Cox calibration slope, coxph(Surv ~ lp), from
# survival_calib_slope_PH(). It does not depend on a time point.
# rf / ranger / xgboost : the slope at a fixed time point t* (the median event
# time), using the predicted log cumulative hazard, log H_i(t*). Random
# survival forests have no proportional-hazards linear predictor: their
# ensemble-mortality score log(sum_t H(t)) ranks subjects (C-index, AUC) but
# is not on a log-hazard scale where the calibrated slope is 1. xgboost uses
# the same time point as rf so that the two are comparable.
survival_calib_slope <- function(data, fit, model, eval_time = NULL) {
if (model %in% c("coxph", "lasso", "ridge")) {
return(survival_calib_slope_PH(data = data, fit = fit, model = model))
}
data <- data[base::order(data$time), ]
# ONE fixed horizon for all models. In a simulation, set this from the DGP
# (a known survival quantile / clinical horizon) and pass it in. Do NOT default
# to the per-replicate observed median: it drifts across replicates and shrinks
# with n, so it couples the metric to sample size -- the very thing you study.
if (is.null(eval_time)) {
ev <- data$time[data$event == 1]
eval_time <- if (length(ev)) stats::median(ev) else stats::median(data$time)
}
# Predicted survival at t* on a common footing (rf: survival matrix;
# coxph/lasso/ridge/xgboost: Breslow baseline applied to the lp).
S <- predicted_survival_at_time(data, fit, model, eval_time)
if (is.null(S) || all(is.na(S))) {
return(NaN)
}
eps <- .Machine$double.eps
S <- pmin(pmax(S, eps), 1 - eps)
eta <- log(-log(S)) # cloglog = log H(t*); log-hazard scale
# Graf/IPCW binary outcome at t*, identical machinery for every model.
iw <- ipcw_binary_at_time(data, eval_time)
suppressWarnings(
binary_calibration_slope(iw$y, eta, link = "cloglog", weights = iw$w)
)
}
# Cox calibration slope, coxph(Surv ~ lp), for the proportional-hazards models.
survival_calib_slope_PH <- function(data, fit, model) {
x <- data[, !(names(data) %in% c("time", "event", "id")), drop = FALSE]
eta <- try(predict_custom(x, NULL, fit, model, type = "lp"), silent = TRUE)
if (inherits(eta, "try-error") || is.null(eta)) {
return(NaN)
}
cox_calibration_slope(data$time, data$event, as.numeric(eta))
}
# Calibration-slope squared error, from the same slope as
# metric = "calibration_slope" (see survival_calib_slope()).
survival_csse <- function(data, fit, model) {
slope <- survival_calib_slope(data = data, fit = fit, model = model)
if (!is.finite(slope)) {
return(NaN)
}
-(1 - slope)^2
}
# Predicted survival probability S_i(t*) at a horizon, for any supported model.
# rf/ranger : taken directly from the predicted survival matrix.
# coxph/lasso/ridge/xgboost : these are proportional-hazards models that yield
# a log-hazard linear predictor (xgboost only after predict_custom() applies
# log() to its hazard-ratio output). We estimate a Breslow baseline with the
# lp held as an offset, then S_i(t*) = exp(-H0(t*) * exp(lp_i)).
predicted_survival_at_time <- function(data, fit, model, eval_time) {
x <- data[, !(names(data) %in% c("time", "event", "id")), drop = FALSE]
if (model %in% c("rf", "ranger")) {
# STEP 1: point prediction S_i(t*) only (see rsf_survival_at_point).
S <- rsf_survival_at_point(fit, x, eval_time)
if (is.null(S)) {
return(NULL)
}
# STEP 2: apply the out-of-bag recalibration map, fitted at this same t*.
rc <- rf_recal_survival(fit, eval_time)
if (!is.null(rc)) {
eps <- .Machine$double.eps
eta <- log(-log(pmin(pmax(S, eps), 1 - eps)))
S <- exp(-exp(rc$a + rc$b * eta))
}
return(S)
}
# PH-lp models: baseline via Breslow with lp as offset
lp <- try(predict_custom(x, NULL, fit, model, type = "lp"), silent = TRUE)
if (inherits(lp, "try-error") || is.null(lp)) {
return(NULL)
}
lp <- as.numeric(lp)
bh <- offset_baseline_hazard(data$time, data$event, lp)
if (is.null(bh)) {
bh <- try(
survival::basehaz(
survival::coxph(survival::Surv(data$time, data$event) ~ offset(lp)),
centered = FALSE
),
silent = TRUE
)
}
if (inherits(bh, "try-error") || is.null(bh)) {
return(NULL)
}
H0 <- stats::approx(bh$time, bh$hazard, xout = eval_time, rule = 2)$y
exp(-H0 * exp(lp))
}
# Direct versions of the two survival-package fits above, which on the
# 30,000-row test set took most of a Cox replicate's time (coxph() also
# computes a concordance nobody uses). Both give survival's results to
# rounding error and return NULL, so that the caller uses survival itself,
# for anything but finite times and 0/1 events.
# Follow-up times as survival's fits see them: coxph() and survfit() first
# merge near-equal times with survival::aeqSurv(). `times` are the merged
# unique times and `group` the index of each observation's time in them.
merged_survival_times <- function(time) {
merged <- survival::aeqSurv(survival::Surv(time))[, 1]
times <- sort(unique(merged))
list(times = times, group = match(merged, times))
}
plain_survival_data <- function(time, event) {
length(time) > 0L &&
length(time) == length(event) &&
is.numeric(time) &&
all(is.finite(time)) &&
all(event %in% c(0, 1))
}
# basehaz(coxph(Surv(time, event) ~ offset(lp)), centered = FALSE): the
# cumulative baseline hazard at every distinct time, as list(hazard, time).
# survfit() centres the offset at its mean, and coxph's default Efron ties
# carry over to the hazard increments at tied event times.
offset_baseline_hazard <- function(time, event, lp) {
if (
!plain_survival_data(time, event) ||
length(lp) != length(time) ||
!all(is.finite(lp)) ||
!any(event == 1)
) {
return(NULL)
}
st <- merged_survival_times(time)
m <- length(st$times)
risk <- exp(lp - mean(lp))
at_risk <- rev(cumsum(rev(rowsum(risk, st$group)[, 1])))
died <- event == 1
deaths <- tabulate(st$group[died], m)
increment <- deaths / at_risk
for (k in which(deaths > 1)) {
# Efron: the dying leave the risk set in equal fractions.
death_risk <- sum(risk[died & st$group == k])
frac <- (seq_len(deaths[k]) - 1) / deaths[k]
increment[k] <- sum(1 / (at_risk[k] - frac * death_risk))
}
list(hazard = cumsum(increment), time = st$times)
}
# survfit(Surv(time, 1 - event) ~ 1): Kaplan-Meier estimate of the censoring
# distribution, as list(time, surv).
censoring_km <- function(time, event) {
if (!plain_survival_data(time, event)) {
return(NULL)
}
st <- merged_survival_times(time)
m <- length(st$times)
n_risk <- rev(cumsum(rev(tabulate(st$group, m))))
censored <- tabulate(st$group[event == 0], m)
list(time = st$times, surv = cumprod(1 - censored / n_risk))
}
# IPCW (Graf) weights and binarised outcome for calibration at a fixed horizon.
# Weight 1/G(T_i-) for events before t*, 1/G(t*) for those still at risk at t*,
# and 0 for subjects censored before t* (their t*-status is unknown). G is the
# Kaplan-Meier estimate of the censoring-time distribution.
ipcw_binary_at_time <- function(data, eval_time) {
cens_fit <- censoring_km(data$time, data$event) %||%
survival::survfit(survival::Surv(data$time, 1 - data$event) ~ 1)
Gfun <- stats::stepfun(cens_fit$time, c(1, cens_fit$surv))
eps <- .Machine$double.eps
y_obs <- as.numeric(data$time <= eval_time & data$event == 1)
w <- numeric(nrow(data))
ev_before <- data$time <= eval_time & data$event == 1
at_risk <- data$time > eval_time
w[ev_before] <- 1 / pmax(Gfun(data$time[ev_before] - 1e-10), eps)
w[at_risk] <- 1 / pmax(Gfun(eval_time), eps)
# censored before t* keep weight 0
list(y = y_obs, w = w)
}
# Previous model-free IPCW calibration-slope method.
#
# This logit-risk formulation has been superseded by
# survival_calib_slope() above, which uses the complementary log-log scale to
# align predicted survival with cumulative hazard. It is retained only for the
# internal "calibration_slope_free" metric selector.
#
# Works for any model that yields a predicted survival probability (rf via its
# survival matrix; coxph/lasso/ridge/xgboost via a Breslow baseline applied to
# the linear predictor). The predicted risk F_i(t*) = 1 - S_i(t*) is mapped to
# logit(F_i) and a censoring-weighted logistic regression of the t*-event status
# on logit(F) is fitted. The slope is 1 when the model is calibrated (predicted
# risks have the correct spread); < 1 indicates over-confident predictions,
# > 1 under-confident. Default horizon t* = median observed event time.
survival_calib_slope_free <- function(data, fit, model, eval_time = NULL) {
data <- data[base::order(data$time), ]
if (is.null(eval_time)) {
ev_times <- data$time[data$event == 1]
eval_time <- if (length(ev_times) > 0) {
stats::median(ev_times)
} else {
stats::median(data$time)
}
}
S <- predicted_survival_at_time(data, fit, model, eval_time)
if (is.null(S) || all(is.na(S))) {
warning(
"survival_calib_slope_free: predicted survival not available for model '",
model,
"'. Returning NaN."
)
return(NaN)
}
eps <- .Machine$double.eps
Frisk <- pmin(pmax(1 - S, eps), 1 - eps) # predicted risk at t*
lp_risk <- stats::qlogis(Frisk) # logit risk -> slope ~1 when calibrated
iw <- ipcw_binary_at_time(data, eval_time)
suppressWarnings(binary_calibration_slope(iw$y, lp_risk, weights = iw$w))
}
survival_auc <- function(data, fit, model) {
x <- data[, !(names(data) %in% c("time", "event", "id")), drop = FALSE]
# get linear predictor / risk score where possible
y_hat <- try(predict_custom(x, NULL, fit, model, type = "lp"), silent = TRUE)
if (inherits(y_hat, "try-error") || is.null(y_hat)) {
return(NaN)
}
# Cumulative/dynamic AUC just before the last event time, i.e. at the end of
# follow-up for the simulated data, where all censoring is administrative.
t_max <- max(data$time[data$event == 1])
ipcw_auc_at_time(data, as.numeric(y_hat), t_max * 0.9999999)
}
# Cumulative/dynamic AUC at time t with inverse-probability-of-censoring
# weights, as timeROC::timeROC(weighting = "marginal") computes it: cases had
# the event by t (weight 1 / G(T_i-)), controls are still at risk after t
# (weight 1 / G(t)), and those censored before t are left out. Higher marker
# values mean higher risk, and ties count one half. Controls share one weight,
# so the AUC is the case-weighted mean of the proportion of controls each case
# outranks.
ipcw_auc_at_time <- function(data, marker, eval_time) {
iw <- ipcw_binary_at_time(data, eval_time)
case <- iw$y == 1
control <- data$time > eval_time
if (!any(case) || !any(control) || !all(is.finite(marker))) {
return(NaN)
}
controls <- sort(marker[control])
below <- findInterval(marker[case], controls, left.open = TRUE)
at_or_below <- findInterval(marker[case], controls)
outranked <- (below + 0.5 * (at_or_below - below)) / length(controls)
sum(iw$w[case] * outranked) / sum(iw$w[case])
}
#### Random-forest point prediction and out-of-bag recalibration ####
#
# WHY THIS SECTION EXISTS
#
# A calibration-slope sample-size question is well posed only for a learner
# whose calibration slope converges to 1. For coxph / lasso / ridge it does: the
# slope sits below 1 at small n because of overfitting and rises to 1 as n
# grows, so "the smallest n at which the slope reaches 0.9" is a real number.
#
# A random forest is different. ranger probability forests are systematically
# UNDER-dispersed on the logit scale -- bagging plus mtry averaging shrinks
# predictions toward the marginal -- so the recalibration slope converges to a
# model-specific constant strictly greater than 1. Measured on the 10-predictor
# binary DGP at C = 0.8 with min.node.size = 15 (test n = 20,000):
#
# n = 179 slope 1.081 n = 2,864 slope 1.289
# n = 716 slope 1.287 n = 5,728 slope 1.272
#
# A target of |slope - 1| <= 0.1 is therefore unreachable at ANY n, and a search
# that keeps doubling n looking for it will never terminate.
#
# The fix is to split the forest into (a) a ranking function and (b) a
# calibration map, and fit the map on OUT-OF-BAG training predictions. ranger
# returns these for free -- fit$predictions for probability forests,
# fit$survival for survival forests -- so this costs no extra model fits and
# uses no test data. The recalibrated forest then behaves like the regression
# models: its slope is below/around 1 at small n (the map is itself estimated
# from finite data) and tightens on 1 as n grows.
#
# REQUIRED HOOK
#
# ranger keeps the OOB predictions but discards the training OUTCOME, and a
# metric function only receives (test_data, fit, model). The training outcome
# must therefore be carried on the fit. Add one line to the model function:
#
# binary: fit$pmsims_train <- list(y = d[["y"]])
# survival: fit$pmsims_train <- list(time = d$time, event = d$event)
#
# If the hook is absent every function below returns NULL and predictions fall
# back to the raw, uncalibrated forest -- i.e. exactly the current behaviour.
# Finest probability resolution the ensemble can express.
rf_prob_eps <- function(fit, default_trees = 300) {
nt <- tryCatch(fit$num.trees, error = function(e) NULL)
if (is.null(nt) || !is.finite(nt) || nt < 1) {
nt <- default_trees
}
1 / (2 * nt)
}
rf_train_outcome <- function(fit) {
tr <- tryCatch(fit$pmsims_train, error = function(e) NULL)
if (is.null(tr)) {
tr <- attr(fit, "pmsims_train", exact = TRUE)
}
tr
}
# Point prediction S_i(t*) for a random survival forest.
#
# ranger materialises a full n_test x n_unique_death_times survival matrix AND a
# matching cumulative-hazard matrix, and n_unique_death_times grows linearly
# with the TRAINING size. We only ever need one column. Left unchunked this is
# an O(test_n * n_train) allocation: at test_n = 30,000 and n_train = 45,000
# that is ~15 GB across the two matrices, which is what kills long jobs.
#
# Predicting in blocks and discarding everything but the horizon column as we go
# holds peak memory at O(chunk * udt) instead.
rsf_survival_at_point <- function(
fit,
x,
eval_time,
chunk = getOption("pmsims.predict_chunk", 2000L)
) {
x_df <- as.data.frame(x)
n_test <- nrow(x_df)
if (n_test == 0L) {
return(numeric(0))
}
chunk <- max(1L, min(n_test, as.integer(chunk)))
times <- fit$forest$unique.death.times
if (inherits(fit, "ranger") && !is.null(times)) {
idx <- which.min(abs(times - eval_time))
chf <- rsf_tree_average(fit, x_df, function(v) v[idx], chunk)
if (!is.null(chf)) {
return(exp(-chf)) # as predict.ranger(): survival = exp(-chf)
}
}
S <- numeric(n_test)
idx <- NA_integer_
for (s in seq(1L, n_test, by = chunk)) {
e <- min(s + chunk - 1L, n_test)
pr <- try(
stats::predict(
fit,
data = x_df[s:e, , drop = FALSE],
num.threads = pmsims_threads()
),
silent = TRUE
)
if (inherits(pr, "try-error") || !is.list(pr) || is.null(pr$survival)) {
return(NULL)
}
times <- pr$unique.death.times
if (is.null(times)) {
times <- fit$unique.death.times
}
if (is.null(times) || length(times) != ncol(pr$survival)) {
return(NULL)
}
if (is.na(idx)) {
idx <- which.min(abs(times - eval_time))
}
S[s:e] <- as.numeric(pr$survival[, idx])
rm(pr)
}
S
}
# A survival forest's prediction from its trees' terminal nodes.
#
# For each test row and death time, ranger's predict() averages the cumulative
# hazards stored at the row's terminal node in each tree (summed in tree
# order, then divided by the number of trees) -- for all death times, though
# the metrics need one time or one summary. Here `node_value` reduces each
# terminal node's cumulative hazard to that one number first (e.g. its value
# at one time), and the trees are averaged in the same order. With
# node_value = function(v) v[j] the result is bit-identical to column j of
# predict()$chf, at a small fraction of the cost (the full prediction took
# 20-75 s per replicate for 30,000 test rows). Returns NULL if the forest
# lacks what is needed or predict() fails.
rsf_tree_average <- function(fit, x_df, node_value, chunk = 2000L) {
node_chf <- fit$forest$chf
if (is.null(node_chf) || length(node_chf) != fit$num.trees) {
return(NULL)
}
# Value per node of each tree (NA for internal nodes, which store nothing).
per_tree <- lapply(node_chf, function(nodes) {
v <- rep(NA_real_, length(nodes))
terminal <- lengths(nodes) > 0L
v[terminal] <- vapply(nodes[terminal], node_value, numeric(1))
v
})
n_test <- nrow(x_df)
out <- numeric(n_test)
for (s in seq(1L, n_test, by = chunk)) {
e <- min(s + chunk - 1L, n_test)
nodes <- try(
stats::predict(
fit,
data = x_df[s:e, , drop = FALSE],
type = "terminalNodes",
num.threads = pmsims_threads()
)$predictions,
silent = TRUE
)
if (
inherits(nodes, "try-error") ||
!is.matrix(nodes) ||
ncol(nodes) != fit$num.trees
) {
return(NULL)
}
total <- 0
for (k in seq_len(fit$num.trees)) {
total <- total + per_tree[[k]][nodes[, k] + 1L] # node ids are 0-based
}
out[s:e] <- total / fit$num.trees
}
out
}
# Logit-scale (Platt) recalibration map from OOB probability predictions.
rf_recal_binary <- function(fit) {
if (!inherits(fit, "ranger")) {
return(NULL)
}
tr <- rf_train_outcome(fit)
if (is.null(tr) || is.null(tr$y)) {
return(NULL)
}
oob <- fit$predictions
if (is.null(oob) || !is.matrix(oob) || ncol(oob) < 2) {
return(NULL)
}
y <- tr$y
if (is.factor(y)) {
y <- as.numeric(as.character(y))
}
y <- as.numeric(y)
p <- as.numeric(oob[, ncol(oob)])
if (length(p) != length(y)) {
return(NULL)
}
ok <- is.finite(p) & is.finite(y)
if (sum(ok) < 20L || length(unique(y[ok])) < 2L) {
return(NULL)
}
eps <- rf_prob_eps(fit)
eta <- stats::qlogis(pmin(pmax(p[ok], eps), 1 - eps))
if (stats::sd(eta) < 1e-10) {
return(NULL)
}
cf <- suppressWarnings(binary_calibration_coef(y[ok], eta))
if (length(cf) < 2L || !all(is.finite(cf))) {
return(NULL)
}
list(a = cf[1], b = cf[2])
}
# Cloglog recalibration map at t*, from OOB survival predictions.
# Fitted at the SAME horizon the metric evaluates at, so the map is never
# applied at a horizon it was not estimated for.
rf_recal_survival <- function(fit, eval_time) {
if (!inherits(fit, "ranger")) {
return(NULL)
}
tr <- rf_train_outcome(fit)
if (is.null(tr) || is.null(tr$time) || is.null(tr$event)) {
return(NULL)
}
S_oob <- fit$survival
times <- fit$unique.death.times
if (is.null(S_oob) || is.null(times) || ncol(S_oob) != length(times)) {
return(NULL)
}
if (nrow(S_oob) != length(tr$time)) {
return(NULL)
}
idx <- which.min(abs(times - eval_time))
eps <- .Machine$double.eps
S <- pmin(pmax(as.numeric(S_oob[, idx]), eps), 1 - eps)
eta <- log(-log(S))
train_df <- data.frame(time = tr$time, event = tr$event)
iw <- try(ipcw_binary_at_time(train_df, eval_time), silent = TRUE)
if (inherits(iw, "try-error")) {
return(NULL)
}
keep <- is.finite(eta) & is.finite(iw$w) & iw$w > 0
if (sum(keep) < 20L || length(unique(iw$y[keep])) < 2L) {
return(NULL)
}
if (stats::sd(eta[keep]) < 1e-10) {
return(NULL)
}
cf <- suppressWarnings(binary_calibration_coef(
iw$y[keep],
eta[keep],
link = "cloglog",
weights = iw$w[keep]
))
if (length(cf) < 2L || !all(is.finite(cf))) {
return(NULL)
}
list(a = cf[1], b = cf[2])
}
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.