tests/testthat/helper-threshold-sync.R

# =============================================================================
# helper-threshold-sync.R
#
# Replicate-equality probe for TASK_threshold_sync_2026-09-18.md, Steps 2 & 5,
# and the fixture behind test-threshold-sync.R.  dev/tasks/probe_threshold_
# sync_2026-09-18.R sources this file; there is no second copy.
#
# NO COMPUTE.  Nothing here fits a model, resamples, or touches data.  The
# probe constructs forestsearch() argument lists and resolves the two effect
# thresholds from them, twice:
#
#   parent     -- the argument list as a user writes it, so an argument the
#                 user did not write is genuinely absent and missing() is TRUE;
#   replicate  -- the argument list as bootstrap_analysis_dofuture.R builds
#                 args_FS_boot from fs.est$args_call_all (and as
#                 forestsearch_cross_validation.R builds cv_args): EVERY
#                 formal is present, so missing() is FALSE for all of them.
#
# The resolution is NOT transcribed.  .probe_resolver() lifts the relevant
# statements out of body(forestsearch) and rebuilds them as a small function
# whose formals are forestsearch()'s own threshold formals -- so missing(),
# the alias merge and every remap branch are the package's code, evaluated
# under the package namespace, and the probe cannot drift from the source.
#
#   forestsearch_main.R  effect_measure default per outcome_type
#                        user_set_* detection + alias merge
#                        threshold resolution (GLM branches)
#                        SECTION 2B-ii threshold sync (present only after the
#                          Step 3 edit; the probe detects its absence)
#                        SECTION 1A3 pair validation / derivation (Directive A;
#                          same treatment -- lifted only when validate = TRUE)
#   bootstrap_analysis_dofuture.R:406, 558, 614       the bootstrap replay
#   forestsearch_cross_validation.R:345, 479; 859, 1001   the CV replay
# =============================================================================

# ---------------------------------------------------------------------------
# Statement lifting
# ---------------------------------------------------------------------------

.probe_deparse1 <- function(e) paste(deparse(e), collapse = " ")

.probe_pick <- function(stmts, pattern, what, n = 1L) {
  d  <- vapply(stmts, .probe_deparse1, character(1))
  ix <- which(grepl(pattern, d, fixed = TRUE))
  if (length(ix) != n)
    stop(sprintf("probe: expected %d statement(s) matching '%s' (%s), found %d",
                 n, pattern, what, length(ix)), call. = FALSE)
  ix
}

#' Build the resolver: forestsearch()'s own threshold code, as a function.
#'
#' @param sync TRUE to include the SECTION 2B-ii sync statements.
#' @param validate TRUE to include the SECTION 1A3 threshold-pair statements
#'   (Directive A: the c2 > c1 stop and the silent-c2 derivation).  Absent
#'   before the Directive A edit, exactly as `sync` was before the sync edit.
#' @return a function(effect.threshold, consistency.threshold, hr.threshold,
#'   hr.consistency, outcome_type, effect_measure, subgroup_method, quiet)
#'   returning the resolved record plus the args_call_all entries a replay
#'   would receive.
.probe_resolver <- function(sync = FALSE, validate = FALSE) {
  fm   <- formals(forestsearch)
  top  <- as.list(body(forestsearch))[-1L]   # drop the `{`

  # -- effect_measure default (top level) ----------------------------------
  i_em <- .probe_pick(top, "is.null(effect_measure)", "effect_measure default")
  s_em <- top[[i_em]]

  # -- user_set_* detection + alias merge (top level, four statements) ------
  i_ust <- .probe_pick(top, "user_set_threshold <- ",   "user_set_threshold")
  i_usc <- .probe_pick(top, "user_set_consistency <- ", "user_set_consistency")
  i_a1  <- .probe_pick(top, "hr.threshold <- effect.threshold",
                       "alias merge c1")
  i_a2  <- .probe_pick(top, "hr.consistency <- consistency.threshold",
                       "alias merge c2")
  s_flags <- top[c(i_ust, i_usc, i_a1, i_a2)]

  # -- SECTION 1A3: the pair validation / derivation (Directive A) ----------
  # The two legacy captures must precede the alias merge (they read the
  # pre-merge values); the resolve call follows it.  Absent before the edit.
  s_pre_flags <- list()
  s_pair      <- list()
  if (isTRUE(validate)) {
    i_l1 <- .probe_pick(top, ".fs_c1_legacy <- ", "legacy c1 capture")
    i_l2 <- .probe_pick(top, ".fs_c2_legacy <- ", "legacy c2 capture")
    i_pr <- .probe_pick(top, "hr.consistency <- .fs_resolve_threshold_pair(",
                        "threshold-pair resolution")
    s_pre_flags <- top[c(i_l1, i_l2)]
    s_pair      <- top[i_pr]
  }

  # -- the GLM resolution block (nested in `if (outcome_type != "survival")`)
  i_glm <- .probe_pick(top, 'if (outcome_type != "survival") {',
                       "GLM branch", n = 1L)
  inner <- as.list(top[[i_glm]][[3L]])[-1L]
  j_e   <- .probe_pick(inner, "effect_threshold <- hr.threshold",
                       "effect_threshold seed")
  j_c   <- .probe_pick(inner, "consistency_threshold <- hr.consistency",
                       "consistency_threshold seed")
  j_br  <- .probe_pick(inner, 'if (effect_measure %in% c("RD", "IRD")) {',
                       "measure branch")
  s_res <- inner[c(j_e, j_c, j_br)]

  # -- the sync block (top level; absent before the Step 3 edit) ------------
  s_sync <- list()
  if (isTRUE(sync)) {
    i_id <- .probe_pick(top, ".fs_identity_scale <- ", "identity-scale flag")
    i_s1 <- .probe_pick(top, "effect.threshold <- if (.fs_identity_scale)",
                        "sync c1")
    i_s2 <- .probe_pick(top, "consistency.threshold <- if (.fs_identity_scale)",
                        "sync c2")
    i_sy <- .probe_pick(top, '"effect.threshold", "consistency.threshold"',
                        "sync call")
    s_sync <- top[c(i_id, i_s1, i_s2, i_sy)]
  }

  body_exprs <- c(
    list(quote(outcome_type <- match.arg(
      outcome_type, c("survival", "binary", "continuous", "count")))),
    list(s_em),
    s_flags[1:2], s_pre_flags, s_flags[3:4], s_pair,
    list(quote(args_call_all <- list(
      effect.threshold      = effect.threshold,
      consistency.threshold = consistency.threshold,
      hr.threshold          = hr.threshold,
      hr.consistency        = hr.consistency,
      outcome_type          = outcome_type,
      effect_measure        = effect_measure,
      subgroup_method       = subgroup_method))),
    list(quote(effect_threshold <- NULL),
         quote(consistency_threshold <- NULL)),
    list(as.call(c(quote(`if`), quote(outcome_type != "survival"),
                   as.call(c(quote(`{`), s_res))))),
    s_sync,
    list(quote(.probe_record(
      outcome_type, effect_measure, hr.threshold, hr.consistency,
      effect_threshold, consistency_threshold,
      user_set_threshold, user_set_consistency, args_call_all)))
  )

  f <- as.function(c(
    fm[c("effect.threshold", "consistency.threshold",
         "hr.threshold", "hr.consistency")],
    alist(outcome_type = "survival", effect_measure = NULL,
          subgroup_method = "consistency", quiet = FALSE),
    as.call(c(quote(`{`), body_exprs))))
  environment(f) <- asNamespace("forestsearch")
  f
}

# ---------------------------------------------------------------------------
# The record compared between parent and replicate.
#
# `screening` / `consistency` are on the comparison scale the search uses;
# the naturals and `scale` mirror threshold_config (forestsearch_main.R,
# SECTION 2B "Store resolved config for the return object").
# ---------------------------------------------------------------------------
.probe_record <- function(outcome_type, effect_measure,
                          hr.threshold, hr.consistency,
                          effect_threshold, consistency_threshold,
                          user_set_threshold, user_set_consistency,
                          args_call_all) {
  if (outcome_type == "survival") {
    out <- list(screening           = log(hr.threshold),
                consistency         = log(max(hr.consistency, 0.001)),
                screening_natural   = hr.threshold,
                consistency_natural = hr.consistency,
                scale               = "log")
  } else {
    is_identity <- effect_measure %in% c("RD", "IRD", "MD")
    out <- list(
      screening           = effect_threshold,
      consistency         = consistency_threshold,
      screening_natural   = if (is_identity) effect_threshold
                            else exp(effect_threshold),
      consistency_natural = if (is_identity) consistency_threshold
                            else exp(consistency_threshold),
      scale               = if (is_identity) "identity" else "log")
  }
  out$user_set_threshold   <- user_set_threshold
  out$user_set_consistency <- user_set_consistency
  out$args_call_all        <- args_call_all
  out
}

.probe_eval <- function(resolver, supplied) {
  warns <- character(0)
  msgs  <- character(0)
  res <- withCallingHandlers(
    tryCatch(do.call(resolver, supplied, quote = TRUE),
             error = function(e) structure(list(error = conditionMessage(e)),
                                           class = "probe_error")),
    warning = function(w) {
      warns <<- c(warns, .probe_warn_tag(conditionMessage(w)))
      invokeRestart("muffleWarning")
    },
    message = function(m) {
      msgs <<- c(msgs, sub("\n$", "", conditionMessage(m)))
      invokeRestart("muffleMessage")
    })
  if (inherits(res, "probe_error")) {
    return(list(screening = NA_real_, consistency = NA_real_,
                screening_natural = NA_real_, consistency_natural = NA_real_,
                scale = NA_character_, args_call_all = NULL,
                warnings = "", messages = paste(msgs, collapse = "|"),
                n_messages = length(msgs), error = res$error))
  }
  res$warnings   <- if (length(warns)) paste(unique(warns), collapse = "|") else ""
  res$messages   <- paste(msgs, collapse = "|")
  res$n_messages <- length(msgs)
  res$error      <- NA_character_
  res
}

.probe_warn_tag <- function(msg) {
  if (grepl("^effect.threshold", msg))      "c1-ratio-scale-remap"
  else if (grepl("^consistency.threshold", msg)) "c2-ratio-scale-remap"
  else "other"
}

# ---------------------------------------------------------------------------
# The matrix
# ---------------------------------------------------------------------------
.PROBE_ESTIMANDS <- list(
  list(key = "survival",       outcome_type = "survival",   effect_measure = NULL),
  list(key = "binary-unset",   outcome_type = "binary",     effect_measure = NULL),
  list(key = "binary-RD",      outcome_type = "binary",     effect_measure = "RD"),
  list(key = "binary-OR",      outcome_type = "binary",     effect_measure = "OR"),
  list(key = "continuous-MD",  outcome_type = "continuous", effect_measure = "MD"),
  list(key = "count-IRR",      outcome_type = "count",      effect_measure = "IRR"),
  list(key = "count-IRD",      outcome_type = "count",      effect_measure = "IRD")
)

.probe_custom_values <- function(key, values = "custom") {
  if (identical(values, "c2gt")) {
    # c2 strictly above c1, per estimand scale.  The ratio pair is the task
    # document's worked example (c1 = 0.90, c2 = 1.00).
    return(switch(key,
      "binary-RD"     = c(c1 = 0.03, c2 = 0.07),
      "count-IRD"     = c(c1 = 0.01, c2 = 0.02),
      "continuous-MD" = c(c1 = 10,   c2 = 30),
      c(c1 = 0.90, c2 = 1.00)))
  }
  if (identical(values, "c2high")) {
    # c2 alone, above the DEFAULT c1 (1.25 on the ratio scale).
    return(switch(key,
      "binary-RD"     = c(c1 = NA, c2 = 0.07),
      "count-IRD"     = c(c1 = NA, c2 = 0.02),
      "continuous-MD" = c(c1 = NA, c2 = 30),
      c(c1 = NA, c2 = 1.50)))
  }
  if (identical(values, "c1derive")) {
    # c1 supplied alone at a value whose 0.80 multiple is not the old default.
    return(switch(key,
      "binary-RD"     = c(c1 = 0.07, c2 = NA),
      "count-IRD"     = c(c1 = 0.02, c2 = NA),
      "continuous-MD" = c(c1 = 30,   c2 = NA),
      c(c1 = 0.90, c2 = NA)))
  }
  switch(key,
    "binary-RD"     = c(c1 = 0.07, c2 = 0.03),
    "count-IRD"     = c(c1 = 0.02, c2 = 0.01),
    "continuous-MD" = c(c1 = 30,   c2 = 10),
    c(c1 = 1.5, c2 = 1.2))
}

.probe_supplied_list <- function(est, spelling, subset, values = "default",
                                 method = "consistency") {
  s <- list(outcome_type = est$outcome_type)
  if (!is.null(est$effect_measure)) s$effect_measure <- est$effect_measure
  if (!identical(method, "consistency")) s$subgroup_method <- method
  if (identical(spelling, "none") || identical(subset, "neither")) return(s)

  fm <- formals(forestsearch)
  v  <- if (identical(values, "default"))
          c(c1 = eval(fm[["hr.threshold"]]), c2 = eval(fm[["hr.consistency"]]))
        else .probe_custom_values(est$key, values)

  # "mixed": BOTH spellings of one quantity, at disagreeing values -- the new
  # and the legacy name differing by one custom step.
  if (identical(spelling, "mixed")) {
    # New spelling at the custom value, legacy spelling at the package
    # default: two spellings of one quantity, disagreeing.
    cus <- .probe_custom_values(est$key, "custom")
    def <- c(c1 = eval(fm[["hr.threshold"]]), c2 = eval(fm[["hr.consistency"]]))
    if (identical(subset, "c1")) {
      s[["effect.threshold"]] <- unname(cus[["c1"]])
      s[["hr.threshold"]]     <- unname(def[["c1"]])
    } else {
      s[["consistency.threshold"]] <- unname(cus[["c2"]])
      s[["hr.consistency"]]        <- unname(def[["c2"]])
    }
    return(s)
  }

  nm_c1 <- if (identical(spelling, "legacy")) "hr.threshold"   else "effect.threshold"
  nm_c2 <- if (identical(spelling, "legacy")) "hr.consistency" else "consistency.threshold"
  if (!identical(subset, "c2")) s[[nm_c1]] <- unname(v[["c1"]])
  if (subset %in% c("both", "c2")) s[[nm_c2]] <- unname(v[["c2"]])
  s
}

.probe_null0 <- function(x, empty = "") if (is.null(x)) empty else x

.probe_fmt <- function(x) {
  if (is.null(x) || length(x) != 1L) return(NA_character_)
  if (is.character(x)) return(x)
  if (is.na(x)) return(NA_character_)
  formatC(as.numeric(x), format = "g", digits = 17)
}

.probe_grid <- function(extended = FALSE) {
  g <- expand.grid(est = seq_along(.PROBE_ESTIMANDS),
                   spelling = c("none", "legacy", "new"),
                   subset   = c("neither", "c1", "both"),
                   values   = c("default", "custom"),
                   method   = "consistency",
                   stringsAsFactors = FALSE)
  keep <- !(g$spelling == "none" & (g$subset != "neither" | g$values != "default"))
  keep <- keep & !(g$spelling != "none" & g$subset == "neither")
  g <- g[keep, , drop = FALSE]
  if (!isTRUE(extended)) return(g)

  # ---- Directive A cells ---------------------------------------------------
  # (A) c1 alone, at a value whose 0.80 multiple differs from the old default
  #     -- the derivation cells.
  gA <- expand.grid(est = seq_along(.PROBE_ESTIMANDS),
                    spelling = c("legacy", "new"), subset = "c1",
                    values = "c1derive", method = "consistency",
                    stringsAsFactors = FALSE)
  # (B) c2 alone, honoured against the DEFAULT c1 (1.25 on the ratio scale);
  #     "c2high" puts it above that default, where the error rule applies.
  gB <- expand.grid(est = seq_along(.PROBE_ESTIMANDS),
                    spelling = c("legacy", "new"), subset = "c2",
                    values = c("default", "custom", "c2high"),
                    method = "consistency", stringsAsFactors = FALSE)
  # (C) both supplied with c2 > c1, under each subgroup_method.  dina and grf
  #     must stay inert: no error, no derivation.
  gC <- expand.grid(est = seq_along(.PROBE_ESTIMANDS),
                    spelling = c("legacy", "new"), subset = "both",
                    values = "c2gt",
                    method = c("consistency", "dina", "grf"),
                    stringsAsFactors = FALSE)
  # (D) both spellings of one quantity, disagreeing.
  gD <- expand.grid(est = seq_along(.PROBE_ESTIMANDS),
                    spelling = "mixed", subset = c("c1", "c2"),
                    values = "disagree", method = "consistency",
                    stringsAsFactors = FALSE)
  rbind(g, gA, gB, gC, gD)
}

# ---------------------------------------------------------------------------
# Surface guard
#
# .probe_resolver() rebuilds forestsearch()'s threshold resolution by lifting
# statements out of body(forestsearch) and matching their DEPARSED TEXT.  That
# holds on the devtools::load_all() surface these tests are gated on (CLAUDE.md:
# "verification per task is the task's own acceptance tests", run with
# load_all() + test_file()).  It does NOT hold against the installed,
# byte-compiled package that `R CMD check` runs tests against: the resolver
# builds but every cell errors, which surfaced as 14 failures and an ERROR in
# the 2026-09-18 certification run (dev/reports/CHECK_ascran_2026-09-18.md).
#
# A check must show these tests as clean SKIPS, never as erroring cells.  Every
# probe-driven test calls skip_if_probe_unavailable() first.
# ---------------------------------------------------------------------------

.probe_state <- new.env(parent = emptyenv())

#' Can the source-lifted resolver be built AND evaluated here?
#'
#' Cached: the answer is a property of the surface, not of the cell.  The
#' maximal variant is tested (`sync = TRUE, validate = TRUE`); the others lift
#' a subset of the same statements, so they cannot fail where it succeeds.
#'
#' @return list(ok = logical, why = character).
.probe_available <- function() {
  if (!is.null(.probe_state$ok)) {
    return(list(ok = .probe_state$ok, why = .probe_state$why))
  }
  res <- tryCatch({
    r <- .probe_resolver(sync = TRUE, validate = TRUE)
    cell <- .probe_eval(r, list(outcome_type = "survival"))
    if (!is.na(cell$error)) {
      stop("resolver evaluated to an error: ", cell$error, call. = FALSE)
    }
    if (is.na(cell$screening)) {
      stop("resolver returned no resolution", call. = FALSE)
    }
    list(ok = TRUE, why = NA_character_)
  }, error = function(e) list(ok = FALSE, why = conditionMessage(e)))
  .probe_state$ok  <- res$ok
  .probe_state$why <- res$why
  res
}

#' Skip, with a clear reason, when the probe cannot run on this surface.
skip_if_probe_unavailable <- function() {
  st <- .probe_available()
  if (!isTRUE(st$ok)) {
    testthat::skip(paste0(
      "threshold probe unavailable on this surface -- it lifts statements out ",
      "of body(forestsearch) by deparsed text and cannot match them against ",
      "the installed, byte-compiled package.  These tests are gated on ",
      "devtools::load_all() per CLAUDE.md.  Cause: ", st$why))
  }
  invisible(TRUE)
}


#' Run the replicate-equality probe.
#'
#' @param sync TRUE to include the SECTION 2B-ii sync statements when building
#'   the resolver -- i.e. to probe the edited tree.
#' @param validate TRUE to include the SECTION 1A3 threshold-pair statements
#'   (Directive A).
#' @param extended TRUE to run the Directive A cells as well as the 63 sync
#'   cells.  The 63 keep their identity, so the sync task's assertions on them
#'   are unaffected.
#' @return a data.frame, one row per cell of the matrix.
probe_threshold_sync <- function(sync = FALSE, validate = FALSE,
                                 extended = FALSE) {
  resolver <- .probe_resolver(sync = sync, validate = validate)
  g <- .probe_grid(extended = extended)

  rows <- lapply(seq_len(nrow(g)), function(i) {
    est <- .PROBE_ESTIMANDS[[g$est[i]]]
    sup <- .probe_supplied_list(est, g$spelling[i], g$subset[i], g$values[i],
                                g$method[i])

    par <- .probe_eval(resolver, sup)

    # The replay: every formal supplied, exactly as args_FS_boot is built.
    rep <- if (is.null(par$args_call_all)) par
           else .probe_eval(resolver, par$args_call_all)

    flds <- c("screening", "consistency", "screening_natural",
              "consistency_natural", "scale")
    pv <- vapply(flds, function(f) .probe_fmt(par[[f]]), character(1))
    rv <- vapply(flds, function(f) .probe_fmt(rep[[f]]), character(1))

    a <- par$args_call_all
    data.frame(
      estimand = est$key,
      spelling = g$spelling[i],
      subset   = if (g$spelling[i] == "none") "neither" else g$subset[i],
      values   = if (g$spelling[i] == "none") "-" else g$values[i],
      method   = g$method[i],
      parent_screening   = pv[["screening"]],
      parent_consistency = pv[["consistency"]],
      parent_scr_natural = pv[["screening_natural"]],
      parent_con_natural = pv[["consistency_natural"]],
      parent_scale       = pv[["scale"]],
      parent_error       = par$error,
      parent_warnings    = par$warnings,
      parent_messages    = .probe_null0(par$messages),
      n_parent_messages  = .probe_null0(par$n_messages, 0L),
      rep_screening      = rv[["screening"]],
      rep_consistency    = rv[["consistency"]],
      rep_scr_natural    = rv[["screening_natural"]],
      rep_con_natural    = rv[["consistency_natural"]],
      rep_scale          = rv[["scale"]],
      rep_error          = rep$error,
      rep_warnings       = rep$warnings,
      rep_messages       = .probe_null0(rep$messages),
      n_rep_messages     = .probe_null0(rep$n_messages, 0L),
      acall_effect.threshold =
        .probe_fmt(if (is.null(a)) NULL else a$effect.threshold),
      acall_consistency.threshold =
        .probe_fmt(if (is.null(a)) NULL else a$consistency.threshold),
      violation = !identical(unname(pv), unname(rv)) ||
                  !identical(par$error, rep$error),
      stringsAsFactors = FALSE)
  })

  out <- do.call(rbind, rows)
  rownames(out) <- NULL
  out
}

Try the forestsearch package in your browser

Any scripts or data that you put into this service are public.

forestsearch documentation built on Nov. 15, 2026, 9:06 a.m.