Nothing
# =============================================================================
# test_report_narrative_asm.R -- Tests for assemble_report_narrative(),
# .calibrate_significance(), .extract_kb_references(), and the
# section_hint= seam in es_advise().
#
# Coverage:
# NARR-01: Provider contacted once per LLM section (3 calls), independent of
# format count (no format arg in assembler)
# NARR-02: Four section keys present + section_sources + report_mode
# NARR-03: References deduped by key, alpha by author, KB-sourced
# NARR-04: .calibrate_significance() four tiers + NA/non-numeric guard
# NARR-05: Joint-hypothesis caveat present in every assembled robustness
# OFFLINE-02: report_mode and section_sources reflect per-section source
# section_hint=: Injection scopes prompt; NULL path unchanged
#
# Fixtures: .make_test_diag() from helper-advice-fixtures.R
# No network, no API keys required.
# =============================================================================
# ---- Helper: mock provider ---------------------------------------------------
.make_mock_provider <- function() {
# Returns a mock provider with call tracking.
# complete() returns a minimal valid JSON response (list with $text field)
# matching the Advice schema that es_advise() expects.
env <- new.env(parent = emptyenv())
env$call_count <- 0L
env$prompts <- character(0L)
list(
source = "mock",
complete = function(prompt, schema) {
env$call_count <- env$call_count + 1L
env$prompts <- c(env$prompts, prompt)
# Return list with $text containing valid Advice JSON
list(
text = '{"interpretation":"Mock LLM narrative prose for testing purposes.","recommendations":[],"caveats":[]}'
)
},
get_call_count = function() env$call_count,
get_prompts = function() env$prompts
)
}
# ---- NARR-02: Four section keys + metadata (offline path) --------------------
test_that("assemble_report_narrative(provider=NULL) returns list with four section keys", {
diag <- .make_test_diag()
result <- EventStudy:::assemble_report_narrative(diag, provider = NULL)
expect_true(is.list(result))
expect_true("exec_summary" %in% names(result))
expect_true("data_methods" %in% names(result))
expect_true("results" %in% names(result))
expect_true("robustness" %in% names(result))
expect_true("section_sources" %in% names(result))
expect_true("report_mode" %in% names(result))
})
test_that("assemble_report_narrative(provider=NULL) section keys are non-empty character scalars", {
diag <- .make_test_diag()
result <- EventStudy:::assemble_report_narrative(diag, provider = NULL)
for (key in c("exec_summary", "data_methods", "results", "robustness")) {
val <- result[[key]]
expect_true(is.character(val), info = paste("key:", key, "should be character"))
expect_length(val, 1L)
expect_true(nzchar(val), info = paste("key:", key, "should be non-empty"))
}
})
test_that("assemble_report_narrative(provider=NULL) report_mode is 'offline'", {
diag <- .make_test_diag()
result <- EventStudy:::assemble_report_narrative(diag, provider = NULL)
expect_identical(result$report_mode, "offline")
})
test_that("assemble_report_narrative(provider=NULL) all section_sources are 'offline'", {
diag <- .make_test_diag()
result <- EventStudy:::assemble_report_narrative(diag, provider = NULL)
for (key in c("exec_summary", "data_methods", "results", "robustness")) {
src <- result$section_sources[[key]]
expect_identical(src, "offline",
info = paste("section_sources$", key, "should be 'offline'"))
}
})
# ---- NARR-05: Joint-hypothesis caveat in every robustness section -----------
test_that("NARR-05: joint-hypothesis caveat present in offline robustness section", {
diag <- .make_test_diag()
result <- EventStudy:::assemble_report_narrative(diag, provider = NULL)
rob <- result$robustness
expect_true(
grepl("joint", rob, ignore.case = TRUE),
info = paste("robustness section should contain 'joint' (caveat). Got:", substr(rob, 1, 200))
)
})
# ---- NARR-01: Provider contacted once per LLM section (not per format) ------
test_that("NARR-01: mock provider contacted exactly 3 times (one per LLM section)", {
diag <- .make_test_diag()
mock <- .make_mock_provider()
result <- EventStudy:::assemble_report_narrative(diag, provider = mock)
# Three LLM-narrated sections: exec_summary, results, robustness
expect_equal(mock$get_call_count(), 3L,
info = "Provider must be called once per LLM section (3), not per format")
})
test_that("NARR-01: data_methods is always offline regardless of provider", {
diag <- .make_test_diag()
mock <- .make_mock_provider()
result <- EventStudy:::assemble_report_narrative(diag, provider = mock)
expect_identical(result$section_sources$data_methods, "offline",
info = "data_methods must always be sourced offline (section_sources$data_methods == 'offline')")
})
# ---- OFFLINE-02: report_mode and section_sources with mock provider ----------
test_that("OFFLINE-02: report_mode is 'ai' when any section used the provider", {
diag <- .make_test_diag()
mock <- .make_mock_provider()
result <- EventStudy:::assemble_report_narrative(diag, provider = mock)
expect_identical(result$report_mode, "ai",
info = "report_mode should be 'ai' when at least one section used the LLM")
})
test_that("OFFLINE-02: section_sources marks LLM-narrated sections as 'ai'", {
diag <- .make_test_diag()
mock <- .make_mock_provider()
result <- EventStudy:::assemble_report_narrative(diag, provider = mock)
# These three sections are LLM-narrated when provider is available
for (key in c("exec_summary", "results", "robustness")) {
src <- result$section_sources[[key]]
expect_identical(src, "ai",
info = paste("section_sources$", key, "should be 'ai' with mock provider"))
}
# data_methods is always offline
expect_identical(result$section_sources$data_methods, "offline")
})
# ---- section_hint= wiring: prompt scoping observable ------------------------
test_that("section_hint= injects section name and 'Write ONLY the' into prompt", {
diag <- .make_test_diag()
mock <- .make_mock_provider()
EventStudy:::assemble_report_narrative(diag, provider = mock)
prompts <- mock$get_prompts()
expect_length(prompts, 3L)
# Each of the three LLM calls must scope to its section
# Map by order: exec_summary (1st), results (2nd), robustness (3rd)
expected_sections <- c("exec_summary", "results", "robustness")
for (i in seq_along(expected_sections)) {
section <- expected_sections[[i]]
prompt <- prompts[[i]]
expect_true(
grepl("Write ONLY the", prompt, fixed = TRUE),
info = paste("Prompt", i, "for", section, "must contain 'Write ONLY the'")
)
expect_true(
grepl(section, prompt, fixed = TRUE),
info = paste("Prompt", i, "must contain section name:", section)
)
}
})
test_that("NULL section_hint path: es_advise report_writing prompt has no 'Write ONLY the'", {
diag <- .make_test_diag()
mock <- .make_mock_provider()
# Call es_advise directly without section_hint (backward-compat path)
# Capture the prompt by using a modified mock that records it
env <- new.env(parent = emptyenv())
env$captured_prompt <- NULL
capture_mock <- list(
source = "capture",
complete = function(prompt, schema) {
env$captured_prompt <- prompt
list(text = '{"interpretation":"test","recommendations":[],"caveats":[]}')
}
)
# es_advise without section_hint (default NULL)
suppressWarnings(
EventStudy::es_advise(diag, task_type = "report_writing", provider = capture_mock)
)
prompt <- env$captured_prompt
if (!is.null(prompt)) {
expect_false(
grepl("Write ONLY the", prompt, fixed = TRUE),
info = "NULL section_hint path must NOT inject 'Write ONLY the' (backward-compat)"
)
}
})
test_that("section_hint= argument exists on es_advise formals", {
expect_true(
"section_hint" %in% names(formals(EventStudy::es_advise)),
info = "es_advise() must have an additive section_hint= argument"
)
})
# ---- Provider error -> fallback to offline for that section -----------------
test_that("provider error causes graceful fallback to offline for that section", {
diag <- .make_test_diag()
error_mock <- list(
source = "error_mock",
complete = function(prompt, schema) {
stop("Simulated provider failure")
}
)
# Should not throw; result should be complete with offline fallback. The
# provider is called once per section that attempts an online path, each
# emitting its own "Provider call failed" warning -- since expect_warning()
# only consumes ONE matching warning per call, muffle every matching
# occurrence explicitly (by message) and assert at least one fired, rather
# than leaving the 2nd/3rd occurrence to leak (C10, 2026-09-24).
provider_warn_count <- 0L
result <- withCallingHandlers(
expect_no_error(
EventStudy:::assemble_report_narrative(diag, provider = error_mock)
),
warning = function(w) {
if (grepl("Simulated provider failure", conditionMessage(w), fixed = TRUE)) {
provider_warn_count <<- provider_warn_count + 1L
invokeRestart("muffleWarning")
}
}
)
expect_gt(provider_warn_count, 0L)
# All four section keys must be non-empty
for (key in c("exec_summary", "data_methods", "results", "robustness")) {
expect_true(nzchar(result[[key]]), info = paste("key:", key, "must be non-empty after fallback"))
}
# All section_sources should be "offline" (since provider failed)
for (key in c("exec_summary", "data_methods", "results", "robustness")) {
expect_identical(result$section_sources[[key]], "offline",
info = paste("section_sources$", key, "should be 'offline' after error fallback"))
}
})
# ---- NARR-04: .calibrate_significance() ------------------------------------
test_that(".calibrate_significance() returns 'strongly significant' for p < 0.01", {
expect_identical(EventStudy:::.calibrate_significance(0.005), "strongly significant")
expect_identical(EventStudy:::.calibrate_significance(0.001), "strongly significant")
expect_identical(EventStudy:::.calibrate_significance(0.0001), "strongly significant")
})
test_that(".calibrate_significance() returns 'significant' for 0.01 <= p < 0.05", {
expect_identical(EventStudy:::.calibrate_significance(0.03), "significant")
expect_identical(EventStudy:::.calibrate_significance(0.01), "significant")
expect_identical(EventStudy:::.calibrate_significance(0.049), "significant")
})
test_that(".calibrate_significance() returns 'marginally significant' for 0.05 <= p < 0.10", {
expect_identical(EventStudy:::.calibrate_significance(0.08), "marginally significant")
expect_identical(EventStudy:::.calibrate_significance(0.05), "marginally significant")
expect_identical(EventStudy:::.calibrate_significance(0.099), "marginally significant")
})
test_that(".calibrate_significance() returns 'not statistically significant' for p >= 0.10", {
expect_identical(EventStudy:::.calibrate_significance(0.5), "not statistically significant")
expect_identical(EventStudy:::.calibrate_significance(0.10), "not statistically significant")
expect_identical(EventStudy:::.calibrate_significance(1.0), "not statistically significant")
})
test_that(".calibrate_significance() returns 'not evaluable' for NA", {
expect_identical(EventStudy:::.calibrate_significance(NA), "not evaluable")
expect_identical(EventStudy:::.calibrate_significance(NA_real_), "not evaluable")
})
test_that(".calibrate_significance() returns 'not evaluable' for non-numeric", {
expect_identical(EventStudy:::.calibrate_significance("0.03"), "not evaluable")
expect_identical(EventStudy:::.calibrate_significance(NULL), "not evaluable")
})
# ---- NARR-03: .extract_kb_references() -------------------------------------
test_that(".extract_kb_references() returns a list", {
diag <- .make_test_diag()
refs <- EventStudy:::.extract_kb_references(diag)
expect_true(is.list(refs))
})
test_that(".extract_kb_references() deduplicates by citation key", {
# Build a diagnostics where at least one rule fires (KB-NORM-PATELL needs
# shapiro_p > 0.05 in >= 70% of events -- our fixture has shapiro_p =
# c(0.12, 0.08, 0.15, 0.20, 0.10), all > 0.05, so 100% pass -> fires)
diag <- .make_test_diag()
refs <- EventStudy:::.extract_kb_references(diag)
keys <- vapply(refs, function(r) r$key, character(1L))
expect_equal(length(keys), length(unique(keys)),
info = "References must not contain duplicate keys")
})
test_that(".extract_kb_references() deduplicates when multiple rules share citation key", {
# The KB has rules KB-NONNORM-NONPAR, KB-AC-WARN, KB-SMALL-N all citing
# BrownWarner1985. Build diagnostics that fire at least two of these:
# - KB-NONNORM-NONPAR: shapiro_p < 0.05 in >= 50% of events
# - KB-SMALL-N: n_valid_events < 30
# - KB-AC-WARN: acf1 > 0.15 in >= 30% of events
multi_fire_diag <- structure(
list(
meta = list(
n_events_total = 5L,
n_events_shown = 5L,
n_events_summarized = 0L,
event_ids_shown = 1:5
),
estimation_window = list(
r2 = c(0.5, 0.4, 0.6, 0.3, 0.5),
sigma = c(0.01, 0.01, 0.01, 0.01, 0.01),
degree_of_freedom = c(120L, 120L, 120L, 120L, 120L),
# acf1 > 0.15 in 3/5 = 60% -> fires KB-AC-WARN (threshold 30%)
acf1 = c(0.20, 0.18, 0.01, 0.25, 0.01),
# shapiro_p < 0.05 in 3/5 = 60% -> fires KB-NONNORM-NONPAR (threshold 50%)
shapiro_p = c(0.01, 0.03, 0.80, 0.02, 0.70),
dw_stat = c(2.0, 2.0, 2.0, 2.0, 2.0),
ljung_box_p = c(0.3, 0.3, 0.3, 0.3, 0.3)
),
event_window = list(
ar_t = c(1.2, 0.8, 2.1, -0.3, 1.5),
ar_p = c(0.23, 0.42, 0.04, 0.77, 0.13),
car_t = c(2.1, 1.5, 3.0, 0.2, 2.5),
car_p = c(0.04, 0.13, 0.003, 0.84, 0.01),
final_car = c(0.03, 0.02, 0.05, 0.001, 0.04)
),
cross_sectional = list(
n_events = 5L,
n_valid_events = 5L, # < 30 -> fires KB-SMALL-N
car_iqr = 0.02,
car_sd = 0.015,
n_overlap_pairs = 0L,
any_overlap = FALSE
),
contract_state = list(
is_fitted = rep(TRUE, 5L),
na_ar_count = rep(0L, 5L),
na_est_count = rep(0L, 5L),
insufficient_obs = rep(FALSE, 5L),
zero_var_index = rep(FALSE, 5L)
),
aggregate_summary = NULL
),
class = "es_diagnostics"
)
refs <- EventStudy:::.extract_kb_references(multi_fire_diag)
# At least 1 rule should fire (KB-SMALL-N fires for n_valid_events=5 < 30)
if (length(refs) > 0L) {
keys <- vapply(refs, function(r) r$key, character(1L))
# KEY INVARIANT: no duplicate keys regardless of how many rules share a key
expect_equal(length(keys), length(unique(keys)),
info = "Dedup by key: duplicate citation keys must be collapsed to one")
# Check alphabetical author order
authors <- vapply(refs, function(r) r$author, character(1L))
expect_equal(authors, sort(authors),
info = "References must be sorted alphabetically by author")
}
})
test_that(".extract_kb_references() returns references ordered alphabetically by author", {
diag <- .make_test_diag()
refs <- EventStudy:::.extract_kb_references(diag)
if (length(refs) >= 2L) {
authors <- vapply(refs, function(r) r$author, character(1L))
expect_equal(authors, sort(authors),
info = "References must be ordered alphabetically by author")
}
})
test_that(".extract_kb_references() returns empty list for all-NA diagnostics", {
# Build diagnostics that will fire no rules
degenerate_diag <- structure(
list(
meta = list(
n_events_total = 0L,
n_events_shown = 0L,
n_events_summarized = 0L,
event_ids_shown = integer(0L)
),
estimation_window = list(
r2 = NA_real_,
sigma = NA_real_,
degree_of_freedom = NA_real_,
acf1 = NA_real_,
shapiro_p = NA_real_,
dw_stat = NA_real_,
ljung_box_p = NA_real_
),
event_window = list(
ar_t = NA_real_,
ar_p = NA_real_,
car_t = NA_real_,
car_p = NA_real_,
final_car = NA_real_
),
cross_sectional = list(
n_events = 0L,
n_valid_events = 0L,
car_iqr = NA_real_,
car_sd = NA_real_,
n_overlap_pairs = 0L,
any_overlap = FALSE
),
contract_state = list(
is_fitted = logical(0L),
na_ar_count = integer(0L),
na_est_count = integer(0L),
insufficient_obs = logical(0L),
zero_var_index = logical(0L)
),
aggregate_summary = NULL
),
class = "es_diagnostics"
)
result <- expect_no_error(EventStudy:::.extract_kb_references(degenerate_diag))
expect_true(is.list(result))
# Either empty or all refs from rules that fired (should be 0 with degenerate)
# The key invariant: no error thrown
})
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.