Nothing
# Tests for Expected Information Gain and PPV-range analyses
test_that("EIG is ~0 for uninformative responses", {
n <- 400
set.seed(1)
ids <- sample(c("suspect", "filler", "reject"), n / 2, replace = TRUE)
confs <- sample(c(30, 60, 90), n / 2, replace = TRUE)
data <- data.frame(
target_present = rep(c(TRUE, FALSE), each = n / 2),
identification = rep(ids, 2),
confidence = rep(confs, 2)
)
result <- compute_eig(data, prior_guilt = 0.5)
expect_lt(result$eig, 0.01)
})
test_that("EIG equals prior entropy (1 bit) for perfect separation at prior 0.5", {
n <- 200
data <- data.frame(
target_present = rep(c(TRUE, FALSE), each = n / 2),
identification = rep(c("suspect", "reject"), each = n / 2),
confidence = rep(90, n)
)
result <- compute_eig(data, prior_guilt = 0.5)
expect_equal(result$eig, 1, tolerance = 1e-8)
expect_equal(result$prior_entropy, 1, tolerance = 1e-8)
})
test_that("EIG is bounded by prior entropy and non-negative", {
data(lineup_example)
result <- compute_eig(lineup_example, confidence_bins = c(0, 60, 80, 100))
expect_gte(result$eig, 0)
expect_lte(result$eig, result$prior_entropy + 1e-10)
})
test_that("ppv corrections are ordered: none <= effective <= nominal for fair lineups", {
data(lineup_example)
lineup_example$identification[
!lineup_example$target_present & lineup_example$identification == "suspect"
] <- "filler"
bins <- c(0, 60, 80, 100)
ppv_none <- ppv_by_confidence(lineup_example, correction = "none",
confidence_bins = bins)
ppv_eff <- ppv_by_confidence(lineup_example, correction = "effective",
confidence_bins = bins)
ppv_nom <- ppv_by_confidence(lineup_example, correction = "nominal",
confidence_bins = bins)
# "none" treats every mistaken ID as an innocent-suspect ID (worst case),
# so it gives the lowest PPV; nominal spreads errors over the full lineup
# (best case); effective sits between them.
expect_lte(ppv_none$overall_ppv, ppv_eff$overall_ppv)
expect_lte(ppv_eff$overall_ppv, ppv_nom$overall_ppv + 1e-10)
})
test_that("effective correction computes a plausible effective size (regression)", {
# Regression for the bug where the fallback tabulated the values of the
# per-position count vector, collapsing E' to ~1.38 regardless of data.
data(lineup_example)
lineup_example$identification[
!lineup_example$target_present & lineup_example$identification == "suspect"
] <- "filler"
ppv_eff <- ppv_by_confidence(lineup_example, correction = "effective",
confidence_bins = c(0, 60, 80, 100))
esizes <- ppv_eff$ppv_data$effective_size
esizes <- esizes[!is.na(esizes)]
expect_true(all(esizes > 2))
expect_true(all(esizes <= 6 + 1e-10))
})
test_that("designated innocent-suspect IDs supersede size approximations", {
data(lineup_example)
bins <- c(0, 60, 80, 100)
nominal <- ppv_by_confidence(lineup_example, correction = "nominal",
confidence_bins = bins)
effective <- ppv_by_confidence(lineup_example, correction = "effective",
confidence_bins = bins)
uncorrected <- ppv_by_confidence(lineup_example, correction = "none",
confidence_bins = bins)
expect_equal(nominal$overall_ppv, effective$overall_ppv)
expect_equal(effective$overall_ppv, uncorrected$overall_ppv)
expect_equal(nominal$innocent_suspect_method, "designated")
})
test_that("EIG splits filler outcomes when no innocent suspect is designated", {
dat <- data.frame(
target_present = c(TRUE, TRUE, FALSE, FALSE),
identification = c("suspect", "reject", "filler", "reject"),
confidence = rep(80, 4)
)
prepared <- make_eig_data(dat, lineup_size = 4)
suspect <- prepared$response_data[
prepared$response_data$identification == "suspect", ]
filler <- prepared$response_data[
prepared$response_data$identification == "filler", ]
expect_equal(suspect$n_innocent, 0.25)
expect_equal(filler$n_innocent, 0.75)
expect_equal(sum(prepared$response_data$p_x_given_innocent), 1)
expect_equal(prepared$innocent_suspect_method, "estimated_from_fillers")
bayes <- make_bayes_curves(dat, lineup_size = 4, prior_grid = 0.5)
suspect_lik <- bayes$likelihoods[
bayes$likelihoods$response == "suspect", ]
expect_equal(suspect_lik$n_innocent, 0.25)
expect_equal(sum(bayes$likelihoods$p_x_given_innocent), 1)
})
test_that("innocent_id_rate helpers compute the documented formulas", {
expect_equal(innocent_id_rate_nominal(0.30, 6), 0.05)
expect_equal(innocent_id_rate_effective(0.30, 4), 0.075)
expect_equal(innocent_id_rate_uncorrected(0.30), 0.30)
expect_error(innocent_id_rate_nominal(0.30, 0))
expect_error(innocent_id_rate_effective(0.30, 0))
})
test_that("ppv_range_by_confidence returns all three corrections", {
data(lineup_example)
result <- ppv_range_by_confidence(lineup_example,
confidence_bins = c(0, 60, 80, 100))
expect_true(all(c("ppv_nominal", "ppv_effective", "ppv_none",
"ppv_range_data") %in% names(result)))
expect_true(all(c("ppv_nominal", "ppv_effective", "ppv_none") %in%
names(result$ppv_range_data)))
})
test_that("PPV keeps confidence bins that contain no target-absent trials", {
# All 90 TP suspect IDs at 100; the 10 TA suspect IDs and all rejections at 50
x <- data.frame(
target_present = rep(c(TRUE, FALSE), each = 100),
identification = c(rep("suspect", 100), rep("suspect", 10), rep("reject", 90)),
confidence = c(rep(100, 90), rep(50, 110))
)
ppv <- ppv_by_confidence(x, confidence_bins = c(0, 60, 80, 100), quiet = TRUE)
expect_equal(ppv$overall_ppv, 100 / 110)
expect_equal(ppv$overall_ppv, make_cacdata(x)$overall_accuracy)
top <- ppv$ppv_data[ppv$ppv_data$confidence == "(80,100]", ]
expect_equal(top$ppv, 1)
expect_true(is.na(top$error_rate))
# The empty middle bin is still dropped
expect_false("(60,80]" %in% ppv$ppv_data$confidence)
# A TA-only bin has PPV 0
y <- x
y$confidence[y$target_present & y$confidence == 50] <- 100
y$confidence[!y$target_present & y$identification == "suspect"] <- 70
ppv_y <- ppv_by_confidence(y, confidence_bins = c(0, 60, 80, 100), quiet = TRUE)
expect_equal(ppv_y$ppv_data$ppv[ppv_y$ppv_data$confidence == "(60,80]"], 0)
# Unbinned PPV agrees with the overall counts
expect_equal(ppv_by_confidence(x, quiet = TRUE)$overall_ppv, 100 / 110)
# The range and its plots cope with the NA rows
rng <- suppressMessages(make_ppv_range(x, confidence_bins = c(0, 60, 80, 100),
show_plots = FALSE))
expect_s3_class(plot_error_rate_conf(rng), "ggplot")
expect_s3_class(plot_ppv_range(rng), "ggplot")
})
test_that("make_bayes_curves() information gain follows the requested measure", {
dat <- data.frame(
target_present = rep(c(TRUE, FALSE), each = 100),
identification = rep(rep(c("suspect", "filler", "reject"), 2),
times = c(60, 20, 20, 10, 30, 60))
)
# Wells & Lindsay (1980), Wells et al. (2015): |posterior - prior|;
# suspect ID at prior .1 -> .4.
prob <- make_bayes_curves(dat, prior_grid = 0.1, innocent_suspect = "designated")
sus <- prob$curves[prob$curves$response == "suspect", ]
rej <- prob$curves[prob$curves$response == "reject", ]
expect_equal(prob$gain, "probability")
expect_equal(sus$posterior, 0.4)
expect_equal(sus$information_gain, 0.3)
# A rejection lowers the probability of guilt; the absolute gain is positive.
expect_lt(rej$posterior, 0.1)
expect_equal(rej$information_gain, 0.1 - rej$posterior)
# "signed" keeps the direction.
sgn <- make_bayes_curves(dat, prior_grid = 0.1, innocent_suspect = "designated",
gain = "signed")
expect_equal(sgn$curves$information_gain[sgn$curves$response == "suspect"], 0.3)
expect_equal(sgn$curves$information_gain[sgn$curves$response == "reject"],
rej$posterior - 0.1)
# Starns et al. (2023): entropy reduction; negative for the same update.
ent <- make_bayes_curves(dat, prior_grid = 0.1, innocent_suspect = "designated",
gain = "entropy")
sus_e <- ent$curves[ent$curves$response == "suspect", ]
h <- function(p) -p * log2(p) - (1 - p) * log2(1 - p)
expect_equal(sus_e$information_gain, h(0.1) - h(0.4))
expect_lt(sus_e$information_gain, 0)
expect_error(make_bayes_curves(dat, gain = "bits"))
})
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.