R/q_residuals.R

Defines functions q_limit influence_table pca_parts plot_influence dmodx q_residuals

Documented in dmodx plot_influence q_residuals

#' @title Hotelling's T-squared and Q Residuals of a PCA Model
#'
#' @author Christian L. Goueguel
#'
#' @description
#' Computes, for each sample, the two distances used to monitor a principal
#' component analysis (PCA) model: Hotelling's \eqn{T^2}, the distance
#' within the model plane of the first `ncomp` components, and the Q residual
#' (squared prediction error, SPE), the squared distance to that plane, with
#' their limits at one or more confidence levels.
#'
#' @details
#' With scores \eqn{t_{ia}} and eigenvalues \eqn{\lambda_a}, and \eqn{k} = `ncomp`
#' components,
#' \deqn{T^2_i = \sum_{a=1}^{k} \frac{t_{ia}^2}{\lambda_a}, \qquad
#'   Q_i = \lVert x_i - \hat{x}_i \rVert^2}
#' where \eqn{\hat{x}_i} is the reconstruction of the (centered and scaled)
#' sample from the \eqn{k} components. A high \eqn{T^2} is an extreme but
#' well-modeled sample (for example, a high concentration); a high Q is a
#' sample the model does not describe (another matrix, a contamination, an
#' instrumental problem).
#'
#' For the samples of the model, \eqn{T^2} and its limits come from
#' [hotelling_t2()] (F or Beta distribution, `t2_method`). For new samples,
#' the limit is that of a new observation,
#' \eqn{k(n+1)(n-1)/(n(n-k))\,F(k, n-k)}. The limit of Q is that of Jackson
#' and Mudholkar (1979), from the eigenvalues of the components left out, or
#' Box's (1954) scaled chi-square approximation (`method = "box"`); both
#' tend to be slightly conservative. The Jackson-Mudholkar limit depends on
#' a power \eqn{h_0} of Q computed from these eigenvalues; when \eqn{h_0} is
#' close to zero (a few large eigenvalues followed by a long tail of small
#' ones, common for spectra), its limit as \eqn{h_0 \to 0}, a lognormal
#' approximation, is used. Samples are classified at the highest confidence
#' level.
#'
#' These are classical estimates, themselves affected by outliers: see
#' [robpca()] and [plot_outlier_map()] for robust score and orthogonal
#' distances. [dmodx()] gives the residual distance as a standard deviation,
#' relative to that of the calibration samples.
#'
#' @param model A [stats::prcomp()] fit that kept all its components (the
#'   default of `prcomp()`), or a numeric matrix or data frame, on which a
#'   PCA is fitted with `center` and `scale`.
#' @param ncomp The number of components of the model.
#' @param newdata Optional new samples (a matrix or data frame with the
#'   variables of the model), whose distances are computed instead of those
#'   of the calibration samples.
#' @param conf_level The confidence level(s) of the limits: one or more
#'   values between 0 and 1. Default is 0.975.
#' @param method The limit of Q: `"jackson"` (default, Jackson-Mudholkar) or
#'   `"box"`.
#' @param t2_method The distribution of the \eqn{T^2} limit of the samples
#'   of the model: `"f"` (default) or `"beta"` (see [hotelling_t2()]).
#' @param center,scale Passed to [stats::prcomp()] when `model` is data.
#'   Default is `TRUE` and `FALSE`.
#' @param k `r lifecycle::badge("deprecated")` Use `ncomp`.
#'
#' @return A tibble of class `specproc_influence`, with one row per sample:
#'   `sample` (row number), `t2`, its limits at each confidence level (in %,
#'   e.g. `t2_limit_97.5`), `q`, its limits (`q_limit_97.5`, ...) and `outlier`,
#'   the type of the sample at the highest confidence level: `"regular"`,
#'   `"extreme"` (high \eqn{T^2} only), `"residual"` (high Q only) or
#'   `"both"`. Draw it with [plot_influence()].
#'
#' @references
#'  - Jackson, J.E., Mudholkar, G.S. (1979). Control procedures for
#'    residuals associated with principal component analysis.
#'    Technometrics, 21(3):341-349.
#'  - Box, G.E.P. (1954). Some theorems on quadratic forms applied in the
#'    study of analysis of variance problems, I. Annals of Mathematical
#'    Statistics, 25(2):290-302.
#'  - Nomikos, P., MacGregor, J.F. (1995). Multivariate SPC charts for
#'    monitoring batch processes. Technometrics, 37(1):41-59.
#'
#' @seealso [dmodx()], [plot_influence()], [hotelling_t2()], [robpca()]
#' @export q_residuals
#'
#' @examples
#' if (rlang::is_installed("HotellingEllipse", version = "1.3.0")) {
#'   data(forageLIBS)
#'   # the 380-430 nm window (Ca II H and K lines)
#'   wl <- suppressWarnings(as.numeric(names(forageLIBS)))
#'   spectra <- forageLIBS[which(wl > 380 & wl < 430)]
#'   pca <- stats::prcomp(spectra)
#'   influence <- q_residuals(pca, ncomp = 3)
#'   influence[influence$outlier != "regular", ]
#'   plot_influence(influence, label = forageLIBS$Measurement)
#'   # several limits
#'   plot_influence(q_residuals(pca, ncomp = 3, conf_level = c(0.95, 0.99)))
#' }
q_residuals <- function(model, ncomp, newdata = NULL, conf_level = 0.975, method = "jackson",
                        t2_method = "f", center = TRUE, scale = FALSE, k = deprecated()) {
  ncomp <- renamed_arg(k, ncomp, "q_residuals", "k", "ncomp")
  method <- match.arg(method, c("jackson", "box"))
  parts <- pca_parts(model, ncomp, newdata, conf_level, t2_method, center, scale)
  q <- parts$sq_residuals
  rest <- parts$lambda[-seq_len(parts$ncomp)]
  q_limits <- stats::setNames(vapply(parts$conf_level, function(l) q_limit(rest, l, method),
                                     numeric(1)), parts$labels)
  influence_table(parts, "q", q, q_limits)
}

#' @title Distance to the Model (DModX) of a PCA Model
#'
#' @author Christian L. Goueguel
#'
#' @description
#' Computes the distance of each sample to a principal component analysis
#' (PCA) model in the space of the variables (DModX): the
#' residual standard deviation of the sample, normalized by that of the
#' calibration samples, with its limits at one or more confidence levels,
#' and Hotelling's \eqn{T^2} for the influence plot.
#'
#' @details
#' With the residuals \eqn{e_{ij}} of the \eqn{K} variables after \eqn{k}
#' components (`ncomp`), for \eqn{N} calibration samples (and \eqn{A_0 = 1} if the
#' data are centered),
#' \deqn{s_i = \sqrt{\frac{\sum_j e_{ij}^2}{K - k}}\,\sqrt{\frac{N}{N - k -
#'   A_0}}, \qquad s_0 = \sqrt{\frac{\sum_i \sum_j e_{ij}^2}{(N - k -
#'   A_0)(K - k)}}}
#' (the correction factor \eqn{\sqrt{N / (N - k - A_0)}} applies to the
#' calibration samples only). DModX is \eqn{s_i / s_0} (normalized, default)
#' or \eqn{s_i}. Its squared ratio follows an F distribution with
#' \eqn{\nu} and \eqn{(N - k - A_0)\nu} degrees of freedom, so the limit of
#' the normalized DModX is \eqn{\sqrt{F_{1-\alpha}}} (times \eqn{s_0} for the
#' absolute DModX).
#'
#' The nominal degrees of freedom are \eqn{\nu = K - k} (`df = "nominal"`), as
#' if the residuals of the variables were independent. For spectra, with far more (correlated)
#' channels than samples, this gives a limit close to 1 that flags a large
#' share of ordinary samples. With `df = "effective"` (default), \eqn{\nu} is
#' the effective number of residual dimensions,
#' \eqn{\theta_1^2 / \theta_2} from the eigenvalues \eqn{\lambda_a} of the
#' components left out (\eqn{\theta_j = \sum_{a > k} \lambda_a^j}, as in
#' Box's approximation of Q), which is \eqn{K - k} when the residuals are
#' independent with equal variances. DModX is then consistent with the Q
#' residuals of [q_residuals()], of which it is a scaled square root.
#'
#' @inheritParams q_residuals
#' @param normalized A logical: DModX relative to the residual standard
#'   deviation of the calibration samples (`TRUE`, default), or absolute.
#' @param df The degrees of freedom of the limit: `"effective"` (default) or
#'   `"nominal"` (see details).
#'
#' @return A tibble of class `specproc_influence`, with one row per sample:
#'   `sample`, `t2` and its limits (as for [q_residuals()]), `dmodx`, its
#'   limits at each confidence level (`dmodx_limit_97.5`, ...), and `outlier`,
#'   the type of the sample at the highest confidence level. Draw it with
#'   [plot_influence()].
#'
#' @references
#'  - Eriksson, L., Johansson, E., Kettaneh-Wold, N., Trygg, J., Wikström,
#'    C., Wold, S. (2006). Multi- and Megavariate Data Analysis, Part I, 2nd
#'    ed. Umetrics Academy, Umeå.
#'  - Box, G.E.P. (1954). Some theorems on quadratic forms applied in the
#'    study of analysis of variance problems, I. Annals of Mathematical
#'    Statistics, 25(2):290-302.
#'
#' @seealso [q_residuals()], [plot_influence()], [hotelling_t2()]
#' @export dmodx
#'
#' @examples
#' if (rlang::is_installed("HotellingEllipse", version = "1.3.0")) {
#'   data(forageLIBS)
#'   # the 380-430 nm window (Ca II H and K lines)
#'   wl <- suppressWarnings(as.numeric(names(forageLIBS)))
#'   spectra <- forageLIBS[which(wl > 380 & wl < 430)]
#'   pca <- stats::prcomp(spectra)
#'   d <- dmodx(pca, ncomp = 3)
#'   d[d$outlier != "regular", ]
#'   plot_influence(d, label = forageLIBS$Measurement)
#' }
dmodx <- function(model, ncomp, newdata = NULL, conf_level = 0.975, normalized = TRUE,
                  df = "effective", t2_method = "f", center = TRUE, scale = FALSE, k = deprecated()) {
  ncomp <- renamed_arg(k, ncomp, "dmodx", "k", "ncomp")
  check_flag(normalized, "normalized")
  df <- match.arg(df, c("effective", "nominal"))
  parts <- pca_parts(model, ncomp, newdata, conf_level, t2_method, center, scale)
  a <- parts$ncomp
  n <- parts$n
  n_var <- parts$n_var
  a0 <- if (parts$centered) 1 else 0
  if (n - a - a0 < 1 || n_var <= a) {
    stop("Too few samples or variables for DModX with ", a, " components.", call. = FALSE)
  }
  s0 <- sqrt(parts$calibration_ss / ((n - a - a0) * (n_var - a)))
  correction <- if (parts$new) 1 else sqrt(n / (n - a - a0))
  s <- sqrt(parts$sq_residuals / (n_var - a)) * correction
  rest <- parts$lambda[-seq_len(a)]
  rest <- rest[rest > 0]
  nu <- if (df == "nominal") n_var - a else sum(rest)^2 / sum(rest^2)
  crit <- sqrt(stats::qf(parts$conf_level, nu, (n - a - a0) * nu))
  value <- if (normalized) s / s0 else s
  limits <- stats::setNames(if (normalized) crit else crit * s0, parts$labels)
  out <- influence_table(parts, "dmodx", value, limits)
  attr(out, "normalized") <- normalized
  attr(out, "df") <- df
  out
}

#' @title Influence Plot of a PCA Model
#'
#' @author Christian L. Goueguel
#'
#' @description
#' Plots the residual distance of each sample to a PCA model (Q residual or
#' DModX) against its Hotelling's \eqn{T^2}, with their limits at each
#' confidence level, from [q_residuals()] or [dmodx()]. It is drawn like the
#' outlier map of a robust PCA, [plot_outlier_map()].
#'
#' @details
#' The limits at the highest confidence level (dashed lines) divide the plot
#' into four types of samples:
#'  - **regular** samples (bottom left);
#'  - **extreme** samples (bottom right): a high \eqn{T^2} but well
#'    described by the model, for example a high concentration;
#'  - **residual** samples (top left): not described by the model (another
#'    matrix, a contamination, an instrumental problem);
#'  - samples beyond **both** limits, the most harmful.
#'
#' The limits at lower confidence levels are dotted, dot-dashed, ... lines.
#'
#' With `relative = TRUE`, each distance is divided by its limit at the
#' highest confidence level, so both limits are at 1 whatever the model,
#' which puts plots of different models on the same scale.
#'
#' @param x The result of [q_residuals()] or [dmodx()].
#' @param label The names of the samples: a vector with one value per
#'   sample. By default, their row numbers.
#' @param labels The number of most outlying samples to label (beyond a
#'   limit at the highest confidence level). Default is 3; use 0 for no
#'   labels.
#' @param relative If `TRUE`, plot the distances divided by their limits at
#'   the highest confidence level. Default is `FALSE`.
#' @param shade If `TRUE`, shade the three outlying regions in grey, darker
#'   for more harmful samples (extreme, residual, both), and name them in
#'   their corners. Default is `FALSE`.
#' @param log If `TRUE`, use logarithmic axes, which spread out the regular
#'   samples when a few are far away. Zero distances are drawn at the
#'   smallest positive distance. Default is `FALSE`.
#' @param color_by The colors of the points: `"type"` (default), by type,
#'   or `"distance"`, by their distance relative to the limits,
#'   \eqn{\max(T^2/T^2_{lim}, Q/Q_{lim})}, on a rainbow scale from dark red to
#'   blue, as in [plot_outlier_map()].
#' @param title The plot title.
#' @param ... Further arguments passed to [ggplot2::geom_point()] to style
#'   the points, such as `stroke` (default 0.4) or `colour` (the outline,
#'   `"black"`).
#' @param xlab,ylab The axis titles. Default are the names of the distances.
#' @param base_size The size of the text, in points. Default is 11.
#' @param point_size The size of the points (default 2.2), or a numeric
#'   vector with one value per sample, such as the concentration of an
#'   element, to vary the size of the points, with a legend.
#' @param point_alpha The opacity of the points, between 0 and 1 (default
#'   0.85).
#' @param colour_by `r lifecycle::badge("deprecated")` Use `color_by`.
#'
#' @return A ggplot object.
#' @seealso [q_residuals()], [dmodx()], [plot_outlier_map()]
#' @export plot_influence
#'
#' @examplesIf rlang::is_installed("HotellingEllipse", version = "1.3.0")
#' data(forageLIBS)
#' pca <- stats::prcomp(forageLIBS[-(1:14)])
#' influence <- q_residuals(pca, ncomp = 3, conf_level = c(0.95, 0.99))
#' plot_influence(influence, label = forageLIBS$Measurement)
#' # the same options as plot_outlier_map(): distances relative to the limits,
#' # shaded outlying regions, logarithmic axes and colors by distance
#' plot_influence(influence, relative = TRUE, shade = TRUE, log = TRUE, color_by = "distance")
#'
#' # a single limit at 97.5%, with the outlying regions shaded
#' influence_975 <- q_residuals(pca, ncomp = 3, conf_level = 0.975)
#' plot_influence(influence_975, label = forageLIBS$Measurement, shade = TRUE)
#'
plot_influence <- function(x, label = NULL, labels = 3, relative = FALSE, shade = FALSE,
                           log = FALSE, title = NULL, ..., color_by = c("type", "distance"),
                           xlab = NULL, ylab = NULL, base_size = 11, point_size = 2.2,
                           point_alpha = 0.85, colour_by = deprecated()) {
  if (!inherits(x, "specproc_influence")) {
    stop("'x' must be returned by q_residuals() or dmodx().", call. = FALSE)
  }
  size_label <- rlang::as_label(rlang::enquo(point_size))
  check_plot_text(title, xlab, ylab, base_size)
  if (!is.null(label) && length(label) != nrow(x)) {
    stop("'label' must have one value per sample (", nrow(x), ").", call. = FALSE)
  }
  check_count(labels, "labels", lower = 0)
  color_by <- match.arg(renamed_arg(colour_by, color_by, "plot_influence", "colour_by",
                                    "color_by"), c("type", "distance"))
  check_map_flags(relative, shade, log)
  distance <- attr(x, "distance") %||% "q"
  levels <- attr(x, "conf_labels")
  new <- isTRUE(attr(x, "new"))
  df <- data.frame(
    id = as.character(if (is.null(label)) x$sample else label), x = x$t2, y = x[[distance]],
    type = x$outlier, set = if (new) "new" else "calibration", stringsAsFactors = FALSE
  )
  cuts <- data.frame(
    level = paste0(levels, "%"),
    x = vapply(levels, function(l) x[[paste0("t2_limit_", l)]][1], numeric(1)),
    y = vapply(levels, function(l) x[[paste0(distance, "_limit_", l)]][1], numeric(1))
  )
  if (is.null(title)) {
    title <- sprintf("Influence plot (%d components)%s", attr(x, "ncomp"), if (new) ", new samples" else "")
  }
  y_lab <- if (distance == "q") "Q residual (SPE)" else if (isTRUE(attr(x, "normalized"))) {
    "DModX (normalized)"
  } else {
    "DModX"
  }
  distance_map(
    df, cuts = cuts, labels = labels, relative = relative, shade = shade, log = log,
    color_by = color_by,
    palette = c(regular = "grey55", extreme = "#1b9e77", residual = "#d95f02", both = "#e7298a"),
    regions = c("extreme", "residual", "both"),
    x_lab = list(expression("Hotelling's" ~ T^2), expression("Reduced Hotelling's" ~ T^2)),
    y_lab = c(y_lab, paste("Reduced", y_lab)),
    title = title,
    point = map_point_args(rlang::enquos(...), nrow(df), "", point_size, point_alpha, size_label),
    show_set = FALSE, style = list(xlab = xlab, ylab = ylab, base_size = base_size)
  )
}

# ---- internals ---------------------------------------------------------------

# T-squared and squared residuals of the calibration samples or of new ones.
# The model keeps all its components, so the squared residual of a
# calibration sample is the sum of its squared scores on the components left
# out: the data need not be reconstructed.
pca_parts <- function(model, ncomp, newdata, conf_level, t2_method, center, scale) {
  conf_level <- check_conf_level(conf_level)
  t2_method <- match.arg(t2_method, c("f", "beta"))
  if (!inherits(model, "prcomp")) {
    model <- stats::prcomp(as_numeric_matrix(model, "model"), center = center, scale. = scale)
  }
  rank <- ncol(model$x)
  if (length(model$sdev) > rank) {
    stop("'model' must keep all its components (do not set `rank.` in prcomp()).", call. = FALSE)
  }
  check_count(ncomp, "ncomp")
  if (ncomp >= rank) {
    stop("'ncomp' must be smaller than the number of components (", rank, ").", call. = FALSE)
  }
  n <- nrow(model$x)
  lambda <- model$sdev^2
  p <- model$rotation[, seq_len(ncomp), drop = FALSE]
  calibration_sq <- rowSums(model$x[, -seq_len(ncomp), drop = FALSE]^2)
  labels <- conf_label(conf_level)
  if (is.null(newdata)) {
    sq_residuals <- calibration_sq
    t2 <- hotelling_t2(as.data.frame(model$x[, seq_len(ncomp), drop = FALSE]),
                       columns = colnames(model$x)[seq_len(ncomp)], conf_level = conf_level,
                       method = t2_method)
    t2_values <- t2$t2
    t2_limits <- vapply(labels, function(l) t2[[paste0("limit_", l)]][1], numeric(1))
  } else {
    x <- as_numeric_matrix(newdata, "newdata")
    if (!is.null(rownames(model$rotation)) && !is.null(colnames(x))) {
      missing_cols <- setdiff(rownames(model$rotation), colnames(x))
      if (length(missing_cols) > 0) {
        stop("'newdata' lacks ", length(missing_cols), " variable(s) of the model, e.g. `",
             missing_cols[1], "`.", call. = FALSE)
      }
      x <- x[, rownames(model$rotation), drop = FALSE]
    }
    x <- scale(x, center = if (isFALSE(model$center)) FALSE else model$center,
               scale = if (isFALSE(model$scale)) FALSE else model$scale)
    scores <- x %*% p
    t2_values <- unname(rowSums(sweep(scores^2, 2, lambda[seq_len(ncomp)], "/")))
    t2_limits <- vapply(conf_level, function(l) t2_limit(l, ncomp, n, "new"), numeric(1))
    sq_residuals <- rowSums((x - scores %*% t(p))^2)
  }
  list(ncomp = ncomp, n = n, n_var = nrow(model$rotation), lambda = lambda,
       sq_residuals = unname(sq_residuals), calibration_ss = sum(calibration_sq), t2 = t2_values,
       t2_limits = stats::setNames(unname(t2_limits), labels), conf_level = conf_level,
       labels = labels, new = !is.null(newdata), centered = !isFALSE(model$center))
}

# The tibble of an influence analysis, classified at the highest level.
influence_table <- function(parts, distance, value, limits) {
  out <- tibble::tibble(sample = seq_along(value), t2 = parts$t2)
  for (l in parts$labels) out[[paste0("t2_limit_", l)]] <- parts$t2_limits[[l]]
  out[[distance]] <- unname(value)
  for (l in parts$labels) out[[paste0(distance, "_limit_", l)]] <- unname(limits[[l]])
  top <- parts$labels[length(parts$labels)]
  high_t2 <- out$t2 > parts$t2_limits[[top]]
  high_d <- out[[distance]] > limits[[top]]
  out$outlier <- factor(ifelse(high_t2 & high_d, "both", ifelse(high_t2, "extreme",
                                                                 ifelse(high_d, "residual", "regular"))),
                        levels = c("regular", "extreme", "residual", "both"))
  attr(out, "ncomp") <- parts$ncomp
  attr(out, "new") <- parts$new
  attr(out, "conf_level") <- parts$conf_level
  attr(out, "conf_labels") <- parts$labels
  attr(out, "distance") <- distance
  class(out) <- c("specproc_influence", class(out))
  out
}

# Upper limit of Q from the eigenvalues of the components left out.
q_limit <- function(rest, level, method) {
  rest <- rest[rest > 0]
  if (length(rest) == 0) return(0)
  theta <- vapply(1:3, function(j) sum(rest^j), numeric(1))
  if (method == "box") {
    g <- theta[2] / theta[1]
    h <- theta[1]^2 / theta[2]
    return(g * stats::qchisq(level, h))
  }
  h0 <- 1 - 2 * theta[1] * theta[3] / (3 * theta[2]^2)
  z <- stats::qnorm(level)
  # Near h0 = 0 (a few large eigenvalues followed by a long tail of small
  # ones, common for spectra), the power 1 / h0 is numerically unstable: use
  # its limit as h0 tends to 0, a lognormal approximation. Elsewhere, h0 keeps
  # its sign in the first term, so that the limit is continuous in h0.
  if (abs(h0) < 1e-3) {
    return(theta[1] * exp(z * sqrt(2 * theta[2]) / theta[1] - theta[2] / theta[1]^2))
  }
  base <- h0 * z * sqrt(2 * theta[2]) / theta[1] + 1 + theta[2] * h0 * (h0 - 1) / theta[1]^2
  theta[1] * base^(1 / h0)
}

Try the specProc package in your browser

Any scripts or data that you put into this service are public.

specProc documentation built on Nov. 20, 2026, 5:11 p.m.