Nothing
#' @title Outlier Map of a Robust PCA or Robust PLS Model
#'
#' @author Christian L. Goueguel
#'
#' @description
#' Plots the orthogonal distance of each observation against its score
#' distance, for a robust PCA fitted by [robpca()], [rospca()],
#' [macropca()] or [cellpca()] (Hubert, Rousseeuw and Vanden Branden, 2005).
#' For [cellpca()], the vertical axis is the norm of the standardized
#' residuals, as in the enhanced outlier map of Centofanti, Hubert and
#' Rousseeuw. For a robust calibration model fitted by [rsimpls()] or
#' [rpcr()], it is by default
#' the regression outlier map of Hubert and Vanden Branden (2003): the
#' residual distance (with one response, the absolute standardized residual)
#' against the score distance. With `map = "score"`, it is the score outlier
#' map of the predictors: the orthogonal distance of each spectrum to the
#' model against its score distance, as for a robust PCA.
#'
#' @details
#' The dashed lines are the cut-offs of the two distances. They divide the
#' map into four types of observations:
#' - **regular** observations (bottom left);
#' - **good leverage** points (bottom right): far from the center, but close
#' to the PCA subspace, so they follow the structure of the data;
#' - **orthogonal outliers** (top left): close to the center once
#' projected, but far from the PCA subspace;
#' - **bad leverage** points (top right): far on both counts, the most
#' harmful outliers.
#'
#' In the regression outlier map of [rsimpls()] or [rpcr()], the top left
#' region holds
#' the **vertical outliers**: observations with a typical spectrum but a
#' large residual, such as a wrong reference value. Bad leverage points have
#' both an outlying spectrum and a large residual; good leverage points have
#' an outlying spectrum that the model still fits.
#'
#' New observations (`newdata`) are projected onto the model with
#' [predict()][predict.specproc_robpca] (or
#' [predict()][predict.specproc_rsimpls] for the score outlier map of an
#' [rsimpls()] or [rpcr()] fit) and shown with the calibration cut-offs,
#' which is how new spectra are screened before prediction.
#'
#' With `relative = TRUE`, each distance is divided by its cut-off (the
#' reduced score and orthogonal distances), so both cut-offs are at 1
#' whatever the model. This puts maps of different models
#' (for example, with different numbers of components) on the same scale.
#' A relative distance tells where an observation falls with respect to the
#' cut-off, not how unlikely it is: twice the cut-off is not equally rare
#' in every model.
#'
#' @param object An object returned by [robpca()], [rospca()],
#' [macropca()], [cellpca()], [rsimpls()] or [rpcr()].
#' @param newdata Optional new observations to add to the map (a numeric
#' matrix or data frame with the calibration variables). Not available for
#' the regression outlier map of [rsimpls()] and [rpcr()] fits.
#' @param labels The number of most outlying observations to label (by
#' their row names, or row numbers). Default is 3; use 0 for no labels.
#' @param relative If `TRUE`, plot the reduced distances (divided by their
#' cut-offs). Default is `FALSE`.
#' @param shade If `TRUE`, shade the three outlying regions in grey, darker
#' for more harmful observations (good leverage, orthogonal outliers, bad
#' leverage), and name them in their corners. The region of regular
#' observations stays white. Default is `FALSE`.
#' @param log If `TRUE`, use logarithmic axes, which spread out the regular
#' observations when a few are far away. Zero distances are drawn at the
#' smallest positive distance. Default is `FALSE`.
#' @param color_by The colors of the points: `"type"` (default), by
#' outlier type, or `"distance"`, by their reduced distance from the
#' origin, \eqn{\max(SD/c_{SD}, OD/c_{OD})}, on a rainbow scale from dark
#' red (the closest) to blue (the farthest). With `relative = TRUE`, where
#' both cut-offs are at 1, yellow marks the cut-offs: dark red through
#' orange for the regular observations, then green and blue for the
#' outlying ones. Otherwise, the colors spread evenly over the distances.
#' @param map For [rsimpls()] and [rpcr()] fits, the map to draw:
#' `"regression"` (default), the regression outlier map, or `"score"`,
#' the score outlier map of the predictors. Not used for robust PCA fits.
#' @param title The plot title.
#' @param ... Further arguments passed to [ggplot2::geom_point()] to style
#' the points, such as `stroke` (default 0.4) or `colour` (the outline,
#' `"black"`).
#' @param xlab,ylab The axis titles. Default are the names of the distances.
#' @param base_size The size of the text, in points. Default is 11.
#' @param point_size The size of the points (default 2.2), or a numeric
#' vector with one value per sample (the calibration samples, then those
#' of `newdata`), such as the concentration of an element, to vary the
#' size of the points, with a legend.
#' @param point_alpha The opacity of the points, between 0 and 1 (default
#' 0.85).
#' @param colour_by `r lifecycle::badge("deprecated")` Use `color_by`.
#'
#' @return A ggplot object.
#'
#' @seealso [robpca()], [rospca()], [macropca()], [rsimpls()], [rpcr()],
#' [plot_cell_map()]
#' @export
#'
#' @examples
#' spectra_id <- forageLIBS |> dplyr::select(1:2) |> names()
#' minerals <- forageLIBS |> dplyr::select(3:14) |> names()
#' set.seed(1)
#' fit <- forageLIBS |>
#' dplyr::select(-dplyr::all_of(c(spectra_id, minerals))) |>
#' center() |>
#' robpca()
#'
#' plot_outlier_map(fit, relative = TRUE, shade = TRUE, log = TRUE)
#' plot_outlier_map(fit, relative = TRUE, shade = TRUE, log = TRUE,
#' labels = 10, color_by = "distance", point_alpha = 0.6)
#'
plot_outlier_map <- function(object, newdata = NULL, labels = 3, relative = FALSE,
shade = FALSE, log = FALSE, title = NULL,
map = c("regression", "score"), ..., color_by = c("type", "distance"),
xlab = NULL, ylab = NULL, base_size = 11, point_size = 2.2,
point_alpha = 0.85, colour_by = deprecated()) {
if (!inherits(object, c("specproc_robpca", "specproc_rsimpls", "specproc_rpcr"))) {
stop("'object' must be returned by robpca(), rospca(), macropca(), cellpca(), rsimpls() ",
"or rpcr().",
call. = FALSE)
}
size_label <- rlang::as_label(rlang::enquo(point_size))
check_count(labels, "labels", lower = 0)
color_by <- match.arg(renamed_arg(colour_by, color_by, "plot_outlier_map", "colour_by",
"color_by"), c("type", "distance"))
map <- match.arg(map)
check_map_flags(relative, shade, log)
check_plot_text(title, xlab, ylab, base_size)
style <- list(xlab = xlab, ylab = ylab, base_size = base_size)
is_calibration <- inherits(object, c("specproc_rsimpls", "specproc_rpcr"))
if (is_calibration && map == "regression") {
point <- map_point_args(rlang::enquos(...), length(object$sd), "", point_size, point_alpha,
size_label)
return(regression_outlier_map(object, newdata, labels, relative, shade, log, color_by, title,
point, style))
}
# the score outlier map of an rsimpls() or rpcr() fit classifies the
# observations like a robust PCA, from its score and orthogonal distances
if (is_calibration) {
ids <- rownames(object$x_scores) %||% as.character(seq_along(object$sd))
type <- outlier_type(object$sd, object$od, object$cutoff_sd, object$cutoff_od)
} else {
ids <- rownames(object$scores) %||% as.character(seq_along(object$sd))
type <- object$outlier_type
}
df <- data.frame(id = ids, x = object$sd, y = object$od, type = type,
set = "calibration", stringsAsFactors = FALSE)
if (!is.null(newdata)) {
if (is_calibration) {
pred <- stats::predict(object, newdata, type = "scores")
pred$outlier_type <- outlier_type(pred$sd, pred$od, object$cutoff_sd, object$cutoff_od)
} else {
pred <- stats::predict(object, newdata)
}
new_ids <- rownames(newdata) %||% paste0("new", seq_len(nrow(pred)))
df <- rbind(df, data.frame(id = new_ids, x = pred$sd, y = pred$od, type = pred$outlier_type,
set = "new", stringsAsFactors = FALSE))
}
df$type <- factor(df$type, levels = levels(type))
if (is.null(title)) {
title <- if (is_calibration) {
paste0(calibration_name(object), " score outlier map (", object$ncomp, " components)")
} else {
paste0(switch(class(object)[1], specproc_robpca = "ROBPCA", specproc_rospca = "ROSPCA",
specproc_macropca = "MacroPCA", specproc_cellpca = "cellPCA"),
" outlier map (", object$ncomp, " components)")
}
}
point <- map_point_args(rlang::enquos(...), nrow(df),
if (!is.null(newdata)) ", calibration then new samples" else "",
point_size, point_alpha, size_label)
distance_map(
df, cuts = data.frame(level = "cut-off", x = object$cutoff_sd, y = object$cutoff_od),
labels = labels, relative = relative, shade = shade, log = log, color_by = color_by,
palette = c(regular = "grey55", `good leverage` = "#1b9e77",
`orthogonal outlier` = "#d95f02", `bad leverage` = "#e7298a"),
# "bad leverage" sits above the OD cut-off rather than in the top right
# corner, where the most outlying observations usually are
regions = c("good leverage", "orthogonal outliers", "bad leverage"),
x_lab = c("Score distance", "Reduced score distance"),
y_lab = if (inherits(object, "specproc_cellpca")) {
c("Norm of the standardized residuals", "Reduced norm of the standardized residuals")
} else {
c("Orthogonal distance", "Reduced orthogonal distance")
},
title = title, point = point, show_set = !is.null(newdata), style = style
)
}
#' @title Cell Map of a MacroPCA or cellPCA Fit
#'
#' @author Christian L. Goueguel
#'
#' @description
#' Shows which cells of the data deviate from a [macropca()] or [cellpca()]
#' fit: one row
#' per observation, the variables (wavelengths) along the horizontal axis,
#' and the flagged cells colored red when the observed value is higher than
#' the fit and blue when it is lower. A strip on the right shows the outlier
#' type of each observation, and a panel on top the share of flagged cells
#' at each wavelength over the mean spectrum, with the most flagged regions
#' labeled, which shows which emission lines hold the cellwise outliers.
#'
#' @details
#' **Resolution.** Spectra have many more cells than a plot has pixels, so
#' adjacent cells are combined into blocks: the numbers of rows and columns
#' of blocks are at most `resolution`, and a map smaller than that shows
#' every cell. Channels are combined only within a detector segment, and the
#' gaps between segments stay empty. The map fills the plot area, whatever
#' the numbers of observations and variables.
#'
#' **Colors.** A flagged cell with standardized residual \eqn{r} has an
#' intensity that grows with \eqn{\log(|r|/c)}, from 0.25 at the flagging
#' cut-off \eqn{c} to 1 at \eqn{10c} and beyond. The color of a block is the
#' mean of these signed intensities over its cells (0 for cells that are not
#' flagged), on a square-root scale so that blocks with a few flagged cells
#' remain visible: pale blocks have few or mixed flagged cells, saturated
#' blocks many cells that deviate strongly in the same direction.
#'
#' **Order.** With `order = "cluster"`, the observations are sorted by a
#' hierarchical clustering (Ward's method) of their flagged cells, at the
#' column resolution of the map, so that observations that deviate in the
#' same regions form bands (for example, a batch or a type of matrix).
#'
#' **Profile.** The top panel shows, for each variable, the share of the
#' (shown) observations whose cell is flagged: above zero when higher than
#' the fit, below zero when lower. The mean spectrum is drawn in grey
#' behind it, rescaled, to show whether the flagged channels are on emission
#' lines, on the continuum or in noise. It is the mean of `spectra` or, by
#' default, of the data imputed by the fit; it is left out when this mean
#' is close to zero, as for centered data (then give the raw spectra in
#' `spectra`). The dashed lines are at `threshold`, and the `labels` flagged
#' regions with the largest share (see [flagged_regions()]) are labeled with
#' their peak wavelength, or with the emission line they match.
#'
#' @param object An object returned by [macropca()] or [cellpca()].
#' @param rows,columns Optional indices or names of the rows and columns to
#' show. Default is all.
#' @param resolution The maximum numbers of rows and columns of blocks.
#' Default is `c(200, 400)`.
#' @param order The order of the rows: `"data"` (default), `"od"` for
#' decreasing orthogonal distance, which puts the most outlying
#' observations at the top, or `"cluster"` to group the observations with
#' similar flagged cells.
#' @param profile If `TRUE` (default), add the panel of the share of flagged
#' cells by variable.
#' @param spectra Optional spectra whose mean is drawn behind the profile (a
#' data frame or matrix with the variables of the model, other columns
#' being ignored, or a single named spectrum). Default is the data imputed
#' by the fit.
#' @param threshold The share of flagged observations above which channels
#' form a flagged region. Default is 0.1.
#' @param labels The number of flagged regions labeled in the profile.
#' Default is 5; use 0 for none.
#' @param lines Optional line list returned by [libs_lines()], to label the
#' regions with the emission lines they match.
#' @param tol The largest distance, in nm, between a region and a line it
#' matches. Default is 0.1.
#' @param title The plot title. Default is "MacroPCA cell map" or "cellPCA cell map".
#' @param wavelength Optional (`NULL` by default). The wavelength range
#' `c(min, max)` to show, in nm: the columns within it (of `columns`, if
#' given). Needs variables named by wavelength.
#' @param xlab,ylab The axis titles of the map. Default are
#' `"Wavelength (nm)"` (or `"Variable"`) and `"Observation"`, with the
#' order of the rows.
#' @param base_size The size of the text, in points. Default is 11.
#'
#' @return A patchwork (ggplot2) object.
#'
#' @seealso [flagged_regions()], [macropca()], [plot_outlier_map()]
#' @export
#'
#' @examples
#' \donttest{
#' data(forageLIBS)
#' # the 380-430 nm window (Ca II H and K lines), faster than all the channels
#' wl <- suppressWarnings(as.numeric(names(forageLIBS)))
#' spectra <- forageLIBS[which(wl > 380 & wl < 430)]
#' set.seed(1)
#' fit <- macropca(center(spectra), ncomp = 3)
#'
#' if (requireNamespace("patchwork", quietly = TRUE)) {
#' plot_cell_map(fit, order = "cluster")
#' # the Ca II H and K lines
#' plot_cell_map(fit, wavelength = c(390, 400), order = "od")
#' }
#' }
#'
plot_cell_map <- function(object, rows = NULL, columns = NULL, resolution = c(200, 400),
order = c("data", "od", "cluster"), profile = TRUE, spectra = NULL,
threshold = 0.1, labels = 5, lines = NULL, tol = 0.1,
title = NULL, wavelength = NULL, xlab = NULL, ylab = NULL,
base_size = 11) {
check_macropca(object)
check_plot_text(title, xlab, ylab, base_size)
title <- title %||% paste(if (inherits(object, "specproc_cellpca")) "cellPCA" else "MacroPCA",
"cell map")
order <- match.arg(order)
check_flag(profile, "profile")
if (!is.numeric(resolution) || length(resolution) != 2 || anyNA(resolution) ||
any(resolution < 1) || any(resolution %% 1 != 0)) {
stop("'resolution' must be two positive integers (rows and columns).", call. = FALSE)
}
check_count(labels, "labels", lower = 0)
rlang::check_installed("patchwork", reason = "to combine the panels of the cell map.")
heading <- split_title(title)
cells <- cell_map_data(object, rows, columns, wavelength_window(wavelength))
check_region_args(threshold, lines, tol, cells$has_wavelength)
# signed intensity of each cell: 0 unless flagged
resid <- cells$resid
flagged <- cells$flagged
cutoff <- if (any(flagged)) min(abs(resid[flagged])) else sqrt(stats::qchisq(0.99, 1))
intensity <- 0.25 + 0.75 * pmin(1, log(pmax(abs(resid), cutoff) / cutoff) / log(10))
cell <- ifelse(flagged, sign(resid) * intensity, 0)
cell[is.na(cell)] <- 0
col_id <- cell_map_columns(cells$segment, ncol(cell), resolution[2])
o <- switch(order,
data = seq_len(nrow(cell)),
od = order(object$od[cells$rows], decreasing = TRUE),
cluster = cell_map_cluster(cell, col_id)
)
blocks <- cell_map_blocks(cell[o, , drop = FALSE], cells$wavelength, col_id, resolution[1])
map <- cell_map_panel(blocks, cells$ids[o], cells$has_wavelength, order, xlab, ylab, base_size)
strip <- cell_map_strip(cells$type[o])
x_limits <- attr(blocks, "extent")[1:2]
map <- map + ggplot2::scale_x_continuous(limits = x_limits, expand = c(0, 0))
if (!profile) {
return(patchwork::wrap_plots(map, strip, widths = c(1, 0.025)) +
patchwork::plot_layout(guides = "collect") +
patchwork::plot_annotation(title = heading$title, subtitle = heading$subtitle, theme = bold_title()) &
ggplot2::theme(legend.position = "bottom", legend.box = "vertical"))
}
if (is.null(spectra)) {
# the mean of the imputed data, unless it is close to zero (centered data)
imputed <- object$imputed[cells$rows, cells$columns, drop = FALSE]
spectrum <- colMeans(imputed, na.rm = TRUE)
spread <- max(apply(imputed, 2, stats::sd, na.rm = TRUE), na.rm = TRUE)
if (!is.finite(spread) || max(abs(spectrum)) < 0.01 * spread) spectrum <- NULL
} else {
spectrum <- mean_spectrum(spectra, cells$variables)
}
regions <- find_flagged_regions(cells, threshold, lines, tol)
top <- cell_map_profile(cells, spectrum, regions, threshold, labels, x_limits, base_size)
patchwork::wrap_plots(top, patchwork::plot_spacer(), map, strip,
ncol = 2, widths = c(1, 0.025),
heights = if (labels > 0 && nrow(regions) > 0) c(1.6, 4) else c(1, 4)) +
patchwork::plot_layout(guides = "collect") +
patchwork::plot_annotation(title = heading$title, subtitle = heading$subtitle, theme = bold_title()) &
ggplot2::theme(legend.position = "bottom", legend.box = "vertical")
}
#' @title Flagged Regions of a MacroPCA or cellPCA Fit
#'
#' @author Christian L. Goueguel
#'
#' @description
#' Lists the wavelength regions where many observations have cells flagged
#' by [macropca()] or [cellpca()]: runs of adjacent channels (on the same detector segment)
#' whose share of flagged observations is at least `threshold`. These are
#' the channels that persistently deviate from the PCA fit, for example
#' emission lines affected by self-absorption, saturation or matrix
#' effects, and candidates to exclude or down-weight. [plot_cell_map()]
#' labels the regions with the largest share.
#'
#' @details
#' Runs separated by at most two channels below the threshold are merged. A
#' region is `"higher"` when at least two thirds of its flagged cells are
#' higher than the fit, `"lower"` when at most one third are, and `"mixed"`
#' otherwise. With `lines`, the lines between `start - tol` and `end + tol`
#' are the candidates, from the nearest to the peak.
#'
#' @inheritParams plot_cell_map
#'
#' @return A tibble with one row per region, sorted by decreasing share,
#' and columns `start`, `end` and `peak` (the wavelength, or variable
#' number, of the largest share), `channels` (the number of channels),
#' `share` (the largest share of flagged observations), `mean_share` and
#' `direction`. With `lines`, it also has the columns `species` and
#' `line_wavelength` of the nearest line (`NA` when none) and
#' `candidates`.
#'
#' @seealso [plot_cell_map()], [macropca()], [libs_lines()]
#' @export
#'
#' @examples
#' \donttest{
#' data(forageLIBS)
#' # the 380-430 nm window (Ca II H and K lines), faster than all the channels
#' wl <- suppressWarnings(as.numeric(names(forageLIBS)))
#' spectra <- forageLIBS[which(wl > 380 & wl < 430)]
#' set.seed(1)
#' fit <- macropca(center(spectra), ncomp = 3)
#'
#' flagged_regions(fit)
#' }
#'
flagged_regions <- function(object, threshold = 0.1, rows = NULL, columns = NULL, lines = NULL,
tol = 0.1) {
check_macropca(object)
cells <- cell_map_data(object, rows, columns)
check_region_args(threshold, lines, tol, cells$has_wavelength)
out <- find_flagged_regions(cells, threshold, lines, tol)
if (is.null(lines)) out <- out[, setdiff(names(out), c("species", "line_wavelength", "candidates"))]
rownames(out) <- NULL
tibble::as_tibble(out)
}
# ---- distance map internals ---------------------------------------------------
#
# plot_outlier_map() and plot_influence() draw the same kind of map: a
# distance within the model (x) against a distance to the model (y), with
# their cut-offs dividing the map into four types of observations.
# The regression outlier map of an rsimpls() or rpcr() fit: residual distance
# against score distance (Hubert and Vanden Branden, 2003).
regression_outlier_map <- function(object, newdata, labels, relative, shade, log, color_by,
title, point, style) {
if (!is.null(newdata)) {
stop("'newdata' is not used in the regression outlier map of rsimpls() and rpcr() ",
"fits: residuals need the responses. Use `map = \"score\"` to screen new spectra.",
call. = FALSE)
}
ids <- rownames(object$x_scores) %||% as.character(seq_along(object$sd))
df <- data.frame(id = ids, x = object$sd, y = object$rd, type = object$outlier_type,
set = "calibration", stringsAsFactors = FALSE)
if (is.null(title)) {
title <- paste0(calibration_name(object), " outlier map (", object$ncomp, " components)")
}
distance_map(
df, cuts = data.frame(level = "cut-off", x = object$cutoff_sd, y = object$cutoff_rd),
labels = labels, relative = relative, shade = shade, log = log, color_by = color_by,
palette = c(regular = "grey55", `good leverage` = "#1b9e77",
`vertical outlier` = "#d95f02", `bad leverage` = "#e7298a"),
regions = c("good leverage", "vertical outliers", "bad leverage"),
x_lab = c("Score distance", "Reduced score distance"),
y_lab = if (length(object$intercept) == 1) {
c("Absolute standardized residual", "Reduced absolute standardized residual")
} else {
c("Residual distance", "Reduced residual distance")
},
title = title, point = point, show_set = FALSE, style = style
)
}
check_map_flags <- function(relative, shade, log) {
for (arg in c("relative", "shade", "log")) {
value <- get(arg)
if (!is.logical(value) || length(value) != 1L || is.na(value)) {
stop("'", arg, "' must be TRUE or FALSE.", call. = FALSE)
}
}
}
# The arguments of geom_point() of a distance map: `point_size` and
# `point_alpha`, and those given in `...` (where `size` and `alpha` replace
# them). A size with one value per sample is returned apart (`sizes`, with
# the expression `size_name` as the legend title), to be mapped.
map_point_args <- function(dots, n, size_hint = "", point_size = 2.2, point_alpha = 0.85,
size_label = "point_size") {
args <- lapply(dots, rlang::eval_tidy)
if (length(args) && (is.null(names(args)) || any(names(args) == ""))) {
stop("The arguments in '...' must be named, such as 'stroke = 0.8'.", call. = FALSE)
}
names(args)[names(args) == "color"] <- "colour"
size_arg <- "point_size"
if (!is.null(args$size)) {
point_size <- args$size
size_arg <- "size"
size_label <- rlang::as_label(dots$size)
args$size <- NULL
}
if (!is.null(args$alpha)) {
point_alpha <- args$alpha
args$alpha <- NULL
}
check_alpha(point_alpha, "point_alpha")
sizes <- size_name <- NULL
if (length(point_size) > 1) {
if (!is.numeric(point_size) || length(point_size) != n) {
stop("`", size_arg, "` must be a number, or a numeric vector with one value per sample (",
n, size_hint, ").", call. = FALSE)
}
sizes <- point_size
size_name <- size_label
} else {
check_size(point_size, size_arg)
args$size <- point_size
}
args <- utils::modifyList(list(colour = "black", stroke = 0.4, alpha = point_alpha), args)
list(args = args, sizes = sizes, size_name = size_name)
}
# Draws a distance map.
# - df: columns id, x, y, type (a factor of the four types, regular first)
# and set ("calibration" or "new").
# - cuts: the cut-offs, columns level, x and y, one row per confidence
# level, the highest (which classifies the observations) last.
# - regions: the names of the three outlying regions (bottom right, top
# left, top right), shown with `shade`.
# - x_lab, y_lab: the axis titles of the distances and of the reduced ones.
# - style: the axis titles given by the user (xlab, ylab), and base_size.
distance_map <- function(df, cuts, labels, relative, shade, log, color_by, palette, regions,
x_lab, y_lab, title, point, show_set, style) {
top <- cuts[nrow(cuts), ]
cut_x <- max(top$x, .Machine$double.eps)
cut_y <- max(top$y, .Machine$double.eps)
df$set <- factor(df$set, levels = c("calibration", "new"))
# most outlying: largest distance relative to the cut-offs, which is the
# distance from the origin in reduced distances (1 on the cut-offs)
df$radius <- pmax(df$x / cut_x, df$y / cut_y)
df$label <- ""
if (labels > 0) {
most <- order(df$radius, decreasing = TRUE)[seq_len(min(labels, nrow(df)))]
most <- most[df$radius[most] > 1]
df$label[most] <- df$id[most]
}
if (!is.null(point$sizes)) df$.size <- point$sizes
x_lab <- style$xlab %||% x_lab[[1 + relative]]
y_lab <- style$ylab %||% y_lab[[1 + relative]]
base_size <- style$base_size
if (relative) {
df$x <- df$x / cut_x
df$y <- df$y / cut_y
cuts$x <- cuts$x / cut_x
cuts$y <- cuts$y / cut_y
}
if (log) {
# plot log10 distances on linear axes labelled in the original units, so
# the shaded regions can still reach the panel edges (-Inf); zeros are
# drawn at the smallest positive distance
for (v in c("x", "y")) {
positive <- df[[v]][df[[v]] > 0]
floor_v <- if (length(positive)) min(positive) else 1
df[[v]] <- log10(pmax(df[[v]], floor_v))
cuts[[v]] <- log10(cuts[[v]])
}
}
cuts$level <- factor(cuts$level, levels = rev(cuts$level))
shade_x <- cuts$x[nrow(cuts)]
shade_y <- cuts$y[nrow(cuts)]
# keys of the absent types too
point_args <- utils::modifyList(list(show.legend = c(fill = TRUE, linetype = FALSE)), point$args)
point_args$mapping <- ggplot2::aes(fill = .data$type, shape = .data$set)
if (color_by == "distance") {
point_args$mapping$fill <- quote(.data$radius)
# the farthest points on top
df <- df[order(df$radius), , drop = FALSE]
}
if (!is.null(point$size_name)) point_args$mapping$size <- quote(.data$.size)
p <- ggplot2::ggplot(df, ggplot2::aes(x = .data$x, y = .data$y))
if (shade) {
rects <- data.frame(
xmin = c(shade_x, -Inf, shade_x), xmax = c(Inf, shade_x, Inf),
ymin = c(-Inf, shade_y, shade_y), ymax = c(shade_y, Inf, Inf),
alpha = c(0.06, 0.13, 0.22)
)
# translucent so that the grid shows through
p <- p +
ggplot2::geom_rect(data = rects, ggplot2::aes(xmin = .data$xmin, xmax = .data$xmax,
ymin = .data$ymin, ymax = .data$ymax),
alpha = rects$alpha, fill = "grey20", inherit.aes = FALSE) +
ggplot2::annotate("text", x = c(Inf, -Inf, Inf), y = c(-Inf, Inf, shade_y), label = regions,
hjust = c(1.05, -0.05, 1.05), vjust = c(-0.6, 1.6, -0.6),
size = 0.77 * base_size / ggplot2::.pt, fontface = "italic",
colour = "grey35")
}
# the highest level dashed, the others dotted, dot-dashed, ...
linetypes <- stats::setNames(
c("dashed", "dotted", "dotdash", "longdash", "twodash", "solid")[seq_len(nrow(cuts))],
levels(cuts$level)
)
p <- p +
ggplot2::geom_vline(data = cuts, ggplot2::aes(xintercept = .data$x, linetype = .data$level),
colour = "grey30", key_glyph = "path") +
ggplot2::geom_hline(data = cuts, ggplot2::aes(yintercept = .data$y, linetype = .data$level),
colour = "grey30", key_glyph = "path") +
do.call(ggplot2::geom_point, point_args) +
repel_labels(base_size) +
(if (color_by == "type") {
ggplot2::scale_fill_manual(values = palette, drop = FALSE, name = NULL,
guide = ggplot2::guide_legend(order = 1, override.aes = list(shape = 21, size = 2.5)))
} else {
distance_fill_scale(range(df$radius, na.rm = TRUE), anchored = relative)
}) +
# the kinds of samples only matter with new samples
ggplot2::scale_shape_manual(values = c(calibration = 21, new = 24), name = NULL, drop = TRUE,
guide = if (show_set) {
ggplot2::guide_legend(order = 2, override.aes = list(size = 2.5))
} else "none") +
# the levels only matter with several of them
ggplot2::scale_linetype_manual(values = linetypes, name = "Limits",
guide = if (nrow(cuts) > 1) {
ggplot2::guide_legend(order = 4, override.aes = list(colour = "grey30"),
keywidth = ggplot2::unit(0.9, "cm"))
} else "none") +
ggplot2::labs(x = x_lab, y = y_lab, title = title) +
ggplot2::theme_bw(base_size = base_size) +
ggplot2::theme(legend.position = "bottom") +
compact_legend(base_size)
if (!is.null(point$size_name)) {
p <- p + ggplot2::theme(legend.box = "vertical")
p <- p + ggplot2::scale_size_continuous(range = c(1.2, 6), name = point$size_name, breaks = three_breaks,
guide = ggplot2::guide_legend(order = 3))
}
# room for the labels above the highest points
x_args <- list(expand = ggplot2::expansion(mult = 0.05))
y_args <- list(expand = ggplot2::expansion(mult = c(0.05, if (any(df$label != "")) 0.08 else 0.05)))
if (log) {
log_breaks <- function(limits) log10(scales::breaks_log(n = 6)(10^limits))
log_labels <- function(breaks) vapply(10^breaks, format, character(1), digits = 3, scientific = FALSE)
log_args <- list(breaks = log_breaks, labels = log_labels, minor_breaks = NULL)
x_args <- c(x_args, log_args)
y_args <- c(y_args, log_args)
}
finish_title(p +
do.call(ggplot2::scale_x_continuous, x_args) +
do.call(ggplot2::scale_y_continuous, y_args))
}
# Labels of the points of a distance map (column `label`, empty for the
# unlabelled points), moved apart so that they overlap neither each other
# nor the points, with a segment to their point when they are moved far.
# The fixed seed gives the same layout at each drawing.
repel_labels <- function(base_size = 11) {
ggrepel::geom_text_repel(ggplot2::aes(label = .data$label),
size = 0.77 * base_size / ggplot2::.pt, colour = "grey20",
box.padding = 0.35, point.padding = 0.2, min.segment.length = 0.2,
segment.colour = "grey50", max.overlaps = Inf, seed = 1,
na.rm = TRUE)
}
# Rainbow of the distances of plot_outlier_map(): dark red at the origin,
# through orange, yellow at the cut-offs (distance 1), then green and blue.
radial_rainbow <- c("#67001f", "#b2182b", "#f46d43", "#fdae61", "#fdd835", "#d9ef8b",
"#66bd63", "#1a9850", "#4575b4", "#313695")
# The fill scale of the distances. Anchored (relative distances, both
# cut-offs at 1): the colors below yellow spread over [0, 1], the others over
# [1, the largest distance], and 1 is labeled. Otherwise, the colors spread
# evenly over the range of the distances.
distance_fill_scale <- function(range, anchored) {
if (!anchored) {
return(ggplot2::scale_fill_gradientn(
colours = radial_rainbow, name = "Reduced distance",
guide = ggplot2::guide_colourbar(order = 1, barwidth = 12, barheight = 0.5, title.vjust = 0.9)
))
}
top <- max(range[2], 1.5)
stops <- c(0, 0.35, 0.65, 0.85, 1, 1 + (top - 1) * c(0.15, 0.35, 0.55, 0.8, 1))
breaks <- pretty(c(0, top), n = 5)
breaks <- sort(c(1, breaks[breaks >= 0 & breaks <= top & abs(breaks - 1) > 0.12 * top]))
ggplot2::scale_fill_gradientn(
colours = radial_rainbow, values = stops / top, limits = c(0, top),
breaks = breaks, labels = ifelse(breaks == 1, "1 (cut-off)", format(breaks, trim = TRUE)),
name = "Reduced distance",
guide = ggplot2::guide_colourbar(order = 1, barwidth = 12, barheight = 0.5, title.vjust = 0.9)
)
}
# ---- cell map internals ------------------------------------------------------
check_macropca <- function(object) {
if (!inherits(object, c("specproc_macropca", "specproc_cellpca"))) {
stop("'object' must be returned by macropca() or cellpca().", call. = FALSE)
}
invisible(object)
}
check_region_args <- function(threshold, lines, tol, has_wavelength) {
check_number(threshold, "threshold", lower = 0, upper = 1, lower_open = TRUE)
check_number(tol, "tol", lower = 0)
if (!is.null(lines)) {
check_lines_table(lines)
if (!has_wavelength) {
stop("Matching 'lines' needs variables named by wavelength.", call. = FALSE)
}
}
invisible(TRUE)
}
# The selected residuals and flags, with the row ids, outlier types,
# variable names, wavelengths and detector segments. `window` keeps the
# columns within a wavelength range.
cell_map_data <- function(object, rows, columns, window = NULL) {
resid <- object$std_resid
n <- nrow(resid)
ids <- rownames(resid) %||% as.character(seq_len(n))
variables <- colnames(resid) %||% attr(object, "variables") %||% as.character(seq_len(ncol(resid)))
keep_rows <- cell_map_index(rows, ids, n, "rows")
keep_cols <- cell_map_index(columns, variables, ncol(resid), "columns")
if (!is.null(window)) {
all_wavelengths <- names_to_wavelength(variables)
if (is.null(all_wavelengths)) {
stop("'wavelength' needs variables named by wavelength.", call. = FALSE)
}
keep_cols <- keep_cols[in_window(all_wavelengths[keep_cols], window)]
if (length(keep_cols) < 2) {
stop("Fewer than 2 variables in the 'wavelength' range.", call. = FALSE)
}
}
variables <- variables[keep_cols]
wavelength <- names_to_wavelength(variables)
has_wavelength <- length(wavelength) == length(variables)
if (!has_wavelength) wavelength <- seq_along(variables)
list(
resid = resid[keep_rows, keep_cols, drop = FALSE],
flagged = object$flagged_cells[keep_rows, keep_cols, drop = FALSE],
rows = keep_rows, columns = keep_cols, ids = ids[keep_rows],
type = object$outlier_type[keep_rows], variables = variables,
wavelength = wavelength, has_wavelength = has_wavelength,
segment = if (has_wavelength) wavelength_segments(wavelength) else rep(1L, length(wavelength))
)
}
cell_map_index <- function(select, names, n, arg) {
if (is.null(select)) return(seq_len(n))
idx <- if (is.character(select)) match(select, names) else select
if (!is.numeric(idx) || length(idx) == 0 || anyNA(idx) || any(idx < 1 | idx > n) ||
any(idx %% 1 != 0)) {
stop("'", arg, "' must be indices or names of the ", arg, " of the data.", call. = FALSE)
}
as.integer(idx)
}
# Column block of each channel: adjacent channels of the same detector
# segment (segments are contiguous in column order).
cell_map_columns <- function(segment, p, max_blocks) {
size <- ceiling(p / max_blocks)
block <- stats::ave(seq_len(p), segment, FUN = function(i) ceiling(seq_along(i) / size))
key <- paste(segment, block)
match(key, unique(key))
}
# Row order of a Ward clustering of the rows, at the column resolution.
cell_map_cluster <- function(cell, col_id) {
if (nrow(cell) < 3) return(seq_len(nrow(cell)))
reduced <- t(rowsum(t(cell), col_id, reorder = TRUE)) / rep(as.vector(table(col_id)), each = nrow(cell))
stats::hclust(stats::dist(reduced), method = "ward.D2")$order
}
# Mean signed intensity of blocks of adjacent rows and of column blocks;
# only the nonzero blocks are returned.
cell_map_blocks <- function(cell, wavelength, col_id, max_rows) {
n <- nrow(cell)
row_block <- ceiling(seq_len(n) / ceiling(n / max_rows))
# sums over row blocks, then over column blocks
by_rows <- rowsum(cell, row_block, reorder = TRUE)
sums <- t(rowsum(t(by_rows), col_id, reorder = TRUE))
counts <- outer(as.vector(table(row_block)), as.vector(table(col_id)))
value <- sums / counts
spacing <- channel_spacing(wavelength)
x_lo <- tapply(wavelength, col_id, min) - spacing / 2
x_hi <- tapply(wavelength, col_id, max) + spacing / 2
r_lo <- tapply(seq_len(n), row_block, min) - 0.5
r_hi <- tapply(seq_len(n), row_block, max) + 0.5
nz <- which(value != 0, arr.ind = TRUE)
out <- data.frame(xmin = x_lo[nz[, 2]], xmax = x_hi[nz[, 2]],
ymin = r_lo[nz[, 1]], ymax = r_hi[nz[, 1]],
value = value[nz])
# keep the extent of the map even without flagged cells
attr(out, "extent") <- c(min(x_lo), max(x_hi), 0.5, n + 0.5)
out$fill <- sign(out$value) * sqrt(abs(out$value))
rownames(out) <- NULL
out
}
channel_spacing <- function(wavelength) {
spacing <- stats::median(abs(diff(wavelength)))
if (!is.finite(spacing) || spacing == 0) 1 else spacing
}
# Runs of channels flagged in at least `threshold` of the observations.
find_flagged_regions <- function(cells, threshold, lines, tol) {
flagged <- cells$flagged
resid <- cells$resid
n <- nrow(flagged)
up <- colSums(flagged & resid > 0, na.rm = TRUE)
down <- colSums(flagged & resid < 0, na.rm = TRUE)
share <- (up + down) / n
above <- which(share >= threshold)
empty <- data.frame(start = numeric(), end = numeric(), peak = numeric(), channels = integer(),
share = numeric(), mean_share = numeric(), direction = character(),
species = character(), line_wavelength = numeric(),
candidates = character(), stringsAsFactors = FALSE)
if (length(above) == 0) return(empty)
# a new region after more than two channels below the threshold, or a new segment
starts <- c(TRUE, diff(above) > 3 | diff(cells$segment[above]) != 0)
region <- cumsum(starts)
wl <- cells$wavelength
out <- do.call(rbind, lapply(split(above, region), function(i) {
i <- seq(min(i), max(i))
top <- i[which.max(share[i])]
higher <- sum(up[i]) / max(1, sum(up[i] + down[i]))
data.frame(start = min(wl[i]), end = max(wl[i]), peak = wl[top], channels = length(i),
share = share[top], mean_share = mean(share[i]),
direction = if (higher >= 2 / 3) "higher" else if (higher <= 1 / 3) "lower" else "mixed",
stringsAsFactors = FALSE)
}))
out <- out[order(out$share, decreasing = TRUE), , drop = FALSE]
out$species <- NA_character_
out$line_wavelength <- NA_real_
out$candidates <- NA_character_
if (!is.null(lines) && nrow(lines) > 0) {
for (r in seq_len(nrow(out))) {
near <- which(lines$wavelength >= out$start[r] - tol & lines$wavelength <= out$end[r] + tol)
if (length(near) == 0) next
near <- near[order(abs(lines$wavelength[near] - out$peak[r]))]
out$species[r] <- lines$species[near[1]]
out$line_wavelength[r] <- lines$wavelength[near[1]]
out$candidates[r] <- paste(lines$species[near], format_wavelength(lines$wavelength[near], 2),
collapse = ", ")
}
}
rownames(out) <- NULL
out
}
cell_map_panel <- function(blocks, ids, has_wavelength, order, xlab = NULL, ylab = NULL,
base_size = 11) {
n <- length(ids)
if (n <= 40) {
breaks <- seq_len(n)
} else {
breaks <- pretty(c(1, n), n = 8)
breaks <- unique(c(1, breaks[breaks >= 1 & breaks <= n]))
}
labels <- ids[breaks]
extent <- attr(blocks, "extent")
ggplot2::ggplot(blocks) +
ggplot2::geom_rect(ggplot2::aes(xmin = .data$xmin, xmax = .data$xmax,
ymin = .data$ymin, ymax = .data$ymax, fill = .data$fill)) +
ggplot2::geom_blank(data = data.frame(x = extent[1:2], y = extent[3:4]),
ggplot2::aes(x = .data$x, y = .data$y)) +
ggplot2::scale_fill_gradient2(low = "#2166ac", mid = "white", high = "#b2182b",
midpoint = 0, limits = c(-1, 1), breaks = c(-1, 1),
labels = c("lower than fit", "higher than fit"),
name = "Flagged cells",
guide = ggplot2::guide_colourbar(order = 1, barwidth = 8)) +
ggplot2::scale_y_reverse(breaks = breaks, labels = labels, expand = c(0, 0)) +
ggplot2::labs(x = xlab %||% if (has_wavelength) "Wavelength (nm)" else "Variable",
y = ylab %||% switch(order, data = "Observation",
od = "Observation (by decreasing OD)",
cluster = "Observation (clustered)")) +
ggplot2::theme_bw(base_size = base_size) +
ggplot2::theme(panel.grid = ggplot2::element_blank())
}
cell_map_strip <- function(type) {
palette <- c(regular = "grey85", `good leverage` = "#1b9e77",
`orthogonal outlier` = "#d95f02", `bad leverage` = "#e7298a")
df <- data.frame(x = 1, y = seq_along(type), type = type)
ggplot2::ggplot(df, ggplot2::aes(x = .data$x, y = .data$y, fill = .data$type)) +
ggplot2::geom_tile(width = 1, height = 1) +
ggplot2::scale_fill_manual(values = palette, drop = FALSE, name = NULL,
guide = ggplot2::guide_legend(order = 2)) +
ggplot2::scale_y_reverse(expand = c(0, 0)) +
ggplot2::scale_x_continuous(expand = c(0, 0)) +
ggplot2::theme_void()
}
cell_map_profile <- function(cells, spectrum, regions, threshold, labels, x_limits,
base_size = 11) {
n <- nrow(cells$resid)
wavelength <- cells$wavelength
up <- colSums(cells$flagged & cells$resid > 0, na.rm = TRUE) / n
down <- colSums(cells$flagged & cells$resid < 0, na.rm = TRUE) / n
# one bar per variable, as wide as its column of the map
spacing <- channel_spacing(wavelength)
df <- data.frame(xmin = rep(wavelength, 2) - spacing / 2, xmax = rep(wavelength, 2) + spacing / 2,
value = c(up, -down),
sign = rep(c("higher", "lower"), each = length(wavelength)))
df <- df[df$value != 0, , drop = FALSE]
top_share <- max(c(up, threshold))
p <- ggplot2::ggplot()
spread <- if (is.null(spectrum)) NA else max(abs(spectrum), na.rm = TRUE)
if (is.finite(spread) && spread > 0) {
bg <- data.frame(wavelength = wavelength, value = spectrum / spread * top_share,
segment = cells$segment)
p <- p + ggplot2::geom_line(data = bg, ggplot2::aes(x = .data$wavelength, y = .data$value,
group = .data$segment),
colour = "grey75", linewidth = 0.3)
}
p <- p +
ggplot2::geom_rect(data = df, ggplot2::aes(xmin = .data$xmin, xmax = .data$xmax, ymin = 0,
ymax = .data$value, fill = .data$sign,
colour = .data$sign), linewidth = 0.1) +
ggplot2::geom_hline(yintercept = 0, colour = "grey30", linewidth = 0.3) +
ggplot2::geom_hline(yintercept = c(-threshold, threshold), colour = "grey40",
linetype = "dashed", linewidth = 0.3)
shown <- utils::head(regions, labels)
if (nrow(shown) > 0) {
digits <- if (cells$has_wavelength) wavelength_digits(wavelength) else 0L
shown$label <- ifelse(is.na(shown$species), format_wavelength(shown$peak, digits),
paste(shown$species, format_wavelength(shown$line_wavelength, 2)))
# horizontal labels beyond the peaks, moved apart along the wavelength
# axis by ggrepel (inside the panel), with a leader line to their peak
lower <- shown$direction == "lower"
shown$y0 <- ifelse(lower, -down[match(shown$peak, wavelength)], up[match(shown$peak, wavelength)])
shown$y <- ifelse(lower, -1, 1) * (pmax(abs(shown$y0), threshold) + 0.12 * top_share)
shown$vjust <- ifelse(lower, 1, 0)
shown$fontface <- ifelse(is.na(shown$species), "plain", "bold")
room <- data.frame(x = shown$peak, y = shown$y + ifelse(lower, -1, 1) * 0.3 * top_share)
p <- p +
ggrepel::geom_text_repel(data = shown,
ggplot2::aes(x = .data$peak, y = .data$y0, label = .data$label,
vjust = .data$vjust, fontface = .data$fontface),
nudge_y = shown$y - shown$y0, direction = "x",
size = 0.67 * base_size / ggplot2::.pt,
colour = "grey15", segment.colour = "grey55", min.segment.length = 0,
max.overlaps = Inf, seed = 1) +
ggplot2::geom_blank(data = room, ggplot2::aes(x = .data$x, y = .data$y))
}
p +
ggplot2::scale_fill_manual(values = c(higher = "#b2182b", lower = "#2166ac"), guide = "none",
aesthetics = c("fill", "colour")) +
ggplot2::scale_x_continuous(limits = x_limits, expand = c(0, 0)) +
ggplot2::scale_y_continuous(labels = function(v) scales::percent(abs(v))) +
ggplot2::labs(x = NULL, y = "Flagged") +
ggplot2::theme_bw(base_size = base_size) +
ggplot2::theme(axis.text.x = ggplot2::element_blank(),
axis.ticks.x = ggplot2::element_blank(),
panel.grid.minor = ggplot2::element_blank())
}
# Name of a robust calibration model in plot titles.
calibration_name <- function(object) {
if (inherits(object, "specproc_rpcr")) "RPCR" else "RSIMPLS"
}
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.