Nothing
#' Tidy a dirty table's column names and category labels before masking
#'
#' Real custodian tables arrive with column names that are not valid R
#' names (`"Yield (t/ha)"`, `"Site Name"`), leading or trailing
#' whitespace in names and factor / character values, and the
#' occasional near-duplicate label (`"north"` vs `"North"` vs
#' `" north"`). `clean_table()` legalises names and trims whitespace, and
#' reports each fix. Near-duplicate labels are only reported, since merging
#' labels that look alike is a judgement call; [conform_table()] makes it.
#'
#' The corrections are returned alongside the cleaned data, so [mask()]
#' records them in the recipe and [apply_recipe()] re-applies the same
#' cleaning to the original.
#'
#' @param df A data frame.
#' @param clean One of `"auto"` (default - legalise names, trim
#' whitespace, report near-duplicates), `"report"` (legalise names,
#' report what whitespace / near-duplicate changes *would* be made but
#' apply none), or `"off"` (legalise names only, skip all other
#' hygiene). Column-name legalisation is applied in **every** mode -- an
#' invalid name silently rewritten downstream corrupts the clone -- and
#' is surfaced as a `masque_name_repaired` warning; only the whitespace
#' and near-duplicate handling is governed by the mode.
#' @param quiet Logical. When `FALSE` (default) a `cli` summary of the
#' fixes and advisories is printed. Set `TRUE` to suppress it (the
#' report object is returned either way).
#'
#' @details Two input columns with the same name are refused with a
#' `masque_duplicate_names_refusal` error: the recipe records renamed
#' columns by their original name, so the two could not be told apart on
#' the round trip. Rename them first.
#'
#' @return An object of class `masque_cleaning`: a list with
#' \itemize{
#' \item `data` - the cleaned (or, under `report` / `off`,
#' unchanged) data.table;
#' \item `name_map` - named character `original -> clean` for every
#' column whose name changed (empty if none);
#' \item `level_fixes` - named list, one entry per column whose
#' values were trimmed, each a named character `original -> clean`;
#' \item `near_duplicates` - a data.table of report-only label pairs
#' (`col`, `a`, `b`, `kind`) that look like typos but were left
#' untouched. Labels that differ only inside a run of digits
#' (`"G01"` and `"G02"`, `"Plot12"` and `"Plot13"`) are different
#' codes and are not listed. The printed report shows the first 20
#' pairs and the count, and a recipe keeps the same;
#' \item `mode` - the `clean` mode applied.
#' }
#'
#' @examples
#' df <- data.frame(
#' `Site Name` = c("north ", "north", "South"),
#' `Yield (t/ha)` = c(3.1, 2.9, 5.0),
#' check.names = FALSE
#' )
#' cl <- clean_table(df, quiet = TRUE)
#' names(cl$data)
#' cl$near_duplicates
#'
#' @seealso [mask()], [propose_roles()].
#' @export
clean_table <- function(df, clean = c("auto", "report", "off"), quiet = FALSE) {
if (!is.data.frame(df)) {
cli::cli_abort("`df` must be a data frame; got {.cls {class(df)[1]}}.")
}
clean <- match.arg(clean)
# Names are legalised in every mode, since a name rewritten later by
# make.names() breaks the round trip. `clean` governs only the steps below.
orig_names <- names(df)
.refuse_duplicate_names(orig_names)
legal <- .legalise_names(orig_names)
name_map <- stats::setNames(legal, orig_names)
name_map <- name_map[orig_names != legal]
if (length(name_map)) {
names(df) <- legal
warning(warningCondition(
.name_repair_message(name_map),
class = "masque_name_repaired"
))
}
apply_fixes <- identical(clean, "auto")
do_hygiene <- !identical(clean, "off")
# Trim whitespace (auto only) and detect near-duplicate labels (auto and
# report). Keyed on the now-legalised column names.
level_fixes <- list()
near_dups <- list()
if (do_hygiene) {
for (j in seq_along(df)) {
x <- df[[j]]
if (!(is.character(x) || is.factor(x))) next
nm_out <- names(df)[j]
fix <- .trim_levels(x)
if (length(fix$map)) {
level_fixes[[nm_out]] <- fix$map
if (apply_fixes) df[[j]] <- fix$x
}
# Near-duplicate detection runs on the (trimmed) vocabulary.
vals <- if (apply_fixes) fix$x else x
nd <- .near_duplicate_pairs(vals)
if (nrow(nd)) {
nd$col <- nm_out
near_dups[[length(near_dups) + 1L]] <- nd
}
}
}
near_duplicates <- if (length(near_dups)) {
out <- do.call(rbind, near_dups)
out[, c("col", "a", "b", "kind")]
} else {
.empty_near_dups()
}
out <- list(
data = .as_owned_dt(df),
name_map = name_map,
level_fixes = level_fixes,
near_duplicates = data.table::as.data.table(near_duplicates),
mode = clean
)
class(out) <- "masque_cleaning"
if (!quiet) .report_cleaning(out)
out
}
#' @export
print.masque_cleaning <- function(x, ...) {
.report_cleaning(x)
invisible(x)
}
.empty_near_dups <- function() {
data.frame(
col = character(), a = character(), b = character(),
kind = character(), stringsAsFactors = FALSE
)
}
# Two input columns with one name cannot be told apart by the name map, so
# the round trip would swap them; refuse before anything is renamed.
.refuse_duplicate_names <- function(nms) {
dups <- unique(nms[duplicated(nms)])
if (!length(dups)) {
return(invisible(nms))
}
cli::cli_abort(c(
paste0(
"{cli::qty(length(dups))}Column name{?s} used more than once: ",
"{.val {dups}}."
),
i = paste0(
"masque records renamed columns by their original name, so two ",
"columns with one name cannot be restored on the round trip."
),
"*" = paste0(
"Give each column its own name first, for example ",
"{.code names(df) <- make.unique(names(df), sep = \"_\")}."
)
), class = c("masque_duplicate_names_refusal", "orchestra_refusal"))
}
# Trim, make.names(), then make.unique() for any collisions this introduces.
.legalise_names <- function(nms) {
trimmed <- trimws(nms)
legal <- make.names(trimmed)
make.unique(legal, sep = "_")
}
# Human-readable summary of a column-name legalisation, shared by the
# clean_table() warning and mask()'s recipe record so the two never drift.
.name_repair_message <- function(name_map) {
pairs <- paste(
sprintf("`%s` -> `%s`", names(name_map), unname(name_map)),
collapse = ", "
)
paste0(
cli::pluralize(
"Renamed {length(name_map)} column name{?s} that R does not accept: "
),
pairs, ". ",
"The map is recorded in the recipe and reversed on the round-trip."
)
}
# Trim leading / trailing whitespace from a character or factor vector.
# Returns the cleaned vector and a map of only the values that changed.
.trim_levels <- function(x) {
if (is.factor(x)) {
lv <- levels(x)
lv_trim <- trimws(lv)
changed <- lv != lv_trim
if (!any(changed)) {
return(list(x = x, map = stats::setNames(character(), character())))
}
# Trimming may collapse levels (" a" and "a"); rebuild the factor on
# the trimmed vocabulary.
new_lv <- unique(lv_trim)
x_new <- factor(lv_trim[match(as.character(x), lv)], levels = new_lv)
map <- stats::setNames(lv_trim[changed], lv[changed])
list(x = x_new, map = map)
} else {
x_trim <- trimws(x)
changed <- !is.na(x) & x != x_trim
if (!any(changed)) {
return(list(x = x, map = stats::setNames(character(), character())))
}
map <- stats::setNames(
x_trim[changed], x[changed]
)
map <- map[!duplicated(names(map))]
list(x = x_trim, map = map)
}
}
# Report only: pairs of distinct labels that differ by case or by one edit
# (utils::adist()). An edit inside a run of digits ("G01" and "G02") is a
# different code, not a typo.
.near_duplicate_pairs <- function(x) {
vals <- sort(unique(stats::na.omit(as.character(x))))
if (length(vals) < 2L) {
return(.empty_near_dups())
}
lower <- tolower(vals)
idx <- which(utils::adist(vals) == 1L, arr.ind = TRUE)
# Case variants can be several edits apart ("NSW", "nsw").
shared <- which(duplicated(lower) | duplicated(lower, fromLast = TRUE))
if (length(shared)) {
same <- which(outer(lower[shared], lower[shared], "=="), arr.ind = TRUE)
idx <- rbind(idx, cbind(shared[same[, 1L]], shared[same[, 2L]]))
}
# Each pair once, listed by `b`, then by `a`.
idx <- unique(idx[idx[, 1L] < idx[, 2L], , drop = FALSE])
idx <- idx[order(idx[, 2L], idx[, 1L]), , drop = FALSE]
if (!nrow(idx)) {
return(.empty_near_dups())
}
a <- vals[idx[, 1L]]
b <- vals[idx[, 2L]]
kind <- ifelse(lower[idx[, 1L]] == lower[idx[, 2L]], "case", "edit1")
keep <- kind == "case" | !.edit_in_digits(a, b)
if (!any(keep)) {
return(.empty_near_dups())
}
data.frame(
a = a[keep], b = b[keep], kind = kind[keep], col = NA_character_,
stringsAsFactors = FALSE
)
}
# For pairs one edit apart: is the edited character a digit, either a digit
# changed for a digit or a digit added or removed?
.edit_in_digits <- function(a, b) {
vapply(seq_along(a), function(i) {
x <- strsplit(a[[i]], "", fixed = TRUE)[[1L]]
y <- strsplit(b[[i]], "", fixed = TRUE)[[1L]]
n <- min(length(x), length(y))
p <- which(x[seq_len(n)] != y[seq_len(n)])[1L]
if (is.na(p)) p <- n + 1L
if (length(x) == length(y)) {
return(grepl("[0-9]", x[[p]]) && grepl("[0-9]", y[[p]]))
}
longer <- if (length(x) > length(y)) x else y
grepl("[0-9]", longer[[p]])
}, logical(1L))
}
# Near-duplicate pairs printed by the hygiene report and kept on a recipe.
.NEAR_DUP_SHOWN <- 20L
# Distil a masque_cleaning object to the reversible record stored on the
# recipe (the data frame itself is not kept). Near-duplicates are advisory:
# the count and the first pairs, as a plain data frame.
.cleaning_record <- function(cl) {
nd <- as.data.frame(cl$near_duplicates)
list(
name_map = cl$name_map,
level_fixes = cl$level_fixes,
near_duplicates = utils::head(nd, .NEAR_DUP_SHOWN),
n_near_duplicates = nrow(nd),
mode = cl$mode
)
}
# Near-duplicate pairs found, including those not kept on the record.
.near_dup_count <- function(rec) {
rec$n_near_duplicates %||% NROW(rec$near_duplicates)
}
# Re-apply a recipe's cleaning record to the original, so a pipeline written
# against the cleaned synthetic runs on it. A clean frame is left as it is.
.apply_cleaning_forward <- function(df, cleaning) {
if (is.null(cleaning)) {
return(df)
}
name_map <- cleaning$name_map
level_fixes <- cleaning$level_fixes
# `original` still has the uncleaned names, so trim through `name_map`.
for (clean_nm in names(level_fixes)) {
orig_nm <- names(name_map)[match(clean_nm, unname(name_map))]
src_nm <- if (!is.na(orig_nm) && length(orig_nm)) orig_nm else clean_nm
if (!(src_nm %in% names(df))) next
df[[src_nm]] <- trimws_keep_class(df[[src_nm]])
}
if (length(name_map)) {
nm <- names(df)
hits <- match(names(name_map), nm)
ok <- !is.na(hits)
nm[hits[ok]] <- unname(name_map)[ok]
names(df) <- nm
}
df
}
# Trim a character / factor vector, preserving its class; pass anything
# else through.
trimws_keep_class <- function(x) {
if (is.factor(x)) {
return(.trim_levels(x)$x)
}
if (is.character(x)) {
return(trimws(x))
}
x
}
# Remap a roles table's `col` column through a name map (original ->
# clean). Preserves all attributes (mode, proposed_actions, design).
.remap_roles_cols <- function(roles, name_map) {
if (!length(name_map) || !("col" %in% names(roles))) {
return(roles)
}
hits <- match(roles$col, names(name_map))
ok <- !is.na(hits)
roles$col[ok] <- unname(name_map)[hits[ok]]
pa <- attr(roles, "proposed_actions")
if (!is.null(pa) && length(pa)) {
nm <- names(pa)
h <- match(nm, names(name_map))
o <- !is.na(h)
nm[o] <- unname(name_map)[h[o]]
names(pa) <- nm
attr(roles, "proposed_actions") <- pa
}
roles
}
# Line a roles table up with the cleaned names. One keyed by them already is
# left alone: with `Site Name` beside `Site.Name` a remap would move it.
.align_roles_cols <- function(roles, name_map, clean_names) {
if (!length(name_map) || !("col" %in% names(roles))) {
return(roles)
}
if (all(roles$col %in% clean_names) && !anyDuplicated(roles$col)) {
return(roles)
}
.remap_roles_cols(roles, name_map)
}
.report_cleaning <- function(cl) {
n_names <- length(cl$name_map)
n_levels <- length(cl$level_fixes)
n_dups <- nrow(cl$near_duplicates)
if (n_names == 0L && n_levels == 0L && n_dups == 0L) {
cli::cli_alert_success("Table is clean: no name or label fixes needed.")
return(invisible(cl))
}
verb <- if (identical(cl$mode, "report")) "would fix" else "fixed"
cli::cli_h2("masque hygiene report")
if (n_names) {
# Names are legalised in every mode, so they are always already fixed.
cli::cli_alert_info("Column names legalised ({n_names}):")
for (i in seq_len(n_names)) {
cli::cli_li("{.val {names(cl$name_map)[i]}} -> {.val {cl$name_map[[i]]}}")
}
}
if (n_levels) {
total <- sum(vapply(cl$level_fixes, length, integer(1L)))
cli::cli_alert_info(
"Whitespace {verb} in {n_levels} column{?s} ({total} label{?s})."
)
}
if (n_dups) {
cli::cli_alert_warning(
"{n_dups} near-duplicate label pair{?s} found - NOT changed:"
)
shown <- min(n_dups, .NEAR_DUP_SHOWN)
for (i in seq_len(shown)) {
row <- cl$near_duplicates[i, ]
cli::cli_li(
"{.field {row$col}}: {.val {row$a}} ~ {.val {row$b}} ({row$kind})"
)
}
if (n_dups > shown) {
cli::cli_text(paste0(
"... and {n_dups - shown} more; the full list is ",
"{.code $near_duplicates}."
))
}
cli::cli_text(
"Resolve these by hand if they are typos; masque will not merge them."
)
}
invisible(cl)
}
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.