R/clean_table.R

Defines functions .report_cleaning .align_roles_cols .remap_roles_cols trimws_keep_class .apply_cleaning_forward .near_dup_count .cleaning_record .edit_in_digits .near_duplicate_pairs .trim_levels .name_repair_message .legalise_names .refuse_duplicate_names .empty_near_dups print.masque_cleaning clean_table

Documented in clean_table

#' Tidy a dirty table's column names and category labels before masking
#'
#' Real custodian tables arrive with column names that are not valid R
#' names (`"Yield (t/ha)"`, `"Site Name"`), leading or trailing
#' whitespace in names and factor / character values, and the
#' occasional near-duplicate label (`"north"` vs `"North"` vs
#' `" north"`). `clean_table()` legalises names and trims whitespace, and
#' reports each fix. Near-duplicate labels are only reported, since merging
#' labels that look alike is a judgement call; [conform_table()] makes it.
#'
#' The corrections are returned alongside the cleaned data, so [mask()]
#' records them in the recipe and [apply_recipe()] re-applies the same
#' cleaning to the original.
#'
#' @param df A data frame.
#' @param clean One of `"auto"` (default - legalise names, trim
#'   whitespace, report near-duplicates), `"report"` (legalise names,
#'   report what whitespace / near-duplicate changes *would* be made but
#'   apply none), or `"off"` (legalise names only, skip all other
#'   hygiene). Column-name legalisation is applied in **every** mode -- an
#'   invalid name silently rewritten downstream corrupts the clone -- and
#'   is surfaced as a `masque_name_repaired` warning; only the whitespace
#'   and near-duplicate handling is governed by the mode.
#' @param quiet Logical. When `FALSE` (default) a `cli` summary of the
#'   fixes and advisories is printed. Set `TRUE` to suppress it (the
#'   report object is returned either way).
#'
#' @details Two input columns with the same name are refused with a
#'   `masque_duplicate_names_refusal` error: the recipe records renamed
#'   columns by their original name, so the two could not be told apart on
#'   the round trip. Rename them first.
#'
#' @return An object of class `masque_cleaning`: a list with
#'   \itemize{
#'     \item `data` - the cleaned (or, under `report` / `off`,
#'       unchanged) data.table;
#'     \item `name_map` - named character `original -> clean` for every
#'       column whose name changed (empty if none);
#'     \item `level_fixes` - named list, one entry per column whose
#'       values were trimmed, each a named character `original -> clean`;
#'     \item `near_duplicates` - a data.table of report-only label pairs
#'       (`col`, `a`, `b`, `kind`) that look like typos but were left
#'       untouched. Labels that differ only inside a run of digits
#'       (`"G01"` and `"G02"`, `"Plot12"` and `"Plot13"`) are different
#'       codes and are not listed. The printed report shows the first 20
#'       pairs and the count, and a recipe keeps the same;
#'     \item `mode` - the `clean` mode applied.
#'   }
#'
#' @examples
#' df <- data.frame(
#'   `Site Name` = c("north ", "north", "South"),
#'   `Yield (t/ha)` = c(3.1, 2.9, 5.0),
#'   check.names = FALSE
#' )
#' cl <- clean_table(df, quiet = TRUE)
#' names(cl$data)
#' cl$near_duplicates
#'
#' @seealso [mask()], [propose_roles()].
#' @export
clean_table <- function(df, clean = c("auto", "report", "off"), quiet = FALSE) {
  if (!is.data.frame(df)) {
    cli::cli_abort("`df` must be a data frame; got {.cls {class(df)[1]}}.")
  }
  clean <- match.arg(clean)

  # Names are legalised in every mode, since a name rewritten later by
  # make.names() breaks the round trip. `clean` governs only the steps below.
  orig_names <- names(df)
  .refuse_duplicate_names(orig_names)
  legal <- .legalise_names(orig_names)
  name_map <- stats::setNames(legal, orig_names)
  name_map <- name_map[orig_names != legal]
  if (length(name_map)) {
    names(df) <- legal
    warning(warningCondition(
      .name_repair_message(name_map),
      class = "masque_name_repaired"
    ))
  }

  apply_fixes <- identical(clean, "auto")
  do_hygiene <- !identical(clean, "off")

  # Trim whitespace (auto only) and detect near-duplicate labels (auto and
  # report). Keyed on the now-legalised column names.
  level_fixes <- list()
  near_dups <- list()
  if (do_hygiene) {
    for (j in seq_along(df)) {
      x <- df[[j]]
      if (!(is.character(x) || is.factor(x))) next
      nm_out <- names(df)[j]
      fix <- .trim_levels(x)
      if (length(fix$map)) {
        level_fixes[[nm_out]] <- fix$map
        if (apply_fixes) df[[j]] <- fix$x
      }
      # Near-duplicate detection runs on the (trimmed) vocabulary.
      vals <- if (apply_fixes) fix$x else x
      nd <- .near_duplicate_pairs(vals)
      if (nrow(nd)) {
        nd$col <- nm_out
        near_dups[[length(near_dups) + 1L]] <- nd
      }
    }
  }

  near_duplicates <- if (length(near_dups)) {
    out <- do.call(rbind, near_dups)
    out[, c("col", "a", "b", "kind")]
  } else {
    .empty_near_dups()
  }

  out <- list(
    data            = .as_owned_dt(df),
    name_map        = name_map,
    level_fixes     = level_fixes,
    near_duplicates = data.table::as.data.table(near_duplicates),
    mode            = clean
  )
  class(out) <- "masque_cleaning"

  if (!quiet) .report_cleaning(out)
  out
}

#' @export
print.masque_cleaning <- function(x, ...) {
  .report_cleaning(x)
  invisible(x)
}

.empty_near_dups <- function() {
  data.frame(
    col = character(), a = character(), b = character(),
    kind = character(), stringsAsFactors = FALSE
  )
}

# Two input columns with one name cannot be told apart by the name map, so
# the round trip would swap them; refuse before anything is renamed.
.refuse_duplicate_names <- function(nms) {
  dups <- unique(nms[duplicated(nms)])
  if (!length(dups)) {
    return(invisible(nms))
  }
  cli::cli_abort(c(
    paste0(
      "{cli::qty(length(dups))}Column name{?s} used more than once: ",
      "{.val {dups}}."
    ),
    i = paste0(
      "masque records renamed columns by their original name, so two ",
      "columns with one name cannot be restored on the round trip."
    ),
    "*" = paste0(
      "Give each column its own name first, for example ",
      "{.code names(df) <- make.unique(names(df), sep = \"_\")}."
    )
  ), class = c("masque_duplicate_names_refusal", "orchestra_refusal"))
}

# Trim, make.names(), then make.unique() for any collisions this introduces.
.legalise_names <- function(nms) {
  trimmed <- trimws(nms)
  legal <- make.names(trimmed)
  make.unique(legal, sep = "_")
}

# Human-readable summary of a column-name legalisation, shared by the
# clean_table() warning and mask()'s recipe record so the two never drift.
.name_repair_message <- function(name_map) {
  pairs <- paste(
    sprintf("`%s` -> `%s`", names(name_map), unname(name_map)),
    collapse = ", "
  )
  paste0(
    cli::pluralize(
      "Renamed {length(name_map)} column name{?s} that R does not accept: "
    ),
    pairs, ". ",
    "The map is recorded in the recipe and reversed on the round-trip."
  )
}

# Trim leading / trailing whitespace from a character or factor vector.
# Returns the cleaned vector and a map of only the values that changed.
.trim_levels <- function(x) {
  if (is.factor(x)) {
    lv <- levels(x)
    lv_trim <- trimws(lv)
    changed <- lv != lv_trim
    if (!any(changed)) {
      return(list(x = x, map = stats::setNames(character(), character())))
    }
    # Trimming may collapse levels (" a" and "a"); rebuild the factor on
    # the trimmed vocabulary.
    new_lv <- unique(lv_trim)
    x_new <- factor(lv_trim[match(as.character(x), lv)], levels = new_lv)
    map <- stats::setNames(lv_trim[changed], lv[changed])
    list(x = x_new, map = map)
  } else {
    x_trim <- trimws(x)
    changed <- !is.na(x) & x != x_trim
    if (!any(changed)) {
      return(list(x = x, map = stats::setNames(character(), character())))
    }
    map <- stats::setNames(
      x_trim[changed], x[changed]
    )
    map <- map[!duplicated(names(map))]
    list(x = x_trim, map = map)
  }
}

# Report only: pairs of distinct labels that differ by case or by one edit
# (utils::adist()). An edit inside a run of digits ("G01" and "G02") is a
# different code, not a typo.
.near_duplicate_pairs <- function(x) {
  vals <- sort(unique(stats::na.omit(as.character(x))))
  if (length(vals) < 2L) {
    return(.empty_near_dups())
  }
  lower <- tolower(vals)
  idx <- which(utils::adist(vals) == 1L, arr.ind = TRUE)
  # Case variants can be several edits apart ("NSW", "nsw").
  shared <- which(duplicated(lower) | duplicated(lower, fromLast = TRUE))
  if (length(shared)) {
    same <- which(outer(lower[shared], lower[shared], "=="), arr.ind = TRUE)
    idx <- rbind(idx, cbind(shared[same[, 1L]], shared[same[, 2L]]))
  }
  # Each pair once, listed by `b`, then by `a`.
  idx <- unique(idx[idx[, 1L] < idx[, 2L], , drop = FALSE])
  idx <- idx[order(idx[, 2L], idx[, 1L]), , drop = FALSE]
  if (!nrow(idx)) {
    return(.empty_near_dups())
  }
  a <- vals[idx[, 1L]]
  b <- vals[idx[, 2L]]
  kind <- ifelse(lower[idx[, 1L]] == lower[idx[, 2L]], "case", "edit1")
  keep <- kind == "case" | !.edit_in_digits(a, b)
  if (!any(keep)) {
    return(.empty_near_dups())
  }
  data.frame(
    a = a[keep], b = b[keep], kind = kind[keep], col = NA_character_,
    stringsAsFactors = FALSE
  )
}

# For pairs one edit apart: is the edited character a digit, either a digit
# changed for a digit or a digit added or removed?
.edit_in_digits <- function(a, b) {
  vapply(seq_along(a), function(i) {
    x <- strsplit(a[[i]], "", fixed = TRUE)[[1L]]
    y <- strsplit(b[[i]], "", fixed = TRUE)[[1L]]
    n <- min(length(x), length(y))
    p <- which(x[seq_len(n)] != y[seq_len(n)])[1L]
    if (is.na(p)) p <- n + 1L
    if (length(x) == length(y)) {
      return(grepl("[0-9]", x[[p]]) && grepl("[0-9]", y[[p]]))
    }
    longer <- if (length(x) > length(y)) x else y
    grepl("[0-9]", longer[[p]])
  }, logical(1L))
}

# Near-duplicate pairs printed by the hygiene report and kept on a recipe.
.NEAR_DUP_SHOWN <- 20L

# Distil a masque_cleaning object to the reversible record stored on the
# recipe (the data frame itself is not kept). Near-duplicates are advisory:
# the count and the first pairs, as a plain data frame.
.cleaning_record <- function(cl) {
  nd <- as.data.frame(cl$near_duplicates)
  list(
    name_map          = cl$name_map,
    level_fixes       = cl$level_fixes,
    near_duplicates   = utils::head(nd, .NEAR_DUP_SHOWN),
    n_near_duplicates = nrow(nd),
    mode              = cl$mode
  )
}

# Near-duplicate pairs found, including those not kept on the record.
.near_dup_count <- function(rec) {
  rec$n_near_duplicates %||% NROW(rec$near_duplicates)
}

# Re-apply a recipe's cleaning record to the original, so a pipeline written
# against the cleaned synthetic runs on it. A clean frame is left as it is.
.apply_cleaning_forward <- function(df, cleaning) {
  if (is.null(cleaning)) {
    return(df)
  }
  name_map <- cleaning$name_map
  level_fixes <- cleaning$level_fixes

  # `original` still has the uncleaned names, so trim through `name_map`.
  for (clean_nm in names(level_fixes)) {
    orig_nm <- names(name_map)[match(clean_nm, unname(name_map))]
    src_nm <- if (!is.na(orig_nm) && length(orig_nm)) orig_nm else clean_nm
    if (!(src_nm %in% names(df))) next
    df[[src_nm]] <- trimws_keep_class(df[[src_nm]])
  }

  if (length(name_map)) {
    nm <- names(df)
    hits <- match(names(name_map), nm)
    ok <- !is.na(hits)
    nm[hits[ok]] <- unname(name_map)[ok]
    names(df) <- nm
  }
  df
}

# Trim a character / factor vector, preserving its class; pass anything
# else through.
trimws_keep_class <- function(x) {
  if (is.factor(x)) {
    return(.trim_levels(x)$x)
  }
  if (is.character(x)) {
    return(trimws(x))
  }
  x
}

# Remap a roles table's `col` column through a name map (original ->
# clean). Preserves all attributes (mode, proposed_actions, design).
.remap_roles_cols <- function(roles, name_map) {
  if (!length(name_map) || !("col" %in% names(roles))) {
    return(roles)
  }
  hits <- match(roles$col, names(name_map))
  ok <- !is.na(hits)
  roles$col[ok] <- unname(name_map)[hits[ok]]

  pa <- attr(roles, "proposed_actions")
  if (!is.null(pa) && length(pa)) {
    nm <- names(pa)
    h <- match(nm, names(name_map))
    o <- !is.na(h)
    nm[o] <- unname(name_map)[h[o]]
    names(pa) <- nm
    attr(roles, "proposed_actions") <- pa
  }
  roles
}

# Line a roles table up with the cleaned names. One keyed by them already is
# left alone: with `Site Name` beside `Site.Name` a remap would move it.
.align_roles_cols <- function(roles, name_map, clean_names) {
  if (!length(name_map) || !("col" %in% names(roles))) {
    return(roles)
  }
  if (all(roles$col %in% clean_names) && !anyDuplicated(roles$col)) {
    return(roles)
  }
  .remap_roles_cols(roles, name_map)
}

.report_cleaning <- function(cl) {
  n_names <- length(cl$name_map)
  n_levels <- length(cl$level_fixes)
  n_dups <- nrow(cl$near_duplicates)

  if (n_names == 0L && n_levels == 0L && n_dups == 0L) {
    cli::cli_alert_success("Table is clean: no name or label fixes needed.")
    return(invisible(cl))
  }

  verb <- if (identical(cl$mode, "report")) "would fix" else "fixed"
  cli::cli_h2("masque hygiene report")

  if (n_names) {
    # Names are legalised in every mode, so they are always already fixed.
    cli::cli_alert_info("Column names legalised ({n_names}):")
    for (i in seq_len(n_names)) {
      cli::cli_li("{.val {names(cl$name_map)[i]}} -> {.val {cl$name_map[[i]]}}")
    }
  }
  if (n_levels) {
    total <- sum(vapply(cl$level_fixes, length, integer(1L)))
    cli::cli_alert_info(
      "Whitespace {verb} in {n_levels} column{?s} ({total} label{?s})."
    )
  }
  if (n_dups) {
    cli::cli_alert_warning(
      "{n_dups} near-duplicate label pair{?s} found - NOT changed:"
    )
    shown <- min(n_dups, .NEAR_DUP_SHOWN)
    for (i in seq_len(shown)) {
      row <- cl$near_duplicates[i, ]
      cli::cli_li(
        "{.field {row$col}}: {.val {row$a}} ~ {.val {row$b}} ({row$kind})"
      )
    }
    if (n_dups > shown) {
      cli::cli_text(paste0(
        "... and {n_dups - shown} more; the full list is ",
        "{.code $near_duplicates}."
      ))
    }
    cli::cli_text(
      "Resolve these by hand if they are typos; masque will not merge them."
    )
  }
  invisible(cl)
}

Try the masque package in your browser

Any scripts or data that you put into this service are public.

masque documentation built on Nov. 19, 2026, 1:06 a.m.