R/scan_storage.R

Defines functions empty_snapshot scan_directory_storage scan_zip_storage extract_storage scan_storage

Documented in scan_storage

#' Observe a filesystem and construct a reproducible snapshot
#'
#' Recursively scans a root folder and returns a data.frame where each row
#' represents one filesystem observation recorded at a specific time.
#'
#' The function implements a read-only filesystem observation model:
#'
#' - it records accessible filesystem state;
#' - it does not interpret file contents;
#' - it does not assume canonical, complete, or authoritative state.
#'
#' Each observation records:
#'
#' - a relative filesystem locator (`rel_path`);
#' - a storage context (`storage_id`);
#' - an observation timestamp (`scan_time`).
#'
#' Additional metadata may include:
#'
#' - filesystem properties (size, timestamps, permissions);
#' - optional content signatures (`quick_sig`);
#' - repository and version-control context
#'   (`repo_root`, `repo_rel_path`, `git_tracked`).
#'
#' The package deliberately records filesystem observations first and
#' postpones documentary interpretation, Record Set construction,
#' and RiC-aligned semantic assertions to later analytical stages.
#'
#' This creates a reproducible observational snapshot suitable for:
#'
#' - forensic analysis of development environments;
#' - reconstruction of activity patterns;
#' - audit and compliance workflows;
#' - alignment with version-controlled repositories.
#'
#' @param root Character. Path to the root folder to observe.
#'
#' @param storage_id Character. Identifier of the storage context.
#'
#' @param person_id Character. Identifier of the observer or operator.
#'
#' @param scan_time POSIXct. Timestamp of the observation.
#'   Defaults to `Sys.time()` if not provided.
#'
#' @param compute_signature Logical. Whether to compute lightweight
#'   content signatures.
#'
#' @param max_signature_size Numeric. Maximum file size (bytes)
#'   for signature computation.
#'
#' @return
#' A `data.frame` where each row represents one filesystem observation.
#'
#' @details
#' The returned dataset at minimum contains:
#'
#' - `rel_path`: relative filesystem locator within the observed root;
#' - `storage_path_id`: deterministic storage-scoped identifier derived
#'   from `storage_id::rel_path`;
#' - `filename`: basename of the observed file;
#' - `mtime`: last modification timestamp;
#' - `extension`: file extension.
#'
#' Additional variables may be present depending on scan configuration.
#'
#' The function is:
#'
#' - read-only and non-destructive;
#' - deterministic for a given filesystem state;
#' - robust to inaccessible files, which are silently skipped.
#'
#' The result represents observed filesystem state rather than complete
#' historical provenance.
#'
#' @seealso [snapshot_storage()]
#'
#' @examples
#' root <- system.file(
#'   "testdata/minimal_R_folder",
#'   package = "fscontext"
#' )
#'
#' snapshot <- scan_storage(root)
#'
#' subset(
#'   snapshot,
#'   rel_path %in% c(
#'     "DESCRIPTION",
#'     "NAMESPACE",
#'     "R/hello_world.R",
#'     "vignettes/demo.Rmd"
#'   )
#' )[, c("rel_path", "extension")]
#'
#' @importFrom utils flush.console
#'
#' @export
scan_storage <- function(root,
                         storage_id = "local-storage",
                         person_id = "local-user",
                         scan_time = Sys.time(),
                         compute_signature = TRUE,
                         max_signature_size =
                           200 * 1024 * 1024) {
  root <- fs::path_abs(root)

  if (fs::dir_exists(root)) {
    return(
      scan_directory_storage(
        root = root,
        storage_id = storage_id,
        person_id = person_id,
        scan_time = scan_time,
        compute_signature = compute_signature,
        max_signature_size = max_signature_size
      )
    )
  }

  ext <- tolower(fs::path_ext(root))

  if (ext %in% c("zip", "wacz")) {
    return(
      scan_zip_storage(
        root = root,
        storage_id = storage_id,
        person_id = person_id,
        scan_time = scan_time,
        compute_signature = compute_signature,
        max_signature_size = max_signature_size
      )
    )
  }

  stop(
    "scan_storage(): unsupported storage type: ",
    root,
    call. = FALSE
  )
}


#' Extract a ZIP-based storage container
#'
#' Internal helper that extracts a ZIP-compatible archive
#' into a temporary directory and returns the extraction path.
#'
#' The function currently supports ZIP-based containers,
#' including `.zip` and `.wacz` files.
#'
#' The extracted directory is intended for immediate use by
#' `scan_directory_storage()` and should be considered temporary.
#'
#' @param archive Character. Path to a ZIP-compatible archive.
#'
#' @param exdir Character. Extraction directory.
#'
#' @details
#' The function returns a character scalar of the extraction directory.
#'
#' @keywords internal
#' @noRd
#' @importFrom utils unzip

extract_storage <- function(
  archive,
  exdir
) {
  archive <- fs::path_abs(archive)

  if (!fs::file_exists(archive)) {
    stop(
      "extract_storage(): archive does not exist: ",
      archive,
      call. = FALSE
    )
  }

  fs::dir_create(exdir)

  utils::unzip(zipfile = archive, exdir = exdir)

  exdir
}


#' Observe a ZIP-based storage container
#'
#' Internal helper used by [scan_storage()] for ZIP-compatible
#' archive containers.
#'
#' The archive is extracted into a temporary directory and then
#' observed using `scan_directory_storage()`. This preserves the
#' existing filesystem observation workflow while allowing
#' archive-backed storage roots to be analysed.
#'
#' Supported formats currently include:
#'
#' - `.zip`
#' - `.wacz`
#'
#' Archive-specific interpretation is deliberately excluded.
#' The function observes the extracted file hierarchy and records
#' the archive container as contextual provenance.
#'
#' @inheritParams scan_storage
#'
#' @details
#' Returns a filesystem observation data frame equivalent to that returned
#' by `scan_directory_storage()`, with additional archive
#' provenance columns.
#'
#' @keywords internal
#' @noRd
#' @importFrom fs path_abs path_ext path dir_create
#' @importFrom utils unzip
#' @importFrom tools file_path_sans_ext

scan_zip_storage <- function(
  root,
  storage_id,
  person_id,
  scan_time = Sys.time(),
  compute_signature = TRUE,
  max_signature_size = 200 * 1024 * 1024
) {
  archive_path <- fs::path_abs(root)

  # Allocate a distinct extraction directory for each scan.
  # Archives may share a basename, and earlier observations must
  # remain accessible after subsequent scans.
  archive_id <- tools::file_path_sans_ext(basename(archive_path))
  extract_root <- tempfile(
    pattern = paste0(archive_id, "_"),
    tmpdir = tempdir()
  )

  fs::dir_create(extract_root)

  # ---------------------------------------------------------
  # Determine observational root.
  #
  # Many ZIP tools (including Windows Explorer) create:
  #
  #   archive.zip
  #     └─ folder/
  #         ├─ DESCRIPTION
  #         ├─ NAMESPACE
  #         └─ ...
  #
  # while direct filesystem scans start at:
  #
  #   folder/
  #       ├─ DESCRIPTION
  #       ├─ NAMESPACE
  #       └─ ...
  #
  # To preserve observational equivalence, descend into the
  # single extracted top-level directory when present.
  # ---------------------------------------------------------


  zip_manifest <- utils::unzip(archive_path, list = TRUE)
  zip_paths <- gsub("\\\\", "/", zip_manifest$Name)
  zip_paths <- zip_paths[nzchar(zip_paths)]

  top_parts <- sub("/.*$", "", zip_paths)
  top_parts <- unique(top_parts[nzchar(top_parts)])

  extract_storage(archive = archive_path, exdir = extract_root)

  archive_scan_root <- extract_root

  if (length(top_parts) == 1) {
    candidate_root <- fs::path(extract_root, top_parts[1])
    if (fs::dir_exists(candidate_root)) {
      archive_scan_root <- candidate_root
    }
  }

  out <- scan_directory_storage(
    root = archive_scan_root,
    storage_id = storage_id,
    person_id = person_id,
    scan_time = scan_time,
    compute_signature = compute_signature,
    max_signature_size = max_signature_size
  )

  out$container_file <- rep(basename(archive_path), nrow(out))
  out$container_type <- rep(tolower(fs::path_ext(archive_path)), nrow(out))

  attr(out, "archive_root") <- archive_path
  attr(out, "extracted_root") <- extract_root
  attr(out, "archive_scan_root") <- archive_scan_root

  out
}

#' @keywords internal
#' @noRd
#' @importFrom fs path_abs dir_exists file_info path_file path_rel path_ext
scan_directory_storage <- function(
  root,
  storage_id,
  person_id,
  scan_time = Sys.time(),
  compute_signature = TRUE,
  max_signature_size = 200 * 1024 * 1024
) {
  start_time <- Sys.time()

  message("Starting scan_storage() on: ", root)

  root <- fs::path_abs(root)

  if (!fs::dir_exists(root)) {
    stop("scan_storage(): root path does not exist: ", root, call. = FALSE)
  }

  # --- initialise the return value ---

  snapshot <- empty_snapshot()

  # --- list files safely ---
  files <- fs::dir_ls(
    path = root,
    recurse = TRUE,
    all = TRUE,
    type = "file",
    fail = FALSE
  )

  # --- early feedback ---
  n_files <- length(files)

  if (n_files == 0) {
    message("No files found.")
  } else if (n_files > 10000) {
    message("Scanning ", n_files, " files, this may take several minutes...")
  } else if (n_files > 1000) {
    message("Scanning ", n_files, " files, this may take 1-2 minutes...")
  } else {
    message("Scanning ", n_files, " files.")
  }

  # --- exclude noise ---
  ignore_patterns <- c(
    "[/\\\\]\\.git[/\\\\]",
    "[/\\\\]\\.Rproj\\.user[/\\\\]",
    "\\$RECYCLE\\.BIN",
    "System Volume Information"
  )

  for (pat in ignore_patterns) {
    files <- files[!grepl(pat, files)]
  }

  # --- safe file_info ---
  info <- tryCatch(
    fs::file_info(files),
    error = function(e) {
      message("file_info() failed, retrying per-file...")

      results <- lapply(files, function(f) {
        tryCatch(
          fs::file_info(f),
          error = function(e) NULL
        )
      })

      results <- Filter(Negate(is.null), results)

      if (length(results) == 0) {
        return(NULL)
      }

      do.call(rbind, results)
    }
  )

  if (is.null(info) || nrow(info) == 0) {
    files <- files[0]
  } else {
    valid <- !is.na(info$size)
    files <- files[valid]
    info <- info[valid, , drop = FALSE]
  }

  if (length(files) > 0) {
    # If there are no files at the relative path, return and empty data.frame

    rel_path <- fs::path_rel(files, start = root)
    filename <- fs::path_file(files)
    ext <- fs::path_ext(files)

    snapshot <- data.frame(
      storage_id = storage_id,
      person_id = person_id,
      full_path = as.character(files),
      rel_path = as.character(rel_path),
      filename = filename,
      stem = tools::file_path_sans_ext(filename),
      extension = ifelse(ext == "", NA_character_, tolower(ext)),
      type = info$type,
      size = as.double(info$size),
      mtime = info$modification_time,
      ctime = info$change_time,
      atime = info$access_time,
      birth_time = info$birth_time,
      depth = lengths(strsplit(rel_path, "[/\\\\]+")),
      links = info$hard_links,
      permissions = info$permissions,
      quick_sig = NA_character_,
      scan_time = scan_time,
      repo_root = rep(NA_character_, length(files)),
      repo_rel_path = rep(NA_character_, length(files)),
      git_tracked = rep(NA, length(files)),
      stringsAsFactors = FALSE
    )

    snapshot$storage_path_id <- paste(
      snapshot$storage_id,
      snapshot$rel_path,
      sep = "::"
    )
  }

  # --- signatures ---
  if (compute_signature && nrow(snapshot) > 0) {
    eligible_idx <- which(snapshot$size <= max_signature_size)

    if (length(eligible_idx) > 0) {
      pb <- progress::progress_bar$new(
        format = "  computing signatures [:bar] :percent (:current/:total)",
        total = length(eligible_idx),
        clear = FALSE,
        width = 60,
        stream = stderr()
      )

      safe_quick_signature <- function(p) {
        pb$tick()
        utils::flush.console()
        tryCatch(quick_signature(p), error = function(e) NA_character_)
      }

      snapshot$quick_sig[eligible_idx] <- vapply(
        snapshot$full_path[eligible_idx],
        safe_quick_signature,
        character(1)
      )
    }
  }

  message("Signatures computed. Detecting repositories and Git status...")


  # --- storage-scoped logical file identity ---
  # storage_path_id approximates a stable Record Resource identity within
  # one storage context. Observation-specific identity is added later
  # via add_snapshot_context().

  snapshot$storage_path_id <- paste(snapshot$storage_id, snapshot$rel_path, sep = "::")

  # --- detect repo roots safely ---
  git_dirs <- tryCatch(
    fs::dir_ls(
      root,
      recurse = TRUE,
      type = "directory",
      all = TRUE,
      regexp = "[/\\\\]\\.git$",
      fail = FALSE
    ),
    error = function(e) character()
  )

  repo_roots <- unique(dirname(git_dirs))
  repo_roots_norm <- fs::path_abs(repo_roots)


  # Find the nearest enclosing Git repository.
  # Match complete directory components, not string prefixes:
  # "project-backup" must not be attributed to "project".
  find_repo_root_safe <- function(path) {
    matches <- repo_roots_norm[
      path == repo_roots_norm |
        startsWith(path, paste0(repo_roots_norm, "/"))
    ]

    if (length(matches) == 0) {
      return(NA_character_)
    }

    matches[which.max(nchar(matches))]
  }


  if (length(repo_roots_norm) > 0) {
    snapshot$repo_root <- vapply(
      snapshot$full_path,
      find_repo_root_safe,
      character(1)
    )

    # --- repo_rel_path ---
    snapshot$repo_rel_path <- vapply(
      seq_len(nrow(snapshot)),
      function(i) {
        if (is.na(snapshot$repo_root[i])) {
          return(NA_character_)
        }
        fs::path_rel(snapshot$full_path[i], start = snapshot$repo_root[i])
      },
      character(1)
    )

    # --- git tracked ---
    message("Checking Git tracked files...")

    for (repo in repo_roots_norm) {
      idx <- which(!is.na(snapshot$repo_root) & snapshot$repo_root == repo)
      if (length(idx) == 0) next

      tracked <- tryCatch(
        system2("git",
          args = c("-C", repo, "ls-files"),
          stdout = TRUE,
          stderr = TRUE
        ),
        warning = function(w) character(),
        error = function(e) character()
      )

      if (length(tracked) == 0) next

      snapshot$git_tracked[idx] <- snapshot$repo_rel_path[idx] %in% tracked
    }
  }

  # --- repo metadata ---
  if (length(repo_roots_norm) > 0) {
    repos_df <- data.frame(
      repo_root = repo_roots_norm,
      git_remote = vapply(repo_roots_norm, get_git_remote, character(1)),
      git_branch = vapply(repo_roots_norm, get_git_branch, character(1)),
      stringsAsFactors = FALSE
    )
  } else {
    repos_df <- data.frame(
      repo_root = character(),
      git_remote = character(),
      git_branch = character(),
      stringsAsFactors = FALSE
    )
  }

  # --- attributes ---
  pkg_version <- tryCatch(
    as.character(utils::packageVersion("fscontext")),
    error = function(e) NA_character_
  )

  attr(snapshot, "created_at") <- scan_time
  attr(snapshot, "created_by") <- "scan_storage"
  attr(snapshot, "package") <- "fscontext"
  attr(snapshot, "package_version") <- pkg_version
  attr(snapshot, "repos") <- repos_df
  attr(snapshot, "schema_version") <- "0.1.3"
  attr(snapshot, "scan_root") <- root

  attr(snapshot, "scan_call") <- paste(
    deparse(match.call()),
    collapse = " "
  )

  attr(snapshot, "signature_enabled") <- compute_signature
  attr(snapshot, "max_signature_size") <- max_signature_size

  end_time <- Sys.time()
  elapsed <- round(as.numeric(difftime(end_time, start_time, units = "secs")), 2)
  skipped_estimate <- n_files - nrow(snapshot)

  message("Files scanned: ", nrow(snapshot))
  message("Files in Git repos: ", sum(!is.na(snapshot$repo_root)))
  message("Files tracked by Git: ", sum(snapshot$git_tracked, na.rm = TRUE))
  message("Skipped approximately ", skipped_estimate, " inaccessible files")
  message("scan_storage completed in ", elapsed, " seconds")

  snapshot
}


#' @keywords internal
#' @noRd
empty_snapshot <- function() {
  data.frame(
    storage_id = character(),
    person_id = character(),
    full_path = character(),
    rel_path = character(),
    filename = character(),
    stem = character(),
    extension = character(),
    type = character(),
    size = double(),
    mtime = as.POSIXct(character()),
    ctime = as.POSIXct(character()),
    atime = as.POSIXct(character()),
    birth_time = as.POSIXct(character()),
    depth = integer(),
    links = integer(),
    permissions = character(),
    quick_sig = character(),
    scan_time = as.POSIXct(character()),
    repo_root = character(),
    repo_rel_path = character(),
    git_tracked = logical(),
    storage_path_id = character(),
    stringsAsFactors = FALSE
  )
}

Try the fscontext package in your browser

Any scripts or data that you put into this service are public.

fscontext documentation built on Nov. 19, 2026, 5:11 p.m.