R/observe_wacz.R

Defines functions read_datapackage match_pages_to_cdx collapse_cdx_versions read_cdx read_pages_jsonl observe_wacz

Documented in observe_wacz

#' Observe a WACZ web archive
#'
#' @description
#' Creates an observational data frame from a WACZ web archive.
#'
#' A WACZ (Web Archive Collection Zipped) file is an open package format
#' for storing and exchanging web archive collections. A WACZ archive is
#' a ZIP container that combines one or more WARC files containing captured
#' web resources with indexes, page metadata, and package metadata needed
#' to locate, describe, validate, and replay the archived content.
#'
#' Unlike a standalone WARC file, which primarily stores captured HTTP
#' requests and responses and their payloads, a WACZ archive provides
#' additional structure around those captures. Typical components include
#' WARC files, a CDX index, a page list, and a `datapackage.json` manifest.
#'
#' `observe_wacz()` extracts this structural information, combines
#' page-level information with WARC index metadata, and returns one
#' observational row for each archived web page.
#'
#' The resulting object represents observations only. It intentionally
#' avoids making semantic assertions about Records, Record Parts,
#' Instantiations, or other archival entities. Such interpretation can
#' be added later with [wacz_to_recordset_df()] or downstream semantic
#' enrichment workflows.
#'
#' @param wacz
#' Path to a `.wacz` archive.
#'
#' @return
#' A tibble containing observations extracted from the archive.
#'
#' The returned object carries two attributes:
#'
#' * `datapackage`, containing the parsed `datapackage.json`
#'   metadata supplied by the WACZ archive;
#' * `wacz`, containing the normalized path to the source archive.
#'
#' Typical variables include:
#'
#' * page identifiers;
#' * resource locators (URLs);
#' * page titles;
#' * timestamps;
#' * extracted text;
#' * text signatures;
#' * MIME types;
#' * WARC digests;
#' * archive offsets;
#' * version counts.
#'
#' @details
#' A WACZ archive commonly contains several complementary sources of
#' observational information. `pages/pages.jsonl` describes archived
#' pages, while `indexes/index.cdx` indexes captured resources stored in
#' the WARC files. The `datapackage.json` manifest describes the package
#' itself and its constituent resources.
#'
#' The function currently performs the following steps:
#'
#' * extracts the WACZ archive into a temporary directory;
#' * reads the archive `datapackage.json`;
#' * parses page metadata from `pages/pages.jsonl`;
#' * parses WARC index metadata from `indexes/index.cdx`;
#' * collapses multiple archived entries for the same resource;
#' * joins page observations with archive metadata.
#'
#' The resulting observations preserve evidence supplied by the archive
#' without interpreting its archival semantics. The underlying WARC,
#' index, page, and package structures may therefore support richer
#' observational analysis than is exposed by the current page-level
#' representation.
#'
#' @references
#' Webrecorder WACZ format specification:
#' \url{https://specs.webrecorder.net/wacz/1.1.1/}
#'
#' @seealso
#' [wacz_to_recordset_df()]
#'
#' @examples
#' wacz <- system.file(
#'   "testdata", "fscontext_020.wacz",
#'   package = "fscontext"
#' )
#'
#' observe_wacz(wacz)
#'
#' @export

observe_wacz <- function(wacz) {
  tmp <- tempfile("wacz")

  extract_storage(
    archive = wacz,
    exdir = tmp
  )

  datapackage <- read_datapackage(tmp)

  pages <- read_pages_jsonl(tmp)

  cdx <- read_cdx(tmp) |>
    collapse_cdx_versions()

  observations <- match_pages_to_cdx(
    pages,
    cdx
  ) |>
    dplyr::mutate(
      archive = basename(wacz),
      full_path = normalizePath(wacz)
    )

  attr(observations, "datapackage") <- datapackage
  attr(observations, "wacz") <- normalizePath(wacz)

  observations
}

#' @keywords internal
#' @importFrom jsonlite stream_in
#' @importFrom dplyr filter mutate rename
#' @importFrom tibble as_tibble
#' @noRd

read_pages_jsonl <- function(path) {
  pages_file <- file.path(path, "pages", "pages.jsonl")

  if (!file.exists(pages_file)) {
    stop(
      "Cannot find 'pages/pages.jsonl' in ",
      path,
      call. = FALSE
    )
  }

  pages <- suppressWarnings(
    jsonlite::stream_in(
      file(pages_file),
      verbose = FALSE
    )
  ) |>
    tibble::as_tibble() |>
    dplyr::rename(
      page_id = id,
      resource_locator = url,
      timestamp = ts,
      has_text = hasText,
      favicon = favIconUrl
    ) |>
    dplyr::filter(!is.na(resource_locator)) |>
    dplyr::mutate(
      text_length = nchar(text),
      quick_sig_text = quick_signature_text(text)
    )
}

#' @keywords internal
#' @importFrom jsonlite fromJSON
#' @importFrom tibble as_tibble tibble
#' @importFrom dplyr mutate rename
#' @importFrom purrr map_dfr
#' @noRd

read_cdx <- function(path) {
  cdx_file <- file.path(path, "indexes", "index.cdx")

  if (!file.exists(cdx_file)) {
    stop(
      "Cannot find 'indexes/index.cdx' in ",
      path,
      call. = FALSE
    )
  }

  lines <- readLines(
    cdx_file,
    warn = FALSE,
    encoding = "UTF-8"
  )

  lines <- lines[nzchar(lines)]

  parsed <- purrr::map_dfr(
    lines,
    function(line) {
      parts <- strsplit(
        line,
        " ",
        fixed = TRUE
      )[[1]]

      if (length(parts) < 3) {
        return(NULL)
      }

      urlkey <- parts[1]
      timestamp <- parts[2]
      json <- paste(
        parts[-c(1, 2)],
        collapse = " "
      )

      meta <- jsonlite::fromJSON(json)

      tibble::tibble(
        urlkey = urlkey,
        cdx_timestamp = timestamp,
        resource_locator = meta$url,
        digest = meta$digest %||% NA_character_,
        mime = meta$mime %||% NA_character_,
        offset = meta$offset %||% NA_real_,
        length = meta$length %||% NA_real_,
        record_digest = meta$recordDigest %||% NA_character_,
        status = meta$status %||% NA_integer_,
        warc_filename = meta$filename %||% NA_character_
      )
    }
  )

  parsed
}

#' Collapse repeated WACZ observations of the same archived resource
#'
#' @keywords internal
#' @noRd

collapse_cdx_versions <- function(cdx) {
  version_count <-
    cdx |>
    dplyr::count(
      resource_locator,
      name = "n_versions"
    )

  cdx |>
    dplyr::filter(
      mime == "text/html"
    ) |>
    dplyr::group_by(resource_locator) |>
    dplyr::slice(1) |>
    dplyr::ungroup() |>
    dplyr::left_join(
      version_count,
      by = "resource_locator"
    )
}


#' Match page observations to archived payload metadata
#'
#' @keywords internal
#' @noRd

match_pages_to_cdx <- function(
  pages,
  cdx
) {
  dplyr::left_join(
    pages,
    cdx,
    by = "resource_locator"
  )
}

#' @keywords internal
#' @importFrom jsonlite read_json
#' @importFrom tibble as_tibble
#' @noRd

read_datapackage <- function(path) {
  datapackage_file <- file.path(path, "datapackage.json")

  if (!file.exists(datapackage_file)) {
    stop(
      "Cannot find 'datapackage.json' in ",
      path,
      call. = FALSE
    )
  }

  dp <- jsonlite::read_json(
    datapackage_file,
    simplifyVector = TRUE
  )

  dp
}

Try the fscontext package in your browser

Any scripts or data that you put into this service are public.

fscontext documentation built on Nov. 19, 2026, 5:11 p.m.