R/validate_specifications.R

Defines functions schema_error_message entry_from_path collect_named read_bib_keys check_specification_consistency validate_specifications read_specifications

#' Read the specifications
#'
#' Reads the manifest and all specification files listed in it. Used by the
#'  tools in `tools/` to validate the specifications and to build datasets and
#'  the Excel template from them. The installed package does not read the
#'  specifications at runtime.
#'
#' @param spec_dir directory with `manifest.json` and the specification files
#'
#' @return a list with the parsed manifest (`manifest`) and one element per component
#'
#' @md
#' @noRd
read_specifications <- function(spec_dir = system.file("spec", package = "SoilManageR")) {
  if (!requireNamespace("jsonlite", quietly = TRUE)) {
    stop("Package 'jsonlite' is needed to read the specifications. Please install it.")
  }
  manifest <- jsonlite::read_json(file.path(spec_dir, "manifest.json"), simplifyVector = FALSE)
  spec <- list(manifest = manifest)
  for (component in manifest$components) {
    path <- file.path(spec_dir, component$file)
    if (file.exists(path)) {
      spec[[component$name]] <- jsonlite::read_json(path, simplifyVector = FALSE)
    }
  }
  spec
}

#' Validate the specifications
#'
#' Checks the specification files against their JSON schemas and checks the
#'  consistency across files: unique identifiers, unambiguous aliases,
#'  references between files, source keys, seed mixture totals, species
#'  information, unused definitions and parameters needed for calculations.
#'
#' Schema validation needs the package `jsonvalidate`. Without it, only the
#'  cross-file checks are done and an info diagnostic (`SM-I001`) is returned.
#'
#' @param spec_dir directory with `manifest.json` and the specification files
#' @param schema_dir directory with the JSON schemas
#' @param bib_file BibTeX file with the keys that may be cited in `sources`
#' @param schema_validation logical, if FALSE the schema validation is skipped
#'
#' @return a data frame of diagnostics (`severity`, `code`, `file`, `entry`,
#'  `attribute`, `message`); no rows if no issues were found
#'
#' @md
#' @noRd
validate_specifications <- function(spec_dir = system.file("spec", package = "SoilManageR"),
                                    schema_dir = system.file("schemas", package = "SoilManageR"),
                                    bib_file = system.file("REFERENCES.bib", package = "SoilManageR"),
                                    schema_validation = TRUE) {

  if (!requireNamespace("jsonlite", quietly = TRUE)) {
    stop("Package 'jsonlite' is needed to validate the specifications. Please install it.")
  }
  diagnostics <- list()
  add <- function(...) diagnostics[[length(diagnostics) + 1]] <<- diagnostic(...)

  # Manifest and files ----------------
  manifest_file <- file.path(spec_dir, "manifest.json")
  if (!file.exists(manifest_file)) {
    return(diagnostic("SM-E001", "manifest.json not found", file = "manifest.json"))
  }
  manifest <- tryCatch(jsonlite::read_json(manifest_file, simplifyVector = FALSE),
                       error = function(e) NULL)
  if (is.null(manifest)) {
    return(diagnostic("SM-E001", "manifest.json is not valid JSON", file = "manifest.json"))
  }

  files <- c(list(list(name = "manifest", file = "manifest.json", schema = "manifest.schema.json")),
             manifest$components)
  spec <- list()
  use_schemas <- schema_validation && requireNamespace("jsonvalidate", quietly = TRUE)
  if (schema_validation && !use_schemas) {
    add("SM-I001", "Package 'jsonvalidate' is not installed, the files were not checked against their schemas.")
  }

  for (component in files) {
    path <- file.path(spec_dir, component$file)
    if (!file.exists(path)) {
      add("SM-E001", "file listed in the manifest does not exist", file = component$file)
      next
    }
    json_text <- paste(readLines(path, encoding = "UTF-8", warn = FALSE), collapse = "\n")
    parsed <- tryCatch(jsonlite::parse_json(json_text, simplifyVector = FALSE), error = function(e) e)
    if (inherits(parsed, "error")) {
      add("SM-E001", paste("file is not valid JSON:", conditionMessage(parsed)), file = component$file)
      next
    }
    spec[[component$name]] <- parsed

    if (use_schemas) {
      schema_file <- file.path(schema_dir, component$schema)
      if (!file.exists(schema_file)) {
        add("SM-E001", paste("schema", component$schema, "does not exist"), file = component$file)
        next
      }
      validator <- jsonvalidate::json_schema$new(schema_file, engine = "ajv")
      result <- validator$validate(json_text, verbose = TRUE, greedy = TRUE)
      if (!isTRUE(as.logical(result))) {
        errors <- attr(result, "errors")
        for (i in seq_len(nrow(errors))) {
          add("SM-E002", schema_error_message(errors[i, ]), file = component$file,
              entry = entry_from_path(parsed, errors$instancePath[i]),
              attribute = errors$instancePath[i])
        }
      }
    }
  }

  listed <- vapply(files, function(x) x$file, character(1))
  for (f in setdiff(list.files(spec_dir, pattern = "\\.json$"), listed)) {
    add("SM-E009", "file is not listed in manifest.json", file = f)
  }

  # Files referenced by excel.json ----------------
  for (f in c(spec$excel$readme$file, vapply(spec$excel$readme$images, function(i) i$file, character(1)))) {
    if (!file.exists(file.path(spec_dir, f))) add("SM-E001", paste("referenced file", f, "does not exist"), file = "excel.json")
  }

  # Cross-file checks ----------------
  diagnostics <- c(diagnostics, check_specification_consistency(spec, read_bib_keys(bib_file)))

  if (length(diagnostics) == 0) return(no_diagnostics())
  out <- do.call(rbind, diagnostics)
  rownames(out) <- NULL
  out
}

#' Consistency checks across specification files
#'
#' @param spec list of parsed specification components
#' @param bib_keys keys available in the bibliography
#'
#' @return list of diagnostics data frames
#'
#' @md
#' @noRd
check_specification_consistency <- function(spec, bib_keys) {
  diagnostics <- list()
  add <- function(...) diagnostics[[length(diagnostics) + 1]] <<- diagnostic(...)

  get_entries <- function(component, key) {
    x <- spec[[component]][[key]]
    if (is.null(x)) list() else x
  }
  field <- function(entries, name) {
    vapply(entries, function(e) if (is.null(e[[name]])) NA_character_ else as.character(e[[name]])[1], character(1))
  }

  catalogues <- list(
    categories = list(file = "operations.json", component = "operations", key = "categories", id = "id"),
    operations = list(file = "operations.json", component = "operations", key = "operations", id = "id"),
    devices = list(file = "devices.json", component = "devices", key = "devices", id = "id"),
    units = list(file = "units.json", component = "units", key = "units", id = "id"),
    attributes = list(file = "attributes.json", component = "attributes", key = "attributes", id = "name"),
    crops = list(file = "crops.json", component = "crops", key = "crops", id = "id"),
    fertilizers = list(file = "fertilizers.json", component = "fertilizers", key = "fertilizers", id = "id"),
    products = list(file = "products.json", component = "products", key = "products", id = "id"),
    seed_mixtures = list(file = "seed_mixtures.json", component = "seed_mixtures", key = "seed_mixtures", id = "id")
  )
  entries <- lapply(catalogues, function(cat) get_entries(cat$component, cat$key))
  ids <- mapply(function(e, cat) field(e, cat$id), entries, catalogues, SIMPLIFY = FALSE)

  # Unique identifiers and unambiguous aliases within each catalogue ----------------
  for (name in names(catalogues)) {
    file <- catalogues[[name]]$file
    dup <- unique(ids[[name]][duplicated(ids[[name]]) & !is.na(ids[[name]])])
    for (d in dup) add("SM-E003", paste0("identifier is used more than once in ", name), file = file, entry = d)
    for (e in entries[[name]]) {
      for (a in unlist(e$aliases)) {
        other <- vapply(entries[[name]], function(o) {
          !identical(o, e) && (a %in% c(o[[catalogues[[name]]$id]], unlist(o$aliases)))
        }, logical(1))
        if (a %in% ids[[name]] || any(other)) {
          add("SM-E004", paste0("alias '", a, "' is also an identifier or alias of another entry"),
              file = file, entry = as.character(e[[catalogues[[name]]$id]]), attribute = "aliases")
        }
      }
    }
  }

  # The product column refers to fertilizers, products and seed mixtures: identifiers must be unique across them
  product_catalogues <- c("fertilizers", "products", "seed_mixtures")
  product_ids <- unlist(lapply(product_catalogues, function(n) stats::setNames(ids[[n]], rep(n, length(ids[[n]])))))
  for (d in unique(product_ids[duplicated(product_ids)])) {
    used_in <- unique(names(product_ids)[product_ids == d])
    if (length(used_in) > 1) {
      add("SM-E003", paste0("product identifier is used in several catalogues: ", paste(used_in, collapse = ", ")),
          file = paste0(used_in[1], ".json"), entry = d)
    }
  }

  # References between files ----------------
  ref_check <- function(values, known, what, file, entry, attribute) {
    for (v in setdiff(unique(stats::na.omit(values)), known)) {
      add("SM-E005", paste0("unknown ", what, " '", v, "'"), file = file, entry = entry, attribute = attribute)
    }
  }
  for (o in entries$operations) {
    ref_check(o$category, ids$categories, "category", "operations.json", o$id, "category")
  }
  for (a in entries$attributes) {
    ref_check(unlist(a$required_for_categories), ids$categories, "category", "attributes.json", a$name, "required_for_categories")
    ref_check(unlist(a$required_with), ids$attributes, "attribute", "attributes.json", a$name, "required_with")
  }
  option_ops <- character()
  option_devices <- character()
  option_units <- character()
  for (opt in get_entries("management_options", "options")) {
    op <- as.character(opt$operation)
    if (op %in% option_ops) add("SM-E003", "operation is listed more than once", file = "management_options.json", entry = op)
    option_ops <- c(option_ops, op)
    ref_check(op, ids$operations, "operation", "management_options.json", op, "operation")
    devices <- field(opt$devices, "device")
    for (d in unique(devices[duplicated(devices)])) {
      add("SM-E003", paste0("device '", d, "' is listed more than once for this operation"),
          file = "management_options.json", entry = op, attribute = "devices")
    }
    ref_check(devices, ids$devices, "device", "management_options.json", op, "devices")
    units <- unlist(lapply(opt$devices, function(d) unlist(d$units)))
    ref_check(units, ids$units, "unit", "management_options.json", op, "units")
    option_devices <- c(option_devices, devices)
    option_units <- c(option_units, units)
  }
  for (name in product_catalogues) {
    for (e in entries[[name]]) {
      ref_check(unlist(e$operations), ids$operations, "operation", catalogues[[name]]$file, e$id, "operations")
    }
  }
  for (m in entries$seed_mixtures) {
    ref_check(field(m$components, "crop"), ids$crops, "crop", "seed_mixtures.json", m$id, "components")
  }

  # Template presentation ----------------
  if (!is.null(spec$excel)) {
    for (s in setdiff(unlist(spec$excel$header_styles), names(spec$excel$styles))) {
      add("SM-E005", paste0("unknown style '", s, "'"), file = "excel.json", attribute = "header_styles")
    }
    groups <- unique(vapply(entries$attributes, function(a) if (is.null(a$excel$header_group)) NA_character_ else a$excel$header_group, character(1)))
    for (g in setdiff(stats::na.omit(groups), names(spec$excel$header_styles))) {
      add("SM-E005", paste0("no header style for header group '", g, "'"), file = "excel.json", attribute = "header_styles")
    }
    for (i in seq_along(spec$excel$examples)) {
      for (a in setdiff(names(spec$excel$examples[[i]]), ids$attributes)) {
        add("SM-E005", paste0("unknown attribute '", a, "' in example ", i), file = "excel.json", attribute = "examples")
      }
    }
    sheet_names <- vapply(spec$excel$sheets, function(s) s$name, character(1))
    if (anyDuplicated(sheet_names)) add("SM-E003", "sheet names are not unique", file = "excel.json", attribute = "sheets")
  }

  # Sources ----------------
  for (name in names(catalogues)) {
    for (e in entries[[name]]) {
      keys <- unique(unlist(collect_named(e, "sources")))
      for (k in setdiff(keys, bib_keys)) {
        add("SM-E006", paste0("source key '", k, "' is not in REFERENCES.bib"),
            file = catalogues[[name]]$file, entry = as.character(e[[catalogues[[name]]$id]]), attribute = "sources")
      }
    }
  }

  # Seed mixture totals ----------------
  totals <- c(seed_mass_percent = 100, seed_mass_g_per_kg_mixture = 1000)
  for (m in entries$seed_mixtures) {
    basis <- as.character(m$basis)
    if (length(basis) == 1 && basis %in% names(totals)) {
      total <- sum(vapply(m$components, function(x) as.numeric(x$amount), numeric(1)))
      if (abs(total - totals[[basis]]) > totals[[basis]] * 0.005) {
        add("SM-E007", paste0("components add up to ", total, " instead of ", totals[[basis]], " (", basis, ")"),
            file = "seed_mixtures.json", entry = m$id, attribute = "components")
      }
    }
  }

  # Species information ----------------
  species <- do.call(rbind, lapply(entries$crops, function(cr) {
    if (is.null(cr$species)) return(NULL)
    data.frame(crop = cr$id, species = cr$species$id, scientific_name = cr$species$scientific_name,
               legume = if (is.null(cr$species$legume)) NA else cr$species$legume, stringsAsFactors = FALSE)
  }))
  if (!is.null(species)) {
    for (s in unique(species$species)) {
      rows <- species[species$species == s, ]
      if (length(unique(rows$scientific_name)) > 1 || length(unique(stats::na.omit(rows$legume))) > 1) {
        add("SM-E008", paste0("crops of species '", s, "' have different scientific names or legume status"),
            file = "crops.json", entry = paste(rows$crop, collapse = ", "), attribute = "species")
      }
    }
  }

  # Draft descriptions ----------------
  for (name in names(catalogues)) {
    for (e in entries[[name]]) {
      if (isTRUE(e$draft)) {
        add("SM-W002", "description is a draft", file = catalogues[[name]]$file,
            entry = as.character(e[[catalogues[[name]]$id]]), attribute = "description")
      }
    }
  }

  # Unused definitions ----------------
  for (category in setdiff(ids$categories, field(entries$operations, "category"))) {
    add("SM-W003", "category has no operations", file = "operations.json", entry = category)
  }
  for (o in setdiff(ids$operations, option_ops)) {
    add("SM-W003", "operation has no devices in management_options.json", file = "operations.json", entry = o)
  }
  for (d in setdiff(ids$devices, option_devices)) {
    add("SM-W003", "device is not used in management_options.json", file = "devices.json", entry = d)
  }
  for (u in setdiff(ids$units, option_units)) {
    add("SM-W003", "unit is not used in management_options.json", file = "units.json", entry = u)
  }

  # Parameters needed for calculations ----------------
  tillage_ops <- ids$operations[field(entries$operations, "category") %in% c("tillage", "sowing")]
  stir_devices <- unique(unlist(lapply(get_entries("management_options", "options"), function(opt) {
    if (as.character(opt$operation) %in% tillage_ops) field(opt$devices, "device")
  })))
  for (d in entries$devices) {
    if (d$id %in% stir_devices && is.null(d$stir) && !isTRUE(d$stir_not_applicable)) {
      add("SM-W004", "device is used in tillage or sowing operations but has no STIR parameters",
          file = "devices.json", entry = d$id, attribute = "stir")
    }
  }
  for (f in entries$fertilizers) {
    type <- as.character(f$type)
    if (length(type) != 1) next
    if (type == "organic" &&
        (is.null(f$DMC) || is.null(f$composition$C) || is.null(f$composition$N))) {
      add("SM-W004", "DMC, C or N content missing, no C and N input can be calculated",
          file = "fertilizers.json", entry = f$id, attribute = "composition")
    }
    if (type == "organic" && is.null(f$ISMO$value)) {
      add("SM-W004", "ISMO missing, no stable organic matter can be calculated",
          file = "fertilizers.json", entry = f$id, attribute = "ISMO")
    }
    if (type == "mineral" && is.null(f$composition$N)) {
      add("SM-W004", "N content missing, no N input can be calculated from product amounts",
          file = "fertilizers.json", entry = f$id, attribute = "composition")
    }
  }
  for (cr in entries$crops) {
    if ("main_crop" %in% unlist(cr$selectable_as)) {
      for (part in c("carbon", "plant_cover")) {
        if (is.null(cr[[part]])) {
          add("SM-W004", paste("main crop has no", part, "parameters"), file = "crops.json", entry = cr$id, attribute = part)
        }
      }
    }
  }

  diagnostics
}

#' Keys of a BibTeX file
#' @noRd
read_bib_keys <- function(bib_file) {
  if (!nzchar(bib_file) || !file.exists(bib_file)) return(character())
  lines <- readLines(bib_file, encoding = "UTF-8", warn = FALSE)
  keys <- regmatches(lines, regexpr("^\\s*@[A-Za-z]+\\s*\\{\\s*[^,\\s]+", lines, perl = TRUE))
  sub("^\\s*@[A-Za-z]+\\s*\\{\\s*", "", keys, perl = TRUE)
}

#' Collect all elements with a given name in a nested list
#' @noRd
collect_named <- function(x, name) {
  if (!is.list(x)) return(list())
  found <- if (!is.null(names(x)) && name %in% names(x)) list(x[[name]]) else list()
  c(found, unlist(lapply(unname(x), collect_named, name = name), recursive = FALSE))
}

#' Identifier of the entry a JSON path points to
#'
#' For a path such as `/devices/3/stir/depth`, returns the `id` (or `name`)
#'  of the fourth device.
#'
#' @noRd
entry_from_path <- function(parsed, path) {
  parts <- strsplit(sub("^/", "", path), "/", fixed = TRUE)[[1]]
  if (length(parts) < 2 || !grepl("^[0-9]+$", parts[2])) return(NA_character_)
  entry <- parsed[[parts[1]]][[as.integer(parts[2]) + 1]]
  id <- if (!is.null(entry$id)) entry$id else if (!is.null(entry$name)) entry$name else if (!is.null(entry$operation)) entry$operation else NULL
  if (is.null(id)) NA_character_ else as.character(id)
}

#' Readable message for a schema error from jsonvalidate
#' @noRd
schema_error_message <- function(error) {
  msg <- error$message
  params <- error$params
  if (is.data.frame(params) || is.list(params)) {
    extra <- unlist(params, use.names = TRUE)
    extra <- extra[!is.na(extra)]
    if (length(extra)) msg <- paste0(msg, " (", paste(names(extra), extra, sep = ": ", collapse = ", "), ")")
  }
  msg
}

Try the SoilManageR package in your browser

Any scripts or data that you put into this service are public.

SoilManageR documentation built on Nov. 17, 2026, 5:08 p.m.