Nothing
#' Read the specifications
#'
#' Reads the manifest and all specification files listed in it. Used by the
#' tools in `tools/` to validate the specifications and to build datasets and
#' the Excel template from them. The installed package does not read the
#' specifications at runtime.
#'
#' @param spec_dir directory with `manifest.json` and the specification files
#'
#' @return a list with the parsed manifest (`manifest`) and one element per component
#'
#' @md
#' @noRd
read_specifications <- function(spec_dir = system.file("spec", package = "SoilManageR")) {
if (!requireNamespace("jsonlite", quietly = TRUE)) {
stop("Package 'jsonlite' is needed to read the specifications. Please install it.")
}
manifest <- jsonlite::read_json(file.path(spec_dir, "manifest.json"), simplifyVector = FALSE)
spec <- list(manifest = manifest)
for (component in manifest$components) {
path <- file.path(spec_dir, component$file)
if (file.exists(path)) {
spec[[component$name]] <- jsonlite::read_json(path, simplifyVector = FALSE)
}
}
spec
}
#' Validate the specifications
#'
#' Checks the specification files against their JSON schemas and checks the
#' consistency across files: unique identifiers, unambiguous aliases,
#' references between files, source keys, seed mixture totals, species
#' information, unused definitions and parameters needed for calculations.
#'
#' Schema validation needs the package `jsonvalidate`. Without it, only the
#' cross-file checks are done and an info diagnostic (`SM-I001`) is returned.
#'
#' @param spec_dir directory with `manifest.json` and the specification files
#' @param schema_dir directory with the JSON schemas
#' @param bib_file BibTeX file with the keys that may be cited in `sources`
#' @param schema_validation logical, if FALSE the schema validation is skipped
#'
#' @return a data frame of diagnostics (`severity`, `code`, `file`, `entry`,
#' `attribute`, `message`); no rows if no issues were found
#'
#' @md
#' @noRd
validate_specifications <- function(spec_dir = system.file("spec", package = "SoilManageR"),
schema_dir = system.file("schemas", package = "SoilManageR"),
bib_file = system.file("REFERENCES.bib", package = "SoilManageR"),
schema_validation = TRUE) {
if (!requireNamespace("jsonlite", quietly = TRUE)) {
stop("Package 'jsonlite' is needed to validate the specifications. Please install it.")
}
diagnostics <- list()
add <- function(...) diagnostics[[length(diagnostics) + 1]] <<- diagnostic(...)
# Manifest and files ----------------
manifest_file <- file.path(spec_dir, "manifest.json")
if (!file.exists(manifest_file)) {
return(diagnostic("SM-E001", "manifest.json not found", file = "manifest.json"))
}
manifest <- tryCatch(jsonlite::read_json(manifest_file, simplifyVector = FALSE),
error = function(e) NULL)
if (is.null(manifest)) {
return(diagnostic("SM-E001", "manifest.json is not valid JSON", file = "manifest.json"))
}
files <- c(list(list(name = "manifest", file = "manifest.json", schema = "manifest.schema.json")),
manifest$components)
spec <- list()
use_schemas <- schema_validation && requireNamespace("jsonvalidate", quietly = TRUE)
if (schema_validation && !use_schemas) {
add("SM-I001", "Package 'jsonvalidate' is not installed, the files were not checked against their schemas.")
}
for (component in files) {
path <- file.path(spec_dir, component$file)
if (!file.exists(path)) {
add("SM-E001", "file listed in the manifest does not exist", file = component$file)
next
}
json_text <- paste(readLines(path, encoding = "UTF-8", warn = FALSE), collapse = "\n")
parsed <- tryCatch(jsonlite::parse_json(json_text, simplifyVector = FALSE), error = function(e) e)
if (inherits(parsed, "error")) {
add("SM-E001", paste("file is not valid JSON:", conditionMessage(parsed)), file = component$file)
next
}
spec[[component$name]] <- parsed
if (use_schemas) {
schema_file <- file.path(schema_dir, component$schema)
if (!file.exists(schema_file)) {
add("SM-E001", paste("schema", component$schema, "does not exist"), file = component$file)
next
}
validator <- jsonvalidate::json_schema$new(schema_file, engine = "ajv")
result <- validator$validate(json_text, verbose = TRUE, greedy = TRUE)
if (!isTRUE(as.logical(result))) {
errors <- attr(result, "errors")
for (i in seq_len(nrow(errors))) {
add("SM-E002", schema_error_message(errors[i, ]), file = component$file,
entry = entry_from_path(parsed, errors$instancePath[i]),
attribute = errors$instancePath[i])
}
}
}
}
listed <- vapply(files, function(x) x$file, character(1))
for (f in setdiff(list.files(spec_dir, pattern = "\\.json$"), listed)) {
add("SM-E009", "file is not listed in manifest.json", file = f)
}
# Files referenced by excel.json ----------------
for (f in c(spec$excel$readme$file, vapply(spec$excel$readme$images, function(i) i$file, character(1)))) {
if (!file.exists(file.path(spec_dir, f))) add("SM-E001", paste("referenced file", f, "does not exist"), file = "excel.json")
}
# Cross-file checks ----------------
diagnostics <- c(diagnostics, check_specification_consistency(spec, read_bib_keys(bib_file)))
if (length(diagnostics) == 0) return(no_diagnostics())
out <- do.call(rbind, diagnostics)
rownames(out) <- NULL
out
}
#' Consistency checks across specification files
#'
#' @param spec list of parsed specification components
#' @param bib_keys keys available in the bibliography
#'
#' @return list of diagnostics data frames
#'
#' @md
#' @noRd
check_specification_consistency <- function(spec, bib_keys) {
diagnostics <- list()
add <- function(...) diagnostics[[length(diagnostics) + 1]] <<- diagnostic(...)
get_entries <- function(component, key) {
x <- spec[[component]][[key]]
if (is.null(x)) list() else x
}
field <- function(entries, name) {
vapply(entries, function(e) if (is.null(e[[name]])) NA_character_ else as.character(e[[name]])[1], character(1))
}
catalogues <- list(
categories = list(file = "operations.json", component = "operations", key = "categories", id = "id"),
operations = list(file = "operations.json", component = "operations", key = "operations", id = "id"),
devices = list(file = "devices.json", component = "devices", key = "devices", id = "id"),
units = list(file = "units.json", component = "units", key = "units", id = "id"),
attributes = list(file = "attributes.json", component = "attributes", key = "attributes", id = "name"),
crops = list(file = "crops.json", component = "crops", key = "crops", id = "id"),
fertilizers = list(file = "fertilizers.json", component = "fertilizers", key = "fertilizers", id = "id"),
products = list(file = "products.json", component = "products", key = "products", id = "id"),
seed_mixtures = list(file = "seed_mixtures.json", component = "seed_mixtures", key = "seed_mixtures", id = "id")
)
entries <- lapply(catalogues, function(cat) get_entries(cat$component, cat$key))
ids <- mapply(function(e, cat) field(e, cat$id), entries, catalogues, SIMPLIFY = FALSE)
# Unique identifiers and unambiguous aliases within each catalogue ----------------
for (name in names(catalogues)) {
file <- catalogues[[name]]$file
dup <- unique(ids[[name]][duplicated(ids[[name]]) & !is.na(ids[[name]])])
for (d in dup) add("SM-E003", paste0("identifier is used more than once in ", name), file = file, entry = d)
for (e in entries[[name]]) {
for (a in unlist(e$aliases)) {
other <- vapply(entries[[name]], function(o) {
!identical(o, e) && (a %in% c(o[[catalogues[[name]]$id]], unlist(o$aliases)))
}, logical(1))
if (a %in% ids[[name]] || any(other)) {
add("SM-E004", paste0("alias '", a, "' is also an identifier or alias of another entry"),
file = file, entry = as.character(e[[catalogues[[name]]$id]]), attribute = "aliases")
}
}
}
}
# The product column refers to fertilizers, products and seed mixtures: identifiers must be unique across them
product_catalogues <- c("fertilizers", "products", "seed_mixtures")
product_ids <- unlist(lapply(product_catalogues, function(n) stats::setNames(ids[[n]], rep(n, length(ids[[n]])))))
for (d in unique(product_ids[duplicated(product_ids)])) {
used_in <- unique(names(product_ids)[product_ids == d])
if (length(used_in) > 1) {
add("SM-E003", paste0("product identifier is used in several catalogues: ", paste(used_in, collapse = ", ")),
file = paste0(used_in[1], ".json"), entry = d)
}
}
# References between files ----------------
ref_check <- function(values, known, what, file, entry, attribute) {
for (v in setdiff(unique(stats::na.omit(values)), known)) {
add("SM-E005", paste0("unknown ", what, " '", v, "'"), file = file, entry = entry, attribute = attribute)
}
}
for (o in entries$operations) {
ref_check(o$category, ids$categories, "category", "operations.json", o$id, "category")
}
for (a in entries$attributes) {
ref_check(unlist(a$required_for_categories), ids$categories, "category", "attributes.json", a$name, "required_for_categories")
ref_check(unlist(a$required_with), ids$attributes, "attribute", "attributes.json", a$name, "required_with")
}
option_ops <- character()
option_devices <- character()
option_units <- character()
for (opt in get_entries("management_options", "options")) {
op <- as.character(opt$operation)
if (op %in% option_ops) add("SM-E003", "operation is listed more than once", file = "management_options.json", entry = op)
option_ops <- c(option_ops, op)
ref_check(op, ids$operations, "operation", "management_options.json", op, "operation")
devices <- field(opt$devices, "device")
for (d in unique(devices[duplicated(devices)])) {
add("SM-E003", paste0("device '", d, "' is listed more than once for this operation"),
file = "management_options.json", entry = op, attribute = "devices")
}
ref_check(devices, ids$devices, "device", "management_options.json", op, "devices")
units <- unlist(lapply(opt$devices, function(d) unlist(d$units)))
ref_check(units, ids$units, "unit", "management_options.json", op, "units")
option_devices <- c(option_devices, devices)
option_units <- c(option_units, units)
}
for (name in product_catalogues) {
for (e in entries[[name]]) {
ref_check(unlist(e$operations), ids$operations, "operation", catalogues[[name]]$file, e$id, "operations")
}
}
for (m in entries$seed_mixtures) {
ref_check(field(m$components, "crop"), ids$crops, "crop", "seed_mixtures.json", m$id, "components")
}
# Template presentation ----------------
if (!is.null(spec$excel)) {
for (s in setdiff(unlist(spec$excel$header_styles), names(spec$excel$styles))) {
add("SM-E005", paste0("unknown style '", s, "'"), file = "excel.json", attribute = "header_styles")
}
groups <- unique(vapply(entries$attributes, function(a) if (is.null(a$excel$header_group)) NA_character_ else a$excel$header_group, character(1)))
for (g in setdiff(stats::na.omit(groups), names(spec$excel$header_styles))) {
add("SM-E005", paste0("no header style for header group '", g, "'"), file = "excel.json", attribute = "header_styles")
}
for (i in seq_along(spec$excel$examples)) {
for (a in setdiff(names(spec$excel$examples[[i]]), ids$attributes)) {
add("SM-E005", paste0("unknown attribute '", a, "' in example ", i), file = "excel.json", attribute = "examples")
}
}
sheet_names <- vapply(spec$excel$sheets, function(s) s$name, character(1))
if (anyDuplicated(sheet_names)) add("SM-E003", "sheet names are not unique", file = "excel.json", attribute = "sheets")
}
# Sources ----------------
for (name in names(catalogues)) {
for (e in entries[[name]]) {
keys <- unique(unlist(collect_named(e, "sources")))
for (k in setdiff(keys, bib_keys)) {
add("SM-E006", paste0("source key '", k, "' is not in REFERENCES.bib"),
file = catalogues[[name]]$file, entry = as.character(e[[catalogues[[name]]$id]]), attribute = "sources")
}
}
}
# Seed mixture totals ----------------
totals <- c(seed_mass_percent = 100, seed_mass_g_per_kg_mixture = 1000)
for (m in entries$seed_mixtures) {
basis <- as.character(m$basis)
if (length(basis) == 1 && basis %in% names(totals)) {
total <- sum(vapply(m$components, function(x) as.numeric(x$amount), numeric(1)))
if (abs(total - totals[[basis]]) > totals[[basis]] * 0.005) {
add("SM-E007", paste0("components add up to ", total, " instead of ", totals[[basis]], " (", basis, ")"),
file = "seed_mixtures.json", entry = m$id, attribute = "components")
}
}
}
# Species information ----------------
species <- do.call(rbind, lapply(entries$crops, function(cr) {
if (is.null(cr$species)) return(NULL)
data.frame(crop = cr$id, species = cr$species$id, scientific_name = cr$species$scientific_name,
legume = if (is.null(cr$species$legume)) NA else cr$species$legume, stringsAsFactors = FALSE)
}))
if (!is.null(species)) {
for (s in unique(species$species)) {
rows <- species[species$species == s, ]
if (length(unique(rows$scientific_name)) > 1 || length(unique(stats::na.omit(rows$legume))) > 1) {
add("SM-E008", paste0("crops of species '", s, "' have different scientific names or legume status"),
file = "crops.json", entry = paste(rows$crop, collapse = ", "), attribute = "species")
}
}
}
# Draft descriptions ----------------
for (name in names(catalogues)) {
for (e in entries[[name]]) {
if (isTRUE(e$draft)) {
add("SM-W002", "description is a draft", file = catalogues[[name]]$file,
entry = as.character(e[[catalogues[[name]]$id]]), attribute = "description")
}
}
}
# Unused definitions ----------------
for (category in setdiff(ids$categories, field(entries$operations, "category"))) {
add("SM-W003", "category has no operations", file = "operations.json", entry = category)
}
for (o in setdiff(ids$operations, option_ops)) {
add("SM-W003", "operation has no devices in management_options.json", file = "operations.json", entry = o)
}
for (d in setdiff(ids$devices, option_devices)) {
add("SM-W003", "device is not used in management_options.json", file = "devices.json", entry = d)
}
for (u in setdiff(ids$units, option_units)) {
add("SM-W003", "unit is not used in management_options.json", file = "units.json", entry = u)
}
# Parameters needed for calculations ----------------
tillage_ops <- ids$operations[field(entries$operations, "category") %in% c("tillage", "sowing")]
stir_devices <- unique(unlist(lapply(get_entries("management_options", "options"), function(opt) {
if (as.character(opt$operation) %in% tillage_ops) field(opt$devices, "device")
})))
for (d in entries$devices) {
if (d$id %in% stir_devices && is.null(d$stir) && !isTRUE(d$stir_not_applicable)) {
add("SM-W004", "device is used in tillage or sowing operations but has no STIR parameters",
file = "devices.json", entry = d$id, attribute = "stir")
}
}
for (f in entries$fertilizers) {
type <- as.character(f$type)
if (length(type) != 1) next
if (type == "organic" &&
(is.null(f$DMC) || is.null(f$composition$C) || is.null(f$composition$N))) {
add("SM-W004", "DMC, C or N content missing, no C and N input can be calculated",
file = "fertilizers.json", entry = f$id, attribute = "composition")
}
if (type == "organic" && is.null(f$ISMO$value)) {
add("SM-W004", "ISMO missing, no stable organic matter can be calculated",
file = "fertilizers.json", entry = f$id, attribute = "ISMO")
}
if (type == "mineral" && is.null(f$composition$N)) {
add("SM-W004", "N content missing, no N input can be calculated from product amounts",
file = "fertilizers.json", entry = f$id, attribute = "composition")
}
}
for (cr in entries$crops) {
if ("main_crop" %in% unlist(cr$selectable_as)) {
for (part in c("carbon", "plant_cover")) {
if (is.null(cr[[part]])) {
add("SM-W004", paste("main crop has no", part, "parameters"), file = "crops.json", entry = cr$id, attribute = part)
}
}
}
}
diagnostics
}
#' Keys of a BibTeX file
#' @noRd
read_bib_keys <- function(bib_file) {
if (!nzchar(bib_file) || !file.exists(bib_file)) return(character())
lines <- readLines(bib_file, encoding = "UTF-8", warn = FALSE)
keys <- regmatches(lines, regexpr("^\\s*@[A-Za-z]+\\s*\\{\\s*[^,\\s]+", lines, perl = TRUE))
sub("^\\s*@[A-Za-z]+\\s*\\{\\s*", "", keys, perl = TRUE)
}
#' Collect all elements with a given name in a nested list
#' @noRd
collect_named <- function(x, name) {
if (!is.list(x)) return(list())
found <- if (!is.null(names(x)) && name %in% names(x)) list(x[[name]]) else list()
c(found, unlist(lapply(unname(x), collect_named, name = name), recursive = FALSE))
}
#' Identifier of the entry a JSON path points to
#'
#' For a path such as `/devices/3/stir/depth`, returns the `id` (or `name`)
#' of the fourth device.
#'
#' @noRd
entry_from_path <- function(parsed, path) {
parts <- strsplit(sub("^/", "", path), "/", fixed = TRUE)[[1]]
if (length(parts) < 2 || !grepl("^[0-9]+$", parts[2])) return(NA_character_)
entry <- parsed[[parts[1]]][[as.integer(parts[2]) + 1]]
id <- if (!is.null(entry$id)) entry$id else if (!is.null(entry$name)) entry$name else if (!is.null(entry$operation)) entry$operation else NULL
if (is.null(id)) NA_character_ else as.character(id)
}
#' Readable message for a schema error from jsonvalidate
#' @noRd
schema_error_message <- function(error) {
msg <- error$message
params <- error$params
if (is.data.frame(params) || is.list(params)) {
extra <- unlist(params, use.names = TRUE)
extra <- extra[!is.na(extra)]
if (length(extra)) msg <- paste0(msg, " (", paste(names(extra), extra, sep = ": ", collapse = ", "), ")")
}
msg
}
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.