Nothing
#' Rules for management records
#'
#' Collects the rules for management records from the specifications: the
#' attributes with their types and requirements, the allowed vocabulary with
#' its aliases and the allowed combinations of category, operation, device and
#' unit. The rules are stored in `R/sysdata.rda` (see
#' `tools/build-specifications.R`) and used both to generate the management
#' schema and to validate management JSON in R.
#'
#' @param spec list of parsed specifications, see `read_specifications()`
#'
#' @return a list with the elements `attributes`, `vocabulary`, `aliases` and `options`
#'
#' @md
#' @noRd
management_rules <- function(spec) {
ids <- function(entries) vapply(entries, function(e) e$id, character(1))
attributes <- lapply(spec$attributes$attributes, function(a) {
list(name = a$name, type = a$type, value_required = a$value_required,
allowed_values = if (is.null(a$allowed_values) || isTRUE(a$suggestions_only)) NA_character_ else a$allowed_values,
minimum = if (is.null(a$minimum)) NA_real_ else as.numeric(a$minimum),
required_for_categories = as.character(unlist(a$required_for_categories)),
required_with = as.character(unlist(a$required_with)))
})
names(attributes) <- vapply(attributes, function(a) a$name, character(1))
main_crops <- Filter(function(cr) "main_crop" %in% unlist(cr$selectable_as), spec$crops$crops)
vocabulary <- list(crops = ids(main_crops),
categories = ids(spec$operations$categories),
operations = ids(spec$operations$operations),
devices = ids(spec$devices$devices),
units = ids(spec$units$units))
# aliases of the controlled attributes, only for allowed targets
alias_rows <- function(entries, attribute, allowed) {
do.call(rbind, c(lapply(entries, function(e) {
if (length(e$aliases) == 0 || !(e$id %in% allowed)) return(NULL)
data.frame(attribute = attribute, alias = unlist(e$aliases), id = e$id, stringsAsFactors = FALSE)
}), list(data.frame(attribute = character(), alias = character(), id = character(), stringsAsFactors = FALSE))))
}
aliases <- rbind(alias_rows(spec$crops$crops, "crop", vocabulary$crops),
alias_rows(spec$operations$categories, "category", vocabulary$categories),
alias_rows(spec$operations$operations, "operation", vocabulary$operations),
alias_rows(spec$devices$devices, "device", vocabulary$devices))
category_of <- stats::setNames(vapply(spec$operations$operations, function(o) o$category, character(1)),
vocabulary$operations)
options <- do.call(rbind, lapply(spec$management_options$options, function(o) {
do.call(rbind, lapply(o$devices, function(d) {
units <- as.character(unlist(d$units))
data.frame(category = category_of[[o$operation]], operation = o$operation, device = d$device,
unit = if (length(units)) units else NA_character_, stringsAsFactors = FALSE)
}))
}))
rownames(options) <- NULL
list(attributes = attributes, vocabulary = vocabulary, aliases = aliases, options = options)
}
#' Generate the JSON Schema for management data
#'
#' Builds the JSON Schema of the management input (`inst/schemas/management.schema.json`)
#' from the rules of `management_rules()`. The same rules are checked in R by
#' `validate_management_records()`.
#'
#' @param rules rules from `management_rules()`
#' @param spec_version version of the specifications
#'
#' @return the schema as nested list, see `format_json()`
#'
#' @md
#' @noRd
build_management_schema <- function(rules, spec_version) {
with_aliases <- function(values, attribute) {
a <- rules$aliases[rules$aliases$attribute == attribute, ]
c(values, a$alias[a$id %in% values])
}
enum <- function(values, attribute, null = TRUE) I(c(with_aliases(values, attribute), if (null) NA))
vocab_attribute <- c(crops = "crop", categories = "category", operations = "operation", devices = "device", units = "unit")
json_type <- c(string = "string", date = "string", number = "number", integer = "integer")
properties <- list()
for (a in rules$attributes) {
required <- a$value_required == "required"
p <- list(type = if (required) json_type[[a$type]] else I(c(json_type[[a$type]], "null")))
if (a$type == "date") p$pattern <- "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
if (!is.na(a$minimum)) p$minimum <- a$minimum
if (!is.na(a$allowed_values)) {
p$enum <- enum(rules$vocabulary[[a$allowed_values]], vocab_attribute[[a$allowed_values]], null = !required)
}
properties[[a$name]] <- p
}
not_null <- list(not = list(type = "null"))
conditions <- list()
for (a in rules$attributes) {
if (length(a$required_for_categories)) {
conditions[[length(conditions) + 1]] <- list(
`if` = list(properties = list(category = list(enum = enum(a$required_for_categories, "category", null = FALSE))),
required = I("category")),
then = list(required = I(a$name), properties = stats::setNames(list(not_null), a$name)))
}
for (w in a$required_with) {
conditions[[length(conditions) + 1]] <- list(
`if` = list(properties = stats::setNames(list(not_null), w), required = I(w)),
then = list(required = I(a$name), properties = stats::setNames(list(not_null), a$name)))
}
}
o <- rules$options
for (category in unique(o$category)) {
conditions[[length(conditions) + 1]] <- list(
`if` = list(properties = list(category = list(enum = enum(category, "category", null = FALSE))), required = I("category")),
then = list(properties = list(operation = list(enum = enum(unique(o$operation[o$category == category]), "operation", null = FALSE)))))
}
for (operation in unique(o$operation)) {
rows <- o[o$operation == operation, ]
conditions[[length(conditions) + 1]] <- list(
`if` = list(properties = list(operation = list(enum = enum(operation, "operation", null = FALSE))), required = I("operation")),
then = list(properties = list(device = list(enum = enum(unique(rows$device), "device")))))
for (device in unique(rows$device)) {
units <- stats::na.omit(rows$unit[rows$device == device])
conditions[[length(conditions) + 1]] <- list(
`if` = list(properties = list(operation = list(enum = enum(operation, "operation", null = FALSE)),
device = list(enum = enum(device, "device", null = FALSE))),
required = I(c("operation", "device"))),
then = list(properties = list(unit = list(enum = I(c(as.character(units), NA))))))
}
}
list(`$schema` = "https://json-schema.org/draft/2020-12/schema",
`$id` = "management.schema.json",
`$comment` = "Generated by tools/build-specifications.R from inst/spec. Do not edit.",
title = "SoilManageR management data",
description = paste("Management records of one field. Generated from the specifications version", spec_version),
type = "object",
additionalProperties = FALSE,
required = I("records"),
properties = list(
`$schema` = list(type = "string"),
spec_version = list(type = "string", description = "Version of the specifications the data was written for."),
records = list(type = "array", minItems = 1, items = list(`$ref` = "#/$defs/record"))),
`$defs` = list(record = list(
type = "object",
additionalProperties = FALSE,
required = I(names(Filter(function(a) a$value_required == "required", rules$attributes))),
properties = properties,
allOf = conditions)))
}
#' Format JSON in the style of the specification files
#'
#' Objects and arrays of objects are indented, arrays of scalars are written on
#' one line. Vectors wrapped in `I()` are always written as arrays, `NA` as `null`.
#'
#' @param x nested list
#' @param indent indentation level
#'
#' @return character string
#'
#' @md
#' @noRd
format_json <- function(x, indent = 0) {
pad <- strrep(" ", indent)
pad1 <- strrep(" ", indent + 1)
scalar <- function(v) as.character(jsonlite::toJSON(v, auto_unbox = TRUE, na = "null", digits = NA))
if (is.list(x) && !is.null(names(x))) {
if (length(x) == 0) return("{}")
items <- vapply(names(x), function(k) paste0(pad1, scalar(k), ": ", format_json(x[[k]], indent + 1)), character(1))
return(paste0("{\n", paste(items, collapse = ",\n"), "\n", pad, "}"))
}
if (is.list(x)) {
if (length(x) == 0) return("[]")
items <- vapply(x, function(v) paste0(pad1, format_json(v, indent + 1)), character(1))
return(paste0("[\n", paste(items, collapse = ",\n"), "\n", pad, "]"))
}
if (inherits(x, "AsIs") || length(x) != 1) {
return(paste0("[", paste(vapply(as.list(unclass(x)), scalar, character(1)), collapse = ", "), "]"))
}
scalar(x)
}
#' Validate management records in R
#'
#' Checks parsed management JSON against the rules of the specifications
#' (`spec_management_rules`): structure, types, dates, required values,
#' vocabulary (aliases allowed) and the allowed combinations of category,
#' operation, device and unit. These are the same rules as in the generated
#' JSON Schema `inst/schemas/management.schema.json`; additionally, dates are
#' checked for existence (e.g. no 30 February).
#'
#' @param x management JSON parsed with `jsonlite::parse_json(simplifyVector = FALSE)`
#' @param rules rules from `management_rules()`
#'
#' @return data frame of diagnostics (see `management_diagnostic()`), `row` is
#' the number of the record
#'
#' @md
#' @noRd
validate_management_records <- function(x, rules = spec_management_rules) {
diagnostics <- list()
add <- function(code, message, record = NA_integer_, attribute = NA_character_) {
diagnostics[[length(diagnostics) + 1]] <<- management_diagnostic(code, message, record, attribute)
}
result <- function() {
if (length(diagnostics) == 0) return(management_diagnostic("SM-E401", "none")[0, ])
do.call(rbind, diagnostics)
}
is_object <- function(v) is.list(v) && !is.null(names(v)) && all(nzchar(names(v)))
is_null <- function(v) is.null(v)
# Structure ----------------
if (!is_object(x) && !(is.list(x) && length(x) == 0)) {
add("SM-E401", "the management JSON must be an object with the element 'records'")
return(result())
}
for (k in setdiff(names(x), c("$schema", "spec_version", "records"))) {
add("SM-E401", paste0("unknown element '", k, "' (allowed: spec_version, records)"))
}
if (!is_null(x$spec_version) && !(is.character(x$spec_version) && length(x$spec_version) == 1)) {
add("SM-E401", "spec_version must be a string")
}
records <- x$records
if (is_null(records) || !is.list(records) || !is.null(names(records)) || length(records) == 0) {
add("SM-E401", "'records' must be a non-empty array of records")
return(result())
}
attributes <- rules$attributes
resolve <- function(value, attribute) {
a <- rules$aliases[rules$aliases$attribute == attribute, ]
i <- match(value, a$alias)
if (is.na(i)) value else a$id[i]
}
vocab_attribute <- c(crops = "crop", categories = "category", operations = "operation", devices = "device", units = "unit")
o <- rules$options
for (r in seq_along(records)) {
rec <- records[[r]]
if (!is_object(rec) && !(is.list(rec) && length(rec) == 0)) {
add("SM-E402", "record is not an object", r)
next
}
for (k in setdiff(names(rec), names(attributes))) add("SM-E402", paste0("unknown attribute '", k, "'"), r, k)
# types, dates, required values and vocabulary
valid <- list()
for (a in attributes) {
v <- rec[[a$name]]
if (is_null(v)) {
if (a$value_required == "required") add("SM-E405", paste0(a$name, " is required"), r, a$name)
next
}
ok <- switch(a$type,
string = is.character(v) && length(v) == 1,
date = is.character(v) && length(v) == 1,
number = is.numeric(v) && length(v) == 1,
integer = is.numeric(v) && length(v) == 1 && v == round(v))
if (!ok) {
add("SM-E403", paste0(a$name, " must be ", if (a$type == "date") "a date (YYYY-MM-DD)" else paste("a", a$type)), r, a$name)
next
}
if (!is.na(a$minimum) && v < a$minimum) {
add("SM-E403", paste0(a$name, " must be at least ", a$minimum), r, a$name)
next
}
if (a$type == "date") {
if (!grepl("^[0-9]{4}-[0-9]{2}-[0-9]{2}$", v)) {
add("SM-E403", paste0(a$name, " must be a date (YYYY-MM-DD)"), r, a$name)
next
}
if (is.na(as.Date(v, format = "%Y-%m-%d", optional = TRUE))) {
add("SM-E404", paste0(a$name, " '", v, "' does not exist"), r, a$name)
next
}
}
if (!is.na(a$allowed_values)) {
v <- resolve(v, vocab_attribute[[a$allowed_values]])
if (!(v %in% rules$vocabulary[[a$allowed_values]])) {
add("SM-E406", paste0("unknown ", a$name, " '", rec[[a$name]], "'"), r, a$name)
next
}
}
valid[[a$name]] <- v
}
# conditional requirements
for (a in attributes) {
if (!is_null(rec[[a$name]])) next
if (length(a$required_for_categories) && !is_null(rec$category) &&
is.character(rec$category) && resolve(rec$category, "category") %in% a$required_for_categories) {
add("SM-E405", paste0(a$name, " is required for the category '", rec$category, "'"), r, a$name)
}
for (w in a$required_with) {
if (!is_null(rec[[w]])) add("SM-E405", paste0(a$name, " is required if ", w, " is given"), r, a$name)
}
}
# allowed combinations
if (!is.null(valid$category) && !is.null(valid$operation) &&
!(valid$operation %in% o$operation[o$category == valid$category])) {
add("SM-E407", paste0("operation '", valid$operation, "' does not belong to the category '", valid$category, "'"), r, "operation")
}
if (!is.null(valid$operation) && !is.null(valid$device) &&
!(valid$device %in% o$device[o$operation == valid$operation])) {
add("SM-E407", paste0("device '", valid$device, "' is not allowed for the operation '", valid$operation, "'"), r, "device")
}
if (!is.null(valid$operation) && !is.null(valid$device) && !is.null(valid$unit) &&
valid$device %in% o$device[o$operation == valid$operation]) {
units <- stats::na.omit(o$unit[o$operation == valid$operation & o$device == valid$device])
if (!(valid$unit %in% units)) {
add("SM-E407", paste0("unit '", valid$unit, "' is not allowed for ", valid$operation, " > ", valid$device,
if (length(units)) paste0(" (allowed: ", paste(units, collapse = ", "), ")") else " (no unit expected)"),
r, "unit")
}
}
}
result()
}
#' Validate management JSON against the generated JSON Schema
#'
#' @param json_text management JSON as text
#' @param schema_file path of the management schema
#'
#' @return data frame of diagnostics with code `SM-E408`, empty if valid
#'
#' @md
#' @noRd
validate_management_schema <- function(json_text,
schema_file = system.file("schemas", "management.schema.json", package = "SoilManageR")) {
validator <- jsonvalidate::json_schema$new(schema_file, engine = "ajv")
result <- validator$validate(json_text, verbose = TRUE, greedy = TRUE)
if (isTRUE(as.logical(result))) return(management_diagnostic("SM-E408", "none")[0, ])
errors <- attr(result, "errors")
# errors of the if/then conditions only repeat the error of the branch
errors <- errors[errors$keyword != "if", , drop = FALSE]
record <- suppressWarnings(as.integer(sub("^/records/([0-9]+).*$", "\\1", errors$instancePath))) + 1L
attribute <- ifelse(grepl("^/records/[0-9]+/", errors$instancePath), sub("^/records/[0-9]+/", "", errors$instancePath), NA_character_)
do.call(rbind, lapply(seq_len(nrow(errors)), function(i) {
management_diagnostic("SM-E408", paste("schema:", schema_error_message(errors[i, ])), record[i], attribute[i])
}))
}
#' Error with diagnostics of management JSON
#'
#' @param diagnostics data frame of diagnostics
#'
#' @md
#' @noRd
stop_management_json <- function(diagnostics) {
shown <- utils::head(diagnostics, 10)
where <- ifelse(is.na(shown$row), "", paste0("record ", shown$row, ifelse(is.na(shown$attribute), "", paste0(", ", shown$attribute)), ": "))
msg <- paste0("The management JSON is not valid (", nrow(diagnostics), " error(s)):\n",
paste0(" ", shown$code, " ", where, shown$message, collapse = "\n"),
if (nrow(diagnostics) > 10) "\n ..." else "",
"\nAll findings are in the element 'diagnostics' of the error condition.")
stop(structure(class = c("management_json_error", "error", "condition"),
list(message = msg, call = NULL, diagnostics = diagnostics)))
}
#' Import a management_df from JSON
#'
#' Reads management data in the SoilManageR JSON format and converts it into a
#' [management_df]. The JSON is an object with the optional element
#' `spec_version` and the element `records`: an array with one object per
#' management operation. The attributes of a record are the columns of a
#' `management_df`; omitted attributes and `null` become missing values.
#'
#' @details
#' The records are validated before the conversion: types, dates (YYYY-MM-DD),
#' required values (crop, date, category, operation, and the device for all
#' categories except "other"), a unit for every value, the vocabulary of the
#' specifications and the allowed combinations of category, operation, device
#' and unit. If the package `jsonvalidate` is installed, the JSON is also
#' validated against the JSON Schema `management.schema.json` (in the folder
#' `schemas` of the installed package), which contains the same rules.
#' If the data is not valid, an error of class `management_json_error` is
#' raised; its element `diagnostics` lists all findings with code, record,
#' attribute and message.
#'
#' Historical names (aliases) of crops, categories, operations, devices and
#' products are replaced by their current identifiers. The year is derived
#' from the date. Seed mixtures are not expanded. Plausibility checks (e.g.
#' order of operations, tillage depths) are done with [check_management_df()].
#'
#' @param json path to a JSON file, or JSON text
#' @param validate_schema logical: if TRUE (default), the JSON is also
#' validated against the JSON Schema when the package `jsonvalidate` is installed
#'
#' @return a management_df
#'
#' @examples
#' path_to_json <- system.file("extdata", "EXAMPLE_data.json", package = "SoilManageR")
#' mgmt_data <- management_df_from_json(path_to_json)
#'
#' # JSON text
#' management_df_from_json('{"records": [{"crop": "wheat, winter", "date": "2021-10-12",
#' "category": "tillage", "operation": "primary_tillage", "device": "plough",
#' "value": 25, "unit": "cm"}]}')
#'
#' @seealso
#' * [management_df_to_json()] to write a management_df as JSON
#' * [management_df_from_excel()] to import management data from the Excel template
#' * [check_management_df()] to check the plausibility of a management_df
#' @md
#'
#' @export
management_df_from_json <- function(json, validate_schema = TRUE) {
# Read and parse ----------------
is_text <- length(json) != 1 || grepl("^\\s*[\\[{]", json)
if (!is_text && !file.exists(json)) stop("File not found: ", json)
json_text <- if (is_text) paste(json, collapse = "\n") else paste(readLines(json, encoding = "UTF-8", warn = FALSE), collapse = "\n")
x <- tryCatch(jsonlite::parse_json(json_text, simplifyVector = FALSE), error = function(e) e)
if (inherits(x, "error")) {
stop_management_json(management_diagnostic("SM-E401", paste("not valid JSON:", conditionMessage(x))))
}
# Validate ----------------
diagnostics <- validate_management_records(x)
if (nrow(diagnostics) > 0) stop_management_json(diagnostics)
if (validate_schema && requireNamespace("jsonvalidate", quietly = TRUE)) {
diagnostics <- validate_management_schema(json_text)
if (nrow(diagnostics) > 0) stop_management_json(diagnostics)
}
if (!is.null(x$spec_version) && x$spec_version != spec_version) {
warning("The data was written for the specifications version ", x$spec_version,
", the package uses version ", spec_version, ".", call. = FALSE)
}
# Convert ----------------
rules <- spec_management_rules
records <- x$records
column <- function(a) {
values <- lapply(records, function(rec) rec[[a$name]])
missing <- vapply(values, is.null, logical(1))
values[missing] <- NA
v <- unlist(values)
switch(a$type,
string = as.character(v),
date = as.Date(as.character(v)),
number = as.numeric(v),
integer = as.integer(v))
}
output_tibble <- tibble::as_tibble(lapply(rules$attributes, column))
output_tibble$year <- as.double(format(output_tibble$date, "%Y"))
# replace historical names
replaced <- character()
for (attribute in c("crop", "category", "operation", "device", "product")) {
new <- resolve_aliases(output_tibble[[attribute]], attribute)
changed <- !is.na(new) & new != output_tibble[[attribute]]
if (any(changed)) replaced <- c(replaced, unique(paste0(output_tibble[[attribute]][changed], " -> ", new[changed])))
output_tibble[[attribute]] <- new
}
if (length(replaced)) message("Historical names replaced: ", paste(replaced, collapse = ", "))
class(output_tibble) <- c("management_df", class(output_tibble))
output_tibble
}
#' Shortest decimal representation of a number that is read back exactly
#'
#' @param x a finite number
#' @return character
#' @noRd
shortest_number <- function(x) {
for (digits in 15:17) {
s <- formatC(x, digits = digits, format = "g")
if (as.numeric(s) == x) break
}
trimws(s)
}
#' Write a management_df as JSON
#'
#' Converts a [management_df] into the SoilManageR JSON format (see
#' [management_df_from_json()]): an object with the specification version
#' and one record per management operation. Missing values and the year
#' (derived from the date) are omitted.
#'
#' @param mgmt_data a management_df
#' @param path optional path of the JSON file to write
#' @param pretty logical: if TRUE (default), the JSON is indented
#'
#' @return the JSON text (invisibly if written to `path`)
#'
#' @examples
#' json <- management_df_to_json(EXAMPLE_data[1:3, ])
#' cat(json)
#'
#' @seealso
#' * [management_df_from_json()] to import management data from JSON
#' @md
#'
#' @export
management_df_to_json <- function(mgmt_data, path = NULL, pretty = TRUE) {
if (!("management_df" %in% class(mgmt_data))) stop("Input is not of the class management_df")
columns <- setdiff(names(spec_management_rules$attributes), "year")
data <- as.data.frame(mgmt_data)[, columns, drop = FALSE]
data$date <- format(data$date, "%Y-%m-%d")
# numbers in the shortest form that is read back exactly
for (col in names(data)[vapply(data, is.double, logical(1))]) {
data[[col]] <- lapply(data[[col]], function(v) if (is.na(v)) NA else structure(shortest_number(v), class = "json"))
}
records <- lapply(seq_len(nrow(data)), function(i) {
row <- lapply(data, function(column) column[[i]])
row[!vapply(row, function(v) length(v) == 1 && is.na(v), logical(1))]
})
json <- jsonlite::toJSON(list(spec_version = spec_version, records = records),
auto_unbox = TRUE, json_verbatim = TRUE, pretty = pretty)
json <- paste0(as.character(json), "\n")
if (!is.null(path)) {
writeLines(json, path, useBytes = TRUE, sep = "")
return(invisible(json))
}
json
}
Any scripts or data that you put into this service are public.
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.