R/management_json.R

Defines functions management_df_to_json shortest_number management_df_from_json stop_management_json validate_management_schema validate_management_records format_json build_management_schema management_rules

Documented in management_df_from_json management_df_to_json

#' Rules for management records
#'
#' Collects the rules for management records from the specifications: the
#'  attributes with their types and requirements, the allowed vocabulary with
#'  its aliases and the allowed combinations of category, operation, device and
#'  unit. The rules are stored in `R/sysdata.rda` (see
#'  `tools/build-specifications.R`) and used both to generate the management
#'  schema and to validate management JSON in R.
#'
#' @param spec list of parsed specifications, see `read_specifications()`
#'
#' @return a list with the elements `attributes`, `vocabulary`, `aliases` and `options`
#'
#' @md
#' @noRd
management_rules <- function(spec) {
  ids <- function(entries) vapply(entries, function(e) e$id, character(1))
  attributes <- lapply(spec$attributes$attributes, function(a) {
    list(name = a$name, type = a$type, value_required = a$value_required,
         allowed_values = if (is.null(a$allowed_values) || isTRUE(a$suggestions_only)) NA_character_ else a$allowed_values,
         minimum = if (is.null(a$minimum)) NA_real_ else as.numeric(a$minimum),
         required_for_categories = as.character(unlist(a$required_for_categories)),
         required_with = as.character(unlist(a$required_with)))
  })
  names(attributes) <- vapply(attributes, function(a) a$name, character(1))

  main_crops <- Filter(function(cr) "main_crop" %in% unlist(cr$selectable_as), spec$crops$crops)
  vocabulary <- list(crops = ids(main_crops),
                     categories = ids(spec$operations$categories),
                     operations = ids(spec$operations$operations),
                     devices = ids(spec$devices$devices),
                     units = ids(spec$units$units))

  # aliases of the controlled attributes, only for allowed targets
  alias_rows <- function(entries, attribute, allowed) {
    do.call(rbind, c(lapply(entries, function(e) {
      if (length(e$aliases) == 0 || !(e$id %in% allowed)) return(NULL)
      data.frame(attribute = attribute, alias = unlist(e$aliases), id = e$id, stringsAsFactors = FALSE)
    }), list(data.frame(attribute = character(), alias = character(), id = character(), stringsAsFactors = FALSE))))
  }
  aliases <- rbind(alias_rows(spec$crops$crops, "crop", vocabulary$crops),
                   alias_rows(spec$operations$categories, "category", vocabulary$categories),
                   alias_rows(spec$operations$operations, "operation", vocabulary$operations),
                   alias_rows(spec$devices$devices, "device", vocabulary$devices))

  category_of <- stats::setNames(vapply(spec$operations$operations, function(o) o$category, character(1)),
                                 vocabulary$operations)
  options <- do.call(rbind, lapply(spec$management_options$options, function(o) {
    do.call(rbind, lapply(o$devices, function(d) {
      units <- as.character(unlist(d$units))
      data.frame(category = category_of[[o$operation]], operation = o$operation, device = d$device,
                 unit = if (length(units)) units else NA_character_, stringsAsFactors = FALSE)
    }))
  }))
  rownames(options) <- NULL

  list(attributes = attributes, vocabulary = vocabulary, aliases = aliases, options = options)
}

#' Generate the JSON Schema for management data
#'
#' Builds the JSON Schema of the management input (`inst/schemas/management.schema.json`)
#'  from the rules of `management_rules()`. The same rules are checked in R by
#'  `validate_management_records()`.
#'
#' @param rules rules from `management_rules()`
#' @param spec_version version of the specifications
#'
#' @return the schema as nested list, see `format_json()`
#'
#' @md
#' @noRd
build_management_schema <- function(rules, spec_version) {
  with_aliases <- function(values, attribute) {
    a <- rules$aliases[rules$aliases$attribute == attribute, ]
    c(values, a$alias[a$id %in% values])
  }
  enum <- function(values, attribute, null = TRUE) I(c(with_aliases(values, attribute), if (null) NA))
  vocab_attribute <- c(crops = "crop", categories = "category", operations = "operation", devices = "device", units = "unit")
  json_type <- c(string = "string", date = "string", number = "number", integer = "integer")

  properties <- list()
  for (a in rules$attributes) {
    required <- a$value_required == "required"
    p <- list(type = if (required) json_type[[a$type]] else I(c(json_type[[a$type]], "null")))
    if (a$type == "date") p$pattern <- "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
    if (!is.na(a$minimum)) p$minimum <- a$minimum
    if (!is.na(a$allowed_values)) {
      p$enum <- enum(rules$vocabulary[[a$allowed_values]], vocab_attribute[[a$allowed_values]], null = !required)
    }
    properties[[a$name]] <- p
  }

  not_null <- list(not = list(type = "null"))
  conditions <- list()
  for (a in rules$attributes) {
    if (length(a$required_for_categories)) {
      conditions[[length(conditions) + 1]] <- list(
        `if` = list(properties = list(category = list(enum = enum(a$required_for_categories, "category", null = FALSE))),
                    required = I("category")),
        then = list(required = I(a$name), properties = stats::setNames(list(not_null), a$name)))
    }
    for (w in a$required_with) {
      conditions[[length(conditions) + 1]] <- list(
        `if` = list(properties = stats::setNames(list(not_null), w), required = I(w)),
        then = list(required = I(a$name), properties = stats::setNames(list(not_null), a$name)))
    }
  }
  o <- rules$options
  for (category in unique(o$category)) {
    conditions[[length(conditions) + 1]] <- list(
      `if` = list(properties = list(category = list(enum = enum(category, "category", null = FALSE))), required = I("category")),
      then = list(properties = list(operation = list(enum = enum(unique(o$operation[o$category == category]), "operation", null = FALSE)))))
  }
  for (operation in unique(o$operation)) {
    rows <- o[o$operation == operation, ]
    conditions[[length(conditions) + 1]] <- list(
      `if` = list(properties = list(operation = list(enum = enum(operation, "operation", null = FALSE))), required = I("operation")),
      then = list(properties = list(device = list(enum = enum(unique(rows$device), "device")))))
    for (device in unique(rows$device)) {
      units <- stats::na.omit(rows$unit[rows$device == device])
      conditions[[length(conditions) + 1]] <- list(
        `if` = list(properties = list(operation = list(enum = enum(operation, "operation", null = FALSE)),
                                      device = list(enum = enum(device, "device", null = FALSE))),
                    required = I(c("operation", "device"))),
        then = list(properties = list(unit = list(enum = I(c(as.character(units), NA))))))
    }
  }

  list(`$schema` = "https://json-schema.org/draft/2020-12/schema",
       `$id` = "management.schema.json",
       `$comment` = "Generated by tools/build-specifications.R from inst/spec. Do not edit.",
       title = "SoilManageR management data",
       description = paste("Management records of one field. Generated from the specifications version", spec_version),
       type = "object",
       additionalProperties = FALSE,
       required = I("records"),
       properties = list(
         `$schema` = list(type = "string"),
         spec_version = list(type = "string", description = "Version of the specifications the data was written for."),
         records = list(type = "array", minItems = 1, items = list(`$ref` = "#/$defs/record"))),
       `$defs` = list(record = list(
         type = "object",
         additionalProperties = FALSE,
         required = I(names(Filter(function(a) a$value_required == "required", rules$attributes))),
         properties = properties,
         allOf = conditions)))
}

#' Format JSON in the style of the specification files
#'
#' Objects and arrays of objects are indented, arrays of scalars are written on
#'  one line. Vectors wrapped in `I()` are always written as arrays, `NA` as `null`.
#'
#' @param x nested list
#' @param indent indentation level
#'
#' @return character string
#'
#' @md
#' @noRd
format_json <- function(x, indent = 0) {
  pad <- strrep("  ", indent)
  pad1 <- strrep("  ", indent + 1)
  scalar <- function(v) as.character(jsonlite::toJSON(v, auto_unbox = TRUE, na = "null", digits = NA))
  if (is.list(x) && !is.null(names(x))) {
    if (length(x) == 0) return("{}")
    items <- vapply(names(x), function(k) paste0(pad1, scalar(k), ": ", format_json(x[[k]], indent + 1)), character(1))
    return(paste0("{\n", paste(items, collapse = ",\n"), "\n", pad, "}"))
  }
  if (is.list(x)) {
    if (length(x) == 0) return("[]")
    items <- vapply(x, function(v) paste0(pad1, format_json(v, indent + 1)), character(1))
    return(paste0("[\n", paste(items, collapse = ",\n"), "\n", pad, "]"))
  }
  if (inherits(x, "AsIs") || length(x) != 1) {
    return(paste0("[", paste(vapply(as.list(unclass(x)), scalar, character(1)), collapse = ", "), "]"))
  }
  scalar(x)
}

#' Validate management records in R
#'
#' Checks parsed management JSON against the rules of the specifications
#'  (`spec_management_rules`): structure, types, dates, required values,
#'  vocabulary (aliases allowed) and the allowed combinations of category,
#'  operation, device and unit. These are the same rules as in the generated
#'  JSON Schema `inst/schemas/management.schema.json`; additionally, dates are
#'  checked for existence (e.g. no 30 February).
#'
#' @param x management JSON parsed with `jsonlite::parse_json(simplifyVector = FALSE)`
#' @param rules rules from `management_rules()`
#'
#' @return data frame of diagnostics (see `management_diagnostic()`), `row` is
#'  the number of the record
#'
#' @md
#' @noRd
validate_management_records <- function(x, rules = spec_management_rules) {
  diagnostics <- list()
  add <- function(code, message, record = NA_integer_, attribute = NA_character_) {
    diagnostics[[length(diagnostics) + 1]] <<- management_diagnostic(code, message, record, attribute)
  }
  result <- function() {
    if (length(diagnostics) == 0) return(management_diagnostic("SM-E401", "none")[0, ])
    do.call(rbind, diagnostics)
  }
  is_object <- function(v) is.list(v) && !is.null(names(v)) && all(nzchar(names(v)))
  is_null <- function(v) is.null(v)

  # Structure ----------------
  if (!is_object(x) && !(is.list(x) && length(x) == 0)) {
    add("SM-E401", "the management JSON must be an object with the element 'records'")
    return(result())
  }
  for (k in setdiff(names(x), c("$schema", "spec_version", "records"))) {
    add("SM-E401", paste0("unknown element '", k, "' (allowed: spec_version, records)"))
  }
  if (!is_null(x$spec_version) && !(is.character(x$spec_version) && length(x$spec_version) == 1)) {
    add("SM-E401", "spec_version must be a string")
  }
  records <- x$records
  if (is_null(records) || !is.list(records) || !is.null(names(records)) || length(records) == 0) {
    add("SM-E401", "'records' must be a non-empty array of records")
    return(result())
  }

  attributes <- rules$attributes
  resolve <- function(value, attribute) {
    a <- rules$aliases[rules$aliases$attribute == attribute, ]
    i <- match(value, a$alias)
    if (is.na(i)) value else a$id[i]
  }
  vocab_attribute <- c(crops = "crop", categories = "category", operations = "operation", devices = "device", units = "unit")
  o <- rules$options

  for (r in seq_along(records)) {
    rec <- records[[r]]
    if (!is_object(rec) && !(is.list(rec) && length(rec) == 0)) {
      add("SM-E402", "record is not an object", r)
      next
    }
    for (k in setdiff(names(rec), names(attributes))) add("SM-E402", paste0("unknown attribute '", k, "'"), r, k)

    # types, dates, required values and vocabulary
    valid <- list()
    for (a in attributes) {
      v <- rec[[a$name]]
      if (is_null(v)) {
        if (a$value_required == "required") add("SM-E405", paste0(a$name, " is required"), r, a$name)
        next
      }
      ok <- switch(a$type,
                   string = is.character(v) && length(v) == 1,
                   date = is.character(v) && length(v) == 1,
                   number = is.numeric(v) && length(v) == 1,
                   integer = is.numeric(v) && length(v) == 1 && v == round(v))
      if (!ok) {
        add("SM-E403", paste0(a$name, " must be ", if (a$type == "date") "a date (YYYY-MM-DD)" else paste("a", a$type)), r, a$name)
        next
      }
      if (!is.na(a$minimum) && v < a$minimum) {
        add("SM-E403", paste0(a$name, " must be at least ", a$minimum), r, a$name)
        next
      }
      if (a$type == "date") {
        if (!grepl("^[0-9]{4}-[0-9]{2}-[0-9]{2}$", v)) {
          add("SM-E403", paste0(a$name, " must be a date (YYYY-MM-DD)"), r, a$name)
          next
        }
        if (is.na(as.Date(v, format = "%Y-%m-%d", optional = TRUE))) {
          add("SM-E404", paste0(a$name, " '", v, "' does not exist"), r, a$name)
          next
        }
      }
      if (!is.na(a$allowed_values)) {
        v <- resolve(v, vocab_attribute[[a$allowed_values]])
        if (!(v %in% rules$vocabulary[[a$allowed_values]])) {
          add("SM-E406", paste0("unknown ", a$name, " '", rec[[a$name]], "'"), r, a$name)
          next
        }
      }
      valid[[a$name]] <- v
    }

    # conditional requirements
    for (a in attributes) {
      if (!is_null(rec[[a$name]])) next
      if (length(a$required_for_categories) && !is_null(rec$category) &&
          is.character(rec$category) && resolve(rec$category, "category") %in% a$required_for_categories) {
        add("SM-E405", paste0(a$name, " is required for the category '", rec$category, "'"), r, a$name)
      }
      for (w in a$required_with) {
        if (!is_null(rec[[w]])) add("SM-E405", paste0(a$name, " is required if ", w, " is given"), r, a$name)
      }
    }

    # allowed combinations
    if (!is.null(valid$category) && !is.null(valid$operation) &&
        !(valid$operation %in% o$operation[o$category == valid$category])) {
      add("SM-E407", paste0("operation '", valid$operation, "' does not belong to the category '", valid$category, "'"), r, "operation")
    }
    if (!is.null(valid$operation) && !is.null(valid$device) &&
        !(valid$device %in% o$device[o$operation == valid$operation])) {
      add("SM-E407", paste0("device '", valid$device, "' is not allowed for the operation '", valid$operation, "'"), r, "device")
    }
    if (!is.null(valid$operation) && !is.null(valid$device) && !is.null(valid$unit) &&
        valid$device %in% o$device[o$operation == valid$operation]) {
      units <- stats::na.omit(o$unit[o$operation == valid$operation & o$device == valid$device])
      if (!(valid$unit %in% units)) {
        add("SM-E407", paste0("unit '", valid$unit, "' is not allowed for ", valid$operation, " > ", valid$device,
                              if (length(units)) paste0(" (allowed: ", paste(units, collapse = ", "), ")") else " (no unit expected)"),
            r, "unit")
      }
    }
  }
  result()
}

#' Validate management JSON against the generated JSON Schema
#'
#' @param json_text management JSON as text
#' @param schema_file path of the management schema
#'
#' @return data frame of diagnostics with code `SM-E408`, empty if valid
#'
#' @md
#' @noRd
validate_management_schema <- function(json_text,
                                       schema_file = system.file("schemas", "management.schema.json", package = "SoilManageR")) {
  validator <- jsonvalidate::json_schema$new(schema_file, engine = "ajv")
  result <- validator$validate(json_text, verbose = TRUE, greedy = TRUE)
  if (isTRUE(as.logical(result))) return(management_diagnostic("SM-E408", "none")[0, ])
  errors <- attr(result, "errors")
  # errors of the if/then conditions only repeat the error of the branch
  errors <- errors[errors$keyword != "if", , drop = FALSE]
  record <- suppressWarnings(as.integer(sub("^/records/([0-9]+).*$", "\\1", errors$instancePath))) + 1L
  attribute <- ifelse(grepl("^/records/[0-9]+/", errors$instancePath), sub("^/records/[0-9]+/", "", errors$instancePath), NA_character_)
  do.call(rbind, lapply(seq_len(nrow(errors)), function(i) {
    management_diagnostic("SM-E408", paste("schema:", schema_error_message(errors[i, ])), record[i], attribute[i])
  }))
}

#' Error with diagnostics of management JSON
#'
#' @param diagnostics data frame of diagnostics
#'
#' @md
#' @noRd
stop_management_json <- function(diagnostics) {
  shown <- utils::head(diagnostics, 10)
  where <- ifelse(is.na(shown$row), "", paste0("record ", shown$row, ifelse(is.na(shown$attribute), "", paste0(", ", shown$attribute)), ": "))
  msg <- paste0("The management JSON is not valid (", nrow(diagnostics), " error(s)):\n",
                paste0("  ", shown$code, " ", where, shown$message, collapse = "\n"),
                if (nrow(diagnostics) > 10) "\n  ..." else "",
                "\nAll findings are in the element 'diagnostics' of the error condition.")
  stop(structure(class = c("management_json_error", "error", "condition"),
                 list(message = msg, call = NULL, diagnostics = diagnostics)))
}

#' Import a management_df from JSON
#'
#' Reads management data in the SoilManageR JSON format and converts it into a
#'  [management_df]. The JSON is an object with the optional element
#'  `spec_version` and the element `records`: an array with one object per
#'  management operation. The attributes of a record are the columns of a
#'  `management_df`; omitted attributes and `null` become missing values.
#'
#' @details
#' The records are validated before the conversion: types, dates (YYYY-MM-DD),
#'  required values (crop, date, category, operation, and the device for all
#'  categories except "other"), a unit for every value, the vocabulary of the
#'  specifications and the allowed combinations of category, operation, device
#'  and unit. If the package `jsonvalidate` is installed, the JSON is also
#'  validated against the JSON Schema `management.schema.json` (in the folder
#'  `schemas` of the installed package), which contains the same rules.
#'  If the data is not valid, an error of class `management_json_error` is
#'  raised; its element `diagnostics` lists all findings with code, record,
#'  attribute and message.
#'
#' Historical names (aliases) of crops, categories, operations, devices and
#'  products are replaced by their current identifiers. The year is derived
#'  from the date. Seed mixtures are not expanded. Plausibility checks (e.g.
#'  order of operations, tillage depths) are done with [check_management_df()].
#'
#' @param json path to a JSON file, or JSON text
#' @param validate_schema logical: if TRUE (default), the JSON is also
#'  validated against the JSON Schema when the package `jsonvalidate` is installed
#'
#' @return a management_df
#'
#' @examples
#' path_to_json <- system.file("extdata", "EXAMPLE_data.json", package = "SoilManageR")
#' mgmt_data <- management_df_from_json(path_to_json)
#'
#' # JSON text
#' management_df_from_json('{"records": [{"crop": "wheat, winter", "date": "2021-10-12",
#'   "category": "tillage", "operation": "primary_tillage", "device": "plough",
#'   "value": 25, "unit": "cm"}]}')
#'
#' @seealso
#' * [management_df_to_json()] to write a management_df as JSON
#' * [management_df_from_excel()] to import management data from the Excel template
#' * [check_management_df()] to check the plausibility of a management_df
#' @md
#'
#' @export
management_df_from_json <- function(json, validate_schema = TRUE) {

  # Read and parse ----------------
  is_text <- length(json) != 1 || grepl("^\\s*[\\[{]", json)
  if (!is_text && !file.exists(json)) stop("File not found: ", json)
  json_text <- if (is_text) paste(json, collapse = "\n") else paste(readLines(json, encoding = "UTF-8", warn = FALSE), collapse = "\n")
  x <- tryCatch(jsonlite::parse_json(json_text, simplifyVector = FALSE), error = function(e) e)
  if (inherits(x, "error")) {
    stop_management_json(management_diagnostic("SM-E401", paste("not valid JSON:", conditionMessage(x))))
  }

  # Validate ----------------
  diagnostics <- validate_management_records(x)
  if (nrow(diagnostics) > 0) stop_management_json(diagnostics)
  if (validate_schema && requireNamespace("jsonvalidate", quietly = TRUE)) {
    diagnostics <- validate_management_schema(json_text)
    if (nrow(diagnostics) > 0) stop_management_json(diagnostics)
  }
  if (!is.null(x$spec_version) && x$spec_version != spec_version) {
    warning("The data was written for the specifications version ", x$spec_version,
            ", the package uses version ", spec_version, ".", call. = FALSE)
  }

  # Convert ----------------
  rules <- spec_management_rules
  records <- x$records
  column <- function(a) {
    values <- lapply(records, function(rec) rec[[a$name]])
    missing <- vapply(values, is.null, logical(1))
    values[missing] <- NA
    v <- unlist(values)
    switch(a$type,
           string = as.character(v),
           date = as.Date(as.character(v)),
           number = as.numeric(v),
           integer = as.integer(v))
  }
  output_tibble <- tibble::as_tibble(lapply(rules$attributes, column))
  output_tibble$year <- as.double(format(output_tibble$date, "%Y"))

  # replace historical names
  replaced <- character()
  for (attribute in c("crop", "category", "operation", "device", "product")) {
    new <- resolve_aliases(output_tibble[[attribute]], attribute)
    changed <- !is.na(new) & new != output_tibble[[attribute]]
    if (any(changed)) replaced <- c(replaced, unique(paste0(output_tibble[[attribute]][changed], " -> ", new[changed])))
    output_tibble[[attribute]] <- new
  }
  if (length(replaced)) message("Historical names replaced: ", paste(replaced, collapse = ", "))

  class(output_tibble) <- c("management_df", class(output_tibble))
  output_tibble
}

#' Shortest decimal representation of a number that is read back exactly
#'
#' @param x a finite number
#' @return character
#' @noRd
shortest_number <- function(x) {
  for (digits in 15:17) {
    s <- formatC(x, digits = digits, format = "g")
    if (as.numeric(s) == x) break
  }
  trimws(s)
}

#' Write a management_df as JSON
#'
#' Converts a [management_df] into the SoilManageR JSON format (see
#'  [management_df_from_json()]): an object with the specification version
#'  and one record per management operation. Missing values and the year
#'  (derived from the date) are omitted.
#'
#' @param mgmt_data a management_df
#' @param path optional path of the JSON file to write
#' @param pretty logical: if TRUE (default), the JSON is indented
#'
#' @return the JSON text (invisibly if written to `path`)
#'
#' @examples
#' json <- management_df_to_json(EXAMPLE_data[1:3, ])
#' cat(json)
#'
#' @seealso
#' * [management_df_from_json()] to import management data from JSON
#' @md
#'
#' @export
management_df_to_json <- function(mgmt_data, path = NULL, pretty = TRUE) {
  if (!("management_df" %in% class(mgmt_data))) stop("Input is not of the class management_df")
  columns <- setdiff(names(spec_management_rules$attributes), "year")
  data <- as.data.frame(mgmt_data)[, columns, drop = FALSE]
  data$date <- format(data$date, "%Y-%m-%d")
  # numbers in the shortest form that is read back exactly
  for (col in names(data)[vapply(data, is.double, logical(1))]) {
    data[[col]] <- lapply(data[[col]], function(v) if (is.na(v)) NA else structure(shortest_number(v), class = "json"))
  }
  records <- lapply(seq_len(nrow(data)), function(i) {
    row <- lapply(data, function(column) column[[i]])
    row[!vapply(row, function(v) length(v) == 1 && is.na(v), logical(1))]
  })
  json <- jsonlite::toJSON(list(spec_version = spec_version, records = records),
                           auto_unbox = TRUE, json_verbatim = TRUE, pretty = pretty)
  json <- paste0(as.character(json), "\n")
  if (!is.null(path)) {
    writeLines(json, path, useBytes = TRUE, sep = "")
    return(invisible(json))
  }
  json
}

Try the SoilManageR package in your browser

Any scripts or data that you put into this service are public.

SoilManageR documentation built on Nov. 17, 2026, 5:08 p.m.