R/fetch_vt.R

In ohvbd: One Health VBD Hub

#' @title Fetch VecTraits dataset/s by ID
#' @description Retrieve VecTraits dataset/s specified by their dataset ID.
#' @author Francis Windram
#'
#' @param ids a numeric ID or numeric vector of ids (preferably in an `ohvbd.ids` object) indicating the particular dataset/s to download.
#' @param rate maximum number of calls to the API per second.
#' @param connections number of simultaneous connections to the server at once. Maximum 8. **Do not enable unless you really need to** as this hits the server significantly harder than usual.
#' @param basereq an [httr2 request][httr2::request()] object, as generated by [vb_basereq()]. If `NA`, uses the default request.
#'
#' @return A list of [httr2 response][httr2::response()] objects, as an `ohvbd.responses` object.
#'
#' @examplesIf interactive()
#' fetch_vt(54)
#'
#' fetch_vt(c(54, 55, 56))
#'
#' ohvbd.ids(c(54, 55, 56), "vt") |>
#'   fetch() # Calls fetch_vt()
#' @concept vectraits
#'
#' @export
#'

fetch_vt <- function(ids, rate = 5, connections = 2, basereq = vb_basereq()) {
  max_conns <- 8

  check_provenance(ids, "vt", altfunc = "fetch")

  # ids_to_find can be a single number or a vector and this works just fine!
  reqs <- ids |>
    lapply(\(id) {
      basereq |>
        req_url_path_append("vectraits-dataset") |>
        req_url_path_append(id) |>
        req_url_query("format" = "json") |>
        req_error(body = vt_error_body) |>
        req_headers(ohvbd = id) |> # Add additional header just so we can nicely handle failures
        req_throttle(rate)
    })

  if (connections > max_conns) {
    cli::cli_alert_warning(
      "No more than {.val {max_conns}} simultaneous connection{?s} allowed!"
    )
    cli::cli_alert_info("Restricting to {.val {max_conns}} connection{?s}.")
    connections <- max_conns
  }
  resps <- reqs |>
    req_perform_parallel(
      on_error = "continue",
      max_active = connections,
      progress = list(
        name = "Vectraits Data",
        format = "Downloading {cli::pb_name} {cli::pb_current}/{cli::pb_total} {cli::pb_bar} {cli::pb_percent} | ETA: {cli::pb_eta}"
      )
    )

  fails <- resps |> httr2::resps_failures()

  # Test if any failures were missing files (404s)
  missing <- find_vb_404s(fails)

  if (!is.null(missing)) {
    cli::cli_alert_info("Incorrect ids:")
    cli::cli_ul(missing)
  }

  # Test to see if we got only errors
  if (length(fails) >= length(resps)) {
    # Only got errors!
    cli::cli_alert_warning(
      "No records retrieved (are you sure the IDs are correct?)."
    )
  }

  resps <- new_ohvbd.responses(l = resps, db = "vt")

  return(resps)
}