R/documents.R

Defines functions rejoin parse_documents resp_documents documents wb_document

Documented in wb_document

#' World Bank document data
#'
#' Search the World Bank Documents & Reports archive from the Documents & Reports API.
#'
#' @param search (`NULL` | `character(1)`)\cr
#'   Free-text search term. Default `NULL`.
#' @param country (`NULL` | `character()`)\cr
#'   Two-character World Bank country code(s) to filter by, e.g. `"BR"` or `c("BR", "IN")`.
#'   Matching is case insensitive, and documents for any of the given codes are returned.
#'   Default `NULL`.
#' @param type (`NULL` | `character()`)\cr
#'   Document type(s) to filter by, e.g. `"Report"` or `"Project Appraisal Document"`. Documents
#'   of any of the given types are returned. Default `NULL`.
#' @param project (`NULL` | `character()`)\cr
#'   Project ID(s) to filter by, e.g. `"P180429"`. Documents for any of the given projects are
#'   returned. Default `NULL`.
#' @param start_date (`NULL` | `character(1)`)\cr
#'   Document start date in `"YYYY-MM-DD"` format. Default `NULL`.
#' @param end_date (`NULL` | `character(1)`)\cr
#'   Document end date in `"YYYY-MM-DD"` format. Default `NULL`.
#' @param id (`NULL` | `character()`)\cr
#'   Document ID(s) to filter by, as returned in the `id` column. Default `NULL`.
#' @param limit (`NULL` | `integer(1)`)\cr
#'   The maximum number of documents to return. Default `NULL`. If `NULL`, all matching documents
#'   are returned, which can take many requests for broad queries. Documents are returned in
#'   descending order of `date`, so the most recent documents come first.
#' @returns A `data.frame()` with World Bank document data. Since a document can have several
#'   types, countries, regions, and projects, `type`, `major_type`, `country_code`, `country`,
#'   `region`, and `project_id` hold all of them separated by `;`, which never occurs within a
#'   value. Use `strsplit(x, ";")` to split them. The columns are:
#' * `id`: The document ID.
#' * `title`: The document title.
#' * `type`: The document types.
#' * `major_type`: The major document types.
#' * `country_code`: The World Bank country codes.
#' * `country`: The country names. They are not in the same order as `country_code`, and a code can
#'   have more than one name, so the two can't be matched by position.
#' * `region`: The administrative region names.
#' * `language`: The document language.
#' * `date`: The document date.
#' * `report_number`: The report number.
#' * `project_id`: The IDs of the associated projects, or `NA` if there are none.
#' * `url`: The document URL.
#' * `pdf_url`: The PDF URL.
#' * `abstract`: The abstract.
#' @source <https://search.worldbank.org/api/v3/wds>
#' @family documents data
#' @export
#' @examplesIf httr2::is_online()
#' \donttest{
#' # project appraisal documents for Brazil published in 2024
#' wb_document(
#'   country = "BR",
#'   type = "Project Appraisal Document",
#'   start_date = "2024-01-01",
#'   end_date = "2024-12-31"
#' )
#'
#' # all documents belonging to a project
#' wb_document(project = "P180429")
#'
#' # the first 100 documents mentioning climate
#' docs <- wb_document(search = "climate", limit = 100)
#'
#' # the ones covering more than one country
#' docs[grepl(";", docs$country_code), c("title", "country_code")]
#' }
wb_document <- function(
  search = NULL,
  country = NULL,
  type = NULL,
  project = NULL,
  start_date = NULL,
  end_date = NULL,
  id = NULL,
  limit = NULL
) {
  stopifnot(
    is_string(search, null_ok = TRUE),
    is_character(country, null_ok = TRUE, n_chars = 2L),
    is_character(type, null_ok = TRUE),
    is_character(project, null_ok = TRUE),
    is_string(start_date, null_ok = TRUE, pattern = "^\\d{4}-\\d{2}-\\d{2}$"),
    is_string(end_date, null_ok = TRUE, pattern = "^\\d{4}-\\d{2}-\\d{2}$"),
    is_character(id, null_ok = TRUE),
    is_count(limit, null_ok = TRUE)
  )

  if (!is.null(start_date) && !is.null(end_date) && start_date > end_date) {
    stop("`start_date` must be earlier than `end_date`.", call. = FALSE)
  }

  data <- documents(
    qterm = search,
    countrycode_exact = collapse_or(toupper(country)),
    docty_exact = collapse_or(type),
    proid = collapse_or(toupper(project)),
    strdate = start_date,
    enddate = end_date,
    id = collapse_or(id),
    limit = limit
  )
  parse_documents(data)
}

document_fields <- c(
  "display_title",
  "docty",
  "majdocty",
  "countrycode",
  "count",
  "admreg",
  "lang",
  "docdt",
  "repnb",
  "projectid",
  "url",
  "pdfurl",
  "abstracts"
)

documents <- function(..., limit = NULL) {
  page_size <- min(limit %||% 1000L, 1000L)
  max_reqs <- if (!is.null(limit)) ceiling(limit / page_size) else Inf

  # the default order by relevance has ties, so paging by offset repeats and skips documents
  req <- wb_request("https://search.worldbank.org/api/v3/wds") |>
    req_url_query(
      ...,
      format = "json",
      rows = page_size,
      fl = document_fields,
      srt = "docdt",
      order = "desc",
      .multi = "comma"
    )

  resps <- req_perform_iterative(
    req,
    next_req = iterate_with_offset(
      "os",
      start = 0L,
      offset = page_size,
      resp_pages = \(resp) resp_total_pages(resp, page_size),
      resp_complete = \(resp) length(resp_documents(resp)) == 0L
    ),
    max_reqs = max_reqs,
    progress = wb_progress()
  )

  data <- resps_data(resps, resp_documents)
  # documents with the same date can still shift between pages
  ids <- map_chr(data, \(x) x$id %||% NA_character_)
  data <- data[is.na(ids) | !duplicated(ids)]
  if (!is.null(limit)) {
    data <- utils::head(data, limit)
  }
  data
}

resp_documents <- function(resp) {
  docs <- resp_body_json(resp, check_type = FALSE)$documents
  docs$facets <- NULL
  unname(docs)
}

parse_documents <- function(data) {
  res <- data.frame(
    id = map_chr(data, \(x) x$id %||% NA_character_),
    title = map_chr(data, \(x) x$display_title %||% NA_character_),
    type = map_chr(data, \(x) x$docty %||% NA_character_),
    major_type = map_chr(data, \(x) x$majdocty %||% NA_character_),
    country_code = map_chr(data, \(x) x$countrycode %||% NA_character_),
    country = map_chr(data, \(x) x$count %||% NA_character_),
    region = map_chr(data, \(x) x$admreg %||% NA_character_),
    language = map_chr(data, \(x) x$lang %||% NA_character_),
    date = map_chr(data, \(x) x$docdt %||% NA_character_),
    report_number = map_chr(data, \(x) x$repnb %||% NA_character_),
    project_id = map_chr(data, \(x) x$projectid %||% NA_character_),
    url = map_chr(data, \(x) x$url %||% NA_character_),
    pdf_url = map_chr(data, \(x) x$pdfurl %||% NA_character_),
    abstract = map_chr(data, \(x) x$abstracts[["cdata!"]] %||% NA_character_),
    check.names = FALSE
  )
  res$date <- as.Date(sub("T.*", "", res$date))
  res <- clean_strings(res)
  res$type <- rejoin(res$type, ";")
  res$major_type <- rejoin(res$major_type, ";")
  res$country_code <- rejoin(res$country_code, ",")
  # names can contain ", " themselves, e.g. "Congo, Democratic Republic of"
  res$country <- rejoin(res$country, ",(?! )")
  res$region <- rejoin(res$region, ";")
  res$project_id <- rejoin(res$project_id, ",")
  res
}

rejoin <- function(x, split) {
  res <- map_chr(x, function(value) {
    if (is.na(value)) {
      return(NA_character_)
    }
    value <- trimws(strsplit(value, split, perl = TRUE)[[1L]])
    paste(value[nzchar(value)], collapse = ";")
  })
  na_if_empty(res)
}

Try the worldbank package in your browser

Any scripts or data that you put into this service are public.

worldbank documentation built on Nov. 17, 2026, 5:06 p.m.