# regulationsFunctions.R
# Mark Hagemann
# 1/19/2016
# Following https://cran.r-project.org/web/packages/httr/vignettes/api-packages.html
# Also hijacks code from rnoaa package.
# Convention Helper functions ---------------------------------------------
#' @importFrom httr GET
regs_GET <- function(path, ..., pat = regs_pat()) {
auth <- regs_auth(pat)
req <- GET("https://api.data.gov/regulations/v3/", path = path, auth, ...)
regs_check(req)
req
}
regs_compact <- function(l) {
out <- Filter(Negate(is.null), l)
out
}
regs_check_key <- function(x) {
tmp <- if (is.null(x))
Sys.getenv("regsgovkey", "")
else x
if (tmp == "")
getOption("regsgovkey", stop("need an API key for regulations.gov data"))
else tmp
}
#' @importFrom httr content
regs_check <- function(x) {
if (!x$status_code == 200) {
stnames <- names(content(x))
if (!is.null(stnames)) {
if ("developerMessage" %in% stnames | "message" %in%
stnames) {
warning(sprintf("Error: (%s) - %s", x$status_code,
regs_compact(list(content(x)$developerMessage,
content(x)$message))))
}
else {
warning(sprintf("Error: (%s)", x$status_code))
}
}
else {
warn_for_status(x)
}
}
else {
stopifnot(x$headers$`content-type` == "application/json")
res <- content(x, as = "text", encoding = "UTF-8")
out <- jsonlite::fromJSON(res, simplifyVector = FALSE)
if (!"results" %in% names(out)) {
if (length(out) == 0) {
warning("Sorry, no data found")
}
}
else {
if (class(try(out$results, silent = TRUE)) == "try-error" |
is.null(try(out$results, silent = TRUE)))
warning("Sorry, no data found")
}
return(out)
}
}
regs_parse <- function(req) {
text <- content(req, as = "text")
if (identical(text, "")) stop("No output to parse", call. = FALSE)
jsonlite::fromJSON(text, simplifyVector = FALSE)
}
regs_makeDf <- function(regslist) {
allnames = Reduce(unique, lapply(regslist, names))
allrows = lapply(regslist, `[`, allnames)
allrows = lapply(allrows, setNames, allnames)
allrows = lapply(allrows, nullToNA)
out <- dplyr::bind_rows(allrows)
}
#' @importFrom assertthat assert_that
regs_datestring <- function(date1, date2) {
null1 <- is.null(date1)
null2 <- is.null(date2)
if (null1 && null2)
return(NULL)
else if (null1)
return(format(as.Date(date2), "%m/%d/%y"))
else if (null2)
return(format(as.Date(date1), "%m/%d/%y"))
else {
assert_that(as.Date(date1) < as.Date(date2))
return(paste(format(as.Date(date1), "%m/%d/%y"),
format(as.Date(date2), "%m/%d/%y"), sep = "-"))
}
}
nullToNA = function(df) {
df[sapply(df, is.null)] = NA
df
}
# Main function -----------------------------------------------------------
#' Retrieve regulations.gov documents
#'
#' @param apikey Access key for accessing the regulations.gov API.
#' Get one at https://regulationsgov.github.io/developers/key/.
#' @param countsOnly 1 (will return only the document count for a search query) 0 (will return documents as well)
#' @param encoded 1 (will accept Regulations.gov style encoded parameters) 0 (will not accept such encoded parameters)
#' @param keywords Keywords associated with the document
#' @param docType Document Type: N: Notice; PR: Proposed Rule; FR: Rule;
#' O: Other; SR: Supporting & Related Material; PS: Public Submission
#' @param docketID Valid Docket ID (ex. SEC-2012-0044)
#' @param docketType "Docket Type: R: Rulemaking; N: Nonrulemaking
#' @param commentPeriod Comment Period: O: Open; C: Closed
#' @param agency Federal Agency: List of accepted Federal Agency values. This field allows multiple values. ex: c(FMCSA, EPA, FDA)
#' @param nresults Results Per Page 10, 25, 100, 500, 1000. Results per page may not exceed 1000.
#' @param offset Page offset (always starts with 0). This is used in
#' conjunction with results per page to provide large data sets. For example,
#' if a search produces 82 results and the result per page is set to 25, this
#' will generate 4 pages. 3 pages will have 25 results and the last page will
#' have 7 results. Page offset values for each page will be: Page 1: po=0
#' Page 2: po=25 Page 3: po=50 Page 4: po=75. The total number of pages
#' is [total results/results per page] and page offset for page X is
#' [X-1 * results per page]"
#' @param closingSoon Comment Period Closing Soon: 0 (closing today);
#' 3 (closing within 3 days); 15 (closing within 15 days);
#' 30 (closing within 30 days); 90 (closing within 90 days)"
#' @param newlyPosted Newly Posted: 0 (posted today);
#' 3 (posted within last 3 days); 15 (posted within last 15 days);
#' 30 (posted within last 30 days); 90 (posted within last 90 days).
#' For periods of time beyond 90-days, please use a date range with the Posted
#' Date parameter.
#' @param comStartDate Comment Period Start Date
#' @param comEndDate Comment Period End Date. Comment Period Start and
#' End Dates are mutually exclusive with the 'closing soon' parameter.
#' If both are provided, 'closing soon' will be ignored.
#' @param createDate1 Beginning of date-of-creation range
#' @param createDate2 End of date-of-creation range
#' @param receivedDate1 Beginning of date-of-reception range
#' @param receivedDate2 End of date-of-reception range
#' @param postedDate1 Beginning of date-of-posting range
#' @param postedDate2 End of date-of-posting range
#' @param category Document Category: AD (Aerospace and Transportation);
#' AEP (Agriculture, Environment, and Public Lands);
#' BFS (Banking and Financial); CT (Commerce and International);
#' LES (Defense, Law Enforcement, and Security);
#' EELS (Education, Labor, Presidential, and Government Services);
#' EUMM (Energy, Natural Resources, and Utilities);
#' HCFP (Food Safety, Health, and Pharmaceutical);
#' PRE (Housing, Development, and Real Estate);
#' ITT (Technology and Telecommunications)
#' @param sortBy Sort By: docketId (Docket ID); docId (Document ID);
#' title (Title); postedDate (Posted Date); agency (Agency);
#' documentType (Document Type); submitterName (Submitter Name);
#' organization (Organization). Sort Order is REQUIRED if this parameter is
#' included.
#' @param sortOrder Sort Order: ASC: Ascending; DESC: Descending
#' @param docketSubtype Docket Subtype: Only one docket subtype at a time may
#' be selected. One or more agency values must be part of the request.
#' Only values valid for the selected agency will be returned.
#' @param docketSubSubtype Docket Sub-subtype: Only one docket sub-subtype at
#' a time may be selected. One or more agency values must be part of the
#' request. Only values valid for the selected agency will be returned.
#' @param documentSubtype: Single or multiple document subtypes may be
#' included. Multiple values should be passed as follows:
#' Certificate+of+Service%2BCorrespondence
#'
#' @details
#' A Docket Type is either Rulemaking or Nonrulemaking. A Rulemaking docket
#' includes the type of regulation that establishes a rule.
#' While a Non-Rulemaking docket does not include a rule.
#'
#' @importFrom assertthat assert_that
#' @importFrom httr GET timeout
#' @export
documents <- function (apikey = NULL, countsOnly = NULL, encoded = NULL,
keywords = NULL, docType = NULL, docketID = NULL,
docketType = NULL, commentPeriod = NULL, agency = NULL,
nresults = 1000, offset = NULL, closingSoon = NULL,
newlyPosted = NULL, comStartDate = NULL,
comEndDate = NULL, createDate1 = NULL,
createDate2 = NULL, receivedDate1 = NULL,
receivedDate2 = NULL,
postedDate1 = NULL, postedDate2 = NULL, category = NULL,
sortBy = NULL, sortOrder = NULL, docketSubtype = NULL,
docketSubSubtype = NULL, docSubtype = NULL, ...)
{
calls <- names(sapply(match.call(), deparse))[-1]
apikey <- regs_check_key(apikey)
countsOnly <- as.integer(as.logical(countsOnly))
encoded <- as.integer(as.logical(encoded))
if(!is.null(docType))
assert_that(all(docType %in% c("N", "PR", "FR", "O", "SR", "PS")))
if (!is.null(docketType))
assert_that(docketType %in% c("R", "N"))
if (!is.null(commentPeriod))
assert_that(commentPeriod) %in% c("O", "C")
if (!is.null(nresults))
assert_that(nresults %in% c(10, 25, 100, 500, 1000))
if (!is.null(closingSoon))
assert_that(closingSoon %in% c(0, 3, 15, 30, 90))
if (!is.null(newlyPosted))
assert_that(newlyPosted %in% c(0, 3, 15, 30, 90))
if(!is.null(agency))
agency <- paste(agency, collapse = "+")
comStartDate <- regs_datestring(comStartDate, NULL)
comEndDate <- regs_datestring(comEndDate, NULL)
createDate <- regs_datestring(createDate1, createDate2)
receivedDate <- regs_datestring(receivedDate1, receivedDate2)
postedDate <- regs_datestring(postedDate1, postedDate2)
keywords <- paste(keywords, collapse = "+")
if (!is.null(category))
assert_that(all(category %in% c("AD", "AEP", "BFS", "LES", "EELS", "EUMM",
"HCFP", "PRE", "ITT")))
if (!is.null(sortBy)) {
assert_that(all(sortBy %in% c("docketId", "docId", "title",
"postedDate", "agency",
"documentType", "submitterName",
"organization")))
assert_that(sortOrder %in% c("ASC", "DESC"))
}
base <- "https://api.data.gov/regulations/v3/documents.json"
args <- regs_compact(list(response_format = "json",
api_key = apikey, countsOnly = countsOnly,
encoded = encoded,
s = keywords, dct = docType, dktid = docketID,
dkt = docketType, cp = commentPeriod,
a = agency,
rpp = nresults, po = offset,
cs = closingSoon,
np = newlyPosted, cmsd = comStartDate,
cmd = comEndDate, crd = createDate,
rd = receivedDate,
pd = postedDate, cat = category,
sb = sortBy, so = sortOrder,
dktst = docketSubtype,
dktst2 = docketSubSubtype, docst = docSubtype))
args <- as.list(unlist(args))
names(args) <- gsub("[0-9]+", "", names(args))
if (length(args) == 0)
args <- NULL
temp <- GET(base, query = args, timeout(getOption("timeout")), ...)
temp
tt <- regs_check(temp)
if (countsOnly) out <- as.data.frame(tt)
else out <- regs_makeDf(tt$documents)
out
}
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.