In bowerth/nsoApi: functions for connecting to various APIs

library(nsoApi)
library(dplyr)
library(tidyr)
library(RCurl)
curl <- getCurlHandle()
n_table <- 5

Providers

Some of the APIs require authentication:

## getwd()
## read.csv(file = file.path("..", "inst", "extdata", "api_overview_table_short.csv")) %>%
system.file("extdata", "api_overview_table_short.csv", package = "nsoApi") %>%
  read.csv() %>%
  mutate(isIO = ifelse(cou == "IO", TRUE, FALSE)) %>%
  arrange(desc(isIO), provider) %>%
  select(cou, provider, label, access) %>%
  filter(provider %in% c("BEA", "CBS", "GENESIS", "ONS", "STATAT", "SSB", "WDS")) %>%
  knitr::kable()

Setup

Create an R script containing a list with your keys or just modify the the example script and save it with gpg encryption at inst/apiKey.R.gpg.

apiKey <- list(
  BEA = list(
    userid = "29UG8ZKT-JN9R5N2Q")
  ,
  GENESIS = list(
    kennung = "8XXHKY78",
    passwort = "3QYYTQ89")
  ,
  ONS = list(
    apikey = "k647uahw")
)

To prevent compromising the API keys, it is suggested to encrypt the file. If you are on Windows, you will need to install GnuPG first. The nsoApiGPG function allows to specify a directory to an executable using the gpg parameter.

file.apiKey.enc <- system.file("apiKey.R.gpg", package = "nsoApi")
nsoApi::nsoApiGPG(file = file.apiKey.enc,
                  gpg = NULL,
                  passphrase = Sys.getenv("NSOAPIGPG"),
                  keep = FALSE
                  )

BEA Bureau of Economic Analysis

api.param <- list(
  DATASETNAME = "GDPbyIndustry",
  METHOD = "GETPARAMETERVALUES",
  RESULTFORMAT = "JSON",
  USERID = apiKey$BEA$userid,
  PARAMETERNAME = "TABLEID"
)
## req.uri <- beaAPI(api.param = api.param, curl = curl, query = TRUE)
## tt <- RCurl::getURL(req.uri, curl = curl)
tablelist <- beaAPI(api.param = api.param, curl = curl)
tablelist$BEAAPI$Request$RequestParam

Show first r n_table tables

knitr::kable(tablelist$BEAAPI$Results$ParamValue[1:n_table, ])

api.param <- list(
  DATASETNAME = "GDPBYINDUSTRY",
  FREQUENCY = "A",
  INDUSTRY = "11",
  METHOD = "GETDATA",
  RESULTFORMAT = "JSON",
  USERID = apiKey$BEA$userid,
  TABLEID = "1",
  YEAR = paste(as.character(c(2008:2015)), collapse = ",")
)
datalist <- beaAPI(api.param = api.param, curl = curl)
knitr::kable(datalist$BEAAPI$Results$Data)

CBS Statistics Netherlands

First, we define the API URI and the dataset ID

api <- "https://opendata.cbs.nl/ODataApi/OData/"
DSD <- "82572ENG" # Input-Output: "83068ENG"

CBS Metadata

Next, we return the members of a dimension

scheme <- "SectorBranchesSIC2008"
## print query
cbsODataAPI(api=api, DSD=DSD, scheme=scheme, query=TRUE)
cbs_dimension <- cbsODataAPI(api=api, DSD=DSD, scheme=scheme, query=FALSE)
knitr::kable(cbs_dimension[1:n_table, 1:2])

CBS Data

Finally, we return a dataset

scheme <- "TypedDataSet"
# print query
cbsODataAPI(api=api, DSD=DSD, scheme=scheme, query=TRUE)
cbs_data <- cbsODataAPI(api=api, DSD=DSD, scheme=scheme, query=FALSE)
knitr::kable(cbs_data[1:n_table, 1:6])

CBS Tables

List existing tables in CBS API. This takes 1-2 minutes to complete

cbs_tables <- nsoApi::cbsODataTables(url = "https://opendata.cbs.nl/ODataCatalog/Tables")
knitr::kable(cbs_tables[1:n_table, ])
## write.table(cbs_tables, file = "~/Downloads/cbs-tables.tsv", sep = "\t", row.names = FALSE)

Destatis GENESIS

Genesis Metadata

api.param.datenexport <- list(
    method = "DatenExport",
    kennung = apiKey$GENESIS$kennung,
    passwort = apiKey$GENESIS$passwort,
    namen = "81000BJ002",
    bereich = "oeffentlich",
    format = "csv",
    werte = "false",
    metadaten = "false",
    zusatz = "true",
    startjahr = "2013",
    endjahr = "2014",
    zeitscheiben = "",
    regionalschluessel = "",
    sachmerkmal = "",
    sachschluessel = "",
    stand = "01.01.2001",
    sprache = "de"
)
## retrieve metadata
xml_list_datenexport_meta <- genesisAPI(api.param = api.param.datenexport,
                                   service = "ExportService",
                                   curl = curl)
meta_df <- genesisXMLtoDF(xml.list = xml_list_datenexport_meta,
                          meta = TRUE)
knitr::kable(meta_df)

Genesis Data

api.param.datenexport <- list(
    method = "DatenExport",
    kennung = apiKey$GENESIS$kennung,
    passwort = apiKey$GENESIS$passwort,
    namen = "81000BJ002",
    bereich = "oeffentlich",
    format = "csv",
    werte = "true",
    metadaten = "false",
    zusatz = "false",
    startjahr = "2013",
    endjahr = "2014",
    zeitscheiben = "",
    regionalschluessel = "",
    sachmerkmal = "",
    sachschluessel = "",
    stand = "01.01.2001",
    sprache = "de"
)
## retrieve data
xml_list_datenexport <- genesisAPI(api.param = api.param.datenexport,
                                   service = "ExportService",
                                   curl = curl)
## convert to data frame
data_df <- genesisXMLtoDF(xml.list = xml_list_datenexport)
knitr::kable(data_df[1:n_table,])

Genesis Tables

genesis_tables <- nsoApi::genesisTables(
                namefilter = 11, # for testing - takes a few minutes to complete
                ## namefilter = c(11:14, 2, 3, 41:49, 5:9),
                fields = c("code", "beschriftungstext"),
                kennung = apiKey$GENESIS$kennung,
                passwort = apiKey$GENESIS$passwort
                )
genesis_tables$beschriftungstext <- gsub("\n", " ", genesis_tables$beschriftungstext)
knitr::kable(genesis_tables)

ONS UK Office for National Statistics

The provider has been taken offline

ONS Data

api.param <- list(context = "Census",
                  geog = "2011WARDH",
                  totals = "false",
                  apikey = "xPuqnMzZ01")
ons_data <- onsCsvData(api.param = api.param, dataset = "QS208EW")
knitr::kable(ons_data[1:n_table, 1:7])

ONS Tables

ons_tables <- onsAPI(method = "contexts", api.key = apiKey$ONS$apikey)
knitr::kable(ons_tables)

ons_collections <- onsAPI(method = "collections", context = "Economic", api.key = apiKey$ONS$apikey)
titles <- sapply(ons_collections$name[, 1], function(x) x[1, 2])
titles <- iconv(titles, "latin1", "ASCII", sub="")
ons_collections_df <- data.frame(ID = ons_collections$id, Title = titles)
knitr::kable(ons_collections_df[1:n_table, ])

STATAT Statistics Austria

STATAT Data

dataset <- "OGD_vgr001_VGRJahresR_1"
statat_data <- statatAPI(dataset = dataset)
knitr::kable(statat_data[1:n_table, ])

STATAT Tables

df <- statatTables()
knitr::kable(df[1:n_table, ])

Tilastokeskus Statistics Finland

Statistics Finland Tables

Obtaining a list of tables for Finland is tedious due to the hierarchical structure of the API. At each of the four levels, we must call pxweb::get_pxweb_levels and interatively concstruct the tree of datasets. This tree is used as input when retrieving the datasets. We must know the path to a dataset a priori.

pxnet2_tables <- read.table(system.file("extdata", "content", "pxnet2_tables.tsv", package = "nsoApi"), header = TRUE)
knitr::kable(pxnet2_tables[1:10,])

Statistics Finland Data

flow <- "070_vtp_tau_071.px"
baseURL <- "http://pxnet2.stat.fi/PXWEB/api/v1/en/StatFin"
path <- as.character(pxnet2_tables$Path[pxnet2_tables$ID==flow])
req.uri <- file.path(baseURL, path, flow)
bottom_node <- pxweb::get_pxweb_metadata(req.uri)
dims <- pxweb::get_pxweb_dims(bottom_node)
## dims_list <- as.list(rep("*", length(names(dims))))
## names(dims_list) <- names(dims)
dims_list <- lapply(dims, function(x) sample(x$values, 3))
pxweb_data <-                           # retrieve data
  pxweb::get_pxweb_data(
           url = req.uri,
           dims = dims_list,
           clean = TRUE)
knitr::kable(pxweb_data[1:10,])

SCB Statistics Sweden

Statistics Sweden Tables

Same as Statistics Finland, SCB is using the pxweb standard for their API.

scb_tables <- read.table(system.file("extdata", "content", "scb_tables.tsv", package = "nsoApi"), header = TRUE)
knitr::kable(scb_tables[1:10,])

Statistics Sweden Data

baseURL <- "http://api.scb.se/OV0104/v1/doris/en/ssd"
flow <- "NR0103ENS2010T08A"
path <- as.character(scb_tables$Path[scb_tables$ID==flow])
req.uri <- file.path(baseURL, path, flow)
bottom_node <- pxweb::get_pxweb_metadata(req.uri)
dims <- pxweb::get_pxweb_dims(bottom_node)
## dims_list <- as.list(rep("*", length(names(dims))))
## names(dims_list) <- names(dims)
dims_list <- lapply(dims, function(x) sample(x$values, min(length(x$values), 3)))
pxweb_data <-                           # retrieve data
  pxweb::get_pxweb_data(
           url = req.uri,
           dims = dims_list,
           clean = TRUE)
knitr::kable(pxweb_data[1:10,])

SBB StatBank Statistics Norway

Statistics Norway Tables

In the latest version, the SSB API is compatible with the pxweb standard.

ssb_tables <- read.table(system.file("extdata", "content", "ssb_tables.tsv", package = "nsoApi"), header = TRUE)
knitr::kable(ssb_tables[1:10,])

Statistics Norway Data

baseURL <- "http://data.ssb.no/api/v0"
## baseURL <- "http://data.ssb.no/api/v0/en/table"
flow <- "OrgsatEksUlonn"
req.uri <- as.character(ssb_tables$Statistic[ssb_tables$ID==flow])[1]
## req.uri <- file.path(baseURL, path, flow)
bottom_node <- pxweb::get_pxweb_metadata(req.uri)
dims <- pxweb::get_pxweb_dims(bottom_node)
## dims_list <- as.list(rep("*", length(names(dims))))
## names(dims_list) <- names(dims)
dims_list <- lapply(dims, function(x) sample(x$values, min(3, length(x$values))))
pxweb_data <-                           # retrieve data
  pxweb::get_pxweb_data(
           url = req.uri,
           dims = dims_list,
           clean = TRUE)
knitr::kable(pxweb_data[1:10,])

bowerth/nsoApi documentation built on June 17, 2019, 4:55 a.m.