In cboettig/neonstore: NEON Data Store

Import

Distribute

library(contentid)
library(fs)
library(dplyr)

meta <- neon_index()
d <- fs::dir_info(neon_dir())

how much data?

dim(d)[[1]]
d %>% summarise(sum(size))

ids <- contentid::store(meta$path, dir = "/home/content-store")

registry <- vroom::vroom("content_id.tsv")

meta <- dplyr::left_join(meta, registry) %>% as_tibble()
vroom::vroom_write(meta, "neonstore.tsv")

## read from the content-store instead
meta <- vroom::vroom("neonstore.tsv")
metab <- meta %>%  mutate(store = retrieve(id, "/home/content-store"))

birds <- metab %>% filter(name == "brd_countdata", type == "basic") %>% pull(store) %>% neon_read()

metab <- metab %>% mutate(mirror = gsub("/home/content-store", "https://data.carlboettiger.info", store))

request <- paste0("https://hash-archive.jetstream.carlboettiger.info/api/enqueue/", metab$mirror)
request2 <- paste0("https://hash-archive.org/api/enqueue/", metab$mirror)
x <- lapply(request2, httr::GET)

#xml2::url_escape("https://data.carlboettiger.info/data/10/c9/10c9e39af827ecdba4255ded08ab7f8e2b6895ce58561236d1e26ff3fa097d20")