In mdsumner/CMIP5files:

knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>",
  fig.path = "README-"
)

Overall focus

https://confluence.csiro.au/display/ACCESS/Home

simulations "piControl" "historical", "rcp45", "rcp85"

experiment "r1i1p1",

See: "standard_output.pdf" and http://cmip-pcmdi.llnl.gov/cmip5/data_description.html

CMIP5files

browseables for CMIP5

Start here: https://github.com/mdsumner/nectar/wiki/Raijin-CMIP5

CMIP5 files

These seem to cause a lot of grief, but NetCDF always does.

SQLite DB built by Python code on /g/data, lists tokenized file paths. This is useful but needs to be run routinely so it's up to date, and really make a full path to all files.

list actual files, find best way to list (i.e. multiple find jobs?)
scan each file and record metadata for each as per rancid::NetCDF or equivalent

Dev Plan

first pass only scan a sample of files, develop a scoping estimate of how long / how much etc.
Use try/catch defenses to be reliable, and feed back lessons into rancid package
Build searchable table of files with DT/Shiny (can we publish this off Raijin?)
Allow select file to deliver metadata DB as constellation of tables (or ncdump, whatevs)
Identify common patterns across all files in variables, dimensions
Use to develope use-case read code

module load   proj/4.9.0  netcdf/4.3.3.1p

#.libPaths("R/x86_64-pc-linux-gnu-library/3.2")
library(dplyr)
library(rancid)
db <- src_sqlite("/g/data1/ua6/unofficial-ESG-replica/tmp/tree/cmip5_raijin_latest.db")
cmip5 <- tbl(db, "cmip5")

access3 <- collect(filter(cmip5, model == "ACCESS1-3"))
id <- sample(access3$id, 1)
print(id)

nc <- NetCDF(sample(list.files(id, full.names = TRUE), 1))

nc$variable



ss <- sample(nrow(access3), min(c(100, nrow(access3))))
vv <- vector("list", length(ss))
system.time({
for (i in seq_along(vv)) {
fs <- sample(list.files(access3$id[i], full.names = TRUE, pattern = "nc$"), 1)
a <- try(NetCDF(fs))
if (!inherits(a, "try-error")) vv[[i]] <- a
}
})

object.size(vv)
# 4.88 Mb

any(unlist(lapply(vv, is.null)))

#save(vv, file = "~/cmip5_metadatasample.rda")

Etc.

Example

1. Use DT to get a user-useable path to a file.

library(DT)
d <- access3[sample(nrow(access3), 100), ]
datatable(d[, c(setdiff(names(d), "id"), "id")], class = 'cell-border stripe', filter = 'top', options = list(
  pageLength = 35, autoWidth = TRUE
))

2. Convert mapped data to model space and make a map

print(nc$variable)
lonlat <- romscoords(nc$file$filename, c("lon", "lat"))
library(maptools)
data(wrld_simpl)
plot(raster(f, varname = "tauuo"))
plot(romsmap(wrld_simpl, lonlat), add = TRUE)
#install.packages(c("rworldmap", "rworldxtra"))


tt <- spbabel::sptable(wrld_simpl)
tt$x_[tt$x_ < 0] <- tt$x_[tt$x_ < 0] + 360
w <- spbabel::sp(tt)
plot(raster(f, varname = "tauuo"))
plot(romsmap(w, lonlat), add = TRUE)

module load proj/4.9.0 netcdf/4.3.3.1p gdal/2.0.0

dp <- "/g/data/ua6/unofficial-ESG-replica/tmp/tree/tds.ucar.edu/thredds/fileServer/datazone/cmip5_data/cmip5/output1/NCAR/CCSM4/rcp85/mon/ocean/Omon/r6i1p1/v20140820/tauuo"
library(raster)
f <- list.files(dp, full.names = TRUE)
library(rancid)
library(angstroms)
nc <- NetCDF(f)
print(nc$variable)

Source: local data frame [6 x 16]

          name ndims natts   prec         units                  longname
         (chr) (int) (int)  (chr)         (chr)                     (chr)
1          lat     2     4  float degrees_north       latitude coordinate
2 lat_vertices     3     1  float degrees_north              lat_vertices
3          lon     2     4  float  degrees_east      longitude coordinate
4 lon_vertices     3     1  float  degrees_east              lon_vertices
5        tauuo     3    12  float         N m-2 Surface Downward X Stress
6    time_bnds     2     0 double                               time_bnds
Variables not shown: group_index (int), storage (dbl), shuffle (lgl),
  compression (lgl), unlim (lgl), make_missing_value (lgl), missval (dbl),
  hasAddOffset (lgl), hasScaleFact (lgl), id (dbl)


lonlat <- romscoords(f, c("lon", "lat"))
#devtools::install_github("AustralianAntarcticDivision/rbgm")
library(rbgm)
fname <- system.file("extdata", "Antarctica_28.bgm", package = "rbgm")
bgm <- bgmfile(fname)
plot(boxSpatial(bgm), col = grey(seq(0, 1, length = nrow(bgm$boxes))))
#devtools::install_github("mdsumner/angstroms")
ar <- romsmap(boxSpatial(bgm), lonlat)


## now extract!!
 cn <- extract(brick(romsdata(f, "tauuo")), ar, cellnumbers = TRUE)