R/aws.R

In leonawicz/snapprep: Prepare And Curate Useful Data Sets From Raw SNAP Data

#' Upload data to Amazon Web Services
#'
#' Upload prepared SNAP data to Amazon Web Services S3 buckets.
#'
#' This function uploads SNAP data to S3 buckets for use primarily by SNAP Shiny apps.
#' The files uploaded includes any recursively listed under \code{in_dir}. The names will include only the base names.
#' A \code{prefix} can be placed before the file base names. Using \code{/} in the prefix allows for the simulation of folders in the S3 bucket.
#' Remember that S3 buckets technically have a flat structure.
#'
#' NOTE: This function will not work unless the AWS keys granting permission to the bucket are loaded into the R session.
#'
#' @param in_dir character, input directory.
#' @param prefix character, optional prefix.
#' @param bkt S3 bucket name, defaults to the author since this is a SNAP developer package.
#' @param pattern file name pattern passed to \code{list.files} for filtering files in \code{in_dir}.
#' @param region_group region group used to further filter files in \code{in_dir}.
#' @param mc.cores number of CPUs when processing years in parallel. Defaults to 32 assuming Atlas compute node context.
#'
#' @export
#'
#' @examples
#' \dontrun{
#' source("aws_key.R") # SEE DETAILS
#' aws_upload(snapdef()$ar5dir_dist_monthly_split, "clim/dist/ar5_2km/monthly")
#' aws_upload(snapdef()$ar5dir_dist_seasonal, "clim/dist/ar5_2km/seasonal")
#' }
aws_upload <- function(in_dir, prefix, bkt = "leonawicz", pattern = NULL, region_group = NULL,
                       mc.cores = 32){
  files <- list.files(in_dir, pattern = pattern, recursive = TRUE)
  if(inherits(region_group, "character"))
    files <- files[basename(dirname(dirname(files))) == region_group]
  objs <- file.path(prefix, files)
  aws_put <- function(i, in_dir, files, objs, bkt){
    cat(paste0("Uploading to AWS: ", files[i], "\n"))
    aws.s3::put_object(file = file.path(in_dir, files[i]), object = objs[i], bucket = bkt)
    invisible()
  }
  parallel::mclapply(seq_along(files), aws_put, in_dir = in_dir, files = files, objs = objs,
                     bkt = bkt, mc.cores = mc.cores)
  cat(paste0("Files uploaded to ", bkt, "/", prefix, ".\n"))
  invisible()
}