R/batchDatawarehouse.R

In blazickjoe/DataScienceLibrary: DKS R Functions

#' Batch Datawarehouse Queries
#'
#' Batch your datawarehouse queries by day, week, or month for large submissions
#'
#' @param fromDate From Date
#' @param toDate To Date
#' @param reportsuite.id Available report suites can be found using RSiteCatalyst's GetReportSuites
#' @param granularity Granularity for submitting DataWarehouse requests. Must be one of "day", "week", or "month"
#' @param rootFileName root of filenames to be saved in FTP
#' @param host url of FTP host
#' @param directory FTP sub directory if needed. Defaults to NULL and stores files in root directory
#' @param username Username for FTP login
#' @param password Password for FTP login
#' @param port defaults to port 21
#' @param metrics Available metrics can be found using RSiteCatalyst's GetMetrics
#' @param elements Available elements can be found using RSiteCatalyst's GetElements
#' @param segment.id Available segments can be found using RSiteCatalyst's GetSegments
#' @param data.current Do not adjust
#' @param expedite Do not adjust
#' @param interval.seconds Do not adjust
#' @param max.attempts Do not adjust
#' @param validate Do not adjust
#' @param enqueueOnly Do not adjust
#' @return Data is delivered to the FTP specified
#' @export

#### Batching DataWarehouse Requests ####

batchDataWarehouse <- function(fromDate,
                               toDate,
                               reportsuite.id,
                               granularity,
                               rootFileName,
                               host,
                               username,
                               password,
                               port = 21,
                               directory = NULL,
                               metrics = NULL,
                               elements = NULL,
                               segment.id = NULL,
                               data.current = FALSE,
                               expedite = FALSE,
                               interval.seconds = 1,
                               max.attempts = 1,
                               validate = TRUE,
                               enqueueOnly = TRUE){

  # Added the arguments that pass into dwArgs as part of the function to make everything self contained
  dwArgs <- list(reportsuite.id = reportsuite.id,
                 date.from = fromDate,
                 date.to = toDate,
                 metrics = metrics,
                 elements = elements,
                 segment.id = segment.id,
                 data.current = data.current,
                 expedite = expedite,
                 interval.seconds = interval.seconds,
                 max.attempts = max.attempts,
                 validate = validate,
                 enqueueOnly = enqueueOnly,
                 ftp = list(host = host,
                            port = port,
                            directory = ifelse(is.null(directory), "/", directory),
                            username = username,
                            password = password))
  #submit your data args as a list, leave out the dates.
  #granularity will take anything from month to day.
  #fromDate and toDate are the start and end date for the batch.
  #rootfile name is the base filename, it will append the date to the end.
  dateList <- seq(as.Date(fromDate), as.Date(toDate), granularity)

  #batching is slightly different for day
  if (granularity == "day") {

    for (i in 1:length(dateList)) {

      dwArgs$date.from <- as.character(dateList[i])
      dwArgs$date.to <- as.character(dateList[i])
      dwArgs$ftp$filename <- paste0(rootFileName,
                                    gsub("\\-","",dwArgs$date.from), ".csv")
      do.call(RSiteCatalyst::QueueDataWarehouse, dwArgs)

    }
  } else {

    if (length(dateList) > 1) {
      #the rest of the granularity possibilities are the same
      for (i in 1:(length(dateList) - 1)) {

        dwArgs$date.from <- as.character(dateList[i])
        dwArgs$date.to <- as.character(dateList[i + 1] - 1)
        dwArgs$ftp$filename <- paste0(rootFileName,
                                      gsub("\\-","",dwArgs$date.from), "-",
                                      gsub("\\-","",dwArgs$date.to), ".csv")

        do.call(RSiteCatalyst::QueueDataWarehouse, dwArgs)

      }
    }
    #handling the granularity not having a good end date
    dwArgs$date.from <- dateList[length(dateList)]
    dwArgs$date.to <- toDate
    dwArgs$ftp$filename <- paste0(rootFileName,
                                  gsub("\\-","",dwArgs$date.from), "-",
                                  gsub("\\-","",dwArgs$date.to), ".csv")
    do.call(RSiteCatalyst::QueueDataWarehouse, dwArgs)
  }
}