R/writeFTPFiles.R

In blazickjoe/DataScienceLibrary: DKS R Functions

#' Fetch and Write FTP Files
#'
#' This function allows you to pull down a series of files from the FTP
#' and immediately save them to a location without loading the complete
#' data into memory.
#'
#' @param host The FTP host url
#' @param userpwd login username and password colon seperated
#' @param fileList List of files to pull down
#' @param file Character string naming the file for writing
#' @param tempPath Optional character string file path
#' @return A single file comprised of the list of files given in the directory of choice
#'
#' @importFrom utils download.file read.csv unzip
#'
#' @export

writeFTPFiles <-
  function(host,
           userpwd,
           fileList,
           file,
           tempPath = NULL) {

    #do a quick check for traling "/"
    host <- ifelse(grepl("\\/$", host), host, paste0(host,"/"))

    #set-up temporary location to save downloaded file
    if(is.null(tempPath)){

      #check OS to define path slashes
      tempPath <- ifelse(grepl("[Ww]indows",Sys.info()['sysname']),
                         paste0(tempdir(), "\\"),
                         paste0(tempdir(), "/"))

      temp_file <- ifelse(grepl("\\.zip", fileList[1]),
                          paste0(tempfile(), ".zip"),
                          paste0(tempfile(), ".csv"))

    } else if (grepl("\\:", tempPath) |
               grepl("dcsg\\.com", tempPath) |
               grepl("\\/", tempPath)) {

      tempPath <- ifelse(grepl("\\/$", tempPath) | grepl("\\\\$", tempPath),
                         tempPath,
                         paste0(tempPath, "/"))
      temp_file <- ifelse(grepl("\\.zip", fileList[1]),
                          paste0(tempPath, "/tempfile", ".zip"),
                          paste0(tempPath, "/tempfile", ".csv"))

    } else {

      stop("Bad temporary file path provided.")

    }


    #get list of files and agg them, handles the i=1 case too.
    zipDir <- ifelse(grepl("[^:](\\/$|\\\\$)", tempPath),
                     gsub("(\\/$|\\\\$)", "", tempPath),
                     tempPath)

    for (i in fileList) {

      url <- paste0("ftp://", userpwd, "@", host, i)

      if (grepl("\\.zip", i)) {

        download.file(url, temp_file, quiet = T)
        Sys.sleep(3)
        unzip(temp_file, exdir = zipDir)

      } else if (grepl("\\.csv", i)) {

        download.file(url, paste0(tempPath,i), quiet = T)

      } else {

        stop("Unsupported filetype provided.  Only accepts 'csv' or 'zip'.")

      }

      unlink(temp_file)

    }

    #write the files out.
    Files <- paste0(tempPath, gsub("\\.zip", ".csv", fileList))
    for(f in Files) {
      d <- suppressWarnings(data.table::fread(f))
      first <- f == Files[1]
      suppressWarnings(data.table::fwrite(d, file, sep = ",", row.names = FALSE,
                  col.names = first, append = !first, showProgress = FALSE, verbose = FALSE))
      unlink(f)
    }

    loc <- ifelse(grepl("\\:", file) | grepl("dcsg\\.com", file) | grepl("\\/", file),
                  file,
                  paste0(getwd(), "/", file))

    return(message("File successfully written to ", loc))
  }