R/read_new_project_files.R

In patelm9/projektoR:

#' Reads a csv using reader::read_csv
#' @param input_dir_names full path to file
#' @importFrom stringr str_remove_all
#' @importFrom mirroR try_catch_error_as_na
#' @importFrom projectloggeR instantiate_project_log
#' @importFrom dplyr mutate_all
#' @importFrom somersaulteR call_mr_clean
#' @export

read_new_project_files <-
        function(input_dir_names) {
                for (i in 1:length(input_dir_names)) {
                        input_dir_name <- input_dir_names[i]
                        path_folder    <- paste0(input_dir_name, "/NEW")

                        filenames <- list.files(paste0("./", input_dir_name, "/NEW"), full.names = TRUE)
                        for (j in 1:length(filenames)) {
                                filename <- filenames[j]
                                file_extension <- tolower(stringr::str_remove_all(filename, ".*[.]{1}"))
                                if ("xlsx" %in% file_extension) {
                                        projektoR::from_xlsx_to_project_csv(xlsx_fn = filename,
                                                                            path_folder = path_folder)
                                }
                        }
                }

                for (i in 1:length(input_dir_names)) {
                        input_dir_name <- input_dir_names[i]
                        filenames <- list.files(paste0("./", input_dir_name, "/NEW"), full.names = TRUE)
                        for (j in 1:length(filenames)) {
                                filename <- filenames[j]
                                file_extension <- tolower(stringr::str_remove_all(filename, ".*[.]{1}"))
                                if (file_extension == "rrf") {
                                        dataframe <- mirroR::try_catch_error_as_na(readr::read_delim(filename, delim = "|", col_names = FALSE, col_types = cols(.default = "c"))) %>%
                                                somersaulteR::call_mr_clean()

                                        if (!(is.vector(dataframe))) {
                                                START_INDEX <- max(as.double(PROJECT_LOG[[input_dir_name]][[length(PROJECT_LOG[[input_dir_name]])]]$LOG_ID), na.rm = TRUE) + 1
                                                LOG_ID <- ifelse(nchar(START_INDEX) == 1,
                                                                 paste0("0", START_INDEX),
                                                                 as.character(START_INDEX)
                                                )

                                                ROBJ_NAME <- paste(toupper(input_dir_name), "DATA", LOG_ID, sep = "_")
                                                RDATA_FN  <- paste0(ROBJ_NAME, ".RData")

                                                saveRDS(dataframe, file = RDATA_FN)
                                                assign(ROBJ_NAME, dataframe, envir = globalenv())


                                                projektoR::append_csv(csv_fn = paste0(input_dir_name, "/", tolower(input_dir_name), "_log.csv"),
                                                                      data.frame(LOAD_TIMESTAMP            = mirroR::get_timestamp(),
                                                                                 LOAD_COMMENT              = "",
                                                                                 PARENT_FN                 = "",
                                                                                 SOURCE_COMMENT            = "",
                                                                                 FN                        = filename,
                                                                                 FN_MD5SUM                 = tools::md5sum(filename),
                                                                                 LOG_ID                    = LOG_ID,
                                                                                 ROBJ_NAME                 = ROBJ_NAME,
                                                                                 RDATA_FN                  = RDATA_FN,
                                                                                 RDATA_FN_MD5SUM           = tools::md5sum(RDATA_FN),
                                                                                 LOG_TIMESTAMP             = "",
                                                                                 LOG_COMMENT               = "",
                                                                                 WRITE_TIMESTAMP           = "",
                                                                                 WRITE_COMMENT             = "",
                                                                                 WRITE_FULL_FN             = "",
                                                                                 WRITE_FULL_FN_MD5SUM      = "",
                                                                                 TO_DB_TIMESTAMP           = "",
                                                                                 TO_DB_COMMENT             = "",
                                                                                 TO_DB_NAME                = "",
                                                                                 TO_DB_SCHEMA              = "",
                                                                                 TO_DB_TABLE               = "",
                                                                                 FROM_DB_TIMESTAMP         = "",
                                                                                 FROM_DB_COMMENT           = "",
                                                                                 FROM_DB_NAME              = "",
                                                                                 FROM_DB_SCHEMA            = "",
                                                                                 FROM_DB_TABLE             = "") %>%
                                                                              dplyr::mutate_all(as.character) %>%
                                                                              dplyr::mutate_all(trimws, "both")
                                                )
                                                projectloggeR::instantiate_project_log(input_dir_names)

                                                old_fn <- filename
                                                new_fn <- paste0(input_dir_name, "/LOADED/", basename(filename))
                                                file.rename(old_fn, new_fn)
                                                gc()
                                        }
                                }
                        }
                }

                for (i in 1:length(input_dir_names)) {
                        input_dir_name <- input_dir_names[i]
                        filenames <- list.files(paste0("./", input_dir_name, "/NEW"), full.names = TRUE)
                        for (j in 1:length(filenames)) {
                                filename <- filenames[j]
                                file_extension <- tolower(stringr::str_remove_all(filename, ".*[.]{1}"))
                                if (file_extension == "csv") {
                                        dataframe <- mirroR::try_catch_error_as_na(readr::read_csv(filename, col_types = cols(.default = "c"))) %>%
                                                                                        somersaulteR::call_mr_clean()

                                        if (!(is.vector(dataframe))) {
                                                START_INDEX <- max(as.double(PROJECT_LOG[[input_dir_name]][[length(PROJECT_LOG[[input_dir_name]])]]$LOG_ID), na.rm = TRUE) + 1
                                                LOG_ID <- ifelse(nchar(START_INDEX) == 1,
                                                                 paste0("0", START_INDEX),
                                                                 as.character(START_INDEX)
                                                                 )

                                                ROBJ_NAME <- paste(toupper(input_dir_name), "DATA", LOG_ID, sep = "_")
                                                RDATA_FN  <- paste0(ROBJ_NAME, ".RData")

                                                saveRDS(dataframe, file = RDATA_FN)
                                                assign(ROBJ_NAME, dataframe, envir = globalenv())

                                                projektoR::append_csv(csv_fn = paste0(input_dir_name, "/", tolower(input_dir_name), "_log.csv"),
                                                                      data.frame(LOAD_TIMESTAMP            = mirroR::get_timestamp(),
                                                                                 LOAD_COMMENT              = "",
                                                                                 PARENT_FN                 = "",
                                                                                 SOURCE_COMMENT            = "",
                                                                                 FN                        = filename,
                                                                                 FN_MD5SUM                 = tools::md5sum(filename),
                                                                                 LOG_ID                    = LOG_ID,
                                                                                 ROBJ_NAME                 = ROBJ_NAME,
                                                                                 RDATA_FN                  = RDATA_FN,
                                                                                 RDATA_FN_MD5SUM           = tools::md5sum(RDATA_FN),
                                                                                 LOG_TIMESTAMP             = "",
                                                                                 LOG_COMMENT               = "",
                                                                                 WRITE_TIMESTAMP           = "",
                                                                                 WRITE_COMMENT             = "",
                                                                                 WRITE_FULL_FN             = "",
                                                                                 WRITE_FULL_FN_MD5SUM      = "",
                                                                                 TO_DB_TIMESTAMP           = "",
                                                                                 TO_DB_COMMENT             = "",
                                                                                 TO_DB_NAME                = "",
                                                                                 TO_DB_SCHEMA              = "",
                                                                                 TO_DB_TABLE               = "",
                                                                                 FROM_DB_TIMESTAMP         = "",
                                                                                 FROM_DB_COMMENT           = "",
                                                                                 FROM_DB_NAME              = "",
                                                                                 FROM_DB_SCHEMA            = "",
                                                                                 FROM_DB_TABLE             = "") %>%
                                                                              dplyr::mutate_all(as.character) %>%
                                                                              dplyr::mutate_all(trimws, "both")
                                                )
                                                projectloggeR::instantiate_project_log(input_dir_names)

                                                old_fn <- filename
                                                new_fn <- paste0(input_dir_name, "/LOADED/", basename(filename))
                                                file.rename(old_fn, new_fn)
                                                gc()
                                        }
                                }
                        }
                }
        }