R/urls_fetch.R

In kongdd/CMIP5tools: Functions to Tidy and Analysis CMIP5/6 Data

.scenarios_cmip6 <- c(
    "hist-GHG", "hist-nat", "hist-aer", "historical", "piControl",
    "ssp126", "ssp245", "ssp370", "ssp460", "ssp585")#[-(1:5)]


#' download urls for a certain CMIP6 variables
#' @export
urls_fetch <- function(variables, frequency, dir_runner = "CMIP6_runner", scenarios = NULL, 
    overwrite_sh = FALSE,
    overwrite_txt = FALSE, ...)
{
    if (is.null(scenarios)) scenarios = .scenarios_cmip6  
    urls_var <- function(variable) {
        url_root <- sprintf("http://esgf-data.dkrz.de/esg-search/wget/?mip_era=CMIP6&frequency=%s&variable=%s&limit=10000", frequency, variable)

        for (scenario in scenarios) {
            print(scenario)
            file_sh  <- glue("{dir_runner}/{variable}_{frequency}/sh/{variable}_{scenario}.sh")
            file_txt <- glue("{dir_runner}/{variable}_{frequency}/{variable}_{scenario}.txt")
            
            outdir_i <- dirname(file_sh)
            # print(outdir_i)
            if (!dir.exists(outdir_i)) dir.create(outdir_i, recursive = TRUE)
            url <- glue("{url_root}&experiment_id={scenario}")
            check_dir(dirname(file_sh))
            # print(file_sh)
            if (!file.exists(file_sh ) || overwrite_sh) wget(url, file_sh)
            if (!file.exists(file_txt) || overwrite_txt) urls_CMIP5_ESGP(file_sh)
        }
    }
    for(variable in variables) urls_var(variable)
}

#' @export
runner <- function(variables, frequency, dir_runner, scenarios = NULL, 
    dir_root = "CMIP6_data", ...)
{
    if (is.null(scenarios)) scenarios = .scenarios_cmip6
    download_var <- function(variable){
        outdir = glue("{root}/{variable}")
        if (!dir.exists(outdir)) dir.create(outdir, recursive = TRUE)

        for (scenario in scenarios) {
            print(scenario)
            infile <- glue("{dir_runner}/{variable}_{frequency}/{variable}_{scenario}.txt")
            outdiri <- glue("{outdir}/{scenario}")
            
            aria2c(infile, outdir = outdiri, ..., verbose = TRUE)
        }
    }
    for(variable in variables) download_var(variable)
}