R/senDownload.R

In spatialstatisticsupna/RGISTools: Handling Multiplatform Satellite Images

#' Download Sentinel images from a search list
#'
#' \code{senDownload} downloads the images from a list of uniform resource
#' locators (URLs) generated by the \code{\link{senSearch}} function from ESA’s
#' `SciHub' web service. The images are saved as GTiff files in the \code{AppRoot}
#' directory.
#' 
#' \code{senDownload} downloads the list of URLs provided by 
#' \code{\link{senSearch}}. In case the process is interrupted, the image file
#' could be corrupted. The function detects the corrupted files and restarts the
#' process. To prevent the computer from crashing, there is a maximum number of
#' attempts to try to donwnload the image (\code{nattempts}). The default number
#' of attempts is set to 3. The function requires an ESA’s `SciHub' account, which
#' can be obtained 
#' \href{https://scihub.copernicus.eu/dhus/#/self-registration}{here}.
#' 
#' When \code{unzip = TRUE}, the function decompresses the imagery. If
#' only a subset of bands is required, band names can be provided through the
#' \code{bFilter} argument. The band names are specified by “B”, followed by 
#' the two-digit band number, and the resolution of the band (either 10m, 20m,
#' or 60m) (e.g, "B01_10m"). Image decompression duplicates the information due
#' to the presence of both, compressed and decompressed images. Set 
#' \code{raw.rm = TRUE} to remove former ones.
#'
#' @param searchres the output from the \code{\link{senSearch}} function.
#' @param username ESA’s `SciHub' username.
#' @param password ESA’s `SciHub' password.
#' @param AppRoot the directory where the outcoming time series are saved.
#' @param unzip logical argument. If \code{TRUE}, unzips the images.
#' @param overwrite logical argument. If \code{TRUE}, overwrites the existing
#' images with the same name.
#' @param nattempts the number of attempts to download an image in case it
#' becomes corrupted.
#' @param omit.md5.error logical argument. If \code{TRUE}, omits md5 errors and 
#' do not removes the downloaded image.
#' @param ... arguments for nested functions.
#'  \itemize{
#'        \item \code{dates} a vector with the capturing dates being considered
#'   for downloading. 
#'        \item \code{bFilter} a vector with the bands to be extracted when \code{unzip=TRUE}. If not
#'   supplied, all bands are extracted.
#' }
#' @return this function does not return anything. It saves the imagery as
#' `zip’ (and JP2 files) in a folder called `raw’ (`unzip’) in the
#'  \code{AppRoot} directory.
#' @examples
#' \dontrun{
#' # load a spatial polygon object of Navarre
#' data(ex.navarre)
#' # download S2MSI1C products sensed by Sentinel-2
#' # in July-August 2018
#' sres <- senSearch(startDate = as.Date("2018-07-29","%Y-%m-%d"),
#'                        endDate = as.Date("2018-08-06","%Y-%m-%d"),
#'                        platform = "Sentinel-2",
#'                        extent = ex.navarre,
#'                        product = "S2MSI1C",
#'                        username = "username",
#'                        password = "password")
#'
#' # filtering the path R094 where Navarre is located
#' names(sres)
#' sres.sen.R094 <- sres[grepl("R094", names(sres))]
#' names(sres.sen.R094)
#' # list the dates in sres
#' senGetDates(names(sres.sen.R094),format="%Y%j")
#' wdir <- file.path(tempdir(),"Path_for_downloading_folder")
#' # donwload the imagery
#' senDownload(searchres = sres,
#'             username = "username",
#'             password = "password",
#'             AppRoot = wdir,
#'             unzip = TRUE)
#' wdir.sen.unzip <- file.path(wdir,"Sentinel","unzip")
#' files.sen.unzip<-list.files(wdir.sen.unzip,
#'                             pattern = "\\TCI.jp2$",
#'                             full.names = TRUE,
#'                             recursive = TRUE)
#' img.sen.rgb<-stack(files.sen.unzip[1])
#' plotRGB(img.sen.rgb)
#' }
senDownload<-function(searchres,
                        AppRoot,
                        username=NULL,
                        password=NULL,
                        nattempts = 5,
                        unzip=FALSE,
                        overwrite=FALSE,
                        omit.md5.error=TRUE,
                        ...){
  arg<-list(...)
  #if(class(searchres)!="senres"){stop("A response from sentinel search function is needed.")}
  if("dates"%in%names(arg)){searchres<-searchres[senGetDates(names(searchres))%in%arg$dates]}
  
  AppRoot<-pathWinLx(AppRoot)
  if(nattempts==0){
    message(paste0("Error downloading ",names(searchres)))
    return(NULL)
  }
  if(is.null(username)|is.null(password)){
    stop("Username and/or password not defined!")
  }
  downFolder<-file.path(AppRoot,"raw")
  dir.create(downFolder,recursive=TRUE,showWarnings = FALSE)
  message(paste0("Downloading the images in: ",downFolder))
  if(unzip){
    unzipFolder<-file.path(AppRoot,"unzip")
    dir.create(unzipFolder,recursive=TRUE,showWarnings = FALSE)
  }
  n.imgs<-length(searchres)
  c.handle = new_handle()
  handle_setopt(c.handle,
                referer=getRGISToolsOpt("SCIHUBAPIURL"),
                useragent = getRGISToolsOpt("USERAGENT"),
                followlocation = TRUE ,
                autoreferer = TRUE ,
                username=username,
                password=password)
  for(i in 1:n.imgs){
    url<-searchres[i]
    file.name<-names(url)
    tryCatch({
      downPath<-file.path(downFolder,paste0(file.name,".zip"))
      if((!file.exists(downPath))|overwrite){
        message(paste0("Downloading image ",file.name," (",i,"/",n.imgs,")"))
        image.url<-URLencode(url)
        if("verbose"%in%names(arg)){if(arg$verbose==T)message(paste0('Trying to download the url: ',image.url))}
        online<-gsub("/$value","/Online/$value",image.url,fixed = T)
        is.online<-curl_fetch_memory(online,handle=c.handle)
        if(rawToChar(is.online$content)=="false"){
          message("The image is archived, ordering...")
          order<-curl_fetch_memory(image.url,handle=c.handle)
          while(order$status_code!=202){
            Sys.sleep(10)
            if(order$status_code==503){
              message("Service Unavailable. The retrieval of offline data is temporarily unavailable, please try again later")
            }else if(order$status_code==403){
              message("Forbidden. User offline products retrieval quota exceeded")
            }else if(order$status_code==500){
              message("Internal Server Error. Unexpected nav segment Navigation Property")
            }
            order<-curl_fetch_memory(image.url,handle=c.handle)
          }
          message("Image ordered!")
          is.online<-curl_fetch_memory(online,handle=c.handle)
          while(rawToChar(is.online$content)=="false"){
            Sys.sleep(10)
            is.online<-curl_fetch_memory(online,handle=c.handle)
          }
          curl_download(image.url, destfile=downPath,handle = c.handle)
          
        }else{
          curl_download(image.url, destfile=downPath,handle = c.handle)
        }
        
      }

      #md5 check
      md5.url<-paste0(gsub("$value","",url,fixed = TRUE),"Checksum/Value/$value")
      message(md5.url)
      repeat{
        response<-curl(md5.url,handle =c.handle)
        
        md5.text<-suppressWarnings(readLines(response))
        close(response)
        if(!grepl("Error",md5.text)){
          message(paste0("Get md5: ",md5.text))
          break
        }else{
          message("md5 not found! trying again.")
          Sys.sleep(10)
        }
      }
      if(!genCheckMD5(downPath,oficial.md5=md5.text,...)){
        message(paste0("Error cheking ",file.name," file md5: ",md5.text))
        if(omit.md5.error){
          Sys.sleep(3)
          next
        }
        file.remove(downPath)
        sres<-searchres[i]
        class(sres)<-"senres"
        senDownload(username=username,
                      password=password,
                      searchres=sres,
                      unzip=unzip,
                      overwrite = overwrite,
                      nattempts=nattempts -1,
                      AppRoot = AppRoot,
                      ...)
      }else{
        message(paste0("OK: cheking ",file.name," file md5."))
        if(unzip){
          message("Unzipping ", basename(downPath)," file.")
          if("bFilter"%in%names(arg)){
            flist<-unzip(downPath,list=TRUE)
            flist<-flist$Name
            flist<-flist[Reduce("|", lapply(paste0(arg$bFilter,"\\.jp2$"),grepl,flist))]
            suppressWarnings(unzip(zipfile=downPath,
                                   files=flist,
                                   exdir = unzipFolder,
                                   overwrite=overwrite))
          }else{
            suppressWarnings(unzip(zipfile=downPath,
                                   exdir = unzipFolder,
                                   overwrite=overwrite))
          }
        }
      }
    }, error = function(e) {
      if(grepl("Operation was aborted",e)){stop(e)}
      message(paste0("ERROR:",e))
      if(!grepl("HTTP error 403.",e)){
        Sys.sleep(5)
        file.remove(downPath)
        sres<-searchres[i]
        class(sres)<-"senres"
        senDownload(username=username,
                    password=password,
                    searchres=sres,
                    unzip=unzip,
                    overwrite = overwrite,
                    nattempts=nattempts -1,
                    AppRoot = AppRoot,
                    ...)
      }
    }, finally = {
    })
  }

  if(unzip){
    message(paste0("The images have been unzipped in: ",unzipFolder))
  }else{
    message(paste0("The images have been downloaded and saved on HDD. \nFile path: ",downFolder))
  }
  rm(c.handle)
}