scripts/view_download_2008_onwards_failures_2.R

In petermeissner/wikipediadumbs: Handling Wikipedia Page View Dumps

# script to download all data for 2011
library(wpd)
library(fs)

dir.create("/data/wpd/failures", showWarnings = FALSE)

failures <-
  c(
    "2008-01-27", "2008-02-01", "2008-02-02", "2008-03-03", "2008-03-04",
    "2008-04-02", "2008-05-20", "2008-10-21", "2008-10-22", "2008-11-09",
    "2008-12-31", "2009-01-01", "2009-01-02", "2009-01-03", "2009-01-04",
    "2009-01-05", "2009-01-06", "2009-01-07", "2009-01-08", "2009-01-09",
    "2009-01-10", "2009-01-11", "2009-01-22", "2009-01-23", "2009-01-24",
    "2009-01-25", "2009-01-26", "2009-01-27", "2009-01-28",
    "2009-01-29", "2009-01-30", "2009-01-31", "2009-02-01", "2009-02-10", "2009-02-11",
    "2009-02-12", "2009-02-13", "2009-02-14", "2009-02-15", "2009-02-16", "2009-02-17",
    "2009-02-18", "2009-02-19", "2009-02-28", "2009-03-01", "2009-03-02", "2009-03-03",
    "2009-03-04", "2009-03-05", "2009-03-06", "2009-05-11", "2009-05-12", "2009-09-21",
    "2009-09-22", "2009-09-23", "2009-09-24", "2009-09-25", "2009-09-26", "2009-09-27",
    "2009-09-28", "2009-09-29", "2009-09-30", "2009-10-01", "2009-10-14", "2009-10-15",
    "2009-10-16", "2009-11-22", "2009-12-11", "2010-01-04", "2010-01-23", "2010-01-24",
    "2010-02-01", "2010-02-02", "2010-02-03", "2010-02-04", "2010-02-05", "2010-02-06",
    "2010-02-07", "2010-02-08", "2010-02-09", "2010-02-10", "2010-02-11", "2010-02-20",
    "2010-02-21", "2010-02-22", "2010-02-23", "2010-02-24", "2010-02-25", "2010-02-26",
    "2010-02-27", "2010-02-28", "2010-03-01", "2010-03-14", "2010-03-15", "2010-03-24",
    "2010-03-25", "2010-03-26", "2010-03-27", "2010-03-28", "2010-03-29", "2010-03-30",
    "2010-03-31", "2010-06-16", "2010-06-27", "2010-06-28", "2010-07-05", "2010-07-07",
    "2010-07-08", "2010-07-09", "2010-07-10", "2010-11-30", "2010-12-28", "2010-12-29",
    "2010-12-30", "2010-12-31"
  )
failures <- gsub("-", "", failures)


for (  i in  failures ){
  wpd_get_dumps(ts = i, directory = "/data/wpd/failures")
  flist <- dir_ls("/data/wpd/failures")
  system("rsync /data/wpd/failures/* pm6:/data/wpd/todo/ -v --progress --ignore-existing")
  file_move(flist, "/data/wpd/todo/")
}