In yuliasidi/condor: HTCondor Cluster usage with R

knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>",
  eval = FALSE
)

Load Libraries

library(condor)
library(ssh)

Preprocessing

Populate package template and create Rcalcpi.condor in example subdir.

condor::build_template(
  file = 'calcpi.R',
  args = c('$(Process)'),
  tag = 'pi',
  jobs = 5,
  init_dir = 'jobs/run',
  template_file = 'example/Rcalcpi.condor',
  job_type = 'test')

Lines in file that will be run on the cluster

readLines(system.file('example/calcpi.R',package='condor'))

Lines in the populated condor file

readLines(system.file('example/Rcalcpi.condor',package='condor'))

Connect to ssh

session <- ssh::ssh_connect(Sys.getenv('UCONN_USER'))

Upload files needed for the job to the cluster

ssh::scp_upload(session,
                files = c('example/calcpi.R',
                          'example/Rcalcpi.condor', 
                          'example/emailMyself.txt'),
                to = '~'
)

Create directories needed for job outputs

condor::create_dirs(session, file = 'example/Rcalcpi.condor')

Submit the jobs

condor::condor_submit(session,'Rcalcpi.condor')

During the job

condor::condor_q(session)

condor::condor_rm(session,'5000.1')

Post Processing

Retrieve the files

condor::pull(session,
             from = c('jobs/run/log',
                      'jobs/run/out',
                      'jobs/run/err',
                      'jobs/run/*.rds'),
             to = c('output',
                    'output',
                    'output',
                    'output/data'))

Remove files from the cluster

condor::cleanup_remote(session)

Collect output from the local into a single object

RETURN <- purrr::map(
  list.files(path = "example/output/data",full.names = TRUE),
  readRDS
  )

Remove files from the local machine

condor::cleanup_local(dir = 'example/output',tag = 'pi')

Close ssh connection

ssh::ssh_disconnect(session)

yuliasidi/condor documentation built on May 17, 2019, 2:15 a.m.