R/motusConstants.R

In jbrzusto/motusServer: Operate a server to process data for http://motus.org

#' Constants for the motusServer package.
#'
#' API entry points:
#'

MOTUS_API_ENTRY_POINTS          = "http://motus.org/data/api/entrypoints.jsp"
MOTUS_API_REGISTER_TAG          = "https://motus.org/api/tag/register"
MOTUS_API_DEPLOY_TAG            = "https://motus.org/api/tag/deploy"
MOTUS_API_SEARCH_TAGS           = "https://motus.org/api/tags/search"
MOTUS_API_USER_VALIDATE         = "https://motus.org/api/user/validate"
MOTUS_API_REGISTER_RECEIVER     = "https://motus.org/api/receiver/register"
MOTUS_API_LIST_PROJECTS         = "https://motus.org/api/projects"
MOTUS_API_LIST_SENSORS          = "https://motus.org/api/receivers/list"
MOTUS_API_LIST_SENSOR_DEPS      = "https://motus.org/api/receiver/sensordeployments"
MOTUS_API_LIST_SPECIES          = "https://motus.org/api/species"

## central repo where we track changes to motus metadata
## this repo is cloned locally, and changes noticed when we cache motus metadata
## are checked in and pushed back; This requires a public/private keypair in the
## /home/XXX/.ssh folder on this server, where XXX is the user running the motusServers,
## and an entry in /home/XXX/.ssh/config for the host 'github-motus-metadata-history' there
## which indicates the ssh key to be used.
## e.g. /home/sg/.ssh/config might contain:
##
##    Host github-motus-metadata-history
##       HostName github.com
##       User git
##       IdentityFile ~/.ssh/id_dsa_github_motus_metadata_history
##       IdentitiesOnly yes

MOTUS_METADATA_HISTORY_REPO     = "git@github-motus-metadata-history:jbrzusto/motus-metadata-history"

# a list of field names which must be formatted as floats so that
# the motus API recognizes them correctly.  This means that if they
# happen to have integer values, a ".0" must be appended to the JSON
# field value.  We do this before sending any query.  This is
# only required due to motus using a crappy JSON parser.

MOTUS_FLOAT_FIELDS = c("tsStart", "tsStartAnticipated", "tsEnd", "regStart", "regEnd",
"offsetFreq", "period", "periodSD", "pulseLen", "param1", "param2",
"param3", "param4", "param5", "param6", "ts", "nomFreq", "deferTime", "lat", "lon", "elev")

## a regular expression for replacing values that need to be floats
## Note: only works for named scalar parameters; i.e. "XXXXX":00000

MOTUS_FLOAT_REGEX = sprintf("((%s):-?[0-9]+)([,}])",
                             paste(sprintf("\"%s\"", MOTUS_FLOAT_FIELDS), collapse="|"))

## format of date/time in logfiles

MOTUS_LOG_TIME_FORMAT = "%Y-%m-%dT%H-%M-%OS6"

## format of date/time in queue entries (file or folder names)

MOTUS_TIMESTAMP_FORMAT = "%Y-%m-%dT%H-%M-%OS6"

## format of date/time in SG filenames
MOTUS_SG_TIMESTAMP_FORMAT = "%Y-%m-%dT%H-%M-%OS4"

## regex to match a leading timestamp
MOTUS_LEADING_TIMESTAMP_REGEX = "^(?<timestamp>[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}\\.[0-9]{1,9})"

## regex to match a full filename for a compressed message
MOTUS_COMPRESSED_EMAIL_REGEX = paste0(MOTUS_LEADING_TIMESTAMP_REGEX, "\\.bz2$")

## separator used in queue filenames
MOTUS_QUEUE_SEP = ","

## regex to match queue items generated by motus package
MOTUS_QUEUEFILE_REGEX = paste0(MOTUS_LEADING_TIMESTAMP_REGEX, MOTUS_QUEUE_SEP, "(?<params>.*)$")

## "From" address for outgoing emails

MOTUS_OUTGOING_EMAIL_ADDRESS = "no-reply@motus.org"

## name of archive to hold "bad" files; this archive is stored in the folder
## for each top-level job; we append "NOAUTO" to the name to prevent it
## being treated as a downloaded file.

MOTUS_BADFILE_ARCHIVE = "bad_files.zip.NOAUTO"

## filesystem layout; dirs end in "/"

#'@export

MOTUS_PATH = list(
    ROOT             = "/sgm",                       ## folder hierarchy on NAS
    ROOT_LOCAL       = "/sgm_local",                 ## folder hierarchy on local HD
    BIN              = "/sgm_local/bin/",            ## executable scripts and programs (store on HD)
    CACHE            = "/sgm_local/",                ## recent results of large queries from motus.org (store on HD)
    CRYPTO           = "/sgm/crypto/",               ## public/private keypairs for ssh etc. by receivers
    DB_BACKUPS       = "/sgm/db_backups/",           ## folder on NAS for daily backups of sqlite databases on local HD
    DONE             = "/sgm/done/",                 ## folders for completed jobs
    ERRORS           = "/sgm/errors/",               ## save dumped call stacks of server errors
    FILE_REPO        = "/sgm/file_repo/",            ## as-is copies of files from all receivers; stored in SERNO/YYYY-MM-DD/ subfolders
    INCOMING         = "/sgm/incoming/",             ## files / dirs moved here are processed by server(); this is the external / asynchronous
                                                     ## access point to the processing queue
    LOGS             = "/sgm/logs/",                 ## processing logs
    METADATA_HISTORY = "/sgm/metadata_history/",     ## .git repo tracking metadata changes
    OUTBOX           = "/sgm/outbox/",               ## copies of all sent emails
    OUTBOX_EMBARGOED = "/sgm/outbox_embargoed/",     ## unsent outgoing emails
    PARAM_OVERRIDES  = "/sgm_local/paramOverrides.sqlite", ## DB with table of receiver-boot-session-specific overrides for the tag finder
    PRODUCTS         = "/sgm/products/",             ## generated products (e.g. hourly summary plots and datasets)
    PRIORITY         = "/sgm/priority/",             ## jobs created here run on a processServer dedicated to short, fast jobs; e.g. updating attached SGs
    PUB              = "/sgm/pub/",                  ## web-visible public content
    QUEUES           = "/sgm/queue/",                ## queues for processing items
    QUEUE0           = "/sgm/queue/0/",              ## queue 0 is watched for new entries by each of the processServers;
                                                     ## at most one of them will claim the new job.
    QUEUE1           = "/sgm/queue/1/",              ## queue 1..8 for processServers
    QUEUE2           = "/sgm/queue/2/",
    QUEUE3           = "/sgm/queue/3/",
    QUEUE4           = "/sgm/queue/4/",
    QUEUE5           = "/sgm/queue/5/",
    QUEUE6           = "/sgm/queue/6/",
    QUEUE7           = "/sgm/queue/7/",
    QUEUE8           = "/sgm/queue/8/",
    QUEUE101         = "/sgm/queue/101/",            ## priority job queues 101..104; for the processServer dedicated to short, fast jobs; e.g. updating attached SGs.
    QUEUE102         = "/sgm/queue/102/",
    QUEUE103         = "/sgm/queue/103/",
    QUEUE104         = "/sgm/queue/104/",
    RECV             = "/sgm/recv/",                 ## receiver databases
    RECVLOG          = "/sgm/recvlog/",              ## logfiles from receivers
    SERVER_DB        = "/sgm_local/server.sqlite",   ## the database used to record processing server activity

    SYNC             = "/sgm_local/sync/",           ## the sync server watches this directory for new empty files named
                                                     ## N:SERNO, where N is a tunnel port and SERNO is a serial number.  When
                                                     ## created, receiver SERNO is rsync'd to the local copy of its data files, and
                                                     ## any new data are processed.  Creation of files in the watch folder,
                                                     ## and mapping of local ports back to SGs is handled
                                                     ## by the server at sensorgnome.org

    TAGS             = "/sgm/tags/",                 ## ??
    TAG_PROJ         = "/sgm/tag_proj/",             ## sqlite databases for all tag projects
    TAGREG_CLEANUP   = "/sgm/tagregCleanup.R",       ## script to provided tag registration cleanups downstream from motus
    TEST_PRODUCTS    = "/sgm/test_products/",        ## products from testing jobs
    TEST_WWW         = structure("/sgm/www/test/",   ## File hierarchy (testing version) to be served by apache to authenticated motus users
                       perm = "g+rsx"),              ## listable/readable
    TMP              = "/sgm/tmp/",                  ## intermediate storage; persistent across reboots
    TRASH            = "/sgm/trash/",                ## files to be deleted once we know they've been processed

    UPLOADS          = "/sgm/uploads/",              ## folder where uploads ultimately go
    UPLOADS_PARTIAL  = structure("/sgm/uploads/partial/",       ## folder where uploads go as they arrive; must be writable by www-data
                       owner="sg:www-data",
                       perm="g+rws"),
    UPLOAD_TESTING   = "/sgm/UPLOAD_TESTING",        ## file whose presence indicates upload jobs are to be marked with "isTesting"

    USERAUTH         = "/sgm_local/user_auth.sqlite",## database of user authentication tokens

    WWW              = structure("/sgm/www/",        ## File hierarchy to be served by apache to authenticated motus users
                       perm = "g+rsx")               ## listable/readable
)

## main logfile name
MOTUS_MAINLOG_NAME = "mainlog.txt"

## default file mode for new files, folders:
MOTUS_DEFAULT_FILEMODE = "0750"

## compressed archives we can handle
MOTUS_ARCHIVE_SUFFIXES = c(
    "zip",
    "7z",
    "rar"
)

MOTUS_ARCHIVE_REGEX = paste0("(?i)\\.(?<suffix>",
                             paste(MOTUS_ARCHIVE_SUFFIXES, collapse="|"),
                             ")$")

## silly dir() can't handle perl-style regex, so make another for that
MOTUS_ARCHIVE_DIR_REGEX = paste0("\\.(", paste(MOTUS_ARCHIVE_SUFFIXES, collapse="|"), ")$")

## regex to match receiver serial numbers (adapted from sgFilenameRegex, which differs
## in not using the 'SG-' prefix).

MOTUS_SG_SERNO_REGEX = "(?i)(?<serno>SG-[0-9A-Z]{4}(?:RPi[123z]|BBBK|(BB[0-9][0-9A-Z]))[0-9A-Z]{4}(?:_[0-9])?)"

## regex to exactly match any receiver serial number
MOTUS_RECV_SERNO_REGEX = "(?i)^(?:(?:(?:SG-[0-9A-Z]{4}(?:RPi[123z]|BBBK|(?:BB[0-9][0-9A-Z]))[0-9A-Z]{4}(?:_[0-9])?))|(?:Lotek-D?[0-9]+(?:_[0-9])?))$"

## regex for matching DOS filenames (names of SG data files which have
## been shortened to 8.3 form)
## see https://en.wikipedia.org/wiki/8.3_filename#VFAT_and_Computer-generated_8.3_filenames

MOTUS_DOS_FILENAME_REGEX = "^[a-zA-Z0-9]+~[0-9].(GZ|TXT)"

## Make the following symbols available outside the package

#'@export

## the unfortunate individual who receives emails when things need attention
## we use a local address, and rely on the forwarding in /home/sg/.forward
## to move it to real users.
MOTUS_ADMIN_EMAIL = "sg@sgdata.motus.org"

## the table used to record locks on arbitrary symbols
MOTUS_SYMBOLIC_LOCK_TABLE = "symLocks"

## regex for files we want to delete.
## Just about every OS these days writes indexing info to
## flash drives, and we often get these files in a transfer.

MOTUS_JUNKFILE_REGEX = "(__MACOSX/|System Volume Information/|.DS_Store|._.DS_Store|.Trashes|._.Trashes)"

## the earliest valid date from a sensorgnome (= as.numeric(ymd("2010-01-01")))
MOTUS_SG_EPOCH = 1262304000

## regular expression for recognizing a tag registration manifest file
## which should be matched case-insensitively
MOTUS_TAGREG_MANIFEST_REGEXP = ".*tagreg.*\\.txt$"

## sprintf-format string for URL of downloadable files; %s is for projectID or other special values
MOTUS_DOWNLOAD_URL_FMT = "https://sgdata.motus.org/download/%s"

## sprintf-format string for URL of downloadable files; %s is for projectID or other special values
MOTUS_TEST_DOWNLOAD_URL_FMT = "https://sgdata.motus.org/download/test/%d"

## location of secret key for mod_auth_tkt authorization used by some
## of the XXXServer() functions in this package
MOTUS_MODAUTHTKT_SECRET_KEYFILE = "/etc/apache2/TKTAuthSecret.inc"

## path to motus metadata db cache
MOTUS_METADB_CACHE = file.path(MOTUS_PATH$CACHE, "motus_meta_db.sqlite")

## authentication options

## lifetime of authorization token in seconds; default: 3 days
MOTUS_AUTH_LIFE = 3 * 24 * 3600

## number of random bits in authorization token;
## gets rounded up to nearest multiple of 8
MOTUS_TOKEN_BITS = 33 * 8