R/update_prices.R

In causality-loop/updateprices: updateprices

.datatable.aware = TRUE 
if (getRversion() >= '2.15.1') 
  utils::globalVariables(c('.'), utils::packageName()) 

#' @title update_prices
#' @description Fetches and manages a collection of Yahoo OHLCV data.
#' @param asset_symbols a character vector of asset symbols
#' @param append_asset_symbols boolean, if TRUE, OHLCV series specified in `asset_symbols` will be added to those already downloaded (see *Note*), Default: TRUE
#' @param include_index_symbols a character vector indicating specific indexes whose components will be added to `asset_symbols`, Default: ''
#' @details The idea behind this function is to continually update a store of cached price data on a VPS.
#' On the server, run `update_prices` once to populate the `prices` directory, then set the function to run via a cronjob.
#' It would be optimal to run daily updates pre- or post-market, and at a time which would not conflict with server maintenance reboots (if scheduled).
#' The OHLCV data is downloaded and stored in `./prices`.
#' If existing symbols are again fed to `asset_symbols` and the function is rerun, only the most recent data will be added to the cached OHLCV data in `./prices`.
#' If a symbol is not included in `asset_symbols` and `update_prices` is run, its OHLCV data will be removed from `./prices`, unless `append_asset_symbols = TRUE`.
#' If `asset_symbols` is left blank, OHLCV already downloaded will be updated.
#' Optionally, components for an index can also be included (see *Examples*).
#' Each time a fetch attempt is made, a record is stored in `./log/fetch_attempts.rds`. Likewise, when a symbol fails to download, a record is stored in `./log/fetch_errors.rds`.
#' @note IMPORTANT: Without `append = TRUE`, the OHLCV series in the `prices` directory will be completely replaced by the symbols indicated in `asset_symbols`
#' @examples 
#' \dontrun{
#' if(interactive()){
#'  ## run through each of the below examples in order
#'  ## download OHLCV series for each major index ETF
#'  update_prices(asset_symbols = c('SPY', 'QQQ', 'DIA', 'IWM'))
#'  ## remove IWM OHLCV data from ./prices
#'  update_prices(asset_symbols = c('SPY', 'QQQ', 'DIA'))
#'  ## add IWM again and update prices for SPY, QQQ, DIA
#'  ## note that, unlike above, this doesn't remove OHLCV previously downloaded
#'  update_prices(asset_symbols = 'IWM', append_asset_symbols = TRUE)
#'  ## update prices for all symbols
#'  update_prices()
#'  ## update prices for all symbols and add all Dow stocks
#'  update_prices(include_index_symbols = 'DOW')
#'  ## remove the Dow stocks
#'  update_prices(asset_symbols = c('SPY', 'QQQ', 'DIA', 'IWM'))
#'  }
#' }
#' @export 
#' @importFrom magrittr '%<>%' 
#' @importFrom tidyquant tq_index
#' @importFrom data.table data.table
#' @importFrom quantmod getSymbols
#' @importFrom zoo index
#' @importFrom xts last
update_prices <- function(asset_symbols, 
                          append_asset_symbols = TRUE,
                          include_index_symbols = '')
{

  if (!dir.exists('log')) dir.create('log', mode = '0744')
  if (file.exists('log/fetch_errors.rds'))
    cat('\nLog contains fetch errors; please inspect\n\n')
  if (!dir.exists('prices')) dir.create('prices', mode = '0744')
  existing_symbols <- list.files('prices')

  if (missing(asset_symbols) & length(existing_symbols) == 0) {
    stop('The argument asset_symbols is missing')
  } else if (missing(asset_symbols)) {
    asset_symbols <- existing_symbols
    append_asset_symbols <- FALSE
  }

  if (append_asset_symbols & length(existing_symbols) != 0) 
    asset_symbols %<>% c(existing_symbols)

  ok_idx_symbols <- c('DOW', 'DOWGLOBAL', 'SP400', 'SP500', 'SP600')
  if (include_index_symbols %in% ok_idx_symbols) {
    idx_symbols <- sort(suppressMessages(
      tidyquant::tq_index(include_index_symbols)$symbol))
    asset_symbols %<>% c(idx_symbols)
  } else if (include_index_symbols != '') {
    stop('Indexes include DOW, DOWGLOBAL, SP400, SP500, SP600')
  }
  
  asset_symbols %<>% unique %>% sort
  new_symbols <- asset_symbols %>% .[. %ni% existing_symbols]
  rm_symbols <- existing_symbols %>% .[. %ni% asset_symbols]

  make_fetch_error_log_entry <- function(symbol) {
    log_entry <- data.table::data.table(DT = Sys.time(), Symbol = symbol)

    if (file.exists('log/fetch_errors.rds')) {
      old_log <- readRDS('log/fetch_errors.rds')
      log_entry %<>% rbind(old_log)
    }

    saveRDS(log_entry, 'log/fetch_errors.rds', compress = 'gzip')
  }

  for (i in rm_symbols) file.remove(file.path('prices', i))

  for (i in existing_symbols) {
    ohlcv_old <- readRDS(file.path('prices', i))
    last_date <- xts::last((zoo::index(ohlcv_old)))
    distance <- as.numeric(Sys.Date() - last_date)
    # if distance=0, still update because prices can change over the course of the trading day
    distance[distance < 5] <- 5
    # to=Sys.Date+1 to avoid today being cut off
    x <- suppressWarnings(try(quantmod::getSymbols(
      i, env = NULL, from = Sys.Date()-distance, to = Sys.Date()+1)))

    if (exists('x')) {
      out <- rbind(ohlcv_old, x) %>% 
        .[!base::duplicated(zoo::index(.), fromLast = TRUE), ]
      saveRDS(out, file.path('prices', i), compress = 'gzip')
    } else {
      make_fetch_error_log_entry(i)
    }
  }

  for (i in new_symbols) {
    x <- suppressWarnings(try(quantmod::getSymbols(
      i, env = NULL, from = '1900-01-01', to = Sys.Date()+1)))

    if (exists('x')) saveRDS(x, file.path('prices', i), compress = 'gzip')
    else make_fetch_error_log_entry(i)
  }

  log_entry <- data.table::data.table(DT = Sys.time(), Status = 'Complete')

  if (file.exists('log/fetch_attempts.rds')) {
    old_log <- readRDS('log/fetch_attempts.rds')
    log_entry %<>% rbind(old_log)
  }

  saveRDS(log_entry, 'log/fetch_attempts.rds', compress = 'gzip')

}