#' Download and save to disk historical Bid, Ask, and Trades data
#'
#' This will make several requests for historical data from IBrokers
#' repecting the historical data request limitations.
#'
#' Experimental code -- see Note section
#'
#' Important: You should have a base directory that contains these
#' sub-directories: \sQuote{BID}, \sQuote{ASK}, \sQuote{TRADES}, and
#' \sQuote{BAT}.
#'
#' The default \code{base_dir} is \dQuote{/mnt/W}. If you do not have a
#' directory there, you should either create
#' it, or not use the default value for \code{base_dir}.
#'
#' \code{reqTBBOhistory} will make requests for 5 days of bid, ask, and trade
#' data and then pause for 30 seconds. In between requests, all BID, ASK, and
#' TRADES data are saved into the respective subdirectories. Data are saved in
#' rda files named with the format YYYY.MM.DD.Symbol.rda. After all requests
#' have been made (approx. 37 minutes), the data are re-read and merged together
#' into BATMV format and (if \code{save=TRUE}) stored in the BAT directory
#'
#' IBrokers does not disseminate TRADES data for FX (CASH). Therefore,
#' \code{reqTBBOhistory} will make 2 requests every 20 seconds
#' (BID and ASK data) instead of 3 requests every 30 seconds.
#'
#' For best results, you should define your instruments before calling this function.
#'
#' This has only been tested on a debian-based linux system.
#'
#' IB limits historical data requests to 6 every 60 seconds
#'
#' @param Symbols names of instruments for which to request data
#' @param base_dir base_dir that contains sub-directories \sQuote{BID},
#' \sQuote{ASK}, \sQuote{TRADES}, and \sQuote{BAT}
#' @param ndays total number of days to retrieve. Default is 95. Max is 365
#' @param endDateTime \sQuote{YYYYMMDD H:M:S}
#' @param tws twsConnection object (Not yet implemented)
#' @param barSize barSize to retrieve. see \code{\link[IBrokers]{reqHistoricalData}}
#' @param useRTH Use regular trading hours? 1 for TRUE, 0 for FALSE
#' @param env where to put the final BATVM or BAM xts object
#' @param save logical. Save to disk the BATVM or BAM object?
#' @param chronological should the requests be made in chronological order.
#' Default is FALSE because, at least for futures and options, requests for old data are more
#' likely to fail than requests for recent data. For stocks, it might make sense
#' to use TRUE so that if a request is interrupted, you will not have gaps in your data.
#' \code{update.data} uses \code{chronological=TRUE}
#' @return called for side-effect. Returns the names of Symbols.
#' @author Garrett See
#' @note Warning: Interactive Brokers *back adjusts* their data for stock splits (but not for dividends).
#' If you are storing stock data, you should probably unadjust the last year's worth of data, then,
#' make sure you download the most recent data each day, being careful not to overwrite any data already
#' stored on disk. This way you could have all unadjusted data that you could adjust (with
#' qmao:::adjustIntraday.yahoo, for example). Otherwise, you'll have some data that is
#' split-adjusted (or partially split-adjusted) and some that isn't.
#' @seealso \code{twsInstrument:::update.data} (unexported due to possibility of name change, and current lack of documentation),
#'
#' \code{\link{getBAT}}, \code{\link{makeBATs}}, \code{\link[IBrokers]{reqHistoricalData}}, \code{\link[IBrokers]{reqHistory}}
#' @references InteractiveBrokers \url{www.interactivebrokers.com}
#'
#' IB API \url{http://interactivebrokers.com/php/apiUsersGuide/apiguide.htm}
#'
#' IB Historic Data Limitations
#' \url{http://individuals.interactivebrokers.com/php/apiUsersGuide/apiguide/api/historical_data_limitations.htm#XREF_93621_Historical_Data}
#'
#' @examples
#' \dontrun{
#' library(twsInstrument)
#' dir.create("tmpdata/BID", recursive=TRUE)
#' dir.create("tmpdata/ASK")
#' dir.create("tmpdata/TRADES")
#' dir.create("tmpdata/BAT")
#' define_stocks("SPY")
#' # get the last 20 days of minutely data
#' # (this will take ~2 minutes)
#' reqTBBOhistory("SPY",
#' base_dir="tmpdata",
#' ndays=20,
#' save=TRUE,
#' chronological=TRUE)
#' # Now to prove it's on disk, delete from workspace
#' # and load it.
#' rm("SPY")
#' getSymbols("SPY", src='FI', dir='tmpdata/BAT',
#' from=Sys.Date() - 20, verbose=TRUE)
#'
#' # If you had called reqTBBOhistory with save=FALSE (the default)
#' # or if your "BAT" directory were wiped out, you can use
#' # the unexported makeBATs function to get the BID, ASK, TRADES
#' # data from disk and rebuild the BAT data.
#' unlink("tmpdata/BAT")
#' twsInstrument:::makeBATs("SPY", base_dir='tmpdata', ndays=20)
#' rm("SPY")
#' getSymbols("SPY", src='FI', dir='tmpdata/BAT',
#' from=Sys.Date() - 20, verbose=TRUE)
#'
#' # Clean up -- Delete everything
#' unlink("tmpdata", recursive=TRUE)
#' }
#' @export
reqTBBOhistory <-
function(Symbols, base_dir='/mnt/W', ndays=95,
endDateTime, tws, barSize='1 min', useRTH='1',
env=.GlobalEnv,save=FALSE, chronological=FALSE) {
if (missing(endDateTime) || is.null(endDateTime)) {
autoDate <- TRUE
} else {
autoDate <- FALSE
endDate <- if (is.character(endDateTime)) {
as.Date(endDateTime, format="%Y%m%d")
} else endDateTime
}
symout <- NULL
lastCompletedHistoricalRequestTime <- Sys.time()
slptime <- 0.1
for (Symbol in Symbols) {
#contract <- Contr_From_Instr(Symbol, verbose = FALSE)
#instr <- Instr_From_Contr(Symbol, verbose = FALSE)
instr <- twsInstrument(Symbol,output="instrument",assign_i=FALSE,verbose=FALSE)
contract <- instr$IB
symbol <- instr$primary_id
symout <- c(symout, symbol)
if (autoDate) {
if (contract$expiry == "" || is.null(contract$expiry)) {
endDate <- Sys.Date()
} else if (as.Date(contract$expiry, format="%Y%m%d") < Sys.Date()) {
endDate <- as.Date(contract$expiry, format="%Y%m%d")
} else endDate <- Sys.Date()
}
if (contract$expiry != "")
endDate <- min(as.Date(contract$expiry, format="%Y%m%d"), endDate)
Ndays <- max(0,ndays[1] - 5 )
if (!is.null(instr$first_traded)) {
tmpfirst <- instr$first_traded
if (is.character(tmpfirst)) tmpfirst <- as.Date(tmpfirst, format="%Y-%m-%d")
Ndays <- min(Ndays,as.numeric(endDate-tmpfirst))
}
if (barSize == "1 min") {
if (chronological) {
endDateTime <- endDate - seq(Ndays, 0, -5)
} else endDateTime <- endDate - seq(0, Ndays, 5)
duration <- "5 D"
}
else if (barSize == "15 mins") {
if (chronological) {
endDateTime <- endDate - seq(Ndays, 0, -10)
} else endDateTime <- endDate - seq(0, Ndays, 10)
duration <- "10 D"
}
if (missing(tws) || is.null(tws) ||
(is.twsConnection(tws) && !isConnected(tws)))
tws <- try(ConnectIB(c(120:122, 150)))
if (isConnected(tws)) {
cat(paste("Connected with clientId ", tws$clientId,
".\n Requesting ", Symbol, "\n", sep = ""))
if (tws$clientId == 150) warning("IB TWS should be restarted.")
} else stop("Could not connect to IB.")
tryCatch( {
if (substr(base_dir,nchar(base_dir),nchar(base_dir)) != "/") base_dir <- paste(base_dir, "/",sep="")
bidenv <- new.env()
askenv <- new.env()
trdenv <- new.env()
if (contract$exch == "IDEALPRO")
hastrades <- FALSE
else hastrades <- TRUE
if (contract$sectype == "IND")
hasbidask <- FALSE
else hasbidask <- TRUE
domakeBATs <- TRUE
slptime <- 0.1
for (i in 1:length(endDateTime)) {
st <- strptime(paste(endDateTime[i],"23:59:59"),"%Y-%m-%d %H:%M:%S")
ceDT <- paste(format(st,"%Y%m%d %H:%M:%S"))
cat(paste("Request ",i, " of ", length(endDateTime), ": request end Date: ", format(st,"%Y-%m-%d"),"\n",sep="")) #debug
if (hasbidask) {
assign(symbol, reqHistoricalData(tws, contract, barSize=barSize,
duration=duration, endDateTime=ceDT,
useRTH=useRTH,whatToShow='BID'),pos=bidenv)
assign(symbol, reqHistoricalData(tws, contract, barSize=barSize,
duration=duration, endDateTime=ceDT,
useRTH=useRTH,whatToShow='ASK'),pos=askenv)
slptime <- slptime + 20
}
if (hastrades) {
assign(symbol, reqHistoricalData(tws, contract, barSize=barSize,
duration=duration, endDateTime=ceDT,
useRTH=useRTH,whatToShow='TRADES'),pos=trdenv)
if (!hasbidask) assign(symbol, trdenv[[symbol]], env)
slptime <- slptime + 10
}
lastCompletedHistoricalRequestTime <- Sys.time()
if (slptime < 20) domakeBATs <- FALSE #if there is only Trade data (IND/synthetic)
if (!is.null(bidenv[[symbol]])) saveSymbols.days(Symbols=symbol,base_dir=paste(base_dir,'BID/',sep=""), env=bidenv)
if (!is.null(askenv[[symbol]])) saveSymbols.days(Symbols=symbol,base_dir=paste(base_dir,'ASK/',sep=""), env=askenv)
if (!is.null(trdenv[[symbol]])) saveSymbols.days(Symbols=symbol,base_dir=paste(base_dir,'TRADES/',sep=""), env=trdenv)
#uncomment if you don't want to sleep after last request.
#(if you don't sleep, then repeated calls to this function will result in pacing violation)
if ((ndays > 5 && i < length(endDateTime)) || (Symbol != Symbols[length(Symbols)])) {
cat("Pausing for", round(slptime,0), "seconds to avoid pacing violation...\n")
Sys.sleep(slptime)
slptime <- 0.1
}
}
},finally={twsDisconnect(tws)} )
}
# tmp <- reqHistoricalData(tws, contract, barSize = barSize, duration = duration,
# endDateTime = endDateTime, useRTH=useRTH)
# assign(symbol, tmp, pos=tmpenv)
# saveSymbols.days(Symbol=symbol, base_dir=base_dir,env=tmpenv)
if (domakeBATs) makeBATs(symout,base_dir,env=env,ndays=ndays)
if (save) saveSymbols.days(symout,base_dir=paste(base_dir,"BAT",sep=""),env=env)
# Determine if sleep is needed
if ((as.numeric(Sys.time()) -
as.numeric(lastCompletedHistoricalRequestTime)) < slptime) {
slptime <- slptime - (as.numeric(Sys.time()) -
as.numeric(lastCompletedHistoricalRequestTime))
if (slptime > 0) {
cat("Pausing for", round(slptime,0), "seconds to avoid pacing violation...\n")
Sys.sleep(slptime)
}
}
symout
}
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.