R/development/review/split_data_table.R

In bfatemi/ninjar: Data Science Utility Belt

library(data.table)

nodes <- 4
cores <- 12
core_rows <- 10000

each_n_rows <- cores*core_rows*nodes
total_len   <- 13432840#nrow(cnDAT$data)
subset_len  <- ceiling(total_len/each_n_rows)


DT <- data.table(V1 = rep(NA_integer_, each_n_rows))
alloc.col(DT, subset_len - 1)
DT[, (paste0("V", 2:subset_len)):=NA_integer_]


for(i in 1:ncol(DT)){
  DT[, i] <- subset_len*(i-1) + 1:each_n_rows
}


floor(subset_len/4) + (subset_len+1) %% 4