R/apply_by_pages.R

In jsonlite: A Simple and Robust JSON Parser and Generator for R

apply_by_pages <- function(x, FUN, pagesize, verbose, ...) {
  stopifnot(is.data.frame(x))
  nr <- nrow(x)
  npages <- nr %/% pagesize
  lastpage <- nr %% pagesize

  out <- as.list(rep(NA, npages + as.logical(lastpage)))
  for (i in seq_len(npages)) {
    from <- pagesize * (i - 1) + 1
    to <- pagesize * i
    out[[i]] <- FUN(x[from:to, , drop = FALSE], ...)
    if (verbose) cat("\rProcessed", i * pagesize, "rows...")
  }

  if (lastpage) {
    from <- nr - lastpage + 1
    out[[npages + 1]] <- FUN(x[from:nr, , drop = FALSE], ...)
  }
  if (verbose) cat("\rComplete! Processed total of", nr, "rows.\n")
  out
}

#this is another slightly slower implementation
apply_by_pages2 <- function(x, FUN, pagesize, verbose, ...) {
  x2 <- split(x, seq_len(nrow(x)) %/% pagesize)
  for (page in x2) {
    if (verbose) message("Writing ", nrow(page), " lines (", ").")
    FUN(page)
  }
  invisible()
}

#' @export
`[.json` <- function(x, i) {
  structure(NextMethod("["), class = c("json", "character"))
}