Description Details Author(s) Examples
Read tables chunk by chunk using a C++ backend and a simple R interface.
This package allows to read long text tables in chunks.
The objects of class "chunker" are the central elements of the chunkR package. These objects can store a data chunk and other information required for the process of reading a file in pieces. A chunker object is created with the chunker
function, that requires the path to a file, and other arguments, as the size of the chunk. Two basic methods are defined to manipulate the object:
- next_chunk
function to read the next chunk
- get_table
to retrieve the data
The functions get_completed
and get_colnames
allow to get the number of rows already read, and the column names of the table.
The program can read data frames (with column type detection) or matrices. The program reads each chunk with the next_chunk
function (that return TRUE), and makes it accessible via the get_table
function. After reading all the file, next_chunk
function returns FALSE and the get_table
function an empty data frame/matrix.
Leandro Roser
Maintainer: Leandro Roser <learoser@gmail.com>
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | data(iris)
# write iris as tab delimited file. Note that quote is set to FALSE
tmp_path <- file.path(tempdir(),"iris.txt")
write.table(iris, tmp_path, quote = FALSE)
#----------------------------------------------------------------#
#--- Reading a data frame with automatic column-type detection ---#
#----------------------------------------------------------------#
# create a 'chunker' object passing the path of the input file.
my_chunker_object <- chunker(tmp_path, chunksize = 30)
# read a chunk
next_chunk(my_chunker_object)
# get the chunk
get_table(my_chunker_object)
# read another chunk
next_chunk(my_chunker_object)
# get the number of lines already read
get_completed(my_chunker_object)
#--- read a csv file ---#
tmp_path_csv <- file.path(tempdir(),"iris.csv")
write.table(iris, tmp_path_csv, quote = FALSE, sep = ",")
# read the csv indicating the value of the sep parameter
my_chunker_object2 <- chunker(tmp_path_csv, chunksize = 30, sep = ",")
# the file can then be processed as with tab delimiters
# remove temporal file
file.remove(tmp_path_csv)
#-------------------------------------------------------#
#--- Reading a data frame using column types argument ---#
#-------------------------------------------------------#
## Four types can be passed : "character", "numeric" (aka "double"), "integer", "logical"
# create a 'chunker' object passing the path of the input file.
my_chunker_object3 <- chunker(tmp_path, chunksize = 120,
columns_classes = c("numeric", "numeric", "numeric","numeric", "character"))
# read a chunk
next_chunk(my_chunker_object3)
# get the chunk
get_table(my_chunker_object3)
# read another chunk
next_chunk(my_chunker_object3)
# get the number of lines already read
get_completed(my_chunker_object3)
#-------------------------#
#--- Reading a matrix ---#
#-------------------------#
my_chunker_object4 <- chunker(tmp_path, chunksize = 30, data_format= "matrix")
# store the chunk as a character matrix in R
this_data <- get_table(my_chunker_object4)
# The package provides a fast generic C++ function for conversion from
# matrix (any R type) to data frame
this_data_as_df2 <- matrix2df(this_data)
# remove temporal file
file.remove(tmp_path)
## Not run:
#----------------------------------#
#--- Example with a big table -----#
#----------------------------------#
### Example with a data frame
# create a large data frame, and write it in a temporal directory
tmp_path <- file.path(tempdir(),"big_table.txt")
out <- data.frame(numeric_data = runif(1000000),
character_data = sample(c("a", "t", "c", "g"), 1000000,
replace = TRUE),
integer_data = sample(1000000),
bool_data = sample(c(TRUE, FALSE), 1000000, replace = TRUE))
write.table(out, tmp_path, quote = FALSE)
# create a chunker object, reading in chunks of 10000 lines
my_chunker_object5 <- chunker(tmp_path, chunksize = 10000)
next_chunk(my_chunker_object5)
data <- get_table(my_chunker_object5)
# check classes
lapply(data,typeof)
file.remove(tmp_path)
### Example with a matrix
# create a large matrix, and write it in a temporal directory
my_table <- tempfile()
write.table(matrix(sample(c("a", "t", "c", "g"), 1000000, replace = TRUE),
100000, 1000), my_table, quote = FALSE)
# create a chunker object, reading in chunks of 10000 lines
my_chunker_object6 <- chunker(my_table, chunksize = 10000, data_format= "matrix")
# create a loop to read all the file and make something with it
lines <- 0
while(next_chunk(my_chunker_object6))
{
data <- get_table(my_chunker_object6)
# do something with data, e.g., convert to data frame first
data <- matrix2df(data)
lines <- lines + nrow(data)
cat("Processed ", lines, "lines\n")
}
# remove the temporal file
file.remove(my_table)
## End(Not run)
|
Add the following code to your website.
For more information on customizing the embed code, read Embedding Snippets.