inst/examples/generate-tokens.R

In wordpredictor: Develop Text Prediction Models Based on N-Grams

# Start of environment setup code
# The level of detail in the information messages
ve <- 2
# The name of the folder that will contain all the files. It will be created in
# the current directory. NULL implies tempdir will be used
fn <- NULL
# The required files. They are default files that are part of the package
rf <- c("test-clean.txt")
# An object of class EnvManager is created
em <- EnvManager$new(ve = ve, rp = "./")
# The required files are downloaded
ed <- em$setup_env(rf, fn)
# End of environment setup code

# The n-gram size
n <- 4
# The test file name
tfn <- paste0(ed, "/test-clean.txt")
# The ngram number is set
tg_opts <- list("n" = n, "save_ngrams" = TRUE, "dir" = ed)
# The TokenGenerator object is created
tg <- TokenGenerator$new(tfn, tg_opts, ve = ve)
# The ngram tokens are generated
tg$generate_tokens()

# The test environment is removed. Comment the below line, so the files
# generated by the function can be viewed
em$td_env()