Files in tok
Fast Text Tokenization
MD5
NEWS.md
README.md
LICENSE.note
NAMESPACE
DESCRIPTION
LICENSE
R/models.R
R/extendr-wrappers.R
R/encoding.R
R/pre_tokenizers.R
R/decoders.R
R/trainers.R
R/normalizers.R
R/post_processors.R
R/tokenizer.R
src/Makevars.win
src/Makevars
src/entrypoint.c
src/Makevars.ucrt
src/tok-win.def
src/rust/Cargo.toml
src/rust/vendor.tar.xz
src/rust/Cargo.lock
src/rust/src/tokenizer.rs
src/rust/src/post_processors.rs
src/rust/src/normalizers.rs
src/rust/src/pre_tokenizers.rs
src/rust/src/decoders.rs
src/rust/src/trainers.rs
src/rust/src/models.rs
src/rust/src/lib.rs
src/rust/config/cargo-config.toml
inst/AUTHORS
tests/testthat.R
tests/testthat/test-post_processors.R
tests/testthat/test-decoders.R
tests/testthat/test-tokenizer.R
tests/testthat/test-models.R
tests/testthat/test-encoding.R
tests/testthat/test-normalizers.R
tests/testthat/test-pre_tokenizers.R
tests/testthat/test-trainers.R
tests/testthat/assets/tokenizer.json
tools/vendor.sh
tools/patch.R
tools/update_authors.R
man/trainer_bpe.Rd
man/processor_byte_level.Rd
man/encoding.Rd
man/tok_decoder.Rd
man/model_wordpiece.Rd
man/pre_tokenizer.Rd
man/model_bpe.Rd
man/trainer_wordpiece.Rd
man/pre_tokenizer_whitespace.Rd
man/model_unigram.Rd
man/trainer_unigram.Rd
man/pre_tokenizer_byte_level.Rd
man/tok_trainer.Rd
man/normalizer_nfkc.Rd
man/normalizer_nfc.Rd
man/tok_model.Rd
man/tok_processor.Rd
man/decoder_byte_level.Rd
man/tok-package.Rd
man/tokenizer.Rd
man/tok_normalizer.Rd
