Files in sentencepiece
Text Tokenization using Byte Pair Encoding and Unigram Modelling

MD5

NEWS.md README.md

LICENSE.note

NAMESPACE

DESCRIPTION

R/RcppExports.R R/wordpiece.R R/sentencepiece.R R/bpemb.R R/pkg.R R/word2vec.R R/AAA.R R/utils.R

src/Makevars

src/RcppExports.cpp

src/rcpp_sentencepiece.cpp

src/rcpp_wordpiece.cpp

src/config.h

src/sentencepiece/src/unicode_script.cc

src/sentencepiece/src/init_test.cc

src/sentencepiece/src/model_factory_test.cc

src/sentencepiece/src/bpe_model_test.cc

src/sentencepiece/src/trainer_interface_test.cc

src/sentencepiece/src/char_model.h

src/sentencepiece/src/unigram_model.cc

src/sentencepiece/src/common.h

src/sentencepiece/src/normalizer.cc

src/sentencepiece/src/normalization_rule.h

src/sentencepiece/src/char_model_test.cc

src/sentencepiece/src/spm_decode_main.cc

src/sentencepiece/src/spm_train_main.cc

src/sentencepiece/src/unigram_model_test.cc

src/sentencepiece/src/char_model_trainer.h

src/sentencepiece/src/util.h

src/sentencepiece/src/unigram_model.h

src/sentencepiece/src/filesystem.h

src/sentencepiece/src/model_interface.h

src/sentencepiece/src/unicode_script_map.h

src/sentencepiece/src/word_model_trainer_test.cc

src/sentencepiece/src/freelist_test.cc

src/sentencepiece/src/compile_charsmap_main.cc

src/sentencepiece/src/bpe_model.cc

src/sentencepiece/src/filesystem.cc

src/sentencepiece/src/normalizer_test.cc

src/sentencepiece/src/model_factory.cc

src/sentencepiece/src/util.cc

src/sentencepiece/src/init.h

src/sentencepiece/src/unigram_model_trainer.h

src/sentencepiece/src/error.cc

src/sentencepiece/src/trainer_factory.cc

src/sentencepiece/src/builder_test.cc

src/sentencepiece/src/char_model_trainer_test.cc

src/sentencepiece/src/sentencepiece_processor_test.cc

src/sentencepiece/src/normalizer.h

src/sentencepiece/src/sentencepiece_model.proto

src/sentencepiece/src/testharness.h

src/sentencepiece/src/sentencepiece_trainer.cc

src/sentencepiece/src/sentencepiece_trainer.h

src/sentencepiece/src/test_main.cc

src/sentencepiece/src/pretokenizer_for_training_test.cc

src/sentencepiece/src/unigram_model_trainer.cc

src/sentencepiece/src/word_model_trainer.cc

src/sentencepiece/src/model_interface_test.cc

src/sentencepiece/src/sentencepiece_processor.cc

src/sentencepiece/src/trainer_factory_test.cc

src/sentencepiece/src/trainer_interface.h

src/sentencepiece/src/builder.h

src/sentencepiece/src/spm_normalize_main.cc

src/sentencepiece/src/spm_encode_main.cc

src/sentencepiece/src/trainer_factory.h

src/sentencepiece/src/CMakeLists.txt

src/sentencepiece/src/char_model_trainer.cc

src/sentencepiece/src/spec_parser.h

src/sentencepiece/src/unicode_script_test.cc

src/sentencepiece/src/word_model_trainer.h

src/sentencepiece/src/sentencepiece_trainer_test.cc

src/sentencepiece/src/builder.cc

src/sentencepiece/src/sentencepiece.proto

src/sentencepiece/src/unigram_model_trainer_test.cc

src/sentencepiece/src/word_model_test.cc

src/sentencepiece/src/pretokenizer_for_training.cc

src/sentencepiece/src/freelist.h

src/sentencepiece/src/bpe_model_trainer_test.cc

src/sentencepiece/src/bpe_model_trainer.cc

src/sentencepiece/src/pretokenizer_for_training.h

src/sentencepiece/src/model_interface.cc

src/sentencepiece/src/bpe_model.h

src/sentencepiece/src/char_model.cc

src/sentencepiece/src/sentencepiece_processor.h

src/sentencepiece/src/filesystem_test.cc

src/sentencepiece/src/util_test.cc

src/sentencepiece/src/testharness.cc

src/sentencepiece/src/unicode_script.h

src/sentencepiece/src/word_model.h

src/sentencepiece/src/bpe_model_trainer.h

src/sentencepiece/src/model_factory.h

src/sentencepiece/src/word_model.cc

src/sentencepiece/src/spm_export_vocab_main.cc

src/sentencepiece/src/trainer_interface.cc

src/sentencepiece/src/builtin_pb/sentencepiece.pb.cc

src/sentencepiece/src/builtin_pb/sentencepiece_model.pb.cc

src/sentencepiece/src/builtin_pb/sentencepiece_model.pb.h

src/sentencepiece/src/builtin_pb/sentencepiece.pb.h

src/third_party/CMakeLists.txt

src/third_party/darts_clone/darts.h

src/third_party/protobuf-lite/coded_stream.cc

src/third_party/protobuf-lite/bytestream.cc

src/third_party/protobuf-lite/message_lite.cc

src/third_party/protobuf-lite/strutil.cc

src/third_party/protobuf-lite/common.cc

src/third_party/protobuf-lite/implicit_weak_message.cc

src/third_party/protobuf-lite/repeated_field.cc

src/third_party/protobuf-lite/stringprintf.cc

src/third_party/protobuf-lite/time.cc

src/third_party/protobuf-lite/statusor.cc

src/third_party/protobuf-lite/stringpiece.cc

src/third_party/protobuf-lite/zero_copy_stream_impl_lite.cc

src/third_party/protobuf-lite/generated_enum_util.cc

src/third_party/protobuf-lite/zero_copy_stream_impl.cc

src/third_party/protobuf-lite/io_win32.cc

src/third_party/protobuf-lite/arenastring.cc

src/third_party/protobuf-lite/generated_message_util.cc

src/third_party/protobuf-lite/arena.cc

src/third_party/protobuf-lite/status.cc

src/third_party/protobuf-lite/parse_context.cc

src/third_party/protobuf-lite/int128.cc

src/third_party/protobuf-lite/structurally_valid.cc

src/third_party/protobuf-lite/zero_copy_stream.cc

src/third_party/protobuf-lite/wire_format_lite.cc

src/third_party/protobuf-lite/generated_message_table_driven_lite.cc

src/third_party/protobuf-lite/extension_set.cc

src/third_party/protobuf-lite/google/protobuf/port_undef.inc

src/third_party/protobuf-lite/google/protobuf/port.h

src/third_party/protobuf-lite/google/protobuf/parse_context.h

src/third_party/protobuf-lite/google/protobuf/descriptor.h

src/third_party/protobuf-lite/google/protobuf/generated_enum_reflection.h

src/third_party/protobuf-lite/google/protobuf/message_lite.h

src/third_party/protobuf-lite/google/protobuf/wire_format_lite.h

src/third_party/protobuf-lite/google/protobuf/any.h

src/third_party/protobuf-lite/google/protobuf/arena_impl.h

src/third_party/protobuf-lite/google/protobuf/arenastring.h

src/third_party/protobuf-lite/google/protobuf/port_def.inc

src/third_party/protobuf-lite/google/protobuf/generated_message_table_driven_lite.h

src/third_party/protobuf-lite/google/protobuf/extension_set.h

src/third_party/protobuf-lite/google/protobuf/map.h

src/third_party/protobuf-lite/google/protobuf/map_field_lite.h

src/third_party/protobuf-lite/google/protobuf/extension_set_inl.h

src/third_party/protobuf-lite/google/protobuf/metadata_lite.h

src/third_party/protobuf-lite/google/protobuf/arena.h

src/third_party/protobuf-lite/google/protobuf/repeated_field.h

src/third_party/protobuf-lite/google/protobuf/has_bits.h

src/third_party/protobuf-lite/google/protobuf/implicit_weak_message.h

src/third_party/protobuf-lite/google/protobuf/generated_message_table_driven.h

src/third_party/protobuf-lite/google/protobuf/generated_message_util.h

src/third_party/protobuf-lite/google/protobuf/map_entry_lite.h

src/third_party/protobuf-lite/google/protobuf/map_type_handler.h

src/third_party/protobuf-lite/google/protobuf/unknown_field_set.h

src/third_party/protobuf-lite/google/protobuf/generated_enum_util.h

src/third_party/protobuf-lite/google/protobuf/io/io_win32.h

src/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl_lite.h

src/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream_impl.h

src/third_party/protobuf-lite/google/protobuf/io/coded_stream.h

src/third_party/protobuf-lite/google/protobuf/io/zero_copy_stream.h

src/third_party/protobuf-lite/google/protobuf/stubs/callback.h

src/third_party/protobuf-lite/google/protobuf/stubs/port.h

src/third_party/protobuf-lite/google/protobuf/stubs/common.h

src/third_party/protobuf-lite/google/protobuf/stubs/once.h

src/third_party/protobuf-lite/google/protobuf/stubs/time.h

src/third_party/protobuf-lite/google/protobuf/stubs/hash.h

src/third_party/protobuf-lite/google/protobuf/stubs/casts.h

src/third_party/protobuf-lite/google/protobuf/stubs/macros.h

src/third_party/protobuf-lite/google/protobuf/stubs/logging.h

src/third_party/protobuf-lite/google/protobuf/stubs/statusor.h

src/third_party/protobuf-lite/google/protobuf/stubs/stl_util.h

src/third_party/protobuf-lite/google/protobuf/stubs/bytestream.h

src/third_party/protobuf-lite/google/protobuf/stubs/stringprintf.h

src/third_party/protobuf-lite/google/protobuf/stubs/strutil.h

src/third_party/protobuf-lite/google/protobuf/stubs/platform_macros.h

src/third_party/protobuf-lite/google/protobuf/stubs/int128.h

src/third_party/protobuf-lite/google/protobuf/stubs/stringpiece.h

src/third_party/protobuf-lite/google/protobuf/stubs/status.h

src/third_party/protobuf-lite/google/protobuf/stubs/map_util.h

src/third_party/protobuf-lite/google/protobuf/stubs/mutex.h

src/third_party/esaxx/sais.hxx

src/third_party/esaxx/esa.hxx

src/third_party/absl/flags/flag.cc

src/third_party/absl/flags/parse.h

src/third_party/absl/flags/flag.h

src/third_party/absl/container/flat_hash_set.h

src/third_party/absl/container/flat_hash_map.h

src/third_party/absl/strings/str_split.h

src/third_party/absl/strings/ascii.h

src/third_party/absl/strings/string_view.h

src/third_party/absl/strings/strip.h

src/third_party/absl/strings/string_view.cc

src/third_party/absl/strings/str_replace.h

src/third_party/absl/strings/str_join.h

src/third_party/absl/strings/numbers.h

src/third_party/absl/strings/match.h

src/third_party/absl/strings/str_format.h

src/third_party/absl/strings/str_cat.h

src/third_party/absl/memory/memory.h

inst/models/nl-fr-dekamer-unigram.model

inst/models/nl.wiki.bpe.vs1000.d25.w2v.txt

inst/models/nl.wiki.bpe.vs1000.model

inst/models/nl-fr-dekamer.model

inst/models/nl.wiki.bpe.vs1000.d25.w2v.bin

inst/spc-help/spm_train

man/sentencepiece.Rd man/BPEembed.Rd man/sentencepiece_decode.Rd man/sentencepiece_download_model.Rd man/predict.BPEembed.Rd man/read_word2vec.Rd man/BPEembedder.Rd man/sentencepiece_encode.Rd man/wordpiece_encode.Rd man/sentencepiece_load_model.Rd man/txt_remove_.Rd

sentencepiece documentation built on Nov. 13, 2022, 5:05 p.m.

rdrr.io home R language documentation Run R code online

CRAN packages Bioconductor packages R-Forge packages GitHub packages

Note that we can't provide technical support on individual packages. You should contact the package authors for that.

sentencepiece
Text Tokenization using Byte Pair Encoding and Unigram Modelling

Files in sentencepiece
Text Tokenization using Byte Pair Encoding and Unigram Modelling

R Package Documentation

Browse R Packages

We want your feedback!

sentencepiece Text Tokenization using Byte Pair Encoding and Unigram Modelling

Files in sentencepiece Text Tokenization using Byte Pair Encoding and Unigram Modelling

R Package Documentation

Browse R Packages

We want your feedback!

sentencepiece
Text Tokenization using Byte Pair Encoding and Unigram Modelling

Files in sentencepiece
Text Tokenization using Byte Pair Encoding and Unigram Modelling