9#include <unordered_map>
31 void pre_tokenize(std::vector<NormalizedString>& splits)
const override;
39 WordLevelModel(std::vector<std::string> vocab,
const std::string& unk_token);
40 [[nodiscard]] std::vector<ModelToken>
tokenize(std::string_view piece)
const override;
41 [[nodiscard]] std::optional<int64_t>
token_to_id(std::string_view token)
const override;
42 [[nodiscard]] std::optional<std::string>
id_to_token(int64_t
id)
const override;
43 [[nodiscard]] int64_t
vocab_size()
const override {
return static_cast<int64_t
>(vocab_.size()); }
46 std::vector<std::string> vocab_;
47 std::unordered_map<std::string, int64_t> ids_;
55 [[nodiscard]] std::vector<ModelToken>
tokenize(std::string_view piece)
const override;
56 [[nodiscard]] std::optional<int64_t>
token_to_id(std::string_view token)
const override;
57 [[nodiscard]] std::optional<std::string>
id_to_token(int64_t
id)
const override;
58 [[nodiscard]] int64_t
vocab_size()
const override {
return 256; }
67 CharModel(std::vector<std::string> chars,
const std::string& unk_token);
68 [[nodiscard]] std::vector<ModelToken>
tokenize(std::string_view piece)
const override;
69 [[nodiscard]] std::optional<int64_t>
token_to_id(std::string_view token)
const override;
70 [[nodiscard]] std::optional<std::string>
id_to_token(int64_t
id)
const override;
71 [[nodiscard]] int64_t
vocab_size()
const override {
return static_cast<int64_t
>(chars_.size()); }
74 std::vector<std::string> chars_;
75 std::unordered_map<std::string, int64_t> ids_;
82 [[nodiscard]] std::string
decode(
const std::vector<std::string>& tokens)
const override;
102 const std::vector<std::string>& special_tokens = {});
Lowercases ASCII letters and leaves every other character alone.
Definition tokenizer_components.hpp:18
void normalize(NormalizedString &text) const override
Every byte is a token whose id is the byte's value (0..255); the token string is the byte itself.
Definition tokenizer_components.hpp:53
std::optional< std::string > id_to_token(int64_t id) const override
int64_t vocab_size() const override
One more than the largest id the model can produce.
Definition tokenizer_components.hpp:58
std::vector< ModelToken > tokenize(std::string_view piece) const override
std::optional< int64_t > token_to_id(std::string_view token) const override
Every character (code point) is a token, looked up in a character list; unknown characters become the...
Definition tokenizer_components.hpp:63
std::optional< std::string > id_to_token(int64_t id) const override
std::optional< int64_t > token_to_id(std::string_view token) const override
CharModel(std::vector< std::string > chars, const std::string &unk_token)
std::vector< ModelToken > tokenize(std::string_view piece) const override
int64_t vocab_size() const override
One more than the largest id the model can produce.
Definition tokenizer_components.hpp:71
Turns token strings back into text.
Definition text_tokenizer.hpp:143
Concatenates the tokens (Hugging Face's Fuse decoder).
Definition tokenizer_components.hpp:80
std::string decode(const std::vector< std::string > &tokens) const override
The decoded text: decode_chain() joined.
Text being normalized and pre-tokenized, remembering where each byte came from in the original input,...
Definition text_tokenizer.hpp:55
Changes text before it is split (lowercasing, NFC, ...).
Definition text_tokenizer.hpp:96
Splits text into the pieces the model tokenizes separately, and may change them (byte- level mapping)...
Definition text_tokenizer.hpp:104
A tokenizer: added tokens are split out of the raw text first (longest match, earliest first); each r...
Definition text_tokenizer.hpp:166
Turns one pre-tokenized piece into tokens (word-level lookup, BPE, ...).
Definition text_tokenizer.hpp:120
Token<->index lookup table. Index 0 is always the reserved "<unk>" token – guaranteed by construction...
Definition vocabulary.hpp:19
Each piece is one token, looked up in a vocabulary; unknown pieces become the unknown token.
Definition tokenizer_components.hpp:36
std::optional< int64_t > token_to_id(std::string_view token) const override
std::vector< ModelToken > tokenize(std::string_view piece) const override
WordLevelModel(std::vector< std::string > vocab, const std::string &unk_token)
std::optional< std::string > id_to_token(int64_t id) const override
int64_t vocab_size() const override
One more than the largest id the model can produce.
Definition tokenizer_components.hpp:43
The split Tokenizer::Tokenize makes: runs of ASCII letters, digits and apostrophes are words,...
Definition tokenizer_components.hpp:29
void pre_tokenize(std::vector< NormalizedString > &splits) const override
Replaces splits with their further-split (or transformed) pieces, in order.
Definition acquisition_functions.hpp:16
TextTokenizer MakeCharTokenizer(std::vector< std::string > chars, const std::vector< std::string > &special_tokens={})
Characters by id from chars, then <unk> (unless chars has it), then special_tokens; decodes by concat...
TextTokenizer MakeWordLevelTokenizer(const Vocabulary &vocab)
The word-level tokenizer over a Vocabulary: AsciiLowercase, WordPunctuationSplit and a WordLevelModel...
TextTokenizer MakeByteTokenizer(const std::vector< std::string > &special_tokens={})
Bytes as ids 0..255, then special_tokens as ids 256, 257, ...; decodes by concatenation.
The tokenizer interface (TOK-1): a pipeline of normalizer, pre-tokenizer, model, post-processor and d...
Token<->index lookup with a reserved <unk> fallback, plus a frequency-ranked builder.