13#include <unordered_map>
35 std::vector<std::shared_ptr<const Normalizer>> normalizers_;
48 void pre_tokenize(std::vector<NormalizedString>& splits)
const override;
52 std::optional<UnicodeRegex> regex_;
62 void pre_tokenize(std::vector<NormalizedString>& splits)
const override;
76 void pre_tokenize(std::vector<NormalizedString>& splits)
const override;
79 bool add_prefix_space_;
80 std::optional<UnicodeRegex> regex_;
87 void pre_tokenize(std::vector<NormalizedString>& splits)
const override;
90 std::vector<std::shared_ptr<const PreTokenizer>> pre_tokenizers_;
119 BpeModel(std::unordered_map<std::string, int64_t> vocab,
const std::vector<std::pair<std::string, std::string>>& merges,
121 [[nodiscard]] std::vector<ModelToken>
tokenize(std::string_view piece)
const override;
122 [[nodiscard]] std::optional<int64_t>
token_to_id(std::string_view token)
const override;
123 [[nodiscard]] std::optional<std::string>
id_to_token(int64_t
id)
const override;
124 [[nodiscard]] int64_t
vocab_size()
const override {
return vocab_size_; }
128 size_t operator()(
const std::pair<int64_t, int64_t>& p)
const noexcept {
129 return std::hash<int64_t>()(p.first * 1000003 + p.second);
132 std::unordered_map<std::string, int64_t> vocab_;
133 std::unordered_map<int64_t, std::string> tokens_;
135 std::unordered_map<std::pair<int64_t, int64_t>, std::pair<int64_t, int64_t>, PairHash> merges_;
137 std::optional<int64_t> unk_id_;
138 int64_t vocab_size_ = 0;
151 bool add_prefix_space_;
168 std::vector<Item> single_;
178 std::vector<std::shared_ptr<const PostProcessor>> processors_;
185 [[nodiscard]] std::string
decode(
const std::vector<std::string>& tokens)
const override;
Byte-pair encoding: start from characters and apply merges, lowest rank first.
Definition byte_level_bpe.hpp:111
std::optional< std::string > id_to_token(int64_t id) const override
int64_t vocab_size() const override
One more than the largest id the model can produce.
Definition byte_level_bpe.hpp:124
BpeModel(std::unordered_map< std::string, int64_t > vocab, const std::vector< std::pair< std::string, std::string > > &merges, BpeOptions options={})
std::optional< int64_t > token_to_id(std::string_view token) const override
std::vector< ModelToken > tokenize(std::string_view piece) const override
Maps byte-level characters back to bytes; invalid UTF-8 becomes U+FFFD, as in Hugging Face.
Definition byte_level_bpe.hpp:183
std::string decode(const std::vector< std::string > &tokens) const override
The decoded text: decode_chain() joined.
Byte-level post-processing (Hugging Face's ByteLevel post-processor): with trim_offsets,...
Definition byte_level_bpe.hpp:145
ByteLevelPostProcessor(bool add_prefix_space, bool trim_offsets)
Encoding process(Encoding encoding, bool add_special_tokens) const override
Byte-level pre-tokenization (Hugging Face's ByteLevel): optionally prepends a space,...
Definition byte_level_bpe.hpp:73
void pre_tokenize(std::vector< NormalizedString > &splits) const override
Replaces splits with their further-split (or transformed) pieces, in order.
ByteLevelPreTokenizer(bool add_prefix_space, bool use_regex)
Turns token strings back into text.
Definition text_tokenizer.hpp:143
Splits off numeric characters (Unicode N), one by one or in runs (Hugging Face's Digits).
Definition byte_level_bpe.hpp:59
void pre_tokenize(std::vector< NormalizedString > &splits) const override
Replaces splits with their further-split (or transformed) pieces, in order.
DigitsPreTokenizer(bool individual_digits)
Unicode Normalization Form C (canonical composition), keeping offsets.
Definition byte_level_bpe.hpp:23
void normalize(NormalizedString &text) const override
Text being normalized and pre-tokenized, remembering where each byte came from in the original input,...
Definition text_tokenizer.hpp:55
Changes text before it is split (lowercasing, NFC, ...).
Definition text_tokenizer.hpp:96
Adds tokens around an encoding (BOS and EOS, sentence pairs, ...).
Definition text_tokenizer.hpp:131
Splits text into the pieces the model tokenizes separately, and may change them (byte- level mapping)...
Definition text_tokenizer.hpp:104
Runs normalizers in order.
Definition byte_level_bpe.hpp:29
SequenceNormalizer(std::vector< std::shared_ptr< const Normalizer > > normalizers)
void normalize(NormalizedString &text) const override
Runs post-processors in order.
Definition byte_level_bpe.hpp:172
Encoding process(Encoding encoding, bool add_special_tokens) const override
SequencePostProcessor(std::vector< std::shared_ptr< const PostProcessor > > processors)
Runs pre-tokenizers in order.
Definition byte_level_bpe.hpp:84
void pre_tokenize(std::vector< NormalizedString > &splits) const override
Replaces splits with their further-split (or transformed) pieces, in order.
SequencePreTokenizer(std::vector< std::shared_ptr< const PreTokenizer > > pre_tokenizers)
Splits on a regex or a literal string (Hugging Face's Split pre-tokenizer).
Definition byte_level_bpe.hpp:42
SplitPreTokenizer(const std::string ®ex, SplitBehavior behavior, bool invert=false)
A regex pattern.
static SplitPreTokenizer Literal(const std::string &text, SplitBehavior behavior, bool invert=false)
A literal string pattern.
void pre_tokenize(std::vector< NormalizedString > &splits) const override
Replaces splits with their further-split (or transformed) pieces, in order.
Special tokens around a single sequence (Hugging Face's TemplateProcessing, single).
Definition byte_level_bpe.hpp:156
Encoding process(Encoding encoding, bool add_special_tokens) const override
TemplatePostProcessor(std::vector< Item > single)
Turns one pre-tokenized piece into tokens (word-level lookup, BPE, ...).
Definition text_tokenizer.hpp:120
Definition acquisition_functions.hpp:16
const std::vector< std::string > & ByteLevelAlphabet()
The GPT-2 byte-to-character table byte-level BPE uses: printable bytes stand for themselves,...
SplitBehavior
What a split does with the pieces that match (Hugging Face's SplitDelimiterBehavior).
Definition byte_level_bpe.hpp:39
BPE options beyond the vocabulary and merges, as tokenizer.json names them.
Definition byte_level_bpe.hpp:94
bool fuse_unk
Consecutive unknown characters become one unknown token.
Definition byte_level_bpe.hpp:107
std::optional< std::string > unk_token
For characters the vocabulary lacks; without one they are dropped, as in Hugging Face.
Definition byte_level_bpe.hpp:102
bool byte_fallback
A character the vocabulary lacks becomes its UTF-8 bytes as <0x41>-style tokens (SentencePiece's byte...
Definition byte_level_bpe.hpp:105
std::string end_of_word_suffix
Added after a piece's last character.
Definition byte_level_bpe.hpp:100
bool ignore_merges
A piece that is a whole vocabulary entry is one token, without merging (Llama 3).
Definition byte_level_bpe.hpp:96
std::string continuing_subword_prefix
Added before every character but a piece's first.
Definition byte_level_bpe.hpp:98
The result of encoding one text.
Definition text_tokenizer.hpp:37
One template item: the sequence itself (special empty), or special tokens.
Definition byte_level_bpe.hpp:159
std::vector< std::string > special_tokens
Definition byte_level_bpe.hpp:161
bool is_sequence() const
Definition byte_level_bpe.hpp:162
std::vector< int64_t > special_ids
Definition byte_level_bpe.hpp:160
The tokenizer interface (TOK-1): a pipeline of normalizer, pre-tokenizer, model, post-processor and d...
A small backtracking regex engine over Unicode code points, for the pre-tokenizer patterns in tokeniz...