pulsatrix 1.1.0
Loading...
Searching...
No Matches
tokenizer_components.hpp
Go to the documentation of this file.
1
6#pragma once
7
8#include <string>
9#include <unordered_map>
10#include <vector>
11
14
15namespace pulsatrix {
16
18class AsciiLowercase : public Normalizer {
19public:
20 void normalize(NormalizedString& text) const override;
21};
22
30public:
31 void pre_tokenize(std::vector<NormalizedString>& splits) const override;
32};
33
36class WordLevelModel : public TokenModel {
37public:
39 WordLevelModel(std::vector<std::string> vocab, const std::string& unk_token);
40 [[nodiscard]] std::vector<ModelToken> tokenize(std::string_view piece) const override;
41 [[nodiscard]] std::optional<int64_t> token_to_id(std::string_view token) const override;
42 [[nodiscard]] std::optional<std::string> id_to_token(int64_t id) const override;
43 [[nodiscard]] int64_t vocab_size() const override { return static_cast<int64_t>(vocab_.size()); }
44
45private:
46 std::vector<std::string> vocab_;
47 std::unordered_map<std::string, int64_t> ids_;
48 int64_t unk_id_ = 0;
49};
50
53class ByteModel : public TokenModel {
54public:
55 [[nodiscard]] std::vector<ModelToken> tokenize(std::string_view piece) const override;
56 [[nodiscard]] std::optional<int64_t> token_to_id(std::string_view token) const override;
57 [[nodiscard]] std::optional<std::string> id_to_token(int64_t id) const override;
58 [[nodiscard]] int64_t vocab_size() const override { return 256; }
59};
60
63class CharModel : public TokenModel {
64public:
67 CharModel(std::vector<std::string> chars, const std::string& unk_token);
68 [[nodiscard]] std::vector<ModelToken> tokenize(std::string_view piece) const override;
69 [[nodiscard]] std::optional<int64_t> token_to_id(std::string_view token) const override;
70 [[nodiscard]] std::optional<std::string> id_to_token(int64_t id) const override;
71 [[nodiscard]] int64_t vocab_size() const override { return static_cast<int64_t>(chars_.size()); }
72
73private:
74 std::vector<std::string> chars_;
75 std::unordered_map<std::string, int64_t> ids_;
76 int64_t unk_id_ = 0;
77};
78
80class FuseDecoder : public Decoder {
81public:
82 [[nodiscard]] std::string decode(const std::vector<std::string>& tokens) const override;
83};
84
91
94[[nodiscard]] TextTokenizer MakeByteTokenizer(const std::vector<std::string>& special_tokens = {});
95
101[[nodiscard]] TextTokenizer MakeCharTokenizer(std::vector<std::string> chars,
102 const std::vector<std::string>& special_tokens = {});
103
104} // namespace pulsatrix
Lowercases ASCII letters and leaves every other character alone.
Definition tokenizer_components.hpp:18
void normalize(NormalizedString &text) const override
Every byte is a token whose id is the byte's value (0..255); the token string is the byte itself.
Definition tokenizer_components.hpp:53
std::optional< std::string > id_to_token(int64_t id) const override
int64_t vocab_size() const override
One more than the largest id the model can produce.
Definition tokenizer_components.hpp:58
std::vector< ModelToken > tokenize(std::string_view piece) const override
std::optional< int64_t > token_to_id(std::string_view token) const override
Every character (code point) is a token, looked up in a character list; unknown characters become the...
Definition tokenizer_components.hpp:63
std::optional< std::string > id_to_token(int64_t id) const override
std::optional< int64_t > token_to_id(std::string_view token) const override
CharModel(std::vector< std::string > chars, const std::string &unk_token)
std::vector< ModelToken > tokenize(std::string_view piece) const override
int64_t vocab_size() const override
One more than the largest id the model can produce.
Definition tokenizer_components.hpp:71
Turns token strings back into text.
Definition text_tokenizer.hpp:143
Concatenates the tokens (Hugging Face's Fuse decoder).
Definition tokenizer_components.hpp:80
std::string decode(const std::vector< std::string > &tokens) const override
The decoded text: decode_chain() joined.
Text being normalized and pre-tokenized, remembering where each byte came from in the original input,...
Definition text_tokenizer.hpp:55
Changes text before it is split (lowercasing, NFC, ...).
Definition text_tokenizer.hpp:96
Splits text into the pieces the model tokenizes separately, and may change them (byte- level mapping)...
Definition text_tokenizer.hpp:104
A tokenizer: added tokens are split out of the raw text first (longest match, earliest first); each r...
Definition text_tokenizer.hpp:166
Turns one pre-tokenized piece into tokens (word-level lookup, BPE, ...).
Definition text_tokenizer.hpp:120
Token<->index lookup table. Index 0 is always the reserved "<unk>" token – guaranteed by construction...
Definition vocabulary.hpp:19
Each piece is one token, looked up in a vocabulary; unknown pieces become the unknown token.
Definition tokenizer_components.hpp:36
std::optional< int64_t > token_to_id(std::string_view token) const override
std::vector< ModelToken > tokenize(std::string_view piece) const override
WordLevelModel(std::vector< std::string > vocab, const std::string &unk_token)
std::optional< std::string > id_to_token(int64_t id) const override
int64_t vocab_size() const override
One more than the largest id the model can produce.
Definition tokenizer_components.hpp:43
The split Tokenizer::Tokenize makes: runs of ASCII letters, digits and apostrophes are words,...
Definition tokenizer_components.hpp:29
void pre_tokenize(std::vector< NormalizedString > &splits) const override
Replaces splits with their further-split (or transformed) pieces, in order.
Definition acquisition_functions.hpp:16
TextTokenizer MakeCharTokenizer(std::vector< std::string > chars, const std::vector< std::string > &special_tokens={})
Characters by id from chars, then <unk> (unless chars has it), then special_tokens; decodes by concat...
TextTokenizer MakeWordLevelTokenizer(const Vocabulary &vocab)
The word-level tokenizer over a Vocabulary: AsciiLowercase, WordPunctuationSplit and a WordLevelModel...
TextTokenizer MakeByteTokenizer(const std::vector< std::string > &special_tokens={})
Bytes as ids 0..255, then special_tokens as ids 256, 257, ...; decodes by concatenation.
The tokenizer interface (TOK-1): a pipeline of normalizer, pre-tokenizer, model, post-processor and d...
Token<->index lookup with a reserved <unk> fallback, plus a frequency-ranked builder.