pulsatrix 1.1.0
Loading...
Searching...
No Matches
text_tokenizer.hpp
Go to the documentation of this file.
1
8#pragma once
9
10#include <cstddef>
11#include <cstdint>
12#include <memory>
13#include <optional>
14#include <string>
15#include <string_view>
16#include <unordered_map>
17#include <vector>
18
19namespace pulsatrix {
20
22struct Offset {
23 size_t begin = 0;
24 size_t end = 0;
25 bool operator==(const Offset& other) const { return begin == other.begin && end == other.end; }
26};
27
37struct Encoding {
38 std::vector<int64_t> ids;
39 std::vector<std::string> tokens;
40 std::vector<Offset> offsets;
43 std::vector<uint8_t> special_tokens_mask;
44
45 [[nodiscard]] size_t size() const { return ids.size(); }
47 void push_back(int64_t id, std::string token, Offset offset, bool special);
48};
49
56public:
57 NormalizedString() = default;
59 explicit NormalizedString(std::string text, size_t offset = 0);
60
61 [[nodiscard]] const std::string& text() const { return text_; }
62 [[nodiscard]] bool empty() const { return text_.empty(); }
63 [[nodiscard]] size_t size() const { return text_.size(); }
64
70 [[nodiscard]] Offset original(size_t begin, size_t end) const;
71
73 [[nodiscard]] NormalizedString slice(size_t begin, size_t end) const;
74
77 struct Piece {
78 std::string text;
79 size_t source_begin = 0;
80 size_t source_end = 0;
81 };
88 void rebuild(const std::vector<Piece>& pieces);
89
90private:
91 std::string text_;
92 std::vector<Offset> origin_;
93};
94
97public:
98 virtual ~Normalizer() = default;
99 virtual void normalize(NormalizedString& text) const = 0;
100};
101
105public:
106 virtual ~PreTokenizer() = default;
108 virtual void pre_tokenize(std::vector<NormalizedString>& splits) const = 0;
109};
110
113 int64_t id = 0;
114 std::string value;
115 size_t begin = 0;
116 size_t end = 0;
117};
118
121public:
122 virtual ~TokenModel() = default;
123 [[nodiscard]] virtual std::vector<ModelToken> tokenize(std::string_view piece) const = 0;
124 [[nodiscard]] virtual std::optional<int64_t> token_to_id(std::string_view token) const = 0;
125 [[nodiscard]] virtual std::optional<std::string> id_to_token(int64_t id) const = 0;
127 [[nodiscard]] virtual int64_t vocab_size() const = 0;
128};
129
132public:
133 virtual ~PostProcessor() = default;
134 [[nodiscard]] virtual Encoding process(Encoding encoding, bool add_special_tokens) const = 0;
135};
136
143class Decoder {
144public:
145 virtual ~Decoder() = default;
147 [[nodiscard]] virtual std::string decode(const std::vector<std::string>& tokens) const;
149 [[nodiscard]] virtual std::vector<std::string> decode_chain(std::vector<std::string> tokens) const;
150};
151
154 std::string content;
155 int64_t id = 0;
157 bool special = true;
158};
159
167public:
168 explicit TextTokenizer(std::shared_ptr<const TokenModel> model);
169
170 TextTokenizer& set_normalizer(std::shared_ptr<const Normalizer> normalizer);
171 TextTokenizer& set_pre_tokenizer(std::shared_ptr<const PreTokenizer> pre_tokenizer);
172 TextTokenizer& set_post_processor(std::shared_ptr<const PostProcessor> post_processor);
173 TextTokenizer& set_decoder(std::shared_ptr<const Decoder> decoder);
176
178 [[nodiscard]] Encoding encode(std::string_view text, bool add_special_tokens = true) const;
184 [[nodiscard]] std::string decode(const std::vector<int64_t>& ids, bool skip_special_tokens = false) const;
185
187 [[nodiscard]] int64_t vocab_size() const;
188 [[nodiscard]] std::optional<int64_t> token_to_id(std::string_view token) const;
189 [[nodiscard]] std::optional<std::string> id_to_token(int64_t id) const;
190 [[nodiscard]] const std::vector<AddedToken>& added_tokens() const { return added_; }
191
192private:
193 void encode_segment(std::string_view text, size_t offset, Encoding& out) const;
194
195 std::shared_ptr<const TokenModel> model_;
196 std::shared_ptr<const Normalizer> normalizer_;
197 std::shared_ptr<const PreTokenizer> pre_tokenizer_;
198 std::shared_ptr<const PostProcessor> post_processor_;
199 std::shared_ptr<const Decoder> decoder_;
200 std::vector<AddedToken> added_;
201 std::unordered_map<std::string, size_t> added_by_content_;
202 std::unordered_map<int64_t, size_t> added_by_id_;
204 std::unordered_map<unsigned char, std::vector<size_t>> added_by_first_byte_;
205};
206
207} // namespace pulsatrix
Turns token strings back into text.
Definition text_tokenizer.hpp:143
virtual ~Decoder()=default
virtual std::string decode(const std::vector< std::string > &tokens) const
The decoded text: decode_chain() joined.
virtual std::vector< std::string > decode_chain(std::vector< std::string > tokens) const
The tokens after this decoder: by default decode()'s text as one token.
Text being normalized and pre-tokenized, remembering where each byte came from in the original input,...
Definition text_tokenizer.hpp:55
const std::string & text() const
Definition text_tokenizer.hpp:61
size_t size() const
Definition text_tokenizer.hpp:63
void rebuild(const std::vector< Piece > &pieces)
Replaces the text with pieces, in order. Every byte of a piece comes from the original span of its so...
NormalizedString slice(size_t begin, size_t end) const
Normalized bytes [begin, end), keeping their origins.
bool empty() const
Definition text_tokenizer.hpp:62
Offset original(size_t begin, size_t end) const
The original bytes that normalized bytes [begin, end) came from. An empty range gives an empty span w...
NormalizedString(std::string text, size_t offset=0)
text as it appears at byte offset of the original input, unchanged.
Changes text before it is split (lowercasing, NFC, ...).
Definition text_tokenizer.hpp:96
virtual void normalize(NormalizedString &text) const =0
virtual ~Normalizer()=default
Adds tokens around an encoding (BOS and EOS, sentence pairs, ...).
Definition text_tokenizer.hpp:131
virtual Encoding process(Encoding encoding, bool add_special_tokens) const =0
virtual ~PostProcessor()=default
Splits text into the pieces the model tokenizes separately, and may change them (byte- level mapping)...
Definition text_tokenizer.hpp:104
virtual void pre_tokenize(std::vector< NormalizedString > &splits) const =0
Replaces splits with their further-split (or transformed) pieces, in order.
virtual ~PreTokenizer()=default
A tokenizer: added tokens are split out of the raw text first (longest match, earliest first); each r...
Definition text_tokenizer.hpp:166
Encoding encode(std::string_view text, bool add_special_tokens=true) const
TextTokenizer & set_pre_tokenizer(std::shared_ptr< const PreTokenizer > pre_tokenizer)
const std::vector< AddedToken > & added_tokens() const
Definition text_tokenizer.hpp:190
int64_t vocab_size() const
One more than the largest id, over the model and the added tokens.
TextTokenizer & set_post_processor(std::shared_ptr< const PostProcessor > post_processor)
TextTokenizer(std::shared_ptr< const TokenModel > model)
std::optional< int64_t > token_to_id(std::string_view token) const
TextTokenizer & set_normalizer(std::shared_ptr< const Normalizer > normalizer)
TextTokenizer & set_decoder(std::shared_ptr< const Decoder > decoder)
TextTokenizer & add_token(AddedToken token)
std::string decode(const std::vector< int64_t > &ids, bool skip_special_tokens=false) const
The text for ids: their token strings through the decoder, or joined with spaces when there is none (...
std::optional< std::string > id_to_token(int64_t id) const
Turns one pre-tokenized piece into tokens (word-level lookup, BPE, ...).
Definition text_tokenizer.hpp:120
virtual ~TokenModel()=default
virtual std::optional< std::string > id_to_token(int64_t id) const =0
virtual int64_t vocab_size() const =0
One more than the largest id the model can produce.
virtual std::vector< ModelToken > tokenize(std::string_view piece) const =0
virtual std::optional< int64_t > token_to_id(std::string_view token) const =0
Definition acquisition_functions.hpp:16
A token matched in the raw input before anything else runs, such as <|im_start|>.
Definition text_tokenizer.hpp:153
bool special
Marked in the special-tokens mask and dropped by decode(skip_special_tokens).
Definition text_tokenizer.hpp:157
std::string content
Definition text_tokenizer.hpp:154
The result of encoding one text.
Definition text_tokenizer.hpp:37
size_t size() const
Definition text_tokenizer.hpp:45
std::vector< uint8_t > special_tokens_mask
1 for special tokens: added tokens marked special, and tokens a post-processor inserted.
Definition text_tokenizer.hpp:43
void push_back(int64_t id, std::string token, Offset offset, bool special)
Appends one token.
std::vector< Offset > offsets
Definition text_tokenizer.hpp:40
std::vector< std::string > tokens
Definition text_tokenizer.hpp:39
std::vector< int64_t > ids
Definition text_tokenizer.hpp:38
One token from a TokenModel: its id and string, and the bytes of the piece it covers.
Definition text_tokenizer.hpp:112
std::string value
Definition text_tokenizer.hpp:114
size_t end
Definition text_tokenizer.hpp:116
size_t begin
Definition text_tokenizer.hpp:115
One piece of a rebuilt text: its new bytes, and the range of the current text they replace (an empty ...
Definition text_tokenizer.hpp:77
size_t source_begin
Definition text_tokenizer.hpp:79
std::string text
Definition text_tokenizer.hpp:78
size_t source_end
Definition text_tokenizer.hpp:80
A byte range [begin, end) of the text passed to TextTokenizer::encode().
Definition text_tokenizer.hpp:22
bool operator==(const Offset &other) const
Definition text_tokenizer.hpp:25
size_t begin
Definition text_tokenizer.hpp:23
size_t end
Definition text_tokenizer.hpp:24