16#include <unordered_map>
38 std::vector<int64_t>
ids;
45 [[nodiscard]]
size_t size()
const {
return ids.size(); }
61 [[nodiscard]]
const std::string&
text()
const {
return text_; }
62 [[nodiscard]]
bool empty()
const {
return text_.empty(); }
63 [[nodiscard]]
size_t size()
const {
return text_.size(); }
88 void rebuild(
const std::vector<Piece>& pieces);
92 std::vector<Offset> origin_;
108 virtual void pre_tokenize(std::vector<NormalizedString>& splits)
const = 0;
123 [[nodiscard]]
virtual std::vector<ModelToken>
tokenize(std::string_view piece)
const = 0;
124 [[nodiscard]]
virtual std::optional<int64_t>
token_to_id(std::string_view token)
const = 0;
125 [[nodiscard]]
virtual std::optional<std::string>
id_to_token(int64_t
id)
const = 0;
147 [[nodiscard]]
virtual std::string
decode(
const std::vector<std::string>& tokens)
const;
149 [[nodiscard]]
virtual std::vector<std::string>
decode_chain(std::vector<std::string> tokens)
const;
178 [[nodiscard]]
Encoding encode(std::string_view text,
bool add_special_tokens =
true)
const;
184 [[nodiscard]] std::string
decode(
const std::vector<int64_t>& ids,
bool skip_special_tokens =
false)
const;
188 [[nodiscard]] std::optional<int64_t>
token_to_id(std::string_view token)
const;
189 [[nodiscard]] std::optional<std::string>
id_to_token(int64_t
id)
const;
190 [[nodiscard]]
const std::vector<AddedToken>&
added_tokens()
const {
return added_; }
193 void encode_segment(std::string_view text,
size_t offset,
Encoding& out)
const;
195 std::shared_ptr<const TokenModel> model_;
196 std::shared_ptr<const Normalizer> normalizer_;
197 std::shared_ptr<const PreTokenizer> pre_tokenizer_;
198 std::shared_ptr<const PostProcessor> post_processor_;
199 std::shared_ptr<const Decoder> decoder_;
200 std::vector<AddedToken> added_;
201 std::unordered_map<std::string, size_t> added_by_content_;
202 std::unordered_map<int64_t, size_t> added_by_id_;
204 std::unordered_map<unsigned char, std::vector<size_t>> added_by_first_byte_;
Turns token strings back into text.
Definition text_tokenizer.hpp:143
virtual ~Decoder()=default
virtual std::string decode(const std::vector< std::string > &tokens) const
The decoded text: decode_chain() joined.
virtual std::vector< std::string > decode_chain(std::vector< std::string > tokens) const
The tokens after this decoder: by default decode()'s text as one token.
Text being normalized and pre-tokenized, remembering where each byte came from in the original input,...
Definition text_tokenizer.hpp:55
const std::string & text() const
Definition text_tokenizer.hpp:61
size_t size() const
Definition text_tokenizer.hpp:63
void rebuild(const std::vector< Piece > &pieces)
Replaces the text with pieces, in order. Every byte of a piece comes from the original span of its so...
NormalizedString slice(size_t begin, size_t end) const
Normalized bytes [begin, end), keeping their origins.
bool empty() const
Definition text_tokenizer.hpp:62
Offset original(size_t begin, size_t end) const
The original bytes that normalized bytes [begin, end) came from. An empty range gives an empty span w...
NormalizedString()=default
NormalizedString(std::string text, size_t offset=0)
text as it appears at byte offset of the original input, unchanged.
Changes text before it is split (lowercasing, NFC, ...).
Definition text_tokenizer.hpp:96
virtual void normalize(NormalizedString &text) const =0
virtual ~Normalizer()=default
Adds tokens around an encoding (BOS and EOS, sentence pairs, ...).
Definition text_tokenizer.hpp:131
virtual Encoding process(Encoding encoding, bool add_special_tokens) const =0
virtual ~PostProcessor()=default
Splits text into the pieces the model tokenizes separately, and may change them (byte- level mapping)...
Definition text_tokenizer.hpp:104
virtual void pre_tokenize(std::vector< NormalizedString > &splits) const =0
Replaces splits with their further-split (or transformed) pieces, in order.
virtual ~PreTokenizer()=default
A tokenizer: added tokens are split out of the raw text first (longest match, earliest first); each r...
Definition text_tokenizer.hpp:166
Encoding encode(std::string_view text, bool add_special_tokens=true) const
TextTokenizer & set_pre_tokenizer(std::shared_ptr< const PreTokenizer > pre_tokenizer)
const std::vector< AddedToken > & added_tokens() const
Definition text_tokenizer.hpp:190
int64_t vocab_size() const
One more than the largest id, over the model and the added tokens.
TextTokenizer & set_post_processor(std::shared_ptr< const PostProcessor > post_processor)
TextTokenizer(std::shared_ptr< const TokenModel > model)
std::optional< int64_t > token_to_id(std::string_view token) const
TextTokenizer & set_normalizer(std::shared_ptr< const Normalizer > normalizer)
TextTokenizer & set_decoder(std::shared_ptr< const Decoder > decoder)
TextTokenizer & add_token(AddedToken token)
std::string decode(const std::vector< int64_t > &ids, bool skip_special_tokens=false) const
The text for ids: their token strings through the decoder, or joined with spaces when there is none (...
std::optional< std::string > id_to_token(int64_t id) const
Turns one pre-tokenized piece into tokens (word-level lookup, BPE, ...).
Definition text_tokenizer.hpp:120
virtual ~TokenModel()=default
virtual std::optional< std::string > id_to_token(int64_t id) const =0
virtual int64_t vocab_size() const =0
One more than the largest id the model can produce.
virtual std::vector< ModelToken > tokenize(std::string_view piece) const =0
virtual std::optional< int64_t > token_to_id(std::string_view token) const =0
Definition acquisition_functions.hpp:16
A token matched in the raw input before anything else runs, such as <|im_start|>.
Definition text_tokenizer.hpp:153
bool special
Marked in the special-tokens mask and dropped by decode(skip_special_tokens).
Definition text_tokenizer.hpp:157
std::string content
Definition text_tokenizer.hpp:154
The result of encoding one text.
Definition text_tokenizer.hpp:37
size_t size() const
Definition text_tokenizer.hpp:45
std::vector< uint8_t > special_tokens_mask
1 for special tokens: added tokens marked special, and tokens a post-processor inserted.
Definition text_tokenizer.hpp:43
void push_back(int64_t id, std::string token, Offset offset, bool special)
Appends one token.
std::vector< Offset > offsets
Definition text_tokenizer.hpp:40
std::vector< std::string > tokens
Definition text_tokenizer.hpp:39
std::vector< int64_t > ids
Definition text_tokenizer.hpp:38
One token from a TokenModel: its id and string, and the bytes of the piece it covers.
Definition text_tokenizer.hpp:112
std::string value
Definition text_tokenizer.hpp:114
size_t end
Definition text_tokenizer.hpp:116
size_t begin
Definition text_tokenizer.hpp:115
One piece of a rebuilt text: its new bytes, and the range of the current text they replace (an empty ...
Definition text_tokenizer.hpp:77
size_t source_begin
Definition text_tokenizer.hpp:79
std::string text
Definition text_tokenizer.hpp:78
size_t source_end
Definition text_tokenizer.hpp:80
A byte range [begin, end) of the text passed to TextTokenizer::encode().
Definition text_tokenizer.hpp:22
bool operator==(const Offset &other) const
Definition text_tokenizer.hpp:25
size_t begin
Definition text_tokenizer.hpp:23
size_t end
Definition text_tokenizer.hpp:24