pulsatrix 1.1.0
Loading...
Searching...
No Matches
sentencepiece_bpe.hpp
Go to the documentation of this file.
1
8#pragma once
9
10#include <memory>
11#include <optional>
12#include <string>
13#include <vector>
14
17
18namespace pulsatrix {
19
22public:
24 ReplaceNormalizer(std::string pattern, std::string content);
26 static ReplaceNormalizer Regex(const std::string& pattern, std::string content);
27 void normalize(NormalizedString& text) const override;
28
29private:
30 ReplaceNormalizer(std::optional<UnicodeRegex> regex, std::string literal, std::string content);
31 std::optional<UnicodeRegex> regex_;
32 std::string literal_;
33 std::string content_;
34};
35
38public:
39 explicit PrependNormalizer(std::string prepend);
40 void normalize(NormalizedString& text) const override;
41
42private:
43 std::string prepend_;
44};
45
48
55public:
56 MetaspacePreTokenizer(std::string replacement, PrependScheme scheme, bool split);
57 void pre_tokenize(std::vector<NormalizedString>& splits) const override;
58
59private:
60 std::string replacement_;
61 PrependScheme scheme_;
62 bool split_;
63};
64
66class ReplaceDecoder : public Decoder {
67public:
68 ReplaceDecoder(std::string pattern, std::string content);
69 static ReplaceDecoder Regex(const std::string& pattern, std::string content);
70 [[nodiscard]] std::vector<std::string> decode_chain(std::vector<std::string> tokens) const override;
71
72private:
73 ReplaceDecoder(std::optional<UnicodeRegex> regex, std::string literal, std::string content);
74 std::optional<UnicodeRegex> regex_;
75 std::string literal_;
76 std::string content_;
77};
78
82public:
83 [[nodiscard]] std::vector<std::string> decode_chain(std::vector<std::string> tokens) const override;
84};
85
88class StripDecoder : public Decoder {
89public:
90 StripDecoder(std::string content, size_t start, size_t stop);
91 [[nodiscard]] std::vector<std::string> decode_chain(std::vector<std::string> tokens) const override;
92
93private:
94 std::string content_;
95 size_t start_;
96 size_t stop_;
97};
98
102class MetaspaceDecoder : public Decoder {
103public:
104 MetaspaceDecoder(std::string replacement, PrependScheme scheme);
105 [[nodiscard]] std::vector<std::string> decode_chain(std::vector<std::string> tokens) const override;
106
107private:
108 std::string replacement_;
109 PrependScheme scheme_;
110};
111
113class SequenceDecoder : public Decoder {
114public:
115 explicit SequenceDecoder(std::vector<std::shared_ptr<const Decoder>> decoders);
116 [[nodiscard]] std::vector<std::string> decode_chain(std::vector<std::string> tokens) const override;
117
118private:
119 std::vector<std::shared_ptr<const Decoder>> decoders_;
120};
121
122} // namespace pulsatrix
Turns runs of <0x41>-style byte tokens back into text; a run that isn't valid UTF-8 becomes one U+FFF...
Definition sentencepiece_bpe.hpp:81
std::vector< std::string > decode_chain(std::vector< std::string > tokens) const override
The tokens after this decoder: by default decode()'s text as one token.
Turns token strings back into text.
Definition text_tokenizer.hpp:143
The replacement character back to spaces (Hugging Face's Metaspace decoder). Unless the scheme is Nev...
Definition sentencepiece_bpe.hpp:102
std::vector< std::string > decode_chain(std::vector< std::string > tokens) const override
The tokens after this decoder: by default decode()'s text as one token.
MetaspaceDecoder(std::string replacement, PrependScheme scheme)
Spaces become a replacement character (▁), one is put in front of each piece (or of the text's first ...
Definition sentencepiece_bpe.hpp:54
void pre_tokenize(std::vector< NormalizedString > &splits) const override
Replaces splits with their further-split (or transformed) pieces, in order.
MetaspacePreTokenizer(std::string replacement, PrependScheme scheme, bool split)
Text being normalized and pre-tokenized, remembering where each byte came from in the original input,...
Definition text_tokenizer.hpp:55
Changes text before it is split (lowercasing, NFC, ...).
Definition text_tokenizer.hpp:96
Splits text into the pieces the model tokenizes separately, and may change them (byte- level mapping)...
Definition text_tokenizer.hpp:104
Puts a string in front of non-empty text (Hugging Face's Prepend normalizer).
Definition sentencepiece_bpe.hpp:37
PrependNormalizer(std::string prepend)
void normalize(NormalizedString &text) const override
Replaces a literal string or regex in every token (Hugging Face's Replace decoder).
Definition sentencepiece_bpe.hpp:66
ReplaceDecoder(std::string pattern, std::string content)
std::vector< std::string > decode_chain(std::vector< std::string > tokens) const override
The tokens after this decoder: by default decode()'s text as one token.
static ReplaceDecoder Regex(const std::string &pattern, std::string content)
Replaces every match of a literal string or a regex (Hugging Face's Replace normalizer).
Definition sentencepiece_bpe.hpp:21
ReplaceNormalizer(std::string pattern, std::string content)
A literal pattern.
void normalize(NormalizedString &text) const override
static ReplaceNormalizer Regex(const std::string &pattern, std::string content)
A regex pattern.
Runs decoders in order, each on the previous one's tokens.
Definition sentencepiece_bpe.hpp:113
std::vector< std::string > decode_chain(std::vector< std::string > tokens) const override
The tokens after this decoder: by default decode()'s text as one token.
SequenceDecoder(std::vector< std::shared_ptr< const Decoder > > decoders)
Removes up to start leading and stop trailing copies of a character from every token (Hugging Face's ...
Definition sentencepiece_bpe.hpp:88
StripDecoder(std::string content, size_t start, size_t stop)
std::vector< std::string > decode_chain(std::vector< std::string > tokens) const override
The tokens after this decoder: by default decode()'s text as one token.
Definition acquisition_functions.hpp:16
PrependScheme
When Metaspace puts its replacement in front of a piece.
Definition sentencepiece_bpe.hpp:47
The tokenizer interface (TOK-1): a pipeline of normalizer, pre-tokenizer, model, post-processor and d...
A small backtracking regex engine over Unicode code points, for the pre-tokenizer patterns in tokeniz...