Replaces a literal string or regex in every token (Hugging Face's Replace decoder).
More...
#include <sentencepiece_bpe.hpp>
|
| | ReplaceDecoder (std::string pattern, std::string content) |
| |
| std::vector< std::string > | decode_chain (std::vector< std::string > tokens) const override |
| | The tokens after this decoder: by default decode()'s text as one token.
|
| |
| virtual | ~Decoder ()=default |
| |
| virtual std::string | decode (const std::vector< std::string > &tokens) const |
| | The decoded text: decode_chain() joined.
|
| |
Replaces a literal string or regex in every token (Hugging Face's Replace decoder).
◆ ReplaceDecoder()
| pulsatrix::ReplaceDecoder::ReplaceDecoder |
( |
std::string |
pattern, |
|
|
std::string |
content |
|
) |
| |
◆ decode_chain()
| std::vector< std::string > pulsatrix::ReplaceDecoder::decode_chain |
( |
std::vector< std::string > |
tokens | ) |
const |
|
overridevirtual |
◆ Regex()
| static ReplaceDecoder pulsatrix::ReplaceDecoder::Regex |
( |
const std::string & |
pattern, |
|
|
std::string |
content |
|
) |
| |
|
static |
The documentation for this class was generated from the following file: