Byte-pair encoding: start from characters and apply merges, lowest rank first.
More...
#include <byte_level_bpe.hpp>
|
| | BpeModel (std::unordered_map< std::string, int64_t > vocab, const std::vector< std::pair< std::string, std::string > > &merges, BpeOptions options={}) |
| |
| std::vector< ModelToken > | tokenize (std::string_view piece) const override |
| |
| std::optional< int64_t > | token_to_id (std::string_view token) const override |
| |
| std::optional< std::string > | id_to_token (int64_t id) const override |
| |
| int64_t | vocab_size () const override |
| | One more than the largest id the model can produce.
|
| |
| virtual | ~TokenModel ()=default |
| |
Byte-pair encoding: start from characters and apply merges, lowest rank first.
◆ BpeModel()
| pulsatrix::BpeModel::BpeModel |
( |
std::unordered_map< std::string, int64_t > |
vocab, |
|
|
const std::vector< std::pair< std::string, std::string > > & |
merges, |
|
|
BpeOptions |
options = {} |
|
) |
| |
- Parameters
-
| vocab | Token strings and their ids. |
| merges | Merge rules, highest priority first. |
- Exceptions
-
| std::invalid_argument | if a merge's parts or result aren't in the vocabulary, or the unknown token isn't. |
◆ id_to_token()
| std::optional< std::string > pulsatrix::BpeModel::id_to_token |
( |
int64_t |
id | ) |
const |
|
overridevirtual |
◆ token_to_id()
| std::optional< int64_t > pulsatrix::BpeModel::token_to_id |
( |
std::string_view |
token | ) |
const |
|
overridevirtual |
◆ tokenize()
| std::vector< ModelToken > pulsatrix::BpeModel::tokenize |
( |
std::string_view |
piece | ) |
const |
|
overridevirtual |
◆ vocab_size()
| int64_t pulsatrix::BpeModel::vocab_size |
( |
| ) |
const |
|
inlineoverridevirtual |
The documentation for this class was generated from the following file: