|
pulsatrix 1.1.0
|
This is the complete list of members for pulsatrix::TextTokenizer, including all inherited members.
| add_token(AddedToken token) | pulsatrix::TextTokenizer | |
| added_tokens() const | pulsatrix::TextTokenizer | inline |
| decode(const std::vector< int64_t > &ids, bool skip_special_tokens=false) const | pulsatrix::TextTokenizer | |
| encode(std::string_view text, bool add_special_tokens=true) const | pulsatrix::TextTokenizer | |
| id_to_token(int64_t id) const | pulsatrix::TextTokenizer | |
| set_decoder(std::shared_ptr< const Decoder > decoder) | pulsatrix::TextTokenizer | |
| set_normalizer(std::shared_ptr< const Normalizer > normalizer) | pulsatrix::TextTokenizer | |
| set_post_processor(std::shared_ptr< const PostProcessor > post_processor) | pulsatrix::TextTokenizer | |
| set_pre_tokenizer(std::shared_ptr< const PreTokenizer > pre_tokenizer) | pulsatrix::TextTokenizer | |
| TextTokenizer(std::shared_ptr< const TokenModel > model) | pulsatrix::TextTokenizer | explicit |
| token_to_id(std::string_view token) const | pulsatrix::TextTokenizer | |
| vocab_size() const | pulsatrix::TextTokenizer |