pulsatrix 1.1.0
Loading...
Searching...
No Matches
pulsatrix::TextTokenizer Member List

This is the complete list of members for pulsatrix::TextTokenizer, including all inherited members.

add_token(AddedToken token)pulsatrix::TextTokenizer
added_tokens() constpulsatrix::TextTokenizerinline
decode(const std::vector< int64_t > &ids, bool skip_special_tokens=false) constpulsatrix::TextTokenizer
encode(std::string_view text, bool add_special_tokens=true) constpulsatrix::TextTokenizer
id_to_token(int64_t id) constpulsatrix::TextTokenizer
set_decoder(std::shared_ptr< const Decoder > decoder)pulsatrix::TextTokenizer
set_normalizer(std::shared_ptr< const Normalizer > normalizer)pulsatrix::TextTokenizer
set_post_processor(std::shared_ptr< const PostProcessor > post_processor)pulsatrix::TextTokenizer
set_pre_tokenizer(std::shared_ptr< const PreTokenizer > pre_tokenizer)pulsatrix::TextTokenizer
TextTokenizer(std::shared_ptr< const TokenModel > model)pulsatrix::TextTokenizerexplicit
token_to_id(std::string_view token) constpulsatrix::TextTokenizer
vocab_size() constpulsatrix::TextTokenizer