pulsatrix 1.1.0
Loading...
Searching...
No Matches
pulsatrix::BpeModel Member List

This is the complete list of members for pulsatrix::BpeModel, including all inherited members.

BpeModel(std::unordered_map< std::string, int64_t > vocab, const std::vector< std::pair< std::string, std::string > > &merges, BpeOptions options={})pulsatrix::BpeModel
id_to_token(int64_t id) const overridepulsatrix::BpeModelvirtual
token_to_id(std::string_view token) const overridepulsatrix::BpeModelvirtual
tokenize(std::string_view piece) const overridepulsatrix::BpeModelvirtual
vocab_size() const overridepulsatrix::BpeModelinlinevirtual
~TokenModel()=defaultpulsatrix::TokenModelvirtual