pulsatrix/api/tokenizer_8hpp_source.html

#pragma once


#include <string>

#include <vector>


namespace pulsatrix {


class Tokenizer {

public:

    [[nodiscard]] static std::vector<std::string> Tokenize(const std::string& text);

};


}  // namespace pulsatrix

pulsatrix::Tokenizer
Splits text into lowercase word/punctuation tokens – pulsatrix's first text primitive (campaign_exai_...
Definition tokenizer.hpp:20

pulsatrix::Tokenizer::Tokenize
static std::vector< std::string > Tokenize(const std::string &text)
Tokenizes text: lowercase-normalizes, splits on whitespace (a pure separator, never emitted),...

pulsatrix
Definition acquisition_functions.hpp:16