text_dataset.hpp

Go to the documentation of this file.

 
#pragma once
 
#include <string>
#include <vector>
 
#include "pulsatrix/dataset.hpp"
#include "pulsatrix/device_backend.hpp"
#include "pulsatrix/vocabulary.hpp"
 
namespace pulsatrix {
 
class TextDataset : public Dataset {
public:
    TextDataset(const std::string& corpus_path, const Vocabulary* vocabulary, DeviceBackend* backend);
 
    [[nodiscard]] int64_t size() const override;
 
    [[nodiscard]] Sample get(int64_t index) const override;
 
private:
    std::vector<std::string> lines_;
    const Vocabulary* vocabulary_;
    DeviceBackend* backend_;
};
 
}  // namespace pulsatrix