rope_module.hpp

Go to the documentation of this file.

 
#pragma once
 
#include <optional>
#include "pulsatrix/module.hpp"
 
namespace pulsatrix {
 
class RoPEModule : public Module {
public:
    RoPEModule(int64_t head_dim, DeviceBackend* backend, float base = 10000.0f);
 
    [[nodiscard]] Tensor backward(const Tensor& grad_output) override;
 
    [[nodiscard]] OpType op_type() const override { return OpType::Elementwise; }
 
    [[nodiscard]] Tensor propagate_relevance(const Tensor& relevance_out, const LRPRuleConfig& config) override;
 
 
    [[nodiscard]] std::optional<DeviceType> compute_device() const override { return backend_->device(); }
 
protected:
    [[nodiscard]] Tensor forward_impl(const Tensor& input) override;
 
private:
    int64_t head_dim_;
    float base_;
    DeviceBackend* backend_;
    Tensor last_input_;   
    Tensor last_output_;  
    bool has_forwarded_ = false;
 
    void ensure_tables(int64_t seq_len);
    Tensor cos_table_;
    Tensor sin_table_;
    int64_t table_seq_len_ = 0;
};
 
}  // namespace pulsatrix