pulsatrix 1.1.0
Loading...
Searching...
No Matches
linear_module.hpp
Go to the documentation of this file.
1
5#pragma once
6
7#include <initializer_list>
8#include <memory>
9#include <optional>
10#include <vector>
11
12#include "pulsatrix/module.hpp"
13
14namespace pulsatrix {
15
16class ReluModule;
17class LoRAAdapter;
18struct LoRAConfig;
19class IA3Adapter;
20
43class LinearModule : public Module {
44public:
60 LinearModule(int64_t in_features, int64_t out_features, DeviceBackend* backend, DeviceType device,
61 bool use_bias = true);
62
69 LinearModule(int64_t in_features, int64_t out_features, DeviceBackend* backend, bool use_bias = true);
70
71 ~LinearModule() override;
73 LinearModule& operator=(LinearModule&&) noexcept;
74
86 void attach_lora(const LoRAConfig& config);
90 [[nodiscard]] LoRAAdapter* lora() { return lora_.get(); }
91 [[nodiscard]] const LoRAAdapter* lora() const { return lora_.get(); }
97 void merge_lora();
107 void set_lora_enabled(bool enabled);
108
116 void attach_ia3(bool feedforward);
119 [[nodiscard]] IA3Adapter* ia3() { return ia3_.get(); }
120 [[nodiscard]] const IA3Adapter* ia3() const { return ia3_.get(); }
123 void merge_ia3();
128 void set_ia3_enabled(bool enabled);
129
144 [[nodiscard]] Tensor backward(const Tensor& grad_output) override;
145
147 [[nodiscard]] OpType op_type() const override { return OpType::Linear; }
148
150 void set_weight(std::initializer_list<float> values);
151
156 void set_bias(std::initializer_list<float> values);
157
159 void set_weight(const std::vector<float>& values);
160
162 void set_bias(const std::vector<float>& values);
163
164 [[nodiscard]] int64_t in_features() const { return in_features_; }
165 [[nodiscard]] int64_t out_features() const { return out_features_; }
167 [[nodiscard]] bool uses_bias() const { return use_bias_; }
168
169 [[nodiscard]] const Tensor& weight() const { return weight_; }
170 [[nodiscard]] const Tensor& bias() const { return bias_; }
171 [[nodiscard]] const Tensor& weight_grad() const { return weight_grad_; }
172 [[nodiscard]] const Tensor& bias_grad() const { return bias_grad_; }
173
196 [[nodiscard]] Tensor propagate_relevance(const Tensor& relevance_out, const LRPRuleConfig& config) override;
197
204 [[nodiscard]] Tensor forward_with_relu(const Tensor& input, ReluModule& relu);
205
207 [[nodiscard]] bool supports_lrp_rule(LRPRule) const override { return true; }
208
211 [[nodiscard]] std::vector<NamedParamRef> named_parameters() override;
212
213
215 [[nodiscard]] std::optional<DeviceType> compute_device() const override { return weight_.device(); }
216
217 void release_activations() override;
218
219protected:
220 [[nodiscard]] Tensor forward_impl(const Tensor& input) override;
221
222private:
223 int64_t in_features_;
224 int64_t out_features_;
225 bool use_bias_;
226 DeviceBackend* backend_;
227 Tensor weight_;
228 Tensor bias_;
229 Tensor weight_grad_;
230 Tensor bias_grad_;
231 Tensor last_input_;
232 Tensor last_pre_bias_output_;
233 bool has_forwarded_ = false;
234 std::unique_ptr<LoRAAdapter> lora_;
235 std::unique_ptr<IA3Adapter> ia3_;
236
238 [[nodiscard]] bool lora_active() const;
240 [[nodiscard]] bool ia3_active() const;
242 [[nodiscard]] Tensor forward_base(const Tensor& input);
243 [[nodiscard]] Tensor backward_base(const Tensor& grad_output);
244};
245
246} // namespace pulsatrix
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
One layer's rescaling vector l: (out) for an output-scaling layer, (in) for a feedforward one....
Definition ia3.hpp:42
y = x @ W + b, batched (x is (N, in_features), y is (N, out_features)) – migrated from the original u...
Definition linear_module.hpp:43
void set_ia3_enabled(bool enabled)
false runs the base layer.
void attach_ia3(bool feedforward)
Adds an IA³ rescaling vector l, all ones (TRN-10, ia3.hpp): the layer then computes (x W + b) ⊙ l,...
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Epsilon-rule LRP relevance propagation (Bach et al. 2015), applied independently per example in the b...
LinearModule(LinearModule &&) noexcept
Tensor backward(const Tensor &grad_output) override
Computes the gradient w.r.t. this module's input, and accumulates the weight/bias gradients internall...
void set_lora_enabled(bool enabled)
false runs the layer as its base layer, x W + b; true adds the update again.
std::optional< DeviceType > compute_device() const override
Where this layer computes, so forward() rejects an input on another device (FND-8).
Definition linear_module.hpp:215
const Tensor & weight() const
Definition linear_module.hpp:169
int64_t out_features() const
Definition linear_module.hpp:165
const LoRAAdapter * lora() const
Definition linear_module.hpp:91
void detach_ia3()
Removes the vector. A merged one stays in the weights. No-op without one.
void unmerge_ia3()
Divides it back out by l + 1e-8, as PEFT's unmerge() does.
LinearModule(int64_t in_features, int64_t out_features, DeviceBackend *backend, DeviceType device, bool use_bias=true)
Constructs a linear layer with zero-initialized weight/bias.
bool supports_lrp_rule(LRPRule) const override
Implements every LRPRule.
Definition linear_module.hpp:207
void unmerge_lora()
Subtracts the update again. W comes back up to float32 rounding, as with PEFT.
std::vector< NamedParamRef > named_parameters() override
weight, bias (unless built without one), then a LoRA adapter's lora_A.weight and lora_B....
void set_bias(const std::vector< float > &values)
Vector overload for runtime-sized sources – see Tensor's own vector ctor.
LoRAAdapter * lora()
The adapter, or nullptr.
Definition linear_module.hpp:90
void set_weight(const std::vector< float > &values)
Vector overload for runtime-sized sources – see Tensor's own vector ctor.
const IA3Adapter * ia3() const
Definition linear_module.hpp:120
void set_bias(std::initializer_list< float > values)
Overwrites the bias buffer – test/initialization use only.
OpType op_type() const override
Linear per charter's closed OpType set.
Definition linear_module.hpp:147
bool uses_bias() const
False when built with use_bias = false: bias() is then all zeros.
Definition linear_module.hpp:167
void merge_ia3()
Folds l into the weights (and, for output scaling, the bias), as PEFT's merge() does.
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
Tensor forward_with_relu(const Tensor &input, ReluModule &relu)
relu.forward(forward(input)) with the bias add and the ReLU in one kernel (HIP-6)....
int64_t in_features() const
Definition linear_module.hpp:164
void detach_lora()
Removes the adapter. A merged one stays in the weight. No-op without one.
Tensor forward_impl(const Tensor &input) override
The actual forward computation. Called by forward() after precondition checks.
void merge_lora()
Adds the update into the weight, W += (alpha / r) A B, so forward() runs at the base layer's cost....
const Tensor & bias_grad() const
Definition linear_module.hpp:172
LinearModule(int64_t in_features, int64_t out_features, DeviceBackend *backend, bool use_bias=true)
As above, on backend's own device (backend->device()).
const Tensor & weight_grad() const
Definition linear_module.hpp:171
IA3Adapter * ia3()
Definition linear_module.hpp:119
const Tensor & bias() const
Definition linear_module.hpp:170
void set_weight(std::initializer_list< float > values)
Overwrites the weight buffer – test/initialization use only.
void attach_lora(const LoRAConfig &config)
Adds a LoRA adapter (TRN-7, lora.hpp): from now on the layer computes x W + b + (alpha / r) x A B....
One layer's low-rank update: A (in, r) and B (r, out), stored like LinearModule's weight,...
Definition lora.hpp:58
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
y = max(x, 0), elementwise. No parameters, no parameter gradients.
Definition relu_module.hpp:16
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
DeviceType device() const
Which device this tensor's buffer conceptually resides on.
Definition tensor.hpp:122
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
Definition acquisition_functions.hpp:16
DeviceType
Which physical device a Tensor's buffer resides on.
Definition device_backend.hpp:17
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
LRPRule
The LRP rule family a module applies. Semantics follow Zennit 1.0.0 exactly (Anders et al....
Definition lrp_rule_config.hpp:21
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72
LoRA's settings. The defaults are PEFT's LoraConfig's.
Definition lora.hpp:34