pulsatrix 1.1.0
Loading...
Searching...
No Matches
sparse_autoencoder.hpp
Go to the documentation of this file.
1
6#pragma once
7
8#include <cmath>
9#include <cstdint>
10#include <random>
11#include <stdexcept>
12#include <string>
13#include <vector>
14
20
21namespace pulsatrix {
22
74class SparseAutoencoder : public Module, public Featurizer {
75public:
94 SparseAutoencoder(int64_t dim, int64_t hidden_dim, float l1_lambda, DeviceBackend* backend)
95 : SparseAutoencoder(dim, hidden_dim, l1_lambda, backend, static_cast<unsigned>(next_seed())) {}
96
97 SparseAutoencoder(int64_t dim, int64_t hidden_dim, float l1_lambda, DeviceBackend* backend, unsigned seed)
98 : dim_(dim),
99 hidden_dim_(hidden_dim),
100 l1_lambda_(l1_lambda),
101 backend_(backend),
102 // Clamped only so a rejected dimension can't reach LinearModule/Shape and throw
103 // *their* message before the check below throws this class's own, clearer one --
104 // member initialization necessarily runs before the constructor body.
105 encoder_(dim > 0 ? dim : 1, hidden_dim > 0 ? hidden_dim : 1, backend),
106 relu_(backend),
107 decoder_(hidden_dim > 0 ? hidden_dim : 1, dim > 0 ? dim : 1, backend),
108 loss_(backend) {
109 if (dim <= 0) {
110 throw std::invalid_argument("SparseAutoencoder: dim must be positive");
111 }
112 if (hidden_dim <= 0) {
113 throw std::invalid_argument("SparseAutoencoder: hidden_dim must be positive");
114 }
115 if (l1_lambda < 0.0f) {
116 throw std::invalid_argument("SparseAutoencoder: l1_lambda must be non-negative");
117 }
118
119 std::mt19937 rng(seed);
120 // Fan-in scaling: each layer's init range shrinks with its own input width, so the
121 // pre-activation magnitude at step 0 stays O(1) regardless of dim/hidden_dim rather
122 // than growing with the layer's width (which for a deliberately overcomplete SAE
123 // would otherwise saturate the decoder's input from the first step).
124 init_layer(encoder_, rng, 1.0f / std::sqrt(static_cast<float>(dim)));
125 init_layer(decoder_, rng, 1.0f / std::sqrt(static_cast<float>(hidden_dim)));
127 }
128
168 template <typename OptimizerT>
169 float train_step(const Tensor& input_batch, OptimizerT& optimizer) {
170 validate_batch(input_batch, "SparseAutoencoder::train_step");
171 optimizer.zero_grad(*this);
172 const FeaturizerLoss loss = loss_and_backward(input_batch);
173 optimizer.step(*this);
174 if (unit_norm_decoder_) normalize_decoder();
175 return loss.reconstruction;
176 }
177
178 // ---- Featurizer ----------------------------------------------------------------------
179
180 [[nodiscard]] int64_t input_dim() const override { return dim_; }
181 [[nodiscard]] int64_t num_features() const override { return hidden_dim_; }
183 [[nodiscard]] Tensor encode(const Tensor& x) override {
184 validate_batch(x, "SparseAutoencoder::encode");
185 return hidden_codes(x);
186 }
188 [[nodiscard]] Tensor decode(const Tensor& codes) override {
189 if (codes.rank() != 2 || codes.shape().dim(1) != hidden_dim_ || codes.shape().dim(0) <= 0) {
190 throw std::invalid_argument("SparseAutoencoder::decode: codes must be (N, hidden_dim) with N > 0");
191 }
192 return decoder_.forward(codes);
193 }
199 FeaturizerLoss loss_and_backward(const Tensor& input_batch, std::vector<float>* codes = nullptr) override {
200 validate_batch(input_batch, "SparseAutoencoder::loss_and_backward");
201 const Tensor hidden = relu_.forward(encoder_.forward(input_batch));
202 const Tensor reconstruction = decoder_.forward(hidden);
203 FeaturizerLoss loss;
204 loss.reconstruction = loss_.forward(reconstruction, input_batch);
205 const Tensor grad_reconstruction = loss_.backward();
206 Tensor grad_hidden = decoder_.backward(grad_reconstruction);
207 const float batch_size = static_cast<float>(input_batch.shape().dim(0));
208 Tensor l1_grad(grad_hidden.shape(), backend_, grad_hidden.device());
209 l1_grad.fill(l1_lambda_ / batch_size);
210 grad_hidden.accumulate(l1_grad);
211 const Tensor grad_pre_activation = relu_.backward(grad_hidden);
212 (void)encoder_.backward(grad_pre_activation); // the input has no upstream to receive this
213 const std::vector<float> h = hidden.to_host_vector();
214 double sum = 0.0;
215 for (float v : h) sum += v;
216 loss.sparsity = static_cast<float>(l1_lambda_ * sum / batch_size);
217 loss.total = loss.reconstruction + loss.sparsity;
218 if (codes != nullptr) *codes = h;
219 return loss;
220 }
224 void normalize_decoder() override {
225 std::vector<float> dec = decoder_.weight().to_host_vector(), enc = encoder_.weight().to_host_vector(),
226 bias = encoder_.bias().to_host_vector();
227 for (int64_t i = 0; i < hidden_dim_; ++i) {
228 double sq = 0.0;
229 for (int64_t j = 0; j < dim_; ++j) sq += static_cast<double>(dec[static_cast<size_t>(i * dim_ + j)]) * dec[static_cast<size_t>(i * dim_ + j)];
230 const double norm = std::sqrt(sq);
231 if (norm == 0.0) continue;
232 for (int64_t j = 0; j < dim_; ++j) dec[static_cast<size_t>(i * dim_ + j)] = static_cast<float>(dec[static_cast<size_t>(i * dim_ + j)] / norm);
233 for (int64_t j = 0; j < dim_; ++j) enc[static_cast<size_t>(j * hidden_dim_ + i)] = static_cast<float>(enc[static_cast<size_t>(j * hidden_dim_ + i)] * norm);
234 bias[static_cast<size_t>(i)] = static_cast<float>(bias[static_cast<size_t>(i)] * norm);
235 }
236 decoder_.set_weight(dec);
237 encoder_.set_weight(enc);
238 encoder_.set_bias(bias);
239 }
240 [[nodiscard]] std::vector<float> decoder_direction(int64_t i) override {
241 if (i < 0 || i >= hidden_dim_) throw std::invalid_argument("SparseAutoencoder::decoder_direction: no such feature");
242 const std::vector<float> dec = decoder_.weight().to_host_vector();
243 return {dec.begin() + static_cast<std::ptrdiff_t>(i * dim_), dec.begin() + static_cast<std::ptrdiff_t>((i + 1) * dim_)};
244 }
245 [[nodiscard]] Module& parameters_module() override { return *this; }
246
248 [[nodiscard]] bool unit_norm_decoder() const { return unit_norm_decoder_; }
249 void set_unit_norm_decoder(bool on) { unit_norm_decoder_ = on; }
250
251 // ---- Module: input to reconstruction --------------------------------------------------
252
255 [[nodiscard]] Tensor backward(const Tensor& grad_output) override {
256 return encoder_.backward(relu_.backward(decoder_.backward(grad_output)));
257 }
259 [[nodiscard]] Tensor propagate_relevance(const Tensor& relevance_out, const LRPRuleConfig& config) override {
260 return encoder_.propagate_relevance(relu_.propagate_relevance(decoder_.propagate_relevance(relevance_out, config), config), config);
261 }
262 [[nodiscard]] OpType op_type() const override { return OpType::Composite; }
263 [[nodiscard]] std::vector<NamedParamRef> named_parameters() override {
264 std::vector<NamedParamRef> out;
265 append_named_parameters(out, "encoder", encoder_);
266 append_named_parameters(out, "decoder", decoder_);
267 return out;
268 }
269 [[nodiscard]] std::optional<DeviceType> compute_device() const override { return backend_->device(); }
270 void set_training(bool training) override {
271 Module::set_training(training);
272 encoder_.set_training(training);
273 decoder_.set_training(training);
274 }
275 void release_activations() override {
276 encoder_.release_activations();
277 relu_.release_activations();
278 decoder_.release_activations();
279 }
280
300 [[nodiscard]] Tensor reconstruct(const Tensor& input_batch) const {
301 validate_batch(input_batch, "SparseAutoencoder::reconstruct");
302 return decoder_.forward(hidden_codes(input_batch));
303 }
304
326 [[nodiscard]] float reconstruction_error(const Tensor& input_batch) const {
327 validate_batch(input_batch, "SparseAutoencoder::reconstruction_error");
328
329 const Tensor reconstruction = reconstruct(input_batch);
330 const int64_t n = reconstruction.numel();
331 float sum_squared = 0.0f;
332 for (int64_t i = 0; i < n; ++i) {
333 const float diff = reconstruction.data()[i] - input_batch.data()[i];
334 sum_squared += diff * diff;
335 }
336 return sum_squared / static_cast<float>(n);
337 }
338
355 [[nodiscard]] float mean_hidden_activation(const Tensor& input_batch) const {
356 validate_batch(input_batch, "SparseAutoencoder::mean_hidden_activation");
357
358 const Tensor hidden = hidden_codes(input_batch);
359 const int64_t n = hidden.numel();
360 float sum = 0.0f;
361 for (int64_t i = 0; i < n; ++i) {
362 sum += hidden.data()[i];
363 }
364 return sum / static_cast<float>(n);
365 }
366
368 [[nodiscard]] int64_t dim() const { return dim_; }
369
371 [[nodiscard]] int64_t hidden_dim() const { return hidden_dim_; }
372
374 [[nodiscard]] float l1_lambda() const { return l1_lambda_; }
375
380 [[nodiscard]] LinearModule& encoder() { return encoder_; }
381
383 [[nodiscard]] const LinearModule& encoder() const { return encoder_; }
384
386 [[nodiscard]] LinearModule& decoder() { return decoder_; }
387
389 [[nodiscard]] const LinearModule& decoder() const { return decoder_; }
390
391protected:
393 [[nodiscard]] Tensor forward_impl(const Tensor& input) override {
394 validate_batch(input, "SparseAutoencoder::forward");
395 return decoder_.forward(relu_.forward(encoder_.forward(input)));
396 }
397
398private:
401 [[nodiscard]] Tensor hidden_codes(const Tensor& input_batch) const { return relu_.forward(encoder_.forward(input_batch)); }
402
404 static void init_layer(LinearModule& layer, std::mt19937& rng, float scale) {
405 std::vector<float> weights(static_cast<size_t>(layer.weight().numel()));
406 for (float& w : weights) {
407 w = uniform_symmetric(rng, scale);
408 }
409 layer.set_weight(weights);
410 layer.set_bias(std::vector<float>(static_cast<size_t>(layer.bias().numel()), 0.0f));
411 }
412
414 static float uniform_symmetric(std::mt19937& rng, float scale) {
415 // std::uniform_real_distribution's mapping is implementation-defined, so identical
416 // seeds would not give identical weights across standard libraries. This mapping is
417 // fixed here, making a seeded SAE reproducible everywhere -- the same convention
418 // LinearProbe already uses.
419 const float unit = static_cast<float>(rng() - std::mt19937::min()) /
420 static_cast<float>(std::mt19937::max() - std::mt19937::min() + 1ull);
421 return (unit * 2.0f - 1.0f) * scale;
422 }
423
439 void validate_batch(const Tensor& input_batch, const char* method) const {
440 const std::string where(method);
441 if (input_batch.rank() != 2) {
442 throw std::invalid_argument(where + ": input_batch must be rank-2 (N, dim)");
443 }
444 if (input_batch.shape().dim(1) != dim_) {
445 throw std::invalid_argument(where + ": input_batch width must equal dim");
446 }
447 if (input_batch.shape().dim(0) <= 0) {
448 throw std::invalid_argument(where + ": batch must not be empty");
449 }
450 }
451
452 int64_t dim_;
453 int64_t hidden_dim_;
454 float l1_lambda_;
455 DeviceBackend* backend_;
456 mutable LinearModule encoder_;
457 mutable ReluModule relu_;
458 mutable LinearModule decoder_;
459 MSELoss loss_;
460 bool unit_norm_decoder_ = true;
461};
462
463} // namespace pulsatrix
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
A featurizer maps activations x, (N, input_dim), to codes f = encode(x), (N, num_features),...
Definition featurizer.hpp:45
y = x @ W + b, batched (x is (N, in_features), y is (N, out_features)) – migrated from the original u...
Definition linear_module.hpp:43
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Epsilon-rule LRP relevance propagation (Bach et al. 2015), applied independently per example in the b...
Tensor backward(const Tensor &grad_output) override
Computes the gradient w.r.t. this module's input, and accumulates the weight/bias gradients internall...
const Tensor & weight() const
Definition linear_module.hpp:169
void set_bias(std::initializer_list< float > values)
Overwrites the bias buffer – test/initialization use only.
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
const Tensor & bias() const
Definition linear_module.hpp:170
void set_weight(std::initializer_list< float > values)
Overwrites the weight buffer – test/initialization use only.
float forward(const Tensor &prediction, const Tensor &target)
Computes the loss value and caches prediction/target for backward().
Tensor backward() const
Computes the gradient w.r.t. the prediction: (2/n) * (prediction - target).
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
virtual void set_training(bool training)
Sets this module's training/eval mode. Defaults to training (matches every mainstream framework's Mod...
Definition module.hpp:271
Tensor forward(const Tensor &input)
Runs this module's forward computation.
Definition module.hpp:93
virtual void release_activations()
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
Definition module.hpp:284
Tensor backward(const Tensor &grad_output) override
Computes the gradient w.r.t. this module's input.
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Pass-through LRP relevance propagation.
int64_t dim(size_t index) const
Size of a single dimension.
Definition shape.hpp:93
A sparse autoencoder (SAE): LinearModule(dim, hidden_dim) -> ReluModule -> LinearModule(hidden_dim,...
Definition sparse_autoencoder.hpp:74
void set_training(bool training) override
Sets this module's training/eval mode. Defaults to training (matches every mainstream framework's Mod...
Definition sparse_autoencoder.hpp:270
OpType op_type() const override
This module's operation-category tag, for ComputationGraph node tagging.
Definition sparse_autoencoder.hpp:262
float reconstruction_error(const Tensor &input_batch) const
Mean squared reconstruction error over the batch – forward-only, no backward, no parameter update.
Definition sparse_autoencoder.hpp:326
const LinearModule & decoder() const
Const overload of decoder().
Definition sparse_autoencoder.hpp:389
int64_t input_dim() const override
Definition sparse_autoencoder.hpp:180
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Each layer's own rule, from the reconstruction back to the input.
Definition sparse_autoencoder.hpp:259
Module & parameters_module() override
The trainable parameters, for an optimizer's step() and zero_grad(), and for checkpoints.
Definition sparse_autoencoder.hpp:245
Tensor decode(const Tensor &codes) override
The reconstruction from codes, (N, dim).
Definition sparse_autoencoder.hpp:188
float mean_hidden_activation(const Tensor &input_batch) const
Mean hidden (post-ReLU) activation over the batch – the sparsity metric.
Definition sparse_autoencoder.hpp:355
int64_t dim() const
Width of the activation vectors this SAE reconstructs.
Definition sparse_autoencoder.hpp:368
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
Definition sparse_autoencoder.hpp:275
SparseAutoencoder(int64_t dim, int64_t hidden_dim, float l1_lambda, DeviceBackend *backend, unsigned seed)
Definition sparse_autoencoder.hpp:97
std::vector< NamedParamRef > named_parameters() override
This module's trainable parameters, each with its hierarchical name – the one place a module declares...
Definition sparse_autoencoder.hpp:263
const LinearModule & encoder() const
Const overload of encoder().
Definition sparse_autoencoder.hpp:383
float l1_lambda() const
Coefficient of the L1 penalty on the hidden activation.
Definition sparse_autoencoder.hpp:374
FeaturizerLoss loss_and_backward(const Tensor &input_batch, std::vector< float > *codes=nullptr) override
Mean squared reconstruction error plus l1_lambda * mean_over_inputs(sum_i f_i), with both gradients a...
Definition sparse_autoencoder.hpp:199
bool unit_norm_decoder() const
Whether train_step() keeps decoder directions at unit norm (the default).
Definition sparse_autoencoder.hpp:248
Tensor reconstruct(const Tensor &input_batch) const
The SAE's reconstruction of a batch – the encoder->ReLU->decoder forward path, forward-only: no loss,...
Definition sparse_autoencoder.hpp:300
LinearModule & encoder()
The encoder – inspection (its columns are the learned feature directions, which is the whole point of...
Definition sparse_autoencoder.hpp:380
int64_t hidden_dim() const
Width of the sparse hidden basis.
Definition sparse_autoencoder.hpp:371
SparseAutoencoder(int64_t dim, int64_t hidden_dim, float l1_lambda, DeviceBackend *backend)
Constructs a sparse autoencoder over activations of a given dimension.
Definition sparse_autoencoder.hpp:94
Tensor encode(const Tensor &x) override
The ReLU codes, (N, hidden_dim).
Definition sparse_autoencoder.hpp:183
void normalize_decoder() override
Unit-norm decoder rows, the norm moved into the encoder's row and bias: the ReLU passes a positive sc...
Definition sparse_autoencoder.hpp:224
Tensor backward(const Tensor &grad_output) override
Backward from the reconstruction's gradient to the input's, through the decoder, the ReLU and the enc...
Definition sparse_autoencoder.hpp:255
LinearModule & decoder()
The decoder – same rationale as encoder().
Definition sparse_autoencoder.hpp:386
float train_step(const Tensor &input_batch, OptimizerT &optimizer)
Runs one training step: forward, MSE reconstruction loss, backward with the L1 penalty gradient injec...
Definition sparse_autoencoder.hpp:169
std::optional< DeviceType > compute_device() const override
The device this module computes on, so forward() can reject an input on another device before any ker...
Definition sparse_autoencoder.hpp:269
std::vector< float > decoder_direction(int64_t i) override
Feature i's decoder direction, output_dim values.
Definition sparse_autoencoder.hpp:240
Tensor forward_impl(const Tensor &input) override
The reconstruction, (N, dim).
Definition sparse_autoencoder.hpp:393
int64_t num_features() const override
Definition sparse_autoencoder.hpp:181
void set_unit_norm_decoder(bool on)
Definition sparse_autoencoder.hpp:249
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
DeviceType device() const
Which device this tensor's buffer conceptually resides on.
Definition tensor.hpp:122
Tensor & fill(float value)
Sets every element to value. Safe no-op on a zero-element tensor.
int64_t rank() const
Number of dimensions – shape().rank().
Definition tensor.hpp:119
Tensor & accumulate(const Tensor &other)
In-place elementwise accumulation: this[i] += other[i] for every element.
int64_t numel() const
Total element count – shape().numel().
Definition tensor.hpp:116
const Shape & shape() const
This tensor's shape.
Definition tensor.hpp:113
std::vector< float > to_host_vector() const
Copies the whole buffer to a host vector through the owning backend, on any device.
const float * data() const
Raw buffer access. nullptr iff numel() == 0.
Definition tensor.hpp:167
One global seed for everything that isn't given its own, and a deterministic mode that forbids nondet...
The featurizer family's common interface (FEAT-1): models that rewrite activations as sparse codes ov...
Dense/fully-connected layer – the reference Module implementation.
Mean squared error loss.
Definition acquisition_functions.hpp:16
uint64_t next_seed()
The next seed in the global stream: a distinct, well-mixed 64-bit value per call, reproducible for a ...
void append_named_parameters(std::vector< NamedParamRef > &out, const std::string &prefix, Module &child)
Appends child's named parameters to out, each renamed to prefix.name – the one step every container's...
Definition module.hpp:301
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
ReLU activation – the second Module subclass, following LinearModule's pattern.
One batch's loss, split into its parts.
Definition featurizer.hpp:18
float reconstruction
Mean squared reconstruction error over every element.
Definition featurizer.hpp:23
float sparsity
The sparsity penalty's contribution (0 for featurizers that set sparsity directly).
Definition featurizer.hpp:25
float total
What was minimized: reconstruction + sparsity, except that a Matryoshka SAE adds every prefix's recon...
Definition featurizer.hpp:21
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72