110 throw std::invalid_argument(
"SparseAutoencoder: dim must be positive");
113 throw std::invalid_argument(
"SparseAutoencoder: hidden_dim must be positive");
116 throw std::invalid_argument(
"SparseAutoencoder: l1_lambda must be non-negative");
119 std::mt19937 rng(seed);
124 init_layer(encoder_, rng, 1.0f / std::sqrt(
static_cast<float>(
dim)));
125 init_layer(decoder_, rng, 1.0f / std::sqrt(
static_cast<float>(
hidden_dim)));
168 template <
typename OptimizerT>
170 validate_batch(input_batch,
"SparseAutoencoder::train_step");
171 optimizer.zero_grad(*
this);
173 optimizer.step(*
this);
180 [[nodiscard]] int64_t
input_dim()
const override {
return dim_; }
181 [[nodiscard]] int64_t
num_features()
const override {
return hidden_dim_; }
184 validate_batch(x,
"SparseAutoencoder::encode");
185 return hidden_codes(x);
190 throw std::invalid_argument(
"SparseAutoencoder::decode: codes must be (N, hidden_dim) with N > 0");
192 return decoder_.
forward(codes);
200 validate_batch(input_batch,
"SparseAutoencoder::loss_and_backward");
207 const float batch_size =
static_cast<float>(input_batch.
shape().
dim(0));
209 l1_grad.
fill(l1_lambda_ / batch_size);
212 (void)encoder_.
backward(grad_pre_activation);
215 for (
float v : h) sum += v;
216 loss.
sparsity =
static_cast<float>(l1_lambda_ * sum / batch_size);
218 if (codes !=
nullptr) *codes = h;
227 for (int64_t i = 0; i < hidden_dim_; ++i) {
229 for (int64_t j = 0; j < dim_; ++j) sq += static_cast<double>(dec[
static_cast<size_t>(i * dim_ + j)]) * dec[
static_cast<size_t>(i * dim_ + j)];
230 const double norm = std::sqrt(sq);
231 if (norm == 0.0)
continue;
232 for (int64_t j = 0; j < dim_; ++j) dec[static_cast<size_t>(i * dim_ + j)] =
static_cast<float>(dec[
static_cast<size_t>(i * dim_ + j)] / norm);
233 for (int64_t j = 0; j < dim_; ++j) enc[static_cast<size_t>(j * hidden_dim_ + i)] =
static_cast<float>(enc[
static_cast<size_t>(j * hidden_dim_ + i)] * norm);
234 bias[
static_cast<size_t>(i)] =
static_cast<float>(bias[
static_cast<size_t>(i)] * norm);
241 if (i < 0 || i >= hidden_dim_)
throw std::invalid_argument(
"SparseAutoencoder::decoder_direction: no such feature");
243 return {dec.begin() +
static_cast<std::ptrdiff_t
>(i * dim_), dec.begin() +
static_cast<std::ptrdiff_t
>((i + 1) * dim_)};
264 std::vector<NamedParamRef> out;
301 validate_batch(input_batch,
"SparseAutoencoder::reconstruct");
302 return decoder_.
forward(hidden_codes(input_batch));
327 validate_batch(input_batch,
"SparseAutoencoder::reconstruction_error");
330 const int64_t n = reconstruction.
numel();
331 float sum_squared = 0.0f;
332 for (int64_t i = 0; i < n; ++i) {
333 const float diff = reconstruction.
data()[i] - input_batch.
data()[i];
334 sum_squared += diff * diff;
336 return sum_squared /
static_cast<float>(n);
356 validate_batch(input_batch,
"SparseAutoencoder::mean_hidden_activation");
358 const Tensor hidden = hidden_codes(input_batch);
359 const int64_t n = hidden.
numel();
361 for (int64_t i = 0; i < n; ++i) {
362 sum += hidden.
data()[i];
364 return sum /
static_cast<float>(n);
368 [[nodiscard]] int64_t
dim()
const {
return dim_; }
371 [[nodiscard]] int64_t
hidden_dim()
const {
return hidden_dim_; }
374 [[nodiscard]]
float l1_lambda()
const {
return l1_lambda_; }
394 validate_batch(input,
"SparseAutoencoder::forward");
404 static void init_layer(LinearModule& layer, std::mt19937& rng,
float scale) {
405 std::vector<float> weights(
static_cast<size_t>(layer.weight().numel()));
406 for (
float& w : weights) {
407 w = uniform_symmetric(rng, scale);
409 layer.set_weight(weights);
410 layer.set_bias(std::vector<float>(
static_cast<size_t>(layer.bias().numel()), 0.0f));
414 static float uniform_symmetric(std::mt19937& rng,
float scale) {
419 const float unit =
static_cast<float>(rng() - std::mt19937::min()) /
420 static_cast<float>(std::mt19937::max() - std::mt19937::min() + 1ull);
421 return (unit * 2.0f - 1.0f) * scale;
439 void validate_batch(
const Tensor& input_batch,
const char* method)
const {
440 const std::string where(method);
441 if (input_batch.rank() != 2) {
442 throw std::invalid_argument(where +
": input_batch must be rank-2 (N, dim)");
444 if (input_batch.shape().dim(1) != dim_) {
445 throw std::invalid_argument(where +
": input_batch width must equal dim");
447 if (input_batch.shape().dim(0) <= 0) {
448 throw std::invalid_argument(where +
": batch must not be empty");
455 DeviceBackend* backend_;
456 mutable LinearModule encoder_;
457 mutable ReluModule relu_;
458 mutable LinearModule decoder_;
460 bool unit_norm_decoder_ =
true;
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
A featurizer maps activations x, (N, input_dim), to codes f = encode(x), (N, num_features),...
Definition featurizer.hpp:45
y = x @ W + b, batched (x is (N, in_features), y is (N, out_features)) – migrated from the original u...
Definition linear_module.hpp:43
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Epsilon-rule LRP relevance propagation (Bach et al. 2015), applied independently per example in the b...
Tensor backward(const Tensor &grad_output) override
Computes the gradient w.r.t. this module's input, and accumulates the weight/bias gradients internall...
const Tensor & weight() const
Definition linear_module.hpp:169
void set_bias(std::initializer_list< float > values)
Overwrites the bias buffer – test/initialization use only.
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
const Tensor & bias() const
Definition linear_module.hpp:170
void set_weight(std::initializer_list< float > values)
Overwrites the weight buffer – test/initialization use only.
float forward(const Tensor &prediction, const Tensor &target)
Computes the loss value and caches prediction/target for backward().
Tensor backward() const
Computes the gradient w.r.t. the prediction: (2/n) * (prediction - target).
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
virtual void set_training(bool training)
Sets this module's training/eval mode. Defaults to training (matches every mainstream framework's Mod...
Definition module.hpp:271
Tensor forward(const Tensor &input)
Runs this module's forward computation.
Definition module.hpp:93
virtual void release_activations()
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
Definition module.hpp:284
Tensor backward(const Tensor &grad_output) override
Computes the gradient w.r.t. this module's input.
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Pass-through LRP relevance propagation.
int64_t dim(size_t index) const
Size of a single dimension.
Definition shape.hpp:93
A sparse autoencoder (SAE): LinearModule(dim, hidden_dim) -> ReluModule -> LinearModule(hidden_dim,...
Definition sparse_autoencoder.hpp:74
void set_training(bool training) override
Sets this module's training/eval mode. Defaults to training (matches every mainstream framework's Mod...
Definition sparse_autoencoder.hpp:270
OpType op_type() const override
This module's operation-category tag, for ComputationGraph node tagging.
Definition sparse_autoencoder.hpp:262
float reconstruction_error(const Tensor &input_batch) const
Mean squared reconstruction error over the batch – forward-only, no backward, no parameter update.
Definition sparse_autoencoder.hpp:326
const LinearModule & decoder() const
Const overload of decoder().
Definition sparse_autoencoder.hpp:389
int64_t input_dim() const override
Definition sparse_autoencoder.hpp:180
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Each layer's own rule, from the reconstruction back to the input.
Definition sparse_autoencoder.hpp:259
Module & parameters_module() override
The trainable parameters, for an optimizer's step() and zero_grad(), and for checkpoints.
Definition sparse_autoencoder.hpp:245
Tensor decode(const Tensor &codes) override
The reconstruction from codes, (N, dim).
Definition sparse_autoencoder.hpp:188
float mean_hidden_activation(const Tensor &input_batch) const
Mean hidden (post-ReLU) activation over the batch – the sparsity metric.
Definition sparse_autoencoder.hpp:355
int64_t dim() const
Width of the activation vectors this SAE reconstructs.
Definition sparse_autoencoder.hpp:368
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
Definition sparse_autoencoder.hpp:275
SparseAutoencoder(int64_t dim, int64_t hidden_dim, float l1_lambda, DeviceBackend *backend, unsigned seed)
Definition sparse_autoencoder.hpp:97
std::vector< NamedParamRef > named_parameters() override
This module's trainable parameters, each with its hierarchical name – the one place a module declares...
Definition sparse_autoencoder.hpp:263
const LinearModule & encoder() const
Const overload of encoder().
Definition sparse_autoencoder.hpp:383
float l1_lambda() const
Coefficient of the L1 penalty on the hidden activation.
Definition sparse_autoencoder.hpp:374
FeaturizerLoss loss_and_backward(const Tensor &input_batch, std::vector< float > *codes=nullptr) override
Mean squared reconstruction error plus l1_lambda * mean_over_inputs(sum_i f_i), with both gradients a...
Definition sparse_autoencoder.hpp:199
bool unit_norm_decoder() const
Whether train_step() keeps decoder directions at unit norm (the default).
Definition sparse_autoencoder.hpp:248
Tensor reconstruct(const Tensor &input_batch) const
The SAE's reconstruction of a batch – the encoder->ReLU->decoder forward path, forward-only: no loss,...
Definition sparse_autoencoder.hpp:300
LinearModule & encoder()
The encoder – inspection (its columns are the learned feature directions, which is the whole point of...
Definition sparse_autoencoder.hpp:380
int64_t hidden_dim() const
Width of the sparse hidden basis.
Definition sparse_autoencoder.hpp:371
SparseAutoencoder(int64_t dim, int64_t hidden_dim, float l1_lambda, DeviceBackend *backend)
Constructs a sparse autoencoder over activations of a given dimension.
Definition sparse_autoencoder.hpp:94
Tensor encode(const Tensor &x) override
The ReLU codes, (N, hidden_dim).
Definition sparse_autoencoder.hpp:183
void normalize_decoder() override
Unit-norm decoder rows, the norm moved into the encoder's row and bias: the ReLU passes a positive sc...
Definition sparse_autoencoder.hpp:224
Tensor backward(const Tensor &grad_output) override
Backward from the reconstruction's gradient to the input's, through the decoder, the ReLU and the enc...
Definition sparse_autoencoder.hpp:255
LinearModule & decoder()
The decoder – same rationale as encoder().
Definition sparse_autoencoder.hpp:386
float train_step(const Tensor &input_batch, OptimizerT &optimizer)
Runs one training step: forward, MSE reconstruction loss, backward with the L1 penalty gradient injec...
Definition sparse_autoencoder.hpp:169
std::optional< DeviceType > compute_device() const override
The device this module computes on, so forward() can reject an input on another device before any ker...
Definition sparse_autoencoder.hpp:269
std::vector< float > decoder_direction(int64_t i) override
Feature i's decoder direction, output_dim values.
Definition sparse_autoencoder.hpp:240
Tensor forward_impl(const Tensor &input) override
The reconstruction, (N, dim).
Definition sparse_autoencoder.hpp:393
int64_t num_features() const override
Definition sparse_autoencoder.hpp:181
void set_unit_norm_decoder(bool on)
Definition sparse_autoencoder.hpp:249
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
DeviceType device() const
Which device this tensor's buffer conceptually resides on.
Definition tensor.hpp:122
Tensor & fill(float value)
Sets every element to value. Safe no-op on a zero-element tensor.
int64_t rank() const
Number of dimensions – shape().rank().
Definition tensor.hpp:119
Tensor & accumulate(const Tensor &other)
In-place elementwise accumulation: this[i] += other[i] for every element.
int64_t numel() const
Total element count – shape().numel().
Definition tensor.hpp:116
const Shape & shape() const
This tensor's shape.
Definition tensor.hpp:113
std::vector< float > to_host_vector() const
Copies the whole buffer to a host vector through the owning backend, on any device.
const float * data() const
Raw buffer access. nullptr iff numel() == 0.
Definition tensor.hpp:167
One global seed for everything that isn't given its own, and a deterministic mode that forbids nondet...
The featurizer family's common interface (FEAT-1): models that rewrite activations as sparse codes ov...
Dense/fully-connected layer – the reference Module implementation.
Definition acquisition_functions.hpp:16
uint64_t next_seed()
The next seed in the global stream: a distinct, well-mixed 64-bit value per call, reproducible for a ...
void append_named_parameters(std::vector< NamedParamRef > &out, const std::string &prefix, Module &child)
Appends child's named parameters to out, each renamed to prefix.name – the one step every container's...
Definition module.hpp:301
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
ReLU activation – the second Module subclass, following LinearModule's pattern.
One batch's loss, split into its parts.
Definition featurizer.hpp:18
float reconstruction
Mean squared reconstruction error over every element.
Definition featurizer.hpp:23
float sparsity
The sparsity penalty's contribution (0 for featurizers that set sparsity directly).
Definition featurizer.hpp:25
float total
What was minimized: reconstruction + sparsity, except that a Matryoshka SAE adds every prefix's recon...
Definition featurizer.hpp:21
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72