pulsatrix 1.1.0
Loading...
Searching...
No Matches
featurizer.hpp
Go to the documentation of this file.
1
7#pragma once
8
9#include <cstdint>
10#include <vector>
11
12#include "pulsatrix/module.hpp"
13#include "pulsatrix/tensor.hpp"
14
15namespace pulsatrix {
16
21 float total = 0.0f;
23 float reconstruction = 0.0f;
25 float sparsity = 0.0f;
26};
27
46public:
47 virtual ~Featurizer() = default;
48
49 [[nodiscard]] virtual int64_t input_dim() const = 0;
50 [[nodiscard]] virtual int64_t num_features() const = 0;
52 [[nodiscard]] virtual int64_t output_dim() const { return input_dim(); }
58 [[nodiscard]] virtual int64_t block_size() const { return 1; }
59
61 [[nodiscard]] virtual Tensor encode(const Tensor& x) = 0;
63 [[nodiscard]] virtual Tensor decode(const Tensor& codes) = 0;
66 [[nodiscard]] virtual Tensor predict(const Tensor& x) { return decode(encode(x)); }
67
74 virtual FeaturizerLoss loss_and_backward(const Tensor& x, std::vector<float>* codes = nullptr) = 0;
79 virtual FeaturizerLoss loss_and_backward_with_target(const Tensor& x, const Tensor& target, std::vector<float>* codes = nullptr);
80
87 virtual void normalize_decoder() = 0;
88
90 [[nodiscard]] virtual std::vector<float> decoder_direction(int64_t i) = 0;
91
94 [[nodiscard]] virtual Module& parameters_module() = 0;
95};
96
103public:
105 explicit FeatureActivityTracker(int64_t num_features);
106
110 void observe(const std::vector<float>& codes, float threshold = 0.0f);
111
113 [[nodiscard]] int64_t inputs_seen() const { return inputs_; }
115 [[nodiscard]] std::vector<double> firing_rates() const;
118 [[nodiscard]] std::vector<int64_t> dead(int64_t window) const;
120 [[nodiscard]] double dead_fraction(int64_t window) const;
122 [[nodiscard]] std::vector<int64_t> dense(double rate = 0.1) const;
124 void reset();
125
126private:
127 int64_t inputs_ = 0;
128 std::vector<int64_t> fired_;
129 std::vector<int64_t> last_fired_;
130};
131
135[[nodiscard]] double MeanL0(const std::vector<float>& codes, int64_t num_features, float threshold = 0.0f);
138[[nodiscard]] double MeanL0(Featurizer& featurizer, const Tensor& x, float threshold = 0.0f);
139
147template <typename Optimizer>
148FeaturizerLoss TrainFeaturizer(Featurizer& featurizer, const Tensor& x, Optimizer& optimizer, bool unit_norm_decoder = true,
149 FeatureActivityTracker* tracker = nullptr) {
150 optimizer.zero_grad(featurizer.parameters_module());
151 std::vector<float> codes;
152 const FeaturizerLoss loss = featurizer.loss_and_backward(x, tracker != nullptr ? &codes : nullptr);
153 optimizer.step(featurizer.parameters_module());
154 if (unit_norm_decoder) featurizer.normalize_decoder();
155 if (tracker != nullptr) tracker->observe(codes);
156 return loss;
157}
158
160template <typename Optimizer>
161FeaturizerLoss TrainFeaturizer(Featurizer& featurizer, const Tensor& x, const Tensor& target, Optimizer& optimizer,
162 bool unit_norm_decoder = true, FeatureActivityTracker* tracker = nullptr) {
163 optimizer.zero_grad(featurizer.parameters_module());
164 std::vector<float> codes;
165 const FeaturizerLoss loss = featurizer.loss_and_backward_with_target(x, target, tracker != nullptr ? &codes : nullptr);
166 optimizer.step(featurizer.parameters_module());
167 if (unit_norm_decoder) featurizer.normalize_decoder();
168 if (tracker != nullptr) tracker->observe(codes);
169 return loss;
170}
171
172} // namespace pulsatrix
How often each feature fires, and how long since it last did. A feature that hasn't fired in a long w...
Definition featurizer.hpp:102
std::vector< double > firing_rates() const
The share of observed inputs each feature fired on.
double dead_fraction(int64_t window) const
dead(window)'s share of all features.
std::vector< int64_t > dead(int64_t window) const
Features that haven't fired in the last window inputs (or ever, if fewer were seen),...
FeatureActivityTracker(int64_t num_features)
void observe(const std::vector< float > &codes, float threshold=0.0f)
Records a batch of codes, row-major (N, num_features); a code above threshold in magnitude counts as ...
int64_t inputs_seen() const
Inputs observed so far.
Definition featurizer.hpp:113
void reset()
Forgets everything observed.
std::vector< int64_t > dense(double rate=0.1) const
Features firing on more than rate of the inputs observed.
A featurizer maps activations x, (N, input_dim), to codes f = encode(x), (N, num_features),...
Definition featurizer.hpp:45
virtual int64_t num_features() const =0
virtual Module & parameters_module()=0
The trainable parameters, for an optimizer's step() and zero_grad(), and for checkpoints.
virtual int64_t output_dim() const
What decode() produces: input_dim() for an autoencoder.
Definition featurizer.hpp:52
virtual Tensor encode(const Tensor &x)=0
The codes for a batch, (N, num_features).
virtual int64_t input_dim() const =0
virtual std::vector< float > decoder_direction(int64_t i)=0
Feature i's decoder direction, output_dim values.
virtual int64_t block_size() const
Codes come in blocks of this many (FEAT-6): a block-sparse featurizer's feature is a block,...
Definition featurizer.hpp:58
virtual FeaturizerLoss loss_and_backward_with_target(const Tensor &x, const Tensor &target, std::vector< float > *codes=nullptr)
The loss for predicting target, (N, output_dim), from x: a transcoder's.
virtual Tensor decode(const Tensor &codes)=0
The reconstruction from codes, (N, output_dim).
virtual FeaturizerLoss loss_and_backward(const Tensor &x, std::vector< float > *codes=nullptr)=0
Computes the training loss on x and adds its gradients to the parameters' (zero them first)....
virtual Tensor predict(const Tensor &x)
The full output for a batch, (N, output_dim): decode(encode(x)), plus any part that bypasses the code...
Definition featurizer.hpp:66
virtual ~Featurizer()=default
virtual void normalize_decoder()=0
Rescales every feature so its decoder direction has unit L2 norm, scaling its code by the inverse whe...
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
Definition acquisition_functions.hpp:16
FeaturizerLoss TrainFeaturizer(Featurizer &featurizer, const Tensor &x, Optimizer &optimizer, bool unit_norm_decoder=true, FeatureActivityTracker *tracker=nullptr)
One training step for any featurizer: zero the gradients, loss_and_backward(), one optimizer step,...
Definition featurizer.hpp:148
double MeanL0(const std::vector< float > &codes, int64_t num_features, float threshold=0.0f)
The mean number of codes above threshold in magnitude per input (L0), for codes row-major (N,...
One batch's loss, split into its parts.
Definition featurizer.hpp:18
float reconstruction
Mean squared reconstruction error over every element.
Definition featurizer.hpp:23
float sparsity
The sparsity penalty's contribution (0 for featurizers that set sparsity directly).
Definition featurizer.hpp:25
float total
What was minimized: reconstruction + sparsity, except that a Matryoshka SAE adds every prefix's recon...
Definition featurizer.hpp:21
N-dimensional tensor – owns a buffer via DeviceBackend*, RAII (Rule of Five).