pulsatrix 1.1.0
Loading...
Searching...
No Matches
transcoder.hpp
Go to the documentation of this file.
1
8#pragma once
9
10#include <cstdint>
11#include <memory>
12#include <optional>
13#include <vector>
14
17
18namespace pulsatrix {
19
22 int64_t k = 32;
25 bool skip = false;
28 int64_t k_aux = 0;
29 float aux_coefficient = 1.0f / 32.0f;
30 int64_t dead_after = 10'000'000;
31 uint64_t seed = 0;
32};
33
54class Transcoder : public Module, public Featurizer {
55public:
59
61 void initialize_bias(const Tensor& target);
62
63 // ---- Featurizer ----------------------------------------------------------------------
64 [[nodiscard]] int64_t input_dim() const override { return in_; }
65 [[nodiscard]] int64_t output_dim() const override { return out_; }
66 [[nodiscard]] int64_t num_features() const override { return m_; }
67 [[nodiscard]] Tensor encode(const Tensor& x) override;
69 [[nodiscard]] Tensor decode(const Tensor& codes) override;
71 [[nodiscard]] Tensor predict(const Tensor& x) override;
73 FeaturizerLoss loss_and_backward(const Tensor& x, std::vector<float>* codes = nullptr) override;
76 FeaturizerLoss loss_and_backward_with_target(const Tensor& x, const Tensor& target, std::vector<float>* codes = nullptr) override;
77 void normalize_decoder() override;
78 [[nodiscard]] std::vector<float> decoder_direction(int64_t i) override;
79 [[nodiscard]] Module& parameters_module() override { return *this; }
80
81 [[nodiscard]] std::vector<int64_t> dead_latents() const;
82 [[nodiscard]] const std::vector<int64_t>& inputs_since_fired() const { return since_fired_; }
84 void set_inputs_since_fired(const std::vector<int64_t>& since);
85 [[nodiscard]] const TranscoderOptions& options() const { return options_; }
86 [[nodiscard]] LinearModule& encoder() { return encoder_; }
87 [[nodiscard]] LinearModule& decoder() { return decoder_; }
89 [[nodiscard]] LinearModule* skip() { return skip_.get(); }
90
91 // ---- Module: input to prediction -----------------------------------------------------
94 [[nodiscard]] Tensor backward(const Tensor& grad_output) override;
98 [[nodiscard]] Tensor propagate_relevance(const Tensor& relevance_out, const LRPRuleConfig& config) override;
99 [[nodiscard]] OpType op_type() const override { return OpType::Composite; }
100 [[nodiscard]] std::vector<NamedParamRef> named_parameters() override;
101 [[nodiscard]] std::optional<DeviceType> compute_device() const override { return backend_->device(); }
102 void set_training(bool training) override;
103 void release_activations() override;
104
105protected:
107 [[nodiscard]] Tensor forward_impl(const Tensor& input) override;
108
109private:
110 void CheckInput(const Tensor& x, const char* who) const;
111 [[nodiscard]] std::vector<float> Codes(const Tensor& pre) const;
113 Tensor BackwardFromCodes(const std::vector<float>& codes, const std::vector<float>& g_codes);
114
115 int64_t in_, out_, m_;
116 DeviceBackend* backend_;
117 TranscoderOptions options_;
118 LinearModule encoder_;
119 LinearModule decoder_;
120 std::unique_ptr<LinearModule> skip_;
121 std::vector<int64_t> since_fired_;
122 std::vector<float> last_codes_;
123 std::vector<float> last_skip_out_;
124 int64_t last_rows_ = 0;
125};
126
127} // namespace pulsatrix
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
A featurizer maps activations x, (N, input_dim), to codes f = encode(x), (N, num_features),...
Definition featurizer.hpp:45
y = x @ W + b, batched (x is (N, in_features), y is (N, out_features)) – migrated from the original u...
Definition linear_module.hpp:43
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
A transcoder: z = ReLU(TopK_k(x W_enc + b_enc)), ŷ = z W_dec + b_dec [+ x W_skip],...
Definition transcoder.hpp:54
Module & parameters_module() override
The trainable parameters, for an optimizer's step() and zero_grad(), and for checkpoints.
Definition transcoder.hpp:79
std::vector< float > decoder_direction(int64_t i) override
Feature i's decoder direction, output_dim values.
int64_t num_features() const override
Definition transcoder.hpp:66
Tensor decode(const Tensor &codes) override
codes W_dec + b_dec: the prediction without the skip connection.
void set_training(bool training) override
Sets this module's training/eval mode. Defaults to training (matches every mainstream framework's Mod...
std::optional< DeviceType > compute_device() const override
The device this module computes on, so forward() can reject an input on another device before any ker...
Definition transcoder.hpp:101
LinearModule & encoder()
Definition transcoder.hpp:86
FeaturizerLoss loss_and_backward_with_target(const Tensor &x, const Tensor &target, std::vector< float > *codes=nullptr) override
The loss and its gradients; reconstruction is the prediction's mean squared error,...
void normalize_decoder() override
Rescales every feature so its decoder direction has unit L2 norm, scaling its code by the inverse whe...
void initialize_bias(const Tensor &target)
Sets b_dec to the mean of target, (N, output_dim).
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Relevance of the input: the output's relevance is split between the latents' part and the skip's in p...
Tensor encode(const Tensor &x) override
The codes for a batch, (N, num_features).
LinearModule * skip()
The skip connection; null unless options.skip.
Definition transcoder.hpp:89
const TranscoderOptions & options() const
Definition transcoder.hpp:85
LinearModule & decoder()
Definition transcoder.hpp:87
Tensor predict(const Tensor &x) override
The full prediction, skip connection included.
Tensor forward_impl(const Tensor &input) override
Tensor backward(const Tensor &grad_output) override
Backward through the latents (and the skip connection), adding to the parameters' gradients.
int64_t output_dim() const override
What decode() produces: input_dim() for an autoencoder.
Definition transcoder.hpp:65
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
FeaturizerLoss loss_and_backward(const Tensor &x, std::vector< float > *codes=nullptr) override
std::vector< NamedParamRef > named_parameters() override
This module's trainable parameters, each with its hierarchical name – the one place a module declares...
std::vector< int64_t > dead_latents() const
void set_inputs_since_fired(const std::vector< int64_t > &since)
int64_t input_dim() const override
Definition transcoder.hpp:64
OpType op_type() const override
This module's operation-category tag, for ComputationGraph node tagging.
Definition transcoder.hpp:99
Transcoder(int64_t input_dim, int64_t output_dim, int64_t num_features, DeviceBackend *backend, TranscoderOptions options={})
const std::vector< int64_t > & inputs_since_fired() const
Definition transcoder.hpp:82
The featurizer family's common interface (FEAT-1): models that rewrite activations as sparse codes ov...
Dense/fully-connected layer – the reference Module implementation.
Definition acquisition_functions.hpp:16
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
One batch's loss, split into its parts.
Definition featurizer.hpp:18
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72
Definition transcoder.hpp:20
int64_t k_aux
AuxK, as TopKSparseAutoencoder: the k_aux largest dead latents predict the error (0: half the output ...
Definition transcoder.hpp:28
bool skip
A skip transcoder: add x W_skip, a linear map from input to output that starts at zero,...
Definition transcoder.hpp:25
float aux_coefficient
Definition transcoder.hpp:29
int64_t dead_after
Definition transcoder.hpp:30
int64_t k
Latents kept per input (a TopK activation, as Paulo et al. use).
Definition transcoder.hpp:22
uint64_t seed
Definition transcoder.hpp:31