pulsatrix 1.1.0
Loading...
Searching...
No Matches
parameter_decomposition.hpp
Go to the documentation of this file.
1
11#pragma once
12
13#include <cstdint>
14#include <functional>
15#include <memory>
16#include <optional>
17#include <vector>
18
20#include "pulsatrix/module.hpp"
21
22namespace pulsatrix {
23
44class ComponentLinear : public Module {
45public:
54 ComponentLinear(LinearModule& target, int64_t num_components, DeviceBackend* backend, int64_t gate_hidden = 16, uint64_t seed = 0);
55
56 [[nodiscard]] int64_t in_features() const { return in_; }
57 [[nodiscard]] int64_t out_features() const { return out_; }
58 [[nodiscard]] int64_t num_components() const { return C_; }
59 [[nodiscard]] int64_t gate_hidden() const { return G_; }
60
62 void use_target();
65 void use_components(std::vector<float> masks, std::vector<float> delta_masks = {});
66 [[nodiscard]] bool uses_target() const { return target_mode_; }
67
69 [[nodiscard]] const std::vector<float>& mask_grad() const { return mask_grad_; }
71 [[nodiscard]] const std::vector<float>& delta_mask_grad() const { return delta_mask_grad_; }
72
76 [[nodiscard]] std::vector<float> gate_outputs();
79 void gate_backward(const std::vector<float>& grad_z);
80
82 [[nodiscard]] std::vector<float> component_weight();
84 [[nodiscard]] const std::vector<float>& target_weight() const { return target_w_; }
85 [[nodiscard]] LinearModule& V() { return v_; }
86 [[nodiscard]] LinearModule& U() { return u_; }
87
88 [[nodiscard]] Tensor backward(const Tensor& grad_output) override;
90 [[nodiscard]] Tensor propagate_relevance(const Tensor& relevance_out, const LRPRuleConfig& config) override;
91 [[nodiscard]] OpType op_type() const override { return OpType::Composite; }
92 [[nodiscard]] std::vector<NamedParamRef> named_parameters() override;
93 [[nodiscard]] std::optional<DeviceType> compute_device() const override { return backend_->device(); }
94 void release_activations() override;
95
96protected:
97 [[nodiscard]] Tensor forward_impl(const Tensor& input) override;
98
99private:
100 int64_t in_, out_, C_, G_;
101 DeviceBackend* backend_;
102 std::vector<float> target_w_, bias_;
103 LinearModule w_;
104 LinearModule v_;
105 LinearModule u_;
106 Tensor gate_in_w_, gate_in_b_, gate_out_w_, gate_out_b_;
107 Tensor gate_in_w_grad_, gate_in_b_grad_, gate_out_w_grad_, gate_out_b_grad_;
108 bool target_mode_ = true;
109 std::vector<float> masks_, delta_masks_;
110 // The last forward (components mode).
111 int64_t rows_ = 0;
112 std::vector<float> h_, xw_, scaled_;
113 std::vector<float> mask_grad_, delta_mask_grad_;
114 // The last target-mode input and gate pass.
115 std::optional<Tensor> target_input_;
116 std::optional<Tensor> gate_input_;
117 std::vector<float> gate_h_;
118 std::vector<float> gate_act_, gate_slope_;
119};
120
125 SPD,
130 VPD,
131};
132
139};
140
166
168 float total = 0, faithfulness = 0, stochastic = 0, layerwise = 0, adversarial = 0, importance = 0;
169};
170
185public:
187 ParameterDecomposition(Module& model, std::vector<ComponentLinear*> layers, DeviceBackend* backend, DecompositionOptions options = {});
188
191
193 [[nodiscard]] Module& parameters_module();
195 void set_p(float p) { options_.p = p; }
196 [[nodiscard]] const DecompositionOptions& options() const { return options_; }
197 [[nodiscard]] const std::vector<ComponentLinear*>& layers() const { return layers_; }
198
200 [[nodiscard]] Tensor target_output(const Tensor& x);
202 [[nodiscard]] std::vector<std::vector<float>> causal_importances(const Tensor& x);
203
210 using NoiseFn = std::function<std::vector<float>(const char* pass, int64_t layer, int64_t count)>;
211 void set_noise(NoiseFn noise) { noise_ = std::move(noise); }
214 [[nodiscard]] std::vector<std::vector<float>>& adversarial_sources() { return sources_; }
215
216private:
217 struct Importance {
218 std::vector<float> z, lower, upper;
219 };
220 std::vector<float> Draw(const char* pass, int64_t layer, int64_t count);
222 float Divergence(const std::vector<float>& pred, const std::vector<float>& target, int64_t rows, std::vector<float>* grad) const;
224 float Pass(const Tensor& x, const std::vector<float>& target, float weight);
225
226 Module& model_;
227 std::vector<ComponentLinear*> layers_;
228 DeviceBackend* backend_;
229 DecompositionOptions options_;
230 std::unique_ptr<Module> params_;
231 uint64_t rng_state_;
232 NoiseFn noise_;
233 std::vector<std::vector<float>> sources_, source_m_, source_v_;
234 int64_t pgd_t_ = 0;
235};
236
238template <typename Optimizer>
240 optimizer.zero_grad(d.parameters_module());
241 const DecompositionLoss loss = d.loss_and_backward(x);
242 optimizer.step(d.parameters_module());
243 return loss;
244}
245
253 double mean_max_cosine = 0;
254 double mean_norm_ratio = 0;
256 std::vector<int64_t> best;
257};
259
263 double l0 = 0;
265 int64_t alive = 0;
267 std::vector<float> max_importance;
268};
269[[nodiscard]] std::vector<ImportanceStats> MeasureImportance(ParameterDecomposition& d, const Tensor& x);
270
271} // namespace pulsatrix
A LinearModule split into C rank-one subcomponents: W ≈ V U, V (in, C) reading the input and U (C,...
Definition parameter_decomposition.hpp:44
bool uses_target() const
Definition parameter_decomposition.hpp:66
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
const std::vector< float > & mask_grad() const
dL/dm from the last backward() in components mode, (N, C).
Definition parameter_decomposition.hpp:69
Tensor backward(const Tensor &grad_output) override
Computes the gradient w.r.t. this module's input, given the gradient w.r.t. its output....
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Through the weight in use: the target's, or the masked subcomponents (and Δ).
LinearModule & U()
Definition parameter_decomposition.hpp:86
std::vector< float > component_weight()
The subcomponents' sum V U, (in, out) as LinearModule stores weights.
void use_components(std::vector< float > masks, std::vector< float > delta_masks={})
Later forward() calls use the subcomponents under masks (N, C), and the Δ weight under delta_masks (N...
int64_t out_features() const
Definition parameter_decomposition.hpp:57
LinearModule & V()
Definition parameter_decomposition.hpp:85
int64_t num_components() const
Definition parameter_decomposition.hpp:58
ComponentLinear(LinearModule &target, int64_t num_components, DeviceBackend *backend, int64_t gate_hidden=16, uint64_t seed=0)
Copies target's weight and bias, frozen.
void gate_backward(const std::vector< float > &grad_z)
Backward from dL/dz (N, C) for the last gate_outputs(): into the gates' parameters and,...
std::optional< DeviceType > compute_device() const override
The device this module computes on, so forward() can reject an input on another device before any ker...
Definition parameter_decomposition.hpp:93
int64_t gate_hidden() const
Definition parameter_decomposition.hpp:59
const std::vector< float > & target_weight() const
The frozen target weight, (in, out).
Definition parameter_decomposition.hpp:84
Tensor forward_impl(const Tensor &input) override
The actual forward computation. Called by forward() after precondition checks.
std::vector< float > gate_outputs()
The gates' outputs z (N, C) for the input of the last target-mode forward(); the decomposition turns ...
std::vector< NamedParamRef > named_parameters() override
This module's trainable parameters, each with its hierarchical name – the one place a module declares...
const std::vector< float > & delta_mask_grad() const
dL/dm_Δ from the last backward() with a Δ mask, (N,).
Definition parameter_decomposition.hpp:71
OpType op_type() const override
This module's operation-category tag, for ComputationGraph node tagging.
Definition parameter_decomposition.hpp:91
void use_target()
Later forward() calls use the target weight.
int64_t in_features() const
Definition parameter_decomposition.hpp:56
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
y = x @ W + b, batched (x is (N, in_features), y is (N, out_features)) – migrated from the original u...
Definition linear_module.hpp:43
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
Trains a decomposition of the ComponentLinear layers inside model against the model itself: the targe...
Definition parameter_decomposition.hpp:184
void set_noise(NoiseFn noise)
Definition parameter_decomposition.hpp:211
std::vector< std::vector< float > > & adversarial_sources()
VPD's persistent adversarial sources per layer, (N, C + 1), the last column the Δ mask's....
Definition parameter_decomposition.hpp:214
Tensor target_output(const Tensor &x)
The target model's output for x.
std::vector< std::vector< float > > causal_importances(const Tensor &x)
Each layer's causal importances (N, C) for x (lower clamp).
const DecompositionOptions & options() const
Definition parameter_decomposition.hpp:196
const std::vector< ComponentLinear * > & layers() const
Definition parameter_decomposition.hpp:197
DecompositionLoss loss_and_backward(const Tensor &x)
Every loss and its gradients, added to the layers' (zero them first).
ParameterDecomposition(Module &model, std::vector< ComponentLinear * > layers, DeviceBackend *backend, DecompositionOptions options={})
Module & parameters_module()
The trainable parameters, for an optimizer's step() and zero_grad().
void set_p(float p)
Importance minimality's exponent, for annealing (VPD: 2 down to 0.4).
Definition parameter_decomposition.hpp:195
std::function< std::vector< float >(const char *pass, int64_t layer, int64_t count)> NoiseFn
Replaces the uniform draws, for tests: called with a pass name ("stochastic", "layerwise",...
Definition parameter_decomposition.hpp:210
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
Dense/fully-connected layer – the reference Module implementation.
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
Definition acquisition_functions.hpp:16
std::vector< ImportanceStats > MeasureImportance(ParameterDecomposition &d, const Tensor &x)
ComponentAlignment AlignComponentsToRows(ComponentLinear &layer)
OutputDivergence
How the masked model's output is compared with the target's.
Definition parameter_decomposition.hpp:134
@ KlOnLogits
KL(softmax(y*) ‖ softmax(y)) over the last dimension, averaged over rows: for logits.
@ MeanSquaredError
mean((y - y*)²) over every value.
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
DecompositionMethod
Definition parameter_decomposition.hpp:121
@ VPD
VPD: the Δ weight is part of the forward pass under its own mask; reconstruction routes each input th...
@ SPD
SPD: masks m = g + (1 - g) r, r uniform; the losses are faithfulness, reconstruction with every layer...
DecompositionLoss TrainDecomposition(ParameterDecomposition &d, const Tensor &x, Optimizer &optimizer)
One training step: zero the gradients, loss_and_backward(), one optimizer step.
Definition parameter_decomposition.hpp:239
How well a layer's subcomponents line up with its target weight's rows, the input features (SPD's toy...
Definition parameter_decomposition.hpp:252
std::vector< int64_t > best
Per input feature: the best subcomponent.
Definition parameter_decomposition.hpp:256
double mean_norm_ratio
Definition parameter_decomposition.hpp:254
double mean_max_cosine
Definition parameter_decomposition.hpp:253
Definition parameter_decomposition.hpp:167
float adversarial
Definition parameter_decomposition.hpp:168
float stochastic
Definition parameter_decomposition.hpp:168
float layerwise
Definition parameter_decomposition.hpp:168
float total
Definition parameter_decomposition.hpp:168
float importance
Definition parameter_decomposition.hpp:168
float faithfulness
Definition parameter_decomposition.hpp:168
Definition parameter_decomposition.hpp:141
float p
Definition parameter_decomposition.hpp:155
DecompositionMethod method
Definition parameter_decomposition.hpp:142
float layerwise_coefficient
SPD: reconstruction with one layer masked at a time, averaged over layers.
Definition parameter_decomposition.hpp:150
OutputDivergence divergence
Definition parameter_decomposition.hpp:143
float adversarial_coefficient
VPD: reconstruction under adversarial masks.
Definition parameter_decomposition.hpp:152
float pgd_beta2
Definition parameter_decomposition.hpp:163
float stochastic_coefficient
Reconstruction with every layer masked (SPD), or k of them (VPD; 0.5 there).
Definition parameter_decomposition.hpp:148
int64_t pgd_steps
VPD's adversary: ascent steps per training step (before the one that uses the main loss's gradient),...
Definition parameter_decomposition.hpp:162
float leak
The leaky slope of the causal importance's clamps.
Definition parameter_decomposition.hpp:159
float frequency_beta
VPD: the frequency term's β in Σ_c [mean_c + β mean_c log2(1 + sum_c)].
Definition parameter_decomposition.hpp:157
float faithfulness_coefficient
Σ_l |W - V U|² / (number of weights): SPD's faithfulness, VPD's Δ loss (whose paper weights it 1e7).
Definition parameter_decomposition.hpp:146
float pgd_lr
Definition parameter_decomposition.hpp:163
float importance_coefficient
Importance minimality's weight and exponent p (set_p() anneals it).
Definition parameter_decomposition.hpp:154
float pgd_beta1
Definition parameter_decomposition.hpp:163
uint64_t seed
Definition parameter_decomposition.hpp:164
Causal importance statistics of one layer over inputs.
Definition parameter_decomposition.hpp:261
std::vector< float > max_importance
Each subcomponent's largest importance.
Definition parameter_decomposition.hpp:267
double l0
Mean number of subcomponents per input with importance above 0.01 (SPD's ci_l0).
Definition parameter_decomposition.hpp:263
int64_t alive
Subcomponents above 0.1 on some input.
Definition parameter_decomposition.hpp:265
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72