65 void use_components(std::vector<float> masks, std::vector<float> delta_masks = {});
66 [[nodiscard]]
bool uses_target()
const {
return target_mode_; }
69 [[nodiscard]]
const std::vector<float>&
mask_grad()
const {
return mask_grad_; }
71 [[nodiscard]]
const std::vector<float>&
delta_mask_grad()
const {
return delta_mask_grad_; }
84 [[nodiscard]]
const std::vector<float>&
target_weight()
const {
return target_w_; }
100 int64_t in_, out_, C_, G_;
102 std::vector<float> target_w_, bias_;
106 Tensor gate_in_w_, gate_in_b_, gate_out_w_, gate_out_b_;
107 Tensor gate_in_w_grad_, gate_in_b_grad_, gate_out_w_grad_, gate_out_b_grad_;
108 bool target_mode_ =
true;
109 std::vector<float> masks_, delta_masks_;
112 std::vector<float> h_, xw_, scaled_;
113 std::vector<float> mask_grad_, delta_mask_grad_;
115 std::optional<Tensor> target_input_;
116 std::optional<Tensor> gate_input_;
117 std::vector<float> gate_h_;
118 std::vector<float> gate_act_, gate_slope_;
197 [[nodiscard]]
const std::vector<ComponentLinear*>&
layers()
const {
return layers_; }
210 using NoiseFn = std::function<std::vector<float>(
const char* pass, int64_t layer, int64_t count)>;
218 std::vector<float> z, lower, upper;
220 std::vector<float> Draw(
const char* pass, int64_t layer, int64_t count);
222 float Divergence(
const std::vector<float>& pred,
const std::vector<float>& target, int64_t rows, std::vector<float>* grad)
const;
224 float Pass(
const Tensor& x,
const std::vector<float>& target,
float weight);
227 std::vector<ComponentLinear*> layers_;
228 DeviceBackend* backend_;
229 DecompositionOptions options_;
230 std::unique_ptr<Module> params_;
233 std::vector<std::vector<float>> sources_, source_m_, source_v_;
238template <
typename Optimizer>
A LinearModule split into C rank-one subcomponents: W ≈ V U, V (in, C) reading the input and U (C,...
Definition parameter_decomposition.hpp:44
bool uses_target() const
Definition parameter_decomposition.hpp:66
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
const std::vector< float > & mask_grad() const
dL/dm from the last backward() in components mode, (N, C).
Definition parameter_decomposition.hpp:69
Tensor backward(const Tensor &grad_output) override
Computes the gradient w.r.t. this module's input, given the gradient w.r.t. its output....
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Through the weight in use: the target's, or the masked subcomponents (and Δ).
LinearModule & U()
Definition parameter_decomposition.hpp:86
std::vector< float > component_weight()
The subcomponents' sum V U, (in, out) as LinearModule stores weights.
void use_components(std::vector< float > masks, std::vector< float > delta_masks={})
Later forward() calls use the subcomponents under masks (N, C), and the Δ weight under delta_masks (N...
int64_t out_features() const
Definition parameter_decomposition.hpp:57
LinearModule & V()
Definition parameter_decomposition.hpp:85
int64_t num_components() const
Definition parameter_decomposition.hpp:58
ComponentLinear(LinearModule &target, int64_t num_components, DeviceBackend *backend, int64_t gate_hidden=16, uint64_t seed=0)
Copies target's weight and bias, frozen.
void gate_backward(const std::vector< float > &grad_z)
Backward from dL/dz (N, C) for the last gate_outputs(): into the gates' parameters and,...
std::optional< DeviceType > compute_device() const override
The device this module computes on, so forward() can reject an input on another device before any ker...
Definition parameter_decomposition.hpp:93
int64_t gate_hidden() const
Definition parameter_decomposition.hpp:59
const std::vector< float > & target_weight() const
The frozen target weight, (in, out).
Definition parameter_decomposition.hpp:84
Tensor forward_impl(const Tensor &input) override
The actual forward computation. Called by forward() after precondition checks.
std::vector< float > gate_outputs()
The gates' outputs z (N, C) for the input of the last target-mode forward(); the decomposition turns ...
std::vector< NamedParamRef > named_parameters() override
This module's trainable parameters, each with its hierarchical name – the one place a module declares...
const std::vector< float > & delta_mask_grad() const
dL/dm_Δ from the last backward() with a Δ mask, (N,).
Definition parameter_decomposition.hpp:71
OpType op_type() const override
This module's operation-category tag, for ComputationGraph node tagging.
Definition parameter_decomposition.hpp:91
void use_target()
Later forward() calls use the target weight.
int64_t in_features() const
Definition parameter_decomposition.hpp:56
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
y = x @ W + b, batched (x is (N, in_features), y is (N, out_features)) – migrated from the original u...
Definition linear_module.hpp:43
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
Trains a decomposition of the ComponentLinear layers inside model against the model itself: the targe...
Definition parameter_decomposition.hpp:184
void set_noise(NoiseFn noise)
Definition parameter_decomposition.hpp:211
std::vector< std::vector< float > > & adversarial_sources()
VPD's persistent adversarial sources per layer, (N, C + 1), the last column the Δ mask's....
Definition parameter_decomposition.hpp:214
Tensor target_output(const Tensor &x)
The target model's output for x.
std::vector< std::vector< float > > causal_importances(const Tensor &x)
Each layer's causal importances (N, C) for x (lower clamp).
const DecompositionOptions & options() const
Definition parameter_decomposition.hpp:196
const std::vector< ComponentLinear * > & layers() const
Definition parameter_decomposition.hpp:197
DecompositionLoss loss_and_backward(const Tensor &x)
Every loss and its gradients, added to the layers' (zero them first).
ParameterDecomposition(Module &model, std::vector< ComponentLinear * > layers, DeviceBackend *backend, DecompositionOptions options={})
Module & parameters_module()
The trainable parameters, for an optimizer's step() and zero_grad().
void set_p(float p)
Importance minimality's exponent, for annealing (VPD: 2 down to 0.4).
Definition parameter_decomposition.hpp:195
std::function< std::vector< float >(const char *pass, int64_t layer, int64_t count)> NoiseFn
Replaces the uniform draws, for tests: called with a pass name ("stochastic", "layerwise",...
Definition parameter_decomposition.hpp:210
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
Dense/fully-connected layer – the reference Module implementation.
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
Definition acquisition_functions.hpp:16
std::vector< ImportanceStats > MeasureImportance(ParameterDecomposition &d, const Tensor &x)
ComponentAlignment AlignComponentsToRows(ComponentLinear &layer)
OutputDivergence
How the masked model's output is compared with the target's.
Definition parameter_decomposition.hpp:134
@ KlOnLogits
KL(softmax(y*) ‖ softmax(y)) over the last dimension, averaged over rows: for logits.
@ MeanSquaredError
mean((y - y*)²) over every value.
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
DecompositionMethod
Definition parameter_decomposition.hpp:121
@ VPD
VPD: the Δ weight is part of the forward pass under its own mask; reconstruction routes each input th...
@ SPD
SPD: masks m = g + (1 - g) r, r uniform; the losses are faithfulness, reconstruction with every layer...
DecompositionLoss TrainDecomposition(ParameterDecomposition &d, const Tensor &x, Optimizer &optimizer)
One training step: zero the gradients, loss_and_backward(), one optimizer step.
Definition parameter_decomposition.hpp:239
How well a layer's subcomponents line up with its target weight's rows, the input features (SPD's toy...
Definition parameter_decomposition.hpp:252
std::vector< int64_t > best
Per input feature: the best subcomponent.
Definition parameter_decomposition.hpp:256
double mean_norm_ratio
Definition parameter_decomposition.hpp:254
double mean_max_cosine
Definition parameter_decomposition.hpp:253
Definition parameter_decomposition.hpp:167
float adversarial
Definition parameter_decomposition.hpp:168
float stochastic
Definition parameter_decomposition.hpp:168
float layerwise
Definition parameter_decomposition.hpp:168
float total
Definition parameter_decomposition.hpp:168
float importance
Definition parameter_decomposition.hpp:168
float faithfulness
Definition parameter_decomposition.hpp:168
Definition parameter_decomposition.hpp:141
float p
Definition parameter_decomposition.hpp:155
DecompositionMethod method
Definition parameter_decomposition.hpp:142
float layerwise_coefficient
SPD: reconstruction with one layer masked at a time, averaged over layers.
Definition parameter_decomposition.hpp:150
OutputDivergence divergence
Definition parameter_decomposition.hpp:143
float adversarial_coefficient
VPD: reconstruction under adversarial masks.
Definition parameter_decomposition.hpp:152
float pgd_beta2
Definition parameter_decomposition.hpp:163
float stochastic_coefficient
Reconstruction with every layer masked (SPD), or k of them (VPD; 0.5 there).
Definition parameter_decomposition.hpp:148
int64_t pgd_steps
VPD's adversary: ascent steps per training step (before the one that uses the main loss's gradient),...
Definition parameter_decomposition.hpp:162
float leak
The leaky slope of the causal importance's clamps.
Definition parameter_decomposition.hpp:159
float frequency_beta
VPD: the frequency term's β in Σ_c [mean_c + β mean_c log2(1 + sum_c)].
Definition parameter_decomposition.hpp:157
float faithfulness_coefficient
Σ_l |W - V U|² / (number of weights): SPD's faithfulness, VPD's Δ loss (whose paper weights it 1e7).
Definition parameter_decomposition.hpp:146
float pgd_lr
Definition parameter_decomposition.hpp:163
float importance_coefficient
Importance minimality's weight and exponent p (set_p() anneals it).
Definition parameter_decomposition.hpp:154
float pgd_beta1
Definition parameter_decomposition.hpp:163
uint64_t seed
Definition parameter_decomposition.hpp:164
Causal importance statistics of one layer over inputs.
Definition parameter_decomposition.hpp:261
std::vector< float > max_importance
Each subcomponent's largest importance.
Definition parameter_decomposition.hpp:267
double l0
Mean number of subcomponents per input with importance above 0.01 (SPD's ci_l0).
Definition parameter_decomposition.hpp:263
int64_t alive
Subcomponents above 0.1 on some input.
Definition parameter_decomposition.hpp:265
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72