pulsatrix 1.1.0
Loading...
Searching...
No Matches
encoder_lm.hpp
Go to the documentation of this file.
1
7#pragma once
8
9#include <functional>
10#include <memory>
11#include <optional>
12#include <string>
13#include <string_view>
14#include <vector>
15
17#include "pulsatrix/causal_lm.hpp" // WeightMapping, LoadWeights
23
24namespace pulsatrix {
25
27enum class EncoderLMHead {
29 Esm,
31 Linear,
32};
33
36 std::string architecture;
37 std::string model_type;
38 int64_t vocab_size = 0;
39 int64_t hidden_size = 0;
40 int64_t intermediate_size = 0;
41 int64_t num_hidden_layers = 0;
43 float layer_norm_eps = 1e-12f;
44 float rope_theta = 10000.0f;
47 bool token_dropout = false;
48 int64_t mask_token_id = -1;
49 int64_t pad_token_id = -1;
50 std::string dtype;
57 std::vector<double> rope_inverse_frequencies;
60 bool attention_bias = true;
63};
64
71[[nodiscard]] EncoderLMConfig ParseEsmConfig(std::string_view json);
73[[nodiscard]] EncoderLMConfig ReadEsmConfig(const std::string& path);
74
84[[nodiscard]] EncoderLMConfig ParseMdlmConfig(std::string_view json);
85
109class EncoderLM : public Module {
110public:
113
115 [[nodiscard]] Tensor backward(const Tensor& grad_output) override;
123 [[nodiscard]] Tensor backward_hidden(const Tensor& grad_hidden);
125 [[nodiscard]] Tensor propagate_relevance(const Tensor& relevance_out, const LRPRuleConfig& config) override;
131 [[nodiscard]] std::vector<Tensor> propagate_relevance_by_layer(const Tensor& relevance_out, const LRPRuleConfig& config);
138 [[nodiscard]] std::vector<Tensor> propagate_hidden_relevance_by_layer(const Tensor& relevance_hidden, const LRPRuleConfig& config);
139 [[nodiscard]] OpType op_type() const override { return OpType::Composite; }
140 [[nodiscard]] std::vector<NamedParamRef> named_parameters() override;
141 [[nodiscard]] std::vector<NamedModuleRef> named_children() override;
142 [[nodiscard]] std::vector<NamedBufferRef> named_buffers() override;
143 void set_training(bool training) override;
144 [[nodiscard]] std::optional<DeviceType> compute_device() const override { return backend_->device(); }
146 void release_activations() override;
153 void set_keep_activations(bool keep) { keep_activations_ = keep; }
154 [[nodiscard]] bool keep_activations() const { return keep_activations_; }
155
158 using AttentionObserver = std::function<void(int64_t layer, const Tensor& attention)>;
164 void set_attention_observer(AttentionObserver observer) { attention_observer_ = std::move(observer); }
168 void set_hidden_state_hook(HiddenStateHook hook) { hidden_state_hook_ = std::move(hook); }
169
175 void set_padding_mask(const Tensor& keep);
178
179 [[nodiscard]] const EncoderLMConfig& config() const { return config_; }
180 [[nodiscard]] int64_t num_layers() const { return static_cast<int64_t>(layers_.size()); }
181 [[nodiscard]] EncoderBlock& layer(int64_t i) { return *layers_.at(static_cast<size_t>(i)); }
182 [[nodiscard]] EmbeddingModule& embed_tokens() { return embed_; }
188 [[nodiscard]] const std::vector<Tensor>& hidden_states() const { return hidden_; }
191 [[nodiscard]] const Tensor& last_hidden_state() const { return last_hidden_; }
192
193protected:
196 [[nodiscard]] Tensor forward_impl(const Tensor& input) override;
197
198private:
199 [[nodiscard]] Tensor Rows(Module& m, const Tensor& x, int64_t width) const;
200 [[nodiscard]] Tensor RowsBackward(Module& m, const Tensor& g, int64_t width) const;
201 [[nodiscard]] Tensor RowsRelevance(Module& m, const Tensor& r, int64_t width, const LRPRuleConfig& config) const;
204 [[nodiscard]] Tensor HeadRelevance(const Tensor& relevance_out, const LRPRuleConfig& config);
206 [[nodiscard]] std::vector<Module*> HeadModules();
207
208 EncoderLMConfig config_;
209 DeviceBackend* backend_;
210 EmbeddingModule embed_;
211 std::vector<std::unique_ptr<EncoderBlock>> layers_;
212 LayerNormModule norm_;
213 // ESM's head; empty for a linear head.
214 std::unique_ptr<LinearModule> lm_dense_;
215 std::unique_ptr<ActivationModule> lm_gelu_;
216 std::unique_ptr<LayerNormModule> lm_norm_;
217 std::unique_ptr<TiedLMHeadModule> lm_decoder_;
218 Tensor lm_bias_;
219 Tensor lm_bias_grad_;
220 // A linear head; empty for ESM's.
221 std::unique_ptr<LinearModule> lm_linear_;
222
223 std::vector<float> padding_keep_;
224 Shape padding_shape_ = Shape({0});
225 Tensor embed_scale_;
226 Tensor last_logits_nobias_;
227 std::vector<Tensor> hidden_;
228 Tensor last_hidden_;
229 Shape last_ids_shape_ = Shape({0});
230 bool has_forwarded_ = false;
231 bool keep_activations_ = true;
232 AttentionObserver attention_observer_;
233 HiddenStateHook hidden_state_hook_;
234};
235
240[[nodiscard]] std::vector<WeightMapping> EsmMapping(const EncoderLMConfig& config);
244[[nodiscard]] std::vector<std::string> EsmIgnoredTensors(const EncoderLMConfig& config);
245
251[[nodiscard]] std::vector<WeightMapping> MdlmMapping(const EncoderLMConfig& config);
252
272[[nodiscard]] std::unique_ptr<EncoderLM> LoadEncoderLM(const std::string& directory, DeviceBackend* backend);
273
274} // namespace pulsatrix
A pointwise activation chosen at construction: ReLU, tanh, sigmoid, SiLU, exact GELU or GELU's tanh a...
A Llama-family decoder-only language model built from a Hugging Face config, and loading its Hugging ...
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
Embedding lookup table, rank-2 input (N, L) of float-encoded indices -> rank-3 output (N,...
Definition embedding_module.hpp:39
One encoder layer. Shape (N, L, d_model) -> (N, L, d_model).
Definition encoder_block.hpp:72
A masked language model: token embeddings (with ESM's token-dropout scaling, when the config has it),...
Definition encoder_lm.hpp:109
const EncoderLMConfig & config() const
Definition encoder_lm.hpp:179
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Relevance per input token, (N, L).
std::vector< Tensor > propagate_hidden_relevance_by_layer(const Tensor &relevance_hidden, const LRPRuleConfig &config)
propagate_relevance_by_layer from relevance at last_hidden_state() (after the final LayerNorm),...
OpType op_type() const override
This module's operation-category tag, for ComputationGraph node tagging.
Definition encoder_lm.hpp:139
void set_attention_observer(AttentionObserver observer)
Calls observer after each layer's forward, before set_keep_activations(false) frees its activations,...
Definition encoder_lm.hpp:164
Tensor backward(const Tensor &grad_output) override
bool keep_activations() const
Definition encoder_lm.hpp:154
EncoderLM(const EncoderLMConfig &config, DeviceBackend *backend)
Tensor backward_hidden(const Tensor &grad_hidden)
backward() from the gradient at last_hidden_state() (after the final LayerNorm), (N,...
void set_hidden_state_hook(HiddenStateHook hook)
Calls hook at every position during forward(): 0 is the scaled embeddings, i layer i's output (before...
Definition encoder_lm.hpp:168
std::optional< DeviceType > compute_device() const override
The device this module computes on, so forward() can reject an input on another device before any ker...
Definition encoder_lm.hpp:144
EncoderBlock & layer(int64_t i)
Definition encoder_lm.hpp:181
const std::vector< Tensor > & hidden_states() const
The last forward pass's hidden states, each (N, L, hidden_size): the scaled embeddings,...
Definition encoder_lm.hpp:188
Tensor forward_impl(const Tensor &input) override
std::vector< NamedModuleRef > named_children() override
This module's direct sub-modules, each under the prefix its parameters carry in named_parameters(),...
std::function< void(int64_t layer, const Tensor &attention)> AttentionObserver
Called with a layer's index and its attention probabilities, (N, heads, L, L) on the model's device,...
Definition encoder_lm.hpp:158
void set_training(bool training) override
Sets this module's training/eval mode. Defaults to training (matches every mainstream framework's Mod...
void set_padding_mask(const Tensor &keep)
Masks padding for the following forward passes: keep is (N, L), 1 for a real token and 0 for padding,...
void set_keep_activations(bool keep)
With keep false, forward() keeps nothing for backward() or the relevance passes: each layer frees its...
Definition encoder_lm.hpp:153
void clear_padding_mask()
Treats every token as real again.
const Tensor & last_hidden_state() const
The last forward pass's final-LayerNorm output, (N, L, hidden_size): the per-residue representations ...
Definition encoder_lm.hpp:191
int64_t num_layers() const
Definition encoder_lm.hpp:180
EmbeddingModule & embed_tokens()
Definition encoder_lm.hpp:182
std::vector< NamedBufferRef > named_buffers() override
This module's buffers: state a checkpoint must save that no optimizer updates.
std::vector< Tensor > propagate_relevance_by_layer(const Tensor &relevance_out, const LRPRuleConfig &config)
propagate_relevance, stopping at every layer boundary.
std::vector< NamedParamRef > named_parameters() override
This module's trainable parameters, each with its hierarchical name – the one place a module declares...
void release_activations() override
Frees every forward cache, last_hidden_state() and hidden_states() included.
y_{n,i} = gamma_i * (x_{n,i} - mu_n)/std_n + beta_i, mu_n = mean_i(x_{n,i}), std_n = sqrt(var_i(x_{n,...
Definition layer_norm_module.hpp:27
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
An N-dimensional shape. A plain aggregate of dimensions with no invariant beyond "non-negative dimens...
Definition shape.hpp:24
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
Lookup-table (row-select) layer – y = W[index], batched (N, L) -> (N, L, embedding_dim).
A configurable transformer encoder layer (PLM-1): LayerNorm or RMSNorm, before or after each sublayer...
Layer normalization (Ba et al., 2016) – mean-centered/scaled RMSNormModule sibling.
Dense/fully-connected layer – the reference Module implementation.
Definition acquisition_functions.hpp:16
std::vector< WeightMapping > EsmMapping(const EncoderLMConfig &config)
The name mapping from an EsmForMaskedLM checkpoint to EncoderLM. Q/K/V, output, MLP and LM-head dense...
std::vector< WeightMapping > MdlmMapping(const EncoderLMConfig &config)
The name mapping from an MDLM checkpoint to EncoderLM. The fused attn_qkv is split into q,...
std::function< Tensor(int64_t position, const Tensor &hidden)> HiddenStateHook
Reads or replaces a language model's residual stream during forward() (FEAT-3): called with a positio...
Definition causal_lm.hpp:37
EncoderLMHead
An EncoderLM's output head.
Definition encoder_lm.hpp:27
@ Linear
One linear layer with a bias, not tied to the embeddings (MDLM).
@ Esm
ESM's: dense, GELU, LayerNorm, then the embedding table transposed plus a bias.
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
EncoderLMConfig ParseEsmConfig(std::string_view json)
Reads an ESM config.json.
std::vector< std::string > EsmIgnoredTensors(const EncoderLMConfig &config)
Checkpoint tensors an ESM-2 EncoderLM doesn't use: the absolute-position table and ids (unused with r...
std::unique_ptr< EncoderLM > LoadEncoderLM(const std::string &directory, DeviceBackend *backend)
Loads a masked LM directory: config.json and safetensors weights (one file or sharded)....
EncoderLMConfig ReadEsmConfig(const std::string &path)
ParseEsmConfig on a file.
EncoderLMConfig ParseMdlmConfig(std::string_view json)
Reads an MDLM config.json (kuleshov-group/mdlm-owt and models trained with the same code): a bidirect...
ElementwiseOp
Unary elementwise operations supported by DeviceBackend::elementwise().
Definition device_backend.hpp:43
@ Gelu
exact GELU, x * Phi(x) with erf – the MLP activation of BERT and ESM (PLM-1)
The settings of a masked LM, from its Hugging Face config.json.
Definition encoder_lm.hpp:35
int64_t intermediate_size
Definition encoder_lm.hpp:40
int64_t pad_token_id
Definition encoder_lm.hpp:49
float layer_norm_eps
Definition encoder_lm.hpp:43
float rope_theta
Definition encoder_lm.hpp:44
int64_t vocab_size
Definition encoder_lm.hpp:38
int64_t num_attention_heads
Definition encoder_lm.hpp:42
ElementwiseOp activation
The MLP's activation: exact GELU for ESM, the tanh approximation for MDLM.
Definition encoder_lm.hpp:62
std::vector< double > rope_inverse_frequencies
RoPE frequencies to use instead of rope_theta's, one per pair. LoadEncoderLM fills them from the chec...
Definition encoder_lm.hpp:57
int64_t num_hidden_layers
Definition encoder_lm.hpp:41
int64_t max_position_embeddings
Definition encoder_lm.hpp:45
int64_t mask_token_id
Definition encoder_lm.hpp:48
int64_t hidden_size
Definition encoder_lm.hpp:39
std::string dtype
Definition encoder_lm.hpp:50
bool attention_bias
Biases on the attention's q, k, v and output projections (ESM's); MDLM has none.
Definition encoder_lm.hpp:60
EncoderLMHead head
Definition encoder_lm.hpp:58
std::string architecture
EsmForMaskedLM, EsmForDPLM, MDLM
Definition encoder_lm.hpp:36
bool token_dropout
ESM's "mask dropout": masked tokens' embeddings are zeroed and the rest rescaled.
Definition encoder_lm.hpp:47
std::string model_type
esm, mdlm
Definition encoder_lm.hpp:37
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72
Language-model head that reuses an EmbeddingModule's table as its weight (LLM-2).