pulsatrix 1.1.0
Loading...
Searching...
No Matches
encoder_block.hpp
Go to the documentation of this file.
1
7#pragma once
8
9#include <memory>
10#include <optional>
11
13#include "pulsatrix/module.hpp"
16
17namespace pulsatrix {
18
20enum class NormType { LayerNorm, RMSNorm };
21
23enum class NormPosition {
25 Pre,
27 Post,
28};
29
31enum class MlpType {
33 Plain,
35 Gated,
36};
37
52
72class EncoderBlock : public Module {
73public:
75 EncoderBlock(const AttentionConfig& attention, int64_t d_ff, DeviceBackend* backend, const EncoderBlockOptions& options = {});
76
78 [[nodiscard]] Tensor backward(const Tensor& grad_output) override;
80 [[nodiscard]] Tensor propagate_relevance(const Tensor& relevance_out, const LRPRuleConfig& config) override;
82 [[nodiscard]] OpType op_type() const override { return OpType::Elementwise; }
83 [[nodiscard]] std::vector<NamedParamRef> named_parameters() override;
84 [[nodiscard]] std::vector<NamedModuleRef> named_children() override;
85 [[nodiscard]] std::vector<NamedBufferRef> named_buffers() override;
86 void set_training(bool training) override;
87 [[nodiscard]] std::optional<DeviceType> compute_device() const override { return backend_->device(); }
88
89 [[nodiscard]] int64_t d_model() const { return d_model_; }
90 [[nodiscard]] const EncoderBlockOptions& options() const { return options_; }
93 [[nodiscard]] Module& norm1() { return *norm1_; }
94 [[nodiscard]] MultiHeadAttentionModule& mha() { return mha_; }
95 [[nodiscard]] Module& norm2() { return *norm2_; }
96 [[nodiscard]] Module& mlp() { return *mlp_; }
99 void set_mlp_hook(MlpHook hook) { mlp_hook_ = std::move(hook); }
101
102 void release_activations() override;
103
104protected:
106 [[nodiscard]] Tensor forward_impl(const Tensor& input) override;
107
108private:
109 [[nodiscard]] Tensor Norm(Module& norm, const Tensor& x) const;
110 [[nodiscard]] Tensor NormBackward(Module& norm, const Tensor& g) const;
111 [[nodiscard]] Tensor NormRelevance(Module& norm, const Tensor& r, const LRPRuleConfig& config) const;
112
113 int64_t d_model_;
114 DeviceBackend* backend_;
115 EncoderBlockOptions options_;
116 std::unique_ptr<Module> norm1_;
118 std::unique_ptr<Module> norm2_;
119 std::unique_ptr<Module> mlp_;
120 MlpHook mlp_hook_;
121
122 // Forward caches: the operands of the two residual splits.
123 Shape last_input_shape_ = Shape({0});
124 Tensor last_x_;
125 Tensor last_attn_out_;
126 Tensor last_y1_;
127 Tensor last_mlp_out_;
128 bool has_forwarded_ = false;
129};
130
131} // namespace pulsatrix
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
One encoder layer. Shape (N, L, d_model) -> (N, L, d_model).
Definition encoder_block.hpp:72
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Tensor forward_impl(const Tensor &input) override
MultiHeadAttentionModule & mha()
Definition encoder_block.hpp:94
EncoderBlock(const AttentionConfig &attention, int64_t d_ff, DeviceBackend *backend, const EncoderBlockOptions &options={})
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
Tensor backward(const Tensor &grad_output) override
Module & norm1()
Definition encoder_block.hpp:93
Module & mlp()
Definition encoder_block.hpp:96
std::vector< NamedModuleRef > named_children() override
This module's direct sub-modules, each under the prefix its parameters carry in named_parameters(),...
Module & norm2()
Definition encoder_block.hpp:95
std::optional< DeviceType > compute_device() const override
The device this module computes on, so forward() can reject an input on another device before any ker...
Definition encoder_block.hpp:87
std::vector< NamedBufferRef > named_buffers() override
This module's buffers: state a checkpoint must save that no optimizer updates.
void set_training(bool training) override
Sets this module's training/eval mode. Defaults to training (matches every mainstream framework's Mod...
OpType op_type() const override
Owns the residual adds, as TransformerBlock does.
Definition encoder_block.hpp:82
const EncoderBlockOptions & options() const
Definition encoder_block.hpp:90
int64_t d_model() const
Definition encoder_block.hpp:89
std::vector< NamedParamRef > named_parameters() override
This module's trainable parameters, each with its hierarchical name – the one place a module declares...
void set_mlp_hook(MlpHook hook)
Calls hook on the MLP's input and output during forward() (MlpHook). An empty function removes it.
Definition encoder_block.hpp:99
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
softmax(Q @ K^T / sqrt(head_dim)) @ V, multi-head, with optional RoPE and optional QK-Norm....
Definition multihead_attention_module.hpp:113
An N-dimensional shape. A plain aggregate of dimensions with no invariant beyond "non-negative dimens...
Definition shape.hpp:24
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
Reads or replaces a transformer block's MLP output during forward() (FEAT-5), for collecting a transc...
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
Multi-head scaled dot-product attention – this codebase's first Module composed out of other real Mod...
Definition acquisition_functions.hpp:16
MlpType
The MLP's form.
Definition encoder_block.hpp:31
@ Plain
Linear -> activation -> Linear (BERT, ESM): FeedForwardModule.
@ Gated
down(act(gate(x)) * up(x)) (Llama's SwiGLU, Gemma's GeGLU): SwiGLUModule.
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
GatedActivation
The gate activation of a gated MLP: SiLU (SwiGLU: Llama, Qwen), GELU's tanh approximation (GeGLU: Gem...
Definition swiglu_module.hpp:16
NormType
The layer's normalization.
Definition encoder_block.hpp:20
std::function< Tensor(const Tensor &mlp_input, const Tensor &mlp_output)> MlpHook
Called with an MLP's input (after the block's norm, where it has one) and its output (the branch adde...
Definition mlp_hook.hpp:23
NormPosition
Where the norms sit.
Definition encoder_block.hpp:23
@ Post
y = norm(x + f(x)) (BERT and the original Transformer).
@ Pre
y = x + f(norm(x)) (ESM, GPT-2, Llama).
ElementwiseOp
Unary elementwise operations supported by DeviceBackend::elementwise().
Definition device_backend.hpp:43
@ Gelu
exact GELU, x * Phi(x) with erf – the MLP activation of BERT and ESM (PLM-1)
Everything that shapes a MultiHeadAttentionModule (LLM-1).
Definition multihead_attention_module.hpp:31
Everything about an EncoderBlock beyond its attention. The defaults are ESM-2's.
Definition encoder_block.hpp:39
float norm_eps
The norms' stabilizer (Hugging Face layer_norm_eps; ESM-2 uses 1e-5).
Definition encoder_block.hpp:42
ElementwiseOp activation
The plain MLP's activation (Gelu for BERT and ESM). Ignored for a gated MLP.
Definition encoder_block.hpp:46
MlpType mlp
Definition encoder_block.hpp:44
bool mlp_bias
Biases on the MLP's projections.
Definition encoder_block.hpp:50
GatedActivation gate_activation
The gated MLP's gate activation. Ignored for a plain MLP.
Definition encoder_block.hpp:48
NormPosition norm_position
Definition encoder_block.hpp:43
NormType norm
Definition encoder_block.hpp:40
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72
SwiGLU gated feedforward block – second module composed from real LinearModule sub-objects,...