pulsatrix 1.1.0
Loading...
Searching...
No Matches
transformer_block.hpp
Go to the documentation of this file.
1
7#pragma once
8
9#include <memory>
10#include <optional>
12#include "pulsatrix/module.hpp"
16
17namespace pulsatrix {
18
33
67class TransformerBlock : public Module {
68public:
82 TransformerBlock(int64_t d_model, int64_t num_heads, int64_t d_ff, DeviceBackend* backend, bool use_rope = true,
83 bool use_qk_norm = false);
84
96 TransformerBlock(const AttentionConfig& attention, int64_t d_ff, DeviceBackend* backend, float norm_eps = 1e-6f,
97 bool mlp_bias = true);
98
101 TransformerBlock(const AttentionConfig& attention, int64_t d_ff, DeviceBackend* backend,
102 const TransformerBlockOptions& options);
103
114 [[nodiscard]] Tensor backward(const Tensor& grad_output) override;
115
117 [[nodiscard]] OpType op_type() const override { return OpType::Elementwise; }
118
130 [[nodiscard]] Tensor propagate_relevance(const Tensor& relevance_out, const LRPRuleConfig& config) override;
131
133 [[nodiscard]] std::vector<NamedParamRef> named_parameters() override;
134 [[nodiscard]] std::vector<NamedModuleRef> named_children() override;
135
137 [[nodiscard]] std::vector<NamedBufferRef> named_buffers() override;
138
140 void set_training(bool training) override;
141
142 [[nodiscard]] int64_t d_model() const { return mha_.d_model(); }
143
150 [[nodiscard]] Tensor forward_cached(const Tensor& input, KVCache& cache);
151
154 [[nodiscard]] RMSNormModule& norm1() { return norm1_; }
155 [[nodiscard]] MultiHeadAttentionModule& mha() { return mha_; }
156 [[nodiscard]] RMSNormModule& norm2() { return norm2_; }
157 [[nodiscard]] SwiGLUModule& swiglu() { return swiglu_; }
160 void set_mlp_hook(MlpHook hook) { mlp_hook_ = std::move(hook); }
162 [[nodiscard]] RMSNormModule* post_attn_norm() { return post_attn_norm_.get(); }
163 [[nodiscard]] RMSNormModule* post_mlp_norm() { return post_mlp_norm_.get(); }
165
166
168 [[nodiscard]] std::optional<DeviceType> compute_device() const override { return backend_->device(); }
169
170protected:
177 [[nodiscard]] Tensor forward_impl(const Tensor& input) override;
178
179private:
180 int64_t d_model_;
181 DeviceBackend* backend_;
182
183 RMSNormModule norm1_;
185 RMSNormModule norm2_;
186 SwiGLUModule swiglu_;
187 std::unique_ptr<RMSNormModule> post_attn_norm_;
188 std::unique_ptr<RMSNormModule> post_mlp_norm_;
189
190 // Forward caches -- the four operands the two residual splits need.
191 Shape last_input_shape_ = Shape({0});
192 Tensor last_x_;
193 Tensor last_attn_out_;
194 Tensor last_y1_;
195 Tensor last_ffn_out_;
196 bool has_forwarded_ = false;
197 MlpHook mlp_hook_;
198};
199
200} // namespace pulsatrix
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
One attention layer's keys and values for the positions processed so far, in buffers allocated once f...
Definition kv_cache.hpp:22
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
softmax(Q @ K^T / sqrt(head_dim)) @ V, multi-head, with optional RoPE and optional QK-Norm....
Definition multihead_attention_module.hpp:113
int64_t d_model() const
Definition multihead_attention_module.hpp:222
y_{n,i} = gamma_i * x_{n,i} / rms(x_n), rms(x_n) = sqrt(mean_i(x_{n,i}^2) + eps), computed independen...
Definition rms_norm_module.hpp:32
An N-dimensional shape. A plain aggregate of dimensions with no invariant beyond "non-negative dimens...
Definition shape.hpp:24
down_proj(silu(gate_proj(x)) * up_proj(x)), the gated feedforward block used in place of a plain two-...
Definition swiglu_module.hpp:53
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
y1 = x + MHA(RMSNorm(x)), y2 = y1 + SwiGLU(RMSNorm(y1)). Shape (N, L, d_model) -> (N,...
Definition transformer_block.hpp:67
std::vector< NamedParamRef > named_parameters() override
norm1_'s, mha_'s, norm2_'s, and swiglu_'s parameters, flattened.
std::vector< NamedBufferRef > named_buffers() override
Every layer's named_buffers(), prefixed the same way as named_parameters().
RMSNormModule & norm1()
Definition transformer_block.hpp:154
RMSNormModule & norm2()
Definition transformer_block.hpp:156
int64_t d_model() const
Definition transformer_block.hpp:142
MultiHeadAttentionModule & mha()
Definition transformer_block.hpp:155
RMSNormModule * post_attn_norm()
The sandwich norms after attention and after the MLP; nullptr without post_norms.
Definition transformer_block.hpp:162
OpType op_type() const override
Elementwise per this module's own op_type() note above.
Definition transformer_block.hpp:117
std::vector< NamedModuleRef > named_children() override
This module's direct sub-modules, each under the prefix its parameters carry in named_parameters(),...
SwiGLUModule & swiglu()
Definition transformer_block.hpp:157
void set_mlp_hook(MlpHook hook)
Calls hook on the MLP's input (after norm2) and output (after any post-MLP norm) during forward(),...
Definition transformer_block.hpp:160
std::optional< DeviceType > compute_device() const override
Where this layer computes, so forward() rejects an input on another device (FND-8).
Definition transformer_block.hpp:168
TransformerBlock(const AttentionConfig &attention, int64_t d_ff, DeviceBackend *backend, float norm_eps=1e-6f, bool mlp_bias=true)
Constructs a block whose attention is configured in full (LLM-1): grouped-query attention,...
Tensor backward(const Tensor &grad_output) override
Gradient w.r.t. this module's input; sub-module parameter gradients accumulate inside norm1_/mha_/nor...
TransformerBlock(int64_t d_model, int64_t num_heads, int64_t d_ff, DeviceBackend *backend, bool use_rope=true, bool use_qk_norm=false)
Constructs a transformer block with zero-initialized sub-module parameters.
TransformerBlock(const AttentionConfig &attention, int64_t d_ff, DeviceBackend *backend, const TransformerBlockOptions &options)
A block with every option (LLM-9): Gemma's GeGLU MLP, sandwich norms and 1 + w norms....
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
LRP relevance propagation: two residual epsilon/z-rule splits composed with norm1_'s/mha_'s/norm2_'s/...
Tensor forward_impl(const Tensor &input) override
Runs: norm1 -> attention -> residual add -> norm2 -> SwiGLU -> residual add.
RMSNormModule * post_mlp_norm()
Definition transformer_block.hpp:163
void set_training(bool training) override
Cascades to every sub-module, the same way SequentialModule/MultiHeadAttentionModule do.
Tensor forward_cached(const Tensor &input, KVCache &cache)
The block for new positions only, with attention against cache (LLM-5); see MultiHeadAttentionModule:...
Reads or replaces a transformer block's MLP output during forward() (FEAT-5), for collecting a transc...
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
Multi-head scaled dot-product attention – this codebase's first Module composed out of other real Mod...
Definition acquisition_functions.hpp:16
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
GatedActivation
The gate activation of a gated MLP: SiLU (SwiGLU: Llama, Qwen), GELU's tanh approximation (GeGLU: Gem...
Definition swiglu_module.hpp:16
std::function< Tensor(const Tensor &mlp_input, const Tensor &mlp_output)> MlpHook
Called with an MLP's input (after the block's norm, where it has one) and its output (the branch adde...
Definition mlp_hook.hpp:23
RMS normalization layer (Zhang & Sennrich, 2019) – this codebase's first normalization Module,...
Everything that shapes a MultiHeadAttentionModule (LLM-1).
Definition multihead_attention_module.hpp:31
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72
The parts of a TransformerBlock beyond its attention (LLM-9 adds Gemma's).
Definition transformer_block.hpp:20
GatedActivation mlp_activation
The MLP's gate activation: SiLU, or GELU-tanh for Gemma.
Definition transformer_block.hpp:26
bool mlp_bias
Biases on the MLP projections (Hugging Face mlp_bias).
Definition transformer_block.hpp:24
float norm_eps
Stabilizer of the RMSNorms (Hugging Face rms_norm_eps).
Definition transformer_block.hpp:22
bool post_norms
Gemma's sandwich norms: the attention's and the MLP's outputs are normalized again before each residu...
Definition transformer_block.hpp:29
float norm_weight_offset
Every RMSNorm scales by offset + gamma (Gemma's 1 + w).
Definition transformer_block.hpp:31
SwiGLU gated feedforward block – second module composed from real LinearModule sub-objects,...