pulsatrix 1.1.0
Loading...
Searching...
No Matches
multihead_attention_module.hpp
Go to the documentation of this file.
1
6#pragma once
7
8#include <optional>
9#include <memory>
10#include <vector>
11
14#include "pulsatrix/module.hpp"
18
19namespace pulsatrix {
20
32 int64_t d_model = 0;
33 int64_t num_heads = 0;
36 int64_t num_kv_heads = 0;
39 int64_t head_dim = 0;
40 bool use_rope = true;
42 float rope_base = 10000.0f;
45 std::vector<double> rope_inverse_frequencies;
46 bool use_qk_norm = false;
48 float qk_norm_eps = 1e-6f;
50 bool qkv_bias = true;
52 bool out_bias = true;
54 bool causal = false;
57 int64_t sliding_window = 0;
60 float score_scale = 0.0f;
63};
64
114public:
127 MultiHeadAttentionModule(int64_t d_model, int64_t num_heads, DeviceBackend* backend, bool use_rope = true,
128 bool use_qk_norm = false);
129
137
155 [[nodiscard]] Tensor backward(const Tensor& grad_output) override;
156
158 [[nodiscard]] OpType op_type() const override { return OpType::Attention; }
159
197 [[nodiscard]] Tensor propagate_relevance(const Tensor& relevance_out, const LRPRuleConfig& config) override;
198
207 [[nodiscard]] const Tensor& key_relevance() const;
209 [[nodiscard]] const Tensor& value_relevance() const;
210
213 [[nodiscard]] std::vector<NamedParamRef> named_parameters() override;
214 [[nodiscard]] std::vector<NamedModuleRef> named_children() override;
215
217 [[nodiscard]] std::vector<NamedBufferRef> named_buffers() override;
218
220 void set_training(bool training) override;
221
222 [[nodiscard]] int64_t d_model() const { return d_model_; }
223 [[nodiscard]] int64_t num_heads() const { return num_heads_; }
224 [[nodiscard]] int64_t num_kv_heads() const { return num_kv_heads_; }
225 [[nodiscard]] int64_t head_dim() const { return head_dim_; }
226 [[nodiscard]] bool uses_rope() const { return use_rope_; }
227 [[nodiscard]] bool uses_qk_norm() const { return use_qk_norm_; }
228 [[nodiscard]] bool is_causal() const { return config_.causal; }
230 [[nodiscard]] const AttentionConfig& config() const { return config_; }
231
240 void set_key_padding_mask(const Tensor& key_keep);
243 [[nodiscard]] bool has_key_padding_mask() const { return has_key_keep_; }
244
251 void set_position_offset(int64_t offset);
252 [[nodiscard]] int64_t position_offset() const { return position_offset_; }
253
275 [[nodiscard]] Tensor forward_cached(const Tensor& input, KVCache& cache);
276
278 [[nodiscard]] KVCache MakeKVCache(int64_t batch, int64_t max_length) const;
279
282 [[nodiscard]] LinearModule& q_proj() { return q_proj_; }
283 [[nodiscard]] LinearModule& k_proj() { return k_proj_; }
284 [[nodiscard]] LinearModule& v_proj() { return v_proj_; }
285 [[nodiscard]] LinearModule& out_proj() { return out_proj_; }
287 [[nodiscard]] RMSNormModule* q_norm() { return q_norm_.get(); }
289 [[nodiscard]] RMSNormModule* k_norm() { return k_norm_.get(); }
291
295 [[nodiscard]] const Tensor& last_attention_weights() const { return last_attn_; }
296
297
299 [[nodiscard]] std::optional<DeviceType> compute_device() const override { return backend_->device(); }
300
301 void release_activations() override;
302
303protected:
312 [[nodiscard]] Tensor forward_impl(const Tensor& input) override;
313
314private:
315 AttentionConfig config_;
316 int64_t d_model_;
317 int64_t num_heads_;
318 int64_t num_kv_heads_;
319 int64_t head_dim_;
320 bool use_rope_;
321 bool use_qk_norm_;
322 DeviceBackend* backend_;
323
324 LinearModule q_proj_;
325 LinearModule k_proj_;
326 LinearModule v_proj_;
327 LinearModule out_proj_;
328 SoftmaxModule softmax_;
329 // Held by pointer rather than std::optional purely so the "disabled" state costs nothing
330 // and needs no move/copy of a Module subclass (Module declares a virtual destructor,
331 // which suppresses implicit move construction -- std::optional's in-place paths would
332 // work but the pointer is unambiguous).
333 std::unique_ptr<RoPEModule> q_rope_;
334 std::unique_ptr<RoPEModule> k_rope_;
335 std::unique_ptr<RMSNormModule> q_norm_;
336 std::unique_ptr<RMSNormModule> k_norm_;
337
338 Tensor key_keep_;
339 bool has_key_keep_ = false;
340 int64_t position_offset_ = 0;
341
344 void repeat_kv(Tensor& t) const;
347 void sum_kv_groups(Tensor& t) const;
349 void fill_masked(Tensor& scores, float value) const;
350 [[nodiscard]] float score_scale() const;
351
352 // Forward caches. Q/K/V are the post-QK-Norm, post-RoPE, head-split values -- exactly
353 // the operands the two matmuls' backward and Eq. 15 rules need. K and V are already
354 // repeated out to num_heads under GQA.
355 int64_t last_N_ = 0;
356 int64_t last_L_ = 0;
357 Tensor last_key_keep_;
358 bool last_has_key_keep_ = false;
359 Tensor last_q_;
360 Tensor last_k_;
361 Tensor last_v_;
362 Tensor last_scores_raw_;
363 Tensor last_attn_;
364 Tensor last_context_;
365 bool has_forwarded_ = false;
366 Tensor last_key_relevance_;
367 Tensor last_value_relevance_;
368 bool has_relevance_ = false;
369};
370
371} // namespace pulsatrix
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
One attention layer's keys and values for the positions processed so far, in buffers allocated once f...
Definition kv_cache.hpp:22
y = x @ W + b, batched (x is (N, in_features), y is (N, out_features)) – migrated from the original u...
Definition linear_module.hpp:43
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
softmax(Q @ K^T / sqrt(head_dim)) @ V, multi-head, with optional RoPE and optional QK-Norm....
Definition multihead_attention_module.hpp:113
bool has_key_padding_mask() const
Definition multihead_attention_module.hpp:243
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
KVCache MakeKVCache(int64_t batch, int64_t max_length) const
An empty cache sized for this layer: (batch, num_kv_heads, max_length, head_dim).
RMSNormModule * q_norm()
Q's QK-Norm sub-module, or nullptr when use_qk_norm is false.
Definition multihead_attention_module.hpp:287
const AttentionConfig & config() const
The configuration with its 0 defaults resolved (num_kv_heads, head_dim).
Definition multihead_attention_module.hpp:230
const Tensor & last_attention_weights() const
Cached attention weights of the last forward, (N, num_heads, L, L) – the softmax output....
Definition multihead_attention_module.hpp:295
std::vector< NamedModuleRef > named_children() override
This module's direct sub-modules, each under the prefix its parameters carry in named_parameters(),...
OpType op_type() const override
Attention per the charter's closed OpType set – see OpType::Attention's own note.
Definition multihead_attention_module.hpp:158
std::vector< NamedParamRef > named_parameters() override
Every sub-module's parameters, flattened – Q/K/V/O weights and biases, plus the two QK-Norm gammas wh...
Tensor forward_impl(const Tensor &input) override
Runs the 9-step pipeline: project -> split heads -> (QK-Norm) -> (RoPE) -> scores -> softmax -> conte...
void set_training(bool training) override
Cascades to every sub-module, the same way SequentialModule does.
int64_t num_kv_heads() const
Definition multihead_attention_module.hpp:224
LinearModule & q_proj()
Definition multihead_attention_module.hpp:282
int64_t num_heads() const
Definition multihead_attention_module.hpp:223
Tensor backward(const Tensor &grad_output) override
Gradient w.r.t. this module's input; sub-module parameter gradients accumulate inside those sub-modul...
int64_t position_offset() const
Definition multihead_attention_module.hpp:252
std::vector< NamedBufferRef > named_buffers() override
Every layer's named_buffers(), prefixed the same way as named_parameters().
Tensor forward_cached(const Tensor &input, KVCache &cache)
Attention for new positions only, against everything in cache (LLM-5): projects the input,...
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
LRP relevance propagation, composed from the sub-modules' own rules plus AttnLRP's Eq....
bool uses_rope() const
Definition multihead_attention_module.hpp:226
LinearModule & v_proj()
Definition multihead_attention_module.hpp:284
LinearModule & k_proj()
Definition multihead_attention_module.hpp:283
const Tensor & key_relevance() const
The last propagate_relevance()'s relevance at the K and V projection outputs, (N * L,...
RMSNormModule * k_norm()
K's QK-Norm sub-module, or nullptr when use_qk_norm is false.
Definition multihead_attention_module.hpp:289
bool uses_qk_norm() const
Definition multihead_attention_module.hpp:227
std::optional< DeviceType > compute_device() const override
Where this layer computes, so forward() rejects an input on another device (FND-8).
Definition multihead_attention_module.hpp:299
int64_t head_dim() const
Definition multihead_attention_module.hpp:225
void clear_key_padding_mask()
Removes the key padding mask.
void set_key_padding_mask(const Tensor &key_keep)
Masks padding keys in every following forward(), until cleared (LLM-1).
int64_t d_model() const
Definition multihead_attention_module.hpp:222
void set_position_offset(int64_t offset)
Position of the input's first token, for RoPE (LLM-1). Forward rotates positions offset ....
MultiHeadAttentionModule(int64_t d_model, int64_t num_heads, DeviceBackend *backend, bool use_rope=true, bool use_qk_norm=false)
Constructs a multi-head attention block with zero-initialized projections.
const Tensor & value_relevance() const
The same at the V projection output.
bool is_causal() const
Definition multihead_attention_module.hpp:228
MultiHeadAttentionModule(const AttentionConfig &config, DeviceBackend *backend)
Constructs attention from a full AttentionConfig, with zero-initialized projections.
LinearModule & out_proj()
Definition multihead_attention_module.hpp:285
y_{n,i} = gamma_i * x_{n,i} / rms(x_n), rms(x_n) = sqrt(mean_i(x_{n,i}^2) + eps), computed independen...
Definition rms_norm_module.hpp:32
Softmax over the tensor's last dimension, applied independently to every "row" (every fixed combinati...
Definition softmax_module.hpp:20
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
Preallocated key/value cache for incremental attention (LLM-5).
Dense/fully-connected layer – the reference Module implementation.
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
Definition acquisition_functions.hpp:16
RoPELayout
Which features RoPE rotates together (LLM-1).
Definition rope_module.hpp:18
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
@ Attention
Multi-head (scaled dot-product) attention – Phase 3's MultiHeadAttentionModule.
RMS normalization layer (Zhang & Sennrich, 2019) – this codebase's first normalization Module,...
Rotary Position Embedding – fixed per-position pair rotation, epsilon-rule LRP.
Rank-agnostic softmax over the last axis, with AttnLRP's Eq. 13 DTD relevance rule.
Everything that shapes a MultiHeadAttentionModule (LLM-1).
Definition multihead_attention_module.hpp:31
std::vector< double > rope_inverse_frequencies
Explicit RoPE frequencies, one per pair (head_dim / 2), overriding rope_base: how scaled RoPE (Llama ...
Definition multihead_attention_module.hpp:45
RoPELayout rope_layout
Definition multihead_attention_module.hpp:41
int64_t head_dim
Definition multihead_attention_module.hpp:39
bool causal
Definition multihead_attention_module.hpp:54
int64_t num_heads
Definition multihead_attention_module.hpp:33
bool qkv_bias
Definition multihead_attention_module.hpp:50
int64_t num_kv_heads
Definition multihead_attention_module.hpp:36
int64_t sliding_window
Definition multihead_attention_module.hpp:57
bool use_rope
Definition multihead_attention_module.hpp:40
float qk_norm_eps
Definition multihead_attention_module.hpp:48
int64_t d_model
Definition multihead_attention_module.hpp:32
float score_scale
Definition multihead_attention_module.hpp:60
bool out_bias
Definition multihead_attention_module.hpp:52
bool use_qk_norm
Definition multihead_attention_module.hpp:46
float rope_base
Definition multihead_attention_module.hpp:42
float qk_norm_weight_offset
Definition multihead_attention_module.hpp:62
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72