pulsatrix 1.1.0
Loading...
Searching...
No Matches
kv_cache.hpp
Go to the documentation of this file.
1
5#pragma once
6
7#include <cstdint>
8
10
11namespace pulsatrix {
12
22class KVCache {
23public:
27 KVCache(int64_t batch, int64_t num_kv_heads, int64_t head_dim, int64_t max_length, DeviceBackend* backend);
28
29 [[nodiscard]] int64_t batch() const { return batch_; }
30 [[nodiscard]] int64_t num_kv_heads() const { return num_kv_heads_; }
31 [[nodiscard]] int64_t head_dim() const { return head_dim_; }
32 [[nodiscard]] int64_t max_length() const { return max_length_; }
34 [[nodiscard]] int64_t length() const { return length_; }
35
37 void reset() { length_ = 0; }
43 void truncate(int64_t length);
44
47 [[nodiscard]] Tensor& keys() { return keys_; }
48 [[nodiscard]] Tensor& values() { return values_; }
49 [[nodiscard]] const Tensor& keys() const { return keys_; }
50 [[nodiscard]] const Tensor& values() const { return values_; }
51
54 void advance(int64_t count);
55
56private:
57 int64_t batch_;
58 int64_t num_kv_heads_;
59 int64_t head_dim_;
60 int64_t max_length_;
61 int64_t length_ = 0;
62 Tensor keys_;
63 Tensor values_;
64};
65
66} // namespace pulsatrix
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
One attention layer's keys and values for the positions processed so far, in buffers allocated once f...
Definition kv_cache.hpp:22
const Tensor & keys() const
Definition kv_cache.hpp:49
void advance(int64_t count)
Records count more positions as written. Called by forward_cached.
int64_t length() const
Positions cached so far; the next forward_cached input starts at this position.
Definition kv_cache.hpp:34
int64_t batch() const
Definition kv_cache.hpp:29
const Tensor & values() const
Definition kv_cache.hpp:50
Tensor & values()
Definition kv_cache.hpp:48
int64_t max_length() const
Definition kv_cache.hpp:32
Tensor & keys()
(batch, num_kv_heads, max_length, head_dim); positions from length() on are unused.
Definition kv_cache.hpp:47
int64_t num_kv_heads() const
Definition kv_cache.hpp:30
void truncate(int64_t length)
Keeps only the first length positions, so a sequence can continue from a shared prefix.
KVCache(int64_t batch, int64_t num_kv_heads, int64_t head_dim, int64_t max_length, DeviceBackend *backend)
void reset()
Forgets every position (the buffers stay allocated).
Definition kv_cache.hpp:37
int64_t head_dim() const
Definition kv_cache.hpp:31
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
Definition acquisition_functions.hpp:16
N-dimensional tensor – owns a buffer via DeviceBackend*, RAII (Rule of Five).