pulsatrix 1.1.0
Loading...
Searching...
No Matches
device_backend.hpp
Go to the documentation of this file.
1
5#pragma once
6
7#include <cstddef>
8#include <cstdint>
9
10namespace pulsatrix {
11
17enum class DeviceType {
18 Cpu,
19 Cuda,
20 Hip
21};
22
30
43enum class ElementwiseOp {
44 Relu,
45 Neg,
46 Tanh,
47 Sigmoid,
48 Silu,
49 Exp,
50 GeluTanh,
51 Gelu
52};
53
55enum class LrpGate {
56 None,
57 Positive,
59};
60
74
89
92 const float* in[10] = {};
93 float* out[6] = {};
94 float eps = 0.0f;
95};
96
156
159 const float* in[12] = {};
160 float* out[10] = {};
161 int64_t n = 0;
162 int64_t l = 0;
163 int64_t d = 0;
164 int64_t s = 0;
165 float eps = 0.0f;
166 float gamma = 0.0f;
167};
168
174 size_t kh;
175 size_t kw;
176 size_t stride_h = 1;
177 size_t stride_w = 1;
178 size_t pad_h = 0;
179 size_t pad_w = 0;
180};
181
206
208struct RlRowArgs {
209 const float* in[5] = {};
210 float* out[4] = {};
211 int64_t rows = 0;
212 int64_t cols = 0;
213 float scale = 0.0f;
214 float lower = 0.0f;
215 float upper = 0.0f;
216 float gamma = 0.0f;
217 float tau = 0.0f;
218 int64_t ignore_index = -100;
219};
220
226 float* param = nullptr;
227 const float* grad = nullptr;
228 float* m = nullptr;
229 float* v = nullptr;
230 size_t n = 0;
231 float lr = 0.0f;
232 float bias_correction1 = 1.0f;
233 float bias_correction2 = 1.0f;
235 float decay = 1.0f;
237 float l2 = 0.0f;
238};
239
249public:
250 virtual ~DeviceBackend() = default;
251
260 [[nodiscard]] virtual DeviceType device() const noexcept = 0;
261
270 [[nodiscard]] virtual void* allocate(size_t bytes) = 0;
271
273 virtual void free(void* ptr) noexcept = 0;
274
283 virtual void copy(void* dst, const void* src, size_t bytes, CopyDirection dir) = 0;
284
291 virtual void fill(void* ptr, float value, size_t n) = 0;
292
303 virtual void gemm(const float* a, const float* b, float* out, size_t m, size_t k, size_t n) = 0;
304
313 virtual void elementwise(ElementwiseOp op, const float* in, float* out, size_t n) = 0;
314
328 virtual void add(const float* a, const float* b, float* out, size_t n) = 0;
329
344 virtual void mul(const float* a, const float* b, float* out, size_t n) = 0;
345
346 // ---- GPU-native-kernels Mission 1: primitives for device-resident training ----------
347 // Every reduction below has a fixed summation order on every backend (no atomics), so
348 // results are reproducible run to run.
349
361 virtual void gemm_ex(const float* a, bool transpose_a, const float* b, bool transpose_b, float* out, size_t m,
362 size_t k, size_t n, float beta) = 0;
363
372 virtual void gemm_strided_batched(const float* a, bool transpose_a, size_t stride_a, const float* b,
373 bool transpose_b, size_t stride_b, float* out, size_t stride_out, size_t m,
374 size_t k, size_t n, size_t batch, float beta = 0.0f) {
375 for (size_t i = 0; i < batch; ++i) {
376 if (!transpose_a && !transpose_b && beta == 0.0f) {
377 gemm(a + i * stride_a, b + i * stride_b, out + i * stride_out, m, k, n);
378 } else {
379 gemm_ex(a + i * stride_a, transpose_a, b + i * stride_b, transpose_b, out + i * stride_out, m, k, n,
380 beta);
381 }
382 }
383 }
384
391 virtual void accumulate_parts(const float* parts, size_t count, size_t n, float* acc) {
392 for (size_t i = 0; i < count; ++i) add(acc, parts + i * n, acc, n);
393 }
394
399 virtual void column_sums(const float* in, float* out, size_t rows, size_t cols, float beta) = 0;
400
405 virtual void add_row_vector(const float* in, const float* row, float* out, size_t rows, size_t cols) = 0;
406
413 virtual void add_row_vector_relu(const float* in, const float* row, float* z, float* out, size_t rows,
414 size_t cols) {
415 add_row_vector(in, row, z, rows, cols);
416 elementwise(ElementwiseOp::Relu, z, out, rows * cols);
417 }
418
428 virtual void elementwise_backward(ElementwiseOp op, const float* x, const float* grad_out, float* grad_in,
429 size_t n) = 0;
430
436 virtual void axpby(float alpha, const float* x, float beta, const float* y, float* out, size_t n) = 0;
437
443 [[nodiscard]] virtual float dot(const float* a, const float* b, size_t n) = 0;
444
450 virtual void dot_into(const float* a, const float* b, size_t n, float* out) {
451 const float result = dot(a, b, n);
452 copy(out, &result, sizeof(float), CopyDirection::HostToDevice);
453 }
454
456 virtual void softmax_rows(const float* in, float* out, size_t rows, size_t cols) = 0;
457
462 virtual void softmax_rows_backward(const float* y, const float* dy, float* dx, size_t rows, size_t cols) = 0;
463
465 virtual void logsumexp_rows(const float* in, float* out, size_t rows, size_t cols) = 0;
466
474 virtual void adam_step(float* param, const float* grad, float* m, float* v, size_t n, float lr, float beta1,
475 float beta2, float eps, float bias_correction1, float bias_correction2) = 0;
476
484 virtual void adam_step_multi(const AdamTensorStep* tensors, size_t count, float beta1, float beta2, float eps) {
485 for (size_t t = 0; t < count; ++t) {
486 const AdamTensorStep& s = tensors[t];
487 if (s.n == 0) continue;
488 if (s.decay != 1.0f) axpby(s.decay, s.param, 0.0f, s.param, s.param, s.n);
489 const float* grad = s.grad;
490 void* decayed = nullptr;
491 if (s.l2 != 0.0f) {
492 decayed = allocate(s.n * sizeof(float));
493 axpby(s.l2, s.param, 1.0f, s.grad, static_cast<float*>(decayed), s.n);
494 grad = static_cast<const float*>(decayed);
495 }
496 adam_step(s.param, grad, s.m, s.v, s.n, s.lr, beta1, beta2, eps, s.bias_correction1, s.bias_correction2);
497 free(decayed);
498 }
499 }
500
501 // ---- GPU-native-kernels Mission 1b ---------------------------------------------------
502
504 [[nodiscard]] virtual float sum(const float* in, size_t n) = 0;
505
516 virtual void dropout_forward(const float* in, float* out, float* mask, size_t n, float p, float scale,
517 uint64_t seed, uint64_t offset) = 0;
518
526 virtual void bce_with_logits(const float* logits, const float* target, float* out, size_t n) = 0;
527
532 virtual void bce_with_logits_grad(const float* logits, const float* target, float* grad, size_t n,
533 float scale) = 0;
534
535 // ---- GPU-native-kernels Mission 2: transformer building blocks -------------------------
536 // The fused row operations below run the same per-row source on every backend (src/
537 // row_math.hpp): CPU loops over rows, GPUs run one thread per row. row_std / row_rms /
538 // log_prob are per-row device buffers (rows entries). Index buffers hold whole numbers as
539 // floats, exact below 2^24.
540
542 virtual void layer_norm_forward(const float* in, const float* gamma, const float* beta, float* xhat, float* out,
543 float* row_std, size_t rows, size_t cols, float eps) = 0;
544
546 virtual void layer_norm_backward(const float* grad_out, const float* gamma, const float* xhat,
547 const float* row_std, float* grad_in, size_t rows, size_t cols) = 0;
548
550 virtual void rms_norm_forward(const float* in, const float* gamma, float* out, float* row_rms, size_t rows,
551 size_t cols, float eps) = 0;
552
557 virtual void rms_norm_backward(const float* grad_out, const float* gamma, const float* in, const float* row_rms,
558 float* grad_in, float* gamma_terms, size_t rows, size_t cols) = 0;
559
568 virtual void rope_rotate(const float* in, const float* cos_table, const float* sin_table, float* out,
569 size_t num_slices, size_t seq_len, size_t head_dim, bool inverse, bool rotate_half) = 0;
570
576 virtual void permute_0213(const float* in, float* out, size_t d0, size_t d1, size_t d2, size_t d3) = 0;
577
592 virtual void attention_mask_fill(float* scores, const float* key_keep, size_t batch, size_t heads, size_t q_len,
593 size_t k_len, bool causal, size_t q_offset, size_t window, float value) = 0;
594
596 virtual void gather_rows(const float* table, const float* indices, float* out, size_t count, size_t dim) = 0;
597
603 virtual void scatter_add_rows(const float* src, const float* indices, float* table, size_t count, size_t dim) = 0;
604
609 virtual void tanh_gaussian_forward(const float* mean, const float* log_std, const float* eps, float* action,
610 float* std_cache, float* log_prob, size_t rows, size_t cols,
611 float stabilizer, double half_log_two_pi) = 0;
612
614 virtual void tanh_gaussian_backward(const float* action, const float* std_cache, const float* eps,
615 const float* grad_action, const float* grad_log_prob, float* grad_mean,
616 float* grad_log_std, size_t n, float stabilizer) = 0;
617
618 // ---- GPU-native-kernels Mission 3: LRP rules and logic modules --------------------------
619 // Shared per-output source in src/lrp_math.hpp (CPU loops, GPU one thread per output).
620 // Every reduction is owned by one thread and runs in the original loop order:
621 // deterministic, no atomics.
622
627 virtual void lrp_linear(const float* x, const float* w, const float* z, const float* r, float* r_in, size_t rows,
628 size_t in_features, size_t out_features, float eps) = 0;
629
631 virtual void lrp_residual_split(const float* a, const float* b, const float* r, float* r_a, float* r_b, size_t n,
632 float eps) = 0;
633
635 virtual void lrp_bilinear_elementwise(const float* a, const float* b, const float* r, float* r_out, size_t n,
636 float eps) = 0;
637
643 virtual void lrp_bilinear_matmul(const float* a, const float* b, const float* o, const float* r_o, float* r_a,
644 float* r_b, size_t slices, size_t m, size_t p, size_t q, float eps,
645 bool b_transposed) = 0;
646
648 virtual void lrp_softmax_rows(const float* x, const float* y, const float* r, float* r_in, size_t rows,
649 size_t cols) = 0;
650
652 virtual void lrp_rope(const float* x, const float* y, const float* r, const float* cos_table,
653 const float* sin_table, float* r_in, size_t slices, size_t seq_len, size_t head_dim,
654 float eps, bool rotate_half) = 0;
655
663 virtual void logic_pointwise(LogicOp op, int norm, const float* a, const float* b, const float* g_or_r,
664 const float* y, float* out_a, float* out_b, size_t n, float eps) = 0;
665
667 virtual void aggregator_forward(const float* x, float* mean_pow, float* out, size_t n, size_t cols, float p) = 0;
669 virtual void aggregator_backward(const float* x, const float* mean_pow, const float* grad_out, float* grad_in,
670 size_t n, size_t cols, float p) = 0;
672 virtual void aggregator_lrp(const float* x, const float* mean_pow, const float* r_out, float* r_in, size_t n,
673 size_t cols, float p, float eps) = 0;
674
675 // ---- GPU-native-kernels Mission 4: convolution, pooling, spatial norms -----------------
676 // Shared per-output source in src/cnn_math.hpp; deterministic, no atomics.
677
683 virtual void im2col(const float* in, float* col, size_t n, size_t c, size_t h, size_t w,
684 const ConvGeometry& geometry) = 0;
685
691 virtual void col2im_add(const float* col, float* out, size_t n, size_t c, size_t h, size_t w,
692 const ConvGeometry& geometry) = 0;
693
695 virtual void add_channel_vector(const float* in, const float* vec, float* out, size_t n, size_t c, size_t inner) =
696 0;
697
702 virtual void lrp_conv(const float* col, const float* kernel, const float* pre_bias, const float* r, float* r_col,
703 size_t n, size_t out_channels, size_t p, size_t q, float eps) = 0;
704
705 // ---- LRP-rules campaign Mission 2: Zennit-compatible Gamma / AlphaBeta / ZBox ----------
712 virtual void lrp_stabilized_divide(const float* r, const float* denom, const float* gate, float* out, size_t n,
713 float eps, LrpGate gate_mode) = 0;
714
720 virtual void max_pool_forward(const float* in, float* out, float* argmax, size_t planes, size_t h, size_t w,
721 size_t kh, size_t kw, size_t sh, size_t sw, size_t ph, size_t pw) = 0;
722
727 virtual void max_unpool(const float* src, const float* argmax, float* dst, size_t planes, size_t h, size_t w,
728 size_t kh, size_t kw, size_t sh, size_t sw, size_t ph, size_t pw) = 0;
729
731 virtual void avg_pool_forward(const float* in, float* out, size_t planes, size_t h, size_t w, size_t kh, size_t kw)
732 = 0;
733
735 virtual void avg_pool_backward(const float* grad_out, float* grad_in, size_t planes, size_t h, size_t w, size_t kh,
736 size_t kw) = 0;
737
739 virtual void lrp_avg_pool(const float* x, const float* r, float* r_in, size_t planes, size_t h, size_t w, size_t kh,
740 size_t kw, float eps) = 0;
741
743 virtual void batch_norm_forward(const float* in, const float* gamma, const float* beta, float* xhat, float* out,
744 float* channel_std, size_t n, size_t c, size_t spatial, float eps) = 0;
745
747 virtual void batch_norm_backward(const float* grad_out, const float* gamma, const float* xhat, const float*
748 channel_std, float* grad_in, float* gamma_grad, float* beta_grad, size_t n, size_t
749 c, size_t spatial) = 0;
750
753 virtual void batch_norm_update_running(const float* in, float* running_mean, float* running_var, size_t n,
754 size_t c, size_t spatial, float momentum) = 0;
755
757 virtual void batch_norm_eval_forward(const float* in, const float* gamma, const float* beta,
758 const float* running_mean, const float* running_var, float* xhat, float* out,
759 float* channel_std, size_t n, size_t c, size_t spatial, float eps) = 0;
760
763 virtual void batch_norm_eval_backward(const float* grad_out, const float* gamma, const float* xhat,
764 const float* channel_std, float* grad_in, float* gamma_grad,
765 float* beta_grad, size_t n, size_t c, size_t spatial) = 0;
766
768 virtual void group_norm_forward(const float* in, const float* gamma, const float* beta, float* xhat, float* out,
769 float* group_std, size_t n, size_t c, size_t spatial, size_t num_groups, float eps)
770 = 0;
771
773 virtual void group_norm_backward(const float* grad_out, const float* gamma, const float* xhat, const float*
774 group_std, float* grad_in, float* gamma_grad, float* beta_grad, size_t n, size_t c,
775 size_t spatial, size_t num_groups) = 0;
776
777 // ---- GPU-native-kernels Mission 5: recurrent networks ------------------------------------
778
783 virtual void copy_2d(float* dst, size_t dst_stride, const float* src, size_t src_stride, size_t rows,
784 size_t cols) = 0;
785
792 virtual void accumulate_rows(const float* in, float* out, size_t rows, size_t cols) = 0;
793
795 virtual void recurrent_cell(RecurrentCellOp op, const RecurrentCellArgs& args, size_t n) = 0;
796
801 virtual void gru_lrp_hprev(const float* h_prev, const float* w_hn, const float* hn, const float* r_term_b,
802 const float* direct, float* r_hprev, size_t rows, size_t hidden, float eps) = 0;
803
804 // ---- GPU-native-kernels Mission 6: state-space models -------------------------------------
805 // Shared per-lane source in src/ssm_math.hpp; deterministic, no atomics.
806
813 virtual void ssm_pass(SsmPassOp op, const SsmPassArgs& args) = 0;
814
815 // ---- GPU-native-kernels Mission 7: reinforcement learning -----------------------------------
816 // Shared per-row source in src/rl_math.hpp; deterministic, no atomics.
817
824 virtual void rl_rows(RlRowOp op, const RlRowArgs& args) = 0;
825
826 // ---- FND-3: selection ------------------------------------------------------------------
827
836 virtual void top_k_rows(const float* in, float* values, float* indices, size_t rows, size_t cols, size_t k,
837 bool largest) = 0;
838};
839
840} // namespace pulsatrix
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual void copy_2d(float *dst, size_t dst_stride, const float *src, size_t src_stride, size_t rows, size_t cols)=0
Strided 2-D copy: rows of cols floats from src (row stride src_stride) to dst (row stride dst_stride)...
virtual void scatter_add_rows(const float *src, const float *indices, float *table, size_t count, size_t dim)=0
table[indices[i]][:] += src[i][:] for i = 0..count-1, in increasing i.
virtual void aggregator_lrp(const float *x, const float *mean_pow, const float *r_out, float *r_in, size_t n, size_t cols, float p, float eps)=0
AggregatorModule epsilon rule, per column.
virtual void bce_with_logits_grad(const float *logits, const float *target, float *grad, size_t n, float scale)=0
BCE-with-logits gradient: grad[i] = (sigmoid(x) - y) * scale, using the overflow-free sigmoid (exp(x)...
virtual void layer_norm_backward(const float *grad_out, const float *gamma, const float *xhat, const float *row_std, float *grad_in, size_t rows, size_t cols)=0
LayerNorm input gradient per row, from the cached xhat and per-row std.
virtual void gemm(const float *a, const float *b, float *out, size_t m, size_t k, size_t n)=0
Row-major matrix multiply: out = a * b.
virtual void * allocate(size_t bytes)=0
Allocates a buffer of the given size.
virtual void column_sums(const float *in, float *out, size_t rows, size_t cols, float beta)=0
Per-column sum of a (rows x cols) row-major matrix: out[j] = beta*out[j] + sum_i in[i][j].
virtual void rms_norm_backward(const float *grad_out, const float *gamma, const float *in, const float *row_rms, float *grad_in, float *gamma_terms, size_t rows, size_t cols)=0
RMSNorm input gradient per row, plus gamma_terms[r][i] = grad_out * x / rms – the per-row contributio...
virtual void rope_rotate(const float *in, const float *cos_table, const float *sin_table, float *out, size_t num_slices, size_t seq_len, size_t head_dim, bool inverse, bool rotate_half)=0
Rotary position embedding over (num_slices, seq_len, head_dim) data.
virtual ~DeviceBackend()=default
virtual void dropout_forward(const float *in, float *out, float *mask, size_t n, float p, float scale, uint64_t seed, uint64_t offset)=0
Inverted dropout with a counter-based RNG: element i is dropped iff uniform(seed, offset + i) < p; ke...
virtual void add_channel_vector(const float *in, const float *vec, float *out, size_t n, size_t c, size_t inner)=0
out[i][ch][k] = in[i][ch][k] + vec[ch] over (n, c, inner). out may alias in.
virtual void layer_norm_forward(const float *in, const float *gamma, const float *beta, float *xhat, float *out, float *row_std, size_t rows, size_t cols, float eps)=0
LayerNorm forward per row: xhat = (x - mean)/sqrt(var + eps), out = gamma*xhat + beta.
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
virtual void recurrent_cell(RecurrentCellOp op, const RecurrentCellArgs &args, size_t n)=0
One fused recurrent-cell pass over n elements (see RecurrentCellOp for slots).
virtual void lrp_rope(const float *x, const float *y, const float *r, const float *cos_table, const float *sin_table, float *r_in, size_t slices, size_t seq_len, size_t head_dim, float eps, bool rotate_half)=0
RoPEModule epsilon rule over (slices, seq_len, head_dim), tables as for rope_rotate.
virtual void permute_0213(const float *in, float *out, size_t d0, size_t d1, size_t d2, size_t d3)=0
Swaps the middle two axes: in (d0, d1, d2, d3) -> out (d0, d2, d1, d3).
virtual void fill(void *ptr, float value, size_t n)=0
Fills every element of a float buffer with a constant value.
virtual void softmax_rows(const float *in, float *out, size_t rows, size_t cols)=0
Row-wise softmax of a (rows x cols) matrix, max-subtracted. out may alias in.
virtual void elementwise_backward(ElementwiseOp op, const float *x, const float *grad_out, float *grad_in, size_t n)=0
Activation backward: grad_in[i] = grad_out[i] * f'(x[i]), f = op, x = the forward input.
virtual void batch_norm_backward(const float *grad_out, const float *gamma, const float *xhat, const float *channel_std, float *grad_in, float *gamma_grad, float *beta_grad, size_t n, size_t c, size_t spatial)=0
BatchNorm input gradient plus this call's gamma/beta gradients (overwritten, per channel).
virtual void axpby(float alpha, const float *x, float beta, const float *y, float *out, size_t n)=0
out[i] = alpha * x[i] + beta * y[i]. out may alias x or y.
virtual void accumulate_rows(const float *in, float *out, size_t rows, size_t cols)=0
out[j] += in[i][j] for i = 0..rows-1 in order, accumulating straight into out.
virtual void lrp_conv(const float *col, const float *kernel, const float *pre_bias, const float *r, float *r_col, size_t n, size_t out_channels, size_t p, size_t q, float eps)=0
Conv2D epsilon rule in patch space: r_col (n, p, q) from the cached patches, the kernel (out_channels...
virtual float sum(const float *in, size_t n)=0
sum_i in[i], returned to the host. Same reduction order as dot(). Synchronizes.
virtual void top_k_rows(const float *in, float *values, float *indices, size_t rows, size_t cols, size_t k, bool largest)=0
Per row of a row-major (rows, cols) matrix: the k largest (or smallest) values in rank order into val...
virtual void attention_mask_fill(float *scores, const float *key_keep, size_t batch, size_t heads, size_t q_len, size_t k_len, bool causal, size_t q_offset, size_t window, float value)=0
Overwrites the masked entries of (batch, heads, q_len, k_len) attention scores with value,...
virtual void dot_into(const float *a, const float *b, size_t n, float *out)
dot() written to out, one float in this backend's memory, instead of returned (HIP-6)....
Definition device_backend.hpp:450
virtual float dot(const float *a, const float *b, size_t n)=0
Dot product sum_i a[i]*b[i], returned to the host.
virtual void lrp_stabilized_divide(const float *r, const float *denom, const float *gate, float *out, size_t n, float eps, LrpGate gate_mode)=0
Gated stabilized division, the one non-gemm step of the affine LRP rules: out[i] = passes(gate[i]) ?...
virtual void add(const float *a, const float *b, float *out, size_t n)=0
Elementwise binary addition: out[i] = a[i] + b[i] for i in [0, n).
virtual void lrp_linear(const float *x, const float *w, const float *z, const float *r, float *r_in, size_t rows, size_t in_features, size_t out_features, float eps)=0
LinearModule epsilon rule: r_in[n][i] = sum_j (x[n][i] w[i][j] / stab(z[n][j])) r[n][j].
virtual void tanh_gaussian_forward(const float *mean, const float *log_std, const float *eps, float *action, float *std_cache, float *log_prob, size_t rows, size_t cols, float stabilizer, double half_log_two_pi)=0
TanhGaussianPolicy sampling per (rows, cols) row: action = tanh(mean + exp(log_std)*eps),...
virtual void copy(void *dst, const void *src, size_t bytes, CopyDirection dir)=0
Copies bytes between buffers.
virtual void rms_norm_forward(const float *in, const float *gamma, float *out, float *row_rms, size_t rows, size_t cols, float eps)=0
RMSNorm forward per row: out = gamma * x / sqrt(mean(x^2) + eps).
virtual void free(void *ptr) noexcept=0
Frees a buffer previously returned by allocate(). Safe to call with nullptr.
virtual void lrp_residual_split(const float *a, const float *b, const float *r, float *r_a, float *r_b, size_t n, float eps)=0
Epsilon split of a residual sum y = a + b: r_a = (a / stab(y)) r, r_b = (b / stab(y)) r.
virtual void softmax_rows_backward(const float *y, const float *dy, float *dx, size_t rows, size_t cols)=0
Softmax backward from its output y: dx[i][j] = y[i][j] * (dy[i][j] - sum_k y[i][k]*dy[i][k]).
virtual void gather_rows(const float *table, const float *indices, float *out, size_t count, size_t dim)=0
out[i][:] = table[indices[i]][:] for count rows of width dim.
virtual void aggregator_forward(const float *x, float *mean_pow, float *out, size_t n, size_t cols, float p)=0
AggregatorModule power mean over the leading axis of an (n, cols) input, per column.
virtual void gemm_strided_batched(const float *a, bool transpose_a, size_t stride_a, const float *b, bool transpose_b, size_t stride_b, float *out, size_t stride_out, size_t m, size_t k, size_t n, size_t batch, float beta=0.0f)
batch independent gemm_ex() products, out_i = op(A_i) op(B_i) (+ beta out_i), with A_i = a + i * stri...
Definition device_backend.hpp:372
virtual void logic_pointwise(LogicOp op, int norm, const float *a, const float *b, const float *g_or_r, const float *y, float *out_a, float *out_b, size_t n, float eps)=0
One elementwise pass of Conjunction/Disjunction for operands a, b.
virtual void aggregator_backward(const float *x, const float *mean_pow, const float *grad_out, float *grad_in, size_t n, size_t cols, float p)=0
AggregatorModule input gradient, per column.
virtual void lrp_avg_pool(const float *x, const float *r, float *r_in, size_t planes, size_t h, size_t w, size_t kh, size_t kw, float eps)=0
AvgPool2D epsilon rule; r_in must be zeroed by the caller.
virtual void batch_norm_eval_forward(const float *in, const float *gamma, const float *beta, const float *running_mean, const float *running_var, float *xhat, float *out, float *channel_std, size_t n, size_t c, size_t spatial, float eps)=0
Eval-mode BatchNorm from the running statistics: a per-channel affine map. FND-5.
virtual void batch_norm_forward(const float *in, const float *gamma, const float *beta, float *xhat, float *out, float *channel_std, size_t n, size_t c, size_t spatial, float eps)=0
BatchNorm over (n, spatial) per channel of (n, c, spatial) data.
virtual void elementwise(ElementwiseOp op, const float *in, float *out, size_t n)=0
Applies a unary elementwise operation to every element of a buffer.
virtual void mul(const float *a, const float *b, float *out, size_t n)=0
Elementwise binary (Hadamard) multiplication: out[i] = a[i] * b[i] for i in [0, n).
virtual void lrp_bilinear_elementwise(const float *a, const float *b, const float *r, float *r_out, size_t n, float eps)=0
Eq. 15 for an elementwise product c = a*b: r_out = (a b / (2c + eps sign c)) r (same for both).
virtual void lrp_bilinear_matmul(const float *a, const float *b, const float *o, const float *r_o, float *r_a, float *r_b, size_t slices, size_t m, size_t p, size_t q, float eps, bool b_transposed)=0
Eq. 15 for slices independent matmuls O = A @ B (A (M x P), B (P x Q), O and r_o (M x Q)).
virtual void ssm_pass(SsmPassOp op, const SsmPassArgs &args)=0
One fused Mamba / RWKV / RetNet pass (see SsmPassOp for lanes and slots).
virtual void logsumexp_rows(const float *in, float *out, size_t rows, size_t cols)=0
Per-row log-sum-exp, max-subtracted: out[i] = log sum_j exp(in[i][j]).
virtual void adam_step_multi(const AdamTensorStep *tensors, size_t count, float beta1, float beta2, float eps)
Adam (or AdamW, or Adam with L2 decay) over many parameter tensors at once (HIP-6)....
Definition device_backend.hpp:484
virtual void tanh_gaussian_backward(const float *action, const float *std_cache, const float *eps, const float *grad_action, const float *grad_log_prob, float *grad_mean, float *grad_log_std, size_t n, float stabilizer)=0
TanhGaussianPolicy gradients w.r.t. mean and log_std, per element.
virtual void avg_pool_forward(const float *in, float *out, size_t planes, size_t h, size_t w, size_t kh, size_t kw)=0
Non-overlapping average pool over planes of (h, w).
virtual void group_norm_forward(const float *in, const float *gamma, const float *beta, float *xhat, float *out, float *group_std, size_t n, size_t c, size_t spatial, size_t num_groups, float eps)=0
GroupNorm per (example, group) of (n, c, spatial) data; group_std is (n, num_groups).
virtual void rl_rows(RlRowOp op, const RlRowArgs &args)=0
One fused RL loss / target / Polyak pass (see RlRowOp for lanes and slots).
virtual void accumulate_parts(const float *parts, size_t count, size_t n, float *acc)
acc += parts[0], then += parts[1], ... (count blocks of n floats, in order): the sums accumulating on...
Definition device_backend.hpp:391
virtual void lrp_softmax_rows(const float *x, const float *y, const float *r, float *r_in, size_t rows, size_t cols)=0
SoftmaxModule rule per row: r_in = x * (r - y * sum(r)).
virtual void im2col(const float *in, float *col, size_t n, size_t c, size_t h, size_t w, const ConvGeometry &geometry)=0
Unfolds (n, c, h, w) into (n, c*kh*kw, out_h*out_w) patches, with out_h = (h + 2*pad_h - kh) / stride...
virtual void avg_pool_backward(const float *grad_out, float *grad_in, size_t planes, size_t h, size_t w, size_t kh, size_t kw)=0
Spreads grad_out / (kh*kw) over each window; grad_in must be zeroed by the caller.
virtual void add_row_vector(const float *in, const float *row, float *out, size_t rows, size_t cols)=0
Broadcast row add: out[i][j] = in[i][j] + row[j] for a (rows x cols) matrix.
virtual void col2im_add(const float *col, float *out, size_t n, size_t c, size_t h, size_t w, const ConvGeometry &geometry)=0
Folds (n, c*kh*kw, out_h*out_w) patches back, adding into out (n, c, h, w).
virtual void batch_norm_update_running(const float *in, float *running_mean, float *running_var, size_t n, size_t c, size_t spatial, float momentum)=0
Folds this batch's per-channel mean and unbiased variance into the running ones (PyTorch's momentum r...
virtual void gemm_ex(const float *a, bool transpose_a, const float *b, bool transpose_b, float *out, size_t m, size_t k, size_t n, float beta)=0
General row-major matrix multiply: out = op(A) * op(B) + beta * out.
virtual void max_unpool(const float *src, const float *argmax, float *dst, size_t planes, size_t h, size_t w, size_t kh, size_t kw, size_t sh, size_t sw, size_t ph, size_t pw)=0
The inverse routing: each input position gets the sum of the pooled values whose argmax it is (window...
virtual void gru_lrp_hprev(const float *h_prev, const float *w_hn, const float *hn, const float *r_term_b, const float *direct, float *r_hprev, size_t rows, size_t hidden, float eps)=0
GRU's R_hprev for (rows, hidden): the recurrent epsilon-rule sum through W_hn, with each element's di...
virtual void max_pool_forward(const float *in, float *out, float *argmax, size_t planes, size_t h, size_t w, size_t kh, size_t kw, size_t sh, size_t sw, size_t ph, size_t pw)=0
Max pool over planes of (h, w) with a (kh, kw) window, stride (sh, sw) and padding (ph,...
virtual void group_norm_backward(const float *grad_out, const float *gamma, const float *xhat, const float *group_std, float *grad_in, float *gamma_grad, float *beta_grad, size_t n, size_t c, size_t spatial, size_t num_groups)=0
GroupNorm input gradient plus this call's gamma/beta gradients (overwritten, per channel).
virtual void batch_norm_eval_backward(const float *grad_out, const float *gamma, const float *xhat, const float *channel_std, float *grad_in, float *gamma_grad, float *beta_grad, size_t n, size_t c, size_t spatial)=0
Eval-mode BatchNorm gradient: grad_out * gamma / std, plus gamma/beta gradients (overwritten,...
virtual void adam_step(float *param, const float *grad, float *m, float *v, size_t n, float lr, float beta1, float beta2, float eps, float bias_correction1, float bias_correction2)=0
One fused Adam update over n parameters.
virtual void add_row_vector_relu(const float *in, const float *row, float *z, float *out, size_t rows, size_t cols)
add_row_vector() into z, then ReLU of z into out: a Linear layer's bias and the ReLU after it (HIP-6)...
Definition device_backend.hpp:413
virtual void bce_with_logits(const float *logits, const float *target, float *out, size_t n)=0
Per-element binary cross-entropy with logits: out[i] = max(x, 0) - x*y + log1p(exp(-|x|)),...
Definition acquisition_functions.hpp:16
LogicOp
Elementwise passes of the fuzzy-logic modules, for DeviceBackend::logic_pointwise.
Definition device_backend.hpp:66
DeviceType
Which physical device a Tensor's buffer resides on.
Definition device_backend.hpp:17
CopyDirection
Direction of a DeviceBackend::copy() call.
Definition device_backend.hpp:24
SsmPassOp
Fused passes of the state-space / linear-recurrence modules (MambaModule, RWKVModule,...
Definition device_backend.hpp:135
RecurrentCellOp
Fused per-element recurrent-cell passes, for DeviceBackend::recurrent_cell. Slots (in[] / out[]),...
Definition device_backend.hpp:88
LrpGate
Elementwise boolean gate for DeviceBackend::lrp_stabilized_divide().
Definition device_backend.hpp:55
@ Positive
passes where gate > 0
@ None
every element passes
@ Negative
passes where gate < 0
RlRowOp
Fused per-row reinforcement-learning passes, for DeviceBackend::rl_rows. One lane per batch row (per ...
Definition device_backend.hpp:205
ElementwiseOp
Unary elementwise operations supported by DeviceBackend::elementwise().
Definition device_backend.hpp:43
@ Silu
x * sigmoid(x) – a.k.a. swish; the gate half of SwiGLU
@ Sigmoid
1 / (1 + exp(-x))
@ GeluTanh
GELU, tanh approximation (gelu_pytorch_tanh) – the gate of Gemma's GeGLU (LLM-9)
@ Exp
exp(x) – GPU-native-kernels Mission 1b (Reparameterize, KL divergence)
@ Gelu
exact GELU, x * Phi(x) with erf – the MLP activation of BERT and ESM (PLM-1)
One parameter tensor's share of DeviceBackend::adam_step_multi (HIP-6): its buffers and its own step'...
Definition device_backend.hpp:225
float lr
Definition device_backend.hpp:231
float * v
Definition device_backend.hpp:229
size_t n
Definition device_backend.hpp:230
float * m
Definition device_backend.hpp:228
float bias_correction1
Definition device_backend.hpp:232
float decay
AdamW: param <- decay * param before the step (1 - lr * wd); 1 for none.
Definition device_backend.hpp:235
float l2
L2 weight decay: the step sees l2 * param + grad; 0 for none.
Definition device_backend.hpp:237
float * param
Definition device_backend.hpp:226
float bias_correction2
Definition device_backend.hpp:233
const float * grad
Definition device_backend.hpp:227
Window geometry for DeviceBackend::im2col / col2im_add: kernel size, stride and zero padding per axis...
Definition device_backend.hpp:173
size_t kh
Definition device_backend.hpp:174
size_t pad_w
Definition device_backend.hpp:179
size_t stride_w
Definition device_backend.hpp:177
size_t kw
Definition device_backend.hpp:175
size_t stride_h
Definition device_backend.hpp:176
size_t pad_h
Definition device_backend.hpp:178
Operand pointers for DeviceBackend::recurrent_cell (passed to kernels by value).
Definition device_backend.hpp:91
float eps
LRP stabilizer (LstmLrp, GruLrp)
Definition device_backend.hpp:94
float * out[6]
Definition device_backend.hpp:93
const float * in[10]
Definition device_backend.hpp:92
Operand pointers and dims for DeviceBackend::rl_rows (passed to kernels by value).
Definition device_backend.hpp:208
float lower
PPO 1 - clip_epsilon.
Definition device_backend.hpp:214
int64_t cols
Definition device_backend.hpp:212
float tau
Polyak blend factor.
Definition device_backend.hpp:217
int64_t rows
Definition device_backend.hpp:211
float * out[4]
Definition device_backend.hpp:210
float gamma
discount factor
Definition device_backend.hpp:216
int64_t ignore_index
TokenCeLoss: the target that marks padding.
Definition device_backend.hpp:218
float upper
PPO 1 + clip_epsilon.
Definition device_backend.hpp:215
const float * in[5]
Definition device_backend.hpp:209
float scale
gradient batch-mean scale (1/N or 2/N)
Definition device_backend.hpp:213
Operand pointers and dims for DeviceBackend::ssm_pass (passed to kernels by value).
Definition device_backend.hpp:158
int64_t d
Definition device_backend.hpp:163
float gamma
RetNet decay.
Definition device_backend.hpp:166
float eps
LRP stabilizer.
Definition device_backend.hpp:165
int64_t l
Definition device_backend.hpp:162
int64_t s
Definition device_backend.hpp:164
int64_t n
Definition device_backend.hpp:161
const float * in[12]
Definition device_backend.hpp:159
float * out[10]
Definition device_backend.hpp:160