pulsatrix 1.1.0
Loading...
Searching...
No Matches
token_cross_entropy_loss.hpp
Go to the documentation of this file.
1
6#pragma once
7
8#include <cstdint>
9
10#include "pulsatrix/tensor.hpp"
11
12namespace pulsatrix {
13
29public:
31 static constexpr int64_t kIgnoreIndex = -100;
32
33 explicit TokenCrossEntropyLoss(DeviceBackend* backend, int64_t ignore_index = kIgnoreIndex);
34
44 [[nodiscard]] float forward(const Tensor& logits, const Tensor& targets, float normalizer = 0.0f);
45
47 [[nodiscard]] Tensor backward() const;
48
50 [[nodiscard]] int64_t num_tokens() const { return num_tokens_; }
51
52private:
54 [[noreturn]] void ThrowForBadTarget(const Tensor& targets, int64_t classes) const;
55
56 DeviceBackend* backend_;
57 int64_t ignore_index_;
58 Tensor probs_;
59 Tensor indices_;
60 Tensor weights_;
61 Shape logits_shape_ = Shape({0});
62 float normalizer_ = 1.0f;
63 int64_t num_tokens_ = 0;
64 bool has_forwarded_ = false;
65};
66
71[[nodiscard]] int64_t CountTargetTokens(const Tensor& targets,
72 int64_t ignore_index = TokenCrossEntropyLoss::kIgnoreIndex);
73
74} // namespace pulsatrix
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
An N-dimensional shape. A plain aggregate of dimensions with no invariant beyond "non-negative dimens...
Definition shape.hpp:24
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
sum over tokens of -log softmax(logits)[target], divided by a normalizer; tokens whose target is the ...
Definition token_cross_entropy_loss.hpp:28
static constexpr int64_t kIgnoreIndex
PyTorch's default ignore_index.
Definition token_cross_entropy_loss.hpp:31
TokenCrossEntropyLoss(DeviceBackend *backend, int64_t ignore_index=kIgnoreIndex)
float forward(const Tensor &logits, const Tensor &targets, float normalizer=0.0f)
Computes the loss and caches what backward() needs.
Tensor backward() const
Gradient w.r.t. the logits, same shape.
int64_t num_tokens() const
Real (not ignored) tokens in the last forward().
Definition token_cross_entropy_loss.hpp:50
Definition acquisition_functions.hpp:16
int64_t CountTargetTokens(const Tensor &targets, int64_t ignore_index=TokenCrossEntropyLoss::kIgnoreIndex)
Number of entries of targets that aren't ignore_index: the normalizer for one accumulation window whe...
N-dimensional tensor – owns a buffer via DeviceBackend*, RAII (Rule of Five).