pulsatrix 1.1.0
Loading...
Searching...
No Matches
adam_optimizer.hpp
Go to the documentation of this file.
1
5#pragma once
6
7#include <stdexcept>
8#include <unordered_map>
9
10#include "pulsatrix/module.hpp"
12
13namespace pulsatrix {
14
23public:
32 explicit AdamOptimizer(float learning_rate, DeviceBackend* backend, float beta1 = 0.9f, float beta2 = 0.999f,
33 float eps = 1e-8f);
34
43 void step(Module& module);
44
49 void zero_grad(Module& module);
50
52 [[nodiscard]] float learning_rate() const { return learning_rate_; }
53
63 void set_learning_rate(float learning_rate) { learning_rate_ = learning_rate; }
64
68 [[nodiscard]] float weight_decay() const { return weight_decay_; }
70 check_optimizer_setting(weight_decay, "AdamOptimizer weight_decay");
71 weight_decay_ = weight_decay;
72 }
73
76 void set_param_groups(std::vector<ParamGroup> groups) { groups_.set(std::move(groups)); }
77 [[nodiscard]] std::vector<ParamGroup>& param_groups() { return groups_.groups(); }
78
80 struct AdamState {
83 int64_t t = 0;
84 };
85
90 [[nodiscard]] const AdamState* state(const Tensor* parameter) const {
91 auto it = state_.find(parameter);
92 return it == state_.end() ? nullptr : &it->second;
93 }
94
100 void set_state(const Tensor* parameter, AdamState state) {
101 if (state.m.shape() != parameter->shape() || state.v.shape() != parameter->shape()) {
102 throw std::invalid_argument("AdamOptimizer::set_state: moments must have the parameter's shape");
103 }
104 if (state.m.device() != parameter->device() || state.v.device() != parameter->device()) {
105 throw std::invalid_argument("AdamOptimizer::set_state: moments must be on the parameter's device");
106 }
107 if (state.t < 0) {
108 throw std::invalid_argument("AdamOptimizer::set_state: step count must be non-negative");
109 }
110 state_.insert_or_assign(parameter, std::move(state));
111 }
112
113protected:
115 void set_decoupled_weight_decay(bool decoupled) { decoupled_weight_decay_ = decoupled; }
116
117private:
118 float learning_rate_;
119 float weight_decay_ = 0.0f;
120 bool decoupled_weight_decay_ = false;
121 ParamGroupSet groups_;
122 DeviceBackend* backend_;
123 float beta1_;
124 float beta2_;
125 float eps_;
126 std::unordered_map<const Tensor*, AdamState> state_;
127};
128
138public:
143 AdamWOptimizer(float learning_rate, DeviceBackend* backend, float weight_decay = 0.01f, float beta1 = 0.9f,
144 float beta2 = 0.999f, float eps = 1e-8f)
145 : AdamOptimizer(learning_rate, backend, beta1, beta2, eps) {
148 }
149};
150
151} // namespace pulsatrix
Adam (Kingma & Ba, 2015): per-parameter moving averages of gradient (m) and squared gradient (v),...
Definition adam_optimizer.hpp:22
void set_weight_decay(float weight_decay)
Definition adam_optimizer.hpp:69
void set_state(const Tensor *parameter, AdamState state)
Replaces the state for parameter, e.g. when resuming from a checkpoint.
Definition adam_optimizer.hpp:100
void step(Module &module)
Applies one Adam update to every parameter the module exposes.
void set_param_groups(std::vector< ParamGroup > groups)
Replaces the parameter groups; see ParamGroup.
Definition adam_optimizer.hpp:76
void set_decoupled_weight_decay(bool decoupled)
Switches weight decay from L2 (Adam) to decoupled (AdamW).
Definition adam_optimizer.hpp:115
std::vector< ParamGroup > & param_groups()
Definition adam_optimizer.hpp:77
void zero_grad(Module &module)
Resets every parameter's gradient to zero. Does not reset Adam's moment state.
void set_learning_rate(float learning_rate)
Overwrites the step size used by every subsequent step() call – necessary infrastructure for any mid-...
Definition adam_optimizer.hpp:63
AdamOptimizer(float learning_rate, DeviceBackend *backend, float beta1=0.9f, float beta2=0.999f, float eps=1e-8f)
Constructs an Adam optimizer.
const AdamState * state(const Tensor *parameter) const
The state for parameter (a value tensor from Module::parameters()), or nullptr if step() hasn't updat...
Definition adam_optimizer.hpp:90
float weight_decay() const
The default group's weight decay: L2 added to the gradient, as PyTorch's Adam does (decoupled decay i...
Definition adam_optimizer.hpp:68
float learning_rate() const
Current step size.
Definition adam_optimizer.hpp:52
AdamW (Loshchilov & Hutter, "Decoupled Weight Decay Regularization", arXiv 1711.05101): Adam whose we...
Definition adam_optimizer.hpp:137
AdamWOptimizer(float learning_rate, DeviceBackend *backend, float weight_decay=0.01f, float beta1=0.9f, float beta2=0.999f, float eps=1e-8f)
Definition adam_optimizer.hpp:143
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
The group machinery SGDOptimizer and AdamOptimizer share.
Definition param_groups.hpp:67
std::vector< ParamGroup > & groups()
Definition param_groups.hpp:84
void set(std::vector< ParamGroup > groups)
Definition param_groups.hpp:77
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
DeviceType device() const
Which device this tensor's buffer conceptually resides on.
Definition tensor.hpp:122
const Shape & shape() const
This tensor's shape.
Definition tensor.hpp:113
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
Definition acquisition_functions.hpp:16
void check_optimizer_setting(float value, const std::string &what)
Throws std::invalid_argument unless value is finite and non-negative.
Definition param_groups.hpp:60
Optimizer parameter groups: a learning rate and weight decay per set of parameters,...
One parameter's Adam state: first and second moments, and its step count.
Definition adam_optimizer.hpp:80
Tensor v
Definition adam_optimizer.hpp:82
int64_t t
Definition adam_optimizer.hpp:83
Tensor m
Definition adam_optimizer.hpp:81