pulsatrix 1.1.0
Loading...
Searching...
No Matches
sgd_optimizer.hpp
Go to the documentation of this file.
1
5#pragma once
6
7#include <stdexcept>
8#include <unordered_map>
9#include <vector>
10
11#include "pulsatrix/module.hpp"
13
14namespace pulsatrix {
15
26public:
36 explicit SGDOptimizer(float learning_rate, float momentum = 0.0f, bool nesterov = false)
37 : learning_rate_(learning_rate), momentum_(momentum), nesterov_(nesterov) {
38 if (!(momentum >= 0.0f && momentum < 1.0f)) {
39 throw std::invalid_argument("SGDOptimizer: momentum must be in [0, 1)");
40 }
41 if (nesterov && momentum == 0.0f) {
42 throw std::invalid_argument("SGDOptimizer: Nesterov momentum needs momentum > 0");
43 }
44 }
45
51 void step(Module& module);
52
57 void zero_grad(Module& module);
58
60 [[nodiscard]] float learning_rate() const { return learning_rate_; }
61 void set_learning_rate(float learning_rate) { learning_rate_ = learning_rate; }
62
65 [[nodiscard]] float weight_decay() const { return weight_decay_; }
67 check_optimizer_setting(weight_decay, "SGDOptimizer weight_decay");
68 weight_decay_ = weight_decay;
69 }
70
73 void set_param_groups(std::vector<ParamGroup> groups) { groups_.set(std::move(groups)); }
74 [[nodiscard]] std::vector<ParamGroup>& param_groups() { return groups_.groups(); }
75
76 [[nodiscard]] float momentum() const { return momentum_; }
77 [[nodiscard]] bool nesterov() const { return nesterov_; }
78
81 [[nodiscard]] const Tensor* momentum_buffer(const Tensor* parameter) const {
82 auto it = buffers_.find(parameter);
83 return it == buffers_.end() ? nullptr : &it->second;
84 }
85
86private:
87 float learning_rate_;
88 float momentum_;
89 bool nesterov_;
90 std::unordered_map<const Tensor*, Tensor> buffers_;
91 float weight_decay_ = 0.0f;
92 ParamGroupSet groups_;
93};
94
95} // namespace pulsatrix
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
The group machinery SGDOptimizer and AdamOptimizer share.
Definition param_groups.hpp:67
std::vector< ParamGroup > & groups()
Definition param_groups.hpp:84
void set(std::vector< ParamGroup > groups)
Definition param_groups.hpp:77
param -= learning_rate * (grad + weight_decay * param), per parameter, for every parameter a Module e...
Definition sgd_optimizer.hpp:25
const Tensor * momentum_buffer(const Tensor *parameter) const
The momentum buffer for parameter, or nullptr before its first step (or with no momentum).
Definition sgd_optimizer.hpp:81
void set_param_groups(std::vector< ParamGroup > groups)
Replaces the parameter groups; see ParamGroup.
Definition sgd_optimizer.hpp:73
void set_weight_decay(float weight_decay)
Definition sgd_optimizer.hpp:66
bool nesterov() const
Definition sgd_optimizer.hpp:77
void set_learning_rate(float learning_rate)
Definition sgd_optimizer.hpp:61
float weight_decay() const
The default group's weight decay. Defaults to 0.
Definition sgd_optimizer.hpp:65
void step(Module &module)
Applies one SGD update to every parameter the module exposes.
SGDOptimizer(float learning_rate, float momentum=0.0f, bool nesterov=false)
Constructs an SGD optimizer.
Definition sgd_optimizer.hpp:36
float momentum() const
Definition sgd_optimizer.hpp:76
void zero_grad(Module &module)
Resets every parameter's gradient to zero.
float learning_rate() const
The default group's learning rate (parameters no group selects).
Definition sgd_optimizer.hpp:60
std::vector< ParamGroup > & param_groups()
Definition sgd_optimizer.hpp:74
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
Definition acquisition_functions.hpp:16
void check_optimizer_setting(float value, const std::string &what)
Throws std::invalid_argument unless value is finite and non-negative.
Definition param_groups.hpp:60
Optimizer parameter groups: a learning rate and weight decay per set of parameters,...