pulsatrix 1.1.0
Loading...
Searching...
No Matches
param_groups.hpp
Go to the documentation of this file.
1
6#pragma once
7
8#include <cmath>
9#include <functional>
10#include <stdexcept>
11#include <string>
12#include <unordered_map>
13#include <utility>
14#include <vector>
15
16#include "pulsatrix/module.hpp"
17
18namespace pulsatrix {
19
21using ParamSelector = std::function<bool(const std::string& name, const Tensor& value)>;
22
30struct ParamGroup {
32 std::string name;
36 float weight_decay = 0.0f;
37};
38
40namespace param_select {
41
43inline ParamSelector name_prefix(std::string prefix) {
44 return [prefix = std::move(prefix)](const std::string& name, const Tensor&) {
45 return name == prefix || name.rfind(prefix + ".", 0) == 0;
46 };
47}
48
54 return [](const std::string&, const Tensor& value) { return value.rank() <= 1; };
55}
56
57} // namespace param_select
58
60inline void check_optimizer_setting(float value, const std::string& what) {
61 if (!std::isfinite(value) || value < 0.0f) {
62 throw std::invalid_argument(what + " must be finite and non-negative");
63 }
64}
65
68public:
75
77 void set(std::vector<ParamGroup> groups) {
78 for (const ParamGroup& g : groups) {
79 check(g);
80 }
81 groups_ = std::move(groups);
82 }
83
84 [[nodiscard]] std::vector<ParamGroup>& groups() { return groups_; }
85 [[nodiscard]] const std::vector<ParamGroup>& groups() const { return groups_; }
86
92 [[nodiscard]] std::vector<Assignment> resolve(Module& module, float default_learning_rate,
93 float default_weight_decay) const {
94 for (const ParamGroup& g : groups_) {
95 check(g);
96 }
97 std::unordered_map<const Tensor*, const ParamGroup*> chosen;
98 if (!groups_.empty()) {
99 for (const NamedParamRef& p : module.named_parameters()) {
100 for (const ParamGroup& g : groups_) {
101 if (g.selects(p.name, *p.ref.value)) {
102 chosen.emplace(p.ref.value, &g);
103 break;
104 }
105 }
106 }
107 }
108 std::vector<Assignment> out;
109 for (ParamRef p : module.parameters()) {
110 auto it = chosen.find(p.value);
111 if (it == chosen.end()) {
112 out.push_back({p, default_learning_rate, default_weight_decay});
113 } else {
114 out.push_back({p, it->second->learning_rate, it->second->weight_decay});
115 }
116 }
117 return out;
118 }
119
120private:
121 static void check(const ParamGroup& g) {
122 if (!g.selects) {
123 throw std::invalid_argument("parameter group \"" + g.name + "\" has no selector");
124 }
125 check_optimizer_setting(g.learning_rate, "parameter group \"" + g.name + "\" learning_rate");
126 check_optimizer_setting(g.weight_decay, "parameter group \"" + g.name + "\" weight_decay");
127 }
128
129 std::vector<ParamGroup> groups_;
130};
131
132} // namespace pulsatrix
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
The group machinery SGDOptimizer and AdamOptimizer share.
Definition param_groups.hpp:67
std::vector< Assignment > resolve(Module &module, float default_learning_rate, float default_weight_decay) const
Every parameter of module, in parameters() order, with its group's settings.
Definition param_groups.hpp:92
const std::vector< ParamGroup > & groups() const
Definition param_groups.hpp:85
std::vector< ParamGroup > & groups()
Definition param_groups.hpp:84
void set(std::vector< ParamGroup > groups)
Definition param_groups.hpp:77
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
ParamSelector one_dimensional()
Parameters of rank 0 or 1: biases and normalization scales. The usual recipe gives these no weight de...
Definition param_groups.hpp:53
ParamSelector name_prefix(std::string prefix)
The parameter named exactly prefix, and every parameter under it (dot boundaries).
Definition param_groups.hpp:43
Definition acquisition_functions.hpp:16
std::function< bool(const std::string &name, const Tensor &value)> ParamSelector
Picks parameters by their named_parameters() name and value.
Definition param_groups.hpp:21
void check_optimizer_setting(float value, const std::string &what)
Throws std::invalid_argument unless value is finite and non-negative.
Definition param_groups.hpp:60
A parameter together with its hierarchical, dot-separated name relative to the module that reported i...
Definition module.hpp:37
One parameter with the settings that apply to it this step.
Definition param_groups.hpp:70
float weight_decay
Definition param_groups.hpp:73
float learning_rate
Definition param_groups.hpp:72
ParamRef ref
Definition param_groups.hpp:71
Optimizer settings for the parameters selects picks.
Definition param_groups.hpp:30
std::string name
For error messages and logs.
Definition param_groups.hpp:32
ParamSelector selects
Definition param_groups.hpp:33
float weight_decay
L2 penalty: the step uses grad + weight_decay * value, as PyTorch's SGD and Adam do.
Definition param_groups.hpp:36
float learning_rate
Definition param_groups.hpp:34
A trainable parameter and its accumulated gradient, as owned by some Module.
Definition module.hpp:24