12#include <unordered_map>
40namespace param_select {
44 return [prefix = std::move(prefix)](
const std::string& name,
const Tensor&) {
45 return name == prefix || name.rfind(prefix +
".", 0) == 0;
54 return [](
const std::string&,
const Tensor& value) {
return value.rank() <= 1; };
61 if (!std::isfinite(value) || value < 0.0f) {
62 throw std::invalid_argument(what +
" must be finite and non-negative");
81 groups_ = std::move(
groups);
84 [[nodiscard]] std::vector<ParamGroup>&
groups() {
return groups_; }
85 [[nodiscard]]
const std::vector<ParamGroup>&
groups()
const {
return groups_; }
92 [[nodiscard]] std::vector<Assignment>
resolve(
Module& module,
float default_learning_rate,
93 float default_weight_decay)
const {
97 std::unordered_map<const Tensor*, const ParamGroup*> chosen;
98 if (!groups_.empty()) {
100 for (const ParamGroup& g : groups_) {
101 if (g.selects(p.name, *p.ref.value)) {
102 chosen.emplace(p.ref.value, &g);
108 std::vector<Assignment> out;
109 for (ParamRef p : module.parameters()) {
110 auto it = chosen.find(p.value);
111 if (it == chosen.end()) {
112 out.push_back({p, default_learning_rate, default_weight_decay});
114 out.push_back({p, it->second->learning_rate, it->second->weight_decay});
121 static void check(
const ParamGroup& g) {
123 throw std::invalid_argument(
"parameter group \"" + g.name +
"\" has no selector");
129 std::vector<ParamGroup> groups_;
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
The group machinery SGDOptimizer and AdamOptimizer share.
Definition param_groups.hpp:67
std::vector< Assignment > resolve(Module &module, float default_learning_rate, float default_weight_decay) const
Every parameter of module, in parameters() order, with its group's settings.
Definition param_groups.hpp:92
const std::vector< ParamGroup > & groups() const
Definition param_groups.hpp:85
std::vector< ParamGroup > & groups()
Definition param_groups.hpp:84
void set(std::vector< ParamGroup > groups)
Definition param_groups.hpp:77
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
ParamSelector one_dimensional()
Parameters of rank 0 or 1: biases and normalization scales. The usual recipe gives these no weight de...
Definition param_groups.hpp:53
ParamSelector name_prefix(std::string prefix)
The parameter named exactly prefix, and every parameter under it (dot boundaries).
Definition param_groups.hpp:43
Definition acquisition_functions.hpp:16
std::function< bool(const std::string &name, const Tensor &value)> ParamSelector
Picks parameters by their named_parameters() name and value.
Definition param_groups.hpp:21
void check_optimizer_setting(float value, const std::string &what)
Throws std::invalid_argument unless value is finite and non-negative.
Definition param_groups.hpp:60
A parameter together with its hierarchical, dot-separated name relative to the module that reported i...
Definition module.hpp:37
One parameter with the settings that apply to it this step.
Definition param_groups.hpp:70
float weight_decay
Definition param_groups.hpp:73
float learning_rate
Definition param_groups.hpp:72
ParamRef ref
Definition param_groups.hpp:71
Optimizer settings for the parameters selects picks.
Definition param_groups.hpp:30
std::string name
For error messages and logs.
Definition param_groups.hpp:32
ParamSelector selects
Definition param_groups.hpp:33
float weight_decay
L2 penalty: the step uses grad + weight_decay * value, as PyTorch's SGD and Adam do.
Definition param_groups.hpp:36
float learning_rate
Definition param_groups.hpp:34
A trainable parameter and its accumulated gradient, as owned by some Module.
Definition module.hpp:24