pulsatrix 1.1.0
Loading...
Searching...
No Matches
lr_scheduler.hpp
Go to the documentation of this file.
1
6#pragma once
7
8#include <algorithm>
9#include <cmath>
10#include <cstdint>
11#include <stdexcept>
12#include <string>
13#include <vector>
14
15namespace pulsatrix {
16
25public:
27 [[nodiscard]] static LRSchedule Constant(int64_t warmup = 0) {
28 if (warmup < 0) {
29 throw std::invalid_argument("LRSchedule::Constant: warmup must be >= 0");
30 }
31 return LRSchedule(Kind::Constant, warmup, 0, 0.0f);
32 }
33
36 [[nodiscard]] static LRSchedule Linear(int64_t warmup, int64_t total) {
37 check(warmup, total, "LRSchedule::Linear");
38 return LRSchedule(Kind::Linear, warmup, total, 0.0f);
39 }
40
43 [[nodiscard]] static LRSchedule Cosine(int64_t warmup, int64_t total, float min_ratio = 0.0f) {
44 check(warmup, total, "LRSchedule::Cosine");
45 if (!(min_ratio >= 0.0f && min_ratio <= 1.0f)) {
46 throw std::invalid_argument("LRSchedule::Cosine: min_ratio must be in [0, 1]");
47 }
48 return LRSchedule(Kind::Cosine, warmup, total, min_ratio);
49 }
50
52 [[nodiscard]] float multiplier(int64_t step) const {
53 if (step < 0) {
54 throw std::invalid_argument("LRSchedule::multiplier: step must be >= 0");
55 }
56 if (step < warmup_) {
57 return static_cast<float>(static_cast<double>(step) / static_cast<double>(warmup_));
58 }
59 const double span = static_cast<double>(std::max<int64_t>(1, total_ - warmup_));
60 switch (kind_) {
61 case Kind::Constant:
62 return 1.0f;
63 case Kind::Linear:
64 return static_cast<float>(std::max(0.0, static_cast<double>(total_ - step) / span));
65 case Kind::Cosine: {
66 constexpr double kPi = 3.14159265358979323846;
67 const double progress = std::min(1.0, static_cast<double>(step - warmup_) / span);
68 return static_cast<float>(min_ratio_ + (1.0 - min_ratio_) * 0.5 * (1.0 + std::cos(kPi * progress)));
69 }
70 }
71 return 1.0f;
72 }
73
74private:
75 enum class Kind { Constant, Linear, Cosine };
76
77 LRSchedule(Kind kind, int64_t warmup, int64_t total, float min_ratio)
78 : kind_(kind), warmup_(warmup), total_(total), min_ratio_(min_ratio) {}
79
80 static void check(int64_t warmup, int64_t total, const char* who) {
81 if (total <= 0 || warmup < 0 || warmup > total) {
82 throw std::invalid_argument(std::string(who) + ": need 0 <= warmup <= total and total > 0");
83 }
84 }
85
86 Kind kind_;
87 int64_t warmup_;
88 int64_t total_;
89 double min_ratio_;
90};
91
99template <typename Optimizer>
101public:
102 LRScheduler(Optimizer& optimizer, LRSchedule schedule)
103 : optimizer_(optimizer), schedule_(schedule), base_(optimizer.learning_rate()) {
104 for (const auto& group : optimizer.param_groups()) {
105 group_bases_.push_back(group.learning_rate);
106 }
107 apply();
108 }
109
111 void step() {
112 ++step_;
113 apply();
114 }
115
117 [[nodiscard]] int64_t last_step() const { return step_; }
118
121 void set_last_step(int64_t step) {
122 if (step < 0) {
123 throw std::invalid_argument("LRScheduler::set_last_step: step must be >= 0");
124 }
125 step_ = step;
126 apply();
127 }
128
129private:
130 void apply() {
131 auto& groups = optimizer_.param_groups();
132 if (groups.size() != group_bases_.size()) {
133 throw std::logic_error("LRScheduler: the optimizer's parameter groups changed after the scheduler was built");
134 }
135 const float m = schedule_.multiplier(step_);
136 optimizer_.set_learning_rate(base_ * m);
137 for (size_t i = 0; i < groups.size(); ++i) {
138 groups[i].learning_rate = group_bases_[i] * m;
139 }
140 }
141
142 Optimizer& optimizer_;
143 LRSchedule schedule_;
144 float base_;
145 std::vector<float> group_bases_;
146 int64_t step_ = 0;
147};
148
149} // namespace pulsatrix
A learning-rate multiplier as a function of the step, with the formulas of Hugging Face transformers'...
Definition lr_scheduler.hpp:24
static LRSchedule Constant(int64_t warmup=0)
1 after an optional linear warmup.
Definition lr_scheduler.hpp:27
static LRSchedule Cosine(int64_t warmup, int64_t total, float min_ratio=0.0f)
Warmup, then cosine decay to min_ratio at total.
Definition lr_scheduler.hpp:43
static LRSchedule Linear(int64_t warmup, int64_t total)
Warmup, then linear decay to 0 at total.
Definition lr_scheduler.hpp:36
float multiplier(int64_t step) const
The multiplier at step.
Definition lr_scheduler.hpp:52
Applies an LRSchedule to an optimizer (SGDOptimizer, AdamOptimizer, AdamWOptimizer): each rate become...
Definition lr_scheduler.hpp:100
LRScheduler(Optimizer &optimizer, LRSchedule schedule)
Definition lr_scheduler.hpp:102
void step()
Advances one step and sets the new rates.
Definition lr_scheduler.hpp:111
int64_t last_step() const
The step the current rates belong to.
Definition lr_scheduler.hpp:117
void set_last_step(int64_t step)
Jumps to step (e.g. when resuming from a checkpoint) and sets its rates.
Definition lr_scheduler.hpp:121
Definition acquisition_functions.hpp:16