29 throw std::invalid_argument(
"LRSchedule::Constant: warmup must be >= 0");
31 return LRSchedule(Kind::Constant, warmup, 0, 0.0f);
37 check(warmup, total,
"LRSchedule::Linear");
38 return LRSchedule(Kind::Linear, warmup, total, 0.0f);
43 [[nodiscard]]
static LRSchedule Cosine(int64_t warmup, int64_t total,
float min_ratio = 0.0f) {
44 check(warmup, total,
"LRSchedule::Cosine");
45 if (!(min_ratio >= 0.0f && min_ratio <= 1.0f)) {
46 throw std::invalid_argument(
"LRSchedule::Cosine: min_ratio must be in [0, 1]");
48 return LRSchedule(Kind::Cosine, warmup, total, min_ratio);
54 throw std::invalid_argument(
"LRSchedule::multiplier: step must be >= 0");
57 return static_cast<float>(
static_cast<double>(step) /
static_cast<double>(warmup_));
59 const double span =
static_cast<double>(std::max<int64_t>(1, total_ - warmup_));
64 return static_cast<float>(std::max(0.0,
static_cast<double>(total_ - step) / span));
66 constexpr double kPi = 3.14159265358979323846;
67 const double progress = std::min(1.0,
static_cast<double>(step - warmup_) / span);
68 return static_cast<float>(min_ratio_ + (1.0 - min_ratio_) * 0.5 * (1.0 + std::cos(kPi * progress)));
75 enum class Kind { Constant, Linear, Cosine };
77 LRSchedule(Kind kind, int64_t warmup, int64_t total,
float min_ratio)
78 : kind_(kind), warmup_(warmup), total_(total), min_ratio_(min_ratio) {}
80 static void check(int64_t warmup, int64_t total,
const char* who) {
81 if (total <= 0 || warmup < 0 || warmup > total) {
82 throw std::invalid_argument(std::string(who) +
": need 0 <= warmup <= total and total > 0");
99template <
typename Optimizer>
103 : optimizer_(optimizer), schedule_(schedule), base_(optimizer.learning_rate()) {
104 for (
const auto& group : optimizer.param_groups()) {
105 group_bases_.push_back(group.learning_rate);
117 [[nodiscard]] int64_t
last_step()
const {
return step_; }
123 throw std::invalid_argument(
"LRScheduler::set_last_step: step must be >= 0");
131 auto& groups = optimizer_.param_groups();
132 if (groups.size() != group_bases_.size()) {
133 throw std::logic_error(
"LRScheduler: the optimizer's parameter groups changed after the scheduler was built");
136 optimizer_.set_learning_rate(base_ * m);
137 for (
size_t i = 0; i < groups.size(); ++i) {
138 groups[i].learning_rate = group_bases_[i] * m;
142 Optimizer& optimizer_;
143 LRSchedule schedule_;
145 std::vector<float> group_bases_;
A learning-rate multiplier as a function of the step, with the formulas of Hugging Face transformers'...
Definition lr_scheduler.hpp:24
static LRSchedule Constant(int64_t warmup=0)
1 after an optional linear warmup.
Definition lr_scheduler.hpp:27
static LRSchedule Cosine(int64_t warmup, int64_t total, float min_ratio=0.0f)
Warmup, then cosine decay to min_ratio at total.
Definition lr_scheduler.hpp:43
static LRSchedule Linear(int64_t warmup, int64_t total)
Warmup, then linear decay to 0 at total.
Definition lr_scheduler.hpp:36
float multiplier(int64_t step) const
The multiplier at step.
Definition lr_scheduler.hpp:52
Applies an LRSchedule to an optimizer (SGDOptimizer, AdamOptimizer, AdamWOptimizer): each rate become...
Definition lr_scheduler.hpp:100
LRScheduler(Optimizer &optimizer, LRSchedule schedule)
Definition lr_scheduler.hpp:102
void step()
Advances one step and sets the new rates.
Definition lr_scheduler.hpp:111
int64_t last_step() const
The step the current rates belong to.
Definition lr_scheduler.hpp:117
void set_last_step(int64_t step)
Jumps to step (e.g. when resuming from a checkpoint) and sets its rates.
Definition lr_scheduler.hpp:121
Definition acquisition_functions.hpp:16