hyperband.hpp

Go to the documentation of this file.

 
#pragma once
 
#include <cmath>
#include <limits>
#include <stdexcept>
#include <vector>
 
#include "pulsatrix/successive_halving.hpp"
 
namespace pulsatrix {
 
struct HyperbandBracket {
    int s;
    size_t num_configs;
    int initial_budget;
};
 
inline std::vector<HyperbandBracket> ComputeHyperbandBrackets(int max_resource, double eta) {
    if (max_resource <= 0) {
        throw std::invalid_argument("ComputeHyperbandBrackets: max_resource must be positive");
    }
    if (eta <= 1.0) {
        throw std::invalid_argument("ComputeHyperbandBrackets: eta must be > 1.0");
    }
 
    int s_max = static_cast<int>(std::floor(std::log(static_cast<double>(max_resource)) / std::log(eta)));
    double b = static_cast<double>(s_max + 1) * static_cast<double>(max_resource);
 
    std::vector<HyperbandBracket> brackets;
    for (int s = s_max; s >= 0; --s) {
        double eta_pow_s = std::pow(eta, s);
        size_t num_configs =
            std::max<size_t>(1, static_cast<size_t>(std::ceil((b / max_resource) * (eta_pow_s / (s + 1)))));
        int initial_budget = std::max(1, static_cast<int>(std::round(max_resource / eta_pow_s)));
        brackets.push_back(HyperbandBracket{s, num_configs, initial_budget});
    }
    return brackets;
}
 
struct HyperbandResult {
    Configuration best_configuration;
    double best_metric;
    size_t total_epochs_trained;
};
 
template <typename RNG>
HyperbandResult RunHyperband(const SearchSpace& space, const TrialFactory& make_trial, int max_resource,
                              double eta, RNG& rng) {
    auto brackets = ComputeHyperbandBrackets(max_resource, eta);
 
    HyperbandResult overall{Configuration{}, -std::numeric_limits<double>::infinity(), 0};
    for (const auto& bracket : brackets) {
        auto result = RunSuccessiveHalving(space, make_trial, bracket.num_configs, bracket.initial_budget, eta, rng);
        overall.total_epochs_trained += result.total_epochs_trained;
        if (result.best_metric > overall.best_metric) {
            overall.best_metric = result.best_metric;
            overall.best_configuration = result.best_configuration;
        }
    }
    return overall;
}
 
}  // namespace pulsatrix