pulsatrix 1.1.0
Loading...
Searching...
No Matches
calibration.hpp
Go to the documentation of this file.
1
18#pragma once
19
20#include <cstdint>
21#include <vector>
22
23#include "pulsatrix/json.hpp"
24
25namespace pulsatrix {
26
29 double temperature = 1.0;
30 double log_loss_before = 0.0;
31 double log_loss_after = 0.0;
32 int64_t iterations = 0;
36 bool converged = true;
37};
38
45[[nodiscard]] TemperatureFit FitBinaryTemperature(const std::vector<double>& logits, const std::vector<int64_t>& labels,
46 const std::vector<double>& weights = {});
47
54[[nodiscard]] TemperatureFit FitTemperature(const std::vector<std::vector<double>>& logits,
55 const std::vector<int64_t>& labels, const std::vector<double>& weights = {});
56
59 double lower = 0.0;
60 double upper = 1.0;
61 int64_t count = 0;
62 double weight = 0.0;
65 double mean_abs = 0.0;
66 double bias = 0.0;
67 double rms = 0.0;
70 double median_abs = 0.0;
71 double p90_abs = 0.0;
72 double max_abs = 0.0;
73};
74
76 double clip = 0.0;
78 std::vector<LogOddsBand> bands;
79};
80
91[[nodiscard]] LogOddsErrorReport LogOddsError(const std::vector<double>& predicted, const std::vector<double>& reference,
92 const std::vector<double>& weights = {},
93 const std::vector<double>& band_edges = {1e-3}, double clip = 1e-6);
94
96[[nodiscard]] JsonValue ToJson(const LogOddsErrorReport& report);
97
98} // namespace pulsatrix
One JSON value: null, a boolean, a number, a string, an array or an object.
Definition json.hpp:22
A small strict JSON value, parser and deterministic writer (RFC 8259).
Definition acquisition_functions.hpp:16
TemperatureFit FitTemperature(const std::vector< std::vector< double > > &logits, const std::vector< int64_t > &labels, const std::vector< double > &weights={})
Fits T for multi-class logits: P(y = k) = softmax(logits / T)_k.
LogOddsErrorReport LogOddsError(const std::vector< double > &predicted, const std::vector< double > &reference, const std::vector< double > &weights={}, const std::vector< double > &band_edges={1e-3}, double clip=1e-6)
The error in log-odds of predicted probabilities against reference ones.
JsonValue ToJson(const AttackResult &result, const AttackOptions &options)
The result as a pulsatrix.attack.v1 document, without the adversarial inputs.
TemperatureFit FitBinaryTemperature(const std::vector< double > &logits, const std::vector< int64_t > &labels, const std::vector< double > &weights={})
Fits T for binary log-odds: P(y = 1) = sigmoid(logit / T).
Log-odds errors over the examples whose reference probability is in [lower, upper).
Definition calibration.hpp:58
int64_t count
Definition calibration.hpp:61
double mean_abs
Weighted means of |e| and e, and sqrt of the weighted mean of e^2, where e is logit(predicted) - logi...
Definition calibration.hpp:65
double upper
the last band includes 1
Definition calibration.hpp:60
double lower
Definition calibration.hpp:59
double median_abs
Weighted quantiles of |e|: the smallest |e| with at least half (or 90%) of the band's weight at or be...
Definition calibration.hpp:70
double rms
Definition calibration.hpp:67
double bias
Definition calibration.hpp:66
double p90_abs
Definition calibration.hpp:71
double max_abs
Definition calibration.hpp:72
double weight
the band's total weight
Definition calibration.hpp:62
Definition calibration.hpp:75
std::vector< LogOddsBand > bands
one per band of the reference probability, low to high
Definition calibration.hpp:78
LogOddsBand overall
every example
Definition calibration.hpp:77
double clip
probabilities were clipped to [clip, 1 - clip]
Definition calibration.hpp:76
A fitted temperature and the weighted mean log loss before and after it.
Definition calibration.hpp:28
bool converged
False when the minimum is at a bound of T in [1e-6, 1e6]: the loss keeps falling as T grows (the logi...
Definition calibration.hpp:36
int64_t iterations
Definition calibration.hpp:32
double temperature
Definition calibration.hpp:29
double log_loss_before
at T = 1
Definition calibration.hpp:30
double log_loss_after
at the fitted T
Definition calibration.hpp:31