pulsatrix 1.1.0
Loading...
Searching...
No Matches
conformal.hpp
Go to the documentation of this file.
1
29#pragma once
30
31#include <cstdint>
32#include <string>
33#include <vector>
34
35#include "pulsatrix/json.hpp"
36#include "pulsatrix/tensor.hpp"
37
38namespace pulsatrix {
39
41enum class ConformalScore {
43 Lac,
45 Aps
46};
47
51 double alpha = 0.1;
57 bool randomized = false;
58 uint64_t seed = 0;
59};
60
64 double alpha = 0.1;
65 bool randomized = false;
66 uint64_t seed = 0;
68 int64_t n = 0;
71 double quantile = 0.0;
72};
73
82[[nodiscard]] ConformalClassifier CalibrateConformalClassifier(const Tensor& probabilities,
83 const std::vector<int64_t>& labels,
84 const ConformalOptions& options = {});
85
91[[nodiscard]] std::vector<std::vector<int64_t>> PredictionSets(const ConformalClassifier& classifier,
92 const Tensor& probabilities);
93
97 std::string score;
98 double alpha = 0.1;
99 int64_t n = 0;
101 double quantile = 0.0;
102};
103
106 double lower = 0.0;
107 double upper = 0.0;
108};
109
115[[nodiscard]] ConformalRegressor CalibrateConformalRegressor(const std::vector<double>& predictions,
116 const std::vector<double>& targets, double alpha = 0.1);
117
123[[nodiscard]] ConformalRegressor CalibrateConformalQuantileRegressor(const std::vector<double>& lower,
124 const std::vector<double>& upper,
125 const std::vector<double>& targets,
126 double alpha = 0.1);
127
132[[nodiscard]] std::vector<PredictionInterval> PredictionIntervals(const ConformalRegressor& regressor,
133 const std::vector<double>& predictions);
134
140[[nodiscard]] std::vector<PredictionInterval> PredictionIntervals(const ConformalRegressor& regressor,
141 const std::vector<double>& lower,
142 const std::vector<double>& upper);
143
146 int64_t n = 0;
148 double coverage = 0.0;
150 double mean_size = 0.0;
152 int64_t empty = 0;
156 std::vector<int64_t> size_counts;
157 std::vector<double> coverage_by_size;
160 std::vector<double> misses;
161};
162
164[[nodiscard]] CoverageReport EvaluateCoverage(const std::vector<std::vector<int64_t>>& sets,
165 const std::vector<int64_t>& labels);
166
168[[nodiscard]] CoverageReport EvaluateCoverage(const std::vector<PredictionInterval>& intervals,
169 const std::vector<double>& targets);
170
172[[nodiscard]] JsonValue ToJson(const CoverageReport& report);
173
174} // namespace pulsatrix
One JSON value: null, a boolean, a number, a string, an array or an object.
Definition json.hpp:22
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
A small strict JSON value, parser and deterministic writer (RFC 8259).
Definition acquisition_functions.hpp:16
std::vector< PredictionInterval > PredictionIntervals(const ConformalRegressor &regressor, const std::vector< double > &predictions)
The intervals for a point regressor's predictions (absolute_residual).
ConformalScore
How a conformal classifier scores an example.
Definition conformal.hpp:41
@ Aps
The mass of the classes ranked at or above the true one: Adaptive Prediction Sets.
@ Lac
1 − p(true class): Least Ambiguous set-valued Classifier.
ConformalClassifier CalibrateConformalClassifier(const Tensor &probabilities, const std::vector< int64_t > &labels, const ConformalOptions &options={})
Calibrates on held-out examples the model wasn't trained on.
ConformalRegressor CalibrateConformalRegressor(const std::vector< double > &predictions, const std::vector< double > &targets, double alpha=0.1)
Calibrates intervals ŷ ± q̂ from a point regressor's predictions on held-out examples.
JsonValue ToJson(const AttackResult &result, const AttackOptions &options)
The result as a pulsatrix.attack.v1 document, without the adversarial inputs.
CoverageReport EvaluateCoverage(const std::vector< std::vector< int64_t > > &sets, const std::vector< int64_t > &labels)
std::vector< std::vector< int64_t > > PredictionSets(const ConformalClassifier &classifier, const Tensor &probabilities)
The prediction set of each input: the classes, most probable first.
ConformalRegressor CalibrateConformalQuantileRegressor(const std::vector< double > &lower, const std::vector< double > &upper, const std::vector< double > &targets, double alpha=0.1)
Calibrates CQR intervals [lo − q̂, hi + q̂] from a quantile regressor's lower and upper predictions,...
A calibrated conformal classifier.
Definition conformal.hpp:62
double quantile
The ⌈(n + 1)(1 − alpha)⌉-th smallest calibration score; +infinity when that rank is above n,...
Definition conformal.hpp:71
double alpha
Definition conformal.hpp:64
int64_t n
The calibration examples.
Definition conformal.hpp:68
uint64_t seed
Definition conformal.hpp:66
ConformalScore score
Definition conformal.hpp:63
bool randomized
Definition conformal.hpp:65
CalibrateConformalClassifier()'s settings.
Definition conformal.hpp:49
uint64_t seed
Definition conformal.hpp:58
double alpha
The miscoverage: sets miss the true class with probability at most alpha.
Definition conformal.hpp:51
bool randomized
APS only: subtract u p(class) from each score, u uniform in [0, 1) and drawn once per example,...
Definition conformal.hpp:57
ConformalScore score
Definition conformal.hpp:52
A calibrated conformal regressor.
Definition conformal.hpp:95
int64_t n
Definition conformal.hpp:99
std::string score
"absolute_residual" or "cqr".
Definition conformal.hpp:97
double quantile
The ⌈(n + 1)(1 − alpha)⌉-th smallest score; +infinity when that rank is above n.
Definition conformal.hpp:101
double alpha
Definition conformal.hpp:98
How often the sets or intervals held the truth, and how large they were.
Definition conformal.hpp:145
int64_t n
Definition conformal.hpp:146
int64_t empty
Classification only: how many sets were empty.
Definition conformal.hpp:152
std::vector< double > misses
Per input: 1 where the truth was missed, 0 where it was held; PerSliceError() on it tests each slice'...
Definition conformal.hpp:160
std::vector< int64_t > size_counts
Classification only: size_counts[s] sets had s classes, and coverage_by_size[s] of them held the trut...
Definition conformal.hpp:156
double coverage
The fraction that held the truth; compare with 1 − alpha.
Definition conformal.hpp:148
std::vector< double > coverage_by_size
Definition conformal.hpp:157
double mean_size
The mean set size, or the mean interval width.
Definition conformal.hpp:150
One prediction interval.
Definition conformal.hpp:105
double upper
Definition conformal.hpp:107
double lower
Definition conformal.hpp:106
N-dimensional tensor – owns a buffer via DeviceBackend*, RAII (Rule of Five).