pulsatrix 1.1.0
Loading...
Searching...
No Matches
provider.hpp
Go to the documentation of this file.
1
5#pragma once
6
7#include <chrono>
8#include <cstdint>
9#include <functional>
10#include <memory>
11#include <optional>
12#include <stdexcept>
13#include <string>
14#include <string_view>
15#include <utility>
16#include <vector>
17
21
22namespace pulsatrix::orch {
23
26 std::function<void(std::string_view text)> on_text = {};
27 std::function<void(std::string_view thinking)> on_thinking = {};
28};
29
31class ProviderError : public std::runtime_error {
32public:
38 ProviderError(const std::string& message, int status, std::string type, std::string request_id)
39 : std::runtime_error(message), status_(status), type_(std::move(type)), request_id_(std::move(request_id)) {}
40
41 [[nodiscard]] int status() const { return status_; }
42 [[nodiscard]] const std::string& type() const { return type_; }
43 [[nodiscard]] const std::string& request_id() const { return request_id_; }
44
45private:
46 int status_;
47 std::string type_;
48 std::string request_id_;
49};
50
54 int max_retries = 2;
57 std::chrono::milliseconds initial_backoff{1000};
58 std::chrono::milliseconds max_backoff{30'000};
60 std::chrono::milliseconds max_retry_after{60'000};
62 std::function<void(std::chrono::milliseconds)> sleep;
64 uint64_t seed = 0x5eed;
65};
66
68class Provider {
69public:
70 virtual ~Provider() = default;
71
78 [[nodiscard]] virtual ChatResponse chat(const ChatRequest& request, const StreamCallbacks& stream = {}) = 0;
79
81 [[nodiscard]] virtual std::string name() const = 0;
82};
83
87 std::string base_url = "https://api.anthropic.com";
88 std::string api_version = "2023-06-01";
90 std::vector<std::string> betas;
93 bool prompt_caching = true;
95 std::chrono::milliseconds read_timeout{600'000};
96
101 [[nodiscard]] static AnthropicOptions FromEnv();
102};
103
108 std::string base_url = "http://127.0.0.1:8080/v1";
110 std::optional<Secret> api_key;
112 std::chrono::milliseconds read_timeout{600'000};
113};
114
116[[nodiscard]] std::unique_ptr<Provider> MakeAnthropicProvider(AnthropicOptions options,
117 std::shared_ptr<HttpTransport> transport = nullptr);
118
120[[nodiscard]] std::unique_ptr<Provider> MakeOpenAiCompatibleProvider(OpenAiCompatibleOptions options,
121 std::shared_ptr<HttpTransport> transport = nullptr);
122
123} // namespace pulsatrix::orch
The provider answered with an error, or failed after its retries.
Definition provider.hpp:31
ProviderError(const std::string &message, int status, std::string type, std::string request_id)
Definition provider.hpp:38
int status() const
Definition provider.hpp:41
const std::string & request_id() const
Definition provider.hpp:43
const std::string & type() const
Definition provider.hpp:42
A model behind an API.
Definition provider.hpp:68
virtual std::string name() const =0
For example anthropic (https://api.anthropic.com).
virtual ~Provider()=default
virtual ChatResponse chat(const ChatRequest &request, const StreamCallbacks &stream={})=0
Sends request and returns the model's answer, streaming it to stream as it arrives.
Holds a credential, such as an LLM provider's API key.
Definition secret.hpp:20
HTTP for the orchestrator: requests, responses and a transport with two backends.
The orchestrator's message format, shared by every LLM provider.
Definition http.hpp:20
std::unique_ptr< Provider > MakeAnthropicProvider(AnthropicOptions options, std::shared_ptr< HttpTransport > transport=nullptr)
The Anthropic provider. transport defaults to MakeHttpTransport().
std::unique_ptr< Provider > MakeOpenAiCompatibleProvider(OpenAiCompatibleOptions options, std::shared_ptr< HttpTransport > transport=nullptr)
The OpenAI-compatible provider. transport defaults to MakeHttpTransport().
A credential such as an API key, which can't be printed by accident.
Settings for the Anthropic Messages API.
Definition provider.hpp:85
Secret api_key
Definition provider.hpp:86
static AnthropicOptions FromEnv()
Options with the key from ANTHROPIC_API_KEY.
bool prompt_caching
Cache the tool definitions, the system prompt and the conversation so far, so each turn of a tool loo...
Definition provider.hpp:93
std::chrono::milliseconds read_timeout
Definition provider.hpp:95
std::string base_url
Definition provider.hpp:87
std::string api_version
Definition provider.hpp:88
std::vector< std::string > betas
Beta features to enable (the anthropic-beta header), for example a dated feature name.
Definition provider.hpp:90
RetryPolicy retry
Definition provider.hpp:94
A request to a provider.
Definition message.hpp:99
A provider's answer.
Definition message.hpp:113
Settings for an OpenAI-compatible chat-completions server (llama-server, vLLM, Ollama).
Definition provider.hpp:105
std::optional< Secret > api_key
Sent as Authorization: Bearer, if the server needs one.
Definition provider.hpp:110
std::chrono::milliseconds read_timeout
Definition provider.hpp:112
std::string base_url
The URL before /chat/completions, for example llama-server's http://127.0.0.1:8080/v1 or Ollama's htt...
Definition provider.hpp:108
RetryPolicy retry
Definition provider.hpp:111
When and how long to retry.
Definition provider.hpp:52
int max_retries
Retries after the first attempt. 0 turns retrying off.
Definition provider.hpp:54
uint64_t seed
The seed for the jitter.
Definition provider.hpp:64
std::chrono::milliseconds initial_backoff
The first backoff. Each retry doubles it up to max_backoff, and the wait is a random fraction of that...
Definition provider.hpp:57
std::function< void(std::chrono::milliseconds)> sleep
Waits between attempts; replaceable so tests don't sleep. Empty: really sleep.
Definition provider.hpp:62
std::chrono::milliseconds max_backoff
Definition provider.hpp:58
std::chrono::milliseconds max_retry_after
The longest retry-after to honor. A longer one fails at once instead of waiting.
Definition provider.hpp:60
Called as a streamed answer arrives. Either may be empty.
Definition provider.hpp:25
std::function< void(std::string_view text)> on_text
Definition provider.hpp:26
std::function< void(std::string_view thinking)> on_thinking
Definition provider.hpp:27