Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions docs/adr/0007-hosted-providers-and-failover.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,12 @@ provider protocol so they sit behind the same gateway as the offline policy.
(0.5 s doubling, capped at 8 s) and failover to the next entry.
* The API maps an exhausted chain to 503 with `Retry-After` and a rejected request to 502,
and counts both in `agent_platform_provider_errors_total`.
* Requests use temperature 0. Token usage reported by the provider is accumulated per
provider instance.
* Defaults are the current small models, `gpt-6-luna` and `claude-haiku-5-5`. OpenAI
requests set `reasoning_effort: "none"` and temperature 0, so answers repeat and no hidden
reasoning tokens are billed. Anthropic requests use API version `2026-10-21` and send no
sampling parameters, because that version and the Claude 5 models accept only the defaults;
their token budget (2048) leaves room for adaptive thinking. Token usage reported by the
provider is accumulated per provider instance.

## Consequences

Expand Down
15 changes: 9 additions & 6 deletions src/agent_platform/providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@

OPENAI_URL = "https://api.openai.com/v1"
ANTHROPIC_URL = "https://api.anthropic.com/v1"
DEFAULT_OPENAI_MODEL = "gpt-4o-mini"
DEFAULT_ANTHROPIC_MODEL = "claude-3-5-haiku-latest"
DEFAULT_OPENAI_MODEL = "gpt-6-luna"
DEFAULT_ANTHROPIC_MODEL = "claude-haiku-5-5"
# Messages API version: 2026-10-21 rejects sampling parameters, so requests send none.
ANTHROPIC_VERSION = "2026-10-21"
PROVIDER_NAMES = ("offline", "openai", "anthropic")


Expand Down Expand Up @@ -118,7 +120,9 @@ def complete(self, system: str, prompt: str) -> str:
{"authorization": f"Bearer {self.api_key}"},
{
"model": self.model,
"max_tokens": self.max_tokens,
"max_completion_tokens": self.max_tokens,
# Reasoning off keeps answers fast and lets temperature 0 make them repeatable.
"reasoning_effort": "none",
"temperature": 0,
"messages": [
{"role": "system", "content": system},
Expand Down Expand Up @@ -146,7 +150,7 @@ class AnthropicProvider:
api_key: str = ""
base_url: str = ANTHROPIC_URL
timeout: float = 30.0
max_tokens: int = 512
max_tokens: int = 2048 # includes adaptive thinking tokens on Claude 5 models
usage: UsageCounter = field(default_factory=UsageCounter)

def __post_init__(self) -> None:
Expand All @@ -161,11 +165,10 @@ def complete(self, system: str, prompt: str) -> str:
payload = _post_json(
"anthropic",
f"{self.base_url.rstrip('/')}/messages",
{"x-api-key": self.api_key, "anthropic-version": "2023-06-01"},
{"x-api-key": self.api_key, "anthropic-version": ANTHROPIC_VERSION},
{
"model": self.model,
"max_tokens": self.max_tokens,
"temperature": 0,
"system": system,
"messages": [{"role": "user", "content": prompt}],
},
Expand Down
9 changes: 7 additions & 2 deletions tests/test_providers.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ def test_openai_request_shape_and_usage(fake: FakeLLM) -> None:
assert headers["authorization"] == "Bearer sk-test"
assert body["model"] == "m1"
assert body["temperature"] == 0
assert body["reasoning_effort"] == "none"
assert body["max_completion_tokens"] == 512
assert "max_tokens" not in body
assert body["messages"] == [
{"role": "system", "content": "be brief"},
{"role": "user", "content": "hi"},
Expand All @@ -121,7 +124,9 @@ def test_anthropic_request_shape_and_text_blocks(fake: FakeLLM) -> None:
assert provider.complete("sys", "hi") == "ab"
_, headers, body = fake.requests[0]
assert headers["x-api-key"] == "ak-test"
assert headers["anthropic-version"] == "2023-06-01"
assert headers["anthropic-version"] == "2026-10-21"
assert body["max_tokens"] == 2048
assert not {"temperature", "top_p", "top_k"} & body.keys()
assert body["system"] == "sys"
assert body["messages"] == [{"role": "user", "content": "hi"}]

Expand Down Expand Up @@ -176,7 +181,7 @@ def test_failover_from_anthropic_to_openai(fake: FakeLLM) -> None:
)
assert chain.complete("s", "p") == "from openai"
assert chain.stats.failovers == 1
assert chain.name == "anthropic:claude-3-5-haiku-latest > openai:gpt-4o-mini"
assert chain.name == "anthropic:claude-haiku-5-5 > openai:gpt-6-luna"


def test_parse_chain_and_build_provider() -> None:
Expand Down
Loading