Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .env.public
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ LITELLM_BASE_URL=
TOKENROUTER_API_KEY=
TOKENROUTER_BASE_URL=https://api.tokenrouter.com/v1

# ── Atlas Cloud (OpenAI-compatible; atlascloud/* models) ──────────────────────────────
ATLASCLOUD_API_KEY=
ATLASCLOUD_API_BASE=https://api.atlascloud.ai/v1

# ── Benchmark code execution
SCENARIO_DIR=
LEADERBOARD_DIR=
Expand Down
12 changes: 12 additions & 0 deletions INSTRUCTIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,13 @@ See [MCP Servers](#mcp-servers) for available tools and [docs/mcp-servers.md](do
| `TOKENROUTER_API_KEY` | _(tokenrouter/* models)_ | TokenRouter API key |
| `TOKENROUTER_BASE_URL` | _(tokenrouter/* models)_ | TokenRouter base URL, e.g. `https://api.tokenrouter.com/v1` |

**Atlas Cloud** — OpenAI-compatible gateway for the direct LLM baseline, selected by the `atlascloud/` prefix

| Variable | Default | Description |
| --------------------- | ------------------------------- | ------------------------------------------------ |
| `ATLASCLOUD_API_KEY` | _(atlascloud/* models)_ | Atlas Cloud API key |
| `ATLASCLOUD_API_BASE` | `https://api.atlascloud.ai/v1` | Atlas Cloud API base URL (optional) |

**OpenCode direct providers** — `opencode-agent` with direct provider routes

| Variable | Default | Description |
Expand Down Expand Up @@ -316,6 +323,11 @@ uv run opencode-agent \
# Direct model-only baseline, no MCP tools
uv run direct-llm-agent --model-id litellm_proxy/Azure/gpt-5-mini-2025-08-07 \
'Return only JSON: {"test": 1}'

# Direct model-only baseline through Atlas Cloud
ATLASCLOUD_API_KEY=... uv run direct-llm-agent \
--model-id atlascloud/openai/gpt-5.4 \
'Return only JSON: {"test": 1}'
```

### OpenCode scenario-suite workspace mode
Expand Down
25 changes: 21 additions & 4 deletions src/llm/openai_compat.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
"""OpenAI-compatible LLM backend (no litellm dependency).

For gateways that expose the standard OpenAI Chat Completions API — such as
`TokenRouter <https://www.tokenrouter.com>`_ — we talk to them with the
`TokenRouter <https://www.tokenrouter.com>`_ and
`Atlas Cloud <https://www.atlascloud.ai>`_ — we talk to them with the
``openai`` SDK directly instead of routing through litellm. litellm only
earns its keep for providers that are *not* OpenAI-shaped (e.g. watsonx).

Expand All @@ -15,7 +16,12 @@
from __future__ import annotations

from .base import LLMBackend, LLMResult
from .routers import is_openai_compat, resolve_model, resolve_router_creds
from .routers import (
ATLASCLOUD_PREFIX,
is_openai_compat,
resolve_model,
resolve_openai_compat_creds,
)

__all__ = ["OpenAICompatBackend", "is_openai_compat"]

Expand All @@ -39,8 +45,19 @@ def generate(self, prompt: str, temperature: float = 0.0) -> str:
def generate_with_usage(self, prompt: str, temperature: float = 0.0) -> LLMResult:
from openai import OpenAI

creds = resolve_router_creds(self._model_id) # strict: clear error if unset
client = OpenAI(base_url=creds.base_url, api_key=creds.api_key)
creds = resolve_openai_compat_creds(self._model_id)
if creds is None:
raise ValueError(
f"missing OpenAI-compatible credentials for {self._model_id!r}"
)
if creds.prefix == ATLASCLOUD_PREFIX:
client = OpenAI(
base_url=creds.base_url,
api_key=creds.api_key,
max_retries=0,
)
else:
client = OpenAI(base_url=creds.base_url, api_key=creds.api_key)
response = client.chat.completions.create(
model=self._model_name,
messages=[{"role": "user", "content": prompt}],
Expand Down
26 changes: 23 additions & 3 deletions src/llm/routers.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

litellm_proxy/<model> LiteLLM proxy (LITELLM_BASE_URL / LITELLM_API_KEY)
tokenrouter/<model> TokenRouter (TOKENROUTER_BASE_URL / TOKENROUTER_API_KEY)
atlascloud/<model> Atlas Cloud (ATLASCLOUD_API_KEY)
"""

from __future__ import annotations
Expand All @@ -21,6 +22,8 @@

LITELLM_PREFIX = "litellm_proxy/"
TOKENROUTER_PREFIX = "tokenrouter/"
ATLASCLOUD_PREFIX = "atlascloud/"
ATLASCLOUD_DEFAULT_BASE_URL = "https://api.atlascloud.ai/v1"


class RouterCreds(NamedTuple):
Expand All @@ -39,12 +42,12 @@ class RouterCreds(NamedTuple):

# Prefixes whose endpoints speak the OpenAI Chat Completions API and can be
# driven by the native ``openai`` SDK (llm.OpenAICompatBackend).
OPENAI_COMPAT_PREFIXES: tuple[str, ...] = (TOKENROUTER_PREFIX,)
OPENAI_COMPAT_PREFIXES: tuple[str, ...] = (TOKENROUTER_PREFIX, ATLASCLOUD_PREFIX)


def router_prefix(model_id: str) -> str | None:
"""Return the proxy-router prefix matching *model_id*, else ``None``."""
for prefix in PROXY_ROUTERS:
for prefix in (*PROXY_ROUTERS, ATLASCLOUD_PREFIX):
if model_id.startswith(prefix):
return prefix
return None
Expand All @@ -65,6 +68,23 @@ def is_openai_compat(model_id: str) -> bool:
return model_id.startswith(OPENAI_COMPAT_PREFIXES)


def resolve_openai_compat_creds(model_id: str) -> RouterCreds | None:
"""Resolve credentials for native OpenAI-compatible backends."""
if model_id.startswith(ATLASCLOUD_PREFIX):
api_key = os.environ.get("ATLASCLOUD_API_KEY")
if not api_key:
raise ValueError(
"ATLASCLOUD_API_KEY must be set when using the 'atlascloud/' model prefix"
)
return RouterCreds(
prefix=ATLASCLOUD_PREFIX,
base_url=os.environ.get("ATLASCLOUD_API_BASE")
or ATLASCLOUD_DEFAULT_BASE_URL,
api_key=api_key,
)
return resolve_router_creds(model_id)


def resolve_router_creds(model_id: str, *, strict: bool = True) -> RouterCreds | None:
"""Resolve endpoint + key for *model_id*, or ``None`` if not proxied.

Expand All @@ -74,7 +94,7 @@ def resolve_router_creds(model_id: str, *, strict: bool = True) -> RouterCreds |
``None`` so the caller can fall back to its own defaults.
"""
prefix = router_prefix(model_id)
if prefix is None:
if prefix is None or prefix not in PROXY_ROUTERS:
return None
base_env, key_env = PROXY_ROUTERS[prefix]
base_url = os.environ.get(base_env)
Expand Down
23 changes: 22 additions & 1 deletion src/llm/tests/test_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,10 @@ def create(**kwargs):
)

class OpenAI:
def __init__(self, base_url=None, api_key=None):
def __init__(self, base_url=None, api_key=None, **kwargs):
captured["base_url"] = base_url
captured["api_key"] = api_key
captured.update(kwargs)
self.chat = types.SimpleNamespace(
completions=types.SimpleNamespace(create=create)
)
Expand All @@ -37,12 +38,14 @@ def __init__(self, base_url=None, api_key=None):

def test_is_openai_compat():
assert is_openai_compat("tokenrouter/MiniMax-M3")
assert is_openai_compat("atlascloud/openai/gpt-5.4")
assert not is_openai_compat("litellm_proxy/aws/claude-opus-4-6")
assert not is_openai_compat("watsonx/meta-llama/llama-3-3-70b-instruct")


def test_make_backend_dispatch():
assert isinstance(make_backend("tokenrouter/MiniMax-M3"), OpenAICompatBackend)
assert isinstance(make_backend("atlascloud/openai/gpt-5.4"), OpenAICompatBackend)
assert isinstance(make_backend("litellm_proxy/aws/claude-opus-4-6"), LiteLLMBackend)
assert isinstance(make_backend("watsonx/meta-llama/x"), LiteLLMBackend)

Expand All @@ -63,10 +66,28 @@ def test_tokenrouter_strips_prefix_and_routes(monkeypatch):
assert captured["model"] == "MiniMax-M3" # bare name, prefix stripped
assert captured["base_url"] == "https://api.tokenrouter.com/v1"
assert captured["api_key"] == "tr-key"
assert "max_retries" not in captured
assert result.text == "hi"
assert (result.input_tokens, result.output_tokens) == (3, 2)


def test_atlascloud_uses_default_endpoint_without_sdk_retries(monkeypatch):
captured: dict = {}
_install_fake_openai(monkeypatch, captured)
monkeypatch.setenv("ATLASCLOUD_API_KEY", "atlas-key") # pragma: allowlist secret
monkeypatch.delenv("ATLASCLOUD_API_BASE", raising=False)

result = make_backend(
"atlascloud/dots-studio/dots-3-note-prev-free"
).generate_with_usage("hello")

assert captured["model"] == "dots-studio/dots-3-note-prev-free"
assert captured["base_url"] == "https://api.atlascloud.ai/v1"
assert captured["api_key"] == "atlas-key" # pragma: allowlist secret
assert captured["max_retries"] == 0
assert result.text == "hi"


def test_model_id_property_keeps_full_string():
assert (
OpenAICompatBackend("tokenrouter/MiniMax-M3").model_id
Expand Down
34 changes: 34 additions & 0 deletions src/llm/tests/test_routers.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@
import pytest

from llm.routers import (
ATLASCLOUD_DEFAULT_BASE_URL,
ATLASCLOUD_PREFIX,
LITELLM_PREFIX,
TOKENROUTER_PREFIX,
is_openai_compat,
resolve_model,
resolve_openai_compat_creds,
resolve_router_creds,
router_prefix,
)
Expand All @@ -17,13 +20,15 @@
def test_prefix_constants():
assert LITELLM_PREFIX == "litellm_proxy/"
assert TOKENROUTER_PREFIX == "tokenrouter/"
assert ATLASCLOUD_PREFIX == "atlascloud/"


@pytest.mark.parametrize(
"model_id,expected",
[
("litellm_proxy/aws/claude-opus-4-6", "aws/claude-opus-4-6"),
("tokenrouter/MiniMax-M3", "MiniMax-M3"),
("atlascloud/openai/gpt-5.4", "openai/gpt-5.4"),
("anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-4-6"),
("gpt-4o", "gpt-4o"),
("", ""),
Expand All @@ -38,6 +43,7 @@ def test_resolve_model(model_id, expected):
[
("litellm_proxy/aws/claude-opus-4-6", "litellm_proxy/"),
("tokenrouter/MiniMax-M3", "tokenrouter/"),
("atlascloud/openai/gpt-5.4", "atlascloud/"),
("anthropic/claude-sonnet-4-6", None),
],
)
Expand All @@ -47,6 +53,7 @@ def test_router_prefix(model_id, expected_prefix):

def test_is_openai_compat():
assert is_openai_compat("tokenrouter/MiniMax-M3")
assert is_openai_compat("atlascloud/openai/gpt-5.4")
assert not is_openai_compat("litellm_proxy/aws/claude-opus-4-6")
assert not is_openai_compat("watsonx/meta-llama/x")

Expand All @@ -62,6 +69,33 @@ def test_resolve_router_creds_tokenrouter(monkeypatch):

def test_resolve_router_creds_native_passthrough():
assert resolve_router_creds("anthropic/claude-sonnet-4-6") is None
assert resolve_router_creds("atlascloud/openai/gpt-5.4") is None


def test_resolve_openai_compat_creds_atlascloud(monkeypatch):
monkeypatch.setenv("ATLASCLOUD_API_KEY", "atlas-key") # pragma: allowlist secret
monkeypatch.delenv("ATLASCLOUD_API_BASE", raising=False)

creds = resolve_openai_compat_creds("atlascloud/openai/gpt-5.4")

assert creds.prefix == ATLASCLOUD_PREFIX
assert creds.base_url == ATLASCLOUD_DEFAULT_BASE_URL
assert creds.api_key == "atlas-key" # pragma: allowlist secret


def test_resolve_openai_compat_creds_atlascloud_custom_base(monkeypatch):
monkeypatch.setenv("ATLASCLOUD_API_KEY", "atlas-key") # pragma: allowlist secret
monkeypatch.setenv("ATLASCLOUD_API_BASE", "https://atlas.example/v1")

creds = resolve_openai_compat_creds("atlascloud/openai/gpt-5.4")

assert creds.base_url == "https://atlas.example/v1"


def test_resolve_openai_compat_creds_atlascloud_requires_key(monkeypatch):
monkeypatch.delenv("ATLASCLOUD_API_KEY", raising=False)
with pytest.raises(ValueError, match="ATLASCLOUD_API_KEY"):
resolve_openai_compat_creds("atlascloud/openai/gpt-5.4")


def test_resolve_router_creds_strict_raises(monkeypatch):
Expand Down