diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 65c5083b6..75d1b1fd1 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -128,6 +128,7 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_ | `POST` | `/v1/chat/completions` | OpenAI Chat Completions | | `POST` | `/v1/messages` | Anthropic Messages | | `POST` | `/v1/responses` | OpenAI Responses | +| `POST` | `/v1/decision` | Resolve selected and fallback targets without a post-routing answer call | | `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target | | `GET` | `/v1/models` | Routes served by this deployment | | `GET` | `/v1/stats` | Per-model usage plus curated algorithm stats | @@ -139,6 +140,14 @@ Requests name a route by its `id`, so `POST /v1/chat/completions` with `"model": routes through the `[routes.general]` entry above. Any of the three request formats can address any route, and the server translates between them. +`POST /v1/decision` accepts `{"input_format": "openai_chat", "request": {...}}`, where the +nested request names the route in `model`. It executes required classifier or judge calls, then +returns the selected target and ordered fallbacks with their model, format, base URL, and +`extra_body`. It does not make a post-routing answer call. Routing-time calls still execute, and +response-dependent algorithms such as escalation and advisor routing may produce an answer while +deciding. When they do, the endpoint includes the buffered answer as `response`, encoded in +`input_format`; otherwise the field is omitted. + For `stage_router`, `algorithm_stats.stage_router` groups routing decisions by source and semantic target and summarizes its score, confidence, and input-dimension histograms. These values reset with `/v1/stats/reset`; the process-lifetime counters on `/metrics` remain cumulative. diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index a78695cd5..420ac5e2b 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -20,7 +20,7 @@ use switchyard_llm_client::{ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, TranslatingLlmClient, }; -use switchyard_protocol::{ModelId, RoutedLlmClient}; +use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat}; use crate::{ CallerAuthKind, CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState, @@ -44,22 +44,31 @@ pub fn load_server_state(path: impl AsRef) -> ServerResult { } fn server_state_from_toml(toml: &str) -> ServerResult { - let config: ServerConfig = toml::from_str(toml) - .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?; - config.build() + let config: Arc = Arc::new( + toml::from_str(toml) + .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?, + ); + let state = config.build()?; + Ok(state.with_config(config)) } #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -struct ServerConfig { +pub(crate) struct ServerConfig { schema_version: u32, #[serde(default)] - llm_clients: BTreeMap, - targets: BTreeMap, + pub(crate) llm_clients: BTreeMap, + pub(crate) targets: BTreeMap, routes: BTreeMap, } impl ServerConfig { + pub(crate) fn routing_target_names(&self, route_name: &str) -> Option> { + self.routes + .get(route_name) + .map(RouteConfig::routing_target_names) + } + fn build(&self) -> ServerResult { if self.schema_version != SUPPORTED_SCHEMA_VERSION { return Err(ServerError::new(format!( @@ -94,6 +103,11 @@ impl ServerConfig { for (route_name, config) in &self.routes { validate_value("route name", route_name)?; validate_value(&format!("route {route_name} id"), config.id())?; + for target_name in config.callable_target_names() { + self.targets.get(target_name).ok_or_else(|| { + ServerError::new(format!("route references unknown target {target_name}")) + })?; + } let capabilities = config.capabilities(); if capabilities.context_window == Some(0) { return Err(ServerError::new(format!( @@ -110,6 +124,7 @@ impl ServerConfig { caller_auth, capabilities, count_tokens_target, + Some(route_name.clone()), )); } ServerState::new_with_capabilities(routes) @@ -246,9 +261,9 @@ fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -struct LlmClientConfig { - format: ClientFormat, - base_url: String, +pub(crate) struct LlmClientConfig { + pub(crate) format: ClientFormat, + pub(crate) base_url: String, api_key_env: Option, #[serde(default)] forward_auth: bool, @@ -260,15 +275,15 @@ struct LlmClientConfig { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -struct TargetConfig { - id: ModelId, - llm_client: String, +pub(crate) struct TargetConfig { + pub(crate) id: ModelId, + pub(crate) llm_client: String, #[serde(default)] - extra_body: BTreeMap, + pub(crate) extra_body: BTreeMap, } #[derive(Clone, Copy, Debug, Deserialize)] -enum ClientFormat { +pub(crate) enum ClientFormat { #[serde(rename = "openai_chat")] OpenAiChat, #[serde(rename = "openai_responses")] @@ -278,6 +293,14 @@ enum ClientFormat { } impl ClientFormat { + pub(crate) const fn wire_format(self) -> WireFormat { + match self { + Self::OpenAiChat => WireFormat::OpenAiChat, + Self::OpenAiResponses => WireFormat::OpenAiResponses, + Self::AnthropicMessages => WireFormat::AnthropicMessages, + } + } + const fn caller_auth_kind(self) -> CallerAuthKind { match self { Self::AnthropicMessages => CallerAuthKind::Anthropic, @@ -1493,9 +1516,10 @@ classifier_magic = true ]; for (toml, expected) in cases { + let error = error_message(&toml); assert!( - error_message(&toml).contains(expected), - "expected error containing {expected}" + error.contains(expected), + "expected error containing {expected}, got {error}" ); } } diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index e302e29dd..483c1f6a6 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -32,9 +32,9 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Extension, Json, Router}; use axum_server::tls_rustls::RustlsConfig; -use libsy::{Algorithm, LibsyError}; +use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive}; use parking_lot::Mutex; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient}; use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage}; @@ -42,8 +42,9 @@ use tokio::net::{TcpListener, TcpSocket}; use tokio::task; use tracing::{Instrument, Level}; -use switchyard_translation::{WireFormat, decode_request}; +use switchyard_translation::{WireFormat, decode_request, encode_aggregated_response}; +use crate::config::ServerConfig; use crate::response::into_http_response; use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env}; @@ -112,6 +113,32 @@ struct RouteEntry { caller_auth: Option, capabilities: ModelCapabilities, count_tokens_target: Option, + config_name: Option, +} + +/// Decision result using the same selected/fallback order as libsy. +#[derive(Serialize)] +struct DecisionResponse<'a> { + selected: DecisionTargetResponse<'a>, + fallbacks: Vec>, + #[serde(skip_serializing_if = "Option::is_none")] + response: Option, +} + +/// Borrowed response view over one target's existing configuration. +#[derive(Serialize)] +struct DecisionTargetResponse<'a> { + target: &'a str, + model: &'a ModelId, + llm_client: DecisionLlmClientResponse<'a>, + extra_body: &'a BTreeMap, +} + +/// Non-secret client settings needed to call a selected model. +#[derive(Serialize)] +struct DecisionLlmClientResponse<'a> { + format: WireFormat, + base_url: &'a str, } /// Caller credential family required by forwarded-auth backends. @@ -158,6 +185,7 @@ impl CountTokensTarget { #[derive(Clone)] pub struct ServerState { routes: Arc>, + config: Option>, metrics: prometheus::Registry, stats: StatsAccumulator, routing_log: Option, @@ -212,6 +240,7 @@ impl ServerState { None, ModelCapabilities::default(), None, + None, ) })) } @@ -225,12 +254,20 @@ impl ServerState { Option, ModelCapabilities, Option, + Option, ), >, ) -> ServerResult { let mut entries = BTreeMap::new(); - for (model, algorithm, target_clients, caller_auth, capabilities, count_tokens_target) in - routes + for ( + model, + algorithm, + target_clients, + caller_auth, + capabilities, + count_tokens_target, + config_name, + ) in routes { let model = ModelId::from(model.trim()); if model.is_empty() { @@ -242,6 +279,7 @@ impl ServerState { caller_auth, capabilities, count_tokens_target, + config_name, }; if entries.insert(model.clone(), entry).is_some() { return Err(ServerError::new(format!("duplicate route model {model}"))); @@ -257,6 +295,7 @@ impl ServerState { ); Ok(Self { routes: Arc::new(entries), + config: None, metrics, stats, routing_log: None, @@ -264,6 +303,12 @@ impl ServerState { }) } + /// Retains the validated configuration used to describe decision results. + fn with_config(mut self, config: Arc) -> Self { + self.config = Some(config); + self + } + /// Enables durable per-request routing records at `path`. pub fn with_routing_log(mut self, path: impl Into) -> ServerResult { self.routing_log = Some(SharedRoutingLog::new(path.into())?); @@ -285,6 +330,42 @@ impl ServerState { fn route_for_model(&self, model: &str) -> Option<&RouteEntry> { self.routes.get(model) } + + /// Resolves the routing outcome through the target names configured for its route. + fn decision_response<'a>( + &'a self, + route: &'a RouteEntry, + outcome: &RoutingOutcome, + response: Option, + ) -> Option> { + let config = self.config.as_ref()?; + let target_names = config.routing_target_names(route.config_name.as_ref()?)?; + let resolve = |model: &ModelId| { + let (target_name, target) = target_names + .iter() + .filter_map(|name| config.targets.get_key_value(*name)) + .find(|(_, target)| target.id == *model)?; + let client = config.llm_clients.get(&target.llm_client)?; + Some(DecisionTargetResponse { + target: target_name, + model: &target.id, + llm_client: DecisionLlmClientResponse { + format: client.format.wire_format(), + base_url: &client.base_url, + }, + extra_body: &target.extra_body, + }) + }; + Some(DecisionResponse { + selected: resolve(&outcome.selected_model_id)?, + fallbacks: outcome + .fallback_models + .iter() + .map(resolve) + .collect::>>()?, + response, + }) + } } /// Runtime options shared by server entry points. @@ -503,6 +584,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router { .route("/v1/chat/completions", post(openai_chat_completions)) .route("/v1/messages", post(anthropic_messages)) .route("/v1/responses", post(openai_responses)) + .route("/v1/decision", post(decision)) .route("/v1/messages/count_tokens", post(anthropic_count_tokens)) .route("/v1/models", get(models)) .route("/v1/stats", get(get_stats)) @@ -564,6 +646,92 @@ async fn openai_responses( handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await } +/// One provider request submitted for a routing decision without an answer-model call. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct DecisionEndpointRequest { + input_format: WireFormat, + request: Value, +} + +/// Selects a target while still allowing the algorithm's classifier and judge calls. +async fn decision( + State(state): State, + headers: HeaderMap, + body: std::result::Result, JsonRejection>, +) -> Response { + let body = match body { + Ok(Json(body)) if body.request.is_object() => body, + Ok(_) => { + return invalid_body_error(StatusCode::BAD_REQUEST, "request must be a JSON object"); + } + Err(error) => { + return invalid_body_error( + error.status(), + format!("Request body must be valid JSON: {error}"), + ); + } + }; + let input_format = body.input_format; + let (route, request) = match resolve_route( + &state, + metadata_from_headers(headers), + body.request, + input_format, + ) { + Ok(resolved) => resolved, + Err(response) => return response, + }; + let mut outcome = match run_decision_only(route, request).await { + Ok(outcome) => outcome, + Err(error) => return algorithm_error(error), + }; + let response = match outcome.response.take() { + Some(response) => { + let aggregate = match response.llm_response.into_agg().await { + Ok(aggregate) => aggregate, + Err(error) => return client_error(&error), + }; + match encode_aggregated_response( + &aggregate, + input_format, + Some(outcome.selected_model_id.as_str()), + ) { + Ok(response) => Some(response), + Err(error) => return server_error(error.to_string()), + } + } + None => None, + }; + match state.decision_response(route, &outcome, response) { + Some(response) => Json(response).into_response(), + None => { + server_error("routing outcome contains a model with no callable target configuration") + } + } +} + +/// Completes routing-time calls and returns the outcome without serving its answer target. +async fn run_decision_only(route: &RouteEntry, request: Request) -> libsy::Result { + drive(Arc::clone(&route.algorithm), request, |call| { + serve_decision_dependency(route, call) + }) + .await +} + +/// Serves a classifier or judge call that the decision depends on. +async fn serve_decision_dependency(route: &RouteEntry, call: CallModel) -> libsy::Result<()> { + let model = call.models.first().cloned().ok_or(LibsyError::NoTargets)?; + let result = match route.target_clients.route(&model) { + Ok(client) => client + .call(call.request.clone()) + .await + .map_err(|source| LibsyError::client_call(model, source)), + Err(source) => Err(LibsyError::client_call(model, source)), + }; + call.respond(result) +} + /// Anthropic token counting against the route's explicitly configured target. async fn anthropic_count_tokens( State(state): State, diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 0a97f90e3..21b1e46da 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -956,6 +956,132 @@ base_threshold = 0.5 Ok(()) } +/// Decision-only routing returns callable metadata and preserves any answer produced while routing. +#[tokio::test] +async fn decision_returns_callable_target_and_routing_answer() -> TestResult { + let judge_upstream = MockUpstream::start().await?; + let model_upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.judge_provider] +format = "openai_chat" +base_url = "{judge_url}" + +[llm_clients.model_provider] +format = "openai_chat" +base_url = "{model_url}" + +[targets.judge] +id = "model/classifier" +llm_client = "judge_provider" + +[targets.quality] +id = "model/strong" +llm_client = "model_provider" + +[targets.economy] +id = "model/weak" +llm_client = "model_provider" +extra_body = {{ service_tier = "priority" }} + +[routes.classify] +id = "switchyard/classify" +type = "llm_classifier" +classifier_target = "judge" +strong_target = "quality" +weak_target = "economy" +base_threshold = 0.5 + +[routes.escalation] +id = "switchyard/escalation" +type = "llm_classifier" +mode = "escalation" +classifier_target = "judge" +strong_target = "quality" +weak_target = "economy" +escalation = {{ confirmations = 1 }} +"#, + judge_url = judge_upstream.base_url, + model_url = model_upstream.base_url, + ))?; + let app = build_switchyard_router(state); + + let response = send( + &app, + "POST", + "/v1/decision", + Some(json!({ + "input_format": "openai_chat", + "request": { + "model": "switchyard/classify", + "messages": [{"role": "user", "content": "bounded task"}] + } + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response.json()?, + json!({ + "selected": { + "target": "economy", + "model": "model/weak", + "llm_client": { + "format": "openai_chat", + "base_url": model_upstream.base_url, + }, + "extra_body": {"service_tier": "priority"}, + }, + "fallbacks": [{ + "target": "quality", + "model": "model/strong", + "llm_client": { + "format": "openai_chat", + "base_url": model_upstream.base_url, + }, + "extra_body": {}, + }], + }) + ); + assert_eq!( + judge_upstream.models().await, + vec!["model/classifier".to_string()] + ); + assert!(model_upstream.models().await.is_empty()); + + judge_upstream.calls.lock().await.clear(); + model_upstream.calls.lock().await.clear(); + let response = send( + &app, + "POST", + "/v1/decision", + Some(json!({ + "input_format": "openai_chat", + "request": { + "model": "switchyard/escalation", + "messages": [{"role": "user", "content": "bounded task"}] + } + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + let response = response.json()?; + assert_eq!(response["selected"]["target"], "economy"); + assert_eq!(response["fallbacks"], json!([])); + assert_eq!(response["response"]["model"], "model/weak"); + assert_eq!( + response["response"]["choices"][0]["message"]["content"], + "ok" + ); + assert_eq!(model_upstream.models().await, ["model/weak"]); + assert_eq!(judge_upstream.models().await, ["model/classifier"]); + Ok(()) +} + /// A critical tool error must reach the stage router's signal scorer, which reads /// the decoded conversation. The endpoint records no inbound wire format, so a /// scorer that parsed the raw body instead would find nothing and route every turn