From 280a38084efc7ab7b14ec0162ff10e041431ccb5 Mon Sep 17 00:00:00 2001 From: nachiketb Date: Mon, 17 Aug 2026 12:42:03 -0700 Subject: [PATCH 1/7] feat(server): add decision-only endpoint Signed-off-by: nachiketb --- crates/switchyard-server/README.md | 5 + crates/switchyard-server/src/config.rs | 49 +++++++++- crates/switchyard-server/src/lib.rs | 119 ++++++++++++++++++++++- crates/switchyard-server/tests/server.rs | 79 +++++++++++++++ 4 files changed, 246 insertions(+), 6 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 65c5083b6..48c53f905 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -128,6 +128,7 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_ | `POST` | `/v1/chat/completions` | OpenAI Chat Completions | | `POST` | `/v1/messages` | Anthropic Messages | | `POST` | `/v1/responses` | OpenAI Responses | +| `POST` | `/v1/decision` | Select a callable target without calling the answer model | | `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target | | `GET` | `/v1/models` | Routes served by this deployment | | `GET` | `/v1/stats` | Per-model usage plus curated algorithm stats | @@ -139,6 +140,10 @@ Requests name a route by its `id`, so `POST /v1/chat/completions` with `"model": routes through the `[routes.general]` entry above. Any of the three request formats can address any route, and the server translates between them. +`POST /v1/decision` accepts `{"input_format": "openai_chat", "request": {...}}`, where the +nested request names the route in `model`. It executes required classifier or judge calls, then +returns the selected target's model, format, base URL, and `extra_body` without calling that model. + For `stage_router`, `algorithm_stats.stage_router` groups routing decisions by source and semantic target and summarizes its score, confidence, and input-dimension histograms. These values reset with `/v1/stats/reset`; the process-lifetime counters on `/metrics` remain cumulative. diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index a78695cd5..55323a813 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -20,10 +20,11 @@ use switchyard_llm_client::{ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, TranslatingLlmClient, }; -use switchyard_protocol::{ModelId, RoutedLlmClient}; +use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat}; use crate::{ - CallerAuthKind, CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState, + CallerAuthKind, CountTokensTarget, DecisionLlmClient, DecisionTarget, ModelCapabilities, + ServerError, ServerResult, ServerState, }; const SUPPORTED_SCHEMA_VERSION: u32 = 1; @@ -103,6 +104,7 @@ impl ServerConfig { let algorithm = build_algorithm(route_name, config, &targets)?; let (client, caller_auth) = self.build_route_clients(route_name, config, &clients)?; let count_tokens_target = self.build_count_tokens_target(config, &clients); + let decision_targets = self.build_decision_targets(config)?; routes.push(( config.id().clone(), algorithm, @@ -110,6 +112,7 @@ impl ServerConfig { caller_auth, capabilities, count_tokens_target, + decision_targets, )); } ServerState::new_with_capabilities(routes) @@ -232,6 +235,40 @@ impl ServerConfig { client: client.clone(), }) } + + /// Retains only the non-secret settings an external caller needs after routing. + fn build_decision_targets( + &self, + route: &RouteConfig, + ) -> ServerResult> { + route + .routing_target_names() + .into_iter() + .map(|target_name| { + let target = self.targets.get(target_name).ok_or_else(|| { + ServerError::new(format!("route references unknown target {target_name}")) + })?; + let client = self.llm_clients.get(&target.llm_client).ok_or_else(|| { + ServerError::new(format!( + "target {target_name} references unknown llm client {}", + target.llm_client + )) + })?; + Ok(( + target.id.clone(), + DecisionTarget { + target: target_name.to_string(), + model: target.id.clone(), + llm_client: DecisionLlmClient { + format: client.format.wire_format(), + base_url: client.base_url.clone(), + }, + extra_body: target.extra_body.clone(), + }, + )) + }) + .collect() + } } // Prefer known Claude families, then preserve the route's target order. @@ -278,6 +315,14 @@ enum ClientFormat { } impl ClientFormat { + const fn wire_format(self) -> WireFormat { + match self { + Self::OpenAiChat => WireFormat::OpenAiChat, + Self::OpenAiResponses => WireFormat::OpenAiResponses, + Self::AnthropicMessages => WireFormat::AnthropicMessages, + } + } + const fn caller_auth_kind(self) -> CallerAuthKind { match self { Self::AnthropicMessages => CallerAuthKind::Anthropic, diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index e302e29dd..7c23107f5 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -32,9 +32,10 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Extension, Json, Router}; use axum_server::tls_rustls::RustlsConfig; -use libsy::{Algorithm, LibsyError}; +use futures_util::StreamExt; +use libsy::{Algorithm, CallModel, LibsyError, Step}; use parking_lot::Mutex; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient}; use switchyard_protocol::{LlmClientError, Metadata, ModelId, Request, Usage}; @@ -112,6 +113,23 @@ struct RouteEntry { caller_auth: Option, capabilities: ModelCapabilities, count_tokens_target: Option, + decision_targets: BTreeMap, +} + +/// Public connection details for a target selected by the decision endpoint. +#[derive(Clone, Serialize)] +struct DecisionTarget { + target: String, + model: ModelId, + llm_client: DecisionLlmClient, + extra_body: BTreeMap, +} + +/// Non-secret client settings needed to call a selected model. +#[derive(Clone, Serialize)] +struct DecisionLlmClient { + format: WireFormat, + base_url: String, } /// Caller credential family required by forwarded-auth backends. @@ -212,6 +230,7 @@ impl ServerState { None, ModelCapabilities::default(), None, + BTreeMap::new(), ) })) } @@ -225,12 +244,20 @@ impl ServerState { Option, ModelCapabilities, Option, + BTreeMap, ), >, ) -> ServerResult { let mut entries = BTreeMap::new(); - for (model, algorithm, target_clients, caller_auth, capabilities, count_tokens_target) in - routes + for ( + model, + algorithm, + target_clients, + caller_auth, + capabilities, + count_tokens_target, + decision_targets, + ) in routes { let model = ModelId::from(model.trim()); if model.is_empty() { @@ -242,6 +269,7 @@ impl ServerState { caller_auth, capabilities, count_tokens_target, + decision_targets, }; if entries.insert(model.clone(), entry).is_some() { return Err(ServerError::new(format!("duplicate route model {model}"))); @@ -503,6 +531,7 @@ pub fn build_switchyard_router(state: ServerState) -> Router { .route("/v1/chat/completions", post(openai_chat_completions)) .route("/v1/messages", post(anthropic_messages)) .route("/v1/responses", post(openai_responses)) + .route("/v1/decision", post(decision)) .route("/v1/messages/count_tokens", post(anthropic_count_tokens)) .route("/v1/models", get(models)) .route("/v1/stats", get(get_stats)) @@ -564,6 +593,88 @@ async fn openai_responses( handle_endpoint(state, started, headers, body, WireFormat::OpenAiResponses).await } +/// One provider request submitted for a routing decision without an answer-model call. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct DecisionEndpointRequest { + input_format: WireFormat, + request: Value, +} + +/// Selects a target while still allowing the algorithm's classifier and judge calls. +async fn decision( + State(state): State, + headers: HeaderMap, + body: std::result::Result, JsonRejection>, +) -> Response { + let body = match body { + Ok(Json(body)) if body.request.is_object() => body, + Ok(_) => return invalid_body_error("request must be a JSON object"), + Err(error) => { + return invalid_body_error(format!("Request body must be valid JSON: {error}")); + } + }; + let (route, request) = match resolve_route( + &state, + metadata_from_headers(headers), + body.request, + body.input_format, + ) { + Ok(resolved) => resolved, + Err(response) => return response, + }; + let selected_model = match run_decision_only(route, request).await { + Ok(model) => model, + Err(error) => return algorithm_error(error), + }; + match route.decision_targets.get(&selected_model) { + Some(target) => Json(target.clone()).into_response(), + None => error_response( + StatusCode::UNPROCESSABLE_ENTITY, + format!("selected model {selected_model} has no callable target configuration"), + "invalid_request_error", + "decision_not_callable", + ), + } +} + +/// Drives an algorithm until its answer call is known, without executing that final call. +async fn run_decision_only(route: &RouteEntry, request: Request) -> libsy::Result { + let stream = Arc::clone(&route.algorithm).run_stream(request); + tokio::pin!(stream); + + while let Some(step) = stream.next().await { + match step? { + // Decision-only behavior: take the selected answer call and stop before model I/O. + Step::CallModel(call) if call.is_answer_call => { + let (_, models) = call.into_parts(); + return models.into_iter().next().ok_or(LibsyError::NoTargets); + } + // Normal algorithm execution: classifiers and judges must answer before a final + // target can be selected. + Step::CallModel(call) => serve_decision_dependency(route, *call).await?, + // Published decisions are observability events. The answer CallModel above is the + // executable selection and therefore the endpoint's source of truth. + Step::Decision(_) => {} + Step::Done(_) => return Err(LibsyError::MissingFinalResponse), + } + } + Err(LibsyError::MissingFinalResponse) +} + +/// Serves a classifier or judge call that the decision depends on. +async fn serve_decision_dependency(route: &RouteEntry, call: CallModel) -> libsy::Result<()> { + let model = call.models.first().cloned().ok_or(LibsyError::NoTargets)?; + let result = match route.target_clients.route(&model) { + Ok(client) => client + .call(call.request.clone()) + .await + .map_err(|source| LibsyError::client_call(model, source)), + Err(source) => Err(LibsyError::client_call(model, source)), + }; + call.respond(result) +} + /// Anthropic token counting against the route's explicitly configured target. async fn anthropic_count_tokens( State(state): State, diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 0a97f90e3..546b34484 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -956,6 +956,85 @@ base_threshold = 0.5 Ok(()) } +/// Decision-only routing executes the judge, stops before the selected model call, and +/// returns only the non-secret connection settings needed by an external caller. +#[tokio::test] +async fn decision_returns_callable_target_without_calling_it() -> TestResult { + let judge_upstream = MockUpstream::start().await?; + let model_upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.judge_provider] +format = "openai_chat" +base_url = "{judge_url}" + +[llm_clients.model_provider] +format = "openai_chat" +base_url = "{model_url}" + +[targets.judge] +id = "model/classifier" +llm_client = "judge_provider" + +[targets.quality] +id = "model/strong" +llm_client = "model_provider" + +[targets.economy] +id = "model/weak" +llm_client = "model_provider" +extra_body = {{ service_tier = "priority" }} + +[routes.classify] +id = "switchyard/classify" +type = "llm_classifier" +classifier_target = "judge" +strong_target = "quality" +weak_target = "economy" +base_threshold = 0.5 +"#, + judge_url = judge_upstream.base_url, + model_url = model_upstream.base_url, + ))?; + let app = build_switchyard_router(state); + + let response = send( + &app, + "POST", + "/v1/decision", + Some(json!({ + "input_format": "openai_chat", + "request": { + "model": "switchyard/classify", + "messages": [{"role": "user", "content": "bounded task"}] + } + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + response.json()?, + json!({ + "target": "economy", + "model": "model/weak", + "llm_client": { + "format": "openai_chat", + "base_url": model_upstream.base_url, + }, + "extra_body": {"service_tier": "priority"}, + }) + ); + assert_eq!( + judge_upstream.models().await, + vec!["model/classifier".to_string()] + ); + assert!(model_upstream.models().await.is_empty()); + Ok(()) +} + /// A critical tool error must reach the stage router's signal scorer, which reads /// the decoded conversation. The endpoint records no inbound wire format, so a /// scorer that parsed the raw body instead would find nothing and route every turn From a1f1562af5586090ed65f5ae2eaab3586d46bc49 Mon Sep 17 00:00:00 2001 From: nachiketb Date: Mon, 17 Aug 2026 13:02:31 -0700 Subject: [PATCH 2/7] refactor(server): resolve decisions from deployment config Signed-off-by: nachiketb --- crates/switchyard-server/src/config.rs | 79 +++++++++----------------- crates/switchyard-server/src/lib.rs | 68 ++++++++++++++++------ 2 files changed, 77 insertions(+), 70 deletions(-) diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index 55323a813..a93db20dc 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -23,8 +23,7 @@ use switchyard_llm_client::{ use switchyard_protocol::{ModelId, RoutedLlmClient, WireFormat}; use crate::{ - CallerAuthKind, CountTokensTarget, DecisionLlmClient, DecisionTarget, ModelCapabilities, - ServerError, ServerResult, ServerState, + CallerAuthKind, CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState, }; const SUPPORTED_SCHEMA_VERSION: u32 = 1; @@ -45,22 +44,31 @@ pub fn load_server_state(path: impl AsRef) -> ServerResult { } fn server_state_from_toml(toml: &str) -> ServerResult { - let config: ServerConfig = toml::from_str(toml) - .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?; - config.build() + let config: Arc = Arc::new( + toml::from_str(toml) + .map_err(|error| ServerError::new(format!("failed to parse TOML: {error}")))?, + ); + let state = config.build()?; + Ok(state.with_config(config)) } #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -struct ServerConfig { +pub(crate) struct ServerConfig { schema_version: u32, #[serde(default)] - llm_clients: BTreeMap, - targets: BTreeMap, + pub(crate) llm_clients: BTreeMap, + pub(crate) targets: BTreeMap, routes: BTreeMap, } impl ServerConfig { + pub(crate) fn routing_target_names(&self, route_name: &str) -> Option> { + self.routes + .get(route_name) + .map(RouteConfig::routing_target_names) + } + fn build(&self) -> ServerResult { if self.schema_version != SUPPORTED_SCHEMA_VERSION { return Err(ServerError::new(format!( @@ -104,7 +112,6 @@ impl ServerConfig { let algorithm = build_algorithm(route_name, config, &targets)?; let (client, caller_auth) = self.build_route_clients(route_name, config, &clients)?; let count_tokens_target = self.build_count_tokens_target(config, &clients); - let decision_targets = self.build_decision_targets(config)?; routes.push(( config.id().clone(), algorithm, @@ -112,7 +119,7 @@ impl ServerConfig { caller_auth, capabilities, count_tokens_target, - decision_targets, + Some(route_name.clone()), )); } ServerState::new_with_capabilities(routes) @@ -235,40 +242,6 @@ impl ServerConfig { client: client.clone(), }) } - - /// Retains only the non-secret settings an external caller needs after routing. - fn build_decision_targets( - &self, - route: &RouteConfig, - ) -> ServerResult> { - route - .routing_target_names() - .into_iter() - .map(|target_name| { - let target = self.targets.get(target_name).ok_or_else(|| { - ServerError::new(format!("route references unknown target {target_name}")) - })?; - let client = self.llm_clients.get(&target.llm_client).ok_or_else(|| { - ServerError::new(format!( - "target {target_name} references unknown llm client {}", - target.llm_client - )) - })?; - Ok(( - target.id.clone(), - DecisionTarget { - target: target_name.to_string(), - model: target.id.clone(), - llm_client: DecisionLlmClient { - format: client.format.wire_format(), - base_url: client.base_url.clone(), - }, - extra_body: target.extra_body.clone(), - }, - )) - }) - .collect() - } } // Prefer known Claude families, then preserve the route's target order. @@ -283,9 +256,9 @@ fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -struct LlmClientConfig { - format: ClientFormat, - base_url: String, +pub(crate) struct LlmClientConfig { + pub(crate) format: ClientFormat, + pub(crate) base_url: String, api_key_env: Option, #[serde(default)] forward_auth: bool, @@ -297,15 +270,15 @@ struct LlmClientConfig { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] -struct TargetConfig { - id: ModelId, - llm_client: String, +pub(crate) struct TargetConfig { + pub(crate) id: ModelId, + pub(crate) llm_client: String, #[serde(default)] - extra_body: BTreeMap, + pub(crate) extra_body: BTreeMap, } #[derive(Clone, Copy, Debug, Deserialize)] -enum ClientFormat { +pub(crate) enum ClientFormat { #[serde(rename = "openai_chat")] OpenAiChat, #[serde(rename = "openai_responses")] @@ -315,7 +288,7 @@ enum ClientFormat { } impl ClientFormat { - const fn wire_format(self) -> WireFormat { + pub(crate) const fn wire_format(self) -> WireFormat { match self { Self::OpenAiChat => WireFormat::OpenAiChat, Self::OpenAiResponses => WireFormat::OpenAiResponses, diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 7c23107f5..5852fa163 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -45,6 +45,7 @@ use tracing::{Instrument, Level}; use switchyard_translation::{WireFormat, decode_request}; +use crate::config::ServerConfig; use crate::response::into_http_response; use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env}; @@ -113,23 +114,23 @@ struct RouteEntry { caller_auth: Option, capabilities: ModelCapabilities, count_tokens_target: Option, - decision_targets: BTreeMap, + config_name: Option, } -/// Public connection details for a target selected by the decision endpoint. -#[derive(Clone, Serialize)] -struct DecisionTarget { - target: String, - model: ModelId, - llm_client: DecisionLlmClient, - extra_body: BTreeMap, +/// Borrowed response view over the selected target's existing configuration. +#[derive(Serialize)] +struct DecisionResponse<'a> { + target: &'a str, + model: &'a ModelId, + llm_client: DecisionLlmClientResponse<'a>, + extra_body: &'a BTreeMap, } /// Non-secret client settings needed to call a selected model. -#[derive(Clone, Serialize)] -struct DecisionLlmClient { +#[derive(Serialize)] +struct DecisionLlmClientResponse<'a> { format: WireFormat, - base_url: String, + base_url: &'a str, } /// Caller credential family required by forwarded-auth backends. @@ -176,6 +177,7 @@ impl CountTokensTarget { #[derive(Clone)] pub struct ServerState { routes: Arc>, + config: Option>, metrics: prometheus::Registry, stats: StatsAccumulator, routing_log: Option, @@ -230,7 +232,7 @@ impl ServerState { None, ModelCapabilities::default(), None, - BTreeMap::new(), + None, ) })) } @@ -244,7 +246,7 @@ impl ServerState { Option, ModelCapabilities, Option, - BTreeMap, + Option, ), >, ) -> ServerResult { @@ -256,7 +258,7 @@ impl ServerState { caller_auth, capabilities, count_tokens_target, - decision_targets, + config_name, ) in routes { let model = ModelId::from(model.trim()); @@ -269,7 +271,7 @@ impl ServerState { caller_auth, capabilities, count_tokens_target, - decision_targets, + config_name, }; if entries.insert(model.clone(), entry).is_some() { return Err(ServerError::new(format!("duplicate route model {model}"))); @@ -285,6 +287,7 @@ impl ServerState { ); Ok(Self { routes: Arc::new(entries), + config: None, metrics, stats, routing_log: None, @@ -292,6 +295,12 @@ impl ServerState { }) } + /// Retains the validated configuration used to describe decision results. + fn with_config(mut self, config: Arc) -> Self { + self.config = Some(config); + self + } + /// Enables durable per-request routing records at `path`. pub fn with_routing_log(mut self, path: impl Into) -> ServerResult { self.routing_log = Some(SharedRoutingLog::new(path.into())?); @@ -313,6 +322,31 @@ impl ServerState { fn route_for_model(&self, model: &str) -> Option<&RouteEntry> { self.routes.get(model) } + + /// Resolves one selected model through the target names configured for its route. + fn decision_response<'a>( + &'a self, + route: &'a RouteEntry, + selected_model: &ModelId, + ) -> Option> { + let config = self.config.as_ref()?; + let (target_name, target) = config + .routing_target_names(route.config_name.as_ref()?)? + .into_iter() + .rev() + .filter_map(|name| config.targets.get_key_value(name)) + .find(|(_, target)| target.id == *selected_model)?; + let client = config.llm_clients.get(&target.llm_client)?; + Some(DecisionResponse { + target: target_name, + model: &target.id, + llm_client: DecisionLlmClientResponse { + format: client.format.wire_format(), + base_url: &client.base_url, + }, + extra_body: &target.extra_body, + }) + } } /// Runtime options shared by server entry points. @@ -627,8 +661,8 @@ async fn decision( Ok(model) => model, Err(error) => return algorithm_error(error), }; - match route.decision_targets.get(&selected_model) { - Some(target) => Json(target.clone()).into_response(), + match state.decision_response(route, &selected_model) { + Some(target) => Json(target).into_response(), None => error_response( StatusCode::UNPROCESSABLE_ENTITY, format!("selected model {selected_model} has no callable target configuration"), From 5f05908d270c9f72bbb6b7b4fc9cbd410a20ad52 Mon Sep 17 00:00:00 2001 From: nachiketb Date: Tue, 18 Aug 2026 16:08:02 -0700 Subject: [PATCH 3/7] fix(server): align decision endpoint with routing outcomes Signed-off-by: nachiketb --- crates/switchyard-server/README.md | 3 +- crates/switchyard-server/src/config.rs | 23 +++++- crates/switchyard-server/src/lib.rs | 100 ++++++++++++----------- crates/switchyard-server/tests/server.rs | 23 ++++-- 4 files changed, 92 insertions(+), 57 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 48c53f905..ce89f8232 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -142,7 +142,8 @@ route, and the server translates between them. `POST /v1/decision` accepts `{"input_format": "openai_chat", "request": {...}}`, where the nested request names the route in `model`. It executes required classifier or judge calls, then -returns the selected target's model, format, base URL, and `extra_body` without calling that model. +returns the selected target and ordered fallbacks with their model, format, base URL, and +`extra_body`, without calling an answer target. For `stage_router`, `algorithm_stats.stage_router` groups routing decisions by source and semantic target and summarizes its score, confidence, and input-dimension histograms. These values reset diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index a93db20dc..1e1b1e3f1 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -103,6 +103,20 @@ impl ServerConfig { for (route_name, config) in &self.routes { validate_value("route name", route_name)?; validate_value(&format!("route {route_name} id"), config.id())?; + let mut target_names_by_model = HashMap::new(); + for target_name in config.callable_target_names() { + let target = self.targets.get(target_name).ok_or_else(|| { + ServerError::new(format!("route references unknown target {target_name}")) + })?; + if let Some(previous_name) = target_names_by_model.insert(&target.id, target_name) + && previous_name != target_name + { + return Err(ServerError::new(format!( + "route {route_name} targets {previous_name} and {target_name} share model id {}; model ids must be unique within a route", + target.id + ))); + } + } let capabilities = config.capabilities(); if capabilities.context_window == Some(0) { return Err(ServerError::new(format!( @@ -1508,12 +1522,17 @@ classifier_magic = true ), "route random context_window must be greater than zero", ), + ( + VALID_CONFIG.replace("id = \"weak/model\"", "id = \"strong/model\""), + "targets weak and strong share model id strong/model", + ), ]; for (toml, expected) in cases { + let error = error_message(&toml); assert!( - error_message(&toml).contains(expected), - "expected error containing {expected}" + error.contains(expected), + "expected error containing {expected}, got {error}" ); } } diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 5852fa163..d95891751 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -32,8 +32,7 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Extension, Json, Router}; use axum_server::tls_rustls::RustlsConfig; -use futures_util::StreamExt; -use libsy::{Algorithm, CallModel, LibsyError, Step}; +use libsy::{Algorithm, CallModel, LibsyError, RoutingOutcome, drive}; use parking_lot::Mutex; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -117,9 +116,16 @@ struct RouteEntry { config_name: Option, } -/// Borrowed response view over the selected target's existing configuration. +/// Decision result using the same selected/fallback order as libsy. #[derive(Serialize)] struct DecisionResponse<'a> { + selected: DecisionTargetResponse<'a>, + fallbacks: Vec>, +} + +/// Borrowed response view over one target's existing configuration. +#[derive(Serialize)] +struct DecisionTargetResponse<'a> { target: &'a str, model: &'a ModelId, llm_client: DecisionLlmClientResponse<'a>, @@ -323,28 +329,37 @@ impl ServerState { self.routes.get(model) } - /// Resolves one selected model through the target names configured for its route. + /// Resolves the routing outcome through the target names configured for its route. fn decision_response<'a>( &'a self, route: &'a RouteEntry, - selected_model: &ModelId, + outcome: &'a RoutingOutcome, ) -> Option> { let config = self.config.as_ref()?; - let (target_name, target) = config - .routing_target_names(route.config_name.as_ref()?)? - .into_iter() - .rev() - .filter_map(|name| config.targets.get_key_value(name)) - .find(|(_, target)| target.id == *selected_model)?; - let client = config.llm_clients.get(&target.llm_client)?; + let target_names = config.routing_target_names(route.config_name.as_ref()?)?; + let resolve = |model: &ModelId| { + let (target_name, target) = target_names + .iter() + .filter_map(|name| config.targets.get_key_value(*name)) + .find(|(_, target)| target.id == *model)?; + let client = config.llm_clients.get(&target.llm_client)?; + Some(DecisionTargetResponse { + target: target_name, + model: &target.id, + llm_client: DecisionLlmClientResponse { + format: client.format.wire_format(), + base_url: &client.base_url, + }, + extra_body: &target.extra_body, + }) + }; Some(DecisionResponse { - target: target_name, - model: &target.id, - llm_client: DecisionLlmClientResponse { - format: client.format.wire_format(), - base_url: &client.base_url, - }, - extra_body: &target.extra_body, + selected: resolve(&outcome.selected_model_id)?, + fallbacks: outcome + .fallback_models + .iter() + .map(resolve) + .collect::>>()?, }) } } @@ -643,9 +658,14 @@ async fn decision( ) -> Response { let body = match body { Ok(Json(body)) if body.request.is_object() => body, - Ok(_) => return invalid_body_error("request must be a JSON object"), + Ok(_) => { + return invalid_body_error(StatusCode::BAD_REQUEST, "request must be a JSON object"); + } Err(error) => { - return invalid_body_error(format!("Request body must be valid JSON: {error}")); + return invalid_body_error( + error.status(), + format!("Request body must be valid JSON: {error}"), + ); } }; let (route, request) = match resolve_route( @@ -657,43 +677,27 @@ async fn decision( Ok(resolved) => resolved, Err(response) => return response, }; - let selected_model = match run_decision_only(route, request).await { - Ok(model) => model, + let outcome = match run_decision_only(route, request).await { + Ok(outcome) => outcome, Err(error) => return algorithm_error(error), }; - match state.decision_response(route, &selected_model) { - Some(target) => Json(target).into_response(), + match state.decision_response(route, &outcome) { + Some(response) => Json(response).into_response(), None => error_response( StatusCode::UNPROCESSABLE_ENTITY, - format!("selected model {selected_model} has no callable target configuration"), + "routing outcome contains a model with no callable target configuration", "invalid_request_error", "decision_not_callable", ), } } -/// Drives an algorithm until its answer call is known, without executing that final call. -async fn run_decision_only(route: &RouteEntry, request: Request) -> libsy::Result { - let stream = Arc::clone(&route.algorithm).run_stream(request); - tokio::pin!(stream); - - while let Some(step) = stream.next().await { - match step? { - // Decision-only behavior: take the selected answer call and stop before model I/O. - Step::CallModel(call) if call.is_answer_call => { - let (_, models) = call.into_parts(); - return models.into_iter().next().ok_or(LibsyError::NoTargets); - } - // Normal algorithm execution: classifiers and judges must answer before a final - // target can be selected. - Step::CallModel(call) => serve_decision_dependency(route, *call).await?, - // Published decisions are observability events. The answer CallModel above is the - // executable selection and therefore the endpoint's source of truth. - Step::Decision(_) => {} - Step::Done(_) => return Err(LibsyError::MissingFinalResponse), - } - } - Err(LibsyError::MissingFinalResponse) +/// Completes routing-time calls and returns the outcome without serving its answer target. +async fn run_decision_only(route: &RouteEntry, request: Request) -> libsy::Result { + drive(Arc::clone(&route.algorithm), request, |call| { + serve_decision_dependency(route, call) + }) + .await } /// Serves a classifier or judge call that the decision depends on. diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 546b34484..7185d2177 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1018,13 +1018,24 @@ base_threshold = 0.5 assert_eq!( response.json()?, json!({ - "target": "economy", - "model": "model/weak", - "llm_client": { - "format": "openai_chat", - "base_url": model_upstream.base_url, + "selected": { + "target": "economy", + "model": "model/weak", + "llm_client": { + "format": "openai_chat", + "base_url": model_upstream.base_url, + }, + "extra_body": {"service_tier": "priority"}, }, - "extra_body": {"service_tier": "priority"}, + "fallbacks": [{ + "target": "quality", + "model": "model/strong", + "llm_client": { + "format": "openai_chat", + "base_url": model_upstream.base_url, + }, + "extra_body": {}, + }], }) ); assert_eq!( From 9a4bf3369cafc689dc907449082e61b37cdfeba4 Mon Sep 17 00:00:00 2001 From: nachiketb Date: Tue, 18 Aug 2026 16:39:31 -0700 Subject: [PATCH 4/7] docs(server): clarify decision routing calls Signed-off-by: nachiketb --- crates/switchyard-server/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index ce89f8232..9142c598f 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -128,7 +128,7 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_ | `POST` | `/v1/chat/completions` | OpenAI Chat Completions | | `POST` | `/v1/messages` | Anthropic Messages | | `POST` | `/v1/responses` | OpenAI Responses | -| `POST` | `/v1/decision` | Select a callable target without calling the answer model | +| `POST` | `/v1/decision` | Resolve selected and fallback targets without a post-routing answer call | | `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target | | `GET` | `/v1/models` | Routes served by this deployment | | `GET` | `/v1/stats` | Per-model usage plus curated algorithm stats | @@ -143,7 +143,9 @@ route, and the server translates between them. `POST /v1/decision` accepts `{"input_format": "openai_chat", "request": {...}}`, where the nested request names the route in `model`. It executes required classifier or judge calls, then returns the selected target and ordered fallbacks with their model, format, base URL, and -`extra_body`, without calling an answer target. +`extra_body`. It does not make a post-routing answer call. Routing-time calls still execute, and +response-dependent algorithms such as escalation and advisor routing may produce an answer while +deciding; that response is not included in the decision endpoint's metadata response. For `stage_router`, `algorithm_stats.stage_router` groups routing decisions by source and semantic target and summarizes its score, confidence, and input-dimension histograms. These values reset From fdade018e955e05bbfb88b12c8be1499728312c5 Mon Sep 17 00:00:00 2001 From: nachiketb Date: Tue, 18 Aug 2026 16:48:19 -0700 Subject: [PATCH 5/7] feat(server): return answers produced while routing Signed-off-by: nachiketb --- crates/switchyard-server/README.md | 3 +- crates/switchyard-server/src/lib.rs | 32 +++++++++++++++--- crates/switchyard-server/tests/server.rs | 42 ++++++++++++++++++++++-- 3 files changed, 68 insertions(+), 9 deletions(-) diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 9142c598f..75d1b1fd1 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -145,7 +145,8 @@ nested request names the route in `model`. It executes required classifier or ju returns the selected target and ordered fallbacks with their model, format, base URL, and `extra_body`. It does not make a post-routing answer call. Routing-time calls still execute, and response-dependent algorithms such as escalation and advisor routing may produce an answer while -deciding; that response is not included in the decision endpoint's metadata response. +deciding. When they do, the endpoint includes the buffered answer as `response`, encoded in +`input_format`; otherwise the field is omitted. For `stage_router`, `algorithm_stats.stage_router` groups routing decisions by source and semantic target and summarizes its score, confidence, and input-dimension histograms. These values reset diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index d95891751..8b91e3f82 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -42,7 +42,7 @@ use tokio::net::{TcpListener, TcpSocket}; use tokio::task; use tracing::{Instrument, Level}; -use switchyard_translation::{WireFormat, decode_request}; +use switchyard_translation::{WireFormat, decode_request, encode_aggregated_response}; use crate::config::ServerConfig; use crate::response::into_http_response; @@ -121,6 +121,8 @@ struct RouteEntry { struct DecisionResponse<'a> { selected: DecisionTargetResponse<'a>, fallbacks: Vec>, + #[serde(skip_serializing_if = "Option::is_none")] + response: Option, } /// Borrowed response view over one target's existing configuration. @@ -333,7 +335,8 @@ impl ServerState { fn decision_response<'a>( &'a self, route: &'a RouteEntry, - outcome: &'a RoutingOutcome, + outcome: &RoutingOutcome, + response: Option, ) -> Option> { let config = self.config.as_ref()?; let target_names = config.routing_target_names(route.config_name.as_ref()?)?; @@ -360,6 +363,7 @@ impl ServerState { .iter() .map(resolve) .collect::>>()?, + response, }) } } @@ -668,20 +672,38 @@ async fn decision( ); } }; + let input_format = body.input_format; let (route, request) = match resolve_route( &state, metadata_from_headers(headers), body.request, - body.input_format, + input_format, ) { Ok(resolved) => resolved, Err(response) => return response, }; - let outcome = match run_decision_only(route, request).await { + let mut outcome = match run_decision_only(route, request).await { Ok(outcome) => outcome, Err(error) => return algorithm_error(error), }; - match state.decision_response(route, &outcome) { + let response = match outcome.response.take() { + Some(response) => { + let aggregate = match response.llm_response.into_agg().await { + Ok(aggregate) => aggregate, + Err(error) => return client_error(&error), + }; + match encode_aggregated_response( + &aggregate, + input_format, + Some(outcome.selected_model_id.as_str()), + ) { + Ok(response) => Some(response), + Err(error) => return server_error(error.to_string()), + } + } + None => None, + }; + match state.decision_response(route, &outcome, response) { Some(response) => Json(response).into_response(), None => error_response( StatusCode::UNPROCESSABLE_ENTITY, diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 7185d2177..21b1e46da 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -956,10 +956,9 @@ base_threshold = 0.5 Ok(()) } -/// Decision-only routing executes the judge, stops before the selected model call, and -/// returns only the non-secret connection settings needed by an external caller. +/// Decision-only routing returns callable metadata and preserves any answer produced while routing. #[tokio::test] -async fn decision_returns_callable_target_without_calling_it() -> TestResult { +async fn decision_returns_callable_target_and_routing_answer() -> TestResult { let judge_upstream = MockUpstream::start().await?; let model_upstream = MockUpstream::start().await?; let state = load_test_config(&format!( @@ -994,6 +993,15 @@ classifier_target = "judge" strong_target = "quality" weak_target = "economy" base_threshold = 0.5 + +[routes.escalation] +id = "switchyard/escalation" +type = "llm_classifier" +mode = "escalation" +classifier_target = "judge" +strong_target = "quality" +weak_target = "economy" +escalation = {{ confirmations = 1 }} "#, judge_url = judge_upstream.base_url, model_url = model_upstream.base_url, @@ -1043,6 +1051,34 @@ base_threshold = 0.5 vec!["model/classifier".to_string()] ); assert!(model_upstream.models().await.is_empty()); + + judge_upstream.calls.lock().await.clear(); + model_upstream.calls.lock().await.clear(); + let response = send( + &app, + "POST", + "/v1/decision", + Some(json!({ + "input_format": "openai_chat", + "request": { + "model": "switchyard/escalation", + "messages": [{"role": "user", "content": "bounded task"}] + } + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + let response = response.json()?; + assert_eq!(response["selected"]["target"], "economy"); + assert_eq!(response["fallbacks"], json!([])); + assert_eq!(response["response"]["model"], "model/weak"); + assert_eq!( + response["response"]["choices"][0]["message"]["content"], + "ok" + ); + assert_eq!(model_upstream.models().await, ["model/weak"]); + assert_eq!(judge_upstream.models().await, ["model/classifier"]); Ok(()) } From 11ba96482b7d16667ee07e29871fecfe1ee8ed38 Mon Sep 17 00:00:00 2001 From: nachiketb Date: Tue, 18 Aug 2026 16:52:05 -0700 Subject: [PATCH 6/7] fix(server): allow shared target model ids Signed-off-by: nachiketb --- crates/switchyard-server/src/config.rs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index 1e1b1e3f1..420ac5e2b 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -103,19 +103,10 @@ impl ServerConfig { for (route_name, config) in &self.routes { validate_value("route name", route_name)?; validate_value(&format!("route {route_name} id"), config.id())?; - let mut target_names_by_model = HashMap::new(); for target_name in config.callable_target_names() { - let target = self.targets.get(target_name).ok_or_else(|| { + self.targets.get(target_name).ok_or_else(|| { ServerError::new(format!("route references unknown target {target_name}")) })?; - if let Some(previous_name) = target_names_by_model.insert(&target.id, target_name) - && previous_name != target_name - { - return Err(ServerError::new(format!( - "route {route_name} targets {previous_name} and {target_name} share model id {}; model ids must be unique within a route", - target.id - ))); - } } let capabilities = config.capabilities(); if capabilities.context_window == Some(0) { @@ -1522,10 +1513,6 @@ classifier_magic = true ), "route random context_window must be greater than zero", ), - ( - VALID_CONFIG.replace("id = \"weak/model\"", "id = \"strong/model\""), - "targets weak and strong share model id strong/model", - ), ]; for (toml, expected) in cases { From 83b024fea7cdc7e36a461a9f3edcb401420344cf Mon Sep 17 00:00:00 2001 From: nachiketb Date: Wed, 19 Aug 2026 10:34:29 -0700 Subject: [PATCH 7/7] fix(server): return 500 for invalid routing outcomes Signed-off-by: nachiketb --- crates/switchyard-server/src/lib.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 8b91e3f82..483c1f6a6 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -705,12 +705,9 @@ async fn decision( }; match state.decision_response(route, &outcome, response) { Some(response) => Json(response).into_response(), - None => error_response( - StatusCode::UNPROCESSABLE_ENTITY, - "routing outcome contains a model with no callable target configuration", - "invalid_request_error", - "decision_not_callable", - ), + None => { + server_error("routing outcome contains a model with no callable target configuration") + } } }