diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 1cd87dca7..04fb29c22 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -166,6 +166,13 @@ async fn upstream_chat( let custom_target_schema = body .pointer("/response_format/json_schema/schema/properties/decision/properties/target") .is_some(); + // The conversation example's scoring-card schema names its tiers + // "efficient"/"capable"; return a capable verdict so the checked-in example + // config can be exercised end-to-end. + let conversation_target_schema = body + .pointer("/response_format/json_schema/schema/properties/decision/properties/target/enum") + .and_then(Value::as_array) + .is_some_and(|values| values == &vec![json!("efficient"), json!("capable")]); let requests_invalid_verdict = body["messages"].as_array().is_some_and(|messages| { messages.iter().any(|message| { message["content"] @@ -173,6 +180,18 @@ async fn upstream_chat( .is_some_and(|content| content.contains("invalid verdict")) }) }); + // The conversation example's regret route escalates when the judge sees the + // user push back; "wrong" stands in for that correction in the condensed + // trajectory the judge reads. System messages are skipped — the regret + // prompt itself names correction phrases like "wrong". + let requests_regret = body["messages"].as_array().is_some_and(|messages| { + messages.iter().any(|message| { + message["role"] == "user" + && message["content"] + .as_str() + .is_some_and(|content| content.contains("wrong")) + }) + }); let requests_schema_invalid_verdict = body["messages"].as_array().is_some_and(|messages| { messages.iter().any(|message| { message["content"] @@ -180,12 +199,21 @@ async fn upstream_chat( .is_some_and(|content| content.contains("schema-invalid verdict")) }) }); - let content = if model == "model/classifier" && custom_target_schema { + let content = if model == "model/classifier" && conversation_target_schema { + r#"{"decision":{"target":"capable"},"crux":"unstated audience","primary_rule":"CONV-4","p_solve":0.3}"# + } else if model == "model/classifier" && custom_target_schema { if requests_invalid_verdict { r#"{"decision":{"target":"unknown"}}"# } else { r#"{"decision":{"target":"premium"}}"# } + } else if model == "model/classifier" + && body + .pointer("/response_format/json_schema/schema/properties/escalate") + .is_some() + && requests_regret + { + r#"{"escalate":true,"reason":"the user says the answer is wrong"}"# } else if model == "model/classifier" && body .pointer("/response_format/json_schema/schema/properties/escalate") @@ -1160,6 +1188,150 @@ selector = "/decision/target" Ok(()) } +/// Model ids the checked-in conversation example config pins its tiers to. +const CONVERSATION_CAPABLE: &str = "anthropic/claude-opus-4.7"; +const CONVERSATION_EFFICIENT: &str = "moonshotai/kimi-k2.7-code"; + +const CONVERSATION_EXAMPLE: &str = + include_str!("../../../examples/conversation-routing/conversation-routing.toml"); + +/// The checked-in conversation example must parse and build both routes as is +/// (the API key env reference is stripped so the build needs no environment). +#[tokio::test] +async fn conversation_example_config_builds_both_routes() -> TestResult { + let toml = CONVERSATION_EXAMPLE.replace("api_key_env = \"OPENROUTER_API_KEY\"", ""); + let state = load_test_config(&toml)?; + let models = state.models().collect::>(); + assert!(models.contains(&"switchyard/conversation"), "{models:?}"); + assert!( + models.contains(&"switchyard/conversation-regret"), + "{models:?}" + ); + Ok(()) +} + +/// The checked-in conversation example drives both config-only routes +/// end-to-end against a mock upstream: the scoring card routes an unstated- +/// audience question to capable, and the regret route escalates on a user +/// correction and stays capable for the rest of the session. +#[tokio::test] +async fn conversation_example_routes_by_card_and_escalates_on_regret() -> TestResult { + let upstream = MockUpstream::start().await?; + // The mock upstream answers classifier calls addressed to "model/classifier"; + // the checked-in example names a real provider model, so swap just that id. + let toml = CONVERSATION_EXAMPLE + .replace("https://openrouter.ai/api/v1", &upstream.base_url) + .replace("api_key_env = \"OPENROUTER_API_KEY\"", "") + .replace( + "id = \"google/gemini-3.5-flash\"", + "id = \"model/classifier\"", + ); + let state = load_test_config(&toml)?; + let app = build_switchyard_router(state); + + let selected_model = |response: &Response| { + response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()) + .map(str::to_string) + }; + + // Scoring-card route: the judge's verdict names capable, so the frontier + // tier answers. + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/conversation", + "messages": [{"role": "user", "content": "explain what a quasar is"}] + })), + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + selected_model(&response).as_deref(), + Some(CONVERSATION_CAPABLE) + ); + + // Regret route: turn 1 has no regret, so the efficient tier answers. + let session: &[(&str, &str)] = &[("x-switchyard-session-id", "conversation-session")]; + let response = send_with_headers( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/conversation-regret", + "messages": [{"role": "user", "content": "explain what a quasar is"}] + })), + session, + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + selected_model(&response).as_deref(), + Some(CONVERSATION_EFFICIENT) + ); + + // Turn 2 corrects the answer; the judge confirms regret and the session + // escalates on this same turn, dropping the efficient reply. + let response = send_with_headers( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/conversation-regret", + "messages": [ + {"role": "user", "content": "explain what a quasar is"}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "no, that's wrong — explain it again"} + ] + })), + session, + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + selected_model(&response).as_deref(), + Some(CONVERSATION_CAPABLE) + ); + + // Turn 3 stays capable without another judge call: escalation is one-way. + upstream.calls.lock().await.clear(); + let response = send_with_headers( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/conversation-regret", + "messages": [ + {"role": "user", "content": "explain what a quasar is"}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "no, that's wrong — explain it again"}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "thanks, now tell me about pulsars"} + ] + })), + session, + ) + .await?; + assert_eq!(response.status, StatusCode::OK); + assert_eq!( + selected_model(&response).as_deref(), + Some(CONVERSATION_CAPABLE) + ); + assert!( + !upstream + .models() + .await + .contains(&"model/classifier".to_string()), + "a latched session must not consult the judge again" + ); + + Ok(()) +} + #[tokio::test] async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult { let upstream = MockUpstream::start().await?; diff --git a/examples/conversation-routing/README.md b/examples/conversation-routing/README.md new file mode 100644 index 000000000..51f3b94fd --- /dev/null +++ b/examples/conversation-routing/README.md @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Conversation routing example + +A two-tier conversation deployment built entirely from the checked-in +`switchyard-server` config surface — no custom code. + +| Route | Mechanism | Behavior | +|---|---|---| +| `switchyard/conversation` | `llm_classifier` custom mode | Pre-hoc scoring card: a judge reads the turn against the conversation capability card (CONV-1..9) and names `efficient` or `capable`. | +| `switchyard/conversation-regret` | `llm_classifier` escalation mode | Regret-driven escalation: the efficient tier answers by default; when the judge sees user regret (correction, re-ask, dissatisfaction), the session latches to capable. | + +Run: + +```bash +export OPENROUTER_API_KEY=sk-or-... +switchyard-server --config examples/conversation-routing/conversation-routing.toml --port 4000 +``` + +Then point an OpenAI-compatible client at `http://localhost:4000/v1` with +`model = "switchyard/conversation"` or `"switchyard/conversation-regret"`. + +Notes: + +- Replace the OpenRouter model ids with your own tiers; the judge target is a + separate small model, not a routing destination. +- The scoring card is uncalibrated. Tune the CONV rules and thresholds against + your own traffic (user-regret logs are the free calibration label). +- Escalation latches one-way for the session and does not decay back to the + efficient tier; a conversation "task" spans the whole session. diff --git a/examples/conversation-routing/conversation-routing.toml b/examples/conversation-routing/conversation-routing.toml new file mode 100644 index 000000000..471361132 --- /dev/null +++ b/examples/conversation-routing/conversation-routing.toml @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Two-tier conversation routing, config-only — no custom code. Two routes +# implement the two zero-code paths from the conversation routing proposal: +# +# - `switchyard/conversation`: a pre-hoc scoring card (`llm_classifier` custom +# mode) that reads the turn and decides efficient vs capable. +# - `switchyard/conversation-regret`: regret-driven escalation (`llm_classifier` +# escalation mode with a conversation judge prompt) — efficient answers by +# default, and the session latches to capable when the judge sees user regret. +# +# Replace the model ids with your own tiers. See README.md. + +schema_version = 1 + +[llm_clients.openrouter] +format = "openai_chat" +base_url = "https://openrouter.ai/api/v1" +api_key_env = "OPENROUTER_API_KEY" + +[targets.classifier] +id = "google/gemini-3.5-flash" # small judge model; replace with your own +llm_client = "openrouter" + +[targets.efficient] +id = "moonshotai/kimi-k2.7-code" # cheap tier; replace with your own +llm_client = "openrouter" + +[targets.capable] +id = "anthropic/claude-opus-4.7" # strong tier; replace with your own +llm_client = "openrouter" + +# Route A — pre-hoc conversation scoring card. The judge reads the turn and the +# capability card below, then names a target; the policy maps that verdict field +# to a routing target. `default_target` is the fallback when the verdict is +# unusable, and it is deliberately conservative. +[routes.conversation] +id = "switchyard/conversation" +type = "llm_classifier" +mode = "custom" +classifier_target = "classifier" +targets = ["efficient", "capable"] +default_target = "capable" +prompt = ''' +You are a router for a two-tier conversation service. You receive the +conversation's opening message and, when present, the latest user follow-up. + +Decide which tier answers this turn: +- EFFICIENT: a cheap model that handles everyday conversation well but is + weaker on fact precision, multi-step reasoning, long structured output, and + low-resource languages. +- CAPABLE: a frontier model, used when the efficient tier would likely get + the answer wrong. + +Use only evidence in the messages. Do not assume a specific audience, hidden +context, or follow-up opportunities. When the evidence is thin, prefer +CAPABLE: a cheap wrong answer costs more than a conservative route. + +# Assessment procedure + +1. State the crux: the hardest requirement of this turn. +2. Select the one capability rule below that best describes the crux. Use + primary_rule=none when no rule applies. +3. Decide the tier last. Length alone does not decide: a long message can be + easy, and a short one can be hard. + +# Conversation capability card + +- CONV-1 [efficient]: The turn has a checkable output contract stated in the + message itself — translate, summarize, reformat, or rewrite the provided + text against an explicit structure. +- CONV-2 [efficient]: Everyday small talk, chit-chat, or common-knowledge + questions where an imperfect answer is cheap to recover from. +- CONV-3 [efficient]: Short, low-stakes writing the user can judge at a + glance — greetings, simple emails, list generation. +- CONV-4 [capable]: Knowledge-intensive questions where factual precision + matters: medicine, law, finance, specific citations, or up-to-date facts. +- CONV-5 [capable]: Multi-step reasoning: mathematics, logic puzzles, + step-by-step analysis, or comparing options on stated criteria. +- CONV-6 [capable]: Long structured output: reports, essays, plans, or + multi-part documents with formatting constraints. +- CONV-7 [capable]: Low-resource languages, dialect-heavy requests, or + requests mixing several languages. +- CONV-8 [capable]: High-stakes or emotionally sensitive situations where a + clumsy answer is costly: grief, medical symptoms, legal rights. +- CONV-9 [capable]: Requests whose audience or depth is unstated ("explain + quantum entanglement", "tell me about the war") — the same question can be + asked by a child or by an expert. + +# Output + +Return exactly one JSON object matching the supplied response schema: +{"decision": {"target": "efficient" or "capable"}, "crux": "...", +"primary_rule": "...", "p_solve": <0.0-1.0>} + +p_solve is your estimate that EFFICIENT alone would produce an acceptable +answer. It is recorded for calibration and does not change the decision. +''' +response_schema = ''' +{ + "type": "object", + "properties": { + "decision": { + "type": "object", + "properties": { + "target": {"type": "string", "enum": ["efficient", "capable"]} + }, + "required": ["target"], + "additionalProperties": false + }, + "crux": {"type": "string"}, + "primary_rule": { + "type": "string", + "enum": ["CONV-1", "CONV-2", "CONV-3", "CONV-4", "CONV-5", "CONV-6", "CONV-7", "CONV-8", "CONV-9", "none"] + }, + "p_solve": {"type": "number", "minimum": 0.0, "maximum": 1.0} + }, + "required": ["decision", "crux", "primary_rule", "p_solve"], + "additionalProperties": false +} +''' + +[routes.conversation.policy] +type = "target_selector" +selector = "/decision/target" + +# Route B — regret-driven escalation. The efficient tier answers every turn +# until the judge confirms user regret; from that turn on the session latches +# to capable. `confirmations = 1` escalates on the first confirmed regret. +[routes.conversation-regret] +id = "switchyard/conversation-regret" +type = "llm_classifier" +mode = "escalation" +classifier_target = "classifier" +strong_target = "capable" +weak_target = "efficient" +escalation = { confirmations = 1 } +prompt = ''' +You are an escalation judge inside a two-tier conversation service. Each turn +is answered by the EFFICIENT tier (a cheap model) unless you escalate; after +escalation the session stays on the STRONG tier. + +You see the conversation so far: the opening message and the most recent +turns, including every user message and the assistant's answers. Judge the +trajectory: is this conversation going wrong in a way the strong tier would +fix? Return exactly one JSON object: + +{"escalate": boolean, "reason": "one short sentence naming the pattern"} + +The user is the only verifier this conversation has. Escalate only on a clear +pattern of trouble, never on a single mild follow-up. When the evidence is +thin, return {"escalate": false}. + +# Escalate on user regret + +- Direct correction: "wrong", "that's not what I asked", "rephrase", "redo + it", or the user repeating a question the assistant already got. +- Re-explaining information the user already gave, after the assistant + ignored or misread it. +- Explicit dissatisfaction: "that doesn't help", "too vague", "too long", + "that makes no sense". + +# Escalate on answer failure (even without user feedback) + +- Non-answer: the assistant dodged the question, refused without reason, or + answered a different question than the one asked. +- Invented specifics the conversation provides no basis for. +- Self-contradiction: the answer contradicts something established earlier + in the conversation. +- Asking the user for information that was already given. + +# Hold — do not escalate on these + +- A single mild "could you say more" or "give an example" — that is a normal + follow-up, not regret. +- Short answers to small talk, greetings, or list requests, even if terse. +- A user asking for more detail, a different angle, or a follow-up question — + engagement is not regret. +- Open-ended or unverifiable requests answered reasonably — no verifier means + no grounds to escalate. +- A single unremarked slip the user has not challenged. + +Do not emit markdown, commentary, or chain-of-thought — only the JSON object. +'''