diff --git a/src/agents/core/BaseAgentAdapter.ts b/src/agents/core/BaseAgentAdapter.ts index 7e8774cc3..858782962 100644 --- a/src/agents/core/BaseAgentAdapter.ts +++ b/src/agents/core/BaseAgentAdapter.ts @@ -1009,10 +1009,26 @@ export abstract class BaseAgentAdapter implements AgentAdapter { branch: branch || undefined, project: env.CODEMIE_PROJECT || undefined, syncApiUrl: env.CODEMIE_SYNC_API_URL || undefined, - syncCodeMieUrl: env.CODEMIE_URL || undefined + syncCodeMieUrl: env.CODEMIE_URL || undefined, + routing: this._resolveRouting(env, profileConfig), }; } + private _resolveRouting( + env: NodeJS.ProcessEnv, + profileConfig: import('../../env/types.js').CodeMieConfigOptions | undefined, + ): 'signal' | 'classifier' | undefined { + const ALLOWED = new Set(['signal', 'classifier']); + const raw = env.CODEMIE_ROUTING ?? profileConfig?.routing; + if (raw === undefined || raw === null || raw === '') return undefined; + const key = String(raw).trim().toLowerCase(); + if (ALLOWED.has(key)) return key as 'signal' | 'classifier'; + logger.warn( + `[BaseAgentAdapter] Invalid CODEMIE_ROUTING=${JSON.stringify(raw)}; allowed: signal, classifier`, + ); + return undefined; + } + /** * Centralized proxy setup * Works for ALL agents based on their metadata diff --git a/src/agents/plugins/claude/plugin/statusline.mjs b/src/agents/plugins/claude/plugin/statusline.mjs index 16dfd705e..67978172b 100644 --- a/src/agents/plugins/claude/plugin/statusline.mjs +++ b/src/agents/plugins/claude/plugin/statusline.mjs @@ -1,6 +1,9 @@ #!/usr/bin/env node // CodeMie statusline — shows model, project, branch, context, session cost/duration, // and (when a CodeMie profile is configured) the CLI budget for the authenticated user. +// When the request was routed to a different backend model (CodeMie Switchyard or the +// LiteLLM router), the actual model is read from the routing headers the proxy injects +// into the transcript and shown alongside the nominal one — see resolveActualModel(). // Deployed to ~/.claude/ by `codemie install statusline` (also triggered by the `--status` // CLI flag, which calls the same installer). Runs standalone — Node builtins only, no // project imports, since it executes via `node ` after the project process exits. @@ -91,6 +94,8 @@ export function extractBasicInfo(ctx) { return { projectName: cwd ? path.basename(cwd) : '', cwd, + transcriptPath: ctx?.transcript_path ?? '', + modelId: ctx?.model?.id ?? '', model: ctx?.model?.display_name ?? '', ctxPct: ctx?.context_window?.used_percentage ?? null, tokIn: ctx?.context_window?.total_input_tokens ?? null, @@ -100,6 +105,134 @@ export function extractBasicInfo(ctx) { }; } +// --- Actual (routed) model resolution --- +// +// Claude Code's own stdin JSON only ever reports the nominal model (`model.id`, the +// alias/tier the session was started with). When CodeMie Switchyard or the LiteLLM router +// dispatches a turn to a different backend model, that can surface two ways in the transcript's +// most recent assistant turn (transcript_path): +// 1. Routing headers — the proxy's routing-header-injector plugin copies the upstream +// router's response headers onto the response body (see +// src/providers/plugins/sso/proxy/plugins/routing-header-injector.plugin.ts), which +// Claude Code then persists verbatim. Authoritative when present: the proxy tags these +// explicitly, so they win over the body-model heuristic below. +// 2. The response body's own `model` field — every Anthropic-compatible response reports +// the model that actually generated it. A router that doesn't emit routing headers (or +// a deployment where this proxy isn't involved at all) still shows the truth here, so +// it's a fallback signal rather than depending on headers alone. +// +// Header field precedence mirrors src/cli/commands/analytics/cost/usage-readers.ts's +// `routedModel` so the statusline and the analytics report agree when both are present. + +const ROUTED_MODEL_TAIL_BYTES = 65_536; // last 64KB — comfortably covers the most recent turn(s) + +// Claude model family names, used to tell "genuinely routed to a different tier" apart from +// "alias resolved to its concrete dated/region-qualified snapshot", which happens on every +// request regardless of routing and must never be shown as if it were routing. +const MODEL_FAMILY_PATTERN = /(opus|sonnet|haiku|fable)/i; + +/** Pulls the actual dispatched model out of a transcript line's `message` object, if present. */ +export function extractRoutedModel(message) { + if (!message || typeof message !== 'object') return null; + return message.x_codemie_routed_model ?? message.x_codemie_routing_capable_model ?? message['x-litellm-router-routed-model'] ?? null; +} + +function modelFamily(modelId) { + const match = modelId ? MODEL_FAMILY_PATTERN.exec(modelId) : null; + return match ? match[1].toLowerCase() : null; +} + +/** Strips Bedrock region/provider qualifiers (`converse/global.anthropic.` / `eu.anthropic.`) and its `-v1:0` suffix. */ +export function normalizeModelId(modelId) { + if (!modelId) return ''; + return modelId + .toLowerCase() + .replace(/^converse\//, '') + .replace(/^[a-z0-9-]+\.anthropic\./, '') + .replace(/-v\d+:\d+$/, ''); +} + +/** + * True when two model identifiers name the same tier — either a recognized family (opus/ + * sonnet/haiku/fable) matches on both sides, or, when neither side matches a known family, + * the Bedrock-normalized identifiers are identical. Used to avoid flagging an alias's normal + * resolution to its concrete provider snapshot as if it were routing. + */ +export function sameModelFamily(a, b) { + if (!a || !b) return false; + const famA = modelFamily(a); + const famB = modelFamily(b); + if (famA && famB) return famA === famB; + return normalizeModelId(a) === normalizeModelId(b); +} + +/** + * Scans transcript JSONL text backwards for the most recent assistant turn and returns the + * response body's own model plus any header-injected routed model. The first (partial) line + * of a tail read is expected to fail JSON.parse when the read didn't start at a line boundary + * — that's normal, not an error, so parse failures are skipped rather than treated as a reason + * to stop scanning. + */ +export function parseLastAssistantTurn(tailText) { + if (!tailText) return null; + const lines = tailText.split('\n'); + for (let i = lines.length - 1; i >= 0; i--) { + const line = lines[i].trim(); + if (!line) continue; + let parsed; + try { + parsed = JSON.parse(line); + } catch { + continue; + } + const message = parsed?.message; + if (parsed?.type === 'assistant' && message?.model) { + return { responseModel: message.model, headerRoutedModel: extractRoutedModel(message) }; + } + } + return null; +} + +async function defaultReadTail(filePath, maxBytes) { + const handle = await fs.open(filePath, 'r'); + try { + const { size } = await handle.stat(); + const start = Math.max(0, size - maxBytes); + const length = size - start; + if (length <= 0) return ''; + const { buffer, bytesRead } = await handle.read({ buffer: Buffer.alloc(length), position: start }); + return buffer.toString('utf8', 0, bytesRead); + } finally { + await handle.close(); + } +} + +/** + * Resolves the actual routed model for the current session, or null when there is nothing + * useful to show — no transcript, an unreadable transcript, or the best available signal + * (routing headers, falling back to the response body's own model) names the same tier as + * `nominalModelId`. Never throws: the statusline must keep rendering even if the transcript + * is mid-write or has already rotated away. + */ +export async function resolveActualModel(transcriptPath, nominalModelId, { readTail = defaultReadTail } = {}) { + if (!transcriptPath) return null; + let tail; + try { + tail = await readTail(transcriptPath, ROUTED_MODEL_TAIL_BYTES); + } catch { + return null; + } + const turn = parseLastAssistantTurn(tail); + if (!turn) return null; + const candidate = turn.headerRoutedModel ?? turn.responseModel; + if (!candidate) return null; + const nominal = nominalModelId || turn.responseModel; + // Display the Bedrock-stripped form — the raw candidate may be a fully qualified backend + // id (e.g. `converse/global.anthropic.claude-haiku-4-5-20251001-v1:0`), which is accurate + // but not what a human wants to read in a one-line statusline. + return sameModelFamily(nominal, candidate) ? null : normalizeModelId(candidate); +} + export function formatDuration(ms) { if (typeof ms !== 'number' || Number.isNaN(ms) || ms < 0) return null; const mins = Math.floor(ms / 60000); @@ -138,14 +271,14 @@ export function ctxBar(pct) { return `${c(color, bar)} ${pct}%`; } -export function buildStatusLine({ projectName, branch, model, ctxPct, tokIn, tokOut, cost, durationMs, budget, budgetError }) { +export function buildStatusLine({ projectName, branch, model, actualModel, ctxPct, tokIn, tokOut, cost, durationMs, budget, budgetError }) { const parts = []; if (projectName) parts.push(c(C.purple, `[${projectName}]`)); if (budget) parts.push(c(budgetColor(budget.pct), budget.text)); else if (budgetError) parts.push(c(C.yellow, `⚠ ${budgetError}`)); if (branch) parts.push(c(C.blue, `(${branch})`)); - if (model) parts.push(c(C.cyan, `[${model}]`)); + if (model) parts.push(c(C.cyan, `[${actualModel ? `${model} → ${actualModel}` : model}]`)); const bar = ctxBar(ctxPct); if (bar) parts.push(bar); @@ -256,9 +389,13 @@ export async function main() { } const branchPromise = basic.cwd ? gitBranch(basic.cwd) : Promise.resolve(''); - const [budgetResult, branch] = await Promise.all([resolveBudget(), branchPromise]); + const [budgetResult, branch, actualModel] = await Promise.all([ + resolveBudget(), + branchPromise, + resolveActualModel(basic.transcriptPath, basic.modelId), + ]); - process.stdout.write(buildStatusLine({ ...basic, branch, ...budgetResult })); + process.stdout.write(buildStatusLine({ ...basic, branch, actualModel, ...budgetResult })); } // Compares decoded paths (not raw strings) so this correctly matches even when the diff --git a/src/cli/commands/analytics/cost/__tests__/cost-enricher.test.ts b/src/cli/commands/analytics/cost/__tests__/cost-enricher.test.ts index be36bf966..87c62d515 100644 --- a/src/cli/commands/analytics/cost/__tests__/cost-enricher.test.ts +++ b/src/cli/commands/analytics/cost/__tests__/cost-enricher.test.ts @@ -569,3 +569,61 @@ describe('buildCostSeries', () => { expect(s[s.length - 1].tokens).toBe(200); // last cumulative total preserved }); }); + +/** + * Session-level rollup of routing classifier cost. Drives the real reader by feeding routing + * headers through parseNative, so these also pin the reader→enricher contract. + */ +describe('enrichCosts — routing classifier cost', () => { + /** One priced assistant turn (1M input @ $3/1M sonnet-4-5 => $3) plus routing headers. */ + const turn = (routing: Record) => ({ + message: { model: 'claude-sonnet-4-5', usage: { input_tokens: 1_000_000, output_tokens: 0 }, ...routing }, + }); + + const depsWith = (turns: unknown[]): EnricherDeps => ({ + ...baseDeps, + parseNative: async () => + ({ sessionId: 's1', agentName: 'claude', metadata: {}, messages: turns }) as never, + }); + + const LITELLM_BILLED = { + 'x-litellm-router-tier': 'SIMPLE', + 'x-litellm-router-cause': 'llm_classifier', + 'x-litellm-classifier-cost': '0.0072204', + 'x-litellm-classifier-prompt-tokens': '6394', + 'x-litellm-classifier-completion-tokens': '34', + }; + + it('sums LiteLLM classifier cost and folds it into the session total', async () => { + const { index } = await enrichCosts(raw, depsWith([turn(LITELLM_BILLED)])); + const c = index.get('s1')!; + expect(c.judgeCostUSD).toBeCloseTo(0.0072204, 8); + expect(c.judgeInputTokens).toBe(6394); + expect(c.judgeOutputTokens).toBe(34); + expect(c.routingCostKnown).toBe(true); + expect(c.costUSD).toBeCloseTo(3 + 0.0072204, 6); // base cost + routing overhead + }); + + it('accumulates classifier cost across turns', async () => { + const second = { ...LITELLM_BILLED, 'x-litellm-classifier-cost': '0.0064691' }; + const { index } = await enrichCosts(raw, depsWith([turn(LITELLM_BILLED), turn(second)])); + expect(index.get('s1')!.judgeCostUSD).toBeCloseTo(0.0072204 + 0.0064691, 8); + }); + + it('reports cost unknown when any routed turn omits classifier headers', async () => { + const bare = { 'x-litellm-router-tier': 'COMPLEX', 'x-litellm-router-cause': 'llm_classifier' }; + const { index } = await enrichCosts(raw, depsWith([turn(LITELLM_BILLED), turn(bare)])); + const c = index.get('s1')!; + expect(c.routingCostKnown).toBe(false); + // The measured turn still contributes — the total is an understatement, not a blank. + expect(c.judgeCostUSD).toBeCloseTo(0.0072204, 8); + }); + + it('leaves routing fields absent for a session with no routed turns', async () => { + const { index } = await enrichCosts(raw, depsWith([turn({})])); + const c = index.get('s1')!; + expect(c.routingCostKnown).toBeUndefined(); + expect(c.judgeCostUSD).toBeUndefined(); + expect(c.costUSD).toBeCloseTo(3, 6); + }); +}); diff --git a/src/cli/commands/analytics/cost/__tests__/usage-readers.test.ts b/src/cli/commands/analytics/cost/__tests__/usage-readers.test.ts index e1d34511d..33ce6aabb 100644 --- a/src/cli/commands/analytics/cost/__tests__/usage-readers.test.ts +++ b/src/cli/commands/analytics/cost/__tests__/usage-readers.test.ts @@ -898,3 +898,106 @@ describe('gatherUsageDeduped / gatherDedupedUsageRecords — pi fork replay', () expect([...viaDedup.entries()]).toEqual([...viaReader.entries()]); }); }); + +/** + * Routing classifier ("judge") cost extraction, for both header families. + * Fixtures mirror the shapes observed in real transcripts: the proxy copies routing response + * headers onto the message object verbatim (`x-litellm-*` keep hyphens, `x-codemie-*` become + * underscores) — see proxy/plugins/routing-header-injector.plugin.ts. + */ +describe('extractClaudeUsageRecords — routing classifier cost', () => { + const usage = { input_tokens: 10, output_tokens: 5 }; + const session = (message: Record) => + ({ + sessionId: 'r1', + agentName: 'Claude Code', + metadata: {}, + messages: [{ message: { model: 'claude-sonnet-4-5', usage, ...message } }], + }) as never; + + const LITELLM_CLASSIFIER = { + 'x-litellm-router-tier': 'SIMPLE', + 'x-litellm-router-cause': 'llm_classifier', + 'x-litellm-router-classifier-model': 'claude-4-5-haiku', + 'x-litellm-classifier-cost': '0.0072204', + 'x-litellm-classifier-prompt-tokens': '6394', + 'x-litellm-classifier-completion-tokens': '34', + 'x-litellm-classifier-total-tokens': '6428', + }; + + it('extracts cost and tokens from LiteLLM classifier headers', () => { + const [r] = extractClaudeUsageRecords(session(LITELLM_CLASSIFIER)); + expect(r.routingFamily).toBe('litellm'); + expect(r.judgeCostUSD).toBeCloseTo(0.0072204, 8); + expect(r.judgeInputTokens).toBe(6394); + expect(r.judgeOutputTokens).toBe(34); + expect(r.classifierModel).toBe('claude-4-5-haiku'); + // LiteLLM emits no cache headers — these stay Switchyard-only. + expect(r.judgeCachedTokens).toBeUndefined(); + expect(r.judgeCacheCreationTokens).toBeUndefined(); + }); + + it('reads prompt+completion consistently with the reported total', () => { + const [r] = extractClaudeUsageRecords(session(LITELLM_CLASSIFIER)); + const total = parseInt(LITELLM_CLASSIFIER['x-litellm-classifier-total-tokens'], 10); + expect((r.judgeInputTokens ?? 0) + (r.judgeOutputTokens ?? 0)).toBe(total); + }); + + it('keeps Switchyard values when both families are present', () => { + const [r] = extractClaudeUsageRecords( + session({ + ...LITELLM_CLASSIFIER, + x_codemie_routing_tier: 'capable', + x_codemie_routing_source: 'judge', + x_codemie_routing_judge_cost_usd: '0.002475', + x_codemie_routing_judge_input_tokens: '1200', + x_codemie_routing_judge_output_tokens: '80', + }) + ); + expect(r.routingFamily).toBe('switchyard'); + expect(r.judgeCostUSD).toBeCloseTo(0.002475, 8); + expect(r.judgeInputTokens).toBe(1200); + expect(r.judgeOutputTokens).toBe(80); + }); + + it('marks routing cost known on Switchyard even when no classifier ran', () => { + const [r] = extractClaudeUsageRecords(session({ x_codemie_routing_tier: 'efficient' })); + expect(r.routingCostKnown).toBe(true); + expect(r.judgeCostUSD).toBeUndefined(); + }); + + it('marks routing cost known on LiteLLM turns that report it', () => { + const [r] = extractClaudeUsageRecords(session(LITELLM_CLASSIFIER)); + expect(r.routingCostKnown).toBe(true); + }); + + it('marks routing cost unknown on LiteLLM turns without classifier headers', () => { + const [r] = extractClaudeUsageRecords( + session({ 'x-litellm-router-tier': 'COMPLEX', 'x-litellm-router-cause': 'llm_classifier' }) + ); + expect(r.routingFamily).toBe('litellm'); + expect(r.routingCostKnown).toBe(false); + expect(r.judgeCostUSD).toBeUndefined(); + }); + + it('treats a malformed classifier cost as unmeasured rather than NaN', () => { + const [r] = extractClaudeUsageRecords( + session({ ...LITELLM_CLASSIFIER, 'x-litellm-classifier-cost': 'n/a' }) + ); + expect(r.judgeCostUSD).toBeUndefined(); + expect(r.routingCostKnown).toBe(false); + }); + + it('classifies a turn carrying only the classifier block as LiteLLM-routed', () => { + const [r] = extractClaudeUsageRecords(session({ 'x-litellm-classifier-cost': '0.0064691' })); + expect(r.routingFamily).toBe('litellm'); + expect(r.routingCostKnown).toBe(true); + }); + + it('leaves non-routed turns free of routing metadata', () => { + const [r] = extractClaudeUsageRecords(session({})); + expect(r.routingFamily).toBeUndefined(); + expect(r.routingCostKnown).toBeUndefined(); + expect(r.judgeCostUSD).toBeUndefined(); + }); +}); diff --git a/src/cli/commands/analytics/cost/cost-enricher.ts b/src/cli/commands/analytics/cost/cost-enricher.ts index 3d7ed714d..473baa5c3 100644 --- a/src/cli/commands/analytics/cost/cost-enricher.ts +++ b/src/cli/commands/analytics/cost/cost-enricher.ts @@ -11,7 +11,7 @@ import { readFile } from 'node:fs/promises'; import type { RawSessionData } from '../data-loader.js'; import type { ParsedSession, SessionAdapter } from '../../../../agents/core/session/BaseSessionAdapter.js'; -import type { SessionCost, SessionCostIndex, CostSummary, ModelCost, TokenUsage, CostSeriesPoint } from './types.js'; +import type { SessionCost, SessionCostIndex, CostSummary, ModelCost, TokenUsage, CostSeriesPoint, ModelTimelinePoint } from './types.js'; import type { DispatchEventRaw } from './types.js'; import { MAX_SERIES_POINTS } from './types.js'; import { emptyUsage, addUsage, costBreakdown } from './cost-calculator.js'; @@ -178,6 +178,50 @@ export function buildCostSeries(records: UsageRecord[]): CostSeriesPoint[] { return downsample(points); } +/** + * Build a per-turn model-usage timeline from ordered usage records. + * Each point captures the actual model, per-turn cost, and token count. + * Returns [] when there are no records. + */ +export function buildModelTimeline(records: UsageRecord[]): ModelTimelinePoint[] { + if (!records.length) return []; + const useTs = records.every((r) => r.ts != null); + return records.map((r, i) => { + const model = normalizeModelName(r.model); + const price = lookupPrice(model); + const costUSD = price ? costBreakdown(r.usage, price).total : 0; + const point: ModelTimelinePoint = { + t: useTs ? (r.ts as number) : i + 1, + model, + costUSD: Math.round(costUSD * 1e8) / 1e8, + tokens: r.usage.total, + }; + if (r.requestedModel != null) point.requestedModel = r.requestedModel; + if (r.capableModel != null) point.capableModel = r.capableModel; + if (r.routingFamily != null) point.routingFamily = r.routingFamily; + if (r.routingTier != null) point.routingTier = r.routingTier; + if (r.routingTierRaw != null) point.routingTierRaw = r.routingTierRaw; + if (r.routedModel != null) point.routedModel = r.routedModel; + if (r.classifierModel != null) point.classifierModel = r.classifierModel; + if (r.routerType != null) point.routerType = r.routerType; + if (r.routerScore != null) point.routerScore = r.routerScore; + if (r.routingConfidence != null) point.routingConfidence = r.routingConfidence; + if (r.routingSource != null) point.routingSource = r.routingSource; + if (r.signalScore != null) point.signalScore = r.signalScore; + if (r.signalConfidence != null) point.signalConfidence = r.signalConfidence; + if (r.signalSeverity != null) point.signalSeverity = r.signalSeverity; + if (r.signalSpinning != null) point.signalSpinning = r.signalSpinning; + if (r.signalExploring != null) point.signalExploring = r.signalExploring; + if (r.signalProductionIntensity != null) point.signalProductionIntensity = r.signalProductionIntensity; + if (r.judgePSolve != null) point.judgePSolve = r.judgePSolve; + if (r.judgeCrux != null) point.judgeCrux = r.judgeCrux; + if (r.judgePrimaryRule != null) point.judgePrimaryRule = r.judgePrimaryRule; + if (r.judgeCapabilityBoundary != null) point.judgeCapabilityBoundary = r.judgeCapabilityBoundary; + if (r.decisionSource != null) point.decisionSource = r.decisionSource; + return point; + }); +} + /** Run async tasks with bounded concurrency (cap open file descriptors). */ async function mapWithConcurrency(items: T[], limit: number, fn: (item: T) => Promise): Promise { const out: R[] = new Array(items.length); @@ -361,6 +405,43 @@ export async function enrichCosts( if (series.length) { cost.costSeries = series; } + if (records.length) { + const timeline = buildModelTimeline(records); + if (timeline.length) cost.modelTimeline = timeline; + // Accumulate classifier (routing LLM) cost and tokens from per-turn metadata. + let judgeCostUSD = 0; + let judgeInputTokens = 0; + let judgeOutputTokens = 0; + let judgeCachedTokens = 0; + let judgeCacheCreationTokens = 0; + // A session is only "cost known" if every routed turn reported its classifier cost. + // One unreported turn makes the session total an understatement, not a measurement. + let routedTurns = 0; + let costKnownTurns = 0; + for (const r of records) { + judgeCostUSD += r.judgeCostUSD ?? 0; + judgeInputTokens += r.judgeInputTokens ?? 0; + judgeOutputTokens += r.judgeOutputTokens ?? 0; + judgeCachedTokens += r.judgeCachedTokens ?? 0; + judgeCacheCreationTokens += r.judgeCacheCreationTokens ?? 0; + if (r.routingFamily != null) { + routedTurns++; + if (r.routingCostKnown) costKnownTurns++; + } + } + if (routedTurns > 0) { + cost.routingCostKnown = costKnownTurns === routedTurns; + } + if (judgeInputTokens > 0 || judgeOutputTokens > 0 || judgeCostUSD > 0) { + cost.judgeCostUSD = Math.round(judgeCostUSD * 1e8) / 1e8; + cost.judgeInputTokens = judgeInputTokens; + cost.judgeOutputTokens = judgeOutputTokens; + if (judgeCachedTokens > 0) cost.judgeCachedTokens = judgeCachedTokens; + if (judgeCacheCreationTokens > 0) cost.judgeCacheCreationTokens = judgeCacheCreationTokens; + // Include routing cost in the session total. + cost.costUSD = Math.round((cost.costUSD + judgeCostUSD) * 1e8) / 1e8; + } + } if (entry.parsed) { // Usage provenance from the adapter: lets the report distinguish "cost is genuinely // zero" from "cost is unmeasurable", and surface a provider's own billing unit. diff --git a/src/cli/commands/analytics/cost/types.ts b/src/cli/commands/analytics/cost/types.ts index d7e54293a..d23dd4474 100644 --- a/src/cli/commands/analytics/cost/types.ts +++ b/src/cli/commands/analytics/cost/types.ts @@ -30,6 +30,36 @@ export interface CostSeriesPoint { tokens: number; // cumulative total tokens up to and including this turn } +/** One point in the per-turn model/tier/decision timeline shown in the session modal. */ +export interface ModelTimelinePoint { + t: number; // epoch ms when all records are timed, else the 1-based turn ordinal + model: string; // normalized model name that was actually used + costUSD: number; // per-turn cost attributed to this model + tokens: number; // per-turn total tokens for this turn + requestedModel?: string; // capable model that was originally requested + capableModel?: string; // alias for requestedModel + routingFamily?: 'switchyard' | 'litellm'; // which header family carried the decision + routingTier?: 'efficient' | 'capable' | string; + routingTierRaw?: string; // tier as emitted, before the two vocabularies are folded + routedModel?: string; // model the router actually dispatched to + classifierModel?: string; // LLM that made the routing decision + routerType?: string; // router strategy, e.g. 'complexity' + routerScore?: number; // numeric score from LiteLLM's heuristic scorer + routingConfidence?: number; + routingSource?: 'stage_router' | 'judge' | 'classifier' | string; + decisionSource?: string; + signalScore?: number; + signalConfidence?: number; + signalSeverity?: number; + signalSpinning?: number; + signalExploring?: number; + signalProductionIntensity?: number; + judgePSolve?: number; + judgeCrux?: string; + judgePrimaryRule?: string; + judgeCapabilityBoundary?: string; +} + /** Max points kept per session series — downsample guard so the embedded payload stays small. */ export const MAX_SERIES_POINTS = 40; @@ -60,10 +90,24 @@ export interface SessionCost { costUSD: number; // summed across models cacheReadCostUSD?: number; // USD attributable to cache reads (subset of costUSD); 0 when unpriced costSeries?: CostSeriesPoint[]; // per-turn cumulative cost/token growth; absent when no per-turn data + modelTimeline?: ModelTimelinePoint[]; // per-turn model + routing metadata; absent when no routing data dispatches?: DispatchEvent[]; // top-level agent/skill/command invocations with timing; absent when none perModel: ModelCost[]; priced: boolean; // true if the native log was found & parsed hadLog: boolean; // true if a native log path was located (priced): UsageReco return out; } +function parseOptInt(s: string | undefined): number | undefined { + if (s == null) return undefined; + const v = parseInt(s, 10); + return Number.isNaN(v) ? undefined : v; +} + +function parseOptFloat(s: string | undefined): number | undefined { + if (s == null) return undefined; + const v = parseFloat(s); + return Number.isNaN(v) ? undefined : v; +} + +function normalizeRoutingTier(raw: string | undefined): string | undefined { + if (raw == null) return undefined; + const key = String(raw).trim().toLowerCase(); + const map: Record = { + 'simple': 'simple', + 'efficient': 'middle', + 'medium': 'middle', + 'capable': 'complex', + 'complex': 'complex', + 'reasoning': 'reasoning', + }; + return map[key] ?? key; +} + +function normalizeDecisionSource(raw: string | undefined): string | undefined { + if (raw == null) return undefined; + return String(raw).trim().toLowerCase().replace(/_/g, '-'); +} + +function parseLitellmSignals(raw: string | string[] | undefined): { decisionSource?: string; routingTier?: string } { + if (raw == null) return {}; + const arr = Array.isArray(raw) ? raw : [raw]; + for (const item of arr) { + const str = String(item).trim(); + const m = str.match(/^([^:]+):(.+)$/); + if (m) { + return { + decisionSource: normalizeDecisionSource(m[1]), + routingTier: normalizeRoutingTier(m[2]), + }; + } + } + return {}; +} + /** * Extract one {@link UsageRecord} per Claude assistant message (skipping ``), * across the main transcript AND every sub-agent transcript in `parsed.subagents`. @@ -171,11 +372,112 @@ export function extractClaudeUsageRecords(parsed: ParsedSession): UsageRecord[] const key = id || reqId ? `${id ?? ''}::${reqId ?? ''}` : null; const parsedTs = raw.timestamp ? Date.parse(raw.timestamp) : NaN; const ts = Number.isFinite(parsedTs) ? parsedTs : null; + + // Switchyard / LiteLLM routing metadata from proxy response headers (stored by Claude Code in message metadata). + const msg = raw.message; + const codemieTier = normalizeRoutingTier(msg?.x_codemie_routing_tier); + const litellmTier = normalizeRoutingTier(msg?.['x-litellm-router-tier']); + const signals = parseLitellmSignals(msg?.['x-litellm-router-signals']); + const codemieDecisionSource = normalizeDecisionSource(msg?.x_codemie_routing_decision_source); + const litellmCause = normalizeDecisionSource(msg?.['x-litellm-router-cause']); + + // Discriminate between the two routing header families so consumers know if routing cost is known. + // A turn carrying only the classifier block (no tier/cause) still counts as LiteLLM-routed. + const litellmClassifierCostRaw = msg?.['x-litellm-classifier-cost']; + const routingFamily: 'switchyard' | 'litellm' | undefined = + codemieTier != null || msg?.x_codemie_routing_source != null + ? 'switchyard' + : litellmTier != null || litellmCause != null || litellmClassifierCostRaw != null + ? 'litellm' + : undefined; + + const requestedModel = msg?.x_codemie_requested_model ?? msg?.['x-litellm-router-model-name']; + const capableModel = msg?.x_codemie_routing_capable_model; + const routingTier = codemieTier ?? litellmTier ?? signals.routingTier; + const routingTierRaw = msg?.x_codemie_routing_tier ?? msg?.['x-litellm-router-tier']; + const decisionSource = codemieDecisionSource ?? litellmCause ?? signals.decisionSource; + const routingSource = msg?.x_codemie_routing_source ?? (decisionSource === 'llm-classifier' ? 'judge' : (decisionSource != null ? 'stage_router' : undefined)); + const routedModel = msg?.x_codemie_routed_model ?? msg?.x_codemie_routing_capable_model ?? msg?.['x-litellm-router-routed-model']; + const classifierModel = msg?.['x-litellm-router-classifier-model']; + const routerType = msg?.['x-litellm-router-type']; + const routerScore = parseOptFloat(msg?.['x-litellm-router-score']); + const rawConfidence = msg?.x_codemie_routing_confidence; + const routingConfidence: number | undefined = (() => { + if (rawConfidence != null) { + const v = parseFloat(rawConfidence); + return Number.isNaN(v) ? undefined : v; + } + // Fallback: extract from routing_reason string ("confidence 0.462") + const reason = raw.message?.x_codemie_routing_reason; + if (reason != null) { + const m = /confidence\s+([\d.]+)/i.exec(reason); + if (m) { + const v = parseFloat(m[1]); + return Number.isNaN(v) ? undefined : v; + } + } + return undefined; + })(); + // Switchyard keys take precedence; LiteLLM's classifier headers are the last fallback. + // The families are mutually exclusive per deployment, so this never blends the two. + // Cached/cache-creation stay Switchyard-only — LiteLLM emits no such header. + const judgeInputTokens = parseOptInt(raw.message?.x_codemie_routing_judge_input_tokens ?? raw.message?.x_codemie_routing_classifier_input_tokens ?? msg?.['x-litellm-classifier-prompt-tokens']); + const judgeOutputTokens = parseOptInt(raw.message?.x_codemie_routing_judge_output_tokens ?? raw.message?.x_codemie_routing_classifier_output_tokens ?? msg?.['x-litellm-classifier-completion-tokens']); + const judgeCachedTokens = parseOptInt(raw.message?.x_codemie_routing_judge_cached_tokens ?? raw.message?.x_codemie_routing_classifier_cached_tokens); + const judgeCacheCreationTokens = parseOptInt(raw.message?.x_codemie_routing_judge_cache_creation_tokens ?? raw.message?.x_codemie_routing_classifier_cache_creation_tokens); + const judgeCostUSD = parseOptFloat(raw.message?.x_codemie_routing_judge_cost_usd ?? raw.message?.x_codemie_routing_classifier_cost_usd ?? litellmClassifierCostRaw); + const judgePSolve = parseOptFloat(raw.message?.x_codemie_routing_judge_p_solve ?? raw.message?.x_codemie_routing_classifier_p_solve); + const signalScore = parseOptFloat(raw.message?.x_codemie_routing_signal_score); + const signalConfidence = parseOptFloat(raw.message?.x_codemie_routing_signal_confidence); + const signalSeverity = parseOptFloat(raw.message?.x_codemie_routing_signal_severity); + const signalSpinning = parseOptFloat(raw.message?.x_codemie_routing_signal_spinning); + const signalExploring = parseOptFloat(raw.message?.x_codemie_routing_signal_exploring); + const signalProductionIntensity = parseOptFloat(raw.message?.x_codemie_routing_signal_production ?? raw.message?.x_codemie_routing_signal_production_intensity); + const judgeCrux = raw.message?.x_codemie_routing_judge_crux ?? raw.message?.x_codemie_routing_classifier_crux; + const judgePrimaryRule = raw.message?.x_codemie_routing_judge_primary_rule ?? raw.message?.x_codemie_routing_classifier_primary_rule; + const judgeCapabilityBoundary = raw.message?.x_codemie_routing_judge_capability_boundary ?? raw.message?.x_codemie_routing_classifier_capability_boundary; + + appendDedupedRecord(records, keyedRecords, { key, ts, model, usage: { input, output, cacheRead, cacheCreation, cacheCreation1h, total: input + output + cacheRead + cacheCreation }, + ...(requestedModel != null && { requestedModel }), + ...(routingFamily != null && { routingFamily }), + ...(routingTier != null && { routingTier }), + ...(routingTierRaw != null && { routingTierRaw }), + ...(capableModel != null && { capableModel }), + ...(routedModel != null && { routedModel }), + ...(classifierModel != null && { classifierModel }), + ...(routerType != null && { routerType }), + ...(routerScore != null && { routerScore }), + ...(routingConfidence != null && { routingConfidence }), + ...(routingSource != null && { routingSource }), + ...(decisionSource != null && { decisionSource }), + // Whether routing cost was actually reported for THIS turn, so a session total doesn't + // read an unmeasured turn as "free". Switchyard always reports it (possibly $0), so + // absence there means "no classifier ran". LiteLLM only reports it on builds that emit + // the classifier headers — and a malformed value parses to undefined, which is not a + // measurement either, so gate on the parsed cost rather than raw header presence. + ...(routingFamily != null && { + routingCostKnown: routingFamily === 'switchyard' || judgeCostUSD != null, + }), + ...(judgeInputTokens != null && { judgeInputTokens }), + ...(judgeOutputTokens != null && { judgeOutputTokens }), + ...(judgeCachedTokens != null && { judgeCachedTokens }), + ...(judgeCacheCreationTokens != null && { judgeCacheCreationTokens }), + ...(judgeCostUSD != null && { judgeCostUSD }), + ...(judgePSolve != null && { judgePSolve }), + ...(signalScore != null && { signalScore }), + ...(signalConfidence != null && { signalConfidence }), + ...(signalSeverity != null && { signalSeverity }), + ...(signalSpinning != null && { signalSpinning }), + ...(signalExploring != null && { signalExploring }), + ...(signalProductionIntensity != null && { signalProductionIntensity }), + ...(judgeCrux != null && { judgeCrux }), + ...(judgePrimaryRule != null && { judgePrimaryRule }), + ...(judgeCapabilityBoundary != null && { judgeCapabilityBoundary }), }); } } diff --git a/src/cli/commands/analytics/report/client/app.js b/src/cli/commands/analytics/report/client/app.js index 3ea9b0e1c..f48811500 100644 --- a/src/cli/commands/analytics/report/client/app.js +++ b/src/cli/commands/analytics/report/client/app.js @@ -55,8 +55,8 @@ } function esc(s) { return String(s == null ? '' : s).replace(/[&<>"]/g, function (c) { return { '&': '&', '<': '<', '>': '>', '"': '"' }[c]; }); } function shortPath(p) { var parts = String(p || '').split('/'); return parts[parts.length - 1] || p; } - // Human-readable session label: the cleaned first-prompt title, falling back to a short id. - function sessTitle(s) { return (s && s.title && s.title.trim()) ? s.title.trim() : ('#' + String((s && s.sessionId) || '').slice(0, 8)); } + // Human-readable session label: AI-generated name when available, else cleaned first-prompt title, else short id. + function sessTitle(s) { if (s && s.sessionName && s.sessionName.trim()) return s.sessionName.trim(); var t = (s && s.title && s.title.trim()) ? s.title.trim() : ''; if (t && t.charAt(0) === '/') { var m = t.match(/^\/\S+\s+([\s\S]+)/); if (m) t = m[1].trim(); } return t || ('#' + String((s && s.sessionId) || '').slice(0, 8)); } function truncStr(s, n) { s = String(s == null ? '' : s); return s.length > n ? s.slice(0, n - 1) + '…' : s; } // First n whitespace-delimited words (the "starting message" preview), '…' when truncated. function firstWords(s, n) { s = String(s == null ? '' : s).trim(); var w = s.split(/\s+/); return w.length > n ? w.slice(0, n).join(' ') + '…' : s; } @@ -732,6 +732,30 @@ }); host.appendChild(grid); + // Routing KPI section — only shown when at least one session has routing. routingCostKnown + // is set (true OR false) whenever a session had at least one routed turn — see + // cost-enricher.ts's `routedTurns > 0` guard — so `!= null` alone identifies "was routed", + // independent of whether a classifier cost was actually incurred (heuristic-only Switchyard + // decisions report zero judge cost but are still routing). + var routedSessions = fs.filter(function (s) { return s.judgeCostUSD != null || s.routingCostKnown != null; }); + if (routedSessions.length > 0) { + host.appendChild(el('h3', 'section-title', 'Routing')); + var rsGrid = el('div', 'kpi-grid'); rsGrid.style.gridTemplateColumns = 'repeat(3,1fr)'; + var measuredSessions = routedSessions.filter(function (s) { return s.judgeCostUSD != null && s.judgeCostUSD > 0; }); + var unmeasuredSessions = routedSessions.filter(function (s) { return s.routingCostKnown === false; }); + var totalJudgeCost = sum(measuredSessions, function (s) { return s.judgeCostUSD || 0; }); + var totalJudgeIn = measuredSessions.reduce(function (acc, s) { return acc + (s.judgeInputTokens || 0); }, 0); + var totalJudgeOut = measuredSessions.reduce(function (acc, s) { return acc + (s.judgeOutputTokens || 0); }, 0); + [ + ['Sessions with routing', fmtNum(routedSessions.length) + ' / ' + fmtNum(fs.length) + (unmeasuredSessions.length ? ' (cost unmeasured on ' + fmtNum(unmeasuredSessions.length) + ')' : '')], + ['Judge routing cost', fmtUSD(totalJudgeCost) + (unmeasuredSessions.length ? ' (unmeasured on ' + fmtNum(unmeasuredSessions.length) + ')' : '')], + ['Judge tokens (in/out)', fmtTokens(totalJudgeIn) + ' / ' + fmtTokens(totalJudgeOut)] + ].forEach(function (k) { + var c = el('div', 'kpi'); c.innerHTML = '
' + k[0] + '
' + k[1] + '
'; rsGrid.appendChild(c); + }); + host.appendChild(rsGrid); + } + // per-agent coverage — answers "which tools' metrics are included?" var cov = DATA.meta.coverage || []; if (cov.length) { @@ -778,14 +802,15 @@ var top = fs.slice().sort(function (a, b) { return b.costUSD - a.costUSD; }).slice(0, 10); topCard._body.style.paddingTop = '0'; topCard._body.innerHTML = '
' + tableHTML( - ['Session', 'Agent', 'Project', 'Input', 'Output', 'Cached', 'Total', 'Cost'], + ['Session', 'Name', 'Agent', 'Project', 'Input', 'Output', 'Cached', 'Total', 'Cost'], top.map(function (s) { return [esc(s.sessionId.slice(0, 8)), + esc(sessTitle(s)), '' + esc(labelFor(s.agentName)) + '', '' + esc(shortPath(s.project)) + '', fmtTokens(tkIn(s)), fmtTokens(tkOut(s)), fmtTokens(tkCached(s)), fmtTokens(s.tokens ? s.tokens.total : 0), fmtUSD(s.costUSD)]; }), - [false, false, false, true, true, true, true, true]) + '
'; + [false, false, false, false, true, true, true, true, true]) + ''; host.appendChild(topCard); }; @@ -808,22 +833,23 @@ var list = fs.slice().sort(function (a, b) { return b.startTime - a.startTime; }); if (q) { var ql = q.toLowerCase(); - list = list.filter(function (s) { return (s.sessionId + ' ' + s.agentName + ' ' + labelFor(s.agentName) + ' ' + s.project + ' ' + s.branch + ' ' + (s.title || '') + ' ' + (s.sessionSource || '')).toLowerCase().indexOf(ql) >= 0; }); + list = list.filter(function (s) { return (s.sessionId + ' ' + s.agentName + ' ' + labelFor(s.agentName) + ' ' + s.project + ' ' + s.branch + ' ' + (s.sessionName || '') + ' ' + (s.title || '') + ' ' + (s.sessionSource || '')).toLowerCase().indexOf(ql) >= 0; }); } var shown = list.slice(0, 300); holder.innerHTML = tableHTML( - ['Date', 'Prompt', 'Agent', 'Project', 'Branch', 'Source', 'Turns', 'Net lines', 'Input', 'Output', 'Cached', 'Cost'], + ['Date', 'Name / Prompt', 'Agent', 'Project', 'Branch', 'Source', 'Turns', 'Routed %', 'Net lines', 'Input', 'Output', 'Cached', 'Cost'], shown.map(function (s) { var branchCell = s.branch ? '' + esc(s.branch) + '' : '—'; - var promptCell = '' + esc(truncStr(s.title || '—', 80)) + ''; + var sessionLabel = s.sessionName || s.title || ''; + var promptCell = '' + esc(truncStr(sessionLabel || '—', 80)) + ''; var sourceCell = '' + esc(s.sessionSource || 'Pure chat') + ''; return [new Date(s.startTime).toISOString().slice(0, 16).replace('T', ' '), promptCell, '' + esc(labelFor(s.agentName)) + '', '' + esc(shortPath(s.project)) + '', branchCell, sourceCell, - fmtNum(s.turns), fmtNum(s.netLines), fmtTokens(tkIn(s)), fmtTokens(tkOut(s)), fmtTokens(tkCached(s)), fmtUSD(s.costUSD)]; + fmtNum(s.turns), s.routedTurnsPct != null ? s.routedTurnsPct + '%' : '—', fmtNum(s.netLines), fmtTokens(tkIn(s)), fmtTokens(tkOut(s)), fmtTokens(tkCached(s)), fmtUSD(s.costUSD)]; }), - [false, false, false, false, false, false, true, true, true, true, true, true], + [false, false, false, false, false, false, true, true, true, true, true, true, true], shown.map(function (s) { return 'class="clickable" data-session="' + esc(s.sessionId) + '"'; })); if (list.length > 300) holder.appendChild(el('p', 'text-muted', 'Showing first 300 of ' + list.length + '.')); } @@ -1135,6 +1161,12 @@ ['Duration', fmtDuration(s.durationMs || 0), ''], ['Started', '' + esc(fmtWhen(s.startTime)) + '', ''] ]; + if (s.judgeCostUSD != null) { + var detail = s.routingCostKnown === false ? 'unmeasured' : 'included in cost'; + costRows.push(['Routing (judge)', s.judgeCostUSD > 0 ? fmtUSD(s.judgeCostUSD) : '—', detail]); + } else if (s.routingCostKnown === false) { + costRows.push(['Routing (judge)', '—', 'unmeasured (no classifier cost headers)']); + } if (s.premiumRequests !== undefined) { costRows.push(['Premium requests', fmtNum(s.premiumRequests), 'provider billing unit']); } @@ -1144,11 +1176,15 @@ } else if (s.usagePartial) { costCard._body.appendChild(el('div', 'text-muted', 'Partial usage — output tokens only; this session recorded no full rollup, so cost is understated.')); } - var tokCard = card('Token usage'); tokCard._body.appendChild(statsEl([ + var tokRows = [ ['Input', fmtTokens(t.input), ''], ['Output', fmtTokens(t.output), ''], ['Cache read', fmtTokens(t.cacheRead), ''], ['Cache create', fmtTokens(t.cacheCreation), ''], ['Total', fmtTokens(t.total), ''] - ])); + ]; + if (s.judgeInputTokens != null || s.judgeOutputTokens != null) { + tokRows.push(['Judge tokens', fmtTokens(s.judgeInputTokens || 0) + ' / ' + fmtTokens(s.judgeOutputTokens || 0), 'routing LLM']); + } + var tokCard = card('Token usage'); tokCard._body.appendChild(statsEl(tokRows)); var actCard = card('Activity'); actCard._body.appendChild(statsEl([ ['Turns / API', fmtNum(s.turns), ''], ['Tool calls', fmtNum(s.toolCallsTotal), (s.toolCallsTotal ? Math.round((s.toolCallsSuccess / s.toolCallsTotal) * 100) + '% ok' : '')], @@ -1192,6 +1228,198 @@ } body.appendChild(growth); + // Model routing decisions chart — bar height = P(capable needed), colour = who decided. + // Old sessions that store routingConfidence but no explicit source are labelled 'scorer'. + var timeline = s.modelTimeline || []; + var tlHasRouting = timeline.some(function (p) { return p.routingTier != null || p.routingConfidence != null; }); + if (timeline.length >= 2 && tlHasRouting && window.Chart) { + var DS_COLORS = { + 'override': '#EF4444', + 'tests-passed': '#10B981', + 'dimensions': '#3B82F6', + 'ambiguous': '#9CA3AF', + 'llm-classifier': '#7C5CFC', + 'fall-open': '#F5A534', + }; + function inferDecisionSource(p) { + if (p.decisionSource) return p.decisionSource; + if (p.routingSource === 'judge' || p.routingSource === 'classifier') return 'llm-classifier'; + if (p.routingSource === 'stage_router') { + if (p.signalSeverity != null && p.signalSeverity >= 1.0) return 'override'; + var score = p.signalScore; + if (score != null) { + var confidence = Math.abs(score); + if (confidence >= 0.5) return 'dimensions'; + if (p.routingTier === 'efficient') return 'tests_passed'; + return 'ambiguous'; + } + } + // Old sessions: has confidence but no explicit source → label as scorer + if (p.routingConfidence != null) return 'dimensions'; + return null; + } + function tlDsColor(p) { var ds = inferDecisionSource(p); return DS_COLORS[ds] || '#9CA3AF'; } + var useEpoch = timeline[0].t > 1e12; + var t0tl = timeline[0].t; + var tlLabels = timeline.map(function (p, i) { return useEpoch ? fmtDuration(Math.max(0, p.t - t0tl)) : ('turn ' + (i + 1)); }); + var tierCounts = timeline.reduce(function (acc, p) { + if (p.routingTier === 'simple' || p.routingTier === 'middle' || p.routingTier === 'complex' || p.routingTier === 'reasoning') { + acc.total++; + if (p.routingTier === 'simple' || p.routingTier === 'middle') acc.simpleOrMiddle++; + } + return acc; + }, { total: 0, simpleOrMiddle: 0 }); + var tierSubtitle = ''; + if (tierCounts.total > 0) { + var simplePct = Math.round((tierCounts.simpleOrMiddle / tierCounts.total) * 100); + tierSubtitle = simplePct + '% simple/middle · ' + (100 - simplePct) + '% complex/reasoning'; + } + var tlCard = card('Routing', tierSubtitle); + + // Legend — show only decision sources present in this session + var dsPresent = {}; + timeline.forEach(function (p) { var ds = inferDecisionSource(p); if (ds) dsPresent[ds] = true; }); + var tlLeg = el('div', ''); + tlLeg.style.cssText = 'display:flex;flex-wrap:wrap;gap:4px 12px;padding:2px 0 8px;align-items:center;'; + Object.keys(DS_COLORS).forEach(function (k) { + if (!dsPresent[k]) return; + var chip = el('span', ''); + chip.style.cssText = 'display:inline-flex;align-items:center;gap:5px;font-size:11px;color:var(--color-text-muted)'; + var dot = el('span', ''); + dot.style.cssText = 'display:inline-block;width:10px;height:10px;border-radius:2px;flex-shrink:0;background:' + DS_COLORS[k]; + chip.appendChild(dot); + chip.appendChild(document.createTextNode(k)); + tlLeg.appendChild(chip); + }); + tlCard._body.appendChild(tlLeg); + + var TIER_LEVELS = { + 'simple': 1, + 'middle': 2, + 'complex': 3, + 'reasoning': 4 + }; + var TIER_LABELS = { + 1: 'simple', + 2: 'middle', + 3: 'complex', + 4: 'reasoning' + }; + function tierLevel(p) { return TIER_LEVELS[p.routingTier] || 0; } + function tierName(p) { return p.routingTier || 'unknown'; } + + var tlCv = canvasIn(tlCard._body, 180); + + var tierData = timeline.map(function (p) { return tierLevel(p); }); + var barBg = timeline.map(function (p) { + var base = tlDsColor(p); + if (base.match(/^#([0-9a-f]{6})$/i)) return base; + return '#9CA3AF'; + }); + var mainDataset = { + type: 'bar', + data: tierData, + backgroundColor: barBg, + borderRadius: 2, + barPercentage: 0.72, + categoryPercentage: 1.0, + yAxisID: 'ytier', + order: 2, + }; + + var confData = timeline.map(function (p) { return p.routingConfidence != null ? p.routingConfidence : null; }); + var confDataset = { + type: 'line', + data: confData, + borderColor: '#F5A534', + backgroundColor: 'transparent', + borderWidth: 2, + pointRadius: 0, + pointHoverRadius: 3, + tension: 0.25, + yAxisID: 'yconf', + order: 1, + }; + + var detailEl = el('div', ''); + detailEl.style.cssText = 'margin-top:10px;font-size:12px;color:var(--color-text-muted);line-height:1.4;'; + detailEl.textContent = 'Click a turn to see judge details.'; + function showTurnDetail(index) { + var p = timeline[index]; + if (!p) return; + var html = '
' + esc(p.model) + ' · ' + esc(tierName(p)) + '
'; + if ((p.routingSource === 'judge' || p.routingSource === 'classifier') && p.judgeCrux) { + html += '
crux: ' + esc(p.judgeCrux) + '
'; + } + if (p.judgePrimaryRule) html += '
rule: ' + esc(p.judgePrimaryRule) + '
'; + if (p.judgeCapabilityBoundary) html += '
boundary: ' + esc(p.judgeCapabilityBoundary) + '
'; + detailEl.innerHTML = html; + } + + makeModalChart(tlCv, { + type: 'bar', + data: { labels: tlLabels, datasets: [mainDataset, confDataset] }, + options: { + plugins: { + legend: { display: false }, + tooltip: { + callbacks: { + title: function () { return ''; }, + label: function (item) { + var p = timeline[item.dataIndex]; + var lines = [p.model]; + lines.push('tier: ' + tierName(p) + ' (' + item.parsed.y + ')'); + var ds = inferDecisionSource(p); + if (ds) lines.push('decision: ' + ds); + if (p.routingSource === 'stage_router') { + if (p.routingConfidence != null) lines.push('confidence: ' + p.routingConfidence.toFixed(2)); + if (p.signalSeverity != null) lines.push('severity: ' + p.signalSeverity.toFixed(2)); + if (p.signalSpinning != null && p.signalSpinning >= 0.5) lines.push('spinning: ✓'); + if (p.signalExploring != null && p.signalExploring >= 0.5) lines.push('exploring: ✓'); + if (p.signalProductionIntensity != null) lines.push('production: ' + p.signalProductionIntensity.toFixed(2)); + } else if (p.routingSource === 'judge' || p.routingSource === 'classifier') { + if (p.judgePrimaryRule) lines.push('rule: ' + p.judgePrimaryRule); + if (p.judgeCapabilityBoundary) lines.push('boundary: ' + p.judgeCapabilityBoundary); + if (p.judgeCrux) lines.push('crux (click to see)'); + } else if (p.routingConfidence != null) { + lines.push('confidence: ' + p.routingConfidence.toFixed(2)); + } + return lines; + } + } + } + }, + onClick: function (e, elements) { + if (elements && elements.length) showTurnDetail(elements[0].index); + }, + scales: { + x: { grid: { display: false }, ticks: { maxTicksLimit: 12 } }, + ytier: { + position: 'left', + min: 0, max: 4, + ticks: { + stepSize: 1, + callback: function (v) { return TIER_LABELS[v] || ''; } + }, + grid: { color: GRID } + }, + yconf: { + position: 'right', + min: 0, max: 1, + ticks: { + stepSize: 0.25, + callback: function (v) { return (v * 100).toFixed(0) + '%'; } + }, + grid: { display: false } + } + } + } + }); + + tlCard._body.appendChild(detailEl); + body.appendChild(tlCard); + } + // Timeline — Gantt of all top-level agent, skill, and command dispatches. var hasDispatches = (s.dispatches || []).length > 0; var hasCostData = (s.dispatches || []).some(function (d) { return d.costUSD != null; }); diff --git a/src/cli/commands/analytics/report/payload-builder.ts b/src/cli/commands/analytics/report/payload-builder.ts index 88dd56618..ab623965a 100644 --- a/src/cli/commands/analytics/report/payload-builder.ts +++ b/src/cli/commands/analytics/report/payload-builder.ts @@ -109,6 +109,13 @@ export function buildPayload( ? { usageUnavailableReason: cost.usageUnavailableReason } : {}), ...(cost?.costSeries && cost.costSeries.length ? { costSeries: cost.costSeries } : {}), + ...(cost?.modelTimeline && cost.modelTimeline.length ? { modelTimeline: cost.modelTimeline } : {}), + ...(cost?.judgeCostUSD != null ? { judgeCostUSD: cost.judgeCostUSD } : {}), + ...(cost?.judgeInputTokens != null ? { judgeInputTokens: cost.judgeInputTokens } : {}), + ...(cost?.judgeOutputTokens != null ? { judgeOutputTokens: cost.judgeOutputTokens } : {}), + ...(cost?.judgeCachedTokens != null ? { judgeCachedTokens: cost.judgeCachedTokens } : {}), + ...(cost?.judgeCacheCreationTokens != null ? { judgeCacheCreationTokens: cost.judgeCacheCreationTokens } : {}), + ...(cost?.routingCostKnown != null ? { routingCostKnown: cost.routingCostKnown } : {}), ...(cost?.dispatches && cost.dispatches.length ? { dispatches: cost.dispatches } : {}), skillInvocations, agentInvocations, diff --git a/src/cli/commands/analytics/report/template.html b/src/cli/commands/analytics/report/template.html index 9aa4c6b7c..1d9199ff3 100644 --- a/src/cli/commands/analytics/report/template.html +++ b/src/cli/commands/analytics/report/template.html @@ -148,9 +148,9 @@ .modal-body { padding: 18px 20px; overflow-y: auto; } .modal-body .card { margin-bottom: 14px; } /* light, borderless stat grid (replaces the heavy box-in-box KPIs) */ - .modal-stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(96px, 1fr)); gap: 14px 16px; } - .mstat .mlabel { font-size: 10px; text-transform: uppercase; letter-spacing: .05em; color: var(--color-text-muted); margin-bottom: 3px; white-space: nowrap; } - .mstat .mval { font-size: 18px; font-weight: 700; color: var(--color-text-primary); white-space: nowrap; line-height: 1.2; } + .modal-stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(78px, 1fr)); gap: 14px 16px; } + .mstat .mlabel { font-size: 10px; text-transform: uppercase; letter-spacing: .05em; color: var(--color-text-muted); margin-bottom: 3px; white-space: normal; overflow-wrap: break-word; } + .mstat .mval { font-size: 18px; font-weight: 700; color: var(--color-text-primary); white-space: normal; line-height: 1.1; } .mstat .mval-sm { font-size: 13px; font-weight: 600; } .mstat .msub { font-size: 11px; color: var(--color-text-muted); margin-top: 2px; } .modal-chips { display: flex; flex-wrap: wrap; gap: 6px; } @@ -194,9 +194,9 @@ .modal-body { padding: 18px 20px; overflow-y: auto; } .modal-body .card { margin-bottom: 14px; } /* light, borderless stat grid (replaces the heavy box-in-box KPIs) */ - .modal-stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(96px, 1fr)); gap: 14px 16px; } - .mstat .mlabel { font-size: 10px; text-transform: uppercase; letter-spacing: .05em; color: var(--color-text-muted); margin-bottom: 3px; white-space: nowrap; } - .mstat .mval { font-size: 18px; font-weight: 700; color: var(--color-text-primary); white-space: nowrap; line-height: 1.2; } + .modal-stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(78px, 1fr)); gap: 14px 16px; } + .mstat .mlabel { font-size: 10px; text-transform: uppercase; letter-spacing: .05em; color: var(--color-text-muted); margin-bottom: 3px; white-space: normal; overflow-wrap: break-word; } + .mstat .mval { font-size: 18px; font-weight: 700; color: var(--color-text-primary); white-space: normal; line-height: 1.1; } .mstat .mval-sm { font-size: 13px; font-weight: 600; } .mstat .msub { font-size: 11px; color: var(--color-text-muted); margin-top: 2px; } .modal-chips { display: flex; flex-wrap: wrap; gap: 6px; } diff --git a/src/cli/commands/analytics/report/types.ts b/src/cli/commands/analytics/report/types.ts index 0338fd16a..55d6af24b 100644 --- a/src/cli/commands/analytics/report/types.ts +++ b/src/cli/commands/analytics/report/types.ts @@ -3,7 +3,7 @@ * report. The client app reads only this and computes every view from it. */ -import type { TokenUsage, ModelCost, AgentCoverage, CostSeriesPoint, DispatchEvent } from '../cost/types.js'; +import type { TokenUsage, ModelCost, AgentCoverage, CostSeriesPoint, DispatchEvent, ModelTimelinePoint } from '../cost/types.js'; import type { ToolStats, NamedInvocationStats } from '../types.js'; /** One flat record per session — the client aggregates everything from these. */ @@ -42,8 +42,23 @@ export interface ReportSessionRecord { perModelCost: ModelCost[]; hadLog: boolean; // a native agent log was located for this session (priced` to every upstream request. + routing?: 'signal' | 'classifier'; + // Metrics configuration metrics?: { enabled?: boolean; // Enable metrics collection (default: true) diff --git a/src/providers/plugins/sso/proxy/plugins/__tests__/routing-header-injector.plugin.test.ts b/src/providers/plugins/sso/proxy/plugins/__tests__/routing-header-injector.plugin.test.ts new file mode 100644 index 000000000..e14238177 --- /dev/null +++ b/src/providers/plugins/sso/proxy/plugins/__tests__/routing-header-injector.plugin.test.ts @@ -0,0 +1,243 @@ +import { describe, it, expect } from 'vitest'; +import { + extractRoutingHeaders, + injectIntoJsonBody, + injectIntoSseChunk, +} from '../routing-header-injector.plugin.js'; + +describe('RoutingHeaderInjectorPlugin', () => { + describe('extractRoutingHeaders', () => { + it('captures x-litellm-* headers with hyphens intact', () => { + const headers = { + 'x-litellm-router-tier': 'COMPLEX', + 'x-litellm-router-cause': 'llm_classifier', + 'x-litellm-model-name': 'claude-sonnet-5', + }; + const result = extractRoutingHeaders(headers); + expect(result).toEqual({ + 'x-litellm-router-tier': 'COMPLEX', + 'x-litellm-router-cause': 'llm_classifier', + 'x-litellm-model-name': 'claude-sonnet-5', + }); + }); + + it('captures x-codemie-routing-* and x-codemie-requested-* headers, converting hyphens to underscores', () => { + const headers = { + 'x-codemie-routing-tier': 'efficient', + 'x-codemie-routing-source': 'judge', + 'x-codemie-requested-model': 'claude-sonnet-4-6', + 'x-codemie-routing-judge-cost-usd': '0.002475', + }; + const result = extractRoutingHeaders(headers); + expect(result).toEqual({ + 'x_codemie_routing_tier': 'efficient', + 'x_codemie_routing_source': 'judge', + 'x_codemie_requested_model': 'claude-sonnet-4-6', + 'x_codemie_routing_judge_cost_usd': '0.002475', + }); + }); + + it('ignores non-routing headers', () => { + const headers = { + 'authorization': 'Bearer secret-token', + 'content-type': 'application/json', + 'x-custom-header': 'value', + 'x-litellm-router-tier': 'SIMPLE', + }; + const result = extractRoutingHeaders(headers); + expect(result).toEqual({ + 'x-litellm-router-tier': 'SIMPLE', + }); + }); + + it('handles array-valued headers by taking the first element', () => { + const headers = { + 'x-litellm-router-signals': ['["llm-classifier:COMPLEX"]', 'ignored'], + }; + const result = extractRoutingHeaders(headers); + expect(result).toEqual({ + 'x-litellm-router-signals': '["llm-classifier:COMPLEX"]', + }); + }); + + it('ignores null and undefined values', () => { + const headers = { + 'x-litellm-router-tier': 'COMPLEX', + 'x-litellm-missing': null as unknown as string, + 'x-codemie-routing-tier': undefined as unknown as string, + }; + const result = extractRoutingHeaders(headers); + expect(result).toEqual({ + 'x-litellm-router-tier': 'COMPLEX', + }); + }); + + it('returns empty object when no routing headers are present', () => { + const headers = { + 'authorization': 'Bearer token', + 'content-type': 'application/json', + }; + const result = extractRoutingHeaders(headers); + expect(result).toEqual({}); + }); + + it('case-insensitive matching for header names', () => { + const headers = { + 'X-LiteLLM-Router-Tier': 'COMPLEX', + 'X-CodeMie-Routing-Tier': 'efficient', + }; + const result = extractRoutingHeaders(headers); + expect(result).toEqual({ + 'x-litellm-router-tier': 'COMPLEX', + 'x_codemie_routing_tier': 'efficient', + }); + }); + }); + + describe('injectIntoJsonBody', () => { + it('merges injections into a JSON object', () => { + const body = Buffer.from(JSON.stringify({ id: 'msg_123', role: 'assistant', content: [] })); + const injections = { + 'x-litellm-router-tier': 'COMPLEX', + 'x-litellm-model-name': 'claude-sonnet-5', + }; + const result = injectIntoJsonBody(body, injections); + const parsed = JSON.parse(result.toString('utf-8')); + expect(parsed).toEqual({ + id: 'msg_123', + role: 'assistant', + content: [], + 'x-litellm-router-tier': 'COMPLEX', + 'x-litellm-model-name': 'claude-sonnet-5', + }); + }); + + it('returns original buffer when injections are empty', () => { + const body = Buffer.from(JSON.stringify({ id: 'msg_123' })); + const result = injectIntoJsonBody(body, {}); + expect(result).toBe(body); + }); + + it('returns original buffer when body is not a JSON object', () => { + const testCases = [ + Buffer.from('not json'), + Buffer.from('[]'), + Buffer.from('null'), + Buffer.from('123'), + Buffer.from('true'), + ]; + const injections = { 'x-litellm-router-tier': 'COMPLEX' }; + for (const body of testCases) { + const result = injectIntoJsonBody(body, injections); + expect(result).toBe(body); + } + }); + + it('returns original buffer on parse error', () => { + const body = Buffer.from('{broken json}'); + const injections = { 'x-litellm-router-tier': 'COMPLEX' }; + const result = injectIntoJsonBody(body, injections); + expect(result).toBe(body); + }); + + it('overwrites existing keys if injections have the same key', () => { + const body = Buffer.from(JSON.stringify({ id: 'msg_123', 'x-litellm-router-tier': 'old' })); + const injections = { 'x-litellm-router-tier': 'COMPLEX' }; + const result = injectIntoJsonBody(body, injections); + const parsed = JSON.parse(result.toString('utf-8')); + expect(parsed['x-litellm-router-tier']).toBe('COMPLEX'); + }); + }); + + describe('injectIntoSseChunk', () => { + it('injects fields into the message object of a message_start event', () => { + const chunk = Buffer.from( + 'data: ' + + JSON.stringify({ + type: 'message_start', + message: { id: 'msg_123', role: 'assistant', model: 'claude-sonnet-5' }, + }) + ); + const injections = { + 'x-litellm-router-tier': 'COMPLEX', + 'x-litellm-model-name': 'claude-sonnet-5', + }; + const result = injectIntoSseChunk(chunk, injections); + const line = result.toString('utf-8'); + const parsed = JSON.parse(line.slice(6)); + expect(parsed.message).toEqual({ + id: 'msg_123', + role: 'assistant', + model: 'claude-sonnet-5', + 'x-litellm-router-tier': 'COMPLEX', + 'x-litellm-model-name': 'claude-sonnet-5', + }); + }); + + it('preserves non-message_start events unchanged', () => { + const chunk = Buffer.from( + 'data: ' + JSON.stringify({ type: 'content_block_delta', delta: { type: 'text_delta' } }) + ); + const injections = { 'x-litellm-router-tier': 'COMPLEX' }; + const result = injectIntoSseChunk(chunk, injections); + expect(result).toBe(chunk); + }); + + it('preserves non-JSON lines unchanged', () => { + const chunk = Buffer.from('data: [DONE]'); + const injections = { 'x-litellm-router-tier': 'COMPLEX' }; + const result = injectIntoSseChunk(chunk, injections); + expect(result).toBe(chunk); + }); + + it('preserves other event types in a multi-line chunk', () => { + const multiLine = `data: ${JSON.stringify({ type: 'message_start', message: { id: 'msg_1' } })} +data: ${JSON.stringify({ type: 'content_block_delta', delta: { type: 'text_delta', text: 'hi' } })} +data: [DONE]`; + const chunk = Buffer.from(multiLine); + const injections = { 'x-litellm-router-tier': 'COMPLEX' }; + const result = injectIntoSseChunk(chunk, injections); + const lines = result.toString('utf-8').split('\n'); + // First line should be modified, others unchanged + expect(JSON.parse(lines[0].slice(6)).message['x-litellm-router-tier']).toBe('COMPLEX'); + expect(lines[1]).toBe( + `data: ${JSON.stringify({ type: 'content_block_delta', delta: { type: 'text_delta', text: 'hi' } })}` + ); + expect(lines[2]).toBe('data: [DONE]'); + }); + + it('returns original buffer when injections are empty', () => { + const chunk = Buffer.from( + 'data: ' + JSON.stringify({ type: 'message_start', message: { id: 'msg_123' } }) + ); + const result = injectIntoSseChunk(chunk, {}); + expect(result).toBe(chunk); + }); + + it('returns original buffer when no message_start is present', () => { + const chunk = Buffer.from('data: ' + JSON.stringify({ type: 'content_block_delta' })); + const injections = { 'x-litellm-router-tier': 'COMPLEX' }; + const result = injectIntoSseChunk(chunk, injections); + expect(result).toBe(chunk); + }); + + it('handles partial lines gracefully (does not parse or modify)', () => { + const chunk = Buffer.from('data: {"type":"message_st'); + const injections = { 'x-litellm-router-tier': 'COMPLEX' }; + const result = injectIntoSseChunk(chunk, injections); + expect(result).toBe(chunk); + }); + + it('injects into all message_start events in a multi-line chunk', () => { + const multiStart = `data: ${JSON.stringify({ type: 'message_start', message: { id: 'msg_1' } })} +data: ${JSON.stringify({ type: 'message_start', message: { id: 'msg_2' } })}`; + const chunk = Buffer.from(multiStart); + const injections = { 'x-litellm-router-tier': 'COMPLEX' }; + const result = injectIntoSseChunk(chunk, injections); + const lines = result.toString('utf-8').split('\n'); + // Both get injected (both are message_start events) + expect(JSON.parse(lines[0].slice(6)).message['x-litellm-router-tier']).toBe('COMPLEX'); + expect(JSON.parse(lines[1].slice(6)).message['x-litellm-router-tier']).toBe('COMPLEX'); + }); + }); +}); diff --git a/src/providers/plugins/sso/proxy/plugins/header-injection.plugin.ts b/src/providers/plugins/sso/proxy/plugins/header-injection.plugin.ts index 5f10ca8a1..ee8852242 100644 --- a/src/providers/plugins/sso/proxy/plugins/header-injection.plugin.ts +++ b/src/providers/plugins/sso/proxy/plugins/header-injection.plugin.ts @@ -79,6 +79,14 @@ class HeaderInjectionInterceptor implements ProxyInterceptor { context.headers['X-CodeMie-Project'] = config.project; } + // Append optional routing mode as a query parameter on every upstream request. + const routingMode = config.routing; + if (routingMode) { + const sep = context.url.includes('?') ? '&' : '?'; + context.url = `${context.url}${sep}routing=${routingMode}`; + logger.debug(`[${this.name}] Appended routing query param: ${routingMode}`); + } + logger.debug(`[${this.name}] Injected CodeMie headers`); } } diff --git a/src/providers/plugins/sso/proxy/plugins/index.ts b/src/providers/plugins/sso/proxy/plugins/index.ts index f1ab3a4a3..f043ecbd6 100644 --- a/src/providers/plugins/sso/proxy/plugins/index.ts +++ b/src/providers/plugins/sso/proxy/plugins/index.ts @@ -20,6 +20,7 @@ import { CodexEncryptedContentSanitizerPlugin } from './codex-encrypted-content- import { CopilotEncryptedContentSanitizerPlugin } from './copilot-encrypted-content-sanitizer.plugin.js'; import { VsCodeRequestNormalizerPlugin } from './vscode-request-normalizer.plugin.js'; import { LoggingPlugin } from './logging.plugin.js'; +import { RoutingHeaderInjectorPlugin } from './routing-header-injector.plugin.js'; import { SSOSessionSyncPlugin } from './sso.session-sync.plugin.js'; /** @@ -44,6 +45,7 @@ export function registerCorePlugins(): void { registry.register(new VsCodeRequestNormalizerPlugin()); // Priority 17 - constrains VS Code user identifiers registry.register(new HeaderInjectionPlugin()); registry.register(new LoggingPlugin()); // Always enabled - logs to log files at INFO level + registry.register(new RoutingHeaderInjectorPlugin()); // Priority 55 - copies router decision headers onto the response body so agents persist them registry.register(new SSOSessionSyncPlugin()); // Priority 100 - syncs sessions via multiple processors } @@ -66,6 +68,7 @@ export { CopilotEncryptedContentSanitizerPlugin, VsCodeRequestNormalizerPlugin, LoggingPlugin, + RoutingHeaderInjectorPlugin, }; export { SSOSessionSyncPlugin } from './sso.session-sync.plugin.js'; export { getPluginRegistry, resetPluginRegistry } from './registry.js'; diff --git a/src/providers/plugins/sso/proxy/plugins/routing-header-injector.plugin.ts b/src/providers/plugins/sso/proxy/plugins/routing-header-injector.plugin.ts new file mode 100644 index 000000000..86adfde21 --- /dev/null +++ b/src/providers/plugins/sso/proxy/plugins/routing-header-injector.plugin.ts @@ -0,0 +1,197 @@ +/** + * Routing Header Injector Plugin + * Priority: 55 (after logging at 50, before session-sync at 100) + * + * The upstream router (CodeMie Switchyard or the LiteLLM router) reports which model + * tier it picked, and why, in HTTP *response headers*. Headers are forwarded downstream + * but agents do not persist them, so the decision is lost the moment the turn ends. + * + * This plugin copies those headers onto the response *body*, where the agent stores them + * verbatim in its own transcript alongside `usage`. The analytics pipeline + * (cost/usage-readers.ts) then reads routing metadata per turn with no join and no + * sidecar file. + * + * Two response paths: + * JSON (non-streaming) — buffer the body, merge fields at the top level (which is the + * message object for the Messages API), return a new response. + * SSE (streaming) — stash headers in `context.metadata` during onUpstreamResponse, + * then merge them into the first `message_start` event's nested + * `message` object as it streams past. + * + * Header → body key mapping (matches what the analytics reader expects): + * x-litellm-* → key keeps its hyphens (`x-litellm-router-tier`) + * x-codemie-* → key converts hyphens to `_` (`x_codemie_routing_tier`) + * + * Only one family is ever present: a deployment routes through Switchyard or the LiteLLM + * router, not both. Capturing by prefix means a new field in either family is recorded + * without a code change here. + */ + +import { IncomingHttpHeaders, IncomingMessage } from 'http'; +import { ProxyPlugin, PluginContext, ProxyInterceptor, UpstreamResponseTools } from './types.js'; +import { ProxyContext } from '../proxy-types.js'; +import { logger } from '../../../../../utils/logger.js'; + +const LITELLM_PREFIX = 'x-litellm-'; +// x-codemie-routed-model is a real Switchyard response header (confirmed against live traffic — +// it always matches the response body's own `model`, i.e. the actually-dispatched model) that +// does NOT share a prefix with x-codemie-routing-*/x-codemie-requested-*, so it needs its own +// entry or it silently never reaches the body (see usage-readers.ts's `routedModel`). +const CODEMIE_ROUTING_PREFIXES = ['x-codemie-routing-', 'x-codemie-requested-', 'x-codemie-routed-']; +const METADATA_HEADERS_KEY = '_routingInjectionHeaders'; +const METADATA_INJECTED_KEY = '_routingInjected'; + +/** Fields extracted from routing headers, keyed as they will appear in the body. */ +export type RoutingInjections = Record; + +/** + * Extract routing-relevant upstream response headers into a flat body-key → value map. + * Returns an empty object when the response carries no routing metadata, which is the + * common case for non-routed deployments and for requests that name a literal model ID. + */ +export function extractRoutingHeaders(headers: IncomingHttpHeaders): RoutingInjections { + const out: RoutingInjections = {}; + for (const [key, value] of Object.entries(headers)) { + if (value == null) continue; + const raw = Array.isArray(value) ? value[0] : value; + if (raw == null) continue; + const lower = key.toLowerCase(); + if (lower.startsWith(LITELLM_PREFIX)) { + out[lower] = raw; + } else if (CODEMIE_ROUTING_PREFIXES.some((p) => lower.startsWith(p))) { + out[lower.replace(/-/g, '_')] = raw; + } + } + return out; +} + +/** + * Merge fields into the top-level JSON object of a body buffer. Returns the buffer + * unchanged when there is nothing to inject, the payload is not a JSON object, or + * parsing fails — a routing annotation must never corrupt a response. + */ +export function injectIntoJsonBody(body: Buffer, injections: RoutingInjections): Buffer { + if (Object.keys(injections).length === 0) return body; + try { + const parsed: unknown = JSON.parse(body.toString('utf-8')); + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + return body; + } + return Buffer.from(JSON.stringify({ ...parsed, ...injections }), 'utf-8'); + } catch { + return body; + } +} + +/** + * Merge fields into the `message` object of a `message_start` SSE event. + * + * Rewrites only whole `data:` lines that parse as a `message_start` event; every other + * line — including partial trailing lines at a chunk boundary — is passed through byte + * for byte. Returns the original buffer when no event was modified. + */ +export function injectIntoSseChunk(chunk: Buffer, injections: RoutingInjections): Buffer { + if (Object.keys(injections).length === 0) return chunk; + const lines = chunk.toString('utf-8').split('\n'); + let modified = false; + const outLines: string[] = []; + + for (const line of lines) { + if (!line.startsWith('data: ')) { + outLines.push(line); + continue; + } + let parsed: unknown; + try { + parsed = JSON.parse(line.slice(6)); + } catch { + // Not JSON (e.g. `[DONE]`) or a line split across chunks — pass through untouched. + outLines.push(line); + continue; + } + const event = parsed as { type?: unknown; message?: unknown }; + if ( + typeof parsed === 'object' && + parsed !== null && + !Array.isArray(parsed) && + event.type === 'message_start' && + typeof event.message === 'object' && + event.message !== null + ) { + const message = { ...(event.message as Record), ...injections }; + outLines.push('data: ' + JSON.stringify({ ...event, message })); + modified = true; + } else { + outLines.push(line); + } + } + + return modified ? Buffer.from(outLines.join('\n'), 'utf-8') : chunk; +} + +export class RoutingHeaderInjectorPlugin implements ProxyPlugin { + id = '@codemie/proxy-routing-header-injector'; + name = 'Routing Header Injector'; + version = '1.0.0'; + priority = 55; // After logging (50), before session-sync (100) + + async createInterceptor(_context: PluginContext): Promise { + return new RoutingHeaderInjectorInterceptor(); + } +} + +class RoutingHeaderInjectorInterceptor implements ProxyInterceptor { + name = 'routing-header-injector'; + + async onUpstreamResponse( + context: ProxyContext, + response: IncomingMessage, + tools: UpstreamResponseTools + ): Promise { + const injections = extractRoutingHeaders(response.headers); + if (Object.keys(injections).length === 0) { + return response; + } + + const contentType = String(response.headers['content-type'] ?? '').toLowerCase(); + if (contentType.includes('text/event-stream')) { + // Streaming: defer to onResponseChunk so the stream is never buffered. + context.metadata[METADATA_HEADERS_KEY] = injections; + logger.debug( + `[${this.name}] Captured ${Object.keys(injections).length} routing header(s) for SSE injection` + ); + return response; + } + + try { + const body = await tools.readBody(response); + const modified = injectIntoJsonBody(body, injections); + if (modified !== body) { + logger.debug( + `[${this.name}] Injected ${Object.keys(injections).length} routing header(s) into JSON response` + ); + } + return tools.fromBuffer(response, modified); + } catch (error) { + logger.debug(`[${this.name}] JSON injection failed, forwarding original response:`, error); + return response; + } + } + + async onResponseChunk(context: ProxyContext, chunk: Buffer): Promise { + if (context.metadata[METADATA_INJECTED_KEY]) { + return chunk; + } + const injections = context.metadata[METADATA_HEADERS_KEY] as RoutingInjections | undefined; + if (!injections || Object.keys(injections).length === 0) { + return chunk; + } + + const modified = injectIntoSseChunk(chunk, injections); + if (modified !== chunk) { + context.metadata[METADATA_INJECTED_KEY] = true; + logger.debug(`[${this.name}] Injected routing headers into SSE message_start`); + } + return modified; + } +} diff --git a/src/providers/plugins/sso/proxy/proxy-types.ts b/src/providers/plugins/sso/proxy/proxy-types.ts index 686d73ca5..ecc0aafc9 100644 --- a/src/providers/plugins/sso/proxy/proxy-types.ts +++ b/src/providers/plugins/sso/proxy/proxy-types.ts @@ -31,6 +31,8 @@ export interface ProxyConfig { syncApiUrl?: string; // Optional CodeMie API URL for analytics/session sync syncCodeMieUrl?: string; // Optional CodeMie org URL for credential lookup gatewayKey?: string; // Static bearer key for gateway/daemon mode + /** Optional routing mode appended as `?routing=` to every upstream request. */ + routing?: 'signal' | 'classifier'; telemetryMode?: 'none' | 'claude-desktop'; telemetryPollIntervalMs?: number; telemetryInactivityTimeoutMs?: number;