Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion src/agents/core/BaseAgentAdapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1009,10 +1009,26 @@ export abstract class BaseAgentAdapter implements AgentAdapter {
branch: branch || undefined,
project: env.CODEMIE_PROJECT || undefined,
syncApiUrl: env.CODEMIE_SYNC_API_URL || undefined,
syncCodeMieUrl: env.CODEMIE_URL || undefined
syncCodeMieUrl: env.CODEMIE_URL || undefined,
routing: this._resolveRouting(env, profileConfig),
};
}

private _resolveRouting(
env: NodeJS.ProcessEnv,
profileConfig: import('../../env/types.js').CodeMieConfigOptions | undefined,
): 'signal' | 'classifier' | undefined {
const ALLOWED = new Set(['signal', 'classifier']);
const raw = env.CODEMIE_ROUTING ?? profileConfig?.routing;
if (raw === undefined || raw === null || raw === '') return undefined;
const key = String(raw).trim().toLowerCase();
if (ALLOWED.has(key)) return key as 'signal' | 'classifier';
logger.warn(
`[BaseAgentAdapter] Invalid CODEMIE_ROUTING=${JSON.stringify(raw)}; allowed: signal, classifier`,
);
return undefined;
}

/**
* Centralized proxy setup
* Works for ALL agents based on their metadata
Expand Down
145 changes: 141 additions & 4 deletions src/agents/plugins/claude/plugin/statusline.mjs
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
#!/usr/bin/env node
// CodeMie statusline — shows model, project, branch, context, session cost/duration,
// and (when a CodeMie profile is configured) the CLI budget for the authenticated user.
// When the request was routed to a different backend model (CodeMie Switchyard or the
// LiteLLM router), the actual model is read from the routing headers the proxy injects
// into the transcript and shown alongside the nominal one — see resolveActualModel().
// Deployed to ~/.claude/ by `codemie install statusline` (also triggered by the `--status`
// CLI flag, which calls the same installer). Runs standalone — Node builtins only, no
// project imports, since it executes via `node <path>` after the project process exits.
Expand Down Expand Up @@ -91,6 +94,8 @@ export function extractBasicInfo(ctx) {
return {
projectName: cwd ? path.basename(cwd) : '',
cwd,
transcriptPath: ctx?.transcript_path ?? '',
modelId: ctx?.model?.id ?? '',
model: ctx?.model?.display_name ?? '',
ctxPct: ctx?.context_window?.used_percentage ?? null,
tokIn: ctx?.context_window?.total_input_tokens ?? null,
Expand All @@ -100,6 +105,134 @@ export function extractBasicInfo(ctx) {
};
}

// --- Actual (routed) model resolution ---
//
// Claude Code's own stdin JSON only ever reports the nominal model (`model.id`, the
// alias/tier the session was started with). When CodeMie Switchyard or the LiteLLM router
// dispatches a turn to a different backend model, that can surface two ways in the transcript's
// most recent assistant turn (transcript_path):
// 1. Routing headers — the proxy's routing-header-injector plugin copies the upstream
// router's response headers onto the response body (see
// src/providers/plugins/sso/proxy/plugins/routing-header-injector.plugin.ts), which
// Claude Code then persists verbatim. Authoritative when present: the proxy tags these
// explicitly, so they win over the body-model heuristic below.
// 2. The response body's own `model` field — every Anthropic-compatible response reports
// the model that actually generated it. A router that doesn't emit routing headers (or
// a deployment where this proxy isn't involved at all) still shows the truth here, so
// it's a fallback signal rather than depending on headers alone.
//
// Header field precedence mirrors src/cli/commands/analytics/cost/usage-readers.ts's
// `routedModel` so the statusline and the analytics report agree when both are present.

const ROUTED_MODEL_TAIL_BYTES = 65_536; // last 64KB — comfortably covers the most recent turn(s)

// Claude model family names, used to tell "genuinely routed to a different tier" apart from
// "alias resolved to its concrete dated/region-qualified snapshot", which happens on every
// request regardless of routing and must never be shown as if it were routing.
const MODEL_FAMILY_PATTERN = /(opus|sonnet|haiku|fable)/i;

/** Pulls the actual dispatched model out of a transcript line's `message` object, if present. */
export function extractRoutedModel(message) {
if (!message || typeof message !== 'object') return null;
return message.x_codemie_routed_model ?? message.x_codemie_routing_capable_model ?? message['x-litellm-router-routed-model'] ?? null;
}

function modelFamily(modelId) {
const match = modelId ? MODEL_FAMILY_PATTERN.exec(modelId) : null;
return match ? match[1].toLowerCase() : null;
}

/** Strips Bedrock region/provider qualifiers (`converse/global.anthropic.` / `eu.anthropic.`) and its `-v1:0` suffix. */
export function normalizeModelId(modelId) {
if (!modelId) return '';
return modelId
.toLowerCase()
.replace(/^converse\//, '')
.replace(/^[a-z0-9-]+\.anthropic\./, '')
.replace(/-v\d+:\d+$/, '');
}

/**
* True when two model identifiers name the same tier — either a recognized family (opus/
* sonnet/haiku/fable) matches on both sides, or, when neither side matches a known family,
* the Bedrock-normalized identifiers are identical. Used to avoid flagging an alias's normal
* resolution to its concrete provider snapshot as if it were routing.
*/
export function sameModelFamily(a, b) {
if (!a || !b) return false;
const famA = modelFamily(a);
const famB = modelFamily(b);
if (famA && famB) return famA === famB;
return normalizeModelId(a) === normalizeModelId(b);
}

/**
* Scans transcript JSONL text backwards for the most recent assistant turn and returns the
* response body's own model plus any header-injected routed model. The first (partial) line
* of a tail read is expected to fail JSON.parse when the read didn't start at a line boundary
* — that's normal, not an error, so parse failures are skipped rather than treated as a reason
* to stop scanning.
*/
export function parseLastAssistantTurn(tailText) {
if (!tailText) return null;
const lines = tailText.split('\n');
for (let i = lines.length - 1; i >= 0; i--) {
const line = lines[i].trim();
if (!line) continue;
let parsed;
try {
parsed = JSON.parse(line);
} catch {
continue;
}
const message = parsed?.message;
if (parsed?.type === 'assistant' && message?.model) {
return { responseModel: message.model, headerRoutedModel: extractRoutedModel(message) };
}
}
return null;
}

async function defaultReadTail(filePath, maxBytes) {
const handle = await fs.open(filePath, 'r');
try {
const { size } = await handle.stat();
const start = Math.max(0, size - maxBytes);
const length = size - start;
if (length <= 0) return '';
const { buffer, bytesRead } = await handle.read({ buffer: Buffer.alloc(length), position: start });
return buffer.toString('utf8', 0, bytesRead);
} finally {
await handle.close();
}
}

/**
* Resolves the actual routed model for the current session, or null when there is nothing
* useful to show — no transcript, an unreadable transcript, or the best available signal
* (routing headers, falling back to the response body's own model) names the same tier as
* `nominalModelId`. Never throws: the statusline must keep rendering even if the transcript
* is mid-write or has already rotated away.
*/
export async function resolveActualModel(transcriptPath, nominalModelId, { readTail = defaultReadTail } = {}) {
if (!transcriptPath) return null;
let tail;
try {
tail = await readTail(transcriptPath, ROUTED_MODEL_TAIL_BYTES);
} catch {
return null;
}
const turn = parseLastAssistantTurn(tail);
if (!turn) return null;
const candidate = turn.headerRoutedModel ?? turn.responseModel;
if (!candidate) return null;
const nominal = nominalModelId || turn.responseModel;
// Display the Bedrock-stripped form — the raw candidate may be a fully qualified backend
// id (e.g. `converse/global.anthropic.claude-haiku-4-5-20251001-v1:0`), which is accurate
// but not what a human wants to read in a one-line statusline.
return sameModelFamily(nominal, candidate) ? null : normalizeModelId(candidate);
}

export function formatDuration(ms) {
if (typeof ms !== 'number' || Number.isNaN(ms) || ms < 0) return null;
const mins = Math.floor(ms / 60000);
Expand Down Expand Up @@ -138,14 +271,14 @@ export function ctxBar(pct) {
return `${c(color, bar)} ${pct}%`;
}

export function buildStatusLine({ projectName, branch, model, ctxPct, tokIn, tokOut, cost, durationMs, budget, budgetError }) {
export function buildStatusLine({ projectName, branch, model, actualModel, ctxPct, tokIn, tokOut, cost, durationMs, budget, budgetError }) {
const parts = [];

if (projectName) parts.push(c(C.purple, `[${projectName}]`));
if (budget) parts.push(c(budgetColor(budget.pct), budget.text));
else if (budgetError) parts.push(c(C.yellow, `⚠ ${budgetError}`));
if (branch) parts.push(c(C.blue, `(${branch})`));
if (model) parts.push(c(C.cyan, `[${model}]`));
if (model) parts.push(c(C.cyan, `[${actualModel ? `${model} → ${actualModel}` : model}]`));

const bar = ctxBar(ctxPct);
if (bar) parts.push(bar);
Expand Down Expand Up @@ -256,9 +389,13 @@ export async function main() {
}

const branchPromise = basic.cwd ? gitBranch(basic.cwd) : Promise.resolve('');
const [budgetResult, branch] = await Promise.all([resolveBudget(), branchPromise]);
const [budgetResult, branch, actualModel] = await Promise.all([
resolveBudget(),
branchPromise,
resolveActualModel(basic.transcriptPath, basic.modelId),
]);

process.stdout.write(buildStatusLine({ ...basic, branch, ...budgetResult }));
process.stdout.write(buildStatusLine({ ...basic, branch, actualModel, ...budgetResult }));
}

// Compares decoded paths (not raw strings) so this correctly matches even when the
Expand Down
58 changes: 58 additions & 0 deletions src/cli/commands/analytics/cost/__tests__/cost-enricher.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -569,3 +569,61 @@ describe('buildCostSeries', () => {
expect(s[s.length - 1].tokens).toBe(200); // last cumulative total preserved
});
});

/**
* Session-level rollup of routing classifier cost. Drives the real reader by feeding routing
* headers through parseNative, so these also pin the reader→enricher contract.
*/
describe('enrichCosts — routing classifier cost', () => {
/** One priced assistant turn (1M input @ $3/1M sonnet-4-5 => $3) plus routing headers. */
const turn = (routing: Record<string, unknown>) => ({
message: { model: 'claude-sonnet-4-5', usage: { input_tokens: 1_000_000, output_tokens: 0 }, ...routing },
});

const depsWith = (turns: unknown[]): EnricherDeps => ({
...baseDeps,
parseNative: async () =>
({ sessionId: 's1', agentName: 'claude', metadata: {}, messages: turns }) as never,
});

const LITELLM_BILLED = {
'x-litellm-router-tier': 'SIMPLE',
'x-litellm-router-cause': 'llm_classifier',
'x-litellm-classifier-cost': '0.0072204',
'x-litellm-classifier-prompt-tokens': '6394',
'x-litellm-classifier-completion-tokens': '34',
};

it('sums LiteLLM classifier cost and folds it into the session total', async () => {
const { index } = await enrichCosts(raw, depsWith([turn(LITELLM_BILLED)]));
const c = index.get('s1')!;
expect(c.judgeCostUSD).toBeCloseTo(0.0072204, 8);
expect(c.judgeInputTokens).toBe(6394);
expect(c.judgeOutputTokens).toBe(34);
expect(c.routingCostKnown).toBe(true);
expect(c.costUSD).toBeCloseTo(3 + 0.0072204, 6); // base cost + routing overhead
});

it('accumulates classifier cost across turns', async () => {
const second = { ...LITELLM_BILLED, 'x-litellm-classifier-cost': '0.0064691' };
const { index } = await enrichCosts(raw, depsWith([turn(LITELLM_BILLED), turn(second)]));
expect(index.get('s1')!.judgeCostUSD).toBeCloseTo(0.0072204 + 0.0064691, 8);
});

it('reports cost unknown when any routed turn omits classifier headers', async () => {
const bare = { 'x-litellm-router-tier': 'COMPLEX', 'x-litellm-router-cause': 'llm_classifier' };
const { index } = await enrichCosts(raw, depsWith([turn(LITELLM_BILLED), turn(bare)]));
const c = index.get('s1')!;
expect(c.routingCostKnown).toBe(false);
// The measured turn still contributes — the total is an understatement, not a blank.
expect(c.judgeCostUSD).toBeCloseTo(0.0072204, 8);
});

it('leaves routing fields absent for a session with no routed turns', async () => {
const { index } = await enrichCosts(raw, depsWith([turn({})]));
const c = index.get('s1')!;
expect(c.routingCostKnown).toBeUndefined();
expect(c.judgeCostUSD).toBeUndefined();
expect(c.costUSD).toBeCloseTo(3, 6);
});
});
103 changes: 103 additions & 0 deletions src/cli/commands/analytics/cost/__tests__/usage-readers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -898,3 +898,106 @@ describe('gatherUsageDeduped / gatherDedupedUsageRecords — pi fork replay', ()
expect([...viaDedup.entries()]).toEqual([...viaReader.entries()]);
});
});

/**
* Routing classifier ("judge") cost extraction, for both header families.
* Fixtures mirror the shapes observed in real transcripts: the proxy copies routing response
* headers onto the message object verbatim (`x-litellm-*` keep hyphens, `x-codemie-*` become
* underscores) — see proxy/plugins/routing-header-injector.plugin.ts.
*/
describe('extractClaudeUsageRecords — routing classifier cost', () => {
const usage = { input_tokens: 10, output_tokens: 5 };
const session = (message: Record<string, unknown>) =>
({
sessionId: 'r1',
agentName: 'Claude Code',
metadata: {},
messages: [{ message: { model: 'claude-sonnet-4-5', usage, ...message } }],
}) as never;

const LITELLM_CLASSIFIER = {
'x-litellm-router-tier': 'SIMPLE',
'x-litellm-router-cause': 'llm_classifier',
'x-litellm-router-classifier-model': 'claude-4-5-haiku',
'x-litellm-classifier-cost': '0.0072204',
'x-litellm-classifier-prompt-tokens': '6394',
'x-litellm-classifier-completion-tokens': '34',
'x-litellm-classifier-total-tokens': '6428',
};

it('extracts cost and tokens from LiteLLM classifier headers', () => {
const [r] = extractClaudeUsageRecords(session(LITELLM_CLASSIFIER));
expect(r.routingFamily).toBe('litellm');
expect(r.judgeCostUSD).toBeCloseTo(0.0072204, 8);
expect(r.judgeInputTokens).toBe(6394);
expect(r.judgeOutputTokens).toBe(34);
expect(r.classifierModel).toBe('claude-4-5-haiku');
// LiteLLM emits no cache headers — these stay Switchyard-only.
expect(r.judgeCachedTokens).toBeUndefined();
expect(r.judgeCacheCreationTokens).toBeUndefined();
});

it('reads prompt+completion consistently with the reported total', () => {
const [r] = extractClaudeUsageRecords(session(LITELLM_CLASSIFIER));
const total = parseInt(LITELLM_CLASSIFIER['x-litellm-classifier-total-tokens'], 10);
expect((r.judgeInputTokens ?? 0) + (r.judgeOutputTokens ?? 0)).toBe(total);
});

it('keeps Switchyard values when both families are present', () => {
const [r] = extractClaudeUsageRecords(
session({
...LITELLM_CLASSIFIER,
x_codemie_routing_tier: 'capable',
x_codemie_routing_source: 'judge',
x_codemie_routing_judge_cost_usd: '0.002475',
x_codemie_routing_judge_input_tokens: '1200',
x_codemie_routing_judge_output_tokens: '80',
})
);
expect(r.routingFamily).toBe('switchyard');
expect(r.judgeCostUSD).toBeCloseTo(0.002475, 8);
expect(r.judgeInputTokens).toBe(1200);
expect(r.judgeOutputTokens).toBe(80);
});

it('marks routing cost known on Switchyard even when no classifier ran', () => {
const [r] = extractClaudeUsageRecords(session({ x_codemie_routing_tier: 'efficient' }));
expect(r.routingCostKnown).toBe(true);
expect(r.judgeCostUSD).toBeUndefined();
});

it('marks routing cost known on LiteLLM turns that report it', () => {
const [r] = extractClaudeUsageRecords(session(LITELLM_CLASSIFIER));
expect(r.routingCostKnown).toBe(true);
});

it('marks routing cost unknown on LiteLLM turns without classifier headers', () => {
const [r] = extractClaudeUsageRecords(
session({ 'x-litellm-router-tier': 'COMPLEX', 'x-litellm-router-cause': 'llm_classifier' })
);
expect(r.routingFamily).toBe('litellm');
expect(r.routingCostKnown).toBe(false);
expect(r.judgeCostUSD).toBeUndefined();
});

it('treats a malformed classifier cost as unmeasured rather than NaN', () => {
const [r] = extractClaudeUsageRecords(
session({ ...LITELLM_CLASSIFIER, 'x-litellm-classifier-cost': 'n/a' })
);
expect(r.judgeCostUSD).toBeUndefined();
expect(r.routingCostKnown).toBe(false);
});

it('classifies a turn carrying only the classifier block as LiteLLM-routed', () => {
const [r] = extractClaudeUsageRecords(session({ 'x-litellm-classifier-cost': '0.0064691' }));
expect(r.routingFamily).toBe('litellm');
expect(r.routingCostKnown).toBe(true);
});

it('leaves non-routed turns free of routing metadata', () => {
const [r] = extractClaudeUsageRecords(session({}));
expect(r.routingFamily).toBeUndefined();
expect(r.routingCostKnown).toBeUndefined();
expect(r.judgeCostUSD).toBeUndefined();
});
});
Loading