From caa53d32dd7e642dc1d2844fbf97bee194cf79ad Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Fri, 16 Oct 2026 19:14:08 +0000 Subject: [PATCH 1/3] feat(pricing): consume the producer cache-validity core for cached-input pricing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Encode the AgentX cache-validity core in ETL and route every cached-input pricing path (calculator, trends, chart and series revenue) through one selector with three policies. The default stays `legacy`, so pricing is unchanged until maintainers choose a policy. - tiers rows need GPU and external, CPU within external and an overall that agrees within the producer's 5-decimal rounding; combined rows need a worker-counter or frontend source and no tier rates. - Rows with any validity data are judged by the contract even without a schema; #3964 ATOM detail rows without the core stay out of actual pricing under an enforced policy. - A frontier mixing a verified point with a point lacking a rate is unavailable instead of zero-hit. - Independent of the policy, trace-replay linking no longer rewrites hit rates on rows carrying validity data, and backfills refuse to rewrite their rates, overall source or validity fields. - Update the API route catalog digests of four shared sources after reviewing the docs; no documented response changes under `legacy`. 中文:缓存输入计价接入 producer 的缓存有效性 core。ETL 将有效性 core 编码为数值指标,计算器、趋势、图表和 series 的收入统一经由同一个选择函数取得命中率,提供三种策略;默认仍为 legacy,维护者选定策略前计价不变。tiers 需同时具备 GPU 与 external、CPU 不大于 external、overall 在 producer 5 位小数舍入误差内一致;combined 需有 worker_counters 或 frontend 来源且不带分层。带任何有效性数据的行即使缺 schema 也按合同判定;只有 #3964 ATOM 明细、没有 core 的行在启用的策略下不计入实测。合格新点与缺命中率旧点混合的前沿判为不可用,不再按零命中计价。与策略无关,trace replay 不再改写带有效性数据的行的命中率,backfill 拒绝改写这类行的命中率、overall 来源和有效性字段。审阅 API 文档后更新了四个共享源文件在 API 目录中的摘要,legacy 下已公开的响应不变。 Refs: SemiAnalysisAI/InferenceX#3964, SemiAnalysisAI/InferenceX#4163, SemiAnalysisAI/InferenceX#4164 --- .../components/calculator/interpolation.ts | 31 ++ .../components/calculator/profit-estimator.ts | 49 +- .../components/calculator/throughput-data.ts | 23 +- .../app/src/components/calculator/types.ts | 20 + .../hooks/interpolated-trend-core.ts | 33 +- .../src/components/inference/token-revenue.ts | 22 +- .../app/src/components/inference/types.ts | 3 +- packages/app/src/lib/api/api-route-catalog.ts | 8 +- .../src/lib/benchmarks/benchmark-api-view.ts | 5 +- .../src/lib/benchmarks/benchmark-transform.ts | 9 + .../app/src/lib/calculator/cache-pricing.ts | 259 ++++++++- .../cache-validity-closeout.test.ts | 437 ++++++++++++++++ .../cache-validity-contract.test.ts | 495 ++++++++++++++++++ packages/constants/src/cache-validity.test.ts | 66 +++ packages/constants/src/cache-validity.ts | 122 +++++ packages/constants/src/index.ts | 1 + packages/constants/src/metric-keys.ts | 4 + .../src/etl/agentic-v3-cache-validity.test.ts | 162 ++++++ packages/db/src/etl/agentic-v3-flatten.ts | 71 ++- .../src/etl/cache-validity-closeout.test.ts | 284 ++++++++++ .../cache-validity-producer-x1ir.json | 141 +++++ .../etl/fixtures/cache-validity-producer.json | 305 +++++++++++ packages/db/src/etl/run-overrides.ts | 6 +- packages/db/src/etl/trace-replay-ingest.ts | 5 + .../db/src/lib/benchmark-point-backfill.ts | 37 ++ 25 files changed, 2559 insertions(+), 39 deletions(-) create mode 100644 packages/app/src/lib/calculator/cache-validity-closeout.test.ts create mode 100644 packages/app/src/lib/calculator/cache-validity-contract.test.ts create mode 100644 packages/constants/src/cache-validity.test.ts create mode 100644 packages/constants/src/cache-validity.ts create mode 100644 packages/db/src/etl/agentic-v3-cache-validity.test.ts create mode 100644 packages/db/src/etl/cache-validity-closeout.test.ts create mode 100644 packages/db/src/etl/fixtures/cache-validity-producer-x1ir.json create mode 100644 packages/db/src/etl/fixtures/cache-validity-producer.json diff --git a/packages/app/src/components/calculator/interpolation.ts b/packages/app/src/components/calculator/interpolation.ts index dbb0217bd..78ac10b0a 100644 --- a/packages/app/src/components/calculator/interpolation.ts +++ b/packages/app/src/components/calculator/interpolation.ts @@ -3,8 +3,34 @@ * Shared by both server-side (compare SSR) and client-side (calculator, trends). */ +import { frontierCacheUnavailableReason } from '@/lib/calculator/cache-pricing'; + import type { CostProvider, GPUDataPoint, InterpolatedResult } from './types'; +/** + * Frontier-level cache verdict and provenance. Empty for points priced by the + * legacy rule, which carry no source, so legacy results are unchanged. + */ +function frontierCache( + points: readonly GPUDataPoint[], +): Pick< + InterpolatedResult, + 'cacheHitUnavailable' | 'cacheHitUnavailableReason' | 'cacheHitSources' +> { + const reason = frontierCacheUnavailableReason( + points.map((p) => ({ + rate: p.cacheHitRate, + source: p.cacheHitSource ?? (p.cacheHitUnavailable ? 'unavailable' : undefined), + reason: p.cacheHitReason, + })), + ); + const sources = [...new Set(points.flatMap((p) => (p.cacheHitSource ? [p.cacheHitSource] : [])))]; + return { + ...(reason ? { cacheHitUnavailable: true, cacheHitUnavailableReason: reason } : {}), + ...(sources.length > 0 ? { cacheHitSources: sources.toSorted() } : {}), + }; +} + /** * Compute the upper-left pareto front for GPUDataPoints. * This is the same algorithm used by the main inference charts for the @@ -390,6 +416,7 @@ export function interpolateForGPU( inputTpPerMw: sorted[0].inputTpPerMw, outputTpPerMw: sorted[0].outputTpPerMw, cacheHitRate: sorted[0].cacheHitRate, + ...frontierCache(sorted), theoreticalCacheHitRate: sorted[0].theoreticalCacheHitRate, inputTokenShare: sorted[0].inputTokenShare, concurrency: sorted[0].concurrency, @@ -456,6 +483,9 @@ export function interpolateForGPU( const cacheHitRate = sorted.every((p) => typeof p.cacheHitRate === 'number') ? buildMetric((p) => p.cacheHitRate!) : undefined; + // One unproven knot, or a verified knot beside one without a rate, leaves the + // operating point without an actual rate rather than a full-price one. + const cache = frontierCache(sorted); // Same all-or-nothing logic for the theoretical (infinite-cache) hit rate. const theoreticalCacheHitRate = sorted.every((p) => typeof p.theoreticalCacheHitRate === 'number') @@ -495,6 +525,7 @@ export function interpolateForGPU( inputTpPerMw, outputTpPerMw, cacheHitRate, + ...cache, theoreticalCacheHitRate, inputTokenShare, concurrency, diff --git a/packages/app/src/components/calculator/profit-estimator.ts b/packages/app/src/components/calculator/profit-estimator.ts index ffcf23085..15221c260 100644 --- a/packages/app/src/components/calculator/profit-estimator.ts +++ b/packages/app/src/components/calculator/profit-estimator.ts @@ -20,7 +20,10 @@ * eats the revenue; the operator's profit can therefore go negative. */ -import { tokenRevenueFromRatesPerGpuHour } from '@/components/inference/token-revenue'; +import { + cacheAffectsPrice, + tokenRevenueFromRatesPerGpuHour, +} from '@/components/inference/token-revenue'; import type { TokenRevenuePricing } from '@/components/inference/types'; import { Model } from '@/lib/catalog/data-mappings'; @@ -252,7 +255,8 @@ export type ProfitEstimatorSkipReason = | 'outside-measured-range' | 'no-power' | 'no-cost' - | 'no-token-mix'; + | 'no-token-mix' + | 'cache-hit-unavailable'; export interface ProfitEstimatorSkipped { hwKey: string; @@ -298,7 +302,14 @@ export interface ProfitEstimatorSpecs { export function estimateSkuProfit( result: Pick< InterpolatedResult, - 'hwKey' | 'resultKey' | 'precision' | 'value' | 'inputTokenShare' | 'cacheHitRate' | 'clamped' + | 'hwKey' + | 'resultKey' + | 'precision' + | 'value' + | 'inputTokenShare' + | 'cacheHitRate' + | 'cacheHitUnavailable' + | 'clamped' > & { date?: string }, specs: ProfitEstimatorSpecs, pricing: TokenRevenuePricing, @@ -315,6 +326,9 @@ export function estimateSkuProfit( const gpuHours = assumptions.basis === 'chip-hour' ? 1 : gpuHoursPerGwYear(specs.powerKwPerGpu); if (gpuHours === null) return { ...base, reason: 'no-power' }; if (!(specs.costPerGpuHour > 0)) return { ...base, reason: 'no-cost' }; + if (result.cacheHitUnavailable && cacheAffectsPrice(pricing)) { + return { ...base, reason: 'cache-hit-unavailable' }; + } const revenuePerGpuHour = tokenRevenueFromRatesPerGpuHour( result.value, @@ -427,17 +441,32 @@ export function modelsWithAgenticData( /** * Remap interpolated results to use the theoretical cache hit rate in place of * the measured one. When the theoretical rate is unavailable, the measured rate - * is kept — the intent is an upper-bound estimate, not a dropout. + * is kept — the intent is an upper-bound estimate, not a dropout. An unproven + * measured rate stays unavailable rather than passing for a theoretical one. */ export function applyCacheHitMode< - T extends { cacheHitRate?: number; theoreticalCacheHitRate?: number }, + T extends Pick< + InterpolatedResult, + | 'cacheHitRate' + | 'theoreticalCacheHitRate' + | 'cacheHitUnavailable' + | 'cacheHitUnavailableReason' + | 'cacheHitSources' + >, >(results: readonly T[], mode: CacheHitMode): T[] { if (mode === 'actual') return results as T[]; - return results.map((r) => - typeof r.theoreticalCacheHitRate === 'number' - ? { ...r, cacheHitRate: r.theoreticalCacheHitRate } - : r, - ); + return results.map((r) => { + if (typeof r.theoreticalCacheHitRate !== 'number') return r; + const next = { + ...r, + cacheHitRate: r.theoreticalCacheHitRate, + // Recorded provenance must name the theoretical rate, not the actual one it replaced. + ...(r.cacheHitSources ? { cacheHitSources: ['theoretical' as const] } : {}), + }; + delete next.cacheHitUnavailable; + delete next.cacheHitUnavailableReason; + return next; + }); } /** diff --git a/packages/app/src/components/calculator/throughput-data.ts b/packages/app/src/components/calculator/throughput-data.ts index f0d0450ad..0a268c21b 100644 --- a/packages/app/src/components/calculator/throughput-data.ts +++ b/packages/app/src/components/calculator/throughput-data.ts @@ -3,7 +3,11 @@ import { rowToSequence } from '@semianalysisai/inferencex-constants'; import type { AggDataEntry, HardwareConfig } from '@/components/inference/types'; import type { BenchmarkRow } from '@/lib/api/api'; import { rowToAggDataEntry } from '@/lib/benchmarks/benchmark-transform'; -import { pricingCacheHitRate } from '@/lib/calculator/cache-pricing'; +import { + type CacheRatePolicy, + DEFAULT_CACHE_RATE_POLICY, + selectPricingCacheBasis, +} from '@/lib/calculator/cache-pricing'; import { getHardwareKey } from '@/lib/charts/chart-utils'; import { DEFAULT_TCO_BASIS, @@ -180,6 +184,8 @@ export function buildGpuGroups( /** Token basis selected by the consumer; applies to official and overlay rows. */ tokenType?: CostType; tcoBasis?: TcoBasis; + /** How strictly actual cache pricing follows the producer's validity core. */ + cacheRatePolicy?: CacheRatePolicy; /** Derive a row's group key + metadata. Return null to drop the row. */ classify: (hwKey: string, row: BenchmarkRow) => { key: string; meta: M } | null; }, @@ -194,6 +200,7 @@ export function buildGpuGroups( percentile = Percentile.P90, tokenType = 'total', tcoBasis = DEFAULT_TCO_BASIS, + cacheRatePolicy = DEFAULT_CACHE_RATE_POLICY, classify, } = options; const grouped: Record = {}; @@ -220,7 +227,10 @@ export function buildGpuGroups( const tput = m.tput_per_gpu ?? 0; const outputTput = m.output_tput_per_gpu ?? tput; const inputTput = m.input_tput_per_gpu ?? 0; - const cacheHitRate = pricingCacheHitRate({ ...m, hw: row.hardware }); + const cache = selectPricingCacheBasis( + { ...m, hw: row.hardware, benchmark_type: row.benchmark_type }, + cacheRatePolicy, + ); const theoreticalRate = typeof m.theoretical_cache_hit_rate === 'number' && Number.isFinite(m.theoretical_cache_hit_rate) @@ -252,7 +262,14 @@ export function buildGpuGroups( throughput: tput, outputThroughput: outputTput, inputThroughput: inputTput, - ...(cacheHitRate === null ? {} : { cacheHitRate }), + ...(cache.rate === null ? {} : { cacheHitRate: cache.rate }), + ...(cache.source === 'unavailable' ? { cacheHitUnavailable: true } : {}), + ...(cacheRatePolicy === 'legacy' + ? {} + : { + cacheHitSource: cache.source, + ...(cache.reason ? { cacheHitReason: cache.reason } : {}), + }), ...(theoreticalRate === null ? {} : { theoreticalCacheHitRate: theoreticalRate }), ...(tokenShare === null ? {} : { inputTokenShare: tokenShare }), concurrency: row.conc, diff --git a/packages/app/src/components/calculator/types.ts b/packages/app/src/components/calculator/types.ts index 5ef917c12..bf53fce1a 100644 --- a/packages/app/src/components/calculator/types.ts +++ b/packages/app/src/components/calculator/types.ts @@ -1,4 +1,5 @@ import type { BenchmarkRow } from '@/lib/api/api'; +import type { PricingCacheSource } from '@/lib/calculator/cache-pricing'; export type CalculatorMode = 'interactivity_to_throughput' | 'throughput_to_interactivity'; @@ -44,6 +45,15 @@ export interface GPUDataPoint { * absence is what makes cached-input billing a no-op outside agentic traces. */ cacheHitRate?: number; + /** + * The validity policy left this point's actual cache hit rate unproven. Not + * the same as an absent `cacheHitRate`, which bills no cache discount. + */ + cacheHitUnavailable?: boolean; + /** Where the rate came from; recorded only when a validity policy is enforced. */ + cacheHitSource?: PricingCacheSource; + /** Why the actual rate is unavailable, when it is. */ + cacheHitReason?: string; /** * Infinite-cache theoretical hit rate (0..1) computed from the trace itself, * representing the maximum reuse potential of the workload with unbounded @@ -100,6 +110,16 @@ export interface InterpolatedResult { * fixed-sequence frontier). See {@link GPUDataPoint.cacheHitRate}. */ cacheHitRate?: number; + /** + * Some frontier point's actual cache hit rate is unproven, or a verified + * point sits next to one without a rate, so actual pricing at this operating + * point is unavailable. See {@link GPUDataPoint.cacheHitUnavailable}. + */ + cacheHitUnavailable?: boolean; + /** Why actual pricing is unavailable at this operating point. */ + cacheHitUnavailableReason?: string; + /** Distinct sources of the frontier points' rates, when a validity policy is enforced. */ + cacheHitSources?: PricingCacheSource[]; /** * Infinite-cache theoretical hit rate at that operating point, or undefined * when the frontier did not carry a theoretical rate on every point. diff --git a/packages/app/src/components/inference/hooks/interpolated-trend-core.ts b/packages/app/src/components/inference/hooks/interpolated-trend-core.ts index 861ed210b..784fa3c51 100644 --- a/packages/app/src/components/inference/hooks/interpolated-trend-core.ts +++ b/packages/app/src/components/inference/hooks/interpolated-trend-core.ts @@ -11,6 +11,7 @@ import { } from '@/components/inference/metric-registry'; import { applyTokenRevenuePricing, + cacheAffectsPrice, inputTokenShareForRevenue, NORMALIZED_TOKEN_REVENUE_PRICING, tokenRevenueFromRatesPerGpuHour, @@ -27,7 +28,12 @@ import { dedupeAgenticHistoryRuns, } from '@/lib/benchmarks/benchmark-run-selection'; import { rowToAggDataEntry } from '@/lib/benchmarks/benchmark-transform'; -import { pricingCacheHitRate } from '@/lib/calculator/cache-pricing'; +import { + type CacheRatePolicy, + DEFAULT_CACHE_RATE_POLICY, + frontierCacheUnavailableReason, + selectPricingCacheBasis, +} from '@/lib/calculator/cache-pricing'; import { buildDerivedChartFields, getHardwareKey, @@ -51,6 +57,7 @@ export function rowToLightweightPoint( requestedMetrics: readonly DerivedMetricKey[], tokenRevenuePricing: TokenRevenuePricing | null = NORMALIZED_TOKEN_REVENUE_PRICING, tcoBasis: TcoBasis = DEFAULT_TCO_BASIS, + cacheRatePolicy: CacheRatePolicy = DEFAULT_CACHE_RATE_POLICY, ): InferenceData | null { const entry = rowToAggDataEntry(row); const hwKey = getHardwareKey(entry); @@ -85,11 +92,21 @@ export function rowToLightweightPoint( server_external_cache_hit_rate: entry.server_external_cache_hit_rate, server_cpu_cache_hit_rate: entry.server_cpu_cache_hit_rate, theoretical_cache_hit_rate: entry.theoretical_cache_hit_rate, + benchmark_type: entry.benchmark_type, + server_overall_cache_hit_rate: entry.server_overall_cache_hit_rate, + server_cache_core_schema_version: entry.server_cache_core_schema_version, + server_cache_detail_schema_code: entry.server_cache_detail_schema_code, + server_cache_status_class_code: entry.server_cache_status_class_code, + server_cache_published: entry.server_cache_published, + server_cache_pricing_eligible: entry.server_cache_pricing_eligible, + server_cache_basis_code: entry.server_cache_basis_code, + server_cache_overall_source_code: entry.server_cache_overall_source_code, + server_cache_invalid_rate_fields: entry.server_cache_invalid_rate_fields, ...buildDerivedChartFields(derivedEntry, hwKey, requestedMetrics, tcoBasis), } as InferenceData; return requestedMetrics.includes('tokenRevenuePerGpuHour') - ? applyTokenRevenuePricing([point], tokenRevenuePricing)[0]! + ? applyTokenRevenuePricing([point], tokenRevenuePricing, cacheRatePolicy)[0]! : point; } @@ -179,6 +196,7 @@ export function interpolateMetricAtInteractivity( targetInteractivity: number, metricKey: YAxisMetricKey, tokenRevenuePricing?: TokenRevenuePricing | null, + cacheRatePolicy: CacheRatePolicy = DEFAULT_CACHE_RATE_POLICY, ): number | null { if (points.length === 0) return null; @@ -238,7 +256,16 @@ export function interpolateMetricAtInteractivity( }; const throughputYs = sorted.map((p) => extractMetric(p, 'tpPerGpu')!); const inputShares = sorted.map(inputTokenShareForRevenue); - const cacheHitRates = sorted.map(pricingCacheHitRate); + const cacheBases = sorted.map((p) => selectPricingCacheBasis(p, cacheRatePolicy)); + // An unproven knot, or a verified knot beside one without a rate, makes the + // operating point unpriceable; splining around it would bill hits as zero. + if ( + frontierCacheUnavailableReason(cacheBases) !== undefined && + cacheAffectsPrice(tokenRevenuePricing) + ) { + return null; + } + const cacheHitRates = cacheBases.map((cache) => cache.rate); const inputShare = inputShares.every((share): share is number => share !== null) ? interpolateBounded(inputShares) : null; diff --git a/packages/app/src/components/inference/token-revenue.ts b/packages/app/src/components/inference/token-revenue.ts index 0dfd9a479..5446eb3ac 100644 --- a/packages/app/src/components/inference/token-revenue.ts +++ b/packages/app/src/components/inference/token-revenue.ts @@ -1,7 +1,9 @@ import type { InferenceData, TokenRevenuePricing } from './types'; import { + type CacheRatePolicy, + DEFAULT_CACHE_RATE_POLICY, DEFAULT_CACHED_INPUT_PRICE_RATIO, - pricingCacheHitRate, + selectPricingCacheBasis, } from '@/lib/calculator/cache-pricing'; const SECONDS_PER_HOUR = 3_600; @@ -28,6 +30,15 @@ export function cachedInputPricePerMillion(pricing: TokenRevenuePricing): number : pricing.inputPerMillion * DEFAULT_CACHED_INPUT_PRICE_RATIO; } +/** + * Whether the cache hit rate can change revenue at all. With the cached-input + * price equal to the input price, (1 - h) * in + h * cached = in for every h, + * so an unproven actual rate does not block pricing. + */ +export function cacheAffectsPrice(pricing: TokenRevenuePricing): boolean { + return cachedInputPricePerMillion(pricing) !== pricing.inputPerMillion; +} + /** * Fraction of a point's per-GPU token throughput that is input. * @@ -95,11 +106,13 @@ export function tokenRevenueFromRatesPerGpuHour( export function tokenRevenuePerGpuHour( point: InferenceData, pricing: TokenRevenuePricing, + cacheRatePolicy: CacheRatePolicy = DEFAULT_CACHE_RATE_POLICY, ): number | null { const total = point.tput_per_gpu ?? 0; - const cacheHitRate = pricingCacheHitRate(point); + const cache = selectPricingCacheBasis(point, cacheRatePolicy); + if (cache.source === 'unavailable' && cacheAffectsPrice(pricing)) return null; const inputShare = inputTokenShareForRevenue(point); - return tokenRevenueFromRatesPerGpuHour(total, inputShare, cacheHitRate, pricing); + return tokenRevenueFromRatesPerGpuHour(total, inputShare, cache.rate, pricing); } /** Whether a Y-axis option depends on the selected normalized or OpenRouter token prices. */ @@ -115,13 +128,14 @@ export function usesTokenSalePricing(metricConfigKey: string): boolean { export function applyTokenRevenuePricing( points: InferenceData[], pricing: TokenRevenuePricing | null, + cacheRatePolicy: CacheRatePolicy = DEFAULT_CACHE_RATE_POLICY, ): InferenceData[] { return points.map((point) => { const next = { ...point }; delete next.tokenRevenuePerGpuHour; if (!pricing) return next; - const revenue = tokenRevenuePerGpuHour(point, pricing); + const revenue = tokenRevenuePerGpuHour(point, pricing, cacheRatePolicy); if (revenue !== null) { next.tokenRevenuePerGpuHour = { y: revenue, roof: false }; } diff --git a/packages/app/src/components/inference/types.ts b/packages/app/src/components/inference/types.ts index dac840a68..c6e2fec31 100644 --- a/packages/app/src/components/inference/types.ts +++ b/packages/app/src/components/inference/types.ts @@ -1,6 +1,7 @@ import type React from 'react'; import type { PowerAudit, WorkerPower } from '@semianalysisai/inferencex-db/queries/benchmarks'; +import type { CacheValidityMetrics } from '@/lib/calculator/cache-pricing'; import type { HardwareEntry } from '@/lib/catalog/constants'; import type { Model, Sequence } from '@/lib/catalog/data-mappings'; import type { PowerTier } from '@/lib/power/power-tier'; @@ -58,7 +59,7 @@ export type WorkerRole = 'prefill' | 'decode' | 'agg' | 'frontend'; * @property {number} std_e2el - Standard deviation of End-to-End Latency. * @property {number} p99_e2el - 99th percentile of End-to-End Latency. */ -export interface AggDataEntry { +export interface AggDataEntry extends CacheValidityMetrics { /** Numeric data-parallel replica count, distinct from DP Attention. */ /** Metric keys present in the source row before missing values are normalized to zero. */ rawMetricKeys?: string[]; diff --git a/packages/app/src/lib/api/api-route-catalog.ts b/packages/app/src/lib/api/api-route-catalog.ts index 66e59c969..6687a7d5f 100644 --- a/packages/app/src/lib/api/api-route-catalog.ts +++ b/packages/app/src/lib/api/api-route-catalog.ts @@ -1240,7 +1240,7 @@ export const apiContractSourceDigests = [ }, { source: 'src/lib/benchmarks/benchmark-transform.ts', - sourceSha256: 'a2968b195179c6da79ea09d9f8c22614ca939bf746e0dc1f5f82c3a526449ced', + sourceSha256: 'b837e9ff66ca76285ca2853bc81af179a0d16db4edb613d8d4031b7f05d0c12c', reviewArea: { en: 'Raw benchmark means and derived reciprocal mean-TPOT interactivity used by Dashboard and read-only views.', zh: '仪表板和只读视图共用的原始 benchmark 均值与 mean TPOT 倒数形式的 interactivity。', @@ -1280,7 +1280,7 @@ export const apiContractSourceDigests = [ }, { source: 'src/components/calculator/profit-estimator.ts', - sourceSha256: 'f91b42f1d75182660c0879801c02f3c0f760bc9a1e53f95a9b84e417df8d4eb3', + sourceSha256: 'ace1007b02755fd067008b0b13b0283d445e7e3cbe985a89ae8bad817374d02c', reviewArea: { en: 'Shared profit-estimator defaults and calculation parity, including model-specific target assumptions.', zh: '利润估算器的共享默认值与计算一致性,包括各模型的目标假设。', @@ -1333,7 +1333,7 @@ export const apiContractSourceDigests = [ { source: 'src/components/inference/hooks/interpolated-trend-core.ts', - sourceSha256: '86ef873a7c7f5542a7e88960c43172e8d29ff648b0404265bf0db6a215568cce', + sourceSha256: 'd56ab550f2edb11e7c3db1b6f14a4def6edfa15ac17bb32877f3b524808febb1', reviewArea: { en: 'Dashboard read-only selector and calculation parity.', zh: '仪表板只读接口的选择项与计算一致性。', @@ -1360,7 +1360,7 @@ export const apiContractSourceDigests = [ { source: 'src/components/calculator/throughput-data.ts', - sourceSha256: '9dbf3e828140b50023724ce129e25db90ce358ceed1e4051871d43a62ec267ef', + sourceSha256: 'bc113ba6e580e457bdbb0b6733687d038f926c465185b5633c02180fc97dc9e8', reviewArea: { en: 'Dashboard read-only selector and calculation parity.', zh: '仪表板只读接口的选择项与计算一致性。', diff --git a/packages/app/src/lib/benchmarks/benchmark-api-view.ts b/packages/app/src/lib/benchmarks/benchmark-api-view.ts index 4b1bd19d0..e16b2752c 100644 --- a/packages/app/src/lib/benchmarks/benchmark-api-view.ts +++ b/packages/app/src/lib/benchmarks/benchmark-api-view.ts @@ -1,4 +1,4 @@ -import { rowToSequence } from '@semianalysisai/inferencex-constants'; +import { CACHE_VALIDITY_METRIC_KEYS, rowToSequence } from '@semianalysisai/inferencex-constants'; const CALCULATOR_METRIC_KEYS = new Set([ 'tput_per_gpu', @@ -18,6 +18,9 @@ const CALCULATOR_METRIC_KEYS = new Set([ // GB300 AgentX rows without a server measurement price cached input from the // trace's theoretical ceiling instead. See `pricingCacheHitRate`. 'theoretical_cache_hit_rate', + // The producer's validity core decides whether those rates are actual. See + // `selectPricingCacheBasis`. + ...CACHE_VALIDITY_METRIC_KEYS, // `ttft` feeds the First-Token Limits page, which reads this same view and // caps rows on time to first token at the percentile `intvty` uses. ...['median', 'p75', 'p90'].flatMap((percentile) => diff --git a/packages/app/src/lib/benchmarks/benchmark-transform.ts b/packages/app/src/lib/benchmarks/benchmark-transform.ts index bc61e1ed9..f66eb3c45 100644 --- a/packages/app/src/lib/benchmarks/benchmark-transform.ts +++ b/packages/app/src/lib/benchmarks/benchmark-transform.ts @@ -332,6 +332,15 @@ export function rowToAggDataEntry(row: BenchmarkRow): AggDataEntry { server_external_cache_hit_rate: m.server_external_cache_hit_rate, server_cpu_cache_hit_rate: m.server_cpu_cache_hit_rate, theoretical_cache_hit_rate: m.theoretical_cache_hit_rate, + server_overall_cache_hit_rate: m.server_overall_cache_hit_rate, + server_cache_core_schema_version: m.server_cache_core_schema_version, + server_cache_detail_schema_code: m.server_cache_detail_schema_code, + server_cache_status_class_code: m.server_cache_status_class_code, + server_cache_published: m.server_cache_published, + server_cache_pricing_eligible: m.server_cache_pricing_eligible, + server_cache_basis_code: m.server_cache_basis_code, + server_cache_overall_source_code: m.server_cache_overall_source_code, + server_cache_invalid_rate_fields: m.server_cache_invalid_rate_fields, num_requests_total: m.num_requests_total, num_requests_successful: m.num_requests_successful, total_prompt_tokens: m.total_prompt_tokens, diff --git a/packages/app/src/lib/calculator/cache-pricing.ts b/packages/app/src/lib/calculator/cache-pricing.ts index ffff0e6ac..3b1ebfed8 100644 --- a/packages/app/src/lib/calculator/cache-pricing.ts +++ b/packages/app/src/lib/calculator/cache-pricing.ts @@ -1,3 +1,14 @@ +import { + CACHE_BASES, + CACHE_DETAIL_SCHEMAS, + CACHE_OVERALL_SOURCES, + CACHE_RATE_DECIMALS, + CACHE_STATUS_CLASSES, + CACHE_VALIDITY_CORE_VERSION, + CACHE_VALIDITY_TRACE_KEYS, + cacheVocabularyName, +} from '@semianalysisai/inferencex-constants'; + /** Default cached-input sale price, matching Fleet Lifecycle's assumption. */ export const DEFAULT_CACHED_INPUT_PRICE_RATIO = 0.1; @@ -38,25 +49,253 @@ export function measuredCacheHitRate(metrics: CacheHitMetrics): number | null { */ export const CACHE_HIT_RATE_FALLBACK_HARDWARE: ReadonlySet = new Set(['gb300']); -export interface PricingCacheHitPoint extends CacheHitMetrics { +/** + * The producer's cache-validity core as ETL encodes it (see + * `CACHE_VALIDITY_METRIC_KEYS`). Every field is absent on rows ingested before + * the core existed. + */ +export interface CacheValidityMetrics { + /** Combined prefix-cache hit rate; the only rate on `combined` rows. */ + server_overall_cache_hit_rate?: number; + /** `N` from `infx-cache-validity-core/N`; 0 for an unrecognised schema. */ + server_cache_core_schema_version?: number; + /** 1-based index into `CACHE_DETAIL_SCHEMAS`; 0 for an unrecognised detail schema. */ + server_cache_detail_schema_code?: number; + /** 1-based index into `CACHE_STATUS_CLASSES`; 0 for an unrecognised class. */ + server_cache_status_class_code?: number; + /** 1 when the producer wrote any standard rate; -1 when not a boolean. Never grants eligibility. */ + server_cache_published?: number; + /** 1 only for a JSON `true`; 0 for `false`; -1 when present but not a boolean. */ + server_cache_pricing_eligible?: number; + /** Bitmask over `CACHE_RATE_METRIC_KEYS` of rates that were present but not finite numbers. */ + server_cache_invalid_rate_fields?: number; + /** 1-based index into `CACHE_BASES`; 0 for an unrecognised basis. */ + server_cache_basis_code?: number; + /** 1-based index into `CACHE_OVERALL_SOURCES`; 0 for an unrecognised source. */ + server_cache_overall_source_code?: number; +} + +export interface PricingCacheHitPoint extends CacheHitMetrics, CacheValidityMetrics { /** Base hardware key, for example `gb300`. */ hw?: string; /** Infinite-cache theoretical hit rate (0..1) computed from the trace. */ theoretical_cache_hit_rate?: number; + /** Only `agentic_traces` rows without a core are subject to the strict policy. */ + benchmark_type?: string; } /** - * Cache hit rate used to price cached input tokens. + * How strictly actual cache pricing follows the producer's validity core. * - * Prefers the measured server rate. When a GB300 point carries none, falls - * back to the trace's theoretical hit rate so cache-aware revenue does not bill - * ~97% cache hits at the uncached price. Other hardware never falls back. + * - `legacy`: the pre-core rule for every row. + * - `enforce-when-present`: rows carrying the core must prove eligibility; + * older rows keep the pre-core rule. + * - `strict`: every agentic row must prove eligibility; older rows have no + * actual rate. */ -export function pricingCacheHitRate(point: PricingCacheHitPoint): number | null { +export type CacheRatePolicy = 'legacy' | 'enforce-when-present' | 'strict'; + +/** Stays `legacy` until the app maintainers choose a policy. */ +export const DEFAULT_CACHE_RATE_POLICY: CacheRatePolicy = 'legacy'; + +/** + * Where a pricing hit rate came from. Only `measured` is a validated actual + * rate. `unavailable` means the actual rate is unproven: it must not be priced + * as zero hits, and `none` (no cache to discount) must not be mistaken for it. + * `theoretical` is the explicit theoretical mode, never an actual rate. + */ +export type PricingCacheSource = + | 'measured' + | 'legacy-measured' + | 'theoretical-fallback' + | 'theoretical' + | 'none' + | 'unavailable'; + +export interface PricingCacheBasis { + rate: number | null; + source: PricingCacheSource; + /** Why the actual rate is unavailable. */ + reason?: string; +} + +/** + * Each stored rate is the producer's value rounded to `CACHE_RATE_DECIMALS`, + * so an identity over k stored rates holds to within k half-steps. Nothing + * looser is accepted as agreement. + */ +const HALF_STEP = 0.5 * 10 ** -CACHE_RATE_DECIMALS; +const FLOAT_SLACK = 1e-12; +const withinRounding = (stored: number) => stored * HALF_STEP + FLOAT_SLACK; + +function isUnitRate(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1; +} + +function legacyCacheBasis(point: PricingCacheHitPoint): PricingCacheBasis { const measured = measuredCacheHitRate(point); - if (measured !== null) return measured; - if (point.hw === undefined || !CACHE_HIT_RATE_FALLBACK_HARDWARE.has(point.hw)) return null; + if (measured !== null) return { rate: measured, source: 'legacy-measured' }; + if (point.hw === undefined || !CACHE_HIT_RATE_FALLBACK_HARDWARE.has(point.hw)) { + return { rate: null, source: 'none' }; + } const theoretical = point.theoretical_cache_hit_rate; - if (typeof theoretical !== 'number' || !Number.isFinite(theoretical)) return null; - return Math.max(0, Math.min(1, theoretical)); + if (typeof theoretical !== 'number' || !Number.isFinite(theoretical)) { + return { rate: null, source: 'none' }; + } + return { rate: Math.max(0, Math.min(1, theoretical)), source: 'theoretical-fallback' }; +} + +/** Rate on a `tiers` row, or why it cannot be priced. */ +function tiersRate(point: PricingCacheHitPoint): number | string { + const gpu = point.server_gpu_cache_hit_rate; + const external = point.server_external_cache_hit_rate; + const cpu = point.server_cpu_cache_hit_rate; + const overall = point.server_overall_cache_hit_rate; + // Producer tiers rows write both GPU and external, with an explicit 0 for a + // tier the deployment does not have. A missing component is never read as 0. + if (typeof gpu !== 'number' || typeof external !== 'number') { + return 'incomplete:tiers-need-gpu-and-external'; + } + const written = [gpu, external, cpu, overall].filter((rate) => typeof rate === 'number'); + if (!written.every(isUnitRate)) return 'invalid:tier-rate-out-of-range'; + // GPU and external hits are disjoint. CPU hits are part of the external tier + // (the same quantity on ATOM), so the three tiers never add up. + if (typeof cpu === 'number' && cpu > external + withinRounding(2)) { + return 'conflict:cpu-exceeds-external'; + } + const sum = gpu + external; + if (sum > 1 + withinRounding(2)) return 'invalid:tier-sum-above-one'; + if (typeof overall === 'number' && Math.abs(overall - sum) > withinRounding(3)) { + return 'conflict:tiers-disagree-with-overall'; + } + // Worker counters are the only overall source consistent with tiers. + const sourceCode = point.server_cache_overall_source_code; + const source = cacheVocabularyName(CACHE_OVERALL_SOURCES, sourceCode); + if (sourceCode !== undefined && source !== 'worker_counters') { + return `conflict:tiers-with-${source ?? 'unknown'}-overall`; + } + return Math.min(1, sum); +} + +/** Rate on a `combined` row, or why it cannot be priced. */ +function combinedRate(point: PricingCacheHitPoint): number | string { + const tiers = [ + point.server_gpu_cache_hit_rate, + point.server_external_cache_hit_rate, + point.server_cpu_cache_hit_rate, + ]; + if (tiers.some((rate) => typeof rate === 'number')) return 'conflict:combined-with-tier-rates'; + const overall = point.server_overall_cache_hit_rate; + if (typeof overall !== 'number') return 'conflict:combined-without-overall'; + if (!isUnitRate(overall)) return 'invalid:overall-out-of-range'; + const source = cacheVocabularyName(CACHE_OVERALL_SOURCES, point.server_cache_overall_source_code); + if (source === undefined) return 'combined-without-source'; + if (source === 'router_estimate') return 'router-estimate'; + return overall; +} + +function unavailable(reason: string): PricingCacheBasis { + return { rate: null, source: 'unavailable', reason }; +} + +/** Whether the producer wrote any validity contract for this row, valid or not. */ +function hasValidityTraces(point: PricingCacheHitPoint): boolean { + return CACHE_VALIDITY_TRACE_KEYS.some((key) => point[key] !== undefined); +} + +function coreCacheBasis(point: PricingCacheHitPoint): PricingCacheBasis { + const version = point.server_cache_core_schema_version; + if (version === undefined) { + // #3964 ships the ATOM detail without the common core; it stays out of + // actual pricing until the core lands rather than passing as pre-core. + const detail = cacheVocabularyName(CACHE_DETAIL_SCHEMAS, point.server_cache_detail_schema_code); + return unavailable( + detail === 'atom-cache-validity/1' + ? 'atom-detail-without-core' + : 'malformed-core:missing-schema', + ); + } + if (version !== CACHE_VALIDITY_CORE_VERSION) return unavailable('unsupported-core-schema'); + if (point.server_cache_invalid_rate_fields) return unavailable('invalid:non-numeric-rate'); + const status = + cacheVocabularyName(CACHE_STATUS_CLASSES, point.server_cache_status_class_code) ?? + 'unknown-status'; + if (point.server_cache_pricing_eligible === -1) { + return unavailable('malformed-core:non-boolean-eligibility'); + } + if (point.server_cache_pricing_eligible !== 1) return unavailable(`not-eligible:${status}`); + if (status !== 'VALID_COMPLETE' && status !== 'VALID_ZERO') { + return unavailable(`conflict:eligible-with-${status}`); + } + if (point.server_cache_published !== 1) return unavailable('conflict:eligible-without-rate'); + const basis = cacheVocabularyName(CACHE_BASES, point.server_cache_basis_code); + const rate = + basis === 'tiers' ? tiersRate(point) : basis === 'combined' ? combinedRate(point) : 'no-basis'; + if (typeof rate === 'string') return unavailable(rate); + if (status === 'VALID_ZERO' && rate !== 0) return unavailable('conflict:valid-zero-with-hits'); + return { rate, source: 'measured' }; +} + +/** + * The one place every agentic revenue consumer gets its cache hit rate from. + * + * Under the core, a rate is actual only when the producer marked it eligible + * with a status that allows exact pricing, the numbers are finite, in range + * and consistent with the basis, and a combined rate comes from counters + * rather than a router estimate. A real zero stays priceable. Everything else + * is `unavailable`, never zero hits and never a silent theoretical stand-in. + * A row with any validity trace is judged by the contract even when its + * schema is missing or broken: deleting the schema cannot undo a verdict. + */ +export function selectPricingCacheBasis( + point: PricingCacheHitPoint, + policy: CacheRatePolicy = DEFAULT_CACHE_RATE_POLICY, +): PricingCacheBasis { + if (policy !== 'legacy' && hasValidityTraces(point)) return coreCacheBasis(point); + if (policy === 'strict' && point.benchmark_type === 'agentic_traces') { + return unavailable('no-validity-core'); + } + return legacyCacheBasis(point); +} + +/** One frontier point's cache standing, as carried by calculator points or a basis. */ +export interface FrontierCachePoint { + rate?: number | null; + source?: PricingCacheSource; + reason?: string; +} + +/** + * Why the points of one frontier cannot be priced with an actual cache rate + * together, or undefined when they can. One unproven point withholds the + * operating point, and so does a contract-verified point next to a point with + * no rate: splining around the gap would bill the verified hits as zero. + * Frontiers without a verified point keep the pre-core all-or-nothing rule. + */ +export function frontierCacheUnavailableReason( + points: readonly FrontierCachePoint[], +): string | undefined { + const unproven = points.find((point) => point.source === 'unavailable'); + if (unproven) return unproven.reason ?? 'unproven-point'; + const verified = points.some((point) => point.source === 'measured'); + if (verified && points.some((point) => typeof point.rate !== 'number')) { + return 'mixed-frontier-missing-rate'; + } + return undefined; +} + +/** + * Cache hit rate used to price cached input tokens. + * + * Under the pre-core rule this prefers the measured server rate. When a GB300 + * point carries none, it falls back to the trace's theoretical hit rate so + * cache-aware revenue does not bill ~97% cache hits at the uncached price. + * Other hardware never falls back. Callers that must tell an unproven rate + * from "no discount" use {@link selectPricingCacheBasis}. + */ +export function pricingCacheHitRate( + point: PricingCacheHitPoint, + policy: CacheRatePolicy = DEFAULT_CACHE_RATE_POLICY, +): number | null { + return selectPricingCacheBasis(point, policy).rate; } diff --git a/packages/app/src/lib/calculator/cache-validity-closeout.test.ts b/packages/app/src/lib/calculator/cache-validity-closeout.test.ts new file mode 100644 index 000000000..992d70fa1 --- /dev/null +++ b/packages/app/src/lib/calculator/cache-validity-closeout.test.ts @@ -0,0 +1,437 @@ +/** + * Closeout cases from the X1-I independent review: incomplete or contradictory + * tiers, mixed new/old frontiers, validity traces without a core schema, the + * #3964 detail-only seam, and eligibility surviving a donor rewrite. Producer + * payloads come from `packages/db/src/etl/fixtures/` (synthetic inputs run + * through the producer itself; not historical results). + */ +import { describe, expect, it } from 'vitest'; +import { mapBenchmarkRow } from '@semianalysisai/inferencex-db/etl/benchmark-mapper'; +import { BENCHMARK_POINT_BACKFILLS } from '@semianalysisai/inferencex-db/etl/run-overrides'; +import { createSkipTracker } from '@semianalysisai/inferencex-db/etl/skip-tracker'; +import { planBenchmarkPointBackfill } from '@semianalysisai/inferencex-db/lib/benchmark-point-backfill'; + +import { interpolateForGPU } from '@/components/calculator/interpolation'; +import { + applyCacheHitMode, + estimateSkuProfit, + isProfitEstimatorRow, + type ProfitEstimatorAssumptions, +} from '@/components/calculator/profit-estimator'; +import { buildGpuGroups } from '@/components/calculator/throughput-data'; +import { + interpolateMetricAtInteractivity, + rowToLightweightPoint, +} from '@/components/inference/hooks/interpolated-trend-core'; +import { applyTokenRevenuePricing } from '@/components/inference/token-revenue'; +import type { TokenRevenuePricing } from '@/components/inference/types'; +import type { BenchmarkRow } from '@/lib/api/api'; +import { DEFAULT_TCO_BASIS } from '@/lib/catalog/constants'; +import { Percentile, Sequence } from '@/lib/catalog/data-mappings'; + +import closeout from '../../../../db/src/etl/fixtures/cache-validity-producer-x1ir.json'; +import producer from '../../../../db/src/etl/fixtures/cache-validity-producer.json'; +import { type CacheRatePolicy, selectPricingCacheBasis } from './cache-pricing'; + +const PRICING: TokenRevenuePricing = { + source: 'openrouter', + inputPerMillion: 2, + cachedInputPerMillion: 0.2, + outputPerMillion: 10, +}; +const FLAT_CACHE_PRICING: TokenRevenuePricing = { ...PRICING, cachedInputPerMillion: 2 }; +const SPECS = { powerKwPerGpu: 1.5, costPerGpuHour: 2 }; +const ASSUMPTIONS: ProfitEstimatorAssumptions = { + basis: 'chip-hour', + utilizationPct: 100, + labCutPct: 0, +}; +/** 1000 tok/s/GPU, 80% input, priced 2 / 0.2 / 10 $/M tok. */ +const REVENUE_AT_H70 = 9.3312; +const REVENUE_AT_H0 = 12.96; + +/** Cache metrics exactly as the real ETL mapper stores them for one producer payload. */ +function etlCacheMetrics(serverMetrics: Record): Record { + const mapped = mapBenchmarkRow( + { + infmax_model_prefix: 'dsv4', + hw: 'cluster:b300-nv', + framework: 'vllm', + precision: 'fp4', + spec_decoding: 'none', + disagg: false, + scenario_type: 'agentic-coding', + is_multinode: false, + tp: 4, + ep: 1, + dp_attention: 'false', + conc: 16, + image: 'vllm/vllm-openai:v0.23.0', + kv_offloading: 'none', + num_requests_total: 100, + num_requests_successful: 100, + request_metrics: { throughput: { per_gpu: { total_tput_tps: 1000 } } }, + server_metrics: { present: true, ...serverMetrics }, + }, + createSkipTracker(), + ); + if (!mapped) throw new Error('mapper dropped the row'); + return Object.fromEntries( + Object.entries(mapped.metrics).filter(([key]) => key.includes('cache')), + ); +} + +function apiRow( + metrics: Record, + overrides: Partial = {}, +): BenchmarkRow { + return { + id: 1, + hardware: 'b300', + framework: 'sglang', + model: 'dsv4', + precision: 'fp4', + spec_method: 'none', + disagg: false, + is_multinode: false, + prefill_tp: 8, + prefill_ep: 8, + prefill_dp_attention: false, + prefill_num_workers: 1, + decode_tp: 8, + decode_ep: 8, + decode_dp_attention: false, + decode_num_workers: 1, + num_prefill_gpu: 8, + num_decode_gpu: 8, + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + conc: 8, + offload_mode: 'off', + image: 'sglang:test', + date: '2026-10-16', + run_url: null, + ...overrides, + metrics: { + median_intvty: 40, + p90_itl: 1 / 40, + tput_per_gpu: 1000, + input_tput_per_gpu: 800, + output_tput_per_gpu: 200, + ...metrics, + }, + }; +} + +/** One config at 40 and 20 tok/s/user (1000 and 1600 tok/s/GPU), plus optional faster points. */ +function frontier(...caches: Record[]): BenchmarkRow[] { + return caches.map((cache, i) => { + const interactivity = 40 - 20 * i; + const throughput = 1000 + 600 * i; + return apiRow( + { + ...cache, + median_intvty: interactivity, + p90_itl: 1 / interactivity, + tput_per_gpu: throughput, + input_tput_per_gpu: throughput * 0.8, + output_tput_per_gpu: throughput * 0.2, + }, + { conc: 8 * 2 ** i }, + ); + }); +} + +const select = (metrics: Record, policy: CacheRatePolicy) => + selectPricingCacheBasis({ ...metrics, hw: 'b300', benchmark_type: 'agentic_traces' }, policy); + +function calculator(rows: BenchmarkRow[], policy: CacheRatePolicy, target = 30, pricing = PRICING) { + const { grouped } = buildGpuGroups(rows, { + sequence: Sequence.AgenticTraces, + precisions: ['fp4'], + percentile: Percentile.P90, + cacheRatePolicy: policy, + classify: (hwKey) => ({ key: hwKey, meta: { hwKey } }), + }); + const points = Object.values(grouped)[0]!; + const interpolated = interpolateForGPU(points, target, 'interactivity_to_throughput', 'costh')!; + const [result] = applyCacheHitMode([interpolated], 'actual'); + return { + points, + result: result!, + estimate: estimateSkuProfit(result!, SPECS, pricing, ASSUMPTIONS), + }; +} + +function trend(rows: BenchmarkRow[], policy: CacheRatePolicy, target = 30, pricing = PRICING) { + const points = rows.map((row) => + rowToLightweightPoint( + { ...row, hardware: 'h200', framework: 'trt', model: 'dsr1' }, + ['tokenRevenuePerGpuHour', 'tpPerGpu'], + pricing, + DEFAULT_TCO_BASIS, + policy, + )!, + ); + return interpolateMetricAtInteractivity( + points, + target, + 'tokenRevenuePerGpuHour', + pricing, + policy, + ); +} + +function chartRevenue(metrics: Record, policy: CacheRatePolicy) { + const row = apiRow(metrics, { hardware: 'h200', framework: 'trt', model: 'dsr1' }); + const point = rowToLightweightPoint(row, ['tpPerGpu'], null, DEFAULT_TCO_BASIS, policy); + return applyTokenRevenuePricing([point!], PRICING, policy)[0]!.tokenRevenuePerGpuHour?.y ?? null; +} + +const CORE = { + server_cache_core_schema_version: 1, + server_cache_status_class_code: 1, + server_cache_published: 1, + server_cache_pricing_eligible: 1, + server_cache_basis_code: 1, +}; +const COMPLETE = { + ...CORE, + server_gpu_cache_hit_rate: 0.6, + server_cpu_cache_hit_rate: 0.1, + server_external_cache_hit_rate: 0.1, + server_overall_cache_hit_rate: 0.7, +}; + +describe('R1: strict tiers need complete, consistent components', () => { + it.each<[string, Record]>([ + ['GPU only', { ...CORE, server_gpu_cache_hit_rate: 0.6 }], + ['CPU only', { ...CORE, server_cpu_cache_hit_rate: 0.1 }], + [ + 'CPU above external', + { + ...CORE, + server_gpu_cache_hit_rate: 0.6, + server_cpu_cache_hit_rate: 0.3, + server_external_cache_hit_rate: 0.2, + server_overall_cache_hit_rate: 0.8, + }, + ], + ['overall 0.004 off the tiers', { ...COMPLETE, server_overall_cache_hit_rate: 0.704 }], + [ + 'a router estimate as the overall source', + { ...COMPLETE, server_cache_overall_source_code: 3 }, + ], + [ + 'rates rewritten under unchanged metadata', + { + ...CORE, + server_gpu_cache_hit_rate: 0.2, + server_cpu_cache_hit_rate: 0.5, + server_overall_cache_hit_rate: 0.7, + }, + ], + ])('refuses %s', (_label, metrics) => { + const basis = select(metrics, 'strict'); + expect(basis.source).toBe('unavailable'); + expect(basis.rate).toBeNull(); + }); + + it('does not turn boolean rates into a priced zero through the real ETL', () => { + const coerced = etlCacheMetrics({ + cache: { + gpu_cache_hit_rate: false, + cpu_cache_hit_rate: false, + external_cache_hit_rate: false, + overall_cache_hit_rate: 'invalid', + }, + cache_validity: producer.cases.atom_complete.cache_validity, + }); + expect(select(coerced, 'strict').source).toBe('unavailable'); + }); + + it('keeps complete, explicit-zero and 5-decimal producer values priceable', () => { + expect(select(COMPLETE, 'strict')).toEqual({ rate: 0.7, source: 'measured' }); + const noCpuTier = etlCacheMetrics(closeout.cases.atom_no_cpu_tier); + expect(select(noCpuTier, 'strict')).toEqual({ rate: 0.6, source: 'measured' }); + const zero = etlCacheMetrics(producer.cases.atom_zero); + expect(select(zero, 'strict')).toEqual({ rate: 0, source: 'measured' }); + // round_floats(5): three independently rounded values may differ by 1.5e-5. + const rounded = { + ...CORE, + server_gpu_cache_hit_rate: 0.33333, + server_cpu_cache_hit_rate: 0.33333, + server_external_cache_hit_rate: 0.33333, + server_overall_cache_hit_rate: 0.66667, + }; + expect(select(rounded, 'strict').source).toBe('measured'); + expect(select(rounded, 'strict').rate).toBeCloseTo(0.66666, 10); + }); +}); + +describe('R2: a mixed frontier never falls back to zero-hit pricing', () => { + const measured = () => etlCacheMetrics(producer.cases.atom_complete); + const missing = {}; + + it.each([40, 30])('withholds actual pricing at interactivity %d', (target) => { + const rows = frontier(measured(), missing); + const calc = calculator(rows, 'enforce-when-present', target); + expect(calc.result.cacheHitUnavailable).toBe(true); + expect(calc.estimate).toMatchObject({ reason: 'cache-hit-unavailable' }); + expect(trend(rows, 'enforce-when-present', target)).toBeNull(); + }); + + it('withholds a three-point frontier with one old point missing its rate', () => { + const rows = frontier(measured(), measured(), missing); + expect(calculator(rows, 'enforce-when-present').estimate).toMatchObject({ + reason: 'cache-hit-unavailable', + }); + }); + + it('prices the all-valid frontier exactly at the node', () => { + const calc = calculator(frontier(measured(), measured()), 'enforce-when-present', 40); + expect(isProfitEstimatorRow(calc.estimate)).toBe(true); + expect(isProfitEstimatorRow(calc.estimate) && calc.estimate.revenuePerGpuHour).toBeCloseTo( + REVENUE_AT_H70, + 8, + ); + expect(trend(frontier(measured(), measured()), 'enforce-when-present', 40)).toBeCloseTo( + REVENUE_AT_H70, + 8, + ); + }); + + it('keeps the legacy-only behaviour as the compatibility control', () => { + const calc = calculator(frontier(measured(), missing), 'legacy', 40); + expect(isProfitEstimatorRow(calc.estimate) && calc.estimate.revenuePerGpuHour).toBeCloseTo( + REVENUE_AT_H0, + 8, + ); + }); + + it('leaves an all-legacy frontier on the legacy rule', () => { + const rated = { server_gpu_cache_hit_rate: 0.5, server_external_cache_hit_rate: 0.2 }; + expect( + isProfitEstimatorRow(calculator(frontier(rated, rated), 'enforce-when-present').estimate), + ).toBe(true); + expect( + isProfitEstimatorRow(calculator(frontier(missing, missing), 'enforce-when-present').estimate), + ).toBe(true); + }); + + it('still prices a mixed frontier when the price list has no cache discount', () => { + const calc = calculator( + frontier(measured(), missing), + 'enforce-when-present', + 30, + FLAT_CACHE_PRICING, + ); + expect(isProfitEstimatorRow(calc.estimate)).toBe(true); + expect(calc.result.cacheHitUnavailable).toBe(true); + }); + + it('keeps point and frontier provenance once the guard is on', () => { + const calc = calculator(frontier(measured(), missing), 'enforce-when-present'); + expect(calc.points.map((p) => p.cacheHitSource)).toEqual(['measured', 'none']); + expect(calc.result.cacheHitSources).toEqual(['measured', 'none']); + expect(calc.result.cacheHitUnavailableReason).toBe('mixed-frontier-missing-rate'); + const legacy = calculator(frontier(measured(), missing), 'legacy'); + expect(legacy.points.every((p) => p.cacheHitSource === undefined)).toBe(true); + }); +}); + +describe('R3: validity traces without a core schema fail closed', () => { + const rejected = producer.cases.trt_rejected_cached_exceeds_prompt; + + it('does not let a deleted core_schema undo an INVALID verdict', () => { + const { core_schema: _dropped, ...withoutSchema } = rejected.cache_validity; + const metrics = etlCacheMetrics({ ...rejected, cache_validity: withoutSchema }); + expect(select(metrics, 'enforce-when-present').source).toBe('unavailable'); + expect(chartRevenue(metrics, 'enforce-when-present')).toBeNull(); + }); + + it('refuses a partial core that lost its schema', () => { + const partial = { + server_cache_status_class_code: 7, + server_cache_published: 1, + server_cache_pricing_eligible: 0, + server_cache_basis_code: 1, + server_overall_cache_hit_rate: 0.7, + }; + expect(select(partial, 'enforce-when-present').source).toBe('unavailable'); + }); + + it.each(['atom3964_complete', 'atom3964_zero', 'atom3964_partial_component'] as const)( + 'gates the #3964 detail-only %s until a common core is present', + (name) => { + const metrics = etlCacheMetrics(closeout.cases[name]); + for (const policy of ['enforce-when-present', 'strict'] as const) { + expect(select(metrics, policy).source).toBe('unavailable'); + } + expect(select(metrics, 'legacy').source).not.toBe('unavailable'); + }, + ); + + it('reads only a canonical core version', () => { + for (const schema of [ + 'infx-cache-validity-core/01', + 'infx-cache-validity-core/1.0', + 'infx-cache-validity-core/1e0', + ]) { + const metrics = etlCacheMetrics({ + ...producer.cases.atom_complete, + cache_validity: { ...producer.cases.atom_complete.cache_validity, core_schema: schema }, + }); + expect(select(metrics, 'strict').source).toBe('unavailable'); + } + }); + + it('treats the real MISSING producer output as unavailable, not zero', () => { + const metrics = etlCacheMetrics(closeout.cases.missing_server_metrics); + expect(select(metrics, 'enforce-when-present').source).toBe('unavailable'); + expect(chartRevenue(metrics, 'enforce-when-present')).toBeNull(); + }); +}); + +describe('eligibility does not survive a donor rewrite', () => { + const donor = BENCHMARK_POINT_BACKFILLS.find( + (backfill) => backfill.id.endsWith('-gb200-cache-hit-rate') && !backfill.set.offloadMode, + )!; + const row = (metrics: Record) => ({ + offload_mode: donor.offloadMode, + recipe_fingerprint: donor.recipeFingerprint ?? null, + metrics, + }); + + it('refuses to write borrowed rates over a validated core', () => { + const validated = etlCacheMetrics(producer.cases.atom_complete); + expect(() => planBenchmarkPointBackfill(row(validated), donor)).toThrow(/validity/); + }); + + it('refuses a source-only rewrite that would turn a router estimate into an actual rate', () => { + const combined = { + ...CORE, + server_cache_basis_code: 2, + server_overall_cache_hit_rate: 0.9, + server_cache_overall_source_code: 3, + }; + expect(select(combined, 'strict').source).toBe('unavailable'); + // Synthetic rule on the donor's identity; the real donor list is unchanged. + const sourceRule = { + ...donor, + id: 'synthetic-overall-source-rule', + set: { metricsMerge: { server_cache_overall_source_code: 1 } }, + }; + expect(() => planBenchmarkPointBackfill(row(combined), sourceRule)).toThrow( + /server_cache_overall_source_code/, + ); + }); + + it('still applies the donor to a pre-core row', () => { + const merged = planBenchmarkPointBackfill(row({ server_gpu_cache_hit_rate: 0 }), donor); + expect(merged?.server_gpu_cache_hit_rate).toBe( + donor.set.metricsMerge?.server_gpu_cache_hit_rate, + ); + }); +}); diff --git a/packages/app/src/lib/calculator/cache-validity-contract.test.ts b/packages/app/src/lib/calculator/cache-validity-contract.test.ts new file mode 100644 index 000000000..18646b2f9 --- /dev/null +++ b/packages/app/src/lib/calculator/cache-validity-contract.test.ts @@ -0,0 +1,495 @@ +/** + * Consumer contract for the producer's cache-validity core, end to end: a v3 + * artifact goes through the real ETL mapper, becomes an API row, and is priced + * by the calculator, revenue, trend and series paths. Producer payloads come + * from `packages/db/src/etl/fixtures/cache-validity-producer.json` (synthetic + * inputs run through the producer itself; not historical results). + */ +import { describe, expect, it } from 'vitest'; +import { mapBenchmarkRow } from '@semianalysisai/inferencex-db/etl/benchmark-mapper'; +import { createSkipTracker } from '@semianalysisai/inferencex-db/etl/skip-tracker'; + +import { interpolateForGPU } from '@/components/calculator/interpolation'; +import { + applyCacheHitMode, + estimateSkuProfit, + isProfitEstimatorRow, + type ProfitEstimatorAssumptions, +} from '@/components/calculator/profit-estimator'; +import { buildGpuGroups } from '@/components/calculator/throughput-data'; +import { + interpolateMetricAtInteractivity, + rowToLightweightPoint, +} from '@/components/inference/hooks/interpolated-trend-core'; +import { applyTokenRevenuePricing } from '@/components/inference/token-revenue'; +import type { TokenRevenuePricing } from '@/components/inference/types'; +import type { BenchmarkRow } from '@/lib/api/api'; +import { DEFAULT_TCO_BASIS } from '@/lib/catalog/constants'; +import { Percentile, Sequence } from '@/lib/catalog/data-mappings'; + +import producer from '../../../../db/src/etl/fixtures/cache-validity-producer.json'; +import { + type CacheRatePolicy, + DEFAULT_CACHE_RATE_POLICY, + measuredCacheHitRate, + selectPricingCacheBasis, +} from './cache-pricing'; + +type CaseName = keyof typeof producer.cases; + +const PRICING: TokenRevenuePricing = { + source: 'openrouter', + inputPerMillion: 2, + cachedInputPerMillion: 0.2, + outputPerMillion: 10, +}; +/** A price list where the cache hit rate cannot move revenue. */ +const FLAT_CACHE_PRICING: TokenRevenuePricing = { ...PRICING, cachedInputPerMillion: 2 }; +const SPECS = { powerKwPerGpu: 1.5, costPerGpuHour: 2 }; +const ASSUMPTIONS: ProfitEstimatorAssumptions = { + basis: 'chip-hour', + utilizationPct: 100, + labCutPct: 0, +}; +const CORE_POLICIES: CacheRatePolicy[] = ['enforce-when-present', 'strict']; + +/** Cache metrics exactly as the real ETL mapper stores them for one producer case. */ +function etlCacheMetrics(name: CaseName): Record { + const mapped = mapBenchmarkRow( + { + infmax_model_prefix: 'dsv4', + hw: 'cluster:b300-nv', + framework: 'vllm', + precision: 'fp4', + spec_decoding: 'none', + disagg: false, + scenario_type: 'agentic-coding', + is_multinode: false, + tp: 4, + ep: 1, + dp_attention: 'false', + conc: 16, + image: 'vllm/vllm-openai:v0.23.0', + kv_offloading: 'none', + num_requests_total: 100, + num_requests_successful: 100, + request_metrics: { throughput: { per_gpu: { total_tput_tps: 1000 } } }, + server_metrics: { present: true, ...producer.cases[name] }, + }, + createSkipTracker(), + ); + if (!mapped) throw new Error(`mapper dropped ${name}`); + return Object.fromEntries( + Object.entries(mapped.metrics).filter(([key]) => key.includes('cache')), + ); +} + +/** API row (the shape the benchmarks API returns) carrying the given metrics. */ +function apiRow( + metrics: Record, + overrides: Partial = {}, +): BenchmarkRow { + return { + id: 1, + hardware: 'b300', + framework: 'sglang', + model: 'dsv4', + precision: 'fp4', + spec_method: 'none', + disagg: false, + is_multinode: false, + prefill_tp: 8, + prefill_ep: 8, + prefill_dp_attention: false, + prefill_num_workers: 1, + decode_tp: 8, + decode_ep: 8, + decode_dp_attention: false, + decode_num_workers: 1, + num_prefill_gpu: 8, + num_decode_gpu: 8, + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + conc: 8, + offload_mode: 'off', + image: 'sglang:test', + date: '2026-10-16', + run_url: null, + ...overrides, + metrics: { + // The calculator derives agentic interactivity as 1 / itl; trends read median_intvty. + median_intvty: 40, + p90_itl: 1 / 40, + tput_per_gpu: 1000, + input_tput_per_gpu: 800, + output_tput_per_gpu: 200, + ...metrics, + }, + }; +} + +/** Two operating points of one config, so interpolation runs the spline path. */ +function frontier(first: Record, second = first): BenchmarkRow[] { + const faster = { + median_intvty: 20, + p90_itl: 1 / 20, + tput_per_gpu: 1600, + input_tput_per_gpu: 1280, + output_tput_per_gpu: 320, + }; + return [apiRow(first, { conc: 8 }), apiRow({ ...second, ...faster }, { conc: 16 })]; +} + +/** The chart and trend paths only price registered GPUs; reuse their test config. */ +const chartRow = (row: BenchmarkRow): BenchmarkRow => ({ + ...row, + hardware: 'h200', + framework: 'trt', + model: 'dsr1', +}); + +const basis = (row: BenchmarkRow, policy: CacheRatePolicy) => + selectPricingCacheBasis( + { ...row.metrics, hw: row.hardware, benchmark_type: row.benchmark_type }, + policy, + ); + +/** Series and inference-chart path: `applyTokenRevenuePricing` on a chart point. */ +function chartRevenue(row: BenchmarkRow, policy: CacheRatePolicy, pricing = PRICING) { + const point = rowToLightweightPoint(chartRow(row), ['tpPerGpu'], null, DEFAULT_TCO_BASIS, policy); + return applyTokenRevenuePricing([point!], pricing, policy)[0]!.tokenRevenuePerGpuHour?.y ?? null; +} + +/** Fleet Lifecycle path: group, interpolate, apply the cache mode, price one SKU. */ +function calculatorEstimate( + rows: BenchmarkRow[], + policy: CacheRatePolicy, + mode: 'actual' | 'theoretical' = 'actual', + pricing = PRICING, +) { + const { grouped } = buildGpuGroups(rows, { + sequence: Sequence.AgenticTraces, + precisions: ['fp4'], + percentile: Percentile.P90, + cacheRatePolicy: policy, + classify: (hwKey) => ({ key: hwKey, meta: { hwKey } }), + }); + const points = Object.values(grouped)[0]!; + const interpolated = interpolateForGPU(points, 30, 'interactivity_to_throughput', 'costh')!; + const [result] = applyCacheHitMode([interpolated], mode); + return { + points, + result: result!, + estimate: estimateSkuProfit(result!, SPECS, pricing, ASSUMPTIONS), + }; +} + +/** Historical Trends path: lightweight points, then the cache-aware revenue spline. */ +function trendRevenue(rows: BenchmarkRow[], policy: CacheRatePolicy, pricing = PRICING) { + const points = rows.map((row) => + rowToLightweightPoint( + chartRow(row), + ['tokenRevenuePerGpuHour', 'tpPerGpu'], + pricing, + DEFAULT_TCO_BASIS, + policy, + )!, + ); + return interpolateMetricAtInteractivity(points, 30, 'tokenRevenuePerGpuHour', pricing, policy); +} + +describe('producer cases through ETL → API row → pricing', () => { + // [case, pre-core rate, rate under a core policy (null = unavailable), unavailable reason] + const table: [CaseName, number | null, number | null, string | undefined][] = [ + ['atom_complete', 0.7, 0.7, undefined], + ['atom_zero', 0, 0, undefined], + ['atom_partial_component', 0.6, null, 'not-eligible:PARTIAL'], + ['atom_request_scope_unresolved', 0.7, null, 'not-eligible:UNRESOLVED'], + ['trt_conditional_interval', null, null, 'not-eligible:VALID_INTERVAL'], + ['trt_rejected_cached_exceeds_prompt', null, null, 'not-eligible:INVALID'], + ['sglang_three_tiers', 0.9, null, 'not-eligible:NOT_EVALUATED'], + ['dynamo_frontend', null, null, 'not-eligible:NOT_EVALUATED'], + ['dynamo_router_estimate', null, null, 'not-eligible:NOT_EVALUATED'], + ['vllm_decode_only', 1, null, 'not-eligible:NOT_EVALUATED'], + ['vllm_conditional_denominator', 1, null, 'not-eligible:NOT_EVALUATED'], + ]; + + it.each(table)('%s', (name, legacyRate, coreRate, reason) => { + const row = apiRow(etlCacheMetrics(name)); + + const legacy = basis(row, 'legacy'); + if (legacyRate === null) { + expect(legacy).toEqual({ rate: null, source: 'none' }); + } else { + expect(legacy.source).toBe('legacy-measured'); + expect(legacy.rate).toBeCloseTo(legacyRate, 10); + } + + for (const policy of CORE_POLICIES) { + const selected = basis(row, policy); + const { estimate } = calculatorEstimate(frontier(row.metrics), policy); + if (coreRate === null) { + expect(selected).toEqual({ rate: null, source: 'unavailable', reason }); + expect(chartRevenue(row, policy)).toBeNull(); + expect(estimate).toMatchObject({ reason: 'cache-hit-unavailable' }); + expect(trendRevenue(frontier(row.metrics), policy)).toBeNull(); + } else { + expect(selected.source).toBe('measured'); + expect(selected.rate).toBeCloseTo(coreRate, 10); + expect(chartRevenue(row, policy)).toBeCloseTo(chartRevenue(row, 'legacy')!, 10); + expect(isProfitEstimatorRow(estimate)).toBe(true); + expect(trendRevenue(frontier(row.metrics), policy)).not.toBeNull(); + } + } + }); + + it('prices a real zero, which is not the same as an unproven rate', () => { + const uncached = chartRevenue(apiRow({}), 'legacy'); + expect(chartRevenue(apiRow(etlCacheMetrics('atom_zero')), 'strict')).toBeCloseTo(uncached!, 10); + expect(chartRevenue(apiRow(etlCacheMetrics('atom_partial_component')), 'strict')).toBeNull(); + }); + + it('keeps the TRT diagnostic 1.0476 out of every pricing path', () => { + const diagnostic = + producer.cases.trt_rejected_cached_exceeds_prompt.validity.cache_rate.raw_rate; + const row = apiRow(etlCacheMetrics('trt_rejected_cached_exceeds_prompt')); + expect(diagnostic).toBeCloseTo(1.0476, 10); + expect(Object.values(row.metrics)).not.toContain(diagnostic); + for (const policy of ['legacy', ...CORE_POLICIES] as const) { + expect(basis(row, policy).rate).toBeNull(); + } + }); +}); + +describe('the exact-pricing contract on synthetic core rows', () => { + const eligibleTiers = { + server_cache_core_schema_version: 1, + server_cache_status_class_code: 1, + server_cache_pricing_eligible: 1, + server_cache_published: 1, + server_cache_basis_code: 1, + server_gpu_cache_hit_rate: 0.5, + server_external_cache_hit_rate: 0.2, + server_overall_cache_hit_rate: 0.7, + }; + const eligibleCombined = { + server_cache_core_schema_version: 1, + server_cache_status_class_code: 1, + server_cache_pricing_eligible: 1, + server_cache_published: 1, + server_cache_basis_code: 2, + server_overall_cache_hit_rate: 0.8, + server_cache_overall_source_code: 1, + }; + const select = ( + base: Record, + override: Record = {}, + ) => { + const merged = Object.entries({ ...base, ...override }).filter(([, v]) => v !== undefined); + return basis(apiRow(Object.fromEntries(merged) as Record), 'strict'); + }; + + it('treats tiers as GPU plus external, never all three', () => { + // SGLang device 600 / host 200 / storage 100 of 1000 prompt tokens. + const sglang = { + server_gpu_cache_hit_rate: 0.6, + server_cpu_cache_hit_rate: 0.2, + server_external_cache_hit_rate: 0.3, + server_overall_cache_hit_rate: 0.9, + }; + expect(select(eligibleTiers, sglang).rate).toBeCloseTo(0.9, 10); + expect(select(eligibleTiers)).toEqual({ rate: 0.7, source: 'measured' }); + }); + + it('accepts a combined rate only from counters or the frontend', () => { + expect(select(eligibleCombined)).toEqual({ rate: 0.8, source: 'measured' }); + expect(select(eligibleCombined, { server_cache_overall_source_code: 2 }).rate).toBe(0.8); + }); + + it.each<[string, Record, string]>([ + [ + 'an eligible partial status', + { server_cache_status_class_code: 4 }, + 'conflict:eligible-with-PARTIAL', + ], + [ + 'an eligible unrecognised status', + { server_cache_status_class_code: 0 }, + 'conflict:eligible-with-unknown-status', + ], + ['explicit ineligibility', { server_cache_pricing_eligible: 0 }, 'not-eligible:VALID_COMPLETE'], + [ + 'eligibility dropped at ETL', + { server_cache_pricing_eligible: undefined }, + 'not-eligible:VALID_COMPLETE', + ], + ['no published rate', { server_cache_published: 0 }, 'conflict:eligible-without-rate'], + [ + 'tiers above one', + { + server_gpu_cache_hit_rate: 0.8, + server_external_cache_hit_rate: 0.5, + server_overall_cache_hit_rate: undefined, + }, + 'invalid:tier-sum-above-one', + ], + [ + 'tiers disagreeing with overall', + { server_overall_cache_hit_rate: 0.9 }, + 'conflict:tiers-disagree-with-overall', + ], + ['a NaN tier', { server_gpu_cache_hit_rate: Number.NaN }, 'invalid:tier-rate-out-of-range'], + [ + 'an infinite tier', + { server_external_cache_hit_rate: Infinity }, + 'invalid:tier-rate-out-of-range', + ], + ['a negative tier', { server_gpu_cache_hit_rate: -0.1 }, 'invalid:tier-rate-out-of-range'], + ['no basis', { server_cache_basis_code: undefined }, 'no-basis'], + ['an unrecognised basis', { server_cache_basis_code: 0 }, 'no-basis'], + ['a newer core', { server_cache_core_schema_version: 2 }, 'unsupported-core-schema'], + ['an unrecognised core', { server_cache_core_schema_version: 0 }, 'unsupported-core-schema'], + [ + 'a valid zero with hits', + { server_cache_status_class_code: 2 }, + 'conflict:valid-zero-with-hits', + ], + ])('refuses a tiers row with %s', (_label, override, reason) => { + expect(select(eligibleTiers, override)).toEqual({ rate: null, source: 'unavailable', reason }); + }); + + it.each<[string, Record, string]>([ + ['a router estimate', { server_cache_overall_source_code: 3 }, 'router-estimate'], + ['no source', { server_cache_overall_source_code: undefined }, 'combined-without-source'], + ['an unrecognised source', { server_cache_overall_source_code: 0 }, 'combined-without-source'], + [ + 'the TRT diagnostic as overall', + { server_overall_cache_hit_rate: 1.0476 }, + 'invalid:overall-out-of-range', + ], + [ + 'no overall', + { server_overall_cache_hit_rate: undefined }, + 'conflict:combined-without-overall', + ], + [ + 'tier rates written later', + { server_gpu_cache_hit_rate: 0.5 }, + 'conflict:combined-with-tier-rates', + ], + ])('refuses a combined row with %s', (_label, override, reason) => { + expect(select(eligibleCombined, override)).toEqual({ + rate: null, + source: 'unavailable', + reason, + }); + }); +}); + +describe('policies, theoretical mode, fallbacks and donors', () => { + const legacyMeasured = { server_gpu_cache_hit_rate: 0.5, server_external_cache_hit_rate: 0.3 }; + + it('defaults to the pre-core rule for every row', () => { + expect(DEFAULT_CACHE_RATE_POLICY).toBe('legacy'); + const names = Object.keys(producer.cases) as CaseName[]; + for (const metrics of [...names.map(etlCacheMetrics), legacyMeasured, {}]) { + const row = apiRow(metrics); + expect(basis(row, DEFAULT_CACHE_RATE_POLICY).rate).toBe(measuredCacheHitRate(row.metrics)); + } + }); + + it('keeps donor-borrowed rates out of strict actual pricing', () => { + // `run-overrides` writes GB200 rates into GB300 rows without a marker; the + // missing core is what tells them apart. + const donor = apiRow(legacyMeasured, { hardware: 'gb300' }); + expect(basis(donor, 'legacy')).toEqual({ rate: 0.8, source: 'legacy-measured' }); + expect(basis(donor, 'enforce-when-present')).toEqual({ rate: 0.8, source: 'legacy-measured' }); + expect(basis(donor, 'strict')).toEqual({ + rate: null, + source: 'unavailable', + reason: 'no-validity-core', + }); + }); + + it('labels the GB300 theoretical fallback and never uses it for a core row', () => { + const legacyGb300 = apiRow({ theoretical_cache_hit_rate: 0.97 }, { hardware: 'gb300' }); + expect(basis(legacyGb300, 'enforce-when-present')).toEqual({ + rate: 0.97, + source: 'theoretical-fallback', + }); + expect(basis(legacyGb300, 'strict').source).toBe('unavailable'); + + const coreGb300 = apiRow( + { ...etlCacheMetrics('trt_conditional_interval'), theoretical_cache_hit_rate: 0.97 }, + { hardware: 'gb300' }, + ); + expect(basis(coreGb300, 'legacy')).toEqual({ rate: 0.97, source: 'theoretical-fallback' }); + expect(basis(coreGb300, 'enforce-when-present').source).toBe('unavailable'); + }); + + it('prices explicit theoretical mode when the theoretical rate exists, and only then', () => { + const sglang = etlCacheMetrics('sglang_three_tiers'); + const withTheory = calculatorEstimate( + frontier({ ...sglang, theoretical_cache_hit_rate: 0.95 }), + 'strict', + 'theoretical', + ); + expect(withTheory.result.cacheHitRate).toBeCloseTo(0.95, 10); + expect(withTheory.result.cacheHitUnavailable).toBeUndefined(); + expect(isProfitEstimatorRow(withTheory.estimate)).toBe(true); + + const withoutTheory = calculatorEstimate(frontier(sglang), 'strict', 'theoretical'); + expect(withoutTheory.estimate).toMatchObject({ reason: 'cache-hit-unavailable' }); + }); + + it('does not block revenue when the price list gives no cache discount', () => { + const row = apiRow(etlCacheMetrics('sglang_three_tiers')); + const strict = chartRevenue(row, 'strict', FLAT_CACHE_PRICING); + expect(strict).toBeCloseTo(chartRevenue(row, 'legacy', FLAT_CACHE_PRICING)!, 10); + const calc = calculatorEstimate(frontier(row.metrics), 'strict', 'actual', FLAT_CACHE_PRICING); + expect(isProfitEstimatorRow(calc.estimate)).toBe(true); + }); + + it('leaves fixed-sequence rows unchanged under every policy', () => { + const fixed = apiRow({}, { benchmark_type: 'single_turn', isl: 1024, osl: 1024 }); + const legacyRevenue = chartRevenue(fixed, 'legacy'); + for (const policy of CORE_POLICIES) { + expect(basis(fixed, policy)).toEqual({ rate: null, source: 'none' }); + expect(chartRevenue(fixed, policy)).toBeCloseTo(legacyRevenue!, 10); + } + }); +}); + +describe('mixed frontiers', () => { + it('lets one unproven knot make the operating point unavailable, not zero-hit', () => { + const rows = frontier(etlCacheMetrics('atom_complete'), etlCacheMetrics('sglang_three_tiers')); + const calc = calculatorEstimate(rows, 'enforce-when-present'); + expect(calc.points.map((p) => p.cacheHitUnavailable ?? false)).toEqual([false, true]); + expect(calc.result.cacheHitRate).toBeUndefined(); + expect(calc.estimate).toMatchObject({ reason: 'cache-hit-unavailable' }); + expect(trendRevenue(rows, 'enforce-when-present')).toBeNull(); + }); + + it('keeps the legacy spline between the two measured rates by default', () => { + const rows = frontier(etlCacheMetrics('atom_complete'), etlCacheMetrics('sglang_three_tiers')); + const { result, estimate } = calculatorEstimate(rows, 'legacy'); + expect(result.cacheHitRate).toBeGreaterThan(0.7); + expect(result.cacheHitRate).toBeLessThan(0.9); + expect(isProfitEstimatorRow(estimate)).toBe(true); + expect(trendRevenue(rows, 'legacy')).not.toBeNull(); + }); + + it('prices old and new points together only while old rows are allowed', () => { + const rows = frontier( + { server_gpu_cache_hit_rate: 0.5, server_external_cache_hit_rate: 0.2 }, + etlCacheMetrics('atom_complete'), + ); + expect(isProfitEstimatorRow(calculatorEstimate(rows, 'enforce-when-present').estimate)).toBe( + true, + ); + expect(calculatorEstimate(rows, 'strict').estimate).toMatchObject({ + reason: 'cache-hit-unavailable', + }); + }); +}); diff --git a/packages/constants/src/cache-validity.test.ts b/packages/constants/src/cache-validity.test.ts new file mode 100644 index 000000000..2167d343d --- /dev/null +++ b/packages/constants/src/cache-validity.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from 'vitest'; + +import { + CACHE_BASES, + CACHE_OVERALL_SOURCES, + CACHE_STATUS_CLASSES, + CACHE_VALIDITY_METRIC_KEYS, + cacheCoreSchemaVersion, + cacheVocabularyCode, + cacheVocabularyName, + strictBooleanCode, +} from './cache-validity'; +import { METRIC_KEYS } from './metric-keys'; + +describe('cache-validity vocabularies', () => { + it('keeps the producer status classes in code order', () => { + // Codes are stored in the DB; reordering would silently remap ingested rows. + expect( + CACHE_STATUS_CLASSES.map((name) => cacheVocabularyCode(CACHE_STATUS_CLASSES, name)), + ).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9]); + expect(cacheVocabularyCode(CACHE_STATUS_CLASSES, 'NOT_EVALUATED')).toBe(9); + expect(cacheVocabularyCode(CACHE_BASES, 'combined')).toBe(2); + expect(cacheVocabularyCode(CACHE_OVERALL_SOURCES, 'router_estimate')).toBe(3); + }); + + it('drops absent values and marks unrecognised ones as 0', () => { + expect(cacheVocabularyCode(CACHE_STATUS_CLASSES, undefined)).toBeUndefined(); + expect(cacheVocabularyCode(CACHE_STATUS_CLASSES, null)).toBeUndefined(); + expect(cacheVocabularyCode(CACHE_STATUS_CLASSES, 'VALID_COMPLETE_V2')).toBe(0); + expect(cacheVocabularyCode(CACHE_BASES, 1)).toBe(0); + }); + + it('decodes only codes it issued', () => { + expect(cacheVocabularyName(CACHE_STATUS_CLASSES, 2)).toBe('VALID_ZERO'); + for (const code of [undefined, 0, -1, 1.5, 10]) { + expect(cacheVocabularyName(CACHE_STATUS_CLASSES, code)).toBeUndefined(); + } + }); + + it('reads the core version and refuses detail schemas as cores', () => { + expect(cacheCoreSchemaVersion('infx-cache-validity-core/1')).toBe(1); + expect(cacheCoreSchemaVersion('infx-cache-validity-core/2')).toBe(2); + expect(cacheCoreSchemaVersion('atom-cache-validity/1')).toBe(0); + expect(cacheCoreSchemaVersion('infx-cache-validity-core/x')).toBe(0); + for (const loose of ['01', '1.0', '1e0', ' 1', '']) { + expect(cacheCoreSchemaVersion(`infx-cache-validity-core/${loose}`)).toBe(0); + } + expect(cacheCoreSchemaVersion(1)).toBe(0); + expect(cacheCoreSchemaVersion(undefined)).toBeUndefined(); + }); + + it('accepts only JSON booleans for eligibility flags and marks other values invalid', () => { + expect(strictBooleanCode(true)).toBe(1); + expect(strictBooleanCode(false)).toBe(0); + for (const value of ['true', 1, 0, 'yes']) { + expect(strictBooleanCode(value)).toBe(-1); + } + for (const value of [null, undefined]) { + expect(strictBooleanCode(value)).toBeUndefined(); + } + }); + + it('registers every key ETL writes', () => { + for (const key of CACHE_VALIDITY_METRIC_KEYS) expect(METRIC_KEYS.has(key)).toBe(true); + }); +}); diff --git a/packages/constants/src/cache-validity.ts b/packages/constants/src/cache-validity.ts new file mode 100644 index 000000000..421241ad7 --- /dev/null +++ b/packages/constants/src/cache-validity.ts @@ -0,0 +1,122 @@ +/** + * The producer's cache-validity core (`infx-cache-validity-core/1`) encoded for + * the numeric `metrics` JSONB. The producer writes strings and booleans; ETL + * stores these codes so API rows stay `Record` and every + * consumer reads one vocabulary. + */ +export const CACHE_VALIDITY_CORE_SCHEMA_PREFIX = 'infx-cache-validity-core/'; + +/** The only core version whose semantics consumers implement. */ +export const CACHE_VALIDITY_CORE_VERSION = 1; + +/** + * Decimal places the producer keeps on every rate (`round_floats` in + * `process_agentic_result.py`), so each stored rate is off by at most half a + * step from the value it was computed as. + */ +export const CACHE_RATE_DECIMALS = 5; + +/** Code = index + 1. Code 0 marks a value this consumer does not recognise. */ +export const CACHE_STATUS_CLASSES = [ + 'VALID_COMPLETE', + 'VALID_ZERO', + 'VALID_INTERVAL', + 'PARTIAL', + 'UNRESOLVED', + 'CONTAMINATED', + 'INVALID', + 'MISSING', + 'NOT_EVALUATED', +] as const; + +/** `tiers`: GPU/CPU/external rates written; `combined`: only the overall rate. */ +export const CACHE_BASES = ['tiers', 'combined'] as const; + +/** Evidence behind a combined overall rate. A router estimate is a prediction. */ +export const CACHE_OVERALL_SOURCES = ['worker_counters', 'frontend', 'router_estimate'] as const; + +/** + * `cache_validity.schema`: the adapter's own detail contract. #3964 ships the + * ATOM detail without the common core; other adapters repeat the core schema. + */ +export const CACHE_DETAIL_SCHEMAS = [ + 'atom-cache-validity/1', + 'infx-cache-validity-core/1', +] as const; + +export type CacheStatusClass = (typeof CACHE_STATUS_CLASSES)[number]; +export type CacheBasis = (typeof CACHE_BASES)[number]; +export type CacheOverallSource = (typeof CACHE_OVERALL_SOURCES)[number]; + +/** Rate keys a validity core vouches for, in `server_metrics.cache` field order. */ +export const CACHE_RATE_METRIC_KEYS = [ + 'server_gpu_cache_hit_rate', + 'server_cpu_cache_hit_rate', + 'server_external_cache_hit_rate', + 'server_overall_cache_hit_rate', +] as const; + +/** + * Keys whose presence means the producer wrote a validity contract for the + * row, valid or not. A row carrying any of them is never a pre-core row. + */ +export const CACHE_VALIDITY_TRACE_KEYS = [ + 'server_cache_core_schema_version', + 'server_cache_detail_schema_code', + 'server_cache_status_class_code', + 'server_cache_published', + 'server_cache_pricing_eligible', + 'server_cache_basis_code', + 'server_cache_invalid_rate_fields', +] as const; + +/** Metric keys ETL writes for the cache-validity core. */ +export const CACHE_VALIDITY_METRIC_KEYS = [ + 'server_overall_cache_hit_rate', + 'server_cache_overall_source_code', + ...CACHE_VALIDITY_TRACE_KEYS, +] as const; + +/** Code for `value` in `vocabulary`: undefined when absent, 0 when unrecognised. */ +export function cacheVocabularyCode( + vocabulary: readonly string[], + value: unknown, +): number | undefined { + if (value === null || value === undefined) return undefined; + return (typeof value === 'string' ? vocabulary.indexOf(value) : -1) + 1; +} + +/** Inverse of {@link cacheVocabularyCode}; undefined for absent, 0 or unknown codes. */ +export function cacheVocabularyName( + vocabulary: readonly T[], + code: number | undefined, +): T | undefined { + return code === undefined || !Number.isInteger(code) || code < 1 + ? undefined + : vocabulary[code - 1]; +} + +const CORE_SCHEMA_PATTERN = /^infx-cache-validity-core\/(?[1-9][0-9]*)$/; + +/** + * Core version from `core_schema`: undefined when absent, 0 unless the string is + * exactly `infx-cache-validity-core/N` with a canonical decimal N, so `01`, + * `1.0` and `1e0` are not read as version 1. + */ +export function cacheCoreSchemaVersion(value: unknown): number | undefined { + if (value === null || value === undefined) return undefined; + const match = typeof value === 'string' ? CORE_SCHEMA_PATTERN.exec(value) : null; + return match?.groups ? Number(match.groups.version) : 0; +} + +/** + * JSON booleans as 1/0. Anything else that is present is -1, never dropped: + * `"true"` or `1` must not pass a numeric parse into eligibility, and a value + * that is there but invalid must stay distinguishable from one that is absent. + */ +export function strictBooleanCode(value: unknown): -1 | 0 | 1 | undefined { + if (value === null || value === undefined) return undefined; + if (value === true) return 1; + if (value === false) return 0; + return -1; +} diff --git a/packages/constants/src/index.ts b/packages/constants/src/index.ts index 3c02c9b04..41c2a0b69 100644 --- a/packages/constants/src/index.ts +++ b/packages/constants/src/index.ts @@ -1,4 +1,5 @@ export * from './benchmark-curve'; +export * from './cache-validity'; export * from './framework-aliases'; export * from './github'; export * from './gpu-keys'; diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index 06d2a7f12..57b5d1a92 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -1,3 +1,5 @@ +import { CACHE_VALIDITY_METRIC_KEYS } from './cache-validity'; + /** * NVL72 Grace-side and compute-module measurements from the srt-slurm CPU power * leg (ACPI hwmon), integrated over the same formal window as GPU energy. They @@ -241,6 +243,8 @@ export const METRIC_KEYS = new Set([ 'server_cpu_cache_hit_rate', 'server_external_cache_hit_rate', 'theoretical_cache_hit_rate', + // cache-validity core and overall rate (v3 server_metrics.cache_validity) + ...CACHE_VALIDITY_METRIC_KEYS, // server KV-cache occupancy — mean GPU KV-cache usage fraction (0-1) over the // profiling window (agentic aiperf; flat in v2 artifacts, mapped from // server_metrics.kv_cache.gpu_usage_pct in v3) diff --git a/packages/db/src/etl/agentic-v3-cache-validity.test.ts b/packages/db/src/etl/agentic-v3-cache-validity.test.ts new file mode 100644 index 000000000..e38e82ad6 --- /dev/null +++ b/packages/db/src/etl/agentic-v3-cache-validity.test.ts @@ -0,0 +1,162 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + CACHE_BASES, + CACHE_OVERALL_SOURCES, + CACHE_STATUS_CLASSES, + cacheVocabularyName, +} from '@semianalysisai/inferencex-constants'; + +import { flattenAgenticAggRow } from './agentic-v3-flatten'; +import { mapBenchmarkRow } from './benchmark-mapper'; +import producer from './fixtures/cache-validity-producer.json'; +import { createSkipTracker } from './skip-tracker'; + +/** Minimal v3 agentic agg row around one producer `server_metrics` payload. */ +function v3Row(serverMetrics: Record): Record { + return { + infmax_model_prefix: 'dsv4', + hw: 'cluster:b300-nv', + framework: 'vllm', + precision: 'fp4', + spec_decoding: 'none', + disagg: false, + scenario_type: 'agentic-coding', + is_multinode: false, + tp: 4, + ep: 1, + dp_attention: 'false', + conc: 16, + image: 'vllm/vllm-openai:v0.23.0', + kv_offloading: 'none', + num_requests_total: 100, + num_requests_successful: 100, + request_metrics: { + throughput: { per_gpu: { total_tput_tps: 2000, input_tput_tps: 1900, output_tput_tps: 100 } }, + }, + server_metrics: { present: true, ...serverMetrics }, + }; +} + +function mappedMetrics(serverMetrics: Record): Record { + const result = mapBenchmarkRow(v3Row(serverMetrics), createSkipTracker()); + expect(result).not.toBeNull(); + return result!.metrics; +} + +const CASES = Object.entries(producer.cases); + +describe('v3 cache-validity core through the real mapper', () => { + afterEach(() => vi.restoreAllMocks()); + + it.each(CASES)('%s round-trips its core without an unregistered key', (_name, payload) => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const m = mappedMetrics(payload); + const core = payload.cache_validity; + + expect(m.server_cache_core_schema_version).toBe(1); + expect(cacheVocabularyName(CACHE_STATUS_CLASSES, m.server_cache_status_class_code)).toBe( + core.status_class, + ); + expect(m.server_cache_pricing_eligible).toBe(core.pricing_eligible ? 1 : 0); + expect(m.server_cache_published).toBe(core.published ? 1 : 0); + expect(cacheVocabularyName(CACHE_BASES, m.server_cache_basis_code)).toBe( + core.basis ?? undefined, + ); + expect(cacheVocabularyName(CACHE_OVERALL_SOURCES, m.server_cache_overall_source_code)).toBe( + payload.cache.overall_source ?? undefined, + ); + expect(m.server_overall_cache_hit_rate).toBe(payload.cache.overall_cache_hit_rate ?? undefined); + expect(warn.mock.calls.flat().join('\n')).not.toMatch(/auto-captured unexpected metric/); + }); + + it('marks only the ATOM complete and zero observations eligible', () => { + const eligible = CASES.filter( + ([, payload]) => mappedMetrics(payload).server_cache_pricing_eligible === 1, + ); + expect(eligible.map(([name]) => name)).toEqual(['atom_complete', 'atom_zero']); + }); + + it('keeps the TRT rejection diagnostic out of every metric', () => { + const payload = producer.cases.trt_rejected_cached_exceeds_prompt; + expect(payload.validity.cache_rate.raw_rate).toBeCloseTo(1.0476, 10); + const m = mappedMetrics(payload); + expect(Object.values(m).some((value) => Math.abs(value - 1.0476) < 1e-6)).toBe(false); + expect(m.server_overall_cache_hit_rate).toBeUndefined(); + expect(m.server_cache_basis_code).toBeUndefined(); + }); + + it('distinguishes the frontend measurement from the router estimate', () => { + expect(mappedMetrics(producer.cases.dynamo_frontend).server_cache_overall_source_code).toBe(2); + expect( + mappedMetrics(producer.cases.dynamo_router_estimate).server_cache_overall_source_code, + ).toBe(3); + }); +}); + +describe('v3 cache-validity edge cases', () => { + const core = { + core_schema: 'infx-cache-validity-core/1', + status_class: 'VALID_COMPLETE', + pricing_eligible: true, + published: true, + basis: 'tiers', + }; + const flat = (serverMetrics: Record) => + flattenAgenticAggRow(v3Row(serverMetrics)); + + it('writes no validity code for a pre-core row', () => { + const row = flat({ cache: { gpu_cache_hit_rate: 0.78, overall_cache_hit_rate: 0.78 } }); + expect(row.server_gpu_cache_hit_rate).toBe(0.78); + expect(row.server_overall_cache_hit_rate).toBe(0.78); + for (const key of Object.keys(row)) expect(key.startsWith('server_cache_')).toBe(false); + }); + + it('marks eligibility that is not a JSON boolean as invalid', () => { + for (const [value, code] of [ + ['true', -1], + [1, -1], + [null, undefined], + ] as const) { + const row = flat({ cache_validity: { ...core, pricing_eligible: value, published: 'yes' } }); + expect(row.server_cache_pricing_eligible).toBe(code); + expect(row.server_cache_published).toBe(-1); + expect(row.server_cache_status_class_code).toBe(1); + } + }); + + it('marks unrecognised values as 0 and reads newer core versions', () => { + const row = flat({ + cache: { overall_source: 'gauge' }, + cache_validity: { + ...core, + core_schema: 'infx-cache-validity-core/2', + status_class: 'VALID_COMPLETE_V2', + basis: 'layers', + }, + }); + expect(row.server_cache_core_schema_version).toBe(2); + expect(row.server_cache_status_class_code).toBe(0); + expect(row.server_cache_basis_code).toBe(0); + expect(row.server_cache_overall_source_code).toBe(0); + }); + + it('does not treat an ATOM detail schema as a core', () => { + const row = flat({ + cache_validity: { + schema: 'atom-cache-validity/1', + status: 'VALID_COMPLETE', + pricing_eligible: true, + }, + }); + expect(row.server_cache_core_schema_version).toBeUndefined(); + expect(row.server_cache_status_class_code).toBeUndefined(); + }); + + it('never writes a non-finite or non-numeric overall rate', () => { + for (const overall of [Infinity, Number.NaN, '0.5']) { + expect( + flat({ cache: { overall_cache_hit_rate: overall } }).server_overall_cache_hit_rate, + ).toBe(undefined); + } + }); +}); diff --git a/packages/db/src/etl/agentic-v3-flatten.ts b/packages/db/src/etl/agentic-v3-flatten.ts index 1349e2263..217acbe81 100644 --- a/packages/db/src/etl/agentic-v3-flatten.ts +++ b/packages/db/src/etl/agentic-v3-flatten.ts @@ -7,6 +7,17 @@ * API / frontend consume, so the rest of the mapper stays version-agnostic. */ +import { + CACHE_BASES, + CACHE_DETAIL_SCHEMAS, + CACHE_OVERALL_SOURCES, + CACHE_RATE_METRIC_KEYS, + CACHE_STATUS_CLASSES, + cacheCoreSchemaVersion, + cacheVocabularyCode, + strictBooleanCode, +} from '@semianalysisai/inferencex-constants'; + import { parseNum } from './normalizers'; /** @@ -71,9 +82,10 @@ const V3_SCALAR_PATHS: [string[], string][] = [ [['server_metrics', 'tokens', 'prompt_total'], 'total_prompt_tokens'], [['server_metrics', 'tokens', 'generation_total'], 'total_generation_tokens'], [['server_metrics', 'tokens', 'requests_completed'], 'total_requests_completed'], - // Deliberately NOT mapped (yet): cache.overall/prefix_cache_hits/queries, + // Deliberately NOT mapped (yet): cache.prefix_cache_hits/queries, // kv_cache.cpu_*, tokens.prompt_by_source, sources[] — new v3 detail we don't - // consume anywhere; add here + METRIC_KEYS when a view needs them. + // consume anywhere; add here + METRIC_KEYS when a view needs them. The + // overall rate and the validity core go through `flattenCacheValidity`. ]; /** Walk a nested object path; returns undefined on any non-object hop. */ @@ -95,6 +107,60 @@ function flattenStatBlock(block: unknown, suffix: string, out: Record, out: Record): void { + const cache = (field: string) => atPath(row, ['server_metrics', 'cache', field]); + const core = (field: string) => atPath(row, ['server_metrics', 'cache_validity', field]); + const validity = atPath(row, ['server_metrics', 'cache_validity']); + const overall = cache('overall_cache_hit_rate'); + const fields = Object.entries({ + server_overall_cache_hit_rate: + typeof overall === 'number' && Number.isFinite(overall) ? overall : undefined, + server_cache_overall_source_code: cacheVocabularyCode( + CACHE_OVERALL_SOURCES, + cache('overall_source'), + ), + server_cache_core_schema_version: cacheCoreSchemaVersion(core('core_schema')), + server_cache_detail_schema_code: cacheVocabularyCode(CACHE_DETAIL_SCHEMAS, core('schema')), + server_cache_status_class_code: cacheVocabularyCode(CACHE_STATUS_CLASSES, core('status_class')), + server_cache_published: strictBooleanCode(core('published')), + server_cache_pricing_eligible: strictBooleanCode(core('pricing_eligible')), + server_cache_basis_code: cacheVocabularyCode(CACHE_BASES, core('basis')), + }); + for (const [key, value] of fields) { + if (value !== undefined) out[key] = value; + } + + // On a row with a validity contract, a rate that is present but not a finite + // number must not reach pricing as `parseNum`'s coercion (`false` -> 0) under + // the producer's eligibility. Pre-core rows keep the existing parse. + if (!validity || typeof validity !== 'object' || Array.isArray(validity)) return; + let invalid = 0; + for (const [bit, field] of CACHE_RATE_FIELDS.entries()) { + const raw = cache(field); + if (raw === null || raw === undefined || (typeof raw === 'number' && Number.isFinite(raw))) { + continue; + } + invalid |= 1 << bit; + delete out[CACHE_RATE_METRIC_KEYS[bit]]; + } + if (invalid !== 0) out.server_cache_invalid_rate_fields = invalid; +} + /** * Flatten a v3 agentic agg row (nested `request_metrics` / `server_metrics` * containers, 2026-07-02+) into the canonical flat metric schema that v1/v2 @@ -134,6 +200,7 @@ export function flattenAgenticAggRow(row: Record): Record): Record { + return { + infmax_model_prefix: 'dsv4', + hw: 'cluster:b300-nv', + framework: 'vllm', + precision: 'fp4', + spec_decoding: 'none', + disagg: false, + scenario_type: 'agentic-coding', + is_multinode: false, + tp: 4, + ep: 1, + dp_attention: 'false', + conc: 16, + image: 'vllm/vllm-openai:v0.23.0', + kv_offloading: 'none', + num_requests_total: 100, + num_requests_successful: 100, + request_metrics: { + throughput: { per_gpu: { total_tput_tps: 2000, input_tput_tps: 1900, output_tput_tps: 100 } }, + }, + server_metrics: { present: true, ...serverMetrics }, + }; +} + +const flat = (serverMetrics: Record) => flattenAgenticAggRow(v3Row(serverMetrics)); +const core = producer.cases.atom_complete.cache_validity; + +describe('ETL keeps "present but invalid" distinct from absent', () => { + it('records the detail schema of a #3964 row that has no common core', () => { + const row = flat(closeout.cases.atom3964_complete); + expect(row.server_cache_detail_schema_code).toBe(1); + expect(row.server_cache_pricing_eligible).toBe(1); + expect(row.server_cache_core_schema_version).toBeUndefined(); + }); + + it('marks non-boolean eligibility as invalid instead of dropping it', () => { + const row = flat({ cache_validity: { ...core, pricing_eligible: 'true', published: 1 } }); + expect(row.server_cache_pricing_eligible).toBe(-1); + expect(row.server_cache_published).toBe(-1); + }); + + it('marks non-numeric rates on a validity row instead of coercing them to zero', () => { + const row = flat({ + cache: { + gpu_cache_hit_rate: false, + cpu_cache_hit_rate: 'x', + external_cache_hit_rate: null, + overall_cache_hit_rate: 'invalid', + }, + cache_validity: core, + }); + expect(row.server_gpu_cache_hit_rate).toBeUndefined(); + expect(row.server_cpu_cache_hit_rate).toBeUndefined(); + expect(row.server_cache_invalid_rate_fields).toBe(1 + 2 + 8); + }); + + it('leaves pre-core rows on the existing numeric parse', () => { + const row = flat({ cache: { gpu_cache_hit_rate: false } }); + expect(row.server_gpu_cache_hit_rate).toBe(0); + expect(row.server_cache_invalid_rate_fields).toBeUndefined(); + }); + + it.each<[string, number]>([ + ['infx-cache-validity-core/1', 1], + ['infx-cache-validity-core/12', 12], + ['infx-cache-validity-core/01', 0], + ['infx-cache-validity-core/1.0', 0], + ['infx-cache-validity-core/1e0', 0], + ['infx-cache-validity-core/ 1', 0], + ['infx-cache-validity-core/', 0], + ])('reads core schema %s as version %d', (schema, version) => { + expect( + flat({ cache_validity: { ...core, core_schema: schema } }).server_cache_core_schema_version, + ).toBe(version); + }); + + it('maps the real MISSING producer output', () => { + const result = mapBenchmarkRow( + v3Row(closeout.cases.missing_server_metrics), + createSkipTracker(), + ); + expect(result).not.toBeNull(); + expect(result!.metrics).toMatchObject({ + server_cache_core_schema_version: 1, + server_cache_status_class_code: 8, + server_cache_pricing_eligible: 0, + server_cache_published: 0, + server_cache_detail_schema_code: 2, + }); + expect(result!.metrics.server_cache_basis_code).toBeUndefined(); + }); +}); + +function preparedWithRates(): PreparedTraceReplay { + return { + profileGz: Buffer.from('profile'), + profileSize: 70, + serverMetricsCsv: Buffer.from('csv'), + serverMetricsCsvSize: 30, + serverMetricsJsonGz: Buffer.from('metrics'), + serverMetricsJsonSize: 700, + aggregateStatsJson: Buffer.from('{"version":1}'), + chartSeriesJson: Buffer.from('{"version":12}'), + requestTimelineJson: Buffer.from('{"version":1}'), + chartWindows: 2, + timelineRequests: 3, + compressionMs: 10, + computeMs: 20, + cacheHitRates: { gpu: 0.2, cpu: 0.5 }, + fullResponseMetrics: {}, + }; +} + +describe('trace replay does not overwrite producer-validated cache rates', () => { + it('excludes rows that carry a validity contract from the chart-series rewrite', async () => { + const calls: { text: string; values: unknown[] }[] = []; + const execute = (strings: TemplateStringsArray, ...values: unknown[]) => { + const text = strings.join('?'); + calls.push({ text, values }); + if (text.includes('for update')) return Promise.resolve([{ id: 41 }]); + if (text.includes('insert into agentic_trace_replay')) return Promise.resolve([{ id: 123 }]); + return Promise.resolve([]); + }; + const tx = Object.assign(execute, { + array: (values: unknown[]) => values, + json: (value: unknown) => value, + }); + const sql = Object.assign(execute, { + array: (values: unknown[]) => values, + begin: (operation: (transaction: typeof tx) => Promise) => operation(tx), + }) as unknown as Parameters[0]; + + await expect(persistPreparedTraceReplay(sql, [41], preparedWithRates())).resolves.toBe(1); + const rewrite = calls.find((call) => call.text.includes("'{server_gpu_cache_hit_rate}'")); + expect(rewrite).toBeDefined(); + expect(rewrite!.text).toContain('?|'); + const guarded = rewrite!.values.find( + (value): value is string[] => + Array.isArray(value) && value.length > 0 && value.every((key) => typeof key === 'string'), + ); + expect(guarded).toEqual( + expect.arrayContaining(['server_cache_core_schema_version', 'server_cache_pricing_eligible']), + ); + }); +}); + +describe('donor backfills do not inherit producer eligibility', () => { + const donor = BENCHMARK_POINT_BACKFILLS.find( + (backfill) => backfill.id.endsWith('-gb200-cache-hit-rate') && !backfill.set.offloadMode, + )!; + const validated = mapBenchmarkRow( + v3Row(producer.cases.atom_complete), + createSkipTracker(), + )!.metrics; + const point = (metrics: Record) => ({ + configId: 1, + config: donor.config, + benchmarkType: donor.benchmarkType, + isl: donor.isl, + osl: donor.osl, + conc: donor.conc, + offloadMode: donor.offloadMode, + recipeFingerprint: donor.recipeFingerprint ?? null, + metrics, + }); + const row = (metrics: Record) => ({ + offload_mode: donor.offloadMode, + recipe_fingerprint: donor.recipeFingerprint ?? null, + metrics, + }); + + it('is a real GB200→GB300 cache-rate donor', () => { + expect(donor.set.metricsMerge).toHaveProperty('server_gpu_cache_hit_rate'); + }); + + it('refuses the replay path over a validated core and applies it to a pre-core row', () => { + expect(() => planBenchmarkPointBackfill(row(validated), donor)).toThrow(/validity/); + const merged = planBenchmarkPointBackfill(row({ server_gpu_cache_hit_rate: 0 }), donor); + expect(merged?.server_gpu_cache_hit_rate).toBe( + donor.set.metricsMerge?.server_gpu_cache_hit_rate, + ); + }); + + it('refuses the ingest path over a validated core and applies it to a pre-core row', () => { + expect(() => + applyBenchmarkPointBackfill(donor.githubRunId, donor.runAttempt, point(validated)), + ).toThrow(/validity/); + const applied = applyBenchmarkPointBackfill(donor.githubRunId, donor.runAttempt, point({})); + expect(applied.backfillId).toBe(donor.id); + expect(applied.point.metrics.server_gpu_cache_hit_rate).toBe( + donor.set.metricsMerge?.server_gpu_cache_hit_rate, + ); + }); + + describe('the overall source a validated row is priced on', () => { + // Synthetic rules on the donor's identity: no real backfill touches the source key. + const rule = (set: BenchmarkPointBackfill['set']): BenchmarkPointBackfill => ({ + ...donor, + id: 'synthetic-overall-source-rule', + set, + }); + const combinedRouter = { + server_cache_core_schema_version: 1, + server_cache_status_class_code: 1, + server_cache_published: 1, + server_cache_pricing_eligible: 1, + server_cache_basis_code: 2, + server_overall_cache_hit_rate: 0.9, + server_cache_overall_source_code: 3, + }; + const tiersRouter = { ...validated, server_cache_overall_source_code: 3 }; + const sourceMerge = rule({ metricsMerge: { server_cache_overall_source_code: 1 } }); + const sourceRemove = rule({ metricsRemove: ['server_cache_overall_source_code'] }); + + it('refuses a source-only merge on a combined row', () => { + expect(() => planBenchmarkPointBackfill(row(combinedRouter), sourceMerge)).toThrow( + /server_cache_overall_source_code/, + ); + }); + + it('refuses a source-only removal on a tiers row', () => { + expect(() => planBenchmarkPointBackfill(row(tiersRouter), sourceRemove)).toThrow( + /server_cache_overall_source_code/, + ); + }); + + it('still refuses rate rewrites and eligibility removal', () => { + const rateMerge = rule({ metricsMerge: { server_gpu_cache_hit_rate: 0.2 } }); + const eligibilityRemove = rule({ metricsRemove: ['server_cache_pricing_eligible'] }); + expect(() => planBenchmarkPointBackfill(row(validated), rateMerge)).toThrow(/validity/); + expect(() => planBenchmarkPointBackfill(row(validated), eligibilityRemove)).toThrow( + /validity/, + ); + }); + + it('does not turn a source-only legacy row into a validated one', () => { + const legacy = { server_gpu_cache_hit_rate: 0.5, server_cache_overall_source_code: 3 }; + expect(planBenchmarkPointBackfill(row(legacy), sourceMerge)).toMatchObject({ + server_cache_overall_source_code: 1, + }); + expect(planBenchmarkPointBackfill(row(legacy), sourceRemove)).not.toHaveProperty( + 'server_cache_overall_source_code', + ); + }); + + it('leaves non-cache fields on a validated row open to backfills', () => { + const offload = rule({ metricsMerge: { kv_offloading: 'dram' } }); + expect(planBenchmarkPointBackfill(row(validated), offload)).toMatchObject({ + kv_offloading: 'dram', + }); + }); + + it('applies the same guard to the replay and ingest entries', () => { + // Ingest only matches the real rule list, so its call into the shared guard is + // exercised with the real donor; the guard itself refuses source-only rules. + expect(() => assertBackfillKeepsValidatedCache(combinedRouter, sourceMerge)).toThrow( + /server_cache_overall_source_code/, + ); + expect(() => + applyBenchmarkPointBackfill(donor.githubRunId, donor.runAttempt, point(combinedRouter)), + ).toThrow(/validity/); + }); + }); +}); diff --git a/packages/db/src/etl/fixtures/cache-validity-producer-x1ir.json b/packages/db/src/etl/fixtures/cache-validity-producer-x1ir.json new file mode 100644 index 000000000..fd8eb0050 --- /dev/null +++ b/packages/db/src/etl/fixtures/cache-validity-producer-x1ir.json @@ -0,0 +1,141 @@ +{ + "scope": "Synthetic producer outputs for the X1-IR consumer closeout tests; not historical results.", + "generators": [ + "lp_fxi_cpu_kvcache/tools/x1ir_make_fixtures.py --tree prb", + "lp_fxi_cpu_kvcache/tools/x1ir_make_fixtures.py --tree 3964" + ], + "cases": { + "missing_server_metrics": { + "producer_commit": "9e3c706b3661f03dd12fdb0cd4e3e7e226cba96e", + "present": false, + "cache": { + "gpu_cache_hit_rate": null, + "cpu_cache_hit_rate": null, + "external_cache_hit_rate": null, + "overall_cache_hit_rate": null, + "overall_source": null, + "role_scope": null + }, + "cache_validity": { + "schema": "infx-cache-validity-core/1", + "core_schema": "infx-cache-validity-core/1", + "status": "MISSING_OBSERVATION", + "status_class": "MISSING", + "pricing_eligible": false, + "published": false, + "basis": null, + "reasons": ["server_metrics_missing"] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "atom_no_cpu_tier": { + "producer_commit": "9e3c706b3661f03dd12fdb0cd4e3e7e226cba96e", + "present": true, + "cache": { + "gpu_cache_hit_rate": 0.6, + "cpu_cache_hit_rate": 0.0, + "external_cache_hit_rate": 0.0, + "overall_cache_hit_rate": 0.6, + "overall_source": null, + "role_scope": null + }, + "cache_validity": { + "schema": "atom-cache-validity/1", + "core_schema": "infx-cache-validity-core/1", + "status": "VALID_COMPLETE", + "status_class": "VALID_COMPLETE", + "pricing_eligible": true, + "published": true, + "basis": "tiers", + "reasons": [] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "atom_complete": { + "producer_commit": "9e3c706b3661f03dd12fdb0cd4e3e7e226cba96e", + "present": true, + "cache": { + "gpu_cache_hit_rate": 0.6, + "cpu_cache_hit_rate": 0.1, + "external_cache_hit_rate": 0.1, + "overall_cache_hit_rate": 0.7, + "overall_source": null, + "role_scope": null + }, + "cache_validity": { + "schema": "atom-cache-validity/1", + "core_schema": "infx-cache-validity-core/1", + "status": "VALID_COMPLETE", + "status_class": "VALID_COMPLETE", + "pricing_eligible": true, + "published": true, + "basis": "tiers", + "reasons": [] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "atom3964_complete": { + "producer_commit": "0842d829", + "present": true, + "cache": { + "gpu_cache_hit_rate": 0.6, + "cpu_cache_hit_rate": 0.1, + "external_cache_hit_rate": 0.1, + "overall_cache_hit_rate": 0.7 + }, + "cache_validity": { + "schema": "atom-cache-validity/1", + "status": "VALID_COMPLETE", + "pricing_eligible": true, + "reasons": [] + } + }, + "atom3964_zero": { + "producer_commit": "0842d829", + "present": true, + "cache": { + "gpu_cache_hit_rate": 0.0, + "cpu_cache_hit_rate": 0.0, + "external_cache_hit_rate": 0.0, + "overall_cache_hit_rate": 0.0 + }, + "cache_validity": { + "schema": "atom-cache-validity/1", + "status": "VALID_ZERO", + "pricing_eligible": true, + "reasons": [] + } + }, + "atom3964_partial_component": { + "producer_commit": "0842d829", + "present": true, + "cache": { + "gpu_cache_hit_rate": 0.6, + "cpu_cache_hit_rate": null, + "external_cache_hit_rate": null, + "overall_cache_hit_rate": null + }, + "cache_validity": { + "schema": "atom-cache-validity/1", + "status": "PARTIAL_COMPONENT", + "pricing_eligible": false, + "reasons": ["CPU reuse counter missing and no confirmed absence of a CPU tier"] + } + } + } +} diff --git a/packages/db/src/etl/fixtures/cache-validity-producer.json b/packages/db/src/etl/fixtures/cache-validity-producer.json new file mode 100644 index 000000000..b5bb63760 --- /dev/null +++ b/packages/db/src/etl/fixtures/cache-validity-producer.json @@ -0,0 +1,305 @@ +{ + "scope": "Synthetic producer outputs for consumer-contract tests; not historical results.", + "producer_commit": "9e3c706b3661f03dd12fdb0cd4e3e7e226cba96e", + "generator": "lp_fxi_cpu_kvcache/tools/x1i_make_fixtures.py", + "cases": { + "atom_complete": { + "cache": { + "gpu_cache_hit_rate": 0.6, + "cpu_cache_hit_rate": 0.1, + "external_cache_hit_rate": 0.1, + "overall_cache_hit_rate": 0.7, + "overall_source": null, + "role_scope": null + }, + "cache_validity": { + "schema": "atom-cache-validity/1", + "core_schema": "infx-cache-validity-core/1", + "status": "VALID_COMPLETE", + "status_class": "VALID_COMPLETE", + "pricing_eligible": true, + "published": true, + "basis": "tiers", + "reasons": [] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "atom_zero": { + "cache": { + "gpu_cache_hit_rate": 0.0, + "cpu_cache_hit_rate": 0.0, + "external_cache_hit_rate": 0.0, + "overall_cache_hit_rate": 0.0, + "overall_source": null, + "role_scope": null + }, + "cache_validity": { + "schema": "atom-cache-validity/1", + "core_schema": "infx-cache-validity-core/1", + "status": "VALID_ZERO", + "status_class": "VALID_ZERO", + "pricing_eligible": true, + "published": true, + "basis": "tiers", + "reasons": [] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "atom_partial_component": { + "cache": { + "gpu_cache_hit_rate": 0.6, + "cpu_cache_hit_rate": null, + "external_cache_hit_rate": null, + "overall_cache_hit_rate": null, + "overall_source": null, + "role_scope": null + }, + "cache_validity": { + "schema": "atom-cache-validity/1", + "core_schema": "infx-cache-validity-core/1", + "status": "PARTIAL_COMPONENT", + "status_class": "PARTIAL", + "pricing_eligible": false, + "published": true, + "basis": "tiers", + "reasons": ["CPU reuse counter missing and no confirmed absence of a CPU tier"] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "atom_request_scope_unresolved": { + "cache": { + "gpu_cache_hit_rate": 0.6, + "cpu_cache_hit_rate": 0.1, + "external_cache_hit_rate": 0.1, + "overall_cache_hit_rate": 0.7, + "overall_source": null, + "role_scope": null + }, + "cache_validity": { + "schema": "atom-cache-validity/1", + "core_schema": "infx-cache-validity-core/1", + "status": "REQUEST_SCOPE_UNRESOLVED", + "status_class": "UNRESOLVED", + "pricing_eligible": false, + "published": true, + "basis": "tiers", + "reasons": [ + "server window admissions/prompt tokens do not match the successful profiling requests (5.0 vs 5; 1000.0 vs 1000)" + ] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "trt_conditional_interval": { + "cache": { + "gpu_cache_hit_rate": null, + "cpu_cache_hit_rate": null, + "external_cache_hit_rate": null, + "overall_cache_hit_rate": 0.9, + "overall_source": null, + "role_scope": null + }, + "cache_validity": { + "schema": "infx-cache-validity-core/1", + "core_schema": "infx-cache-validity-core/1", + "status": "CONDITIONAL_INTERVAL", + "status_class": "VALID_INTERVAL", + "pricing_eligible": false, + "published": true, + "basis": "combined", + "reasons": ["request_scope_not_bound", "expected_instances_not_verified"] + }, + "validity": { + "cache_rate": { + "status": "CONDITIONAL_INTERVAL", + "reasons": ["request_scope_not_bound", "expected_instances_not_verified"], + "raw_rate": 0.9 + } + } + }, + "trt_rejected_cached_exceeds_prompt": { + "cache": { + "gpu_cache_hit_rate": null, + "cpu_cache_hit_rate": null, + "external_cache_hit_rate": null, + "overall_cache_hit_rate": null, + "overall_source": null, + "role_scope": null + }, + "cache_validity": { + "schema": "infx-cache-validity-core/1", + "core_schema": "infx-cache-validity-core/1", + "status": "REJECTED", + "status_class": "INVALID", + "pricing_eligible": false, + "published": false, + "basis": null, + "reasons": [ + "cached_exceeds_prompt", + "request_scope_not_bound", + "expected_instances_not_verified" + ] + }, + "validity": { + "cache_rate": { + "status": "REJECTED", + "reasons": [ + "cached_exceeds_prompt", + "request_scope_not_bound", + "expected_instances_not_verified" + ], + "raw_rate": 1.0475999999999999 + } + } + }, + "sglang_three_tiers": { + "cache": { + "gpu_cache_hit_rate": 0.6, + "cpu_cache_hit_rate": 0.2, + "external_cache_hit_rate": 0.3, + "overall_cache_hit_rate": 0.9, + "overall_source": null, + "role_scope": null + }, + "cache_validity": { + "schema": "infx-cache-validity-core/1", + "core_schema": "infx-cache-validity-core/1", + "status": "NOT_EVALUATED", + "status_class": "NOT_EVALUATED", + "pricing_eligible": false, + "published": true, + "basis": "tiers", + "reasons": ["adapter_does_not_assess_cache_rate"] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "dynamo_frontend": { + "cache": { + "gpu_cache_hit_rate": null, + "cpu_cache_hit_rate": null, + "external_cache_hit_rate": null, + "overall_cache_hit_rate": 0.8, + "overall_source": "frontend", + "role_scope": "combined" + }, + "cache_validity": { + "schema": "infx-cache-validity-core/1", + "core_schema": "infx-cache-validity-core/1", + "status": "NOT_EVALUATED", + "status_class": "NOT_EVALUATED", + "pricing_eligible": false, + "published": true, + "basis": "combined", + "reasons": ["adapter_does_not_assess_cache_rate"] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "dynamo_router_estimate": { + "cache": { + "gpu_cache_hit_rate": null, + "cpu_cache_hit_rate": null, + "external_cache_hit_rate": null, + "overall_cache_hit_rate": 0.7, + "overall_source": "router_estimate", + "role_scope": "combined" + }, + "cache_validity": { + "schema": "infx-cache-validity-core/1", + "core_schema": "infx-cache-validity-core/1", + "status": "NOT_EVALUATED", + "status_class": "NOT_EVALUATED", + "pricing_eligible": false, + "published": true, + "basis": "combined", + "reasons": ["adapter_does_not_assess_cache_rate"] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "vllm_decode_only": { + "cache": { + "gpu_cache_hit_rate": null, + "cpu_cache_hit_rate": 1.0, + "external_cache_hit_rate": 1.0, + "overall_cache_hit_rate": 1.0, + "overall_source": "worker_counters", + "role_scope": "combined" + }, + "cache_validity": { + "schema": "infx-cache-validity-core/1", + "core_schema": "infx-cache-validity-core/1", + "status": "NOT_EVALUATED", + "status_class": "NOT_EVALUATED", + "pricing_eligible": false, + "published": true, + "basis": "tiers", + "reasons": ["adapter_does_not_assess_cache_rate"] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + }, + "vllm_conditional_denominator": { + "cache": { + "gpu_cache_hit_rate": 0.6, + "cpu_cache_hit_rate": 0.5, + "external_cache_hit_rate": 0.5, + "overall_cache_hit_rate": 0.6, + "overall_source": "worker_counters", + "role_scope": "prefill" + }, + "cache_validity": { + "schema": "infx-cache-validity-core/1", + "core_schema": "infx-cache-validity-core/1", + "status": "NOT_EVALUATED", + "status_class": "NOT_EVALUATED", + "pricing_eligible": false, + "published": true, + "basis": "tiers", + "reasons": ["adapter_does_not_assess_cache_rate"] + }, + "validity": { + "cache_rate": { + "status": "NOT_EVALUATED", + "reasons": ["adapter_does_not_assess_cache_rate"] + } + } + } + } +} diff --git a/packages/db/src/etl/run-overrides.ts b/packages/db/src/etl/run-overrides.ts index 94483bbef..b764e88d9 100644 --- a/packages/db/src/etl/run-overrides.ts +++ b/packages/db/src/etl/run-overrides.ts @@ -33,7 +33,10 @@ import { type ConfigParams, configCacheKey } from './config-cache'; import { QWEN35_P90_POWER_BACKFILLS } from './power-p90-backfills'; -import { matchesExpectedBackfillMetrics } from '../lib/benchmark-point-backfill'; +import { + assertBackfillKeepsValidatedCache, + matchesExpectedBackfillMetrics, +} from '../lib/benchmark-point-backfill'; export const CONCLUSION_OVERRIDES: ReadonlyMap = new Map([ [22806827144, 'success'], // 2026-03-07 | dsr1 fp8 h200 SGLang 0.5.7→0.5.9 bump | Reason: database upload step failed @@ -1242,6 +1245,7 @@ export function applyBenchmarkPointBackfill( if (!backfill || !matchesExpectedBackfillMetrics(point.metrics, backfill)) { return { point, backfillId: null, sourceIdentity, desiredIdentity: sourceIdentity }; } + assertBackfillKeepsValidatedCache(point.metrics, backfill); const metrics = { ...point.metrics }; for (const key of backfill.set.metricsRemove ?? []) delete metrics[key]; diff --git a/packages/db/src/etl/trace-replay-ingest.ts b/packages/db/src/etl/trace-replay-ingest.ts index caa5edb47..040b0cc8f 100644 --- a/packages/db/src/etl/trace-replay-ingest.ts +++ b/packages/db/src/etl/trace-replay-ingest.ts @@ -12,6 +12,7 @@ import { createReadStream } from 'node:fs'; import { readFile, stat } from 'node:fs/promises'; import { createGzip, gzipSync } from 'node:zlib'; +import { CACHE_VALIDITY_TRACE_KEYS } from '@semianalysisai/inferencex-constants'; import type postgres from 'postgres'; import { PROFILE_INLINE_BYTES, writeTraceProfileChunks } from '../lib/trace-profile-storage'; @@ -371,6 +372,9 @@ export async function persistPreparedTraceReplay( log(`linked benchmark rows (${elapsed(updateStart)})`); if (cacheHitRates) { + // Rows carrying a producer validity contract keep the producer's rates: + // its eligibility vouches for those values, not for this chart-series + // recomputation. await tx` update benchmark_results set metrics = jsonb_set( @@ -386,6 +390,7 @@ export async function persistPreparedTraceReplay( to_jsonb(${cacheHitRates.gpu}::numeric) ) where id = any(${tx.array(unlinked.map((row) => row.id))}::bigint[]) + and not (metrics ?| ${tx.array([...CACHE_VALIDITY_TRACE_KEYS])}::text[]) `; log('updated cache-hit metrics from chart series'); } diff --git a/packages/db/src/lib/benchmark-point-backfill.ts b/packages/db/src/lib/benchmark-point-backfill.ts index 3d4fc77c7..944495df8 100644 --- a/packages/db/src/lib/benchmark-point-backfill.ts +++ b/packages/db/src/lib/benchmark-point-backfill.ts @@ -1,5 +1,10 @@ import { isDeepStrictEqual } from 'node:util'; +import { + CACHE_RATE_METRIC_KEYS, + CACHE_VALIDITY_TRACE_KEYS, +} from '@semianalysisai/inferencex-constants'; + import type { BenchmarkPointBackfill } from '../etl/run-overrides.js'; import { isRecord } from './records.js'; @@ -26,6 +31,36 @@ export function matchesExpectedBackfillMetrics( ); } +const VALIDATED_CACHE_KEYS: ReadonlySet = new Set([ + ...CACHE_RATE_METRIC_KEYS, + // Decides whether a combined overall is actual, so it is protected on a + // validated row; on its own it does not make a row validated. + 'server_cache_overall_source_code', + ...CACHE_VALIDITY_TRACE_KEYS, +]); + +/** + * Refuse a backfill that would rewrite cache-hit metrics, their overall + * source, or the validity fields vouching for them, on a row carrying a + * producer validity contract: the producer's eligibility covers its own + * values, not borrowed ones. Rows without a contract (every donor target + * today) are unaffected. + */ +export function assertBackfillKeepsValidatedCache( + metrics: Record, + backfill: BenchmarkPointBackfill, +): void { + const touched = [ + ...Object.keys(backfill.set.metricsMerge ?? {}), + ...(backfill.set.metricsRemove ?? []), + ].filter((key) => VALIDATED_CACHE_KEYS.has(key)); + if (touched.length === 0) return; + if (!CACHE_VALIDITY_TRACE_KEYS.some((key) => Object.hasOwn(metrics, key))) return; + throw new Error( + `${backfill.id}: refusing to rewrite ${touched.join(', ')} on a row whose cache validity the producer vouched for`, + ); +} + function metricsPatch(backfill: BenchmarkPointBackfill): Record { return { ...backfill.set.metricsMerge, @@ -74,6 +109,8 @@ export function planBenchmarkPointBackfill( if (!matchesExpectedBackfillMetrics(row.metrics, backfill)) { throw new Error(`${backfill.id}: benchmark source metrics differ from the audited measurement`); } + const currentMetrics = asMetricsRecord(row.metrics); + if (currentMetrics) assertBackfillKeepsValidatedCache(currentMetrics, backfill); if (isApplied(row, backfill)) return null; const desiredOffloadMode = backfill.set.offloadMode ?? backfill.offloadMode; From e4054ea283f7fa8540f391424b0303be83175f8a Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Thu, 22 Oct 2026 22:06:30 +0000 Subject: [PATCH 2/3] fix(calculator): retain agentic token totals in compact rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preserve recorded prompt and generation totals so prefill/decode rows keep their structural token mix after projection. Add raw/compact parity coverage and synchronize the public API contract notes. 中文:在计算器精简数据中保留 agentic token 总量,使 prefill/decode 数据投影后仍可确定输入输出占比。补充原始与精简数据一致性的测试,并同步公开 API 契约说明。 --- .../calculator/useThroughputData.test.ts | 39 +++++++++++++++++++ packages/app/src/lib/api/api-documentation.ts | 4 +- packages/app/src/lib/api/api-route-catalog.ts | 8 ++++ .../src/lib/benchmarks/benchmark-api-view.ts | 2 + .../skills/inferencex-api/integrity.json | 2 +- .../references/public-api-examples.md | 3 ++ 6 files changed, 55 insertions(+), 3 deletions(-) diff --git a/packages/app/src/components/calculator/useThroughputData.test.ts b/packages/app/src/components/calculator/useThroughputData.test.ts index af5d6e0f6..c3141e09f 100644 --- a/packages/app/src/components/calculator/useThroughputData.test.ts +++ b/packages/app/src/components/calculator/useThroughputData.test.ts @@ -7,6 +7,7 @@ import type { BenchmarkRow } from '@/lib/api/api'; import { Percentile, Sequence } from '@/lib/catalog/data-mappings'; import { overlayRunIndex } from '@/lib/charts/overlay-run-style'; import { SUPPLEMENTAL_BENCHMARK_ROWS } from '@/lib/benchmarks/supplemental-benchmarks'; +import { toCalculatorBenchmarkRows } from '@/lib/benchmarks/benchmark-api-view'; import type { GPUDataPoint } from './types'; import { @@ -1396,6 +1397,44 @@ describe('buildGpuGroups', () => { // tokens instead of inventing an 80% input mix. expect(Object.values(grouped)[0][0].inputTokenShare).toBeUndefined(); }); + + it.each([true, false])( + 'preserves the agentic P/D token mix through the calculator projection (totals=%s)', + (hasTotals) => { + const row = makeRow({ + disagg: true, + is_multinode: true, + prefill_tp: 4, + prefill_ep: 1, + decode_tp: 16, + decode_ep: 1, + num_prefill_gpu: 4, + num_decode_gpu: 16, + benchmark_type: 'agentic_traces', + isl: null, + osl: null, + metrics: { + p90_itl: 1 / 50, + tput_per_gpu: 300, + input_tput_per_gpu: 750, + output_tput_per_gpu: 187.5, + ...(hasTotals ? { total_prompt_tokens: 1000, total_generation_tokens: 1000 } : {}), + }, + }); + const options = { + sequence: Sequence.AgenticTraces, + precisions: ['fp4'], + classify: singlePrecisionClassify, + }; + const raw = Object.values(buildGpuGroups([row], options).grouped)[0][0]; + const compact = Object.values( + buildGpuGroups(toCalculatorBenchmarkRows([row], 'agentic-traces'), options).grouped, + )[0][0]; + expect(compact.inputTokenShare).toBe(raw.inputTokenShare); + if (hasTotals) expect(compact.inputTokenShare).toBeCloseTo(0.5, 9); + else expect(compact.inputTokenShare).toBeUndefined(); + }, + ); }); it('applies Internal TPUv7 owner cost to every calculator token basis', () => { diff --git a/packages/app/src/lib/api/api-documentation.ts b/packages/app/src/lib/api/api-documentation.ts index 2e3442e50..5a87c2d39 100644 --- a/packages/app/src/lib/api/api-documentation.ts +++ b/packages/app/src/lib/api/api-documentation.ts @@ -1017,8 +1017,8 @@ export const apiOperations: readonly ApiOperation[] = [ path: '/api/v1/benchmarks', summary: text('Read benchmark results', '读取基准结果'), description: text( - 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId for that run’s logical snapshot. An AgentX curve is scoped by model, hardware, framework, precision, and workload: a normal run replaces all prior topology, speculative-decoding, and offload variants in that curve. Explicit append-only runs may include the preceding same-image snapshot; producer IDs and URLs are preserved. Historical snapshots remain accessible. view=calculator returns a trimmed page-owned projection (measured power metrics and workers are removed; its allowlist may change). powerValid=strictV2 selects validated schema-v2 power measurements and cannot be combined with view=calculator. Omit powerValid to keep general benchmark results regardless of power validity.', - '返回指定展示模型的原始基准测试数据行。使用 date 可获取截至指定日期的快照;exact=true 仅返回该日期的数据;runId 用于限定最新结果的查询范围;将 exactRun=true 与数值型 runId 搭配使用,可获取该次运行对应的逻辑快照。AgentX 按模型、硬件、框架、精度和工作负载划分曲线;常规运行会整体替换同一曲线中此前的所有拓扑、推测解码和 offload 变体。显式标记为 append-only 的运行可继承此前使用相同镜像的快照,并保留各数据点来源运行的 ID 和 URL。历史快照仍可查询。view=calculator 返回页面专用的裁剪投影(会移除实测功率指标和 workers,其允许列表可能变化)。powerValid=strictV2 仅返回采用 schema v2 且功率测量通过验证的数据行,不能与 view=calculator 组合使用。省略 powerValid 则保留常规基准测试结果,不按功率有效性筛选。', + 'Returns raw benchmark rows for a display model. Use date for an as-of snapshot, exact=true for that exact date, runId to constrain the latest lookup, or exactRun=true with a numeric runId for that run’s logical snapshot. An AgentX curve is scoped by model, hardware, framework, precision, and workload: a normal run replaces all prior topology, speculative-decoding, and offload variants in that curve. Explicit append-only runs may include the preceding same-image snapshot; producer IDs and URLs are preserved. Historical snapshots remain accessible. view=calculator returns a trimmed page-owned projection (measured power metrics and workers are removed; its allowlist may change). It preserves recorded total_prompt_tokens and total_generation_tokens for the agentic P/D token mix; missing totals stay missing. powerValid=strictV2 selects validated schema-v2 power measurements and cannot be combined with view=calculator. Omit powerValid to keep general benchmark results regardless of power validity.', + '返回指定展示模型的原始基准测试数据行。使用 date 可获取截至指定日期的快照;exact=true 仅返回该日期的数据;runId 用于限定最新结果的查询范围;将 exactRun=true 与数值型 runId 搭配使用,可获取该次运行对应的逻辑快照。AgentX 按模型、硬件、框架、精度和工作负载划分曲线;常规运行会整体替换同一曲线中此前的所有拓扑、推测解码和 offload 变体。显式标记为 append-only 的运行可继承此前使用相同镜像的快照,并保留各数据点来源运行的 ID 和 URL。历史快照仍可查询。view=calculator 返回页面专用的裁剪投影(会移除实测功率指标和 workers,其允许列表可能变化)。若数据中有 total_prompt_tokens 和 total_generation_tokens,投影会保留这两个字段,用于计算智能体 P/D 工作负载的输入输出 token 比例;缺失时不补值。powerValid=strictV2 仅返回采用 schema v2 且功率测量通过验证的数据行,不能与 view=calculator 组合使用。省略 powerValid 则保留常规基准测试结果,不按功率有效性筛选。', ), audience: 'public', stability: 'stable', diff --git a/packages/app/src/lib/api/api-route-catalog.ts b/packages/app/src/lib/api/api-route-catalog.ts index 6687a7d5f..31ee207ce 100644 --- a/packages/app/src/lib/api/api-route-catalog.ts +++ b/packages/app/src/lib/api/api-route-catalog.ts @@ -1150,6 +1150,14 @@ export interface ApiContractSourceDigest { * touching a route module. Digest changes require an explicit documentation review. */ export const apiContractSourceDigests = [ + { + source: 'src/lib/benchmarks/benchmark-api-view.ts', + sourceSha256: '82197a032d055fdc572f95d9fc680649342223c4ed57f01f0739846ac8de3aeb', + reviewArea: { + en: 'Calculator metric projection, including recorded prompt/generation totals for the agentic P/D token mix.', + zh: '计算器指标投影,包括用于智能体 P/D 输入输出 token 比例的 prompt/generation 总量。', + }, + }, { source: 'src/app/api/gpu-metrics/artifact-selection.ts', sourceSha256: 'ab1efa13ee06e467a73b3a89946a8e58968c9f3dee1eea533b33fe4770aff08e', diff --git a/packages/app/src/lib/benchmarks/benchmark-api-view.ts b/packages/app/src/lib/benchmarks/benchmark-api-view.ts index e16b2752c..1a7e8423b 100644 --- a/packages/app/src/lib/benchmarks/benchmark-api-view.ts +++ b/packages/app/src/lib/benchmarks/benchmark-api-view.ts @@ -4,6 +4,8 @@ const CALCULATOR_METRIC_KEYS = new Set([ 'tput_per_gpu', 'input_tput_per_gpu', 'output_tput_per_gpu', + 'total_prompt_tokens', + 'total_generation_tokens', 'prefill_pp', 'decode_pp', // Cached-input billing in the Fleet Lifecycle section reads these. Only diff --git a/packages/skills/skills/inferencex-api/integrity.json b/packages/skills/skills/inferencex-api/integrity.json index a8e1efd04..25f37cd85 100644 --- a/packages/skills/skills/inferencex-api/integrity.json +++ b/packages/skills/skills/inferencex-api/integrity.json @@ -13,7 +13,7 @@ "references/pareto.md": "1b4d2d163f982e3f2d97addce310789eae9c1e5badd82dd7501708c9e0385027", "references/powerx.md": "769d793727ca0e89e301d95dd8b73d6b0592b703d5e6c0e1f808b4f0b73ba5cf", "references/provenance.md": "25b8771ab64b490c8d6523c4370313bd2663ae461250d30c6a1c367cbe4f9041", - "references/public-api-examples.md": "f263df8b7568aff0a3b5c7ac98c196a0aebde0c52791c97ae1c2917388f86eb5", + "references/public-api-examples.md": "2977f9f66b05bd10cebc21c5ce4609085aaf6b148d5cd77722d05465183f4b21", "references/releases.md": "b858c44360dc9fc394b11c7318245b167bc6d0f0f95e46c68f6990eaf90496d8", "references/tco.md": "78ba1cfec0f2e040a114674af4380c5c2b8a92b24c776ae1fea8c51a904297e0", "schemas.json": "623ee414f39aede8e81faf5981f2a90c49ad75e918923ad3e64c991da2d9c51a", diff --git a/packages/skills/skills/inferencex-api/references/public-api-examples.md b/packages/skills/skills/inferencex-api/references/public-api-examples.md index 73b0d6b20..6d72d6600 100644 --- a/packages/skills/skills/inferencex-api/references/public-api-examples.md +++ b/packages/skills/skills/inferencex-api/references/public-api-examples.md @@ -271,6 +271,9 @@ The history operation accepts `model`, `isl`, and `osl`. It has no documented GP date-range or `powerValid` filter. This recipe filters the received array locally by the exact `hardware` key and inclusive **original observation `date`**, not an as-of cutoff or `curve_date`. Omit `view=calculator` to retain all stored metrics. +The compact calculator view preserves recorded `total_prompt_tokens` and +`total_generation_tokens` for an agentic P/D token mix. Missing totals stay +missing; do not infer the fleet-wide mix from rates per prefill/decode GPU. Edit `scope` to match the user's request; discover raw hardware keys from the API. ```bash From 12f7c15dcb5ce99c562fb86e3cdf4f1c1c01165f Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Fri, 23 Oct 2026 00:28:18 +0000 Subject: [PATCH 3/3] test(db): align cache closeout fixture with gzip timeline input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the current PreparedTraceReplay requestTimelineGz field and a real gzip payload so the existing closeout test passes type checking. 中文:将 cache closeout 测试数据改为当前 PreparedTraceReplay 的 requestTimelineGz 字段,并使用真实 gzip 内容,修复既有类型检查错误。 --- packages/db/src/etl/cache-validity-closeout.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/db/src/etl/cache-validity-closeout.test.ts b/packages/db/src/etl/cache-validity-closeout.test.ts index 011694c48..92acb8ef5 100644 --- a/packages/db/src/etl/cache-validity-closeout.test.ts +++ b/packages/db/src/etl/cache-validity-closeout.test.ts @@ -1,3 +1,5 @@ +import { gzipSync } from 'node:zlib'; + import { describe, expect, it } from 'vitest'; import { @@ -121,7 +123,7 @@ function preparedWithRates(): PreparedTraceReplay { serverMetricsJsonSize: 700, aggregateStatsJson: Buffer.from('{"version":1}'), chartSeriesJson: Buffer.from('{"version":12}'), - requestTimelineJson: Buffer.from('{"version":1}'), + requestTimelineGz: gzipSync(Buffer.from('{"version":1}')), chartWindows: 2, timelineRequests: 3, compressionMs: 10,