From 811fdd423eb6c36b8f6d9b7a5794ed360ea8b675 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:12:53 -0400 Subject: [PATCH 1/5] feat: verify immutable measurement receipts before publication MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bind receipt-required imports to trusted issuer revisions, exact artifact IDs, archive digests, complete point contracts, and immutable accepted snapshots. Share strict eval sample projection and verify later publication records. 在发布前验证不可变测量凭据，将导入绑定到受信任的签发版本、精确制品 ID、 归档摘要及完整测量点契约。统一严格评测样本投影，支持同一快照恢复， 并验证后续发布记录。 Validation: workspace unit suites, DB tests, typecheck, lint, and all 128 Cypress integration tests plus component specs pass. 验证：工作区单元测试、数据库测试、类型检查、代码检查及全部 128 项 Cypress 集成测试和组件测试通过。 --- .github/workflows/ingest-agentic-results.yml | 146 ++++++ .github/workflows/ingest-results.yml | 35 ++ .github/workflows/stage-results.yml | 102 +++- bun.lock | 2 +- docs/index.md | 1 + docs/measurement-publication.md | 21 + docs/measurement-publication_zh.md | 21 + packages/app/src/lib/eval-samples-live.ts | 5 +- .../migrations/016_measurement_snapshots.sql | 13 + packages/db/package.json | 2 +- packages/db/src/etl/eval-mapper.test.ts | 42 ++ packages/db/src/etl/eval-mapper.ts | 38 ++ packages/db/src/etl/eval-samples-ingest.ts | 6 +- .../db/src/etl/eval-samples-mapper.test.ts | Bin 5789 -> 7259 bytes packages/db/src/etl/eval-samples-mapper.ts | Bin 6686 -> 7980 bytes .../src/etl/measurement-publication.test.ts | 120 +++++ .../db/src/etl/measurement-publication.ts | 95 ++++ .../db/src/etl/measurement-snapshot.test.ts | 57 +++ packages/db/src/etl/measurement-snapshot.ts | 44 ++ packages/db/src/ingest-ci-run.ts | 98 ++-- packages/db/src/lib/artifact-archive.test.ts | 42 ++ packages/db/src/lib/artifact-archive.ts | 97 ++++ .../lib/fixtures/measurement-receipt/101.zip | Bin 0 -> 1055 bytes .../lib/fixtures/measurement-receipt/102.zip | Bin 0 -> 1585 bytes .../measurement-receipt/bmk_pilot/agg.json | 1 + .../bmk_pilot/execution.json | 1 + .../eval_results_all/agg.json | 1 + .../eval_results_all/execution.json | 1 + .../eval_results_all/samples_gsm8k.jsonl | 4 + .../fixtures/measurement-receipt/receipt.json | 154 ++++++ .../measurement-receipt/receipt.sha256 | 1 + packages/db/src/lib/github-artifacts.ts | 93 ++-- .../db/src/lib/measurement-receipt.test.ts | 150 ++++++ packages/db/src/lib/measurement-receipt.ts | 478 ++++++++++++++++++ .../lib/receipt-artifact-preparation.test.ts | 101 ++++ .../src/lib/receipt-artifact-preparation.ts | 45 ++ packages/db/src/lib/receipt-transport.test.ts | 98 ++++ packages/db/src/lib/receipt-transport.ts | 126 +++++ packages/db/src/prepare-ci-artifacts.ts | 62 +++ packages/db/src/prepare-receipt-transport.ts | 11 + .../db/src/verify-measurement-publication.ts | 92 ++++ 41 files changed, 2336 insertions(+), 70 deletions(-) create mode 100644 docs/measurement-publication.md create mode 100644 docs/measurement-publication_zh.md create mode 100644 packages/db/migrations/016_measurement_snapshots.sql create mode 100644 packages/db/src/etl/measurement-publication.test.ts create mode 100644 packages/db/src/etl/measurement-publication.ts create mode 100644 packages/db/src/etl/measurement-snapshot.test.ts create mode 100644 packages/db/src/etl/measurement-snapshot.ts create mode 100644 packages/db/src/lib/artifact-archive.test.ts create mode 100644 packages/db/src/lib/artifact-archive.ts create mode 100644 packages/db/src/lib/fixtures/measurement-receipt/101.zip create mode 100644 packages/db/src/lib/fixtures/measurement-receipt/102.zip create mode 100644 packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/agg.json create mode 100644 packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/execution.json create mode 100644 packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/agg.json create mode 100644 packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/execution.json create mode 100644 packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/samples_gsm8k.jsonl create mode 100644 packages/db/src/lib/fixtures/measurement-receipt/receipt.json create mode 100644 packages/db/src/lib/fixtures/measurement-receipt/receipt.sha256 create mode 100644 packages/db/src/lib/measurement-receipt.test.ts create mode 100644 packages/db/src/lib/measurement-receipt.ts create mode 100644 packages/db/src/lib/receipt-artifact-preparation.test.ts create mode 100644 packages/db/src/lib/receipt-artifact-preparation.ts create mode 100644 packages/db/src/lib/receipt-transport.test.ts create mode 100644 packages/db/src/lib/receipt-transport.ts create mode 100644 packages/db/src/prepare-receipt-transport.ts create mode 100644 packages/db/src/verify-measurement-publication.ts diff --git a/.github/workflows/ingest-agentic-results.yml b/.github/workflows/ingest-agentic-results.yml index b81cd68b0..bf81e86f5 100644 --- a/.github/workflows/ingest-agentic-results.yml +++ b/.github/workflows/ingest-agentic-results.yml @@ -51,6 +51,61 @@ on: required: false default: '' type: string + receipt-required: + description: Immutable publication transport receipt-required + required: false + type: string + default: 'false' + receipt-artifact-id: + description: Immutable publication transport receipt-artifact-id + required: false + type: string + default: '' + receipt-artifact-sha256: + description: Immutable publication transport receipt-artifact-sha256 + required: false + type: string + default: '' + receipt-sha256: + description: Immutable publication transport receipt-sha256 + required: false + type: string + default: '' + receipt-issuer-run-id: + description: Immutable publication transport receipt-issuer-run-id + required: false + type: string + default: '' + receipt-issuer-sha: + description: Immutable publication transport receipt-issuer-sha + required: false + type: string + default: '' + publication-artifact-id: + description: Immutable publication transport publication-artifact-id + required: false + type: string + default: '' + publication-artifact-sha256: + description: Immutable publication transport publication-artifact-sha256 + required: false + type: string + default: '' + publication-sha256: + description: Immutable publication transport publication-sha256 + required: false + type: string + default: '' + publication-issuer-run-id: + description: Immutable publication transport publication-issuer-run-id + required: false + type: string + default: '' + publication-issuer-sha: + description: Immutable publication transport publication-issuer-sha + required: false + type: string + default: '' secrets: DATABASE_WRITE_URL: description: Production database write connection for direct ingests @@ -111,6 +166,62 @@ on: default: '' type: string + receipt-required: + description: Immutable publication transport receipt-required + required: false + type: string + default: 'false' + receipt-artifact-id: + description: Immutable publication transport receipt-artifact-id + required: false + type: string + default: '' + receipt-artifact-sha256: + description: Immutable publication transport receipt-artifact-sha256 + required: false + type: string + default: '' + receipt-sha256: + description: Immutable publication transport receipt-sha256 + required: false + type: string + default: '' + receipt-issuer-run-id: + description: Immutable publication transport receipt-issuer-run-id + required: false + type: string + default: '' + receipt-issuer-sha: + description: Immutable publication transport receipt-issuer-sha + required: false + type: string + default: '' + publication-artifact-id: + description: Immutable publication transport publication-artifact-id + required: false + type: string + default: '' + publication-artifact-sha256: + description: Immutable publication transport publication-artifact-sha256 + required: false + type: string + default: '' + publication-sha256: + description: Immutable publication transport publication-sha256 + required: false + type: string + default: '' + publication-issuer-run-id: + description: Immutable publication transport publication-issuer-run-id + required: false + type: string + default: '' + publication-issuer-sha: + description: Immutable publication transport publication-issuer-sha + required: false + type: string + default: '' + permissions: {} concurrency: @@ -266,6 +377,27 @@ jobs: echo "Selected ingest target: $REQUESTED_DATABASE_TARGET" echo "Cache invalidate URL: $cache_invalidate_url" + - name: Verify immutable receipt transport + env: + GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }} + INGEST_REPO: SemiAnalysisAI/InferenceX + SOURCE_RUN_ID: ${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id || inputs.run-id }} + RECEIPT_TRANSPORT_PATH: ${{ github.workspace }}/receipt-transport + ALLOWED_RECEIPT_ISSUER_SHAS: ${{ vars.INFX_RECEIPT_ISSUER_SHAS }} + ALLOWED_RECEIPT_ISSUER_WORKFLOW: ${{ vars.INFX_RECEIPT_ISSUER_WORKFLOW }} + RECEIPT_REQUIRED: ${{ github.event.client_payload.receipt-required || inputs.receipt-required || 'false' }} + RECEIPT_ARTIFACT_ID: ${{ github.event.client_payload.receipt-artifact-id || inputs.receipt-artifact-id }} + RECEIPT_ARTIFACT_SHA256: ${{ github.event.client_payload.receipt-artifact-sha256 || inputs.receipt-artifact-sha256 }} + RECEIPT_SHA256: ${{ github.event.client_payload.receipt-sha256 || inputs.receipt-sha256 }} + RECEIPT_ISSUER_RUN_ID: ${{ github.event.client_payload.receipt-issuer-run-id || inputs.receipt-issuer-run-id }} + RECEIPT_ISSUER_SHA: ${{ github.event.client_payload.receipt-issuer-sha || inputs.receipt-issuer-sha }} + PUBLICATION_ARTIFACT_ID: ${{ github.event.client_payload.publication-artifact-id || inputs.publication-artifact-id }} + PUBLICATION_ARTIFACT_SHA256: ${{ github.event.client_payload.publication-artifact-sha256 || inputs.publication-artifact-sha256 }} + PUBLICATION_SHA256: ${{ github.event.client_payload.publication-sha256 || inputs.publication-sha256 }} + PUBLICATION_ISSUER_RUN_ID: ${{ github.event.client_payload.publication-issuer-run-id || inputs.publication-issuer-run-id }} + PUBLICATION_ISSUER_SHA: ${{ github.event.client_payload.publication-issuer-sha || inputs.publication-issuer-sha }} + run: bun packages/db/src/prepare-receipt-transport.ts + - name: Prepare artifacts from InferenceX id: artifacts env: @@ -317,6 +449,20 @@ jobs: - name: Verify PowerX source, database and public API run: bun packages/db/src/verify-power-publication.ts power-publication.json "${CACHE_INVALIDATE_URL%/api/v1/invalidate}" + - name: Verify accepted measurement publication + if: env.INGEST_RECEIPT_REQUIRED == '1' + env: + INGEST_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts + run: bun packages/db/src/verify-measurement-publication.ts "${CACHE_INVALIDATE_URL%/api/v1/invalidate}" measurement-publication.json + + - name: Retain measurement publication verification + if: always() && env.INGEST_RECEIPT_REQUIRED == '1' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: measurement-publication-${{ steps.artifacts.outputs.source-run-id }} + path: measurement-publication.json + if-no-files-found: warn + - name: Retain PowerX publication receipt if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/.github/workflows/ingest-results.yml b/.github/workflows/ingest-results.yml index 04b253e34..4c9c2ac40 100644 --- a/.github/workflows/ingest-results.yml +++ b/.github/workflows/ingest-results.yml @@ -50,6 +50,27 @@ jobs: env: CYPRESS_INSTALL_BINARY: '0' + - name: Verify immutable receipt transport + env: + GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }} + INGEST_REPO: SemiAnalysisAI/InferenceX + SOURCE_RUN_ID: ${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id }} + RECEIPT_TRANSPORT_PATH: ${{ github.workspace }}/receipt-transport + ALLOWED_RECEIPT_ISSUER_SHAS: ${{ vars.INFX_RECEIPT_ISSUER_SHAS }} + ALLOWED_RECEIPT_ISSUER_WORKFLOW: ${{ vars.INFX_RECEIPT_ISSUER_WORKFLOW }} + RECEIPT_REQUIRED: ${{ github.event.client_payload.receipt-required || 'false' }} + RECEIPT_ARTIFACT_ID: ${{ github.event.client_payload.receipt-artifact-id }} + RECEIPT_ARTIFACT_SHA256: ${{ github.event.client_payload.receipt-artifact-sha256 }} + RECEIPT_SHA256: ${{ github.event.client_payload.receipt-sha256 }} + RECEIPT_ISSUER_RUN_ID: ${{ github.event.client_payload.receipt-issuer-run-id }} + RECEIPT_ISSUER_SHA: ${{ github.event.client_payload.receipt-issuer-sha }} + PUBLICATION_ARTIFACT_ID: ${{ github.event.client_payload.publication-artifact-id }} + PUBLICATION_ARTIFACT_SHA256: ${{ github.event.client_payload.publication-artifact-sha256 }} + PUBLICATION_SHA256: ${{ github.event.client_payload.publication-sha256 }} + PUBLICATION_ISSUER_RUN_ID: ${{ github.event.client_payload.publication-issuer-run-id }} + PUBLICATION_ISSUER_SHA: ${{ github.event.client_payload.publication-issuer-sha }} + run: bun packages/db/src/prepare-receipt-transport.ts + - name: Prepare artifacts from InferenceX id: artifacts env: @@ -102,6 +123,20 @@ jobs: DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }} run: bun packages/db/src/verify-power-publication.ts power-publication.json + - name: Verify accepted measurement publication + if: env.INGEST_RECEIPT_REQUIRED == '1' + env: + INGEST_ARTIFACTS_PATH: ${{ github.workspace }}/artifacts + run: bun packages/db/src/verify-measurement-publication.ts "https://inferencex.semianalysis.com" measurement-publication.json + + - name: Retain measurement publication verification + if: always() && env.INGEST_RECEIPT_REQUIRED == '1' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: measurement-publication-${{ steps.artifacts.outputs.source-run-id }} + path: measurement-publication.json + if-no-files-found: warn + - name: Retain PowerX publication receipt if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/.github/workflows/stage-results.yml b/.github/workflows/stage-results.yml index 7079abe4d..88356bed3 100644 --- a/.github/workflows/stage-results.yml +++ b/.github/workflows/stage-results.yml @@ -38,6 +38,62 @@ on: default: false type: boolean + receipt-required: + description: Immutable publication transport receipt-required + required: false + type: string + default: 'false' + receipt-artifact-id: + description: Immutable publication transport receipt-artifact-id + required: false + type: string + default: '' + receipt-artifact-sha256: + description: Immutable publication transport receipt-artifact-sha256 + required: false + type: string + default: '' + receipt-sha256: + description: Immutable publication transport receipt-sha256 + required: false + type: string + default: '' + receipt-issuer-run-id: + description: Immutable publication transport receipt-issuer-run-id + required: false + type: string + default: '' + receipt-issuer-sha: + description: Immutable publication transport receipt-issuer-sha + required: false + type: string + default: '' + publication-artifact-id: + description: Immutable publication transport publication-artifact-id + required: false + type: string + default: '' + publication-artifact-sha256: + description: Immutable publication transport publication-artifact-sha256 + required: false + type: string + default: '' + publication-sha256: + description: Immutable publication transport publication-sha256 + required: false + type: string + default: '' + publication-issuer-run-id: + description: Immutable publication transport publication-issuer-run-id + required: false + type: string + default: '' + publication-issuer-sha: + description: Immutable publication transport publication-issuer-sha + required: false + type: string + default: '' + permissions: {} concurrency: @@ -47,6 +103,8 @@ concurrency: jobs: validate: name: Validate staging request + permissions: + contents: read runs-on: ubuntu-latest outputs: run-id: ${{ steps.request.outputs.run-id }} @@ -112,6 +170,37 @@ jobs: echo "reset-staging-database=$RESET_STAGING_DATABASE" } >> "$GITHUB_OUTPUT" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: package.json + - name: Install receipt reader + run: bun install --frozen-lockfile + env: + CYPRESS_INSTALL_BINARY: '0' + - name: Verify immutable receipt transport + env: + GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }} + INGEST_REPO: SemiAnalysisAI/InferenceX + SOURCE_RUN_ID: ${{ steps.request.outputs.run-id }} + RECEIPT_TRANSPORT_PATH: ${{ github.workspace }}/receipt-transport + ALLOWED_RECEIPT_ISSUER_SHAS: ${{ vars.INFX_RECEIPT_ISSUER_SHAS }} + ALLOWED_RECEIPT_ISSUER_WORKFLOW: ${{ vars.INFX_RECEIPT_ISSUER_WORKFLOW }} + RECEIPT_REQUIRED: ${{ github.event.client_payload.receipt-required || inputs.receipt-required || 'false' }} + RECEIPT_ARTIFACT_ID: ${{ github.event.client_payload.receipt-artifact-id || inputs.receipt-artifact-id }} + RECEIPT_ARTIFACT_SHA256: ${{ github.event.client_payload.receipt-artifact-sha256 || inputs.receipt-artifact-sha256 }} + RECEIPT_SHA256: ${{ github.event.client_payload.receipt-sha256 || inputs.receipt-sha256 }} + RECEIPT_ISSUER_RUN_ID: ${{ github.event.client_payload.receipt-issuer-run-id || inputs.receipt-issuer-run-id }} + RECEIPT_ISSUER_SHA: ${{ github.event.client_payload.receipt-issuer-sha || inputs.receipt-issuer-sha }} + PUBLICATION_ARTIFACT_ID: ${{ github.event.client_payload.publication-artifact-id || inputs.publication-artifact-id }} + PUBLICATION_ARTIFACT_SHA256: ${{ github.event.client_payload.publication-artifact-sha256 || inputs.publication-artifact-sha256 }} + PUBLICATION_SHA256: ${{ github.event.client_payload.publication-sha256 || inputs.publication-sha256 }} + PUBLICATION_ISSUER_RUN_ID: ${{ github.event.client_payload.publication-issuer-run-id || inputs.publication-issuer-run-id }} + PUBLICATION_ISSUER_SHA: ${{ github.event.client_payload.publication-issuer-sha || inputs.publication-issuer-sha }} + run: bun packages/db/src/prepare-receipt-transport.ts + - name: Report staging start env: RUN_ID: ${{ steps.request.outputs.run-id }} @@ -254,11 +343,22 @@ jobs: needs: [validate, prepare-staging-database] permissions: contents: read - uses: $/.github/workflows/ingest-agentic-results.yml + uses: ./.github/workflows/ingest-agentic-results.yml with: run-id: ${{ needs.validate.outputs.run-id }} run-attempt: ${{ needs.validate.outputs.run-attempt }} database-target: staging + receipt-required: ${{ github.event.client_payload.receipt-required || inputs.receipt-required || 'false' }} + receipt-artifact-id: ${{ github.event.client_payload.receipt-artifact-id || inputs.receipt-artifact-id }} + receipt-artifact-sha256: ${{ github.event.client_payload.receipt-artifact-sha256 || inputs.receipt-artifact-sha256 }} + receipt-sha256: ${{ github.event.client_payload.receipt-sha256 || inputs.receipt-sha256 }} + receipt-issuer-run-id: ${{ github.event.client_payload.receipt-issuer-run-id || inputs.receipt-issuer-run-id }} + receipt-issuer-sha: ${{ github.event.client_payload.receipt-issuer-sha || inputs.receipt-issuer-sha }} + publication-artifact-id: ${{ github.event.client_payload.publication-artifact-id || inputs.publication-artifact-id }} + publication-artifact-sha256: ${{ github.event.client_payload.publication-artifact-sha256 || inputs.publication-artifact-sha256 }} + publication-sha256: ${{ github.event.client_payload.publication-sha256 || inputs.publication-sha256 }} + publication-issuer-run-id: ${{ github.event.client_payload.publication-issuer-run-id || inputs.publication-issuer-run-id }} + publication-issuer-sha: ${{ github.event.client_payload.publication-issuer-sha || inputs.publication-issuer-sha }} secrets: DATABASE_STAGING_WRITE_URL: ${{ secrets.DATABASE_STAGING_WRITE_URL }} VERCEL_STAGING_BYPASS_SECRET: ${{ secrets.VERCEL_STAGING_BYPASS_SECRET }} diff --git a/bun.lock b/bun.lock index 21d318a47..aa8bd6d1b 100644 --- a/bun.lock +++ b/bun.lock @@ -108,6 +108,7 @@ "@neondatabase/serverless": "^1.1.0", "@noble/ciphers": "^2.4.0", "@semianalysisai/inferencex-constants": "workspace:*", + "adm-zip": "^0.6.0", "postgres": "^3.4.9", "stream-chain": "^4.2.5", "stream-json": "^3.6.0", @@ -118,7 +119,6 @@ "@types/node": "^26.4.0", "@types/stream-json": "^1.7.8", "@vitest/coverage-v8": "^4.1.11", - "adm-zip": "^0.6.0", "typescript": "^6.0.3", "vitest": "^4.1.11", }, diff --git a/docs/index.md b/docs/index.md index 73e2c2b43..42807995a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,6 +13,7 @@ Design rationale and non-obvious conventions. See [CLAUDE.md](../CLAUDE.md) for - [Architecture](./architecture.md) — Why client-first, route navigation, URL state, provider nesting, server-side caching (unstable_cache + blob), in-memory client cache, color system, analytics enforcement - [D3 Charts](./d3-charts.md) — Why 4 effects, in-place mutation, refs for zoom, rAF throttling, HTML tooltips, Pareto directions, gradient labels +- [Immutable Measurement Publication](./measurement-publication.md) / [中文](./measurement-publication_zh.md) — Receipt authority, exact artifact transport, immutable replay and run-scoped acceptance - [Data Pipeline](./data-pipeline.md) — DB schema reasoning, ETL design, transform pipeline, spline method choice, normalizer resolution order (model/GPU/framework) - [Pitfalls](./pitfalls.md) — Failure modes: token type consistency, schema evolution, empty objects, zoom loss, stale closures, disaggregated metrics, negative splines, date stamping, ref stability, cost inheritance - [GPU Specs](./gpu-specs.md) — Unit conventions, topology invariants, SVG layout rationale, hardware gotchas diff --git a/docs/measurement-publication.md b/docs/measurement-publication.md new file mode 100644 index 000000000..265c72b29 --- /dev/null +++ b/docs/measurement-publication.md @@ -0,0 +1,21 @@ +# Immutable measurement publication + +[中文](./measurement-publication_zh.md) + +Phase 1 readers accept a version-1 source measurement receipt produced by the independently trusted InferenceX hosted issuer. A later publication record references the receipt and adds merge, changelog, ingest and public-app revisions. The source receipt never changes when staging becomes production. This implementation is a reader prerequisite; local tests do not establish deployed-reader or GPU qualification. + +`prepare-receipt-transport.ts` validates the source artifact inventory independently. Any `native-execution-*` upload makes a receipt mandatory, including failed native executions. The trusted dispatcher also supplies `receipt-required`; omission of native evidence cannot qualify a run. Legacy inventory without the native capability retains its weaker selection behavior, and an already accepted run cannot later bypass its receipt via legacy ingest. + +The app repository must configure `INFX_RECEIPT_ISSUER_SHAS` as a comma-separated allowlist of reviewed issuer revisions, and `INFX_RECEIPT_ISSUER_WORKFLOW` as the exact `.github/workflows/.yml` path. Keep prior accepted revisions while their receipts remain supported. The receiver checks successful completed issuer runs, exact repository/run ownership, API digest, ZIP bytes and member paths. Payloads cannot choose arbitrary download URLs or trusted code. Dispatch publication only after its issuer finishes successfully. + +Dispatch fields are `receipt-required`, `receipt-artifact-id`, `receipt-artifact-sha256`, `receipt-sha256`, `receipt-issuer-run-id`, and `receipt-issuer-sha`. ZIP and JSON digests are distinct. A receipt ZIP contains `receipt.json`. Production/recovery adds `publication-artifact-id`, `publication-artifact-sha256`, `publication-sha256`, `publication-issuer-run-id`, and `publication-issuer-sha`; that later ZIP contains `publication.json`. Both staging and ingest workflows forward these fields. Staging checks transport before any optional database reset. + +The reader uses separately versioned `aiperf-1.4`, `agentx-v1` and publication contract 1. Unsupported required versions fail before import. Every point binds its original execution/attempt, prepared bundle, native manifest, physical topology, canonical model/hardware/framework/precision, required metrics, full dataset identity and exact artifact/member references. Normalized AgentX JSON and per-job raw lm-eval results plus `meta_env.json` are supported. Evaluation requires complete `(task, doc_id, filter)` coverage. Both live preview and stored document-level samples select strict-match independent of line order and reject conflicting copies. Aggregate metadata maps to one physical serving role with compatibility worker counts 0/0. + +Artifacts are downloaded with argv-based `gh` calls and extracted under numeric ID roots. Validation rejects escaping paths, links, duplicate normalized names, file/directory collisions, overwrites and changed member sets. Checked compatibility views retain existing consumer discovery names. All snapshot bytes and normalized requirements are verified before the first database write. + +Apply migration `016_measurement_snapshots.sql` before deploying these readers. The table retains the compact receipt and binds each source repository/run/attempt to one accepted snapshot. An interrupted progressive import remains `writing` and resumes only that same receipt; successful replay remains stable. Replacing measurement bytes requires a separately versioned source execution. Execution rollback does not reverse prior database writes. Preserve a compatible reader for all retained receipt versions. + +After cache refresh, the workflows execute the read-only `packages/db/src/verify-measurement-publication.ts `. It compares exact-run and latest curve metrics/topology with the accepted snapshot, checks eval summary visibility and strict sample counts, and verifies trace-detail availability. It uses `INGEST_ARTIFACTS_PATH` and the receipt environment emitted by transport. Retain its report alongside existing PowerX and database diagnostics. Run-specific verification does not silently imply that every fleet lane is qualified. + +Local regression coverage includes Python/TypeScript receipt interoperability, unsafe archives, omitted/wrong-owner exact IDs, untrusted issuers, rehashed semantic corruption, full sample/filter coverage, actual PGlite partial-import/replay guards and a fresh-process verifier against controlled HTTP APIs. Deployment, the complete H100 eight-point/c28 run, staging and post-merge production verification remain explicit release steps. diff --git a/docs/measurement-publication_zh.md b/docs/measurement-publication_zh.md new file mode 100644 index 000000000..117c8dce1 --- /dev/null +++ b/docs/measurement-publication_zh.md @@ -0,0 +1,21 @@ +# 不可变测量结果发布 + +[English](./measurement-publication.md) + +Phase 1 的读取端接受由 InferenceX 独立受信托管签发流程生成的 version 1 源测量回执。之后生成的发布记录引用该回执，并补充 merge、changelog、ingest 和公开应用的版本。结果从 staging 进入生产时，源回执保持不变。当前实现属于读取端前置条件；本地测试通过不代表读取端已经部署，也不代表 GPU 验收已经完成。 + +`prepare-receipt-transport.ts` 独立检查源运行的产物清单。只要存在 `native-execution-*` 上传项，就必须提供回执；失败的 native 执行也适用。受信调度端同时传递 `receipt-required`。缺少 native 证据的运行不能通过 native 资格验收。不含该能力标识的旧产物继续使用较弱的兼容选择逻辑，但已接受回执的运行不能再通过旧入口绕过回执。 + +应用仓库必须配置 `INFX_RECEIPT_ISSUER_SHAS`，其中以逗号分隔经过审查的签发流程版本；同时将 `INFX_RECEIPT_ISSUER_WORKFLOW` 设置为准确的 `.github/workflows/.yml` 路径。只要旧回执仍受支持，就应保留对应的允许版本。接收端检查签发运行已成功完成、仓库及运行归属准确、API digest、ZIP 字节和成员路径一致。payload 不能选择任意下载 URL 或受信代码。签发流程成功完成后，才能调度发布。 + +调度字段为 `receipt-required`、`receipt-artifact-id`、`receipt-artifact-sha256`、`receipt-sha256`、`receipt-issuer-run-id` 和 `receipt-issuer-sha`。ZIP digest 与 JSON digest 分别校验。回执 ZIP 包含 `receipt.json`。生产发布及恢复还需提供 `publication-artifact-id`、`publication-artifact-sha256`、`publication-sha256`、`publication-issuer-run-id` 和 `publication-issuer-sha`；后续 ZIP 包含 `publication.json`。staging 与 ingest 工作流都会转发这些字段。staging 会先验证传输，再执行可选的数据库重置。 + +读取端分别支持 `aiperf-1.4`、`agentx-v1` 和 publication contract 1。不支持的必需版本会在导入前报错。每个点都绑定其原始执行及 attempt、prepared bundle、native manifest、物理拓扑、规范化后的模型/硬件/framework/precision、必需指标、完整数据集身份，以及准确的产物和文件引用。输入支持规范化 AgentX JSON，也支持每个任务的原始 lm-eval 结果和 `meta_env.json`。评估必须完整覆盖 `(task, doc_id, filter)`。live preview 与数据库中的文档级样本统一选择 strict-match，不受行顺序影响；同一过滤器的冲突副本会被拒绝。聚合部署元数据映射到一个实际 serving role，兼容字段中的 worker 数量为 0/0。 + +产物下载使用 argv 形式调用 `gh`，并在以数字 ID 命名的目录中解压。校验会拒绝越界路径、链接、规范化后重名的成员、文件与目录冲突、覆盖写入以及成员集合变化。经过验证的兼容视图保留已有消费端的发现名称。在第一次数据库写入之前，必须验证完整 snapshot 的文件字节和规范化要求。 + +部署读取端之前，先应用 `016_measurement_snapshots.sql`。该表保留紧凑回执，并将源仓库/run/attempt 绑定到唯一的已接受 snapshot。渐进式导入中断后，状态保持 `writing`，恢复时只能使用同一回执；成功后的重复导入保持结果稳定。替换测量字节需要独立版本的源执行。执行回滚不会撤销既有数据库写入。对保留的回执版本，必须继续提供兼容读取端。 + +缓存刷新后，工作流执行只读校验命令 `packages/db/src/verify-measurement-publication.ts `。它将 exact-run 和最新曲线中的指标与拓扑同已接受的 snapshot 比较，检查评估汇总和 strict 样本计数，并验证 trace 明细可用性。该命令使用 `INGEST_ARTIFACTS_PATH` 及传输步骤生成的回执环境变量。应将其报告与现有 PowerX、数据库诊断一起保留。单次运行通过不代表整个集群覆盖范围均已完成验收。 + +本地回归覆盖 Python/TypeScript 回执互操作、不安全归档、缺失或归属错误的准确 ID、不受信签发流程、重新计算 digest 后仍不合法的结果、完整样本与过滤器覆盖、实际 PGlite 中断恢复/重复导入保护，以及独立进程对受控 HTTP API 的发布校验。部署、H100 全部八个吞吐量点与 c28 评估、staging 和 merge 后的生产验证仍是明确的发布步骤。 diff --git a/packages/app/src/lib/eval-samples-live.ts b/packages/app/src/lib/eval-samples-live.ts index 8b5258893..382a56124 100644 --- a/packages/app/src/lib/eval-samples-live.ts +++ b/packages/app/src/lib/eval-samples-live.ts @@ -17,6 +17,7 @@ import { } from '@/lib/github-artifacts'; import { mapEvalSamples, + projectEvalSamples, type EvalSampleParams, } from '@semianalysisai/inferencex-db/etl/eval-samples-mapper'; import { createSkipTracker } from '@semianalysisai/inferencex-db/etl/skip-tracker'; @@ -136,7 +137,5 @@ export async function fetchAndParseSamples( // (`strict-match`, `flexible-extract`) post-process the same response, and // re-run scenarios can also produce multiple samples files in one zip. The DB // ingest dedups via `(eval_result_id, doc_id)`; mirror that here so totals match. - const seen = new Map(); - for (const s of collected) if (!seen.has(s.docId)) seen.set(s.docId, s); - return [...seen.values()].toSorted((a, b) => a.docId - b.docId); + return projectEvalSamples(collected); } diff --git a/packages/db/migrations/016_measurement_snapshots.sql b/packages/db/migrations/016_measurement_snapshots.sql new file mode 100644 index 000000000..89340e4da --- /dev/null +++ b/packages/db/migrations/016_measurement_snapshots.sql @@ -0,0 +1,13 @@ +-- A receipt is an immutable measurement snapshot. Interrupted imports resume the same bytes. +create table if not exists measurement_snapshots ( + source_repo text not null, + source_run_id bigint not null, + source_attempt integer not null check (source_attempt > 0), + receipt_id text not null check (receipt_id ~ '^[a-f0-9]{64}$'), + bundle_digest text not null check (bundle_digest ~ '^[a-f0-9]{64}$'), + receipt jsonb not null, + state text not null check (state in ('writing', 'complete')), + created_at timestamptz not null default now(), + updated_at timestamptz not null default now(), + primary key (source_repo, source_run_id, source_attempt) +); diff --git a/packages/db/package.json b/packages/db/package.json index c6c05c57b..f813b3d9f 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -41,6 +41,7 @@ "@neondatabase/serverless": "^1.1.0", "@noble/ciphers": "^2.4.0", "@semianalysisai/inferencex-constants": "workspace:*", + "adm-zip": "^0.6.0", "postgres": "^3.4.9", "stream-chain": "^4.2.5", "stream-json": "^3.6.0" @@ -51,7 +52,6 @@ "@types/node": "^26.4.0", "@types/stream-json": "^1.7.8", "@vitest/coverage-v8": "^4.1.11", - "adm-zip": "^0.6.0", "typescript": "^6.0.3", "vitest": "^4.1.11" } diff --git a/packages/db/src/etl/eval-mapper.test.ts b/packages/db/src/etl/eval-mapper.test.ts index 079f3ffd8..44378e685 100644 --- a/packages/db/src/etl/eval-mapper.test.ts +++ b/packages/db/src/etl/eval-mapper.test.ts @@ -598,3 +598,45 @@ describe('mapAggEvalRow', () => { expect(result!.conc).toBeNull(); }); }); + +describe('single-node aggregate deployment contract', () => { + it.each([8, 4])('maps both readers to one physical %i-GPU serving role', (gpus) => { + const deployment = { kind: 'aggregate', nodes: 1, serving_gpus: gpus, tp: gpus, ep: 1 }; + const topology = { + deployment, + disagg: false, + is_multinode: false, + prefill_tp: gpus, + decode_tp: gpus, + prefill_num_workers: 1, + decode_num_workers: 1, + }; + const detail = mapEvalRow(makeMeta(topology), makeResults(), createSkipTracker())[0]; + const aggregate = mapAggEvalRow(makeAggRow(topology), createSkipTracker())!; + for (const row of [detail, aggregate]) { + expect(row.config).toMatchObject({ + disagg: false, + isMultinode: false, + prefillNumWorkers: 0, + decodeNumWorkers: 0, + numPrefillGpu: gpus, + numDecodeGpu: gpus, + prefillTp: gpus, + decodeTp: gpus, + }); + } + expect(configCacheKey(detail.config)).toBe(configCacheKey(aggregate.config)); + }); + it('rejects a contradictory explicit deployment before mapping', () => { + expect(() => + mapEvalRow( + makeMeta({ + deployment: { kind: 'aggregate', nodes: 1, serving_gpus: 8, tp: 8, ep: 1 }, + disagg: true, + }), + makeResults(), + createSkipTracker(), + ), + ).toThrow('Invalid single-node aggregate'); + }); +}); diff --git a/packages/db/src/etl/eval-mapper.ts b/packages/db/src/etl/eval-mapper.ts index 05419dcf4..45bb60ede 100644 --- a/packages/db/src/etl/eval-mapper.ts +++ b/packages/db/src/etl/eval-mapper.ts @@ -243,6 +243,44 @@ function buildEvalConfig( specMethod: string, disaggFromFw: boolean, ): ConfigParams { + // New aggregate producers describe runtime topology explicitly. Split-role fields are + // compatibility columns, and must not turn one serving role into two GPU pools. + if (src.deployment !== undefined) { + const deployment = src.deployment; + if ( + !deployment || + deployment.kind !== 'aggregate' || + deployment.nodes !== 1 || + !Number.isSafeInteger(deployment.serving_gpus) || + deployment.serving_gpus < 1 || + !Number.isSafeInteger(deployment.tp) || + deployment.tp < 1 || + !Number.isSafeInteger(deployment.ep) || + deployment.ep < 1 || + parseOptionalBool(src.disagg) !== false || + parseBool(src.is_multinode) + ) + throw new Error('Invalid single-node aggregate evaluation deployment'); + return { + hardware, + framework, + model, + precision, + specMethod, + disagg: false, + isMultinode: false, + prefillTp: deployment.tp, + decodeTp: deployment.tp, + prefillEp: deployment.ep, + decodeEp: deployment.ep, + prefillDpAttn: parseBool(src.dp_attention), + decodeDpAttn: parseBool(src.dp_attention), + prefillNumWorkers: 0, + decodeNumWorkers: 0, + numPrefillGpu: deployment.serving_gpus, + numDecodeGpu: deployment.serving_gpus, + }; + } const isMultinode = parseBool(src.is_multinode); let prefillTp: number, prefillEp: number, prefillDpAttn: boolean, prefillNumWorkers: number; diff --git a/packages/db/src/etl/eval-samples-ingest.ts b/packages/db/src/etl/eval-samples-ingest.ts index c5c9d23fd..ad192b905 100644 --- a/packages/db/src/etl/eval-samples-ingest.ts +++ b/packages/db/src/etl/eval-samples-ingest.ts @@ -4,7 +4,7 @@ */ import type postgres from 'postgres'; -import type { EvalSampleParams } from './eval-samples-mapper'; +import { projectEvalSamples, type EvalSampleParams } from './eval-samples-mapper'; type Sql = ReturnType; @@ -29,9 +29,7 @@ export async function bulkIngestEvalSamples( if (samples.length === 0) return { newCount: 0 }; // Dedupe within the batch on doc_id to avoid ON CONFLICT collisions in one statement. - const seen = new Map(); - for (const s of samples) seen.set(s.docId, s); - const deduped = [...seen.values()]; + const deduped = projectEvalSamples(samples); let newCount = 0; for (let i = 0; i < deduped.length; i += CHUNK_SIZE) { diff --git a/packages/db/src/etl/eval-samples-mapper.test.ts b/packages/db/src/etl/eval-samples-mapper.test.ts index 548050d9fdbeffa0fffaa01e437cdc9760439ae1..20b7324050c0733566443c6045d236d0c11251f7 100644 GIT binary patch delta 1126 zcmbVLL2DC16s9$3jHsYVT6)k2BD)Kl(B5in3L+vNMQ<_4Wan)QGdP zWHStdgmA%GLNcHtDHg~zD6+^itP7anK@4F8<3l*>05BZ}3NzbEfzpyCwjdD_fp7{E zRW7(fdzceRGgJyh3MtDpO2`;T5Rz`9G#9sfZC^e%8`pa~)teCnRdlc*IynkrJJI~y zcxZmN9&azBr}hev*by4i3{Tn20xjzwH1&0SfTvoLMAtp~?Gq{ZJebDjSL;!y;1p8} zh`MXn9B$rU?Ye0I^_Wyh2&-9)=Y`$Zt}-c~08A>FaRXyo728Y=Aol$D%rq+bTQA@h zK?>uN?7NoJ7avLXrsQ9R=AzxZH5?AB;ZbbJweC*jIRL=Z+!lmQ7ju4Zt9Lm(*`1o# zceYpH1b#(F;J0)*YPl@yPfD6x5FjT?p?#@kf%R5UsXT(cb%YX4MsVik{Z~3z)-N5Q z6W4fY$*b6PE|TaBshpf?W_`r?9B7f-Bhl!as?t_uF=O*W6v{^jf@hx^!zyQ{BxTZ? zVhnS&*gvi)46lafS@ay1?)JiZKK~ZNXTY4d`-5^$Y`3<`_;Y1Fr0*S{Y6O#DZ#S6Q zO-sq%MKurp5Dm3>dnCn47(8FE9{eAq&o{POj#~4*-Q92g0TQ_BS8*|O*#?y>w$3`J TQ35on%YdKNN>p&2nC|-nc{Eh5 delta 10 Rcmca@F;{nj`^M-nF#sA31T6pn diff --git a/packages/db/src/etl/eval-samples-mapper.ts b/packages/db/src/etl/eval-samples-mapper.ts index 82e0a2811a261079a4cf9a7c5b1c2254bbf12213..589cab9db913788b8e71859b90b95e0da38934aa 100644 GIT binary patch delta 1270 zcmZ`(PjAyO6c;;|3ld^n-=wi7RhNOpsnAY{6^R4V#EzxX+~l=2a_nF`-9}aAL+s8c z;DE#zfN#N>Pr|dEWUVGnapLFSpZ9*wx0l~XKmP7LTLU*|h{c7~JhkI5B@?iLxmG!J zrOI$nS}uP6*?af2>YVg?aIV#7OfAg0u&9p(u22B3c!n}X;L<9vO9Z^;#&WrUOr>Rx z($C-tavTUe1^L3*MYsFe`moiwYu`c##$>KU2aFD37C@g z$w8Z1I9!Jo2YZ`-6Q09$khi(41#Wz+8vTa)LhFjINGm5abE|o-XdAr&Hy+XfQou_s zVUi@t);YROnlc^5Q_q=Lb)mGyEQ%&<+K0((8pCh|42~<9#i1r=1#SBowKSnDB&V5$ ziv->7-AVYMq&8gDZf!WsB-F=^((oFbQb9#iYg@voIHH`YC iPk!?op@Dc4ZQpmA)Nr{!hHKEH>E!t|sM(P0P3IriNUc2p delta 12 TcmZ2uH_v3lBc{!Q%r@cxBHILO diff --git a/packages/db/src/etl/measurement-publication.test.ts b/packages/db/src/etl/measurement-publication.test.ts new file mode 100644 index 000000000..cd9aca411 --- /dev/null +++ b/packages/db/src/etl/measurement-publication.test.ts @@ -0,0 +1,120 @@ +import fs from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { createServer } from 'node:http'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import os from 'node:os'; +import path from 'node:path'; +import { expect, it } from 'vitest'; +import { parseMeasurementReceipt } from '../lib/measurement-receipt'; +import { expectedPublication, verifyPublishedMeasurements } from './measurement-publication'; + +const fixture = new URL('../lib/fixtures/measurement-receipt/', import.meta.url); +const hash = '00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6'; +const raw = fs.readFileSync(new URL('receipt.json', fixture)); +const config = { + model: 'dsr1', + hardware: 'h200', + framework: 'vllm', + precision: 'fp8', + disagg: false, + is_multinode: false, + decode_tp: 8, + decode_ep: 1, + num_decode_gpu: 8, + num_prefill_gpu: 8, +}; +it('compares hand-worked metric values and point multiplicity independently of power', () => { + const expected = expectedPublication( + parseMeasurementReceipt(raw, hash, 'd'.repeat(40)), + fileURLToPath(fixture), + ); + expect(expected.map((point) => [point.metrics, point.strictPassed])).toEqual([ + [{ output_tput_tps: 100, duration_seconds: 60 }, null], + [{ em_strict: 0.5, n_eff: 2 }, 1], + ]); + const row = { + ...config, + id: 10, + conc: 1, + metrics: { output_tput_tps: 100, duration_seconds: 60 }, + }; + expect(verifyPublishedMeasurements(expected, [row], 'throughput', 'API')).toEqual([]); + expect( + verifyPublishedMeasurements( + expected, + [{ ...row, metrics: { ...row.metrics, output_tput_tps: 90 } }], + 'throughput', + 'API', + )[0], + ).toContain('output_tput_tps differs (90 vs 100)'); + expect(verifyPublishedMeasurements(expected, [row, row], 'throughput', 'API')[0]).toContain( + 'found 2', + ); + expect(verifyPublishedMeasurements(expected, [], 'eval', 'API')[0]).toContain('found 0'); +}); +it('runs the read-only CLI through exact-run/latest curves and trace/sample detail APIs', async () => { + const requests: string[] = []; + const server = createServer((request, response) => { + requests.push(request.url!); + const url = new URL(request.url!, 'http://localhost'); + const data = url.pathname.endsWith('/benchmarks') + ? [{ ...config, id: 10, conc: 1, metrics: { output_tput_tps: 100, duration_seconds: 60 } }] + : url.pathname.endsWith('/evaluations') + ? [ + { + ...config, + id: 20, + conc: 28, + task: 'gsm8k', + metrics: { em_strict: 0.5, n_eff: 2 }, + run_url: 'https://github.com/org/repo/actions/runs/100', + }, + ] + : url.pathname.endsWith('/trace-availability') + ? { '10': true } + : url.searchParams.get('eval_result_id') === '20' + ? { total: 2, passedTotal: 1, failedTotal: 1, samples: [] } + : {}; + response.setHeader('Content-Type', 'application/json'); + response.end(JSON.stringify(data)); + }); + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'verify-publication-')); + try { + const address = server.address() as { port: number }; + const output = path.join(root, 'verification.json'); + await promisify(execFile)( + 'bun', + [ + fileURLToPath(new URL('../verify-measurement-publication.ts', import.meta.url)), + `http://127.0.0.1:${address.port}`, + output, + ], + { + env: { + ...process.env, + INGEST_RECEIPT_REQUIRED: '1', + INGEST_RECEIPT_PATH: fileURLToPath(new URL('receipt.json', fixture)), + INGEST_RECEIPT_SHA256: hash, + INGEST_RECEIPT_ISSUER_SHA: 'd'.repeat(40), + INGEST_ARTIFACTS_PATH: fileURLToPath(fixture), + }, + }, + ); + expect(JSON.parse(fs.readFileSync(output, 'utf8'))).toMatchObject({ + status: 'matched', + points: 2, + errors: [], + }); + expect(requests.some((request) => request.includes('exactRun=true'))).toBe(true); + expect(requests.some((request) => request.includes('eval_result_id=20'))).toBe(true); + } finally { + await new Promise((resolve) => { + server.close(() => resolve()); + }); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/packages/db/src/etl/measurement-publication.ts b/packages/db/src/etl/measurement-publication.ts new file mode 100644 index 000000000..a80a701fd --- /dev/null +++ b/packages/db/src/etl/measurement-publication.ts @@ -0,0 +1,95 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { + verifyMeasurementSnapshot, + mapReceiptPointRows, + type MeasurementReceipt, + type ReceiptPoint, +} from '../lib/measurement-receipt'; +import { createSkipTracker } from './skip-tracker'; +import { mapEvalSamples, projectEvalSamples } from './eval-samples-mapper'; + +export interface ExpectedPublishedPoint { + point: ReceiptPoint; + metrics: Record; + strictPassed: number | null; +} +export interface PublishedMeasurement extends Record { + id: number; + conc: number; + metrics: Record; + task?: string; +} +const CONFIG_COLUMNS: Record = { + specMethod: 'spec_method', + recipeFingerprint: 'recipe_fingerprint', +}; + +export function expectedPublication( + receipt: MeasurementReceipt, + root: string, +): ExpectedPublishedPoint[] { + verifyMeasurementSnapshot(receipt, root); + const names = new Map(receipt.artifacts.map((artifact) => [artifact.id, artifact.name])); + return receipt.points.map((point) => { + const mapped = mapReceiptPointRows(receipt, root, point).mapped.find( + (row) => + row?.conc === point.concurrency && + (point.kind === 'throughput' || ('task' in row && row.task === point.task)), + )!; + let strictPassed: number | null = null; + if (point.kind === 'eval') { + const text = fs.readFileSync( + path.join(root, names.get(point.samples_artifact_id!)!, point.samples_path!), + 'utf8', + ); + strictPassed = projectEvalSamples(mapEvalSamples(text, createSkipTracker())).filter( + (sample) => sample.passed, + ).length; + } + return { + point, + metrics: Object.fromEntries(point.required_metrics.map((key) => [key, mapped.metrics[key]])), + strictPassed, + }; + }); +} +export function publishedPointMatches(point: ReceiptPoint, row: PublishedMeasurement): boolean { + return ( + row.conc === point.concurrency && + (point.kind === 'throughput' || row.task === point.task) && + Object.entries(point.config).every(([key, value]) => row[CONFIG_COLUMNS[key] ?? key] === value) + ); +} +export function verifyPublishedMeasurements( + expected: readonly ExpectedPublishedPoint[], + actual: readonly PublishedMeasurement[], + kind: ReceiptPoint['kind'], + source: string, +): string[] { + const errors: string[] = []; + for (const item of expected.filter((entry) => entry.point.kind === kind)) { + const rows = actual.filter((row) => publishedPointMatches(item.point, row)); + if (rows.length !== 1) { + errors.push(`${source}: ${item.point.point_id}: expected one point, found ${rows.length}`); + continue; + } + const row = rows[0]; + const topology = item.point.topology; + if ( + row.disagg !== false || + row.is_multinode !== false || + row.decode_tp !== topology.tp || + row.decode_ep !== topology.ep || + row.num_decode_gpu !== topology.serving_gpus || + row.num_prefill_gpu !== topology.serving_gpus + ) + errors.push(`${source}: ${item.point.point_id}: topology mismatch`); + for (const [metric, value] of Object.entries(item.metrics)) + if (row.metrics[metric] !== value) + errors.push( + `${source}: ${item.point.point_id}: ${metric} differs (${row.metrics[metric]} vs ${value})`, + ); + } + return errors; +} diff --git a/packages/db/src/etl/measurement-snapshot.test.ts b/packages/db/src/etl/measurement-snapshot.test.ts new file mode 100644 index 000000000..a71367542 --- /dev/null +++ b/packages/db/src/etl/measurement-snapshot.test.ts @@ -0,0 +1,57 @@ +import { PGlite } from '@electric-sql/pglite'; +import fs from 'node:fs'; +import type postgres from 'postgres'; +import { expect, it } from 'vitest'; +import { + claimMeasurementSnapshot, + completeMeasurementSnapshot, + assertLegacySnapshotUnclaimed, +} from './measurement-snapshot'; +import type { MeasurementReceipt } from '../lib/measurement-receipt'; + +it('resumes partial imports with the same receipt and rejects replacement after completion', async () => { + const db = await PGlite.create(); + try { + await db.exec( + fs.readFileSync( + new URL('../../migrations/016_measurement_snapshots.sql', import.meta.url), + 'utf8', + ), + ); + const sql = (async (strings: TemplateStringsArray, ...values: unknown[]) => { + const query = strings.reduce((text, part, i) => text + (i ? `$${i}` : '') + part, ''); + const result = await db.query(query, values); + return result.rows; + }) as unknown as ReturnType; + const receipt = JSON.parse( + fs.readFileSync( + new URL('../lib/fixtures/measurement-receipt/receipt.json', import.meta.url), + 'utf8', + ), + ) as MeasurementReceipt; + await assertLegacySnapshotUnclaimed(sql, 'org/repo', '100', 2); + await claimMeasurementSnapshot(sql, receipt); + await expect(assertLegacySnapshotUnclaimed(sql, 'org/repo', '100', 2)).rejects.toThrow( + 'previously accepted', + ); + await claimMeasurementSnapshot(sql, receipt); + const partial = await db.query( + 'select source_run_id, source_attempt, state from measurement_snapshots', + ); + expect(partial.rows).toEqual([{ source_run_id: 100, source_attempt: 2, state: 'writing' }]); + await completeMeasurementSnapshot(sql, receipt); + await claimMeasurementSnapshot(sql, receipt); + await expect( + claimMeasurementSnapshot(sql, { ...receipt, receipt_id: 'f'.repeat(64) }), + ).rejects.toThrow('different accepted'); + const complete = await db.query('select state, receipt_id from measurement_snapshots'); + expect(complete.rows).toEqual([ + { + state: 'complete', + receipt_id: '2e12626c30ec11cf1738c7541a36444af53c29bad4f59f927930618e298e7a01', + }, + ]); + } finally { + await db.close(); + } +}, 20_000); diff --git a/packages/db/src/etl/measurement-snapshot.ts b/packages/db/src/etl/measurement-snapshot.ts new file mode 100644 index 000000000..421ae6717 --- /dev/null +++ b/packages/db/src/etl/measurement-snapshot.ts @@ -0,0 +1,44 @@ +import type postgres from 'postgres'; +import type { MeasurementReceipt } from '../lib/measurement-receipt'; + +type Sql = ReturnType; + +/** Claim before writes. A partial import resumes only the same immutable accepted snapshot. */ +export async function claimMeasurementSnapshot( + sql: Sql, + receipt: MeasurementReceipt, +): Promise { + const claimed = await sql` + insert into measurement_snapshots (source_repo, source_run_id, source_attempt, receipt_id, bundle_digest, receipt, state) + values (${receipt.repository}, ${receipt.source_run_id}, ${receipt.source_attempt}, ${receipt.receipt_id}, ${receipt.bundle_digest}, ${JSON.stringify(receipt)}::jsonb, 'writing') + on conflict (source_repo, source_run_id, source_attempt) do update + set updated_at = now() + where measurement_snapshots.receipt_id = excluded.receipt_id + and measurement_snapshots.bundle_digest = excluded.bundle_digest + returning receipt_id + `; + if (claimed.length !== 1) + throw new Error('Run already belongs to a different accepted measurement snapshot'); +} + +export async function completeMeasurementSnapshot( + sql: Sql, + receipt: MeasurementReceipt, +): Promise { + await sql`update measurement_snapshots set state = 'complete', updated_at = now() + where source_repo = ${receipt.repository} and source_run_id = ${receipt.source_run_id} + and source_attempt = ${receipt.source_attempt} and receipt_id = ${receipt.receipt_id}`; +} + +/** Legacy compatibility cannot bypass an already accepted immutable snapshot. */ +export async function assertLegacySnapshotUnclaimed( + sql: Sql, + repository: string, + runId: string, + attempt: number, +): Promise { + const rows = await sql`select receipt_id from measurement_snapshots + where source_repo = ${repository} and source_run_id = ${runId} and source_attempt = ${attempt}`; + if (rows.length > 0) + throw new Error('This run requires its previously accepted measurement receipt'); +} diff --git a/packages/db/src/ingest-ci-run.ts b/packages/db/src/ingest-ci-run.ts index a27225bc9..d24c30713 100644 --- a/packages/db/src/ingest-ci-run.ts +++ b/packages/db/src/ingest-ci-run.ts @@ -23,6 +23,17 @@ */ import fs from 'fs'; +import { + receiptFromEnvironment, + publicationFromEnvironment, + verifyMeasurementSnapshot, +} from './lib/measurement-receipt'; +import { prepareReceiptArtifacts } from './lib/receipt-artifact-preparation'; +import { + claimMeasurementSnapshot, + completeMeasurementSnapshot, + assertLegacySnapshotUnclaimed, +} from './etl/measurement-snapshot'; import { createHash } from 'node:crypto'; import { powerPublicationPoint, @@ -97,6 +108,7 @@ const DEFAULT_REPO = 'SemiAnalysisAI/InferenceX'; const powerPublicationPoints = new Map(); const powerPublicationErrors: string[] = []; const tracker = createSkipTracker(); +const measurementReceipt = receiptFromEnvironment(); const isDownloadMode = process.argv[2] === '--download'; let artifactsDir: string; @@ -160,42 +172,49 @@ if (isDownloadMode) { console.log(` Repo: ${REPO}`); console.log(`\n--- Downloading artifacts to ${artifactsDir} ---`); - // Retried configs produce artifacts on multiple runners — keep only the - // most recent per logical name (see RUNNER_SUFFIX_RE in github-artifacts) - // so a failed attempt's empty metrics can't overwrite the good one via - // ON CONFLICT DO UPDATE. - const artifacts = listRunArtifacts(REPO, runIdStr); - const byLogical = dedupeArtifactsByLogicalName(artifacts); - // Server-log artifacts from eval and benchmark jobs can share a logical - // config but differ by runner suffix. Keep the exact server-log sibling for - // each selected bmk artifact instead of letting latest-created eval logs win. - for (const [key, artifact] of byLogical) { - if ( - artifact.name.startsWith('server_logs_') || - artifact.name.startsWith('multinode_server_logs_') - ) { - byLogical.delete(key); + if (measurementReceipt) { + if (measurementReceipt.repository !== REPO || measurementReceipt.source_run_id !== runIdStr) + throw new Error('Receipt source differs from requested run'); + prepareReceiptArtifacts(measurementReceipt, artifactsDir); + runAttemptNum = measurementReceipt.source_attempt; + } else { + // Retried configs produce artifacts on multiple runners — keep only the + // most recent per logical name (see RUNNER_SUFFIX_RE in github-artifacts) + // so a failed attempt's empty metrics can't overwrite the good one via + // ON CONFLICT DO UPDATE. + const artifacts = listRunArtifacts(REPO, runIdStr); + const byLogical = dedupeArtifactsByLogicalName(artifacts); + // Server-log artifacts from eval and benchmark jobs can share a logical + // config but differ by runner suffix. Keep the exact server-log sibling for + // each selected bmk artifact instead of letting latest-created eval logs win. + for (const [key, artifact] of byLogical) { + if ( + artifact.name.startsWith('server_logs_') || + artifact.name.startsWith('multinode_server_logs_') + ) { + byLogical.delete(key); + } } - } - const selectedBenchmarkNames = new Set( - [...byLogical.values()] - .filter((artifact) => artifact.name.startsWith('bmk_')) - .map((artifact) => artifact.name), - ); - for (const pair of pairServerLogArtifacts(artifacts)) { - if (selectedBenchmarkNames.has(pair.benchmarks.name)) { - byLogical.set(`server-log:${pair.serverLogs.name}`, pair.serverLogs); + const selectedBenchmarkNames = new Set( + [...byLogical.values()] + .filter((artifact) => artifact.name.startsWith('bmk_')) + .map((artifact) => artifact.name), + ); + for (const pair of pairServerLogArtifacts(artifacts)) { + if (selectedBenchmarkNames.has(pair.benchmarks.name)) { + byLogical.set(`server-log:${pair.serverLogs.name}`, pair.serverLogs); + } } - } - for (const artifact of byLogical.values()) { - console.log(` ${artifact.name}`); - downloadArtifact(artifact, artifactsDir); - } + for (const artifact of byLogical.values()) { + console.log(` ${artifact.name}`); + downloadArtifact(artifact, artifactsDir); + } - console.log(`\n Downloaded ${byLogical.size} artifact(s)`); + console.log(`\n Downloaded ${byLogical.size} artifact(s)`); - runAttemptNum = fetchRunAttempt(REPO, runIdStr); + runAttemptNum = fetchRunAttempt(REPO, runIdStr); + } } else { // CI mode — read from env vars for (const key of [ @@ -231,6 +250,17 @@ if (reusedIngestMetadata) { runAttemptNum = reusedIngestMetadata.sourceRunAttempt; } +if (measurementReceipt) { + if ( + measurementReceipt.repository !== REPO || + measurementReceipt.source_run_id !== runIdStr || + measurementReceipt.source_attempt !== runAttemptNum + ) + throw new Error('Receipt source/attempt differs from ingest request'); + publicationFromEnvironment(measurementReceipt, requestedRunIdStr); + verifyMeasurementSnapshot(measurementReceipt, artifactsDir); +} + const runIdNum = parseInt(runIdStr, 10); const GITHUB_TOKEN = process.env.GITHUB_TOKEN!; @@ -285,6 +315,9 @@ async function main(): Promise { return; } + await (measurementReceipt + ? claimMeasurementSnapshot(sql, measurementReceipt) + : assertLegacySnapshotUnclaimed(sql, REPO, runIdStr, runAttemptNum)); validateRunBackfills(); const configCache = createConfigCache(sql); const { getOrCreateConfig, preloadConfigs } = configCache; @@ -1030,7 +1063,10 @@ async function main(): Promise { ); } + if (measurementReceipt && Object.values(tracker.skips).some((count) => count > 0)) + throw new Error('Accepted snapshot ingestion skipped required input'); await refreshLatestBenchmarks(sql); + if (measurementReceipt) await completeMeasurementSnapshot(sql, measurementReceipt); console.log('\n=== ingest-ci-run complete ==='); console.log(' Invalidate API cache: bun run admin:cache:invalidate'); diff --git a/packages/db/src/lib/artifact-archive.test.ts b/packages/db/src/lib/artifact-archive.test.ts new file mode 100644 index 000000000..2f20c70e4 --- /dev/null +++ b/packages/db/src/lib/artifact-archive.test.ts @@ -0,0 +1,42 @@ +import AdmZip from 'adm-zip'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, expect, it } from 'vitest'; +import { extractVerifiedArchive, inspectArchive } from './artifact-archive'; + +const roots: string[] = []; +function root() { + const value = fs.mkdtempSync(path.join(os.tmpdir(), 'receipt-archive-')); + roots.push(value); + return value; +} +afterEach(() => roots.forEach((value) => fs.rmSync(value, { recursive: true, force: true }))); +it('extracts checked bytes and refuses overwrites and altered members', () => { + const zip = new AdmZip(); + zip.addFile('raw/result.json', Buffer.from('{"ok":true}')); + const bytes = zip.toBuffer(); + const target = path.join(root(), '123'); + const expected = [ + { + path: 'raw/result.json', + size: 11, + sha256: '4062edaf750fb8074e7e83e0c9028c94a8e32468a8b6e6c0fc692be0f1cc83875', + }, + ]; + expect(() => extractVerifiedArchive(bytes, target, expected)).toThrow('digest/size mismatch'); + expect(fs.existsSync(target)).toBe(false); + extractVerifiedArchive(bytes, target); + expect(fs.readFileSync(path.join(target, 'raw/result.json'), 'utf8')).toBe('{"ok":true}'); + expect(() => extractVerifiedArchive(bytes, target)).toThrow('overwrite'); +}); +it('rejects links and file-directory collisions before filesystem writes', () => { + const links = new AdmZip(); + links.addFile('link', Buffer.from('/tmp/target')); + links.getEntries()[0].attr = (0o120777 << 16) >>> 0; + expect(() => inspectArchive(links.toBuffer())).toThrow('Unsafe'); + const collision = new AdmZip(); + collision.addFile('raw', Buffer.from('file')); + collision.addFile('raw/result.json', Buffer.from('{}')); + expect(() => inspectArchive(collision.toBuffer())).toThrow('collision'); +}); diff --git a/packages/db/src/lib/artifact-archive.ts b/packages/db/src/lib/artifact-archive.ts new file mode 100644 index 000000000..b590ebff0 --- /dev/null +++ b/packages/db/src/lib/artifact-archive.ts @@ -0,0 +1,97 @@ +import AdmZip from 'adm-zip'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; + +export interface ArchiveMember { + path: string; + sha256: string; + size: number; +} +export function sha256(bytes: Buffer | string): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +/** Validate every entry before writing anything. Extraction never follows ZIP links. */ +export function inspectArchive(bytes: Buffer): { + members: ArchiveMember[]; + files: Map; +} { + const zip = new AdmZip(bytes); + const files = new Map(); + const names = new Set(); + let total = 0; + for (const entry of zip.getEntries()) { + const name = entry.entryName; + const segments = name.replace(/\/$/u, '').split('/'); + const mode = (entry.attr >>> 16) & 0o170000; + if ( + !name || + name.includes('\\') || + name.includes('\0') || + name.startsWith('/') || + /^[A-Za-z]:/u.test(name) || + segments.some((part) => !part || part === '.' || part === '..') || + (mode !== 0 && mode !== 0o100000 && mode !== 0o040000) + ) { + throw new Error(`Unsafe archive member: ${name}`); + } + const normalized = segments.join('/'); + if (names.has(normalized)) throw new Error(`Duplicate archive member: ${normalized}`); + names.add(normalized); + if (entry.isDirectory) continue; + total += entry.header.size; + if (total > 20 * 1024 ** 3 || entry.header.size > 10 * 1024 ** 3) { + throw new Error('Artifact exceeds extraction size budget'); + } + files.set(normalized, entry.getData()); + } + for (const name of files.keys()) { + const segments = name.split('/'); + segments.pop(); + while (segments.length > 0) { + if (files.has(segments.join('/'))) + throw new Error(`Archive file/directory collision: ${name}`); + segments.pop(); + } + } + return { + files, + members: [...files].map(([name, data]) => ({ + path: name, + size: data.length, + sha256: sha256(data), + })), + }; +} + +export function extractVerifiedArchive( + bytes: Buffer, + destination: string, + expected?: readonly ArchiveMember[], +): void { + const { files, members } = inspectArchive(bytes); + if (expected) { + const selected = new Map(expected.map((member) => [member.path, member])); + if (selected.size !== expected.length || selected.size !== members.length) + throw new Error('Archive member set mismatch'); + for (const member of members) { + const match = selected.get(member.path); + if (!match || match.sha256 !== member.sha256 || match.size !== member.size) { + throw new Error(`Archive member digest/size mismatch: ${member.path}`); + } + } + } + if (fs.existsSync(destination)) throw new Error(`Refusing artifact overwrite: ${destination}`); + fs.mkdirSync(destination, { recursive: true }); + try { + for (const [name, data] of files) { + const target = path.join(destination, name); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, data, { flag: 'wx', mode: 0o600 }); + } + } catch (error) { + fs.rmSync(destination, { recursive: true, force: true }); + throw error; + } +} diff --git a/packages/db/src/lib/fixtures/measurement-receipt/101.zip b/packages/db/src/lib/fixtures/measurement-receipt/101.zip new file mode 100644 index 0000000000000000000000000000000000000000..afcb2e6c539db94a0753c6e62f3a3111d328d74d GIT binary patch literal 1055 zcmbtTO>fgc5OoqjoOv zpFo}GghDHjkTQ{-o!Omv&u?b)+33lB0{e9P>hkC3y$?r&B*F8XVxq2JUU_T2te7%Y zE0(;qrPRE0vQkU-`jD00W$fsXHP`T(Nt(Ve3j``BT6t~VSHu@uw+K>Rp}evN!Kyn& zEOg|Y(n4a9(Za-**Io46c&R+`Ag2=AOWIeSw^0XWh|3@0o|M)c=>V1v6;etlWkE;( zvQb!5&>4U#udzxe$K==tP7GZPeEUMpEbXgCCjxy7_f+1L*f>eom-z!le|$gd4u(mB zXO9$>OIbu@^}4Q@FB;hjz96z2S-_>U$^@=Ts>re(4HDmjZfjaskDcu7m_B5j(YtTq zJepEVUaDI90OKx^F1{ivte1Xg16?!io@N+kpbaXrw5)Zv>LW ze;Qj;B1RQSp>a0W4fPOMhI+P;oCZ;<9%IF>?0hTCbc!RbFsdRAfI69;phmT$^-=^8 zdO~OK-{o)5FV5feGz+aHbz3e~;22YaOV{~qFx(&g`(on4!fxoI{=TM?V{%MJhy0me zA!SkwN#_e(3~>T9>EXnbnx{qSmikya#DR-MCy=5cHO@uamO7DfMv8GNB&j}c4?!1+erzo94qe&E0a;hkAuS2m7ha{mGi^+Jc zmtTk1-}iQcfW{;2B~%cV9&4{VI%h7*Knp%Ym`y{d^dgtWa~Wem*56X<8dgb{F>{-C zEfjgT7lz#)e*0uKElsR|$1(*+)w1c9LP1?#+KBozXOz)NpW6=?NDv z22>*Qys^WRSUmj8M5{Y;2$i=O>YtMl3I0=6;aXL9Nwi&6<56p8zy0Tz7EidAOFZtI ucX!ik^G#`dg5a`sf`02S%%<-qp>BLXwQr@@c(i+;jN177hX2_vt$qUHP|eK% literal 0 HcmV?d00001 diff --git a/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/agg.json b/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/agg.json new file mode 100644 index 000000000..197ad20c0 --- /dev/null +++ b/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/agg.json @@ -0,0 +1 @@ +[{"infmax_model_prefix": "dsr1", "hw": "h200-nv", "framework": "vllm", "precision": "fp8", "tp": 8, "ep": 1, "num_gpus": 8, "disagg": false, "is_multinode": false, "prefill_num_workers": 0, "decode_num_workers": 0, "conc": 1, "isl": 1024, "osl": 1024, "output_tput_tps": 100, "duration_seconds": 60}] \ No newline at end of file diff --git a/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/execution.json b/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/execution.json new file mode 100644 index 000000000..3cb53cd83 --- /dev/null +++ b/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/execution.json @@ -0,0 +1 @@ +{"schema_version": 1, "point_id": "1111111111111111111111111111111111111111111111111111111111111111", "execution_id": "org/repo/100/1/101", "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "source": {"repository": "org/repo", "run_id": 100, "attempt": 1, "head_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "mode": "throughput", "native_receipt": {"job_id": "55", "manifest_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "state": "COMPLETED"}, "client_exit_code": 0} \ No newline at end of file diff --git a/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/agg.json b/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/agg.json new file mode 100644 index 000000000..14f25647d --- /dev/null +++ b/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/agg.json @@ -0,0 +1 @@ +[{"infmax_model_prefix": "dsr1", "hw": "h200-nv", "framework": "vllm", "precision": "fp8", "tp": 8, "ep": 1, "num_gpus": 8, "disagg": false, "is_multinode": false, "prefill_num_workers": 0, "decode_num_workers": 0, "conc": 28, "task": "gsm8k", "em_strict": 0.5, "em_flexible": 1.0, "n_eff": 2}] \ No newline at end of file diff --git a/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/execution.json b/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/execution.json new file mode 100644 index 000000000..aee7b4253 --- /dev/null +++ b/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/execution.json @@ -0,0 +1 @@ +{"schema_version": 1, "point_id": "2222222222222222222222222222222222222222222222222222222222222222", "execution_id": "org/repo/100/1/102", "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "source": {"repository": "org/repo", "run_id": 100, "attempt": 1, "head_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "mode": "eval", "native_receipt": {"job_id": "55", "manifest_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "state": "COMPLETED"}, "client_exit_code": 0} \ No newline at end of file diff --git a/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/samples_gsm8k.jsonl b/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/samples_gsm8k.jsonl new file mode 100644 index 000000000..0c03f3983 --- /dev/null +++ b/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/samples_gsm8k.jsonl @@ -0,0 +1,4 @@ +{"doc_id": 0, "task_name": "gsm8k", "filter": "strict-match", "filtered_resps": ["9"], "exact_match": 1} +{"doc_id": 0, "task_name": "gsm8k", "filter": "flexible-extract", "filtered_resps": ["9"], "exact_match": 1} +{"doc_id": 1, "task_name": "gsm8k", "filter": "strict-match", "filtered_resps": ["9"], "exact_match": 0} +{"doc_id": 1, "task_name": "gsm8k", "filter": "flexible-extract", "filtered_resps": ["9"], "exact_match": 1} diff --git a/packages/db/src/lib/fixtures/measurement-receipt/receipt.json b/packages/db/src/lib/fixtures/measurement-receipt/receipt.json new file mode 100644 index 000000000..5ccb8f204 --- /dev/null +++ b/packages/db/src/lib/fixtures/measurement-receipt/receipt.json @@ -0,0 +1,154 @@ +{ + "repository": "org/repo", + "source_run_id": "100", + "source_attempt": 2, + "source_head_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "contracts": { + "raw": "aiperf-1.4", + "normalized": "agentx-v1", + "publication": 1 + }, + "points": [ + { + "point_id": "1111111111111111111111111111111111111111111111111111111111111111", + "execution_id": "org/repo/100/1/101", + "source_run_id": "100", + "source_attempt": 1, + "kind": "throughput", + "concurrency": 1, + "topology": { + "kind": "aggregate", + "nodes": 1, + "serving_gpus": 8, + "tp": 8, + "ep": 1 + }, + "artifact_ids": [ + 101 + ], + "execution_artifact_id": 101, + "execution_path": "execution.json", + "native_manifest_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "normalized_artifact_id": 101, + "normalized_path": "agg.json", + "required_metrics": [ + "output_tput_tps", + "duration_seconds" + ], + "config": { + "model": "dsr1", + "hardware": "h200", + "framework": "vllm", + "precision": "fp8" + }, + "task": null, + "filters": [], + "sample_count": 0, + "samples_artifact_id": null, + "samples_path": null, + "dataset": {}, + "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "normalized_format": "normalized", + "metadata_path": null + }, + { + "point_id": "2222222222222222222222222222222222222222222222222222222222222222", + "execution_id": "org/repo/100/1/102", + "source_run_id": "100", + "source_attempt": 1, + "kind": "eval", + "concurrency": 28, + "topology": { + "kind": "aggregate", + "nodes": 1, + "serving_gpus": 8, + "tp": 8, + "ep": 1 + }, + "artifact_ids": [ + 102 + ], + "execution_artifact_id": 102, + "execution_path": "execution.json", + "native_manifest_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "normalized_artifact_id": 102, + "normalized_path": "agg.json", + "required_metrics": [ + "em_strict", + "n_eff" + ], + "config": { + "model": "dsr1", + "hardware": "h200", + "framework": "vllm", + "precision": "fp8" + }, + "task": "gsm8k", + "filters": [ + "strict-match", + "flexible-extract" + ], + "sample_count": 2, + "samples_artifact_id": 102, + "samples_path": "samples_gsm8k.jsonl", + "dataset": {}, + "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "normalized_format": "normalized", + "metadata_path": null + } + ], + "kind": "source-measurement-receipt", + "version": 1, + "issuer": { + "repository": "org/repo", + "run_id": "200", + "job": "issuer", + "workflow_sha": "dddddddddddddddddddddddddddddddddddddddd", + "collector_sha": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + }, + "artifacts": [ + { + "id": 101, + "name": "bmk_pilot", + "sha256": "b644c12d559428df1845d5ff8d3483aac35e7fdb694d8ce1a7ac891239300fd3", + "run_id": "100", + "members": [ + { + "path": "agg.json", + "sha256": "a142da2f49fa4f706c801bd824eea698740b567ad689269fbdd181a9648dd849", + "size": 300 + }, + { + "path": "execution.json", + "sha256": "41746d1d444ec14e440d5d75e1144a5636d7ef7acaaef7b4957217cde4503c5b", + "size": 537 + } + ] + }, + { + "id": 102, + "name": "eval_results_all", + "sha256": "8c2f42b955bfdb85fac00d6bfca0442475803f2aa7aefd93b1842cacdff7e94c", + "run_id": "100", + "members": [ + { + "path": "agg.json", + "sha256": "2a05f61da6ac2bd48a9a5b8c40a4032dbd1f711581fcd8e1df429c0ce619a031", + "size": 294 + }, + { + "path": "execution.json", + "sha256": "479ff4fc073b1f73508bd76fb13be0fb6fb8a740f113e4f2404bd4139345ecbb", + "size": 531 + }, + { + "path": "samples_gsm8k.jsonl", + "sha256": "910488b869e78c51dd00e5b956515d2e232bba3669f426246e6384c0b7554256", + "size": 428 + } + ] + } + ], + "receipt_id": "2e12626c30ec11cf1738c7541a36444af53c29bad4f59f927930618e298e7a01" +} diff --git a/packages/db/src/lib/fixtures/measurement-receipt/receipt.sha256 b/packages/db/src/lib/fixtures/measurement-receipt/receipt.sha256 new file mode 100644 index 000000000..7bd6f61be --- /dev/null +++ b/packages/db/src/lib/fixtures/measurement-receipt/receipt.sha256 @@ -0,0 +1 @@ +00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6 diff --git a/packages/db/src/lib/github-artifacts.ts b/packages/db/src/lib/github-artifacts.ts index 67827855e..c809894e6 100644 --- a/packages/db/src/lib/github-artifacts.ts +++ b/packages/db/src/lib/github-artifacts.ts @@ -4,8 +4,8 @@ * `gh` CLI, which picks up GITHUB_TOKEN from the environment. */ -import { execSync } from 'node:child_process'; -import fs from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { extractVerifiedArchive, sha256, type ArchiveMember } from './artifact-archive.js'; import path from 'node:path'; export interface ArtifactMeta { @@ -14,6 +14,8 @@ export interface ArtifactMeta { archive_download_url: string; created_at: string; expired?: boolean; + digest?: string; + workflow_run?: { id: number }; } /** @@ -34,19 +36,15 @@ export const RUNNER_SUFFIX_RE = /_[a-zA-Z][a-zA-Z0-9.-]*_\d+$/u; /** List a workflow run's artifacts via `gh api` (paginated). Malformed lines are skipped. */ export function listRunArtifacts(repo: string, runId: string): ArtifactMeta[] { - const json = execSync( - `gh api "repos/${repo}/actions/runs/${runId}/artifacts" --paginate --jq '.artifacts[]'`, + validateRun(repo, runId); + const json = execFileSync( + 'gh', + ['api', `repos/${repo}/actions/runs/${runId}/artifacts`, '--paginate', '--slurp'], { encoding: 'utf8', maxBuffer: 50 * 1024 * 1024 }, ); - const out: ArtifactMeta[] = []; - for (const line of json.trim().split('\n')) { - if (!line) continue; - try { - out.push(JSON.parse(line) as ArtifactMeta); - } catch { - // skip malformed line - } - } + const pages = JSON.parse(json) as { artifacts: ArtifactMeta[] }[]; + const out = pages.flatMap((page) => page.artifacts); + return out; } @@ -66,25 +64,62 @@ export function dedupeArtifactsByLogicalName( return byLogical; } -/** Download + unzip one artifact into `/`; returns that dir. */ -export function downloadArtifact(artifact: ArtifactMeta, destRoot: string): string { - const zipPath = path.join(destRoot, 'artifact.zip'); - execSync(`gh api "${artifact.archive_download_url}" > "${zipPath}"`, { - stdio: ['pipe', 'pipe', 'inherit'], +export function validateRun(repo: string, runId: string): void { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repo) || !/^[1-9]\d*$/u.test(runId)) { + throw new Error('Invalid repository or run ID'); + } +} + +export interface DownloadOptions { + repo?: string; + sha256?: string; + members?: readonly ArchiveMember[]; + isolated?: boolean; +} + +/** New receipts use numeric ID roots. The legacy view accepts only safe single path components. */ +export function downloadArtifact( + artifact: ArtifactMeta, + destRoot: string, + options: DownloadOptions = {}, +): string { + if (!Number.isSafeInteger(artifact.id) || artifact.id! <= 0 || artifact.expired) + throw new Error('Invalid/expired artifact ID'); + const match = artifact.archive_download_url.match( + /^https:\/\/api\.github\.com\/repos\/(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)\/actions\/artifacts\/(?\d+)\/zip$/u, + ); + const repo = options.repo ?? match?.[1]; + if (!repo || (match && Number(match[2]) !== artifact.id)) + throw new Error('Invalid GitHub artifact owner/ID'); + validateRun(repo, String(artifact.id)); + const bytes = execFileSync('gh', ['api', `repos/${repo}/actions/artifacts/${artifact.id}/zip`], { + maxBuffer: 20 * 1024 ** 3, + stdio: ['ignore', 'pipe', 'inherit'], }); - const destDir = path.join(destRoot, artifact.name); - fs.mkdirSync(destDir, { recursive: true }); - execSync(`unzip -oq "${zipPath}" -d "${destDir}"`, { stdio: 'inherit' }); - fs.unlinkSync(zipPath); - return destDir; + const expected = options.sha256 ?? artifact.digest?.replace(/^sha256:/u, ''); + if (expected && (!/^[a-f0-9]{64}$/u.test(expected) || sha256(bytes) !== expected)) + throw new Error(`Artifact digest mismatch: ${artifact.id}`); + if ( + !options.isolated && + (!/^[A-Za-z0-9_.-]+$/u.test(artifact.name) || ['.', '..'].includes(artifact.name)) + ) { + throw new Error('Unsafe legacy artifact display name'); + } + const destination = path.join(destRoot, options.isolated ? String(artifact.id) : artifact.name); + extractVerifiedArchive(bytes, destination, options.members); + return destination; } -/** Fetch a run's current attempt number via `gh api` (defaults to 1). */ +/** Fetch a run's current attempt for the explicitly weaker legacy path only. */ export function fetchRunAttempt(repo: string, runId: string): number { - const attemptStr = execSync(`gh api "repos/${repo}/actions/runs/${runId}" --jq '.run_attempt'`, { - encoding: 'utf8', - }).trim(); - return parseInt(attemptStr || '1', 10); + validateRun(repo, runId); + const attempt = Number( + execFileSync('gh', ['api', `repos/${repo}/actions/runs/${runId}`, '--jq', '.run_attempt'], { + encoding: 'utf8', + }).trim(), + ); + if (!Number.isSafeInteger(attempt) || attempt < 1) throw new Error('Invalid run attempt'); + return attempt; } export interface RunMeta { @@ -116,7 +151,7 @@ export function fetchRunMeta(repo: string, runId: string): RunMeta { if (!/^\d+$/u.test(runId)) { throw new Error(`Invalid run id: ${runId}`); } - const json = execSync(`gh api "repos/${repo}/actions/runs/${runId}"`, { + const json = execFileSync('gh', ['api', `repos/${repo}/actions/runs/${runId}`], { encoding: 'utf8', maxBuffer: 10 * 1024 * 1024, }); diff --git a/packages/db/src/lib/measurement-receipt.test.ts b/packages/db/src/lib/measurement-receipt.test.ts new file mode 100644 index 000000000..58a64b55f --- /dev/null +++ b/packages/db/src/lib/measurement-receipt.test.ts @@ -0,0 +1,150 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, expect, it } from 'vitest'; +import { sha256 } from './artifact-archive'; +import { + parseMeasurementReceipt, + receiptFromEnvironment, + verifyMeasurementSnapshot, + publicationFromEnvironment, +} from './measurement-receipt'; +const fixture = new URL('fixtures/measurement-receipt/', import.meta.url); +const raw = fs.readFileSync(new URL('receipt.json', fixture)); +const expectedDigest = '00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6'; +const issuerSha = 'd'.repeat(40); +const roots: string[] = []; +function copy() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'receipt-')); + roots.push(root); + fs.cpSync(fixture, root, { recursive: true }); + return root; +} +afterEach(() => + roots.splice(0).forEach((root) => fs.rmSync(root, { force: true, recursive: true })), +); +it('reads a Python-sealed snapshot and verifies both mapped families and complete sample coverage', () => { + const receipt = parseMeasurementReceipt(raw, expectedDigest, issuerSha); + expect(receipt.receipt_id).toBe( + '2e12626c30ec11cf1738c7541a36444af53c29bad4f59f927930618e298e7a01', + ); + expect( + receipt.points.map((point) => [point.kind, point.concurrency, point.source_attempt]), + ).toEqual([ + ['throughput', 1, 1], + ['eval', 28, 1], + ]); + const root = copy(); + expect(() => verifyMeasurementSnapshot(receipt, root)).not.toThrow(); + const row = JSON.parse(fs.readFileSync(path.join(root, 'bmk_pilot/agg.json'), 'utf8')); + row[0].output_tput_tps = 999; + fs.writeFileSync(path.join(root, 'bmk_pilot/agg.json'), JSON.stringify(row)); + expect(() => verifyMeasurementSnapshot(receipt, root)).toThrow('Changed receipt member'); +}); +it('rejects missing required receipt, unsupported contracts and untrusted issuers', () => { + expect(() => receiptFromEnvironment({ INGEST_RECEIPT_REQUIRED: '1' })).toThrow( + 'Required measurement receipt missing', + ); + expect(receiptFromEnvironment({})).toBeNull(); + expect(() => parseMeasurementReceipt(raw, expectedDigest, 'f'.repeat(40))).toThrow('Untrusted'); + const value = JSON.parse(raw.toString()); + value.version = 2; + const changed = Buffer.from(JSON.stringify(value)); + expect(() => parseMeasurementReceipt(changed, sha256(changed), issuerSha)).toThrow('Unsupported'); +}); +it.each([ + 'duplicate', + 'wrong-topology', + 'missing-metric', + 'wrong-model', + 'filter-gap', + 'wrong-execution', +])('rejects self-consistent but semantically invalid %s artifacts', (kind) => { + const receipt = parseMeasurementReceipt(raw, expectedDigest, issuerSha); + const root = copy(); + const artifact = receipt.artifacts[kind === 'filter-gap' ? 1 : 0]; + const name = + kind === 'filter-gap' + ? 'samples_gsm8k.jsonl' + : kind === 'wrong-execution' + ? 'execution.json' + : 'agg.json'; + const file = path.join(root, artifact.name, name); + let contents: string; + if (kind === 'filter-gap') + contents = fs.readFileSync(file, 'utf8').trim().split('\n').slice(1).join('\n'); + else { + const value = JSON.parse(fs.readFileSync(file, 'utf8')); + if (kind === 'duplicate') value.push(value[0]); + if (kind === 'wrong-topology') value[0].tp = 4; + if (kind === 'missing-metric') delete value[0].output_tput_tps; + if (kind === 'wrong-model') value[0].infmax_model_prefix = 'glm5'; + if (kind === 'wrong-execution') value.native_receipt.state = 'CANCELLED'; + contents = JSON.stringify(value); + } + fs.writeFileSync(file, contents); + const member = artifact.members.find((entry) => entry.path === name)!; + member.sha256 = sha256(contents); + member.size = Buffer.byteLength(contents); + expect(() => verifyMeasurementSnapshot(receipt, root)).toThrow(); +}); +it('requires a later, immutable publication record when source and merge differ', () => { + const receipt = parseMeasurementReceipt(raw, expectedDigest, issuerSha); + expect(publicationFromEnvironment(receipt, '100', {})).toBeNull(); + expect(() => publicationFromEnvironment(receipt, '200', {})).toThrow( + 'requires a publication record', + ); + const record = { + kind: 'publication-record', + version: 1, + receipt_id: receipt.receipt_id, + receipt_artifact_id: 301, + receipt_artifact_sha256: 'a'.repeat(64), + source_run_id: '100', + merge_run_id: '200', + merge_sha: 'e'.repeat(40), + changelog_artifact_id: 302, + changelog_artifact_sha256: 'b'.repeat(64), + ingest_sha: 'f'.repeat(40), + app_sha: 'c'.repeat(40), + }; + const file = path.join(copy(), 'publication.json'); + const bytes = JSON.stringify(record); + fs.writeFileSync(file, bytes); + const env = { + INGEST_PUBLICATION_RECORD_PATH: file, + INGEST_PUBLICATION_RECORD_SHA256: sha256(bytes), + }; + expect(publicationFromEnvironment(receipt, '200', env)?.receipt_id).toBe(receipt.receipt_id); + expect(() => publicationFromEnvironment(receipt, '201', env)).toThrow('Invalid/unsupported'); +}); +it('accepts per-job lm-eval results through the real detail mapper without an aggregate collector', () => { + const receipt = parseMeasurementReceipt(raw, expectedDigest, issuerSha); + const root = copy(); + const point = receipt.points[1]; + const artifact = receipt.artifacts[1]; + point.normalized_format = 'lm-eval'; + point.metadata_path = 'meta_env.json'; + const old = JSON.parse( + fs.readFileSync(path.join(root, artifact.name, point.normalized_path), 'utf8'), + )[0]; + const results = { + lm_eval_version: '0.4.7', + results: { gsm8k: { 'exact_match,strict-match': 0.5 } }, + 'n-samples': { gsm8k: { effective: 2 } }, + }; + for (const [name, object] of [ + [point.normalized_path, results], + ['meta_env.json', old], + ] as const) { + const value = JSON.stringify(object); + fs.writeFileSync(path.join(root, artifact.name, name), value); + const existing = artifact.members.find((member) => member.path === name); + const member = { path: name, size: Buffer.byteLength(value), sha256: sha256(value) }; + if (existing) Object.assign(existing, member); + else artifact.members.push(member); + } + expect(() => verifyMeasurementSnapshot(receipt, root)).not.toThrow(); + fs.writeFileSync(path.join(root, artifact.name, 'unexpected.json'), '[]'); + expect(() => verifyMeasurementSnapshot(receipt, root)).toThrow('member set differs'); +}); diff --git a/packages/db/src/lib/measurement-receipt.ts b/packages/db/src/lib/measurement-receipt.ts new file mode 100644 index 000000000..f57f74fd4 --- /dev/null +++ b/packages/db/src/lib/measurement-receipt.ts @@ -0,0 +1,478 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { isDeepStrictEqual } from 'node:util'; +import { sha256, type ArchiveMember } from './artifact-archive'; +import { mapBenchmarkRow } from '../etl/benchmark-mapper'; +import { mapAggEvalRow, mapEvalRow } from '../etl/eval-mapper'; +import { createSkipTracker } from '../etl/skip-tracker'; + +export interface ReceiptArtifact { + id: number; + name: string; + sha256: string; + run_id: string; + members: ArchiveMember[]; +} +export interface ReceiptPoint { + point_id: string; + bundle_digest: string; + execution_id: string; + source_run_id: string; + source_attempt: number; + kind: 'throughput' | 'eval'; + concurrency: number; + topology: { kind: 'aggregate'; nodes: 1; serving_gpus: number; tp: number; ep: number }; + artifact_ids: number[]; + normalized_artifact_id: number; + normalized_path: string; + normalized_format: 'normalized' | 'lm-eval'; + metadata_path: string | null; + execution_artifact_id: number; + execution_path: string; + native_manifest_sha256: string; + config: Record; + required_metrics: string[]; + task: string | null; + filters: string[]; + sample_count: number; + samples_artifact_id: number | null; + samples_path: string | null; + dataset: Record; +} +export interface MeasurementReceipt { + kind: 'source-measurement-receipt'; + version: 1; + receipt_id: string; + repository: string; + source_run_id: string; + source_attempt: number; + source_head_sha: string; + bundle_digest: string; + contracts: { raw: string; normalized: 'agentx-v1'; publication: 1 }; + issuer: { + repository: string; + run_id: string; + job: string; + workflow_sha: string; + collector_sha: string; + }; + points: ReceiptPoint[]; + artifacts: ReceiptArtifact[]; +} + +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value && typeof value === 'object') + return `{${Object.entries(value) + .toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, item]) => `${JSON.stringify(key)}:${canonical(item)}`) + .join(',')}}`; + if (typeof value === 'number' && !Number.isSafeInteger(value)) + throw new Error('Receipt numbers must be safe integers'); + return JSON.stringify(value); +} +function positive(value: unknown): boolean { + return Number.isSafeInteger(value) && Number(value) > 0; +} +function digest(value: unknown): boolean { + return typeof value === 'string' && /^[a-f0-9]{64}$/u.test(value); +} +function safePath(value: string): boolean { + return ( + Boolean(value) && + !value.includes('\\') && + !value.includes('\0') && + !value.startsWith('/') && + !/^[A-Za-z]:/u.test(value) && + value.split('/').every((part) => Boolean(part) && part !== '.' && part !== '..') + ); +} +function finiteJson(value: unknown): void { + if (typeof value === 'number' && !Number.isFinite(value)) + throw new Error('Non-finite result value'); + if (value && typeof value === 'object') Object.values(value).forEach(finiteJson); +} + +/** A hash establishes immutability; the transport's trusted issuer pin establishes authority. */ +export function parseMeasurementReceipt( + bytes: Buffer, + expectedSha256: string, + issuerSha: string, +): MeasurementReceipt { + if (!digest(expectedSha256) || sha256(bytes) !== expectedSha256) + throw new Error('Receipt transport digest mismatch'); + const receipt = JSON.parse(bytes.toString('utf8')) as MeasurementReceipt; + if ( + receipt.kind !== 'source-measurement-receipt' || + receipt.version !== 1 || + receipt.contracts?.publication !== 1 || + receipt.contracts?.normalized !== 'agentx-v1' || + receipt.contracts?.raw !== 'aiperf-1.4' + ) + throw new Error('Unsupported required receipt version/contract'); + if (!/^[a-f0-9]{40}$/u.test(issuerSha) || receipt.issuer?.workflow_sha !== issuerSha) + throw new Error('Untrusted receipt issuer revision'); + const { receipt_id, ...payload } = receipt; + if (!digest(receipt_id) || sha256(canonical(payload)) !== receipt_id) + throw new Error('Receipt content digest mismatch'); + if ( + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(receipt.repository) || + !/^[1-9]\d*$/u.test(receipt.source_run_id) || + !positive(receipt.source_attempt) || + !digest(receipt.bundle_digest) + ) + throw new Error('Invalid receipt source identity'); + if ( + !Array.isArray(receipt.points) || + receipt.points.length === 0 || + !Array.isArray(receipt.artifacts) + ) + throw new Error('Empty receipt points/artifacts'); + const points = new Set(); + const required = new Set(); + for (const point of receipt.points) { + if ( + !digest(point.point_id) || + !digest(point.bundle_digest) || + points.has(point.point_id) || + !point.execution_id || + !positive(point.concurrency) || + !positive(point.source_attempt) || + !/^[1-9]\d*$/u.test(point.source_run_id) || + !['throughput', 'eval'].includes(point.kind) + ) + throw new Error('Invalid/duplicate receipt point'); + points.add(point.point_id); + const topology = point.topology; + if ( + topology?.kind !== 'aggregate' || + topology.nodes !== 1 || + ![topology.tp, topology.ep, topology.serving_gpus].every(positive) + ) + throw new Error('Unsupported receipt topology'); + if ( + !Array.isArray(point.artifact_ids) || + !point.artifact_ids.every(positive) || + new Set(point.artifact_ids).size !== point.artifact_ids.length || + !point.artifact_ids.includes(point.normalized_artifact_id) || + !safePath(point.normalized_path) || + !point.required_metrics?.length + ) + throw new Error('Invalid point artifact/metric requirements'); + if ( + !point.artifact_ids.includes(point.execution_artifact_id) || + !safePath(point.execution_path) || + !digest(point.native_manifest_sha256) || + !point.config || + Object.keys(point.config).length === 0 + ) + throw new Error('Missing execution/config contract'); + point.artifact_ids.forEach((id) => required.add(id)); + if ( + !['normalized', 'lm-eval'].includes(point.normalized_format) || + (point.normalized_format === 'lm-eval' && + (point.kind !== 'eval' || !point.metadata_path || !safePath(point.metadata_path))) + ) + throw new Error('Invalid normalized format/metadata requirement'); + if ( + point.kind === 'eval' && + (!point.task || + !positive(point.sample_count) || + !point.filters?.length || + !point.artifact_ids.includes(point.samples_artifact_id!) || + !point.samples_path || + !safePath(point.samples_path)) + ) + throw new Error('Missing eval sample contract'); + } + const ids = new Set(); + const names = new Set(); + for (const artifact of receipt.artifacts) { + if ( + !positive(artifact.id) || + ids.has(artifact.id) || + !required.has(artifact.id) || + !digest(artifact.sha256) || + !/^[A-Za-z0-9_.-]+$/u.test(artifact.name) || + ['.', '..'].includes(artifact.name) || + names.has(artifact.name) + ) + throw new Error('Invalid/duplicate receipt artifact'); + ids.add(artifact.id); + names.add(artifact.name); + const members = new Set(); + for (const member of artifact.members) { + if ( + !safePath(member.path) || + members.has(member.path) || + !digest(member.sha256) || + !Number.isSafeInteger(member.size) || + member.size < 0 + ) + throw new Error('Invalid receipt archive member'); + members.add(member.path); + } + for (const point of receipt.points.filter((candidate) => + candidate.artifact_ids.includes(artifact.id), + )) { + if (point.source_run_id !== artifact.run_id) throw new Error('Point artifact owner mismatch'); + if (point.execution_artifact_id === artifact.id && !members.has(point.execution_path)) + throw new Error('Missing execution member'); + if (point.normalized_artifact_id === artifact.id && !members.has(point.normalized_path)) + throw new Error('Missing normalized member'); + if ( + point.normalized_artifact_id === artifact.id && + point.metadata_path && + !members.has(point.metadata_path) + ) + throw new Error('Missing lm-eval metadata member'); + if (point.samples_artifact_id === artifact.id && !members.has(point.samples_path!)) + throw new Error('Missing samples member'); + } + } + if (ids.size !== required.size) throw new Error('Receipt artifact set incomplete'); + return receipt; +} + +export function receiptFromEnvironment( + env: Record = process.env, +): MeasurementReceipt | null { + const required = env.INGEST_RECEIPT_REQUIRED === '1'; + if (!env.INGEST_RECEIPT_PATH) { + if (required) throw new Error('Required measurement receipt missing'); + return null; + } + if (!env.INGEST_RECEIPT_SHA256 || !env.INGEST_RECEIPT_ISSUER_SHA) + throw new Error('Receipt requires trusted digest and issuer revision'); + return parseMeasurementReceipt( + fs.readFileSync(env.INGEST_RECEIPT_PATH), + env.INGEST_RECEIPT_SHA256, + env.INGEST_RECEIPT_ISSUER_SHA, + ); +} + +export function mapReceiptPointRows( + receipt: MeasurementReceipt, + root: string, + point: ReceiptPoint, +) { + const artifact = receipt.artifacts.find((item) => item.id === point.normalized_artifact_id)!; + const value = JSON.parse( + fs.readFileSync(path.join(root, artifact.name, point.normalized_path), 'utf8'), + ); + finiteJson(value); + const tracker = createSkipTracker(); + if (point.normalized_format === 'lm-eval') { + const meta = JSON.parse( + fs.readFileSync(path.join(root, artifact.name, point.metadata_path!), 'utf8'), + ); + finiteJson(meta); + return { rows: [meta], mapped: mapEvalRow(meta, value, tracker) }; + } + const rows: Record[] = Array.isArray(value) ? value : [value]; + const mapped = rows.map((row) => + point.kind === 'throughput' ? mapBenchmarkRow(row, tracker) : mapAggEvalRow(row, tracker), + ); + if (mapped.some((row) => !row) || Object.values(tracker.skips).some((count) => count > 0)) + throw new Error('Unmapped required normalized input'); + return { rows, mapped }; +} + +/** Validate the complete snapshot before the first database write, including raw filter coverage. */ +export function verifyMeasurementSnapshot(receipt: MeasurementReceipt, root: string): void { + const artifacts = new Map(receipt.artifacts.map((artifact) => [artifact.id, artifact])); + const readMember = (id: number, member: string) => + fs.readFileSync(path.join(root, artifacts.get(id)!.name, member), 'utf8'); + const allowedRoots = new Set([ + ...receipt.artifacts.map((artifact) => artifact.name), + '.receipt-objects', + 'changelog-metadata', + 'reused-ingest-metadata', + ]); + for (const entry of fs.readdirSync(root)) { + // Fixture/transport files outside ingest discovery do not produce rows. + if ( + (entry.startsWith('bmk_') || + entry.startsWith('results_') || + entry.startsWith('eval_') || + entry.startsWith('agentic_') || + entry.startsWith('server_logs_')) && + !allowedRoots.has(entry) + ) + throw new Error(`Unaccepted artifact directory: ${entry}`); + } + for (const artifact of receipt.artifacts) { + const files: string[] = []; + const walk = (directory: string, prefix: string) => { + if (fs.lstatSync(directory).isSymbolicLink()) + throw new Error('Symlink in receipt artifact path'); + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const relative = prefix ? `${prefix}/${entry.name}` : entry.name; + if (entry.isSymbolicLink()) throw new Error('Symlink in receipt artifact path'); + if (entry.isDirectory()) walk(path.join(directory, entry.name), relative); + else if (entry.isFile()) files.push(relative); + else throw new Error('Special file in receipt artifact'); + } + }; + walk(path.join(root, artifact.name), ''); + if ( + !isDeepStrictEqual(files.toSorted(), artifact.members.map((member) => member.path).toSorted()) + ) + throw new Error('Receipt member set differs from extracted files'); + for (const member of artifact.members) { + const file = path.join(root, artifact.name, member.path); + const stat = fs.lstatSync(file); + if (!stat.isFile() || stat.isSymbolicLink()) throw new Error(`Invalid receipt file: ${file}`); + const bytes = fs.readFileSync(file); + if (bytes.length !== member.size || sha256(bytes) !== member.sha256) + throw new Error(`Changed receipt member: ${file}`); + } + } + const expectedByFile = new Map>(); + const actualByFile = new Map(); + for (const point of receipt.points) { + const execution = JSON.parse(readMember(point.execution_artifact_id, point.execution_path)); + if ( + execution.schema_version !== 1 || + execution.point_id !== point.point_id || + execution.execution_id !== point.execution_id || + execution.bundle_digest !== point.bundle_digest || + execution.mode !== point.kind || + execution.client_exit_code !== 0 || + execution.source?.repository !== receipt.repository || + String(execution.source?.run_id) !== point.source_run_id || + execution.source?.attempt !== point.source_attempt || + execution.source?.head_sha !== receipt.source_head_sha || + execution.native_receipt?.state !== 'COMPLETED' || + execution.native_receipt?.manifest_sha256 !== point.native_manifest_sha256 || + !/^[1-9]\d*$/u.test(String(execution.native_receipt?.job_id)) + ) + throw new Error('Execution evidence differs from expected contract'); + const { rows, mapped: allMapped } = mapReceiptPointRows(receipt, root, point); + const mapped = allMapped.filter( + (row) => + row && + row.conc === point.concurrency && + (point.kind === 'throughput' || ('task' in row && row.task === point.task)), + ); + if (mapped.length !== 1) + throw new Error( + `Expected exactly one normalized point ${point.point_id}, got ${mapped.length}`, + ); + const row = mapped[0]!; + const config = row.config; + for (const [field, expected] of Object.entries(point.config)) { + const actual = + field === 'recipeFingerprint' && 'recipeFingerprint' in row + ? row.recipeFingerprint + : config[field as keyof typeof config]; + if (actual !== expected) throw new Error(`Configuration identity mismatch: ${field}`); + } + if ( + config.disagg || + config.isMultinode || + config.decodeTp !== point.topology.tp || + config.decodeEp !== point.topology.ep || + config.numDecodeGpu !== point.topology.serving_gpus || + config.numPrefillGpu !== point.topology.serving_gpus + ) + throw new Error(`Topology mismatch: ${point.point_id}`); + for (const metric of point.required_metrics) + if (!Number.isFinite(row.metrics[metric]) || row.metrics[metric] < 0) + throw new Error(`Missing/non-finite metric: ${metric}`); + if (point.dataset && Object.keys(point.dataset).length > 0) { + const selected = rows.find( + (item: Record) => Number(item.conc ?? item.users) === point.concurrency, + ); + if (!isDeepStrictEqual(selected?.dataset, point.dataset)) + throw new Error('Dataset identity mismatch'); + } + const key = `${point.normalized_artifact_id}/${point.normalized_path}`; + actualByFile.set(key, allMapped.length); + const covered = expectedByFile.get(key) ?? new Set(); + covered.add(`${point.kind}:${point.concurrency}:${point.task ?? ''}`); + expectedByFile.set(key, covered); + if (point.kind === 'eval') { + const records = readMember(point.samples_artifact_id!, point.samples_path!) + .trim() + .split('\n') + .map((line) => JSON.parse(line)); + const documents = new Map>(); + for (const sample of records) { + finiteJson(sample); + if ( + !Number.isSafeInteger(sample.doc_id) || + sample.doc_id < 0 || + !point.filters.includes(sample.filter) || + (sample.task_name && sample.task_name !== point.task) + ) + throw new Error('Invalid evaluation sample identity'); + const filters = documents.get(sample.doc_id) ?? new Set(); + if (filters.has(sample.filter)) throw new Error('Duplicate evaluation sample/filter'); + filters.add(sample.filter); + documents.set(sample.doc_id, filters); + } + if (row.metrics.n_eff !== point.sample_count) + throw new Error('Evaluation summary sample count mismatch'); + if ( + documents.size !== point.sample_count || + [...documents.keys()].some((id) => id >= point.sample_count) || + [...documents.values()].some((filters) => filters.size !== point.filters.length) + ) + throw new Error('Incomplete evaluation sample/filter coverage'); + } + } + for (const [key, covered] of expectedByFile) { + if (actualByFile.get(key) !== covered.size) + throw new Error('Unexpected/conflicting normalized point multiplicity'); + } +} + +export interface PublicationRecord { + kind: 'publication-record'; + version: 1; + receipt_id: string; + receipt_artifact_id: number; + receipt_artifact_sha256: string; + source_run_id: string; + merge_run_id: string; + merge_sha: string; + changelog_artifact_id: number; + changelog_artifact_sha256: string; + ingest_sha: string; + app_sha: string; +} +export function publicationFromEnvironment( + receipt: MeasurementReceipt, + mergeRunId: string, + env: Record = process.env, +): PublicationRecord | null { + if (!env.INGEST_PUBLICATION_RECORD_PATH) { + if (mergeRunId !== receipt.source_run_id) + throw new Error('Production/recovery requires a publication record'); + return null; + } + const bytes = fs.readFileSync(env.INGEST_PUBLICATION_RECORD_PATH); + if ( + !digest(env.INGEST_PUBLICATION_RECORD_SHA256) || + sha256(bytes) !== env.INGEST_PUBLICATION_RECORD_SHA256 + ) + throw new Error('Publication record digest mismatch'); + const record = JSON.parse(bytes.toString('utf8')) as PublicationRecord; + if ( + record.kind !== 'publication-record' || + record.version !== 1 || + record.receipt_id !== receipt.receipt_id || + record.source_run_id !== receipt.source_run_id || + record.merge_run_id !== mergeRunId || + !positive(record.receipt_artifact_id) || + !digest(record.receipt_artifact_sha256) || + !positive(record.changelog_artifact_id) || + !digest(record.changelog_artifact_sha256) || + ![record.merge_sha, record.ingest_sha, record.app_sha].every((value) => + /^[a-f0-9]{40}$/u.test(value), + ) + ) + throw new Error('Invalid/unsupported publication record'); + return record; +} diff --git a/packages/db/src/lib/receipt-artifact-preparation.test.ts b/packages/db/src/lib/receipt-artifact-preparation.test.ts new file mode 100644 index 000000000..654b8ecd7 --- /dev/null +++ b/packages/db/src/lib/receipt-artifact-preparation.test.ts @@ -0,0 +1,101 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { afterEach, expect, it, vi } from 'vitest'; +import { prepareReceiptArtifacts } from './receipt-artifact-preparation'; +import { parseMeasurementReceipt } from './measurement-receipt'; +import { downloadArtifact } from './github-artifacts'; + +vi.mock('node:child_process', () => ({ execFileSync: vi.fn() })); +const fixture = new URL('fixtures/measurement-receipt/', import.meta.url); +const bytes = fs.readFileSync(new URL('receipt.json', fixture)); +const roots: string[] = []; +function root() { + const value = fs.mkdtempSync(path.join(os.tmpdir(), 'receipt-download-')); + roots.push(value); + return value; +} +afterEach(() => { + vi.resetAllMocks(); + roots.splice(0).forEach((value) => fs.rmSync(value, { recursive: true, force: true })); +}); +function receipt() { + return parseMeasurementReceipt( + bytes, + '00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6', + 'd'.repeat(40), + ); +} +it('downloads exact retained IDs and verifies every member before preparing legacy discovery views', () => { + const accepted = receipt(); + vi.mocked(execFileSync).mockImplementation((command, args) => { + if (command !== 'gh') throw new Error('unexpected executable'); + const endpoint = (args as string[])[1]; + const id = Number(endpoint.match(/artifacts\/(?\d+)/u)?.[1]); + const artifact = accepted.artifacts.find((item) => item.id === id); + if (!artifact) throw new Error('unrequested newer artifact'); + if (endpoint.endsWith('/zip')) return fs.readFileSync(new URL(`${id}.zip`, fixture)); + return JSON.stringify({ + id, + name: artifact.name, + expired: false, + workflow_run: { id: 100 }, + digest: `sha256:${artifact.sha256}`, + archive_download_url: `https://api.github.com/repos/org/repo/actions/artifacts/${id}/zip`, + }); + }); + const destination = root(); + prepareReceiptArtifacts(accepted, destination); + expect(fs.readdirSync(path.join(destination, '.receipt-objects')).toSorted()).toEqual([ + '101', + '102', + ]); + expect( + JSON.parse(fs.readFileSync(path.join(destination, 'bmk_pilot/agg.json'), 'utf8'))[0] + .output_tput_tps, + ).toBe(100); + expect( + fs + .readFileSync(path.join(destination, 'eval_results_all/samples_gsm8k.jsonl'), 'utf8') + .trim() + .split('\n'), + ).toHaveLength(4); +}); +it('fails on missing/wrong ownership even if a same-name artifact exists', () => { + const accepted = receipt(); + vi.mocked(execFileSync).mockReturnValue( + JSON.stringify({ + id: 999, + name: 'bmk_pilot', + workflow_run: { id: 100 }, + digest: `sha256:${accepted.artifacts[0].sha256}`, + }), + ); + const destination = root(); + expect(() => prepareReceiptArtifacts(accepted, destination)).toThrow( + 'ownership or digest mismatch', + ); + expect(fs.existsSync(path.join(destination, 'bmk_pilot'))).toBe(false); +}); +it('treats hostile display names as data under numeric roots and never interprets their URL', () => { + vi.mocked(execFileSync).mockReturnValue(fs.readFileSync(new URL('101.zip', fixture))); + const destination = root(); + const output = downloadArtifact( + { + id: 101, + name: '$(touch injected)', + created_at: '', + archive_download_url: 'https://attacker.invalid/$(touch injected)', + }, + destination, + { repo: 'org/repo', isolated: true }, + ); + expect(output).toBe(path.join(destination, '101')); + expect(fs.readdirSync(destination)).toEqual(['101']); + expect(JSON.parse(fs.readFileSync(path.join(output, 'agg.json'), 'utf8'))[0].conc).toBe(1); + expect(vi.mocked(execFileSync).mock.calls[0].slice(0, 2)).toEqual([ + 'gh', + ['api', 'repos/org/repo/actions/artifacts/101/zip'], + ]); +}); diff --git a/packages/db/src/lib/receipt-artifact-preparation.ts b/packages/db/src/lib/receipt-artifact-preparation.ts new file mode 100644 index 000000000..697884393 --- /dev/null +++ b/packages/db/src/lib/receipt-artifact-preparation.ts @@ -0,0 +1,45 @@ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { downloadArtifact, type ArtifactMeta } from './github-artifacts'; +import { verifyMeasurementSnapshot, type MeasurementReceipt } from './measurement-receipt'; + +/** Fetch exact IDs individually, including retained uploads excluded by latest-only APIs. */ +export function prepareReceiptArtifacts(receipt: MeasurementReceipt, root: string): void { + const objects = path.join(root, '.receipt-objects'); + fs.mkdirSync(objects, { recursive: true }); + for (const artifact of receipt.artifacts) { + const metadata = JSON.parse( + execFileSync('gh', ['api', `repos/${receipt.repository}/actions/artifacts/${artifact.id}`], { + encoding: 'utf8', + }), + ) as ArtifactMeta; + if ( + metadata.id !== artifact.id || + metadata.name !== artifact.name || + metadata.expired || + String(metadata.workflow_run?.id) !== artifact.run_id || + metadata.digest !== `sha256:${artifact.sha256}` + ) + throw new Error(`Receipt/API ownership or digest mismatch: ${artifact.id}`); + downloadArtifact(metadata, objects, { + repo: receipt.repository, + sha256: artifact.sha256, + members: artifact.members, + isolated: true, + }); + } + // Preserve existing consumer discovery while extraction itself is isolated by immutable ID. + // Names have already been checked as unique, single, non-special path components. + for (const artifact of receipt.artifacts) { + const target = path.join(root, artifact.name); + if (fs.existsSync(target)) + throw new Error(`Refusing artifact view overwrite: ${artifact.name}`); + fs.cpSync(path.join(objects, String(artifact.id)), target, { + recursive: true, + errorOnExist: true, + force: false, + }); + } + verifyMeasurementSnapshot(receipt, root); +} diff --git a/packages/db/src/lib/receipt-transport.test.ts b/packages/db/src/lib/receipt-transport.test.ts new file mode 100644 index 000000000..f3230ad29 --- /dev/null +++ b/packages/db/src/lib/receipt-transport.test.ts @@ -0,0 +1,98 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import AdmZip from 'adm-zip'; +import { execFileSync } from 'node:child_process'; +import { afterEach, expect, it, vi } from 'vitest'; +import { prepareReceiptTransport } from './receipt-transport'; +import { sha256 } from './artifact-archive'; +vi.mock('node:child_process', () => ({ execFileSync: vi.fn() })); +const roots: string[] = []; +function root() { + const value = fs.mkdtempSync(path.join(os.tmpdir(), 'receipt-transport-')); + roots.push(value); + return value; +} +afterEach(() => { + vi.resetAllMocks(); + roots.splice(0).forEach((value) => fs.rmSync(value, { recursive: true, force: true })); +}); +const raw = fs.readFileSync(new URL('fixtures/measurement-receipt/receipt.json', import.meta.url)); +const zip = new AdmZip(); +zip.addFile('receipt.json', raw); +const archive = Buffer.from(zip.toBuffer()); +const env = { + INGEST_REPO: 'org/repo', + SOURCE_RUN_ID: '100', + RECEIPT_REQUIRED: 'false', + RECEIPT_ARTIFACT_ID: '301', + RECEIPT_ARTIFACT_SHA256: sha256(archive), + RECEIPT_SHA256: '00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6', + RECEIPT_ISSUER_RUN_ID: '200', + RECEIPT_ISSUER_SHA: 'd'.repeat(40), + ALLOWED_RECEIPT_ISSUER_SHAS: 'd'.repeat(40), + ALLOWED_RECEIPT_ISSUER_WORKFLOW: '.github/workflows/issue-receipt.yml', +}; +function api(endpoint: string) { + if (endpoint.endsWith('/runs/100/artifacts')) + return JSON.stringify([ + { artifacts: [{ id: 101, name: 'native-execution-point', created_at: '' }] }, + ]); + if (endpoint.endsWith('/runs/200')) + return JSON.stringify({ + id: 200, + head_sha: 'd'.repeat(40), + path: '.github/workflows/issue-receipt.yml', + status: 'completed', + conclusion: 'success', + }); + if (endpoint.endsWith('/301/zip')) return archive; + if (endpoint.endsWith('/artifacts/301')) + return JSON.stringify({ + id: 301, + name: 'receipt-display', + workflow_run: { id: 200 }, + expired: false, + digest: `sha256:${sha256(archive)}`, + archive_download_url: 'https://api.github.com/repos/org/repo/actions/artifacts/301/zip', + }); + throw new Error('unrequested endpoint'); +} +it('enforces native capability from API inventory and accepts only the pinned successful issuer', () => { + vi.mocked(execFileSync).mockImplementation((_command, args) => api((args as string[])[1])); + expect(() => prepareReceiptTransport({ ...env, RECEIPT_ARTIFACT_ID: '' }, root())).toThrow( + 'Required source receipt', + ); + const values = prepareReceiptTransport(env, root()); + expect(values.INGEST_RECEIPT_REQUIRED).toBe('1'); + expect(JSON.parse(fs.readFileSync(values.INGEST_RECEIPT_PATH, 'utf8')).source_attempt).toBe(2); + expect(() => + prepareReceiptTransport({ ...env, RECEIPT_ISSUER_SHA: 'f'.repeat(40) }, root()), + ).toThrow('not deployed/allowed'); +}); +it('does not promote a same-named candidate workflow artifact to issuer authority', () => { + vi.mocked(execFileSync).mockImplementation((_command, args) => { + const endpoint = (args as string[])[1]; + if (endpoint.endsWith('/runs/200')) + return JSON.stringify({ + id: 200, + head_sha: 'd'.repeat(40), + path: '.github/workflows/candidate.yml', + status: 'completed', + conclusion: 'success', + }); + return api(endpoint); + }); + const destination = root(); + expect(() => prepareReceiptTransport(env, destination)).toThrow('successful allowed workflow'); + expect(fs.readdirSync(destination)).toEqual([]); +}); +it('keeps explicit legacy behavior only when API inventory does not identify the native lane', () => { + vi.mocked(execFileSync).mockReturnValue('[{"artifacts":[]}]'); + expect( + prepareReceiptTransport( + { INGEST_REPO: 'org/repo', SOURCE_RUN_ID: '100', RECEIPT_REQUIRED: 'false' }, + root(), + ), + ).toEqual({ INGEST_RECEIPT_REQUIRED: '0' }); +}); diff --git a/packages/db/src/lib/receipt-transport.ts b/packages/db/src/lib/receipt-transport.ts new file mode 100644 index 000000000..548fb51ee --- /dev/null +++ b/packages/db/src/lib/receipt-transport.ts @@ -0,0 +1,126 @@ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { downloadArtifact, listRunArtifacts, type ArtifactMeta } from './github-artifacts'; +import { parseMeasurementReceipt } from './measurement-receipt'; +import { sha256 } from './artifact-archive'; + +function required(value: string | undefined, name: string, pattern: RegExp): string { + if (!value || !pattern.test(value)) throw new Error(`Invalid or missing ${name}`); + return value; +} +const ID = /^[1-9]\d*$/u; +const HASH = /^[a-f0-9]{64}$/u; +const SHA = /^[a-f0-9]{40}$/u; + +/** Executed from the deployed app checkout. Allowed issuer pins are app configuration, not payload policy. */ +export function prepareReceiptTransport( + env: Record, + destination: string, +): Record { + if (env.RECEIPT_REQUIRED !== 'true' && env.RECEIPT_REQUIRED !== 'false') + throw new Error('receipt-required must be explicitly true or false'); + const repo = required(env.INGEST_REPO, 'INGEST_REPO', /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u); + const sourceRun = required(env.SOURCE_RUN_ID, 'source run ID', ID); + const nativeRequired = listRunArtifacts(repo, sourceRun).some((artifact) => + artifact.name.startsWith('native-execution-'), + ); + if (!env.RECEIPT_ARTIFACT_ID) { + if (env.RECEIPT_REQUIRED === 'true' || nativeRequired) + throw new Error('Required source receipt transport missing'); + return { INGEST_RECEIPT_REQUIRED: '0' }; + } + const allowed = (env.ALLOWED_RECEIPT_ISSUER_SHAS ?? '').split(',').map((value) => value.trim()); + if (allowed.length === 0 || allowed.some((value) => !SHA.test(value))) + throw new Error('Missing/invalid deployed issuer revision allowlist'); + const issuerSha = required(env.RECEIPT_ISSUER_SHA, 'issuer revision', SHA); + if (!allowed.includes(issuerSha)) + throw new Error('Requested issuer revision is not deployed/allowed'); + const issuerWorkflow = required( + env.ALLOWED_RECEIPT_ISSUER_WORKFLOW, + 'deployed issuer workflow', + /^\.github\/workflows\/[A-Za-z0-9_.-]+\.ya?ml$/u, + ); + const issuerRun = required(env.RECEIPT_ISSUER_RUN_ID, 'issuer run ID', ID); + function verifyIssuer(runId: string, revision: string): void { + if (!allowed.includes(revision)) throw new Error('Publisher revision is not deployed/allowed'); + const run = JSON.parse( + execFileSync('gh', ['api', `repos/${repo}/actions/runs/${runId}`], { encoding: 'utf8' }), + ); + if ( + String(run.id) !== runId || + run.head_sha !== revision || + run.path !== issuerWorkflow || + run.status !== 'completed' || + run.conclusion !== 'success' + ) + throw new Error('Receipt issuer is not the successful allowed workflow execution'); + } + verifyIssuer(issuerRun, issuerSha); + fs.mkdirSync(destination, { recursive: true }); + function download(id: string, digest: string, ownerRun = issuerRun): string { + const metadata = JSON.parse( + execFileSync('gh', ['api', `repos/${repo}/actions/artifacts/${id}`], { encoding: 'utf8' }), + ) as ArtifactMeta; + if ( + String(metadata.id) !== id || + String(metadata.workflow_run?.id) !== ownerRun || + metadata.expired || + metadata.digest !== `sha256:${digest}` + ) + throw new Error('Receipt transport artifact ownership/digest mismatch'); + return downloadArtifact(metadata, destination, { repo, sha256: digest, isolated: true }); + } + const id = required(env.RECEIPT_ARTIFACT_ID, 'receipt artifact ID', ID); + const archiveDigest = required(env.RECEIPT_ARTIFACT_SHA256, 'receipt archive digest', HASH); + const contentDigest = required(env.RECEIPT_SHA256, 'receipt JSON digest', HASH); + const directory = download(id, archiveDigest); + const file = path.join(directory, 'receipt.json'); + const receipt = parseMeasurementReceipt(fs.readFileSync(file), contentDigest, issuerSha); + if ( + receipt.issuer.repository !== repo || + receipt.issuer.run_id !== issuerRun || + receipt.repository !== repo || + receipt.source_run_id !== env.SOURCE_RUN_ID + ) + throw new Error('Receipt source or issuer identity differs from requested transport'); + const out: Record = { + INGEST_RECEIPT_REQUIRED: '1', + INGEST_RECEIPT_PATH: file, + INGEST_RECEIPT_SHA256: contentDigest, + INGEST_RECEIPT_ISSUER_SHA: issuerSha, + }; + if (env.PUBLICATION_ARTIFACT_ID) { + const publicationId = required(env.PUBLICATION_ARTIFACT_ID, 'publication artifact ID', ID); + const publicationDigest = required( + env.PUBLICATION_ARTIFACT_SHA256, + 'publication archive digest', + HASH, + ); + const publicationSha = required(env.PUBLICATION_SHA256, 'publication JSON digest', HASH); + const publicationRun = required(env.PUBLICATION_ISSUER_RUN_ID, 'publication issuer run ID', ID); + const publicationRevision = required( + env.PUBLICATION_ISSUER_SHA, + 'publication issuer revision', + SHA, + ); + verifyIssuer(publicationRun, publicationRevision); + const publicationFile = path.join( + download(publicationId, publicationDigest, publicationRun), + 'publication.json', + ); + const publicationBytes = fs.readFileSync(publicationFile); + const publication = JSON.parse(publicationBytes.toString('utf8')); + if ( + publication.receipt_id !== receipt.receipt_id || + publication.receipt_artifact_id !== Number(id) || + publication.receipt_artifact_sha256 !== archiveDigest + ) + throw new Error('Publication refers to a different accepted source receipt'); + if (sha256(publicationBytes) !== publicationSha) + throw new Error('Publication JSON digest mismatch'); + out.INGEST_PUBLICATION_RECORD_PATH = publicationFile; + out.INGEST_PUBLICATION_RECORD_SHA256 = publicationSha; + } + return out; +} diff --git a/packages/db/src/prepare-ci-artifacts.ts b/packages/db/src/prepare-ci-artifacts.ts index fbf834d70..e9eb3553e 100644 --- a/packages/db/src/prepare-ci-artifacts.ts +++ b/packages/db/src/prepare-ci-artifacts.ts @@ -4,6 +4,9 @@ import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; +import { receiptFromEnvironment, publicationFromEnvironment } from './lib/measurement-receipt'; +import { prepareReceiptArtifacts } from './lib/receipt-artifact-preparation'; + import { buildArtifactPlan } from './lib/ci-artifact-preparation.js'; import { downloadArtifact, listRunArtifacts, type ArtifactMeta } from './lib/github-artifacts.js'; @@ -33,6 +36,8 @@ function fetchRunMetadata(repo: string, runId: string): GithubRunMetadata { } function downloadWithRetries(artifact: ArtifactMeta, artifactsPath: string, attempt = 1): void { + if (!/^[A-Za-z0-9_.-]+$/u.test(artifact.name) || ['.', '..'].includes(artifact.name)) + throw new Error('Unsafe legacy artifact display name'); const zipPath = path.join(artifactsPath, 'artifact.zip'); const artifactPath = path.join(artifactsPath, artifact.name); try { @@ -108,6 +113,63 @@ function main(): void { const repo = process.env.INGEST_REPO ?? DEFAULT_REPO; const artifactsPath = process.env.ARTIFACTS_PATH ?? path.resolve('artifacts'); + const receipt = receiptFromEnvironment(); + if (receipt) { + if (receipt.repository !== repo || receipt.source_run_id !== sourceRunId) + throw new Error('Receipt source differs from requested source'); + const publication = publicationFromEnvironment(receipt, mergeRunId); + if (dryRun) { + console.log( + `Verified receipt ${receipt.receipt_id}; ${receipt.artifacts.length} immutable artifacts`, + ); + return; + } + fs.mkdirSync(artifactsPath, { recursive: true }); + if (fs.readdirSync(artifactsPath).length > 0) throw new Error('ARTIFACTS_PATH must be empty'); + prepareReceiptArtifacts(receipt, artifactsPath); + let mergeAttempt = receipt.source_attempt; + if (publication) { + const metadata = JSON.parse( + execFileSync( + 'gh', + ['api', `repos/${repo}/actions/artifacts/${publication.changelog_artifact_id}`], + { encoding: 'utf8' }, + ), + ) as ArtifactMeta; + if ( + metadata.id !== publication.changelog_artifact_id || + metadata.name !== 'changelog-metadata' || + String(metadata.workflow_run?.id) !== mergeRunId || + metadata.digest !== `sha256:${publication.changelog_artifact_sha256}` + ) + throw new Error('Publication changelog mismatch'); + downloadArtifact(metadata, artifactsPath, { + repo, + sha256: publication.changelog_artifact_sha256, + }); + const merge = fetchRunMetadata(repo, mergeRunId); + mergeAttempt = merge.run_attempt ?? 1; + if (merge.head_sha !== publication.merge_sha) + throw new Error('Publication merge revision mismatch'); + writeReuseMetadata( + artifactsPath, + sourceRunId, + mergeRunId, + { head_sha: receipt.source_head_sha, run_attempt: receipt.source_attempt }, + merge, + ); + } + writeOutputs({ + 'source-run-id': sourceRunId, + 'source-run-attempt': receipt.source_attempt, + 'merge-run-id': mergeRunId, + 'merge-run-attempt': mergeAttempt, + reused: sourceRunId !== mergeRunId, + 'receipt-id': receipt.receipt_id, + }); + return; + } + const sourceMetadata = fetchRunMetadata(repo, sourceRunId); const mergeMetadata = mergeRunId === sourceRunId ? sourceMetadata : fetchRunMetadata(repo, mergeRunId); diff --git a/packages/db/src/prepare-receipt-transport.ts b/packages/db/src/prepare-receipt-transport.ts new file mode 100644 index 000000000..68b07ec4a --- /dev/null +++ b/packages/db/src/prepare-receipt-transport.ts @@ -0,0 +1,11 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { prepareReceiptTransport } from './lib/receipt-transport'; +const root = process.env.RECEIPT_TRANSPORT_PATH; +const output = process.env.GITHUB_ENV; +if (!root || !output) throw new Error('RECEIPT_TRANSPORT_PATH and GITHUB_ENV are required'); +const values = prepareReceiptTransport(process.env, path.resolve(root)); +for (const [key, value] of Object.entries(values)) { + if (/[\r\n]/u.test(value)) throw new Error('Invalid transport environment value'); + fs.appendFileSync(output, `${key}=${value}\n`); +} diff --git a/packages/db/src/verify-measurement-publication.ts b/packages/db/src/verify-measurement-publication.ts new file mode 100644 index 000000000..bc3c46b01 --- /dev/null +++ b/packages/db/src/verify-measurement-publication.ts @@ -0,0 +1,92 @@ +/** Read-only, receipt-driven check of exact-run and latest visibility plus raw/sample details. */ +import fs from 'node:fs'; +import { DB_MODEL_TO_DISPLAY } from '@semianalysisai/inferencex-constants'; +import { receiptFromEnvironment } from './lib/measurement-receipt'; +import { + expectedPublication, + publishedPointMatches, + verifyPublishedMeasurements, + type PublishedMeasurement, +} from './etl/measurement-publication'; + +const receipt = receiptFromEnvironment(); +if (!receipt) throw new Error('An immutable measurement receipt is required'); +const root = process.env.INGEST_ARTIFACTS_PATH; +const origin = process.argv[2]; +const output = process.argv[3]; +if (!root || !origin || !output) + throw new Error( + 'Usage: verify-measurement-publication.ts ; INGEST_ARTIFACTS_PATH and receipt environment are required', + ); +const expected = expectedPublication(receipt, root); +async function readApi(route: string, params: Record = {}): Promise { + const url = new URL(route, origin); + url.search = new URLSearchParams(params).toString(); + const response = await fetch(url, { + signal: AbortSignal.timeout(30_000), + headers: process.env.CACHE_PROTECTION_BYPASS_SECRET + ? { 'x-vercel-protection-bypass': process.env.CACHE_PROTECTION_BYPASS_SECRET } + : undefined, + }); + if (!response.ok) throw new Error(`Publication request failed: ${url} HTTP ${response.status}`); + return response.json(); +} +const errors: string[] = []; +const benchmarks: PublishedMeasurement[] = []; +const latest: PublishedMeasurement[] = []; +for (const modelKey of new Set(receipt.points.map((point) => point.config.model))) { + const model = DB_MODEL_TO_DISPLAY[modelKey]; + if (!model) throw new Error(`Unmapped public model: ${modelKey}`); + benchmarks.push( + ...(await readApi('/api/v1/benchmarks', { + model, + runId: receipt.source_run_id, + exactRun: 'true', + })), + ); + latest.push(...(await readApi('/api/v1/benchmarks', { model }))); +} +errors.push( + ...verifyPublishedMeasurements(expected, benchmarks, 'throughput', 'exact run'), + ...verifyPublishedMeasurements(expected, latest, 'throughput', 'latest curve'), +); +const evals = ((await readApi('/api/v1/evaluations')) as PublishedMeasurement[]).filter( + (row) => + row.run_url === + `https://github.com/${receipt.repository}/actions/runs/${receipt.source_run_id}`, +); +errors.push(...verifyPublishedMeasurements(expected, evals, 'eval', 'evaluation API')); +for (const item of expected) { + const rows = (item.point.kind === 'throughput' ? benchmarks : evals).filter((row) => + publishedPointMatches(item.point, row), + ); + if (rows.length !== 1) continue; + if (item.point.kind === 'throughput') { + const availability = await readApi('/api/v1/trace-availability', { ids: String(rows[0].id) }); + if (availability[String(rows[0].id)] !== true) + errors.push(`${item.point.point_id}: missing raw trace detail`); + } else { + const detail = await readApi('/api/v1/eval-samples', { + eval_result_id: String(rows[0].id), + filter: 'all', + limit: '1', + }); + if ( + detail.total !== item.point.sample_count || + detail.passedTotal !== item.strictPassed || + detail.failedTotal !== item.point.sample_count - item.strictPassed! + ) + errors.push(`${item.point.point_id}: strict sample detail coverage differs`); + } +} +const result = { + version: 1, + receipt_id: receipt.receipt_id, + checked_at: new Date().toISOString(), + points: expected.length, + status: errors.length > 0 ? 'failed' : 'matched', + errors, +}; +fs.writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`); +console.log(JSON.stringify(result, null, 2)); +if (errors.length > 0) process.exitCode = 1; From be943e7824c0be6dbcfd73f842d0b2a4c15af505 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:21:01 -0400 Subject: [PATCH 2/5] fix: require publication provenance for every native production import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Derive publication requirements from the receiving database target, verify original source attempts and trusted issuer events, and reject records whose ingest revision differs from the executing checkout. Preserve repository token semantics and document the auditor-only credential advisories without suppression. 中文：为所有 native 生产导入强制验证发布来源。根据接收端实际数据库目标 确定发布记录要求，校验源运行原始 attempt 和受信签发事件，并拒绝与执行 checkout 版本不一致的发布记录。保留现有仓库凭据语义，在文档中说明仅由 auditor 模式报告的凭据提示，不增加抑制规则。 Validation: 880 DB tests, typecheck, lint, formatting/diff checks pass. The repository-wide configured zizmor audit has zero findings; the separate auditor persona retains 17 known repository-secret advisories (15 baseline). 验证：880 项数据库测试、类型检查、代码检查及格式检查通过；仓库实际 配置的 zizmor 检查结果为零，单独运行 auditor 模式仍报告 17 项仓库级凭据 提示，其中 15 项在原有基线中已存在。 --- .github/workflows/ingest-agentic-results.yml | 1 + .github/workflows/ingest-results.yml | 2 + .github/workflows/stage-results.yml | 4 +- docs/measurement-publication.md | 4 + docs/measurement-publication_zh.md | 4 + .../db/src/lib/measurement-receipt.test.ts | 6 ++ packages/db/src/lib/measurement-receipt.ts | 4 +- packages/db/src/lib/receipt-transport.test.ts | 92 ++++++++++++++++++- packages/db/src/lib/receipt-transport.ts | 22 +++++ 9 files changed, 136 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ingest-agentic-results.yml b/.github/workflows/ingest-agentic-results.yml index bf81e86f5..87b0b4596 100644 --- a/.github/workflows/ingest-agentic-results.yml +++ b/.github/workflows/ingest-agentic-results.yml @@ -386,6 +386,7 @@ jobs: ALLOWED_RECEIPT_ISSUER_SHAS: ${{ vars.INFX_RECEIPT_ISSUER_SHAS }} ALLOWED_RECEIPT_ISSUER_WORKFLOW: ${{ vars.INFX_RECEIPT_ISSUER_WORKFLOW }} RECEIPT_REQUIRED: ${{ github.event.client_payload.receipt-required || inputs.receipt-required || 'false' }} + PUBLICATION_REQUIRED: ${{ env.REQUESTED_DATABASE_TARGET == 'production' && 'true' || 'false' }} RECEIPT_ARTIFACT_ID: ${{ github.event.client_payload.receipt-artifact-id || inputs.receipt-artifact-id }} RECEIPT_ARTIFACT_SHA256: ${{ github.event.client_payload.receipt-artifact-sha256 || inputs.receipt-artifact-sha256 }} RECEIPT_SHA256: ${{ github.event.client_payload.receipt-sha256 || inputs.receipt-sha256 }} diff --git a/.github/workflows/ingest-results.yml b/.github/workflows/ingest-results.yml index 4c9c2ac40..40fd8c0b8 100644 --- a/.github/workflows/ingest-results.yml +++ b/.github/workflows/ingest-results.yml @@ -52,6 +52,7 @@ jobs: - name: Verify immutable receipt transport env: + # Existing cross-repository credential reads pinned issuer metadata and receipt artifacts. GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }} INGEST_REPO: SemiAnalysisAI/InferenceX SOURCE_RUN_ID: ${{ github.event.client_payload.source-run-id || github.event.client_payload.run-id }} @@ -59,6 +60,7 @@ jobs: ALLOWED_RECEIPT_ISSUER_SHAS: ${{ vars.INFX_RECEIPT_ISSUER_SHAS }} ALLOWED_RECEIPT_ISSUER_WORKFLOW: ${{ vars.INFX_RECEIPT_ISSUER_WORKFLOW }} RECEIPT_REQUIRED: ${{ github.event.client_payload.receipt-required || 'false' }} + PUBLICATION_REQUIRED: 'true' RECEIPT_ARTIFACT_ID: ${{ github.event.client_payload.receipt-artifact-id }} RECEIPT_ARTIFACT_SHA256: ${{ github.event.client_payload.receipt-artifact-sha256 }} RECEIPT_SHA256: ${{ github.event.client_payload.receipt-sha256 }} diff --git a/.github/workflows/stage-results.yml b/.github/workflows/stage-results.yml index 88356bed3..885870dc1 100644 --- a/.github/workflows/stage-results.yml +++ b/.github/workflows/stage-results.yml @@ -182,6 +182,7 @@ jobs: CYPRESS_INSTALL_BINARY: '0' - name: Verify immutable receipt transport env: + # Existing cross-repository credential reads pinned issuer metadata and receipt artifacts. GH_TOKEN: ${{ secrets.INFX_MAIN_PAT }} INGEST_REPO: SemiAnalysisAI/InferenceX SOURCE_RUN_ID: ${{ steps.request.outputs.run-id }} @@ -189,6 +190,7 @@ jobs: ALLOWED_RECEIPT_ISSUER_SHAS: ${{ vars.INFX_RECEIPT_ISSUER_SHAS }} ALLOWED_RECEIPT_ISSUER_WORKFLOW: ${{ vars.INFX_RECEIPT_ISSUER_WORKFLOW }} RECEIPT_REQUIRED: ${{ github.event.client_payload.receipt-required || inputs.receipt-required || 'false' }} + PUBLICATION_REQUIRED: 'false' RECEIPT_ARTIFACT_ID: ${{ github.event.client_payload.receipt-artifact-id || inputs.receipt-artifact-id }} RECEIPT_ARTIFACT_SHA256: ${{ github.event.client_payload.receipt-artifact-sha256 || inputs.receipt-artifact-sha256 }} RECEIPT_SHA256: ${{ github.event.client_payload.receipt-sha256 || inputs.receipt-sha256 }} @@ -343,7 +345,7 @@ jobs: needs: [validate, prepare-staging-database] permissions: contents: read - uses: ./.github/workflows/ingest-agentic-results.yml + uses: $/.github/workflows/ingest-agentic-results.yml with: run-id: ${{ needs.validate.outputs.run-id }} run-attempt: ${{ needs.validate.outputs.run-attempt }} diff --git a/docs/measurement-publication.md b/docs/measurement-publication.md index 265c72b29..aa7bfd5b2 100644 --- a/docs/measurement-publication.md +++ b/docs/measurement-publication.md @@ -8,8 +8,12 @@ Phase 1 readers accept a version-1 source measurement receipt produced by the in The app repository must configure `INFX_RECEIPT_ISSUER_SHAS` as a comma-separated allowlist of reviewed issuer revisions, and `INFX_RECEIPT_ISSUER_WORKFLOW` as the exact `.github/workflows/.yml` path. Keep prior accepted revisions while their receipts remain supported. The receiver checks successful completed issuer runs, exact repository/run ownership, API digest, ZIP bytes and member paths. Payloads cannot choose arbitrary download URLs or trusted code. Dispatch publication only after its issuer finishes successfully. +Receipt reads use the existing repository-scoped `INFX_MAIN_PAT` with read access to source Actions runs and artifacts. Keep it confined to the steps that need it; moving secrets into a protected GitHub Environment requires migrating the stored credential and its policy, not adding an empty workflow `environment`. The app's configured zizmor check passes; the separate auditor persona additionally reports this existing credential architecture, including the new receipt consumers. These advisories are not suppressed. + Dispatch fields are `receipt-required`, `receipt-artifact-id`, `receipt-artifact-sha256`, `receipt-sha256`, `receipt-issuer-run-id`, and `receipt-issuer-sha`. ZIP and JSON digests are distinct. A receipt ZIP contains `receipt.json`. Production/recovery adds `publication-artifact-id`, `publication-artifact-sha256`, `publication-sha256`, `publication-issuer-run-id`, and `publication-issuer-sha`; that later ZIP contains `publication.json`. Both staging and ingest workflows forward these fields. Staging checks transport before any optional database reset. +The receiving workflow derives `PUBLICATION_REQUIRED` from its actual database target. Native production requires a later record even when source and merge run IDs are equal; staging can use only the source receipt. Issuers must be completed successful `workflow_dispatch` runs on `main`, and the receiver independently verifies the original source attempt and head through the API. In GitHub Actions, the record's `ingest_sha` must match the executing app checkout's `GITHUB_SHA`. Pin the source repository's `INFX_PHASE1_READER_REVISION` to that deployed app commit before issuing publication records, and retain deployment evidence that the public app uses the recorded `app_sha`. + The reader uses separately versioned `aiperf-1.4`, `agentx-v1` and publication contract 1. Unsupported required versions fail before import. Every point binds its original execution/attempt, prepared bundle, native manifest, physical topology, canonical model/hardware/framework/precision, required metrics, full dataset identity and exact artifact/member references. Normalized AgentX JSON and per-job raw lm-eval results plus `meta_env.json` are supported. Evaluation requires complete `(task, doc_id, filter)` coverage. Both live preview and stored document-level samples select strict-match independent of line order and reject conflicting copies. Aggregate metadata maps to one physical serving role with compatibility worker counts 0/0. Artifacts are downloaded with argv-based `gh` calls and extracted under numeric ID roots. Validation rejects escaping paths, links, duplicate normalized names, file/directory collisions, overwrites and changed member sets. Checked compatibility views retain existing consumer discovery names. All snapshot bytes and normalized requirements are verified before the first database write. diff --git a/docs/measurement-publication_zh.md b/docs/measurement-publication_zh.md index 117c8dce1..1c690002b 100644 --- a/docs/measurement-publication_zh.md +++ b/docs/measurement-publication_zh.md @@ -8,8 +8,12 @@ Phase 1 的读取端接受由 InferenceX 独立受信托管签发流程生成的 应用仓库必须配置 `INFX_RECEIPT_ISSUER_SHAS`，其中以逗号分隔经过审查的签发流程版本；同时将 `INFX_RECEIPT_ISSUER_WORKFLOW` 设置为准确的 `.github/workflows/.yml` 路径。只要旧回执仍受支持，就应保留对应的允许版本。接收端检查签发运行已成功完成、仓库及运行归属准确、API digest、ZIP 字节和成员路径一致。payload 不能选择任意下载 URL 或受信代码。签发流程成功完成后，才能调度发布。 +读取回执沿用仓库级凭据 `INFX_MAIN_PAT`，并要求它具有读取源 Actions 运行及产物的权限。凭据只传给实际需要它的步骤；迁移到受保护的 GitHub Environment 时，必须同时迁移已存储凭据及其策略，仅在工作流里添加空的 `environment` 无法完成迁移。应用当前配置的 zizmor 检查通过；单独运行 auditor persona 时，会额外提示这种既有凭据架构，包括新增的回执读取步骤。这些提示未被抑制。 + 调度字段为 `receipt-required`、`receipt-artifact-id`、`receipt-artifact-sha256`、`receipt-sha256`、`receipt-issuer-run-id` 和 `receipt-issuer-sha`。ZIP digest 与 JSON digest 分别校验。回执 ZIP 包含 `receipt.json`。生产发布及恢复还需提供 `publication-artifact-id`、`publication-artifact-sha256`、`publication-sha256`、`publication-issuer-run-id` 和 `publication-issuer-sha`；后续 ZIP 包含 `publication.json`。staging 与 ingest 工作流都会转发这些字段。staging 会先验证传输，再执行可选的数据库重置。 +接收工作流根据实际数据库目标确定 `PUBLICATION_REQUIRED`。native 生产发布即使源运行与 merge 运行 ID 相同，也必须提供后续发布记录；staging 可以只使用源回执。签发运行必须由 `main` 上的 `workflow_dispatch` 触发并成功完成，接收端还会通过 API 独立验证源运行的原始 attempt 和 head。在 GitHub Actions 中，记录的 `ingest_sha` 必须匹配应用执行 checkout 的 `GITHUB_SHA`。签发发布记录前，应将源仓库的 `INFX_PHASE1_READER_REVISION` 固定到这个已部署的应用 commit，并保留公开应用确实使用所记录 `app_sha` 的部署证据。 + 读取端分别支持 `aiperf-1.4`、`agentx-v1` 和 publication contract 1。不支持的必需版本会在导入前报错。每个点都绑定其原始执行及 attempt、prepared bundle、native manifest、物理拓扑、规范化后的模型/硬件/framework/precision、必需指标、完整数据集身份，以及准确的产物和文件引用。输入支持规范化 AgentX JSON，也支持每个任务的原始 lm-eval 结果和 `meta_env.json`。评估必须完整覆盖 `(task, doc_id, filter)`。live preview 与数据库中的文档级样本统一选择 strict-match，不受行顺序影响；同一过滤器的冲突副本会被拒绝。聚合部署元数据映射到一个实际 serving role，兼容字段中的 worker 数量为 0/0。 产物下载使用 argv 形式调用 `gh`，并在以数字 ID 命名的目录中解压。校验会拒绝越界路径、链接、规范化后重名的成员、文件与目录冲突、覆盖写入以及成员集合变化。经过验证的兼容视图保留已有消费端的发现名称。在第一次数据库写入之前，必须验证完整 snapshot 的文件字节和规范化要求。 diff --git a/packages/db/src/lib/measurement-receipt.test.ts b/packages/db/src/lib/measurement-receipt.test.ts index 58a64b55f..c57e4a02f 100644 --- a/packages/db/src/lib/measurement-receipt.test.ts +++ b/packages/db/src/lib/measurement-receipt.test.ts @@ -91,6 +91,9 @@ it.each([ it('requires a later, immutable publication record when source and merge differ', () => { const receipt = parseMeasurementReceipt(raw, expectedDigest, issuerSha); expect(publicationFromEnvironment(receipt, '100', {})).toBeNull(); + expect(() => + publicationFromEnvironment(receipt, '100', { INGEST_PUBLICATION_REQUIRED: '1' }), + ).toThrow('requires a publication record'); expect(() => publicationFromEnvironment(receipt, '200', {})).toThrow( 'requires a publication record', ); @@ -117,6 +120,9 @@ it('requires a later, immutable publication record when source and merge differ' }; expect(publicationFromEnvironment(receipt, '200', env)?.receipt_id).toBe(receipt.receipt_id); expect(() => publicationFromEnvironment(receipt, '201', env)).toThrow('Invalid/unsupported'); + expect(() => + publicationFromEnvironment(receipt, '200', { ...env, GITHUB_SHA: '9'.repeat(40) }), + ).toThrow('executing app checkout'); }); it('accepts per-job lm-eval results through the real detail mapper without an aggregate collector', () => { const receipt = parseMeasurementReceipt(raw, expectedDigest, issuerSha); diff --git a/packages/db/src/lib/measurement-receipt.ts b/packages/db/src/lib/measurement-receipt.ts index f57f74fd4..c6c537c76 100644 --- a/packages/db/src/lib/measurement-receipt.ts +++ b/packages/db/src/lib/measurement-receipt.ts @@ -448,7 +448,7 @@ export function publicationFromEnvironment( env: Record = process.env, ): PublicationRecord | null { if (!env.INGEST_PUBLICATION_RECORD_PATH) { - if (mergeRunId !== receipt.source_run_id) + if (env.INGEST_PUBLICATION_REQUIRED === '1' || mergeRunId !== receipt.source_run_id) throw new Error('Production/recovery requires a publication record'); return null; } @@ -474,5 +474,7 @@ export function publicationFromEnvironment( ) ) throw new Error('Invalid/unsupported publication record'); + if (env.GITHUB_SHA && record.ingest_sha !== env.GITHUB_SHA) + throw new Error('Publication ingest revision differs from the executing app checkout'); return record; } diff --git a/packages/db/src/lib/receipt-transport.test.ts b/packages/db/src/lib/receipt-transport.test.ts index f3230ad29..82926fbbc 100644 --- a/packages/db/src/lib/receipt-transport.test.ts +++ b/packages/db/src/lib/receipt-transport.test.ts @@ -25,6 +25,7 @@ const env = { INGEST_REPO: 'org/repo', SOURCE_RUN_ID: '100', RECEIPT_REQUIRED: 'false', + PUBLICATION_REQUIRED: 'false', RECEIPT_ARTIFACT_ID: '301', RECEIPT_ARTIFACT_SHA256: sha256(archive), RECEIPT_SHA256: '00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6', @@ -43,6 +44,16 @@ function api(endpoint: string) { id: 200, head_sha: 'd'.repeat(40), path: '.github/workflows/issue-receipt.yml', + head_branch: 'main', + event: 'workflow_dispatch', + status: 'completed', + conclusion: 'success', + }); + if (endpoint.endsWith('/runs/100/attempts/2')) + return JSON.stringify({ + id: 100, + run_attempt: 2, + head_sha: 'a'.repeat(40), status: 'completed', conclusion: 'success', }); @@ -91,8 +102,87 @@ it('keeps explicit legacy behavior only when API inventory does not identify the vi.mocked(execFileSync).mockReturnValue('[{"artifacts":[]}]'); expect( prepareReceiptTransport( - { INGEST_REPO: 'org/repo', SOURCE_RUN_ID: '100', RECEIPT_REQUIRED: 'false' }, + { + INGEST_REPO: 'org/repo', + SOURCE_RUN_ID: '100', + RECEIPT_REQUIRED: 'false', + PUBLICATION_REQUIRED: 'true', + }, root(), ), ).toEqual({ INGEST_RECEIPT_REQUIRED: '0' }); }); + +it('requires later publication for native production even when source and merge IDs are equal', () => { + vi.mocked(execFileSync).mockImplementation((_command, args) => api((args as string[])[1])); + expect(() => prepareReceiptTransport({ ...env, PUBLICATION_REQUIRED: 'true' }, root())).toThrow( + 'Native production requires a later publication record', + ); +}); + +it('carries the immutable later record and mandatory-publication flag into production ingest', () => { + const publication = { + kind: 'publication-record', + version: 1, + receipt_id: JSON.parse(raw.toString('utf8')).receipt_id, + receipt_artifact_id: 301, + receipt_artifact_sha256: sha256(archive), + source_run_id: '100', + merge_run_id: '100', + merge_sha: 'a'.repeat(40), + changelog_artifact_id: 501, + changelog_artifact_sha256: 'b'.repeat(64), + ingest_sha: 'e'.repeat(40), + app_sha: 'e'.repeat(40), + }; + const content = Buffer.from(JSON.stringify(publication)); + const publicationZip = new AdmZip(); + publicationZip.addFile('publication.json', content); + const publicationArchive = Buffer.from(publicationZip.toBuffer()); + vi.mocked(execFileSync).mockImplementation((_command, args) => { + const endpoint = (args as string[])[1]; + if (endpoint.endsWith('/401/zip')) return publicationArchive; + if (endpoint.endsWith('/artifacts/401')) + return JSON.stringify({ + id: 401, + name: 'publication-record', + workflow_run: { id: 200 }, + expired: false, + digest: `sha256:${sha256(publicationArchive)}`, + archive_download_url: 'https://api.github.com/repos/org/repo/actions/artifacts/401/zip', + }); + return api(endpoint); + }); + const values = prepareReceiptTransport( + { + ...env, + PUBLICATION_REQUIRED: 'true', + PUBLICATION_ARTIFACT_ID: '401', + PUBLICATION_ARTIFACT_SHA256: sha256(publicationArchive), + PUBLICATION_SHA256: sha256(content), + PUBLICATION_ISSUER_RUN_ID: '200', + PUBLICATION_ISSUER_SHA: 'd'.repeat(40), + }, + root(), + ); + expect(values.INGEST_PUBLICATION_REQUIRED).toBe('1'); + expect( + JSON.parse(fs.readFileSync(values.INGEST_PUBLICATION_RECORD_PATH, 'utf8')).merge_run_id, + ).toBe('100'); +}); + +it.each([ + ['/runs/200', { head_branch: 'candidate' }, 'successful allowed workflow'], + ['/runs/200', { event: 'pull_request' }, 'successful allowed workflow'], + ['/runs/100/attempts/2', { head_sha: 'f'.repeat(40) }, 'completed original attempt'], + ['/runs/100/attempts/2', { run_attempt: 3 }, 'completed original attempt'], +])('rejects wrong issuer branch/event or original attempt: %s %j', (suffix, change, message) => { + vi.mocked(execFileSync).mockImplementation((_command, args) => { + const endpoint = (args as string[])[1]; + const result = api(endpoint); + return endpoint.endsWith(suffix) + ? JSON.stringify({ ...JSON.parse(result as string), ...change }) + : result; + }); + expect(() => prepareReceiptTransport(env, root())).toThrow(message); +}); diff --git a/packages/db/src/lib/receipt-transport.ts b/packages/db/src/lib/receipt-transport.ts index 548fb51ee..4462b9cd6 100644 --- a/packages/db/src/lib/receipt-transport.ts +++ b/packages/db/src/lib/receipt-transport.ts @@ -20,6 +20,8 @@ export function prepareReceiptTransport( ): Record { if (env.RECEIPT_REQUIRED !== 'true' && env.RECEIPT_REQUIRED !== 'false') throw new Error('receipt-required must be explicitly true or false'); + if (env.PUBLICATION_REQUIRED !== 'true' && env.PUBLICATION_REQUIRED !== 'false') + throw new Error('publication-required must be explicitly true or false'); const repo = required(env.INGEST_REPO, 'INGEST_REPO', /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u); const sourceRun = required(env.SOURCE_RUN_ID, 'source run ID', ID); const nativeRequired = listRunArtifacts(repo, sourceRun).some((artifact) => @@ -51,6 +53,8 @@ export function prepareReceiptTransport( String(run.id) !== runId || run.head_sha !== revision || run.path !== issuerWorkflow || + run.head_branch !== 'main' || + run.event !== 'workflow_dispatch' || run.status !== 'completed' || run.conclusion !== 'success' ) @@ -84,11 +88,29 @@ export function prepareReceiptTransport( receipt.source_run_id !== env.SOURCE_RUN_ID ) throw new Error('Receipt source or issuer identity differs from requested transport'); + const attempt = JSON.parse( + execFileSync( + 'gh', + ['api', `repos/${repo}/actions/runs/${sourceRun}/attempts/${receipt.source_attempt}`], + { encoding: 'utf8' }, + ), + ); + if ( + String(attempt.id) !== sourceRun || + attempt.run_attempt !== receipt.source_attempt || + attempt.head_sha !== receipt.source_head_sha || + attempt.status !== 'completed' || + attempt.conclusion !== 'success' + ) + throw new Error('Receipt source does not match its completed original attempt'); + if (env.PUBLICATION_REQUIRED === 'true' && !env.PUBLICATION_ARTIFACT_ID) + throw new Error('Native production requires a later publication record'); const out: Record = { INGEST_RECEIPT_REQUIRED: '1', INGEST_RECEIPT_PATH: file, INGEST_RECEIPT_SHA256: contentDigest, INGEST_RECEIPT_ISSUER_SHA: issuerSha, + INGEST_PUBLICATION_REQUIRED: env.PUBLICATION_REQUIRED === 'true' ? '1' : '0', }; if (env.PUBLICATION_ARTIFACT_ID) { const publicationId = required(env.PUBLICATION_ARTIFACT_ID, 'publication artifact ID', ID); From 5e2b92be433ed8c03f259ea9541a5ec3dbaf30b8 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:26:57 -0400 Subject: [PATCH 3/5] test: format and reseal measurement receipt fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the GitHub format gate by formatting the controlled JSON snapshots, rebuilding deterministic ZIPs, and resealing their member and receipt digests with the Python validator. Update fixed test expectations without weakening byte-level validation. 中文：格式化并重新封存测量回执测试样例，修复 GitHub 格式检查。 重新生成可复现 ZIP，使用 Python 验证器更新成员与回执摘要，并同步 固定测试预期，保留逐字节校验。 Validation: complete format check and all 880 DB tests pass. 验证：完整格式检查及全部 880 项数据库测试通过。 --- .../src/etl/measurement-publication.test.ts | 2 +- .../db/src/etl/measurement-snapshot.test.ts | 2 +- .../lib/fixtures/measurement-receipt/101.zip | Bin 1055 -> 642 bytes .../lib/fixtures/measurement-receipt/102.zip | Bin 1585 -> 882 bytes .../measurement-receipt/bmk_pilot/agg.json | 21 +++++- .../bmk_pilot/execution.json | 20 +++++- .../eval_results_all/agg.json | 21 +++++- .../eval_results_all/execution.json | 20 +++++- .../fixtures/measurement-receipt/receipt.json | 63 +++++++----------- .../measurement-receipt/receipt.sha256 | 2 +- .../db/src/lib/measurement-receipt.test.ts | 4 +- .../lib/receipt-artifact-preparation.test.ts | 2 +- packages/db/src/lib/receipt-transport.test.ts | 2 +- 13 files changed, 110 insertions(+), 49 deletions(-) diff --git a/packages/db/src/etl/measurement-publication.test.ts b/packages/db/src/etl/measurement-publication.test.ts index cd9aca411..7d5b1afdb 100644 --- a/packages/db/src/etl/measurement-publication.test.ts +++ b/packages/db/src/etl/measurement-publication.test.ts @@ -10,7 +10,7 @@ import { parseMeasurementReceipt } from '../lib/measurement-receipt'; import { expectedPublication, verifyPublishedMeasurements } from './measurement-publication'; const fixture = new URL('../lib/fixtures/measurement-receipt/', import.meta.url); -const hash = '00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6'; +const hash = '7112fa0765669dc24b375946b136231df3fcff53d756ab1ce56ea6a83914dd4e'; const raw = fs.readFileSync(new URL('receipt.json', fixture)); const config = { model: 'dsr1', diff --git a/packages/db/src/etl/measurement-snapshot.test.ts b/packages/db/src/etl/measurement-snapshot.test.ts index a71367542..2c4dc441b 100644 --- a/packages/db/src/etl/measurement-snapshot.test.ts +++ b/packages/db/src/etl/measurement-snapshot.test.ts @@ -48,7 +48,7 @@ it('resumes partial imports with the same receipt and rejects replacement after expect(complete.rows).toEqual([ { state: 'complete', - receipt_id: '2e12626c30ec11cf1738c7541a36444af53c29bad4f59f927930618e298e7a01', + receipt_id: '924d4040a396a3bd964600c6d1706eb6a51c798e3f43a150751fb4e8a554fa08', }, ]); } finally { diff --git a/packages/db/src/lib/fixtures/measurement-receipt/101.zip b/packages/db/src/lib/fixtures/measurement-receipt/101.zip index afcb2e6c539db94a0753c6e62f3a3111d328d74d..70893fdbf6cf9e7cb092b05ccae3da8401e3cad8 100644 GIT binary patch literal 642 zcmWIWW@Zs#U|`??V#SzjG1eRVfUHtRh)803x?WaseqL$+T`nd?0k`+xHB;7e>nJS> z?-aOs)%O#>`}WoscW2+c)3J!V)zaPG-k&$QJf>&sv21nqhfeF)ba-YOZ*{-+ZGzd` zSEWyil#6Z_?fdqasdSRxU8h}>q}P^qALu#Pz&Wp_uf6qo$AbWN*Tc*B_E{JnVUpix zuF%fdI87|1m90hFOu9dI|CN|KQI3HjUaDS?y=T4pa7z9r>;2j(h6$OEZpX4dWS=7A zEmqnbxi4PJGwsiAhMG&?RldtyU2JLYAAsUV{}t;wUIG0Wz{J472c%OgQj<$dGV}Am zK3zI#W7ZJ^p0@8ct{-ANf;`&Sz0I|~wQzS~PVAzhjm^69;eFNLPxU!li!D01ApX<( z^M|&tTIYH}_>5QF0rrwGo9lO-CyBRDQ;`yL*>3Do)}sBw!)EagD|<6JvFjzdw@YfK z?Nf{@X3H#SaadV+LvnR)!Nu%^{nF`kmS1E%`SbmLgPrD~$p=$*C}}Fq5Wi)1@!W*AMf+m&X$1^*(G)1{qFurtTepbl(<1}w-aqa%ZbEh74UZ0{N z_j`GW&D)-{RudMT`TnWyPla`jTwMLduab@R>@gFU#Ld4L_Wc!mfHxzP2s7@u2L=ro sENKK$u#|wV3q3R;dKeg%G(G_`QNuRCo0SbD&jf@;K>8Ojl`${?0N$+h{r~^~ literal 1055 zcmbtTO>fgc5OoqjoOv zpFo}GghDHjkTQ{-o!Omv&u?b)+33lB0{e9P>hkC3y$?r&B*F8XVxq2JUU_T2te7%Y zE0(;qrPRE0vQkU-`jD00W$fsXHP`T(Nt(Ve3j``BT6t~VSHu@uw+K>Rp}evN!Kyn& zEOg|Y(n4a9(Za-**Io46c&R+`Ag2=AOWIeSw^0XWh|3@0o|M)c=>V1v6;etlWkE;( zvQb!5&>4U#udzxe$K==tP7GZPeEUMpEbXgCCjxy7_f+1L*f>eom-z!le|$gd4u(mB zXO9$>OIbu@^}4Q@FB;hjz96z2S-_>U$^@=Ts>re(4HDmjZfjaskDcu7m_B5j(YtTq zJepEVUaDI90OKx^F1{ivte1Xg16?!io@N+kpbaXrw5)Zv>LW ze;Qj;B1RQSp>a0W4fPOMhI+P;oCZ;<9%IF>?0hTCbc!RbFsdRAfI69;phmT$^-=^8 zdO~OK-{o)5FV5feGz+aHbz3e~;22YaOV{~qFx(&g`(on4!fxoI{=TM|_XmY{5HMjhkC(Ki}hF#yH4pPn7&c@Ffg7ZU>m zACOM1NKGy+$;{6K`+4c4jaf$wc-p?(xPGXa@I*m&*S76jw>XzSzOk#@D~Uatvs~u; zyQ#A6WjZd+j{Bd)&u@M28rij=T4HL<0rrwGo9Sm2Pfd3A7UJc8w8Pk?s73q5%nG-% z(tkmfGh(h@DZTn(=I>oMXE820$j#E#d-q&KZu9KDY~OYlAFGs+oiJzr-&W7P8?DlP z+D=bWYARfm>%8X6f*zi|U2L!BEB<(;%-DO*MTW6kD$Hdyhxi0JlLV+35tx9mgj9exD;r1= OGZ1bD(kp?=7#INeWKmTB literal 1585 zcmcIk&2AGh5Z*KtAr45WeSqQvCz@=6f&y3CQ;Ns|PE}+%yR*A)ZSTtVri-FTJp=cO zXP`I4JMkFI>?V{%MJhy0me zA!SkwN#_e(3~>T9>EXnbnx{qSmikya#DR-MCy=5cHO@uamO7DfMv8GNB&j}c4?!1+erzo94qe&E0a;hkAuS2m7ha{mGi^+Jc zmtTk1-}iQcfW{;2B~%cV9&4{VI%h7*Knp%Ym`y{d^dgtWa~Wem*56X<8dgb{F>{-C zEfjgT7lz#)e*0uKElsR|$1(*+)w1c9LP1?#+KBozXOz)NpW6=?NDv z22>*Qys^WRSUmj8M5{Y;2$i=O>YtMl3I0=6;aXL9Nwi&6<56p8zy0Tz7EidAOFZtI ucX!ik^G#`dg5a`sf`02S%%<-qp>BLXwQr@@c(i+;jN177hX2_vt$qUHP|eK% diff --git a/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/agg.json b/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/agg.json index 197ad20c0..7addfd8b3 100644 --- a/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/agg.json +++ b/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/agg.json @@ -1 +1,20 @@ -[{"infmax_model_prefix": "dsr1", "hw": "h200-nv", "framework": "vllm", "precision": "fp8", "tp": 8, "ep": 1, "num_gpus": 8, "disagg": false, "is_multinode": false, "prefill_num_workers": 0, "decode_num_workers": 0, "conc": 1, "isl": 1024, "osl": 1024, "output_tput_tps": 100, "duration_seconds": 60}] \ No newline at end of file +[ + { + "infmax_model_prefix": "dsr1", + "hw": "h200-nv", + "framework": "vllm", + "precision": "fp8", + "tp": 8, + "ep": 1, + "num_gpus": 8, + "disagg": false, + "is_multinode": false, + "prefill_num_workers": 0, + "decode_num_workers": 0, + "conc": 1, + "isl": 1024, + "osl": 1024, + "output_tput_tps": 100, + "duration_seconds": 60 + } +] diff --git a/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/execution.json b/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/execution.json index 3cb53cd83..dfc684c1e 100644 --- a/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/execution.json +++ b/packages/db/src/lib/fixtures/measurement-receipt/bmk_pilot/execution.json @@ -1 +1,19 @@ -{"schema_version": 1, "point_id": "1111111111111111111111111111111111111111111111111111111111111111", "execution_id": "org/repo/100/1/101", "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "source": {"repository": "org/repo", "run_id": 100, "attempt": 1, "head_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "mode": "throughput", "native_receipt": {"job_id": "55", "manifest_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "state": "COMPLETED"}, "client_exit_code": 0} \ No newline at end of file +{ + "schema_version": 1, + "point_id": "1111111111111111111111111111111111111111111111111111111111111111", + "execution_id": "org/repo/100/1/101", + "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source": { + "repository": "org/repo", + "run_id": 100, + "attempt": 1, + "head_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "mode": "throughput", + "native_receipt": { + "job_id": "55", + "manifest_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "state": "COMPLETED" + }, + "client_exit_code": 0 +} diff --git a/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/agg.json b/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/agg.json index 14f25647d..4cf2cda4e 100644 --- a/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/agg.json +++ b/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/agg.json @@ -1 +1,20 @@ -[{"infmax_model_prefix": "dsr1", "hw": "h200-nv", "framework": "vllm", "precision": "fp8", "tp": 8, "ep": 1, "num_gpus": 8, "disagg": false, "is_multinode": false, "prefill_num_workers": 0, "decode_num_workers": 0, "conc": 28, "task": "gsm8k", "em_strict": 0.5, "em_flexible": 1.0, "n_eff": 2}] \ No newline at end of file +[ + { + "infmax_model_prefix": "dsr1", + "hw": "h200-nv", + "framework": "vllm", + "precision": "fp8", + "tp": 8, + "ep": 1, + "num_gpus": 8, + "disagg": false, + "is_multinode": false, + "prefill_num_workers": 0, + "decode_num_workers": 0, + "conc": 28, + "task": "gsm8k", + "em_strict": 0.5, + "em_flexible": 1.0, + "n_eff": 2 + } +] diff --git a/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/execution.json b/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/execution.json index aee7b4253..42487d736 100644 --- a/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/execution.json +++ b/packages/db/src/lib/fixtures/measurement-receipt/eval_results_all/execution.json @@ -1 +1,19 @@ -{"schema_version": 1, "point_id": "2222222222222222222222222222222222222222222222222222222222222222", "execution_id": "org/repo/100/1/102", "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "source": {"repository": "org/repo", "run_id": 100, "attempt": 1, "head_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "mode": "eval", "native_receipt": {"job_id": "55", "manifest_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "state": "COMPLETED"}, "client_exit_code": 0} \ No newline at end of file +{ + "schema_version": 1, + "point_id": "2222222222222222222222222222222222222222222222222222222222222222", + "execution_id": "org/repo/100/1/102", + "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "source": { + "repository": "org/repo", + "run_id": 100, + "attempt": 1, + "head_sha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "mode": "eval", + "native_receipt": { + "job_id": "55", + "manifest_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "state": "COMPLETED" + }, + "client_exit_code": 0 +} diff --git a/packages/db/src/lib/fixtures/measurement-receipt/receipt.json b/packages/db/src/lib/fixtures/measurement-receipt/receipt.json index 5ccb8f204..a6bacec50 100644 --- a/packages/db/src/lib/fixtures/measurement-receipt/receipt.json +++ b/packages/db/src/lib/fixtures/measurement-receipt/receipt.json @@ -12,6 +12,7 @@ "points": [ { "point_id": "1111111111111111111111111111111111111111111111111111111111111111", + "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "execution_id": "org/repo/100/1/101", "source_run_id": "100", "source_attempt": 1, @@ -24,18 +25,15 @@ "tp": 8, "ep": 1 }, - "artifact_ids": [ - 101 - ], + "artifact_ids": [101], "execution_artifact_id": 101, "execution_path": "execution.json", "native_manifest_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "normalized_artifact_id": 101, "normalized_path": "agg.json", - "required_metrics": [ - "output_tput_tps", - "duration_seconds" - ], + "normalized_format": "normalized", + "metadata_path": null, + "required_metrics": ["output_tput_tps", "duration_seconds"], "config": { "model": "dsr1", "hardware": "h200", @@ -47,13 +45,11 @@ "sample_count": 0, "samples_artifact_id": null, "samples_path": null, - "dataset": {}, - "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "normalized_format": "normalized", - "metadata_path": null + "dataset": {} }, { "point_id": "2222222222222222222222222222222222222222222222222222222222222222", + "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "execution_id": "org/repo/100/1/102", "source_run_id": "100", "source_attempt": 1, @@ -66,18 +62,15 @@ "tp": 8, "ep": 1 }, - "artifact_ids": [ - 102 - ], + "artifact_ids": [102], "execution_artifact_id": 102, "execution_path": "execution.json", "native_manifest_sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "normalized_artifact_id": 102, "normalized_path": "agg.json", - "required_metrics": [ - "em_strict", - "n_eff" - ], + "normalized_format": "normalized", + "metadata_path": null, + "required_metrics": ["em_strict", "n_eff"], "config": { "model": "dsr1", "hardware": "h200", @@ -85,21 +78,16 @@ "precision": "fp8" }, "task": "gsm8k", - "filters": [ - "strict-match", - "flexible-extract" - ], + "filters": ["strict-match", "flexible-extract"], "sample_count": 2, "samples_artifact_id": 102, "samples_path": "samples_gsm8k.jsonl", - "dataset": {}, - "bundle_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "normalized_format": "normalized", - "metadata_path": null + "dataset": {} } ], "kind": "source-measurement-receipt", "version": 1, + "receipt_id": "924d4040a396a3bd964600c6d1706eb6a51c798e3f43a150751fb4e8a554fa08", "issuer": { "repository": "org/repo", "run_id": "200", @@ -111,36 +99,36 @@ { "id": 101, "name": "bmk_pilot", - "sha256": "b644c12d559428df1845d5ff8d3483aac35e7fdb694d8ce1a7ac891239300fd3", + "sha256": "8d54db4e2a112e85bd942732aec2c07a73746a388e7ce0cf7192f016994ddc30", "run_id": "100", "members": [ { "path": "agg.json", - "sha256": "a142da2f49fa4f706c801bd824eea698740b567ad689269fbdd181a9648dd849", - "size": 300 + "sha256": "7021f75ab64617a361af57e749ea26a4700dbfc6e7b336ad5b69fee5a5eddf5c", + "size": 373 }, { "path": "execution.json", - "sha256": "41746d1d444ec14e440d5d75e1144a5636d7ef7acaaef7b4957217cde4503c5b", - "size": 537 + "sha256": "25b6262ae1299a88e4d67e4b473c63a19e02213309695a1fe5d4fa3aa38c1336", + "size": 592 } ] }, { "id": 102, "name": "eval_results_all", - "sha256": "8c2f42b955bfdb85fac00d6bfca0442475803f2aa7aefd93b1842cacdff7e94c", + "sha256": "9dc97db53cd14d7361e8202419cf338e948212c8142ac7ff95b72b1a70c695dc", "run_id": "100", "members": [ { "path": "agg.json", - "sha256": "2a05f61da6ac2bd48a9a5b8c40a4032dbd1f711581fcd8e1df429c0ce619a031", - "size": 294 + "sha256": "f9115ace4cb9bb3dc663d6243b4a1ac40bf4af2cc417140be7bae54a053ca782", + "size": 367 }, { "path": "execution.json", - "sha256": "479ff4fc073b1f73508bd76fb13be0fb6fb8a740f113e4f2404bd4139345ecbb", - "size": 531 + "sha256": "339c79f0bd16508d24dbb6049f1632fc66c20e6ee9ff2c6bf10251246b25ead1", + "size": 586 }, { "path": "samples_gsm8k.jsonl", @@ -149,6 +137,5 @@ } ] } - ], - "receipt_id": "2e12626c30ec11cf1738c7541a36444af53c29bad4f59f927930618e298e7a01" + ] } diff --git a/packages/db/src/lib/fixtures/measurement-receipt/receipt.sha256 b/packages/db/src/lib/fixtures/measurement-receipt/receipt.sha256 index 7bd6f61be..7aa479644 100644 --- a/packages/db/src/lib/fixtures/measurement-receipt/receipt.sha256 +++ b/packages/db/src/lib/fixtures/measurement-receipt/receipt.sha256 @@ -1 +1 @@ -00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6 +7112fa0765669dc24b375946b136231df3fcff53d756ab1ce56ea6a83914dd4e diff --git a/packages/db/src/lib/measurement-receipt.test.ts b/packages/db/src/lib/measurement-receipt.test.ts index c57e4a02f..88ad92769 100644 --- a/packages/db/src/lib/measurement-receipt.test.ts +++ b/packages/db/src/lib/measurement-receipt.test.ts @@ -11,7 +11,7 @@ import { } from './measurement-receipt'; const fixture = new URL('fixtures/measurement-receipt/', import.meta.url); const raw = fs.readFileSync(new URL('receipt.json', fixture)); -const expectedDigest = '00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6'; +const expectedDigest = '7112fa0765669dc24b375946b136231df3fcff53d756ab1ce56ea6a83914dd4e'; const issuerSha = 'd'.repeat(40); const roots: string[] = []; function copy() { @@ -26,7 +26,7 @@ afterEach(() => it('reads a Python-sealed snapshot and verifies both mapped families and complete sample coverage', () => { const receipt = parseMeasurementReceipt(raw, expectedDigest, issuerSha); expect(receipt.receipt_id).toBe( - '2e12626c30ec11cf1738c7541a36444af53c29bad4f59f927930618e298e7a01', + '924d4040a396a3bd964600c6d1706eb6a51c798e3f43a150751fb4e8a554fa08', ); expect( receipt.points.map((point) => [point.kind, point.concurrency, point.source_attempt]), diff --git a/packages/db/src/lib/receipt-artifact-preparation.test.ts b/packages/db/src/lib/receipt-artifact-preparation.test.ts index 654b8ecd7..efeb85243 100644 --- a/packages/db/src/lib/receipt-artifact-preparation.test.ts +++ b/packages/db/src/lib/receipt-artifact-preparation.test.ts @@ -23,7 +23,7 @@ afterEach(() => { function receipt() { return parseMeasurementReceipt( bytes, - '00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6', + '7112fa0765669dc24b375946b136231df3fcff53d756ab1ce56ea6a83914dd4e', 'd'.repeat(40), ); } diff --git a/packages/db/src/lib/receipt-transport.test.ts b/packages/db/src/lib/receipt-transport.test.ts index 82926fbbc..bf90af8db 100644 --- a/packages/db/src/lib/receipt-transport.test.ts +++ b/packages/db/src/lib/receipt-transport.test.ts @@ -28,7 +28,7 @@ const env = { PUBLICATION_REQUIRED: 'false', RECEIPT_ARTIFACT_ID: '301', RECEIPT_ARTIFACT_SHA256: sha256(archive), - RECEIPT_SHA256: '00465d715d63cd4df2f55d6662eb4b10982330f14e8e0d35872b88efb2e619d6', + RECEIPT_SHA256: '7112fa0765669dc24b375946b136231df3fcff53d756ab1ce56ea6a83914dd4e', RECEIPT_ISSUER_RUN_ID: '200', RECEIPT_ISSUER_SHA: 'd'.repeat(40), ALLOWED_RECEIPT_ISSUER_SHAS: 'd'.repeat(40), From 0dc98d251624ac8f98fc3bb0345bc881a23e42a2 Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sat, 19 Sep 2026 19:45:39 -0400 Subject: [PATCH 4/5] fix: ingest bound receipt data and stream artifact archives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Select benchmark and eval/sample inputs from accepted receipt member bindings. Reject conflicting eval-only scope before writes and require successful per-file point counts before refreshing curves or completing a snapshot. Stream ZIP payloads and member hashes with bounded memory, preserving exact-byte checks. Validate real PGlite insertion, interruption recovery and immutable replay, large archive memory behavior, production Bun execution and download cleanup. All 6,886 workspace unit tests pass (four existing skips), along with typecheck, lint, formatting and the browser smoke coverage. 中文：修复回执数据导入，并以流式方式处理产物压缩包。 根据已接受回执的成员绑定选择基准测试、评估与样本数据。在写入前拒绝 与回执冲突的 evals-only 范围，并逐文件检查实际持久化的测量点数量， 通过后才刷新曲线并完成 snapshot。压缩包与成员哈希均分块处理，避免 大型 trace 文件占满内存，同时保留完整性校验。 使用真实 PGlite 数据库验证写入、中断恢复和不可变回执重放，并验证 大型压缩包的内存上限、Bun 执行及下载失败清理。单元测试、类型检查、 lint、格式检查和浏览器冒烟测试均已通过。 --- bun.lock | 1 + docs/measurement-publication.md | 6 +- docs/measurement-publication_zh.md | 6 +- packages/db/package.json | 3 +- packages/db/src/etl/receipt-ingest.test.ts | 194 ++++++++++++++++++ packages/db/src/etl/receipt-ingest.ts | 100 +++++++++ packages/db/src/ingest-ci-run.ts | 77 ++++--- .../db/src/lib/artifact-archive-worker.mjs | 159 ++++++++++++++ packages/db/src/lib/artifact-archive.test.ts | 170 ++++++++++++++- packages/db/src/lib/artifact-archive.ts | 135 ++++++------ packages/db/src/lib/github-artifacts.test.ts | 110 +++++++++- packages/db/src/lib/github-artifacts.ts | 34 ++- packages/db/src/lib/measurement-receipt.ts | 5 +- .../lib/receipt-artifact-preparation.test.ts | 29 ++- packages/db/src/lib/receipt-transport.test.ts | 39 ++-- 15 files changed, 930 insertions(+), 138 deletions(-) create mode 100644 packages/db/src/etl/receipt-ingest.test.ts create mode 100644 packages/db/src/etl/receipt-ingest.ts create mode 100644 packages/db/src/lib/artifact-archive-worker.mjs diff --git a/bun.lock b/bun.lock index aa8bd6d1b..f940505d7 100644 --- a/bun.lock +++ b/bun.lock @@ -112,6 +112,7 @@ "postgres": "^3.4.9", "stream-chain": "^4.2.5", "stream-json": "^3.6.0", + "yauzl": "3.4.0", }, "devDependencies": { "@electric-sql/pglite": "^0.5.8", diff --git a/docs/measurement-publication.md b/docs/measurement-publication.md index aa7bfd5b2..e9252b9f2 100644 --- a/docs/measurement-publication.md +++ b/docs/measurement-publication.md @@ -16,7 +16,11 @@ The receiving workflow derives `PUBLICATION_REQUIRED` from its actual database t The reader uses separately versioned `aiperf-1.4`, `agentx-v1` and publication contract 1. Unsupported required versions fail before import. Every point binds its original execution/attempt, prepared bundle, native manifest, physical topology, canonical model/hardware/framework/precision, required metrics, full dataset identity and exact artifact/member references. Normalized AgentX JSON and per-job raw lm-eval results plus `meta_env.json` are supported. Evaluation requires complete `(task, doc_id, filter)` coverage. Both live preview and stored document-level samples select strict-match independent of line order and reject conflicting copies. Aggregate metadata maps to one physical serving role with compatibility worker counts 0/0. -Artifacts are downloaded with argv-based `gh` calls and extracted under numeric ID roots. Validation rejects escaping paths, links, duplicate normalized names, file/directory collisions, overwrites and changed member sets. Checked compatibility views retain existing consumer discovery names. All snapshot bytes and normalized requirements are verified before the first database write. +Artifacts are downloaded with argv-based `gh` calls into private temporary ZIP files and extracted under numeric ID roots. Archive and member hashes use bounded reads. A Node/Bun worker streams every ZIP member once for validation, then streams extraction and rechecks the accepted member set; large trace payloads are never retained together in memory. Validation rejects escaping paths, links, duplicate normalized names, file/directory collisions, overwrites and changed member sets. Checked compatibility views retain existing consumer discovery names. All snapshot bytes and normalized requirements are verified before the first database write. + +Receipt ingestion selects benchmark JSON and evaluation samples from the receipt's exact member bindings. Execution metadata and other JSON sidecars are retained as evidence without entering benchmark row mapping. Both normalized aggregate evals and raw lm-eval outputs persist their explicitly bound strict-match samples; sample attachment does not depend on directory or timestamp naming conventions. + +An `evals-only` changelog cannot narrow a receipt that requires throughput points: the reader rejects it before database writes. Before refreshing the published curve or marking a snapshot complete, every accepted benchmark file must return the receipt's exact point count from database inserts, including existing rows on replay. Per-point purge skips therefore leave the snapshot incomplete. Apply migration `016_measurement_snapshots.sql` before deploying these readers. The table retains the compact receipt and binds each source repository/run/attempt to one accepted snapshot. An interrupted progressive import remains `writing` and resumes only that same receipt; successful replay remains stable. Replacing measurement bytes requires a separately versioned source execution. Execution rollback does not reverse prior database writes. Preserve a compatible reader for all retained receipt versions. diff --git a/docs/measurement-publication_zh.md b/docs/measurement-publication_zh.md index 1c690002b..4e7536715 100644 --- a/docs/measurement-publication_zh.md +++ b/docs/measurement-publication_zh.md @@ -16,7 +16,11 @@ Phase 1 的读取端接受由 InferenceX 独立受信托管签发流程生成的 读取端分别支持 `aiperf-1.4`、`agentx-v1` 和 publication contract 1。不支持的必需版本会在导入前报错。每个点都绑定其原始执行及 attempt、prepared bundle、native manifest、物理拓扑、规范化后的模型/硬件/framework/precision、必需指标、完整数据集身份，以及准确的产物和文件引用。输入支持规范化 AgentX JSON，也支持每个任务的原始 lm-eval 结果和 `meta_env.json`。评估必须完整覆盖 `(task, doc_id, filter)`。live preview 与数据库中的文档级样本统一选择 strict-match，不受行顺序影响；同一过滤器的冲突副本会被拒绝。聚合部署元数据映射到一个实际 serving role，兼容字段中的 worker 数量为 0/0。 -产物下载使用 argv 形式调用 `gh`，并在以数字 ID 命名的目录中解压。校验会拒绝越界路径、链接、规范化后重名的成员、文件与目录冲突、覆盖写入以及成员集合变化。经过验证的兼容视图保留已有消费端的发现名称。在第一次数据库写入之前，必须验证完整 snapshot 的文件字节和规范化要求。 +产物下载使用 argv 形式调用 `gh`，先写入私有临时 ZIP 文件，再在以数字 ID 命名的目录中解压。压缩包和成员的哈希计算均按固定大小分块读取。Node/Bun 工作进程先流式读取所有 ZIP 成员完成校验，再流式解压并复核已接受的成员集合；大型 trace 文件不会同时驻留内存。校验会拒绝越界路径、链接、规范化后重名的成员、文件与目录冲突、覆盖写入以及成员集合变化。经过验证的兼容视图保留已有消费端的发现名称。在第一次数据库写入之前，必须验证完整 snapshot 的文件字节和规范化要求。 + +回执导入根据回执中准确的成员绑定选择基准测试 JSON 和评估样本。执行元数据及其他 JSON 附属文件保留为证据，不进入基准测试行映射。规范化聚合评估与原始 lm-eval 输出都会写入明确绑定的 strict-match 样本；样本关联不依赖目录或时间戳命名约定。 + +如果回执要求导入吞吐量测量点，`evals-only` changelog 不能缩小该范围，读取端会在数据库写入前拒绝这一冲突。在刷新已发布曲线或将 snapshot 标记为完成之前，每个已接受的基准测试文件都必须从数据库写入中返回回执规定的测量点数量，重复导入时已有的数据行也计入。因此，按测量点执行的清除规则若跳过必需数据，snapshot 会保持未完成状态。 部署读取端之前，先应用 `016_measurement_snapshots.sql`。该表保留紧凑回执，并将源仓库/run/attempt 绑定到唯一的已接受 snapshot。渐进式导入中断后，状态保持 `writing`，恢复时只能使用同一回执；成功后的重复导入保持结果稳定。替换测量字节需要独立版本的源执行。执行回滚不会撤销既有数据库写入。对保留的回执版本，必须继续提供兼容读取端。 diff --git a/packages/db/package.json b/packages/db/package.json index f813b3d9f..3653c5c96 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -44,7 +44,8 @@ "adm-zip": "^0.6.0", "postgres": "^3.4.9", "stream-chain": "^4.2.5", - "stream-json": "^3.6.0" + "stream-json": "^3.6.0", + "yauzl": "3.4.0" }, "devDependencies": { "@electric-sql/pglite": "^0.5.8", diff --git a/packages/db/src/etl/receipt-ingest.test.ts b/packages/db/src/etl/receipt-ingest.test.ts new file mode 100644 index 000000000..dfedcfa0f --- /dev/null +++ b/packages/db/src/etl/receipt-ingest.test.ts @@ -0,0 +1,194 @@ +import { PGlite } from '@electric-sql/pglite'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import type postgres from 'postgres'; +import { afterAll, afterEach, beforeAll, beforeEach, expect, it } from 'vitest'; +import { sha256 } from '../lib/artifact-archive'; +import type { MeasurementReceipt } from '../lib/measurement-receipt'; +import { mapBenchmarkRow } from './benchmark-mapper'; +import { bulkIngestBenchmarkRows } from './benchmark-ingest'; +import { createSkipTracker } from './skip-tracker'; +import { + prepareReceiptIngestInputs, + ingestReceiptEvaluations, + assertReceiptIngestMode, + completeReceiptIngest, +} from './receipt-ingest'; +import { claimMeasurementSnapshot } from './measurement-snapshot'; + +type Sql = postgres.Sql; +let db: PGlite; +let sql: Sql; +const roots: string[] = []; +const fixture = new URL('../lib/fixtures/measurement-receipt/', import.meta.url); + +function queryClient(database: Pick) { + return Object.assign( + async (strings: TemplateStringsArray, ...values: unknown[]) => { + const query = strings.reduce((text, part, i) => text + (i ? `$${i}` : '') + part, ''); + // postgres.js supplies typed array parameters; PGlite needs their OIDs + // for unnest calls whose SQL deliberately omits a redundant cast. + const result = await database.query(query, values, { + paramTypes: values.map((value) => (Array.isArray(value) ? 1009 : 0)), + }); + return result.rows; + }, + { json: JSON.stringify, array: (values: unknown[]) => values }, + ); +} + +beforeAll(async () => { + db = await PGlite.create(); + const migrations = new URL('../../migrations/', import.meta.url); + for (const name of fs + .readdirSync(migrations) + .filter((file) => file.endsWith('.sql')) + .toSorted()) + await db.exec(fs.readFileSync(new URL(name, migrations), 'utf8')); + sql = Object.assign(queryClient(db), { + begin: (fn: (tx: Sql) => Promise) => + db.transaction((tx) => fn(queryClient(tx) as unknown as Sql)), + }) as unknown as Sql; +}, 20_000); + +beforeEach(async () => { + await db.exec(`TRUNCATE workflow_runs, configs, measurement_snapshots RESTART IDENTITY CASCADE; + INSERT INTO workflow_runs (id, github_run_id, run_attempt, name, status, conclusion, created_at, date) + VALUES (1, 100, 2, 'Run Sweep', 'completed', 'success', '2026-09-19', '2026-09-19'); + INSERT INTO configs (id, model, hardware, framework, precision, spec_method, disagg, + prefill_tp, decode_tp, num_prefill_gpu, num_decode_gpu) + VALUES (1, 'dsr1', 'h200', 'vllm', 'fp8', 'none', false, 8, 8, 8, 8);`); +}); +afterEach(() => + roots.splice(0).forEach((root) => fs.rmSync(root, { recursive: true, force: true })), +); +afterAll(() => db?.close()); + +it.each(['normalized', 'lm-eval'] as const)( + 'ingests bound benchmark data and recovers strict samples from %s receipts through replay', + async (format) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'receipt-ingest-')); + roots.push(root); + fs.cpSync(fixture, root, { recursive: true }); + const receipt = JSON.parse( + fs.readFileSync(path.join(root, 'receipt.json'), 'utf8'), + ) as MeasurementReceipt; + if (format === 'lm-eval') { + const point = receipt.points[1]; + const artifact = receipt.artifacts[1]; + const directory = path.join(root, artifact.name); + const meta = JSON.parse(fs.readFileSync(path.join(directory, 'agg.json'), 'utf8'))[0]; + fs.unlinkSync(path.join(directory, 'agg.json')); + point.normalized_format = 'lm-eval'; + point.normalized_path = 'results_2026-09-19_conc28.json'; + point.metadata_path = 'meta_env.json'; + fs.writeFileSync(path.join(directory, point.metadata_path), JSON.stringify(meta)); + fs.writeFileSync( + path.join(directory, point.normalized_path), + JSON.stringify({ + results: { + gsm8k: { 'exact_match,strict-match': 0.5, 'exact_match,flexible-extract': 1 }, + }, + 'n-samples': { gsm8k: { effective: 2 } }, + }), + ); + // The binding must work with the actual timestamp/concurrency naming shape. + const sampleName = 'samples_gsm8k_2026-09-19_conc28.jsonl'; + fs.renameSync(path.join(directory, point.samples_path!), path.join(directory, sampleName)); + point.samples_path = sampleName; + artifact.members = fs.readdirSync(directory).map((name) => { + const bytes = fs.readFileSync(path.join(directory, name)); + return { path: name, size: bytes.length, sha256: sha256(bytes) }; + }); + } + + const tracker = createSkipTracker(); + const inputs = prepareReceiptIngestInputs(receipt, root, tracker); + expect(inputs.benchmarkFiles.map((file) => path.relative(root, file))).toEqual([ + 'bmk_pilot/agg.json', + ]); + const rows = inputs.benchmarkFiles + .flatMap((file) => JSON.parse(fs.readFileSync(file, 'utf8'))) + .map((row) => mapBenchmarkRow(row, tracker)); + expect(rows).toHaveLength(1); + expect(rows[0]?.config.numDecodeGpu).toBe(8); + expect(Object.values(tracker.skips).every((count) => count === 0)).toBe(true); + expect(() => assertReceiptIngestMode(inputs, true)).toThrow('evals-only'); + const unclaimed = await db.query('select count(*) as count from measurement_snapshots'); + expect(unclaimed.rows).toEqual([{ count: 0 }]); + assertReceiptIngestMode(inputs, false); + await claimMeasurementSnapshot(sql, receipt); + const resolveConfig = async (config: (typeof inputs.evaluations)[0]['params']['config']) => { + const matching = await db.query<{ id: number }>( + 'select id from configs where model=$1 and hardware=$2', + [config.model, config.hardware], + ); + if (matching.rows.length !== 1) throw new Error('unmapped test config'); + return matching.rows[0].id; + }; + // Fail the real sample INSERT after the summary has committed, then resume + // the accepted snapshot. Incomplete samples must not look like completion. + await db.exec('ALTER TABLE eval_samples RENAME TO interrupted_eval_samples'); + try { + await expect( + ingestReceiptEvaluations(sql, inputs, resolveConfig, 1, '2026-09-19'), + ).rejects.toThrow('eval_samples'); + } finally { + await db.exec('ALTER TABLE interrupted_eval_samples RENAME TO eval_samples'); + } + const partialResults = await db.query('select count(*) as count from eval_results'); + expect(partialResults.rows).toEqual([{ count: 1 }]); + const partialSnapshot = await db.query('select state from measurement_snapshots'); + expect(partialSnapshot.rows).toEqual([{ state: 'writing' }]); + expect(await ingestReceiptEvaluations(sql, inputs, resolveConfig, 1, '2026-09-19')).toEqual({ + newEvals: 0, + newSamples: 2, + sampleFiles: 1, + }); + // Skipping throughput cannot refresh the curve or complete an eval-only + // partial import. Successful replay counts existing benchmark rows too. + await expect(completeReceiptIngest(sql, receipt, inputs, new Map())).rejects.toThrow( + 'throughput points were skipped', + ); + const incomplete = await db.query('select state from measurement_snapshots'); + expect(incomplete.rows).toEqual([{ state: 'writing' }]); + const benchmarkRows = rows.map((row) => ({ ...row!, configId: 1 })); + const inserted = await bulkIngestBenchmarkRows(sql, benchmarkRows, 1, '2026-09-19'); + expect(inserted.newCount).toBe(1); + await completeReceiptIngest( + sql, + receipt, + inputs, + new Map([[inputs.benchmarkFiles[0], inserted.insertedIds.length]]), + ); + expect(await ingestReceiptEvaluations(sql, inputs, resolveConfig, 1, '2026-09-19')).toEqual({ + newEvals: 0, + newSamples: 0, + sampleFiles: 1, + }); + const replayed = await bulkIngestBenchmarkRows(sql, benchmarkRows, 1, '2026-09-19'); + expect(replayed.newCount).toBe(0); + expect(replayed.dupCount).toBe(1); + await completeReceiptIngest( + sql, + receipt, + inputs, + new Map([[inputs.benchmarkFiles[0], replayed.insertedIds.length]]), + ); + const benchmark = await db.query('select conc, metrics from latest_benchmarks'); + expect(benchmark.rows).toHaveLength(1); + expect(benchmark.rows[0]).toMatchObject({ + conc: 1, + metrics: { output_tput_tps: 100, duration_seconds: 60 }, + }); + const stored = await db.query(`select e.task, e.conc, s.doc_id, s.passed, s.score + from eval_results e join eval_samples s on s.eval_result_id=e.id order by s.doc_id`); + expect(stored.rows).toEqual([ + { task: 'gsm8k', conc: 28, doc_id: 0, passed: true, score: '1' }, + { task: 'gsm8k', conc: 28, doc_id: 1, passed: false, score: '0' }, + ]); + const snapshot = await db.query('select state from measurement_snapshots'); + expect(snapshot.rows).toEqual([{ state: 'complete' }]); + }, +); diff --git a/packages/db/src/etl/receipt-ingest.ts b/packages/db/src/etl/receipt-ingest.ts new file mode 100644 index 000000000..3bbae022b --- /dev/null +++ b/packages/db/src/etl/receipt-ingest.ts @@ -0,0 +1,100 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import type postgres from 'postgres'; +import { + mapReceiptPointRows, + verifyMeasurementSnapshot, + type MeasurementReceipt, +} from '../lib/measurement-receipt'; +import type { ConfigParams } from './config-cache'; +import type { EvalParams } from './eval-mapper'; +import { mapEvalSamples, projectEvalSamples, type EvalSampleParams } from './eval-samples-mapper'; +import { ingestEvalRow } from './eval-ingest'; +import { bulkIngestEvalSamples } from './eval-samples-ingest'; +import type { SkipTracker } from './skip-tracker'; +import { completeMeasurementSnapshot } from './measurement-snapshot'; +import { refreshLatestBenchmarks } from './db-utils'; + +export interface ReceiptIngestInputs { + benchmarkFiles: string[]; + benchmarkPointCounts: Map; + evaluations: { params: EvalParams; samples: EvalSampleParams[] }[]; +} + +/** Use accepted member bindings, never discovery names, to select required ingest inputs. */ +export function prepareReceiptIngestInputs( + receipt: MeasurementReceipt, + root: string, + tracker: SkipTracker, +): ReceiptIngestInputs { + verifyMeasurementSnapshot(receipt, root); + const member = (id: number, name: string) => + path.join(root, receipt.artifacts.find((artifact) => artifact.id === id)!.name, name); + const benchmarkFiles = new Set(); + const benchmarkPointCounts = new Map(); + const evaluations: ReceiptIngestInputs['evaluations'] = []; + for (const point of receipt.points) { + if (point.kind === 'throughput') { + const file = member(point.normalized_artifact_id, point.normalized_path); + benchmarkFiles.add(file); + benchmarkPointCounts.set(file, (benchmarkPointCounts.get(file) ?? 0) + 1); + continue; + } + const mapped = [...mapReceiptPointRows(receipt, root, point).mapped].filter( + (row): row is EvalParams => + row !== null && 'task' in row && row.task === point.task && row.conc === point.concurrency, + ); + if (mapped.length !== 1) throw new Error('Required eval point is missing or ambiguous'); + const samples = projectEvalSamples( + mapEvalSamples( + fs.readFileSync(member(point.samples_artifact_id!, point.samples_path!), 'utf8'), + tracker, + ), + ); + if (samples.length !== point.sample_count) + throw new Error('Required eval sample projection is incomplete'); + evaluations.push({ params: mapped[0], samples }); + } + return { benchmarkFiles: [...benchmarkFiles], benchmarkPointCounts, evaluations }; +} + +/** Changelog modes cannot narrow the independently accepted measurement scope. */ +export function assertReceiptIngestMode(inputs: ReceiptIngestInputs, evalsOnly: boolean): void { + if (evalsOnly && inputs.benchmarkFiles.length > 0) + throw new Error('Accepted throughput points cannot be imported as an evals-only run'); +} + +/** Count successful persisted point identities per accepted file, including replay. */ +export async function completeReceiptIngest( + sql: postgres.Sql, + receipt: MeasurementReceipt, + inputs: ReceiptIngestInputs, + persistedBenchmarks: ReadonlyMap, +): Promise { + for (const [file, expected] of inputs.benchmarkPointCounts) { + if ((persistedBenchmarks.get(file) ?? 0) !== expected) + throw new Error(`Accepted throughput points were skipped or collapsed: ${file}`); + } + await refreshLatestBenchmarks(sql); + await completeMeasurementSnapshot(sql, receipt); +} + +/** Persist both supported eval shapes through the same summary-and-samples path. */ +export async function ingestReceiptEvaluations( + sql: postgres.Sql, + inputs: ReceiptIngestInputs, + getOrCreateConfig: (config: ConfigParams) => Promise, + workflowRunId: number, + date: string, +): Promise<{ newEvals: number; newSamples: number; sampleFiles: number }> { + let newEvals = 0; + let newSamples = 0; + for (const { params, samples } of inputs.evaluations) { + const configId = await getOrCreateConfig(params.config); + const result = await ingestEvalRow(sql, configId, params, workflowRunId, date); + if (result.outcome === 'new') newEvals++; + const stored = await bulkIngestEvalSamples(sql, result.id, samples); + newSamples += stored.newCount; + } + return { newEvals, newSamples, sampleFiles: inputs.evaluations.length }; +} diff --git a/packages/db/src/ingest-ci-run.ts b/packages/db/src/ingest-ci-run.ts index d24c30713..4900c047b 100644 --- a/packages/db/src/ingest-ci-run.ts +++ b/packages/db/src/ingest-ci-run.ts @@ -23,15 +23,17 @@ */ import fs from 'fs'; +import { receiptFromEnvironment, publicationFromEnvironment } from './lib/measurement-receipt'; import { - receiptFromEnvironment, - publicationFromEnvironment, - verifyMeasurementSnapshot, -} from './lib/measurement-receipt'; + prepareReceiptIngestInputs, + assertReceiptIngestMode, + completeReceiptIngest, + ingestReceiptEvaluations, + type ReceiptIngestInputs, +} from './etl/receipt-ingest'; import { prepareReceiptArtifacts } from './lib/receipt-artifact-preparation'; import { claimMeasurementSnapshot, - completeMeasurementSnapshot, assertLegacySnapshotUnclaimed, } from './etl/measurement-snapshot'; import { createHash } from 'node:crypto'; @@ -250,6 +252,7 @@ if (reusedIngestMetadata) { runAttemptNum = reusedIngestMetadata.sourceRunAttempt; } +let receiptInputs: ReceiptIngestInputs | null = null; if (measurementReceipt) { if ( measurementReceipt.repository !== REPO || @@ -258,7 +261,7 @@ if (measurementReceipt) { ) throw new Error('Receipt source/attempt differs from ingest request'); publicationFromEnvironment(measurementReceipt, requestedRunIdStr); - verifyMeasurementSnapshot(measurementReceipt, artifactsDir); + receiptInputs = prepareReceiptIngestInputs(measurementReceipt, artifactsDir, tracker); } const runIdNum = parseInt(runIdStr, 10); @@ -315,9 +318,6 @@ async function main(): Promise { return; } - await (measurementReceipt - ? claimMeasurementSnapshot(sql, measurementReceipt) - : assertLegacySnapshotUnclaimed(sql, REPO, runIdStr, runAttemptNum)); validateRunBackfills(); const configCache = createConfigCache(sql); const { getOrCreateConfig, preloadConfigs } = configCache; @@ -374,9 +374,6 @@ async function main(): Promise { if (requiredPowerPoints.length > 0) console.log(` Required power: ${requiredPowerPoints.length} source benchmark points verified`); - await preloadConfigs(); - console.log(` ${configCache.size} configs preloaded`); - if (!fs.existsSync(artifactsDir)) { throw new Error(`Artifacts directory does not exist: ${artifactsDir}`); } @@ -439,9 +436,16 @@ async function main(): Promise { } const appendOnly = hasAppendOnlyFlag(changelogs); const evalsOnly = hasEvalsOnlyFlag(changelogs); + if (receiptInputs) assertReceiptIngestMode(receiptInputs, evalsOnly); if (evalsOnly && requiredPowerPoints.length > 0) throw new Error('Required power: benchmark scope cannot be published as an evals-only run'); + await (measurementReceipt + ? claimMeasurementSnapshot(sql, measurementReceipt) + : assertLegacySnapshotUnclaimed(sql, REPO, runIdStr, runAttemptNum)); + await preloadConfigs(); + console.log(` ${configCache.size} configs preloaded`); + const workflowRunId = await getOrCreateWorkflowRun({ githubRunId: runId, runAttempt: runAttemptNum, @@ -499,6 +503,7 @@ async function main(): Promise { console.log('\n--- Benchmark Results ---'); const retainedPowerPoints: BenchmarkParams[] = []; + const persistedReceiptBenchmarks = new Map(); if (evalsOnly) { console.log(' Skipped (evals-only run)'); } else { @@ -539,7 +544,10 @@ async function main(): Promise { ); } - const allBmkFiles = [...bmkFiles, ...allBmkDirs.flatMap((d) => findJsonFiles(d))]; + const allBmkFiles = receiptInputs?.benchmarkFiles ?? [ + ...bmkFiles, + ...allBmkDirs.flatMap((d) => findJsonFiles(d)), + ]; const seenPointIdentities = new Map(); console.log(` Found ${allBmkFiles.length} benchmark JSON file(s)`); @@ -653,6 +661,7 @@ async function main(): Promise { ); totalNewBmk += newCount; totalDupBmk += dupCount; + if (receiptInputs) persistedReceiptBenchmarks.set(file, insertedIds.length); if (requiredPowerPoints.length > 0) retainedPowerPoints.push(...toInsert); // Build availability only after successful insert @@ -888,8 +897,20 @@ async function main(): Promise { // `metrics`. Samples then attach to the resolved row id. console.log('\n--- Eval Results ---'); + if (receiptInputs) { + const result = await ingestReceiptEvaluations( + sql, + receiptInputs, + getOrCreateConfig, + workflowRunId, + date, + ); + totalEvals += result.newEvals; + totalSamples += result.newSamples; + totalSampleFiles += result.sampleFiles; + } const evalDir = path.join(artifactsDir, ARTIFACT_NAMES.evals); - const evalFiles = findJsonFiles(evalDir); + const evalFiles = receiptInputs ? [] : findJsonFiles(evalDir); for (const file of evalFiles) { const data = readJson(file); @@ -914,17 +935,18 @@ async function main(): Promise { // Per-config eval dirs (`eval_*`) — same on-disk shape as the eval ZIPs // handled by `ingest-gcs-backup.ts`, but already unzipped. Each dir holds // one config's meta_env.json, results JSON, and samples JSONL. - const perConfigEvalDirs = fs.existsSync(artifactsDir) - ? fs - .readdirSync(artifactsDir) - .filter( - (d) => - d.startsWith('eval_') && - !d.startsWith(ARTIFACT_NAMES.evals) && - fs.statSync(path.join(artifactsDir, d)).isDirectory(), - ) - .map((d) => path.join(artifactsDir, d)) - : []; + const perConfigEvalDirs = + !receiptInputs && fs.existsSync(artifactsDir) + ? fs + .readdirSync(artifactsDir) + .filter( + (d) => + d.startsWith('eval_') && + !d.startsWith(ARTIFACT_NAMES.evals) && + fs.statSync(path.join(artifactsDir, d)).isDirectory(), + ) + .map((d) => path.join(artifactsDir, d)) + : []; if (perConfigEvalDirs.length > 0) { console.log(` Found ${perConfigEvalDirs.length} per-config eval dir(s)`); @@ -1065,8 +1087,9 @@ async function main(): Promise { if (measurementReceipt && Object.values(tracker.skips).some((count) => count > 0)) throw new Error('Accepted snapshot ingestion skipped required input'); - await refreshLatestBenchmarks(sql); - if (measurementReceipt) await completeMeasurementSnapshot(sql, measurementReceipt); + await (measurementReceipt && receiptInputs + ? completeReceiptIngest(sql, measurementReceipt, receiptInputs, persistedReceiptBenchmarks) + : refreshLatestBenchmarks(sql)); console.log('\n=== ingest-ci-run complete ==='); console.log(' Invalidate API cache: bun run admin:cache:invalidate'); diff --git a/packages/db/src/lib/artifact-archive-worker.mjs b/packages/db/src/lib/artifact-archive-worker.mjs new file mode 100644 index 000000000..edd8b56a9 --- /dev/null +++ b/packages/db/src/lib/artifact-archive-worker.mjs @@ -0,0 +1,159 @@ +// Node/Bun worker for the synchronous ingestion boundary. Payloads stay in +// bounded ZIP streams; only the bounded member inventory crosses stdout. +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { crc32 } from 'node:zlib'; +import yauzl from 'yauzl'; + +const MAX_ARCHIVE_BYTES = 20 * 1024 ** 3; +const MAX_MEMBER_BYTES = 10 * 1024 ** 3; +const MAX_MEMBERS = 100_000; +const MAX_NAME_BYTES = 16 * 1024 ** 2; + +async function scanArchive(archive, destination) { + const zip = await yauzl.openPromise(archive, { + autoClose: false, + strictFileNames: true, + validateEntrySizes: true, + }); + const members = []; + const names = new Set(); + const files = new Set(); + let total = 0; + let nameBytes = 0; + try { + for await (const entry of zip.eachEntry()) { + const name = entry.fileName; + const segments = name.replace(/\/$/u, '').split('/'); + const mode = (entry.externalFileAttributes >>> 16) & 0o170000; + if ( + !name || + name.includes('\\') || + name.includes('\0') || + name.startsWith('/') || + /^[A-Za-z]:/u.test(name) || + segments.some((part) => !part || part === '.' || part === '..') || + (mode !== 0 && mode !== 0o100000 && mode !== 0o040000) + ) { + throw new Error(`Unsafe archive member: ${name}`); + } + const normalized = segments.join('/'); + if (names.has(normalized)) throw new Error(`Duplicate archive member: ${normalized}`); + names.add(normalized); + nameBytes += Buffer.byteLength(name); + if (names.size > MAX_MEMBERS || nameBytes > MAX_NAME_BYTES) { + throw new Error('Artifact member metadata exceeds size budget'); + } + if (name.endsWith('/')) continue; + if (!Number.isSafeInteger(entry.uncompressedSize) || entry.uncompressedSize < 0) { + throw new Error(`Invalid archive member size: ${normalized}`); + } + total += entry.uncompressedSize; + if (total > MAX_ARCHIVE_BYTES || entry.uncompressedSize > MAX_MEMBER_BYTES) { + throw new Error('Artifact exceeds extraction size budget'); + } + files.add(normalized); + const hash = createHash('sha256'); + let size = 0; + let checksum = 0; + let output; + try { + if (destination !== undefined) { + const target = path.join(destination, normalized); + fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 }); + output = fs.openSync(target, 'wx', 0o600); + } + const stream = await zip.openReadStreamPromise(entry); + for await (const chunk of stream) { + size += chunk.length; + if (size > entry.uncompressedSize || size > MAX_MEMBER_BYTES) { + throw new Error(`Archive member size mismatch: ${normalized}`); + } + hash.update(chunk); + checksum = crc32(chunk, checksum); + if (output !== undefined) { + let offset = 0; + while (offset < chunk.length) { + const written = fs.writeSync(output, chunk, offset, chunk.length - offset); + if (written === 0) throw new Error(`Cannot write archive member: ${normalized}`); + offset += written; + } + } + } + } finally { + if (output !== undefined) fs.closeSync(output); + } + if (size !== entry.uncompressedSize || checksum !== entry.crc32) { + throw new Error(`Archive member CRC/size mismatch: ${normalized}`); + } + members.push({ path: normalized, size, sha256: hash.digest('hex') }); + } + for (const name of names) { + const segments = name.split('/'); + segments.pop(); + while (segments.length > 0) { + if (files.has(segments.join('/'))) { + throw new Error(`Archive file/directory collision: ${name}`); + } + segments.pop(); + } + } + return members; + } finally { + zip.close(); + } +} + +function verifyMembers(members, expected) { + const selected = new Map(expected.map((member) => [member.path, member])); + if (selected.size !== expected.length || selected.size !== members.length) { + throw new Error('Archive member set mismatch'); + } + for (const member of members) { + const match = selected.get(member.path); + if (!match || match.sha256 !== member.sha256 || match.size !== member.size) { + throw new Error(`Archive member digest/size mismatch: ${member.path}`); + } + } +} + +export async function processArchive({ archive, destination, expected }) { + const stat = fs.statSync(archive); + if (!stat.isFile() || stat.size > MAX_ARCHIVE_BYTES) { + throw new Error('Artifact exceeds archive size budget'); + } + // First pass verifies every payload, even in the legacy path. A late unsafe + // member, damaged CRC, or receipt mismatch cannot leave extraction writes. + const members = await scanArchive(archive); + if (expected !== undefined) verifyMembers(members, expected); + if (destination !== undefined) { + fs.mkdirSync(path.dirname(destination), { recursive: true }); + try { + fs.mkdirSync(destination, { mode: 0o700 }); + } catch (error) { + if (error.code === 'EEXIST') + throw new Error(`Refusing artifact overwrite: ${destination}`, { cause: error }); + throw error; + } + try { + // The second pass streams to disk and rechecks hashes, defending against + // archive changes between validation and extraction. + verifyMembers(await scanArchive(archive, destination), members); + } catch (error) { + fs.rmSync(destination, { recursive: true, force: true }); + throw error; + } + } + return { members }; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === import.meta.filename) { + try { + const request = JSON.parse(fs.readFileSync(0, 'utf8')); + process.stdout.write(JSON.stringify(await processArchive(request))); + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } +} diff --git a/packages/db/src/lib/artifact-archive.test.ts b/packages/db/src/lib/artifact-archive.test.ts index 2f20c70e4..962fcd564 100644 --- a/packages/db/src/lib/artifact-archive.test.ts +++ b/packages/db/src/lib/artifact-archive.test.ts @@ -1,9 +1,15 @@ import AdmZip from 'adm-zip'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { fileURLToPath } from 'node:url'; +import { crc32, createDeflateRaw } from 'node:zlib'; import { afterEach, expect, it } from 'vitest'; -import { extractVerifiedArchive, inspectArchive } from './artifact-archive'; +import { extractVerifiedArchive, inspectArchive, sha256, sha256File } from './artifact-archive'; const roots: string[] = []; function root() { @@ -40,3 +46,165 @@ it('rejects links and file-directory collisions before filesystem writes', () => collision.addFile('raw/result.json', Buffer.from('{}')); expect(() => inspectArchive(collision.toBuffer())).toThrow('collision'); }); + +it('verifies a disk archive and resolves its worker independently of the current directory', () => { + const zip = new AdmZip(); + const payload = Buffer.from('verified trace\n'); + zip.addFile('raw/trace.jsonl', payload); + const directory = root(); + const archive = path.join(directory, 'archive.zip'); + const target = path.join(directory, 'output'); + fs.writeFileSync(archive, zip.toBuffer()); + const expected = [{ path: 'raw/trace.jsonl', size: payload.length, sha256: sha256(payload) }]; + expect(sha256File(archive)).toBe(sha256(zip.toBuffer())); + const previous = process.cwd(); + try { + process.chdir(directory); + extractVerifiedArchive(archive, target, expected); + } finally { + process.chdir(previous); + } + expect(fs.readFileSync(path.join(target, 'raw/trace.jsonl'))).toEqual(payload); +}); + +it('checks the complete member inventory before creating any extraction directory', () => { + const zip = new AdmZip(); + zip.addFile('first.json', Buffer.from('{}')); + zip.addFile('last.json', Buffer.from('{}')); + const directory = root(); + const expected = [{ path: 'first.json', size: 2, sha256: sha256('{}') }]; + const target = path.join(directory, 'output'); + expect(() => extractVerifiedArchive(zip.toBuffer(), target, expected)).toThrow('member set'); + expect(fs.existsSync(target)).toBe(false); + + zip.getEntry('last.json')!.attr = (0o120777 << 16) >>> 0; + expect(() => extractVerifiedArchive(zip.toBuffer(), target)).toThrow('Unsafe'); + expect(fs.existsSync(target)).toBe(false); +}); + +it('rejects damaged payload CRCs before extraction and preserves existing symlink destinations', () => { + const zip = new AdmZip(); + zip.addFile('trace.jsonl', Buffer.from('valid bytes')); + const bytes = Buffer.from(zip.toBuffer()); + // Mutate only the ZIP's recorded central-directory CRC, preserving deflate data. + const central = bytes.indexOf(Buffer.from('504b0102', 'hex')); + bytes.writeUInt32LE((bytes.readUInt32LE(central + 16) ^ 1) >>> 0, central + 16); + const directory = root(); + const target = path.join(directory, 'output'); + expect(() => extractVerifiedArchive(bytes, target)).toThrow('CRC/size mismatch'); + expect(fs.existsSync(target)).toBe(false); + fs.symlinkSync(path.join(directory, 'missing'), target); + expect(() => extractVerifiedArchive(zip.toBuffer(), target)).toThrow('overwrite'); + expect(fs.lstatSync(target).isSymbolicLink()).toBe(true); +}); + +it('runs the synchronous archive API and streaming worker under the production Bun runtime', () => { + const zip = new AdmZip(); + zip.addFile('raw/trace.jsonl', Buffer.from('Bun verified bytes')); + const directory = root(); + const archive = path.join(directory, 'archive.zip'); + const target = path.join(directory, 'output'); + fs.writeFileSync(archive, zip.toBuffer()); + const module = fileURLToPath(new URL('artifact-archive.ts', import.meta.url)); + const result = spawnSync( + 'bun', + [ + '--eval', + `import { extractVerifiedArchive } from ${JSON.stringify(module)}; + extractVerifiedArchive(${JSON.stringify(archive)}, ${JSON.stringify(target)});`, + ], + { cwd: directory, encoding: 'utf8' }, + ); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + expect(fs.readFileSync(path.join(target, 'raw/trace.jsonl'), 'utf8')).toBe('Bun verified bytes'); +}); + +/** Write a large, valid fixture without allocating its uncompressed payload. */ +async function writeLargeArchive(filename: string, megabytes: number) { + const name = Buffer.from('raw/large-trace.jsonl'); + const local = Buffer.alloc(30 + name.length); + Buffer.from('504b0304', 'hex').copy(local); + local.writeUInt16LE(20, 4); + local.writeUInt16LE(8, 8); + local.writeUInt16LE(name.length, 26); + name.copy(local, 30); + fs.writeFileSync(filename, local); + const chunk = Buffer.alloc(1024 ** 2, 'x'); + const hash = createHash('sha256'); + let checksum = 0; + const input = Readable.from( + (function* () { + for (let index = 0; index < megabytes; index++) { + hash.update(chunk); + checksum = crc32(chunk, checksum); + yield chunk; + } + })(), + ); + await pipeline( + input, + createDeflateRaw(), + fs.createWriteStream(filename, { flags: 'r+', start: local.length }), + ); + const compressedSize = fs.statSync(filename).size - local.length; + local.writeUInt32LE(checksum, 14); + local.writeUInt32LE(compressedSize, 18); + local.writeUInt32LE(megabytes * chunk.length, 22); + const descriptor = fs.openSync(filename, 'r+'); + try { + fs.writeSync(descriptor, local, 0, local.length, 0); + } finally { + fs.closeSync(descriptor); + } + const central = Buffer.alloc(46 + name.length); + Buffer.from('504b0102', 'hex').copy(central); + central.writeUInt16LE(20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt16LE(8, 10); + central.writeUInt32LE(checksum, 16); + central.writeUInt32LE(compressedSize, 20); + central.writeUInt32LE(megabytes * chunk.length, 24); + central.writeUInt16LE(name.length, 28); + name.copy(central, 46); + const end = Buffer.alloc(22); + Buffer.from('504b0506', 'hex').copy(end); + end.writeUInt16LE(1, 8); + end.writeUInt16LE(1, 10); + end.writeUInt32LE(central.length, 12); + end.writeUInt32LE(local.length + compressedSize, 16); + fs.appendFileSync(filename, Buffer.concat([central, end])); + return { path: name.toString(), size: megabytes * chunk.length, sha256: hash.digest('hex') }; +} + +it('extracts a 256 MiB trace without retaining the payload in memory', async () => { + const directory = root(); + const archive = path.join(directory, 'large.zip'); + const target = path.join(directory, 'output'); + const expected = await writeLargeArchive(archive, 256); + const worker = new URL('artifact-archive-worker.mjs', import.meta.url); + const program = ` + import { processArchive } from ${JSON.stringify(worker.href)}; + const result = await processArchive(JSON.parse(process.argv[1])); + process.stdout.write(JSON.stringify({ ...result, maxRSS: process.resourceUsage().maxRSS })); + `; + const result = spawnSync( + process.execPath, + [ + '--input-type=module', + '--eval', + program, + JSON.stringify({ archive, destination: target, expected: [expected] }), + ], + { cwd: directory, encoding: 'utf8', timeout: 25_000 }, + ); + expect(result.error).toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + const report = JSON.parse(result.stdout); + expect(report.members).toEqual([expected]); + // maxRSS is in KiB on Node's supported Linux/macOS runners. Allow ample + // runtime overhead while detecting retention of even one entire payload. + expect(report.maxRSS).toBeLessThan(192 * 1024); + expect(fs.statSync(path.join(target, expected.path)).size).toBe(expected.size); + expect(sha256File(path.join(target, expected.path))).toBe(expected.sha256); +}, 30_000); diff --git a/packages/db/src/lib/artifact-archive.ts b/packages/db/src/lib/artifact-archive.ts index b590ebff0..64c362d88 100644 --- a/packages/db/src/lib/artifact-archive.ts +++ b/packages/db/src/lib/artifact-archive.ts @@ -1,97 +1,86 @@ -import AdmZip from 'adm-zip'; +import { spawnSync } from 'node:child_process'; import { createHash } from 'node:crypto'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; +import { fileURLToPath } from 'node:url'; export interface ArchiveMember { path: string; sha256: string; size: number; } + export function sha256(bytes: Buffer | string): string { return createHash('sha256').update(bytes).digest('hex'); } -/** Validate every entry before writing anything. Extraction never follows ZIP links. */ -export function inspectArchive(bytes: Buffer): { - members: ArchiveMember[]; - files: Map; -} { - const zip = new AdmZip(bytes); - const files = new Map(); - const names = new Set(); - let total = 0; - for (const entry of zip.getEntries()) { - const name = entry.entryName; - const segments = name.replace(/\/$/u, '').split('/'); - const mode = (entry.attr >>> 16) & 0o170000; - if ( - !name || - name.includes('\\') || - name.includes('\0') || - name.startsWith('/') || - /^[A-Za-z]:/u.test(name) || - segments.some((part) => !part || part === '.' || part === '..') || - (mode !== 0 && mode !== 0o100000 && mode !== 0o040000) - ) { - throw new Error(`Unsafe archive member: ${name}`); - } - const normalized = segments.join('/'); - if (names.has(normalized)) throw new Error(`Duplicate archive member: ${normalized}`); - names.add(normalized); - if (entry.isDirectory) continue; - total += entry.header.size; - if (total > 20 * 1024 ** 3 || entry.header.size > 10 * 1024 ** 3) { - throw new Error('Artifact exceeds extraction size budget'); +/** Hash archive bytes without retaining the download in the ingestion process. */ +export function sha256File(filename: string): string { + const descriptor = fs.openSync(filename, 'r'); + try { + const hash = createHash('sha256'); + const chunk = Buffer.allocUnsafe(1024 * 1024); + let length; + while ((length = fs.readSync(descriptor, chunk, 0, chunk.length, null)) > 0) { + hash.update(chunk.subarray(0, length)); } - files.set(normalized, entry.getData()); + return hash.digest('hex'); + } finally { + fs.closeSync(descriptor); } - for (const name of files.keys()) { - const segments = name.split('/'); - segments.pop(); - while (segments.length > 0) { - if (files.has(segments.join('/'))) - throw new Error(`Archive file/directory collision: ${name}`); - segments.pop(); +} + +function processArchive( + archive: Buffer | string, + destination?: string, + expected?: readonly ArchiveMember[], +): { members: ArchiveMember[] } { + // Only callers that already own a small Buffer use this compatibility path. + // Production downloads pass a private on-disk ZIP directly to the worker. + const temporary = Buffer.isBuffer(archive) + ? fs.mkdtempSync(path.join(os.tmpdir(), 'artifact-inspect-')) + : undefined; + try { + const filename = temporary + ? path.join(temporary, 'archive.zip') + : path.resolve(archive as string); + if (temporary) fs.writeFileSync(filename, archive, { flag: 'wx', mode: 0o600 }); + const input = JSON.stringify({ + archive: filename, + destination: destination === undefined ? undefined : path.resolve(destination), + expected, + }); + if (Buffer.byteLength(input) > 32 * 1024 ** 2) { + throw new Error('Artifact member metadata exceeds size budget'); + } + // Keep the public ingestion API synchronous while the isolated worker uses + // bounded asynchronous ZIP streams. Only member metadata crosses stdout. + const result = spawnSync( + process.execPath, + [fileURLToPath(new URL('artifact-archive-worker.mjs', import.meta.url))], + { input, encoding: 'utf8', maxBuffer: 32 * 1024 ** 2 }, + ); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error(result.stderr.trim() || 'Artifact verification failed'); } + return JSON.parse(result.stdout) as { members: ArchiveMember[] }; + } finally { + if (temporary) fs.rmSync(temporary, { recursive: true, force: true }); } - return { - files, - members: [...files].map(([name, data]) => ({ - path: name, - size: data.length, - sha256: sha256(data), - })), - }; } +/** Validate and hash every entry, retaining member metadata rather than payloads. */ +export function inspectArchive(archive: Buffer | string): { members: ArchiveMember[] } { + return processArchive(archive); +} + +/** Verify the complete archive before creating its exclusive extraction directory. */ export function extractVerifiedArchive( - bytes: Buffer, + archive: Buffer | string, destination: string, expected?: readonly ArchiveMember[], ): void { - const { files, members } = inspectArchive(bytes); - if (expected) { - const selected = new Map(expected.map((member) => [member.path, member])); - if (selected.size !== expected.length || selected.size !== members.length) - throw new Error('Archive member set mismatch'); - for (const member of members) { - const match = selected.get(member.path); - if (!match || match.sha256 !== member.sha256 || match.size !== member.size) { - throw new Error(`Archive member digest/size mismatch: ${member.path}`); - } - } - } - if (fs.existsSync(destination)) throw new Error(`Refusing artifact overwrite: ${destination}`); - fs.mkdirSync(destination, { recursive: true }); - try { - for (const [name, data] of files) { - const target = path.join(destination, name); - fs.mkdirSync(path.dirname(target), { recursive: true }); - fs.writeFileSync(target, data, { flag: 'wx', mode: 0o600 }); - } - } catch (error) { - fs.rmSync(destination, { recursive: true, force: true }); - throw error; - } + processArchive(archive, destination, expected); } diff --git a/packages/db/src/lib/github-artifacts.test.ts b/packages/db/src/lib/github-artifacts.test.ts index 571643a5e..f86ccebb0 100644 --- a/packages/db/src/lib/github-artifacts.test.ts +++ b/packages/db/src/lib/github-artifacts.test.ts @@ -1,6 +1,15 @@ -import { describe, expect, it } from 'vitest'; +import AdmZip from 'adm-zip'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; -import { RUNNER_SUFFIX_RE, dedupeArtifactsByLogicalName } from './github-artifacts.js'; +import { sha256 } from './artifact-archive.js'; +import { + RUNNER_SUFFIX_RE, + dedupeArtifactsByLogicalName, + downloadArtifact, +} from './github-artifacts.js'; const art = (name: string, created_at: string) => ({ name, @@ -40,3 +49,100 @@ describe('dedupeArtifactsByLogicalName', () => { expect(deduped.get('run-stats')?.name).toBe('run-stats'); }); }); + +const roots: string[] = []; +afterEach(() => { + vi.unstubAllEnvs(); + for (const directory of roots.splice(0)) fs.rmSync(directory, { recursive: true, force: true }); +}); + +function downloadFixture() { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'artifact-download-test-')); + roots.push(directory); + const scratch = path.join(directory, 'scratch'); + fs.mkdirSync(scratch); + const zip = new AdmZip(); + const payload = Buffer.from('downloaded trace\n'); + zip.addFile('raw/trace.jsonl', payload); + const bytes = zip.toBuffer(); + const archive = path.join(directory, 'fixture.zip'); + fs.writeFileSync(archive, bytes); + // Exercise the real process/file-descriptor boundary without a network call. + // This executable streams its fixture exactly as `gh api .../zip` does. + fs.writeFileSync( + path.join(directory, 'gh'), + `#!/usr/bin/env node +const fs = require('node:fs'); +fs.writeFileSync(process.env.ARTIFACT_TEST_CALL, JSON.stringify({ + args: process.argv.slice(2), file: fs.fstatSync(1).isFile(), +})); +if (process.env.ARTIFACT_TEST_FAIL === '1') { + fs.writeSync(1, Buffer.from('partial ZIP')); + process.exit(7); +} +const fd = fs.openSync(process.env.ARTIFACT_TEST_ZIP, 'r'); +const chunk = Buffer.alloc(32 * 1024); +let count; +while ((count = fs.readSync(fd, chunk, 0, chunk.length, null)) > 0) { + let offset = 0; + while (offset < count) offset += fs.writeSync(1, chunk, offset, count - offset); +} +fs.closeSync(fd); +`, + { mode: 0o700 }, + ); + const call = path.join(directory, 'call.json'); + vi.stubEnv('PATH', `${directory}${path.delimiter}${process.env.PATH}`); + vi.stubEnv('TMPDIR', scratch); + vi.stubEnv('ARTIFACT_TEST_ZIP', archive); + vi.stubEnv('ARTIFACT_TEST_CALL', call); + return { + scratch, + call, + output: path.join(directory, 'downloads'), + bytes, + payload, + artifact: { + id: 123, + name: 'benchmark', + archive_download_url: + 'https://api.github.com/repos/example/project/actions/artifacts/123/zip', + created_at: '2026-06-01T00:00:00Z', + }, + }; +} + +it('streams gh stdout to disk and verifies archive and member digests before extraction', () => { + const fixture = downloadFixture(); + const destination = downloadArtifact(fixture.artifact, fixture.output, { + isolated: true, + sha256: sha256(fixture.bytes), + members: [ + { path: 'raw/trace.jsonl', size: fixture.payload.length, sha256: sha256(fixture.payload) }, + ], + }); + expect(destination).toBe(path.join(fixture.output, '123')); + expect(fs.readFileSync(path.join(destination, 'raw/trace.jsonl'))).toEqual(fixture.payload); + expect(JSON.parse(fs.readFileSync(fixture.call, 'utf8'))).toEqual({ + args: ['api', 'repos/example/project/actions/artifacts/123/zip'], + file: true, + }); + expect(fs.readdirSync(fixture.scratch)).toEqual([]); +}); + +it('removes partial ZIP downloads after gh fails without creating an extraction root', () => { + const fixture = downloadFixture(); + vi.stubEnv('ARTIFACT_TEST_FAIL', '1'); + expect(() => downloadArtifact(fixture.artifact, fixture.output)).toThrow(); + expect(fs.existsSync(fixture.output)).toBe(false); + expect(fs.readdirSync(fixture.scratch)).toEqual([]); +}); + +it('removes a completed download whose archive digest fails before extracting it', () => { + const fixture = downloadFixture(); + expect(() => + downloadArtifact(fixture.artifact, fixture.output, { sha256: '0'.repeat(64) }), + ).toThrow('digest mismatch'); + expect(fs.existsSync(fixture.output)).toBe(false); + expect(fs.readdirSync(fixture.scratch)).toEqual([]); +}); diff --git a/packages/db/src/lib/github-artifacts.ts b/packages/db/src/lib/github-artifacts.ts index c809894e6..60c60000c 100644 --- a/packages/db/src/lib/github-artifacts.ts +++ b/packages/db/src/lib/github-artifacts.ts @@ -5,8 +5,10 @@ */ import { execFileSync } from 'node:child_process'; -import { extractVerifiedArchive, sha256, type ArchiveMember } from './artifact-archive.js'; +import { extractVerifiedArchive, sha256File, type ArchiveMember } from './artifact-archive.js'; import path from 'node:path'; +import fs from 'node:fs'; +import os from 'node:os'; export interface ArtifactMeta { id?: number; @@ -92,12 +94,8 @@ export function downloadArtifact( if (!repo || (match && Number(match[2]) !== artifact.id)) throw new Error('Invalid GitHub artifact owner/ID'); validateRun(repo, String(artifact.id)); - const bytes = execFileSync('gh', ['api', `repos/${repo}/actions/artifacts/${artifact.id}/zip`], { - maxBuffer: 20 * 1024 ** 3, - stdio: ['ignore', 'pipe', 'inherit'], - }); const expected = options.sha256 ?? artifact.digest?.replace(/^sha256:/u, ''); - if (expected && (!/^[a-f0-9]{64}$/u.test(expected) || sha256(bytes) !== expected)) + if (expected && !/^[a-f0-9]{64}$/u.test(expected)) throw new Error(`Artifact digest mismatch: ${artifact.id}`); if ( !options.isolated && @@ -106,8 +104,28 @@ export function downloadArtifact( throw new Error('Unsafe legacy artifact display name'); } const destination = path.join(destRoot, options.isolated ? String(artifact.id) : artifact.name); - extractVerifiedArchive(bytes, destination, options.members); - return destination; + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'artifact-download-')); + const archive = path.join(temporary, 'archive.zip'); + try { + const descriptor = fs.openSync(archive, 'wx', 0o600); + try { + execFileSync('gh', ['api', `repos/${repo}/actions/artifacts/${artifact.id}/zip`], { + stdio: ['ignore', descriptor, 'inherit'], + }); + } finally { + fs.closeSync(descriptor); + } + if (fs.statSync(archive).size > 20 * 1024 ** 3) { + throw new Error('Artifact exceeds archive size budget'); + } + if (expected && sha256File(archive) !== expected) { + throw new Error(`Artifact digest mismatch: ${artifact.id}`); + } + extractVerifiedArchive(archive, destination, options.members); + return destination; + } finally { + fs.rmSync(temporary, { recursive: true, force: true }); + } } /** Fetch a run's current attempt for the explicitly weaker legacy path only. */ diff --git a/packages/db/src/lib/measurement-receipt.ts b/packages/db/src/lib/measurement-receipt.ts index c6c537c76..d3dfb7bde 100644 --- a/packages/db/src/lib/measurement-receipt.ts +++ b/packages/db/src/lib/measurement-receipt.ts @@ -1,7 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { isDeepStrictEqual } from 'node:util'; -import { sha256, type ArchiveMember } from './artifact-archive'; +import { sha256, sha256File, type ArchiveMember } from './artifact-archive'; import { mapBenchmarkRow } from '../etl/benchmark-mapper'; import { mapAggEvalRow, mapEvalRow } from '../etl/eval-mapper'; import { createSkipTracker } from '../etl/skip-tracker'; @@ -323,8 +323,7 @@ export function verifyMeasurementSnapshot(receipt: MeasurementReceipt, root: str const file = path.join(root, artifact.name, member.path); const stat = fs.lstatSync(file); if (!stat.isFile() || stat.isSymbolicLink()) throw new Error(`Invalid receipt file: ${file}`); - const bytes = fs.readFileSync(file); - if (bytes.length !== member.size || sha256(bytes) !== member.sha256) + if (stat.size !== member.size || sha256File(file) !== member.sha256) throw new Error(`Changed receipt member: ${file}`); } } diff --git a/packages/db/src/lib/receipt-artifact-preparation.test.ts b/packages/db/src/lib/receipt-artifact-preparation.test.ts index efeb85243..6034c0b73 100644 --- a/packages/db/src/lib/receipt-artifact-preparation.test.ts +++ b/packages/db/src/lib/receipt-artifact-preparation.test.ts @@ -1,13 +1,16 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { execFileSync } from 'node:child_process'; +import * as childProcess from 'node:child_process'; import { afterEach, expect, it, vi } from 'vitest'; import { prepareReceiptArtifacts } from './receipt-artifact-preparation'; import { parseMeasurementReceipt } from './measurement-receipt'; import { downloadArtifact } from './github-artifacts'; -vi.mock('node:child_process', () => ({ execFileSync: vi.fn() })); +vi.mock('node:child_process', async (importOriginal) => ({ + ...(await importOriginal()), + execFileSync: vi.fn(), +})); const fixture = new URL('fixtures/measurement-receipt/', import.meta.url); const bytes = fs.readFileSync(new URL('receipt.json', fixture)); const roots: string[] = []; @@ -29,13 +32,19 @@ function receipt() { } it('downloads exact retained IDs and verifies every member before preparing legacy discovery views', () => { const accepted = receipt(); - vi.mocked(execFileSync).mockImplementation((command, args) => { + vi.mocked(childProcess.execFileSync).mockImplementation((command, args, options) => { if (command !== 'gh') throw new Error('unexpected executable'); const endpoint = (args as string[])[1]; const id = Number(endpoint.match(/artifacts\/(?\d+)/u)?.[1]); const artifact = accepted.artifacts.find((item) => item.id === id); if (!artifact) throw new Error('unrequested newer artifact'); - if (endpoint.endsWith('/zip')) return fs.readFileSync(new URL(`${id}.zip`, fixture)); + if (endpoint.endsWith('/zip')) { + fs.writeSync( + (options as { stdio: number[] }).stdio[1], + fs.readFileSync(new URL(`${id}.zip`, fixture)), + ); + return ''; + } return JSON.stringify({ id, name: artifact.name, @@ -64,7 +73,7 @@ it('downloads exact retained IDs and verifies every member before preparing lega }); it('fails on missing/wrong ownership even if a same-name artifact exists', () => { const accepted = receipt(); - vi.mocked(execFileSync).mockReturnValue( + vi.mocked(childProcess.execFileSync).mockReturnValue( JSON.stringify({ id: 999, name: 'bmk_pilot', @@ -79,7 +88,13 @@ it('fails on missing/wrong ownership even if a same-name artifact exists', () => expect(fs.existsSync(path.join(destination, 'bmk_pilot'))).toBe(false); }); it('treats hostile display names as data under numeric roots and never interprets their URL', () => { - vi.mocked(execFileSync).mockReturnValue(fs.readFileSync(new URL('101.zip', fixture))); + vi.mocked(childProcess.execFileSync).mockImplementation((_command, _args, options) => { + fs.writeSync( + (options as { stdio: number[] }).stdio[1], + fs.readFileSync(new URL('101.zip', fixture)), + ); + return ''; + }); const destination = root(); const output = downloadArtifact( { @@ -94,7 +109,7 @@ it('treats hostile display names as data under numeric roots and never interpret expect(output).toBe(path.join(destination, '101')); expect(fs.readdirSync(destination)).toEqual(['101']); expect(JSON.parse(fs.readFileSync(path.join(output, 'agg.json'), 'utf8'))[0].conc).toBe(1); - expect(vi.mocked(execFileSync).mock.calls[0].slice(0, 2)).toEqual([ + expect(vi.mocked(childProcess.execFileSync).mock.calls[0].slice(0, 2)).toEqual([ 'gh', ['api', 'repos/org/repo/actions/artifacts/101/zip'], ]); diff --git a/packages/db/src/lib/receipt-transport.test.ts b/packages/db/src/lib/receipt-transport.test.ts index bf90af8db..0c846723a 100644 --- a/packages/db/src/lib/receipt-transport.test.ts +++ b/packages/db/src/lib/receipt-transport.test.ts @@ -2,11 +2,14 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import AdmZip from 'adm-zip'; -import { execFileSync } from 'node:child_process'; +import * as childProcess from 'node:child_process'; import { afterEach, expect, it, vi } from 'vitest'; import { prepareReceiptTransport } from './receipt-transport'; import { sha256 } from './artifact-archive'; -vi.mock('node:child_process', () => ({ execFileSync: vi.fn() })); +vi.mock('node:child_process', async (importOriginal) => ({ + ...(await importOriginal()), + execFileSync: vi.fn(), +})); const roots: string[] = []; function root() { const value = fs.mkdtempSync(path.join(os.tmpdir(), 'receipt-transport-')); @@ -34,7 +37,11 @@ const env = { ALLOWED_RECEIPT_ISSUER_SHAS: 'd'.repeat(40), ALLOWED_RECEIPT_ISSUER_WORKFLOW: '.github/workflows/issue-receipt.yml', }; -function api(endpoint: string) { +function zipResponse(bytes: Buffer, options: unknown): string { + fs.writeSync((options as { stdio: number[] }).stdio[1], bytes); + return ''; +} +function api(endpoint: string, options?: unknown) { if (endpoint.endsWith('/runs/100/artifacts')) return JSON.stringify([ { artifacts: [{ id: 101, name: 'native-execution-point', created_at: '' }] }, @@ -57,7 +64,7 @@ function api(endpoint: string) { status: 'completed', conclusion: 'success', }); - if (endpoint.endsWith('/301/zip')) return archive; + if (endpoint.endsWith('/301/zip')) return zipResponse(archive, options); if (endpoint.endsWith('/artifacts/301')) return JSON.stringify({ id: 301, @@ -70,7 +77,9 @@ function api(endpoint: string) { throw new Error('unrequested endpoint'); } it('enforces native capability from API inventory and accepts only the pinned successful issuer', () => { - vi.mocked(execFileSync).mockImplementation((_command, args) => api((args as string[])[1])); + vi.mocked(childProcess.execFileSync).mockImplementation((_command, args, options) => + api((args as string[])[1], options), + ); expect(() => prepareReceiptTransport({ ...env, RECEIPT_ARTIFACT_ID: '' }, root())).toThrow( 'Required source receipt', ); @@ -82,7 +91,7 @@ it('enforces native capability from API inventory and accepts only the pinned su ).toThrow('not deployed/allowed'); }); it('does not promote a same-named candidate workflow artifact to issuer authority', () => { - vi.mocked(execFileSync).mockImplementation((_command, args) => { + vi.mocked(childProcess.execFileSync).mockImplementation((_command, args, options) => { const endpoint = (args as string[])[1]; if (endpoint.endsWith('/runs/200')) return JSON.stringify({ @@ -92,14 +101,14 @@ it('does not promote a same-named candidate workflow artifact to issuer authorit status: 'completed', conclusion: 'success', }); - return api(endpoint); + return api(endpoint, options); }); const destination = root(); expect(() => prepareReceiptTransport(env, destination)).toThrow('successful allowed workflow'); expect(fs.readdirSync(destination)).toEqual([]); }); it('keeps explicit legacy behavior only when API inventory does not identify the native lane', () => { - vi.mocked(execFileSync).mockReturnValue('[{"artifacts":[]}]'); + vi.mocked(childProcess.execFileSync).mockReturnValue('[{"artifacts":[]}]'); expect( prepareReceiptTransport( { @@ -114,7 +123,9 @@ it('keeps explicit legacy behavior only when API inventory does not identify the }); it('requires later publication for native production even when source and merge IDs are equal', () => { - vi.mocked(execFileSync).mockImplementation((_command, args) => api((args as string[])[1])); + vi.mocked(childProcess.execFileSync).mockImplementation((_command, args, options) => + api((args as string[])[1], options), + ); expect(() => prepareReceiptTransport({ ...env, PUBLICATION_REQUIRED: 'true' }, root())).toThrow( 'Native production requires a later publication record', ); @@ -139,9 +150,9 @@ it('carries the immutable later record and mandatory-publication flag into produ const publicationZip = new AdmZip(); publicationZip.addFile('publication.json', content); const publicationArchive = Buffer.from(publicationZip.toBuffer()); - vi.mocked(execFileSync).mockImplementation((_command, args) => { + vi.mocked(childProcess.execFileSync).mockImplementation((_command, args, options) => { const endpoint = (args as string[])[1]; - if (endpoint.endsWith('/401/zip')) return publicationArchive; + if (endpoint.endsWith('/401/zip')) return zipResponse(publicationArchive, options); if (endpoint.endsWith('/artifacts/401')) return JSON.stringify({ id: 401, @@ -151,7 +162,7 @@ it('carries the immutable later record and mandatory-publication flag into produ digest: `sha256:${sha256(publicationArchive)}`, archive_download_url: 'https://api.github.com/repos/org/repo/actions/artifacts/401/zip', }); - return api(endpoint); + return api(endpoint, options); }); const values = prepareReceiptTransport( { @@ -177,9 +188,9 @@ it.each([ ['/runs/100/attempts/2', { head_sha: 'f'.repeat(40) }, 'completed original attempt'], ['/runs/100/attempts/2', { run_attempt: 3 }, 'completed original attempt'], ])('rejects wrong issuer branch/event or original attempt: %s %j', (suffix, change, message) => { - vi.mocked(execFileSync).mockImplementation((_command, args) => { + vi.mocked(childProcess.execFileSync).mockImplementation((_command, args, options) => { const endpoint = (args as string[])[1]; - const result = api(endpoint); + const result = api(endpoint, options); return endpoint.endsWith(suffix) ? JSON.stringify({ ...JSON.parse(result as string), ...change }) : result; From 9003c0bcc2a46d52167ca007d50dd26cc0c911ac Mon Sep 17 00:00:00 2001 From: functionstackx <47992694+functionstackx@users.noreply.github.com> Date: Sat, 19 Sep 2026 20:05:26 -0400 Subject: [PATCH 5/5] feat(db): verify explicit migration-only deployments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a master-only migration workflow with explicit staging/production targets and an exact reviewed commit guard. Apply the existing migrations, verify receipt schema in a read-only transaction, and retain a credential-free readiness report. Cover preservation, replay and schema drift with real PGlite tests. 中文：新增仅从 master 执行的独立数据库迁移流程，显式选择 staging 或 production，并校验准确的已审查提交。沿用现有迁移命令，在只读事务中验证回执 schema，保留不含凭证的就绪报告，并通过真实 PGlite 测试覆盖数据保留、重复执行及 schema 不一致。 --- .github/workflows/migrate-database.yml | 82 ++++++++++++++ docs/measurement-publication.md | 19 +++- docs/measurement-publication_zh.md | 19 +++- .../db/src/lib/measurement-schema.test.ts | 102 ++++++++++++++++++ packages/db/src/lib/measurement-schema.ts | 64 +++++++++++ packages/db/src/verify-measurement-schema.ts | 36 +++++++ 6 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/migrate-database.yml create mode 100644 packages/db/src/lib/measurement-schema.test.ts create mode 100644 packages/db/src/lib/measurement-schema.ts create mode 100644 packages/db/src/verify-measurement-schema.ts diff --git a/.github/workflows/migrate-database.yml b/.github/workflows/migrate-database.yml new file mode 100644 index 000000000..a6a901bfa --- /dev/null +++ b/.github/workflows/migrate-database.yml @@ -0,0 +1,82 @@ +name: Migrate Database + +on: + workflow_dispatch: + inputs: + database-target: + description: Explicit database target; existing measurements are preserved + required: true + type: choice + options: [staging, production] + expected-sha: + description: Exact reviewed 40-character app commit to migrate and verify + required: true + type: string + +permissions: {} + +concurrency: + group: database-migrations-${{ inputs.database-target }} + cancel-in-progress: false + +jobs: + migrate: + name: Migrate and verify selected database + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + env: + MIGRATION_DATABASE_TARGET: ${{ inputs.database-target }} + MIGRATION_EXPECTED_SHA: ${{ inputs.expected-sha }} + steps: + - name: Validate exact migration request + shell: python + run: | + import os + import re + + if os.environ['GITHUB_REF'] != 'refs/heads/master': + raise ValueError('Database migrations must execute the reviewed master branch') + if os.environ['MIGRATION_DATABASE_TARGET'] not in {'staging', 'production'}: + raise ValueError('An explicit staging or production database target is required') + expected = os.environ['MIGRATION_EXPECTED_SHA'] + if not re.fullmatch(r'[0-9a-f]{40}', expected) or expected != os.environ['GITHUB_SHA']: + raise ValueError('The workflow commit must equal the exact reviewed expected-sha') + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: package.json + - name: Install dependencies + run: bun install --frozen-lockfile + env: + CYPRESS_INSTALL_BINARY: '0' + + - name: Apply staging migrations and verify receipt schema + if: inputs.database-target == 'staging' + env: + DATABASE_WRITE_URL: ${{ secrets.DATABASE_STAGING_WRITE_URL }} + run: | + bun run admin:db:migrate --yes + bun packages/db/src/verify-measurement-schema.ts migration-report.json + + - name: Apply production migrations and verify receipt schema + if: inputs.database-target == 'production' + env: + DATABASE_WRITE_URL: ${{ secrets.DATABASE_WRITE_URL }} + run: | + bun run admin:db:migrate --yes + bun packages/db/src/verify-measurement-schema.ts migration-report.json + + - name: Retain migration verification + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: measurement-schema-${{ inputs.database-target }}-${{ github.run_id }}-${{ github.run_attempt }} + path: migration-report.json + if-no-files-found: ignore diff --git a/docs/measurement-publication.md b/docs/measurement-publication.md index e9252b9f2..44454344c 100644 --- a/docs/measurement-publication.md +++ b/docs/measurement-publication.md @@ -22,7 +22,24 @@ Receipt ingestion selects benchmark JSON and evaluation samples from the receipt An `evals-only` changelog cannot narrow a receipt that requires throughput points: the reader rejects it before database writes. Before refreshing the published curve or marking a snapshot complete, every accepted benchmark file must return the receipt's exact point count from database inserts, including existing rows on replay. Per-point purge skips therefore leave the snapshot incomplete. -Apply migration `016_measurement_snapshots.sql` before deploying these readers. The table retains the compact receipt and binds each source repository/run/attempt to one accepted snapshot. An interrupted progressive import remains `writing` and resumes only that same receipt; successful replay remains stable. Replacing measurement bytes requires a separately versioned source execution. Execution rollback does not reverse prior database writes. Preserve a compatible reader for all retained receipt versions. +Apply migration `016_measurement_snapshots.sql` before enabling receipt ingestion or declaring the deployed reader ready. The table retains the compact receipt and binds each source repository/run/attempt to one accepted snapshot. An interrupted progressive import remains `writing` and resumes only that same receipt; successful replay remains stable. Replacing measurement bytes requires a separately versioned source execution. Execution rollback does not reverse prior database writes. Preserve a compatible reader for all retained receipt versions. + +## Migration-only readiness + +The [Migrate Database workflow](../.github/workflows/migrate-database.yml) applies pending checked-in migrations through `bun run admin:db:migrate --yes`, using only the existing writer secret for the explicitly selected `staging` or `production` target. It preserves the database and existing measurements; it does not reset a Neon branch, ingest a run or refresh public caches. The exact `expected-sha` must match the workflow commit before the migration step can access a database credential. + +After this workflow lands on `master`, dispatch staging first and inspect its successful verification artifact, then repeat for production with the same reviewed app commit: + +```bash +gh workflow run migrate-database.yml --repo SemiAnalysisAI/InferenceX-app --ref master \ + -f database-target=staging -f expected-sha=REVIEWED_MASTER_SHA +gh workflow run migrate-database.yml --repo SemiAnalysisAI/InferenceX-app --ref master \ + -f database-target=production -f expected-sha=REVIEWED_MASTER_SHA +``` + +The read-only verifier checks that migration `016` is recorded, the receipt table has its required column types and nullability, and its primary key binds source repository/run/attempt. A successful `measurement-schema---` artifact contains `migration-report.json` with the target, exact app SHA, workflow identity and verified schema; it contains no connection string or measurement data. Missing or incompatible schema fails without producing a success report. Keep both successful run links and artifacts as readiness evidence. + +Normal app merges trigger Vercel deployment separately. Keep native receipt ingestion disabled until production deployment and migration verification both succeed for the reviewed revision, then set `INFX_PHASE1_READER_REVISION` to that deployed app SHA and configure the issuer allowlist. A successful preview or migration-only run alone does not establish publication readiness. The new workflow is not dispatchable until it exists on the default branch. After cache refresh, the workflows execute the read-only `packages/db/src/verify-measurement-publication.ts `. It compares exact-run and latest curve metrics/topology with the accepted snapshot, checks eval summary visibility and strict sample counts, and verifies trace-detail availability. It uses `INGEST_ARTIFACTS_PATH` and the receipt environment emitted by transport. Retain its report alongside existing PowerX and database diagnostics. Run-specific verification does not silently imply that every fleet lane is qualified. diff --git a/docs/measurement-publication_zh.md b/docs/measurement-publication_zh.md index 4e7536715..678aa0e73 100644 --- a/docs/measurement-publication_zh.md +++ b/docs/measurement-publication_zh.md @@ -22,7 +22,24 @@ Phase 1 的读取端接受由 InferenceX 独立受信托管签发流程生成的 如果回执要求导入吞吐量测量点，`evals-only` changelog 不能缩小该范围，读取端会在数据库写入前拒绝这一冲突。在刷新已发布曲线或将 snapshot 标记为完成之前，每个已接受的基准测试文件都必须从数据库写入中返回回执规定的测量点数量，重复导入时已有的数据行也计入。因此，按测量点执行的清除规则若跳过必需数据，snapshot 会保持未完成状态。 -部署读取端之前，先应用 `016_measurement_snapshots.sql`。该表保留紧凑回执，并将源仓库/run/attempt 绑定到唯一的已接受 snapshot。渐进式导入中断后，状态保持 `writing`，恢复时只能使用同一回执；成功后的重复导入保持结果稳定。替换测量字节需要独立版本的源执行。执行回滚不会撤销既有数据库写入。对保留的回执版本，必须继续提供兼容读取端。 +启用回执导入或声明已部署读取端就绪之前，先应用 `016_measurement_snapshots.sql`。该表保留紧凑回执，并将源仓库/run/attempt 绑定到唯一的已接受 snapshot。渐进式导入中断后，状态保持 `writing`，恢复时只能使用同一回执；成功后的重复导入保持结果稳定。替换测量字节需要独立版本的源执行。执行回滚不会撤销既有数据库写入。对保留的回执版本，必须继续提供兼容读取端。 + +## 独立迁移与就绪验证 + +[Migrate Database workflow](../.github/workflows/migrate-database.yml) 通过 `bun run admin:db:migrate --yes` 应用已纳入版本控制但尚未执行的迁移，仅使用显式选择的 `staging` 或 `production` 目标对应的现有 writer secret。该流程保留数据库及已有测量，不重置 Neon 分支、不导入运行结果，也不刷新公共缓存。迁移步骤获得数据库凭证之前，必须确认准确的 `expected-sha` 与 workflow 提交一致。 + +此 workflow 合入 `master` 后，先对 staging 执行迁移并检查成功的验证产物，再以同一个已审查应用提交对 production 执行： + +```bash +gh workflow run migrate-database.yml --repo SemiAnalysisAI/InferenceX-app --ref master \ + -f database-target=staging -f expected-sha=REVIEWED_MASTER_SHA +gh workflow run migrate-database.yml --repo SemiAnalysisAI/InferenceX-app --ref master \ + -f database-target=production -f expected-sha=REVIEWED_MASTER_SHA +``` + +只读验证器确认 migration `016` 已记录、回执表具有必需的列类型与非空约束，并确认主键绑定源仓库/run/attempt。成功的 `measurement-schema---` 产物包含 `migration-report.json`，记录目标、准确的 app SHA、workflow 身份及已验证 schema，不包含连接字符串或测量数据。缺失或不兼容的 schema 会使流程失败，且不生成成功报告。保留两个成功运行的链接及产物作为就绪证据。 + +应用的正常合并会独立触发 Vercel 部署。在 production 部署和迁移验证都针对已审查版本成功之前，继续禁用原生回执导入；随后才将 `INFX_PHASE1_READER_REVISION` 设置为实际部署的 app SHA，并配置 issuer allowlist。仅 preview 或独立迁移成功不能证明发布就绪。新的 workflow 必须先存在于默认分支，才能调度。 缓存刷新后，工作流执行只读校验命令 `packages/db/src/verify-measurement-publication.ts `。它将 exact-run 和最新曲线中的指标与拓扑同已接受的 snapshot 比较，检查评估汇总和 strict 样本计数，并验证 trace 明细可用性。该命令使用 `INGEST_ARTIFACTS_PATH` 及传输步骤生成的回执环境变量。应将其报告与现有 PowerX、数据库诊断一起保留。单次运行通过不代表整个集群覆盖范围均已完成验收。 diff --git a/packages/db/src/lib/measurement-schema.test.ts b/packages/db/src/lib/measurement-schema.test.ts new file mode 100644 index 000000000..5a34ee5ab --- /dev/null +++ b/packages/db/src/lib/measurement-schema.test.ts @@ -0,0 +1,102 @@ +import { PGlite } from '@electric-sql/pglite'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, expect, it } from 'vitest'; +import type { Sql } from '../etl/db-utils'; +import { runMigrations } from './migration-runner'; +import { verifyMeasurementSchema } from './measurement-schema'; + +let db: PGlite; +let sql: Sql; +let directory: string; + +function queryClient(database: Pick) { + return Object.assign( + async (strings: TemplateStringsArray, ...values: unknown[]) => { + const query = strings.reduce((text, part, i) => text + (i ? `$${i}` : '') + part, ''); + const result = await database.query(query, values); + return result.rows; + }, + { + unsafe: async (query: string, values: unknown[] = []) => { + const result = await database.query(query, values); + return result.rows; + }, + }, + ); +} + +beforeEach(async () => { + db = await PGlite.create(); + directory = fs.mkdtempSync(path.join(os.tmpdir(), 'measurement-schema-')); + fs.copyFileSync( + new URL('../../migrations/016_measurement_snapshots.sql', import.meta.url), + path.join(directory, '016_measurement_snapshots.sql'), + ); + sql = Object.assign(queryClient(db), { + begin: (fn: (tx: Sql) => Promise) => + db.transaction((tx) => fn(queryClient(tx) as unknown as Sql)), + }) as unknown as Sql; +}); + +afterEach(async () => { + await db.close(); + fs.rmSync(directory, { recursive: true, force: true }); +}); + +it('applies the receipt migration, preserves existing data and verifies replay in read-only mode', async () => { + await db.exec( + 'create table existing_measurements (value integer); insert into existing_measurements values (42)', + ); + expect(await runMigrations(sql, directory)).toBe(1); + await db.query( + `insert into measurement_snapshots + (source_repo, source_run_id, source_attempt, receipt_id, bundle_digest, receipt, state) + values ('org/repo', 100, 1, $1, $2, '{"retained":true}', 'complete')`, + ['a'.repeat(64), 'b'.repeat(64)], + ); + expect(await runMigrations(sql, directory)).toBe(0); + const report = await db.transaction(async (tx) => { + await tx.query('set transaction read only'); + return verifyMeasurementSchema(queryClient(tx) as unknown as Sql); + }); + expect(report).toMatchObject({ + migration: '016_measurement_snapshots.sql', + table: 'public.measurement_snapshots', + columns: { source_run_id: 'bigint', receipt: 'jsonb', state: 'text' }, + primary_key: ['source_repo', 'source_run_id', 'source_attempt'], + }); + const existing = await db.query('select value from existing_measurements'); + expect(existing.rows).toEqual([{ value: 42 }]); + const snapshots = await db.query( + 'select source_run_id, receipt, state from measurement_snapshots', + ); + expect(snapshots.rows).toEqual([ + { source_run_id: 100, receipt: { retained: true }, state: 'complete' }, + ]); +}); + +it('rejects a database with no receipt schema', async () => { + await expect(verifyMeasurementSchema(sql)).rejects.toThrow('table is missing'); +}); + +it.each([ + ['delete from schema_migrations', 'migration is not recorded'], + [ + 'alter table measurement_snapshots alter column source_run_id type text', + 'non-null bigint column', + ], + [ + 'alter table measurement_snapshots alter column bundle_digest drop not null', + 'non-null text column', + ], + [ + 'alter table measurement_snapshots drop constraint measurement_snapshots_pkey', + 'uniquely bind source repository, run and attempt', + ], +])('rejects incompatible deployed schema after %s', async (change, error) => { + await runMigrations(sql, directory); + await db.exec(change); + await expect(verifyMeasurementSchema(sql)).rejects.toThrow(error); +}); diff --git a/packages/db/src/lib/measurement-schema.ts b/packages/db/src/lib/measurement-schema.ts new file mode 100644 index 000000000..0981b13e8 --- /dev/null +++ b/packages/db/src/lib/measurement-schema.ts @@ -0,0 +1,64 @@ +import type { Sql } from '../etl/db-utils'; + +/** Read-only readiness evidence for the receipt migration on the selected database. */ +export async function verifyMeasurementSchema(sql: Pick) { + const tables = await sql.unsafe(` + select to_regclass('public.schema_migrations')::text as ledger, + to_regclass('public.measurement_snapshots')::text as snapshots + `); + if (!tables[0]?.ledger || !tables[0]?.snapshots) { + throw new Error('Receipt migration ledger or measurement_snapshots table is missing'); + } + + const migration = '016_measurement_snapshots.sql'; + const applied = await sql.unsafe( + 'select filename from public.schema_migrations where filename = $1', + [migration], + ); + if (applied.length !== 1) throw new Error(`Receipt migration is not recorded: ${migration}`); + + const columns = await sql.unsafe(` + select attname as name, format_type(atttypid, atttypmod) as type, attnotnull as required + from pg_attribute + where attrelid = 'public.measurement_snapshots'::regclass and attnum > 0 and not attisdropped + order by attnum + `); + const expected: Record = { + source_repo: 'text', + source_run_id: 'bigint', + source_attempt: 'integer', + receipt_id: 'text', + bundle_digest: 'text', + receipt: 'jsonb', + state: 'text', + created_at: 'timestamp with time zone', + updated_at: 'timestamp with time zone', + }; + for (const [name, type] of Object.entries(expected)) { + if ( + !columns.some((column) => column.name === name && column.type === type && column.required) + ) { + throw new Error(`Receipt schema requires a non-null ${type} column: ${name}`); + } + } + + const keys = await sql.unsafe(` + select array_agg(attribute.attname order by key.ordinality) as columns + from pg_constraint as definition + cross join lateral unnest(definition.conkey) with ordinality as key(attnum, ordinality) + join pg_attribute as attribute + on attribute.attrelid = definition.conrelid and attribute.attnum = key.attnum + where definition.conrelid = 'public.measurement_snapshots'::regclass and definition.contype = 'p' + group by definition.oid + `); + const primaryKey = ['source_repo', 'source_run_id', 'source_attempt']; + if (keys.length !== 1 || JSON.stringify(keys[0].columns) !== JSON.stringify(primaryKey)) { + throw new Error('Receipt schema must uniquely bind source repository, run and attempt'); + } + return { + migration, + table: 'public.measurement_snapshots', + columns: expected, + primary_key: primaryKey, + }; +} diff --git a/packages/db/src/verify-measurement-schema.ts b/packages/db/src/verify-measurement-schema.ts new file mode 100644 index 000000000..9fef5704f --- /dev/null +++ b/packages/db/src/verify-measurement-schema.ts @@ -0,0 +1,36 @@ +import fs from 'node:fs'; +import { createAdminSql } from './etl/db-utils'; +import { verifyMeasurementSchema } from './lib/measurement-schema'; + +const target = process.env.MIGRATION_DATABASE_TARGET; +const sourceSha = process.env.GITHUB_SHA; +const output = process.argv[2]; +if ( + !output || + !['staging', 'production'].includes(target ?? '') || + !sourceSha?.match(/^[a-f0-9]{40}$/) || + sourceSha !== process.env.MIGRATION_EXPECTED_SHA +) { + throw new Error( + 'Schema verification requires an output path, explicit target and exact workflow commit', + ); +} + +const sql = createAdminSql({ max: 1 }); +try { + const schema = await sql.begin('read only', (tx) => verifyMeasurementSchema(tx)); + const report = { + schema_version: 1, + status: 'verified', + database_target: target, + app_sha: sourceSha, + workflow_run_id: process.env.GITHUB_RUN_ID, + workflow_run_attempt: process.env.GITHUB_RUN_ATTEMPT, + verified_at: new Date().toISOString(), + schema, + }; + fs.writeFileSync(output, `${JSON.stringify(report, null, 2)}\n`, { flag: 'wx' }); + console.log(`Verified receipt schema on ${target}; evidence: ${output}`); +} finally { + await sql.end(); +}