From 275de85d18b6ab894f10932170b1e83e83963e8d Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Fri, 9 Oct 2026 14:24:52 -0500 Subject: [PATCH] fix(db): ingest server logs from srt-slurm single-node archives Native srt-slurm single-node runs upload the job's logs directory, including the server worker log _agg_w0.out, inside srt-single-node-logs.tar.gz in the server_logs_* artifact. Ingest only unpacked the multinode archive, so those runs stored just benchmark.log and srt-setup.log. Extract the single-node archive in place so its .log/.out files are stored alongside the root files. Artifacts without it, and multinode artifacts, read as before; the primary log choice is unchanged. Wire the existing --force flag in the server-log backfill so bundles already marked complete can pick up the newly visible files. --- docs/data-pipeline.md | 11 +++- packages/db/src/backfill-server-log-files.ts | 8 ++- .../db/src/etl/server-log-artifacts.test.ts | 60 +++++++++++++++++++ packages/db/src/etl/server-log-artifacts.ts | 29 ++++++--- 4 files changed, 96 insertions(+), 12 deletions(-) diff --git a/docs/data-pipeline.md b/docs/data-pipeline.md index 07df2b1c3..2eec137bb 100644 --- a/docs/data-pipeline.md +++ b/docs/data-pipeline.md @@ -335,7 +335,12 @@ Server-log artifacts are stored as filename-keyed bundles rather than as a hard- router, worker, or benchmark roles. Ingest recursively retains every regular `.log` and `.out` file and preserves its artifact-relative path. This applies to both `server_logs_*` ZIPs and `multinode_server_logs_*` artifacts whose files are nested inside -`multinode_server_logs.tar.gz`. +`multinode_server_logs.tar.gz`. Native srt-slurm single-node runs (fixed-sequence from +2026-09-24, AgentX from 2026-09-26) keep the client logs at the `server_logs_*` artifact root and +put the job's `logs/` directory, including the server worker log `_agg_w0.out`, in +`srt-single-node-logs.tar.gz`. Ingest extracts that archive in place, so its files are stored +under `srt-single-node-logs/` next to the root files; artifacts without the archive read as +before. The primary file remains in `server_logs.server_log` for compatibility and KV-cache metadata extraction; `server_log_files` holds the remaining files. `server_logs.file_name` records the @@ -346,7 +351,9 @@ for complete DB history, `--source gcs` to exercise only the backup path, and `- inventory pair counts and compressed download size without writing. `--run `, `--limit `, and `--yes` remain available for targeted, idempotent recovery. Bundles already marked `files_complete` are skipped after their small benchmark artifact is mapped, avoiding repeated -large log downloads. +large log downloads. `--force` revisits those bundles and inserts only filenames they do not +already hold, which recovers files a newer artifact reader exposes, such as the single-node +srt-slurm archive. Full-bundle search stays inside PostgreSQL: each file is scanned through overlapping 16 MiB character slices, and the API returns at most 50 match contexts rather than transferring the diff --git a/packages/db/src/backfill-server-log-files.ts b/packages/db/src/backfill-server-log-files.ts index 8047e0db5..c907dce24 100644 --- a/packages/db/src/backfill-server-log-files.ts +++ b/packages/db/src/backfill-server-log-files.ts @@ -12,6 +12,10 @@ * bun run --cwd packages/db db:backfill-server-log-files --all --from-run 26606969606 --yes * bun run --cwd packages/db db:backfill-server-log-files --all --source gcs --dry-run * bun run --cwd packages/db db:backfill-server-log-files --all --source auto --yes + * bun run --cwd packages/db db:backfill-server-log-files --run 36470443155 --force --yes + * + * --force revisits bundles already marked complete; only missing filenames are + * inserted, so it picks up files a newer artifact reader can see. */ import fs from 'node:fs'; @@ -204,7 +208,7 @@ async function main(): Promise { throw new Error('--run and --from-run cannot be combined'); } const flags = parseBackfillFlags(); - const { limit } = parseLimitForceFlags(); + const { limit, force } = parseLimitForceFlags(); const runs = await sql` select distinct wr.github_run_id, wr.run_attempt, wr.html_url, wr.date::text as date from workflow_runs wr @@ -318,7 +322,7 @@ async function main(): Promise { console.warn(` [WARN] ${pair.serverLogs.name}: no matching benchmark rows`); continue; } - if (await resultLogsAreComplete(resultIds)) { + if (!force && (await resultLogsAreComplete(resultIds))) { completeSkipped++; continue; } diff --git a/packages/db/src/etl/server-log-artifacts.test.ts b/packages/db/src/etl/server-log-artifacts.test.ts index 78f23588b..68eee9f35 100644 --- a/packages/db/src/etl/server-log-artifacts.test.ts +++ b/packages/db/src/etl/server-log-artifacts.test.ts @@ -1,3 +1,4 @@ +import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -8,7 +9,9 @@ import { cleanLogText, listServerLogFilePaths, primaryServerLogFile, + readServerLogArtifact, readServerLogFiles, + serverLogArtifactRoot, serverLogArtifactSuffix, } from './server-log-artifacts.js'; @@ -24,6 +27,16 @@ function tempRoot(): string { return root; } +/** Write `files` under a scratch `logs/` tree and gzip it to `archivePath`. */ +function writeLogsArchive(archivePath: string, files: Record): void { + const staging = tempRoot(); + for (const [name, text] of Object.entries(files)) { + fs.mkdirSync(path.dirname(path.join(staging, name)), { recursive: true }); + fs.writeFileSync(path.join(staging, name), text); + } + execFileSync('tar', ['-czf', archivePath, '-C', staging, '.']); +} + describe('server-log artifact discovery', () => { it('recursively keeps .log and .out files with relative paths', () => { const root = tempRoot(); @@ -56,6 +69,53 @@ describe('server-log artifact discovery', () => { expect(serverLogArtifactSuffix('agentic_config-c')).toBeNull(); }); + it('lists native srt-slurm single-node logs alongside the client logs', () => { + const artifactDir = tempRoot(); + fs.mkdirSync(path.join(artifactDir, 'results')); + fs.writeFileSync(path.join(artifactDir, 'results', 'benchmark.log'), 'client\n'); + fs.writeFileSync(path.join(artifactDir, 'srt-setup.log'), 'setup\n'); + writeLogsArchive(path.join(artifactDir, 'srt-single-node-logs.tar.gz'), { + 'logs/host-1_agg_w0.out': 'max_total_num_tokens=2461120\n', + 'logs/agentic/aiperf_artifacts/server_metrics_export.json': '{}', + }); + + const artifact = { artifactName: 'server_logs_config-a', artifactDir }; + const files = readServerLogArtifact(artifact); + expect(files.map((file) => file.fileName)).toEqual([ + 'results/benchmark.log', + 'srt-setup.log', + 'srt-single-node-logs/logs/host-1_agg_w0.out', + ]); + // The primary stream is unchanged from the pre-archive layout. + expect(primaryServerLogFile(files)?.fileName).toBe('results/benchmark.log'); + // A second read reuses the extraction instead of failing on an existing directory. + expect(readServerLogArtifact(artifact)).toEqual(files); + }); + + it('reads legacy single-node artifacts without an archive unchanged', () => { + const artifactDir = tempRoot(); + fs.mkdirSync(path.join(artifactDir, 'results')); + fs.writeFileSync(path.join(artifactDir, 'results', 'server.log'), 'server\n'); + + expect(serverLogArtifactRoot(artifactDir, 'server_logs_config-a')).toBe(artifactDir); + expect(fs.existsSync(path.join(artifactDir, 'srt-single-node-logs'))).toBe(false); + expect(readServerLogArtifact({ artifactName: 'server_logs_config-a', artifactDir })).toEqual([ + { fileName: 'results/server.log', logText: 'server\n' }, + ]); + }); + + it('roots multinode artifacts at their extracted archive', () => { + const artifactDir = tempRoot(); + writeLogsArchive(path.join(artifactDir, 'multinode_server_logs.tar.gz'), { + 'host-1_decode_w0.out': 'decode\n', + }); + + expect( + readServerLogArtifact({ artifactName: 'multinode_server_logs_config-b', artifactDir }), + ).toEqual([{ fileName: 'host-1_decode_w0.out', logText: 'decode\n' }]); + expect(serverLogArtifactRoot(tempRoot(), 'multinode_server_logs_config-c')).toBeNull(); + }); + it('removes PostgreSQL-incompatible null bytes', () => { expect(cleanLogText('a\u0000b')).toBe('ab'); }); diff --git a/packages/db/src/etl/server-log-artifacts.ts b/packages/db/src/etl/server-log-artifacts.ts index fc9e5f5de..63561ad61 100644 --- a/packages/db/src/etl/server-log-artifacts.ts +++ b/packages/db/src/etl/server-log-artifacts.ts @@ -81,21 +81,34 @@ export function primaryServerLogFile(files: readonly ServerLogFile[]): ServerLog ); } +/** Extract `archivePath` into `extractedDir` once; returns whether `extractedDir` exists. */ +function extractArchiveOnce(archivePath: string, extractedDir: string): boolean { + if (!fs.existsSync(extractedDir) && fs.existsSync(archivePath)) { + fs.mkdirSync(extractedDir, { recursive: true }); + execFileSync('tar', ['-xzf', archivePath, '-C', extractedDir], { stdio: 'ignore' }); + } + return fs.existsSync(extractedDir) && fs.statSync(extractedDir).isDirectory(); +} + /** * Resolve the root containing logs for an extracted GitHub artifact. - * Multinode uploads wrap their files in multinode_server_logs.tar.gz. + * Multinode uploads wrap their files in multinode_server_logs.tar.gz. Native + * srt-slurm single-node uploads keep the client logs at the artifact root and + * wrap the job's logs directory, including the server worker logs, in + * srt-single-node-logs.tar.gz; it is extracted in place so both are listed. */ export function serverLogArtifactRoot(artifactDir: string, artifactName: string): string | null { - if (artifactName.startsWith('server_logs_')) return artifactDir; + if (artifactName.startsWith('server_logs_')) { + extractArchiveOnce( + path.join(artifactDir, 'srt-single-node-logs.tar.gz'), + path.join(artifactDir, 'srt-single-node-logs'), + ); + return artifactDir; + } if (!artifactName.startsWith('multinode_server_logs_')) return null; - const archivePath = path.join(artifactDir, 'multinode_server_logs.tar.gz'); const extractedDir = path.join(artifactDir, 'multinode_server_logs'); - if (!fs.existsSync(extractedDir) && fs.existsSync(archivePath)) { - fs.mkdirSync(extractedDir, { recursive: true }); - execFileSync('tar', ['-xzf', archivePath, '-C', extractedDir], { stdio: 'ignore' }); - } - return fs.existsSync(extractedDir) && fs.statSync(extractedDir).isDirectory() + return extractArchiveOnce(path.join(artifactDir, 'multinode_server_logs.tar.gz'), extractedDir) ? extractedDir : null; }