fix(db): ingest server logs from srt-slurm single-node archives

docs/data-pipeline.md

@@ -335,7 +335,12 @@ Server-log artifacts are stored as filename-keyed bundles rather than as a hard-
 router, worker, or benchmark roles. Ingest recursively retains every regular `.log` and `.out`
 file and preserves its artifact-relative path. This applies to both `server_logs_*` ZIPs and
 `multinode_server_logs_*` artifacts whose files are nested inside
-`multinode_server_logs.tar.gz`.
+`multinode_server_logs.tar.gz`. Native srt-slurm single-node runs (fixed-sequence from
+2026-09-24, AgentX from 2026-09-26) keep the client logs at the `server_logs_*` artifact root and
+put the job's `logs/` directory, including the server worker log `<host>_agg_w0.out`, in
+`srt-single-node-logs.tar.gz`. Ingest extracts that archive in place, so its files are stored
+under `srt-single-node-logs/` next to the root files; artifacts without the archive read as
+before.
 
 The primary file remains in `server_logs.server_log` for compatibility and KV-cache metadata
 extraction; `server_log_files` holds the remaining files. `server_logs.file_name` records the
@@ -346,7 +351,9 @@ for complete DB history, `--source gcs` to exercise only the backup path, and `-
 inventory pair counts and compressed download size without writing. `--run <id>`, `--limit <n>`,
 and `--yes` remain available for targeted, idempotent recovery. Bundles already marked
 `files_complete` are skipped after their small benchmark artifact is mapped, avoiding repeated
-large log downloads.
+large log downloads. `--force` revisits those bundles and inserts only filenames they do not
+already hold, which recovers files a newer artifact reader exposes, such as the single-node
+srt-slurm archive.
 
 Full-bundle search stays inside PostgreSQL: each file is scanned through overlapping 16 MiB
 character slices, and the API returns at most 50 match contexts rather than transferring the

packages/db/src/backfill-server-log-files.ts

@@ -12,6 +12,10 @@
  *   bun run --cwd packages/db db:backfill-server-log-files --all --from-run 26606969606 --yes
  *   bun run --cwd packages/db db:backfill-server-log-files --all --source gcs --dry-run
  *   bun run --cwd packages/db db:backfill-server-log-files --all --source auto --yes
+ *   bun run --cwd packages/db db:backfill-server-log-files --run 36470443155 --force --yes
+ *
+ * --force revisits bundles already marked complete; only missing filenames are
+ * inserted, so it picks up files a newer artifact reader can see.
  */
 
 import fs from 'node:fs';
@@ -204,7 +208,7 @@ async function main(): Promise<void> {
     throw new Error('--run and --from-run cannot be combined');
   }
   const flags = parseBackfillFlags();
-  const { limit } = parseLimitForceFlags();
+  const { limit, force } = parseLimitForceFlags();
   const runs = await sql<CandidateRun[]>`
     select distinct wr.github_run_id, wr.run_attempt, wr.html_url, wr.date::text as date
     from workflow_runs wr
@@ -318,7 +322,7 @@ async function main(): Promise<void> {
             console.warn(`  [WARN] ${pair.serverLogs.name}: no matching benchmark rows`);
             continue;
           }
-          if (await resultLogsAreComplete(resultIds)) {
+          if (!force && (await resultLogsAreComplete(resultIds))) {
             completeSkipped++;
             continue;
           }

packages/db/src/etl/server-log-artifacts.test.ts

@@ -1,3 +1,4 @@
+import { execFileSync } from 'node:child_process';
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
@@ -8,7 +9,9 @@ import {
   cleanLogText,
   listServerLogFilePaths,
   primaryServerLogFile,
+  readServerLogArtifact,
   readServerLogFiles,
+  serverLogArtifactRoot,
   serverLogArtifactSuffix,
 } from './server-log-artifacts.js';
 
@@ -24,6 +27,16 @@ function tempRoot(): string {
   return root;
 }
 
+/** Write `files` under a scratch `logs/` tree and gzip it to `archivePath`. */
+function writeLogsArchive(archivePath: string, files: Record<string, string>): void {
+  const staging = tempRoot();
+  for (const [name, text] of Object.entries(files)) {
+    fs.mkdirSync(path.dirname(path.join(staging, name)), { recursive: true });
+    fs.writeFileSync(path.join(staging, name), text);
+  }
+  execFileSync('tar', ['-czf', archivePath, '-C', staging, '.']);
+}
+
 describe('server-log artifact discovery', () => {
   it('recursively keeps .log and .out files with relative paths', () => {
     const root = tempRoot();
@@ -56,6 +69,53 @@ describe('server-log artifact discovery', () => {
     expect(serverLogArtifactSuffix('agentic_config-c')).toBeNull();
   });
 
+  it('lists native srt-slurm single-node logs alongside the client logs', () => {
+    const artifactDir = tempRoot();
+    fs.mkdirSync(path.join(artifactDir, 'results'));
+    fs.writeFileSync(path.join(artifactDir, 'results', 'benchmark.log'), 'client\n');
+    fs.writeFileSync(path.join(artifactDir, 'srt-setup.log'), 'setup\n');
+    writeLogsArchive(path.join(artifactDir, 'srt-single-node-logs.tar.gz'), {
+      'logs/host-1_agg_w0.out': 'max_total_num_tokens=2461120\n',
+      'logs/agentic/aiperf_artifacts/server_metrics_export.json': '{}',
+    });
+
+    const artifact = { artifactName: 'server_logs_config-a', artifactDir };
+    const files = readServerLogArtifact(artifact);
+    expect(files.map((file) => file.fileName)).toEqual([
+      'results/benchmark.log',
+      'srt-setup.log',
+      'srt-single-node-logs/logs/host-1_agg_w0.out',
+    ]);
+    // The primary stream is unchanged from the pre-archive layout.
+    expect(primaryServerLogFile(files)?.fileName).toBe('results/benchmark.log');
+    // A second read reuses the extraction instead of failing on an existing directory.
+    expect(readServerLogArtifact(artifact)).toEqual(files);
+  });
+
+  it('reads legacy single-node artifacts without an archive unchanged', () => {
+    const artifactDir = tempRoot();
+    fs.mkdirSync(path.join(artifactDir, 'results'));
+    fs.writeFileSync(path.join(artifactDir, 'results', 'server.log'), 'server\n');
+
+    expect(serverLogArtifactRoot(artifactDir, 'server_logs_config-a')).toBe(artifactDir);
+    expect(fs.existsSync(path.join(artifactDir, 'srt-single-node-logs'))).toBe(false);
+    expect(readServerLogArtifact({ artifactName: 'server_logs_config-a', artifactDir })).toEqual([
+      { fileName: 'results/server.log', logText: 'server\n' },
+    ]);
+  });
+
+  it('roots multinode artifacts at their extracted archive', () => {
+    const artifactDir = tempRoot();
+    writeLogsArchive(path.join(artifactDir, 'multinode_server_logs.tar.gz'), {
+      'host-1_decode_w0.out': 'decode\n',
+    });
+
+    expect(
+      readServerLogArtifact({ artifactName: 'multinode_server_logs_config-b', artifactDir }),
+    ).toEqual([{ fileName: 'host-1_decode_w0.out', logText: 'decode\n' }]);
+    expect(serverLogArtifactRoot(tempRoot(), 'multinode_server_logs_config-c')).toBeNull();
+  });
+
   it('removes PostgreSQL-incompatible null bytes', () => {
     expect(cleanLogText('a\u0000b')).toBe('ab');
   });

packages/db/src/etl/server-log-artifacts.ts

@@ -81,21 +81,34 @@ export function primaryServerLogFile(files: readonly ServerLogFile[]): ServerLog
   );
 }
 
+/** Extract `archivePath` into `extractedDir` once; returns whether `extractedDir` exists. */
+function extractArchiveOnce(archivePath: string, extractedDir: string): boolean {
+  if (!fs.existsSync(extractedDir) && fs.existsSync(archivePath)) {
+    fs.mkdirSync(extractedDir, { recursive: true });
+    execFileSync('tar', ['-xzf', archivePath, '-C', extractedDir], { stdio: 'ignore' });
+  }
+  return fs.existsSync(extractedDir) && fs.statSync(extractedDir).isDirectory();
+}
+
 /**
  * Resolve the root containing logs for an extracted GitHub artifact.
- * Multinode uploads wrap their files in multinode_server_logs.tar.gz.
+ * Multinode uploads wrap their files in multinode_server_logs.tar.gz. Native
+ * srt-slurm single-node uploads keep the client logs at the artifact root and
+ * wrap the job's logs directory, including the server worker logs, in
+ * srt-single-node-logs.tar.gz; it is extracted in place so both are listed.
  */
 export function serverLogArtifactRoot(artifactDir: string, artifactName: string): string | null {
-  if (artifactName.startsWith('server_logs_')) return artifactDir;
+  if (artifactName.startsWith('server_logs_')) {
+    extractArchiveOnce(
+      path.join(artifactDir, 'srt-single-node-logs.tar.gz'),
+      path.join(artifactDir, 'srt-single-node-logs'),
+    );
+    return artifactDir;
+  }
   if (!artifactName.startsWith('multinode_server_logs_')) return null;
 
-  const archivePath = path.join(artifactDir, 'multinode_server_logs.tar.gz');
   const extractedDir = path.join(artifactDir, 'multinode_server_logs');
-  if (!fs.existsSync(extractedDir) && fs.existsSync(archivePath)) {
-    fs.mkdirSync(extractedDir, { recursive: true });
-    execFileSync('tar', ['-xzf', archivePath, '-C', extractedDir], { stdio: 'ignore' });
-  }
-  return fs.existsSync(extractedDir) && fs.statSync(extractedDir).isDirectory()
+  return extractArchiveOnce(path.join(artifactDir, 'multinode_server_logs.tar.gz'), extractedDir)
     ? extractedDir
     : null;
 }