From 2920de49fa1c07a92b6ac8d8b1ff9bdb31019e9c Mon Sep 17 00:00:00 2001 From: Peter Olom Date: Thu, 15 Oct 2026 07:58:35 +0000 Subject: [PATCH 1/2] perf: build the trace span covering index in gated maintenance Why: - Deduplication scans large trace groups using a two-column index and fetches table rows for every span comparison, saturating the storage worker. What: - Add an explicitly requested index-only operation to coordinated maintenance. - Build a nonunique covering index on drop, trace and span behind the paused writer and request gate, with database/temp disk and queue headroom checks. - Preserve deduplication SQL, NULL handling, existing duplicates, receipts, source timestamps, FULL ingestion durability and retention scheduling. - Document queue backpressure, safe cancellation, recovery and rollback. Notes: - Startup and scheduled retention do not create the index implicitly. - Verified HTTP ingestion, durable queue replay, deduplication, full queue, process interruption and disk-headroom failure with disposable fixtures. - No persistent tests were added or changed. No deployment is included. --- docs/telemetry-maintenance.md | 6 ++++++ src/server/retention.ts | 14 ++++++++------ src/server/routes/api.ts | 3 ++- src/server/storageWorker.ts | 31 ++++++++++++++++++++++++++++++- 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/docs/telemetry-maintenance.md b/docs/telemetry-maintenance.md index 9c23b37..4a62dd2 100644 --- a/docs/telemetry-maintenance.md +++ b/docs/telemetry-maintenance.md @@ -43,6 +43,12 @@ During maintenance: `POST /api/maintenance/compact` remains admin-only and now returns HTTP 202 with status. It starts the coordinated prune/threshold-based compaction cycle. Repeated triggers do not create overlapping cycles. Disabling the scheduler still permits a manual cycle. +An explicitly requested index-only operation creates the nonunique covering index `idx_traces_drop_trace_span` on `(drop_id, trace_id, span_id)` when missing. It accelerates ingestion deduplication without changing the existing `span_id IS` predicate, historical duplicates, NULL handling, source timestamps, or FULL ingestion durability. Index creation runs only after the writer has paused and the request gate has closed; it never runs in database startup or ordinary scheduled retention. The worker reserves four times the logical database size plus minimum free disk and the queue byte allowance on both the database and SQLite temporary filesystems, conservatively covering index, WAL, sort files, and continued queue admissions. + +For an index-only maintenance window, send the JSON body `{"index_only":true}` to the same admin endpoint. This skips pruning and vacuuming and preserves the scheduled retention deadline and last successful retention time. Observe `phase: indexing` through `/health`, then inspect `last_operation.trace_span_index` through `/api/admin/retention` after the gate reopens. Repeating the operation reuses the index. Query and uncached authentication requests receive retryable 503s during the window; existing-drop ingestion remains bounded by the durable queue and returns retryable 503s when full. Keep upstream persistent exporter queues and retries enabled, particularly when starting with a full Raphael queue. + +Before a large build, verify database and temporary disk capacity, preserve a consistent database/queue backup, and measure incoming bytes and upstream queue capacity against the planned window. Native index creation has no application deadline or in-process cancellation: monitor health and disk externally. If cancellation is necessary, stop the container and restart it, using the previous pinned image if code rollback is also needed. SQLite recovers or rolls back interrupted index DDL on reopen; queued files and commit receipts remain the replay authority. An interrupted index-only operation is not automatically repeated by scheduled retention. Do not unlink the queue, owner file, WAL, or SHM, and do not drop an already completed additive index as part of code rollback. Recheck integrity and queue replay before explicitly retrying the operation. + The default two-day interval is the nominal extra retention allowance between successful cycles. Pruning runtime, preferred-time alignment, failures, disk shortages, or downtime can extend it; it is not a hard two-day age limit. An incomplete prune, failed cycle, or compaction skipped for insufficient disk retries within at most one minute. Compaction below the threshold, disabled by policy, or unavailable for a legacy incremental layout waits for the normal next cycle. The drain resumes during ongoing arrivals rather than waiting for ingestion to stop. ## Configuration diff --git a/src/server/retention.ts b/src/server/retention.ts index e10f4cb..d01e38e 100644 --- a/src/server/retention.ts +++ b/src/server/retention.ts @@ -63,8 +63,9 @@ function schedule(delayMs: number, align = true) { }; arm(); } -export async function runRetentionOnce(reason = 'manual') { +export async function runRetentionOnce(reason = 'manual', options: { index_only?: boolean } = {}) { if (running) { skippedOverlappingRuns++; return getRetentionSchedulerStatus(); } + const scheduledAt = nextRunAt ?? (lastSuccessAt === null ? Date.now() + maintenanceConfig.initial_delay_ms : lastSuccessAt + maintenanceConfig.interval_ms); running = true; phase = 'preparing'; lastStartedAt = Date.now(); nextRunAt = null; if (timer) { clearTimeout(timer); timer = null; } let gateClosed = false; @@ -75,11 +76,11 @@ export async function runRetentionOnce(reason = 'manual') { gateClosed = true; await closeDatabaseGate(); captureMaintenanceSnapshot(); - lastOperation = await storageCommand('maintenance', { reason }); + lastOperation = await storageCommand('maintenance', { reason, ...options }); lastResults = lastOperation.results; lastError = lastOperation.vacuum_skipped === 'insufficient_free_disk' ? 'Compaction skipped: insufficient free disk; pruning completed' : null; completedRuns++; - if (lastOperation.complete && !lastError) lastSuccessAt = Date.now(); + if (!options.index_only && lastOperation.complete && !lastError) lastSuccessAt = Date.now(); } catch (error) { errorRuns++; if ((error as any).partial_result) { lastOperation = (error as any).partial_result; lastResults = lastOperation.results; } @@ -91,13 +92,14 @@ export async function runRetentionOnce(reason = 'manual') { await storageCommand('resume'); running = false; phase = lastError ? 'failed' : 'idle'; lastFinishedAt = Date.now(); lastDurationMs = lastStartedAt === null ? null : lastFinishedAt - lastStartedAt; - schedule(lastError || lastOperation?.complete === false ? Math.min(60_000, maintenanceConfig.interval_ms) : maintenanceConfig.interval_ms, !lastError && lastOperation?.complete !== false); + const delay = options.index_only ? Math.max(0, scheduledAt - Date.now()) : maintenanceConfig.interval_ms; + schedule(lastError || lastOperation?.complete === false ? Math.min(60_000, maintenanceConfig.interval_ms) : delay, !options.index_only && !lastError && lastOperation?.complete !== false); try { await persist(); } catch (error) { lastError = `Could not persist maintenance status: ${(error as Error).message}`; phase = 'failed'; } } return getRetentionSchedulerStatus(); } -export function requestMaintenance(reason = 'manual') { - if (!running) setImmediate(() => { void runRetentionOnce(reason); }); +export function requestMaintenance(reason = 'manual', options: { index_only?: boolean } = {}) { + if (!running) setImmediate(() => { void runRetentionOnce(reason, options); }); return getRetentionSchedulerStatus(); } export function startRetentionScheduler() { diff --git a/src/server/routes/api.ts b/src/server/routes/api.ts index 2c3112b..48ccd56 100644 --- a/src/server/routes/api.ts +++ b/src/server/routes/api.ts @@ -420,7 +420,8 @@ apiRouter.post('/dashboards/generate', async (req: Request, res: Response) => { apiRouter.post('/maintenance/compact', (_req: Request, res: Response) => { if (!requireAdmin(_req, res)) return; try { - res.status(202).json({ accepted: true, maintenance: requestMaintenance('manual compact') }); + const indexOnly = _req.body?.index_only === true; + res.status(202).json({ accepted: true, maintenance: requestMaintenance(indexOnly ? 'manual trace span index' : 'manual compact', { index_only: indexOnly }) }); } catch (error) { res.status(500).json({ error: (error as Error).message || 'Failed to compact database' }); } diff --git a/src/server/storageWorker.ts b/src/server/storageWorker.ts index b4f09dd..5eb880a 100644 --- a/src/server/storageWorker.ts +++ b/src/server/storageWorker.ts @@ -150,11 +150,40 @@ async function drain() { } function pragmaNumber(name: string) { return Number(db.pragma(name, { simple: true })); } let maintenanceProgress: any = null; -async function maintain(options: { reason?: string; force?: boolean }) { +async function maintain(options: { reason?: string; force?: boolean; index_only?: boolean }) { const started = Date.now(); const cutoffTime = started; const results: RetentionPruneResult[] = []; maintenanceProgress = { success: false, complete: false, reason: options.reason || 'manual', results, phase: 'pruning' }; + if (options.index_only) { + // Explicitly requested, behind the paused-writer/request gate. Never at + // startup or as an unexpected addition to scheduled retention. + // Nonunique preserves historical duplicates and span_id IS NULL semantics. + const indexStarted = Date.now(); + const indexExists = db.prepare("SELECT 1 FROM sqlite_schema WHERE type = 'index' AND name = 'idx_traces_drop_trace_span'").get(); + if (!indexExists) { + maintenanceProgress.phase = 'indexing'; + port.postMessage({ type: 'phase', phase: 'indexing' }); + const size = pragmaNumber('page_count') * pragmaNumber('page_size'); + const disk = await fs.statfs(path.dirname(dbPath)); + const temporaryDisk = await fs.statfs(process.env.SQLITE_TMPDIR!); + // Conservative allowance for the new index, WAL, sort files and queue growth. + const reserve = config.min_free_disk_bytes + config.queue_max_bytes; + if (Number(disk.bavail) * Number(disk.bsize) < size * 4 + reserve || + Number(temporaryDisk.bavail) * Number(temporaryDisk.bsize) < size * 4 + reserve) { + throw new Error('Insufficient free disk for trace span index; database and durable queue retained'); + } + db.exec('CREATE INDEX IF NOT EXISTS idx_traces_drop_trace_span ON traces(drop_id, trace_id, span_id)'); + } + const traceSpanIndex = { created: !indexExists, duration_ms: Date.now() - indexStarted }; + maintenanceProgress.trace_span_index = traceSpanIndex; + maintenanceProgress.phase = 'checkpointing'; + port.postMessage({ type: 'phase', phase: 'checkpointing' }); + const checkpoint = db.pragma('wal_checkpoint(TRUNCATE)') as any[]; + if (checkpoint.some((row) => row.busy)) throw new Error('SQLite checkpoint is busy; external connection must release its transaction'); + return { success: true, complete: true, reason: options.reason || 'manual', results, + index_only: true, trace_span_index: traceSpanIndex, duration_ms: Date.now() - started }; + } port.postMessage({ type: 'phase', phase: 'pruning' }); let complete = true; const deadline = started + config.prune_max_runtime_ms; From 225437c45ff43d320470402bc90ad4353fdc7cb7 Mon Sep 17 00:00:00 2001 From: Peter Olom Date: Thu, 15 Oct 2026 08:06:37 +0000 Subject: [PATCH 2/2] fix: preserve retention deadline after index-only failure Why: - A failed index-only operation must not unexpectedly schedule ordinary pruning or vacuuming within the normal maintenance retry interval. What: - Restore the preexisting retention deadline after index-only success or failure, while keeping ordinary maintenance retry behavior unchanged. Notes: - Verified disk-headroom failure through the HTTP maintenance flow: zero deadline drift and a resumed writer. Ordinary failure still retries in 60s. - No persistent tests were added or changed. Not published or deployed. --- src/server/retention.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/retention.ts b/src/server/retention.ts index d01e38e..0f2072a 100644 --- a/src/server/retention.ts +++ b/src/server/retention.ts @@ -92,8 +92,8 @@ export async function runRetentionOnce(reason = 'manual', options: { index_only? await storageCommand('resume'); running = false; phase = lastError ? 'failed' : 'idle'; lastFinishedAt = Date.now(); lastDurationMs = lastStartedAt === null ? null : lastFinishedAt - lastStartedAt; - const delay = options.index_only ? Math.max(0, scheduledAt - Date.now()) : maintenanceConfig.interval_ms; - schedule(lastError || lastOperation?.complete === false ? Math.min(60_000, maintenanceConfig.interval_ms) : delay, !options.index_only && !lastError && lastOperation?.complete !== false); + if (options.index_only) schedule(Math.max(0, scheduledAt - Date.now()), false); + else schedule(lastError || lastOperation?.complete === false ? Math.min(60_000, maintenanceConfig.interval_ms) : maintenanceConfig.interval_ms, !lastError && lastOperation?.complete !== false); try { await persist(); } catch (error) { lastError = `Could not persist maintenance status: ${(error as Error).message}`; phase = 'failed'; } } return getRetentionSchedulerStatus();