From 71a22048fe6f1f5df2744c40e9f916fe06d96ba8 Mon Sep 17 00:00:00 2001 From: Vikas Singhal Date: Tue, 22 Sep 2026 17:17:03 +0530 Subject: [PATCH] fix(server): a hung tmux call or a big memory dedupe no longer freezes the tenant (v0.448.5) Every tmux spawnSync in LocalSessionBackend is now bounded (5s, SIGKILL): one send-keys held a live server for 1h55m. planConsolidation normalises once and finds exact dups via a map, so vector-less groups are linear (13.8s -> 65ms at 5.2k rows, identical plans); maintain yields between groups. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 13 ++++ package-lock.json | 4 +- package.json | 4 +- scripts/event-loop-freeze-test.cjs | 114 +++++++++++++++++++++++++++++ src/edge/session-backend.ts | 25 +++++-- src/memory/embedding.ts | 28 +++++-- src/memory/sqlite-provider.ts | 3 + 7 files changed, 172 insertions(+), 19 deletions(-) create mode 100644 scripts/event-loop-freeze-test.cjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c696e61..ac0fde5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ new version heading in the same commit. ## [Unreleased] +## [0.448.5] - 2026-09-22 +### Fixed +- **Two synchronous paths could freeze a whole tenant's server.** Both hit globex on 2026-09-22: `/health` + dead, listen backlog full, no DB write for two hours. (1) Every `tmux` call in `LocalSessionBackend` is a + `spawnSync`, and one `send-keys -l` (a task-reconcile poke-back) waited 1h55m on a tmux server reply that + never came, holding every request, gate check and scheduler tick behind it. All tmux calls now time out + after 5s (SIGKILL). A timed-out call reads as "unknown / not delivered", which every caller already + handles. (2) Memory maintenance's `planConsolidation` normalised content inside its inner loop (n²/2 + regex passes). After the unblock, one agent's 5.2k vector-less memories pinned the process at 100% CPU for + ~5 minutes. Content is now normalised once and exact duplicates come from a map, so a group with no + vectors is linear: 13.8s → 65ms at that size, with identical merge plans. `maintain` also yields between + groups. Pinned by `scripts/event-loop-freeze-test.cjs`. + ## [0.448.4] - 2026-09-18 ### Fixed - **`make-live.sh` takes a lock: one deploy at a time per box.** Every live checkout is shared by whoever diff --git a/package-lock.json b/package-lock.json index dbef313c..7901ff19 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "agent-os", - "version": "0.448.4", + "version": "0.448.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agent-os", - "version": "0.448.4", + "version": "0.448.5", "license": "MIT", "bin": { "agent-os": "bin/agent-os" diff --git a/package.json b/package.json index 5f0bee6a..ac3e1226 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "agent-os", - "version": "0.448.4", + "version": "0.448.5", "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.", "license": "MIT", "type": "commonjs", @@ -27,7 +27,7 @@ "check-deps": "bash scripts/install-deps.sh --check", "dev": "ts-node src/cli.ts serve", "demo:dev": "ts-node src/demo.ts", - "test:governance": "node scripts/version-sync-test.cjs && node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/policy-baseline-test.cjs && node scripts/heredoc-intent-test.cjs && node scripts/capability-registry-test.cjs && node scripts/composio-envelope-test.cjs && node scripts/composio-identity-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/telegram-dm-lane-test.cjs && node scripts/cli-link-origin-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/email-identity-guard-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-usage-refresh-test.cjs && node scripts/keychain-credential-test.cjs && node scripts/credential-preflight-test.cjs && node scripts/capacity-queue-test.cjs && node scripts/runtime-login-test.cjs && node scripts/rotate-on-reload-test.cjs && node scripts/headless-resumable-test.cjs && node scripts/session-revive-gates-test.cjs && node scripts/session-pause-test.cjs && node scripts/make-live-lock-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs && node scripts/output-style-test.cjs && node scripts/session-cost-test.cjs && node scripts/chain-model-test.cjs && node scripts/task-workers-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-pr-links-test.cjs && node scripts/task-draft-delete-test.cjs && node scripts/task-discussion-delivery-test.cjs && node scripts/task-resume-test.cjs && node scripts/task-unblock-test.cjs && node scripts/audience-session-access-test.cjs && node scripts/warm-chat-test.cjs && node scripts/poke-warm-caller-test.cjs && node scripts/wakeup-queue-test.cjs && node scripts/stranded-human-stop-test.cjs && node scripts/inject-submit-test.cjs && node scripts/blocked-routing-test.cjs && node scripts/self-dispatch-guard-test.cjs && node scripts/task-proposals-test.cjs && node scripts/npm-boundary-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/per-agent-context-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs && node scripts/episode-quality-test.cjs && node scripts/memory-upkeep-test.cjs && node scripts/automem-health-test.cjs && node scripts/memory-store-switch-test.cjs && node scripts/memory-preload-test.cjs && node scripts/turn-lifecycle-test.cjs && node scripts/resume-seed-test.cjs && node scripts/outcome-vocabulary-test.cjs && node scripts/skill-presets-test.cjs && node scripts/skill-edit-proposal-test.cjs && node scripts/notify-hook-route-test.cjs && node scripts/review-notify-test.cjs && node scripts/turn-idle-background-guard-test.cjs && node scripts/waiting-brief-test.cjs && node scripts/runtime-death-alert-test.cjs && node scripts/github-per-member-test.cjs && node scripts/github-multi-org-test.cjs && node scripts/card-measurement-test.cjs && node scripts/scheduler-admission-test.cjs && node scripts/tick-liveness-test.cjs && node scripts/audit-mirror-test.cjs && node scripts/request-metrics-test.cjs && node scripts/tool-usage-test.cjs && node scripts/sessions-list-perf-test.cjs && node scripts/summarizer-degradation-test.cjs && node scripts/agent-history-scope-test.cjs && node scripts/webhook-ingress-test.cjs && node scripts/slack-content-filter-test.cjs && node scripts/slack-ingress-test.cjs && node scripts/discord-ingress-test.cjs && node scripts/chat-attachments-test.cjs && node scripts/clickup-task-bridge-test.cjs && node scripts/agentric-commands-test.cjs && node scripts/whats-new-test.cjs && node scripts/opencode-gate-test.cjs && node scripts/protected-path-guard-test.cjs && node scripts/attach-grace-test.cjs && node scripts/attach-file-liveness-test.cjs && node scripts/feed-smoke.cjs && node scripts/activity-classify-test.cjs && node scripts/goal-room-test.cjs && node scripts/secret-rotation-test.cjs && node scripts/update-watch-test.cjs && node scripts/runtime-update-watch-test.cjs && node scripts/setup-wizard-test.cjs && node scripts/md-pdf-test.cjs && node scripts/proposal-surfacing-test.cjs && node scripts/process-janitor-test.cjs && node scripts/detached-work-steer-test.cjs && node scripts/statusline-install-test.cjs && node scripts/docs-create-agent-test.cjs && node scripts/agent-stats-rollup-test.cjs && node scripts/task-discussion-rollup-test.cjs && node scripts/session-insights-stamp-test.cjs && node scripts/loop-stall-attribution-test.cjs && node scripts/session-progress-test.cjs && node scripts/drift-nudge-test.cjs && node scripts/goal-metric-review-test.cjs && node scripts/capability-gap-test.cjs && node scripts/workflow-proposal-test.cjs && node scripts/automation-edit-proposal-test.cjs && node scripts/app-card-close-test.cjs", + "test:governance": "node scripts/version-sync-test.cjs && node scripts/event-loop-freeze-test.cjs && node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/policy-baseline-test.cjs && node scripts/heredoc-intent-test.cjs && node scripts/capability-registry-test.cjs && node scripts/composio-envelope-test.cjs && node scripts/composio-identity-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/telegram-dm-lane-test.cjs && node scripts/cli-link-origin-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/email-identity-guard-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-usage-refresh-test.cjs && node scripts/keychain-credential-test.cjs && node scripts/credential-preflight-test.cjs && node scripts/capacity-queue-test.cjs && node scripts/runtime-login-test.cjs && node scripts/rotate-on-reload-test.cjs && node scripts/headless-resumable-test.cjs && node scripts/session-revive-gates-test.cjs && node scripts/session-pause-test.cjs && node scripts/make-live-lock-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs && node scripts/output-style-test.cjs && node scripts/session-cost-test.cjs && node scripts/chain-model-test.cjs && node scripts/task-workers-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-pr-links-test.cjs && node scripts/task-draft-delete-test.cjs && node scripts/task-discussion-delivery-test.cjs && node scripts/task-resume-test.cjs && node scripts/task-unblock-test.cjs && node scripts/audience-session-access-test.cjs && node scripts/warm-chat-test.cjs && node scripts/poke-warm-caller-test.cjs && node scripts/wakeup-queue-test.cjs && node scripts/stranded-human-stop-test.cjs && node scripts/inject-submit-test.cjs && node scripts/blocked-routing-test.cjs && node scripts/self-dispatch-guard-test.cjs && node scripts/task-proposals-test.cjs && node scripts/npm-boundary-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/per-agent-context-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs && node scripts/episode-quality-test.cjs && node scripts/memory-upkeep-test.cjs && node scripts/automem-health-test.cjs && node scripts/memory-store-switch-test.cjs && node scripts/memory-preload-test.cjs && node scripts/turn-lifecycle-test.cjs && node scripts/resume-seed-test.cjs && node scripts/outcome-vocabulary-test.cjs && node scripts/skill-presets-test.cjs && node scripts/skill-edit-proposal-test.cjs && node scripts/notify-hook-route-test.cjs && node scripts/review-notify-test.cjs && node scripts/turn-idle-background-guard-test.cjs && node scripts/waiting-brief-test.cjs && node scripts/runtime-death-alert-test.cjs && node scripts/github-per-member-test.cjs && node scripts/github-multi-org-test.cjs && node scripts/card-measurement-test.cjs && node scripts/scheduler-admission-test.cjs && node scripts/tick-liveness-test.cjs && node scripts/audit-mirror-test.cjs && node scripts/request-metrics-test.cjs && node scripts/tool-usage-test.cjs && node scripts/sessions-list-perf-test.cjs && node scripts/summarizer-degradation-test.cjs && node scripts/agent-history-scope-test.cjs && node scripts/webhook-ingress-test.cjs && node scripts/slack-content-filter-test.cjs && node scripts/slack-ingress-test.cjs && node scripts/discord-ingress-test.cjs && node scripts/chat-attachments-test.cjs && node scripts/clickup-task-bridge-test.cjs && node scripts/agentric-commands-test.cjs && node scripts/whats-new-test.cjs && node scripts/opencode-gate-test.cjs && node scripts/protected-path-guard-test.cjs && node scripts/attach-grace-test.cjs && node scripts/attach-file-liveness-test.cjs && node scripts/feed-smoke.cjs && node scripts/activity-classify-test.cjs && node scripts/goal-room-test.cjs && node scripts/secret-rotation-test.cjs && node scripts/update-watch-test.cjs && node scripts/runtime-update-watch-test.cjs && node scripts/setup-wizard-test.cjs && node scripts/md-pdf-test.cjs && node scripts/proposal-surfacing-test.cjs && node scripts/process-janitor-test.cjs && node scripts/detached-work-steer-test.cjs && node scripts/statusline-install-test.cjs && node scripts/docs-create-agent-test.cjs && node scripts/agent-stats-rollup-test.cjs && node scripts/task-discussion-rollup-test.cjs && node scripts/session-insights-stamp-test.cjs && node scripts/loop-stall-attribution-test.cjs && node scripts/session-progress-test.cjs && node scripts/drift-nudge-test.cjs && node scripts/goal-metric-review-test.cjs && node scripts/capability-gap-test.cjs && node scripts/workflow-proposal-test.cjs && node scripts/automation-edit-proposal-test.cjs && node scripts/app-card-close-test.cjs", "test:alert-staleness": "node scripts/alert-staleness-test.cjs", "test:deps": "node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-login-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs", "test:dm-continuity": "node scripts/dm-continuity-test.cjs", diff --git a/scripts/event-loop-freeze-test.cjs b/scripts/event-loop-freeze-test.cjs new file mode 100644 index 00000000..459ed8f7 --- /dev/null +++ b/scripts/event-loop-freeze-test.cjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +/* Event-loop freeze regressions — two synchronous paths that each took a live tenant's server down whole + * (globex 2026-09-22: /health dead, listen backlog full, no DB write for two hours). + * + * 1. A tmux client that never exits. `LocalSessionBackend` calls tmux with `spawnSync`, which holds the + * single-threaded process until the child is gone; one hung `send-keys -l` held it for 1h55m. Every + * tmux call is now bounded — pinned here with a fake `tmux` that sleeps forever. + * 2. `planConsolidation` normalised content inside its inner loop (n²/2 regex passes), and one agent's + * 5.2k vector-less memories pinned the process at 100% CPU for ~5 minutes. Pinned by equivalence + * against the original algorithm on random data, plus a wall-clock bound at that live size. + * No tmux, claude or DB needed. */ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const assert = require('assert'); + +const ROOT = path.resolve(__dirname, '..'); +const { LocalSessionBackend } = require(path.join(ROOT, 'dist/edge/session-backend')); +const { planConsolidation, cosine } = require(path.join(ROOT, 'dist/memory/embedding')); + +let pass = 0; +const ok = (name) => { pass++; console.log(` ✓ ${name}`); }; + +// ── 1. a hung tmux client is bounded ──────────────────────────────────────────────────────────────── +const BIN = fs.mkdtempSync(path.join(os.tmpdir(), 'aos-freeze-test-')); +fs.writeFileSync(path.join(BIN, 'tmux'), '#!/bin/sh\nexec sleep 600\n', { mode: 0o755 }); +const savedPath = process.env.PATH; +process.env.PATH = `${BIN}:${savedPath}`; +try { + const be = new LocalSessionBackend(path.join(BIN, 'sock'), () => {}); + let t = Date.now(); + const delivered = be.injectText('', 'aos-x', 'hello', false); + const took = Date.now() - t; + assert.strictEqual(delivered, false, 'a timed-out send-keys must report not-delivered'); + assert.ok(took < 8_000, `send-keys must be bounded, took ${took}ms`); + ok(`hung send-keys returns not-delivered in ${took}ms`); + + t = Date.now(); + assert.strictEqual(be.aliveNames(), null, 'a timed-out liveness poll is UNKNOWN, never "no sessions"'); + assert.strictEqual(be.hasClient('', 'aos-x'), null, 'a timed-out client poll is unknown'); + assert.strictEqual(be.capturePane('', 'aos-x'), null, 'a timed-out capture is null'); + assert.ok(Date.now() - t < 20_000); + ok('hung liveness/client/capture polls read as unknown, not as dead'); +} finally { + process.env.PATH = savedPath; + fs.rmSync(BIN, { recursive: true, force: true }); +} + +// ── 2. planConsolidation: same answer, no longer quadratic in regex work ──────────────────────────── +// The pre-fix algorithm, verbatim, as the oracle. +function reference(rows, dedupeThreshold) { + const norm = (s) => s.trim().replace(/\s+/g, ' ').toLowerCase(); + const sorted = [...rows].sort((a, b) => + (b.importance ?? 0.5) - (a.importance ?? 0.5) || b.recallCount - a.recallCount || b.ts - a.ts); + const taken = new Set(); + const ops = []; + for (let i = 0; i < sorted.length; i++) { + const anchor = sorted[i]; + if (taken.has(anchor.id)) continue; + const aContent = norm(anchor.content); + const dups = []; + for (let j = i + 1; j < sorted.length; j++) { + const cand = sorted[j]; + if (taken.has(cand.id)) continue; + const exact = norm(cand.content) === aContent; + const near = dedupeThreshold != null && !!anchor.vec && !!cand.vec && cosine(anchor.vec, cand.vec) >= dedupeThreshold; + if (exact || near) { dups.push(cand); taken.add(cand.id); } + } + if (dups.length) { + taken.add(anchor.id); + ops.push({ + keepId: anchor.id, dropIds: dups.map((d) => d.id), + importance: Math.max(anchor.importance ?? 0.5, ...dups.map((d) => d.importance ?? 0.5)), + recallCount: anchor.recallCount + dups.reduce((s, d) => s + d.recallCount, 0), + }); + } + } + return ops; +} + +let seed = 42; +const rnd = () => ((seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31); +const vecNear = (base) => Float32Array.from(base, (x) => x + (rnd() - 0.5) * 0.05); +function randomRows(n, vecShare) { + const bases = Array.from({ length: 6 }, () => Float32Array.from({ length: 8 }, () => rnd() - 0.5)); + const texts = ['Deploy failed on box', 'User asked for refund', ' deploy FAILED on box ', 'Site name taken', 'unique']; + return Array.from({ length: n }, (_, i) => ({ + id: `m${i}`, + content: rnd() < 0.5 ? texts[Math.floor(rnd() * texts.length)] : `note ${Math.floor(rnd() * n)}`, + importance: rnd() < 0.3 ? undefined : Math.round(rnd() * 10) / 10, + recallCount: Math.floor(rnd() * 4), + ts: Math.floor(rnd() * 1000), + vec: rnd() < vecShare ? vecNear(bases[Math.floor(rnd() * bases.length)]) : undefined, + })); +} +for (const [n, share, thr] of [[60, 0, 0.95], [60, 0.5, 0.95], [80, 1, 0.9], [80, 0.7, undefined], [5, 1, 0.99], [0, 0, 0.9]]) { + for (let k = 0; k < 20; k++) { + const rows = randomRows(n, share); + assert.deepStrictEqual(planConsolidation(rows, thr), reference(rows, thr), `n=${n} share=${share} thr=${thr}`); + } +} +ok('identical merge plans to the original algorithm across 120 random groups (exact, near, mixed, no threshold)'); + +// Live shape: 5.2k memories, ~1.1KB each, no vectors. Pre-fix this was minutes of solid CPU. +const body = 'x '.repeat(550); +const big = Array.from({ length: 5200 }, (_, i) => ({ id: `b${i}`, content: `${i % 700} ${body}`, recallCount: 0, ts: i })); +const t0 = Date.now(); +const ops = planConsolidation(big, 0.95); +const took = Date.now() - t0; +assert.strictEqual(ops.length, 700); +assert.ok(took < 2_000, `5.2k vector-less rows must plan fast, took ${took}ms`); +ok(`5.2k × 1.1KB vector-less rows planned in ${took}ms`); + +console.log(`\nevent-loop-freeze-test: ${pass} passed`); diff --git a/src/edge/session-backend.ts b/src/edge/session-backend.ts index aec957a1..0c8a2b71 100644 --- a/src/edge/session-backend.ts +++ b/src/edge/session-backend.ts @@ -100,6 +100,15 @@ const SUBMIT_ATTEMPTS = 2; * Invalidated eagerly on spawn/kill, so this bounds only how stale an UNCHANGED-by-us world may read. */ const ALIVE_POLL_TTL_MS = 1_000; +/** Every synchronous tmux call is bounded. They answer in single-digit ms, but `spawnSync` holds the + * whole single-threaded server until the child exits, and a tmux client can wait forever on a server + * reply that never comes: globex 2026-09-22, one `send-keys -l` (a task-reconcile poke-back) sat in + * `poll` for 1h55m while every request, gate check and scheduler tick queued behind it (listen backlog + * full, `/health` dead, no DB write for two hours). A timed-out call reports `error` — which every + * caller already reads as "unknown / not delivered" — instead of freezing the process. SIGKILL because + * the stuck client is in a blocking poll and a polite signal buys nothing. */ +const TMUX_SYNC = { timeout: 5_000, killSignal: 'SIGKILL' as const }; + /** Block this thread for `ms`. The backend contract is synchronous, and the settle has to happen BETWEEN * two tmux calls, so there is nowhere to await. Bounded by PASTE_SETTLE_MS × SUBMIT_ATTEMPTS (~1s). */ function sleepSync(ms: number): void { @@ -159,12 +168,12 @@ export class LocalSessionBackend implements SessionBackend { ['set', '-g', 'mode-style', 'bg=#2563eb,fg=#ffffff'], ['bind', '-T', 'copy-mode', 'MouseDragEnd1Pane', 'send-keys', '-X', 'copy-selection-no-clear'], ['bind', '-T', 'copy-mode-vi', 'MouseDragEnd1Pane', 'send-keys', '-X', 'copy-selection-no-clear']]) { - spawnSync('tmux', ['-S', this.tmuxSocket, ...opt], { stdio: 'ignore' }); + spawnSync('tmux', ['-S', this.tmuxSocket, ...opt], { ...TMUX_SYNC, stdio: 'ignore' }); } } kill(_space: string, tmuxName: string): void { - spawnSync('tmux', ['-S', this.tmuxSocket, 'kill-session', '-t', tmuxName], { stdio: 'ignore' }); + spawnSync('tmux', ['-S', this.tmuxSocket, 'kill-session', '-t', tmuxName], { ...TMUX_SYNC, stdio: 'ignore' }); this.alivePoll = undefined; // we just changed the world — the next reader must see it } @@ -191,12 +200,12 @@ export class LocalSessionBackend implements SessionBackend { injectText(_space: string, tmuxName: string, text: string, submit: boolean, _verify = true, enterPresses = SUBMIT_ATTEMPTS): boolean { // `-l` = literal: send the bytes as typed, not as tmux key names (a path could contain `;`, `-`, // etc.). Submit is a SEPARATE send-keys with the `Enter` key name so it's interpreted as a return. - const r = spawnSync('tmux', ['-S', this.tmuxSocket, 'send-keys', '-t', tmuxName, '-l', text], { stdio: 'ignore' }); + const r = spawnSync('tmux', ['-S', this.tmuxSocket, 'send-keys', '-t', tmuxName, '-l', text], { ...TMUX_SYNC, stdio: 'ignore' }); if (r.status !== 0) return false; if (!submit) return true; for (let attempt = 1; attempt <= Math.max(1, enterPresses); attempt++) { sleepSync(PASTE_SETTLE_MS * attempt); // let the paste finish assembling before the Enter lands - spawnSync('tmux', ['-S', this.tmuxSocket, 'send-keys', '-t', tmuxName, 'Enter'], { stdio: 'ignore' }); + spawnSync('tmux', ['-S', this.tmuxSocket, 'send-keys', '-t', tmuxName, 'Enter'], { ...TMUX_SYNC, stdio: 'ignore' }); } return true; } @@ -226,7 +235,7 @@ export class LocalSessionBackend implements SessionBackend { } private pollAliveNames(): Set | null { - const r = spawnSync('tmux', ['-S', this.tmuxSocket, 'list-sessions', '-F', '#S'], { encoding: 'utf8' }); + const r = spawnSync('tmux', ['-S', this.tmuxSocket, 'list-sessions', '-F', '#S'], { ...TMUX_SYNC, encoding: 'utf8' }); // Distinguish "couldn't run the poll" from "tmux answered, no sessions". A transient spawn // failure (EAGAIN/ENOMEM/EMFILE under fork/memory pressure) sets r.error; treat that as UNKNOWN // (null) so the caller does NOT reap — otherwise one hiccup flips every live session to idle and, @@ -241,7 +250,7 @@ export class LocalSessionBackend implements SessionBackend { // One tmux call maps every live pane to its root PID; one `ps` snapshot gives the whole process // table. We then sum RSS over each pane's subtree (the shell → node/claude → MCP children). // `-Ao pid=,ppid=,rss=` is portable across BSD (macOS) and GNU (Linux) ps; RSS is KiB on both. - const panes = spawnSync('tmux', ['-S', this.tmuxSocket, 'list-panes', '-a', '-F', '#{session_name} #{pane_pid}'], { encoding: 'utf8' }); + const panes = spawnSync('tmux', ['-S', this.tmuxSocket, 'list-panes', '-a', '-F', '#{session_name} #{pane_pid}'], { ...TMUX_SYNC, encoding: 'utf8' }); if (panes.error) return null; // couldn't poll tmux → unknown if (panes.status !== 0) return new Map(); // no server / no sessions → nothing to measure const ps = spawnSync('ps', ['-Ao', 'pid=,ppid=,rss='], { encoding: 'utf8' }); @@ -285,7 +294,7 @@ export class LocalSessionBackend implements SessionBackend { hasClient(_space: string, tmuxName: string): boolean | null { // `list-clients -t ` prints one line per attached ttyd/xterm client; empty → nobody watching. - const r = spawnSync('tmux', ['-S', this.tmuxSocket, 'list-clients', '-t', tmuxName, '-F', '#{client_name}'], { encoding: 'utf8' }); + const r = spawnSync('tmux', ['-S', this.tmuxSocket, 'list-clients', '-t', tmuxName, '-F', '#{client_name}'], { ...TMUX_SYNC, encoding: 'utf8' }); if (r.error) return null; // couldn't poll → unknown (don't reap on a hiccup) if (r.status !== 0) return false; // no such session / no server → nothing attached return (r.stdout || '').split('\n').some(Boolean); @@ -293,7 +302,7 @@ export class LocalSessionBackend implements SessionBackend { capturePane(_space: string, tmuxName: string): string | null { // -p: print to stdout; -J: join wrapped lines; -S -: from the start of the scrollback history. - const r = spawnSync('tmux', ['-S', this.tmuxSocket, 'capture-pane', '-p', '-J', '-S', '-', '-t', tmuxName], { encoding: 'utf8' }); + const r = spawnSync('tmux', ['-S', this.tmuxSocket, 'capture-pane', '-p', '-J', '-S', '-', '-t', tmuxName], { ...TMUX_SYNC, encoding: 'utf8' }); if (r.error || r.status !== 0) return null; return r.stdout || ''; } diff --git a/src/memory/embedding.ts b/src/memory/embedding.ts index 1a81714b..7bc9b1df 100644 --- a/src/memory/embedding.ts +++ b/src/memory/embedding.ts @@ -93,19 +93,33 @@ export function planConsolidation(rows: ConsolidateRow[], dedupeThreshold?: numb const norm = (s: string) => s.trim().replace(/\s+/g, ' ').toLowerCase(); const sorted = [...rows].sort((a, b) => (b.importance ?? 0.5) - (a.importance ?? 0.5) || b.recallCount - a.recallCount || b.ts - a.ts); + // Normalise ONCE per row. This used to run inside the inner loop — n²/2 regex passes over ~1KB bodies — + // and since the whole plan is synchronous it held the server's event loop: globex 2026-09-22, one agent + // with 5.2k vector-less memories pinned the process at 100% CPU for ~5 minutes (every request dead). + // Exact duplicates now come from a content→indices map, so a group with no vectors is O(n); the pairwise + // scan runs only where a near-dup comparison is actually possible. + const normed = sorted.map((r) => norm(r.content)); + const byContent = new Map(); + normed.forEach((c, i) => { const a = byContent.get(c); if (a) a.push(i); else byContent.set(c, [i]); }); + const nearOn = dedupeThreshold != null; const taken = new Set(); const ops: MergeOp[] = []; for (let i = 0; i < sorted.length; i++) { const anchor = sorted[i]; if (taken.has(anchor.id)) continue; - const aContent = norm(anchor.content); const dups: ConsolidateRow[] = []; - for (let j = i + 1; j < sorted.length; j++) { - const cand = sorted[j]; - if (taken.has(cand.id)) continue; - const exact = norm(cand.content) === aContent; - const near = dedupeThreshold != null && !!anchor.vec && !!cand.vec && cosine(anchor.vec, cand.vec) >= dedupeThreshold; - if (exact || near) { dups.push(cand); taken.add(cand.id); } + const exactIdx = new Set((byContent.get(normed[i]) ?? []).filter((j) => j > i)); + if (nearOn && anchor.vec) { + for (let j = i + 1; j < sorted.length; j++) { + const cand = sorted[j]; + if (taken.has(cand.id)) continue; + if (exactIdx.has(j) || (!!cand.vec && cosine(anchor.vec, cand.vec) >= dedupeThreshold!)) { dups.push(cand); taken.add(cand.id); } + } + } else { + for (const j of exactIdx) { + const cand = sorted[j]; + if (!taken.has(cand.id)) { dups.push(cand); taken.add(cand.id); } + } } if (dups.length) { taken.add(anchor.id); diff --git a/src/memory/sqlite-provider.ts b/src/memory/sqlite-provider.ts index f7f23f3f..de4f20f7 100644 --- a/src/memory/sqlite-provider.ts +++ b/src/memory/sqlite-provider.ts @@ -231,6 +231,9 @@ export class SqliteMemoryProvider implements MemoryProvider { }); } for (const [key, arr] of groups) { + // Yield between groups: a vector-carrying group is still an O(n²) cosine scan, and one agent's + // plan must not hold every other request behind it for the whole sweep. + await new Promise((r) => setImmediate(r)); const [gTenant, gAgent] = key.split('\0'); for (const op of planConsolidation(arr, opts.dedupeThreshold)) { this.db.prepare('UPDATE memories SET importance = ?, recall_count = ? WHERE id = ?')