fix(server): a hung tmux call or a big memory dedupe no longer freezes the tenant (v0.448.5)

CHANGELOG.md

@@ -8,6 +8,19 @@ new version heading in the same commit.
 
 ## [Unreleased]
 
+## [0.448.5] - 2026-09-22
+### Fixed
+- **Two synchronous paths could freeze a whole tenant's server.** Both hit globex on 2026-09-22: `/health`
+  dead, listen backlog full, no DB write for two hours. (1) Every `tmux` call in `LocalSessionBackend` is a
+  `spawnSync`, and one `send-keys -l` (a task-reconcile poke-back) waited 1h55m on a tmux server reply that
+  never came, holding every request, gate check and scheduler tick behind it. All tmux calls now time out
+  after 5s (SIGKILL). A timed-out call reads as "unknown / not delivered", which every caller already
+  handles. (2) Memory maintenance's `planConsolidation` normalised content inside its inner loop (n²/2
+  regex passes). After the unblock, one agent's 5.2k vector-less memories pinned the process at 100% CPU for
+  ~5 minutes. Content is now normalised once and exact duplicates come from a map, so a group with no
+  vectors is linear: 13.8s → 65ms at that size, with identical merge plans. `maintain` also yields between
+  groups. Pinned by `scripts/event-loop-freeze-test.cjs`.
+
 ## [0.448.4] - 2026-09-18
 ### Fixed
 - **`make-live.sh` takes a lock: one deploy at a time per box.** Every live checkout is shared by whoever

package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-os",
-  "version": "0.448.4",
+  "version": "0.448.5",
   "description": "A generic, governed operating system for running autonomous agents safely across brands. Ships with a local web console.",
   "license": "MIT",
   "type": "commonjs",
@@ -27,7 +27,7 @@
     "check-deps": "bash scripts/install-deps.sh --check",
     "dev": "ts-node src/cli.ts serve",
     "demo:dev": "ts-node src/demo.ts",
-    "test:governance": "node scripts/version-sync-test.cjs && node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/policy-baseline-test.cjs && node scripts/heredoc-intent-test.cjs && node scripts/capability-registry-test.cjs && node scripts/composio-envelope-test.cjs && node scripts/composio-identity-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/telegram-dm-lane-test.cjs && node scripts/cli-link-origin-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/email-identity-guard-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-usage-refresh-test.cjs && node scripts/keychain-credential-test.cjs && node scripts/credential-preflight-test.cjs && node scripts/capacity-queue-test.cjs && node scripts/runtime-login-test.cjs && node scripts/rotate-on-reload-test.cjs && node scripts/headless-resumable-test.cjs && node scripts/session-revive-gates-test.cjs && node scripts/session-pause-test.cjs && node scripts/make-live-lock-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs && node scripts/output-style-test.cjs && node scripts/session-cost-test.cjs && node scripts/chain-model-test.cjs && node scripts/task-workers-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-pr-links-test.cjs && node scripts/task-draft-delete-test.cjs && node scripts/task-discussion-delivery-test.cjs && node scripts/task-resume-test.cjs && node scripts/task-unblock-test.cjs && node scripts/audience-session-access-test.cjs && node scripts/warm-chat-test.cjs && node scripts/poke-warm-caller-test.cjs && node scripts/wakeup-queue-test.cjs && node scripts/stranded-human-stop-test.cjs && node scripts/inject-submit-test.cjs && node scripts/blocked-routing-test.cjs && node scripts/self-dispatch-guard-test.cjs && node scripts/task-proposals-test.cjs && node scripts/npm-boundary-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/per-agent-context-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs && node scripts/episode-quality-test.cjs && node scripts/memory-upkeep-test.cjs && node scripts/automem-health-test.cjs && node scripts/memory-store-switch-test.cjs && node scripts/memory-preload-test.cjs && node scripts/turn-lifecycle-test.cjs && node scripts/resume-seed-test.cjs && node scripts/outcome-vocabulary-test.cjs && node scripts/skill-presets-test.cjs && node scripts/skill-edit-proposal-test.cjs && node scripts/notify-hook-route-test.cjs && node scripts/review-notify-test.cjs && node scripts/turn-idle-background-guard-test.cjs && node scripts/waiting-brief-test.cjs && node scripts/runtime-death-alert-test.cjs && node scripts/github-per-member-test.cjs && node scripts/github-multi-org-test.cjs && node scripts/card-measurement-test.cjs && node scripts/scheduler-admission-test.cjs && node scripts/tick-liveness-test.cjs && node scripts/audit-mirror-test.cjs && node scripts/request-metrics-test.cjs && node scripts/tool-usage-test.cjs && node scripts/sessions-list-perf-test.cjs && node scripts/summarizer-degradation-test.cjs && node scripts/agent-history-scope-test.cjs && node scripts/webhook-ingress-test.cjs && node scripts/slack-content-filter-test.cjs && node scripts/slack-ingress-test.cjs && node scripts/discord-ingress-test.cjs && node scripts/chat-attachments-test.cjs && node scripts/clickup-task-bridge-test.cjs && node scripts/agentric-commands-test.cjs && node scripts/whats-new-test.cjs && node scripts/opencode-gate-test.cjs && node scripts/protected-path-guard-test.cjs && node scripts/attach-grace-test.cjs && node scripts/attach-file-liveness-test.cjs && node scripts/feed-smoke.cjs && node scripts/activity-classify-test.cjs && node scripts/goal-room-test.cjs && node scripts/secret-rotation-test.cjs && node scripts/update-watch-test.cjs && node scripts/runtime-update-watch-test.cjs && node scripts/setup-wizard-test.cjs && node scripts/md-pdf-test.cjs && node scripts/proposal-surfacing-test.cjs && node scripts/process-janitor-test.cjs && node scripts/detached-work-steer-test.cjs && node scripts/statusline-install-test.cjs && node scripts/docs-create-agent-test.cjs && node scripts/agent-stats-rollup-test.cjs && node scripts/task-discussion-rollup-test.cjs && node scripts/session-insights-stamp-test.cjs && node scripts/loop-stall-attribution-test.cjs && node scripts/session-progress-test.cjs && node scripts/drift-nudge-test.cjs && node scripts/goal-metric-review-test.cjs && node scripts/capability-gap-test.cjs && node scripts/workflow-proposal-test.cjs && node scripts/automation-edit-proposal-test.cjs && node scripts/app-card-close-test.cjs",
+    "test:governance": "node scripts/version-sync-test.cjs && node scripts/event-loop-freeze-test.cjs && node scripts/governance-conformance.cjs && node scripts/tier-a-policy-test.cjs && node scripts/policy-baseline-test.cjs && node scripts/heredoc-intent-test.cjs && node scripts/capability-registry-test.cjs && node scripts/composio-envelope-test.cjs && node scripts/composio-identity-test.cjs && node scripts/idle-reaper-test.cjs && node scripts/dm-continuity-test.cjs && node scripts/telegram-dm-lane-test.cjs && node scripts/cli-link-origin-test.cjs && node scripts/alert-staleness-test.cjs && node scripts/run-as-identity-test.cjs && node scripts/email-identity-guard-test.cjs && node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-usage-refresh-test.cjs && node scripts/keychain-credential-test.cjs && node scripts/credential-preflight-test.cjs && node scripts/capacity-queue-test.cjs && node scripts/runtime-login-test.cjs && node scripts/rotate-on-reload-test.cjs && node scripts/headless-resumable-test.cjs && node scripts/session-revive-gates-test.cjs && node scripts/session-pause-test.cjs && node scripts/make-live-lock-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs && node scripts/output-style-test.cjs && node scripts/session-cost-test.cjs && node scripts/chain-model-test.cjs && node scripts/task-workers-test.cjs && node scripts/tuning-patch-test.cjs && node scripts/task-runs-test.cjs && node scripts/task-pr-links-test.cjs && node scripts/task-draft-delete-test.cjs && node scripts/task-discussion-delivery-test.cjs && node scripts/task-resume-test.cjs && node scripts/task-unblock-test.cjs && node scripts/audience-session-access-test.cjs && node scripts/warm-chat-test.cjs && node scripts/poke-warm-caller-test.cjs && node scripts/wakeup-queue-test.cjs && node scripts/stranded-human-stop-test.cjs && node scripts/inject-submit-test.cjs && node scripts/blocked-routing-test.cjs && node scripts/self-dispatch-guard-test.cjs && node scripts/task-proposals-test.cjs && node scripts/npm-boundary-test.cjs && node scripts/agent-edit-guard-test.cjs && node scripts/per-agent-context-test.cjs && node scripts/goal-update-guard-test.cjs && node scripts/insights-signal-test.cjs && node scripts/outcome-derivation-test.cjs && node scripts/episode-quality-test.cjs && node scripts/memory-upkeep-test.cjs && node scripts/automem-health-test.cjs && node scripts/memory-store-switch-test.cjs && node scripts/memory-preload-test.cjs && node scripts/turn-lifecycle-test.cjs && node scripts/resume-seed-test.cjs && node scripts/outcome-vocabulary-test.cjs && node scripts/skill-presets-test.cjs && node scripts/skill-edit-proposal-test.cjs && node scripts/notify-hook-route-test.cjs && node scripts/review-notify-test.cjs && node scripts/turn-idle-background-guard-test.cjs && node scripts/waiting-brief-test.cjs && node scripts/runtime-death-alert-test.cjs && node scripts/github-per-member-test.cjs && node scripts/github-multi-org-test.cjs && node scripts/card-measurement-test.cjs && node scripts/scheduler-admission-test.cjs && node scripts/tick-liveness-test.cjs && node scripts/audit-mirror-test.cjs && node scripts/request-metrics-test.cjs && node scripts/tool-usage-test.cjs && node scripts/sessions-list-perf-test.cjs && node scripts/summarizer-degradation-test.cjs && node scripts/agent-history-scope-test.cjs && node scripts/webhook-ingress-test.cjs && node scripts/slack-content-filter-test.cjs && node scripts/slack-ingress-test.cjs && node scripts/discord-ingress-test.cjs && node scripts/chat-attachments-test.cjs && node scripts/clickup-task-bridge-test.cjs && node scripts/agentric-commands-test.cjs && node scripts/whats-new-test.cjs && node scripts/opencode-gate-test.cjs && node scripts/protected-path-guard-test.cjs && node scripts/attach-grace-test.cjs && node scripts/attach-file-liveness-test.cjs && node scripts/feed-smoke.cjs && node scripts/activity-classify-test.cjs && node scripts/goal-room-test.cjs && node scripts/secret-rotation-test.cjs && node scripts/update-watch-test.cjs && node scripts/runtime-update-watch-test.cjs && node scripts/setup-wizard-test.cjs && node scripts/md-pdf-test.cjs && node scripts/proposal-surfacing-test.cjs && node scripts/process-janitor-test.cjs && node scripts/detached-work-steer-test.cjs && node scripts/statusline-install-test.cjs && node scripts/docs-create-agent-test.cjs && node scripts/agent-stats-rollup-test.cjs && node scripts/task-discussion-rollup-test.cjs && node scripts/session-insights-stamp-test.cjs && node scripts/loop-stall-attribution-test.cjs && node scripts/session-progress-test.cjs && node scripts/drift-nudge-test.cjs && node scripts/goal-metric-review-test.cjs && node scripts/capability-gap-test.cjs && node scripts/workflow-proposal-test.cjs && node scripts/automation-edit-proposal-test.cjs && node scripts/app-card-close-test.cjs",
     "test:alert-staleness": "node scripts/alert-staleness-test.cjs",
     "test:deps": "node scripts/deps-freshness-test.cjs && node scripts/runtime-account-test.cjs && node scripts/runtime-account-misattribution-test.cjs && node scripts/runtime-login-test.cjs && node scripts/claude-config-seed-test.cjs && node scripts/claude-config-isolation-test.cjs",
     "test:dm-continuity": "node scripts/dm-continuity-test.cjs",

scripts/event-loop-freeze-test.cjs

@@ -0,0 +1,114 @@
+#!/usr/bin/env node
+/* Event-loop freeze regressions — two synchronous paths that each took a live tenant's server down whole
+ * (globex 2026-09-22: /health dead, listen backlog full, no DB write for two hours).
+ *
+ *   1. A tmux client that never exits. `LocalSessionBackend` calls tmux with `spawnSync`, which holds the
+ *      single-threaded process until the child is gone; one hung `send-keys -l` held it for 1h55m. Every
+ *      tmux call is now bounded — pinned here with a fake `tmux` that sleeps forever.
+ *   2. `planConsolidation` normalised content inside its inner loop (n²/2 regex passes), and one agent's
+ *      5.2k vector-less memories pinned the process at 100% CPU for ~5 minutes. Pinned by equivalence
+ *      against the original algorithm on random data, plus a wall-clock bound at that live size.
+ * No tmux, claude or DB needed. */
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const assert = require('assert');
+
+const ROOT = path.resolve(__dirname, '..');
+const { LocalSessionBackend } = require(path.join(ROOT, 'dist/edge/session-backend'));
+const { planConsolidation, cosine } = require(path.join(ROOT, 'dist/memory/embedding'));
+
+let pass = 0;
+const ok = (name) => { pass++; console.log(`  ✓ ${name}`); };
+
+// ── 1. a hung tmux client is bounded ────────────────────────────────────────────────────────────────
+const BIN = fs.mkdtempSync(path.join(os.tmpdir(), 'aos-freeze-test-'));
+fs.writeFileSync(path.join(BIN, 'tmux'), '#!/bin/sh\nexec sleep 600\n', { mode: 0o755 });
+const savedPath = process.env.PATH;
+process.env.PATH = `${BIN}:${savedPath}`;
+try {
+  const be = new LocalSessionBackend(path.join(BIN, 'sock'), () => {});
+  let t = Date.now();
+  const delivered = be.injectText('', 'aos-x', 'hello', false);
+  const took = Date.now() - t;
+  assert.strictEqual(delivered, false, 'a timed-out send-keys must report not-delivered');
+  assert.ok(took < 8_000, `send-keys must be bounded, took ${took}ms`);
+  ok(`hung send-keys returns not-delivered in ${took}ms`);
+
+  t = Date.now();
+  assert.strictEqual(be.aliveNames(), null, 'a timed-out liveness poll is UNKNOWN, never "no sessions"');
+  assert.strictEqual(be.hasClient('', 'aos-x'), null, 'a timed-out client poll is unknown');
+  assert.strictEqual(be.capturePane('', 'aos-x'), null, 'a timed-out capture is null');
+  assert.ok(Date.now() - t < 20_000);
+  ok('hung liveness/client/capture polls read as unknown, not as dead');
+} finally {
+  process.env.PATH = savedPath;
+  fs.rmSync(BIN, { recursive: true, force: true });
+}
+
+// ── 2. planConsolidation: same answer, no longer quadratic in regex work ────────────────────────────
+// The pre-fix algorithm, verbatim, as the oracle.
+function reference(rows, dedupeThreshold) {
+  const norm = (s) => s.trim().replace(/\s+/g, ' ').toLowerCase();
+  const sorted = [...rows].sort((a, b) =>
+    (b.importance ?? 0.5) - (a.importance ?? 0.5) || b.recallCount - a.recallCount || b.ts - a.ts);
+  const taken = new Set();
+  const ops = [];
+  for (let i = 0; i < sorted.length; i++) {
+    const anchor = sorted[i];
+    if (taken.has(anchor.id)) continue;
+    const aContent = norm(anchor.content);
+    const dups = [];
+    for (let j = i + 1; j < sorted.length; j++) {
+      const cand = sorted[j];
+      if (taken.has(cand.id)) continue;
+      const exact = norm(cand.content) === aContent;
+      const near = dedupeThreshold != null && !!anchor.vec && !!cand.vec && cosine(anchor.vec, cand.vec) >= dedupeThreshold;
+      if (exact || near) { dups.push(cand); taken.add(cand.id); }
+    }
+    if (dups.length) {
+      taken.add(anchor.id);
+      ops.push({
+        keepId: anchor.id, dropIds: dups.map((d) => d.id),
+        importance: Math.max(anchor.importance ?? 0.5, ...dups.map((d) => d.importance ?? 0.5)),
+        recallCount: anchor.recallCount + dups.reduce((s, d) => s + d.recallCount, 0),
+      });
+    }
+  }
+  return ops;
+}
+
+let seed = 42;
+const rnd = () => ((seed = (seed * 1103515245 + 12345) % 2 ** 31) / 2 ** 31);
+const vecNear = (base) => Float32Array.from(base, (x) => x + (rnd() - 0.5) * 0.05);
+function randomRows(n, vecShare) {
+  const bases = Array.from({ length: 6 }, () => Float32Array.from({ length: 8 }, () => rnd() - 0.5));
+  const texts = ['Deploy failed on box', 'User asked for refund', '  deploy FAILED   on box ', 'Site name taken', 'unique'];
+  return Array.from({ length: n }, (_, i) => ({
+    id: `m${i}`,
+    content: rnd() < 0.5 ? texts[Math.floor(rnd() * texts.length)] : `note ${Math.floor(rnd() * n)}`,
+    importance: rnd() < 0.3 ? undefined : Math.round(rnd() * 10) / 10,
+    recallCount: Math.floor(rnd() * 4),
+    ts: Math.floor(rnd() * 1000),
+    vec: rnd() < vecShare ? vecNear(bases[Math.floor(rnd() * bases.length)]) : undefined,
+  }));
+}
+for (const [n, share, thr] of [[60, 0, 0.95], [60, 0.5, 0.95], [80, 1, 0.9], [80, 0.7, undefined], [5, 1, 0.99], [0, 0, 0.9]]) {
+  for (let k = 0; k < 20; k++) {
+    const rows = randomRows(n, share);
+    assert.deepStrictEqual(planConsolidation(rows, thr), reference(rows, thr), `n=${n} share=${share} thr=${thr}`);
+  }
+}
+ok('identical merge plans to the original algorithm across 120 random groups (exact, near, mixed, no threshold)');
+
+// Live shape: 5.2k memories, ~1.1KB each, no vectors. Pre-fix this was minutes of solid CPU.
+const body = 'x '.repeat(550);
+const big = Array.from({ length: 5200 }, (_, i) => ({ id: `b${i}`, content: `${i % 700} ${body}`, recallCount: 0, ts: i }));
+const t0 = Date.now();
+const ops = planConsolidation(big, 0.95);
+const took = Date.now() - t0;
+assert.strictEqual(ops.length, 700);
+assert.ok(took < 2_000, `5.2k vector-less rows must plan fast, took ${took}ms`);
+ok(`5.2k × 1.1KB vector-less rows planned in ${took}ms`);
+
+console.log(`\nevent-loop-freeze-test: ${pass} passed`);