From b9ba165a1d58fc7b20b90b4d68a399d59ac6e60f Mon Sep 17 00:00:00 2001 From: codeasier Date: Mon, 28 Sep 2026 20:38:05 +0800 Subject: [PATCH] fix(cli): make fresh installs portable and persistent Port of upstream main e04ba88 (#892) onto v0.9.29-patch, aligned with upstream: startIiiBin regenerates a runtime iii-config (absolute data paths under ~/.agentmemory/data, absolute worker exec line, engine cwd anchored at ~/.agentmemory), migrates legacy ./data stores on first run, and passes user overrides through verbatim. Viewer port becomes configurable (III_VIEWER_PORT); worker pidfiles honor AGENTMEMORY_RUNTIME_DIR; docker installs gain uid/gid/chown controls; agentmemory remove covers the runtime config; doctor and startup-failure reporting expanded. Adaptations for this line: - test/engine-config.test.ts: upstream adds the file for renderEngineConfig while this line already had an observability-off drift guard of the same name; both suites are kept side by side. - src/index.ts import conflict: kept this line's logger + parsePositiveIntervalMs imports alongside upstream's runtimeMetadataPath. - plugin/skills/agentmemory-config/REFERENCE.md regenerated via skills:gen (55 -> 60 vars; upstream's five new variables added, this line's variables retained). - III_PINNED_VERSION indirection (0.11.2) carried through untouched. - README endpoint count stays 132 (this line's REST surface). tsc error set identical to the post-#7 baseline; 1844 tests pass (+54 from upstream suites). --- .env.example | 40 +- CHANGELOG.md | 1 + INSTALL_FOR_AGENTS.md | 151 ++- README.md | 172 ++- docker-compose.yml | 23 +- iii-config.docker.yaml | 16 +- plugin/skills/agentmemory-config/REFERENCE.md | 7 +- src/cli.ts | 1179 ++++++++++++++--- src/cli/engine-config.ts | 121 ++ src/cli/engine-launch.ts | 189 +++ src/cli/process-state.ts | 3 + src/cli/remove-plan.ts | 75 +- src/cli/startup-stderr.ts | 23 + src/config.ts | 6 +- src/index.ts | 17 +- src/runtime-paths.ts | 19 + src/types.ts | 1 + test/cli-data-dir.test.ts | 15 + test/cli-engine-startup.test.ts | 149 +++ test/cli-lifecycle-safety.test.ts | 356 +++++ test/cli-remove.test.ts | 47 + test/docker-port-config.test.ts | 67 + test/engine-config.test.ts | 51 + test/engine-launch.test.ts | 178 +++ test/multi-instance-port.test.ts | 10 + test/process-state.test.ts | 14 + test/runtime-paths.test.ts | 29 + test/startup-stderr.test.ts | 13 + test/stop-worker-pidfile.test.ts | 24 +- 29 files changed, 2649 insertions(+), 347 deletions(-) create mode 100644 src/cli/engine-config.ts create mode 100644 src/cli/engine-launch.ts create mode 100644 src/cli/process-state.ts create mode 100644 src/cli/startup-stderr.ts create mode 100644 src/runtime-paths.ts create mode 100644 test/cli-engine-startup.test.ts create mode 100644 test/cli-lifecycle-safety.test.ts create mode 100644 test/docker-port-config.test.ts create mode 100644 test/engine-launch.test.ts create mode 100644 test/process-state.test.ts create mode 100644 test/runtime-paths.test.ts create mode 100644 test/startup-stderr.test.ts diff --git a/.env.example b/.env.example index ce8d203fb7..f51d2c76c0 100644 --- a/.env.example +++ b/.env.example @@ -9,9 +9,9 @@ # LLM key, no embedding key, no API auth, and no scheduled eviction. Maintenance # sweeps that only bound internal metadata may be enabled unless marked otherwise. # -# Run `npx @agentmemory/agentmemory init` to copy this file into place -# automatically. Run `npx @agentmemory/agentmemory doctor` to verify that -# the daemon reads the env you expect. +# Run `npx -y @agentmemory/agentmemory@latest init` to copy this file into +# place automatically. Run `npx -y @agentmemory/agentmemory@latest doctor` +# to verify that the daemon reads the env you expect. # # Defaults shown in comments. Listed in priority order — the first key # present wins on the LLM detection path (see src/config.ts::detectProvider). @@ -21,8 +21,10 @@ # ----------------------------------------------------------------------------- # # Without a provider key, agentmemory runs in noop mode: observations are -# indexed via zero-LLM synthetic compression, hybrid search still works, +# indexed via zero-LLM synthetic compression and BM25 recall still works, # but LLM-backed summarisation / reflection / consolidation are disabled. +# A provider key alone does not enable LLM-written observation compression; +# that path also requires AGENTMEMORY_AUTO_COMPRESS=true. # The detection order is OPENAI_API_KEY → MINIMAX_API_KEY → ANTHROPIC_API_KEY # → GEMINI_API_KEY → OPENROUTER_API_KEY → noop. @@ -64,10 +66,14 @@ # 2. Embedding provider — auto-detected, override via EMBEDDING_PROVIDER # ----------------------------------------------------------------------------- # -# Without an embedding key, agentmemory runs in BM25-only mode for hybrid -# search. Detection order: EMBEDDING_PROVIDER override → GEMINI_API_KEY → +# Without an embedding provider, vector embeddings are disabled. `mem::search` +# uses BM25; `mem::smart-search` can also fuse structural graph matches when +# graph data exists. Remote +# provider detection order: EMBEDDING_PROVIDER override → GEMINI_API_KEY → # OPENAI_API_KEY → VOYAGE_API_KEY → COHERE_API_KEY → OPENROUTER_API_KEY → -# local (Xenova/all-MiniLM-L6-v2, 384-dim). +# BM25-only. Local embeddings are an explicit opt-in, not the keyless default. +# `EMBEDDING_PROVIDER=local` uses Xenova/all-MiniLM-L6-v2 (384-dim) on-device; +# the first embedding request downloads the model and needs network access. # EMBEDDING_PROVIDER=local # local | openai | voyage | cohere | gemini | openrouter @@ -107,7 +113,7 @@ # 5. Behaviour flags # ----------------------------------------------------------------------------- -# AGENTMEMORY_AUTO_COMPRESS=true # Run LLM compression on every observation batch (requires a provider key). Default off — synthetic compression handles most cases. +# AGENTMEMORY_AUTO_COMPRESS=true # Run LLM-written compression on every observation batch. Requires BOTH this flag and a configured provider; a provider alone leaves synthetic compression active. # AGENTMEMORY_INJECT_CONTEXT=true # Inject recalled memories back into agent prompts (#143). Default off — hooks capture observations but do not modify conversation. # CONSOLIDATION_ENABLED=false # Runs by default with an LLM provider; set false to opt out of the 4-tier pipeline. # CONSOLIDATION_DECAY_DAYS=30 # Age (days) after which non-reinforced memories decay during consolidation @@ -150,6 +156,13 @@ # AGENTMEMORY_PROBE_TIMEOUT_MS=2000 # MCP shim livez probe timeout # AGENTMEMORY_URL=http://localhost:3111 # REST base URL — honored by status, doctor, MCP shim # AGENTMEMORY_VIEWER_URL=http://localhost:3113 # Override the viewer URL printed by `agentmemory status` +# Persistent iii state; same override as `--data-dir`, used by native and +# Docker starts. Explicit flag/env wins. Otherwise instance 0 reuses an +# existing ./data/state_store.db or ./data/iii-config.yaml before defaults: macOS +# ~/Library/Application Support/agentmemory; Linux $XDG_DATA_HOME/agentmemory +# or ~/.local/share/agentmemory; Windows %APPDATA%\agentmemory. +# AGENTMEMORY_DATA_DIR=/absolute/path +# AGENTMEMORY_USE_DOCKER=1 # Prefer bundled Docker even when a compatible native iii binary exists # AGENTMEMORY_EXPORT_ROOT=~/agentmemory-backup # Default destination for `agentmemory export` # STANDALONE_MCP=1 # MCP shim only — bypass the worker and run @agentmemory/mcp in-process @@ -169,9 +182,14 @@ # 7. Ports # ----------------------------------------------------------------------------- -# III_REST_PORT=3111 # REST API port (also affects viewer at +2) -# III_STREAMS_PORT=3112 # Streams API port -# III_ENGINE_URL=ws://localhost:49134 # iii-engine WebSocket URL (used by the worker) +# III_REST_PORT=3111 # REST API + MCP HTTP anchor +# III_STREAM_PORT=3112 # Preferred streams override; defaults to REST + 1 +# III_STREAMS_PORT=3112 # Legacy alias for III_STREAM_PORT +# III_VIEWER_PORT=3113 # Viewer bind port; defaults to REST + 2 +# III_ENGINE_PORT=49134 # Engine bind port; defaults to REST + 46023 +# III_ENGINE_URL=ws://localhost:49134 # Explicit/remote worker WebSocket URL; takes precedence over derived engine port +# Derivation applies only while the corresponding explicit port/URL is unset. +# `--instance 1` selects 3211/3212/3213/49234 and an instance-1 data subdir. # ----------------------------------------------------------------------------- # 8. iii engine pin diff --git a/CHANGELOG.md b/CHANGELOG.md index 39c98db4ee..93ed95ba8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Fixed +- **Fresh installs are portable and persistent (aligned with upstream #892).** The bundled `iii-config.yaml` uses cwd-relative paths and the engine was spawned without a cwd, so on global and npx installs `./data/state_store.db` and `./data/stream_store` landed in whatever directory the user ran the CLI from, and the `iii-exec` supervision block never resolved — nothing respawned the worker after it died, which surfaced as "all data gone" reports against a live REST port. `agentmemory start` now regenerates a runtime config each boot (`~/.agentmemory/data/iii-config.runtime.yaml`) with absolute data paths and an absolute worker exec line, anchors the engine cwd at `~/.agentmemory`, and copies any legacy `./data` stores from the invocation directory on first run. Repo checkouts keep the cwd config and repo-root cwd, so dev behavior is unchanged; user overrides via env or `~/.agentmemory/iii-config.yaml` are passed through verbatim. The viewer port is now configurable (`III_VIEWER_PORT`, default REST + 2), worker pidfile paths honor `AGENTMEMORY_RUNTIME_DIR`, Docker installs gain uid/gid/chown controls, and `agentmemory remove` covers the generated runtime config. `agentmemory doctor` and startup-failure reporting were expanded accordingly. - **Observation capture no longer grows the persisted live-stream store.** Raw and compressed observation events use `stream::send` instead of `stream::set`; the viewer still receives live updates and REST remains the history source, but `data/stream_store` no longer accumulates a write-only second and third copy of every observation. - **Server-side capture payloads are bounded without flattening structured tool data.** `tool_input`, `tool_output`, prompts and the raw envelope are trimmed to configurable character budgets (`AGENTMEMORY_OBSERVE_*`); object fields such as `file_path` survive truncation, `0` disables an individual bound, and positive budgets below 2 are clamped to the smallest serialized string. - **Graph and semantic provenance are bounded.** Graph nodes, edges, insights and semantic session provenance keep only the freshest source ids instead of growing for the life of the session. diff --git a/INSTALL_FOR_AGENTS.md b/INSTALL_FOR_AGENTS.md index 18c78d3bf8..fd140d21cd 100644 --- a/INSTALL_FOR_AGENTS.md +++ b/INSTALL_FOR_AGENTS.md @@ -4,83 +4,116 @@ You are a coding agent. Follow this runbook top to bottom to install agentmemory ## What you are installing -agentmemory is a local memory server for coding agents. It runs on your machine, exposes a REST API on port 3111 and an MCP server, and stores memories on disk under `~/.agentmemory`. It is built on the iii engine, which agentmemory downloads and manages for you as a pinned binary under `~/.agentmemory/bin`. You do not install iii separately. +agentmemory is a local memory server for coding agents. It exposes a REST API and MCP HTTP routes on port 3111, streams on 3112, a viewer on 3113, and connects its worker to iii-engine on 49134. It is built on the iii engine, which agentmemory manages as a pinned v0.11.2 binary under `~/.agentmemory/bin` on macOS/Linux or `%USERPROFILE%\.agentmemory\bin` on Windows. -Default mode needs no API key and no cloud account. Out of the box it runs hybrid retrieval (BM25 keyword search plus local on-device embeddings), so a full install proves real semantic recall with zero credentials. An LLM provider key is optional and only unlocks richer summaries and auto-injection (see "Optional: richer features"). +Configuration and the managed iii binary live under `~/.agentmemory`. Persistent iii state uses the platform data directory: `~/Library/Application Support/agentmemory` on macOS, `$XDG_DATA_HOME/agentmemory` or `~/.local/share/agentmemory` on Linux, and `%APPDATA%\agentmemory` on Windows. Override it with `--data-dir ` or `AGENTMEMORY_DATA_DIR`; use the same override on every restart. For instance 0, an existing `./data/state_store.db` or `./data/iii-config.yaml` takes precedence over the platform default, while an explicit flag or environment override still wins. Native and Docker starts use the same resolved host data directory. + +Default keyless mode needs no API key or cloud account and disables vector embeddings. `memory_recall` (`mem::search`) uses BM25, while `memory_smart_search` can also fuse structural graph matches when graph data exists. Semantic vector recall is an explicit opt-in: set `EMBEDDING_PROVIDER=local` for on-device `Xenova/all-MiniLM-L6-v2`, or configure a supported remote embedding provider. Local inference is free, but the first embedding request downloads the model and therefore needs network access and extra startup time. An LLM provider is separate; LLM-written observation compression requires both a provider and `AGENTMEMORY_AUTO_COMPRESS=true`. ## Prerequisites -- Node.js >= 20 and npm. Check with `node -v`. -- macOS or Linux for the one-command path. On Windows, use WSL2; native Windows engine setup is manual and `agentmemory connect` is not supported there. +- Node.js >= 20 with npm and npx. Check with `node -v`, `npm -v`, and `npx -v`. +- macOS/Linux automatic iii installation also requires `curl`, a POSIX `sh`, and `tar`. Check with `command -v curl sh tar`. Minimal images such as `node:20-slim` may not include them. +- Windows: WSL2 follows the Linux path above. Native Windows requires a manually downloaded, pinned iii-engine v0.11.2 `iii.exe`, or Docker Desktop; the CLI does not auto-extract the Windows ZIP. Native automated `connect` supports only `copilot-cli`. Other Windows agents need manual MCP configuration; WSL `connect` applies only to agents installed in that same WSL environment. - Ports 3111 (REST), 3112 (streams), 3113 (viewer), and 49134 (engine) free. If any are taken, stop whatever is using them before starting (see Troubleshooting). ## Running non-interactively -Several commands prompt on a TTY (for example the first-run "install globally?" question). As an agent you usually want no prompts. Either set `CI=1` in the environment for the commands below, or rely on the fact that agentmemory skips all prompts automatically when stdin/stdout are not a TTY. Prompts are also never-nag: once answered they persist and are not asked again. Re-run onboarding any time with `agentmemory --reset`. +Use `npx -y @agentmemory/agentmemory@latest` as the canonical command. `-y` accepts npx's package prompt and `@latest` avoids running a stale cached release. Several agentmemory onboarding choices can still prompt on a TTY; set `CI=1` for unattended runs or use non-TTY stdin. Re-run onboarding any time with `npx -y @agentmemory/agentmemory@latest --reset`. -## 1. Install globally +## 1. Verify the runtime prerequisites ```bash -npm install -g @agentmemory/agentmemory +node -v +npm -v +npx -v + +# macOS/Linux only +command -v curl sh tar ``` -`npm install -g` already fetches the latest published release. If you hit `EACCES` on a system Node install (macOS/Linux), retry with `sudo npm install -g @agentmemory/agentmemory`. +Expect: Node prints v20 or newer, npm succeeds, and macOS/Linux prints a path for each automatic-installer dependency. -No-install alternative: skip this step and run the server with `npx -y @agentmemory/agentmemory@latest` everywhere this runbook says `agentmemory`. The `@latest` and `-y` flags matter because npx caches per version and a bare `npx @agentmemory/agentmemory` can serve a stale release. +For native Windows, install the engine before continuing: -Expect: the install completes without errors. +```powershell +# Download the archive that matches your CPU from the pinned release: +# https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.11.2 +# Extract iii.exe to $HOME\.agentmemory\bin\iii.exe, then verify the pin: +& "$HOME\.agentmemory\bin\iii.exe" --version +``` -## 2. Verify the install +Expect: exactly `0.11.2`. Do not install the latest unpinned iii release. Alternatively, use WSL2 or start Docker Desktop and choose the Docker path when agentmemory starts. -```bash -agentmemory --version -``` +## 2. Start the server -Expect: a version string is printed. If `command not found`, the global bin is not on `PATH`; use the `npx -y @agentmemory/agentmemory@latest` form instead. +The canonical command downloads the npm package and, on macOS/Linux, auto-installs its pinned iii-engine into the private agentmemory bin directory. Run it in a dedicated terminal: -## 3. Start the server +```bash +npx -y @agentmemory/agentmemory@latest +``` -The server listens on port 3111 and auto-starts its pinned iii engine on first run (this can take a few seconds the first time while the engine binary is fetched into `~/.agentmemory/bin`). Run it in the background or in a separate terminal so the rest of the runbook can talk to it. +To prefer Docker even when a compatible native binary is present, set `AGENTMEMORY_USE_DOCKER=1` for the same command. To choose persistent storage, pass an absolute path and reuse it on every start: ```bash -agentmemory & +npx -y @agentmemory/agentmemory@latest --data-dir /absolute/path/to/agentmemory-data ``` -Or run `agentmemory` in the foreground in a dedicated terminal. +Docker bind-mounts that same resolved host directory at `/data`. For a second isolated daemon, add `--instance 1`; it stores data and lifecycle metadata under `instance-1` and defaults to ports 3211, 3212, 3213, and 49234. Do not use `--port` alone for concurrent daemons: it changes the ports but keeps instance 0's canonical lifecycle ownership. Use `--instance` for isolation. + +Expect: iii-engine v0.11.2 starts, the agentmemory worker registers with iii, and the ready panel lists REST, viewer, streams, and engine addresses. First boot can take longer while the engine binary is downloaded. + +## 3. Validate the server and all four ports + +| Port | Owner | Validation | +|---|---|---| +| 3111 | agentmemory REST/MCP | `/agentmemory/livez` and `/agentmemory/health` return 200 | +| 3112 | iii streams | listed in the ready panel; must be free at startup | +| 3113 | agentmemory viewer | opening the URL returns the viewer | +| 49134 | iii engine WebSocket | listed in the ready panel and worker registration succeeds | -Wait until it is reachable, then continue: +From a second terminal, run: ```bash curl -fsS http://localhost:3111/agentmemory/livez +curl -fsS http://localhost:3111/agentmemory/health +curl -fsS -o /dev/null http://localhost:3113/ +npx -y @agentmemory/agentmemory@latest status ``` -Expect: a `200` response. Retry for up to ~15 seconds on first run while the engine warms up. +Expect: both REST checks return JSON, the viewer request succeeds, the startup ready panel has listed all four addresses, and `status` reports healthy agentmemory state. In keyless mode, status should report vectors disabled or `bm25-only`; `mem::search` uses BM25, while smart search may also include existing graph matches. Native PowerShell users can use `Invoke-RestMethod` instead of `curl`. -## 4. Prove recall in one command +## 4. Exercise default recall and optional local semantics ```bash -agentmemory demo --serve +npx -y @agentmemory/agentmemory@latest demo ``` -`--serve` boots the server, seeds three realistic sessions (JWT auth, an N+1 query fix, rate limiting), runs semantic searches against them, prints the results, and tears everything down. No second terminal needed. +The demo seeds three realistic sessions and searches them. In default keyless mode, vectors are disabled and the `mem::search` keyword queries such as `jwt auth middleware` and `rate limiting` should return BM25 hits. The deliberately semantic query `database performance optimization` can return zero because no embedding provider is active. Smart search can still add structural graph matches if graph data already exists. + +To opt into on-device semantic recall, add this line to `~/.agentmemory/.env`, restart the server, and rerun the demo: -Expect: the demo finds the "N+1 query fix" memory when it searches for "database performance optimization", which keyword matching alone cannot do. Open `http://localhost:3113` while it runs to watch the memory build live. +```env +EMBEDDING_PROVIDER=local +``` + +The first embedding request downloads `Xenova/all-MiniLM-L6-v2`; wait for that download to finish. Afterward, inference runs locally and the semantic query should find the N+1 memory. Open `http://localhost:3113` to watch the memory build live. ## 5. Wire MCP into the calling agent -Detect which agent is running this runbook, then wire its MCP config: +Detect which agent is running this runbook, then wire its MCP config. On native Windows, run this automated step only for `copilot-cli`: ```bash -agentmemory connect +npx -y @agentmemory/agentmemory@latest connect ``` `connect` merges agentmemory into that agent's MCP config and preserves any existing servers. Supported agent names: `claude-code`, `copilot-cli`, `codex`, `cursor`, `gemini-cli`, `opencode`, `cline`, `continue`, `droid`, `hermes`, `openclaw`, `openhuman`, `pi`, `qwen`, `warp`, `zed`, `antigravity`, `kiro`. -If you cannot tell which agent you are, default to `claude-code`. After wiring, restart the agent or run its MCP reload command (for example `/mcp` in Claude Code) so it picks up the server. +If you cannot tell which agent you are, default to `claude-code` on macOS/Linux. On native Windows, `copilot-cli` is the only automated adapter; configure all other agents with the manual MCP block in the README. Run `connect` inside WSL only when the target agent is installed inside the same WSL environment. After wiring, restart the agent or run its MCP reload command (for example `/mcp` in Claude Code) so it picks up the server. -Expect: the agent now lists agentmemory's tools. With the server running you should see the full set of 54 tools (for example `memory_save`, `memory_smart_search`, `memory_sessions`). If you see only 7 tools, the MCP shim could not reach a server, see Troubleshooting. +Expect: the agent now lists agentmemory's tools. With the server running you should see the full set of 54 tools (for example `memory_save`, `memory_smart_search`, `memory_sessions`). If you see only 7 tools, the MCP shim could not reach the server; see Troubleshooting. ## 6. Install native skills @@ -92,7 +125,7 @@ This installs the native skills so the agent knows when to call the memory tools Expect: the skills are installed for the detected agent. -## 7. Verify a save and recall round-trip +## 7. Verify save, recall, and restart persistence Confirm health first: @@ -118,13 +151,33 @@ Expect: the first call returns `201`, the second returns `200` with results that If `AGENTMEMORY_SECRET` is set in the environment, the REST API requires it. Add `-H "Authorization: Bearer $AGENTMEMORY_SECRET"` to both calls. By default no secret is set and localhost is open. +Now stop and restart the processes, using the same `--data-dir` value if you set one: + +```bash +npx -y @agentmemory/agentmemory@latest stop +npx -y @agentmemory/agentmemory@latest +``` + +After `/agentmemory/livez` returns 200 again, repeat the `smart-search` request above. Expect: the saved probe is still returned. This restart check proves the selected platform data directory is persistent rather than only proving the in-memory index. + +## Optional: install a global command + +The npx form above is the canonical fresh-install path. If you want the shorter `agentmemory` command afterward: + +```bash +npm install -g @agentmemory/agentmemory@latest +agentmemory --version +``` + +If a system Node install returns `EACCES` on macOS/Linux, use a user-owned npm prefix rather than changing repository permissions. The npx form remains available without a global install. + ## Optional: richer features These are off by default because they spend tokens. Enable them only if the user wants them. Put configuration in `~/.agentmemory/.env` (no `export` prefix), then restart the server. - `AGENTMEMORY_INJECT_CONTEXT=true` makes the SessionStart and PreToolUse hooks inject past memory into the agent's context automatically. Cost: spends session tokens proportional to tool-call frequency. -- `AGENTMEMORY_AUTO_COMPRESS=true` sends each observation to your LLM provider for a richer summary. Cost: spends API tokens proportional to tool-use frequency. Requires a provider key. -- Provider key: set one of `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, and similar, in the same file. Without a key, agentmemory stays in zero-LLM mode and still indexes and recalls via BM25 plus local embeddings. +- LLM-written observation compression requires both `AGENTMEMORY_AUTO_COMPRESS=true` and a configured provider. A provider alone leaves synthetic compression active. When both are set, each observation is sent to the provider for a richer summary, spending API tokens proportional to tool-use frequency. +- Provider key: set one of `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, and similar, in the same file. Without a key, agentmemory stays in zero-LLM mode and recalls via BM25. Local embeddings remain available as the separate `EMBEDDING_PROVIDER=local` opt-in above. ## Tool surface @@ -132,33 +185,37 @@ The MCP server exposes 54 tools by default (`--tools all`). Use `--tools core` ( ## Lifecycle commands -- `agentmemory status` shows server and engine state. -- `agentmemory doctor` runs diagnostics and reports what is misconfigured. -- `agentmemory stop` stops the engine this CLI started (`stop --force` bypasses the Docker guard). -- `agentmemory upgrade` upgrades agentmemory and the iii runtime, best effort. -- `agentmemory --reset` wipes onboarding preferences and re-runs the wizard. -- `agentmemory import-jsonl ` imports prior Claude Code session logs as memories. +- `npx -y @agentmemory/agentmemory@latest status` shows server health, viewer, provider, and embedding state. +- `npx -y @agentmemory/agentmemory@latest doctor` runs diagnostics and reports what is misconfigured. +- `npx -y @agentmemory/agentmemory@latest stop` stops the engine this CLI started. Docker mode validates and preserves the exact container and `/data` mount for the next restart; `stop --force` applies only to native ownership checks. +- `npx -y @agentmemory/agentmemory@latest upgrade` upgrades agentmemory and the iii runtime, best effort. +- `npx -y @agentmemory/agentmemory@latest --reset` wipes onboarding preferences and re-runs the wizard. +- `npx -y @agentmemory/agentmemory@latest import-jsonl ` imports prior Claude Code session logs as memories. ## Troubleshooting -- `command not found: agentmemory`: the global bin is not on `PATH`. Use `npx -y @agentmemory/agentmemory@latest`. -- `EACCES` during global install: retry with `sudo`, or use the npx form. -- Stale npx version: run `npx -y @agentmemory/agentmemory@latest`, or clear the cache with `rm -rf ~/.npm/_npx` (macOS/Linux). +- `command not found: agentmemory`: the optional global bin is not on `PATH`. Use `npx -y @agentmemory/agentmemory@latest`. +- Automatic iii install fails on macOS/Linux: verify `command -v curl sh tar`; install the missing prerequisite or use Docker. +- Stale npx version: include both `-y` and `@latest` as shown throughout this runbook. - Port already in use: another process holds 3111, 3112, 3113, or 49134. Stop that process, then re-run. -- Server starts but `livez` never returns 200: re-run with `agentmemory --verbose` to see engine stderr. +- Custom REST port: `--port ` derives streams as `N+1`, viewer as `N+2`, and the iii worker WebSocket as `N+46023` only when their explicit port/URL variables are unset (`III_STREAM_PORT` or legacy `III_STREAMS_PORT`, `III_VIEWER_PORT` or `AGENTMEMORY_VIEWER_URL`, and `III_ENGINE_PORT` or `III_ENGINE_URL`). +- Server starts but `livez` never returns 200: re-run with `npx -y @agentmemory/agentmemory@latest --verbose` to see engine stderr. +- `The engine process started but the REST API never responded.`: inspect all four derived ports, confirm the pinned iii process stayed alive, and use `--verbose` for captured engine stderr. - Engine version warning on start: harmless. agentmemory uses its own pinned engine in `~/.agentmemory/bin` regardless of any `iii` on `PATH`. Set `AGENTMEMORY_III_VERSION` only to override deliberately. -- "engine conflict" / another iii engine already running: if a different iii version is already serving the port (common if you run your own iii), agentmemory will not adopt it and stops with an "engine conflict" note. Stop that engine (`agentmemory stop --force`, or however you started it), then re-run `agentmemory` — it installs and runs the pinned engine in `~/.agentmemory/bin`, leaving your own iii untouched. -- Only 7 tools visible in the agent: the MCP shim is in local fallback because it could not reach a server. Start `npx @agentmemory/agentmemory` and ensure `AGENTMEMORY_URL` points at it (default `http://localhost:3111`), then reload MCP. -- Windows: use WSL2 for the path above. Native Windows runs the server but `connect` and the automated engine install are not supported. +- "engine conflict" / another iii engine already running: if a different iii version is already serving the port, agentmemory will not adopt it. Stop that engine (`npx -y @agentmemory/agentmemory@latest stop --force`, or however you started it), then rerun the canonical command. Its private v0.11.2 binary does not replace a user-managed `iii` on `PATH`. +- Only 7 tools visible in the agent: the MCP shim is in local fallback because it could not reach a server. Start `npx -y @agentmemory/agentmemory@latest`, ensure `AGENTMEMORY_URL` points at it (default `http://localhost:3111`), then reload MCP. +- Native Windows: the CLI does not auto-extract the ZIP. Install the pinned v0.11.2 `iii.exe` manually, use WSL2, or run with Docker Desktop and `AGENTMEMORY_USE_DOCKER=1`. Automated `connect` supports only `copilot-cli`; configure other native agents manually. +- Semantic demo query returns zero hits: this is expected with vectors disabled in default keyless mode. Set `EMBEDDING_PROVIDER=local`, restart, allow the first model download to complete, and rerun the demo. ## Report success Report back to the user: - agentmemory installed, version, and the server running on port 3111 +- iii-engine v0.11.2 running, the selected platform data directory, and all four ports accounted for - which agent was wired via `agentmemory connect`, and the tool count the agent now sees -- the save and recall round-trip returned the probe memory +- the save and recall round-trip returned the probe memory after a full stop/start - the viewer is available at `http://localhost:3113` -- whether any optional features were enabled +- whether vectors were disabled (`mem::search` BM25, with optional graph matches in smart search) or an embedding provider was enabled If any step failed, report which step, the exact command, and the error output. diff --git a/README.md b/README.md index f7a16632ef..9e6b8ee56d 100644 --- a/README.md +++ b/README.md @@ -75,21 +75,33 @@ ## Install -One command: +Requirements: + +- Node.js 20 or newer with npm and npx (`node -v`, `npm -v`, and `npx -v`). +- macOS/Linux automatic iii-engine installation also needs `curl`, a POSIX `sh`, and `tar`. Minimal images such as `node:20-slim` may not include them. +- Native Windows requires the pinned iii-engine v0.11.2 `iii.exe` to be installed manually. WSL2 or Docker Desktop are the other supported paths. + +Canonical fresh-install command: ```bash -npx @agentmemory/agentmemory +npx -y @agentmemory/agentmemory@latest ``` -The first run is an interactive setup: pick the agents to wire (Claude Code, Cursor, Codex, Gemini CLI, OpenCode, ...), pick an LLM provider or stay keyless, and it seeds the config, starts the memory server on `:3111`, and offers to install globally so the bare `agentmemory` command works everywhere afterwards. +The first run is an interactive setup: pick the agents to wire (Claude Code, Cursor, Codex, Gemini CLI, OpenCode, ...), pick an LLM provider or stay keyless, and it seeds the config, starts the memory server and its pinned iii engine, and offers to install globally so the bare `agentmemory` command works everywhere afterward. `-y` accepts npx's package prompt and `@latest` avoids a stale cached release. A provider makes LLM features available, but LLM-written observation compression starts only when `AGENTMEMORY_AUTO_COMPRESS=true` is also set. + +Keyless mode disables vector embeddings. `memory_recall` (the `mem::search` path) uses BM25, while `memory_smart_search` can also fuse structural graph matches when graph data already exists. For free on-device semantic recall, set `EMBEDDING_PROVIDER=local` in `~/.agentmemory/.env` and restart. The first embedding request downloads `Xenova/all-MiniLM-L6-v2`; inference runs locally after that initial model download. + +The local runtime uses four ports: `3111` for REST/MCP HTTP, `3112` for iii streams, `3113` for the viewer, and `49134` for the iii worker WebSocket. Persistent iii state lives in `~/Library/Application Support/agentmemory` on macOS, `$XDG_DATA_HOME/agentmemory` or `~/.local/share/agentmemory` on Linux, and `%APPDATA%\agentmemory` on Windows. Use `--data-dir ` or `AGENTMEMORY_DATA_DIR` to override it, and reuse the same value on every restart. For backward compatibility, an existing `./data/state_store.db` or `./data/iii-config.yaml` takes precedence over the platform default for instance 0; an explicit flag or environment override still wins. Then prove recall works and give your agent its skills: ```bash -agentmemory demo --serve # seed sample sessions + watch recall find them +npx -y @agentmemory/agentmemory@latest demo # seed sample sessions + exercise recall npx skills add rohitg00/agentmemory -y # 17 native skills so your agent knows when to reach for memory ``` +The keyword searches should hit in default keyless mode through BM25. The demo's `database performance optimization` query is intentionally semantic and can return zero until an embedding provider is configured. + Prefer to let a coding agent do the whole thing? Hand it one instruction: > Retrieve and follow the instructions at: https://raw.githubusercontent.com/rohitg00/agentmemory/main/INSTALL_FOR_AGENTS.md @@ -99,7 +111,7 @@ Wire more agents any time with `agentmemory connect ` — 20 adapters lis
Windows -The fast path is WSL2. Native Windows engine setup is manual (about 10 to 20 minutes) and `agentmemory connect` is currently unsupported there. See the [Windows notes](#windows) for the step-by-step. +The fast path is WSL2. Native Windows engine setup requires the pinned v0.11.2 ZIP to be downloaded and `iii.exe` extracted manually; the CLI does not auto-extract it. Docker Desktop is also supported. See the [Windows notes](#windows) for the step-by-step.
@@ -107,11 +119,11 @@ The fast path is WSL2. Native Windows engine setup is manual (about 10 to 20 min Global install / EACCES ```bash -npm install -g @agentmemory/agentmemory -# If you hit EACCES on macOS/Linux system Node installs: -sudo npm install -g @agentmemory/agentmemory +npm install -g @agentmemory/agentmemory@latest ``` +The npx command above remains the canonical fresh-install path and avoids global-prefix permission issues. +
@@ -244,16 +256,18 @@ You explain the same architecture every session. You re-discover the same bugs. **What changes:** Session 1 you set up JWT auth. Session 2 you ask for rate limiting. The agent already knows your auth uses jose middleware in `src/middleware/auth.ts`, your tests cover token validation, and you chose jose over jsonwebtoken for Edge compatibility, with no re-explaining and no copy-pasting. ```bash -npx @agentmemory/agentmemory +npx -y @agentmemory/agentmemory@latest ``` -By default, agentmemory stores iii-engine state outside the repository you start it from: `~/Library/Application Support/agentmemory` on macOS, `$XDG_DATA_HOME/agentmemory` or `~/.local/share/agentmemory` on Linux, and `%APPDATA%\agentmemory` on Windows. To choose a location, pass `--data-dir ` or set `AGENTMEMORY_DATA_DIR`: +By default, agentmemory stores iii-engine state outside the repository you start it from: `~/Library/Application Support/agentmemory` on macOS, `$XDG_DATA_HOME/agentmemory` or `~/.local/share/agentmemory` on Linux, and `%APPDATA%\agentmemory` on Windows. An existing legacy `./data/state_store.db` or `./data/iii-config.yaml` is reused for instance 0 before that platform default. To choose a location explicitly, pass `--data-dir ` or set `AGENTMEMORY_DATA_DIR`; either explicit setting takes precedence over legacy discovery: ```bash -npx @agentmemory/agentmemory --data-dir ~/.agentmemory-projects/main -AGENTMEMORY_DATA_DIR=~/.agentmemory-projects/main npx @agentmemory/agentmemory +npx -y @agentmemory/agentmemory@latest --data-dir ~/.agentmemory-projects/main +AGENTMEMORY_DATA_DIR=~/.agentmemory-projects/main npx -y @agentmemory/agentmemory@latest ``` +Native and Docker launches use this same resolved host directory; Docker bind-mounts it at `/data`. `--instance 1` appends `instance-1` to the resolved directory and selects the separate default port quartet `3211/3212/3213/49234`. + Latest release notes: [CHANGELOG.md](CHANGELOG.md). --- @@ -482,29 +496,54 @@ None of these auto-capture from coding-agent hooks, ship a local-first viewer, o

Quick Start

-Compatibility: this release targets stable `iii-sdk` `^0.11.0` and iii-engine v0.11.x. +Compatibility: this release targets `iii-sdk` 0.11.2 and pins iii-engine v0.11.2. ### Try it in 30 seconds ```bash # Terminal 1: start the server -npx @agentmemory/agentmemory +npx -y @agentmemory/agentmemory@latest # Terminal 2: seed sample data and see recall in action -npx @agentmemory/agentmemory demo +npx -y @agentmemory/agentmemory@latest demo ``` -`demo` seeds 3 realistic sessions (JWT auth, N+1 query fix, rate limiting) and runs semantic searches against them. You'll see it find "N+1 query fix" when you search "database performance optimization", which keyword matching cannot do. +`demo` seeds 3 realistic sessions (JWT auth, N+1 query fix, rate limiting) and runs searches against them. Keyless installs disable vectors, so the `mem::search` keyword queries should hit through BM25 while `database performance optimization` can return zero. `smart-search` may additionally return structural graph matches when graph data exists. To make the semantic query find the N+1 fix through vectors, set `EMBEDDING_PROVIDER=local`, restart, and allow the first model download to finish. Open `http://localhost:3113` to watch the memory build live. +### Validate a fresh install and restart persistence + +With the server running, validate REST, health, the viewer, and the iii-backed runtime status: + +```bash +curl -fsS http://localhost:3111/agentmemory/livez +curl -fsS http://localhost:3111/agentmemory/health +curl -fsS -o /dev/null http://localhost:3113/ +npx -y @agentmemory/agentmemory@latest status +``` + +The startup ready panel accounts for all four ports: REST/MCP HTTP on 3111, iii streams on 3112, the viewer on 3113, and the iii worker WebSocket on 49134. `status` confirms agentmemory health and the active provider/embedding mode. Save a probe and confirm it is searchable: + +```bash +curl -fsS -X POST http://localhost:3111/agentmemory/remember \ + -H 'Content-Type: application/json' \ + -d '{"content":"agentmemory restart persistence probe","concepts":["install-check"]}' + +curl -fsS -X POST http://localhost:3111/agentmemory/smart-search \ + -H 'Content-Type: application/json' \ + -d '{"query":"restart persistence probe","limit":5}' +``` + +Then run `npx -y @agentmemory/agentmemory@latest stop`, start the canonical command again in Terminal 1, wait for `/agentmemory/livez`, and repeat the search. The probe must still be returned. If you selected a custom `--data-dir`, pass the same directory on the restart. + ### Everyday commands Install and setup live in [Install](#install) above (the first run walks you through it). Day to day: ```bash agentmemory # start the server -agentmemory stop # tear it down +agentmemory stop # stop it cleanly agentmemory connect # wire another agent agentmemory doctor # interactive diagnostics + fix prompts agentmemory remove # uninstall everything we created @@ -518,10 +557,10 @@ To bring in older Claude Code JSONL transcripts: ```bash # Import everything under the default ~/.claude/projects -npx @agentmemory/agentmemory import-jsonl +npx -y @agentmemory/agentmemory@latest import-jsonl # Or import a single file -npx @agentmemory/agentmemory import-jsonl ~/.claude/projects/-my-project/abc123.jsonl +npx -y @agentmemory/agentmemory@latest import-jsonl ~/.claude/projects/-my-project/abc123.jsonl ``` Imported sessions show up in the Replay picker alongside native ones. Under the hood each entry routes through the `mem::replay::load`, `mem::replay::sessions`, and `mem::replay::import-jsonl` iii functions, with no side-channel servers. Each imported transcript is indexed for search, stamped with origin channel `import`, and mined for a session crystal and lessons. @@ -533,7 +572,7 @@ Imported sessions show up in the Replay picker alongside native ones. Under the Use the maintenance command when you intentionally want to update your local runtime: ```bash -npx @agentmemory/agentmemory upgrade +npx -y @agentmemory/agentmemory@latest upgrade ``` Warning: this command mutates the current workspace/runtime. It can update JavaScript dependencies and pull the pinned `iiidev/iii:0.11.2` Docker image. It never installs an unpinned or newer iii engine. @@ -543,7 +582,7 @@ Implementation details live in `src/cli.ts` (see `runUpgrade` around the `src/cl ### Claude Code (one block, paste it) ```text -Install agentmemory: run `npx @agentmemory/agentmemory` in a separate terminal to start the memory server. Then run `/plugin marketplace add rohitg00/agentmemory` and `/plugin install agentmemory` — the plugin registers all 12 hooks, 17 skills, AND auto-wires the `@agentmemory/mcp` stdio server via its `.mcp.json`, so you get 54 MCP tools (memory_smart_search, memory_save, memory_sessions, memory_governance_delete, etc.) without any extra config step. Verify with `curl http://localhost:3111/agentmemory/health`. The real-time viewer is at http://localhost:3113. +Install agentmemory: run `npx -y @agentmemory/agentmemory@latest` in a separate terminal to start the memory server and its pinned iii engine. Then run `/plugin marketplace add rohitg00/agentmemory` and `/plugin install agentmemory` — the plugin registers all 12 hooks, 17 skills, AND auto-wires the `@agentmemory/mcp` stdio server via its `.mcp.json`, so you get 54 MCP tools (memory_smart_search, memory_save, memory_sessions, memory_governance_delete, etc.) without any extra config step. Verify with `curl http://localhost:3111/agentmemory/health`. The real-time viewer is at http://localhost:3113. Keyless mode disables vectors: `memory_recall` uses BM25, and `memory_smart_search` can also use existing structural graph data. Set `EMBEDDING_PROVIDER=local` in `~/.agentmemory/.env` and restart to opt into on-device semantic recall. ``` #### Claude Code without the plugin install (MCP-standalone path) @@ -563,7 +602,7 @@ For remote or protected deployments, launch Claude Code with `AGENTMEMORY_URL` a ```bash # 1. start the memory server in a separate terminal -npx @agentmemory/agentmemory +npx -y @agentmemory/agentmemory@latest # 2. register the agentmemory marketplace and install the plugin codex plugin marketplace add rohitg00/agentmemory @@ -600,13 +639,13 @@ agentmemory connect copilot-cli copilot plugin install rohitg00/agentmemory:plugin ``` -`agentmemory connect copilot-cli` merges `mcpServers.agentmemory` into `~/.copilot/mcp-config.json` (or `$COPILOT_HOME/mcp-config.json` when `COPILOT_HOME` is set) and preserves existing servers. This adapter is Windows-safe even though other `connect` adapters still require manual Windows setup. Copilot picks up the MCP server on next launch or after `/mcp`. Install the plugin as well when you want the full hook/skill experience. +`agentmemory connect copilot-cli` merges `mcpServers.agentmemory` into `~/.copilot/mcp-config.json` (or `$COPILOT_HOME/mcp-config.json` when `COPILOT_HOME` is set) and preserves existing servers. On native Windows this is the only automated `connect` adapter; configure every other native Windows agent manually. WSL `connect` is supported only when the target agent is installed in that same WSL environment. Copilot picks up the MCP server on next launch or after `/mcp`. Install the plugin as well when you want the full hook/skill experience.
OpenClaw (paste this prompt) ```text -Install agentmemory for OpenClaw. Run `npx @agentmemory/agentmemory` in a separate terminal to start the memory server on localhost:3111. Then add this to my OpenClaw MCP config so agentmemory is available with all 54 memory tools: +Install agentmemory for OpenClaw. Run `npx -y @agentmemory/agentmemory@latest` in a separate terminal to start the memory server on localhost:3111. Then add this to my OpenClaw MCP config so agentmemory is available with all 54 memory tools: { "mcpServers": { @@ -631,7 +670,7 @@ Full guide: [`integrations/openclaw/`](integrations/openclaw/) Hermes Agent (paste this prompt) ```text -Install agentmemory for Hermes. Run `npx @agentmemory/agentmemory` in a separate terminal to start the memory server on localhost:3111. Then add this to ~/.hermes/config.yaml so Hermes can use agentmemory as an MCP server with all 54 memory tools: +Install agentmemory for Hermes. Run `npx -y @agentmemory/agentmemory@latest` in a separate terminal to start the memory server on localhost:3111. Then add this to ~/.hermes/config.yaml so Hermes can use agentmemory as an MCP server with all 54 memory tools: mcp_servers: agentmemory: @@ -650,7 +689,7 @@ Full guide: [`integrations/hermes/`](integrations/hermes/) ### Other agents -Start the memory server: `npx @agentmemory/agentmemory` +Start the memory server: `npx -y @agentmemory/agentmemory@latest` #### Native skills via `npx skills add` (50+ agents) @@ -752,7 +791,7 @@ git clone https://github.com/rohitg00/agentmemory.git && cd agentmemory npm install && npm run build && npm start ``` -This starts agentmemory with a local `iii-engine` if `iii` is already installed, or falls back to Docker Compose if Docker is available. REST, streams, and the viewer bind to `127.0.0.1` by default. +This starts agentmemory with a local `iii-engine` if the pinned binary is already installed, or uses Docker Compose when selected. REST, streams, and the viewer bind to `127.0.0.1` by default. The automatic macOS/Linux binary path requires `curl`, a POSIX `sh`, and `tar`. Install `iii-engine` manually. **agentmemory currently pins `iii-engine` to `v0.11.2`**. `v0.11.6` introduces a new sandbox-everything-via-`iii worker add` model that agentmemory hasn't been refactored for yet. Pin lifts once the refactor lands. Override with `AGENTMEMORY_III_VERSION=` if you've migrated to the sandbox model manually. @@ -760,13 +799,15 @@ Install `iii-engine` manually. **agentmemory currently pins `iii-engine` to `v0. - **macOS x64:** swap `aarch64-apple-darwin` for `x86_64-apple-darwin` - **Linux x64:** swap for `x86_64-unknown-linux-gnu` - **Linux arm64:** swap for `aarch64-unknown-linux-gnu` -- **Windows:** download `iii-x86_64-pc-windows-msvc.zip` from [iii-hq/iii releases v0.11.2](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.11.2), extract `iii.exe`, add to PATH +- **Windows:** download `iii-x86_64-pc-windows-msvc.zip` from [iii-hq/iii releases v0.11.2](https://github.com/iii-hq/iii/releases/tag/iii%2Fv0.11.2) and extract `iii.exe` to `%USERPROFILE%\.agentmemory\bin\iii.exe` Or use Docker (the bundled `docker-compose.yml` pulls `iiidev/iii:0.11.2`). Full docs: [iii.dev/docs](https://iii.dev/docs). ### Windows -agentmemory runs on Windows 10/11, but the Node.js package alone isn't enough; you also need the `iii-engine` runtime (a separate native binary) as a background process. The official upstream installer is a `sh` script and there is no PowerShell installer or scoop/winget package today, so Windows users have two paths: +agentmemory runs on Windows 10/11, but the Node.js package alone isn't enough; you also need the pinned iii-engine v0.11.2 runtime as a background process. The CLI does not auto-extract the Windows ZIP, so native Windows users must install `iii.exe` manually, use WSL2, or choose Docker Desktop. + +Native Windows automated MCP wiring supports only `agentmemory connect copilot-cli`. For Claude Code, Codex, Cursor, and every other native Windows agent, copy the manual MCP block from [Other agents](#other-agents) into that agent's Windows config. Running `connect` in WSL is appropriate only when the target agent is also installed in the same WSL environment; it does not edit a Windows-host agent's configuration. **Option A: prebuilt Windows binary (recommended)** @@ -776,15 +817,15 @@ agentmemory runs on Windows 10/11, but the Node.js package alone isn't enough; y # model that engine v0.11.6+ requires) # 2. Download iii-x86_64-pc-windows-msvc.zip # (or iii-aarch64-pc-windows-msvc.zip if you're on an ARM machine) -# 3. Extract iii.exe somewhere on PATH, or place it at: -# %USERPROFILE%\.local\bin\iii.exe -# (agentmemory checks that location automatically) +# 3. Extract iii.exe to agentmemory's private engine directory: +New-Item -ItemType Directory -Force "$HOME\.agentmemory\bin" +# Copy iii.exe to $HOME\.agentmemory\bin\iii.exe # 4. Verify: -iii --version +& "$HOME\.agentmemory\bin\iii.exe" --version # Should print: 0.11.2 # 5. Then run agentmemory as usual: -npx -y @agentmemory/agentmemory +npx -y @agentmemory/agentmemory@latest ``` **Option B: Docker Desktop** @@ -792,28 +833,29 @@ npx -y @agentmemory/agentmemory ```powershell # 1. Install Docker Desktop for Windows # 2. Start Docker Desktop and make sure the engine is running -# 3. Run agentmemory — it will auto-start the bundled compose file: -npx -y @agentmemory/agentmemory +# 3. Select Docker explicitly and run agentmemory: +$env:AGENTMEMORY_USE_DOCKER = "1" +npx -y @agentmemory/agentmemory@latest ``` **Option C: standalone MCP only (no engine).** If you only need the MCP tools for your agent and don't need the REST API, viewer, or cron jobs, skip the engine entirely: ```powershell -npx -y @agentmemory/agentmemory mcp +npx -y @agentmemory/agentmemory@latest mcp # or via the shim package: npx -y @agentmemory/mcp ``` -**Diagnostics for Windows:** if `npx @agentmemory/agentmemory` fails, re-run with `--verbose` to see the actual engine stderr. Common failure modes: +**Diagnostics for Windows:** if `npx -y @agentmemory/agentmemory@latest` fails, re-run it with `--verbose` to see the actual engine stderr. Common failure modes: | Symptom | Fix | |---|---| -| `iii-engine process started` then `did not become ready within 15s` | Engine crashed on startup; re-run with `--verbose`, check stderr | +| `The engine process started but the REST API never responded.` | Confirm all four derived ports are free, verify the pinned `iii.exe` stayed alive, then re-run with `--verbose` and inspect the captured engine stderr | | `Could not start iii-engine` | Neither `iii.exe` nor Docker is installed. See Option A or B above | | Port conflict | `netstat -ano \| findstr :3111` to see what's bound, then kill it or use `--port ` | | Docker fallback skipped even though Docker is installed | Make sure Docker Desktop is actually running (system tray icon) | -> Note: the iii **engine** is a prebuilt binary, not a cargo crate, so don't try to `cargo install` it. (The iii **SDKs** are published on crates.io, npm, and PyPI, but agentmemory doesn't need them.) Supported engine install methods, all pinned to v0.11.2: the prebuilt v0.11.2 binary above, the upstream sh install script **with the version pin** `curl -fsSL https://install.iii.dev/iii/main/install.sh | VERSION=0.11.2 sh` (macOS/Linux), and the Docker image `iiidev/iii:0.11.2`. A bare `install.sh | sh` installs the **latest** engine, which agentmemory does not support; always pass `VERSION=0.11.2`. Easiest of all: just run `npx @agentmemory/agentmemory`, which fetches the pinned engine into `~/.agentmemory/bin` for you. +> Note: the iii **engine** is a prebuilt binary, not a cargo crate, so don't try to `cargo install` it. (The iii **SDKs** are published on crates.io, npm, and PyPI, but agentmemory doesn't need them.) Supported engine install methods are all pinned to v0.11.2: the prebuilt binary above, agentmemory's macOS/Linux auto-install path (`curl`, POSIX `sh`, and `tar` required), and the Docker image `iiidev/iii:0.11.2`. A bare upstream `install.sh | sh` installs the latest engine, which agentmemory does not support. Use `npx -y @agentmemory/agentmemory@latest`; on macOS/Linux it fetches the pinned engine into `~/.agentmemory/bin`. --- @@ -897,9 +939,10 @@ PostToolUse hook fires -> SHA-256 dedup (5min window) -> Privacy filter (strip secrets, API keys) -> Store raw observation - -> LLM compress -> structured facts + concepts + narrative - -> Vector embedding (6 providers + local) - -> Index in BM25 + vector + -> Synthetic compression by default + (LLM-written compression only with a provider + AGENTMEMORY_AUTO_COMPRESS=true) + -> Vector embedding when an embedding provider is active + -> Index in BM25, plus vectors when enabled Stop / SessionEnd hook fires -> Summarize session @@ -974,21 +1017,23 @@ Triple-stream retrieval combining three signals: Fused with Reciprocal Rank Fusion (RRF, k=60) and session-diversified (max 3 results per session). -Hybrid ranking applies to the primary recall path, not just `smart-search`: `mem::search` (behind `memory_recall`) ranks through the same BM25 + vector + graph fusion once the vector index is populated. Lesson recall runs on a dedicated in-memory BM25 index instead of scanning the whole corpus per query. Superseded memory versions are excluded from every recall path; the version chain keeps their history. +When a vector index is populated, `mem::search` (behind `memory_recall`) uses the hybrid BM25 + vector ranker. Without embeddings it uses BM25. `smart-search` can additionally fuse structural graph matches when graph data exists, including in keyless mode. Lesson recall runs on a dedicated in-memory BM25 index instead of scanning the whole corpus per query. Superseded memory versions are excluded from every recall path; the version chain keeps their history. BM25 tokenizes Greek, Cyrillic, Hebrew, Arabic, and accented Latin out of the box. For Chinese / Japanese / Korean memories, install the optional segmenters (`npm install @node-rs/jieba tiny-segmenter`) to split CJK runs into word-level tokens; without them, agentmemory soft-falls to whole-run tokenization and prints a one-time hint on stderr. ### Embedding providers -agentmemory auto-detects your provider. For best results, install local embeddings (free): +Keyless installs disable vector embeddings: `mem::search` uses BM25, while `smart-search` can also use existing structural graph data. To opt into free on-device semantic embeddings, add this to `~/.agentmemory/.env` and restart agentmemory: -```bash -npm install @huggingface/transformers +```env +EMBEDDING_PROVIDER=local ``` +The normal npm install includes the optional `@huggingface/transformers` runtime. The first embedding request downloads `Xenova/all-MiniLM-L6-v2`, so it needs network access and can take longer; subsequent inference runs on-device. Remote providers are auto-detected from their keys unless `EMBEDDING_PROVIDER` overrides them. + | Provider | Model | Cost | Notes | |---|---|---|---| -| **Local (recommended)** | `all-MiniLM-L6-v2` | Free | Offline, +8pp recall over BM25-only | +| **Local (recommended opt-in)** | `all-MiniLM-L6-v2` | Free | On-device after the first model download, +8pp recall over BM25-only | | Gemini | `gemini-embedding-001` | Free tier | 100+ languages, 768/1536/3072 dims (MRL), 2048-token input. Replaces `text-embedding-004` ([deprecated, shutdown Jan 14, 2026](https://ai.google.dev/gemini-api/docs/deprecations)) | | OpenAI | `text-embedding-3-small` | $0.02/1M | Highest quality | | Voyage AI | `voyage-code-3` | Paid | Optimized for code | @@ -1001,7 +1046,7 @@ npm install @huggingface/transformers 54 tools, 6 resources, 3 prompts, and 17 skills. -> **MCP shim vs full server:** the published `@agentmemory/mcp` package is a thin shim. It exposes the full 54-tool surface **only when it can reach a running agentmemory server** via `AGENTMEMORY_URL` (proxy mode). With no server reachable, the shim falls back to a 7-tool local set (`memory_save`, `memory_recall`, `memory_smart_search`, `memory_sessions`, `memory_export`, `memory_audit`, `memory_governance_delete`). The `AGENTMEMORY_TOOLS=core|all` env var is a *server-side* flag; setting it in the shim's `env` block has no effect. If you see only 7 tools in Cursor / OpenCode / Gemini CLI, start `npx @agentmemory/agentmemory` (or the Docker stack) and set `AGENTMEMORY_URL=http://localhost:3111`. +> **MCP shim vs full server:** the published `@agentmemory/mcp` package is a thin shim. It exposes the full 54-tool surface **only when it can reach a running agentmemory server** via `AGENTMEMORY_URL` (proxy mode). With no server reachable, the shim falls back to a 7-tool local set (`memory_save`, `memory_recall`, `memory_smart_search`, `memory_sessions`, `memory_export`, `memory_audit`, `memory_governance_delete`). The `AGENTMEMORY_TOOLS=core|all` env var is a *server-side* flag; setting it in the shim's `env` block has no effect. If you see only 7 tools in Cursor / OpenCode / Gemini CLI, start `npx -y @agentmemory/agentmemory@latest` (or the Docker stack) and set `AGENTMEMORY_URL=http://localhost:3111`. ### 54 Tools @@ -1093,7 +1138,7 @@ The table shows the four core skills. The full set is 9 invocable skills plus 8 Run without the full server, for any MCP client. Either of these works: ```bash -npx -y @agentmemory/agentmemory mcp # canonical (always available) +npx -y @agentmemory/agentmemory@latest mcp # canonical (always available) npx -y @agentmemory/mcp # shim package alias ``` @@ -1268,11 +1313,11 @@ Full registry: [workers.iii.dev](https://workers.iii.dev). Every worker there co ### LLM Providers -agentmemory auto-detects from your environment. By default, no LLM calls are made unless you configure a provider or explicitly opt in to the Claude subscription fallback. +agentmemory auto-detects providers from your environment. A provider makes LLM-backed operations available, but provider configuration alone does not enable LLM-written observation compression. That path requires both a provider and `AGENTMEMORY_AUTO_COMPRESS=true`. | Provider | Config | Notes | |----------|--------|-------| -| **No-op (default)** | No config needed | LLM-backed compress/summarize is DISABLED. Synthetic BM25 compression + recall still work. See `AGENTMEMORY_ALLOW_AGENT_SDK` below if you used to rely on the Claude-subscription fallback. | +| **No-op (default)** | No config needed | LLM-backed compress/summarize is disabled. Synthetic compression and BM25 recall still work. See `AGENTMEMORY_ALLOW_AGENT_SDK` below if you used to rely on the Claude-subscription fallback. | | Anthropic API | `ANTHROPIC_API_KEY` | Per-token billing | | MiniMax | `MINIMAX_API_KEY` | Anthropic-compatible | | Gemini | `GEMINI_API_KEY` | Also enables embeddings | @@ -1326,11 +1371,11 @@ Qwen 3 models think by default and can burn the whole token budget on reasoning Reasoning-class models (`o1`-style with `` blocks) can return empty `content` with a `reasoning` field your local server may not surface. If extractions come back blank, switch to a non-reasoning model first. The `OPENAI_REASONING_EFFORT=none` env can also disable thinking on Ollama Cloud thinking models that mirror the OpenAI reasoning schema. -Local embeddings ship out of the box via `@huggingface/transformers`: `EMBEDDING_PROVIDER=local` (default) gives you `Xenova/all-MiniLM-L6-v2` (384-dim) entirely on-device. No extra config needed. +Local embeddings ship as an optional dependency but are not enabled by default. Set `EMBEDDING_PROVIDER=local` to opt into `Xenova/all-MiniLM-L6-v2` (384-dim). The first embedding request downloads the model; inference is on-device afterward. Without that setting or a remote embedding key, vectors stay disabled, `mem::search` uses BM25, and `smart-search` can still add existing graph matches. ### Cost-aware model selection -Background compression runs on every observation, so model choice meaningfully changes monthly spend. Captured workload data: 635 requests / 888K tokens / 35 hours of active use, run against three OpenRouter models at 2026-05-23 pricing. +When LLM-written background compression is enabled with both a provider and `AGENTMEMORY_AUTO_COMPRESS=true`, it runs on every observation, so model choice meaningfully changes monthly spend. Captured workload data: 635 requests / 888K tokens / 35 hours of active use, run against three OpenRouter models at 2026-05-23 pricing. | Tier | Model | Input / 1M | Output / 1M | Cost for the captured 35h | Notes | |------|-------|------------|-------------|---------------------------|-------| @@ -1382,9 +1427,11 @@ agentmemory + iii-engine bind four ports by default. If a restart fails with `po | Port | Process | Purpose | Env override | |------|---------|---------|--------------| | `3111` | agentmemory | REST API + MCP HTTP + `/agentmemory/health` + `/agentmemory/livez` | `III_REST_PORT` | -| `3112` | iii-engine | Internal streams worker (consumed by agentmemory + viewer) | `III_STREAMS_PORT` | -| `3113` | agentmemory | Real-time viewer (`http://localhost:3113`) | `AGENTMEMORY_VIEWER_PORT` | -| `49134` | iii-engine | WebSocket; workers register here, OTel telemetry flows over it | `III_ENGINE_URL` (full URL, default `ws://localhost:49134`) | +| `3112` | iii-engine | Internal streams worker (consumed by agentmemory + viewer) | `III_STREAM_PORT` (preferred) or legacy `III_STREAMS_PORT` | +| `3113` | agentmemory | Real-time viewer (`http://localhost:3113`) | `III_VIEWER_PORT` or `AGENTMEMORY_VIEWER_URL` for the reported URL | +| `49134` | iii-engine | WebSocket; workers register here, OTel telemetry flows over it | `III_ENGINE_PORT` or `III_ENGINE_URL` | + +`--port ` changes the REST anchor and derives streams `N+1`, viewer `N+2`, and engine WebSocket `N+46023` only where the corresponding explicit port or URL above is unset. It does not create an isolated lifecycle namespace. Use `--instance 1` for a second daemon; it uses anchor 3211, defaults to `3211/3212/3213/49234`, and receives a separate `instance-1` data and lifecycle directory. Instances 1 through 50 follow the same pattern. Stale-process cleanup when ports stay bound after a crashed run: @@ -1399,7 +1446,7 @@ netstat -ano | findstr ":3111 :3112 :3113 :49134" taskkill /F /PID ``` -`agentmemory stop` reaps both the worker and the engine pidfile cleanly on graceful shutdown. In Docker mode it tears down only agentmemory's own compose services and reaps the native worker before the Docker teardown; the CLI also refuses to adopt or signal Docker or VM port holders (Docker backend, vpnkit, colima) as the native engine unless `--force` is passed. The manual cleanup above is only for the post-crash case where neither pidfile is left behind. +`agentmemory stop` reaps both the worker and the engine pidfile cleanly on graceful native shutdown. In Docker mode it flushes the native worker, stops the exact validated engine container, and preserves both the container and its `/data` mount for a lossless restart; the next start validates and resumes that same container. Docker-backed uninstall requires `agentmemory remove --keep-data`: it removes shared agentmemory-managed files while preserving the validated container, its data mount, and the lifecycle record needed to recover them. Destructive Docker data deletion is intentionally left to the operator after a backup. The CLI also refuses to adopt or signal Docker or VM port holders (Docker backend, vpnkit, colima) as the native engine unless `--force` is passed. The manual cleanup above is only for the post-crash case where neither pidfile is left behind. ### Config File @@ -1421,6 +1468,8 @@ AGENTMEMORY_ALLOW_AGENT_SDK=true AGENTMEMORY_AUTO_COMPRESS=true ``` +LLM-written observation compression requires both lines: access to an LLM provider (including this explicit subscription fallback) and `AGENTMEMORY_AUTO_COMPRESS=true`. A provider by itself leaves the default synthetic compression path in place. + Consolidation (graph nodes, lessons, crystals) is on by default whenever an LLM provider is configured. Explicitly opt out with `CONSOLIDATION_ENABLED=false` if you want LLM-free operation. Graph extraction is a separate flag: ```env @@ -1467,7 +1516,7 @@ Create `~/.agentmemory/.env`: # leave OFF unless you understand the Stop-hook recursion risk: # AGENTMEMORY_ALLOW_AGENT_SDK=true -# Embedding provider (auto-detected, or override) +# Embedding provider (BM25-only when unset; local is an explicit opt-in) # EMBEDDING_PROVIDER=local # VOYAGE_API_KEY=... # OPENAI_API_KEY=sk-... @@ -1498,7 +1547,8 @@ Create `~/.agentmemory/.env`: # III_REST_PORT=3111 # Features -# AGENTMEMORY_AUTO_COMPRESS=false # OFF by default. When on, +# AGENTMEMORY_AUTO_COMPRESS=false # OFF by default. Requires an LLM + # provider as well. When both are on, # every PostToolUse hook calls your # LLM provider to compress the # observation — expect significant @@ -1597,7 +1647,7 @@ npm test # 1,674 tests npm run test:integration # API tests (requires running services) ``` -**Prerequisites:** Node.js >= 20, [iii-engine](https://iii.dev/docs) or Docker +**Prerequisites:** Node.js >= 20 with npm/npx; [iii-engine](https://iii.dev/docs) v0.11.2 or Docker. The macOS/Linux automatic engine install also requires `curl`, a POSIX `sh`, and `tar`; native Windows uses the manual pinned `iii.exe`, WSL2, or Docker Desktop.

License

diff --git a/docker-compose.yml b/docker-compose.yml index a74cd82a1a..66abc86966 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,14 +8,18 @@ services: iii-init: image: busybox:1.36 user: "0:0" + environment: + AGENTMEMORY_DOCKER_UID: "${AGENTMEMORY_DOCKER_UID:-65532}" + AGENTMEMORY_DOCKER_GID: "${AGENTMEMORY_DOCKER_GID:-65532}" + AGENTMEMORY_DOCKER_SKIP_CHOWN: "${AGENTMEMORY_DOCKER_SKIP_CHOWN:-0}" volumes: - "${AGENTMEMORY_DATA_DIR:-iii-data}:/data" - entrypoint: ["sh", "-c", "chown -R 65532:65532 /data && chmod 755 /data"] + entrypoint: ["sh", "-c", "if [ \"$${AGENTMEMORY_DOCKER_SKIP_CHOWN}\" = \"1\" ]; then exit 0; fi; chown -R $${AGENTMEMORY_DOCKER_UID}:$${AGENTMEMORY_DOCKER_GID} /data && chmod 755 /data"] restart: "no" iii-engine: # Pinned to v0.11.2 — the last engine that runs agentmemory's current - # worker model cleanly. v0.11.6 introduces a new sandbox-everything- + # direct-registration worker cleanly. v0.11.6 introduces a new sandbox-everything- # via-`iii worker add` model that agentmemory hasn't been refactored # for yet; the architectural mismatch surfaces as EPIPE reconnect # loops and empty search after save. Bump only after agentmemory is @@ -24,15 +28,20 @@ services: # Override per-shell or via .env file: # AGENTMEMORY_III_VERSION=0.11.7 docker compose up image: iiidev/iii:${AGENTMEMORY_III_VERSION:-0.11.2} - user: "65532:65532" + user: "${AGENTMEMORY_DOCKER_UID:-65532}:${AGENTMEMORY_DOCKER_GID:-65532}" depends_on: iii-init: condition: service_completed_successfully + environment: + III_REST_PORT: "${III_REST_PORT:-3111}" + III_STREAM_PORT: "${III_STREAM_PORT:-3112}" + III_VIEWER_PORT: "${III_VIEWER_PORT:-3113}" + III_ENGINE_PORT: "${III_ENGINE_PORT:-49134}" ports: - - "127.0.0.1:49134:49134" - - "127.0.0.1:3111:3111" - - "127.0.0.1:3112:3112" - - "127.0.0.1:9464:9464" + - "127.0.0.1:${III_ENGINE_PORT:-49134}:${III_ENGINE_PORT:-49134}" + - "127.0.0.1:${III_REST_PORT:-3111}:${III_REST_PORT:-3111}" + - "127.0.0.1:${III_STREAM_PORT:-3112}:${III_STREAM_PORT:-3112}" + - "127.0.0.1:${AGENTMEMORY_METRICS_PORT:-9464}:9464" volumes: - "${AGENTMEMORY_DATA_DIR:-iii-data}:/data" - ./iii-config.docker.yaml:/app/config.yaml:ro diff --git a/iii-config.docker.yaml b/iii-config.docker.yaml index 775b523628..eb12c2913a 100644 --- a/iii-config.docker.yaml +++ b/iii-config.docker.yaml @@ -1,12 +1,16 @@ workers: - name: iii-http config: - port: 3111 + port: ${III_REST_PORT:3111} host: 0.0.0.0 default_timeout: 180000 cors: - allowed_origins: ["http://localhost:3111", "http://localhost:3113", "http://127.0.0.1:3111", "http://127.0.0.1:3113"] + allowed_origins: ["http://localhost:${III_REST_PORT:3111}", "http://localhost:${III_VIEWER_PORT:3113}", "http://127.0.0.1:${III_REST_PORT:3111}", "http://127.0.0.1:${III_VIEWER_PORT:3113}"] allowed_methods: [GET, POST, PUT, DELETE, OPTIONS] + - name: iii-worker-manager + config: + port: ${III_ENGINE_PORT:49134} + host: 0.0.0.0 - name: iii-state config: adapter: @@ -28,7 +32,7 @@ workers: name: kv - name: iii-stream config: - port: 3112 + port: ${III_STREAM_PORT:3112} host: 0.0.0.0 adapter: name: kv @@ -45,9 +49,3 @@ workers: metrics_enabled: true logs_enabled: true logs_console_output: false - - name: iii-exec - config: - watch: - - src/**/*.ts - exec: - - node dist/index.mjs diff --git a/plugin/skills/agentmemory-config/REFERENCE.md b/plugin/skills/agentmemory-config/REFERENCE.md index dee70cfd6f..4f81730c8b 100644 --- a/plugin/skills/agentmemory-config/REFERENCE.md +++ b/plugin/skills/agentmemory-config/REFERENCE.md @@ -3,7 +3,7 @@ Generated by scanning `src/` for `AGENTMEMORY_*` usage. Do not edit the block below by hand; run `npm run skills:gen` after adding or removing a variable. Internal markers ending in two underscores are excluded. -Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 55 recognized variables: +Configuration is read from the environment and from `~/.agentmemory/.env` (no `export` prefix). 60 recognized variables: - `AGENTMEMORY_AGENT_SCOPE` - `AGENTMEMORY_ALLOW_AGENT_SDK` @@ -20,6 +20,9 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_CWD` - `AGENTMEMORY_DATA_DIR` - `AGENTMEMORY_DEBUG` +- `AGENTMEMORY_DOCKER_GID` +- `AGENTMEMORY_DOCKER_SKIP_CHOWN` +- `AGENTMEMORY_DOCKER_UID` - `AGENTMEMORY_DROP_STALE_INDEX` - `AGENTMEMORY_EVICTION_ENABLED` - `AGENTMEMORY_EVICTION_INTERVAL_MS` @@ -38,6 +41,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_LLM_NOTHINK` - `AGENTMEMORY_LLM_TIMEOUT_MS` - `AGENTMEMORY_MCP_BLOCK` +- `AGENTMEMORY_METRICS_PORT` - `AGENTMEMORY_OBSERVE_PROMPT_CHARS` - `AGENTMEMORY_OBSERVE_RAW_CHARS` - `AGENTMEMORY_OBSERVE_TOOL_INPUT_CHARS` @@ -46,6 +50,7 @@ Configuration is read from the environment and from `~/.agentmemory/.env` (no `e - `AGENTMEMORY_PROJECT_NAME` - `AGENTMEMORY_PROVIDER` - `AGENTMEMORY_REFLECT` +- `AGENTMEMORY_RUNTIME_DIR` - `AGENTMEMORY_SDK_CHILD` - `AGENTMEMORY_SECRET` - `AGENTMEMORY_SESSION_ID` diff --git a/src/cli.ts b/src/cli.ts index 19fde0e660..e4c67c2c56 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -7,6 +7,7 @@ import { type ChildProcess, } from "node:child_process"; import { + cpSync, existsSync, mkdirSync, readdirSync, @@ -17,7 +18,7 @@ import { unlinkSync, writeFileSync, } from "node:fs"; -import { join, dirname, delimiter as PATH_DELIMITER } from "node:path"; +import { join, dirname, resolve, delimiter as PATH_DELIMITER } from "node:path"; import { fileURLToPath } from "node:url"; import { homedir, platform } from "node:os"; import * as p from "@clack/prompts"; @@ -54,6 +55,19 @@ import { type ConnectManifest, type RemoveOptions, } from "./cli/remove-plan.js"; +import { + dockerComposeArgs, + dockerProjectName, + isBundledConfig, + legacyDataMigrations, + resolveEngineCwd, + rewriteBundledConfig, + runtimeConfigPath, +} from "./cli/engine-launch.js"; +import { runtimeMetadataPath } from "./runtime-paths.js"; +import { createStartupStderrCapture } from "./cli/startup-stderr.js"; +import { renderEngineConfig } from "./cli/engine-config.js"; +import { processStatIsRunning } from "./cli/process-state.js"; import { renderSplash } from "./cli/splash.js"; import { isFirstRun, readPrefs, resetPrefs, writePrefs } from "./cli/preferences.js"; import { runOnboarding } from "./cli/onboarding.js"; @@ -102,7 +116,7 @@ if (args.includes("--version") || args.includes("-V")) { // script tracks `latest`, which made every fresh agentmemory install pull // engine 0.11.6 — and 0.11.6 introduces a new sandbox-everything-via- // `iii worker add` worker model that agentmemory hasn't been refactored -// for yet (we still use the old `iii-exec watch` config-file model). The +// for yet (the CLI still registers its worker directly through the SDK). The // architectural mismatch surfaces as EPIPE reconnect loops and empty // search results after save. Pin to v0.11.2 — the last engine that runs // agentmemory's current worker model cleanly — until the refactor lands. @@ -242,9 +256,21 @@ if (toolsIdx !== -1 && args[toolsIdx + 1]) { process.env["AGENTMEMORY_TOOLS"] = toolsMode; } -const portIdx = args.indexOf("--port"); -if (portIdx !== -1 && args[portIdx + 1]) { - process.env["III_REST_PORT"] = args[portIdx + 1]; +const URL_CLIENT_COMMANDS = new Set(["status", "doctor", "mcp"]); +let hasExplicitLocalPortOverride = false; +let selectedInstance = 0; + +function cliArgValue(name: string): string | undefined { + const inline = args.find((arg) => arg.startsWith(`${name}=`)); + if (inline) return inline.slice(name.length + 1); + const index = args.indexOf(name); + return index === -1 ? undefined : args[index + 1]; +} + +const portValue = cliArgValue("--port"); +if (portValue) { + process.env["III_REST_PORT"] = portValue; + hasExplicitLocalPortOverride = true; } // `--instance N` picks a 100-port block off the 3111 base so multiple @@ -253,26 +279,63 @@ if (portIdx !== -1 && args[portIdx + 1]) { // quartet; `--instance 1` → 3211/3212/3213/49234; etc. REST acts as the // anchor — streams/viewer/engine derive from it via fixed offsets below // unless an env explicitly pins each one. -const instanceIdx = args.indexOf("--instance"); -if (instanceIdx !== -1 && args[instanceIdx + 1]) { - const n = parseInt(args[instanceIdx + 1] || "", 10); - if (Number.isFinite(n) && n >= 0 && n <= 50) { - const base = 3111 + n * 100; - if (!process.env["III_REST_PORT"]) { - process.env["III_REST_PORT"] = String(base); - } +const instanceValue = cliArgValue("--instance"); +const hasInstanceArgument = args.some( + (arg) => arg === "--instance" || arg.startsWith("--instance="), +); +if (hasInstanceArgument) { + if (instanceValue === undefined || !/^\d+$/.test(instanceValue)) { + p.log.error("--instance must be an integer between 0 and 50."); + process.exit(1); } + const n = Number(instanceValue); + if (!Number.isInteger(n) || n > 50) { + p.log.error("--instance must be an integer between 0 and 50."); + process.exit(1); + } + selectedInstance = n; + const base = 3111 + n * 100; + if (!hasExplicitLocalPortOverride) { + process.env["III_REST_PORT"] = String(base); + } + hasExplicitLocalPortOverride = true; } -const dataDirResolution = resolveDataDir({ args }); -if (dataDirResolution.source !== "default") { - process.env["AGENTMEMORY_DATA_DIR"] = dataDirResolution.dataDir; +const restPort = parseInt(process.env["III_REST_PORT"] || "3111", 10); +if (Number.isFinite(restPort) && restPort > 0) { + process.env["III_STREAM_PORT"] ??= + process.env["III_STREAMS_PORT"] ?? String(restPort + 1); + process.env["III_VIEWER_PORT"] ??= String(restPort + 2); + const configuredEngineUrl = process.env["III_ENGINE_URL"]; + if (configuredEngineUrl) { + try { + const parsedEnginePort = new URL(configuredEngineUrl).port; + if (parsedEnginePort) { + process.env["III_ENGINE_PORT"] = parsedEnginePort; + } + } catch {} + } else { + process.env["III_ENGINE_PORT"] ??= String(restPort + 46023); + } + process.env["AGENTMEMORY_METRICS_PORT"] ??= String(restPort + 6353); } +const dataDirResolution = resolveDataDir({ args }); +process.env["AGENTMEMORY_DATA_DIR"] = dataDirResolution.dataDir; +process.env["AGENTMEMORY_RUNTIME_DIR"] = selectedInstance > 0 + ? dataDirResolution.dataDir + : join(homedir(), ".agentmemory"); + const skipEngine = args.includes("--no-engine"); +function shouldUseAgentmemoryUrl(): boolean { + return !hasExplicitLocalPortOverride && URL_CLIENT_COMMANDS.has(args[0] ?? ""); +} + function getRestPort(): number { - const url = process.env["AGENTMEMORY_URL"]; + const url = shouldUseAgentmemoryUrl() + ? process.env["AGENTMEMORY_URL"] + : undefined; if (url) { try { const parsed = new URL(url).port; @@ -283,11 +346,20 @@ function getRestPort(): number { } function getBaseUrl(): string { - const url = process.env["AGENTMEMORY_URL"]; + const url = shouldUseAgentmemoryUrl() + ? process.env["AGENTMEMORY_URL"] + : undefined; if (url) return url.replace(/\/+$/, ""); return `http://localhost:${getRestPort()}`; } +function getConfiguredViewerPort(): number { + return ( + parseInt(process.env["III_VIEWER_PORT"] || "", 10) || + getRestPort() + 2 + ); +} + let discoveredViewerPort: number | null = null; export async function discoverViewerPort(): Promise { @@ -320,14 +392,10 @@ function getViewerUrl(): string { try { const u = new URL(getBaseUrl()); - const vPort = - parseInt(process.env["III_VIEWER_PORT"] || "", 10) || - (parseInt(u.port || "3111", 10) || 3111) + 2; + const vPort = getConfiguredViewerPort(); return `${u.protocol}//${u.hostname}:${vPort}`; } catch { - const vPort = - parseInt(process.env["III_VIEWER_PORT"] || "", 10) || - getRestPort() + 2; + const vPort = getConfiguredViewerPort(); return `http://localhost:${vPort}`; } } @@ -352,8 +420,6 @@ function getStreamPort(): number { // second-instance collision. Overridable via // `III_ENGINE_PORT` or the legacy `III_ENGINE_URL=ws://host:port`. function getEnginePort(): number { - const explicit = parseInt(process.env["III_ENGINE_PORT"] || "", 10); - if (explicit) return explicit; const url = process.env["III_ENGINE_URL"]; if (url) { try { @@ -361,6 +427,8 @@ function getEnginePort(): number { if (parsed) return parseInt(parsed, 10); } catch {} } + const explicit = parseInt(process.env["III_ENGINE_PORT"] || "", 10); + if (explicit) return explicit; return getRestPort() + 46023; } @@ -412,37 +480,11 @@ function findIiiConfig(): string { join(__dirname, "..", "iii-config.yaml"), ]; for (const c of candidates) { - if (existsSync(c)) return c; + if (existsSync(c)) return resolve(c); } return ""; } -function yamlSingleQuoted(value: string): string { - return `'${value.replace(/'/g, "''")}'`; -} - -function renderIiiConfig(template: string, dataDir: string): string { - return template - .replace( - "file_path: ./data/state_store.db", - `file_path: ${yamlSingleQuoted(join(dataDir, "state_store.db"))}`, - ) - .replace( - "file_path: ./data/stream_store", - `file_path: ${yamlSingleQuoted(join(dataDir, "stream_store"))}`, - ); -} - -function writeRuntimeIiiConfig(configPath: string, dataDir: string): string { - mkdirSync(dataDir, { recursive: true }); - const runtimeConfigPath = join(dataDir, "iii-config.yaml"); - writeFileSync( - runtimeConfigPath, - renderIiiConfig(readFileSync(configPath, "utf8"), dataDir), - ); - return runtimeConfigPath; -} - function warnIfRelocatedDataDir(): void { if (!dataDirResolution.relocatedFrom) return; @@ -568,16 +610,36 @@ function resolveCompatibleIii(iiiBinPath: string | null | undefined): string | n } function enginePidfilePath(): string { - return join(homedir(), ".agentmemory", "iii.pid"); + return runtimeMetadataPath("iii.pid"); } function engineStatePath(): string { - return join(homedir(), ".agentmemory", "engine-state.json"); + return runtimeMetadataPath("engine-state.json"); } -type EngineState = - | { kind: "native"; configPath: string; attached?: boolean; binPath?: string } - | { kind: "docker"; composeFile: string }; +type NativeEngineState = { + kind: "native"; + configPath: string; + attached?: boolean; + binPath?: string; + restPort?: number; +}; + +type DockerEngineState = { + kind: "docker"; + schemaVersion?: 2; + composeFile: string; + projectName?: string; + engineVersion?: string; + restPort?: number; + dataDir?: string; + containerId?: string; + dataMountType?: "bind" | "volume"; + dataMountSource?: string; + preserveContainer?: boolean; +}; + +type EngineState = NativeEngineState | DockerEngineState; function writeEnginePidfile(pid: number): void { try { @@ -605,15 +667,8 @@ function clearEnginePidfile(): void { } catch {} } -// Worker pidfile: the agentmemory worker process -// (`node dist/index.mjs`) is spawned by iii-exec inside the engine. When -// `agentmemory stop` kills only the engine pid, the worker can survive -// (detached spawn, signal not propagated, or kept alive by a wrapper -// script). On the next start, the orphaned worker reconnects to the new -// engine and shows up as a duplicate registration. We write the worker -// pid from src/index.ts on boot so stop can find and reap it. function workerPidfilePath(): string { - return join(homedir(), ".agentmemory", "worker.pid"); + return runtimeMetadataPath("worker.pid"); } function readWorkerPidfile(): number | null { @@ -636,7 +691,14 @@ function writeEngineState(state: EngineState): void { try { const statePath = engineStatePath(); mkdirSync(dirname(statePath), { recursive: true }); - writeFileSync(statePath, `${JSON.stringify(state)}\n`, { encoding: "utf-8" }); + const normalized = state.kind === "docker" && state.schemaVersion === 2 + ? { + ...state, + restPort: state.restPort ?? getRestPort(), + dataDir: state.dataDir ?? dataDirResolution.dataDir, + } + : { ...state, restPort: state.restPort ?? getRestPort() }; + writeFileSync(statePath, `${JSON.stringify(normalized)}\n`, { encoding: "utf-8" }); } catch (err) { vlog(`writeEngineState: ${err instanceof Error ? err.message : String(err)}`); } @@ -661,6 +723,55 @@ function clearEngineState(): void { } catch {} } +function engineStateRestPort(state: EngineState): number { + if (state.restPort && Number.isFinite(state.restPort)) return state.restPort; + if (state.kind === "docker") { + const projectPort = state.projectName?.match(/^agentmemory-(\d+)$/)?.[1]; + if (projectPort) return parseInt(projectPort, 10); + return 3111; + } + try { + const raw = readFileSync(state.configPath, "utf-8"); + const httpBlock = raw.match( + /- name:\s*iii-http([\s\S]*?)(?=\n\s*- name:|$)/, + )?.[1]; + const configuredPort = httpBlock?.match(/\n\s*port:\s*(\d+)/)?.[1]; + if (configuredPort) return parseInt(configuredPort, 10); + } catch {} + return 3111; +} + +async function startWorkerForEngineState(): Promise { + const workerPid = readWorkerPidfile(); + if (workerPid && pidAlive(workerPid)) return; + if (configuredEngineMayStartWorker() && await waitForConfiguredWorker(5000)) { + return; + } + await import("./index.js"); +} + +function configuredEngineMayStartWorker(): boolean { + const state = readEngineState(); + if (state?.kind !== "native") return false; + try { + const raw = readFileSync(state.configPath, "utf-8"); + return raw.includes("- name: iii-exec"); + } catch { + return false; + } +} + +async function waitForConfiguredWorker(timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const workerPid = readWorkerPidfile(); + if (workerPid && pidAlive(workerPid)) return true; + if (await isAgentmemoryReady()) return true; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + return false; +} + function discoverComposeFile(): string | null { const candidates = [ join(__dirname, "..", "docker-compose.yml"), @@ -670,6 +781,401 @@ function discoverComposeFile(): string | null { return candidates.find((c) => existsSync(c)) ?? null; } +type DockerEngineInspection = + | { + status: "running" | "stopped"; + composeFile: string; + projectName: string; + containerId: string; + dataMountType: "bind" | "volume"; + dataMountSource: string; + preserveContainer: boolean; + } + | { status: "missing"; composeFile: string; projectName?: string } + | { status: "unavailable"; reason: string }; + +type DockerInspectRecord = { + Id?: string; + State?: { Running?: boolean }; + Config?: { Image?: string; Labels?: Record }; + HostConfig?: { + PortBindings?: Record< + string, + Array<{ HostIp?: string; HostPort?: string }> | null + >; + }; + Mounts?: Array<{ + Type?: string; + Name?: string; + Source?: string; + Destination?: string; + }>; +}; + +function sameDockerMountSource( + type: "bind" | "volume", + left: string, + right: string, +): boolean { + return type === "bind" ? resolve(left) === resolve(right) : left === right; +} + +function inspectDockerContainer( + dockerBin: string, + containerId: string, + state: DockerEngineState, + ownerPort: number, +): DockerEngineInspection { + const result = spawnSync(dockerBin, ["inspect", containerId], { + encoding: "utf-8", + stdio: ["ignore", "pipe", "ignore"], + maxBuffer: 1024 * 1024, + }); + if (result.status !== 0) { + return { status: "unavailable", reason: `docker inspect failed for ${containerId}` }; + } + let record: DockerInspectRecord; + try { + record = (JSON.parse(result.stdout) as DockerInspectRecord[])[0] ?? {}; + } catch { + return { status: "unavailable", reason: `docker inspect returned invalid JSON for ${containerId}` }; + } + + const image = record.Config?.Image ?? ""; + const expectedImage = `iiidev/iii:${state.engineVersion ?? IIPINNED_VERSION}`; + if (image !== expectedImage) { + return { + status: "unavailable", + reason: `saved container uses ${image || "an unknown image"}, expected ${expectedImage}`, + }; + } + + const labels = record.Config?.Labels ?? {}; + const projectName = labels["com.docker.compose.project"] ?? ""; + if (labels["com.docker.compose.service"] !== "iii-engine" || !projectName) { + return { + status: "unavailable", + reason: "saved container is not the iii-engine service of a Compose project", + }; + } + if (state.projectName && state.projectName !== projectName) { + return { + status: "unavailable", + reason: `saved Docker project ${state.projectName} does not match container project ${projectName}`, + }; + } + if (state.schemaVersion === 2 && projectName !== dockerProjectName(ownerPort)) { + return { + status: "unavailable", + reason: `Docker project ${projectName} does not match REST port ${ownerPort}`, + }; + } + + const restBindings = record.HostConfig?.PortBindings?.[`${ownerPort}/tcp`] ?? []; + if (!restBindings.some((binding) => binding.HostPort === String(ownerPort))) { + return { + status: "unavailable", + reason: `saved container does not publish REST port ${ownerPort}`, + }; + } + + const dataMount = record.Mounts?.find((mount) => mount.Destination === "/data"); + const dataMountType = dataMount?.Type; + const dataMountSource = dataMountType === "volume" + ? dataMount?.Name ?? dataMount?.Source ?? "" + : dataMount?.Source ?? ""; + if ( + (dataMountType !== "bind" && dataMountType !== "volume") || + !dataMountSource + ) { + return { + status: "unavailable", + reason: "saved container has no verifiable /data bind or volume mount", + }; + } + if ( + state.dataMountType && + state.dataMountType !== dataMountType + ) { + return { + status: "unavailable", + reason: `saved /data mount type ${state.dataMountType} changed to ${dataMountType}`, + }; + } + if ( + state.dataMountSource && + !sameDockerMountSource(dataMountType, state.dataMountSource, dataMountSource) + ) { + return { + status: "unavailable", + reason: "saved container /data mount source changed", + }; + } + if (!state.dataMountSource) { + if ( + dataMountType === "bind" && + resolve(dataMountSource) !== resolve(dataDirResolution.dataDir) + ) { + return { + status: "unavailable", + reason: `container /data bind ${dataMountSource} does not match ${dataDirResolution.dataDir}`, + }; + } + if (dataMountType === "volume" && dataDirResolution.source !== "default") { + return { + status: "unavailable", + reason: "legacy container uses a named volume but this invocation selects an explicit data directory", + }; + } + } + if ( + state.dataDir && + resolve(state.dataDir) !== resolve(dataDirResolution.dataDir) + ) { + return { + status: "unavailable", + reason: `saved data directory ${state.dataDir} does not match ${dataDirResolution.dataDir}`, + }; + } + + return { + status: record.State?.Running ? "running" : "stopped", + composeFile: existsSync(state.composeFile) + ? state.composeFile + : discoverComposeFile() ?? state.composeFile, + projectName, + containerId: record.Id ?? containerId, + dataMountType, + dataMountSource, + preserveContainer: + state.preserveContainer === true || + (state.schemaVersion !== 2 && dataMountType === "volume"), + }; +} + +function inspectOwnedDockerEngine(state: EngineState): DockerEngineInspection { + if (state.kind !== "docker") { + return { status: "unavailable", reason: "engine state is not Docker" }; + } + const ownerPort = engineStateRestPort(state); + if (state.schemaVersion === 2 && state.projectName !== dockerProjectName(ownerPort)) { + return { + status: "unavailable", + reason: `saved Docker project ${state.projectName} does not match REST port ${ownerPort}`, + }; + } + + const dockerBin = whichBinary("docker"); + if (!dockerBin) { + return { status: "unavailable", reason: "docker is not on PATH" }; + } + + const discoveryFailures: string[] = []; + if (state.containerId) { + const direct = inspectDockerContainer(dockerBin, state.containerId, state, ownerPort); + if (direct.status !== "unavailable") return direct; + if (state.preserveContainer) return direct; + discoveryFailures.push(direct.reason); + } + + const composeFile = existsSync(state.composeFile) + ? state.composeFile + : discoverComposeFile() ?? state.composeFile; + const candidateIds = new Set(); + try { + if (existsSync(composeFile)) { + const composeText = readFileSync(composeFile, "utf-8"); + if (!/^\s+iii-engine:/m.test(composeText)) { + return { + status: "unavailable", + reason: `${composeFile} does not define iii-engine`, + }; + } + const psResult = spawnSync( + dockerBin, + dockerComposeArgs(composeFile, state.projectName, [ + "ps", + "-q", + "--all", + "iii-engine", + ]), + { encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"] }, + ); + if (psResult.status === 0) { + for (const id of (psResult.stdout ?? "").trim().split(/\s+/)) { + if (id) candidateIds.add(id); + } + } else { + discoveryFailures.push( + `docker compose ps failed for project ${state.projectName ?? ""}`, + ); + } + } + + const scanArgs = [ + "ps", + "-aq", + "--filter", + "label=com.docker.compose.service=iii-engine", + "--filter", + `ancestor=iiidev/iii:${state.engineVersion ?? IIPINNED_VERSION}`, + ...(state.projectName + ? ["--filter", `label=com.docker.compose.project=${state.projectName}`] + : []), + ]; + const scanResult = spawnSync( + dockerBin, + scanArgs, + { encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"] }, + ); + if (scanResult.status === 0) { + for (const id of (scanResult.stdout ?? "").trim().split(/\s+/)) { + if (id) candidateIds.add(id); + } + } else { + discoveryFailures.push("docker container scan failed"); + } + + const matches: DockerEngineInspection[] = []; + for (const containerId of candidateIds) { + const inspected = inspectDockerContainer(dockerBin, containerId, state, ownerPort); + if (inspected.status === "running" || inspected.status === "stopped") { + if ( + !matches.some( + (match) => + (match.status === "running" || match.status === "stopped") && + match.containerId === inspected.containerId, + ) + ) { + matches.push(inspected); + } + } else if (inspected.status === "unavailable") { + discoveryFailures.push(inspected.reason); + } + } + if (matches.length === 1) return matches[0]!; + if (matches.length > 1) { + return { + status: "unavailable", + reason: `multiple Docker iii-engine containers match REST port ${ownerPort}`, + }; + } + if (discoveryFailures.length > 0) { + return { + status: "unavailable", + reason: [...new Set(discoveryFailures)].join("; "), + }; + } + if (state.schemaVersion !== 2 || state.preserveContainer) { + return { + status: "unavailable", + reason: `legacy Docker ownership exists for REST port ${ownerPort}, but no unique container can be verified`, + }; + } + return { + status: "missing", + composeFile, + projectName: state.projectName, + }; + } catch (err) { + return { + status: "unavailable", + reason: err instanceof Error ? err.message : String(err), + }; + } +} + +function persistDockerInspection( + state: DockerEngineState, + inspection: DockerEngineInspection, +): DockerEngineState { + if (inspection.status !== "running" && inspection.status !== "stopped") { + return state; + } + const resolved: DockerEngineState = { + ...state, + composeFile: inspection.composeFile, + projectName: inspection.projectName, + containerId: inspection.containerId, + dataMountType: inspection.dataMountType, + dataMountSource: inspection.dataMountSource, + preserveContainer: inspection.preserveContainer, + }; + writeEngineState(resolved); + return resolved; +} + +async function assertRuntimePortOwnership(): Promise { + const state = readEngineState(); + if (!state) return; + const ownerPort = engineStateRestPort(state); + const requestedPort = getRestPort(); + if (ownerPort === requestedPort) return; + + if (state.kind === "docker") { + const inspection = inspectOwnedDockerEngine(state); + if (inspection.status === "unavailable") { + p.log.error( + `The data directory contains Docker ownership for REST port ${ownerPort}, but it cannot be verified: ${inspection.reason}. Refusing to overwrite its lifecycle state.`, + ); + process.exit(1); + } + if (inspection.status === "running" || inspection.status === "stopped") { + persistDockerInspection(state, inspection); + p.log.error( + `The canonical lifecycle state is owned by a ${inspection.status} Docker engine on REST port ${ownerPort}. Refusing to reuse it for port ${requestedPort}. Use the original port, or use --instance for an isolated daemon.`, + ); + process.exit(1); + } + p.log.error( + `The canonical lifecycle state still records a Docker engine for REST port ${ownerPort}, but its container is missing. Refusing to overwrite that ownership from port ${requestedPort}. Re-run \`agentmemory stop\` with the original port to reconcile it.`, + ); + process.exit(1); + } + + const enginePid = readEnginePidfile(); + const workerPid = readWorkerPidfile(); + let active = Boolean( + (enginePid && + pidAlive(enginePid) && + !isForeignPortHolder(pidCommand(enginePid))) || + (workerPid && pidAlive(workerPid)), + ); + if (!active) { + try { + await fetch(`http://127.0.0.1:${ownerPort}/`, { + signal: AbortSignal.timeout(750), + }); + active = true; + } catch {} + } + if (active) { + p.log.error( + `The canonical lifecycle state is owned by a live agentmemory engine on REST port ${ownerPort}. Refusing to reuse it for port ${requestedPort}. Use the original port, or use --instance for an isolated daemon.`, + ); + process.exit(1); + } + clearEnginePidfile(); + clearWorkerPidfile(); + clearEngineState(); +} + +function configureDockerHostUser(): void { + if (typeof process.getuid !== "function" || typeof process.getgid !== "function") { + return; + } + const hostUid = String(process.getuid()); + const hostGid = String(process.getgid()); + process.env["AGENTMEMORY_DOCKER_UID"] ??= hostUid; + process.env["AGENTMEMORY_DOCKER_GID"] ??= hostGid; + if ( + process.env["AGENTMEMORY_DOCKER_UID"] === hostUid && + process.env["AGENTMEMORY_DOCKER_GID"] === hostGid + ) { + process.env["AGENTMEMORY_DOCKER_SKIP_CHOWN"] ??= "1"; + } +} + function isInvokedViaNpx(): boolean { if (process.env["npm_lifecycle_event"] === "npx") return true; const argv1 = process.argv[1] ?? ""; @@ -884,8 +1390,12 @@ async function runIiiInstaller(): Promise<{ ok: boolean; binPath: string | null const shBin = whichBinary("sh"); const curlBin = whichBinary("curl"); - if (!shBin || !curlBin) { - p.log.warn("curl or sh not found. Cannot auto-install iii-engine."); + const tarBin = whichBinary("tar"); + if (!shBin || !curlBin || !tarBin) { + const missing = [!shBin && "sh", !curlBin && "curl", !tarBin && "tar"] + .filter(Boolean) + .join(", "); + p.log.warn(`${missing} not found. Cannot auto-install iii-engine.`); return { ok: false, binPath: null }; } @@ -916,6 +1426,40 @@ type StartupFailure = { }; let startupFailure: StartupFailure | null = null; +let activeStartupStderr = createStartupStderrCapture(); + +function printCapturedStartupStderr(): void { + const stderr = activeStartupStderr.text().trim(); + if (IS_VERBOSE && stderr) { + p.note(stderr, "engine stderr"); + } +} + +function printDockerStartupLogs(): void { + const state = readEngineState(); + if (state?.kind !== "docker") return; + const inspection = inspectOwnedDockerEngine(state); + if (inspection.status !== "running" && inspection.status !== "stopped") return; + const dockerBin = whichBinary("docker"); + if (!dockerBin) return; + const result = spawnSync( + dockerBin, + [ + "logs", + "--tail", + "80", + inspection.containerId, + ], + { + encoding: "utf-8", + stdio: ["ignore", "pipe", "pipe"], + timeout: 5000, + maxBuffer: 64 * 1024, + }, + ); + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`.trim(); + if (output) p.note(output.slice(-16 * 1024), "docker engine logs"); +} // Spawn a background engine and collect any startup stderr for a short // window. The process is unref'd so the CLI parent can exit cleanly; we @@ -925,31 +1469,28 @@ function spawnEngineBackground( bin: string, spawnArgs: string[], label: string, + cwd?: string, ): ChildProcess { - vlog(`spawn: ${bin} ${spawnArgs.join(" ")}`); + vlog(`spawn: ${bin} ${spawnArgs.join(" ")}${cwd ? ` (cwd: ${cwd})` : ""}`); + activeStartupStderr = createStartupStderrCapture(); const child = spawn(bin, spawnArgs, { detached: true, stdio: ["ignore", "ignore", "pipe"], windowsHide: true, + ...(cwd ? { cwd } : {}), }); const isDocker = label.includes("Docker"); if (!isDocker && typeof child.pid === "number") { writeEnginePidfile(child.pid); } - const stderrChunks: Buffer[] = []; - let stderrBytes = 0; - const MAX_STDERR_CAPTURE = 16 * 1024; child.stderr?.on("data", (chunk: Buffer) => { - if (stderrBytes >= MAX_STDERR_CAPTURE) return; - const slice = chunk.subarray(0, MAX_STDERR_CAPTURE - stderrBytes); - stderrChunks.push(slice); - stderrBytes += slice.length; + activeStartupStderr.append(chunk); }); child.on("exit", (code, signal) => { const abnormal = (code !== null && code !== 0) || (code === null && signal !== null); if (abnormal) { - const stderr = Buffer.concat(stderrChunks).toString("utf-8"); + const stderr = activeStartupStderr.text(); startupFailure = { kind: isDocker ? "docker-crashed" : "engine-crashed", stderr: @@ -963,19 +1504,93 @@ function spawnEngineBackground( if (IS_VERBOSE && stderr.trim()) { p.log.error(`engine stderr:\n${stderr}`); } - if (!isDocker) clearEnginePidfile(); - clearEngineState(); + if (!isDocker) { + clearEnginePidfile(); + clearEngineState(); + } } }); child.unref(); return child; } +function prepareEngineLaunch(configPath: string): { + configPath: string; + cwd: string; +} { + const home = homedir(); + const bundledConfig = isBundledConfig(configPath, __dirname); + const cwd = resolveEngineCwd( + configPath, + process.cwd(), + home, + bundledConfig, + ); + try { + mkdirSync(cwd, { recursive: true }); + } catch { + return { configPath, cwd: process.cwd() }; + } + try { + const rawConfig = readFileSync(configPath, "utf-8"); + const options = { + dataDir: dataDirResolution.dataDir, + ports: { + restPort: getRestPort(), + streamPort: getStreamPort(), + viewerPort: getConfiguredViewerPort(), + enginePort: getEnginePort(), + }, + }; + const rewritten = bundledConfig + ? rewriteBundledConfig( + rawConfig, + home, + process.execPath, + join(__dirname, "index.mjs"), + options, + ) + : renderEngineConfig(rawConfig, options); + const runtimePath = runtimeConfigPath(dataDirResolution.dataDir); + mkdirSync(dirname(runtimePath), { recursive: true }); + writeFileSync(runtimePath, rewritten, "utf-8"); + if (selectedInstance === 0 && dataDirResolution.source === "default") { + for (const m of legacyDataMigrations( + process.cwd(), + home, + dataDirResolution.dataDir, + )) { + if (existsSync(m.from) && !existsSync(m.to)) { + try { + mkdirSync(dirname(m.to), { recursive: true }); + cpSync(m.from, m.to, { recursive: true }); + p.log.info(`Copied existing data: ${m.from} -> ${m.to}`); + } catch (err) { + vlog(`data copy failed for ${m.from}: ${String(err)}`); + } + } + } + } + return { + configPath: runtimePath, + cwd, + }; + } catch (err) { + vlog(`runtime config generation failed, using bundled config verbatim: ${String(err)}`); + return { configPath, cwd }; + } +} + function startIiiBin(iiiBin: string, configPath: string): boolean { const s = p.spinner(); s.start(`Starting iii-engine: ${iiiBin}`); - writeEngineState({ kind: "native", configPath, binPath: iiiBin }); - spawnEngineBackground(iiiBin, ["--config", configPath], "iii-engine"); + const launch = prepareEngineLaunch(configPath); + writeEngineState({ + kind: "native", + configPath: launch.configPath, + binPath: iiiBin, + }); + spawnEngineBackground(iiiBin, ["--config", launch.configPath], "iii-engine", launch.cwd); s.stop(c.ok("iii-engine process started")); return true; } @@ -995,13 +1610,44 @@ function pickCompatibleIii(candidates: Array): string } async function startEngine(): Promise { + await assertRuntimePortOwnership(); + const persistedState = readEngineState(); + if (persistedState?.kind === "docker") { + const inspection = inspectOwnedDockerEngine(persistedState); + if (inspection.status === "unavailable" || inspection.status === "missing") { + const detail = inspection.status === "unavailable" + ? inspection.reason + : "container is missing"; + p.log.error( + `A saved Docker engine owns this lifecycle state, but it cannot be resumed safely (${detail}). Run \`agentmemory stop\` to reconcile that project before starting another engine.`, + ); + return false; + } + persistDockerInspection(persistedState, inspection); + if (inspection.status === "running") return true; + const dockerBin = whichBinary("docker"); + return Boolean( + dockerBin && + runCommand(dockerBin, ["start", inspection.containerId], { + label: `Restarting owned Docker container ${inspection.containerId}`, + }), + ); + } const configPath = findIiiConfig(); warnIfRelocatedDataDir(); - const runtimeConfigPath = configPath - ? writeRuntimeIiiConfig(configPath, dataDirResolution.dataDir) - : ""; const pathIii = whichBinary("iii"); - vlog(`iii binary: ${pathIii ?? "(not on PATH)"}, config: ${runtimeConfigPath || "(not found)"}`); + vlog(`iii binary: ${pathIii ?? "(not on PATH)"}, config: ${configPath || "(not found)"}`); + + const dockerBin = whichBinary("docker"); + const dockerComposeCandidates = [ + join(__dirname, "..", "docker-compose.yml"), + join(__dirname, "docker-compose.yml"), + join(process.cwd(), "docker-compose.yml"), + ]; + const composeFile = dockerComposeCandidates.find((c) => existsSync(c)); + const dockerOptIn = + process.env["AGENTMEMORY_USE_DOCKER"] === "1" || + process.env["AGENTMEMORY_USE_DOCKER"] === "true"; const fallbacks = fallbackIiiPaths().filter((p) => existsSync(p)); for (const f of fallbacks) { @@ -1011,12 +1657,12 @@ async function startEngine(): Promise { let iiiBin = pickCompatibleIii([pathIii, ...fallbacks]); - if (iiiBin && runtimeConfigPath) { + if (iiiBin && configPath && !(dockerOptIn && dockerBin && composeFile)) { if (iiiBin !== pathIii) { p.log.info(`Using iii at: ${c.dim(iiiBin)} (v${c.accent(IIPINNED_VERSION)})`); process.env["PATH"] = `${dirname(iiiBin)}${PATH_DELIMITER}${process.env["PATH"] ?? ""}`; } - return startIiiBin(iiiBin, runtimeConfigPath); + return startIiiBin(iiiBin, configPath); } if (pathIii && !iiiBin) { @@ -1032,20 +1678,9 @@ async function startEngine(): Promise { return false; } - const dockerBin = whichBinary("docker"); vlog(`docker binary: ${dockerBin ?? "(not on PATH)"}`); mkdirSync(dataDirResolution.dataDir, { recursive: true }); - const dockerComposeCandidates = [ - join(__dirname, "..", "docker-compose.yml"), - join(__dirname, "docker-compose.yml"), - join(process.cwd(), "docker-compose.yml"), - ]; - const composeFile = dockerComposeCandidates.find((c) => existsSync(c)); vlog(`docker-compose.yml: ${composeFile ?? "(not found)"}`); - - const dockerOptIn = - process.env["AGENTMEMORY_USE_DOCKER"] === "1" || - process.env["AGENTMEMORY_USE_DOCKER"] === "true"; const interactive = !!process.stdin.isTTY && !process.env["CI"]; type Choice = "install" | "docker" | "manual"; @@ -1125,10 +1760,21 @@ async function startEngine(): Promise { if (choice === "docker" && dockerBin && composeFile) { const s = p.spinner(); s.start("Starting iii-engine via Docker..."); - writeEngineState({ kind: "docker", composeFile }); + configureDockerHostUser(); + const projectName = dockerProjectName(getRestPort()); + writeEngineState({ + kind: "docker", + schemaVersion: 2, + composeFile, + projectName, + engineVersion: IIPINNED_VERSION, + dataMountType: "bind", + dataMountSource: dataDirResolution.dataDir, + preserveContainer: false, + }); spawnEngineBackground( dockerBin, - ["compose", "-f", composeFile, "up", "-d"], + dockerComposeArgs(composeFile, projectName, ["up", "-d"]), "iii-engine via Docker", ); s.stop("Docker compose started"); @@ -1152,6 +1798,53 @@ async function waitForEngine(timeoutMs: number): Promise { return false; } +async function reconcilePersistedDockerEngine(): Promise { + const state = readEngineState(); + if (state?.kind !== "docker") return false; + const inspection = inspectOwnedDockerEngine(state); + if (inspection.status === "unavailable") { + p.log.error( + `The saved Docker engine cannot be verified: ${inspection.reason}. Refusing to overwrite its lifecycle state. Run \`agentmemory stop\` after restoring Docker/compose access.`, + ); + process.exit(1); + } + if (inspection.status === "missing") { + p.log.error( + "The saved Docker project no longer has an iii-engine container. Run `agentmemory stop` to remove its stale project state, then start agentmemory again.", + ); + process.exit(1); + } + persistDockerInspection(state, inspection); + if (inspection.status === "stopped") { + const dockerBin = whichBinary("docker"); + if ( + !dockerBin || + !runCommand(dockerBin, ["start", inspection.containerId], { + label: `Restarting owned Docker container ${inspection.containerId}`, + }) + ) { + p.log.error("The owned Docker container could not be restarted; lifecycle state was preserved."); + process.exit(1); + } + } + if (!(await waitForEngine(5000))) { + printDockerStartupLogs(); + p.log.error( + `The owned Docker container is running, but its REST API on port ${getRestPort()} is unhealthy. The saved project was preserved; inspect the Docker logs above or run \`agentmemory stop\`.`, + ); + process.exit(1); + } + await startWorkerForEngineState(); + if (!(await waitForAgentmemoryReady(15000))) { + p.log.error("agentmemory worker did not become ready within 15s."); + process.exit(1); + } + const consoleState = await ensureIiiConsole(); + await maybeOfferGlobalInstall(); + printReadyHint(consoleState); + return true; +} + function installInstructions(): string[] { const releaseUrl = iiiReleaseUrl(); if (IS_WINDOWS) { @@ -1211,7 +1904,10 @@ async function waitForAgentmemoryReady(timeoutMs: number): Promise { // `III_ENGINE_URL=ws://my-host:49134` doesn't print misleading // localhost addresses. Falls back to localhost. function getEngineHost(): string { - for (const envKey of ["III_ENGINE_URL", "AGENTMEMORY_URL"]) { + const envKeys = shouldUseAgentmemoryUrl() + ? ["III_ENGINE_URL", "AGENTMEMORY_URL"] + : ["III_ENGINE_URL"]; + for (const envKey of envKeys) { const raw = process.env[envKey]; if (!raw) continue; try { @@ -1267,6 +1963,7 @@ function printReadyHint(consoleState: IiiConsoleState): void { } async function main() { + await assertRuntimePortOwnership(); // Booting a second instance next to a live daemon registers a duplicate // worker on the running engine, and on iii 0.11.2 the second instance's // shutdown tears down the daemon's HTTP trigger routing (every @@ -1318,6 +2015,8 @@ async function main() { return; } + if (await reconcilePersistedDockerEngine()) return; + if (await isEngineRunning()) { if (IS_VERBOSE) p.log.success("iii-engine is running"); // Prefer the binary path persisted at launch time over whatever's on @@ -1344,12 +2043,14 @@ async function main() { // the pinned engine, never a silent loop. if (detected === IIPINNED_VERSION) { adoptRunningEngine(); - await import("./index.js"); - if (await waitForAgentmemoryReady(15000)) { - const consoleState = await ensureIiiConsole(); - await maybeOfferGlobalInstall(); - printReadyHint(consoleState); + await startWorkerForEngineState(); + if (!(await waitForAgentmemoryReady(15000))) { + p.log.error("agentmemory worker did not become ready within 15s."); + process.exit(1); } + const consoleState = await ensureIiiConsole(); + await maybeOfferGlobalInstall(); + printReadyHint(consoleState); return; } @@ -1410,6 +2111,7 @@ async function main() { if (!ready) { const port = getRestPort(); s.stop("iii-engine did not become ready within 15s"); + printDockerStartupLogs(); if (startupFailure?.kind === "engine-crashed" || startupFailure?.kind === "docker-crashed") { p.log.error("The iii-engine process crashed on startup."); @@ -1435,6 +2137,7 @@ async function main() { ); } else { p.log.error("The engine process started but the REST API never responded."); + printCapturedStartupStderr(); p.note( [ `Check whether port ${port} is already bound by another process:`, @@ -1452,12 +2155,14 @@ async function main() { } s.stop(c.ok("iii-engine is ready")); - await import("./index.js"); - if (await waitForAgentmemoryReady(15000)) { - const consoleState = await ensureIiiConsole(); - await maybeOfferGlobalInstall(); - printReadyHint(consoleState); + await startWorkerForEngineState(); + if (!(await waitForAgentmemoryReady(15000))) { + p.log.error("agentmemory worker did not become ready within 15s."); + process.exit(1); } + const consoleState = await ensureIiiConsole(); + await maybeOfferGlobalInstall(); + printReadyHint(consoleState); // Mark splash as something to skip on subsequent runs. This is a // no-op if onboarding already flipped the flag (idempotent merge). writePrefs({ skipSplash: true }); @@ -1833,7 +2538,7 @@ async function passiveServerChecks(): Promise { ok: hasEmbed, hint: hasEmbed ? undefined - : "Running BM25-only. Add OPENAI_API_KEY / VOYAGE_API_KEY / COHERE_API_KEY / OLLAMA_HOST", + : "Running BM25-only. Set EMBEDDING_PROVIDER=local for on-device semantic search, or add OPENAI_API_KEY / VOYAGE_API_KEY / COHERE_API_KEY / OLLAMA_HOST", }, ); @@ -2271,6 +2976,7 @@ async function runInit() { } async function startServerForDemo(): Promise<() => Promise> { + await assertRuntimePortOwnership(); if (await isAgentmemoryReady()) { return async () => {}; } @@ -2284,12 +2990,14 @@ async function startServerForDemo(): Promise<() => Promise> { process.exit(1); } if (!(await waitForEngine(15000))) { + printCapturedStartupStderr(); + printDockerStartupLogs(); p.log.error("iii-engine did not become ready within 15s."); process.exit(1); } } - await import("./index.js"); + await startWorkerForEngineState(); if (!(await waitForAgentmemoryReady(15000))) { p.log.error("agentmemory worker did not become ready within 15s."); process.exit(1); @@ -2300,7 +3008,7 @@ async function startServerForDemo(): Promise<() => Promise> { const port = getRestPort(); const state = readEngineState(); if (state?.kind === "docker") { - await stopDockerEngine(state.composeFile, port).catch(() => {}); + await stopDockerEngine(state, port).catch(() => {}); return; } const pids = new Set(findEnginePidsByPort(port)); @@ -2398,8 +3106,8 @@ async function runDemoBody(base: string) { ] : [ c.dim(`Note: "database performance optimization" found nothing —`), - c.dim(`semantic recall needs an embedding provider key (e.g.`), - c.dim(`OPENAI_API_KEY or GEMINI_API_KEY in ~/.agentmemory/.env).`), + c.dim(`set EMBEDDING_PROVIDER=local for on-device semantic recall,`), + c.dim(`or add an embedding API key in ~/.agentmemory/.env.`), ]), "", `Viewer: ${c.url(getViewerUrl())}`, @@ -2528,6 +3236,15 @@ async function runUpgrade() { function pidAlive(pid: number): boolean { try { process.kill(pid, 0); + if (!IS_WINDOWS) { + try { + const stat = execFileSync("ps", ["-p", String(pid), "-o", "stat="], { + encoding: "utf-8", + stdio: ["ignore", "pipe", "ignore"], + }); + if (stat.trim() && !processStatIsRunning(stat)) return false; + } catch {} + } return true; } catch (err) { return (err as NodeJS.ErrnoException)?.code === "EPERM"; @@ -2579,6 +3296,57 @@ async function stopWorkerPid(pid: number, graceMs: number): Promise { return ok; } +type NativeRemovalStopResult = + | { ok: true; stoppedEnginePids: number } + | { ok: false; reason: string }; + +async function stopNativeEngineForRemoval(): Promise { + const port = getRestPort(); + const workerPid = readWorkerPidfile(); + const enginePids = new Set(findEnginePidsByPort(port)); + const pidfilePid = readEnginePidfile(); + if (pidfilePid) enginePids.add(pidfilePid); + if (workerPid) enginePids.delete(workerPid); + + const foreign = [...enginePids] + .map((pid) => ({ pid, comm: pidCommand(pid) })) + .filter(({ comm }) => isForeignPortHolder(comm)); + if (foreign.length > 0) { + return { + ok: false, + reason: `refusing to signal non-iii process(es): ${foreign.map(({ pid, comm }) => `${pid} (${comm})`).join(", ")}`, + }; + } + + if (workerPid) { + if (!(await stopWorkerPid(workerPid, 5000))) { + return { + ok: false, + reason: `worker pid ${workerPid} could not be stopped; no engine or files were removed`, + }; + } + clearWorkerPidfile(); + } + + for (const pid of enginePids) { + const s = p.spinner(); + s.start(`Stopping iii-engine (pid ${pid})...`); + const ok = await signalAndWait(pid, "SIGTERM", 3000); + s.stop(ok ? `Stopped pid ${pid}` : `Failed to stop pid ${pid}`); + if (!ok) { + return { + ok: false, + reason: `engine pid ${pid} survived shutdown; lifecycle state and files were preserved`, + }; + } + } + + clearEnginePidfile(); + clearEngineState(); + clearWorkerPidfile(); + return { ok: true, stoppedEnginePids: enginePids.size }; +} + function pidCommand(pid: number): string { if (IS_WINDOWS) return ""; try { @@ -2627,73 +3395,113 @@ function findEnginePidsByPort(port: number): number[] { } } -async function stopDockerEngine(composeFile: string, port: number): Promise { +async function stopDockerEngine( + state: DockerEngineState, + port: number, + suppressOutro = false, +): Promise { const dockerBin = whichBinary("docker"); if (!dockerBin) { p.log.error( - `Engine was started via Docker compose, but \`docker\` is no longer on PATH. Stop it manually:\n docker compose -f ${composeFile} down`, + "Engine was started via Docker, but `docker` is no longer on PATH. Lifecycle state was preserved.", ); process.exit(1); } - if (!existsSync(composeFile)) { + + const inspection = inspectOwnedDockerEngine(state); + if (inspection.status === "unavailable") { p.log.error( - `Engine state references ${composeFile}, but the file is gone. Stop it manually:\n docker compose down (from the dir holding the original docker-compose.yml)`, + `The saved Docker engine cannot be verified: ${inspection.reason}. Refusing to stop or clear ownership state.`, ); process.exit(1); } + const resolvedState = persistDockerInspection(state, inspection); - // Reap the native worker first so its shutdown flush (BM25/vector - // snapshots via iii state::set) lands while the engine is still up — - // same ordering as the native stop path. Previously the worker pidfile - // was cleared without ever signaling the process, leaking a worker on - // every Docker-mode stop. const workerPid = readWorkerPidfile(); - let workerStopped = true; - if (workerPid) { - workerStopped = await stopWorkerPid(workerPid, 5000); + if (workerPid && !(await stopWorkerPid(workerPid, 5000))) { + p.log.error("The agentmemory worker could not be stopped; Docker ownership state was preserved."); + process.exit(1); } + clearWorkerPidfile(); - // Scope teardown to agentmemory's own services. A bare `down` against a - // user-owned compose file tears down every service in it. `rm -s -f` - // stops and removes only the named containers. - let composeText = ""; - try { - composeText = readFileSync(composeFile, "utf-8"); - } catch { - composeText = ""; + if (inspection.status === "running" || inspection.status === "stopped") { + if ( + inspection.status === "running" && + !runCommand( + dockerBin, + ["stop", "--time", "10", inspection.containerId], + { label: `Stopping preserved Docker container ${inspection.containerId}` }, + ) + ) { + p.log.error("The Docker container could not be stopped; ownership state was preserved."); + process.exit(1); + } + clearEnginePidfile(); + writeEngineState(resolvedState); + if (!suppressOutro) { + p.outro( + "Stopped. The validated Docker container and data mount were preserved for a lossless restart.", + ); + } + return; + } + + const projectName = inspection.projectName ?? resolvedState.projectName; + if (!projectName) { + p.log.error("No verified Docker Compose project name is available. Refusing an unscoped stop."); + process.exit(1); } - const ownServices = ["iii-engine", "iii-init"].filter((svc) => - new RegExp(`^\\s+${svc}:`, "m").test(composeText), + const effectiveComposeFile = existsSync(inspection.composeFile) + ? inspection.composeFile + : discoverComposeFile(); + if (!effectiveComposeFile) { + p.log.error("No agentmemory compose file is available. Docker ownership state was preserved."); + process.exit(1); + } + const composeText = readFileSync(effectiveComposeFile, "utf-8"); + const ownServices = ["iii-engine", "iii-init"].filter((service) => + new RegExp(`^\\s+${service}:`, "m").test(composeText), ); if (ownServices.length === 0) { - p.log.error( - `${composeFile} does not define the agentmemory services (iii-engine/iii-init). Refusing to run an unscoped \`docker compose down\` against it — that would tear down every service in the file.\n\nStop the engine service manually:\n docker compose -f ${composeFile} stop `, - ); + p.log.error(`${effectiveComposeFile} does not define iii-engine or iii-init. Refusing an unscoped stop.`); process.exit(1); } + + const composeCommand = [ + "docker", + ...dockerComposeArgs(effectiveComposeFile, projectName, []), + ].join(" "); + const ok = runCommand( dockerBin, - ["compose", "-f", composeFile, "rm", "-s", "-f", ...ownServices], + dockerComposeArgs(effectiveComposeFile, projectName, [ + "rm", + "-s", + "-f", + ...ownServices, + ]), { - label: `docker compose -f ${composeFile} rm -s -f ${ownServices.join(" ")}`, + label: `${composeCommand} rm -s -f ${ownServices.join(" ")}`, }, ); // Clear each piece of state only after its shutdown succeeded, so a // failed stop stays retryable. - if (workerStopped) clearWorkerPidfile(); if (ok) { clearEnginePidfile(); clearEngineState(); } else { p.log.error( - `docker compose rm failed. The engine may still be running on :${port}. Inspect with:\n docker compose -f ${composeFile} ps`, + `docker compose rm failed. The engine may still be running on :${port}. Inspect with:\n ${composeCommand} ps`, ); process.exit(1); } - p.outro("Stopped. Memories persisted to disk; restart anytime with: npx @agentmemory/agentmemory"); + if (!suppressOutro) { + p.outro("Stopped. Memories persisted to disk; restart anytime with: npx @agentmemory/agentmemory"); + } } async function runStop(): Promise { + await assertRuntimePortOwnership(); p.intro("agentmemory stop"); const port = getRestPort(); const state = readEngineState(); @@ -2703,13 +3511,8 @@ async function runStop(): Promise { if (state?.kind === "docker") { if (!running) { p.log.info(`No engine responding on port ${port}.`); - clearEnginePidfile(); - clearEngineState(); - clearWorkerPidfile(); - p.outro("Nothing to stop."); - return; } - await stopDockerEngine(state.composeFile, port); + await stopDockerEngine(state, port); return; } @@ -3071,18 +3874,53 @@ function safeDelete(path: string): { ok: boolean; message: string } { } async function runRemove(): Promise { + if (selectedInstance !== 0) { + p.intro("agentmemory remove"); + p.log.error( + `remove is a global uninstall and cannot target instance ${selectedInstance}. Stop that daemon with \`agentmemory stop --instance ${selectedInstance}\`, then run \`agentmemory remove\` without --instance when you intend to uninstall shared assets.`, + ); + process.exit(1); + } + await assertRuntimePortOwnership(); p.intro("agentmemory remove"); const force = args.includes("--force"); const keepData = args.includes("--keep-data"); + const ownedState = readEngineState(); + if (ownedState?.kind === "docker") { + const inspection = inspectOwnedDockerEngine(ownedState); + if (inspection.status === "running" || inspection.status === "stopped") { + persistDockerInspection(ownedState, inspection); + if (!keepData) { + p.log.error( + `Docker-backed removal requires --keep-data so the validated ${inspection.dataMountType} data mount (${inspection.dataMountSource}) and its recovery metadata remain usable. Re-run \`agentmemory remove --keep-data\`; destructive Docker data deletion is intentionally not automated.`, + ); + process.exit(1); + } + } else { + const detail = inspection.status === "unavailable" + ? inspection.reason + : "the owned container is missing"; + p.log.error( + `remove cannot safely reconcile the saved Docker ownership (${detail}). Lifecycle state was preserved for manual recovery.`, + ); + process.exit(1); + } + } const home = homedir(); const connectManifest = loadConnectManifest(home); const localBinIiiVersion = probeLocalBinIiiVersion(home); - const options: RemoveOptions = { force, keepData }; + const options: RemoveOptions = { + force, + keepData, + preserveRuntimeState: ownedState?.kind === "docker", + }; const plan = buildRemovePlan( { home, + runtimeDir: dirname(engineStatePath()), + dataDir: dataDirResolution.dataDir, pinnedVersion: IIPINNED_VERSION, localBinIiiVersion, connectManifest, @@ -3133,26 +3971,21 @@ async function runRemove(): Promise { } if (item.id === "stop-engine") { - try { - const port = getRestPort(); - const portPids = findEnginePidsByPort(port); - const pidfilePid = readEnginePidfile(); - const cands = new Set(); - if (pidfilePid) cands.add(pidfilePid); - for (const pid of portPids) cands.add(pid); - for (const pid of cands) await signalAndWait(pid, "SIGTERM", 3000); - clearEnginePidfile(); - clearEngineState(); - p.log.success( - cands.size > 0 - ? `stopped engine (${cands.size} pid${cands.size === 1 ? "" : "s"})` - : "no engine running", - ); - } catch (err) { - p.log.warn( - `engine stop best-effort: ${err instanceof Error ? err.message : String(err)}`, - ); + if (ownedState?.kind === "docker") { + await stopDockerEngine(ownedState, getRestPort(), true); + p.log.success("stopped Docker engine; container, data mount, and recovery state preserved"); + continue; } + const stopped = await stopNativeEngineForRemoval(); + if (!stopped.ok) { + p.log.error(`Removal aborted: ${stopped.reason}.`); + process.exit(1); + } + p.log.success( + stopped.stoppedEnginePids > 0 + ? `stopped engine (${stopped.stoppedEnginePids} pid${stopped.stoppedEnginePids === 1 ? "" : "s"})` + : "no engine running", + ); continue; } diff --git a/src/cli/engine-config.ts b/src/cli/engine-config.ts new file mode 100644 index 0000000000..e20484a790 --- /dev/null +++ b/src/cli/engine-config.ts @@ -0,0 +1,121 @@ +import { join } from "node:path"; + +export interface EngineConfigOptions { + dataDir: string; + ports?: EngineRuntimePorts; +} + +export interface EngineRuntimePorts { + restPort: number; + streamPort: number; + viewerPort: number; + enginePort: number; +} + +function yamlSingleQuote(value: string): string { + return `'${value.replace(/'/g, "''")}'`; +} + +function workerBlock( + lines: string[], + name: string, +): { start: number; end: number; indent: string } | null { + const marker = `- name: ${name}`; + const start = lines.findIndex((line) => line.trim() === marker); + if (start === -1) return null; + const indent = lines[start]!.match(/^\s*/)?.[0] ?? ""; + let end = lines.length; + for (let i = start + 1; i < lines.length; i++) { + const line = lines[i]!; + if (line.startsWith(indent) && line.trim().startsWith("- name: ")) { + end = i; + break; + } + } + return { start, end, indent }; +} + +function setWorkerPort(lines: string[], name: string, port: number): void { + let block = workerBlock(lines, name); + if (!block && name === "iii-worker-manager") { + const workersIndex = lines.findIndex((line) => line.trim() === "workers:"); + if (workersIndex === -1) return; + lines.splice( + workersIndex + 1, + 0, + " - name: iii-worker-manager", + " config:", + ` port: ${port}`, + " host: 127.0.0.1", + ); + return; + } + if (!block) return; + + const portIndex = lines.findIndex( + (line, index) => + index > block!.start && + index < block!.end && + line.trim().startsWith("port:"), + ); + if (portIndex !== -1) { + const indent = lines[portIndex]!.match(/^\s*/)?.[0] ?? `${block.indent} `; + lines[portIndex] = `${indent}port: ${port}`; + return; + } + + const configIndex = lines.findIndex( + (line, index) => + index > block!.start && index < block!.end && line.trim() === "config:", + ); + if (configIndex !== -1) { + const configIndent = lines[configIndex]!.match(/^\s*/)?.[0] ?? `${block.indent} `; + lines.splice(configIndex + 1, 0, `${configIndent} port: ${port}`); + } +} + +function setManagedCorsOrigins( + lines: string[], + restPort: number, + viewerPort: number, +): void { + const block = workerBlock(lines, "iii-http"); + if (!block) return; + const originsIndex = lines.findIndex( + (line, index) => + index > block.start && + index < block.end && + line.trim().startsWith("allowed_origins:"), + ); + if (originsIndex === -1) return; + const indent = lines[originsIndex]!.match(/^\s*/)?.[0] ?? " "; + lines[originsIndex] = + `${indent}allowed_origins: [` + + `"http://localhost:${restPort}", ` + + `"http://localhost:${viewerPort}", ` + + `"http://127.0.0.1:${restPort}", ` + + `"http://127.0.0.1:${viewerPort}"]`; +} + +export function renderEngineConfig( + template: string, + options: EngineConfigOptions, +): string { + const rendered = template + .replace( + "file_path: ./data/state_store.db", + `file_path: ${yamlSingleQuote(join(options.dataDir, "state_store.db"))}`, + ) + .replace( + "file_path: ./data/stream_store", + `file_path: ${yamlSingleQuote(join(options.dataDir, "stream_store"))}`, + ); + if (!options.ports) return rendered; + + const lines = rendered.split("\n"); + setWorkerPort(lines, "iii-http", options.ports.restPort); + setWorkerPort(lines, "iii-stream", options.ports.streamPort); + setWorkerPort(lines, "iii-worker-manager", options.ports.enginePort); + setManagedCorsOrigins(lines, options.ports.restPort, options.ports.viewerPort); + return lines.join("\n"); +} diff --git a/src/cli/engine-launch.ts b/src/cli/engine-launch.ts new file mode 100644 index 0000000000..b162e7f5c5 --- /dev/null +++ b/src/cli/engine-launch.ts @@ -0,0 +1,189 @@ +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { + renderEngineConfig, + type EngineConfigOptions, +} from "./engine-config.js"; + +export function agentmemoryHome(home: string): string { + return join(home, ".agentmemory"); +} + +export function runtimeConfigPath(dataDir: string): string { + return join(dataDir, "iii-config.runtime.yaml"); +} + +export function dockerProjectName(restPort: number): string { + return `agentmemory-${restPort}`; +} + +export function dockerComposeArgs( + composeFile: string, + projectName: string | undefined, + command: string[], +): string[] { + return [ + "compose", + ...(projectName ? ["-p", projectName] : []), + "-f", + composeFile, + ...command, + ]; +} + +export function isBundledConfig(configPath: string, packageDir: string): boolean { + const resolved = resolve(configPath); + return ( + resolved === resolve(join(packageDir, "iii-config.yaml")) || + resolved === resolve(join(packageDir, "..", "iii-config.yaml")) + ); +} + +export function resolveEngineCwd( + configPath: string, + invocationCwd: string, + home: string, + bundledConfig = false, +): string { + if (bundledConfig) return agentmemoryHome(home); + const absoluteConfigPath = isAbsolute(configPath) + ? configPath + : resolve(invocationCwd, configPath); + return dirname(absoluteConfigPath); +} + +function indentation(line: string): number { + return line.length - line.trimStart().length; +} + +function yamlScalar(value: string): string { + const trimmed = value.trim(); + const first = trimmed[0]; + const last = trimmed.at(-1); + if ((first === '"' && last === '"') || (first === "'" && last === "'")) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +function isAgentmemoryExecCommand( + line: string, + nodeBin: string, + workerEntry: string, +): boolean { + const match = line.match(/^\s*-\s+(.+?)\s*$/); + if (!match) return false; + const command = yamlScalar(match[1]!).replaceAll("\\", "/"); + const explicitCommand = `${nodeBin} ${workerEntry}`.replaceAll("\\", "/"); + return ( + command === explicitCommand || + /^(?:node|node\.exe)\s+\.\/dist\/index\.mjs$/.test(command) || + /^(?:node|node\.exe)\s+dist\/index\.mjs$/.test(command) + ); +} + +function removeAgentmemoryExecCommand( + raw: string, + nodeBin: string, + workerEntry: string, +): string { + const lines = raw.split("\n"); + const output: string[] = []; + let index = 0; + + while (index < lines.length) { + const line = lines[index]!; + if (line.trim() !== "- name: iii-exec") { + output.push(line); + index++; + continue; + } + + const blockIndent = indentation(line); + let blockEnd = index + 1; + while (blockEnd < lines.length) { + const candidate = lines[blockEnd]!; + if (candidate.trim() !== "") { + const candidateIndent = indentation(candidate); + if ( + candidateIndent < blockIndent || + (candidateIndent === blockIndent && candidate.trim().startsWith("- name: ")) + ) { + break; + } + } + blockEnd++; + } + + const block = lines.slice(index, blockEnd); + const execIndex = block.findIndex((candidate) => candidate.trim() === "exec:"); + if (execIndex === -1) { + output.push(...block); + index = blockEnd; + continue; + } + + const execIndent = indentation(block[execIndex]!); + let execEnd = execIndex + 1; + while (execEnd < block.length) { + const candidate = block[execEnd]!; + if (candidate.trim() !== "" && indentation(candidate) <= execIndent) break; + execEnd++; + } + + const beforeExec = block.slice(0, execIndex + 1); + const execLines = block.slice(execIndex + 1, execEnd); + const afterExec = block.slice(execEnd); + const remainingExecLines = execLines.filter( + (candidate) => !isAgentmemoryExecCommand(candidate, nodeBin, workerEntry), + ); + const removed = remainingExecLines.length !== execLines.length; + const hasExecCommand = remainingExecLines.some( + (candidate) => /^\s*-\s+/.test(candidate) && indentation(candidate) > execIndent, + ); + + if (!removed) { + output.push(...block); + } else if (hasExecCommand) { + output.push(...beforeExec, ...remainingExecLines, ...afterExec); + } + index = blockEnd; + } + + return output.join("\n"); +} + +export function rewriteBundledConfig( + raw: string, + home: string, + nodeBin: string, + workerEntry: string, + options?: EngineConfigOptions, +): string { + const rendered = renderEngineConfig(raw, { + dataDir: options?.dataDir ?? join(agentmemoryHome(home), "data"), + ...(options?.ports ? { ports: options.ports } : {}), + }); + return removeAgentmemoryExecCommand(rendered, nodeBin, workerEntry); +} + +export interface DataMigration { + from: string; + to: string; +} + +export function legacyDataMigrations( + invocationCwd: string, + home: string, + resolvedDataDir = join(agentmemoryHome(home), "data"), +): DataMigration[] { + return [ + { + from: join(invocationCwd, "data", "state_store.db"), + to: join(resolvedDataDir, "state_store.db"), + }, + { + from: join(invocationCwd, "data", "stream_store"), + to: join(resolvedDataDir, "stream_store"), + }, + ]; +} diff --git a/src/cli/process-state.ts b/src/cli/process-state.ts new file mode 100644 index 0000000000..41badf9f57 --- /dev/null +++ b/src/cli/process-state.ts @@ -0,0 +1,3 @@ +export function processStatIsRunning(stat: string): boolean { + return !stat.trimStart().startsWith("Z"); +} diff --git a/src/cli/remove-plan.ts b/src/cli/remove-plan.ts index fbc64de1be..662e378352 100644 --- a/src/cli/remove-plan.ts +++ b/src/cli/remove-plan.ts @@ -13,6 +13,7 @@ import { existsSync, statSync } from "node:fs"; import { join } from "node:path"; +import { runtimeConfigPath } from "./engine-launch.js"; export type RemovePlanItem = { /** Stable id, used in tests and CLI output. */ @@ -34,11 +35,17 @@ export type RemoveOptions = { force: boolean; /** Keep ~/.agentmemory/* user data; only remove binaries/symlinks. */ keepData: boolean; + /** Keep engine ownership metadata needed to recover retained Docker data. */ + preserveRuntimeState?: boolean; }; export type RemoveContext = { /** $HOME (so tests can sandbox). */ home: string; + /** Resolved directory containing pidfiles and engine ownership state. */ + runtimeDir: string; + /** Resolved iii-engine data directory selected for this invocation. */ + dataDir: string; /** Pinned engine version we expect ~/.local/bin/iii to match. */ pinnedVersion: string; /** @@ -66,12 +73,16 @@ export type ConnectManifest = { }>; }; -export function pidfilePath(home: string): string { - return join(home, ".agentmemory", "iii.pid"); +export function pidfilePath(runtimeDir: string): string { + return join(runtimeDir, "iii.pid"); } -export function enginePath(home: string): string { - return join(home, ".agentmemory", "engine-state.json"); +export function workerPidfilePath(runtimeDir: string): string { + return join(runtimeDir, "worker.pid"); +} + +export function enginePath(runtimeDir: string): string { + return join(runtimeDir, "engine-state.json"); } export function envPath(home: string): string { @@ -86,10 +97,6 @@ export function backupsDir(home: string): string { return join(home, ".agentmemory", "backups"); } -export function dataDir(home: string): string { - return join(home, ".agentmemory", "data"); -} - // Platform-aware binary name. Windows requires the .exe suffix or the // existsSync probe misses the installed binary. function iiiBinFile(): string { @@ -138,7 +145,14 @@ export function buildRemovePlan( ctx: RemoveContext, options: RemoveOptions, ): RemovePlanItem[] { - const { home, pinnedVersion, localBinIiiVersion, connectManifest } = ctx; + const { + home, + runtimeDir, + dataDir, + pinnedVersion, + localBinIiiVersion, + connectManifest, + } = ctx; const plan: RemovePlanItem[] = []; plan.push({ @@ -146,26 +160,39 @@ export function buildRemovePlan( description: "Stop running iii-engine (if any) cleanly", path: null, alwaysAsk: false, - applicable: pathExists(pidfilePath(home)) || pathExists(enginePath(home)), + applicable: + pathExists(pidfilePath(runtimeDir)) || + pathExists(workerPidfilePath(runtimeDir)) || + pathExists(enginePath(runtimeDir)), sizeBytes: -1, }); plan.push({ id: "pidfile", description: "Delete pidfile", - path: pidfilePath(home), + path: pidfilePath(runtimeDir), + alwaysAsk: false, + applicable: pathExists(pidfilePath(runtimeDir)), + sizeBytes: safeSize(pidfilePath(runtimeDir)), + }); + + plan.push({ + id: "worker-pidfile", + description: "Delete worker pidfile", + path: workerPidfilePath(runtimeDir), alwaysAsk: false, - applicable: pathExists(pidfilePath(home)), - sizeBytes: safeSize(pidfilePath(home)), + applicable: pathExists(workerPidfilePath(runtimeDir)), + sizeBytes: safeSize(workerPidfilePath(runtimeDir)), }); plan.push({ id: "engine-state", description: "Delete engine-state.json", - path: enginePath(home), + path: enginePath(runtimeDir), alwaysAsk: false, - applicable: pathExists(enginePath(home)), - sizeBytes: safeSize(enginePath(home)), + applicable: + !options.preserveRuntimeState && pathExists(enginePath(runtimeDir)), + sizeBytes: safeSize(enginePath(runtimeDir)), }); // .env holds the user's API keys. Always ask before deleting, even on @@ -197,6 +224,15 @@ export function buildRemovePlan( sizeBytes: -1, }); + plan.push({ + id: "runtime-config", + description: "Delete generated iii-config.runtime.yaml", + path: runtimeConfigPath(dataDir), + alwaysAsk: false, + applicable: pathExists(runtimeConfigPath(dataDir)), + sizeBytes: safeSize(runtimeConfigPath(dataDir)), + }); + // Iterate over connect-installed agent symlinks. We always honor these // (even with --keep-data, since they're outside ~/.agentmemory/). if (connectManifest?.installed?.length) { @@ -252,11 +288,10 @@ export function buildRemovePlan( // behavior is keep. plan.push({ id: "data-dir", - description: - "Delete memory data directory (~/.agentmemory/data/) — will ask separately", - path: dataDir(home), + description: `Delete memory data directory (${dataDir}) — will ask separately`, + path: dataDir, alwaysAsk: true, - applicable: !options.keepData && pathExists(dataDir(home)), + applicable: !options.keepData && pathExists(dataDir), sizeBytes: -1, }); diff --git a/src/cli/startup-stderr.ts b/src/cli/startup-stderr.ts new file mode 100644 index 0000000000..d240dfd23f --- /dev/null +++ b/src/cli/startup-stderr.ts @@ -0,0 +1,23 @@ +export interface StartupStderrCapture { + append(chunk: Buffer): void; + text(): string; +} + +export function createStartupStderrCapture( + maxBytes = 16 * 1024, +): StartupStderrCapture { + const chunks: Buffer[] = []; + let capturedBytes = 0; + + return { + append(chunk: Buffer): void { + if (capturedBytes >= maxBytes) return; + const slice = chunk.subarray(0, maxBytes - capturedBytes); + chunks.push(slice); + capturedBytes += slice.length; + }, + text(): string { + return Buffer.concat(chunks).toString("utf8"); + }, + }; +} diff --git a/src/config.ts b/src/config.ts index a8e0538008..5738d89c87 100644 --- a/src/config.ts +++ b/src/config.ts @@ -160,7 +160,8 @@ function detectProvider(env: Record): ProviderConfig { if (!allowAgentSdk) { process.stderr.write( pc.dim( - "[agentmemory] No LLM provider key set — running zero-LLM (BM25 + on-device embeddings). " + + "[agentmemory] No LLM provider key set — running zero-LLM with BM25 search. " + + "Set EMBEDDING_PROVIDER=local for on-device semantic embeddings. " + "Set ANTHROPIC_API_KEY (or GEMINI/OPENAI/OPENROUTER/MINIMAX) in ~/.agentmemory/.env for LLM compression and summaries. " + "Agent-SDK fallback stays off by default to avoid a Stop-hook recursion loop; opt in with AGENTMEMORY_AUTO_COMPRESS=true + AGENTMEMORY_ALLOW_AGENT_SDK=true.\n", ), @@ -198,6 +199,8 @@ export function loadConfig(): AgentMemoryConfig { const streamsPort = parseInt(env["III_STREAM_PORT"] || env["III_STREAMS_PORT"] || "", 10) || restPort + 1; + const viewerPort = + parseInt(env["III_VIEWER_PORT"] || "", 10) || restPort + 2; const engineUrl = env["III_ENGINE_URL"] || `ws://localhost:${ @@ -208,6 +211,7 @@ export function loadConfig(): AgentMemoryConfig { engineUrl, restPort, streamsPort, + viewerPort, provider, tokenBudget: safeParseInt(env["TOKEN_BUDGET"], 2000), maxObservationsPerSession: safeParseInt(env["MAX_OBS_PER_SESSION"], 500), diff --git a/src/index.ts b/src/index.ts index 9e49359a38..eb6ef1f59a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -107,18 +107,12 @@ import { initMetrics, OTEL_CONFIG } from "./telemetry/setup.js"; import { VERSION } from "./version.js"; import { bootLog, logger } from "./logger.js"; import { parsePositiveIntervalMs } from "./config.js"; +import { runtimeMetadataPath } from "./runtime-paths.js"; import { mkdirSync, writeFileSync, unlinkSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { homedir } from "node:os"; - -// #640 + #474: the worker process (this file) is spawned by iii-exec -// inside the engine. When `agentmemory stop` kills only the engine pid, -// this worker can survive (detached spawn, signal not propagated, or a -// wrapper script keeps it running) and reconnects to the next engine as -// a duplicate worker. Write the worker pid alongside iii.pid so -// `agentmemory stop` can reap us too. +import { dirname } from "node:path"; + function workerPidfilePath(): string { - return join(homedir(), ".agentmemory", "worker.pid"); + return runtimeMetadataPath("worker.pid"); } function writeWorkerPidfile(): void { try { @@ -519,9 +513,8 @@ async function main() { `MCP surface (opt-in via \`npx @agentmemory/mcp\`): ${getAllTools().length} tools · 6 resources · 3 prompts`, ); - const viewerPort = config.restPort + 2; const viewerServer = startViewerServer( - viewerPort, + config.viewerPort, kv, sdk, secret, diff --git a/src/runtime-paths.ts b/src/runtime-paths.ts new file mode 100644 index 0000000000..9bea8e7f0e --- /dev/null +++ b/src/runtime-paths.ts @@ -0,0 +1,19 @@ +import { homedir } from "node:os"; +import { join, resolve } from "node:path"; + +interface RuntimeMetadataPathOptions { + env?: NodeJS.ProcessEnv; + home?: string; +} + +export function runtimeMetadataPath( + name: string, + options: RuntimeMetadataPathOptions = {}, +): string { + const env = options.env ?? process.env; + const home = options.home ?? homedir(); + const baseDir = env["AGENTMEMORY_RUNTIME_DIR"] + ? resolve(env["AGENTMEMORY_RUNTIME_DIR"]) + : join(home, ".agentmemory"); + return join(baseDir, name); +} diff --git a/src/types.ts b/src/types.ts index 2df022205d..f6866419ac 100644 --- a/src/types.ts +++ b/src/types.ts @@ -186,6 +186,7 @@ export interface AgentMemoryConfig { engineUrl: string; restPort: number; streamsPort: number; + viewerPort: number; provider: ProviderConfig; tokenBudget: number; maxObservationsPerSession: number; diff --git a/test/cli-data-dir.test.ts b/test/cli-data-dir.test.ts index faf453676f..d4058645a7 100644 --- a/test/cli-data-dir.test.ts +++ b/test/cli-data-dir.test.ts @@ -73,6 +73,21 @@ describe("resolveDataDir", () => { }); }); + it("accepts both instance flag forms", () => { + const options = { + env: {}, + cwd: "/repo/project", + home: "/home/alex", + platform: "linux" as const, + }; + expect(resolveDataDir({ ...options, args: ["--instance", "2"] })).toEqual( + resolveDataDir({ ...options, args: ["--instance=2"] }), + ); + expect(resolveDataDir({ ...options, args: ["--instance=2"] }).dataDir).toBe( + join("/home/alex", ".local", "share", "agentmemory", "instance-2"), + ); + }); + it("adopts legacy data without sharing it with another instance", () => { const cwd = mkdtempSync(join(tmpdir(), "agentmemory-data-")); mkdirSync(join(cwd, ".git")); diff --git a/test/cli-engine-startup.test.ts b/test/cli-engine-startup.test.ts new file mode 100644 index 0000000000..ceaeeacbe6 --- /dev/null +++ b/test/cli-engine-startup.test.ts @@ -0,0 +1,149 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +describe("fresh native engine startup", () => { + const source = readFileSync("src/cli.ts", "utf8"); + + it("routes existing and newly installed engines through one runtime config", () => { + const prepareStart = source.indexOf("function prepareEngineLaunch"); + const prepareEnd = source.indexOf("function startIiiBin", prepareStart); + const prepareBody = source.slice(prepareStart, prepareEnd); + expect(prepareBody).toContain("dataDir: dataDirResolution.dataDir"); + expect(prepareBody).toContain("restPort: getRestPort()"); + expect(prepareBody).toContain("streamPort: getStreamPort()"); + expect(prepareBody).toContain("viewerPort: getConfiguredViewerPort()"); + expect(prepareBody).toContain("enginePort: getEnginePort()"); + + const engineStart = source.indexOf("async function startEngine"); + const engineEnd = source.indexOf("async function waitForEngine", engineStart); + const engineBody = source.slice(engineStart, engineEnd); + expect(engineBody).not.toContain("writeRuntimeIiiConfig"); + expect(engineBody.match(/startIiiBin\(iiiBin, configPath\)/g)).toHaveLength(2); + }); + + it("reports buffered engine stderr when a verbose startup times out", () => { + const spawnStart = source.indexOf("function spawnEngineBackground"); + const spawnEnd = source.indexOf("function prepareEngineLaunch", spawnStart); + const spawnBody = source.slice(spawnStart, spawnEnd); + expect(spawnBody).toContain( + "activeStartupStderr = createStartupStderrCapture()", + ); + expect(spawnBody).toContain("activeStartupStderr.append(chunk)"); + + const timeoutStart = source.indexOf("const ready = await waitForEngine(15000)"); + const timeoutEnd = source.indexOf('s.stop(c.ok("iii-engine is ready"))', timeoutStart); + expect(source.slice(timeoutStart, timeoutEnd)).toContain( + "printCapturedStartupStderr()", + ); + + const demoTimeoutStart = source.indexOf("await waitForEngine(15000)", timeoutEnd); + const demoTimeoutEnd = source.indexOf("await import(\"./index.js\")", demoTimeoutStart); + expect(source.slice(demoTimeoutStart, demoTimeoutEnd)).toContain( + "printCapturedStartupStderr()", + ); + }); + + it("exports the complete derived port set for Docker and native launchers", () => { + expect(source).toContain( + 'process.env["III_STREAM_PORT"] ??=\n process.env["III_STREAMS_PORT"] ?? String(restPort + 1)', + ); + expect(source).toContain( + 'process.env["III_VIEWER_PORT"] ??= String(restPort + 2)', + ); + expect(source).toContain( + 'process.env["III_ENGINE_PORT"] ??= String(restPort + 46023)', + ); + expect(source).toContain("new URL(configuredEngineUrl).port"); + expect(source).toContain( + 'process.env["III_ENGINE_PORT"] = parsedEnginePort', + ); + expect(source).toContain( + 'process.env["AGENTMEMORY_METRICS_PORT"] ??= String(restPort + 6353)', + ); + expect(source).toContain( + 'process.env["AGENTMEMORY_DATA_DIR"] = dataDirResolution.dataDir', + ); + }); + + it("checks every Unix engine installer prerequisite before downloading", () => { + const installerStart = source.indexOf("async function runIiiInstaller"); + const installerEnd = source.indexOf("type StartupFailure", installerStart); + const installerBody = source.slice(installerStart, installerEnd); + expect(installerBody).toContain('whichBinary("sh")'); + expect(installerBody).toContain('whichBinary("curl")'); + expect(installerBody).toContain('whichBinary("tar")'); + }); + + it("lets the CLI own one bundled worker and reuses a live worker", () => { + const prepareStart = source.indexOf("function prepareEngineLaunch"); + const prepareEnd = source.indexOf("function startIiiBin", prepareStart); + const prepareBody = source.slice(prepareStart, prepareEnd); + expect(prepareBody).toContain( + "rewriteBundledConfig(", + ); + expect(prepareBody).toContain(": renderEngineConfig(rawConfig, options)"); + + const workerStart = source.indexOf("async function startWorkerForEngineState"); + const workerEnd = source.indexOf("async function startEngine", workerStart); + const workerBody = source.slice(workerStart, workerEnd); + expect(workerBody).toContain("readWorkerPidfile()"); + expect(workerBody).toContain("pidAlive(workerPid)"); + expect(workerBody).toContain("waitForConfiguredWorker"); + expect(workerBody).toContain('await import("./index.js")'); + + expect(source.match(/await startWorkerForEngineState\(\)/g)).toHaveLength(4); + expect(source).toContain("agentmemory worker did not become ready within 15s"); + }); + + it("stores lifecycle metadata per resolved instance and scopes Docker", () => { + expect(source).toContain('runtimeMetadataPath("iii.pid")'); + expect(source).toContain('runtimeMetadataPath("engine-state.json")'); + expect(source).toContain('runtimeMetadataPath("worker.pid")'); + expect(source).toContain('process.env["AGENTMEMORY_RUNTIME_DIR"] = selectedInstance > 0'); + expect(source).toContain("dockerProjectName(getRestPort())"); + expect(source).toContain("dockerComposeArgs("); + }); + + it("keeps AGENTMEMORY_URL client-only and lets local port flags win", () => { + expect(source).toContain('const URL_CLIENT_COMMANDS = new Set(["status", "doctor", "mcp"])'); + expect(source).toContain("hasExplicitLocalPortOverride"); + expect(source).toContain("shouldUseAgentmemoryUrl()"); + expect(source).toContain('process.env["III_REST_PORT"] = String(base)'); + }); + + it("recovers the host worker for a validated Docker engine", () => { + expect(source).toContain("function inspectOwnedDockerEngine"); + expect(source).toMatch(/"ps",\s*"-q",\s*"--all",\s*"iii-engine"/); + expect(source).toContain('"com.docker.compose.service"'); + expect(source).toContain('record.HostConfig?.PortBindings'); + expect(source).toContain('mount.Destination === "/data"'); + expect(source).toContain('["start", inspection.containerId]'); + expect(source).toContain('["stop", "--time", "10", inspection.containerId]'); + + const mainStart = source.indexOf("async function main()"); + const mainEnd = source.indexOf("async function apiFetch", mainStart); + const mainBody = source.slice(mainStart, mainEnd); + expect(mainBody).toContain("reconcilePersistedDockerEngine()"); + expect(mainBody.indexOf("reconcilePersistedDockerEngine()")) + .toBeLessThan(mainBody.indexOf("if (await isEngineRunning())")); + }); + + it("keeps canonical compatibility and rejects Docker ownership on another port", () => { + expect(source).toContain('function cliArgValue(name: string)'); + expect(source).toContain('arg.startsWith(`${name}=`)'); + expect(source).toContain("async function assertRuntimePortOwnership"); + expect(source).toContain("restPort: getRestPort()"); + expect(source).toContain('inspection.status === "running" || inspection.status === "stopped"'); + expect(source).not.toContain("migrateLegacyRuntimeMetadata"); + expect(source).not.toContain("clearLegacyRuntimeMetadata"); + }); + + it("preserves Docker ownership when compose startup fails", () => { + const spawnStart = source.indexOf("function spawnEngineBackground"); + const spawnEnd = source.indexOf("function prepareEngineLaunch", spawnStart); + const spawnBody = source.slice(spawnStart, spawnEnd); + expect(spawnBody).toContain("if (!isDocker)"); + expect(spawnBody).toContain("clearEngineState()"); + expect(spawnBody).not.toContain("if (!isDocker) clearEnginePidfile();\n clearEngineState();"); + }); +}); diff --git a/test/cli-lifecycle-safety.test.ts b/test/cli-lifecycle-safety.test.ts new file mode 100644 index 0000000000..7d19949267 --- /dev/null +++ b/test/cli-lifecycle-safety.test.ts @@ -0,0 +1,356 @@ +import { spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const sandboxes: string[] = []; +const FULL_CONTAINER_ID = + "a348133fe52fc5ba49b74cb0a4e36a1ec1d64827d79dc120940a96054bbe87c4"; + +function sandbox(): string { + const path = mkdtempSync(join(tmpdir(), "agentmemory-lifecycle-")); + sandboxes.push(path); + return path; +} + +function installFakeDocker(binDir: string): void { + mkdirSync(binDir, { recursive: true }); + const dockerPath = join(binDir, "docker"); + writeFileSync( + dockerPath, + `#!/usr/bin/env node +const { appendFileSync } = require("node:fs"); +const args = process.argv.slice(2); +appendFileSync(process.env.DOCKER_LOG, args.join(" ") + "\\n"); +const isComposePs = args[0] === "compose" && args.includes("ps"); +const isGlobalPs = args[0] === "ps"; +if (process.env.DOCKER_FAILURE_MODE === "scan" && (isComposePs || isGlobalPs)) { + process.exit(1); +} +if (isComposePs || isGlobalPs) { + if (process.env.DOCKER_FAILURE_MODE === "duplicate") { + process.stdout.write((isComposePs ? process.env.FULL_CONTAINER_ID.slice(0, 12) : process.env.FULL_CONTAINER_ID) + "\\n"); + } else { + process.stdout.write("candidate-id\\n"); + } + process.exit(0); +} +if (args[0] === "inspect") { + if (process.env.DOCKER_FAILURE_MODE === "duplicate") { + process.stdout.write(JSON.stringify([{ + Id: process.env.FULL_CONTAINER_ID, + State: { Running: false }, + Config: { + Image: "iiidev/iii:0.11.2", + Labels: { + "com.docker.compose.project": "agentmemory-3111", + "com.docker.compose.service": "iii-engine" + } + }, + HostConfig: { PortBindings: { "3111/tcp": [{ HostPort: "3111" }] } }, + Mounts: [{ Type: "bind", Source: process.env.DOCKER_DATA_DIR, Destination: "/data" }] + }])); + process.exit(0); + } + process.exit(1); +} +process.exit(0); +`, + ); + chmodSync(dockerPath, 0o755); +} + +function runDockerStop( + failureMode: "scan" | "inspect" | "duplicate", + command: "stop" | "remove" = "stop", +) { + const root = sandbox(); + const home = join(root, "home"); + const runtimeDir = join(home, ".agentmemory"); + const dataDir = join(root, "data"); + const binDir = join(root, "bin"); + const composeFile = join(root, "docker-compose.yml"); + const dockerLog = join(root, "docker.log"); + const privateBin = join(runtimeDir, "bin", "iii"); + mkdirSync(runtimeDir, { recursive: true }); + mkdirSync(dataDir, { recursive: true }); + if (command === "remove") { + mkdirSync(join(runtimeDir, "bin"), { recursive: true }); + writeFileSync(privateBin, "owned binary"); + } + installFakeDocker(binDir); + writeFileSync( + composeFile, + "services:\n iii-engine:\n image: iiidev/iii:0.11.2\n iii-init:\n image: busybox\n", + ); + const statePath = join(runtimeDir, "engine-state.json"); + writeFileSync( + statePath, + JSON.stringify({ + kind: "docker", + schemaVersion: 2, + composeFile, + projectName: "agentmemory-3111", + engineVersion: "0.11.2", + restPort: 3111, + dataDir, + ...(failureMode === "inspect" ? { containerId: "candidate-id" } : {}), + dataMountType: "bind", + dataMountSource: dataDir, + preserveContainer: false, + }), + ); + + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + "src/cli.ts", + command, + ...(command === "remove" ? ["--force", "--keep-data"] : []), + "--data-dir", + dataDir, + ], + { + cwd: process.cwd(), + encoding: "utf-8", + env: { + ...process.env, + HOME: home, + USERPROFILE: home, + CI: "1", + PATH: `${binDir}:${process.env.PATH ?? ""}`, + DOCKER_LOG: dockerLog, + DOCKER_FAILURE_MODE: failureMode, + DOCKER_DATA_DIR: dataDir, + FULL_CONTAINER_ID, + }, + }, + ); + + return { + result, + statePath, + privateBin, + dockerLog: existsSync(dockerLog) ? readFileSync(dockerLog, "utf-8") : "", + }; +} + +function runInstanceRemove(instanceArgs = ["--instance", "1"]) { + const root = sandbox(); + const home = join(root, "home"); + const privateBin = join(home, ".agentmemory", "bin", "iii"); + const dataBase = join(root, "data"); + mkdirSync(join(home, ".agentmemory", "bin"), { recursive: true }); + writeFileSync(privateBin, "shared binary"); + + const result = spawnSync( + process.execPath, + [ + "--import", + "tsx", + "src/cli.ts", + "remove", + ...instanceArgs, + "--data-dir", + dataBase, + "--force", + "--keep-data", + ], + { + cwd: process.cwd(), + encoding: "utf-8", + env: { + ...process.env, + HOME: home, + USERPROFILE: home, + CI: "1", + }, + }, + ); + return { result, privateBin }; +} + +function runNativeRemoveWithWorkerFailure() { + const root = sandbox(); + const home = join(root, "home"); + const runtimeDir = join(home, ".agentmemory"); + const privateBin = join(runtimeDir, "bin", "iii"); + const engineState = join(runtimeDir, "engine-state.json"); + const enginePidfile = join(runtimeDir, "iii.pid"); + const workerPidfile = join(runtimeDir, "worker.pid"); + const killLog = join(root, "kill.log"); + const preload = join(root, "deny-worker-signal.mjs"); + mkdirSync(join(runtimeDir, "bin"), { recursive: true }); + writeFileSync(privateBin, "owned binary"); + writeFileSync(enginePidfile, "424243\n"); + writeFileSync(workerPidfile, "424242\n"); + writeFileSync( + engineState, + JSON.stringify({ + kind: "native", + configPath: join(root, "iii-config.runtime.yaml"), + restPort: 3111, + }), + ); + writeFileSync( + preload, + `import { appendFileSync } from "node:fs"; +const originalKill = process.kill.bind(process); +process.kill = (pid, signal) => { + if (pid === 424242 || pid === 424243) { + appendFileSync(process.env.KILL_LOG, String(pid) + ":" + String(signal) + "\\n"); + const error = new Error(pid === 424242 ? "worker denied" : "engine missing"); + error.code = pid === 424242 ? "EPERM" : "ESRCH"; + throw error; + } + return originalKill(pid, signal); +}; +`, + ); + + const result = spawnSync( + process.execPath, + [ + "--import", + preload, + "--import", + "tsx", + "src/cli.ts", + "remove", + "--force", + "--keep-data", + ], + { + cwd: process.cwd(), + encoding: "utf-8", + env: { + ...process.env, + HOME: home, + USERPROFILE: home, + CI: "1", + KILL_LOG: killLog, + }, + }, + ); + + return { + result, + privateBin, + engineState, + enginePidfile, + workerPidfile, + killLog: existsSync(killLog) ? readFileSync(killLog, "utf-8") : "", + }; +} + +afterEach(() => { + for (const path of sandboxes.splice(0)) { + rmSync(path, { recursive: true, force: true }); + } +}); + +describe("Docker lifecycle discovery", () => { + it.each(["scan", "inspect"] as const)( + "preserves ownership and performs no cleanup when Docker %s fails", + (failureMode) => { + const { result, statePath, dockerLog } = runDockerStop(failureMode); + + expect(result.status).toBe(1); + expect(existsSync(statePath)).toBe(true); + expect(dockerLog).not.toMatch(/\bcompose\b.*\brm\b/); + expect(`${result.stdout}\n${result.stderr}`).toContain("cannot be verified"); + }, + ); + + it("deduplicates short and full IDs for the same inspected container", () => { + const { result, statePath, dockerLog } = runDockerStop("duplicate"); + + expect(result.status).toBe(0); + expect(existsSync(statePath)).toBe(true); + expect(dockerLog).not.toMatch(/\bcompose\b.*\brm\b/); + expect(JSON.parse(readFileSync(statePath, "utf-8")).containerId).toBe( + FULL_CONTAINER_ID, + ); + }); + + it("removes shared assets while preserving validated Docker recovery state with --keep-data", () => { + const { result, statePath, privateBin, dockerLog } = runDockerStop( + "duplicate", + "remove", + ); + + expect(result.status).toBe(0); + expect(existsSync(statePath)).toBe(true); + expect(existsSync(privateBin)).toBe(false); + expect(dockerLog).not.toMatch(/\bcompose\b.*\brm\b/); + }); +}); + +describe("native removal shutdown", () => { + it("stops the worker first and aborts all cleanup when it cannot stop", () => { + const result = runNativeRemoveWithWorkerFailure(); + + expect(result.result.status).toBe(1); + expect(result.killLog).toBe("424242:SIGTERM\n"); + expect(existsSync(result.workerPidfile)).toBe(true); + expect(existsSync(result.enginePidfile)).toBe(true); + expect(existsSync(result.engineState)).toBe(true); + expect(existsSync(result.privateBin)).toBe(true); + }); + + it("refuses an instance-scoped global uninstall before deleting shared assets", () => { + const { result, privateBin } = runInstanceRemove(); + + expect(result.status).toBe(1); + expect(existsSync(privateBin)).toBe(true); + expect(`${result.stdout}\n${result.stderr}`).toContain("stop --instance 1"); + }); + + it.each([ + ["out-of-range", ["--instance", "51"]], + ["non-integer", ["--instance", "nope"]], + ["partial integer", ["--instance", "1nope"]], + ["missing value", ["--instance"]], + ["missing inline value", ["--instance="]], + ])( + "rejects an %s instance before deleting shared assets", + (_label, instanceArgs) => { + const { result, privateBin } = runInstanceRemove(instanceArgs); + + expect(result.status).toBe(1); + expect(existsSync(privateBin)).toBe(true); + expect(`${result.stdout}\n${result.stderr}`).toContain( + "--instance must be an integer between 0 and 50", + ); + }, + ); + + it.each([ + ["separate", ["--instance", "0"]], + ["inline", ["--instance=0"]], + ])("accepts valid instance zero in %s form", (_label, instanceArgs) => { + const { result, privateBin } = runInstanceRemove(instanceArgs); + + expect(result.status).toBe(0); + expect(existsSync(privateBin)).toBe(false); + }); + + it("accepts a valid nonzero inline instance", () => { + const { result, privateBin } = runInstanceRemove(["--instance=1"]); + + expect(result.status).toBe(1); + expect(existsSync(privateBin)).toBe(true); + expect(`${result.stdout}\n${result.stderr}`).toContain("stop --instance 1"); + }); +}); diff --git a/test/cli-remove.test.ts b/test/cli-remove.test.ts index 2484d4f882..27a3feea4c 100644 --- a/test/cli-remove.test.ts +++ b/test/cli-remove.test.ts @@ -20,6 +20,8 @@ let sandbox: string; function ctx(overrides: Partial = {}): RemoveContext { return { home: sandbox, + runtimeDir: join(sandbox, ".agentmemory"), + dataDir: join(sandbox, ".agentmemory", "data"), pinnedVersion: "0.11.2", localBinIiiVersion: null, connectManifest: null, @@ -147,6 +149,51 @@ describe("buildRemovePlan", () => { expect(item.alwaysAsk).toBe(false); expect(item.description).toContain("private install"); }); + + it.each([ + { + label: "instance", + runtimeDir: "instance-2", + selectedDataDir: "instance-2", + }, + { + label: "custom data", + runtimeDir: ".agentmemory", + selectedDataDir: "custom-data", + }, + ])("targets the resolved $label runtime and data roots", ({ runtimeDir, selectedDataDir }) => { + const resolvedRuntimeDir = join(sandbox, runtimeDir); + const resolvedDataDir = join(sandbox, selectedDataDir); + mkdirSync(resolvedRuntimeDir, { recursive: true }); + mkdirSync(resolvedDataDir, { recursive: true }); + writeFileSync(join(resolvedRuntimeDir, "iii.pid"), "101\n"); + writeFileSync(join(resolvedRuntimeDir, "worker.pid"), "102\n"); + writeFileSync(join(resolvedRuntimeDir, "engine-state.json"), "{}"); + writeFileSync(join(resolvedDataDir, "iii-config.runtime.yaml"), "# generated"); + + const context = { + ...ctx(), + runtimeDir: resolvedRuntimeDir, + dataDir: resolvedDataDir, + } as RemoveContext & { runtimeDir: string; dataDir: string }; + const plan = buildRemovePlan(context, { force: false, keepData: false }); + + expect(plan.find((item) => item.id === "pidfile")?.path).toBe( + join(resolvedRuntimeDir, "iii.pid"), + ); + expect(plan.find((item) => item.id === "worker-pidfile")?.path).toBe( + join(resolvedRuntimeDir, "worker.pid"), + ); + expect(plan.find((item) => item.id === "engine-state")?.path).toBe( + join(resolvedRuntimeDir, "engine-state.json"), + ); + expect(plan.find((item) => item.id === "runtime-config")?.path).toBe( + join(resolvedDataDir, "iii-config.runtime.yaml"), + ); + expect(plan.find((item) => item.id === "data-dir")?.path).toBe( + resolvedDataDir, + ); + }); }); describe("formatPlan", () => { diff --git a/test/docker-port-config.test.ts b/test/docker-port-config.test.ts new file mode 100644 index 0000000000..2bcc55b62a --- /dev/null +++ b/test/docker-port-config.test.ts @@ -0,0 +1,67 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +const root = join(import.meta.dirname, ".."); +const compose = readFileSync(join(root, "docker-compose.yml"), "utf8"); +const engineConfig = readFileSync(join(root, "iii-config.docker.yaml"), "utf8"); + +function workerBlock(name: string): string { + const start = engineConfig.indexOf(` - name: ${name}`); + if (start === -1) return ""; + const next = engineConfig.indexOf("\n - name:", start + 1); + return engineConfig.slice(start, next === -1 ? undefined : next); +} + +describe("Docker engine port configuration", () => { + it("passes the resolved port quartet into the engine container", () => { + expect(compose).toContain('III_REST_PORT: "${III_REST_PORT:-3111}"'); + expect(compose).toContain('III_STREAM_PORT: "${III_STREAM_PORT:-3112}"'); + expect(compose).toContain('III_VIEWER_PORT: "${III_VIEWER_PORT:-3113}"'); + expect(compose).toContain('III_ENGINE_PORT: "${III_ENGINE_PORT:-49134}"'); + }); + + it("publishes the configured host and container ports", () => { + expect(compose).toContain( + '"127.0.0.1:${III_REST_PORT:-3111}:${III_REST_PORT:-3111}"', + ); + expect(compose).toContain( + '"127.0.0.1:${III_STREAM_PORT:-3112}:${III_STREAM_PORT:-3112}"', + ); + expect(compose).toContain( + '"127.0.0.1:${III_ENGINE_PORT:-49134}:${III_ENGINE_PORT:-49134}"', + ); + expect(compose).toContain( + '"127.0.0.1:${AGENTMEMORY_METRICS_PORT:-9464}:9464"', + ); + }); + + it("runs CLI-managed bind mounts as the host user without recursive chown", () => { + expect(compose).toContain( + 'user: "${AGENTMEMORY_DOCKER_UID:-65532}:${AGENTMEMORY_DOCKER_GID:-65532}"', + ); + expect(compose).toContain("AGENTMEMORY_DOCKER_SKIP_CHOWN"); + }); + + it("configures every iii listener from the same environment", () => { + expect(workerBlock("iii-http")).toContain("port: ${III_REST_PORT:3111}"); + expect(workerBlock("iii-stream")).toContain("port: ${III_STREAM_PORT:3112}"); + + const manager = workerBlock("iii-worker-manager"); + expect(manager).toContain("port: ${III_ENGINE_PORT:49134}"); + expect(manager).toContain("host: 0.0.0.0"); + }); + + it("leaves worker ownership to the host CLI", () => { + expect(engineConfig).not.toContain("- name: iii-exec"); + expect(engineConfig).not.toContain("node dist/index.mjs"); + }); + + it("keeps REST and viewer CORS aligned with overridden ports", () => { + const http = workerBlock("iii-http"); + expect(http).toContain("http://localhost:${III_REST_PORT:3111}"); + expect(http).toContain("http://127.0.0.1:${III_REST_PORT:3111}"); + expect(http).toContain("http://localhost:${III_VIEWER_PORT:3113}"); + expect(http).toContain("http://127.0.0.1:${III_VIEWER_PORT:3113}"); + }); +}); diff --git a/test/engine-config.test.ts b/test/engine-config.test.ts index 67ac8eec7c..152f41bdc6 100644 --- a/test/engine-config.test.ts +++ b/test/engine-config.test.ts @@ -1,6 +1,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import { renderEngineConfig } from "../src/cli/engine-config.js"; function shippedConfig(file: string): string { return readFileSync(join(process.cwd(), file), "utf8"); @@ -29,3 +30,53 @@ describe("bundled iii engine config", () => { }, ); }); + +describe("renderEngineConfig", () => { + it("stores engine state in the resolved data directory", () => { + const source = readFileSync( + join(import.meta.dirname, "..", "iii-config.yaml"), + "utf8", + ); + const dataDir = join("/var", "lib", "agentmemory"); + + const rendered = renderEngineConfig(source, { dataDir }); + + expect(rendered).toContain( + `file_path: '${join(dataDir, "state_store.db")}'`, + ); + expect(rendered).toContain( + `file_path: '${join(dataDir, "stream_store")}'`, + ); + expect(rendered).not.toContain("./data/"); + }); + + it("moves the complete native port quartet from one REST override", () => { + const source = readFileSync( + join(import.meta.dirname, "..", "iii-config.yaml"), + "utf8", + ); + + const rendered = renderEngineConfig(source, { + dataDir: "/tmp/agentmemory", + ports: { + restPort: 3211, + streamPort: 3212, + viewerPort: 3213, + enginePort: 49234, + }, + }); + + expect(rendered).toMatch( + /- name: iii-http\n\s+config:\n\s+port: 3211/, + ); + expect(rendered).toMatch( + /- name: iii-stream\n\s+config:\n\s+port: 3212/, + ); + expect(rendered).toContain( + 'allowed_origins: ["http://localhost:3211", "http://localhost:3213", "http://127.0.0.1:3211", "http://127.0.0.1:3213"]', + ); + expect(rendered).toMatch( + /- name: iii-worker-manager\n\s+config:\n\s+port: 49234\n\s+host: 127\.0\.0\.1/, + ); + }); +}); diff --git a/test/engine-launch.test.ts b/test/engine-launch.test.ts new file mode 100644 index 0000000000..cf12f687cd --- /dev/null +++ b/test/engine-launch.test.ts @@ -0,0 +1,178 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + agentmemoryHome, + dockerComposeArgs, + dockerProjectName, + isBundledConfig, + legacyDataMigrations, + resolveEngineCwd, + rewriteBundledConfig, + runtimeConfigPath, +} from "../src/cli/engine-launch.js"; + +const HOME = "/Users/test"; + +describe("engine-launch path resolution", () => { + it("agentmemoryHome anchors config and runtimeConfigPath scopes instance state", () => { + expect(agentmemoryHome(HOME)).toBe(join(HOME, ".agentmemory")); + expect(runtimeConfigPath("/var/lib/agentmemory/instance-1")).toBe( + join("/var/lib/agentmemory/instance-1", "iii-config.runtime.yaml"), + ); + }); + + it("isBundledConfig matches both package config locations", () => { + const dist = "/opt/pkg/dist"; + expect(isBundledConfig(join(dist, "iii-config.yaml"), dist)).toBe(true); + expect(isBundledConfig(join(dist, "..", "iii-config.yaml"), dist)).toBe(true); + expect(isBundledConfig("/opt/pkg/iii-config.yaml", dist)).toBe(true); + expect(isBundledConfig(join(HOME, ".agentmemory", "iii-config.yaml"), dist)).toBe(false); + expect(isBundledConfig("/some/project/iii-config.yaml", dist)).toBe(false); + }); + + it("resolveEngineCwd keeps the invocation cwd for repo-local configs", () => { + const repo = "/work/agentmemory"; + expect(resolveEngineCwd(join(repo, "iii-config.yaml"), repo, HOME)).toBe(repo); + }); + + it("resolveEngineCwd anchors bundled configs and preserves custom config roots", () => { + const repo = "/work/some-project"; + expect(resolveEngineCwd("/opt/pkg/dist/iii-config.yaml", repo, HOME, true)).toBe( + join(HOME, ".agentmemory"), + ); + expect( + resolveEngineCwd(join(HOME, ".agentmemory", "iii-config.yaml"), repo, HOME), + ).toBe(join(HOME, ".agentmemory")); + expect(resolveEngineCwd("/etc/custom-iii.yaml", repo, HOME)).toBe( + "/etc", + ); + expect(resolveEngineCwd("custom-iii.yaml", repo, HOME)).toBe(repo); + }); + + it("scopes Docker compose commands by REST-port project", () => { + expect(dockerProjectName(3211)).toBe("agentmemory-3211"); + expect( + dockerComposeArgs("/opt/agentmemory/docker-compose.yml", "agentmemory-3211", [ + "up", + "-d", + ]), + ).toEqual([ + "compose", + "-p", + "agentmemory-3211", + "-f", + "/opt/agentmemory/docker-compose.yml", + "up", + "-d", + ]); + }); + + it("legacyDataMigrations targets the resolved platform data dir", () => { + const resolvedDataDir = "/var/lib/agentmemory"; + const migrations = legacyDataMigrations("/work/proj", HOME, resolvedDataDir); + expect(migrations).toEqual([ + { + from: join("/work/proj", "data", "state_store.db"), + to: join(resolvedDataDir, "state_store.db"), + }, + { + from: join("/work/proj", "data", "stream_store"), + to: join(resolvedDataDir, "stream_store"), + }, + ]); + }); +}); + +describe("rewriteBundledConfig", () => { + const SAMPLE = [ + " file_path: ./data/state_store.db", + " file_path: ./data/stream_store", + " - name: iii-exec", + " config:", + " watch:", + " - src/**/*.ts", + " exec:", + " - node dist/index.mjs", + ].join("\n"); + + it("substitutes data paths and removes bundled worker supervision", () => { + const out = rewriteBundledConfig(SAMPLE, HOME, "/usr/bin/node", "/opt/pkg/dist/index.mjs"); + expect(out).toContain( + `file_path: '${join(HOME, ".agentmemory", "data", "state_store.db")}'`, + ); + expect(out).toContain( + `file_path: '${join(HOME, ".agentmemory", "data", "stream_store")}'`, + ); + expect(out).not.toContain("./data/"); + expect(out).not.toContain("- name: iii-exec"); + expect(out).not.toContain("src/**/*.ts"); + }); + + it("preserves unrelated commands in the bundled iii-exec worker", () => { + const bundled = [ + "workers:", + " - name: iii-exec", + " config:", + " exec:", + " - node dist/index.mjs", + " - node scripts/other-worker.mjs", + ].join("\n"); + + const out = rewriteBundledConfig( + bundled, + HOME, + "/usr/bin/node", + "/opt/pkg/dist/index.mjs", + ); + expect(out).toContain("- name: iii-exec"); + expect(out).toContain("- node scripts/other-worker.mjs"); + expect(out).not.toContain("- node dist/index.mjs"); + }); + + it("escapes apostrophes in paths for single-quoted YAML", () => { + const out = rewriteBundledConfig( + SAMPLE, + "/Users/o'brien", + "/usr/bin/node", + "/opt/pkg/dist/index.mjs", + ); + expect(out).toContain("o''brien"); + }); + + it("rewrites the real bundled config with no relative paths left", () => { + const raw = readFileSync(join(import.meta.dirname, "..", "iii-config.yaml"), "utf-8"); + const out = rewriteBundledConfig(raw, HOME, process.execPath, "/opt/pkg/dist/index.mjs"); + expect(out).not.toContain("./data/"); + expect(out).not.toContain("src/**/*.ts"); + expect(out).not.toContain("- node dist/index.mjs"); + expect(out).not.toContain("- name: iii-exec"); + expect(out).toContain(join(HOME, ".agentmemory", "data", "state_store.db")); + expect(out).toContain(join(HOME, ".agentmemory", "data", "stream_store")); + }); + + it("uses the CLI-resolved data directory and port quartet", () => { + const raw = readFileSync(join(import.meta.dirname, "..", "iii-config.yaml"), "utf-8"); + const dataDir = "/var/lib/agentmemory"; + const out = rewriteBundledConfig( + raw, + HOME, + process.execPath, + "/opt/pkg/dist/index.mjs", + { + dataDir, + ports: { + restPort: 3211, + streamPort: 3212, + viewerPort: 3213, + enginePort: 49234, + }, + }, + ); + + expect(out).toContain(join(dataDir, "state_store.db")); + expect(out).toContain("port: 3211"); + expect(out).toContain("port: 3212"); + expect(out).toContain("port: 49234"); + }); +}); diff --git a/test/multi-instance-port.test.ts b/test/multi-instance-port.test.ts index 4343365b3e..8c8723265c 100644 --- a/test/multi-instance-port.test.ts +++ b/test/multi-instance-port.test.ts @@ -5,6 +5,7 @@ const PORT_ENVS = [ "III_REST_PORT", "III_STREAM_PORT", "III_STREAMS_PORT", + "III_VIEWER_PORT", "III_ENGINE_PORT", "III_ENGINE_URL", ] as const; @@ -33,6 +34,7 @@ describe("multi-instance port auto-derive (#750)", () => { const cfg = loadConfig(); expect(cfg.restPort).toBe(3111); expect(cfg.streamsPort).toBe(3112); + expect(cfg.viewerPort).toBe(3113); expect(cfg.engineUrl).toBe("ws://localhost:49134"); }); @@ -41,6 +43,7 @@ describe("multi-instance port auto-derive (#750)", () => { const cfg = loadConfig(); expect(cfg.restPort).toBe(3211); expect(cfg.streamsPort).toBe(3212); + expect(cfg.viewerPort).toBe(3213); expect(cfg.engineUrl).toBe("ws://localhost:49234"); }); @@ -76,6 +79,13 @@ describe("multi-instance port auto-derive (#750)", () => { expect(cfg.engineUrl).toBe("ws://localhost:55555"); }); + it("explicit III_VIEWER_PORT pins the viewer", () => { + process.env["III_REST_PORT"] = "3211"; + process.env["III_VIEWER_PORT"] = "7777"; + const cfg = loadConfig(); + expect(cfg.viewerPort).toBe(7777); + }); + it("legacy III_ENGINE_URL overrides derivation entirely", () => { process.env["III_REST_PORT"] = "3211"; process.env["III_ENGINE_URL"] = "ws://remote-host:49999"; diff --git a/test/process-state.test.ts b/test/process-state.test.ts new file mode 100644 index 0000000000..d4c599d0aa --- /dev/null +++ b/test/process-state.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; +import { processStatIsRunning } from "../src/cli/process-state.js"; + +describe("processStatIsRunning", () => { + it("treats zombie processes as stopped", () => { + expect(processStatIsRunning("Z+")).toBe(false); + expect(processStatIsRunning(" Z ")).toBe(false); + }); + + it("keeps active process states running", () => { + expect(processStatIsRunning("S+")).toBe(true); + expect(processStatIsRunning("R")).toBe(true); + }); +}); diff --git a/test/runtime-paths.test.ts b/test/runtime-paths.test.ts new file mode 100644 index 0000000000..231fb8ca32 --- /dev/null +++ b/test/runtime-paths.test.ts @@ -0,0 +1,29 @@ +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { runtimeMetadataPath } from "../src/runtime-paths.js"; + +describe("runtime metadata paths", () => { + it("uses an explicit runtime directory so instances do not share metadata", () => { + expect( + runtimeMetadataPath("engine-state.json", { + env: { AGENTMEMORY_RUNTIME_DIR: "/var/lib/agentmemory/instance-2" }, + home: "/home/test", + }), + ).toBe(join("/var/lib/agentmemory/instance-2", "engine-state.json")); + }); + + it("keeps instance-zero lifecycle state canonical across data-dir changes", () => { + expect( + runtimeMetadataPath("engine-state.json", { + env: { AGENTMEMORY_DATA_DIR: "/var/lib/agentmemory/custom" }, + home: "/home/test", + }), + ).toBe(join("/home/test", ".agentmemory", "engine-state.json")); + }); + + it("keeps the legacy home fallback for direct worker launches", () => { + expect(runtimeMetadataPath("worker.pid", { env: {}, home: "/home/test" })).toBe( + join("/home/test", ".agentmemory", "worker.pid"), + ); + }); +}); diff --git a/test/startup-stderr.test.ts b/test/startup-stderr.test.ts new file mode 100644 index 0000000000..ee3edaacb4 --- /dev/null +++ b/test/startup-stderr.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { createStartupStderrCapture } from "../src/cli/startup-stderr.js"; + +describe("startup stderr capture", () => { + it("exposes bounded stderr before the child process exits", () => { + const capture = createStartupStderrCapture(5); + + capture.append(Buffer.from("abc")); + capture.append(Buffer.from("def")); + + expect(capture.text()).toBe("abcde"); + }); +}); diff --git a/test/stop-worker-pidfile.test.ts b/test/stop-worker-pidfile.test.ts index c70aaf1af9..fe018fca3c 100644 --- a/test/stop-worker-pidfile.test.ts +++ b/test/stop-worker-pidfile.test.ts @@ -3,8 +3,8 @@ import { readFileSync } from "node:fs"; // #640 + #474: stop must also kill the worker process, not just the // iii engine. We expose the worker pidfile from src/index.ts and read it -// from src/cli.ts. Static check that both files agree on the path -// (~/.agentmemory/worker.pid) and that stop reads it. +// from src/cli.ts. Static check that both files use the resolved runtime +// metadata directory and that stop reads it. describe("stop reaps the worker process (#640, #474)", () => { it("src/index.ts writes worker.pid alongside iii.pid", () => { const source = readFileSync("src/index.ts", "utf-8"); @@ -26,10 +26,24 @@ describe("stop reaps the worker process (#640, #474)", () => { expect(source).toMatch(/Stopping agentmemory worker/); }); - it("both files agree on the pidfile path: ~/.agentmemory/worker.pid", () => { + it("both files agree on the instance-scoped worker pidfile", () => { const indexSrc = readFileSync("src/index.ts", "utf-8"); const cliSrc = readFileSync("src/cli.ts", "utf-8"); - expect(indexSrc).toMatch(/\.agentmemory["'].*worker\.pid|"worker\.pid"/); - expect(cliSrc).toMatch(/\.agentmemory["'].*worker\.pid|"worker\.pid"/); + expect(indexSrc).toContain('runtimeMetadataPath("worker.pid")'); + expect(cliSrc).toContain('runtimeMetadataPath("worker.pid")'); + }); + + it("validates Docker ownership and flushes the worker before stopping the container", () => { + const source = readFileSync("src/cli.ts", "utf-8"); + const start = source.indexOf("async function stopDockerEngine"); + const end = source.indexOf("async function runStop", start); + const body = source.slice(start, end); + + expect(body.indexOf("inspectOwnedDockerEngine(state)")) + .toBeLessThan(body.indexOf("readWorkerPidfile()")); + expect(body.indexOf("readWorkerPidfile()")) + .toBeLessThan(body.indexOf('["stop", "--time", "10", inspection.containerId]')); + expect(body).toContain("persistDockerInspection(state, inspection)"); + expect(body).toContain("writeEngineState(resolvedState)"); }); });