diff --git a/.github/scripts/detect-rebake-lanes.py b/.github/scripts/detect-rebake-lanes.py index 5e9036236..d5578cab9 100755 --- a/.github/scripts/detect-rebake-lanes.py +++ b/.github/scripts/detect-rebake-lanes.py @@ -178,7 +178,7 @@ # exactly mirroring `web/`'s treatment. A source-only orchestrator change # rebakes the orchestrator (and the other images in the matrix; that's the same # coarse behavior `web/` has always had). -IMAGES_PATHS = ["docker/", "web/", "orchestrator/", "deploy/migrations/"] + BINARY_COMMON +IMAGES_PATHS = ["docker/", "web/", "orchestrator/", "mcp-worker/", "deploy/migrations/"] + BINARY_COMMON # The build INSTRUCTIONS, as opposed to the sources they copy in. Only a change # here can break an image in a way a native build does not already catch, so # only a change here needs the PR-time `build-images` lane: `tests (linux)` @@ -249,7 +249,7 @@ ] -# ── PR test-lane gating (ci.yml + ci-macos-vz.yml) ───────────────────── +# ── PR test-lane gating (ci.yml) ───────────────────── # These gate the CI *test* lanes (not the bake lanes above) so a change only # runs the lanes it can affect. Protos live in the engram-protocol crate, so a # proto change is already in every Rust binary's closure (→ cc, host_binaries, @@ -263,7 +263,10 @@ # touching ci.yml (ci-macos-vz.yml was folded into # ci.yml; its old entry here was a dead path). ".github/actions/", - ".github/scripts/detect-rebake-lanes.py"] + ".github/scripts/detect-rebake-lanes.py", + ".github/scripts/tests/", + ".github/scripts/mcp-acceptance-", + ".github/scripts/mcp-stack-"] PROTO_PATHS = ["crates/engram-protocol/proto/", "buf.gen.yaml"] WEB_PATHS = ["web/", "orchestrator/packages/spec-document/"] # The public docs + landing site (site/, Astro + Starlight). Its content is @@ -279,10 +282,11 @@ # version it ships on. They were absent here, so a change to either skipped # this lane entirely — which is half of how prod came to run a Bun the tests # never saw (the other half was the Dockerfile floating on `oven/bun:1`). -ORCH_PATHS = ["orchestrator/", +MCP_STACK_FIXTURE_PATHS = ["tests/fixtures/mcp-stack/"] +ORCH_PATHS = ["orchestrator/", "mcp-worker/", "docker/orchestrator.Dockerfile", ".bun-version", - "crates/engram-egress-proxy/policy/google-credential-denylist.json"] + "crates/engram-egress-proxy/policy/google-credential-denylist.json"] + MCP_STACK_FIXTURE_PATHS # A lockfile/manifest/toolchain bump recompiles the whole workspace. RUST_COMMON = ["Cargo.lock", "Cargo.toml", "rust-toolchain.toml"] @@ -499,6 +503,7 @@ def main(): e2e_core = ( bool(cc & e2e_closure) or any_path(changed, E2E_PATHS) + or any_path(changed, ["sandbox-netstack/"]) or any_path(changed, HARNESS_PATHS) ) e2e_orchestrator = e2e_core or any_path(changed, ORCH_PATHS) @@ -550,14 +555,27 @@ def main(): # lint + tests(linux) + the two macOS lanes build/test the whole workspace: # any workspace crate (cc — protos included via engram-protocol), a # lockfile/toolchain bump, or a CI-definition change. + fixture_changed = any_path(changed, ["tests/fixtures/mcp-model-cli/"]) test_rust = ( - ci_self + fixture_changed + or ci_self or bool(cc) or any_path(changed, RUST_COMMON) or any_path(changed, HARNESS_PATHS) + or any_path(changed, ["mcp-worker/", "sandbox-netstack/"]) + ) + test_mcp_worker = ci_self or proto or any_path(changed, ["mcp-worker/", "docker/mcp-worker.Dockerfile"] + MCP_STACK_FIXTURE_PATHS) + # Self-contained producers use one selector: no skipped artifact dependency. + test_mcp_physical = ( + ci_self or proto or test_rust or any_path(changed, ORCH_PATHS + [ + "crates/", ".sqlx/", "deploy/migrations/", "docker/", "deploy/bundles/", + "deploy/demo/", "deploy/kernel/", "third_party/firecracker", + "sandbox-netstack/", ".bun-version", ".dockerignore", ".cargo/config.toml", + ]) ) + test_netstack = test_rust # musl cross-compile of the FC host binaries — same closure as host_binaries. - test_cross = ci_self or host_binaries + test_cross = ci_self or host_binaries or fixture_changed or harness_changed # Firecracker integration lane. Gated on the e2e binary CLOSURE (the FC # backend + host-agent + their deps — the Rust the FC integration tests # exercise; engram-sandbox-firecracker is in host-agent's closure, and the @@ -567,7 +585,8 @@ def main(): # fixtures and mount bundles via the sandbox crates already in the closure; # tripping FC on a bundle-payload edit was a needless ~KVM lane. The # e2e-STACK lane still gates on `e2e`, so bundle staging is validated there. - test_fc = ci_self or bool(cc & e2e_closure) or fc_fork or any_path(changed, FC_POOL_PATHS) + test_fc = ci_self or bool(cc & e2e_closure) or fc_fork or any_path( + changed, FC_POOL_PATHS + ["sandbox-netstack/"]) # web / orchestrator: their own sources or the protos they codegen from. # (The `engrams` CLI is a workspace crate: test_rust covers it.) test_web = ci_self or proto or any_path(changed, WEB_PATHS) @@ -576,9 +595,9 @@ def main(): test_buf = ci_self or proto test_integrations_cli = ci_self or any_path(changed, INTEGRATIONS_CLI_PATHS) # ADR 0122: the deploy-config lane (helm lint/template + terraform - # fmt/validate). Same path set as the bake-side tf_or_helm flag, - # plus ci_self like every test lane. - test_deploy = ci_self or tf_or_helm + # fmt/validate + actual nginx callback canary). The web runtime image + # also changes the effective nginx behavior, without changing the chart. + test_deploy = ci_self or tf_or_helm or any_path(changed, ["docker/web.Dockerfile"]) # The site lane: leak lint + astro build, build-only (no deploy on PRs). test_site = ci_self or any_path(changed, SITE_PATHS) # ADR 0098 R-CoSim: the coordinator↔host boundary sim — its own (spanning) @@ -634,14 +653,17 @@ def image_flag(name): ]) if name == "orchestrator": return bake_all_l or proto_l or any_path( - cs, ["orchestrator/", "docker/orchestrator.Dockerfile"]) + cs, ["orchestrator/", "mcp-worker/src/", "mcp-worker/package.json", + "mcp-worker/package-lock.json", "docker/orchestrator.Dockerfile"]) + if name == "mcp-worker": + return bake_all_l or any_path(cs, ["mcp-worker/", "docker/mcp-worker.Dockerfile"]) raise ValueError(name) image_flags = {name: image_flag(name) for name in - ["coordinator", "host-agent", "host-operator", "web", "orchestrator"]} + ["coordinator", "host-agent", "host-operator", "web", "orchestrator", "mcp-worker"]} # Stable matrix order; the bake job consumes this as `fromJSON`. images_matrix = [name for name in - ["coordinator", "host-agent", "host-operator", "web", "orchestrator"] + ["coordinator", "host-agent", "host-operator", "web", "orchestrator", "mcp-worker"] if image_flags[name]] # The deploy-dispatch gate must fire whenever ANY container lane trips — # level-triggered lanes can trip when the push range alone would not @@ -687,8 +709,11 @@ def b(v): f.write(f"e2e={b(e2e)}\n") f.write(f"e2e_matrix={json.dumps({'include': e2e_matrix})}\n") f.write(f"e2e_full_matrix={json.dumps({'include': e2e_full_matrix})}\n") - # PR test lanes (ci.yml + ci-macos-vz.yml gate on these). + # PR test lanes (ci.yml gates on these). f.write(f"test_rust={b(test_rust)}\n") + f.write(f"test_mcp_worker={b(test_mcp_worker)}\n") + f.write(f"test_mcp_physical={b(test_mcp_physical)}\n") + f.write(f"test_netstack={b(test_netstack)}\n") f.write(f"test_cross={b(test_cross)}\n") f.write(f"test_fc={b(test_fc)}\n") f.write(f"test_web={b(test_web)}\n") diff --git a/.github/scripts/mcp-acceptance-fc-closed.sh b/.github/scripts/mcp-acceptance-fc-closed.sh new file mode 100644 index 000000000..9276afce8 --- /dev/null +++ b/.github/scripts/mcp-acceptance-fc-closed.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Run the installed CLOSED namespace check on the isolated Linux CI worker. +set -euo pipefail +[[ $(uname -s) == Linux ]] || { echo 'CLOSED namespace gate requires Linux' >&2; exit 1; } +for tool in cargo python3 sudo ip sysctl nft; do + command -v "$tool" >/dev/null || { echo "Missing required tool: $tool" >&2; exit 1; } +done +name=managed::network::linux::tests::actual_closed_namespace_readback_detects_route_tampering +state=$(mktemp -d) +trap 'rm -rf "$state"' EXIT +# Cargo builds as the runner; only its current libtest executable runs as root. +export CARGO_TARGET_DIR="$PWD/target" +cargo test --locked -p engram-sandbox-firecracker --lib --no-run --message-format=json > "$state/artifacts.jsonl" +artifact=$(ARTIFACT_LIST="$state/artifacts.jsonl" python3 - "$PWD" "${CARGO_TARGET_DIR:-$PWD/target}" < /dev/null <<'PY' +import json, os, pathlib, stat, sys +workspace, target = map(pathlib.Path, sys.argv[1:]) +workspace = workspace.resolve() +target = (workspace / target).resolve() +assert target.is_relative_to(workspace), 'target must be inside this checkout' +with open(os.environ['ARTIFACT_LIST']) as stream: + paths = [m['executable'] for line in stream if (m := json.loads(line)).get('reason') == 'compiler-artifact' + and m.get('executable') and m['target']['name'] == 'engram_sandbox_firecracker' + and pathlib.Path(m['target']['src_path']).resolve() == workspace / 'crates/engram-sandbox-firecracker/src/lib.rs' + and m['profile']['test']] +assert len(paths) == 1, 'one current Firecracker libtest artifact is required' +p = pathlib.Path(paths[0]).resolve(strict=True) +s = p.stat() +assert p.is_relative_to(target) and stat.S_ISREG(s.st_mode) and s.st_uid == os.getuid() +assert not s.st_mode & 0o022 and os.access(p, os.X_OK), 'unsafe test artifact' +print(p) +PY +) +# Check selection before sudo, then require a positive result from the same artifact. +"$artifact" --ignored --exact "$name" --list > "$state/selected" +grep -Fxq "$name: test" "$state/selected" +grep -Fxq '1 test, 0 benchmarks' "$state/selected" +[[ $(grep -c ': test$' "$state/selected") == 1 ]] || { echo 'Expected exactly one CLOSED namespace test' >&2; exit 1; } +sudo env PATH=/usr/sbin:/usr/bin:/sbin:/bin "$artifact" --ignored --exact "$name" --nocapture 2>&1 | tee "$state/result" +grep -Fxq 'running 1 test' "$state/result" +grep -Fxq "test $name ... ok" "$state/result" +grep -Eq '^test result: ok\. 1 passed; 0 failed; 0 ignored; 0 measured; [0-9]+ filtered out;' "$state/result" diff --git a/.github/scripts/mcp-acceptance-helper.sh b/.github/scripts/mcp-acceptance-helper.sh new file mode 100755 index 000000000..7b00646ea --- /dev/null +++ b/.github/scripts/mcp-acceptance-helper.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +root="$PWD" +artifact="$root/var/mcp-netstack-artifact" +(cd "$artifact" && test -s engram-sandbox-netstack && shasum -a 256 -c SHA256SUMS) +export ENGRAM_SANDBOX_NETSTACK_TEST_BIN="$artifact/engram-sandbox-netstack" +test -x "$ENGRAM_SANDBOX_NETSTACK_TEST_BIN" +mkdir -p var/mcp-acceptance-ci +log=$(mktemp "$root/var/mcp-acceptance-ci/helper.XXXXXXXX") +trap 'rm -f "$log"' EXIT +cargo nextest run -p engram-sandbox-vz --lib --run-ignored ignored-only \ + -E 'test(=managed::helper::tests::actual_helper_closed_lifetime_and_descriptor_isolation) or test(=managed::helper::tests::actual_helper_release_ack_and_active_dns_cleanup) or test(=managed::helper::tests::actual_helper_rejects_wrong_epoch_and_canceled_release)' \ + --no-tests fail --test-threads 1 2>&1 | tee "$log" +grep -Eq "(^|[[:space:]])3 tests run: 3 passed, [0-9]+ skipped([[:space:]]|$)" "$log" diff --git a/.github/scripts/mcp-acceptance-linux.sh b/.github/scripts/mcp-acceptance-linux.sh new file mode 100644 index 000000000..444b0b6f5 --- /dev/null +++ b/.github/scripts/mcp-acceptance-linux.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Real Linux wrappers and SDK transport. No VM, provider, or upstream server. +set -euo pipefail +[[ $(uname -s) == Linux ]] || { echo 'Linux wrapper gate requires Linux' >&2; exit 1; } +: "${CARGO_TARGET_DIR:?Set an owned absolute target directory}" +[[ "$CARGO_TARGET_DIR" == /* ]] || exit 1 +cargo build --locked --manifest-path tests/fixtures/mcp-model-cli/Cargo.toml --bin engram-mcp-model-fixture +cargo build --locked -p engram-harness-claude -p engram-harness-codex --bins +export ENGRAM_FIXTURE_BIN="$CARGO_TARGET_DIR/debug/engram-mcp-model-fixture" +export ENGRAM_TEST_CLAUDE_BIN="$CARGO_TARGET_DIR/debug/engram-harness-claude" +export ENGRAM_TEST_CODEX_BIN="$CARGO_TARGET_DIR/debug/engram-harness-codex" +for artifact in "$ENGRAM_FIXTURE_BIN" "$ENGRAM_TEST_CLAUDE_BIN" "$ENGRAM_TEST_CODEX_BIN"; do + test -x "$artifact" + file "$artifact" | grep -Fq 'ELF 64-bit' +done +test "$("$ENGRAM_FIXTURE_BIN" --fixture-version)" = 'engram-mcp-model-fixture 1' +# An example is an explicitly built test utility, never a production binary +# or a harness=false target that breaks workspace nextest discovery. +cargo build --locked -p engram-harness-claude --example linux_mcp_supervisor +# Keep generated Unix socket paths below Linux's sockaddr_un limit. +state=$(mktemp -d /tmp/mcp-linux.XXXXXXXX) +trap 'if [[ -f "$state/cleanup-complete" ]]; then rm -rf "$state"; else echo "Cleanup unconfirmed; retained owned state: $state" >&2; fi' EXIT +export ENGRAM_TEST_STATE_ROOT="$state" +cargo test --locked -p engram-harness-claude --test linux_mcp_entrypoint \ + --no-run --message-format=json > "$state/artifacts.jsonl" +export ENGRAM_TEST_DRIVER_BIN +ENGRAM_TEST_DRIVER_BIN=$(python3 -c ' +import json, sys +paths = [m["executable"] for line in sys.stdin if (m := json.loads(line)).get("reason") == "compiler-artifact" and m.get("executable") and m["target"]["name"] == "linux_mcp_entrypoint"] +assert len(paths) == 1, "one current Linux test driver is required" +print(paths[0]) +' < "$state/artifacts.jsonl") +test -x "$ENGRAM_TEST_DRIVER_BIN" +# The supervisor has an earlier bounded deadline and reaps all descendants. +# This hard outer kill is only a final bound if the supervisor itself fails. +timeout --signal=KILL 240s "$CARGO_TARGET_DIR/debug/examples/linux_mcp_supervisor" \ + --check-cleanup-and-run 2>&1 | tee "$state/entrypoint.log" +grep -Fq 'forced deadline cleanup: passed; all descendants reaped before state removal' "$state/entrypoint.log" +grep -Fq 'running 9 tests' "$state/entrypoint.log" +grep -Fq 'test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 1 filtered out' "$state/entrypoint.log" +test "$(cat "$state/cleanup-complete")" = complete diff --git a/.github/scripts/mcp-acceptance-native.sh b/.github/scripts/mcp-acceptance-native.sh new file mode 100755 index 000000000..c3cc13507 --- /dev/null +++ b/.github/scripts/mcp-acceptance-native.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${CARGO_TARGET_DIR:?Set an owned target directory}" +root="$PWD" +mkdir -p var/mcp-acceptance-ci +state=$(mktemp -d "$root/var/mcp-acceptance-ci/native.XXXXXXXX") +trap 'rm -rf "$state"' EXIT +unset ENGRAM_FIXTURE_BARRIER +cargo test --locked --manifest-path tests/fixtures/mcp-model-cli/Cargo.toml --bins --test protocol 2>&1 | tee "$state/protocol.log" +# Binary unit results cannot satisfy the protocol suite's positive-count guard. +awk ' + /Running tests\/protocol.rs / { protocol = 1; next } + protocol && /^test result: ok\. [1-9][0-9]* passed; 0 failed; 0 ignored/ { passed = 1 } + END { exit !passed } +' "$state/protocol.log" +grep -Fxq 'test evidence::tests::expired_deadline_cannot_consume_a_release ... ok' "$state/protocol.log" +cargo build --locked --manifest-path tests/fixtures/mcp-model-cli/Cargo.toml --bin engram-mcp-model-fixture +export ENGRAM_FIXTURE_BIN="$CARGO_TARGET_DIR/debug/engram-mcp-model-fixture" +test -x "$ENGRAM_FIXTURE_BIN" +test "$("$ENGRAM_FIXTURE_BIN" --fixture-version)" = 'engram-mcp-model-fixture 1' +export ENGRAM_FIXTURE_TOOL=remote_inventory +for mode in read held-call; do + export ENGRAM_FIXTURE_MODE="$mode" + export ENGRAM_FIXTURE_STATE_DIR="$state/$mode" + mkdir -m 700 "$ENGRAM_FIXTURE_STATE_DIR" + export ENGRAM_FIXTURE_PROOF="$ENGRAM_FIXTURE_STATE_DIR/proof" + cargo test --locked -p engram-harness-codex --bin engram-harness-codex \ + tests::external_mcp_static_fixture_adapter_gate -- --ignored --exact 2>&1 | tee "$state/$mode.log" + grep -Fq 'test result: ok. 1 passed; 0 failed; 0 ignored' "$state/$mode.log" +done diff --git a/.github/scripts/mcp-stack-assets.py b/.github/scripts/mcp-stack-assets.py new file mode 100644 index 000000000..80eec89f1 --- /dev/null +++ b/.github/scripts/mcp-stack-assets.py @@ -0,0 +1,950 @@ +#!/usr/bin/env python3 +"""Fixed physical MCP producer. No default engine, inherited credentials or skip path.""" +import argparse +import hashlib +import io +import ipaddress +import json +import os +from pathlib import Path +import platform +import re +import secrets +import shutil +import signal +import stat +import struct +import subprocess +import sys +import tarfile +import time + +REPO = Path(__file__).resolve().parents[2] +CI = 'tests/fixtures/mcp-stack/platform/ci' +LABEL = 'io.engrams.mcp.ci' +LEGACY_RUNTIME = 'crates/engram-coordinator/var' +DIGEST = re.compile(r'sha256:[a-f0-9]{64}') +SOURCE_IMAGES = { + 'coordinator': 'docker/coordinator.Dockerfile', + 'orchestrator': 'docker/orchestrator.Dockerfile', + 'worker': 'docker/mcp-worker.Dockerfile', + 'fixture': 'tests/fixtures/mcp-stack/platform/fixture.Dockerfile', + 'guard': 'tests/fixtures/mcp-stack/platform/guard.Dockerfile', + 'guest': 'tests/fixtures/mcp-stack/platform/guest.Dockerfile', +} +TOOLS = {'node', 'bun', 'docker', 'openssl', 'skopeo', 'gh', 'oras', 'npm'} + + +def require(ok, seam): + if not ok: + raise RuntimeError(seam) + + +def canonical(path, kind='file', owner=True): + p = Path(path) + require(p.is_absolute() and str(p) == path and p.resolve() == p, 'canonical_path') + for part in [p, *p.parents]: + require(not part.is_symlink(), 'symlink_path') + s = p.lstat() + require(not owner or s.st_uid == os.getuid(), 'path_owner') + if kind == 'file': + require(stat.S_ISREG(s.st_mode) and s.st_nlink == 1 and s.st_size > 0, 'regular_file') + elif kind == 'socket': + require(stat.S_ISSOCK(s.st_mode), 'engine_socket') + elif kind == 'directory': + require(stat.S_ISDIR(s.st_mode), 'owned_directory') + return p + + +def validate_thin_pool(value): + require(isinstance(value, dict) and set(value) == {'name', 'directory'}, 'operator_thin_pool_keys') + require(isinstance(value['name'], str) and re.fullmatch(r'[A-Za-z][A-Za-z0-9_.-]{0,126}', value['name']), 'operator_thin_pool_name') + require(isinstance(value['directory'], str), 'operator_thin_pool_directory') + directory = canonical(value['directory'], 'directory') + require(directory.stat().st_mode & 0o777 == 0o700, 'operator_thin_pool_mode') + return value + + +def read_operator(path, backend): + p = canonical(path, owner=False) + s = p.stat() + require(s.st_uid in {0, os.getuid()} and s.st_mode & 0o022 == 0 and s.st_size <= 16384, 'operator_file') + o = json.loads(p.read_bytes()) + require(set(o) == {'version', 'backend', 'engineSocket', 'nativeHostIp', 'subnet', + 'registryPort', 'buildRegistryPort', 'tools'} | ({'thinPool'} if backend == 'firecracker' else set()), 'operator_keys') + require(o['version'] == 1 and o['backend'] == backend, 'operator_backend') + if backend == 'firecracker': + validate_thin_pool(o['thinPool']) + need = TOOLS | ({'cargo', 'rustc', 'cc', 'go'} if backend == 'vz' else set()) + require(isinstance(o['tools'], dict) and set(o['tools']) == need, 'operator_tools') + for path in o['tools'].values(): + # Tools can be installed by root. Only staged runtime copies need run ownership. + p = canonical(path, owner=False) + require(os.access(p, os.X_OK) and p.stat().st_mode & 0o022 == 0, 'tool_mode') + canonical(o['engineSocket'], 'socket') + net = ipaddress.ip_network(o['subnet'], strict=True) + ip = ipaddress.ip_address(o['nativeHostIp']) + require(net.version == 4 and net.prefixlen == 24 and str(net).startswith('10.') + and str(net).split('.')[1] != '0' and ip.version == 4 and ip.is_private + and any(ip in ipaddress.ip_network(cidr) for cidr in + ['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16']) + and not ip.is_loopback and ip not in net, 'operator_network') + for key in ['registryPort', 'buildRegistryPort']: + require(type(o[key]) is int and 20000 <= o[key] <= 59997, 'operator_port') + require(o['buildRegistryPort'] not in range(o['registryPort'], o['registryPort'] + 4), 'operator_port_overlap') + return o + + +def source_paths(names): + for name in names: + parts = Path(name).parts + require(not name.startswith('/') and '..' not in parts and + not {'var', '.git', '.superpowers'}.intersection(parts) and + not any(part == '.env' or (part.startswith('.env.') and + not part.endswith(('.example', '.sample', '.template'))) for part in parts), + 'secret_source_context') + + +def digest(data): + return 'sha256:' + hashlib.sha256(data).hexdigest() + + +def boot_json(data): + def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, 'boot_json_duplicate') + result[key] = value + return result + value = json.loads(data, object_pairs_hook=unique) + require(isinstance(value, dict), 'boot_json_object') + return value + + +def boot_file(path, directory, limit): + require(isinstance(path, str) and Path(path).parent == directory, 'boot_file_directory') + p = canonical(path) + parent = canonical(str(directory), 'directory') + parent_before = parent.stat() + require(parent_before.st_mode & 0o7777 == 0o700, 'boot_directory_mode') + before = p.lstat() + require(before.st_mode & 0o7777 == 0o400, 'boot_file_mode') + require(0 < before.st_size <= limit, 'boot_file_size') + identity = lambda s: (s.st_dev, s.st_ino, s.st_uid, s.st_mode, s.st_nlink, + s.st_size, s.st_mtime_ns, s.st_ctime_ns) + # Open through the retained private directory and compare named/opened + # identities, as the existing Inputs pin reader does for staged files. + folder = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + opened_parent = os.fstat(folder) + require((opened_parent.st_dev, opened_parent.st_ino, opened_parent.st_uid, opened_parent.st_mode) == + (parent_before.st_dev, parent_before.st_ino, parent_before.st_uid, parent_before.st_mode), 'boot_directory_changed') + fd = os.open(p.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=folder) + try: + require(identity(os.fstat(fd)) == identity(before), 'boot_file_changed') + chunks, size = [], 0 + while True: + chunk = os.read(fd, min(1024 * 1024, limit + 1 - size)) + if not chunk: + break + chunks.append(chunk) + size += len(chunk) + require(size <= limit, 'boot_file_size') + require(size == before.st_size and identity(os.fstat(fd)) == identity(before), 'boot_file_changed') + require(identity(canonical(path).lstat()) == identity(before), 'boot_file_changed') + current_parent = canonical(str(directory), 'directory').stat() + require((current_parent.st_dev, current_parent.st_ino, current_parent.st_uid, current_parent.st_mode) == + (opened_parent.st_dev, opened_parent.st_ino, opened_parent.st_uid, opened_parent.st_mode), 'boot_directory_changed') + return b''.join(chunks) + finally: + os.close(fd) + finally: + os.close(folder) + + +def static_amd64_firecracker(data): + require(len(data) >= 64 and data[:7] == b'\x7fELF\x02\x01\x01', 'boot_fork_elf') + (_, kind, machine, version, entry, offset, _, _, header_size, program_size, + count, _, _, _) = struct.unpack_from('<16sHHIQQQIHHHHHH', data) + require(kind in {2, 3} and machine == 62 and version == 1 and header_size == 64 and + program_size == 56 and 0 < count <= 1024 and offset >= 64 and + offset + count * program_size <= len(data), 'boot_fork_elf') + executable = False + for i in range(count): + kind, flags, start, address, _, size, memory_size, _ = struct.unpack_from('= size and address <= entry < address + memory_size: + executable = True + if kind == 2: + require(size <= 65536 and size % 16 == 0 and not any(struct.unpack_from('= deadline: + return False + time.sleep(0.01) + if self.group_exists(child.pid): + send(signal.SIGTERM) + if not wait_group(grace): + send(signal.SIGKILL) + if not wait_group(2): + self.group_event(child, 'survivor') + raise RuntimeError('producer_process_survivor') + child.wait(timeout=2) + self.group_event(child, 'absent') + del self.groups[child.pid] + if self.child is child: + self.child = None + + def call(self, argv, *, env=None, data=None, timeout=7200, cleanup=False, cwd=None, grace=15): + require(cleanup or not self.cancelled, 'cancelled') + self.counter += 1 + number = self.counter + log = self.root / f'command-{number:04d}.log' + with log.open('xb') as err: + child = subprocess.Popen([str(x) for x in argv], cwd=cwd or REPO, env=env or self.env, start_new_session=True, + stdin=subprocess.PIPE if data is not None else subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=err) + self.child = child + self.groups[child.pid] = child + self.group_event(child, 'created') + out, failure = b'', None + try: + out, _ = child.communicate(data, timeout=timeout) + except BaseException as error: + failure = error + try: + # Leader/pipe completion never proves that this retained session + # group has no descendant. Drain on success and every failure. + self.drain_group(child, grace) + remaining, _ = child.communicate(timeout=2) + if failure is not None: + out = remaining + finally: + with (self.root / 'commands.jsonl').open('a') as journal: + journal.write(json.dumps({'command': number, 'exit': child.returncode, + 'outputBytes': len(out), + 'groupDrained': child.pid not in self.groups}) + '\n') + if failure is not None: + raise failure + require(child.returncode == 0, 'producer_command') + return out + + def private_checkout(self, source): + source = Path(source) + require(source.is_absolute() and source.resolve() == source and source.is_dir(), 'source_checkout') + # Trust only this exact original directory in these bootstrap commands. + # No safe.directory value reaches the private repository or the adapter. + flags = ['/usr/bin/git', '-c', 'safe.directory=' + str(source), + '-c', 'core.fsmonitor=false', '-c', 'core.hooksPath=/dev/null'] + head = self.call([*flags, '-C', source, 'rev-parse', 'HEAD']).decode().strip() + require(re.fullmatch('[a-f0-9]{40}', head), 'source_head') + require(not self.call([*flags, '-C', source, 'status', '--porcelain', '--untracked-files=no']), 'source_dirty') + template = self.root / 'git-empty-template' + template.mkdir(mode=0o700) + checkout = self.root / 'checkout' + self.call(['/usr/bin/git', 'init', '--template=' + str(template), checkout]) + checkout.chmod(0o700) + self.call([*flags, '-c', 'protocol.file.allow=always', '-C', checkout, + 'fetch', '--depth=1', '--no-tags', '--no-write-fetch-head', source, head]) + self.call(['/usr/bin/git', '-C', checkout, '-c', 'core.hooksPath=/dev/null', + 'checkout', '--detach', head]) + # Every subsequent Git read uses only this principal-owned exact HEAD. + require(self.call(['/usr/bin/git', '-C', checkout, 'rev-parse', 'HEAD']).decode().strip() == head, + 'private_checkout_head') + require(not self.call(['/usr/bin/git', '-C', checkout, 'status', '--porcelain']), 'private_checkout_dirty') + require(self.call([*flags, '-C', source, 'rev-parse', 'HEAD']).decode().strip() == head, 'source_head_changed') + for directory, dirs, files in os.walk(checkout, followlinks=False): + for path in [Path(directory), *(Path(directory) / name for name in [*dirs, *files])]: + state = path.lstat() + require(state.st_uid == os.getuid() and not stat.S_ISLNK(state.st_mode), 'private_checkout_owner') + config = self.call(['/usr/bin/git', '-C', checkout, 'config', '--local', '--name-only', '--list']).decode().splitlines() + require(all(key in {'core.repositoryformatversion', 'core.filemode', 'core.bare', 'core.logallrefupdates', + 'core.ignorecase', 'core.precomposeunicode'} + for key in config), 'private_checkout_config') + return checkout, head + + def run_private_checkout(self, prepare_only=False, boot_artifacts=None): + checkout, head = self.private_checkout(REPO) + script = '.github/scripts/mcp-stack-assets.py' + require((checkout / script).read_bytes() == Path(__file__).read_bytes(), 'producer_source_changed') + self.nested_started = True + env = dict(self.env) + if boot_artifacts is None and os.environ.get('GITHUB_TOKEN'): + env['GITHUB_TOKEN'] = os.environ['GITHUB_TOKEN'] + self.call([sys.executable, '-I', checkout / script, '--backend', self.backend, + '--operator', self.operator_path, '--work-root', self.root / 'execution', + '--private-checkout', *(['--prepare-only'] if prepare_only else []), + *(['--boot-artifacts', boot_artifacts] if boot_artifacts is not None else [])], env=env, cwd=checkout, grace=240) + return prepared_result(self.root / 'execution', self.backend, head) if prepare_only else matrix_result(self.root / 'execution/run', self.backend, head) + + def engine(self, *args, cleanup=False): + s = canonical(self.o['engineSocket'], 'socket').stat() + require((s.st_dev, s.st_ino) == (self.socket.st_dev, self.socket.st_ino), 'engine_socket_changed') + if args[0] != 'info' and self.engine_identity is not None: + before = json.loads(self.engine('info', '--format', '{{json .}}', cleanup=cleanup)) + require(before['ID'] == self.engine_identity, 'engine_identity_changed') + out = self.call([self.tools['docker'], '--config', self.config, '--host', + 'unix://' + self.o['engineSocket'], *args], cleanup=cleanup) + after = canonical(self.o['engineSocket'], 'socket').stat() + require((after.st_dev, after.st_ino) == (s.st_dev, s.st_ino), 'engine_socket_changed') + # Every mutation is bracketed by identity queries; do not recursively call engine. + if self.engine_identity is not None and args[0] != 'info': + info = json.loads(self.engine('info', '--format', '{{json .}}', cleanup=cleanup)) + require(info['ID'] == self.engine_identity, 'engine_identity_changed') + return out + + def skopeo(self, *args): + command, *rest = args + require(command in {'copy', 'inspect'}, 'skopeo_command') + auth = ['--authfile', str(self.auth)] + if command == 'copy': + auth += ['--src-no-creds', '--dest-no-creds', '--src-cert-dir', str(self.root / 'certs'), + '--dest-cert-dir', str(self.root / 'certs')] + else: + auth += ['--no-creds', '--cert-dir', str(self.root / 'certs')] + return self.call([self.tools['skopeo'], '--policy', self.root / 'policy.json', + '--registries.d', self.root / 'registries.d', '--registries-conf', self.root / 'registries.conf', + '--override-os', 'linux', '--override-arch', self.arch, + command, *auth, *rest], env={**self.env, 'REGISTRY_AUTH_FILE': str(self.auth), + 'XDG_CONFIG_HOME': str(self.config)}) + + def own_container(self, role, image, args=(), command=()): + name = self.run + '-' + role + self.names.append(name) # Recover exact planned name after a lost create response. + self.put(self.root / f'planned-{role}.json', json.dumps({'name': name, 'label': self.run}).encode()) + identity = self.engine('create', '--name', name, '--label', f'{LABEL}={self.run}', + '--pull', 'never', '--restart', 'no', '--pids-limit', '256', + '--memory', '512m', '--cpus', '2', '--security-opt', 'no-new-privileges', + *args, image, *command).decode().strip() + require(re.fullmatch('[a-f0-9]{64}', identity), 'container_identity') + return identity + + def archive(self, role, dockerfile, context=None, build_args=None, local=False): + if role == 'guest': + # Append the fixture layer without copying the production recipe. + recipe = self.root / 'guest.Dockerfile' + self.put(recipe, (self.source / 'deploy/demo/Dockerfile').read_bytes() + b'\n' + + (self.source / dockerfile).read_bytes()) + dockerfile = recipe + output = self.files / role if local else self.files / f'{role}.docker.tar' + metadata = self.root / f'build-{role}.json' + args = ['buildx', 'build', '--builder', 'default', '--platform', 'linux/' + self.arch, + '--provenance=false', '--sbom=false', '--label', 'org.opencontainers.image.revision=' + self.head, + '--label', f'{LABEL}={self.run}', '--build-arg', 'SOURCE_COMMIT=' + self.head, + '--file', str((context or self.source) / dockerfile)] + args += (['--output', 'type=local,dest=' + str(output)] if local else + ['--load', '--metadata-file', str(metadata)]) + for key, value in (build_args or {}).items(): + args += ['--build-arg', key + '=' + value] + args += [str(context or self.source)] + self.engine(*args) + if local: + return output + built = json.loads(canonical(str(metadata)).read_bytes()) + identity = built.get('containerimage.config.digest') if isinstance(built, dict) else None + require(isinstance(identity, str) and DIGEST.fullmatch(identity), 'build_image_config_digest') + inspected = json.loads(self.engine('image', 'inspect', identity)) + require(isinstance(inspected, list) and len(inspected) == 1 and isinstance(inspected[0], dict), + 'build_image_inspect') + actual = inspected[0] + require(actual.get('Id') == identity and actual.get('Os') == 'linux' and + actual.get('Architecture') == self.arch, 'build_image_platform') + config = actual.get('Config') + labels = config.get('Labels') if isinstance(config, dict) else None + require(isinstance(labels, dict) and labels.get('org.opencontainers.image.revision') == self.head and + labels.get(LABEL) == self.run, 'build_image_labels') + self.engine('image', 'save', '--output', str(output), identity) + canonical(str(output)) + saved = self.skopeo('inspect', '--raw', '--config', 'docker-archive:' + str(output)) + require(digest(saved) == identity, 'build_saved_config') + archive = self.files / f'{role}.oci.tar' + self.skopeo('copy', 'docker-archive:' + str(output), 'oci-archive:' + str(archive)) + config = self.skopeo('inspect', '--raw', '--config', 'oci-archive:' + str(archive)) + normalized = [json.dumps(json.loads(raw), sort_keys=True, separators=(',', ':'), allow_nan=False) + for raw in (saved, config)] + require(normalized[0] == normalized[1], 'build_archive_config') + return archive + + def import_image(self, role, archive): + raw = self.skopeo('inspect', '--raw', 'oci-archive:' + str(archive)) + config = self.skopeo('inspect', '--raw', '--config', 'oci-archive:' + str(archive)) + facts = image_facts(raw, config, self.arch) + load = self.files / f'{role}.load.tar' + self.skopeo('copy', 'oci-archive:' + str(archive), 'docker-archive:' + str(load)) + self.engine('load', '--input', str(load)) + actual = json.loads(self.engine('image', 'inspect', facts['id']))[0] + require(actual['Id'] == facts['id'] and actual['Os'] == 'linux' and actual['Architecture'] == self.arch, + 'loaded_image_platform') + if role in {*SOURCE_IMAGES, 'hostAgent', 'hostBase'}: + require(actual['Config'].get('Labels', {}).get('org.opencontainers.image.revision') == self.head, + 'loaded_image_revision') + self.images[role] = {**facts, 'archive': {'path': str(archive)}} + return facts + + def stage(self, source, name, executable=False): + p = self.files / name + # Copy bytes to a fresh inode; executable pin must be the actual staged file. + with Path(source).open('rb') as src, p.open('xb') as dst: + shutil.copyfileobj(src, dst) + p.chmod(0o700 if executable else 0o600) + canonical(str(p)) + return {'path': str(p)} + + def snapshot(self): + require(not self.call(['/usr/bin/git', 'status', '--porcelain', '--untracked-files=no']), 'source_dirty') + self.head = self.call(['/usr/bin/git', 'rev-parse', 'HEAD']).decode().strip() + require(re.fullmatch('[a-f0-9]{40}', self.head), 'source_head') + entries = {} + for entry in self.call(['/usr/bin/git', 'ls-tree', '-r', '-z', self.head]).split(b'\0')[:-1]: + metadata, name = entry.split(b'\t', 1) + name = os.fsdecode(name) + # This exact upstream directory contains legacy runtime files. + if name == LEGACY_RUNTIME or name.startswith(LEGACY_RUNTIME + '/'): + continue + mode = metadata.split()[0].decode() + require(mode in {'100644', '100755', '160000'}, 'source_entry_type') + entries[name] = mode + source_paths(entries) + require(entries, 'source_empty') + # Literal names keep selection and archive bytes identical, including + # names with newlines or Git pathspec metacharacters. + archive = self.call(['/usr/bin/git', '--literal-pathspecs', 'archive', '--format=tar', + self.head, '--', *entries]) + with tarfile.open(fileobj=io.BytesIO(archive)) as tar: + expected = {name: mode == '160000' for name, mode in entries.items()} + for name in entries: + for parent in Path(name).parents: + if parent != Path('.'): + expected[str(parent)] = True + seen = set() + for member in tar.getmembers(): + source_paths([member.name]) + require(member.name in expected and member.name not in seen and + (member.isdir() if expected[member.name] else member.isfile()), 'source_archive_member') + seen.add(member.name) + require(seen == set(expected), 'source_archive_selection') + self.source.mkdir(mode=0o700) + tar.extractall(self.source, filter='data') + # The data filter leaves directory modes to the private process umask. + # Set public source modes only; keep the containing export root private. + for name, directory in expected.items(): + (self.source / name).chmod(0o755 if directory or entries[name] == '100755' else 0o644) + # Outputs are private and outside this tracked-only build context. + for role in [*SOURCE_IMAGES.values(), '.github/scripts/mcp-stack-inputs.mts', + 'deploy/demo/Dockerfile', + '.github/scripts/mcp-stack-fc.sh', '.github/scripts/mcp-stack-vz.sh', + 'tests/fixtures/mcp-stack/platform/main.ts', + 'tests/fixtures/mcp-stack/platform/host-agent.Dockerfile']: + require((self.source / role).is_file(), 'source_recipe') + + def fork_gitlink(self): + gitlink = self.call(['/usr/bin/git', 'ls-tree', 'HEAD', 'third_party/firecracker']).decode().split() + require(len(gitlink) == 4 and gitlink[0] == '160000' and gitlink[1] == 'commit' and + re.fullmatch('[a-f0-9]{40}', gitlink[2]) and gitlink[3] == 'third_party/firecracker', 'fork_gitlink') + return gitlink[2] + + def acquire_boot_artifacts(self, path): + require(self.backend == 'firecracker' and self.arch == 'amd64', 'boot_artifacts_backend') + directory = Path(path).parent + require(not directory.is_relative_to(REPO) and not directory.is_relative_to(self.source), 'boot_source_context') + raw = boot_file(path, directory, 16384) + manifest = boot_json(raw) + pins = {'firecracker': 256 * 1024 * 1024, 'kernel': 128 * 1024 * 1024, + 'kernelChecksum': 4096, 'provenance': 16384} + require(set(manifest) == {'version', 'kind', 'backend', 'architecture', 'forkGitlink', 'kernelRelease'} | pins.keys(), + 'boot_manifest_keys') + require(type(manifest['version']) is int and manifest['version'] == 1 and + manifest['kind'] == 'mcp-physical-boot-assets' and manifest['backend'] == 'firecracker' and + manifest['architecture'] == 'amd64' and manifest['kernelRelease'] == 'fc-kernel-6.1.102-1', 'boot_manifest_identity') + require(manifest['forkGitlink'] == self.fork_gitlink(), 'boot_fork_gitlink') + blobs = {} + for key, limit in pins.items(): + pin = manifest[key] + require(isinstance(pin, dict) and set(pin) == {'path', 'sha256'} and + isinstance(pin['sha256'], str) and re.fullmatch('[a-f0-9]{64}', pin['sha256']), 'boot_pin') + blobs[key] = boot_file(pin['path'], directory, limit) + require(hashlib.sha256(blobs[key]).hexdigest() == pin['sha256'], 'boot_pin_hash') + static_amd64_firecracker(blobs['firecracker']) + checksum = blobs['kernelChecksum'].decode('ascii').split() + require(len(checksum) in {1, 2} and checksum[0] == manifest['kernel']['sha256'] and + (len(checksum) == 1 or checksum[1] in {'vmlinux-engram-6.1.102-1', '*vmlinux-engram-6.1.102-1'}), + 'boot_kernel_checksum') + receipt = boot_json(blobs['provenance']) + require(set(receipt) == {'version', 'kind', 'forkGitlink', 'target', 'profile', 'firecrackerSha256'} and + type(receipt['version']) is int and receipt['version'] == 1 and + receipt['kind'] == 'operator-built-firecracker' and receipt['forkGitlink'] == manifest['forkGitlink'] and + receipt['target'] == 'x86_64-unknown-linux-musl' and receipt['profile'] == 'release' and + receipt['firecrackerSha256'] == manifest['firecracker']['sha256'], 'boot_provenance') + downloads = self.root / 'boot-download' + downloads.mkdir(mode=0o700) + for key, data in {'manifest': raw, **blobs}.items(): + target = downloads / key + self.put(target, data) + require(hashlib.sha256(target.read_bytes()).hexdigest() == hashlib.sha256(data).hexdigest(), 'boot_staged_hash') + self.put(self.root / 'fork-provenance.json', json.dumps({ + 'kind': 'operator-built', 'gitlink': manifest['forkGitlink'], + 'firecrackerSha256': manifest['firecracker']['sha256'], + 'manifestSha256': hashlib.sha256(raw).hexdigest(), + 'provenanceSha256': manifest['provenance']['sha256']}).encode()) + return self.stage(downloads / 'kernel', 'kernel'), downloads / 'firecracker' + + def acquire_boot(self, boot_artifacts=None): + if boot_artifacts is not None: + return self.acquire_boot_artifacts(boot_artifacts) + downloads = self.root / 'boot-download' + downloads.mkdir(mode=0o700) + ghconfig = self.root / 'gh-config' + ghconfig.mkdir(mode=0o700) + # The only credential read. It reaches fixed GitHub/GHCR artifact children only. + token = os.environ.get('GITHUB_TOKEN', '') + gh_env = {**self.env, 'GH_CONFIG_DIR': str(ghconfig)} + if token: + gh_env['GH_TOKEN'] = token + asset = ('vmlinux-engram-' if self.backend == 'firecracker' else 'Image-engram-') + '6.1.102-1' + for suffix in ['', '.sha256']: + self.call([self.tools['gh'], 'release', 'download', 'fc-kernel-6.1.102-1', + '--repo', 'cortexapps/engrams', '--pattern', asset + suffix, + '--output', downloads / (asset + suffix)], env=gh_env) + want = (downloads / (asset + '.sha256')).read_text().split()[0] + require(re.fullmatch('[a-f0-9]{64}', want) and + hashlib.sha256((downloads / asset).read_bytes()).hexdigest() == want, 'kernel_checksum') + kernel = self.stage(downloads / asset, 'kernel') + if self.backend != 'firecracker': + shutil.rmtree(ghconfig) + return kernel, None + gitlink = self.fork_gitlink() + auth = self.root / 'fork-registry.json' + self.put(auth, b'{"auths":{}}') + try: + if token: + self.call([self.tools['oras'], 'login', '--registry-config', auth, '--username', 'x-access-token', + '--password-stdin', 'ghcr.io'], data=token.encode()) + reference = 'ghcr.io/cortexapps/engrams/firecracker:' + gitlink + immutable = self.call([self.tools['oras'], 'resolve', '--registry-config', auth, reference]).decode().strip() + require(DIGEST.fullmatch(immutable), 'fork_manifest') + self.call([self.tools['oras'], 'pull', '--registry-config', auth, + reference.split(':')[0] + '@' + immutable, '--output', downloads / 'fork']) + self.put(self.root / 'fork-provenance.json', json.dumps({'gitlink': gitlink, 'manifest': immutable}).encode()) + finally: + auth.unlink(missing_ok=True) + # gh may create config state. Delete only this new task-owned config directory. + shutil.rmtree(ghconfig) + return kernel, downloads / 'fork/firecracker' + + def final_host(self, fork): + base_archive = self.archive('hostBase', 'docker/host-agent.Dockerfile') + base = self.import_image('hostBase', base_archive) + image = self.images['registry']['id'] + port = self.o['buildRegistryPort'] + registry = self.own_container('build-registry', image, + ['--network', 'bridge', '--publish', f'127.0.0.1:{port}:5000', '--cap-drop', 'ALL', + '--read-only', '--tmpfs', '/var/lib/registry:rw,nosuid,nodev,size=8g', + '--tmpfs', '/tmp:rw,nosuid,nodev,size=16m']) + self.engine('start', registry) + uri = f'127.0.0.1:{port}/{self.run}/host' + # Probe registry with bounded skopeo copy retries; never select another server. + for attempt in range(20): + try: + self.skopeo('copy', '--preserve-digests', '--dest-tls-verify=false', + 'oci-archive:' + str(base_archive), 'docker://' + uri + ':base') + break + except RuntimeError: + if attempt == 19 or self.cancelled: + raise + time.sleep(0.5) + raw = self.skopeo('inspect', '--raw', '--tls-verify=false', 'docker://' + uri + '@' + base['manifestDigest']) + require(digest(raw) == base['manifestDigest'], 'host_base_registry') + context = self.root / 'host-context' + context.mkdir(mode=0o700) + shutil.copyfile(self.source / 'tests/fixtures/mcp-stack/platform/host-agent.Dockerfile', context / 'Dockerfile') + shutil.copyfile(fork, context / 'firecracker') + checksum = hashlib.sha256((context / 'firecracker').read_bytes()).hexdigest() + archive = self.archive('hostAgent', 'Dockerfile', context, {'HOST_IMAGE': uri + '@' + base['manifestDigest'], + 'FIRECRACKER_SHA256': checksum}) + final = self.import_image('hostAgent', archive) + container = self.own_container('extract-host', final['id'], ['--network', 'none', '--cap-drop', 'ALL']) + result = {} + for key, name in [('hostAgent', 'engram-host-agent'), ('helper', 'engram-uffd-handler'), + ('proxy', 'engram-egress-proxyd'), ('firecracker', 'firecracker')]: + target = self.files / key + self.engine('cp', container + ':/usr/local/bin/' + name, str(target)) + target.chmod(0o700) + canonical(str(target)) + result[key] = {'path': str(target)} + require(hashlib.sha256((self.files / 'firecracker').read_bytes()).hexdigest() == checksum, 'baked_fork_bytes') + self.images.pop('hostBase') + return result + + def discover_native_sdk(self): + try: + output = self.call(['/usr/bin/xcrun', '--sdk', 'macosx', '--show-sdk-path'], timeout=30) + lines = output.decode().splitlines() + require(len(lines) == 1 and lines[0] == lines[0].strip() and + '\0' not in lines[0] and Path(lines[0]).is_absolute(), 'native_macos_sdk_path') + # The selected system SDK can be an alias to its installed version. + sdk = Path(lines[0]).resolve(strict=True) + info = sdk.stat() + header = sdk / 'usr/include/TargetConditionals.h' + require(stat.S_ISDIR(info.st_mode) and info.st_uid in {0, os.getuid()} and + info.st_mode & 0o022 == 0 and header.is_file() and + header.stat().st_size > 0 and os.access(header, os.R_OK), 'native_macos_sdk_unusable') + except (OSError, ValueError, RuntimeError) as error: + raise RuntimeError('native_macos_sdk_unusable') from error + return str(sdk) + + def native_host(self): + build = self.root / 'native-build' + build.mkdir(mode=0o700) + for name in ['cargo', 'target', 'netstack']: + (build / name).mkdir(mode=0o700) + # Real cargo/rustc toolchain binaries, never credential-reading rustup shims. + env = {**self.env, 'CARGO_HOME': str(build / 'cargo'), 'CARGO_TARGET_DIR': str(build / 'target'), + 'RUSTC': self.tools['rustc'], 'SQLX_OFFLINE': 'true', + 'SDKROOT': self.native_sdk, 'CARGO_BUILD_JOBS': '2'} + version = self.call([self.tools['rustc'], '--version']).decode().split() + require(version[1] == '1.99.0', 'native_rust_toolchain') + fp = self.call([sys.executable, '-I', self.source / 'docker/egress-proxyd-fingerprint.py'], env=env, cwd=self.source).decode().strip() + require(re.fullmatch('[a-f0-9]{64}', fp), 'proxy_fingerprint') + self.call([self.tools['cargo'], 'build', '--locked', '--release', '--manifest-path', self.source / 'Cargo.toml', + '-p', 'engram-host-agent', '-p', 'engram-egress-proxyd'], + env={**env, 'ENGRAM_EGRESS_PROXYD_FINGERPRINT': fp}, cwd=self.source) + host = self.stage(build / 'target/release/engram-host-agent', 'hostAgent', True) + proxy = self.stage(build / 'target/release/engram-egress-proxyd', 'proxy', True) + self.call(['/usr/bin/codesign', '--force', '--sign', '-', '--entitlements', + self.source / 'crates/engram-sandbox-vz/entitlements.plist', host['path']]) + self.call(['/usr/bin/codesign', '--verify', '--strict', host['path']]) + self.call([self.tools['cc'], '-std=c11', '-Wall', '-Wextra', '-Werror', '-O2', + self.source / 'tests/fixtures/mcp-stack/platform/native-process-identity.c', + '-o', self.files / 'nativeProbe'], env=env) + (self.files / 'nativeProbe').chmod(0o700) + # build.sh creates its own exact private caches and verifies Go1.26.2. + (build / 'netstack').rmdir() + self.call(['/bin/bash', self.source / 'sandbox-netstack/build.sh', 'build', build / 'netstack'], cwd=self.source) + helper = self.stage(build / 'netstack/engram-sandbox-netstack', 'helper', True) + return {'hostAgent': host, 'proxy': proxy, 'helper': helper, 'nativeProbe': {'path': str(self.files / 'nativeProbe')}} + + def produce(self, prepare_only=False, boot_artifacts=None): + require(platform.system() == ('Linux' if self.backend == 'firecracker' else 'Darwin') and + platform.machine() in (['x86_64'] if self.backend == 'firecracker' else ['arm64']), 'physical_platform') + require(self.backend != 'firecracker' or os.getuid() == 0, 'fc_root') + info = json.loads(self.engine('info', '--format', '{{json .}}')) + require(info.get('OSType') == 'linux' and info.get('Architecture') in + [self.arch, 'x86_64' if self.arch == 'amd64' else 'aarch64'] and + re.fullmatch('[A-Za-z0-9:_-]{12,128}', info.get('ID', '')), 'engine_platform') + self.engine_identity = info['ID'] + version = json.loads(self.engine('version', '--format', '{{json .}}')) + server = version.get('Server', {}) + require(int(str(server.get('Version', '0')).split('.')[0]) >= 26 and + tuple(map(int, str(server.get('ApiVersion', '0.0')).split('.'))) >= (1, 45), 'engine_version') + if self.backend == 'firecracker': + for device in ['/dev/kvm', '/dev/net/tun']: + p = canonical(device, kind='device', owner=False) + require(stat.S_ISCHR(p.stat().st_mode) and os.access(p, os.R_OK | os.W_OK), 'fc_device') + else: + require(self.call(['/usr/sbin/sysctl', '-n', 'kern.hv_support']).strip() == b'1', 'vz_hypervisor') + self.native_sdk = self.discover_native_sdk() + require(self.call([self.tools['node'], '--version']).decode().startswith('v24.'), 'node24') + require(self.call([self.tools['openssl'], 'version']).decode().startswith('OpenSSL 3.'), 'openssl3') + expected_bun = (REPO / '.bun-version').read_text().strip() + require(self.call([self.tools['bun'], '--version']).decode().strip() == expected_bun, 'bun_version') + self.put(self.root / 'engine-provenance.json', json.dumps({ + 'id': info['ID'], 'os': info['OSType'], 'architecture': info['Architecture'], + 'socketDevice': self.socket.st_dev, 'socketInode': self.socket.st_ino}).encode()) + self.snapshot() + boot = self.acquire_boot(boot_artifacts) if boot_artifacts is not None else None + bins = {name: self.stage(self.tools[name], name, True) for name in ['node', 'bun', 'docker', 'openssl']} + upstream = json.loads((self.source / CI / 'upstream-images.json').read_bytes()) + for role, value in upstream.items(): + archive = self.files / f'{role}.oci.tar' + reference = 'docker://' + value['repository'] + '@' + value[self.arch] + raw = self.skopeo('inspect', '--raw', reference) + require(digest(raw) == value[self.arch], 'upstream_manifest_changed') + # OCI and Docker-save use different manifest media types. Conversion + # may change manifest bytes; final Inputs use actual OCI facts only. + self.skopeo('copy', reference, 'oci-archive:' + str(archive)) + self.put(self.root / f'upstream-{role}.json', json.dumps( + {'reference': reference, 'sourceManifest': digest(raw)}).encode()) + self.import_image(role, archive) + for role, recipe in SOURCE_IMAGES.items(): + self.import_image(role, self.archive(role, recipe)) + linux = self.archive('linux-assets', CI + '/linux-assets.Dockerfile', local=True) + for key in ['claudeHarness', 'codexHarness', 'modelCli', 'agentdBundle', 'sentinelBundle', 'guestToolsBundle']: + bins[key] = self.stage(linux / key, key, key.endswith('Harness') or key == 'modelCli') + match = re.search(r'pub const DEFAULT_INIT_SHIM: &str = r#"([\s\S]*?)"#;', + (self.source / 'crates/engram-rootfs-materializer/src/inject.rs').read_text()) + require(match is not None, 'production_init') + self.put(self.files / 'guestInit', match.group(1).encode()) + (self.files / 'guestInit').chmod(0o700) + bins['guestInit'] = {'path': str(self.files / 'guestInit')} + bins['kernel'], fork = boot if boot is not None else self.acquire_boot() + host = self.final_host(fork) if self.backend == 'firecracker' else self.native_host() + for key in ['hostAgent', 'helper']: + bins[key] = host.pop(key) + template = {'version': 1, 'backend': self.backend, 'architecture': self.arch, 'sourceCommit': self.head, + 'engineSocket': self.o['engineSocket'], 'engineConfig': str(self.adapter_config), + 'engineId': self.engine_identity, 'registryPort': self.o['registryPort'], + 'subnet': self.o['subnet'], 'nativeHostIp': self.o['nativeHostIp'], + 'binaries': bins, 'images': self.images, **host, + **({'thinPool': self.o['thinPool']} if self.backend == 'firecracker' else {})} + self.put(self.root / 'template.json', json.dumps(template).encode()) + self.call([bins['node']['path'], '--experimental-transform-types', REPO / '.github/scripts/mcp-stack-inputs.mts', + '--backend', self.backend, '--template', self.root / 'template.json', '--output', self.root / 'inputs.json']) + deps = self.root / 'dependency-cache' + deps.mkdir(mode=0o700) + self.put(deps / '.npmrc', b'') + self.put(deps / '.bunfig.toml', b'') + depenv = {**self.env, 'PATH': str(self.files) + ':' + self.env['PATH'], + 'NPM_CONFIG_USERCONFIG': str(deps / '.npmrc'), 'NPM_CONFIG_GLOBALCONFIG': '/dev/null', + 'NPM_CONFIG_CACHE': str(deps / 'npm'), 'BUN_INSTALL_CACHE_DIR': str(deps / 'bun'), + 'XDG_CONFIG_HOME': str(deps), 'BUN_INSTALL': str(deps / 'bun-install')} + self.call([self.tools['npm'], 'ci', '--ignore-scripts'], env=depenv, cwd=REPO / 'mcp-worker') + self.call([bins['bun']['path'], 'install', '--frozen-lockfile', '--ignore-scripts'], env=depenv, cwd=REPO / 'orchestrator') + require(not self.call(['/usr/bin/git', 'status', '--porcelain', '--untracked-files=no']), 'source_dirty') + return self.dispatch(prepare_only) + + def dispatch(self, prepare_only=False): + if prepare_only: + inputs = self.root / 'inputs.json' + inputs.chmod(0o400) + self.put(self.root / 'assets-prepared.json', json.dumps({ + 'version': 1, 'kind': 'assets-prepared', 'backend': self.backend, 'sourceCommit': self.head, + 'inputs': {'path': str(inputs), 'sha256': hashlib.sha256(inputs.read_bytes()).hexdigest()}, + 'matrixCompleted': False, 'physicalAcceptance': False}).encode()) + return prepared_result(self.root, self.backend, self.head) + wrapper = '.github/scripts/mcp-stack-' + ('fc' if self.backend == 'firecracker' else 'vz') + '.sh' + # The adapter output root MUST NOT exist; its parent is this canonical0700 root. + require(not (self.root / 'run').exists(), 'run_root_exists') + self.adapter_started = True + self.call(['/bin/bash', REPO / wrapper, '--run-root', self.root / 'run', + '--artifacts', self.root / 'inputs.json'], env={**self.env, 'PATH': str(self.files) + ':' + self.env['PATH']}, grace=240) + return matrix_result(self.root / 'run', self.backend, self.head) + + def cleanup(self): + failed = [] + for child in list(getattr(self, 'groups', {}).values()): + try: + self.drain_group(child, 0.1) + child.communicate(timeout=2) + except Exception: + failed.append('process-group-' + str(child.pid)) + if getattr(self, 'groups', {}): + failed.append('process-groups') + if getattr(self, 'nested_started', False): + try: + actual = json.loads((self.root / 'execution/summary.json').read_bytes()) + require(actual.get('cleanup') is True, 'private_producer_cleanup') + except Exception: + failed.append('private-producer') + if getattr(self, 'adapter_started', False): + try: + run = self.root / 'run' + completions = list(run.glob('completion-*.json')) + require(len(completions) == 1, 'adapter_completion_missing') + done = json.loads(completions[0].read_bytes()) + require(done.get('backend') == self.backend and done.get('sourceCommit') == self.head and + done.get('cleanupFailed') == [], 'adapter_cleanup_failed') + actual = json.loads((run / 'cleanup.json').read_bytes()) + require(actual.get('success') is True and actual.get('containersRemaining') == 0 and + actual.get('networkRemaining') is False and + re.fullmatch('mcp-[a-f0-9]{24}', actual.get('runId', '')), 'adapter_cleanup_receipt') + label = 'label=engrams.mcp.run=' + actual['runId'] + require(not self.engine('ps', '-aq', '--filter', label, cleanup=True).strip(), 'adapter_survivor') + require(not self.engine('network', 'ls', '-q', '--filter', label, cleanup=True).strip(), 'adapter_network_survivor') + self.adapter_clean = True + except Exception: + failed.append('adapter') + for name in reversed(self.names): + try: + # Query by exact name through a successful daemon list, never assume query failure means absent. + ids = self.engine('ps', '-aq', '--no-trunc', '--filter', 'name=^/' + name + '$', cleanup=True).decode().split() + require(len(ids) <= 1, 'cleanup_identity') + for identity in ids: + item = json.loads(self.engine('inspect', identity, cleanup=True))[0] + require(item['Name'] == '/' + name and item['Config'].get('Labels', {}).get(LABEL) == self.run, + 'cleanup_foreign_resource') + self.engine('rm', '--force', identity, cleanup=True) + require(not self.engine('ps', '-aq', '--filter', 'name=^/' + name + '$', cleanup=True).strip(), 'cleanup_survivor') + except Exception: + failed.append(name) + self.put(self.root / 'producer-cleanup.json', json.dumps({'version': 1, 'failed': failed, + 'adapterStarted': getattr(self, 'adapter_started', False), + 'adapterCleanup': getattr(self, 'adapter_clean', False)}).encode()) + require(not failed, 'producer_cleanup_failed') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--backend', choices=['firecracker', 'vz'], required=True) + parser.add_argument('--operator', required=True) + parser.add_argument('--work-root', required=True) + parser.add_argument('--prepare-only', action='store_true', help='Prepare pinned assets without starting the acceptance adapter') + parser.add_argument('--boot-artifacts', help='Use a private, pinned Firecracker boot manifest instead of downloading boot artifacts') + parser.add_argument('--private-checkout', action='store_true', help=argparse.SUPPRESS) + args = parser.parse_args() + require(args.boot_artifacts is None or args.backend == 'firecracker', 'boot_artifacts_backend') + os.umask(0o077) + root = Path(args.work_root) + require(root.is_absolute() and root.resolve() == root and root.parent.resolve() == root.parent and not root.exists() and + not root.is_symlink(), 'work_root') + root.mkdir(mode=0o700) + producer, passed, cleanup, completed = None, 0, False, False + try: + o = read_operator(args.operator, args.backend) + producer = Producer(args.backend, root, o) + producer.operator_path = args.operator + if args.private_checkout: + canonical(str(REPO), kind='directory') + canonical(str(REPO / '.git'), kind='directory') + require(REPO.name == 'checkout' and REPO.parent == root.parent and + (REPO.parent.stat().st_mode & 0o777) == 0o700, 'private_checkout_layout') + def cancel(_signum, _frame): + if not producer.cancelled: + producer.cancelled = True + raise RuntimeError('cancelled') + signal.signal(signal.SIGTERM, cancel) + signal.signal(signal.SIGINT, cancel) + try: + passed = producer.produce(args.prepare_only, args.boot_artifacts) if args.private_checkout else producer.run_private_checkout(args.prepare_only, args.boot_artifacts) + completed = True + finally: + # Drain the interrupted child in call() before owned-resource cleanup. + signal.signal(signal.SIGTERM, signal.SIG_IGN) + signal.signal(signal.SIGINT, signal.SIG_IGN) + producer.cleanup() + cleanup = True + finally: + summary = {'version': 1, 'backend': args.backend, 'selected': 0 if args.prepare_only else 24, 'passed': passed, + 'cleanup': cleanup, 'physicalAcceptance': not args.prepare_only and passed == 24 and cleanup, + 'status': ('ASSETS_PREPARED' if args.prepare_only else 'PASS') if completed and cleanup and (args.prepare_only or passed == 24) else 'FAIL'} + path = root / 'summary.json' + with path.open('x') as out: + json.dump(summary, out) + print(json.dumps(summary)) + require(completed and cleanup and (args.prepare_only or passed == 24), 'physical_matrix_failed') + + +if __name__ == '__main__': + try: + main() + except BaseException: + # Private logs carry controlled evidence; CI output must not expose payloads or secrets. + print('FAILED:mcp_physical_ci', file=sys.stderr) + sys.exit(1) diff --git a/.github/scripts/mcp-stack-fc.sh b/.github/scripts/mcp-stack-fc.sh new file mode 100755 index 000000000..082fd788b --- /dev/null +++ b/.github/scripts/mcp-stack-fc.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +bun="$(command -v bun || true)" +if [[ -z "$bun" || "$bun" != /* ]]; then + echo 'NEEDS_CONTEXT:bun' >&2 + exit 2 +fi +exec env -i PATH=/usr/bin:/bin:/usr/sbin:/sbin "$bun" "$root/tests/fixtures/mcp-stack/platform/main.ts" --backend firecracker "$@" diff --git a/.github/scripts/mcp-stack-inputs.mts b/.github/scripts/mcp-stack-inputs.mts new file mode 100644 index 000000000..4e7aad32c --- /dev/null +++ b/.github/scripts/mcp-stack-inputs.mts @@ -0,0 +1,135 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, open, realpath } from "node:fs/promises"; +import { dirname, isAbsolute, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + check, + ownedPath, + PlatformFailure, + privateRoot, + put, + readOwned, + record, + type Pin, +} from "../../tests/fixtures/mcp-stack/platform/owned.ts"; +import { + BINARIES, + IMAGES, + parseInputs, + type Inputs, +} from "../../tests/fixtures/mcp-stack/platform/preflight.ts"; + +function object(value: unknown): Record { + check(value !== null && typeof value === "object" && !Array.isArray(value), "object"); + return value as Record; +} + +async function pin(value: unknown): Promise { + const source = record(value, ["path"]); + check(typeof source.path === "string", "artifact_path"); + const path = source.path; + await ownedPath(path); + check((await realpath(path)) === path, "artifact_path"); + const namedBefore = await lstat(path, { bigint: true }); + check(namedBefore.isFile(), "artifact_identity"); + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const before = await file.stat({ bigint: true }); + check( + before.isFile() && + before.dev === namedBefore.dev && + before.ino === namedBefore.ino && + before.uid === BigInt(process.getuid?.() ?? -1) && + before.nlink === 1n && + before.size > 0n, + "artifact_identity", + ); + const digest = createHash("sha256"); + let size = 0n; + for await (const bytes of file.createReadStream({ autoClose: false })) { + digest.update(bytes); + size += BigInt(bytes.length); + } + const after = await file.stat({ bigint: true }); + await ownedPath(path); + check((await realpath(path)) === path, "artifact_path"); + const named = await lstat(path, { bigint: true }); + for (const state of [after, named]) + check( + state.isFile() && + state.dev === before.dev && + state.ino === before.ino && + state.uid === before.uid && + state.mode === before.mode && + state.nlink === 1n && + state.size === before.size && + state.mtimeNs === before.mtimeNs && + state.ctimeNs === before.ctimeNs, + "artifact_changed", + ); + check(size === before.size, "artifact_changed"); + return { path, sha256: digest.digest("hex") }; + } finally { + await file.close(); + } +} + +/** Hash staged files, validate the complete Inputs object, then create its private manifest. */ +export async function materializeInputs( + backend: Inputs["backend"], + template: string, + output: string, +): Promise { + check(backend === "firecracker" || backend === "vz", "backend"); + await privateRoot(dirname(output)); + await ownedPath(output, true); + const source = object(JSON.parse((await readOwned(template, 1 << 20)).toString("utf8"))); + const binaries = { ...object(source.binaries) }; + for (const key of BINARIES) binaries[key] = await pin(binaries[key]); + const images = { ...object(source.images) }; + for (const key of [...IMAGES, "hostAgent"]) { + if (!Object.hasOwn(images, key)) continue; + const image = object(images[key]); + images[key] = { ...image, archive: await pin(image.archive) }; + } + const final: Record = { + ...source, + binaries, + images, + proxy: await pin(source.proxy), + }; + for (const key of ["firecracker", "nativeProbe"] as const) + if (Object.hasOwn(source, key)) final[key] = await pin(source[key]); + const inputs = parseInputs(final, backend); + await put(output, JSON.stringify(inputs, null, 2) + "\n"); + return inputs; +} + +export function cli(args: string[]): { + backend: Inputs["backend"]; + template: string; + output: string; +} { + check( + args.length === 6 && + args[0] === "--backend" && + ["firecracker", "vz"].includes(args[1]!) && + args[2] === "--template" && + args[4] === "--output", + "arguments", + ); + for (const path of [args[3]!, args[5]!]) + check(isAbsolute(path) && resolve(path) === path, "arguments_path"); + return { backend: args[1] as Inputs["backend"], template: args[3]!, output: args[5]! }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + try { + const args = cli(process.argv.slice(2)); + await materializeInputs(args.backend, args.template, args.output); + } catch (error) { + console.error(error instanceof PlatformFailure ? error.message : "INVALID_CONTEXT:inputs"); + process.exitCode = 1; + } +} diff --git a/.github/scripts/mcp-stack-vz.sh b/.github/scripts/mcp-stack-vz.sh new file mode 100755 index 000000000..d1aad2738 --- /dev/null +++ b/.github/scripts/mcp-stack-vz.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +bun="$(command -v bun || true)" +if [[ -z "$bun" || "$bun" != /* ]]; then + echo 'NEEDS_CONTEXT:bun' >&2 + exit 2 +fi +exec env -i PATH=/usr/bin:/bin:/usr/sbin:/sbin "$bun" "$root/tests/fixtures/mcp-stack/platform/main.ts" --backend vz "$@" diff --git a/.github/scripts/release-plan.py b/.github/scripts/release-plan.py index 1f0dd35fe..f67a0b681 100644 --- a/.github/scripts/release-plan.py +++ b/.github/scripts/release-plan.py @@ -36,9 +36,9 @@ DETECTOR = Path(__file__).with_name("detect-rebake-lanes.py") -# The five container images whose lanes the detector evaluates against a +# The container images whose lanes the detector evaluates against a # registry baseline (`--image-baseline`), in the detector's own order. -BASELINE_IMAGES = ["coordinator", "host-agent", "host-operator", "web", "orchestrator"] +BASELINE_IMAGES = ["coordinator", "host-agent", "host-operator", "web", "orchestrator", "mcp-worker"] # Artifacts with a push-range lane: the detector output that says "the inputs # changed in this range". RANGE_LANES = {"node-assets": "node_assets", "cli-tools": "cli_tools"} diff --git a/.github/scripts/tests/mcp-stack-inputs.test.mts b/.github/scripts/tests/mcp-stack-inputs.test.mts new file mode 100644 index 000000000..754e965ee --- /dev/null +++ b/.github/scripts/tests/mcp-stack-inputs.test.mts @@ -0,0 +1,413 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + access, + chmod, + chown, + cp, + link, + lstat, + mkdir, + mkdtemp, + readFile, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { cli, materializeInputs } from "../mcp-stack-inputs.mts"; +import { + BINARIES, + IMAGES, + parseInputs, + REPO, + type Inputs, +} from "../../../tests/fixtures/mcp-stack/platform/preflight.ts"; + +type PathPin = { path: string }; +type Template = Omit & { + binaries: Record<(typeof BINARIES)[number], PathPin>; + images: Record; + proxy: PathPin; + firecracker?: PathPin; + nativeProbe?: PathPin; +}; +async function fixture( + backend: Inputs["backend"], + run: (f: { + root: string; + template: Template; + source: string; + output: string; + save: () => Promise; + }) => Promise, +) { + const root = await mkdtemp(join(await realpath(tmpdir()), "mcp-inputs-")); + try { + await chmod(root, 0o700); + const staged = async (name: string): Promise => { + const path = join(root, name); + await writeFile(path, `staged-${name}\n`, { mode: 0o600, flag: "wx" }); + return { path }; + }; + const binaries = Object.fromEntries( + await Promise.all(BINARIES.map(async (key) => [key, await staged(key)])), + ) as Template["binaries"]; + const images: Template["images"] = {}; + for (const key of [...IMAGES, ...(backend === "firecracker" ? ["hostAgent"] : [])]) + images[key] = { + archive: await staged(`image-${key}`), + id: `sha256:${"c".repeat(64)}`, + manifestDigest: `sha256:${"d".repeat(64)}`, + }; + const template: Template = { + version: 1, + backend, + architecture: backend === "vz" ? "arm64" : "amd64", + sourceCommit: "b".repeat(40), + engineSocket: join(root, "engine.sock"), + engineConfig: join(root, "engine-config"), + engineId: "fixture-engine-identity", + registryPort: 24567, + subnet: "10.80.1.0/24", + nativeHostIp: "192.168.50.2", + binaries, + images, + proxy: await staged("proxy"), + ...(backend === "vz" + ? { nativeProbe: await staged("native-probe") } + : { + firecracker: await staged("firecracker"), + thinPool: { name: "fixture-pool", directory: root }, + }), + }; + const source = join(root, "template.json"), + output = join(root, "inputs.json"); + const save = () => writeFile(source, JSON.stringify(template), { mode: 0o600 }); + await save(); + await run({ root, template, source, output, save }); + } finally { + await rm(root, { recursive: true }); + } +} +async function absent(path: string) { + await assert.rejects(access(path), { code: "ENOENT" }); +} + +for (const backend of ["firecracker", "vz"] as const) { + test(`${backend}: hashes fresh staged files and preserves distinct producer facts`, async () => { + await fixture(backend, async ({ root, template, source, output }) => { + const inputs = await materializeInputs(backend, source, output); + assert.deepEqual(parseInputs(JSON.parse(await readFile(output, "utf8")), backend), inputs); + assert.equal((await lstat(output)).mode & 0o777, 0o600); + assert.equal(inputs.sourceCommit, template.sourceCommit); + if (backend === "firecracker") + assert.deepEqual(inputs.thinPool, { name: "fixture-pool", directory: root }); + else assert.equal(Object.hasOwn(inputs, "thinPool"), false); + const pins = [ + ...Object.values(inputs.binaries), + ...Object.values(inputs.images).map((i) => i.archive), + inputs.proxy, + inputs.firecracker ?? inputs.nativeProbe!, + ]; + for (const pin of pins) + assert.equal( + pin.sha256, + createHash("sha256") + .update(await readFile(pin.path)) + .digest("hex"), + ); + for (const image of Object.values(inputs.images)) { + assert.equal(image.id, `sha256:${"c".repeat(64)}`); + assert.equal(image.manifestDigest, `sha256:${"d".repeat(64)}`); + assert.notEqual(`sha256:${image.archive.sha256}`, image.id); + assert.notEqual(image.id, image.manifestDigest); + } + }); + }); +} + +const invalidTemplates: Record void> = { + "extra top-level field": (t) => Object.assign(t, { approved: true }), + "missing top-level field": (t) => { + Reflect.deleteProperty(t, "sourceCommit"); + }, + "missing binary": (t) => { + Reflect.deleteProperty(t.binaries, "node"); + }, + "extra binary": (t) => Object.assign(t.binaries, { unknown: t.proxy }), + "missing image": (t) => { + Reflect.deleteProperty(t.images, "guest"); + }, + "extra image": (t) => { + t.images.unknown = t.images.guest!; + }, + "extra image fact": (t) => Object.assign(t.images.guest!, { passed: true }), + "missing manifest digest": (t) => { + Reflect.deleteProperty(t.images.guest!, "manifestDigest"); + }, + "invalid config digest": (t) => { + t.images.guest!.id = "latest"; + }, + "backend mismatch": (t) => { + t.backend = "firecracker"; + }, + "wrong architecture": (t) => { + t.architecture = "amd64"; + }, + "missing conditional pin": (t) => { + delete t.nativeProbe; + }, + "wrong conditional pin": (t) => { + t.firecracker = t.proxy; + }, + "wrong conditional image": (t) => { + t.images.hostAgent = t.images.guest!; + }, + "binary hash supplied": (t) => Object.assign(t.binaries.node, { sha256: "a".repeat(64) }), + "proxy hash supplied": (t) => Object.assign(t.proxy, { sha256: "a".repeat(64) }), + "probe hash supplied": (t) => Object.assign(t.nativeProbe!, { sha256: "a".repeat(64) }), + "archive hash supplied": (t) => + Object.assign(t.images.guest!.archive, { sha256: "a".repeat(64) }), +}; +for (const [name, mutate] of Object.entries(invalidTemplates)) + test(`rejects ${name} without output`, async () => { + await fixture("vz", async ({ template, source, output, save }) => { + mutate(template); + await save(); + await assert.rejects(materializeInputs("vz", source, output)); + await absent(output); + }); + }); + +test("FC rejects a missing host image, a native probe, and a supplied firecracker hash", async () => { + const cases: [(t: Template) => void, string][] = [ + [ + (t) => { + delete t.images.hostAgent; + }, + "object_keys", + ], + [ + (t) => { + t.nativeProbe = t.proxy; + }, + "fc_artifacts", + ], + [(t) => Object.assign(t.firecracker!, { sha256: "a".repeat(64) }), "object_keys"], + ]; + for (const [mutate, seam] of cases) + await fixture("firecracker", async ({ template, source, output, save }) => { + mutate(template); + await save(); + await assert.rejects(materializeInputs("firecracker", source, output), { + code: "INVALID_CONTEXT", + seam, + }); + await absent(output); + }); +}); + +for (const [kind, seam] of [ + ["missing", "fc_pool_required"], + ["invalid-name", "fc_pool_name"], + ["noncanonical", "fc_pool_directory"], + ["public", "fc_pool_directory_owner_mode"], + ["symlink", "fc_pool_directory_symlink"], +] as const) + test(`FC rejects ${kind} thin pool without output`, async () => { + await fixture("firecracker", async ({ root, template, source, output, save }) => { + if (kind === "missing") delete template.thinPool; + else if (kind === "invalid-name") template.thinPool!.name = "../pool"; + else if (kind === "noncanonical") template.thinPool!.directory = `${root}/.`; + else { + const directory = join(root, "pool"); + if (kind === "symlink") await symlink(root, directory); + else { + await mkdir(directory); + await chmod(directory, 0o755); + } + template.thinPool!.directory = directory; + } + await save(); + await assert.rejects(materializeInputs("firecracker", source, output), { + code: "INVALID_CONTEXT", + seam, + }); + await absent(output); + }); + }); + +test("VZ rejects a thin pool without output", async () => { + await fixture("vz", async ({ root, template, source, output, save }) => { + template.thinPool = { name: "fixture-pool", directory: root }; + await save(); + await assert.rejects(materializeInputs("vz", source, output), { + code: "INVALID_CONTEXT", + seam: "vz_architecture", + }); + await absent(output); + }); +}); + +for (const kind of [ + "missing", + "empty", + "symlink", + "hardlink", + "directory", + "noncanonical", + "symlink-parent", +] as const) + test(`rejects ${kind} artifact without output`, async () => { + await fixture("vz", async ({ root, template, source, output, save }) => { + const path = join(root, "bad"); + if (kind === "empty") await writeFile(path, ""); + if (kind === "symlink") await symlink(template.proxy.path, path); + if (kind === "hardlink") await link(template.proxy.path, path); + if (kind === "directory") await mkdir(path); + if (kind === "symlink-parent") await symlink(root, path); + template.binaries.node.path = + kind === "noncanonical" + ? `${root}/../${root.split("/").at(-1)}/proxy` + : kind === "symlink-parent" + ? join(path, "proxy") + : path; + await save(); + await assert.rejects(materializeInputs("vz", source, output)); + await absent(output); + }); + }); + +test( + "rejects an unowned artifact when file-owner changes are available", + { skip: process.getuid?.() !== 0 }, + async () => { + await fixture("vz", async ({ template, source, output }) => { + await chown(template.binaries.node.path, 1, 1); + await assert.rejects(materializeInputs("vz", source, output)); + await absent(output); + }); + }, +); + +for (const kind of ["public", "oversize", "symlink", "hardlink"] as const) + test(`rejects ${kind} template without output`, async () => { + await fixture("vz", async ({ root, source, output }) => { + if (kind === "public") await chmod(source, 0o644); + if (kind === "oversize") await writeFile(source, " ".repeat((1 << 20) + 1)); + let input = source; + if (kind === "symlink" || kind === "hardlink") { + input = join(root, "linked-template"); + await (kind === "symlink" ? symlink : link)(source, input); + } + await assert.rejects(materializeInputs("vz", input, output)); + await absent(output); + }); + }); + +for (const kind of ["existing", "symlink", "hardlink", "public-parent", "noncanonical"] as const) + test(`rejects ${kind} output`, async () => { + await fixture("vz", async ({ root, source, output }) => { + if (kind === "existing") await writeFile(output, "retain", { mode: 0o600 }); + if (kind === "symlink") await symlink(source, output); + if (kind === "hardlink") await link(source, output); + if (kind === "public-parent") await chmod(root, 0o755); + const target = kind === "noncanonical" ? `${root}/./inputs.json` : output; + await assert.rejects(materializeInputs("vz", source, target)); + if (kind === "existing") assert.equal(await readFile(output, "utf8"), "retain"); + if (kind === "public-parent" || kind === "noncanonical") await absent(output); + }); + }); + +test("CLI accepts exact flags, rejects unknown backend and extra arguments, and runs with a cleared environment", async () => { + assert.deepEqual( + cli(["--backend", "vz", "--template", "/private/template", "--output", "/private/output"]), + { backend: "vz", template: "/private/template", output: "/private/output" }, + ); + for (const args of [ + [], + ["--backend", "process", "--template", "/t", "--output", "/o"], + ["--backend", "vz", "--template", "relative", "--output", "/o"], + ["--backend", "vz", "--output", "/o", "--template", "/t"], + ["--backend", "vz", "--template", "/t", "--output", "/o", "--extra"], + ]) + assert.throws(() => cli(args)); + await fixture("vz", async ({ root, source, output }) => { + const result = spawnSync( + process.execPath, + [ + "--experimental-transform-types", + join(REPO, ".github/scripts/mcp-stack-inputs.mts"), + "--backend", + "vz", + "--template", + source, + "--output", + output, + ], + { + cwd: REPO, + env: { PATH: dirname(process.execPath), TMPDIR: root }, + encoding: "utf8", + timeout: 10000, + }, + ); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, ""); + parseInputs(JSON.parse(await readFile(output, "utf8")), "vz"); + }); +}); + +test("CLI writes Inputs from a source snapshot before dependencies are installed", async () => { + await fixture("vz", async ({ root, template, source, output }) => { + const snapshot = join(root, "source"); + for (const path of [ + ".github/scripts/mcp-stack-inputs.mts", + "tests/fixtures/mcp-stack/platform", + "orchestrator/scripts/mcp-stack", + ]) { + const target = join(snapshot, path); + await mkdir(dirname(target), { recursive: true }); + await cp(join(REPO, path), target, { recursive: true }); + } + await absent(join(snapshot, "node_modules")); + await absent(join(snapshot, "orchestrator/node_modules")); + const result = spawnSync( + process.execPath, + [ + "--experimental-transform-types", + join(snapshot, ".github/scripts/mcp-stack-inputs.mts"), + "--backend", + "vz", + "--template", + source, + "--output", + output, + ], + { + cwd: snapshot, + env: { PATH: dirname(process.execPath), TMPDIR: root }, + encoding: "utf8", + timeout: 10000, + }, + ); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, ""); + const inputs = parseInputs(JSON.parse(await readFile(output, "utf8")), "vz"); + assert.equal((await lstat(output)).mode & 0o777, 0o600); + assert.equal(inputs.sourceCommit, template.sourceCommit); + assert.equal( + inputs.binaries.node.sha256, + createHash("sha256") + .update(await readFile(template.binaries.node.path)) + .digest("hex"), + ); + assert.equal(inputs.images.guest.id, template.images.guest!.id); + assert.equal(inputs.images.guest.manifestDigest, template.images.guest!.manifestDigest); + }); +}); diff --git a/.github/scripts/tests/test_mcp_helper_gate.py b/.github/scripts/tests/test_mcp_helper_gate.py new file mode 100644 index 000000000..54b693046 --- /dev/null +++ b/.github/scripts/tests/test_mcp_helper_gate.py @@ -0,0 +1,100 @@ +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / '.github/scripts/mcp-acceptance-helper.sh' +TARGETS = [ + 'managed::helper::tests::actual_helper_closed_lifetime_and_descriptor_isolation', + 'managed::helper::tests::actual_helper_release_ack_and_active_dns_cleanup', + 'managed::helper::tests::actual_helper_rejects_wrong_epoch_and_canceled_release', +] +FILTER = ' or '.join(f'test(={target})' for target in TARGETS) + + +class HelperGate(unittest.TestCase): + def test_required_count_and_command_contract(self): + # Fake cargo output only: no Rust test or helper process runs here. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifact = root / 'var/mcp-netstack-artifact' + artifact.mkdir(parents=True) + helper = artifact / 'engram-sandbox-netstack' + helper.write_text('#!/bin/sh\nexit 97\n') + helper.chmod(0o700) + digest = hashlib.sha256(helper.read_bytes()).hexdigest() + (artifact / 'SHA256SUMS').write_text(f'{digest} {helper.name}\n') + commands = root / 'bin' + commands.mkdir() + cargo = commands / 'cargo' + cargo.write_text(f'#!{sys.executable}\n' + 'import json, os, sys\n' + 'from pathlib import Path\n' + 'Path(os.environ["FAKE_CARGO_ARGS"]).write_text(json.dumps(sys.argv[1:]))\n' + 'print(os.environ["FAKE_CARGO_SUMMARY"])\n' + 'sys.exit(int(os.environ.get("FAKE_CARGO_EXIT", "0")))\n') + cargo.chmod(0o700) + args_file = root / 'args.json' + env = dict(os.environ, PATH=f'{commands}{os.pathsep}{os.environ["PATH"]}', + FAKE_CARGO_ARGS=str(args_file)) + for count in [3, 2, 1, 0, 13]: + with self.subTest(count=count): + env['FAKE_CARGO_SUMMARY'] = f'Summary [0.01s] {count} tests run: {count} passed, 37 skipped' + result = subprocess.run(['bash', str(SCRIPT)], cwd=root, env=env, + capture_output=True, text=True) + self.assertEqual(result.returncode == 0, count == 3, result.stdout + result.stderr) + self.assertEqual(json.loads(args_file.read_text()), [ + 'nextest', 'run', '-p', 'engram-sandbox-vz', '--lib', + '--run-ignored', 'ignored-only', '-E', FILTER, + '--no-tests', 'fail', '--test-threads', '1', + ]) + env['FAKE_CARGO_SUMMARY'] = 'Summary [0.01s] 3 tests run: 3 passed, 37 skipped' + env['FAKE_CARGO_EXIT'] = '42' + result = subprocess.run(['bash', str(SCRIPT)], cwd=root, env=env, capture_output=True) + self.assertNotEqual(result.returncode, 0) + del env['FAKE_CARGO_EXIT'] + for invalid in ['checksum', 'executable']: + with self.subTest(artifact=invalid): + args_file.unlink(missing_ok=True) + if invalid == 'checksum': + helper.write_text('#!/bin/sh\nexit 98\n') + else: + helper.write_text('#!/bin/sh\nexit 97\n') + helper.chmod(0o600) + result = subprocess.run(['bash', str(SCRIPT)], cwd=root, env=env, capture_output=True) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(args_file.exists(), 'invalid artifact must fail before cargo') + + def test_workflow_owns_all_three_targets_once(self): + workflow = (ROOT / '.github/workflows/ci.yml').read_text() + lane = workflow.split(' vz:', 1)[1].split(' test-macos:', 1)[0] + ignored = lane.split(' - name: Run ignored VZ tests', 1)[1].split(' - name:', 1)[0] + exclusions = ' and '.join(f'not test(={target})' for target in TARGETS) + self.assertIn(f"-E 'not test(e2e_vz) and {exclusions}'", ignored) + gate = lane.split(' - name: Run verified packaged helper tests', 1)[1].split(' - name:', 1)[0] + self.assertIn(' if: always()', gate) + self.assertIn('run: bash .github/scripts/mcp-acceptance-helper.sh', gate) + self.assertIn('needs: [detect, netstack]', lane) + self.assertIn("if: needs.detect.outputs.test_rust == 'true'", lane) + self.assertIn('name: sandbox-netstack-macos', lane) + self.assertIn('shasum -a 256 -c SHA256SUMS', lane) + self.assertIn('chmod +x engram-sandbox-netstack', lane) + self.assertIn(' - vz', workflow.split(' ci-gate:', 1)[1]) + producer = workflow.split(' netstack:', 1)[1].split(' orchestrator:', 1)[0] + self.assertIn('sandbox-netstack/build.sh build', producer) + self.assertIn('shasum -a 256 engram-sandbox-netstack > SHA256SUMS', producer) + self.assertIn('name: sandbox-netstack-${{ matrix.platform }}', producer) + self.assertIn('if-no-files-found: error', producer) + discovery = "python3 -m unittest discover -s .github/scripts/tests -p 'test_mcp*.py'" + self.assertEqual(workflow.count(discovery), 1) + detector = workflow.split(' detect:', 1)[1].split(' - name: Detect changed lanes', 1)[0] + self.assertIn(f'run: {discovery}', detector) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/tests/test_mcp_lanes.py b/.github/scripts/tests/test_mcp_lanes.py new file mode 100644 index 000000000..2b999f4b7 --- /dev/null +++ b/.github/scripts/tests/test_mcp_lanes.py @@ -0,0 +1,222 @@ +import contextlib +import importlib.util +import io +import json +import os +from pathlib import Path +import sys +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +SCRIPT = Path(__file__).resolve().parents[1] / 'detect-rebake-lanes.py' +spec = importlib.util.spec_from_file_location('lanes', SCRIPT) +lanes = importlib.util.module_from_spec(spec) +spec.loader.exec_module(lanes) + + +def detect(path, crates=None, closure=None): + with tempfile.NamedTemporaryFile() as output, patch.object(lanes, 'cargo_meta', return_value={}), \ + patch.object(lanes, 'crate_dirs', return_value={}), \ + patch.object(lanes, 'changed_crates', return_value=crates or set()), \ + patch.object(lanes, 'release_closure', return_value=closure or set()), \ + patch.object(sys, 'argv', ['detect', '--stdin']), \ + patch.object(sys, 'stdin', io.StringIO(path + '\n')), \ + patch.dict(os.environ, {'GITHUB_OUTPUT': output.name}), \ + contextlib.redirect_stderr(io.StringIO()): + lanes.main() + return dict(line.split('=', 1) for line in Path(output.name).read_text().splitlines()) + + +class McpLanes(unittest.TestCase): + def test_worker_runs_consumers_and_rebuilds_images(self): + result = detect('mcp-worker/src/contracts.ts') + self.assertEqual(result['test_rust'], 'true') + self.assertEqual(result['test_orchestrator'], 'true') + self.assertEqual(result['test_mcp_worker'], 'true') + self.assertEqual(json.loads(result['images_matrix']), ['orchestrator', 'mcp-worker']) + self.assertEqual(result['node_assets'], 'false') + + def test_worker_dependency_files_rebuild_both_images(self): + for path in ['mcp-worker/package.json', 'mcp-worker/package-lock.json']: + with self.subTest(path=path): + result = detect(path) + self.assertEqual(json.loads(result['images_matrix']), ['orchestrator', 'mcp-worker']) + + def test_worker_test_changes_do_not_rebuild_orchestrator(self): + result = detect('mcp-worker/test/deployment.test.ts') + self.assertEqual(json.loads(result['images_matrix']), ['mcp-worker']) + + def test_fixture_selects_native_and_static_gates(self): + result = detect('tests/fixtures/mcp-model-cli/src/main.rs') + self.assertEqual(result['test_rust'], 'true') + self.assertEqual(result['test_cross'], 'true') + self.assertEqual(result['node_assets'], 'false') + + def test_harness_and_shared_protocol_select_fixture_consumers(self): + for name in ['engram-harness-codex', 'engram-harness-claude', 'engram-harness-proto']: + with self.subTest(name=name): + result = detect(f'crates/{name}/src/main.rs', {name}, {name}) + self.assertEqual(result['test_rust'], 'true') + self.assertEqual(result['test_cross'], 'true') + + def test_linux_wrapper_target_and_gate_select_linux_lane(self): + result = detect('crates/engram-harness-claude/tests/linux_mcp_entrypoint.rs', + {'engram-harness-claude'}, {'engram-harness-claude'}) + self.assertEqual(result['test_rust'], 'true') + self.assertEqual(result['test_cross'], 'true') + self.assertEqual(detect('.github/scripts/mcp-acceptance-linux.sh')['test_rust'], 'true') + + def test_closed_namespace_source_and_gate_select_firecracker_lane(self): + source = 'crates/engram-sandbox-firecracker/src/managed/network/linux.rs' + result = detect(source, {'engram-sandbox-firecracker'}, {'engram-sandbox-firecracker'}) + self.assertEqual(result['test_fc'], 'true') + self.assertEqual(detect('.github/scripts/mcp-acceptance-fc-closed.sh')['test_fc'], 'true') + + def test_closed_namespace_gate_is_required_and_unconditional(self): + workflow = (SCRIPT.parents[1] / 'workflows/ci.yml').read_text() + lane = workflow.split(' test-fc-root:', 1)[1].split(' test-fc-e2e:', 1)[0] + step = lane.split(' - name: CLOSED namespace installed-state test (root)', 1)[1].split(' - name:', 1)[0] + self.assertIn('run: bash .github/scripts/mcp-acceptance-fc-closed.sh', step) + self.assertNotIn(' if:', step) + self.assertIn(' - test-fc-root', workflow.split(' ci-gate:', 1)[1]) + + def test_pool_paths_select_firecracker_lane(self): + for path in ['deploy/helm/engram-host-fleet/files/thin-pool.sh', + 'scripts/thin-pool-states-test.sh']: + with self.subTest(path=path): + self.assertEqual(detect(path)['test_fc'], 'true') + + def test_helper_runs_native_gate(self): + result = detect('sandbox-netstack/go.sum') + self.assertEqual(result['test_netstack'], 'true') + self.assertEqual(result['test_rust'], 'true') + self.assertEqual(result['test_fc'], 'true') + self.assertEqual(result['e2e'], 'true') + self.assertEqual(result['node_assets'], 'false') + + def test_helper_source_and_gate_select_artifact_and_vz_lanes(self): + for path, crates in [ + ('crates/engram-sandbox-vz/src/managed/helper.rs', {'engram-sandbox-vz'}), + ('.github/scripts/mcp-acceptance-helper.sh', set()), + ('.github/workflows/ci.yml', set()), + ]: + with self.subTest(path=path): + result = detect(path, crates, crates) + self.assertEqual(result['test_rust'], 'true') + self.assertEqual(result['test_netstack'], 'true') + + def test_stack_fixture_selects_worker_and_orchestrator_gates(self): + for path in ['tests/fixtures/mcp-stack/main.mts', + 'tests/fixtures/mcp-stack/durable.test.mts', + 'tests/fixtures/mcp-stack/process.test.mts', + 'tests/fixtures/mcp-stack/tsconfig.json', + 'tests/fixtures/mcp-stack/platform/supervisor.ts', + 'tests/fixtures/mcp-stack/platform/platform.test.ts', + 'tests/fixtures/mcp-stack/platform/composition.test.ts', + 'tests/fixtures/mcp-stack/platform/fc-thin.ts', + 'tests/fixtures/mcp-stack/platform/fc-thin.test.ts', + 'tests/fixtures/mcp-stack/platform/tsconfig.json']: + with self.subTest(path=path): + result = detect(path) + self.assertEqual(result['test_mcp_worker'], 'true') + self.assertEqual(result['test_orchestrator'], 'true') + + def test_source_gate_paths_select_required_jobs(self): + for path in ['orchestrator/scripts/mcp-stack/run.ts', + 'orchestrator/scripts/mcp-stack/tsconfig.json', + '.github/scripts/mcp-acceptance-native.sh', + '.github/scripts/tests/test_mcp_source_gates.py', + '.github/workflows/ci.yml']: + with self.subTest(path=path): + self.assertEqual(detect(path)['test_orchestrator'], 'true') + workflow = (SCRIPT.parents[1] / 'workflows/ci.yml').read_text() + worker = workflow.split(' mcp-worker:', 1)[1].split(' netstack:', 1)[0] + self.assertIn("if: needs.detect.outputs.test_mcp_worker == 'true'", worker) + self.assertIn('node-version: 24.22.2', worker) + self.assertIn('mkdir -p var/mcp-stack-fixture', worker) + self.assertIn('node --test --test-concurrency=1 --test-timeout=30000', worker) + self.assertIn('tests/fixtures/mcp-stack/durable.test.mts', worker) + self.assertIn('tests/fixtures/mcp-stack/process.test.mts', worker) + self.assertNotIn('mcp-worker/test/fixture.test.ts', worker) + orch = workflow.split(' orchestrator:', 1)[1].split(' buf:', 1)[0] + self.assertIn("if: needs.detect.outputs.test_orchestrator == 'true'", orch) + self.assertIn('run: bun run typecheck', orch) + self.assertIn('run: bun run tsc -p scripts/mcp-stack/tsconfig.json --noEmit', orch) + self.assertIn('bun test scripts/mcp-stack', orch) + self.assertIn('0 fail', orch) + self.assertIn('run: bun run tsc -p ../tests/fixtures/mcp-stack/platform/tsconfig.json --noEmit', orch) + platform = orch.split(' - name: MCP platform source unit tests', 1)[1].split(' - name:', 1)[0] + self.assertIn(' working-directory: ${{ github.workspace }}\n', platform) + self.assertIn(' umask 077\n mkdir -p var/mcp-platform-composition\n', platform) + self.assertIn('bun test tests/fixtures/mcp-stack/platform', platform) + self.assertNotIn(' if:', platform) + project = json.loads((SCRIPT.parents[2] / 'tests/fixtures/mcp-stack/platform/tsconfig.json').read_text()) + self.assertEqual(project['extends'], '../../../../orchestrator/tsconfig.json') + self.assertEqual(project['include'], ['./*.ts']) + settings = json.loads((SCRIPT.parents[2] / 'orchestrator/tsconfig.json').read_text()) + self.assertTrue(settings['compilerOptions']['strict']) + gate = workflow.split(' ci-gate:', 1)[1] + for lane in ['orchestrator', 'mcp-worker', 'test-linux', 'test-macos']: + self.assertIn(f' - {lane}\n', gate) + + def test_fc_boot_manifest_is_an_optional_literal_argument_from_admin_environment(self): + workflow = (SCRIPT.parents[1] / 'workflows/ci.yml').read_text() + lane = workflow.split(' mcp-physical-fc:', 1)[1].split(' mcp-physical-vz:', 1)[0] + self.assertIn('MCP_FC_BOOT_ARTIFACTS: ${{ vars.MCP_FC_BOOT_ARTIFACTS }}', lane) + run = lane.split(' run: |\n', 1)[1].split(' - uses:', 1)[0] + self.assertNotIn('${{', run) + argv = run[run.index(' args=('):] + script = 'set -euo pipefail\nroot=/owned/run\nsudo() { printf "%s\\0" "$@"; }\n' + argv + expected = ['--preserve-env=GITHUB_TOKEN', 'python3', '-I', '.github/scripts/mcp-stack-assets.py', + '--backend', 'firecracker', '--operator', '/etc/engrams/mcp-physical-firecracker.json', + '--work-root', '/owned/run'] + for manifest in [None, '', '/owned/manifest with spaces.json', '/owned/$(false);manifest.json']: + with self.subTest(manifest=manifest): + env = {'PATH': '/usr/bin:/bin'} + if manifest is not None: + env['MCP_FC_BOOT_ARTIFACTS'] = manifest + result = subprocess.run(['/bin/bash', '-c', script], env=env, capture_output=True, check=True) + actual = result.stdout.decode().rstrip('\0').split('\0') + self.assertEqual(actual, expected + (['--boot-artifacts', manifest] if manifest else [])) + self.assertIn('runs-on: [self-hosted, Linux, X64, engram-mcp-firecracker]', lane) + self.assertIn('cancel-in-progress: false', lane) + self.assertIn('MCP physical Firecracker (24 cases)', lane) + self.assertIn(' - mcp-physical-fc\n', workflow.split(' ci-gate:', 1)[1]) + + def test_complete_physical_producer_selector(self): + for path in ['tests/fixtures/mcp-stack/platform/fc-thin.ts', + 'tests/fixtures/mcp-stack/platform/fc-thin.test.ts', + 'tests/fixtures/mcp-stack/platform/ci/linux-assets.Dockerfile', + '.github/scripts/mcp-stack-assets.py', '.github/scripts/mcp-stack-inputs.mts', + '.github/scripts/mcp-stack-fc.sh', 'docker/coordinator.Dockerfile', + 'deploy/bundles/agentd/build.sh', 'deploy/demo/Dockerfile', + 'deploy/kernel/build-fc-kernel.sh', 'third_party/firecracker', + 'sandbox-netstack/build.sh', '.sqlx/query.json', 'deploy/migrations/100.sql', + 'orchestrator/scripts/mcp-stack/run.ts', 'mcp-worker/package-lock.json', + 'crates/engram-host-agent/src/lib.rs', '.bun-version', '.github/workflows/ci.yml', + '.dockerignore', '.cargo/config.toml']: + with self.subTest(path=path): + self.assertEqual(detect(path)['test_mcp_physical'], 'true') + for path in ['README.md', 'web/src/App.tsx', 'site/src/content/docs/index.mdx']: + with self.subTest(path=path): + self.assertEqual(detect(path)['test_mcp_physical'], 'false') + + def test_unrelated_path_does_not_run_new_lanes(self): + result = detect('README.md') + self.assertEqual(result['test_mcp_worker'], 'false') + self.assertEqual(result['test_netstack'], 'false') + + def test_callback_canary_runs_for_its_profiles_runtime_and_script(self): + for path in ['deploy/helm/engram/templates/web-configmap.yaml', + 'deploy/helm/engram/values-aws.yaml.example', + 'deploy/terraform/gcp/quickstart/eso.tf', + 'deploy/helm/engram/ci/callback-ingress-canary.py', + 'docker/web.Dockerfile']: + with self.subTest(path=path): + self.assertEqual(detect(path)['test_deploy'], 'true') + self.assertEqual(detect('README.md')['test_deploy'], 'false') + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/tests/test_mcp_physical_gates.py b/.github/scripts/tests/test_mcp_physical_gates.py new file mode 100644 index 000000000..b00367995 --- /dev/null +++ b/.github/scripts/tests/test_mcp_physical_gates.py @@ -0,0 +1,1391 @@ +import importlib.util +from contextlib import contextmanager +import hashlib +import io +import json +import os +import signal +import shutil +import struct +import time +from pathlib import Path +import socket +import subprocess +import sys +import tempfile +import tarfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[3] +BUILD_CONFIG = b'{"os":"linux","architecture":"arm64","config":{"Tty":true,"StopTimeout":1,"Env":["A=1","B=2"]}}' +BUILD_CONFIG_ID = 'sha256:e1c93a3ed1766bed27c966f53476dd47fde4ae1f2cd16f933f3d989a55d0f789' +SPEC = importlib.util.spec_from_file_location('physical_assets', ROOT / '.github/scripts/mcp-stack-assets.py') +assets = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(assets) + + +class PhysicalGates(unittest.TestCase): + @contextmanager + def boot_fixture(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + p, source, git = self.source_fixture(root) + git('update-index', '--add', '--cacheinfo', '160000,' + 'a' * 40 + ',third_party/firecracker') + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-m', 'fixture fork') + p.backend, p.arch = 'firecracker', 'amd64' + p.files = p.root / 'files' + p.files.mkdir(mode=0o700) + p.tools = {'gh': 'forbidden-gh', 'oras': 'forbidden-oras'} + call = p.call + def git_only(argv, **kwargs): + self.assertEqual(argv[0], '/usr/bin/git', 'Boot input must not start a download or executable') + return call(argv, **kwargs) + p.call = git_only + bundle = root / 'boot' + bundle.mkdir(mode=0o700) + ident = b'\x7fELF\x02\x01\x01' + bytes(9) + elf = struct.pack('<16sHHIQQQIHHHHHH', ident, 2, 62, 1, 0x400000, 64, 0, 0, 64, 56, 1, 0, 0, 0) + elf += struct.pack(' /etc/apt/sources.list; \\\n', 1)[1].split( + ' rm -rf /var/lib/apt/lists/', 1)[0].replace('\\\n', '') + shell = 'set -eu;\n' + commands + subprocess.run(['/bin/sh', '-n'], input=shell, text=True, check=True) + cases = [(0, 0, 0, 1, 1), (2, 1, 0, 3, 2), (3, 3, 0, 4, 4), + (4, 0, 1, 4, 0), (0, 4, 1, 1, 4)] + for update_failures, install_failures, failed, updates, installs in cases: + with self.subTest(update_failures=update_failures, install_failures=install_failures): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + log = root / 'calls.json' + stub = root / 'apt-get' + stub.write_text(f'#!{sys.executable} -I\n' + + 'import json, os, sys\n' + 'from pathlib import Path\n' + 'log = Path(os.environ["APT_STUB_LOG"])\n' + 'calls = json.loads(log.read_text()) if log.exists() else []\n' + 'args = sys.argv[1:]\n' + 'operation = "update" if "update" in args else "install"\n' + 'calls.append({"operation": operation, "args": args})\n' + 'log.write_text(json.dumps(calls))\n' + 'attempt = sum(c["operation"] == operation for c in calls)\n' + 'sys.exit(100 if attempt <= int(os.environ[operation.upper() + "_FAILURES"]) else 0)\n') + stub.chmod(0o700) + result = subprocess.run(['/bin/sh', '-c', shell], capture_output=True, + text=True, timeout=5, env={ + 'PATH': directory, 'APT_STUB_LOG': str(log), + 'UPDATE_FAILURES': str(update_failures), + 'INSTALL_FAILURES': str(install_failures), + }) + self.assertEqual(result.returncode != 0, bool(failed), result.stderr) + calls = json.loads(log.read_text()) + self.assertEqual([call['operation'] for call in calls], + ['update'] * updates + ['install'] * installs) + for call in calls: + args = call['args'] + self.assertEqual(args[:4], ['-o', 'Acquire::Retries=0', + '-o', 'Acquire::http::Timeout=30']) + if call['operation'] == 'update': + self.assertEqual(args[4:], ['--error-on=any', 'update']) + + def test_thin_pool_operator_rejects_unknown_keys_names_paths_and_modes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + valid = {'name': 'suite', 'directory': str(root)} + self.assertEqual(assets.validate_thin_pool(valid), valid) + for value in [None, {}, {**valid, 'extra': True}, {**valid, 'name': 'a' * 128}, + {**valid, 'name': '../pool'}, {**valid, 'name': 'pool;true'}, + {**valid, 'directory': str(root) + '/..'}]: + with self.subTest(value=value), self.assertRaises((RuntimeError, TypeError)): + assets.validate_thin_pool(value) + root.chmod(0o755) + with self.assertRaisesRegex(RuntimeError, 'operator_thin_pool_mode'): + assets.validate_thin_pool(valid) + root.chmod(0o700) + with patch.object(assets.os, 'getuid', return_value=os.getuid() + 1): + with self.assertRaisesRegex(RuntimeError, 'path_owner'): + assets.validate_thin_pool(valid) + + def test_source_context_rejects_secrets(self): + assets.source_paths(['docker/host-agent.Dockerfile', '.env.example', 'orchestrator/src/index.ts']) + for path in ['.env', 'orchestrator/.env.local', 'var/key', '.git/config', '.superpowers/private', + 'crates/engram-coordinator/var/key', '../key', '/tmp/key']: + with self.subTest(path=path), self.assertRaises(RuntimeError): + assets.source_paths([path]) + + def source_fixture(self, root, extra=()): + source, work = root / 'fixture', root / 'work' + source.mkdir() + work.mkdir(mode=0o700) + env = {'PATH': '/usr/bin:/bin', 'LANG': 'C', 'GIT_CONFIG_NOSYSTEM': '1', + 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_CONFIG_SYSTEM': '/dev/null'} + def git(*args): + return subprocess.run(['/usr/bin/git', '-C', str(source), *args], env=env, + capture_output=True, check=True) + git('init', '--template=') + recipes = ['docker/coordinator.Dockerfile', 'docker/orchestrator.Dockerfile', + 'docker/mcp-worker.Dockerfile', 'deploy/demo/Dockerfile', + '.github/scripts/mcp-stack-inputs.mts', '.github/scripts/mcp-stack-fc.sh', + '.github/scripts/mcp-stack-vz.sh', 'tests/fixtures/mcp-stack/platform/main.ts', + 'tests/fixtures/mcp-stack/platform/fixture.Dockerfile', + 'tests/fixtures/mcp-stack/platform/guest.Dockerfile', + 'tests/fixtures/mcp-stack/platform/guard.Dockerfile', + 'tests/fixtures/mcp-stack/platform/host-agent.Dockerfile'] + for name in [*recipes, 'tracked.txt', *extra]: + path = source / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('public fixture source\n') + git('add', '.') + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-m', 'fixture') + p = object.__new__(assets.Producer) + p.root, p.source, p.env = work, work / 'source', env + p.counter, p.cancelled, p.child, p.groups = 0, False, None, {} + call = p.call + p.call = lambda argv, **kwargs: call(argv, cwd=source, **kwargs) + return p, source, git + + def test_snapshot_excludes_only_known_legacy_runtime_directory(self): + with tempfile.TemporaryDirectory() as directory: + legacy = 'crates/engram-coordinator/var/snapshots/fixture.bin' + safe = ['literal[1].txt', 'line\nbreak.txt', + 'crates/engram-coordinator/var-source/fixture.rs'] + p, source, git = self.source_fixture(Path(directory), [legacy, *safe]) + (source / 'third_party/fixture').mkdir(parents=True) + git('update-index', '--add', '--cacheinfo', + '160000,' + git('rev-parse', 'HEAD').stdout.decode().strip() + ',third_party/fixture') + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-m', 'fixture gitlink') + (source / '.env').write_text('untracked fixture secret\n') + archives = [] + call = p.call + def capture(argv, **kwargs): + result = call(argv, **kwargs) + if 'archive' in argv: + archives.append(result) + return result + p.call = capture + p.snapshot() + self.assertEqual(p.head, git('rev-parse', 'HEAD').stdout.decode().strip()) + self.assertEqual((p.source / 'tracked.txt').read_text(), 'public fixture source\n') + for name in safe: + self.assertTrue((p.source / name).is_file()) + self.assertFalse((p.source / 'crates/engram-coordinator/var').exists()) + self.assertFalse((p.source / '.env').exists()) + self.assertTrue((p.source / 'third_party/fixture').is_dir()) + self.assertTrue((source / legacy).is_file()) + self.assertEqual(len(archives), 1) + with tarfile.open(fileobj=io.BytesIO(archives[0])) as tar: + members = {m.name for m in tar.getmembers()} + self.assertIn('tracked.txt', members) + self.assertNotIn(legacy, members) + self.assertNotIn('crates/engram-coordinator/var', members) + + def test_snapshot_rejects_other_forbidden_tracked_paths(self): + for name in ['var/key', 'crates/other/var/key', '.env', 'orchestrator/.env.local', + '.superpowers/private', 'crates/engram-coordinator/var-old/var/key']: + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + p, _, _ = self.source_fixture(Path(directory), [name]) + with self.assertRaisesRegex(RuntimeError, 'secret_source_context'): + p.snapshot() + self.assertFalse(p.source.exists()) + + def test_snapshot_preserves_public_source_modes_under_private_umask(self): + with tempfile.TemporaryDirectory() as directory: + files = ['orchestrator/src/db/migrate.ts', 'orchestrator/drizzle/meta/_journal.json', + 'scripts/run.sh'] + p, source, git = self.source_fixture(Path(directory), files) + (source / 'scripts/run.sh').chmod(0o755) + git('update-index', '--chmod=+x', 'scripts/run.sh') + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-m', 'fixture executable') + (source / '.env').write_text('untracked fixture secret\n') + previous_umask = os.umask(0o077) + try: + p.snapshot() + finally: + os.umask(previous_umask) + for path in [p.root, p.source]: + self.assertEqual(path.stat().st_mode & 0o7777, 0o700) + for name in ['orchestrator', 'orchestrator/src', 'orchestrator/src/db', + 'orchestrator/drizzle', 'orchestrator/drizzle/meta', 'scripts']: + with self.subTest(directory=name): + self.assertEqual((p.source / name).stat().st_mode & 0o7777, 0o755) + for name, mode in [('tracked.txt', 0o644), (files[0], 0o644), + (files[1], 0o644), (files[2], 0o755)]: + with self.subTest(file=name): + self.assertEqual((p.source / name).stat().st_mode & 0o7777, mode) + self.assertEqual((p.source / name).read_bytes(), b'public fixture source\n') + self.assertFalse((p.source / '.env').exists()) + + def test_snapshot_checks_all_archive_members_before_extraction(self): + for name, kind in [('unexpected.txt', tarfile.REGTYPE), ('var/key', tarfile.REGTYPE), + ('.env', tarfile.REGTYPE), ('.git/config', tarfile.REGTYPE), + ('.superpowers/private', tarfile.REGTYPE), + ('crates/engram-coordinator/var/key', tarfile.REGTYPE), + ('../escape', tarfile.REGTYPE), ('/tmp/escape', tarfile.REGTYPE), + ('safe/../../escape', tarfile.REGTYPE), ('./tracked.txt', tarfile.REGTYPE), + ('tracked.txt', tarfile.REGTYPE), ('unexpected', tarfile.DIRTYPE), + ('tracked.txt', tarfile.SYMTYPE), ('tracked.txt', tarfile.LNKTYPE), + ('tracked.txt', tarfile.FIFOTYPE), (None, None)]: + with self.subTest(name=name, kind=kind), tempfile.TemporaryDirectory() as directory: + p, _, _ = self.source_fixture(Path(directory)) + call = p.call + def corrupt(argv, **kwargs): + result = call(argv, **kwargs) + if 'archive' not in argv: + return result + output = io.BytesIO() + with (tarfile.open(fileobj=io.BytesIO(result)) as original, + tarfile.open(fileobj=output, mode='w') as tar): + for member in original.getmembers(): + if member.name == 'tracked.txt' and (name is None or kind != tarfile.REGTYPE): + continue + tar.addfile(member, original.extractfile(member) if member.isfile() else None) + if name is not None: + member = tarfile.TarInfo(name) + member.type = kind + if kind in {tarfile.SYMTYPE, tarfile.LNKTYPE}: + member.linkname = 'tracked.txt' + tar.addfile(member) + return output.getvalue() + p.call = corrupt + with self.assertRaises(RuntimeError): + p.snapshot() + self.assertFalse((p.source / 'tracked.txt').exists()) + self.assertFalse((p.source / 'docker').exists()) + + def test_archive_identity_rejects_index_wrong_arch_and_digest(self): + config = json.dumps({'os': 'linux', 'architecture': 'arm64'}).encode() + raw = json.dumps({'schemaVersion': 2, 'config': {'digest': assets.digest(config)}}).encode() + facts = assets.image_facts(raw, config, 'arm64') + self.assertNotEqual(facts['id'], facts['manifestDigest']) + for value, cfg, arch in [ + (raw, config, 'amd64'), + (json.dumps({'schemaVersion': 2, 'manifests': []}).encode(), config, 'arm64'), + (raw.replace(assets.digest(config).encode(), b'sha256:' + b'a' * 64), config, 'arm64'), + ]: + with self.assertRaises(RuntimeError): + assets.image_facts(value, cfg, arch) + + @contextmanager + def archive_fixture(self): + # Only controlled executables run. No Docker daemon or image is used. + with tempfile.TemporaryDirectory() as directory, socket.socket(socket.AF_UNIX) as sock: + root = Path(directory).resolve() + sock.bind(str(root / 'engine.sock')) + script = f'''#!{sys.executable} +import hashlib, json, os, sys +from pathlib import Path +root = Path({str(root)!r}) +kind, args = Path(sys.argv[0]).name, sys.argv[1:] +with (root / 'calls').open('a') as out: + out.write(json.dumps([kind, *args]) + '\\n') +operation = (args[4:6] if kind == 'docker' else + [kind, 'inspect-' + ('docker' if args[-1].startswith('docker-archive:') else 'oci') + if 'inspect' in args else 'copy']) +if (root / 'fail').exists() and operation == json.loads((root / 'fail').read_text()): + raise SystemExit(42) +def emit(path, data, stage): + mode = (root / (stage + '-kind')).read_text() if (root / (stage + '-kind')).exists() else 'normal' + if mode in ['symlink', 'hardlink']: + target = root / (stage + '-target') + target.write_bytes(data) + path.symlink_to(target) if mode == 'symlink' else os.link(target, path) + else: + path.write_bytes(b'' if mode == 'empty' else data) +if kind == 'docker': + assert not {{'HOME', 'DOCKER_HOST', 'DOCKER_CONTEXT', 'GITHUB_TOKEN', 'REGISTRY_AUTH_FILE'}}.intersection(os.environ) + assert args[:4] == ['--config', str(root / 'producer-engine-config'), '--host', 'unix://' + str(root / 'engine.sock')] + args = args[4:] + if args[0] == 'info': + print(json.dumps({{'ID': 'controlled-engine-identity'}})) + elif args[:2] == ['buildx', 'build']: + assert args[2:4] == ['--builder', 'default'] + if '--load' in args: + path = root / 'metadata' + if path.exists(): + emit(Path(args[args.index('--metadata-file') + 1]), path.read_bytes(), 'metadata') + else: + output = args[args.index('--output') + 1] + assert output.startswith('type=local,dest=') + Path(output.split('dest=', 1)[1]).mkdir() + elif args[:2] == ['image', 'inspect']: + actual = json.loads((root / 'inspect').read_bytes()) + identity = {BUILD_CONFIG_ID!r} + if (root / 'loaded').exists(): + identity = 'sha256:' + hashlib.sha256((root / 'config-bytes').read_bytes()).hexdigest() + actual[0]['Id'] = identity + assert args[2:] == [identity] + print(json.dumps(actual)) + elif args[:2] == ['image', 'save']: + assert args[2] == '--output' and args[4:] == [{BUILD_CONFIG_ID!r}] + emit(Path(args[3]), b'controlled-docker-archive', 'save') + elif args[0] == 'load': + assert args[1] == '--input' and Path(args[2]).read_bytes() == b'controlled-docker-load' + (root / 'loaded').touch() + else: + raise AssertionError(args) +else: + assert kind == 'skopeo' + if 'inspect' in args: + assert '--raw' in args + if args[-1].startswith('docker-archive:'): + assert '--config' in args + sys.stdout.buffer.write((root / 'saved-config-bytes').read_bytes()) + elif '--config' in args: + sys.stdout.buffer.write((root / 'config-bytes').read_bytes()) + else: + config = (root / 'config-bytes').read_bytes() + print(json.dumps({{'schemaVersion': 2, 'config': {{'digest': 'sha256:' + hashlib.sha256(config).hexdigest()}}}})) + else: + assert 'copy' in args + if args[-2].startswith('docker-archive:'): + assert Path(args[-2].removeprefix('docker-archive:')).read_bytes() == b'controlled-docker-archive' + Path(args[-1].removeprefix('oci-archive:')).write_bytes(b'controlled-oci-archive') + else: + assert Path(args[-2].removeprefix('oci-archive:')).read_bytes() == b'controlled-oci-archive' + Path(args[-1].removeprefix('docker-archive:')).write_bytes(b'controlled-docker-load') +''' + tools = {} + for name in ['docker', 'skopeo']: + path = root / name + path.write_text(script) + path.chmod(0o700) + tools[name] = str(path) + p = assets.Producer('vz', root, {'engineSocket': str(root / 'engine.sock'), 'tools': tools}) + p.engine_identity, p.head = 'controlled-engine-identity', 'b' * 40 + (root / 'config-bytes').write_bytes(BUILD_CONFIG) + (root / 'saved-config-bytes').write_bytes(BUILD_CONFIG) + (root / 'metadata').write_text(json.dumps({ + 'containerimage.config.digest': BUILD_CONFIG_ID, + 'containerimage.digest': 'sha256:' + 'c' * 64})) + actual = {'Id': BUILD_CONFIG_ID, 'Os': 'linux', 'Architecture': 'arm64', + 'Config': {'Labels': {'org.opencontainers.image.revision': p.head, assets.LABEL: p.run}}} + (root / 'inspect').write_text(json.dumps([actual])) + yield p, actual + + def test_archive_loads_then_saves_exact_config_before_oci_conversion(self): + with self.archive_fixture() as (p, _): + context = p.root / 'custom-context' + archive = p.archive('hostAgent', 'Dockerfile', context, {'HOST_IMAGE': 'controlled-base'}) + self.assertEqual(archive.read_bytes(), b'controlled-oci-archive') + calls = [json.loads(line) for line in (p.root / 'calls').read_text().splitlines()] + operations = [call[5:] for call in calls if call[0] == 'docker' and call[5] != 'info'] + self.assertEqual([call[:2] for call in operations], + [['buildx', 'build'], ['image', 'inspect'], ['image', 'save']]) + build, inspect, save = operations + self.assertIn('--load', build) + self.assertNotIn('--output', build) + self.assertNotIn('--push', build) + self.assertIn('--provenance=false', build) + self.assertIn('--sbom=false', build) + self.assertIn('org.opencontainers.image.revision=' + p.head, build) + self.assertIn(assets.LABEL + '=' + p.run, build) + self.assertIn('SOURCE_COMMIT=' + p.head, build) + self.assertEqual(build[build.index('--platform') + 1], 'linux/arm64') + metadata = Path(build[build.index('--metadata-file') + 1]) + self.assertEqual(metadata.parent, p.root) + self.assertEqual(metadata.read_bytes(), (p.root / 'metadata').read_bytes()) + self.assertEqual(build[build.index('--file') + 1], str(context / 'Dockerfile')) + self.assertIn('HOST_IMAGE=controlled-base', build) + self.assertEqual(build[-1], str(context)) + self.assertEqual(inspect[2:], [BUILD_CONFIG_ID]) + self.assertEqual(save, ['image', 'save', '--output', str(p.files / 'hostAgent.docker.tar'), + BUILD_CONFIG_ID]) + self.assertEqual(calls[-1][0], 'skopeo') + for call in operations: + self.assertNotIn('--tag', call) + self.assertNotIn('-t', call) + self.assertNotIn('--iidfile', call) + self.assertEqual(p.names, []) + + def test_guest_archive_creates_private_state_before_start_and_keeps_existing_proof(self): + with self.archive_fixture() as (p, _): + demo = 'deploy/demo/Dockerfile' + selected = assets.SOURCE_IMAGES['guest'] + for name in {demo, selected}: + target = p.source / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((ROOT / name).read_bytes()) + p.archive('guest', selected) + calls = [json.loads(line) for line in (p.root / 'calls').read_text().splitlines()] + build = next(call for call in calls if call[0] == 'docker' and call[5:7] == ['buildx', 'build']) + recipe = Path(build[build.index('--file') + 1]).read_bytes() + self.assertTrue(recipe.startswith((ROOT / demo).read_bytes())) + self.assertEqual(recipe, (ROOT / demo).read_bytes() + b'\n' + (ROOT / selected).read_bytes()) + self.assertEqual(build[-1], str(p.source)) + self.assertIn('SOURCE_COMMIT=' + p.head, build) + self.assertIn('org.opencontainers.image.revision=' + p.head, build) + self.assertIn(assets.LABEL + '=' + p.run, build) + instruction = recipe.decode().strip().splitlines()[-1] + self.assertTrue(instruction.startswith('RUN ')) + state = p.root / 'state' + command = instruction.removeprefix('RUN ').replace('/tmp/mcp-stack-fixture', str(state)) + command = command.replace('/workspace/.engrams', str(p.root / 'workspace')) + def setup(): + subprocess.run(['/bin/sh', '-c', 'umask 022; ' + command], check=True) + setup() + self.assertTrue(state.is_dir()) + self.assertFalse(state.is_symlink()) + self.assertEqual(state.stat().st_mode & 0o777, 0o700) + self.assertEqual(state.stat().st_uid, os.getuid()) + proof = state / 'proof.jsonl' + proof.write_bytes(b'existing proof\n') + setup() + self.assertEqual(proof.read_bytes(), b'existing proof\n') + self.assertEqual(state.stat().st_mode & 0o777, 0o700) + self.assertEqual(state.stat().st_uid, os.getuid()) + + def test_archive_rejects_missing_or_invalid_config_metadata_before_inspect(self): + for metadata in [None, b'not-json', b'[]', b'{}', + json.dumps({'containerimage.digest': BUILD_CONFIG_ID}).encode(), + *[json.dumps({'containerimage.config.digest': value}).encode() + for value in [None, 7, 'a' * 64, 'sha256:' + 'A' * 64, + 'sha256:' + 'a' * 63, BUILD_CONFIG_ID + '\n']]]: + with self.subTest(metadata=metadata), self.archive_fixture() as (p, _): + if metadata is None: + (p.root / 'metadata').unlink() + else: + (p.root / 'metadata').write_bytes(metadata) + with self.assertRaises((RuntimeError, ValueError, FileNotFoundError)) as failure: + p.archive('worker', 'docker/mcp-worker.Dockerfile') + if isinstance(failure.exception, RuntimeError): + self.assertEqual(str(failure.exception), 'build_image_config_digest') + calls = [json.loads(line) for line in (p.root / 'calls').read_text().splitlines()] + self.assertFalse(any(call[0] == 'skopeo' or call[5:7] in + [['image', 'inspect'], ['image', 'save']] for call in calls)) + self.assertFalse((p.files / 'worker.docker.tar').exists()) + + def test_archive_rejects_inspect_mismatch_before_save_or_conversion(self): + for field, value in [('Id', 'sha256:' + 'c' * 64), ('Os', 'darwin'), ('Architecture', 'amd64'), + ('revision', 'c' * 40), ('run', 'foreign'), ('Config', {}), + ('Labels', None)]: + with self.subTest(field=field), self.archive_fixture() as (p, actual): + if field in ['revision', 'run']: + key = 'org.opencontainers.image.revision' if field == 'revision' else assets.LABEL + actual['Config']['Labels'][key] = value + elif field == 'Labels': + actual['Config']['Labels'] = value + else: + actual[field] = value + (p.root / 'inspect').write_text(json.dumps([actual])) + seam = 'build_image_platform' if field in ['Id', 'Os', 'Architecture'] else 'build_image_labels' + with self.assertRaisesRegex(RuntimeError, seam): + p.archive('worker', 'docker/mcp-worker.Dockerfile') + calls = [json.loads(line) for line in (p.root / 'calls').read_text().splitlines()] + self.assertFalse(any(call[0] == 'skopeo' or call[5:7] == ['image', 'save'] for call in calls)) + self.assertFalse((p.files / 'worker.docker.tar').exists()) + + def test_archive_local_directory_output_keeps_default_driver_path(self): + with self.archive_fixture() as (p, _): + output = p.archive('linux-assets', assets.CI + '/linux-assets.Dockerfile', local=True) + self.assertEqual(output, p.files / 'linux-assets') + self.assertTrue(output.is_dir()) + calls = [json.loads(line) for line in (p.root / 'calls').read_text().splitlines()] + builds = [call[5:] for call in calls if call[0] == 'docker' and call[5] != 'info'] + self.assertEqual(len(builds), 1) + build = builds[0] + self.assertEqual(build[build.index('--output') + 1], 'type=local,dest=' + str(output)) + self.assertNotIn('--load', build) + self.assertNotIn('--metadata-file', build) + self.assertFalse(any(call[0] == 'skopeo' for call in calls)) + + def test_archive_requires_one_inspected_image(self): + for response in [[], [{}, {}]]: + with self.subTest(response=response), self.archive_fixture() as (p, actual): + if response: + response = [actual, actual] + (p.root / 'inspect').write_text(json.dumps(response)) + with self.assertRaisesRegex(RuntimeError, 'build_image_inspect'): + p.archive('worker', 'docker/mcp-worker.Dockerfile') + self.assertFalse((p.files / 'worker.docker.tar').exists()) + + def test_archive_requires_owned_regular_metadata_and_saved_file(self): + for stage in ['metadata', 'save']: + for kind in ['symlink', 'hardlink', 'empty']: + with self.subTest(stage=stage, kind=kind), self.archive_fixture() as (p, _): + (p.root / (stage + '-kind')).write_text(kind) + with self.assertRaises(RuntimeError): + p.archive('worker', 'docker/mcp-worker.Dockerfile') + calls = [json.loads(line) for line in (p.root / 'calls').read_text().splitlines()] + self.assertFalse(any(call[0] == 'skopeo' for call in calls)) + if stage == 'metadata': + self.assertFalse((p.files / 'worker.docker.tar').exists()) + + def test_archive_rejects_converted_config_mismatch(self): + with self.archive_fixture() as (p, _): + (p.root / 'config-bytes').write_bytes(b'{"os":"foreign"}') + with self.assertRaisesRegex(RuntimeError, 'build_archive_config'): + p.archive('worker', 'docker/mcp-worker.Dockerfile') + + def test_archive_saved_config_digest_must_match_before_conversion(self): + with self.archive_fixture() as (p, _): + # The same values in a different byte representation do not have the build ID. + (p.root / 'saved-config-bytes').write_bytes(json.dumps(json.loads(BUILD_CONFIG)).encode()) + with self.assertRaisesRegex(RuntimeError, 'build_saved_config'): + p.archive('worker', 'docker/mcp-worker.Dockerfile') + calls = [json.loads(line) for line in (p.root / 'calls').read_text().splitlines()] + self.assertFalse(any(call[0] == 'skopeo' and 'copy' in call for call in calls)) + self.assertFalse((p.files / 'worker.oci.tar').exists()) + + def test_archive_accepts_object_key_order_and_imports_actual_converted_id(self): + with self.archive_fixture() as (p, _): + converted = json.dumps(json.loads(BUILD_CONFIG), sort_keys=True, separators=(',', ':')).encode() + self.assertNotEqual(converted, BUILD_CONFIG) + converted_id = 'sha256:' + hashlib.sha256(converted).hexdigest() + self.assertNotEqual(converted_id, BUILD_CONFIG_ID) + (p.root / 'config-bytes').write_bytes(converted) + archive = p.archive('worker', 'docker/mcp-worker.Dockerfile') + facts = p.import_image('worker', archive) + self.assertEqual(facts['id'], converted_id) + self.assertEqual(p.images['worker']['id'], converted_id) + calls = [json.loads(line) for line in (p.root / 'calls').read_text().splitlines()] + inspections = [call[7:] for call in calls if call[0] == 'docker' and call[5:7] == ['image', 'inspect']] + self.assertEqual(inspections, [[BUILD_CONFIG_ID], [converted_id]]) + + def test_archive_rejects_config_value_type_or_list_order_changes(self): + for key, value in [('Tty', 1), ('StopTimeout', True), ('StopTimeout', 1.0), + ('StopTimeout', '1'), ('Env', ['B=2', 'A=1']), ('Env', 'A=1'), + ('NewField', None), ('StopTimeout', float('nan'))]: + with self.subTest(key=key, value=value), self.archive_fixture() as (p, _): + config = json.loads(BUILD_CONFIG) + config['config'][key] = value + (p.root / 'config-bytes').write_text(json.dumps(config)) + with self.assertRaises((RuntimeError, ValueError)): + p.archive('worker', 'docker/mcp-worker.Dockerfile') + + def test_archive_command_errors_stop_without_fallback(self): + operations = [['buildx', 'build'], ['image', 'inspect'], ['image', 'save'], + ['skopeo', 'inspect-docker'], ['skopeo', 'copy'], ['skopeo', 'inspect-oci']] + for index, operation in enumerate(operations): + with self.subTest(operation=operation), self.archive_fixture() as (p, _): + (p.root / 'fail').write_text(json.dumps(operation)) + with self.assertRaisesRegex(RuntimeError, 'producer_command'): + p.archive('worker', 'docker/mcp-worker.Dockerfile') + calls = [json.loads(line) for line in (p.root / 'calls').read_text().splitlines()] + actual = [call[5:7] if call[0] == 'docker' else + ['skopeo', 'inspect-' + ('docker' if call[-1].startswith('docker-archive:') else 'oci') + if 'inspect' in call else 'copy'] for call in calls + if call[0] != 'docker' or call[5] != 'info'] + self.assertEqual(actual, operations[:index + 1]) + + def test_prepare_dispatch_pins_inputs_without_adapter_or_matrix(self): + with tempfile.TemporaryDirectory() as directory: + p = object.__new__(assets.Producer) + p.root, p.backend, p.head = Path(directory), 'vz', 'a' * 40 + p.env, p.files = {}, p.root + path = p.root / 'inputs.json' + path.write_text('{"version":1}') + def forbidden(*args, **kwargs): + self.fail('preparation must not dispatch an adapter or inspect a matrix') + p.call = forbidden + with patch.object(assets, 'matrix_result', forbidden): + self.assertEqual(p.dispatch(prepare_only=True), 0) + receipt = json.loads((p.root / 'assets-prepared.json').read_bytes()) + self.assertEqual(receipt['kind'], 'assets-prepared') + self.assertFalse(receipt['physicalAcceptance']) + self.assertFalse(receipt['matrixCompleted']) + self.assertEqual(receipt['inputs']['sha256'], hashlib.sha256(path.read_bytes()).hexdigest()) + self.assertEqual(path.stat().st_mode & 0o777, 0o400) + self.assertFalse(getattr(p, 'adapter_started', False)) + self.assertEqual(assets.prepared_result(p.root, 'vz', p.head), 0) + path.chmod(0o600) + path.write_text('changed') + with self.assertRaises(RuntimeError): + assets.prepared_result(p.root, 'vz', p.head) + + def test_private_preparation_propagates_mode_and_verifies_nested_receipt(self): + with tempfile.TemporaryDirectory() as directory: + p = object.__new__(assets.Producer) + p.root, p.backend, p.operator_path = Path(directory), 'vz', '/owned/operator.json' + p.env = {'PATH': '/usr/bin:/bin'} + p.private_checkout = lambda source: (assets.REPO, 'a' * 40) + calls = [] + def call(argv, **kwargs): + calls.append((list(map(str, argv)), kwargs)) + nested = object.__new__(assets.Producer) + nested.root, nested.backend, nested.head = p.root / 'execution', p.backend, 'a' * 40 + nested.root.mkdir() + (nested.root / 'inputs.json').write_text('{"version":1}') + nested.dispatch(prepare_only=True) + p.call = call + self.assertEqual(p.run_private_checkout(prepare_only=True), 0) + argv, kwargs = calls[0] + self.assertEqual(argv[-2:], ['--private-checkout', '--prepare-only']) + self.assertEqual(kwargs['grace'], 240) + self.assertTrue(p.nested_started) + self.assertFalse((p.root / 'execution/run').exists()) + + def test_prepared_and_filtered_receipts_cannot_satisfy_matrix_gate(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'completion-1.json').write_text(json.dumps({ + 'backend':'vz','sourceCommit':'a' * 40,'matrixCompleted':True, + 'matrix':'all','physicalAcceptance':True,'stageCompleted':False,'cleanupFailed':[]})) + for receipt in [ + {'status':'ASSETS_PREPARED','selected':0,'passed':0,'physicalAcceptance':False}, + {'status':'PASS','selected':24,'passed':24,'matrix':'claude/catalog-fidelity','stageCompleted':True,'matrixCompleted':False,'physicalAcceptance':False}, + {'status':'PASS','selected':24,'passed':24,'matrix':'all','stageCompleted':True,'matrixCompleted':True,'physicalAcceptance':True}, + ]: + (root / 'matrix.log').write_text(json.dumps(receipt) + '\n') + with self.assertRaises(RuntimeError): + assets.matrix_result(root, 'vz', 'a' * 40) + + def test_matrix_requires_actual_nonzero_output_and_cleanup(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + completion = {'backend': 'vz', 'sourceCommit': 'a' * 40, + 'matrixCompleted': True, 'physicalAcceptance': True, 'stageCompleted': False, 'matrix': 'all', 'cleanupFailed': []} + (root / 'completion-1.json').write_text(json.dumps(completion)) + for passed, selected, ok in [(24, 24, True), (0, 0, False), (23, 24, False), (24, 0, False)]: + (root / 'matrix.log').write_text(json.dumps({'status': 'PASS', 'selected': selected, 'passed': passed, 'matrix': 'all', 'physicalAcceptance': True, 'matrixCompleted': True}) + '\n') + if ok: + self.assertEqual(assets.matrix_result(root, 'vz', 'a' * 40), 24) + else: + with self.assertRaises(RuntimeError): + assets.matrix_result(root, 'vz', 'a' * 40) + (root / 'matrix.log').write_text(json.dumps({'status':'PASS','selected':24,'passed':24,'matrix':'all','physicalAcceptance':True,'matrixCompleted':True}) + '\n') + for field, value in [('matrixCompleted', False), ('matrix', 'claude/catalog-fidelity'), ('stageCompleted', True), ('physicalAcceptance', False), ('cleanupFailed', ['host']), ('sourceCommit', 'b' * 40)]: + (root / 'completion-1.json').write_text(json.dumps({**completion, field: value})) + with self.assertRaises(RuntimeError): + assets.matrix_result(root, 'vz', 'a' * 40) + + def test_operator_rejects_commands_credentials_and_noncanonical_socket(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + sock = socket.socket(socket.AF_UNIX) + try: + sock.bind(str(root / 'engine.sock')) + tool = root / 'tool' + tool.write_text('#!/bin/sh\nexit 0\n') + tool.chmod(0o700) + config = {'version': 1, 'backend': 'firecracker', 'engineSocket': str(root / 'engine.sock'), + 'nativeHostIp': '192.168.1.2', 'subnet': '10.88.2.0/24', 'registryPort': 30000, + 'buildRegistryPort': 31000, 'tools': {key: str(tool) for key in assets.TOOLS}, + 'thinPool': {'name': 'suite', 'directory': str(root)}} + path = root / 'operator.json' + path.write_text(json.dumps(config)) + path.chmod(0o600) + self.assertEqual(assets.read_operator(str(path), 'firecracker'), config) + for extra in [{'command': 'anything'}, {'token': 'secret'}, {'PASS': True}]: + path.write_text(json.dumps({**config, **extra})) + with self.assertRaises(RuntimeError): + assets.read_operator(str(path), 'firecracker') + for key, value in [('engineSocket', str(root / 'missing')), ('subnet', '10.0.2.0/24'), + ('nativeHostIp', '127.0.0.1'), ('buildRegistryPort', 30002)]: + path.write_text(json.dumps({**config, key: value})) + with self.assertRaises((RuntimeError, FileNotFoundError)): + assets.read_operator(str(path), 'firecracker') + finally: + sock.close() + + @contextmanager + def native_sdk_fixture(self, backend='vz'): + # Controlled command responses only; no compiler, engine or VM runs. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + sdk = root / 'MacOSX.sdk' + header = sdk / 'usr/include/TargetConditionals.h' + header.parent.mkdir(parents=True) + header.write_text('/* installed SDK header */\n') + alias = root / 'selected.sdk' + alias.symlink_to(sdk, target_is_directory=True) + work = root / 'work' + work.mkdir(mode=0o700) + with socket.socket(socket.AF_UNIX) as sock: + sock.bind(str(root / 'engine.sock')) + tools = assets.TOOLS | ({'cargo', 'rustc', 'cc', 'go'} if backend == 'vz' else set()) + p = assets.Producer(backend, work, {'engineSocket': str(root / 'engine.sock'), + 'tools': {key: '/tools/' + key for key in tools}}) + p.sdk_output = (str(alias) + '\n').encode() + p.commands = [] + def call(argv, **kwargs): + p.commands.append((argv, dict(kwargs.get('env', p.env)))) + if argv[0] == '/usr/bin/xcrun': + if isinstance(p.sdk_output, Exception): + raise p.sdk_output + return p.sdk_output + if argv[0] == '/usr/sbin/sysctl': + return b'1\n' + if argv[0] == '/tools/rustc': + return b'rustc 1.99.0 (fixture)\n' + if argv[0] == sys.executable: + return b'a' * 64 + if argv[0] == '/tools/cc': + (p.files / 'nativeProbe').write_bytes(b'fixture probe') + return {'/tools/node': b'v24.0.0', '/tools/openssl': b'OpenSSL 3.0', + '/tools/bun': (ROOT / '.bun-version').read_bytes()}.get(argv[0], b'') + p.call = call + p.engine = lambda command, *args: json.dumps( + {'OSType': 'linux', 'Architecture': p.arch, 'ID': 'fixture-engine-id'} if command == 'info' + else {'Server': {'Version': '29.0', 'ApiVersion': '1.45'}}).encode() + p.stage = lambda path, name, executable: {'path': str(path)} + with patch.object(assets.platform, 'system', return_value='Darwin' if backend == 'vz' else 'Linux'), \ + patch.object(assets.platform, 'machine', return_value='arm64' if backend == 'vz' else 'x86_64'), \ + patch.object(p, 'snapshot', side_effect=RuntimeError('before_images')) as snapshot: + yield p, sdk, header, snapshot + + def test_native_sdk_selected_alias_reaches_cargo_and_probe_without_ambient_config(self): + with patch.dict(os.environ, {'SDKROOT': '/poison/sdk', 'CC': '/poison/cc', 'CFLAGS': '-poison', + 'DEVELOPER_DIR': '/poison/developer', 'CARGO_BUILD_JOBS': '99'}), \ + self.native_sdk_fixture() as (p, sdk, header, snapshot): + with self.assertRaisesRegex(RuntimeError, 'before_images'): + p.produce() + snapshot.assert_called_once() + p.native_host() + discovery = [entry for entry in p.commands if entry[0][0] == '/usr/bin/xcrun'] + self.assertEqual(len(discovery), 1) + self.assertEqual(discovery[0][0], ['/usr/bin/xcrun', '--sdk', 'macosx', '--show-sdk-path']) + self.assertFalse({'SDKROOT', 'CC', 'CFLAGS', 'DEVELOPER_DIR'}.intersection(discovery[0][1])) + cargo = next(env for argv, env in p.commands if argv[0] == '/tools/cargo') + probe = next(env for argv, env in p.commands if argv[0] == '/tools/cc') + self.assertEqual(cargo.get('SDKROOT'), str(sdk)) + self.assertEqual(probe.get('SDKROOT'), str(sdk)) + self.assertEqual(cargo.get('CARGO_BUILD_JOBS'), '2') + self.assertEqual(cargo['RUSTC'], '/tools/rustc') + self.assertEqual(cargo['ENGRAM_EGRESS_PROXYD_FINGERPRINT'], 'a' * 64) + self.assertFalse({'CC', 'CFLAGS', 'DEVELOPER_DIR'}.intersection(cargo)) + self.assertNotIn('SDKROOT', p.env) + + def test_native_sdk_rejects_bad_selection_before_images(self): + for case in ['empty', 'relative', 'multiple', 'encoding', 'nul', 'missing', 'file', 'writable', + 'foreign_owner', 'missing_header', 'empty_header', 'header_directory', 'unreadable_header', + 'symlink_loop', 'xcrun_missing', 'xcrun_failure']: + with self.subTest(case=case), self.native_sdk_fixture() as (p, sdk, header, snapshot): + if case in ['empty', 'relative', 'multiple', 'encoding', 'nul', 'missing', 'file']: + p.sdk_output = {'empty': b'', 'relative': b'relative.sdk\n', 'multiple': b'/one\n/two\n', + 'encoding': b'\xff', 'nul': b'/sdk\x00', 'missing': str(sdk / 'missing').encode(), + 'file': str(header).encode()}[case] + elif case == 'writable': + sdk.chmod(0o777) + elif case == 'missing_header': + header.unlink() + elif case == 'empty_header': + header.write_bytes(b'') + elif case == 'header_directory': + header.unlink() + header.mkdir() + elif case == 'symlink_loop': + loop = sdk.parent / 'loop.sdk' + loop.symlink_to(loop) + p.sdk_output = str(loop).encode() + elif case in ['xcrun_missing', 'xcrun_failure']: + p.sdk_output = FileNotFoundError('xcrun') if case == 'xcrun_missing' else RuntimeError('producer_command') + real_stat, real_access = Path.stat, os.access + def sdk_stat(path, *args, **kwargs): + value = real_stat(path, *args, **kwargs) + if case == 'foreign_owner' and path == sdk: + fields = list(value) + fields[4] = os.getuid() + 10000 + return os.stat_result(fields) + return value + with patch.object(Path, 'stat', autospec=True, side_effect=sdk_stat), \ + patch.object(assets.os, 'access', side_effect=lambda path, mode: + False if case == 'unreadable_header' and path == header else real_access(path, mode)): + with self.assertRaisesRegex(RuntimeError, 'native_macos_sdk|producer_command'): + p.produce() + snapshot.assert_not_called() + self.assertFalse(any(argv[0] == '/tools/cargo' for argv, env in p.commands)) + + def test_firecracker_preflight_does_not_discover_or_export_native_sdk(self): + with self.native_sdk_fixture('firecracker') as (p, sdk, header, snapshot), \ + patch.object(assets.os, 'getuid', return_value=0), \ + patch.object(assets, 'canonical', return_value=header), \ + patch.object(assets.stat, 'S_ISCHR', return_value=True): + with self.assertRaisesRegex(RuntimeError, 'before_images'): + p.produce() + snapshot.assert_called_once() + self.assertFalse(any(argv[0] == '/usr/bin/xcrun' for argv, env in p.commands)) + self.assertNotIn('SDKROOT', p.env) + self.assertNotIn('CARGO_BUILD_JOBS', p.env) + + def test_command_failure_and_cancellation_stop_production(self): + # Fake CLI child only. No engine, application, helper or VM is executed. + with tempfile.TemporaryDirectory() as directory: + p = object.__new__(assets.Producer) + p.root = Path(directory) + p.counter, p.cancelled, p.child, p.groups = 0, False, None, {} + p.env = {'PATH': '/usr/bin:/bin', 'LANG': 'C'} + with self.assertRaises(RuntimeError): + p.call([sys.executable, '-c', 'raise SystemExit(42)']) + p.cancelled = True + with self.assertRaises(RuntimeError): + p.call([sys.executable, '-c', 'raise SystemExit(0)']) + self.assertEqual(p.counter, 1) + self.assertIsNone(p.child) + p.cancelled = False + with self.assertRaises(subprocess.TimeoutExpired): + p.call([sys.executable, '-c', 'import time; time.sleep(60)'], timeout=0.02) + self.assertIsNone(p.child) + + def test_private_checkout_covers_git_ownership_without_shared_trust(self): + # Actual Git's ownership-test switch models runner UID -> root UID. + # No sudo/chown/foreign account or global configuration is used. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + source = root / 'runner-checkout' + source.mkdir() + env = {'PATH': '/usr/bin:/bin', 'LANG': 'C', 'GIT_CONFIG_NOSYSTEM': '1', + 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_CONFIG_SYSTEM': '/dev/null', + 'GIT_OPTIONAL_LOCKS': '0', 'GIT_NO_LAZY_FETCH': '1'} + def git(*args, **kwargs): + return subprocess.run(['/usr/bin/git', *args], env=kwargs.pop('env', env), + capture_output=True, text=True, check=True, **kwargs) + empty = root / 'empty-template' + empty.mkdir() + git('init', '--template=' + str(empty), str(source)) + (source / 'tracked.txt').write_text('exact public source\n') + git('-C', str(source), 'add', 'tracked.txt') + git('-C', str(source), '-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', + '-c', 'commit.gpgsign=false', 'commit', '-m', 'fixture') + head = git('-C', str(source), 'rev-parse', 'HEAD').stdout.strip() + # A configuration credential and hook are not part of the checkout. + git('-C', str(source), 'config', '--local', 'http.extraHeader', 'fixture-sensitive-header') + (source / '.git/hooks').mkdir(exist_ok=True) + (source / '.git/hooks/post-checkout').write_text('fixture hook must not copy') + config_before = (source / '.git/config').read_bytes() + index_before = (source / '.git/index').read_bytes() + refused = subprocess.run(['/usr/bin/git', '-C', str(source), 'status', '--porcelain'], + env={**env, 'GIT_TEST_ASSUME_DIFFERENT_OWNER': '1'}, capture_output=True) + self.assertNotEqual(refused.returncode, 0) + self.assertIn(b'dubious ownership', refused.stderr) + work = root / 'private' + work.mkdir(mode=0o700) + p = object.__new__(assets.Producer) + p.root, p.env = work, env + p.counter, p.cancelled, p.child, p.groups = 0, False, None, {} + upload = root / 'owned-upload-pack' + upload.write_text('#!/bin/sh\nexec env GIT_TEST_ASSUME_DIFFERENT_OWNER=1 /usr/bin/git upload-pack "$@"\n') + upload.chmod(0o700) + original_call = p.call + def ownership_call(argv, **kwargs): + words = [str(x) for x in argv] + if '-C' in words and words[words.index('-C') + 1] == str(source): + kwargs['env'] = {**env, 'GIT_TEST_ASSUME_DIFFERENT_OWNER': '1'} + if 'fetch' in words: + words.insert(words.index('fetch') + 1, '--upload-pack=' + str(upload)) + return original_call(words, **kwargs) + p.call = ownership_call + checkout, actual = p.private_checkout(source) + self.assertEqual(actual, head) + self.assertEqual(checkout.stat().st_uid, os.getuid()) + self.assertEqual(checkout.stat().st_mode & 0o777, 0o700) + self.assertEqual(git('-C', str(checkout), 'rev-parse', 'HEAD').stdout.strip(), head) + self.assertFalse(git('-C', str(checkout), 'status', '--porcelain').stdout) + self.assertEqual((checkout / 'tracked.txt').read_bytes(), (source / 'tracked.txt').read_bytes()) + self.assertFalse((checkout / '.git/hooks/post-checkout').exists()) + self.assertEqual((source / '.git/config').read_bytes(), config_before) + self.assertEqual((source / '.git/index').read_bytes(), index_before) + self.assertNotEqual((source / 'tracked.txt').stat().st_ino, (checkout / 'tracked.txt').stat().st_ino) + self.assertEqual((checkout / 'tracked.txt').stat().st_nlink, 1) + local = git('-C', str(checkout), 'config', '--local', '--list').stdout + self.assertNotIn('fixture-sensitive-header', local) + self.assertNotIn('safe.directory', local) + # Same cleared environment the adapter uses succeeds without trust. + self.assertEqual(git('-C', str(checkout), 'show', 'HEAD:tracked.txt').stdout, 'exact public source\n') + self.assertEqual(git('-C', str(source), '-c', 'safe.directory=' + str(source), + 'config', '--local', '--get', 'http.extraHeader').stdout.strip(), 'fixture-sensitive-header') + + def test_owned_group_drains_redirected_term_ignoring_descendant(self): + # Controlled Python children only. The descendant closes every stream. + for mode in ['success', 'error', 'timeout']: + with self.subTest(mode=mode), tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + p = object.__new__(assets.Producer) + p.root, p.env = root, {'PATH': '/usr/bin:/bin', 'LANG': 'C'} + p.counter, p.cancelled, p.child, p.groups = 0, False, None, {} + pidfile = root / 'descendant.pid' + descendant = ('import os,signal,time; from pathlib import Path; ' + 'signal.signal(signal.SIGTERM,signal.SIG_IGN); ' + f'Path({str(pidfile)!r}).write_text(str(os.getpid())); ' + 'time.sleep(20)') + leader = ('import subprocess,sys,time; from pathlib import Path; ' + f'p=subprocess.Popen([sys.executable,"-c",{descendant!r}], ' + 'stdin=subprocess.DEVNULL,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL); ' + f'path=Path({str(pidfile)!r}); ' + '\nwhile not path.exists(): time.sleep(0.005)\n' + + ('time.sleep(20)' if mode == 'timeout' else + 'raise SystemExit(42)' if mode == 'error' else 'raise SystemExit(0)')) + try: + if mode == 'success': + p.call([sys.executable, '-c', leader], grace=0.05, timeout=0.5) + else: + with self.assertRaises((RuntimeError, subprocess.TimeoutExpired)): + p.call([sys.executable, '-c', leader], grace=0.05, timeout=0.5) + pid = int(pidfile.read_text()) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + self.assertEqual(p.groups, {}) + self.assertIsNone(p.child) + finally: + # This exact test-owned PID/group is retained for RED cleanup. + if pidfile.exists(): + try: + pgid = os.getpgid(int(pidfile.read_text())) + os.killpg(pgid, signal.SIGKILL) + except ProcessLookupError: + pass + if p.child is not None: + p.child.communicate(timeout=2) + if pidfile.exists(): + deadline = time.monotonic() + 2 + while True: + try: + os.kill(int(pidfile.read_text()), 0) + except ProcessLookupError: + break + self.assertLess(time.monotonic(), deadline, 'controlled descendant survived test cleanup') + time.sleep(0.01) + + def test_inaccessible_group_is_remaining_not_absent(self): + p = object.__new__(assets.Producer) + p.groups = {123: object()} + with patch.object(os, 'killpg', side_effect=PermissionError): + self.assertTrue(p.group_exists(123)) + with patch.object(os, 'killpg', side_effect=ProcessLookupError): + self.assertFalse(p.group_exists(123)) + self.assertIn(123, p.groups) + + def test_failed_group_drain_retains_owner_and_blocks_cleanup(self): + # A forced drain error tests retained ownership, without any signal call. + with tempfile.TemporaryDirectory() as directory: + p = object.__new__(assets.Producer) + p.root, p.names = Path(directory), [] + child = type('RetainedChild', (), {'pid': 123})() + p.groups, p.child = {123: child}, child + with patch.object(p, 'drain_group', side_effect=RuntimeError('controlled drain failure')): + with self.assertRaises(RuntimeError): + p.cleanup() + self.assertIs(p.groups[123], child) + self.assertIs(p.child, child) + failed = json.loads((p.root / 'producer-cleanup.json').read_bytes())['failed'] + self.assertIn('process-group-123', failed) + self.assertIn('process-groups', failed) + + def test_fake_engine_uses_explicit_selection_and_rejects_identity_change(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + sock = socket.socket(socket.AF_UNIX) + try: + sock.bind(str(root / 'engine.sock')) + fake = root / 'docker' + fake.write_text(f'#!{sys.executable}\n' + 'import json, os, sys\n' + 'from pathlib import Path\n' + f'root = Path({str(root)!r})\n' + 'assert "HOME" not in os.environ and "GITHUB_TOKEN" not in os.environ\n' + 'with (root / "calls").open("a") as out: out.write(json.dumps(sys.argv[1:]) + "\\n")\n' + 'if "info" in sys.argv: print(json.dumps({"ID": (root / "id").read_text()}))\n' + 'else: print("[]")\n') + fake.chmod(0o700) + (root / 'id').write_text('engine-one-identity') + o = {'engineSocket': str(root / 'engine.sock'), 'tools': {'docker': str(fake)}} + p = assets.Producer('firecracker', root, o) + p.engine_identity = 'engine-one-identity' + p.engine('image', 'inspect', 'sha256:' + 'a' * 64) + calls = [json.loads(line) for line in (root / 'calls').read_text().splitlines()] + self.assertEqual(len(calls), 3) + for call in calls: + self.assertEqual(call[:4], ['--config', str(p.config), '--host', 'unix://' + str(root / 'engine.sock')]) + (root / 'id').write_text('engine-two-identity') + with self.assertRaises(RuntimeError): + p.engine('create', 'anything') + calls = [json.loads(line) for line in (root / 'calls').read_text().splitlines()] + self.assertFalse(any('create' in call for call in calls)) + finally: + sock.close() + + def test_cleanup_refuses_foreign_resource_and_survivor(self): + with tempfile.TemporaryDirectory() as directory: + for foreign, survivor in [(True, False), (False, True), (False, False)]: + root = Path(directory) / f'{foreign}-{survivor}' + root.mkdir() + p = object.__new__(assets.Producer) + p.root, p.run, p.names = root, 'mcp-ci-owned', ['mcp-ci-owned-registry'] + calls = [] + def fake(*args, **kwargs): + calls.append(args) + if args[0] == 'ps': + return b'a' * 64 if len(calls) == 1 or survivor else b'' + if args[0] == 'inspect': + return json.dumps([{'Name': '/mcp-ci-owned-registry', 'Config': {'Labels': { + assets.LABEL: 'foreign' if foreign else p.run}}}]).encode() + return b'' + p.engine = fake + if foreign or survivor: + with self.assertRaises(RuntimeError): + p.cleanup() + else: + p.cleanup() + self.assertEqual(any(call[0] == 'rm' for call in calls), not foreign) + + def test_adapter_cleanup_requires_actual_receipt_and_survivor_queries(self): + with tempfile.TemporaryDirectory() as directory: + for mode in ['good', 'missing', 'failed', 'survivor']: + root = Path(directory) / mode + root.mkdir() + (root / 'run').mkdir() + p = object.__new__(assets.Producer) + p.root, p.run, p.names = root, 'mcp-ci-owned', [] + p.backend, p.head = 'vz', 'a' * 40 + p.adapter_started, p.adapter_clean = True, False + if mode != 'missing': + (root / 'run/completion-1.json').write_text(json.dumps({ + 'backend': 'vz', 'sourceCommit': p.head, + 'cleanupFailed': ['host'] if mode == 'failed' else []})) + (root / 'run/cleanup.json').write_text(json.dumps({ + 'success': True, 'containersRemaining': 0, 'networkRemaining': False, + 'runId': 'mcp-' + 'b' * 24})) + queries = [] + def fake(*args, **kwargs): + queries.append(args) + return b'owned-survivor' if mode == 'survivor' else b'' + p.engine = fake + if mode == 'good': + p.cleanup() + self.assertTrue(p.adapter_clean) + self.assertEqual(len(queries), 2) + else: + with self.assertRaises(RuntimeError): + p.cleanup() + self.assertFalse(p.adapter_clean) + self.assertTrue((root / 'producer-cleanup.json').is_file()) + + def test_workflow_graph_has_no_skipped_producers_and_safe_uploads(self): + workflow = (ROOT / '.github/workflows/ci.yml').read_text() + for backend, name, label, operator in [ + ('firecracker', 'fc', 'engram-mcp-firecracker', '/etc/engrams/mcp-physical-firecracker.json'), + ('vz', 'vz', 'engram-mcp-vz', '/Library/Application Support/engrams/mcp-physical-vz.json'), + ]: + # Extract to next two-space job key, not nested YAML fields. + lane = workflow.split(f' mcp-physical-{name}:', 1)[1].split('\n mcp-physical-', 1)[0].split('\n ci-gate:', 1)[0] + self.assertIn('needs: detect', lane) + self.assertNotIn('needs: [', lane) + self.assertIn("if: needs.detect.outputs.test_mcp_physical == 'true'", lane) + self.assertIn(label, lane) + self.assertIn(f'--backend {backend}', lane) + self.assertIn(operator, lane) + self.assertIn('persist-credentials: false', lane) + self.assertIn('if-no-files-found: error', lane) + self.assertIn(f' - mcp-physical-{name}\n', workflow.split(' ci-gate:', 1)[1]) + self.assertNotIn('continue-on-error', lane) + self.assertNotIn('fc-setup', lane) + source = (ROOT / '.github/scripts/mcp-stack-assets.py').read_text() + self.assertIn("'--run-root', self.root / 'run'", source) + self.assertIn("'--artifacts', self.root / 'inputs.json'", source) + self.assertIn("'--experimental-transform-types'", source) + self.assertNotIn("os.environ['HOME']", source) + self.assertNotIn('prune', source) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/scripts/tests/test_mcp_source_gates.py b/.github/scripts/tests/test_mcp_source_gates.py new file mode 100644 index 000000000..60b26ec95 --- /dev/null +++ b/.github/scripts/tests/test_mcp_source_gates.py @@ -0,0 +1,194 @@ +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import textwrap +import unittest + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / '.github/scripts/mcp-acceptance-native.sh' +DEADLINE = 'evidence::tests::expired_deadline_cannot_consume_a_release' + + +class NativeSourceGate(unittest.TestCase): + def test_binary_protocol_and_deadline_guards(self): + # Command stubs only: no Rust test, fixture, or harness process runs. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + commands = root / 'bin' + commands.mkdir() + target = root / 'target' + (target / 'debug').mkdir(parents=True) + fixture = target / 'debug/engram-mcp-model-fixture' + fixture.write_text('#!/bin/sh\nprintf "engram-mcp-model-fixture 1\\n"\n') + fixture.chmod(0o700) + cargo = commands / 'cargo' + cargo.write_text(f'#!{sys.executable}\n' + 'import json, os, sys\n' + 'from pathlib import Path\n' + 'with Path(os.environ["FAKE_CARGO_ARGS"]).open("a") as log:\n' + ' log.write(json.dumps(sys.argv[1:]) + "\\n")\n' + 'if "--test" in sys.argv:\n' + ' print(os.environ["FAKE_CARGO_SUMMARY"])\n' + ' sys.exit(int(os.environ.get("FAKE_CARGO_EXIT", "0")))\n' + 'if "test" in sys.argv:\n' + ' print("test result: ok. 1 passed; 0 failed; 0 ignored")\n') + cargo.chmod(0o700) + args_file = root / 'args.jsonl' + env = dict(PATH=f'{commands}{os.pathsep}/usr/bin:/bin', + CARGO_TARGET_DIR=str(target), FAKE_CARGO_ARGS=str(args_file)) + good = ('Running unittests src/main.rs (target/debug/deps/fixture)\n' + f'test {DEADLINE} ... ok\n' + 'test result: ok. 4 passed; 0 failed; 0 ignored\n' + 'Running tests/protocol.rs (target/debug/deps/protocol)\n' + 'test result: ok. 38 passed; 0 failed; 0 ignored\n') + for name, summary, exit_code, expected in [ + ('positive', good, 0, True), + ('zero-protocol', good.replace('38 passed', '0 passed'), 0, False), + ('missing-deadline', good.replace(f'test {DEADLINE} ... ok\n', ''), 0, False), + ('failed-protocol', good.replace('38 passed; 0 failed', '37 passed; 1 failed'), 0, False), + ('failed-command', good, 42, False), + ]: + with self.subTest(name=name): + args_file.unlink(missing_ok=True) + env.update(FAKE_CARGO_SUMMARY=summary, FAKE_CARGO_EXIT=str(exit_code)) + result = subprocess.run(['bash', str(SCRIPT)], cwd=root, env=env, + capture_output=True, text=True) + self.assertEqual(result.returncode == 0, expected, result.stdout + result.stderr) + args = [json.loads(line) for line in args_file.read_text().splitlines()] + self.assertEqual(args[0], [ + 'test', '--locked', '--manifest-path', + 'tests/fixtures/mcp-model-cli/Cargo.toml', '--bins', '--test', 'protocol', + ]) + self.assertEqual(len(args), 4 if expected else 1) + + def test_bun_source_gate_rejects_zero_and_failed_runs(self): + workflow = (ROOT / '.github/workflows/ci.yml').read_text() + step = workflow.split(' - name: MCP acceptance scripts unit tests', 1)[1] + body = textwrap.dedent(step.split(' run: |\n', 1)[1].split(' - name:', 1)[0]) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'orchestrator').mkdir() + commands = root / 'bin' + commands.mkdir() + bun = commands / 'bun' + bun.write_text(f'#!{sys.executable}\n' + 'import os, sys\n' + 'assert sys.argv[1:] == ["test", "scripts/mcp-stack"]\n' + 'print(os.environ["FAKE_BUN_SUMMARY"])\n' + 'sys.exit(int(os.environ["FAKE_BUN_EXIT"]))\n') + bun.chmod(0o700) + env = dict(PATH=f'{commands}{os.pathsep}/usr/bin:/bin', TMPDIR=str(root)) + for summary, exit_code, expected in [ + (' 64 pass\n 0 fail\nRan 64 tests across 6 files. [80ms]', 0, True), + (' 0 pass\n 0 fail\nRan 0 tests across 6 files. [80ms]', 0, False), + (' 63 pass\n 1 fail\nRan 64 tests across 6 files. [80ms]', 0, False), + ('No tests found', 0, False), + (' 64 pass\n 0 fail\nRan 64 tests across 6 files. [80ms]', 42, False), + ]: + with self.subTest(summary=summary, exit_code=exit_code): + env.update(FAKE_BUN_SUMMARY=summary, FAKE_BUN_EXIT=str(exit_code)) + result = subprocess.run(['bash', '-c', body], cwd=root / 'orchestrator', + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, expected, result.stdout + result.stderr) + self.assertTrue((root / 'var/mcp-stack-runner').is_dir()) + + def test_inputs_writer_gate_rejects_zero_failure_and_cancel(self): + # Source contract: select TAP for the shell summary guards. + workflow = (ROOT / '.github/workflows/ci.yml').read_text() + step = workflow.split(' - name: MCP Inputs writer source tests', 1)[1] + body = textwrap.dedent(step.split(' run: |\n', 1)[1].split(' - name:', 1)[0]) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + commands = root / 'bin' + commands.mkdir() + node = commands / 'node' + node.write_text(f'#!{sys.executable}\n' + 'import os, sys\n' + 'args = sys.argv[1:]\n' + 'tap = "--test-reporter=tap" in args\n' + 'if tap:\n' + ' args.remove("--test-reporter=tap")\n' + 'assert args == ["--experimental-transform-types", "--test", "--test-isolation=none", "../.github/scripts/tests/mcp-stack-inputs.test.mts"]\n' + 'summary = os.environ["FAKE_SUMMARY"]\n' + 'print(summary if tap else summary.replace("# ", "ℹ "))\n' + 'sys.exit(int(os.environ["FAKE_EXIT"]))\n') + node.chmod(0o700) + env = dict(PATH=f'{commands}{os.pathsep}/usr/bin:/bin', TMPDIR=str(root)) + good = '# pass 38\n# fail 0\n# cancelled 0' + for summary, code, expected in [(good, 0, True), (good.replace('38', '0'), 0, False), + (good.replace('fail 0', 'fail 1'), 0, False), + (good.replace('cancelled 0', 'cancelled 1'), 0, False), + (good, 42, False), ('No tests', 0, False)]: + env.update(FAKE_SUMMARY=summary, FAKE_EXIT=str(code)) + result = subprocess.run(['bash', '-c', body], cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, expected, result.stdout + result.stderr) + + def test_platform_gate_requires_openssl_three_and_nonzero_tests(self): + # Source contract with command stubs; actual platform tests run in CI. + workflow = (ROOT / '.github/workflows/ci.yml').read_text() + step = workflow.split(' - name: MCP platform source unit tests', 1)[1] + header = step.split(' run: |\n', 1)[0] + body = textwrap.dedent(step.split(' run: |\n', 1)[1].split(' - name:', 1)[0]) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + (root / 'orchestrator').mkdir() + cwd = (root if ' working-directory: ${{ github.workspace }}\n' in header + else root / 'orchestrator') + commands = root / 'bin' + commands.mkdir() + openssl = commands / 'openssl' + openssl.write_text(f'#!{sys.executable}\n' + 'import os, sys\n' + 'assert sys.argv[1:] == ["version"]\n' + 'print(os.environ["FAKE_OPENSSL_VERSION"])\n' + 'sys.exit(int(os.environ["FAKE_OPENSSL_EXIT"]))\n') + openssl.chmod(0o700) + bun = commands / 'bun' + bun.write_text(f'#!{sys.executable}\n' + 'import os, sys\n' + 'from pathlib import Path\n' + 'assert Path.cwd() == Path(os.environ["FAKE_REPO_ROOT"]), "platform tests require repository root"\n' + 'assert sys.argv[1:] == ["test", "tests/fixtures/mcp-stack/platform"]\n' + 'scratch = Path("var/mcp-platform-composition")\n' + 'assert scratch.is_dir(), "fresh checkout requires scratch parent"\n' + 'assert scratch.stat().st_mode & 0o777 == 0o700, "scratch parent must be private"\n' + 'import tempfile\n' + 'with tempfile.TemporaryDirectory(dir=scratch):\n' + ' pass\n' + 'assert os.environ["MCP_PLATFORM_TEST_OPENSSL"] == str(Path(__file__).with_name("openssl"))\n' + 'print(os.environ["FAKE_BUN_SUMMARY"])\n' + 'sys.exit(int(os.environ["FAKE_BUN_EXIT"]))\n') + bun.chmod(0o700) + env = dict(PATH=f'{commands}{os.pathsep}/usr/bin:/bin', TMPDIR=str(root), + FAKE_REPO_ROOT=str(root)) + good = ' 22 pass\n 0 fail\nRan 22 tests across 1 file. [80ms]' + for name, version, openssl_exit, summary, bun_exit, expected in [ + ('positive', 'OpenSSL 3.0.13', 0, good, 0, True), + ('libressl', 'LibreSSL 3.3.6', 0, good, 0, False), + ('old-openssl', 'OpenSSL 1.1.1', 0, good, 0, False), + ('failed-openssl', 'OpenSSL 3.0.13', 42, good, 0, False), + ('zero-tests', 'OpenSSL 3.0.13', 0, ' 0 pass\n 0 fail\nRan 0 tests across 1 file.', 0, False), + ('failed-test', 'OpenSSL 3.0.13', 0, ' 21 pass\n 1 fail\nRan 22 tests across 1 file.', 0, False), + ('no-tests', 'OpenSSL 3.0.13', 0, 'No tests found', 0, False), + ('zero-count', 'OpenSSL 3.0.13', 0, good.replace('Ran 22 tests', 'Ran 0 tests'), 0, False), + ('zero-files', 'OpenSSL 3.0.13', 0, good.replace('1 file.', '0 files.'), 0, False), + ('missing-count', 'OpenSSL 3.0.13', 0, ' 22 pass\n 0 fail', 0, False), + ('failed-command', 'OpenSSL 3.0.13', 0, good, 42, False), + ]: + with self.subTest(name=name): + scratch = root / 'var/mcp-platform-composition' + if scratch.exists(): + scratch.rmdir() + self.assertFalse(scratch.exists()) + env.update(FAKE_OPENSSL_VERSION=version, FAKE_OPENSSL_EXIT=str(openssl_exit), + FAKE_BUN_SUMMARY=summary, FAKE_BUN_EXIT=str(bun_exit)) + result = subprocess.run(['bash', '-c', 'umask 022\n' + body], cwd=cwd, env=env, + capture_output=True, text=True) + self.assertEqual(result.returncode == 0, expected, result.stdout + result.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/bake-images.yml b/.github/workflows/bake-images.yml index 09f66c10e..6d69d16e8 100644 --- a/.github/workflows/bake-images.yml +++ b/.github/workflows/bake-images.yml @@ -97,7 +97,7 @@ jobs: run: | set -euo pipefail args="" - for image in coordinator host-agent host-operator web orchestrator; do + for image in coordinator host-agent host-operator web orchestrator mcp-worker; do versions="/orgs/${{ github.repository_owner }}/packages/container/engrams%2F$image/versions?per_page=100" sha=$(gh api "$versions" \ --jq 'first(.[] | .metadata.container.tags[]? | select(test("^[0-9a-f]{40}$")))' \ @@ -137,8 +137,8 @@ jobs: fail-fast: false matrix: # Only the affected images (from the detector), EXCEPT a manual dispatch - # bakes all five. `if:` above guarantees this is never the empty array. - image: ${{ (github.event_name == 'workflow_dispatch' && fromJSON('["coordinator","host-agent","host-operator","web","orchestrator"]')) || fromJSON(needs.detect.outputs.images_matrix) }} + # bakes all images. `if:` above guarantees this is never the empty array. + image: ${{ (github.event_name == 'workflow_dispatch' && fromJSON('["coordinator","host-agent","host-operator","web","orchestrator","mcp-worker"]')) || fromJSON(needs.detect.outputs.images_matrix) }} steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f391d3fd7..c60421f9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,9 +46,8 @@ name: CI # ls; cold session with harness=claude + exec # ls; cold session with harness=claude + bogus # API key + prompt → assert auth failure event. -# Closes the gap that let prod session -# 8725648d's empty-Binary-frame bug ship (all -# existing FC tests bypass the coord HTTP path). +# Exercises the coordinator path that the lower +# host-agent FC tests do not cover. # # All jobs run on Blacksmith Linux runners # (https://docs.blacksmith.sh/blacksmith-runners/overview) — faster @@ -82,9 +81,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} -# Default to read-only repo access. No job here writes to the repo -# (no GH releases, no commit-back, no PR comments), so least -# privilege. `packages: read` is load-bearing: the test-fc-* jobs +# Default to read-only repo access. Jobs that publish artifacts or +# report protobuf changes request their own write permissions. +# `packages: read` is load-bearing: the test-fc-* jobs # oras-pull the forked firecracker artifact from GHCR — without # it the pull is DENIED and every fork-gated FC test (the ADR 0045 R2 # stock<->fork compat gate, the substrate suites, vmstate_only) @@ -126,6 +125,9 @@ jobs: # detector change sets all of these (see detect-rebake-lanes.py), so a # change to how CI runs re-runs everything. test_rust: ${{ steps.lanes.outputs.test_rust }} + test_mcp_worker: ${{ steps.lanes.outputs.test_mcp_worker }} + test_mcp_physical: ${{ steps.lanes.outputs.test_mcp_physical }} + test_netstack: ${{ steps.lanes.outputs.test_netstack }} test_cross: ${{ steps.lanes.outputs.test_cross }} test_fc: ${{ steps.lanes.outputs.test_fc }} test_web: ${{ steps.lanes.outputs.test_web }} @@ -147,6 +149,8 @@ jobs: # Full history so the PR base SHA is present for the diff. fetch-depth: 0 - uses: dtolnay/rust-toolchain@stable + - name: Test MCP path selection + run: python3 -m unittest discover -s .github/scripts/tests -p 'test_mcp*.py' - name: Detect changed lanes id: lanes run: | @@ -262,6 +266,26 @@ jobs: fi - name: terraform fmt -check run: terraform fmt -check -recursive -diff deploy/terraform + - name: Callback ingress profiles and effective nginx canary + timeout-minutes: 5 + run: | + set -euo pipefail + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends ruby + python3 -m unittest discover \ + -s deploy/helm/engram/ci/tests -p 'test_callback_profiles.py' + image="$(sed -n 's/^FROM \(nginx:[^ ]*\)$/\1/p' docker/web.Dockerfile)" + test -n "$image" + docker pull "$image" + python3 deploy/helm/engram/ci/callback-ingress-canary.py \ + --stage var/mcp-callback-canary + - name: Keep callback canary evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: callback-ingress-canary + path: var/mcp-callback-canary + if-no-files-found: error - name: terraform validate (every root) run: | set -euo pipefail @@ -438,6 +462,19 @@ jobs: AWS_REGION: us-east-1 steps: - uses: actions/checkout@v7 + - uses: oven-sh/setup-bun@v2 + with: + bun-version-file: .bun-version + - uses: actions/setup-node@v7 + with: + node-version: 24.22.2 + - name: Install worker test dependencies + working-directory: mcp-worker + run: npm ci --ignore-scripts + - name: Check TLS fixture prerequisites + run: | + node --version + openssl version - uses: dtolnay/rust-toolchain@stable # Install protoc via apt: arduino/setup-protoc@v3 with a pinned # version *still* called the GH releases API to validate and @@ -527,6 +564,16 @@ jobs: # `--no-fail-fast` reports every failing test instead of stopping # at the first; `if: always()` keeps the ha_listener steps below # running even if this one fails. + - name: Run native MCP fixture and adapter gates + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + run: bash .github/scripts/mcp-acceptance-native.sh + + - name: Run production Linux MCP wrapper entrypoints + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + run: bash .github/scripts/mcp-acceptance-linux.sh + - name: cargo nextest run --workspace if: always() run: cargo nextest run --workspace --no-fail-fast @@ -686,6 +733,22 @@ jobs: --release \ -p engram-agentd + - name: Build x86_64 static MCP model fixture + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + run: | + cargo build --locked --release --target x86_64-unknown-linux-musl --manifest-path tests/fixtures/mcp-model-cli/Cargo.toml --bin engram-mcp-model-fixture + mkdir -p var/mcp-fixture-x86_64 + cp "$CARGO_TARGET_DIR/x86_64-unknown-linux-musl/release/engram-mcp-model-fixture" var/mcp-fixture-x86_64/ + file var/mcp-fixture-x86_64/engram-mcp-model-fixture | grep -E 'ELF 64-bit.*x86-64.*static(ally|-pie)? linked' + cd var/mcp-fixture-x86_64 + shasum -a 256 engram-mcp-model-fixture > SHA256SUMS + - uses: actions/upload-artifact@v7 + with: + name: mcp-fixture-x86_64-linux-musl + path: var/mcp-fixture-x86_64/ + if-no-files-found: error + - name: build harness wrappers (musl) run: | cargo build \ @@ -826,6 +889,72 @@ jobs: # and (b) the ADR 0060 DBOS engine — DBOS keeps its system tables in a `dbos` # schema of this same database. The .live smoke test still self-skips without a # live coordinator. + mcp-worker: + name: MCP worker (Node 24) + needs: detect + if: needs.detect.outputs.test_mcp_worker == 'true' + runs-on: blacksmith-2vcpu-ubuntu-2404 + defaults: + run: + working-directory: mcp-worker + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24.22.2 + - run: npm ci --ignore-scripts + - run: npm run format:check + - run: npm run typecheck + - run: npm test + - name: Standalone MCP stack fixture tests + working-directory: ${{ github.workspace }} + run: | + mkdir -p var/mcp-stack-fixture + node --test --test-concurrency=1 --test-timeout=30000 \ + tests/fixtures/mcp-stack/durable.test.mts \ + tests/fixtures/mcp-stack/process.test.mts + - run: npm run build + + netstack: + name: sandbox netstack (${{ matrix.platform }}) + needs: detect + if: needs.detect.outputs.test_netstack == 'true' + strategy: + fail-fast: false + matrix: + include: + - platform: linux + runner: blacksmith-2vcpu-ubuntu-2404 + - platform: macos + runner: blacksmith-6vcpu-macos-15 + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v6 + with: + go-version: "1.26.2" + cache: false + - name: Test, vet, verify modules, and race + run: | + mkdir -p var + sandbox-netstack/build.sh test + sandbox-netstack/build.sh race + - name: Build and stamp current helper + run: | + sandbox-netstack/build.sh build + cd var/mcp-netstack-build + test -s engram-sandbox-netstack + shasum -a 256 engram-sandbox-netstack > SHA256SUMS + shasum -a 256 -c SHA256SUMS + - uses: actions/upload-artifact@v7 + with: + name: sandbox-netstack-${{ matrix.platform }} + path: | + var/mcp-netstack-build/engram-sandbox-netstack + var/mcp-netstack-build/SHA256SUMS + if-no-files-found: error + + # Postgres backs live tests and the isolated DBOS restart fixtures. orchestrator: name: orchestrator (bun) needs: detect @@ -836,8 +965,8 @@ jobs: image: postgres:16 env: POSTGRES_PASSWORD: engram - POSTGRES_DB: engram_orchestrator_test - ports: ["5432:5432"] + POSTGRES_DB: engram_mcp_1_1_1 + ports: ["5435:5432"] options: >- --health-cmd pg_isready --health-interval 10s @@ -853,10 +982,23 @@ jobs: CONTROL_PLANE_BEARER: ci-test-token # Reachable Postgres → the DB-gated tests + the DBOS smoke test run here # (instead of self-skipping). `bun run migrate` below applies the Drizzle - # schema before the suite; DBOS creates its own `dbos` schema on launch. - ORCHESTRATOR_DATABASE_URL: postgres://postgres:engram@localhost:5432/engram_orchestrator_test + # schema before the suite; DBOS restart fixtures use a second database. + ORCHESTRATOR_DATABASE_URL: postgres://postgres:engram@127.0.0.1:5435/engram_mcp_1_1_1 + MCP_TEST_DATABASE_NAME: engram_mcp_1_1_1 + MCP_TEST_DBOS_DATABASE_URL: postgres://postgres:engram@127.0.0.1:5435/engram_mcp_dbos_test_ci steps: - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24.22.2 + - name: Install worker test dependencies + working-directory: mcp-worker + run: npm ci --ignore-scripts + - name: Check TLS fixture prerequisites + working-directory: ${{ github.workspace }} + run: | + node --version + openssl version - uses: oven-sh/setup-bun@v2 with: # `.bun-version` is the ONE place the runtime version lives; the @@ -898,6 +1040,8 @@ jobs: exit 1 fi echo "Bun pinned to $want in both." + - name: Create isolated DBOS test database + run: docker exec ${{ job.services.postgres.id }} createdb -U postgres engram_mcp_dbos_test_ci - name: bun install run: bun install --frozen-lockfile - name: DBOS workflow hash warning (non-blocking, ADR 0104) @@ -910,6 +1054,44 @@ jobs: NODE_ENV=test bun scripts/dbos-workflow-hashes.ts - name: bun typecheck run: bun run typecheck + - name: MCP acceptance scripts typecheck + run: bun run tsc -p scripts/mcp-stack/tsconfig.json --noEmit + - name: MCP acceptance scripts unit tests + run: | + set -euo pipefail + mkdir -p ../var/mcp-stack-runner + result=$(mktemp) + trap 'rm -f "$result"' EXIT + bun test scripts/mcp-stack 2>&1 | tee "$result" + grep -Eq '^[[:space:]]*[1-9][0-9]* pass$' "$result" + grep -Eq '^[[:space:]]*0 fail$' "$result" + grep -Eq '^Ran [1-9][0-9]* tests across [1-9][0-9]* files\.' "$result" + - name: MCP platform source typecheck + run: bun run tsc -p ../tests/fixtures/mcp-stack/platform/tsconfig.json --noEmit + - name: MCP platform source unit tests + working-directory: ${{ github.workspace }} + run: | + set -euo pipefail + umask 077 + mkdir -p var/mcp-platform-composition + MCP_PLATFORM_TEST_OPENSSL=$(command -v openssl) + export MCP_PLATFORM_TEST_OPENSSL + "$MCP_PLATFORM_TEST_OPENSSL" version | grep -Eq '^OpenSSL 3\.' + result=$(mktemp) + trap 'rm -f "$result"' EXIT + bun test tests/fixtures/mcp-stack/platform 2>&1 | tee "$result" + grep -Eq '^[[:space:]]*[1-9][0-9]* pass$' "$result" + grep -Eq '^[[:space:]]*0 fail$' "$result" + grep -Eq '^Ran [1-9][0-9]* tests across [1-9][0-9]* files?\.' "$result" + - name: MCP Inputs writer source tests + run: | + set -euo pipefail + result=$(mktemp) + trap 'rm -f "$result"' EXIT + node --experimental-transform-types --test --test-isolation=none --test-reporter=tap ../.github/scripts/tests/mcp-stack-inputs.test.mts 2>&1 | tee "$result" + grep -Eq '^# pass [1-9][0-9]*$' "$result" + grep -Eq '^# fail 0$' "$result" + grep -Eq '^# cancelled 0$' "$result" - name: apply orchestrator migrations run: bun run migrate - name: bun test @@ -921,9 +1103,8 @@ jobs: # fails this job. Breaking-change detection is ADVISORY: it runs against # the PR base, but a detected break does NOT fail the job — it surfaces # as a `::warning::` plus a detailed PR comment so a reviewer can confirm - # the wire-compat impact is intended. Codegen-drift activates only once - # the generated TS bindings are committed (ADR 0051 Tasks 4-5) — until - # then this PR ships only the contract + Rust stubs. + # the wire-compat impact is intended. Codegen drift checks the committed + # TypeScript bindings. buf: name: buf (proto lint + breaking + codegen drift) needs: detect @@ -987,9 +1168,7 @@ jobs: if: github.event_name == 'pull_request' && steps.breaking.outcome == 'failure' run: | echo "::warning title=Proto breaking change detected::buf found a wire-incompatible change to crates/engram-protocol/proto against the PR base. This is non-blocking — see the buf comment on the PR for details. Confirm the break is intended before merging." - # TS codegen drift check. Guarded on the committed output dir so it - # stays inert in this PR (protos + Rust stubs only) and activates - # once web/src/gen lands with the connect-es deps (ADR 0051 Task 4). + # Check all committed TypeScript outputs for schema drift. - name: Codegen drift check (buf generate) run: | if [ -d web/src/gen/engram/app ]; then @@ -1145,6 +1324,14 @@ jobs: gh-token: ${{ secrets.GITHUB_TOKEN }} rust-cache-shared-key: fc-root + - name: Install CLOSED namespace test tools + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends iproute2 nftables procps + + - name: CLOSED namespace installed-state test (root) + run: bash .github/scripts/mcp-acceptance-fc-closed.sh + # Root-only lanes: the pagemap and idle-page scans, the post-copy # moves that use them, the dm-thin pool, and the TAP/netns tests. # `sudo -E env "PATH=$PATH"` keeps the cargo env sudo scrubs. @@ -1473,7 +1660,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v2 with: - rustflags: '' + rustflags: "" target: x86_64-unknown-linux-musl - name: Install protoc @@ -1751,7 +1938,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v2 with: - rustflags: '' + rustflags: "" target: x86_64-unknown-linux-musl # Install protoc via apt: arduino/setup-protoc@v3 with a pinned @@ -1928,7 +2115,7 @@ jobs: - uses: actions-rust-lang/setup-rust-toolchain@v2 with: - rustflags: '' + rustflags: "" # nextest is what every other test step in this workflow uses. - uses: taiki-e/install-action@nextest @@ -2271,11 +2458,21 @@ jobs: # surface; Linux/FC remains the canonical production gate. vz: name: engram-sandbox-vz unit tests - needs: detect + needs: [detect, netstack] if: needs.detect.outputs.test_rust == 'true' - runs-on: blacksmith-6vcpu-macos-15 # arm64, has Xcode CLT pre-installed + runs-on: blacksmith-6vcpu-macos-15 # arm64, has Xcode CLT pre-installed steps: - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: sandbox-netstack-macos + path: var/mcp-netstack-artifact + - name: Verify current helper artifact + run: | + cd var/mcp-netstack-artifact + test -s engram-sandbox-netstack + shasum -a 256 -c SHA256SUMS + chmod +x engram-sandbox-netstack - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -2336,7 +2533,27 @@ jobs: run: touch /tmp/engram-vz-rootfs.ext4 - name: Run ignored VZ tests (config validation; e2e_vz runs live below) if: always() - run: cargo nextest run -p engram-sandbox-vz --run-ignored ignored-only --no-fail-fast -E 'not test(e2e_vz)' + run: cargo nextest run -p engram-sandbox-vz --run-ignored ignored-only --no-fail-fast -E 'not test(e2e_vz) and not test(=managed::helper::tests::actual_helper_closed_lifetime_and_descriptor_isolation) and not test(=managed::helper::tests::actual_helper_release_ack_and_active_dns_cleanup) and not test(=managed::helper::tests::actual_helper_rejects_wrong_epoch_and_canceled_release)' + - name: Run verified packaged helper tests + if: always() + run: bash .github/scripts/mcp-acceptance-helper.sh + - name: Build arm64 static MCP model fixture + run: | + brew install messense/macos-cross-toolchains/aarch64-unknown-linux-musl + rustup target add aarch64-unknown-linux-musl + export CARGO_TARGET_DIR="$PWD/target" + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-unknown-linux-musl-gcc + cargo build --locked --release --target aarch64-unknown-linux-musl --manifest-path tests/fixtures/mcp-model-cli/Cargo.toml --bin engram-mcp-model-fixture + mkdir -p var/mcp-fixture-aarch64 + cp "$CARGO_TARGET_DIR/aarch64-unknown-linux-musl/release/engram-mcp-model-fixture" var/mcp-fixture-aarch64/ + file var/mcp-fixture-aarch64/engram-mcp-model-fixture | grep -E 'ELF 64-bit.*ARM aarch64.*static(ally|-pie)? linked' + cd var/mcp-fixture-aarch64 + shasum -a 256 engram-mcp-model-fixture > SHA256SUMS + - uses: actions/upload-artifact@v7 + with: + name: mcp-fixture-aarch64-linux-musl + path: var/mcp-fixture-aarch64/ + if-no-files-found: error - name: Probe nested virtualization id: hv if: always() @@ -2371,6 +2588,19 @@ jobs: runs-on: blacksmith-6vcpu-macos-15 steps: - uses: actions/checkout@v7 + - uses: oven-sh/setup-bun@v2 + with: + bun-version-file: .bun-version + - uses: actions/setup-node@v7 + with: + node-version: 24.22.2 + - name: Install worker test dependencies + working-directory: mcp-worker + run: npm ci --ignore-scripts + - name: Check TLS fixture prerequisites + run: | + node --version + openssl version - uses: dtolnay/rust-toolchain@stable - uses: arduino/setup-protoc@v3 with: @@ -2379,6 +2609,10 @@ jobs: - uses: taiki-e/install-action@nextest # Same default set as Linux minus engram-sandbox-vz (the `vz` job runs that # codesigned) — catches macOS-only regressions in cross-platform crates. + - name: Run native MCP fixture and adapter gates + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + run: bash .github/scripts/mcp-acceptance-native.sh - name: cargo nextest run --workspace --exclude engram-sandbox-vz if: always() run: cargo nextest run --workspace --exclude engram-sandbox-vz --no-fail-fast @@ -2517,6 +2751,82 @@ jobs: # never exports to gha anyway; it stays warm through Blacksmith's # colocated BuildKit cache. + # Each dedicated runner has a qualified existing Linux engine and native + # DNS/routes. See platform/ci/README.md. Missing prerequisites fail hard. + mcp-physical-fc: + name: MCP physical Firecracker (24 cases) + concurrency: + group: mcp-physical-fc-${{ github.repository }} + cancel-in-progress: false + needs: detect + if: needs.detect.outputs.test_mcp_physical == 'true' + runs-on: [self-hosted, Linux, X64, engram-mcp-firecracker] + timeout-minutes: 180 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Produce and run physical Firecracker assets + env: + GITHUB_TOKEN: ${{ github.token }} + MCP_FC_BOOT_ARTIFACTS: ${{ vars.MCP_FC_BOOT_ARTIFACTS }} + run: | + set -euo pipefail + root="${RUNNER_TEMP}/mcp-physical-fc-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + safe="var/mcp-physical-safe-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -p "$safe" + finish() { + if sudo test -f "$root/summary.json"; then + sudo install -m 0644 "$root/summary.json" "$safe/firecracker.json" + fi + } + trap finish EXIT + args=(--backend firecracker --operator /etc/engrams/mcp-physical-firecracker.json --work-root "$root") + if [[ -n "${MCP_FC_BOOT_ARTIFACTS:-}" ]]; then + args+=(--boot-artifacts "$MCP_FC_BOOT_ARTIFACTS") + fi + sudo --preserve-env=GITHUB_TOKEN python3 -I .github/scripts/mcp-stack-assets.py "${args[@]}" + - uses: actions/upload-artifact@v7 + if: always() + with: + name: mcp-physical-fc-safe-summary + path: var/mcp-physical-safe-${{ github.run_id }}-${{ github.run_attempt }}/firecracker.json + if-no-files-found: error + + mcp-physical-vz: + name: MCP physical VZ (24 cases) + concurrency: + group: mcp-physical-vz-${{ github.repository }} + cancel-in-progress: false + needs: detect + if: needs.detect.outputs.test_mcp_physical == 'true' + runs-on: [self-hosted, macOS, ARM64, engram-mcp-vz] + timeout-minutes: 180 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Produce and run physical VZ assets + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + root="${RUNNER_TEMP}/mcp-physical-vz-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -p var/mcp-physical-safe-${{ github.run_id }}-${{ github.run_attempt }} + finish() { + if test -f "$root/summary.json"; then + install -m 0644 "$root/summary.json" var/mcp-physical-safe-${{ github.run_id }}-${{ github.run_attempt }}/vz.json + fi + } + trap finish EXIT + python3 -I .github/scripts/mcp-stack-assets.py --backend vz --operator '/Library/Application Support/engrams/mcp-physical-vz.json' --work-root "$root" + - uses: actions/upload-artifact@v7 + if: always() + with: + name: mcp-physical-vz-safe-summary + path: var/mcp-physical-safe-${{ github.run_id }}-${{ github.run_attempt }}/vz.json + if-no-files-found: error + ci-gate: name: CI Gate if: always() @@ -2529,6 +2839,10 @@ jobs: - web - site - orchestrator + - mcp-worker + - mcp-physical-fc + - mcp-physical-vz + - netstack - buf - test-fc-unprivileged - test-fc-pooled diff --git a/Cargo.lock b/Cargo.lock index e91ce87b6..80d640230 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3082,11 +3082,16 @@ dependencies = [ "dashmap", "engram-core", "engram-telemetry", + "engram-tls", "futures", + "hex", "proptest", "prost", + "rustls", + "rustls-pemfile", "serde", "serde_json", + "sha2 0.11.1", "tokio", "tokio-stream", "tonic", @@ -3154,8 +3159,10 @@ dependencies = [ "engram-agentd", "engram-chunk-store", "engram-core", + "engram-egress-proto", "engram-egress-proxy", "engram-harness-proto", + "engram-host-core", "engram-oci", "engram-rootfs-materializer", "engram-storage-local", @@ -3223,8 +3230,10 @@ dependencies = [ "dispatch2", "engram-agentd", "engram-core", + "engram-egress-proto", "engram-harness-proto", "futures", + "hex", "libc", "objc2", "objc2-foundation", @@ -3232,6 +3241,7 @@ dependencies = [ "parking_lot 0.13.0", "serde", "serde_json", + "sha2 0.11.1", "tempfile", "tokio", "tokio-stream", @@ -3407,6 +3417,7 @@ dependencies = [ "engram-core", "engram-postgres", "futures", + "serde_json", "sha2 0.11.1", "sqlx", "tokio", @@ -8384,6 +8395,7 @@ dependencies = [ "socket2", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-stream", "tokio-util", "tower", diff --git a/DESIGN.md b/DESIGN.md index 9d0600dd8..67abc23bb 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -134,7 +134,7 @@ Snapshot store has **two tiers** (ADR 0005). **Hot tier** lives on each host's l - `engram-sandbox-firecracker` — Linux + KVM. Production. Firecracker's HTTP-over-Unix-socket API + UFFD-backed memory restore. - `engram-sandbox-vz` — macOS Apple Silicon. Apple Virtualization.framework via `objc2-virtualization` bindings. APFS clone-based snapshots. Mac dev with real microVM isolation. (ADR 0003.) - `engram-sandbox-process` — anywhere. Subprocesses, no isolation. Fastest iteration loop for orchestration-layer work. -- Maintains the chunked-OCI image cache + tiered chunk resolver (host-side `PooledBackend` wrapper, agnostic of which `SandboxBackend` is wrapped), `HarnessHub` TCP listener for in-VM harness adapters dialing back. Heartbeats `(capacity, utilization, draining)` to the coordinator. +- Maintains the image cache, chunk store, and fill engine (host-side `PooledBackend` wrapper, agnostic of which `SandboxBackend` is wrapped), `HarnessHub` TCP listener for in-VM harness adapters dialing back. Heartbeats `(capacity, utilization, draining)` to the coordinator. - **`engram-agentd`**: in-VM exec daemon + harness supervisor (PID 1 after the init shim). Length-prefixed bincode over the configured transport. Verbs: `Exec` (streaming), `Stat`, `Upload`, `Download`, `StartShell`, `Ping`, `Shutdown`, `SpawnHarness`. First-frame token handshake (server side) gates non-trivial verbs. Owns the harness child process; each `SpawnHarness` kills the previous child and exec's a fresh one — clean re-spawn point on resume. On startup, dials the host's per-sandbox ready UDS so the host can block on `accept()` rather than poll for "is the in-VM listener bound." - **`engram-transport`**: backend-agnostic transport trait. `VsockTransport` (FC) and `ConsoleTransport` (VZ); chosen at runtime via `ENGRAM_TRANSPORT` set by the stage-1 init shim (injected by the materializer, ADR 0080). - **`engram-rootfs-materializer`**: host-side OCI→rootfs materializer (ADR 0080). Pulls a standard OCI/Docker image, applies whiteout-aware layer semantics to an in-memory namespace, declares the stage-1 `/sbin/engram-init` shim (the only engrams-owned file baked in), seals a deterministic pure-Rust ext4 layout (`mkext4`, ADR 0093), and streams the fill directly into chunked `BlobStorage` — no tree, no image file. Driven by the `MaterializeImage` host RPC at enable/rebase time — agentd, the harness, and ttyd are *not* injected; they ride host-staged bundle slots. @@ -184,8 +184,8 @@ Git is **not** in this table. Agents that want their work to land in a remote do ### Host agent (`engram-host-agent`) **Responsibilities:** -- Host-side `PooledBackend` wrapper composes the image cache, the chunk store, egress proxy, and the session-disk pool (see [Session disk](#session-disk)) onto any `SandboxBackend` (FC / VZ / process). Each session-create resolves the image's rootfs through the chunk store before delegating to the underlying backend. A chunk comes from the local NVMe cache, a peer host, or the object store; the host's fill engine (`engram-fill`) admits and schedules the object-store fetches. The OCI registry is not a chunk tier: no session read goes to it. The `PooledBackend` is what the host-agent's `LocalHostClient` wraps as its `SandboxBackend` inner; `LocalHostClient` itself adds harness routing (`bind_session`/`unbind_session`/`send_prompt`) by referencing a shared `HarnessHub` (ADR 0011). -- Drive sandbox lifecycle through `SandboxBackend` (`create`/`destroy`/`exec_stream`/`snapshot`/`restore`/`start_agent`/`set_harness_sink`); harness routing through `HarnessHub` (`bind_session`/`unbind_session`/`send_prompt`/`accept_via_session_lookup`). The hub's `EventSink` ships every per-session event over the WS as `NotifyKind::HarnessEvent`; the coord's read loop re-emits those into its in-proc hub so SSE subscribers see the same stream as mode=all. +- Host-side `PooledBackend` wrapper composes the image cache, the chunk store, egress proxy, and the session-disk pool (see [Session disk](#session-disk)) onto any `SandboxBackend` (FC / VZ / process). Each ordinary session-create resolves the image's rootfs through the chunk store before delegating to the underlying backend. A chunk comes from the local NVMe cache, a peer host, or the object store; the host's fill engine (`engram-fill`) admits and schedules the object-store fetches. The OCI registry is not a chunk tier: no session read goes to it. The `PooledBackend` is what the host-agent's `LocalHostClient` wraps as its `SandboxBackend` inner; `LocalHostClient` itself adds harness routing (`bind_session`/`unbind_session`/`send_prompt`) by referencing a shared `HarnessHub` (ADR 0011). +- Drive ordinary sandbox lifecycle through `SandboxBackend` (`create`/`destroy`/`exec_stream`/`snapshot`/`restore`/`start_agent`/`set_harness_sink`); harness routing through `HarnessHub` (`bind_session`/`unbind_session`/`send_prompt`/`accept_via_session_lookup`). The hub's `EventSink` ships every per-session event over the WS as `NotifyKind::HarnessEvent`; the coord's read loop re-emits those into its in-proc hub so SSE subscribers see the same stream as mode=all. - Run snapshot manager (host-local; see below). - Run the `HarnessHub` TCP listener: in-VM harness adapters dial back via `engram-transport` (vsock or virtio-console) → host-side TCP forwarding → `session_events` ingestion. - Enforce per-VM resource limits via the production backend (Firecracker enforces RAM/CPU/disk at the VMM boundary; the dev backend ignores them with a documented caveat). @@ -203,10 +203,12 @@ Git is **not** in this table. Agents that want their work to land in a remote do A session's disk is a volume the kernel owns. On Linux it is a thin volume in a device-mapper pool that node-prep builds (`thinPool` in the host-fleet chart; a Firecracker host agent refuses to start without one); on macOS -it is a sparse file under the work dir. Both sit behind `engram_host_core::SessionDisk` -and the two flows in `engram_host_core::disk_flows`: `stage` turns a chunk-store manifest -into a volume, and `checkpoint` turns a volume into a manifest. The guard for each fact is -the test named beside it. +it is a sparse file under the work dir. Ordinary FC and VZ sessions use +`engram_host_core::SessionDisk` and the two flows in `engram_host_core::disk_flows`: +`stage` turns a chunk-store manifest into a volume, and `checkpoint` turns a volume +into a manifest. The facts below describe ordinary session lifecycles and shared pool +behavior. Managed external-MCP sessions have the narrower lifecycle described after +these facts. The guard for each fact is the test named beside it. - No userspace process is in the data path. The host agent holds a volume only while it stages or checkpoints it; a volume outlives every handle to it and reopens by name @@ -341,8 +343,44 @@ the test named beside it. ADR 0125 records the decision and what it replaced. +**Managed external-MCP sessions:** Managed Firecracker uses the same production +`SessionDisk` instance, but imports the exact verified cold disk into a new root +volume and attaches its block device. Capture stops and reaps the VM before it +exports a transient volume snapshot. The host durably records the exported file's +hash and size before it can release the source +(`exported_capture_keeps_source_until_host_records_its_hash_and_size`). + +Both managed backends permit only explicit disk-only cold recovery with the exact +stored capture authority. They do not restore saved memory or device state, or +replay pending external effects. Ordinary checkpoint, eviction, resume, and migration +paths remain closed. A missing managed owner fails closed; a host restart does not +adopt that owner. Durable markers protect managed FC volumes from the ordinary +startup sweep (`startup_keeps_managed_volumes_and_sweeps_unowned_volumes`). Failed +or uncertain capture retains its source; an authorized normal FC destroy releases +its volumes only after owner cleanup succeeds +(`started_normal_destroy_releases_disks_but_uncertain_capture_does_not`). + +An incompatible-upgrade drain stops before deployment changes if it cannot +checkpoint or evict a managed session. An unchanged reattach generation and wire +version checks do not establish managed-session upgrade or adoption support. +External MCP remains disabled pending its upgrade, recovery, and rollback gates, +source-matched Firecracker and VZ qualification, current-head CI, and final +acceptance review. A manual Firecracker matrix passed all 24 cases at +`c5dc8e7b`, with verified cleanup. That result and the earlier VZ result at +`b75b0d7f` qualify only their recorded source, not the later upstream integration. +Ordinary coordinator/host traffic uses the version-38 policy contract. Its wire +conversion refuses managed deny data. Dedicated managed RPCs additionally require +the exact managed protocol-43 marker and pinned mTLS; neither the ordinary +version nor that marker supplies authority. The ordinary mixed-release path +keeps its existing transport with managed creation disabled. A TLS-only managed +listener cutover and managed-session rollback require separate qualification. +Proxy replacement can preserve a VM while closing its active proxy streams. +VZ parity is a separate gate. ADR 0127 records these limits. + ### Snapshot manager (lives inside host agent) +The snapshot and eviction flows in this section apply to ordinary sessions. + **Two tiers (ADR 0005)**: hot tier on local NVMe for same-host fast resume, cold tier in `BlobStorage` for cross-host durability. ADR 0001 retired the cold tier; ADR 0005 reintroduced it once the premise shifted from "30-second-preemption replication" (infeasible math) to "minutes-budget disk-pressure flush" (trivial math). - **Hot — FC backend (Linux production)**: `PATCH /vm Paused` + `PUT /snapshot/create` writes a state file + `memory.bin` to local NVMe. Restore via `PUT /snapshot/load` with `backend_type=Uffd` for sub-100ms hot resume on the same host. @@ -370,7 +408,7 @@ ADR 0125 records the decision and what it replaced. ADR 0080 retired the `engram-image-builder` bake path: a session image is now a plain `docker build && docker push` of a standard OCI image, and the OCI→rootfs conversion moved host-side, off the per-session path. -- Runs at **enable/rebase time** as the `MaterializeImage` host RPC (a sibling of `build_base_snapshot`), never per-session — so per-session latency is zero. +- Runs at **enable/rebase time** as the `MaterializeImage` host RPC, outside the per-session path. - Pulls the standard OCI/Docker image (platform by host arch), whiteout-aware-flattens the layers (xattrs / hardlinks / setuid / ownership, `FollowSymlinkInScope` semantics), extracts the OCI config blob → `OciRuntimeDefaults` (Dockerfile `ENV`/`WORKDIR`), declares the stage-1 `/sbin/engram-init` shim, seals a deterministic pure-Rust ext4 layout (`mkext4`, ADR 0093), and stream-fills the result into the host chunk store / `BlobStorage`. - Host safeguards keep host disk bounded: ADR 0078 disk-veto host pick, a ~2.5× image-size scratch budget integrated with chunk-cache accounting, scrub on all exit paths, an enable-time image-size cap, and ≤1 concurrent materialize per host. - The only engrams-owned file it bakes into the rootfs is the init shim; agentd, the harness, and ttyd ride host-staged bundle slots and are swapped in per-session (zero image re-bakes when they change). @@ -849,7 +887,7 @@ A small set of Dockerfile concepts don't translate cleanly to a microVM rootfs, ### Materialization (`engram-rootfs-materializer` + `MaterializeImage`) -There is no bake step and no `engram-image-builder` (ADR 0080). The OCI→rootfs conversion runs **host-side, at enable/rebase time** as the server-streaming `MaterializeImage` host RPC (a sibling of `build_base_snapshot`), off the per-session path: +The OCI→rootfs conversion runs **host-side, at enable/rebase time** as the server-streaming `MaterializeImage` host RPC, outside the per-session path: 1. **Pull** the standard OCI/Docker image, platform-selected by host arch. 2. **Flatten** the layers, whiteout-aware (`.wh.` / opaque dirs, xattrs / hardlinks / setuid, ownership; `FollowSymlinkInScope` symlink resolution). diff --git a/buf.gen.yaml b/buf.gen.yaml index ce7b49048..0fddb637d 100644 --- a/buf.gen.yaml +++ b/buf.gen.yaml @@ -1,12 +1,6 @@ -# TS codegen for the ADR 0051 app contract (`just gen-proto`). -# -# Only `engram/app/**` is generated for TS — host_service.proto is the -# coord↔host transport (tonic via crates/engram-protocol/build.rs) and -# never reaches the web/orchestrator tier. The `paths` filter values are -# cwd-relative and must include the input-directory prefix (buf v2 -# semantics). The app dir lands with Tasks 4-5; until then `buf -# generate` produces no output (the CI drift check guards on the dir -# existing). +# TS codegen for the application and dedicated internal MCP control contracts. +# host_service.proto remains coordinator-to-host transport, generated by tonic. +# Input paths are cwd-relative and include the input-directory prefix (buf v2). version: v2 # Wipe the `out` dirs before generating so deleted/renamed protos don't # leave stale bindings behind for the drift check to miss. @@ -14,7 +8,8 @@ clean: true inputs: - directory: crates/engram-protocol/proto paths: - - crates/engram-protocol/proto/engram/app # TS needs only the app contract + - crates/engram-protocol/proto/engram/app + - crates/engram-protocol/proto/engram/mcp # Dedicated internal mTLS control # Asymmetry is deliberate: query-es (TanStack Query hooks) is web-only — # the orchestrator is a server and needs plain messages/clients only. plugins: diff --git a/crates/engram-agentd/src/file_transfer.rs b/crates/engram-agentd/src/file_transfer.rs index 5c364ea3e..7051f9fbb 100644 --- a/crates/engram-agentd/src/file_transfer.rs +++ b/crates/engram-agentd/src/file_transfer.rs @@ -94,37 +94,58 @@ where } pub async fn read_file( + connection: S, + path: String, +) -> Result<(SessionFileMetadata, SessionFileStream), SandboxError> +where + S: AsyncRead + AsyncWrite + Unpin + Send + 'static, +{ + read_file_until_closed(connection, path, std::future::pending()).await +} + +/// Keep the connection bound to the supplied owner until the download ends. +pub async fn read_file_until_closed( mut connection: S, path: String, + closed: impl std::future::Future + Send + 'static, ) -> Result<(SessionFileMetadata, SessionFileStream), SandboxError> where S: AsyncRead + AsyncWrite + Unpin + Send + 'static, { - write_msg( - &mut connection, - &WireRequest::DownloadStream { path: path.clone() }, - ) - .await - .map_err(|error| vm_error("send DownloadStream request", error))?; - let (size_bytes, sha256) = match read_msg::<_, WireResponse>(&mut connection).await { - Ok(WireResponse::DownloadStreamReady { size_bytes, sha256 }) => (size_bytes, sha256), - Ok(WireResponse::Error { kind, message }) => { - return Err(SandboxError::Vm( - format!("agentd rejected DownloadStream ({kind}): {message}").into(), - )); - } - Ok(other) => { - return Err(SandboxError::Vm( - format!("unexpected DownloadStream response: {other:?}").into(), - )); + let mut closed = Box::pin(closed); + let setup = async { + write_msg( + &mut connection, + &WireRequest::DownloadStream { path: path.clone() }, + ) + .await + .map_err(|error| vm_error("send DownloadStream request", error))?; + let (size_bytes, sha256) = match read_msg::<_, WireResponse>(&mut connection).await { + Ok(WireResponse::DownloadStreamReady { size_bytes, sha256 }) => (size_bytes, sha256), + Ok(WireResponse::Error { kind, message }) => { + return Err(SandboxError::Vm( + format!("agentd rejected DownloadStream ({kind}): {message}").into(), + )); + } + Ok(other) => { + return Err(SandboxError::Vm( + format!("unexpected DownloadStream response: {other:?}").into(), + )); + } + Err(error) => return Err(vm_error("read DownloadStream metadata", error)), + }; + if size_bytes > MAX_SESSION_FILE_BYTES { + return Err(SandboxError::InvalidSpec(format!( + "file exceeds {MAX_SESSION_FILE_BYTES} bytes" + ))); } - Err(error) => return Err(vm_error("read DownloadStream metadata", error)), + Ok((size_bytes, sha256)) }; - if size_bytes > MAX_SESSION_FILE_BYTES { - return Err(SandboxError::InvalidSpec(format!( - "file exceeds {MAX_SESSION_FILE_BYTES} bytes" - ))); - } + let (size_bytes, sha256) = tokio::select! { + biased; + _ = &mut closed => Err(SandboxError::Unavailable("file transfer owner closed".into())), + result = setup => result, + }?; let metadata = SessionFileMetadata { path, size_bytes, @@ -132,41 +153,58 @@ where }; let (tx, rx) = mpsc::channel(8); tokio::spawn(async move { - let mut received = 0u64; - let mut hasher = Sha256::new(); - while received < size_bytes { - let WireFileChunk { bytes } = match read_msg(&mut connection).await { - Ok(frame) => frame, - Err(error) => { - let _ = tx - .send(Err(vm_error("read DownloadStream chunk", error))) - .await; - return; - } - }; - let next = match received.checked_add(bytes.len() as u64) { - Some(next) if next <= size_bytes => next, - _ => { - let _ = tx - .send(Err(SandboxError::Unavailable( - "download stream exceeds declared size".into(), - ))) - .await; + let download = async { + let mut received = 0u64; + let mut hasher = Sha256::new(); + while received < size_bytes { + let WireFileChunk { bytes } = match read_msg(&mut connection).await { + Ok(frame) => frame, + Err(error) => { + let _ = tx + .send(Err(vm_error("read DownloadStream chunk", error))) + .await; + return; + } + }; + let next = match received.checked_add(bytes.len() as u64) { + Some(next) if next <= size_bytes => next, + _ => { + let _ = tx + .send(Err(SandboxError::Unavailable( + "download stream exceeds declared size".into(), + ))) + .await; + return; + } + }; + hasher.update(&bytes); + received = next; + if tx.send(Ok(Bytes::from(bytes))).await.is_err() { return; } - }; - hasher.update(&bytes); - received = next; - if tx.send(Ok(Bytes::from(bytes))).await.is_err() { - return; } - } - let actual_sha256 = digest_hex(hasher.finalize()); - if actual_sha256 != sha256 { + let actual_sha256 = digest_hex(hasher.finalize()); + if actual_sha256 != sha256 { + let _ = tx + .send(Err(SandboxError::Unavailable(format!( + "download SHA-256 mismatch: expected {sha256}, received {actual_sha256}" + )))) + .await; + } + }; + let owner_closed = tokio::select! { + biased; + _ = tx.closed() => false, + _ = closed => true, + () = download => false, + }; + // Close I/O before a terminal error can wait on receiver backpressure. + drop(connection); + if owner_closed { let _ = tx - .send(Err(SandboxError::Unavailable(format!( - "download SHA-256 mismatch: expected {sha256}, received {actual_sha256}" - )))) + .send(Err(SandboxError::Unavailable( + "file transfer owner closed".into(), + ))) .await; } }); @@ -182,6 +220,78 @@ mod tests { use crate::proto::{read_msg, write_msg, WireFileChunk, WireRequest, WireResponse}; use futures::stream; + #[tokio::test] + async fn owner_close_stops_stalled_download_socket_before_receiver_drop() { + use tokio::io::AsyncReadExt; + let (client, mut peer) = tokio::net::UnixStream::pair().unwrap(); + let server = tokio::spawn(async move { + assert!(matches!( + read_msg::<_, WireRequest>(&mut peer).await.unwrap(), + WireRequest::DownloadStream { .. } + )); + write_msg( + &mut peer, + &WireResponse::DownloadStreamReady { + size_bytes: 3, + sha256: digest_hex(Sha256::digest(b"abc")), + }, + ) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_millis(100), peer.read(&mut [0])).await + }); + let (owner, mut closed) = tokio::sync::watch::channel(()); + let (_, mut stream) = + read_file_until_closed(client, "/tmp/stalled.bin".into(), async move { + let _ = closed.changed().await; + }) + .await + .unwrap(); + owner.send_replace(()); + let eof = server + .await + .unwrap() + .expect("owner close must close the socket before receiver drop") + .unwrap(); + assert_eq!(eof, 0); + assert!(matches!( + stream.next().await, + Some(Err(SandboxError::Unavailable(_))) + )); + assert!(stream.next().await.is_none()); + } + + #[tokio::test] + async fn dropping_stalled_download_closes_owned_socket() { + use tokio::io::AsyncReadExt; + let (client, mut peer) = tokio::net::UnixStream::pair().unwrap(); + let server = tokio::spawn(async move { + assert!(matches!( + read_msg::<_, WireRequest>(&mut peer).await.unwrap(), + WireRequest::DownloadStream { .. } + )); + write_msg( + &mut peer, + &WireResponse::DownloadStreamReady { + size_bytes: 3, + sha256: digest_hex(Sha256::digest(b"abc")), + }, + ) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_millis(100), peer.read(&mut [0])).await + }); + let (_, stream) = read_file(client, "/tmp/stalled.bin".into()).await.unwrap(); + tokio::task::yield_now().await; + drop(stream); + let eof = server + .await + .unwrap() + .expect("dropping the download must close its socket without peer data") + .unwrap(); + assert_eq!(eof, 0); + } + #[tokio::test] async fn upload_streams_more_than_the_old_unary_limit_with_exact_bytes() { let bytes = vec![0x5a; 3 * 1024 * 1024 + 17]; diff --git a/crates/engram-chunk-store/src/gc.rs b/crates/engram-chunk-store/src/gc.rs index 6c8d4cf90..1283eaf1a 100644 --- a/crates/engram-chunk-store/src/gc.rs +++ b/crates/engram-chunk-store/src/gc.rs @@ -379,6 +379,7 @@ async fn collect_manifest_refs(meta: &dyn MetadataStore) -> Result, snap_swap: Vec, snap_mem: Vec, + managed_disk: Vec, // ADR 0022 sources #4 + #5. enabled_base_mem: Vec, enabled_base_disk: Vec, @@ -649,6 +651,9 @@ mod tests { .chain(self.snap_swap.iter().map(|r| (None, Some(*r)))) .collect()) } + async fn list_managed_disk_manifests(&self) -> Result, MetaError> { + Ok(self.managed_disk.clone()) + } async fn list_recoverable_snapshot_memory_manifests( &self, ) -> Result, MetaError> { @@ -726,6 +731,19 @@ mod tests { assert!(pin_set.is_empty(), "empty meta must produce empty pin set"); } + #[tokio::test] + async fn managed_cold_inputs_pin_chunks_without_snapshot_rows() { + let (store, _dir) = fresh_store(); + let manifest = seed_manifest(&store, &[b"managed-cold-input"], ManifestKind::Disk).await; + let meta = PinSetMockMeta { + managed_disk: vec![manifest], + ..Default::default() + }; + let pins = PinSet::collect(&meta, &store).await.expect("collect"); + assert!(pins.contains(&ChunkHash::of(b"managed-cold-input"))); + assert_eq!(pins.len(), 1); + } + #[tokio::test] async fn sources_union_with_dedup() { let (store, _dir) = fresh_store(); diff --git a/crates/engram-coordinator/Cargo.toml b/crates/engram-coordinator/Cargo.toml index 0579cb347..857784b33 100644 --- a/crates/engram-coordinator/Cargo.toml +++ b/crates/engram-coordinator/Cargo.toml @@ -68,7 +68,7 @@ sqlx.workspace = true reqwest.workspace = true tokio.workspace = true tokio-stream.workspace = true -tonic.workspace = true +tonic = { workspace = true, features = ["tls-ring"] } tonic-reflection.workspace = true tower.workspace = true tower-http.workspace = true diff --git a/crates/engram-coordinator/src/api/admin.rs b/crates/engram-coordinator/src/api/admin.rs index 8cad7ffc5..9e7a48479 100644 --- a/crates/engram-coordinator/src/api/admin.rs +++ b/crates/engram-coordinator/src/api/admin.rs @@ -91,11 +91,11 @@ pub async fn pause_session( "session has no live sandbox to pause — it is idle or not yet started".into(), ) })?; - // ADR 0079: epoch threaded by the op executor; 0 until the verb migrates. + // ADR 0079: preserve identity without claiming an op epoch. state .services .host - .pause(sandbox_id, SessionFence::unfenced()) + .pause(sandbox_id, SessionFence::new(session_id, 0)) .await .map_err(|e| ApiError::Internal(format!("pause sandbox: {e}")))?; tracing::info!(%session_id, %sandbox_id, "admin: froze microVM in place"); @@ -118,11 +118,11 @@ pub async fn resume_session( "session has no live sandbox to resume in place — it is idle or not yet started".into(), ) })?; - // ADR 0079: epoch threaded by the op executor; 0 until the verb migrates. + // ADR 0079: preserve identity without claiming an op epoch. state .services .host - .resume(sandbox_id, SessionFence::unfenced()) + .resume(sandbox_id, SessionFence::new(session_id, 0)) .await .map_err(|e| ApiError::Internal(format!("resume sandbox: {e}")))?; tracing::info!(%session_id, %sandbox_id, "admin: unfroze microVM in place"); diff --git a/crates/engram-coordinator/src/api/exec.rs b/crates/engram-coordinator/src/api/exec.rs index 1b8638381..05ffc3dec 100644 --- a/crates/engram-coordinator/src/api/exec.rs +++ b/crates/engram-coordinator/src/api/exec.rs @@ -184,6 +184,8 @@ pub async fn exec_stream_core( ), ApiError, > { + crate::managed_boot::gate(state, id, true).await?; + let (base_env, default_workdir) = session_exec_env(state, id).await; let wake = req.wake.unwrap_or(true); let caller_supplied_exec_id = req.exec_id.is_some(); diff --git a/crates/engram-coordinator/src/api/prompt.rs b/crates/engram-coordinator/src/api/prompt.rs index 76df62114..2da8ab80f 100644 --- a/crates/engram-coordinator/src/api/prompt.rs +++ b/crates/engram-coordinator/src/api/prompt.rs @@ -226,6 +226,14 @@ pub(crate) async fn complete_tool_call_core( if tool_call_id.is_empty() { return Err(ApiError::BadRequest("`tool_call_id` is required".into())); } + if state + .services + .meta + .managed_tool_target_gone(id, &tool_call_id) + .await? + { + return Ok("delivery target gone"); + } let session = state.services.meta.get_session(id).await?; if session.status.is_terminal() { return Err(ApiError::Gone(format!( @@ -272,7 +280,7 @@ pub(crate) async fn complete_tool_call_core( delivered_at: None, acked_at: None, }; - state + let accepted = state .emit_with_outbox( id, SessionEvent::ToolResultSubmitted { @@ -282,7 +290,19 @@ pub(crate) async fn complete_tool_call_core( }, &row, ) - .await?; + .await; + if let Err(error) = accepted { + let call = row.payload["tool_call_id"].as_str().unwrap_or_default(); + if state + .services + .meta + .managed_tool_target_gone(id, call) + .await? + { + return Ok("delivery target gone"); + } + return Err(error); + } crate::outbox_delivery::enqueue_deliver_op(state, id).await; state.outbox_wake.notify_one(); Ok("tool result queued") @@ -451,6 +471,73 @@ pub(crate) async fn dequeue_queued_prompt_core( Ok("queued prompt dequeued") } +/// Same durable prompt/event path, included in the managed creation transaction. +pub(crate) fn managed_initial_prompt( + state: &SharedState, + id: SessionId, + text: Option, + mode: Option, +) -> Result, ApiError> { + let Some(text) = text.filter(|t| !t.is_empty()) else { + return Ok(None); + }; + if text.len() > MAX_PROMPT_BYTES { + return Err(ApiError::BadRequest( + "initial prompt exceeds size limit".into(), + )); + } + let now = state.services.clock.now_utc(); + let prompt_id = format!("create:{id}"); + let mut events = vec![ + SessionEvent::PromptReceived { + prompt_id: prompt_id.clone(), + at: now, + }, + SessionEvent::HarnessAgentMessage { + run_id: String::new(), + message_id: format!("user-{prompt_id}"), + role: AgentRole::User, + text: text.clone(), + prompt_id: Some(prompt_id.clone()), + at: now, + }, + ]; + if let Some(mode) = &mode { + events.push(SessionEvent::HarnessModeChanged { + mode: mode.clone(), + at: now, + }); + } + let events = events + .into_iter() + .map(|e| { + Ok(( + e.kind().to_string(), + serde_json::to_value(e) + .map_err(|_| ApiError::Internal("prompt event encoding failed".into()))?, + )) + }) + .collect::, ApiError>>()?; + let payload = match mode { + Some(mode) => serde_json::json!({"text":text,"mode":mode}), + None => serde_json::json!({"text":text}), + }; + Ok(Some(engram_core::traits::metadata::ManagedInitialPrompt { + events, + row: engram_core::types::outbox::OutboxRow { + prompt_id, + session_id: id, + kind: engram_core::types::outbox::OutboxKind::Prompt, + payload, + created_at: now, + not_before: now, + attempts: 0, + delivered_at: None, + acked_at: None, + }, + })) +} + #[cfg(test)] // tests drive a live system; wall clock/OS entropy here is input, not a decision source (ADR 0098 D1) #[allow(clippy::disallowed_methods)] diff --git a/crates/engram-coordinator/src/api/sessions.rs b/crates/engram-coordinator/src/api/sessions.rs index 531ac09cc..caa5b1ec0 100644 --- a/crates/engram-coordinator/src/api/sessions.rs +++ b/crates/engram-coordinator/src/api/sessions.rs @@ -32,16 +32,14 @@ pub(crate) const DEFAULT_DISK_GIB: u32 = 20; /// org-secret backend layered ahead of the deployment one). A `literal` secret /// is placed in the guest env; a `broker` secret installs a per-session /// placeholder in the env + an egress entry the proxy substitutes on the -/// secret's `allow_hosts` (the guest never holds the value). A ref that doesn't -/// resolve is skipped + warn-logged — the session still boots, that one secret -/// is just absent (mirrors `resolve_inject_entries`). +/// secret's `allow_hosts` (the guest never holds the value). Missing refs and +/// secret-store errors block launch and resume. /// /// Issue #535 (c): the per-secret `SecretStore` round trips are independent /// (no secret's resolution depends on another's), so they run concurrently /// via `join_all` instead of one-at-a-time — the result folds back into the /// SAME order-insensitive (env map + entries vec) shape a serial loop would /// have produced. -/// Missing secrets and store errors block launch and resume. pub(crate) async fn resolve_policy_secrets( state: &SharedState, policy: Option<&engram_core::types::IntegrationPolicy>, @@ -304,11 +302,10 @@ fn stamp_committer_env(cfg: &crate::config::CoordinatorConfig, env: &mut HashMap /// /// Async only because of `host.guest_ip`. Pure assembly delegated to /// [`assemble_resume_egress_policy`] for testability. -/// Build the egress policy for a resume. Lossy by design: a -/// resolution failure degrades to a reduced policy (the session must -/// come up; the user is present to retry, and the mint refresh seam -/// re-mints stale entries on first use). `None` = no guest IP for -/// this sandbox (yet). ADR 0111 removed the strict survivor face: +/// Policy reads, policy parsing, and required secret resolution fail closed. +/// Individual inject failures omit those entries. `None` = no guest IP for +/// an ordinary session (yet). Managed policies require a guest IP. +/// ADR 0111 removed the strict survivor face: /// the host persists every APPLIED policy beside its binding record /// and rebuilds its registry locally, so no post-restart rebuild /// happens on the coordinator at all. @@ -318,13 +315,33 @@ pub(crate) async fn build_resume_egress_policy( sandbox_id: engram_core::SandboxId, image: &str, ) -> Result, ApiError> { + let policy = load_session_policy(state, session_id).await?; let Some(guest_ip) = state.services.host.guest_ip(sandbox_id).await else { + if policy.as_ref().is_some_and(|p| p.managed_deny.is_some()) { + return Err(ApiError::BadRequest( + "managed policy requires an enforceable guest identity".into(), + )); + } return Ok(None); }; + build_egress_policy_for_guest(state, session_id, sandbox_id, image, guest_ip, policy) + .await + .map(Some) +} + +/// Compile persisted policy with a guest identity obtained from the host authority. +pub(crate) async fn build_egress_policy_for_guest( + state: &SharedState, + session_id: SessionId, + sandbox_id: engram_core::SandboxId, + image: &str, + guest_ip: std::net::Ipv4Addr, + policy: Option, +) -> Result { // ADR 0057: re-read the persisted session policy once → network + secrets + // injects (resolved host-side) + observes (pure), so a resumed session // re-derives its whole egress policy on the new host (same as create). - let policy = load_session_policy(state, session_id).await?; + let (repo, tag) = split_image_ref(image); let secret_ctx = SecretContext { repo, @@ -351,7 +368,7 @@ pub(crate) async fn build_resume_egress_policy( .await? .map(|rs| rs.apps) .unwrap_or_default(); - Ok(Some(assemble_resume_egress_policy( + let mut result = assemble_resume_egress_policy( session_id, sandbox_id, guest_ip, @@ -368,7 +385,9 @@ pub(crate) async fn build_resume_egress_policy( .map(|policy| policy.tunnels.clone()) .unwrap_or_default(), apps, - ))) + ); + result.managed_deny = policy.and_then(|p| p.managed_deny); + Ok(result) } /// Pure synchronous assembly path for the resume egress policy. @@ -391,6 +410,7 @@ pub(crate) fn assemble_resume_egress_policy( apps: Vec, ) -> engram_core::types::egress::SessionEgressPolicy { engram_core::types::egress::SessionEgressPolicy { + managed_deny: None, session_id, sandbox_id, guest_ip, @@ -453,6 +473,11 @@ pub(crate) async fn load_session_secrets( #[derive(Deserialize)] pub struct CreateSessionRequest { + /// Supplied only by the dedicated pinned mTLS creation service. + #[serde(skip)] + pub(crate) managed_creation: Option, + #[serde(skip)] + pub(crate) mcp_creation: Option, /// ADR 0109: trusted app-gRPC callers can reserve an ID so their external /// authorization snapshot exists before the VM starts. The public JSON /// surface cannot set this field. @@ -595,7 +620,15 @@ pub(crate) async fn create_session_core( return result; } }; - let result = boot_prepared(state, prepared, start, req.oauth_credential.clone()).await; + let result = boot_prepared( + state, + prepared, + start, + req.oauth_credential.clone(), + req.mcp_creation.clone(), + req.managed_creation.clone(), + ) + .await; let kind = match &result { Ok(body) => body.kind, Err(_) => "unknown", @@ -637,8 +670,11 @@ async fn boot_prepared( // (now-concurrent, host-side) restore work. create_start: std::time::Duration, oauth_credential: Option, + mcp_creation: Option, + managed_creation: Option, ) -> Result { let crate::session_boot::PreparedBoot { + managed_selection, inputs, memory_mib, cpu_budget_vcpus, @@ -730,7 +766,19 @@ async fn boot_prepared( // Issue #535 (b): the KEK seal is async crypto with no place inside a DB // transaction — do it here, once, and hand the SEALED row (not the // plaintext) to `reserve_and_persist_create`. - let sealed_secrets = match inputs.deferred_session_secrets.as_ref() { + // Managed retries use the complete original environment, never current image secrets. + let creation_secrets = if managed_creation.is_some() { + Some( + &inputs + .agent + .as_ref() + .ok_or_else(|| ApiError::Unavailable("managed harness unavailable".into()))? + .session_env, + ) + } else { + inputs.deferred_session_secrets.as_ref() + }; + let sealed_secrets = match creation_secrets { Some(overrides) => Some(seal_session_secrets(state, session_id, overrides).await?), None => None, }; @@ -742,7 +790,36 @@ async fn boot_prepared( None => None, }; + if mcp_creation.is_some() + && (!matches!(inputs.selected_harness.as_deref(), Some("claude" | "codex")) + || inputs.spec.mode != SessionMode::Agent) + { + return Err(ApiError::BadRequest( + "external write policy requires a supported agent harness".into(), + )); + } let write_set = engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: if managed_creation.is_some() { + let selection = managed_selection.ok_or_else(|| { + ApiError::Unavailable("approved cold boot inputs unavailable".into()) + })?; + selection.validate()?; + Some(selection) + } else { + None + }, + managed_initial_prompt: if managed_creation.is_some() { + crate::api::prompt::managed_initial_prompt( + state, + session_id, + inputs.prompt.clone(), + inputs.harness_mode.clone(), + )? + } else { + None + }, + managed_creation: managed_creation.clone(), + mcp_policy: mcp_creation, session_id, spec: inputs.spec.clone(), mem_budget_mib: memory_mib as i64, @@ -778,6 +855,17 @@ async fn boot_prepared( .await .map_err(|e| ApiError::Internal(format!("reserve_and_persist_create: {e}")))?; + // Held managed creates cannot reach prompt delivery or ordinary boot. The + // later physical boot consumer must claim this same durable reservation. + if managed_creation.is_some() { + return Ok(CreateSessionResponse { + session_id, + status: SessionState::Queued.as_str(), + image_version: image_tag, + kind: "queued", + }); + } + // ADR 0073 → ADR 0108 A6: the create-time initial prompt gets its // durable accept here (receipt + user echo + outbox row, committed // before any boot), but its DELIVERY rides the spawn: @@ -899,14 +987,14 @@ async fn boot_prepared( async move { // ADR 0079: the direct (capacity-available) create pipeline is // deliberately out-of-op — see the epoch-0 disposition note in - // the host's `check_session_epoch` — so the boot RPCs are - // unfenced here. The queued-create path fences via the - // create_boot verb. + // the host's `check_session_epoch`. Keep the session identity + // with epoch zero; this does not claim an op epoch. The + // queued-create path fences via the create_boot verb. match crate::session_boot::boot_on_reserved_host( &st, inputs, host_id, - SessionFence::unfenced(), + SessionFence::new(session_id, 0), ) .await { @@ -1015,13 +1103,7 @@ pub(crate) async fn prepare_from_row( state: &SharedState, session: &Session, ) -> Result { - let overrides = load_session_secrets(state, session.id) - .await - .unwrap_or_else(|e| { - tracing::warn!(session_id = %session.id, error = %e, - "prepare_from_row: secret overrides unavailable; continuing without them"); - None - }); + let overrides = load_session_secrets(state, session.id).await?; // Issue #535 (a): the queued path stays tolerant of a soft-deleted image // (the lineage is still pinned) — the cache's `_any` fill is exactly this. // Propagate the cache's own error verbatim (missing row / bad manifest / @@ -1033,25 +1115,8 @@ pub(crate) async fn prepare_from_row( // ADR 0056 (B′): re-read the persisted integration policy so the queued // boot re-injects (`resolve_inject_entries` resolves its refs again on the // new host, via `boot_on_reserved_host`'s overlapped env/egress leg). - // A malformed/absent blob → None (no injection). - let integration_policy = match state - .services - .meta - .get_session_integration_policy(session.id) - .await - { - Ok(Some(json)) => engram_core::types::IntegrationPolicy::parse(&json).unwrap_or_else(|e| { - tracing::warn!(session_id = %session.id, error = %e, - "persisted integration policy failed to parse; booting without injection"); - None - }), - Ok(None) => None, - Err(e) => { - tracing::warn!(session_id = %session.id, error = %e, - "integration policy lookup failed; booting without injection"); - None - } - }; + // Read and parse errors block boot; a missing policy is valid absence. + let integration_policy = load_session_policy(state, session.id).await?; // ADR 0077 phase 3: read the persisted RuntimeSpec so the queued / // resumed boot re-resolves the session's dynamic skill NAMES against // the current fleet stamp — fixing the ADR 0055 TODO(P1-D) where the @@ -1408,8 +1473,8 @@ async fn prepare_inner( // App committer identity (deployment config): stamped after every other // layer with `or_insert`, so anything explicit above wins. Resume takes - // the same stamp via `resolve_session_env` — deliberately NOT sealed - // into `session_secrets` (config is durable and hot-swappable). + // the same stamp via `resolve_session_env` for ordinary sessions. Managed + // reservations seal the complete original environment in `boot_prepared`. stamp_committer_env(&state.cfg, &mut session_env); let deferred_session_secrets: Option> = if deferred_map.is_empty() { @@ -1432,7 +1497,7 @@ async fn prepare_inner( // ADR 0062: resolve the per-session harness from the catalog → the AgentSpec // the backend execs + the `dyn_0` mount carrying the current catalog // generation. `None` for a dev VM (no harness, dyn_0 stays sentinel). - let (agent, harness_mount, harness_egress) = match resolve_harness( + let (agent, harness_mount, harness_egress, managed_launch) = match resolve_harness( state, selected_harness.as_deref(), mode, @@ -1443,11 +1508,10 @@ async fn prepare_inner( ) .await? { - // The CREATE flavor consumes all three fields BY NAME (ADR 0116 - // B2): the agent spec, the dyn_0 mount for the slot plan, and - // the egress to merge into the policy it persists. - Some(h) => (Some(h.agent), Some(h.mount), h.egress), - None => (None, None, Default::default()), + // CREATE retains the launch metadata with dyn_0 and merges its egress + // into the session policy before reservation. + Some(h) => (Some(h.agent), Some(h.mount), h.egress, Some(h.launch)), + None => (None, None, Default::default(), None), }; // ADR 0063 addendum: fold the selected harness's declared egress @@ -1492,7 +1556,36 @@ async fn prepare_inner( // the reserve-side `ScheduleContext` on `required_image_digest`. let manifest_digest = bundle.enabled.manifest_digest.clone(); + let managed_selection = + bundle + .base_snapshot + .disk_manifest + .zip(managed_launch) + .map(|(base_manifest, launch)| { + let mut mounts = selected_mounts.clone(); + mounts.sort_by_key(|d| { + (0..engram_core::types::sandbox::AuxRoDrive::RESERVED_SLOTS) + .position(|i| { + d.drive_id == engram_core::types::sandbox::AuxRoDrive::slot_drive_id(i) + }) + .unwrap_or(usize::MAX) + }); + engram_core::types::managed_bridge::ManagedBootSelection { + launch, + image: image_uri.into(), + image_digest: manifest_digest.clone(), + base_manifest, + aux_ro_drives: mounts, + vcpus: cpu_budget_vcpus, + memory_mib, + disk_gib: config + .resources + .suggested_disk_gib + .unwrap_or(DEFAULT_DISK_GIB), + } + }); Ok(crate::session_boot::PreparedBoot { + managed_selection, inputs: crate::session_boot::BootInputs { session_id, spec, @@ -1983,8 +2076,33 @@ pub(crate) async fn resolve_harness( )); }; - let (descriptor, exec, mount) = resolve_harness_mount(state, name).await?; - validate_harness_mode(&descriptor, name, harness_mode)?; + let (descriptor, _exec, mount) = resolve_harness_mount(state, name).await?; + let launch = engram_core::types::managed_bridge::ManagedHarnessLaunch { + descriptor, + workdir, + }; + Ok(Some(resolve_pinned_harness( + state, + launch, + mount, + session_id, + session_env, + harness_mode, + )?)) +} + +pub(crate) fn resolve_pinned_harness( + state: &SharedState, + launch: engram_core::types::managed_bridge::ManagedHarnessLaunch, + mount: engram_core::types::sandbox::AuxRoDrive, + session_id: SessionId, + session_env: HashMap, + harness_mode: Option<&str>, +) -> Result { + let descriptor = &launch.descriptor; + validate_harness_mode(descriptor, &descriptor.name, harness_mode)?; + let exec = format!("{}/{}", mount.guest_mount.display(), descriptor.exec_path()); + let workdir = launch.workdir.clone(); // Harness-only extras, layered on top of `session_env` for the harness // child. `session_env` already carries ENGRAM_SESSION_ID + the image env + @@ -2048,11 +2166,12 @@ pub(crate) async fn resolve_harness( session_env, host_ca_pem: None, }; - Ok(Some(ResolvedHarness { + Ok(ResolvedHarness { agent, mount, - egress: descriptor.egress, - })) + egress: descriptor.egress.clone(), + launch, + }) } /// ADR 0116 B2: `resolve_harness`'s output, with NAMED fields so each @@ -2060,6 +2179,7 @@ pub(crate) async fn resolve_harness( /// tuple-dropping (`let (agent, _mount, _egress) = …`) is how the /// cold-boot harness wedge survived review, and it is retired. pub(crate) struct ResolvedHarness { + pub launch: engram_core::types::managed_bridge::ManagedHarnessLaunch, /// The spec the backend execs (argv points into `dyn_0`). pub agent: engram_core::types::sandbox::AgentSpec, /// The `dyn_0` catalog mount. Consumed by the CREATE flavor's slot @@ -2140,6 +2260,159 @@ mod tests { use super::*; use engram_core::SandboxId; + fn seed_boot_image(meta: &engram_sim::SimMetadataStore, session: &Session) { + let id = engram_core::SnapshotId::from(uuid::Uuid::from_u128(72)); + let now = session.created_at; + meta.with_db_mut(|db| { + db.enabled_images.insert( + session.image.clone(), + ( + engram_core::types::EnabledImage { + id: uuid::Uuid::from_u128(73), + image_uri: session.image.clone(), + image_config: Default::default(), + oci_defaults: Default::default(), + manifest_digest: "sha256:test".into(), + disk_manifest: None, + base_snapshot_id: Some(id), + base_snapshot_disk_manifest: None, + base_snapshot_memory_manifest: None, + last_refreshed_at: now, + created_at: now, + updated_at: None, + soft_deleted_at: None, + }, + None, + ), + ); + db.snapshots.insert( + id, + engram_core::types::SnapshotRecord { + id, + session_id: None, + host_id: None, + image_version: "test".into(), + size_bytes: 0, + created_at: now, + last_accessed_at: now, + swap_manifest: None, + disk_manifest: None, + memory_manifest: None, + recoverable: true, + aux_bundles: Vec::new(), + events_cursor: None, + fc_snapshot_version: None, + }, + ); + }); + } + + #[tokio::test(start_paused = true)] + async fn queued_boot_propagates_secret_lookup_failure() { + let (state, meta, session, _dir) = crate::session_boot::credential_tests::fixture(); + meta.set_outage(true); + let expected = load_session_secrets(&state, session.id) + .await + .unwrap_err() + .to_string(); + let error = prepare_from_row(&state, &session) + .await + .err() + .expect("must reject failed secret read"); + assert_eq!( + error.to_string(), + expected, + "do not proceed to image lookup after a failed secret read" + ); + } + + #[tokio::test(start_paused = true)] + async fn queued_boot_rejects_unreadable_sealed_secrets() { + let (state, meta, session, _dir) = crate::session_boot::credential_tests::fixture(); + let cipher = engram_crypto::CredCipher::new(state.services.kek.as_ref()); + // A valid envelope containing the wrong JSON shape exercises decoding; + // corrupting its ciphertext exercises authentication/decryption. + let sealed = cipher.seal(b"[]").await.unwrap(); + let mut row = engram_core::types::SessionSecrets { + session_id: session.id, + wrapped_dek: sealed.wrapped_dek, + nonce: sealed.nonce.to_vec(), + ciphertext: sealed.ciphertext, + key_id: sealed.key_id, + created_at: session.created_at, + }; + for failure in [ + "deserialize session secrets", + "open session secrets", + "nonce wrong length", + ] { + meta.with_db_mut(|db| { + db.session_secrets.insert(session.id, row.clone()); + }); + let error = prepare_from_row(&state, &session) + .await + .err() + .expect("must reject unreadable secrets"); + assert!(error.to_string().contains(failure), "{error}"); + row.ciphertext[0] ^= 1; + if failure == "open session secrets" { + row.nonce.clear(); + } + } + } + + #[tokio::test(start_paused = true)] + async fn queued_boot_rejects_malformed_policy_and_preserves_absence() { + let (state, meta, session, _dir) = crate::session_boot::credential_tests::fixture(); + seed_boot_image(&meta, &session); + meta.with_db_mut(|db| { + db.session_integration_policy.insert(session.id, "{".into()); + }); + let error = prepare_from_row(&state, &session) + .await + .err() + .expect("must reject malformed policy"); + assert!(error.to_string().contains("session policy:"), "{error}"); + meta.with_db_mut(|db| { + db.session_integration_policy.remove(&session.id); + }); + let prepared = prepare_from_row(&state, &session) + .await + .expect("absent secrets and policy are valid"); + assert!(prepared.inputs.integration_policy.is_none()); + assert!(prepared.inputs.deferred_session_secrets.is_none()); + + let overrides = HashMap::from([("API_TOKEN".into(), "session-token".into())]); + let sealed = seal_session_secrets(&state, session.id, &overrides) + .await + .unwrap(); + meta.with_db_mut(|db| { + db.session_secrets.insert(session.id, sealed); + db.session_integration_policy + .insert(session.id, "{}".into()); + }); + let prepared = prepare_from_row(&state, &session) + .await + .expect("valid credentials and policy still prepare"); + assert_eq!( + prepared.inputs.session_env.get("API_TOKEN"), + overrides.get("API_TOKEN") + ); + assert_eq!(prepared.inputs.deferred_session_secrets, Some(overrides)); + assert!(prepared.inputs.integration_policy.is_some()); + } + + #[tokio::test(start_paused = true)] + async fn persisted_policy_lookup_failure_is_not_absence() { + let (state, meta, session, _dir) = crate::session_boot::credential_tests::fixture(); + assert!(load_session_policy(&state, session.id) + .await + .unwrap() + .is_none()); + meta.set_outage(true); + assert!(load_session_policy(&state, session.id).await.is_err()); + } + struct ExplicitRefStore; #[async_trait::async_trait] diff --git a/crates/engram-coordinator/src/api/snapshot.rs b/crates/engram-coordinator/src/api/snapshot.rs index 3bb974af7..1eb388459 100644 --- a/crates/engram-coordinator/src/api/snapshot.rs +++ b/crates/engram-coordinator/src/api/snapshot.rs @@ -53,6 +53,7 @@ pub(crate) fn placeholder_egress_policy( sandbox_id: SandboxId, ) -> engram_core::types::egress::SessionEgressPolicy { engram_core::types::egress::SessionEgressPolicy { + managed_deny: None, session_id, sandbox_id, guest_ip: std::net::Ipv4Addr::UNSPECIFIED, @@ -108,6 +109,8 @@ pub(crate) async fn reattach_harness_in_place( sandbox_id: SandboxId, fence: SessionFence, ) -> Result { + crate::managed_boot::gate(state, session_id, false).await?; + let session = state.services.meta.get_session(session_id).await?; // Re-confirm the session is still bound to the sandbox the caller resolved — // a concurrent eviction/resume may have moved it, and we must never re-issue @@ -161,6 +164,8 @@ pub(crate) async fn snapshot_core( state: &SharedState, id: SessionId, ) -> Result { + crate::managed_boot::gate(state, id, false).await?; + state.services.meta.get_session(id).await?; let sandbox_id = state.resolve_sandbox(id).await.ok_or_else(|| { @@ -422,6 +427,8 @@ pub(crate) async fn resume_core( state: &SharedState, id: SessionId, ) -> Result { + crate::managed_boot::gate(state, id, false).await?; + let observed = enqueue_and_observe_resume(state, id).await?; Ok(SnapshotResponse { session_id: id, @@ -550,6 +557,8 @@ async fn observe_resume_op( /// return 410; terminal `Completed` / `Failed` return 409 (no work /// is left to dispatch). pub async fn ensure_active(state: &SharedState, id: SessionId) -> Result<(), ApiError> { + crate::managed_boot::gate(state, id, true).await?; + let session = state.services.meta.get_session(id).await?; match session.status { SessionState::Active => { diff --git a/crates/engram-coordinator/src/config.rs b/crates/engram-coordinator/src/config.rs index e1361a40c..a92ae6b27 100644 --- a/crates/engram-coordinator/src/config.rs +++ b/crates/engram-coordinator/src/config.rs @@ -24,13 +24,15 @@ pub struct CoordinatorConfig { // `engram-auth` AuthConfig) is removed along with the whole `engram-auth` // crate. The coordinator no longer resolves human identity — `auth_tokens` // (the deployment bearer) and `app_grpc_tokens` (the app-gRPC bearer) are - // the only machine-caller credentials it knows about. + // the generic machine-caller credentials. MCP uses separate mTLS pins. /// Local address the harness-channel TCP listener binds to. /// Harnesses spawned via `start_agent` on the Process backend dial /// this from the same host. `127.0.0.1:0` (default) lets the OS /// pick a free port; the coordinator reads back the bound address /// and plumbs it into the agent's env at session-create time. pub harness_listen_addr: std::net::SocketAddr, + /// Optional operator file for the separate MCP mTLS listener. Unset disables it. + pub mcp_control_config: Option, /// Address the orchestrator-facing app gRPC server binds to /// (ADR 0051 §2.3). Serves Session/ShellRelay/Fleet/Image/Secret /// beside the axum API. Loopback by default — the orchestrator is @@ -77,6 +79,7 @@ impl Default for CoordinatorConfig { harness_listen_addr: "127.0.0.1:0" .parse() .expect("default harness_listen_addr must parse"), + mcp_control_config: None, app_grpc_addr: "127.0.0.1:50061" .parse() .expect("default app_grpc_addr must parse"), diff --git a/crates/engram-coordinator/src/grpc_app/convert.rs b/crates/engram-coordinator/src/grpc_app/convert.rs index 3fb5670d8..aaabfe537 100644 --- a/crates/engram-coordinator/src/grpc_app/convert.rs +++ b/crates/engram-coordinator/src/grpc_app/convert.rs @@ -183,7 +183,11 @@ pub(crate) fn create_request_from_proto( Ok(app::OauthSubjectKind::Connector) => { engram_core::types::oauth::OAuthSubjectKind::Connector } - Ok(app::OauthSubjectKind::Mcp) => engram_core::types::oauth::OAuthSubjectKind::Mcp, + Ok(app::OauthSubjectKind::Mcp) => { + return Err(ApiError::BadRequest( + "MCP credentials cannot be bound to a guest session".into(), + )) + } Ok(app::OauthSubjectKind::UserConnector) => { engram_core::types::oauth::OAuthSubjectKind::UserConnector } @@ -235,6 +239,8 @@ pub(crate) fn create_request_from_proto( }) .collect::, ApiError>>()?; Ok(CreateSessionRequest { + managed_creation: None, + mcp_creation: None, requested_session_id, image: image_uri, mode, @@ -885,6 +891,22 @@ pub(super) fn retirement_to_proto( // tests drive a live system; wall clock/OS entropy here is input, not a decision source (ADR 0098 D1) #[allow(clippy::disallowed_methods)] mod tests { + #[test] + fn mcp_guest_create_is_rejected() { + let request = engram_protocol::app::CreateSessionRequest { + image_uri: "registry.test/example:latest".into(), + oauth_credential: Some(engram_protocol::app::OAuthCredentialBinding { + subject: Some(engram_protocol::app::OAuthSubject { + kind: engram_protocol::app::OauthSubjectKind::Mcp as i32, + id: "connection-a".into(), + }), + provider: "mcp".into(), + }), + ..Default::default() + }; + assert!(super::create_request_from_proto(request).is_err()); + } + use super::*; use engram_core::types::session::SessionMode; use engram_core::types::{Session, SessionState}; diff --git a/crates/engram-coordinator/src/grpc_app/mod.rs b/crates/engram-coordinator/src/grpc_app/mod.rs index a83682714..a1bd59d19 100644 --- a/crates/engram-coordinator/src/grpc_app/mod.rs +++ b/crates/engram-coordinator/src/grpc_app/mod.rs @@ -11,7 +11,7 @@ //! authz lives over there, not here. pub mod auth; -mod convert; +pub(crate) mod convert; mod fleet; mod harness_catalog; mod image; diff --git a/crates/engram-coordinator/src/grpc_app/oauth.rs b/crates/engram-coordinator/src/grpc_app/oauth.rs index 2650572be..4be1ce20c 100644 --- a/crates/engram-coordinator/src/grpc_app/oauth.rs +++ b/crates/engram-coordinator/src/grpc_app/oauth.rs @@ -33,7 +33,9 @@ fn subject_from_proto( let kind = match app::OauthSubjectKind::try_from(subject.kind) { Ok(app::OauthSubjectKind::User) => OAuthSubjectKind::User, Ok(app::OauthSubjectKind::Connector) => OAuthSubjectKind::Connector, - Ok(app::OauthSubjectKind::Mcp) => OAuthSubjectKind::Mcp, + Ok(app::OauthSubjectKind::Mcp) => { + return Err("MCP credentials require the dedicated control service") + } Ok(app::OauthSubjectKind::UserConnector) => OAuthSubjectKind::UserConnector, _ => return Err("subject kind is required"), }; @@ -395,7 +397,11 @@ impl app::o_auth_credential_service_server::OAuthCredentialService for AppOAuthC let kind = match app::OauthSubjectKind::try_from(subject.kind) { Ok(app::OauthSubjectKind::User) => OAuthSubjectKind::User, Ok(app::OauthSubjectKind::Connector) => OAuthSubjectKind::Connector, - Ok(app::OauthSubjectKind::Mcp) => OAuthSubjectKind::Mcp, + Ok(app::OauthSubjectKind::Mcp) => { + return Err(Status::permission_denied( + "MCP credentials require the dedicated control service", + )) + } Ok(app::OauthSubjectKind::UserConnector) => OAuthSubjectKind::UserConnector, _ => return Err(Status::invalid_argument("subject kind is required")), }; @@ -557,3 +563,16 @@ impl app::o_auth_credential_service_server::OAuthCredentialService for AppOAuthC })) } } + +#[cfg(test)] +mod mcp_boundary_tests { + use super::*; + #[test] + fn mcp_subject_rejects_all_generic_app_oauth_operations() { + assert!(subject_from_proto(Some(app::OAuthSubject { + kind: app::OauthSubjectKind::Mcp as i32, + id: "connection-a".into() + })) + .is_err()); + } +} diff --git a/crates/engram-coordinator/src/grpc_app/session.rs b/crates/engram-coordinator/src/grpc_app/session.rs index e99b1f91c..4b908de0f 100644 --- a/crates/engram-coordinator/src/grpc_app/session.rs +++ b/crates/engram-coordinator/src/grpc_app/session.rs @@ -73,6 +73,16 @@ impl app::session_service_server::SessionService for AppSessionService { })) } + async fn get_mcp_write_policy( + &self, + req: Request, + ) -> Result, Status> { + self.auth.check(&req)?; + Err(Status::failed_precondition( + "trusted policy transport required", + )) + } + async fn get_session( &self, req: Request, @@ -171,6 +181,11 @@ impl app::session_service_server::SessionService for AppSessionService { ) -> Result, Status> { self.auth.check(&req)?; let r = req.into_inner(); + if r.expected_mcp_plan_review.is_some() { + return Err(Status::failed_precondition( + "trusted policy transport required", + )); + } let id = parse_session_id(&r.session_id)?; let note = crate::api::prompt::complete_tool_call_core( &self.state, @@ -476,7 +491,7 @@ impl app::session_service_server::SessionService for AppSessionService { self.auth.check(&req)?; let request = req.into_inner(); let session_id = parse_session_id(&request.session_id)?; - let (metadata, mut bytes) = + let (metadata, bytes) = crate::api::session_files::read_file_core(&self.state, session_id, request.path) .await .map_err(into_status)?; @@ -486,8 +501,7 @@ impl app::session_service_server::SessionService for AppSessionService { .next() .unwrap_or_default() .to_owned(); - let (tx, rx) = tokio::sync::mpsc::channel(8); - tx.send(Ok(app::ReadFileResponse { + let metadata = app::ReadFileResponse { frame: Some(app::read_file_response::Frame::Metadata( app::ReadFileMetadata { path: metadata.path, @@ -496,24 +510,17 @@ impl app::session_service_server::SessionService for AppSessionService { file_name, }, )), - })) - .await - .map_err(|_| Status::cancelled("read client closed before metadata"))?; - tokio::spawn(async move { - while let Some(item) = bytes.next().await { - let response = item - .map(|chunk| app::ReadFileResponse { - frame: Some(app::read_file_response::Frame::Chunk(chunk.to_vec())), - }) - .map_err(|error| into_status(error.into())); - if tx.send(response).await.is_err() { - return; - } - } - }); - Ok(Response::new(Box::pin( - tokio_stream::wrappers::ReceiverStream::new(rx), - ))) + }; + // Keep ownership of the source in the response. A stalled source must + // also be dropped when the downstream client closes. + let response = + futures::stream::once(async move { Ok(metadata) }).chain(bytes.map(|item| { + item.map(|chunk| app::ReadFileResponse { + frame: Some(app::read_file_response::Frame::Chunk(chunk.to_vec())), + }) + .map_err(|error| into_status(error.into())) + })); + Ok(Response::new(Box::pin(response))) } async fn copy_files( diff --git a/crates/engram-coordinator/src/host_registry.rs b/crates/engram-coordinator/src/host_registry.rs index a3b328e75..2e6aa017d 100644 --- a/crates/engram-coordinator/src/host_registry.rs +++ b/crates/engram-coordinator/src/host_registry.rs @@ -117,6 +117,36 @@ pub struct HostRegistry { } impl HostRegistry { + async fn managed_sandbox_gate( + &self, + id: SandboxId, + allow_started: bool, + ) -> Result<(), SandboxError> { + match self + .meta + .managed_sandbox_open(id) + .await + .map_err(|_| SandboxError::Unavailable("managed authority unavailable".into()))? + { + None => Ok(()), + Some(true) if allow_started => Ok(()), + _ => Err(SandboxError::Unavailable("managed boot gate closed".into())), + } + } + async fn managed_session_gate(&self, id: SessionId) -> Result<(), SandboxError> { + if self + .meta + .managed_boot_open(id) + .await + .map_err(|_| SandboxError::Unavailable("managed authority unavailable".into()))? + .is_some() + { + return Err(SandboxError::Unavailable( + "managed session requires registered boot commands".into(), + )); + } + Ok(()) + } /// Construct a registry backed by `meta` for read-through cache /// misses. TTL is read from `ENGRAM_HOST_REGISTRY_TTL_SECS` /// (default 60s). @@ -326,6 +356,8 @@ impl HostRegistry { selected_mounts: Vec, fence: SessionFence, ) -> Result { + self.managed_session_gate(fence.session_id).await?; + // The capacity decision already happened in `reserve_placement`; just // resolve the chosen host's backend (dialing through PG if this // replica hasn't seen the host yet — ADR 0047). @@ -511,6 +543,8 @@ impl HostClient for HostRegistry { id: SandboxId, cmd: ExecRequest, ) -> Result { + self.managed_sandbox_gate(id, true).await?; + // ADR 0050 C: the gRPC pool dials lazily and defers per-RPC // retry to the call site. A freshly-scaled host's first call // can come back `Unavailable` (connect not yet established); @@ -548,6 +582,8 @@ impl HostClient for HostRegistry { spec: SessionFileSpec, bytes: SessionFileStream, ) -> Result { + self.managed_sandbox_gate(id, true).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.write_file(id, spec, bytes).await } @@ -557,6 +593,8 @@ impl HostClient for HostRegistry { id: SandboxId, path: String, ) -> Result<(SessionFileMetadata, SessionFileStream), SandboxError> { + self.managed_sandbox_gate(id, true).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.read_file(id, path).await } @@ -566,6 +604,8 @@ impl HostClient for HostRegistry { id: SandboxId, fence: SessionFence, ) -> Result { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.snapshot(id, fence).await } @@ -575,6 +615,8 @@ impl HostClient for HostRegistry { id: SandboxId, fence: SessionFence, ) -> Result { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.snapshot_hold(id, fence).await } @@ -584,6 +626,8 @@ impl HostClient for HostRegistry { id: SandboxId, fence: SessionFence, ) -> Result { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.snapshot_begin(id, fence).await } @@ -593,6 +637,8 @@ impl HostClient for HostRegistry { id: SandboxId, fence: SessionFence, ) -> Result { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.migration_presetup(id, fence).await } @@ -603,6 +649,8 @@ impl HostClient for HostRegistry { export_id: &str, fence: SessionFence, ) -> Result { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend .migration_capture_postcopy(id, export_id, fence) @@ -613,6 +661,8 @@ impl HostClient for HostRegistry { &self, id: SandboxId, ) -> Result { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.migration_drain_wait(id).await } @@ -623,6 +673,8 @@ impl HostClient for HostRegistry { export_id: &str, fence: SessionFence, ) -> Result<(), SandboxError> { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.migration_commit(id, export_id, fence).await } @@ -633,6 +685,8 @@ impl HostClient for HostRegistry { export_id: &str, fence: SessionFence, ) -> Result<(), SandboxError> { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.migration_abort(id, export_id, fence).await } @@ -652,11 +706,15 @@ impl HostClient for HostRegistry { id: SandboxId, fence: SessionFence, ) -> Result<(), SandboxError> { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.commit_snapshot(id, fence).await } async fn abort_snapshot(&self, id: SandboxId, fence: SessionFence) -> Result<(), SandboxError> { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.abort_snapshot(id, fence).await } @@ -666,6 +724,8 @@ impl HostClient for HostRegistry { metadata: SnapshotMetadata, fence: SessionFence, ) -> Result { + self.managed_session_gate(fence.session_id).await?; + let (host_id, backend) = self.pick_any().ok_or_else(Self::no_host_error)?; let sandbox_id = backend.restore(metadata, fence).await?; self.sandbox_owner.insert(sandbox_id, host_id); @@ -706,6 +766,87 @@ impl HostClient for HostRegistry { result } + async fn read_managed_cold_assets( + &self, + request: engram_core::types::managed_assets::ManagedColdAssetsRequest, + ) -> Result { + self.backend_for(request.host_id) + .await? + .read_managed_cold_assets(request) + .await + } + async fn prepare_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootPrepare, + ) -> Result { + self.backend_for(request.key.context.target.host_id) + .await? + .prepare_managed_boot(request) + .await + } + async fn read_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootKey, + ) -> Result { + self.backend_for(request.context.target.host_id) + .await? + .read_managed_boot(request) + .await + } + async fn finalize_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootFinalize, + ) -> Result { + self.backend_for( + request + .finalization + .prepared + .request + .key + .context + .target + .host_id, + ) + .await? + .finalize_managed_boot(request) + .await + } + async fn start_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootStart, + ) -> Result { + self.backend_for(request.receipt.key.context.target.host_id) + .await? + .start_managed_boot(request) + .await + } + async fn capture_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedCaptureRequest, + ) -> Result { + self.backend_for(request.key.context.target.host_id) + .await? + .capture_managed_boot(request) + .await + } + async fn read_managed_policy( + &self, + target: engram_core::types::managed_deny::ManagedPolicyTarget, + ) -> Result, SandboxError> { + self.backend_for(target.host_id) + .await? + .read_managed_policy(target) + .await + } + async fn apply_managed_policy( + &self, + request: engram_core::types::managed_deny::ManagedPolicyApply, + ) -> Result { + self.backend_for(request.context.target.host_id) + .await? + .apply_managed_policy(request) + .await + } async fn start_agent( &self, id: SandboxId, @@ -713,6 +854,8 @@ impl HostClient for HostRegistry { policy: engram_core::types::egress::SessionEgressPolicy, fence: SessionFence, ) -> Result<(), SandboxError> { + self.managed_sandbox_gate(id, false).await?; + let (_, backend) = self.resolve_owner(id).await?; backend.start_agent(id, agent, policy, fence).await } @@ -767,6 +910,8 @@ impl HostClient for HostRegistry { sandbox_id: SandboxId, binding_epoch: u64, ) -> Result<(), engram_core::SandboxError> { + self.managed_session_gate(session_id).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend .bind_session(session_id, sandbox_id, binding_epoch) @@ -799,6 +944,8 @@ impl HostClient for HostRegistry { text: String, mode: Option, ) -> Result<(), SandboxError> { + self.managed_sandbox_gate(sandbox_id, true).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend.send_prompt(sandbox_id, prompt_id, text, mode).await } @@ -809,6 +956,8 @@ impl HostClient for HostRegistry { prompt_id: String, text: String, ) -> Result<(), SandboxError> { + self.managed_sandbox_gate(sandbox_id, true).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend .edit_queued_prompt(sandbox_id, prompt_id, text) @@ -820,6 +969,8 @@ impl HostClient for HostRegistry { sandbox_id: SandboxId, prompt_id: String, ) -> Result<(), SandboxError> { + self.managed_sandbox_gate(sandbox_id, true).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend.dequeue_queued_prompt(sandbox_id, prompt_id).await } @@ -830,6 +981,8 @@ impl HostClient for HostRegistry { tool_call_id: String, result_json: String, ) -> Result<(), SandboxError> { + self.managed_sandbox_gate(sandbox_id, true).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend .tool_result(sandbox_id, tool_call_id, result_json) @@ -842,11 +995,15 @@ impl HostClient for HostRegistry { } async fn pause(&self, sandbox_id: SandboxId, fence: SessionFence) -> Result<(), SandboxError> { + self.managed_sandbox_gate(sandbox_id, false).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend.pause(sandbox_id, fence).await } async fn resume(&self, sandbox_id: SandboxId, fence: SessionFence) -> Result<(), SandboxError> { + self.managed_sandbox_gate(sandbox_id, false).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend.resume(sandbox_id, fence).await } @@ -855,6 +1012,8 @@ impl HostClient for HostRegistry { &self, sandbox_id: SandboxId, ) -> Result { + self.managed_sandbox_gate(sandbox_id, true).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend.start_browser(sandbox_id).await } @@ -865,6 +1024,8 @@ impl HostClient for HostRegistry { } async fn start_ide(&self, sandbox_id: SandboxId) -> Result { + self.managed_sandbox_gate(sandbox_id, true).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend.start_ide(sandbox_id).await } @@ -878,6 +1039,8 @@ impl HostClient for HostRegistry { &self, sandbox_id: SandboxId, ) -> Result { + self.managed_sandbox_gate(sandbox_id, true).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend.proxy_shell(sandbox_id).await } @@ -887,6 +1050,8 @@ impl HostClient for HostRegistry { sandbox_id: SandboxId, port: u16, ) -> Result { + self.managed_sandbox_gate(sandbox_id, true).await?; + let (_, backend) = self.resolve_owner(sandbox_id).await?; backend.proxy_port(sandbox_id, port).await } @@ -989,6 +1154,18 @@ mod tests { ) -> Result<(), engram_core::MetaError> { Ok(()) } + async fn managed_sandbox_open( + &self, + _id: SandboxId, + ) -> Result, engram_core::MetaError> { + Ok(None) + } + async fn managed_boot_open( + &self, + _id: SessionId, + ) -> Result, engram_core::MetaError> { + Ok(None) + } async fn host_for_sandbox( &self, sandbox_id: SandboxId, diff --git a/crates/engram-coordinator/src/integrations.rs b/crates/engram-coordinator/src/integrations.rs index 02d787277..c405c2066 100644 --- a/crates/engram-coordinator/src/integrations.rs +++ b/crates/engram-coordinator/src/integrations.rs @@ -112,6 +112,18 @@ impl IntegrationBroker { return Some(e.clone()); } let desc = self.by_provider.get(provider)?; + // Cache notifications are not an authorization fence. Check each ref + // before returning an engine that already holds a plaintext value. + for field in &desc.fields { + let name = format!("{}.{}", desc.kind, field.name); + if secrets + .check_access(&name, &SecretSchema::default()) + .await + .is_err() + { + return None; + } + } let version = self.version.load(Ordering::Acquire); if let Some((v, eng)) = self.lazy.lock().get(provider) { if *v == version { @@ -170,6 +182,34 @@ impl IntegrationBroker { mod tests { // tests drive a live system; wall clock/OS entropy here is input, not a decision source (ADR 0098 D1) #![allow(clippy::disallowed_methods)] + #[tokio::test] + async fn mcp_reservation_blocks_cached_mint_before_notification() { + use engram_core::traits::{LayeredSecretStore, MetadataStore, StaticSecretStore}; + use engram_core::types::org_secret::McpSecretRole; + let meta = engram_sim::SimMetadataStore::new( + engram_sim::ManualClock::new(), + Arc::new(engram_sim::SimEntropy::seeded(4)), + ); + let secrets: Arc = Arc::new(LayeredSecretStore::new(vec![ + Arc::new(crate::org_secrets::OrgSecretBackend::new( + meta.clone(), + Arc::new(engram_crypto::EnvVarKeyProvider::from_bytes( + [7; 32], "test", + )), + )), + Arc::new(StaticSecretStore::new(HashMap::from([( + "fake_kind.token".into(), + "cached-value".into(), + )]))), + ])); + let broker = broker(); + assert!(broker.resolve("fake", &secrets).await.is_some()); + meta.reserve_mcp_org_secret("fake_kind.token", "connection-a", McpSecretRole::ApiKey) + .await + .unwrap(); + assert!(broker.resolve("fake", &secrets).await.is_none()); + } + use super::*; use async_trait::async_trait; use engram_core::error::{IntegrationError, SecretError}; diff --git a/crates/engram-coordinator/src/lib.rs b/crates/engram-coordinator/src/lib.rs index 30353e314..e76c9f1ff 100644 --- a/crates/engram-coordinator/src/lib.rs +++ b/crates/engram-coordinator/src/lib.rs @@ -30,6 +30,7 @@ pub mod idle_detector; pub mod idle_evictor; pub mod integration_ops; pub mod integrations; +pub mod mcp; pub mod metrics; pub mod oauth; pub mod oauth_client_credentials; @@ -217,6 +218,7 @@ pub async fn run_with_registry_and_local( // Push-driven via `queue_wake` (see above); polling is the fallback. // ADR 0073 phase 2: the outbox delivery driver — resume-behind- // enqueue + at-least-once forward + redelivery-until-acked. + let _managed_boot = managed_boot::spawn(state.clone()); let _outbox_delivery = outbox_delivery::spawn(state.clone(), state.outbox_wake.clone()); // ADR 0079: the session-op executor — the lifecycle kernel. let _session_ops = session_ops::spawn(state.clone(), state.session_ops_wake.clone()); @@ -373,6 +375,20 @@ pub async fn run_with_registry_and_local( state.services.host.set_upload_sink(sink); } + // The optional credential service has its own authenticated listener. + let mcp_control = mcp::startup::start_with_policy( + cfg.mcp_control_config.as_deref(), + state.oauth.clone(), + state.subscribe_shutdown(), + Some(state.clone()), + ) + .await + .map_err(|_| { + CoordinatorError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "invalid MCP control configuration", + )) + })?; // ADR 0051 B: the orchestrator-facing app gRPC surface, served // BESIDE the axum API (additive — no REST route changes). Spawned so // a runtime serve crash doesn't take the web surface down with it @@ -443,10 +459,12 @@ pub async fn run_with_registry_and_local( .await .map_err(CoordinatorError::Io)?; - // Graceful shutdown drains both servers: axum has returned, now - // wait for the app gRPC serve loop to finish its own drain. A + // Axum has drained. Wait for both control listeners to finish. A // JoinError here means the spawned task panicked — log it rather // than turning a clean web-side shutdown into a process error. + if let Some(task) = mcp_control { + let _ = task.await; + } if let Err(e) = app_grpc.await { tracing::error!(error = %e, "app gRPC server task panicked"); } @@ -617,6 +635,8 @@ impl std::error::Error for CoordinatorError { } } +pub mod managed_boot; +mod managed_bridge; impl From for CoordinatorError { fn from(e: engram_upgrade_drain::DrainError) -> Self { Self::UpgradeDrain(e) diff --git a/crates/engram-coordinator/src/main.rs b/crates/engram-coordinator/src/main.rs index 77621b85c..6de34cf7d 100644 --- a/crates/engram-coordinator/src/main.rs +++ b/crates/engram-coordinator/src/main.rs @@ -77,6 +77,10 @@ struct Cli { #[arg(long, env = "ENGRAM_APP_GRPC_ADDR", default_value = "127.0.0.1:50061")] app_grpc_addr: std::net::SocketAddr, + /// Operator configuration file for the separate MCP credential control listener. + #[arg(long, env = "ENGRAM_MCP_CONTROL_CONFIG")] + mcp_control_config: Option, + /// Comma-separated bearer tokens accepted on the app gRPC surface /// (ADR 0051 §5) — the orchestrator's machine credential, separate /// from `--auth-tokens` (different caller, different blast radius, @@ -374,6 +378,7 @@ async fn main() -> Result<(), CoordinatorError> { .collect(), harness_listen_addr: cli.harness_listen_addr, app_grpc_addr: cli.app_grpc_addr, + mcp_control_config: cli.mcp_control_config.clone(), // Same empty-entry stripping as `auth_tokens` above (clap's // value_delimiter turns an unset env into one "" entry), plus // trim so `a, b` rotation lists don't mint a " b" token. An @@ -767,7 +772,13 @@ async fn main() -> Result<(), CoordinatorError> { // ADR 0013: per-pod gRPC pool. Empty at construction; populated // on `/api/hosts/register` POSTs and (in `run_with_registry`) at // startup from already-registered `hosts` rows. - let host_pool = Arc::new(engram_protocol::grpc_pool::GrpcHostPool::new()); + let mut host_pool = engram_protocol::grpc_pool::GrpcHostPool::new(); + if let Some(tls) = + engram_protocol::host_tls::HostClientTls::from_env().map_err(CoordinatorError::Config)? + { + host_pool = host_pool.with_tls(tls); + } + let host_pool = Arc::new(host_pool); // ADR 0047: let the registry dial hosts this replica has never // fielded a heartbeat from (multi-replica routing read-through). host_registry.set_dialer(host_pool.clone()); diff --git a/crates/engram-coordinator/src/managed_boot.rs b/crates/engram-coordinator/src/managed_boot.rs new file mode 100644 index 000000000..c4bcec101 --- /dev/null +++ b/crates/engram-coordinator/src/managed_boot.rs @@ -0,0 +1,464 @@ +//! Durable managed boot driver. Every transaction ends before a host command. +use crate::{error::ApiError, state::SharedState}; +use engram_core::{ + types::{managed_boot::*, managed_create::digest, mcp_policy::WriteEligibility}, + SandboxError, SessionId, +}; +use engram_protocol::grpc_client::restore_rpc_timeout; +use std::{future::Future, time::Duration}; + +// Expiry leaves the host outcome unknown. Detached commands can still complete; +// the durable phase decides whether to retry the same command or only read it. +pub(crate) async fn host_rpc( + budget: Duration, + call: impl Future>, +) -> Result { + tokio::time::timeout(budget, call).await.map_err(|_| { + SandboxError::Unavailable("managed host request deadline exceeded; outcome unknown".into()) + })? +} + +pub(crate) async fn gate( + state: &SharedState, + id: SessionId, + allow_started: bool, +) -> Result<(), ApiError> { + match state.services.meta.managed_boot_open(id).await? { + None => Ok(()), + Some(true) if allow_started => Ok(()), + _ => Err(ApiError::Unavailable( + "managed session requires its registered boot authority".into(), + )), + } +} +/// Called by the trusted durable preparation driver after physical input verification. +pub async fn register( + state: &SharedState, + request: ManagedBootPrepare, +) -> Result { + Ok(state.services.meta.claim_managed_boot(request).await?) +} + +pub(crate) async fn finalize( + state: &SharedState, + input: ManagedBootFinalizeInput, +) -> Result { + let meta = &state.services.meta; + let sid = input.key.creation.session_id; + let old = meta.read_managed_boot(&input.key).await?; + if let Some(f) = &old.finalization { + if f.prepared.receipt_id == input.prepared_receipt_id + && f.policy == input.policy + && f.native_version == input.native_version + && f.combined_manifest == input.combined_manifest + && f.combined_manifest_digest == input.combined_manifest_digest + { + return Ok(old); + } + return Err(conflict().into()); + } + let prepared = old.prepared.clone().ok_or_else(conflict)?; + if old.phase != ManagedBootPhase::Prepared || prepared.receipt_id != input.prepared_receipt_id { + return Err(conflict().into()); + } + let applied = host_rpc( + restore_rpc_timeout(), + state + .services + .host + .read_managed_policy(input.key.context.target.clone()), + ) + .await?; + if applied.as_ref() != Some(&input.policy) { + return Err(conflict().into()); + } + let native = meta + .read_managed_creation(&input.key.creation) + .await? + .native; + validate_combined_manifest(&native, &input.combined_manifest)?; + let policy = meta.read_mcp_write_policy(sid).await?; + if !policy.current { + return Err(conflict().into()); + } + let selection = meta + .read_managed_boot_selection(&engram_core::types::managed_bridge::ManagedCreationQuery { + identity: input.key.creation.clone(), + native_version: native.version, + native_digest: native.digest.clone(), + }) + .await?; + if selection.aux_ro_drives != old.request.cold.aux_ro_drives + || meta.get_session_harness(sid).await?.as_deref() + != Some(selection.launch.descriptor.name.as_str()) + { + return Err(conflict().into()); + } + let mut session_env = crate::api::sessions::load_session_secrets(state, sid) + .await? + .ok_or_else(conflict)?; + session_env.insert("ENGRAM_SESSION_ID".into(), sid.to_string()); + let mode = (policy.write_eligibility != WriteEligibility::WriteEligible).then_some("plan"); + let resolved = crate::api::sessions::resolve_pinned_harness( + state, + selection.launch, + selection + .aux_ro_drives + .first() + .ok_or_else(conflict)? + .clone(), + sid, + session_env, + mode, + )?; + let mut agent = resolved.agent; + crate::api::sessions::inject_harness_env(state, sid, &mut agent.env).await?; + bind_agent_target(&mut agent, &input.key)?; + agent.session_env.remove("ENGRAM_TOOLS"); + agent.env.insert( + "ENGRAM_TOOLS".into(), + String::from_utf8(input.combined_manifest.clone()).map_err(|_| conflict())?, + ); + let agent_bytes = serde_json::to_vec(&serde_json::to_value(&agent).map_err(|_| conflict())?) + .map_err(|_| conflict())?; + let finalization = ManagedBootFinalization { + prepared, + policy: input.policy, + native_version: input.native_version, + combined_manifest: input.combined_manifest, + combined_manifest_digest: input.combined_manifest_digest, + agent_digest: digest(&agent_bytes), + }; + finalization.validate(&native)?; + ManagedBootFinalize { + finalization: finalization.clone(), + agent, + } + .validate_agent()?; + let sealed = engram_crypto::CredCipher::new(state.services.kek.as_ref()) + .seal(&agent_bytes) + .await + .map_err(|_| ApiError::Unavailable("managed environment sealing unavailable".into()))?; + let sealed_agent = ManagedBootSealedAgent { + wrapped_dek: sealed.wrapped_dek, + nonce: sealed.nonce.to_vec(), + ciphertext: sealed.ciphertext, + key_id: sealed.key_id, + }; + Ok(meta + .advance_managed_boot( + &input.key, + old.revision, + ManagedBootAdvance::Finalize { + finalization: Box::new(finalization), + sealed_agent, + write_policy: policy, + }, + ) + .await?) +} + +/// Bind attach identity before canonical hashing or sealing. Session-wide values +/// must not override the harness-only token, even when the strings happen to match. +fn bind_agent_target( + agent: &mut engram_core::types::sandbox::AgentSpec, + key: &ManagedBootKey, +) -> Result<(), engram_core::MetaError> { + key.validate()?; + let token = engram_harness_proto::attach_token::AttachToken { + sandbox_id: key.context.target.sandbox_id, + binding_epoch: key.context.target.binding_epoch, + }; + let env = token.env(); + if env + .iter() + .any(|(name, _)| agent.session_env.contains_key(name)) + { + return Err(conflict()); + } + agent.binding_epoch = token.binding_epoch; + agent.env.extend(env); + Ok(()) +} + +async fn open_finalize( + state: &SharedState, + row: &ManagedBootRecord, +) -> Result { + let sealed = row.sealed_agent.as_ref().ok_or_else(conflict)?; + let sealed = engram_crypto::SealedCred { + wrapped_dek: sealed.wrapped_dek.clone(), + nonce: sealed.nonce.as_slice().try_into().map_err(|_| conflict())?, + ciphertext: sealed.ciphertext.clone(), + key_id: sealed.key_id.clone(), + }; + let bytes = engram_crypto::CredCipher::new(state.services.kek.as_ref()) + .open(&sealed) + .await + .map_err(|_| ApiError::Unavailable("managed environment unavailable".into()))?; + let command = ManagedBootFinalize { + finalization: row.finalization.clone().ok_or_else(conflict)?, + agent: serde_json::from_slice(&bytes).map_err(|_| conflict())?, + }; + command.validate_agent()?; + Ok(command) +} + +/// Each call performs at most one phase. A failed host response leaves authority closed. +pub async fn drive(state: &SharedState, key: &ManagedBootKey) -> Result<(), ApiError> { + drive_with_timeout(state, key, restore_rpc_timeout()).await +} + +async fn drive_with_timeout( + state: &SharedState, + key: &ManagedBootKey, + rpc_timeout: Duration, +) -> Result<(), ApiError> { + let row = state.services.meta.read_managed_boot(key).await?; + if matches!( + row.phase, + ManagedBootPhase::Prepared | ManagedBootPhase::Finalizing | ManagedBootPhase::Finalized + ) { + // Final policy installation can retire a prepared owner. Preserve its + // exact nonrunning proof even when no finalization/start reply succeeds. + if let ManagedBootHostState::Retired { receipt } = host_rpc( + rpc_timeout, + state.services.host.read_managed_boot(key.clone()), + ) + .await? + { + state + .services + .meta + .advance_managed_boot(key, row.revision, ManagedBootAdvance::Retired(receipt)) + .await?; + return Ok(()); + } + } + match row.phase { + ManagedBootPhase::Claimed => { + let receipt = host_rpc( + rpc_timeout, + state + .services + .host + .prepare_managed_boot(row.request.clone()), + ) + .await?; + state + .services + .meta + .advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::Prepared(Box::new(receipt)), + ) + .await?; + } + ManagedBootPhase::Finalizing => { + let command = open_finalize(state, &row).await?; + let receipt = host_rpc( + rpc_timeout, + state.services.host.finalize_managed_boot(command), + ) + .await?; + state + .services + .meta + .advance_managed_boot(key, row.revision, ManagedBootAdvance::Finalized(receipt)) + .await?; + } + ManagedBootPhase::Finalized => { + let receipt = row.receipt.clone().ok_or_else(conflict)?; + let start = ManagedBootStart { + start_id: digest(format!("{}:{}", key.boot_attempt, receipt.receipt_id).as_bytes()), + receipt, + }; + let issued = state + .services + .meta + .advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::IssueStart(start.clone()), + ) + .await?; + // Only this successful CAS holder can issue start. Failure or lost reply + // is resolved by the read branch, never by another start command. + let observed = + host_rpc(rpc_timeout, state.services.host.start_managed_boot(start)).await?; + confirm_started(state, &issued, observed).await?; + } + ManagedBootPhase::StartIssued => { + let observed = host_rpc( + rpc_timeout, + state.services.host.read_managed_boot(key.clone()), + ) + .await?; + confirm_started(state, &row, observed).await?; + } + ManagedBootPhase::Started => { + let observed = match host_rpc( + rpc_timeout, + state.services.host.read_managed_boot(key.clone()), + ) + .await + { + Ok(observed) => observed, + Err(error) => { + state + .services + .meta + .advance_managed_boot(key, row.revision, ManagedBootAdvance::Interrupt) + .await?; + return Err(error.into()); + } + }; + match observed { + ManagedBootHostState::Started { intent } if Some(&intent) == row.start.as_ref() => { + } + ManagedBootHostState::Retired { receipt } => { + state + .services + .meta + .advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::Retired(receipt), + ) + .await?; + } + _ => { + state + .services + .meta + .advance_managed_boot(key, row.revision, ManagedBootAdvance::Interrupt) + .await?; + } + } + } + ManagedBootPhase::Prepared | ManagedBootPhase::Interrupted => {} + } + Ok(()) +} +async fn confirm_started( + state: &SharedState, + row: &ManagedBootRecord, + observed: ManagedBootHostState, +) -> Result<(), ApiError> { + match observed { + ManagedBootHostState::Retired { receipt } => { + state + .services + .meta + .advance_managed_boot( + &row.request.key, + row.revision, + ManagedBootAdvance::Retired(receipt), + ) + .await?; + Ok(()) + } + ManagedBootHostState::Started { intent } if Some(&intent) == row.start.as_ref() => { + state + .services + .meta + .advance_managed_boot( + &row.request.key, + row.revision, + ManagedBootAdvance::Started(intent), + ) + .await?; + state.outbox_wake.notify_one(); + Ok(()) + } + ManagedBootHostState::StartPending { intent } if Some(&intent) == row.start.as_ref() => { + Ok(()) + } + _ => Err(conflict().into()), + } +} +pub async fn run_once(state: &SharedState) -> Result<(), ApiError> { + run_once_with_timeout(state, restore_rpc_timeout()).await +} + +async fn run_once_with_timeout(state: &SharedState, rpc_timeout: Duration) -> Result<(), ApiError> { + crate::managed_bridge::run_once(state).await?; + let mut after = None; + loop { + let keys = state.services.meta.list_managed_boots(after, 128).await?; + if keys.is_empty() { + break; + } + for key in keys { + after = Some(key.creation.session_id); + if let Err(error) = drive_with_timeout(state, &key, rpc_timeout).await { + tracing::debug!(session_id=%key.creation.session_id,error=%error,"managed boot remains closed"); + } + } + } + Ok(()) +} +pub(crate) fn spawn(state: SharedState) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + loop { + if let Err(error) = run_once(&state).await { + tracing::warn!(error=%error,"managed boot scan failed"); + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + }) +} +/// Explicit trusted recovery action. Durable capture identity precedes host I/O. +pub async fn capture( + state: &SharedState, + request: ManagedCaptureRequest, +) -> Result { + let row = state + .services + .meta + .get_managed_boot(&request.key.creation) + .await? + .ok_or_else(conflict)?; + if row.request.key != request.key { + return state + .services + .meta + .reserve_managed_capture(&request) + .await? + .ok_or_else(|| conflict().into()); + } + if row.phase != ManagedBootPhase::Interrupted { + state + .services + .meta + .advance_managed_boot(&request.key, row.revision, ManagedBootAdvance::Interrupt) + .await?; + } + if let Some(authority) = state + .services + .meta + .reserve_managed_capture(&request) + .await? + { + return Ok(authority); + } + let authority = host_rpc( + restore_rpc_timeout(), + state.services.host.capture_managed_boot(request.clone()), + ) + .await?; + if authority.request != request { + return Err(conflict().into()); + } + state + .services + .meta + .record_managed_capture(&authority) + .await?; + Ok(authority) +} + +#[cfg(test)] +#[path = "managed_boot_tests.rs"] +pub(crate) mod tests; diff --git a/crates/engram-coordinator/src/managed_boot_tests.rs b/crates/engram-coordinator/src/managed_boot_tests.rs new file mode 100644 index 000000000..e0767abd2 --- /dev/null +++ b/crates/engram-coordinator/src/managed_boot_tests.rs @@ -0,0 +1,1895 @@ +//! Synthetic host fault tests. Public admission and actual host execution are not exercised. +use super::*; +use async_trait::async_trait; +use engram_core::types::sandbox::ExecRequest as HostExecRequest; +use engram_core::{ + traits::{HostClient, MetadataStore, SessionFence}, + types::{ + egress::SessionEgressPolicy, managed_cold::*, managed_create::*, managed_deny::*, + mcp_policy::*, runtime_spec::RuntimeSpec, sandbox::*, snapshot::SnapshotMetadata, + }, + HostId, SandboxError, SandboxId, +}; +use engram_sim::{ManualClock, MemBlobStorage, SimEntropy, SimMetadataStore}; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, Mutex, +}; +struct LossyHost { + prepare_stall: Mutex>, + start_stall: AtomicBool, + read_stall: AtomicBool, + chunks: engram_chunk_store::ChunkStore, + captures: AtomicUsize, + state: Mutex>, + starts: AtomicUsize, + policy_calls: AtomicUsize, + policy: Mutex>, + prepared: ManagedBootPreparedReceipt, + apply_mode: AtomicUsize, + entered: tokio::sync::Notify, + release: tokio::sync::Semaphore, +} +#[async_trait] +impl HostClient for LossyHost { + fn harness_dial(&self) -> engram_core::traits::HarnessDial { + engram_core::traits::HarnessDial::Vsock + } + async fn capture_managed_boot( + &self, + request: ManagedCaptureRequest, + ) -> Result { + let prepared = match self.state.lock().unwrap().clone() { + Some(ManagedBootHostState::Captured { authority }) if authority.request == request => { + return Ok(authority) + } + Some(ManagedBootHostState::Prepared { receipt }) + if receipt.request.key == request.key => + { + *receipt + } + _ => return Err(SandboxError::Unavailable("source unavailable".into())), + }; + let manifest = self + .chunks + .get_manifest(prepared.request.cold.disk.manifest) + .await + .map_err(|e| SandboxError::Vm(e.to_string().into()))?; + self.chunks + .put_manifest(request.disk_manifest(), &manifest) + .await + .map_err(|e| SandboxError::Vm(e.to_string().into()))?; + let mut authority = ManagedCaptureAuthority { + version: 1, + disk_manifest: request.disk_manifest(), + request, + host_incarnation: prepared.host_incarnation, + disk_sha256: prepared.request.cold.disk.sha256, + disk_size_bytes: prepared.request.cold.disk.size_bytes, + image: prepared.request.cold.image, + kernel_sha256: prepared.request.cold.kernel_sha256, + aux_ro_drives: prepared.request.cold.aux_ro_drives, + consistency: ManagedDiskConsistency::CrashConsistentDiskCutV1, + source_state: ManagedCapturedSourceState::Destroyed, + record_digest: String::new(), + }; + authority.record_digest = authority.digest().unwrap(); + self.captures.fetch_add(1, Ordering::SeqCst); + *self.state.lock().unwrap() = Some(ManagedBootHostState::Captured { authority }); + Err(SandboxError::Unavailable( + "capture reply lost after artifact commit".into(), + )) + } + async fn read_managed_cold_assets( + &self, + request: engram_core::types::managed_assets::ManagedColdAssetsRequest, + ) -> Result { + request + .validate() + .map_err(|e| SandboxError::Vm(e.to_string().into()))?; + Ok( + engram_core::types::managed_assets::ManagedColdAssetsReceipt { + request, + host_incarnation: "host".into(), + backend: engram_core::types::managed_assets::ManagedColdBackend::Firecracker, + kernel_sha256: "c".repeat(64), + disk_sha256: "b".repeat(64), + disk_size_bytes: 4096, + }, + ) + } + async fn prepare_managed_boot( + &self, + request: ManagedBootPrepare, + ) -> Result { + if *self.prepare_stall.lock().unwrap() == Some(request.key.creation.session_id) { + return std::future::pending().await; + } + if let Some(ManagedBootHostState::Prepared { receipt }) = + self.state.lock().unwrap().as_ref() + { + if receipt.request == request { + return Ok(*receipt.clone()); + } + } + request + .spec() + .map_err(|e| SandboxError::Vm(e.to_string().into()))?; + let receipt = ManagedBootPreparedReceipt { + request, + host_incarnation: "host".into(), + network_identity: digest(b"network"), + guest_ip: "192.0.2.2".parse().unwrap(), + receipt_id: digest(b"prepared"), + }; + *self.state.lock().unwrap() = Some(ManagedBootHostState::Prepared { + receipt: Box::new(receipt), + }); + Err(SandboxError::Unavailable("reply lost after prepare".into())) + } + async fn read_managed_policy( + &self, + _: ManagedPolicyTarget, + ) -> Result, SandboxError> { + self.policy_calls.fetch_add(1, Ordering::SeqCst); + Ok(self.policy.lock().unwrap().clone()) + } + async fn apply_managed_policy( + &self, + request: ManagedPolicyApply, + ) -> Result { + self.policy_calls.fetch_add(1, Ordering::SeqCst); + let mode = self.apply_mode.load(Ordering::SeqCst); + if mode == 0 { + return Err(SandboxError::Unavailable("policy unavailable".into())); + } + assert_eq!(*self.policy.lock().unwrap(), request.expected); + assert_eq!( + request.context.target, + self.prepared.request.key.context.target + ); + assert!( + request.context.operator_generation + > self.prepared.request.key.context.operator_generation + ); + assert_eq!(request.egress.guest_ip, self.prepared.guest_ip); + if request.expected.is_none() { + assert!(request.context.binding_hash.is_none()); + } + self.entered.notify_one(); + if mode == 2 { + let _permit = self.release.acquire().await.unwrap(); + } + let retained = request.egress.managed_deny.unwrap(); + assert!(retained.contains(&self.prepared.request.bootstrap_policy)); + *self.state.lock().unwrap() = Some(ManagedBootHostState::Retired { + receipt: Box::new(ManagedBootRetiredReceipt { + prepared: self.prepared.clone(), + retained_policy: retained, + receipt_id: digest(b"operator retired"), + }), + }); + Err(SandboxError::Unavailable( + "reply lost after retirement".into(), + )) + } + async fn start_managed_boot( + &self, + start: ManagedBootStart, + ) -> Result { + self.starts.fetch_add(1, Ordering::SeqCst); + *self.state.lock().unwrap() = Some(ManagedBootHostState::Started { intent: start }); + if self.start_stall.load(Ordering::SeqCst) { + return std::future::pending().await; + } + Err(SandboxError::Unavailable("reply lost after spawn".into())) + } + async fn read_managed_boot( + &self, + _: ManagedBootKey, + ) -> Result { + if self.read_stall.load(Ordering::SeqCst) { + return std::future::pending().await; + } + self.state + .lock() + .unwrap() + .clone() + .ok_or_else(|| SandboxError::Unavailable("host state unavailable".into())) + } + async fn create(&self, _spec: SandboxSpec) -> Result { + unreachable!("exec-core tests never create sandboxes") + } + + async fn destroy(&self, _id: SandboxId, _fence: SessionFence) -> Result<(), SandboxError> { + unreachable!("exec-core tests never destroy sandboxes") + } + + async fn list(&self) -> Result, SandboxError> { + unreachable!("exec-core tests never list sandboxes") + } + + async fn probe_sandbox(&self, _id: SandboxId) -> Result { + unreachable!("exec-core tests never probe sandboxes") + } + + async fn exec_stream( + &self, + id: SandboxId, + cmd: HostExecRequest, + ) -> Result { + let _ = (id, cmd); + unreachable!("readiness never executes commands") + } + + async fn snapshot( + &self, + _id: SandboxId, + _fence: SessionFence, + ) -> Result { + unreachable!("exec-core tests never snapshot sandboxes") + } + + async fn restore( + &self, + _metadata: SnapshotMetadata, + _fence: SessionFence, + ) -> Result { + unreachable!("exec-core tests never restore sandboxes") + } + + async fn start_agent( + &self, + _id: SandboxId, + _agent: AgentSpec, + _policy: SessionEgressPolicy, + _fence: SessionFence, + ) -> Result<(), SandboxError> { + unreachable!("policy control never starts guests") + } + + async fn guest_ip(&self, _id: SandboxId) -> Option { + unreachable!("boot scanner does not resolve another network") + } + + async fn bind_session( + &self, + _session_id: SessionId, + _sandbox_id: SandboxId, + _binding_epoch: u64, + ) -> Result<(), engram_core::SandboxError> { + Err(SandboxError::Vm("bind rejected".into())) + } + + async fn unbind_session(&self, _session_id: SessionId) {} + + async fn send_prompt( + &self, + _sandbox_id: SandboxId, + _prompt_id: String, + _text: String, + _mode: Option, + ) -> Result<(), SandboxError> { + unreachable!("exec-core tests never send prompts") + } +} + +async fn fixture() -> ( + SharedState, + Arc, + Arc, + ManagedBootRecord, +) { + let clock = ManualClock::new(); + let entropy = Arc::new(SimEntropy::seeded(700)); + let meta = SimMetadataStore::new(clock.clone(), entropy.clone()); + let sid = SessionId::from(uuid::Uuid::from_u128(700)); + let sbx = SandboxId::from(uuid::Uuid::from_u128(701)); + let host_id = HostId::from(uuid::Uuid::from_u128(702)); + let policy = McpPolicyCreation { + actor_id: "owner".into(), + input_id: "create".into(), + parent_session_id: None, + eligibility: WriteEligibility::ReadOnly, + }; + let creation = ManagedCreation { + identity: ManagedCreateIdentity { + session_id: sid, + task_id: "task".into(), + policy: policy.clone(), + input_digest: digest(b"input"), + }, + native: FrozenNativeManifest { + version: 1, + bytes: b"[]".to_vec(), + digest: digest(b"[]"), + }, + }; + let ws = engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: None, + managed_creation: Some(creation.clone()), + managed_initial_prompt: None, + mcp_policy: Some(policy), + session_id: sid, + spec: engram_core::types::SessionSpec { + image: "test.invalid/managed:latest".into(), + mode: Default::default(), + }, + mem_budget_mib: 128, + cpu_budget_vcpus: 1, + sealed_secrets: None, + capabilities: vec![], + integration_policy_json: None, + runtime_spec: RuntimeSpec::new(vec![], Some("claude".into()), None, vec![]), + oauth_binding: None, + }; + engram_testkit::operator::publish_empty(meta.as_ref()).await; + meta.reserve_and_persist_create(ws, &[], 0).await.unwrap(); + let disk = engram_core::types::manifest::ManifestRef { + manifest_id: uuid::Uuid::from_u128(703), + version: 1, + }; + let drives = (0..3) + .map(|i| AuxRoDrive { + drive_id: AuxRoDrive::slot_drive_id(i), + guest_mount: AuxRoDrive::slot_guest_mount(i), + fs_type: "squashfs".into(), + sha256: Some("a".repeat(64)), + }) + .collect::>(); + let spec = SandboxSpec { + image: "test.invalid/managed:latest".into(), + rootfs_source: Some("disk.ext4".into()), + image_uri: None, + rootfs_manifest: Some(disk), + cpu: CpuLimit { vcpus: 1 }, + memory: MemoryLimit { max_mib: 128 }, + disk: DiskLimit { max_gib: 1 }, + ttl: None, + env: Default::default(), + workdir: None, + network: Default::default(), + aux_ro_drives: drives.clone(), + swap_mib: None, + swap_source: None, + swap_manifest: None, + }; + let key = ManagedBootKey { + creation: creation.identity, + context: ManagedPolicyContext { + target: ManagedPolicyTarget { + session_id: sid, + sandbox_id: sbx, + host_id, + binding_epoch: 1, + }, + operator_generation: 1, + operator_identity: engram_testkit::operator::candidate().identity().unwrap(), + binding_hash: None, + }, + boot_attempt: "boot-one".into(), + native_digest: creation.native.digest.clone(), + }; + let request = ManagedBootPrepare { + key: key.clone(), + capture_authority: None, + bootstrap_policy: ManagedDenyPolicy::new(1, ["tools.example".into()]).unwrap(), + sandbox_spec_json: serde_json::to_string(&spec).unwrap(), + cold: ManagedColdPrepare { + version: 1, + session_id: sid, + sandbox_id: sbx, + boot_attempt: key.boot_attempt.clone(), + image: spec.image, + disk: ManagedColdDisk { + manifest: disk, + sha256: "b".repeat(64), + size_bytes: 4096, + source: ManagedColdSource::FreshImage { + base_manifest: engram_core::types::manifest::ManifestRef { + manifest_id: uuid::Uuid::from_u128(704), + version: 1, + }, + }, + }, + kernel_sha256: "c".repeat(64), + aux_ro_drives: drives, + }, + }; + request.spec().unwrap(); + // Explicit store fixture: exercise scanner authority over a durable placement. + // Actual publication and host receipts have separate integration gates. + meta.with_db_mut(|db| { + db.managed_operator_gate = Some((1, key.context.operator_identity.clone(), true)); + let s = db.sessions.get_mut(&sid).unwrap(); + s.session.status = engram_core::types::SessionState::Created; + s.session.host_id = Some(host_id); + s.session.sandbox_id = Some(sbx); + s.binding_epoch = 1; + db.managed_boots + .insert(sid, ManagedBootRecord::claimed(request.clone())); + }); + let prepared = ManagedBootPreparedReceipt { + request, + host_incarnation: "host".into(), + network_identity: digest(b"network"), + guest_ip: "192.0.2.2".parse().unwrap(), + receipt_id: digest(b"prepared"), + }; + let row = meta + .advance_managed_boot( + &key, + 1, + ManagedBootAdvance::Prepared(Box::new(prepared.clone())), + ) + .await + .unwrap(); + let mut context = key.context.clone(); + context.binding_hash = Some(digest(b"binding")); + let applied = ManagedAppliedState::new( + context.clone(), + prepared.request.bootstrap_policy.clone(), + "host".into(), + "proxy".into(), + Some(ManagedNetworkProof { + target: context.target, + guest_ip: prepared.guest_ip, + network_identity: prepared.network_identity.clone(), + }), + ) + .unwrap(); + let write_policy = meta.read_mcp_write_policy(sid).await.unwrap(); + let f = ManagedBootFinalization { + prepared: prepared.clone(), + policy: applied, + native_version: 1, + combined_manifest: b"[]".to_vec(), + combined_manifest_digest: digest(b"[]"), + agent_digest: digest(b"fixture agent"), + }; + let row = meta + .advance_managed_boot( + &key, + row.revision, + ManagedBootAdvance::Finalize { + finalization: Box::new(f.clone()), + write_policy, + sealed_agent: ManagedBootSealedAgent { + wrapped_dek: vec![1], + nonce: vec![2; 12], + ciphertext: vec![3], + key_id: "fixture".into(), + }, + }, + ) + .await + .unwrap(); + let receipt = ManagedBootFinalReceipt { + key: key.clone(), + prepared_receipt_id: prepared.receipt_id, + finalization_digest: f.digest().unwrap(), + host_incarnation: prepared.host_incarnation, + network_identity: prepared.network_identity, + receipt_id: digest(b"final"), + }; + let row = meta + .advance_managed_boot(&key, row.revision, ManagedBootAdvance::Finalized(receipt)) + .await + .unwrap(); + let blob = Arc::new(MemBlobStorage::new()); + let host = Arc::new(LossyHost { + prepare_stall: Mutex::new(None), + start_stall: AtomicBool::new(false), + read_stall: AtomicBool::new(false), + chunks: engram_chunk_store::ChunkStore::new(blob.clone()), + captures: AtomicUsize::new(0), + state: Mutex::new(Some(ManagedBootHostState::Finalized { + receipt: row.receipt.clone().unwrap(), + })), + starts: AtomicUsize::new(0), + policy_calls: AtomicUsize::new(0), + policy: Mutex::new(Some(row.finalization.as_ref().unwrap().policy.clone())), + prepared: row.prepared.clone().unwrap(), + apply_mode: AtomicUsize::new(0), + entered: Default::default(), + release: tokio::sync::Semaphore::new(0), + }); + let services = crate::Services { + meta: meta.clone(), + host: host.clone(), + host_pool: Arc::new(engram_protocol::grpc_pool::GrpcHostPool::new()), + secrets: Arc::new(engram_secrets_dev::InMemorySecretStore::new()), + kek: Arc::new(engram_crypto::EnvVarKeyProvider::from_bytes( + [0; 32], "fixture", + )), + oci: Arc::new(engram_oci::OciClient::new(Arc::new( + engram_oci::AnonymousResolver, + ))), + auth_resolver: Arc::new(engram_oci::AnonymousResolver), + blob: blob.clone(), + chunk_store: engram_chunk_store::ChunkStore::new(blob), + materialize_dir: None, + clock, + entropy, + }; + ( + Arc::new(crate::AppState::new( + crate::CoordinatorConfig::default(), + services, + )), + meta, + host, + row, + ) +} +/// Shared prepared authority for the policy control regression tests. +/// Keep the claimed request, exact validated prepared receipt, revision, and +/// created physical placement. Remove only the later finalization/start fields; +/// the synthetic host is replaced by the caller and supplies no authority. +pub(crate) async fn prepared_policy_fixture( +) -> (SharedState, Arc, ManagedBootRecord) { + let (state, meta, _host, mut row) = fixture().await; + row.phase = ManagedBootPhase::Prepared; + row.revision = 2; + row.finalization = None; + row.sealed_agent = None; + row.write_policy = None; + row.receipt = None; + row.start = None; + meta.with_db_mut(|db| { + db.managed_boots + .insert(row.request.key.creation.session_id, row.clone()); + }); + (state, meta, row) +} + +#[tokio::test] +async fn managed_boot_lost_start_reply_is_read_only_reconciled() { + let (state, meta, host, row) = fixture().await; + let key = &row.request.key; + assert!(drive(&state, key).await.is_err()); + assert_eq!(host.starts.load(Ordering::SeqCst), 1); + assert_eq!( + meta.read_managed_boot(key).await.unwrap().phase, + ManagedBootPhase::StartIssued + ); + assert!(gate(&state, key.creation.session_id, true).await.is_err()); + // A new scanner step has no in-memory authority and recovers only the exact intent. + run_once(&state).await.unwrap(); + assert_eq!( + meta.read_managed_boot(key).await.unwrap().phase, + ManagedBootPhase::Started + ); + run_once(&state).await.unwrap(); + assert_eq!(host.starts.load(Ordering::SeqCst), 1); + gate(&state, key.creation.session_id, true).await.unwrap(); + assert!(gate(&state, key.creation.session_id, false).await.is_err()); + let retired = ManagedBootRetiredReceipt { + prepared: row.prepared.unwrap(), + retained_policy: ManagedDenyPolicy::new(2, ["tools.example".into(), "next.example".into()]) + .unwrap(), + receipt_id: digest(b"retired"), + }; + *host.state.lock().unwrap() = Some(ManagedBootHostState::Retired { + receipt: Box::new(retired.clone()), + }); + run_once(&state).await.unwrap(); + let closed = meta.read_managed_boot(key).await.unwrap(); + assert_eq!(closed.phase, ManagedBootPhase::Interrupted); + assert_eq!(closed.retired, Some(retired)); + assert!(gate(&state, key.creation.session_id, true).await.is_err()); + assert_eq!(host.starts.load(Ordering::SeqCst), 1); +} +#[tokio::test] +async fn managed_boot_missing_or_changed_host_state_never_reissues_start() { + let (state, meta, host, row) = fixture().await; + let key = &row.request.key; + let start = ManagedBootStart { + receipt: row.receipt.unwrap(), + start_id: digest(b"start"), + }; + meta.advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::IssueStart(start.clone()), + ) + .await + .unwrap(); + // Crash after durable intent but before its RPC: absence cannot authorize a new spawn. + *host.state.lock().unwrap() = Some(ManagedBootHostState::Missing { key: key.clone() }); + assert!(drive(&state, key).await.is_err()); + let mut wrong = start.clone(); + wrong.start_id = digest(b"different"); + *host.state.lock().unwrap() = Some(ManagedBootHostState::Started { intent: wrong }); + assert!(drive(&state, key).await.is_err()); + *host.state.lock().unwrap() = Some(ManagedBootHostState::StartPending { intent: start }); + drive(&state, key).await.unwrap(); + assert_eq!(host.starts.load(Ordering::SeqCst), 0); + assert_eq!( + meta.read_managed_boot(key).await.unwrap().phase, + ManagedBootPhase::StartIssued + ); + assert!(gate(&state, key.creation.session_id, true).await.is_err()); +} + +#[tokio::test] +async fn managed_boot_generic_entry_points_stay_closed() { + let (state, meta, host, row) = fixture().await; + let sid = row.request.key.creation.session_id; + let sbx = row.request.key.context.target.sandbox_id; + let fence = SessionFence::new(sid, 1); + assert!(crate::api::snapshot::ensure_active(&state, sid) + .await + .is_err()); + assert!(crate::api::snapshot::snapshot_core(&state, sid) + .await + .is_err()); + assert!(crate::api::snapshot::resume_core(&state, sid) + .await + .is_err()); + assert!( + crate::api::snapshot::reattach_harness_in_place(&state, sid, sbx, fence) + .await + .is_err() + ); + assert!(crate::teleport::admit_for_rpc(&state, sid, None) + .await + .is_err()); + crate::api::prompt::send_prompt_core( + &state, + sid, + "pending".into(), + "later".into(), + None, + crate::api::prompt::PromptDelivery::Enqueue, + ) + .await + .unwrap(); + assert!(meta + .outbox_get("pending") + .await + .unwrap() + .unwrap() + .delivered_at + .is_none()); + assert!(!meta + .op_pending_exists(sid, engram_core::types::session_op::OpKind::Deliver) + .await + .unwrap()); + let registry = crate::host_registry::HostRegistry::new(meta.clone()); + registry.register(row.request.key.context.target.host_id, host.clone()); + let exec = HostExecRequest { + command: vec!["true".into()], + stdin: None, + env: Default::default(), + workdir: None, + timeout: None, + exec_id: None, + stdout_offset: None, + stderr_offset: None, + wake: None, + }; + assert!(registry.exec_stream(sbx, exec).await.is_err()); + assert!(registry + .send_prompt(sbx, "p".into(), "prompt".into(), None) + .await + .is_err()); + assert!(registry.snapshot(sbx, fence).await.is_err()); + assert!(registry.bind_session(sid, sbx, 1).await.is_err()); + assert_eq!(host.starts.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn managed_boot_prestart_retirement_is_durably_observed() { + for phase in [ + ManagedBootPhase::Prepared, + ManagedBootPhase::Finalizing, + ManagedBootPhase::Finalized, + ] { + let (state, meta, host, row) = fixture().await; + let key = &row.request.key; + // Store fixture selects the crash boundary; no actual host proof is minted. + meta.with_db_mut(|db| { + db.managed_boots + .get_mut(&key.creation.session_id) + .unwrap() + .phase = phase + }); + let receipt = ManagedBootRetiredReceipt { + prepared: row.prepared.clone().unwrap(), + retained_policy: row.finalization.as_ref().unwrap().policy.policy.clone(), + receipt_id: digest(b"prestart-retired"), + }; + *host.state.lock().unwrap() = Some(ManagedBootHostState::Retired { + receipt: Box::new(receipt.clone()), + }); + run_once(&state).await.unwrap(); + let closed = meta.read_managed_boot(key).await.unwrap(); + assert_eq!(closed.phase, ManagedBootPhase::Interrupted); + assert_eq!(closed.retired, Some(receipt)); + assert_eq!(host.starts.load(Ordering::SeqCst), 0); + } +} + +#[tokio::test] +async fn managed_boot_attach_environment_is_exact_and_sealed() { + let (state, _, _, mut row) = fixture().await; + let target = &row.request.key.context.target; + let mut agent = AgentSpec { + argv: vec!["harness".into()], + env: [ + ("ENGRAM_TOOLS".into(), "[]".into()), + ("ENGRAM_SANDBOX_ID".into(), target.sandbox_id.to_string()), + ( + "ENGRAM_BINDING_EPOCH".into(), + target.binding_epoch.to_string(), + ), + ] + .into_iter() + .collect(), + session_env: Default::default(), + binding_epoch: target.binding_epoch, + host_ca_pem: None, + }; + // Producer replaces stale harness-local values with the durable claim. + agent.env.insert("ENGRAM_SANDBOX_ID".into(), "stale".into()); + agent.env.insert("ENGRAM_BINDING_EPOCH".into(), "0".into()); + bind_agent_target(&mut agent, &row.request.key).unwrap(); + assert_eq!( + agent.env["ENGRAM_SANDBOX_ID"], + target.sandbox_id.to_string() + ); + assert_eq!( + agent.env["ENGRAM_BINDING_EPOCH"], + target.binding_epoch.to_string() + ); + for name in ["ENGRAM_SANDBOX_ID", "ENGRAM_BINDING_EPOCH"] { + let mut override_agent = agent.clone(); + override_agent + .session_env + .insert(name.into(), agent.env[name].clone()); + assert!(bind_agent_target(&mut override_agent, &row.request.key).is_err()); + } + let mut command = ManagedBootFinalize { + finalization: row.finalization.clone().unwrap(), + agent, + }; + let canonical = + |agent: &AgentSpec| serde_json::to_vec(&serde_json::to_value(agent).unwrap()).unwrap(); + command.finalization.agent_digest = digest(&canonical(&command.agent)); + command.validate_agent().unwrap(); + for name in ["ENGRAM_SANDBOX_ID", "ENGRAM_BINDING_EPOCH"] { + for change in 0..3 { + let mut wrong = command.clone(); + match change { + 0 => { + wrong.agent.env.remove(name); + } + 1 => { + wrong.agent.env.insert(name.into(), "wrong".into()); + } + _ => { + wrong + .agent + .session_env + .insert(name.into(), command.agent.env[name].clone()); + } + } + // Recompute the hash: valid integrity cannot authorize an incorrect target. + wrong.finalization.agent_digest = digest(&canonical(&wrong.agent)); + assert!( + wrong.validate_agent().is_err(), + "accepted invalid {name}, case {change}" + ); + } + } + let cipher = engram_crypto::CredCipher::new(state.services.kek.as_ref()); + let sealed = cipher.seal(&canonical(&command.agent)).await.unwrap(); + row.finalization = Some(command.finalization); + row.sealed_agent = Some(ManagedBootSealedAgent { + wrapped_dek: sealed.wrapped_dek, + nonce: sealed.nonce.to_vec(), + ciphertext: sealed.ciphertext, + key_id: sealed.key_id, + }); + let opened = open_finalize(&state, &row).await.unwrap(); + assert_eq!(opened.agent.env, command.agent.env); + assert_eq!(opened.agent.session_env, command.agent.session_env); +} + +#[tokio::test] +async fn managed_boot_event_entry_points_reject_interrupted_authority() { + use crate::state::{emit_harness_event, SessionEvent}; + use engram_harness_proto::HarnessEvent; + use engram_host_agent::harness::EventDelivery; + let (state, meta, _, row) = fixture().await; + let key = row.request.key.clone(); + let sid = key.creation.session_id; + let sbx = key.context.target.sandbox_id; + let start = ManagedBootStart { + receipt: row.receipt.unwrap(), + start_id: digest(b"event-start"), + }; + let row = meta + .advance_managed_boot( + &key, + row.revision, + ManagedBootAdvance::IssueStart(start.clone()), + ) + .await + .unwrap(); + let row = meta + .advance_managed_boot(&key, row.revision, ManagedBootAdvance::Started(start)) + .await + .unwrap(); + let now = state.services.clock.now_utc(); + let delivery = Some(EventDelivery { + binding_epoch: 1, + incarnation: 1, + seq: 1, + }); + let run = || HarnessEvent::RunStarted { + run_id: "old-run".into(), + prompt_summary: None, + prompt_id: None, + }; + emit_harness_event(&state, sid, sbx, run(), now, delivery) + .await + .unwrap(); + // Replay through the other entry point uses the same transactional key. + state + .harness_hub + .emit_external(sid, sbx, run(), delivery) + .await + .unwrap(); + let pending = engram_core::types::outbox::OutboxRow { + prompt_id: "pending-boundary".into(), + session_id: sid, + kind: engram_core::types::outbox::OutboxKind::Prompt, + payload: serde_json::json!({"text":"pending"}), + created_at: now, + not_before: now, + attempts: 0, + delivered_at: None, + acked_at: None, + }; + meta.outbox_enqueue(&pending).await.unwrap(); + let mut live = state.events.subscribe(sid); + meta.advance_managed_boot(&key, row.revision, ManagedBootAdvance::Interrupt) + .await + .unwrap(); + let late = || HarnessEvent::RunStarted { + run_id: "late-run".into(), + prompt_summary: None, + prompt_id: Some(pending.prompt_id.clone()), + }; + let delivery = Some(EventDelivery { + binding_epoch: 1, + incarnation: 1, + seq: 2, + }); + assert!(emit_harness_event(&state, sid, sbx, late(), now, delivery) + .await + .is_err()); + assert!(state + .harness_hub + .emit_external(sid, sbx, late(), delivery) + .await + .is_err()); + assert!(live.try_recv().is_err()); + assert!(meta + .outbox_get(&pending.prompt_id) + .await + .unwrap() + .unwrap() + .acked_at + .is_none()); + let rows = meta.list_session_events_since(sid, -1, 100).await.unwrap(); + assert_eq!(rows.iter().filter(|r| r.kind == "run_started").count(), 1); + let interrupted = rows + .into_iter() + .find(|r| r.kind == "run_interrupted") + .unwrap(); + assert!( + matches!(serde_json::from_value::(interrupted.payload).unwrap(),SessionEvent::HarnessRunInterrupted {run_id,..} if run_id=="old-run") + ); +} + +#[tokio::test] +async fn managed_boot_interruption_publishes_through_pg_listener() { + use crate::state::SessionEvent; + use engram_core::types::host::{HostCapacity, HostRecord, HostStatus}; + use std::time::Duration; + let Some(db) = engram_testkit::pg::fresh_db().await else { + return; + }; + let (state, sim, _, row) = fixture().await; + let key = &row.request.key; + let sid = key.creation.session_id; + let target = &key.context.target; + let now = state.services.clock.now_utc(); + let meta = Arc::new(db.store.clone()); + engram_testkit::operator::publish_empty(meta.as_ref()).await; + let creation = sim.read_managed_creation(&key.creation).await.unwrap(); + meta.reserve_and_persist_create( + engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: None, + session_id: sid, + spec: engram_core::types::SessionSpec { + image: row.request.cold.image.clone(), + mode: Default::default(), + }, + mem_budget_mib: 128, + cpu_budget_vcpus: 1, + sealed_secrets: None, + capabilities: vec![], + integration_policy_json: None, + runtime_spec: RuntimeSpec::new(vec![], Some("claude".into()), None, vec![]), + oauth_binding: None, + mcp_policy: Some(creation.identity.policy.clone()), + managed_creation: Some(creation), + managed_initial_prompt: None, + }, + &[], + 0, + ) + .await + .unwrap(); + meta.upsert_host(HostRecord { + id: target.host_id, + hostname: "managed-host".into(), + cloud_metadata: Default::default(), + capacity: HostCapacity { + total_gb: 0, + used_gb: 0, + total_mib: 1024, + used_mib: 0, + running_sandboxes: 0, + }, + utilization: Default::default(), + status: HostStatus::Ready, + last_heartbeat_at: now, + host_addr: None, + ready_images: vec![], + current_bundles: vec![], + sandbox_bundles: vec![], + cordoned: false, + cordon_owner: None, + cordon_reason: None, + retire_requested_at: None, + retired_at: None, + total_vcpus: 2, + wire_version: 0, + stages_images: false, + capabilities: Default::default(), + lease_expires_at: None, + lease_state: Default::default(), + lease_epoch: 0, + }) + .await + .unwrap(); + // Explicit fixture imports a finalized source. This test proves event storage + // and publication, not a real host receipt or public admission. + sqlx::query("UPDATE sessions SET status='created',host_id=$2,sandbox_id=$3,binding_epoch=$4 WHERE id=$1") + .bind(sid.as_uuid()).bind(target.host_id.as_uuid()).bind(target.sandbox_id.as_uuid()).bind(target.binding_epoch as i64) + .execute(meta.pool()).await.unwrap(); + sqlx::query("INSERT INTO session_managed_boots(session_id,host_id,sandbox_id,binding_epoch,boot_attempt,revision,phase,record,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)") + .bind(sid.as_uuid()).bind(target.host_id.as_uuid()).bind(target.sandbox_id.as_uuid()).bind(target.binding_epoch as i64) + .bind(&key.boot_attempt).bind(row.revision as i64).bind(row.phase.as_str()).bind(serde_json::to_value(&row).unwrap()).bind(now) + .execute(meta.pool()).await.unwrap(); + let mut live = state.events.subscribe(sid); + let listener = crate::pg_listener::spawn( + db.url.clone(), + meta.clone(), + state.events.clone(), + state.host_registry.clone(), + state.integrations.clone(), + state.boot_bundles.clone(), + Arc::new(tokio::sync::Notify::new()), + Arc::new(tokio::sync::Notify::new()), + Arc::new(tokio::sync::Notify::new()), + state.op_finished.clone(), + ); + // No local publish: receiving this marker proves LISTEN is established. + tokio::time::timeout(Duration::from_secs(5), async { + loop { + meta.append_session_event( + sid, + "run_started", + serde_json::to_value(SessionEvent::HarnessRunStarted { + run_id: "old-run".into(), + prompt_summary: None, + prompt_id: None, + at: now, + }) + .unwrap(), + ) + .await + .unwrap(); + if let Ok(Ok(_)) = tokio::time::timeout(Duration::from_millis(50), live.recv()).await { + break; + } + } + }) + .await + .expect("listener must receive readiness marker"); + meta.advance_managed_boot(key, row.revision, ManagedBootAdvance::Interrupt) + .await + .unwrap(); + let published = tokio::time::timeout(Duration::from_secs(3),async { + loop { let event = live.recv().await.unwrap(); if matches!(&event.event,SessionEvent::HarnessRunInterrupted {run_id,..} if run_id=="old-run") { break event; } } + }).await.expect("committed interruption must reach the live bus through NOTIFY"); + let stored = meta + .list_session_events_since(sid, published.idx - 1, 1) + .await + .unwrap() + .remove(0); + assert_eq!(stored.kind, "run_interrupted"); + assert_eq!(stored.idx, published.idx); + assert!( + matches!(serde_json::from_value::(stored.payload).unwrap(),SessionEvent::HarnessRunInterrupted {run_id,..} if run_id=="old-run") + ); + listener.abort(); + let _ = listener.await; +} + +#[tokio::test] +async fn operator_scanner_requires_retirement_and_retains_before_host_io() { + let (state, meta, host, row) = fixture().await; + let mut candidate = engram_testkit::operator::candidate(); + candidate.network_identity = digest(b"new network same authority hosts"); + let g = meta + .stage_operator_publication(Some(1), candidate.clone()) + .await + .unwrap() + .pending + .unwrap(); + // Error and Missing are not proof. The scanner remains closed on both. + *host.state.lock().unwrap() = None; + crate::mcp::operator::run_once(&state, &candidate, None) + .await + .unwrap(); + assert!( + !meta + .read_operator_publication() + .await + .unwrap() + .admission_open + ); + *host.state.lock().unwrap() = Some(ManagedBootHostState::Missing { + key: row.request.key.clone(), + }); + crate::mcp::operator::run_once(&state, &candidate, None) + .await + .unwrap(); + assert!( + !meta + .read_operator_publication() + .await + .unwrap() + .admission_open + ); + *host.state.lock().unwrap() = Some(ManagedBootHostState::Finalized { + receipt: row.receipt.clone().unwrap(), + }); + host.apply_mode.store(2, Ordering::SeqCst); + let scanner = tokio::spawn({ + let state = state.clone(); + let candidate = candidate.clone(); + async move { crate::mcp::operator::run_once(&state, &candidate, None).await } + }); + tokio::time::timeout(std::time::Duration::from_secs(2), host.entered.notified()) + .await + .unwrap(); + let retained = meta + .get_session_integration_policy(row.request.key.creation.session_id) + .await + .unwrap() + .unwrap(); + let value: serde_json::Value = serde_json::from_str(&retained).unwrap(); + assert_eq!( + value["managed_deny"]["revision"], 2, + "same-host generation must invalidate the old policy receipt" + ); + // The same-row retention transaction can complete while the host is blocked. + tokio::time::timeout( + std::time::Duration::from_secs(2), + meta.bind_session_integration_policy(row.request.key.creation.session_id, &retained), + ) + .await + .unwrap() + .unwrap(); + assert!(meta.publish_operator_generation(&g).await.is_err()); + scanner.abort(); + let _ = scanner.await; + assert!( + !meta + .read_operator_publication() + .await + .unwrap() + .admission_open + ); + // A fresh pass recovers the persistent candidate/union, obtains a fresh host + // observation after a lost apply reply, and never asks the host to start. + host.apply_mode.store(1, Ordering::SeqCst); + crate::mcp::operator::run_once(&state, &candidate, None) + .await + .unwrap(); + assert!( + meta.read_operator_publication() + .await + .unwrap() + .admission_open + ); + assert_eq!(host.starts.load(Ordering::SeqCst), 0); + assert_eq!( + meta.managed_boot_open(row.request.key.creation.session_id) + .await + .unwrap(), + Some(false) + ); +} + +#[tokio::test] +async fn operator_scanner_retires_closed_owner_without_fabricating_binding() { + let (state, meta, host, row) = fixture().await; + *host.state.lock().unwrap() = Some(ManagedBootHostState::Prepared { + receipt: Box::new(row.prepared.clone().unwrap()), + }); + *host.policy.lock().unwrap() = None; + host.apply_mode.store(1, Ordering::SeqCst); + // Model a prepared reply not yet persisted: the registered claim still owns it. + meta.with_db_mut(|db| { + db.managed_boots.insert( + row.request.key.creation.session_id, + ManagedBootRecord::claimed(row.request.clone()), + ); + }); + let mut candidate = engram_testkit::operator::candidate(); + candidate.network_identity = digest(b"new closed network"); + meta.stage_operator_publication(Some(1), candidate.clone()) + .await + .unwrap(); + crate::mcp::operator::run_once(&state, &candidate, None) + .await + .unwrap(); + assert!( + meta.read_operator_publication() + .await + .unwrap() + .admission_open + ); + assert_eq!(host.starts.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn operator_scanner_keeps_policy_only_lineage_blocked_without_host_calls() { + let (state, meta, host, row) = fixture().await; + *host.state.lock().unwrap() = Some(ManagedBootHostState::Retired { + receipt: Box::new(ManagedBootRetiredReceipt { + prepared: row.prepared.clone().unwrap(), + retained_policy: row.request.bootstrap_policy.clone(), + receipt_id: digest(b"permanent owner"), + }), + }); + let sid = meta + .create_session(engram_core::types::SessionSpec { + image: "test.invalid/policy-only:latest".into(), + mode: Default::default(), + }) + .await + .unwrap(); + let hid = row.request.key.context.target.host_id; + meta.assign_session_host(sid, Some(hid)).await.unwrap(); + let sandbox = SandboxId::from(uuid::Uuid::from_u128(991)); + let epoch = meta.transition_session_created(sid, sandbox).await.unwrap(); + let target = ManagedPolicyTarget { + session_id: sid, + host_id: hid, + sandbox_id: sandbox, + binding_epoch: epoch, + }; + meta.retain_managed_deny( + &target, + &ManagedDenyPolicy::new(1, ["tools.example".into(), "history.example".into()]).unwrap(), + ) + .await + .unwrap(); + let candidate = engram_testkit::operator::candidate(); + let g = meta + .stage_operator_publication(Some(1), candidate.clone()) + .await + .unwrap() + .pending + .unwrap(); + for (host_inc, proxy_inc) in [ + ("old-host", "old-proxy"), + ("next-host", "old-proxy"), + ("next-host", "next-proxy"), + ] { + let receipt = ManagedAppliedState::new( + ManagedPolicyContext { + target: target.clone(), + operator_generation: g.generation, + operator_identity: g.identity.clone(), + binding_hash: None, + }, + ManagedDenyPolicy::new(2, ["tools.example".into(), "history.example".into()]).unwrap(), + host_inc.into(), + proxy_inc.into(), + Some(ManagedNetworkProof { + target: target.clone(), + guest_ip: "192.0.2.2".parse().unwrap(), + network_identity: digest(b"network"), + }), + ) + .unwrap(); + *host.policy.lock().unwrap() = Some(receipt); + crate::mcp::operator::run_once(&state, &candidate, None) + .await + .unwrap(); + assert_eq!( + host.policy_calls.load(Ordering::SeqCst), + 0, + "unsupported lineage must not authorize host I/O" + ); + assert_eq!(host.starts.load(Ordering::SeqCst), 0); + assert!( + !meta + .read_operator_publication() + .await + .unwrap() + .admission_open + ); + meta.with_db_mut(|db| { + assert!( + db.operator_readiness + .get(&(g.generation, row.request.key.creation.session_id)) + .unwrap() + .1 + .is_some(), + "current immutable source still reaches permanent readiness" + ); + let (participant, proof) = db.operator_readiness.get(&(g.generation, sid)).unwrap(); + assert!(proof.is_none()); + assert!(participant.policy.denies("history.example")); + assert!(participant.policy.denies("tools.example")); + assert!(participant.boot.is_none()); + }); + } +} + +#[tokio::test] +async fn managed_bridge_real_driver_claims_once_and_reconciles_prepared_reply() { + managed_bridge_driver(false).await; +} + +#[tokio::test] +async fn managed_bridge_driver_rechecks_pool_capacity() { + managed_bridge_driver(true).await; +} + +async fn managed_bridge_driver(pool_capacity: bool) { + use engram_core::types::{host::*, managed_bridge::*}; + let (mut state, meta, host, row) = fixture().await; + let sid = row.request.key.creation.session_id; + let host_id = row.request.key.context.target.host_id; + let base = match row.request.cold.disk.source { + ManagedColdSource::FreshImage { base_manifest } => base_manifest, + _ => unreachable!(), + }; + let mut selection = ManagedBootSelection { + launch: ManagedHarnessLaunch { + descriptor: crate::builtin_harness::builtin("claude") + .unwrap() + .descriptor() + .unwrap(), + workdir: Some("/work".into()), + }, + image: row.request.cold.image.clone(), + image_digest: digest(b"approved image"), + base_manifest: base, + aux_ro_drives: row.request.cold.aux_ro_drives.clone(), + vcpus: 1, + memory_mib: 128, + disk_gib: 1, + }; + // Explicit held-reservation fixture. The test drives actual coordinator selection, + // blob fork, claim and prepare code; its host is controlled, not physical proof. + meta.with_db_mut(|db| { + db.managed_boots.remove(&sid); + let s = db.sessions.get_mut(&sid).unwrap(); + s.session.status = engram_core::types::SessionState::Queued; + s.session.host_id = None; + s.session.sandbox_id = None; + s.binding_epoch = 0; + db.managed_boot_selections.insert(sid, selection.clone()); + }); + *host.state.lock().unwrap() = Some(ManagedBootHostState::Missing { + key: row.request.key.clone(), + }); + meta.upsert_host(HostRecord { + id: host_id, + hostname: "bridge-host".into(), + cloud_metadata: Default::default(), + capacity: HostCapacity { + total_gb: 10, + used_gb: 0, + total_mib: 8192, + used_mib: 0, + running_sandboxes: 0, + }, + utilization: Default::default(), + status: HostStatus::Ready, + last_heartbeat_at: state.services.clock.now_utc(), + host_addr: None, + ready_images: vec![selection.image_digest.clone()], + current_bundles: vec![], + sandbox_bundles: vec![], + cordoned: false, + cordon_owner: None, + cordon_reason: None, + retire_requested_at: None, + retired_at: None, + total_vcpus: 4, + wire_version: engram_protocol::WIRE_VERSION, + stages_images: true, + capabilities: HostCapabilities { + backend: "firecracker".into(), + ..Default::default() + }, + lease_expires_at: None, + lease_state: Default::default(), + lease_epoch: 0, + }) + .await + .unwrap(); + let registered = meta.get_host(host_id).await.unwrap().unwrap(); + meta.touch_host_heartbeat( + host_id, + HostHeartbeat { + status: HostStatus::Ready, + capacity: registered.capacity, + utilization: Default::default(), + ready_images: vec![selection.image_digest.clone()], + current_bundles: vec![], + sandbox_bundles: vec![], + total_vcpus: 4, + wire_version: engram_protocol::WIRE_VERSION, + stages_images: true, + capabilities: registered.capabilities, + lease_renew_until: None, + }, + ) + .await + .unwrap(); + let free_host = HostId::from(uuid::Uuid::from_u128(705)); + let mut second = meta.get_host(host_id).await.unwrap().unwrap(); + second.id = free_host; + meta.upsert_host(second.clone()).await.unwrap(); + for (id, capacity) in [(host_id, 64), (free_host, 1024)] { + meta.touch_host_heartbeat( + id, + HostHeartbeat { + status: HostStatus::Ready, + capacity: second.capacity.clone(), + utilization: HostUtilization { + allocatable_mib: if pool_capacity { 1024 } else { capacity }, + pool_data_total_mib: if pool_capacity { 200_000 } else { 0 }, + pool_data_free_mib: if id == host_id { 0 } else { 100_000 }, + pool_metadata_free_mib: 1_000, + ..Default::default() + }, + ready_images: vec![selection.image_digest.clone()], + current_bundles: vec![], + sandbox_bundles: vec![], + total_vcpus: 4, + wire_version: engram_protocol::WIRE_VERSION, + stages_images: true, + capabilities: second.capabilities.clone(), + lease_renew_until: None, + }, + ) + .await + .unwrap(); + } + // Resolve launch A through the real trusted catalog path, then retain it with the drive. + meta.with_db_mut(|db| { + db.hosts.get_mut(&host_id).unwrap().current_bundles = vec![AuxBundleRef { + drive_id: "harness-claude".into(), + sha256: "a".repeat(64), + }] + }); + let resolved = crate::api::sessions::resolve_harness( + &state, + Some("claude"), + Default::default(), + sid, + [("ORIGINAL_SECRET".into(), "original-value".into())].into(), + Some("/work".into()), + Some("plan"), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(resolved.mount, selection.aux_ro_drives[0]); + let original_argv = resolved.agent.argv.clone(); + selection.launch = resolved.launch; + let sealed = + crate::api::sessions::seal_session_secrets(&state, sid, &resolved.agent.session_env) + .await + .unwrap(); + meta.with_db_mut(|db| { + db.managed_boot_selections.insert(sid, selection.clone()); + db.session_secrets.insert(sid, sealed.clone()); + }); + let hash = state + .services + .chunk_store + .put_chunk(&[7; 4096]) + .await + .unwrap(); + let manifest = engram_chunk_store::Manifest { + schema_version: 1, + kind: engram_chunk_store::ManifestKind::Disk, + chunk_size: engram_chunk_store::ChunkSize::bytes(4096), + total_bytes: 4096, + chunks: vec![engram_chunk_store::ChunkRef { offset: 0, hash }], + parent: None, + annotations: serde_json::json!({ "image_tag": "approved" }), + }; + state + .services + .chunk_store + .put_manifest(base, &manifest) + .await + .unwrap(); + let creation = meta + .read_managed_creation(&row.request.key.creation) + .await + .unwrap(); + let q = ManagedCreationQuery { + identity: creation.identity, + native_version: creation.native.version, + native_digest: creation.native.digest, + }; + let first = crate::managed_bridge::prepare(&state, q.clone()) + .await + .unwrap(); + let retry = crate::managed_bridge::prepare(&state, q.clone()) + .await + .unwrap(); + assert_eq!(first.operation, retry.operation); + let initial = meta + .read_managed_boot_intent(&q, "initial") + .await + .unwrap() + .unwrap(); + let lost = Arc::new(LostManifestReply { + inner: state.services.blob.clone(), + target: initial.disk_manifest.storage_key(), + lost: AtomicUsize::new(0), + }); + Arc::get_mut(&mut state).unwrap().services.chunk_store = + engram_chunk_store::ChunkStore::new(lost.clone()); + // Accepted prompts remain queued during preparation; no Deliver op may exist. + crate::api::prompt::send_prompt_core( + &state, + sid, + "held-prompt".into(), + "queued".into(), + None, + crate::api::prompt::PromptDelivery::Enqueue, + ) + .await + .unwrap(); + assert!(!meta + .op_pending_exists(sid, engram_core::types::session_op::OpKind::Deliver) + .await + .unwrap()); + for step in 0..5 { + let current = meta + .read_managed_boot_intent(&q, "initial") + .await + .unwrap() + .unwrap(); + if step == 1 { + assert_eq!( + current.host_id, + Some(free_host), + "skip full first candidate" + ); + } + if step == 2 { + assert_eq!(current.host_id, Some(free_host)); + assert!(current.assets.is_some()); + meta.set_host_cordon( + free_host, + Some(CordonOwner::Admin), + Some("fixture host loss"), + ) + .await + .unwrap(); + meta.with_db_mut(|db| { + let host = db.hosts.get_mut(&host_id).unwrap(); + host.utilization.allocatable_mib = 1024; + if pool_capacity { + host.utilization.pool_data_free_mib = 100_000; + } + }); + } + if step == 3 { + assert_eq!(current.host_id, Some(host_id)); + assert!(current.assets.is_none(), "old host assets are fenced"); + assert_eq!(current.disk_manifest, initial.disk_manifest); + assert_eq!(current.sandbox_id, initial.sandbox_id); + } + crate::managed_bridge::drive(&state, current) + .await + .unwrap_or_else(|e| panic!("initial phase {step}: {e:?}")); + } + crate::managed_bridge::run_once(&state).await.unwrap(); + let claimed = meta.get_managed_boot(&q.identity).await.unwrap().unwrap(); + assert_eq!(claimed.request.key.boot_attempt, initial.intent_id); + assert_eq!( + claimed.request.key.context.target.sandbox_id, + initial.sandbox_id + ); + assert_eq!(claimed.phase, ManagedBootPhase::Claimed); + let fork = state + .services + .chunk_store + .get_manifest(initial.disk_manifest) + .await + .unwrap(); + assert_eq!( + lost.lost.load(Ordering::SeqCst), + 1, + "one committed private manifest despite a lost PUT reply" + ); + let expected = engram_chunk_store::Manifest { + parent: Some(base), + ..manifest + }; + assert_eq!(fork, expected); + // Replaying the old intent after a lost claim reply does not allocate or fork again. + let committed = meta + .read_managed_boot_intent(&q, "initial") + .await + .unwrap() + .unwrap(); + assert!(committed.completed); + crate::managed_bridge::drive(&state, committed) + .await + .unwrap(); + *host.state.lock().unwrap() = Some(ManagedBootHostState::Missing { + key: claimed.request.key.clone(), + }); + assert!(drive(&state, &claimed.request.key).await.is_err()); + drive(&state, &claimed.request.key).await.unwrap(); + let prepared = meta.read_managed_boot(&claimed.request.key).await.unwrap(); + assert_eq!(prepared.phase, ManagedBootPhase::Prepared); + assert!(prepared.sealed_agent.is_none()); + assert!(meta + .outbox_get("held-prompt") + .await + .unwrap() + .unwrap() + .acked_at + .is_none()); + assert_eq!(host.starts.load(Ordering::SeqCst), 0); + meta.with_db_mut(|db| { + db.hosts.get_mut(&host_id).unwrap().current_bundles[0].sha256 = "f".repeat(64) + }); + state.boot_bundles.invalidate_fleet_catalog(); + let (_, _, current_mount) = crate::api::sessions::resolve_harness_mount(&state, "claude") + .await + .unwrap(); + assert_ne!( + current_mount, selection.aux_ro_drives[0], + "current catalog is B" + ); + let finalized = finalize_bridge_fixture(&state, &host, &prepared).await; + let opened = open_finalize(&state, &finalized).await.unwrap(); + assert_eq!(opened.agent.argv, original_argv); + assert_eq!(opened.finalization.combined_manifest_digest, digest(b"[]")); + assert_eq!( + opened.finalization.prepared.request.cold.aux_ro_drives, + selection.aux_ro_drives + ); + assert_eq!( + opened.agent.session_env.get("ORIGINAL_SECRET").unwrap(), + "original-value" + ); + assert_eq!( + opened + .agent + .env + .get(engram_harness_proto::HARNESS_CWD_ENV) + .unwrap(), + "/work" + ); + assert_eq!( + meta.get_session_secrets(sid) + .await + .unwrap() + .unwrap() + .ciphertext, + sealed.ciphertext + ); + if pool_capacity { + meta.set_host_cordon(free_host, None, None).await.unwrap(); + } + let input = ManagedRecoveryInput { + query: q.clone(), + expected_source_key: claimed.request.key.clone(), + recovery_id: "recover-one".into(), + }; + let recovering = crate::managed_bridge::recover(&state, input.clone()) + .await + .unwrap(); + let retry = crate::managed_bridge::recover(&state, input.clone()) + .await + .unwrap(); + assert_eq!(recovering.operation, retry.operation); + let recovery = meta + .read_managed_boot_intent(&q, "recover:recover-one") + .await + .unwrap() + .unwrap(); + assert!( + crate::managed_bridge::drive(&state, recovery.clone()) + .await + .is_err(), + "lost capture reply" + ); + assert_eq!(host.captures.load(Ordering::SeqCst), 1); + assert!(meta + .list_managed_disk_manifests() + .await + .unwrap() + .contains(&recovery.disk_manifest)); + for _ in 0..3 { + crate::managed_bridge::run_once(&state).await.unwrap(); + } + let next = meta.get_managed_boot(&q.identity).await.unwrap().unwrap(); + assert_eq!(next.request.key.context.target.binding_epoch, 2); + assert_eq!(next.request.cold.disk.manifest, recovery.disk_manifest); + assert!(next.request.capture_authority.is_some()); + assert_eq!(host.captures.load(Ordering::SeqCst), 1); + assert!(meta.read_managed_boot(&claimed.request.key).await.is_err()); + let reconciled = crate::managed_bridge::recover(&state, input).await.unwrap(); + assert!(reconciled.operation.unwrap().completed); + assert_eq!(reconciled.boot.unwrap().request.key, next.request.key); + assert!( + drive(&state, &next.request.key).await.is_err(), + "lost recovery prepare reply" + ); + drive(&state, &next.request.key).await.unwrap(); + let recovered = meta.read_managed_boot(&next.request.key).await.unwrap(); + let final_recovery = finalize_bridge_fixture(&state, &host, &recovered).await; + let opened = open_finalize(&state, &final_recovery).await.unwrap(); + assert_eq!(opened.agent.argv, original_argv); + assert_eq!( + opened.agent.session_env.get("ORIGINAL_SECRET").unwrap(), + "original-value" + ); + assert_eq!( + opened.finalization.prepared.request.cold.aux_ro_drives, + selection.aux_ro_drives + ); + assert_eq!( + meta.get_session_secrets(sid) + .await + .unwrap() + .unwrap() + .ciphertext, + sealed.ciphertext + ); + + assert!(meta + .outbox_get("held-prompt") + .await + .unwrap() + .unwrap() + .acked_at + .is_none()); +} + +async fn finalize_bridge_fixture( + state: &SharedState, + host: &LossyHost, + prepared: &ManagedBootRecord, +) -> ManagedBootRecord { + let receipt = prepared.prepared.clone().unwrap(); + let mut context = prepared.request.key.context.clone(); + context.binding_hash = Some(digest(b"binding")); + let applied = ManagedAppliedState::new( + context.clone(), + receipt.request.bootstrap_policy.clone(), + receipt.host_incarnation.clone(), + "proxy".into(), + Some(ManagedNetworkProof { + target: context.target, + guest_ip: receipt.guest_ip, + network_identity: receipt.network_identity.clone(), + }), + ) + .unwrap(); + *host.policy.lock().unwrap() = Some(applied.clone()); + finalize( + state, + ManagedBootFinalizeInput { + key: prepared.request.key.clone(), + prepared_receipt_id: receipt.receipt_id, + policy: applied, + native_version: 1, + combined_manifest: b"[]".to_vec(), + combined_manifest_digest: digest(b"[]"), + }, + ) + .await + .expect("original A must finalize after catalog changes to B") +} + +struct LostManifestReply { + inner: Arc, + target: String, + lost: AtomicUsize, +} +#[async_trait] +impl engram_core::traits::BlobStorage for LostManifestReply { + async fn put_streaming( + &self, + key: &str, + body: engram_core::traits::ByteStream, + ) -> Result { + let size = self.inner.put_streaming(key, body).await?; + if key == self.target && self.lost.fetch_add(1, Ordering::SeqCst) == 0 { + return Err(engram_core::BlobError::Protocol( + "reply lost after manifest commit".into(), + )); + } + Ok(size) + } + async fn get_streaming( + &self, + key: &str, + ) -> Result { + self.inner.get_streaming(key).await + } + async fn head( + &self, + key: &str, + ) -> Result { + self.inner.head(key).await + } + async fn delete(&self, key: &str) -> Result<(), engram_core::BlobError> { + self.inner.delete(key).await + } + async fn list_prefix(&self, prefix: &str) -> Result, engram_core::BlobError> { + self.inner.list_prefix(prefix).await + } + async fn list_prefix_page( + &self, + prefix: &str, + cursor: Option<&str>, + limit: usize, + ) -> Result { + self.inner.list_prefix_page(prefix, cursor, limit).await + } +} + +const RPC_TEST_BUDGET: std::time::Duration = std::time::Duration::from_millis(10); +const SCAN_TEST_LIMIT: std::time::Duration = std::time::Duration::from_secs(1); + +#[tokio::test] +async fn managed_boot_prepare_deadline_allows_next_row_without_replacing_attempt() { + let (state, meta, host, row) = fixture().await; + let key = &row.request.key; + let first = ManagedBootRecord::claimed(row.request.clone()); + let mut second_request = row.request.clone(); + let second_sid = SessionId::from(uuid::Uuid::from_u128(800)); + let second_sandbox = SandboxId::from(uuid::Uuid::from_u128(801)); + second_request.key.creation.session_id = second_sid; + second_request.key.context.target.session_id = second_sid; + second_request.key.context.target.sandbox_id = second_sandbox; + second_request.key.boot_attempt = "boot-two".into(); + second_request.cold.session_id = second_sid; + second_request.cold.sandbox_id = second_sandbox; + second_request.cold.boot_attempt = second_request.key.boot_attempt.clone(); + let second_key = second_request.key.clone(); + meta.with_db_mut(|db| { + let mut session = db.sessions[&key.creation.session_id].clone(); + session.session.id = second_sid; + session.session.sandbox_id = Some(second_sandbox); + let mut creation = db.managed_creations[&key.creation.session_id].clone(); + creation.identity = second_request.key.creation.clone(); + db.managed_creations.insert(second_sid, creation); + db.sessions.insert(second_sid, session); + db.managed_boots + .insert(key.creation.session_id, first.clone()); + db.managed_boots.insert( + second_sid, + ManagedBootRecord::claimed(second_request.clone()), + ); + }); + *host.prepare_stall.lock().unwrap() = Some(key.creation.session_id); + let mut prepared = host.prepared.clone(); + prepared.request = second_request; + *host.state.lock().unwrap() = Some(ManagedBootHostState::Prepared { + receipt: Box::new(prepared), + }); + for _ in 0..2 { + tokio::time::timeout( + SCAN_TEST_LIMIT, + run_once_with_timeout(&state, RPC_TEST_BUDGET), + ) + .await + .expect("one host must not block the next row") + .unwrap(); + assert_eq!( + serde_json::to_value(meta.read_managed_boot(key).await.unwrap()).unwrap(), + serde_json::to_value(&first).unwrap() + ); + assert_eq!( + meta.read_managed_boot(&second_key).await.unwrap().phase, + ManagedBootPhase::Prepared + ); + } + assert_eq!(host.starts.load(Ordering::SeqCst), 0); + assert!(gate(&state, key.creation.session_id, true).await.is_err()); +} + +#[tokio::test] +async fn managed_boot_start_deadline_reconciles_without_another_start() { + let (state, meta, host, row) = fixture().await; + let key = &row.request.key; + host.start_stall.store(true, Ordering::SeqCst); + let error = tokio::time::timeout( + SCAN_TEST_LIMIT, + drive_with_timeout(&state, key, RPC_TEST_BUDGET), + ) + .await + .expect("start reply must have a deadline") + .unwrap_err(); + assert!(matches!(error, ApiError::Unavailable(_)), "{error:?}"); + let issued = meta.read_managed_boot(key).await.unwrap(); + assert_eq!(issued.phase, ManagedBootPhase::StartIssued); + assert_eq!(issued.request, row.request); + assert!(gate(&state, key.creation.session_id, true).await.is_err()); + drive_with_timeout(&state, key, RPC_TEST_BUDGET) + .await + .unwrap(); + assert_eq!( + meta.read_managed_boot(key).await.unwrap().phase, + ManagedBootPhase::Started + ); + assert_eq!(host.starts.load(Ordering::SeqCst), 1); + gate(&state, key.creation.session_id, true).await.unwrap(); +} + +#[tokio::test] +async fn managed_boot_started_read_deadline_interrupts_without_releasing_owner() { + let (state, meta, host, row) = fixture().await; + let key = &row.request.key; + assert!(drive(&state, key).await.is_err()); + drive(&state, key).await.unwrap(); + host.read_stall.store(true, Ordering::SeqCst); + let error = tokio::time::timeout( + SCAN_TEST_LIMIT, + drive_with_timeout(&state, key, RPC_TEST_BUDGET), + ) + .await + .expect("started read must have a deadline") + .unwrap_err(); + assert!(matches!(error, ApiError::Unavailable(_)), "{error:?}"); + let interrupted = meta.read_managed_boot(key).await.unwrap(); + assert_eq!(interrupted.phase, ManagedBootPhase::Interrupted); + assert_eq!(interrupted.request, row.request); + assert_eq!(host.starts.load(Ordering::SeqCst), 1); + let session = meta.get_session(key.creation.session_id).await.unwrap(); + assert_eq!(session.host_id, Some(key.context.target.host_id)); + assert_eq!(session.sandbox_id, Some(key.context.target.sandbox_id)); + assert!(gate(&state, key.creation.session_id, true).await.is_err()); +} diff --git a/crates/engram-coordinator/src/managed_bridge.rs b/crates/engram-coordinator/src/managed_bridge.rs new file mode 100644 index 000000000..35616c406 --- /dev/null +++ b/crates/engram-coordinator/src/managed_bridge.rs @@ -0,0 +1,410 @@ +//! Identity-only durable creation and explicit cold-recovery producer. +use crate::{error::ApiError, state::SharedState}; +use engram_core::types::managed_bridge::conflict; +use engram_core::{ + types::{managed_boot::*, managed_bridge::*, managed_cold::*, managed_deny::*, sandbox::*}, + SessionId, SnapshotId, +}; + +pub(crate) async fn prepare( + state: &SharedState, + q: ManagedCreationQuery, +) -> Result { + q.validate()?; + let meta = &state.services.meta; + let view = meta.read_managed_creation_state(&q).await?; + if matches!( + view.state, + ManagedCreationPhase::Absent | ManagedCreationPhase::Terminal + ) { + return Err(conflict().into()); + } + if view.boot.is_none() + && meta + .read_managed_boot_intent(&q, "initial") + .await? + .is_none() + { + let selection = meta.read_managed_boot_selection(&q).await?; + let intent = ManagedBootIntent { + query: q.clone(), + intent_id: state.services.entropy.uuid().to_string(), + revision: 1, + selection, + sandbox_id: state.services.entropy.uuid().into(), + binding_epoch: 1, + disk_manifest: engram_core::types::ManifestRef { + manifest_id: state.services.entropy.uuid(), + version: 1, + }, + recovery: None, + host_id: None, + assets: None, + completed: false, + }; + meta.begin_managed_boot_intent(intent).await?; + } + meta.read_managed_creation_state(&q) + .await + .map_err(Into::into) +} +pub(crate) async fn recover( + state: &SharedState, + input: ManagedRecoveryInput, +) -> Result { + input.validate()?; + let meta = &state.services.meta; + let q = &input.query; + let operation = format!("recover:{}", input.recovery_id); + if let Some(i) = meta.read_managed_boot_intent(q, &operation).await? { + if i.recovery + .as_ref() + .is_none_or(|r| r.source != input.expected_source_key) + { + return Err(conflict().into()); + } + let mut view = meta.read_managed_creation_state(q).await?; + view.operation = Some(i.view()); + return Ok(view); + } + let row = meta.read_managed_boot(&input.expected_source_key).await?; + if row.prepared.is_none() { + return Err(conflict().into()); + } + let mut selection = meta.read_managed_boot_selection(q).await?; + selection.image = row.request.cold.image.clone(); + selection.aux_ro_drives = row.request.cold.aux_ro_drives.clone(); + let spec = row.request.spec()?; + selection.vcpus = spec.cpu.vcpus; + selection.memory_mib = spec.memory.max_mib; + selection.disk_gib = spec.disk.max_gib; + let capture_id = SnapshotId::from(state.services.entropy.uuid()); + let intent = ManagedBootIntent { + query: q.clone(), + intent_id: state.services.entropy.uuid().to_string(), + revision: 1, + selection, + sandbox_id: state.services.entropy.uuid().into(), + binding_epoch: input + .expected_source_key + .context + .target + .binding_epoch + .checked_add(1) + .ok_or_else(conflict)?, + disk_manifest: engram_core::types::ManifestRef { + manifest_id: capture_id.as_uuid(), + version: 1, + }, + recovery: Some(ManagedRecoveryIntent { + recovery_id: input.recovery_id, + source: input.expected_source_key, + capture_id, + }), + host_id: None, + assets: None, + completed: false, + }; + meta.begin_managed_boot_intent(intent).await?; + meta.read_managed_creation_state(q) + .await + .map_err(Into::into) +} +async fn publication( + state: &SharedState, +) -> Result { + let p = state.services.meta.read_operator_publication().await?; + if !p.admission_open || p.pending.is_some() { + return Err(conflict().into()); + } + p.published.ok_or_else(|| conflict().into()) +} +/// One durable phase per pass. All host/blob work occurs after short store transactions. +pub(crate) async fn drive(state: &SharedState, i: ManagedBootIntent) -> Result<(), ApiError> { + if i.completed { + return Ok(()); + } + i.validate()?; + let meta = &state.services.meta; + let generation = publication(state).await?; + // Lost claim replies are reconciled before any capture or private disk operation. + if let Some(b) = meta.get_managed_boot(&i.query.identity).await? { + if b.request.key.boot_attempt == i.intent_id { + if b.request.key.context.target.sandbox_id != i.sandbox_id { + return Err(conflict().into()); + } + meta.advance_managed_boot_intent( + &i.intent_id, + i.revision, + ManagedIntentAdvance::Complete, + ) + .await?; + return Ok(()); + } + } + let capture = if let Some(r) = &i.recovery { + Some( + crate::managed_boot::capture( + state, + ManagedCaptureRequest { + key: r.source.clone(), + capture_id: r.capture_id, + }, + ) + .await?, + ) + } else { + None + }; + { + i.query + .matches(&meta.read_managed_creation(&i.query.identity).await?)?; + let (repo, tag) = engram_core::types::session::split_image_ref(&i.selection.image); + let bundles: Vec<_> = i + .selection + .aux_ro_drives + .iter() + .map(|d| AuxBundleRef { + drive_id: d.drive_id.clone(), + sha256: d.sha256.clone().expect("validated selection"), + }) + .collect(); + let context = crate::placement::ScheduleContext { + repo, + image_version: tag, + snapshot_host: None, + memory_mib: Some(i.selection.memory_mib), + cpu_budget_vcpus: Some(i.selection.vcpus), + required_image_digest: Some(engram_protocol::heartbeat::ManifestDigest::new( + i.selection.image_digest.clone(), + )), + exclude_host: None, + prefer_host: None, + caps: Default::default(), + prefer_bundles: &bundles, + }; + let candidates = crate::placement::candidates_for( + meta.as_ref(), + &context, + state.services.clock.now_utc(), + ) + .await + .map_err(engram_core::SandboxError::from)?; + let fits = meta + .placement_no_fit_details( + &candidates.hosts, + i64::from(i.selection.memory_mib), + i32::try_from(i.selection.vcpus).map_err(|_| conflict())?, + ) + .await?; + let mut hosts = candidates.hosts; + // Keep a usable selection stable; only read-only asset inspection precedes claim. + hosts.sort_by_key(|h| Some(*h) != i.host_id); + let mut selected = false; + for host in hosts { + if !fits + .iter() + .any(|f| f.host_id == host && matches!(f.reason, "fits_now" | "unmeasured")) + { + continue; + } + let h = meta.get_host(host).await?.ok_or_else(conflict)?; + if !matches!(h.capabilities.backend.as_str(), "firecracker" | "vz") { + continue; + } + if i.host_id == Some(host) { + selected = true; + break; + } + match meta + .advance_managed_boot_intent( + &i.intent_id, + i.revision, + ManagedIntentAdvance::SelectHost(host), + ) + .await + { + Ok(_) => return Ok(()), + Err(engram_core::MetaError::Conflict(_)) => continue, + Err(e) => return Err(e.into()), + } + } + if !selected { + return Ok(()); // Retry the same immutable intent when capacity returns. + } + } + if capture.is_none() { + fork(state, &i).await?; + } + if i.assets.is_none() { + let request = i.assets_request()?; + let assets = crate::managed_boot::host_rpc( + engram_protocol::grpc_client::restore_rpc_timeout(), + state + .services + .host + .read_managed_cold_assets(request.clone()), + ) + .await?; + assets.validate_for(&request)?; + let h = meta.get_host(request.host_id).await?.ok_or_else(conflict)?; + let backend = match assets.backend { + engram_core::types::managed_assets::ManagedColdBackend::Firecracker => "firecracker", + engram_core::types::managed_assets::ManagedColdBackend::Vz => "vz", + }; + if h.capabilities.backend != backend { + return Err(conflict().into()); + } + if let Some(a) = &capture { + if assets.disk_sha256 != a.disk_sha256 + || assets.disk_size_bytes != a.disk_size_bytes + || assets.kernel_sha256 != a.kernel_sha256 + { + return Err(conflict().into()); + } + } + meta.advance_managed_boot_intent( + &i.intent_id, + i.revision, + ManagedIntentAdvance::Assets(Box::new(assets)), + ) + .await?; + return Ok(()); + } + let assets = i.assets.as_ref().ok_or_else(conflict)?; + let prior = + crate::api::sessions::load_session_policy(state, i.query.identity.session_id).await?; + let old = prior + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|_| conflict())?; + let (_, deny) = engram_core::types::managed_operator::retain( + old.as_deref(), + generation.candidate.hosts()?, + )?; + let key = ManagedBootKey { + creation: i.query.identity.clone(), + native_digest: i.query.native_digest.clone(), + boot_attempt: i.intent_id.clone(), + context: ManagedPolicyContext { + target: ManagedPolicyTarget { + session_id: i.query.identity.session_id, + host_id: i.host_id.ok_or_else(conflict)?, + sandbox_id: i.sandbox_id, + binding_epoch: i.binding_epoch, + }, + operator_generation: generation.generation, + operator_identity: generation.identity, + binding_hash: None, + }, + }; + let source = match &capture { + None => ManagedColdSource::FreshImage { + base_manifest: i.selection.base_manifest, + }, + Some(a) => ManagedColdSource::Capture { + capture_id: a.request.capture_id, + session_id: i.query.identity.session_id, + sandbox_id: a.request.key.context.target.sandbox_id, + binding_epoch: a.request.key.context.target.binding_epoch, + record_digest: a.record_digest.clone(), + }, + }; + let cold = ManagedColdPrepare { + version: 1, + session_id: i.query.identity.session_id, + sandbox_id: i.sandbox_id, + boot_attempt: i.intent_id.clone(), + image: i.selection.image.clone(), + disk: ManagedColdDisk { + manifest: i.disk_manifest, + sha256: assets.disk_sha256.clone(), + size_bytes: assets.disk_size_bytes, + source, + }, + kernel_sha256: assets.kernel_sha256.clone(), + aux_ro_drives: i.selection.aux_ro_drives.clone(), + }; + // A logical manifest locator, replaced by authenticated host materialization; never a local path supplied by O. + let spec = SandboxSpec { + image: i.selection.image.clone(), + rootfs_source: Some(i.disk_manifest.storage_key().into()), + rootfs_manifest: Some(i.disk_manifest), + image_uri: None, + cpu: CpuLimit { + vcpus: i.selection.vcpus, + }, + memory: MemoryLimit { + max_mib: i.selection.memory_mib, + }, + disk: DiskLimit { + max_gib: i.selection.disk_gib, + }, + ttl: None, + env: Default::default(), + workdir: None, + network: Default::default(), + aux_ro_drives: i.selection.aux_ro_drives.clone(), + swap_mib: None, + swap_source: None, + swap_manifest: None, + }; + let request = ManagedBootPrepare { + key, + cold, + bootstrap_policy: deny, + capture_authority: capture, + sandbox_spec_json: serde_json::to_string(&spec).map_err(|_| conflict())?, + }; + crate::managed_boot::register(state, request).await?; + meta.advance_managed_boot_intent(&i.intent_id, i.revision, ManagedIntentAdvance::Complete) + .await?; + Ok(()) +} +async fn fork(state: &SharedState, i: &ManagedBootIntent) -> Result<(), ApiError> { + let store = &state.services.chunk_store; + let mut expected = store + .get_manifest(i.selection.base_manifest) + .await + .map_err(|_| unavailable())?; + if expected.kind != engram_chunk_store::ManifestKind::Disk { + return Err(conflict().into()); + } + expected.parent = Some(i.selection.base_manifest); + // PUT is create-only. A lost reply is reconciled by complete immutable content. + let written = store.put_manifest(i.disk_manifest, &expected).await; + if written.is_err() { + let actual = store + .get_manifest(i.disk_manifest) + .await + .map_err(|_| unavailable())?; + if actual != expected { + return Err(conflict().into()); + } + } + Ok(()) +} +fn unavailable() -> ApiError { + ApiError::Unavailable("managed disk storage unavailable".into()) +} +pub(crate) async fn run_once(state: &SharedState) -> Result<(), ApiError> { + let mut after: Option = None; + loop { + let rows = state + .services + .meta + .list_managed_boot_intents(after, 128) + .await?; + if rows.is_empty() { + break; + } + for i in rows { + after = Some(i.query.identity.session_id); + if let Err(error) = drive(state, i).await { + tracing::debug!(error=%error,"managed preparation remains closed"); + } + } + } + Ok(()) +} diff --git a/crates/engram-coordinator/src/mcp/control.rs b/crates/engram-coordinator/src/mcp/control.rs new file mode 100644 index 000000000..6b3c394c2 --- /dev/null +++ b/crates/engram-coordinator/src/mcp/control.rs @@ -0,0 +1,447 @@ +//! Dedicated mTLS service. Never add this router to the app gRPC listener. +use super::{model as domain, oauth::Callback, worker, McpAuthority, McpError}; +use engram_core::types::org_secret::{McpSecretReservation, McpSecretRole}; +use engram_protocol::mcp as wire; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeSet, sync::Arc, time::Duration}; +use tonic::{Request, Response, Status}; +use wire::credential_control_service_server::{ + CredentialControlService, CredentialControlServiceServer, +}; + +type RpcResult = std::result::Result, Status>; +pub struct ControlTls { + pub certificate: Vec, + pub private_key: Vec, + pub client_ca: Vec, + pub orchestrator_fingerprints: Vec, +} +#[derive(Clone)] +pub(super) struct Control { + pub(super) authority: Arc, + pins: Arc>, + admission: Arc, +} +pub fn server( + authority: Arc, + tls: ControlTls, +) -> super::Result { + server_with_policy(authority, tls, None) +} +pub fn server_with_policy( + authority: Arc, + tls: ControlTls, + state: Option, +) -> super::Result { + if tls.orchestrator_fingerprints.is_empty() + || tls.orchestrator_fingerprints.len() > 8 + || tls.orchestrator_fingerprints.iter().any(|s| { + s.len() != 95 + || s.split(':').any(|p| { + p.len() != 2 + || !p + .bytes() + .all(|b| b.is_ascii_digit() || (b'A'..=b'F').contains(&b)) + }) + }) + { + return Err(McpError::Invalid); + } + engram_tls::install_provider(); + let service = Control { + authority, + pins: Arc::new(tls.orchestrator_fingerprints.into_iter().collect()), + admission: Arc::new(tokio::sync::Semaphore::new(32)), + }; + Ok(tonic::transport::Server::builder() + .tls_config( + tonic::transport::ServerTlsConfig::new() + .identity(tonic::transport::Identity::from_pem( + tls.certificate, + tls.private_key, + )) + .client_ca_root(tonic::transport::Certificate::from_pem(tls.client_ca)), + ) + .map_err(|_| McpError::Invalid)? + .concurrency_limit_per_connection(16) + .timeout(Duration::from_secs(30)) + .max_concurrent_streams(16) + .add_service( + CredentialControlServiceServer::new(service.clone()) + .max_decoding_message_size(32768) + .max_encoding_message_size(32768), + ) + .add_service( + wire::session_policy_service_server::SessionPolicyServiceServer::new( + super::policy::PolicyControl { + control: service, + state, + }, + ) + .max_decoding_message_size(9 * 1024 * 1024) + .max_encoding_message_size(9 * 1024 * 1024), + )) +} +impl Control { + pub(super) fn check( + &self, + req: &Request, + protocol: u32, + ) -> std::result::Result { + let certs = req + .peer_certs() + .ok_or_else(|| Status::unauthenticated("client certificate required"))?; + let cert = certs + .first() + .ok_or_else(|| Status::unauthenticated("client certificate required"))?; + let pin = Sha256::digest(cert.as_ref()) + .iter() + .map(|b| format!("{b:02X}")) + .collect::>() + .join(":"); + if !self.pins.contains(&pin) { + return Err(Status::permission_denied("caller denied")); + } + if protocol != wire::PROTOCOL { + return Err(Status::invalid_argument("unsupported control protocol")); + } + self.admission + .clone() + .try_acquire_owned() + .map_err(|_| Status::resource_exhausted("control capacity exceeded")) + } +} +fn error(e: McpError) -> Status { + match e { + McpError::NotFound => Status::not_found("credential not found"), + McpError::Invalid => Status::invalid_argument("invalid credential request"), + McpError::Denied => Status::permission_denied("credential request denied"), + McpError::Stale => Status::failed_precondition("credential context changed"), + _ => Status::unavailable("credential unavailable"), + } +} +fn required(value: Option) -> std::result::Result { + value.ok_or_else(|| Status::invalid_argument("required field missing")) +} +fn actor(value: &str) -> std::result::Result<(), Status> { + if value.is_empty() || value.len() > 256 { + Err(Status::invalid_argument("actor required")) + } else { + Ok(()) + } +} +fn destination( + value: Option, +) -> std::result::Result { + let v = required(value)?; + Ok(domain::Destination { + zone_id: v.zone_id, + endpoint: v.endpoint, + }) +} +fn change( + value: Option, +) -> std::result::Result { + let v = required(value)?; + Ok(domain::ChangeContext { + connection_id: v.connection_id, + config_revision: v.config_revision, + previous_authorization_generation: v.previous_authorization_generation, + resource: destination(v.resource)?, + }) +} +fn context( + value: Option, +) -> std::result::Result { + let v = required(value)?; + Ok(domain::AuthorityContext { + connection_id: v.connection_id, + config_revision: v.config_revision, + authorization_generation: v.authorization_generation, + credential_identity: v.credential_identity, + resource: destination(v.resource)?, + policy_digest: v.policy_digest, + policy_revision: v.policy_revision, + }) +} +fn reference( + value: wire::CredentialReference, +) -> std::result::Result { + match wire::CredentialKind::try_from(value.kind) + .map_err(|_| Status::invalid_argument("invalid credential kind"))? + { + wire::CredentialKind::ApiKey | wire::CredentialKind::ClientSecret + if value.registration_id.is_empty() + && !value.name.is_empty() + && !value.connection_id.is_empty() + && value.generation >= 1 => + { + let role = if value.kind == wire::CredentialKind::ApiKey as i32 { + McpSecretRole::ApiKey + } else { + McpSecretRole::ClientSecret + }; + let reservation = McpSecretReservation { + name: value.name, + connection_id: value.connection_id, + role, + generation: value.generation, + }; + Ok(if role == McpSecretRole::ApiKey { + domain::CredentialReference::ApiKey { reservation } + } else { + domain::CredentialReference::ClientSecret { reservation } + }) + } + wire::CredentialKind::Oauth + if !value.registration_id.is_empty() + && value.name.is_empty() + && value.connection_id.is_empty() + && value.generation == 0 => + { + Ok(domain::CredentialReference::OAuth { + registration_id: value.registration_id, + }) + } + _ => Err(Status::invalid_argument("invalid credential reference")), + } +} +fn context_wire(v: domain::AuthorityContext) -> wire::AuthorityContext { + wire::AuthorityContext { + connection_id: v.connection_id, + config_revision: v.config_revision, + authorization_generation: v.authorization_generation, + credential_identity: v.credential_identity, + resource: Some(wire::Destination { + zone_id: v.resource.zone_id, + endpoint: v.resource.endpoint, + }), + policy_digest: v.policy_digest, + policy_revision: v.policy_revision, + } +} +fn reference_wire(v: domain::CredentialReference) -> wire::CredentialReference { + match v { + domain::CredentialReference::ApiKey { reservation } + | domain::CredentialReference::ClientSecret { reservation } => wire::CredentialReference { + kind: if reservation.role == McpSecretRole::ApiKey { + wire::CredentialKind::ApiKey + } else { + wire::CredentialKind::ClientSecret + } as i32, + name: reservation.name, + connection_id: reservation.connection_id, + generation: reservation.generation, + registration_id: String::new(), + }, + domain::CredentialReference::OAuth { registration_id } => wire::CredentialReference { + kind: wire::CredentialKind::Oauth as i32, + registration_id, + ..Default::default() + }, + } +} +fn status_wire(v: domain::CredentialStatus) -> wire::CredentialStatus { + wire::CredentialStatus { + protocol: wire::PROTOCOL, + context: Some(context_wire(v.context)), + reference: Some(reference_wire(v.reference)), + version: v.version, + status: v.status, + } +} +fn status(v: wire::CredentialStatus) -> std::result::Result { + if v.protocol != wire::PROTOCOL { + return Err(Status::invalid_argument("unsupported control protocol")); + } + Ok(domain::CredentialStatus { + context: context(v.context)?, + reference: reference(required(v.reference)?)?, + version: v.version, + status: v.status, + }) +} +#[tonic::async_trait] +impl CredentialControlService for Control { + async fn put_secret( + &self, + req: Request, + ) -> RpcResult { + let _permit = self.check(&req, req.get_ref().protocol)?; + let v = req.into_inner(); + actor(&v.actor_id)?; + let role = match wire::CredentialKind::try_from(v.kind) { + Ok(wire::CredentialKind::ApiKey) => McpSecretRole::ApiKey, + Ok(wire::CredentialKind::ClientSecret) => McpSecretRole::ClientSecret, + _ => return Err(Status::invalid_argument("invalid secret kind")), + }; + let expected = v.expected.map(reference).transpose()?; + Ok(Response::new(wire::PutSecretResponse { + status: Some(status_wire( + self.authority + .put_secret( + &change(v.context)?, + role, + expected.as_ref(), + &v.header, + &v.value, + ) + .await + .map_err(error)?, + )), + })) + } + async fn begin_authorization( + &self, + req: Request, + ) -> RpcResult { + let _permit = self.check(&req, req.get_ref().protocol)?; + let v = req.into_inner(); + actor(&v.actor_id)?; + let secret = v.client_secret.map(reference).transpose()?; + let b = self + .authority + .begin( + &change(v.context)?, + &v.registration_id, + secret.as_ref(), + v.expected_version, + &v.actor_id, + v.sharing_consent, + ) + .await + .map_err(error)?; + Ok(Response::new(wire::BeginAuthorizationResponse { + protocol: wire::PROTOCOL, + flow_id: b.flow_id.to_string(), + context: Some(context_wire(b.context)), + authorization_url: b.authorization_url, + expires_at: b.expires_at.timestamp_millis(), + })) + } + async fn complete_authorization( + &self, + req: Request, + ) -> RpcResult { + let _permit = self.check(&req, req.get_ref().protocol)?; + let v = req.into_inner(); + actor(&v.actor_id)?; + let callback = Callback { + context: context(v.context)?, + registration_id: v.registration_id, + flow_id: v + .flow_id + .parse() + .map_err(|_| Status::invalid_argument("invalid flow id"))?, + state: v.state, + redirect_uri: v.redirect_uri, + issuer: v.issuer, + code: v.code, + actor_id: v.actor_id, + }; + Ok(Response::new(wire::CompleteAuthorizationResponse { + status: Some(status_wire( + self.authority.callback(&callback).await.map_err(error)?, + )), + })) + } + async fn cancel_authorization( + &self, + req: Request, + ) -> RpcResult { + let _permit = self.check(&req, req.get_ref().protocol)?; + let v = req.into_inner(); + actor(&v.actor_id)?; + let id = v + .flow_id + .parse() + .map_err(|_| Status::invalid_argument("invalid flow id"))?; + self.authority + .cancel_authorization(id, &v.connection_id, &v.registration_id) + .await + .map_err(error)?; + Ok(Response::new(wire::CancelAuthorizationResponse { + protocol: wire::PROTOCOL, + flow_id: v.flow_id, + connection_id: v.connection_id, + registration_id: v.registration_id, + })) + } + async fn get_status( + &self, + req: Request, + ) -> RpcResult { + let _permit = self.check(&req, req.get_ref().protocol)?; + let v = req.into_inner(); + Ok(Response::new(wire::GetStatusResponse { + status: Some(status_wire( + self.authority + .status(&v.connection_id, &reference(required(v.reference)?)?) + .await + .map_err(error)?, + )), + })) + } + async fn revoke(&self, req: Request) -> RpcResult { + let _permit = self.check(&req, req.get_ref().protocol)?; + let v = req.into_inner(); + actor(&v.actor_id)?; + Ok(Response::new(wire::RevokeResponse { + status: Some(status_wire( + self.authority + .revoke(&status(required(v.expected)?)?) + .await + .map_err(error)?, + )), + })) + } + async fn lease(&self, req: Request) -> RpcResult { + let _permit = self.check(&req, req.get_ref().protocol)?; + let v = req.into_inner(); + let i = required(v.worker)?; + let a = required(v.admission)?; + let scope = match required(v.scope)? { + wire::lease_request::Scope::Administration(s) => worker::WorkerScope::Administration { + request_id: s.request_id, + actor_id: s.actor_id, + }, + wire::lease_request::Scope::Execution(s) => worker::WorkerScope::Execution { + session_id: s.session_id, + run_id: s.run_id, + tool_call_id: s.tool_call_id, + binding_hash: s.binding_hash, + }, + }; + let lease = domain::ResourceLease { + context: context(v.context)?, + reference: reference(required(v.reference)?)?, + worker: worker::WorkerIdentity { + protocol: i.protocol, + boot_id: i.boot_id, + zone_id: i.zone_id, + policy_digest: i.policy_digest, + policy_revision: i.policy_revision, + }, + intent_id: v.intent_id, + request_digest: v.request_digest, + operation: v.operation, + expires_at: v.expires_at, + operation_expires_at: v.operation_expires_at, + admission: worker::WorkerLease { + id: a.id, + deployment_id: a.deployment_id, + expires_at: a.expires_at, + }, + scope, + }; + let credential = self.authority.lease(&lease).await.map_err(error)?; + Ok(Response::new(wire::LeaseResponse { + protocol: wire::PROTOCOL, + header: credential.header, + value: credential.value, + expires_at: lease.expires_at, + operation_expires_at: lease.operation_expires_at, + intent_id: lease.intent_id, + })) + } +} diff --git a/crates/engram-coordinator/src/mcp/io.rs b/crates/engram-coordinator/src/mcp/io.rs new file mode 100644 index 000000000..6bc942702 --- /dev/null +++ b/crates/engram-coordinator/src/mcp/io.rs @@ -0,0 +1,125 @@ +//! Bounded worker I/O with durable deployment admission and refresh dispatch fencing. +use super::*; +use chrono::Duration; +use engram_core::types::mcp::{McpOAuthAdmission, McpRefreshClaim}; +use serde_json::{json, Value}; + +impl McpAuthority { + pub(super) async fn oauth_io( + &self, + context: &AuthorityContext, + destination: &Destination, + actor: &str, + mut operation: WorkerOperation, + secret: Option<&McpSecretReservation>, + claim: Option<&McpRefreshClaim>, + ) -> Result { + self.enabled()?; + if actor.is_empty() || actor.len() > 256 { + return Err(McpError::Invalid); + } + let now = self.manager.clock.now_utc(); + let deadline = claim + .map(|c| c.expires_at - Duration::seconds(5)) + .unwrap_or(now + Duration::seconds(10)) + .min(now + Duration::seconds(10)); + if deadline <= now { + return Err(McpError::Stale); + } + let admission = McpOAuthAdmission { + id: self.manager.entropy.uuid(), + deployment_id: self.registry.deployment_id.clone(), + connection_id: context.connection_id.clone(), + expires_at: deadline + Duration::seconds(5), + }; + if !self + .manager + .meta + .acquire_mcp_oauth_admission(&admission) + .await? + { + return Err(McpError::Unavailable); + } + let result = async { + let identity = self.identity(destination, None).await?; + if let Some(secret) = secret { + let payload = self.open_secret(secret).await?; + if payload.revoked + || secret.role != McpSecretRole::ClientSecret + || payload.context.connection_id != context.connection_id + || payload.context.config_revision != context.config_revision + { + return Err(McpError::Stale); + } + match &mut operation { + WorkerOperation::Token { grant, .. } => match grant { + TokenGrant::AuthorizationCode { client_secret, .. } + | TokenGrant::RefreshToken { client_secret, .. } => { + *client_secret = Some(payload.value); + } + }, + _ => return Err(McpError::Denied), + } + } + let intent = WorkerIntent { + id: self.manager.entropy.uuid().to_string(), + boot_id: identity.boot_id.clone(), + zone_id: identity.zone_id.clone(), + policy_digest: identity.policy_digest.clone(), + connection_id: context.connection_id.clone(), + connection_revision: context.config_revision, + credential_identity: context.credential_identity.clone(), + scope: WorkerScope::Administration { + request_id: admission.id.to_string(), + actor_id: actor.into(), + }, + operation: operation.kind().into(), + destination: destination.endpoint.clone(), + request_digest: digest(&json!({ + "context": context, + "admission": admission.id.to_string(), + "operation": operation.kind(), + "destination": destination, + }))?, + operation_digest: digest( + &serde_json::to_value(&operation).map_err(|_| McpError::Invalid)?, + )?, + expires_at: deadline.timestamp_millis(), + lease: WorkerLease { + id: admission.id.to_string(), + deployment_id: admission.deployment_id.clone(), + expires_at: admission.expires_at.timestamp_millis(), + }, + }; + // Admission, worker identity, and client-secret checks precede the + // irreversible rotating-token dispatch claim. + if self.manager.clock.now_utc() >= deadline { + return Err(McpError::Stale); + } + if let Some(claim) = claim { + if !self.manager.meta.claim_mcp_refresh(claim).await? { + return Err(McpError::Unavailable); + } + } + // A slow claim transaction must not extend the token RPC deadline. + let remaining = deadline - self.manager.clock.now_utc(); + if remaining <= Duration::zero() { + return Err(McpError::Stale); + } + tokio::time::timeout( + remaining.to_std().map_err(|_| McpError::Stale)?, + self.worker(destination)? + .rpc(&identity, &intent, &operation), + ) + .await + .map_err(|_| McpError::Transport)? + } + .await; + // Release only after the bounded RPC has ended. Expiry covers a crashed caller. + self.manager + .meta + .release_mcp_oauth_admission(&admission) + .await?; + result + } +} diff --git a/crates/engram-coordinator/src/mcp/lifecycle.rs b/crates/engram-coordinator/src/mcp/lifecycle.rs new file mode 100644 index 000000000..5fba7859f --- /dev/null +++ b/crates/engram-coordinator/src/mcp/lifecycle.rs @@ -0,0 +1,116 @@ +//! Reference-only status and expected-version revocation. +use super::*; +use engram_core::types::oauth::{OAuthCredentialKey, OAuthSubjectKind}; +impl McpAuthority { + pub async fn status( + &self, + connection: &str, + reference: &CredentialReference, + ) -> Result { + self.enabled()?; + match reference { + CredentialReference::ApiKey { reservation } + | CredentialReference::ClientSecret { reservation } => { + let current = self + .manager + .meta + .get_mcp_org_secret_reservation(&reservation.name) + .await? + .ok_or(McpError::NotFound)?; + if current.connection_id != connection || current.role != reservation.role { + return Err(McpError::Denied); + } + let p = self.open_secret(¤t).await?; + let reference = if current.role == McpSecretRole::ApiKey { + CredentialReference::ApiKey { + reservation: current.clone(), + } + } else { + CredentialReference::ClientSecret { + reservation: current.clone(), + } + }; + Ok(CredentialStatus { + context: p.context, + reference, + version: current.generation, + status: if p.revoked { "revoked" } else { "connected" }.into(), + }) + } + CredentialReference::OAuth { registration_id } => { + let row = self + .manager + .meta + .get_oauth_credential(&OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: connection.into(), + provider: registration_id.clone(), + }) + .await? + .ok_or(McpError::NotFound)?; + let bundle = self.decode_bundle(&row).await?; + Ok(CredentialStatus { + context: bundle.context, + reference: reference.clone(), + version: row.version, + status: row.status(self.manager.clock.now_utc()).as_str().into(), + }) + } + } + } + pub async fn revoke(&self, expected: &CredentialStatus) -> Result { + self.enabled()?; + if &self + .status(&expected.context.connection_id, &expected.reference) + .await? + != expected + { + return Err(McpError::Stale); + } + match &expected.reference { + CredentialReference::ApiKey { reservation } + | CredentialReference::ClientSecret { reservation } => { + let mut p = self.open_secret(reservation).await?; + p.revoked = true; + p.value.clear(); + p.reservation.generation = p + .reservation + .generation + .checked_add(1) + .ok_or(McpError::Invalid)?; + p.context.authorization_generation = p + .context + .authorization_generation + .checked_add(1) + .ok_or(McpError::Invalid)?; + p.context.credential_identity = self.random()?; + let sealed = crate::org_secrets::seal_org_secret( + self.manager.kek.as_ref(), + &reservation.name, + &serde_json::to_vec(&p).map_err(|_| McpError::Invalid)?, + ) + .await + .map_err(|_| McpError::Crypto)?; + self.manager + .meta + .put_mcp_org_secret(sealed, reservation) + .await?; + } + CredentialReference::OAuth { registration_id } => { + self.manager + .meta + .revoke_oauth_credential( + &OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: expected.context.connection_id.clone(), + provider: registration_id.clone(), + }, + expected.version, + ) + .await?; + } + } + self.status(&expected.context.connection_id, &expected.reference) + .await + } +} diff --git a/crates/engram-coordinator/src/mcp/mod.rs b/crates/engram-coordinator/src/mcp/mod.rs new file mode 100644 index 000000000..06a46f2b4 --- /dev/null +++ b/crates/engram-coordinator/src/mcp/mod.rs @@ -0,0 +1,536 @@ +//! Dedicated coordinator credential authority. Never registered on app gRPC. +pub mod worker; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum McpError { + Invalid, + Denied, + Unavailable, + NotFound, + Stale, + Crypto, + Transport, +} +pub type Result = std::result::Result; +impl From for McpError { + fn from(error: engram_core::MetaError) -> Self { + use engram_core::MetaError; + match error { + MetaError::NotFound | MetaError::Conflict(_) => Self::Stale, + MetaError::Db(_) | MetaError::Migration(_) | MetaError::Serialization(_) => { + Self::Unavailable + } + } + } +} + +pub mod control; +mod io; +mod lifecycle; +pub mod model; +pub mod oauth; +mod policy; +pub mod startup; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use engram_core::types::org_secret::{McpSecretReservation, McpSecretRole}; +use engram_crypto::{CredCipher, SealedCred}; +use model::*; +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, sync::Arc}; +use worker::*; + +pub trait SecurityEntropy: Send + Sync { + fn bytes(&self) -> Result<[u8; 32]>; +} +pub struct SystemSecurityEntropy; +impl SecurityEntropy for SystemSecurityEntropy { + fn bytes(&self) -> Result<[u8; 32]> { + engram_crypto::security_random_bytes().map_err(|_| McpError::Crypto) + } +} +pub struct McpAuthority { + pub(crate) manager: Arc, + pub(crate) security: Arc, + pub(crate) registry: Registry, + workers: BTreeMap>, +} +#[derive(Serialize, Deserialize)] +struct SecretPayload { + v: u32, + context: AuthorityContext, + reservation: McpSecretReservation, + header: String, + value: String, + revoked: bool, +} +impl McpAuthority { + pub fn new( + manager: Arc, + security: Arc, + registry: Registry, + workers: BTreeMap>, + ) -> Result { + if registry.deployment_id.is_empty() || registry.deployment_id.len() > 256 { + return Err(McpError::Invalid); + } + let this = Self { + manager, + security, + registry, + workers, + }; + for (id, zone) in &this.registry.zones { + if !this.workers.contains_key(id) + || zone.policy_digest.len() != 64 + || zone.policy_revision.is_empty() + { + return Err(McpError::Invalid); + } + for endpoint in &zone.endpoints { + Self::canonical_endpoint(endpoint)?; + } + } + for (id, r) in &this.registry.registrations { + if id != &r.id + || r.client_id.is_empty() + || r.client_id.len() > 8192 + || r.scopes + .iter() + .any(|s| s.is_empty() || s.len() > 256 || s.chars().any(char::is_whitespace)) + || r.scopes.len() > 128 + { + return Err(McpError::Invalid); + } + Self::canonical_issuer(&r.issuer)?; + Self::canonical_endpoint(&r.redirect_uri)?; + for d in [ + &r.resource, + &r.resource_metadata, + &r.authorization_metadata, + &r.authorization, + &r.token, + ] { + this.destination(d)?; + } + } + Ok(this) + } + pub(crate) fn enabled(&self) -> Result<()> { + if !self.registry.enabled { + Err(McpError::Unavailable) + } else { + Ok(()) + } + } + pub(crate) fn random(&self) -> Result { + Ok(URL_SAFE_NO_PAD.encode(self.security.bytes()?)) + } + fn canonical_issuer(issuer: &str) -> Result<()> { + let url = url::Url::parse(issuer).map_err(|_| McpError::Invalid)?; + // Issuer identifiers can be canonical origins without a trailing slash. + // Validate the URL, but keep the registered bytes for exact OAuth checks. + if url.origin().ascii_serialization() == issuer { + Self::canonical_endpoint(url.as_str()) + } else { + Self::canonical_endpoint(issuer) + } + } + pub(crate) fn canonical_endpoint(endpoint: &str) -> Result<()> { + let url = url::Url::parse(endpoint).map_err(|_| McpError::Invalid)?; + if url.scheme() != "https" + || !matches!(url.host(), Some(url::Host::Domain(_))) + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || url.as_str() != endpoint + { + return Err(McpError::Invalid); + } + Ok(()) + } + pub(crate) fn destination(&self, d: &Destination) -> Result<&Zone> { + let z = self + .registry + .zones + .get(&d.zone_id) + .ok_or(McpError::Denied)?; + if !z.endpoints.contains(&d.endpoint) { + return Err(McpError::Denied); + } + Ok(z) + } + pub(crate) fn worker(&self, d: &Destination) -> Result<&Arc> { + self.destination(d)?; + self.workers.get(&d.zone_id).ok_or(McpError::Denied) + } + pub(crate) async fn identity( + &self, + d: &Destination, + expected: Option<&WorkerIdentity>, + ) -> Result { + let z = self.destination(d)?; + let actual = self.worker(d)?.identity().await?; + if actual.protocol != 1 + || actual.zone_id != d.zone_id + || actual.policy_digest != z.policy_digest + || actual.policy_revision != z.policy_revision + || expected.is_some_and(|v| v != &actual) + { + return Err(McpError::Stale); + } + Ok(actual) + } + pub(crate) fn context(&self, change: &ChangeContext) -> Result { + self.enabled()?; + self.destination(&change.resource)?; + if change.connection_id.is_empty() + || change.connection_id.len() > 256 + || change.config_revision < 1 + || change.config_revision > 9_007_199_254_740_991 + || change.previous_authorization_generation < 0 + { + return Err(McpError::Invalid); + } + let generation = change + .previous_authorization_generation + .checked_add(1) + .filter(|v| *v <= 9_007_199_254_740_991) + .ok_or(McpError::Invalid)?; + let zone = self.destination(&change.resource)?; + Ok(AuthorityContext { + policy_digest: zone.policy_digest.clone(), + policy_revision: zone.policy_revision.clone(), + connection_id: change.connection_id.clone(), + config_revision: change.config_revision, + authorization_generation: generation, + credential_identity: self.random()?, + resource: change.resource.clone(), + }) + } + pub async fn put_secret( + &self, + change: &ChangeContext, + role: McpSecretRole, + expected: Option<&CredentialReference>, + header: &str, + value: &str, + ) -> Result { + let context = self.context(change)?; + if value.is_empty() + || value.len() > 8192 + || value.contains(['\r', '\n']) + || !["authorization", "x-api-key", "api-key"].contains(&header) + { + return Err(McpError::Invalid); + } + let reservation = match expected { + None => { + self.manager + .meta + .reserve_mcp_org_secret( + &format!("mcp-{}", self.random()?), + &context.connection_id, + role, + ) + .await? + } + Some( + CredentialReference::ApiKey { reservation } + | CredentialReference::ClientSecret { reservation }, + ) => { + if reservation.connection_id != context.connection_id || reservation.role != role { + return Err(McpError::Denied); + } + let previous = self.open_secret(reservation).await?; + if previous.context.authorization_generation + != change.previous_authorization_generation + { + return Err(McpError::Stale); + } + reservation.clone() + } + _ => return Err(McpError::Denied), + }; + let mut next = reservation.clone(); + next.generation = next.generation.checked_add(1).ok_or(McpError::Invalid)?; + let payload = SecretPayload { + v: 1, + context: context.clone(), + reservation: next.clone(), + header: header.into(), + value: value.into(), + revoked: false, + }; + let bytes = serde_json::to_vec(&payload).map_err(|_| McpError::Invalid)?; + let sealed = crate::org_secrets::seal_org_secret( + self.manager.kek.as_ref(), + &reservation.name, + &bytes, + ) + .await + .map_err(|_| McpError::Crypto)?; + self.manager + .meta + .put_mcp_org_secret(sealed, &reservation) + .await?; + let reference = match role { + McpSecretRole::ApiKey => CredentialReference::ApiKey { + reservation: next.clone(), + }, + McpSecretRole::ClientSecret => CredentialReference::ClientSecret { + reservation: next.clone(), + }, + }; + Ok(CredentialStatus { + context, + reference, + version: next.generation, + status: "connected".into(), + }) + } + async fn open_secret(&self, expected: &McpSecretReservation) -> Result { + let row = self + .manager + .meta + .get_org_secret_sealed(&expected.name) + .await? + .ok_or(McpError::Unavailable)?; + // Read the current permanent reservation after the captured ciphertext. + if self + .manager + .meta + .get_mcp_org_secret_reservation(&expected.name) + .await? + .as_ref() + != Some(expected) + { + return Err(McpError::Stale); + } + let envelope = SealedCred { + wrapped_dek: row.wrapped_dek, + nonce: row.nonce.try_into().map_err(|_| McpError::Crypto)?, + ciphertext: row.ciphertext, + key_id: row.key_id, + }; + let bytes = CredCipher::new(self.manager.kek.as_ref()) + .open(&envelope) + .await + .map_err(|_| McpError::Crypto)?; + let payload: SecretPayload = + serde_json::from_slice(&bytes).map_err(|_| McpError::Invalid)?; + if payload.v != 1 + || &payload.reservation != expected + || payload.context.connection_id != expected.connection_id + { + return Err(McpError::Denied); + } + Ok(payload) + } + pub async fn lease(&self, lease: &ResourceLease) -> Result { + self.enabled()?; + let now = self.manager.clock.now_utc().timestamp_millis(); + if lease.expires_at <= now + || lease.expires_at > now + 10000 + || lease.operation_expires_at < lease.expires_at + || lease.operation_expires_at > now + 300000 + || lease.admission.expires_at < lease.operation_expires_at + || lease.admission.expires_at > now + 300000 + || lease.admission.deployment_id != self.registry.deployment_id + || lease.admission.id.is_empty() + || lease.intent_id.is_empty() + || lease.request_digest.len() != 64 + || !lease.request_digest.bytes().all(|b| b.is_ascii_hexdigit()) + { + return Err(McpError::Denied); + } + if !matches!( + (&*lease.operation, &lease.scope), + ("discover", WorkerScope::Administration { .. }) + | ("execute", WorkerScope::Execution { .. }) + ) { + return Err(McpError::Denied); + } + if lease.context.policy_digest != lease.worker.policy_digest + || lease.context.policy_revision != lease.worker.policy_revision + { + return Err(McpError::Stale); + } + self.identity(&lease.context.resource, Some(&lease.worker)) + .await?; + let credential = match &lease.reference { + CredentialReference::ApiKey { reservation } => { + if reservation.role != McpSecretRole::ApiKey { + return Err(McpError::Denied); + } + let p = self.open_secret(reservation).await?; + if p.context != lease.context || p.revoked { + return Err(McpError::Stale); + } + ProviderCredential { + header: p.header, + value: p.value, + } + } + CredentialReference::OAuth { registration_id } => { + self.oauth_lease(&lease.context, registration_id).await? + } + _ => return Err(McpError::Denied), + }; + if self.manager.clock.now_utc().timestamp_millis() >= lease.expires_at { + return Err(McpError::Stale); + } + Ok(credential) + } +} + +pub mod operator; + +#[cfg(test)] +mod tests { + use super::*; + use engram_core::traits::StaticSecretStore; + use engram_sim::{ManualClock, SimEntropy, SimMetadataStore}; + use std::collections::{BTreeSet, HashMap}; + + #[test] + fn metadata_failures_do_not_report_changed_authority() { + use engram_core::MetaError; + for error in [ + MetaError::Db(std::io::Error::other("database unavailable").into()), + MetaError::Migration("migration unavailable".into()), + MetaError::Serialization("invalid stored metadata".into()), + ] { + assert_eq!(McpError::from(error), McpError::Unavailable); + } + assert_eq!( + McpError::from(MetaError::Conflict("version changed".into())), + McpError::Stale + ); + assert_eq!(McpError::from(MetaError::NotFound), McpError::Stale); + } + + struct NoWorkerIo; + #[async_trait::async_trait] + impl WorkerTransport for NoWorkerIo { + async fn identity(&self) -> Result { + panic!("authority construction must not contact a worker") + } + async fn rpc( + &self, + _: &WorkerIdentity, + _: &WorkerIntent, + _: &WorkerOperation, + ) -> Result { + panic!("authority construction must not contact a worker") + } + } + + fn authority_with_issuer(issuer: &str) -> Result { + let clock = ManualClock::new(); + let entropy = Arc::new(SimEntropy::seeded(9)); + let manager = crate::oauth::OAuthManager::new( + SimMetadataStore::new(clock.clone(), entropy.clone()), + Arc::new(engram_crypto::EnvVarKeyProvider::from_bytes( + [8; 32], "test", + )), + clock, + entropy, + Arc::new(StaticSecretStore::new(HashMap::new())), + ); + let destination = Destination { + zone_id: "public".into(), + endpoint: "https://resource.example/mcp".into(), + }; + let registration = ClientRegistration { + id: "registration-a".into(), + client_id: "client-a".into(), + confidential: false, + redirect_uri: "https://client.example/callback".into(), + issuer: issuer.into(), + resource: destination.clone(), + resource_metadata: destination.clone(), + authorization_metadata: destination.clone(), + authorization: destination.clone(), + token: destination, + scopes: BTreeSet::new(), + issuer_parameter_required: true, + account_field: None, + }; + McpAuthority::new( + manager, + Arc::new(SystemSecurityEntropy), + Registry { + enabled: true, + deployment_id: "deployment-a".into(), + zones: BTreeMap::from([( + "public".into(), + Zone { + policy_digest: "a".repeat(64), + policy_revision: "1".into(), + endpoints: BTreeSet::from(["https://resource.example/mcp".into()]), + }, + )]), + registrations: BTreeMap::from([("registration-a".into(), registration)]), + }, + BTreeMap::from([( + "public".into(), + Arc::new(NoWorkerIo) as Arc, + )]), + ) + } + + #[test] + fn constructor_preserves_canonical_issuer_identifiers() { + for issuer in [ + "https://issuer.fixture.example:8443", + "https://issuer.fixture.example", + "https://issuer.fixture.example:8443/", + "https://issuer.fixture.example:8443/tenant", + ] { + let authority = authority_with_issuer(issuer) + .unwrap_or_else(|error| panic!("canonical issuer {issuer} rejected: {error:?}")); + assert_eq!( + authority.registry.registrations["registration-a"].issuer, + issuer + ); + } + } + + #[test] + fn constructor_denies_invalid_or_noncanonical_issuers() { + for issuer in [ + "not a URL", + "http://issuer.fixture.example", + "https://user@issuer.fixture.example", + "https://user:pass@issuer.fixture.example/", + "https://issuer.fixture.example?query", + "https://issuer.fixture.example/#fragment", + "https://127.0.0.1", + "https://[::1]/", + "https://%69ssuer.fixture.example", + "https://Issuer.fixture.example", + "HTTPS://issuer.fixture.example/", + "https://issuer.fixture.example:443", + "https://issuer.fixture.example:443/", + "https://issuer.fixture.example/a/../tenant", + ] { + assert!( + matches!(authority_with_issuer(issuer), Err(McpError::Invalid)), + "accepted {issuer}" + ); + } + } + + #[test] + fn endpoints_still_require_exact_canonical_urls() { + assert_eq!( + McpAuthority::canonical_endpoint("https://issuer.fixture.example:8443"), + Err(McpError::Invalid) + ); + assert_eq!( + McpAuthority::canonical_endpoint("https://issuer.fixture.example:8443/"), + Ok(()) + ); + } +} diff --git a/crates/engram-coordinator/src/mcp/model.rs b/crates/engram-coordinator/src/mcp/model.rs new file mode 100644 index 000000000..33f5294bb --- /dev/null +++ b/crates/engram-coordinator/src/mcp/model.rs @@ -0,0 +1,101 @@ +//! Metadata shared with the trusted control caller. No credential values. +use super::worker::{WorkerIdentity, WorkerLease, WorkerScope}; +use engram_core::types::org_secret::McpSecretReservation; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Destination { + pub zone_id: String, + pub endpoint: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ChangeContext { + pub connection_id: String, + pub config_revision: i64, + pub previous_authorization_generation: i64, + pub resource: Destination, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct AuthorityContext { + pub policy_digest: String, + pub policy_revision: String, + pub connection_id: String, + pub config_revision: i64, + pub authorization_generation: i64, + pub credential_identity: String, + pub resource: Destination, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum CredentialReference { + ApiKey { reservation: McpSecretReservation }, + ClientSecret { reservation: McpSecretReservation }, + OAuth { registration_id: String }, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CredentialStatus { + pub context: AuthorityContext, + pub reference: CredentialReference, + pub version: i64, + pub status: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ResourceLease { + pub context: AuthorityContext, + pub reference: CredentialReference, + pub worker: WorkerIdentity, + pub intent_id: String, + pub request_digest: String, + pub operation: String, + pub scope: WorkerScope, + /// Credential delivery deadline, at most ten seconds. + pub expires_at: i64, + /// Frozen worker operation deadline; admission must cover the full operation. + pub operation_expires_at: i64, + pub admission: WorkerLease, +} +// Raw values live only in an authenticated request or one operation's memory. +#[derive(Serialize, Deserialize)] +pub struct ProviderCredential { + pub header: String, + pub value: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Zone { + pub policy_digest: String, + pub policy_revision: String, + pub endpoints: BTreeSet, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ClientRegistration { + pub id: String, + pub client_id: String, + pub confidential: bool, + pub redirect_uri: String, + pub issuer: String, + pub resource: Destination, + pub resource_metadata: Destination, + pub authorization_metadata: Destination, + pub authorization: Destination, + pub token: Destination, + pub scopes: BTreeSet, + pub issuer_parameter_required: bool, + pub account_field: Option, +} +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Registry { + #[serde(default)] + pub enabled: bool, + pub deployment_id: String, + pub zones: BTreeMap, + pub registrations: BTreeMap, +} diff --git a/crates/engram-coordinator/src/mcp/oauth.rs b/crates/engram-coordinator/src/mcp/oauth.rs new file mode 100644 index 000000000..fb2796a2c --- /dev/null +++ b/crates/engram-coordinator/src/mcp/oauth.rs @@ -0,0 +1,603 @@ +//! Durable encrypted PKCE and strict, exclusive refresh for registered MCP clients. +use super::*; +use chrono::{DateTime, Duration, Utc}; +use engram_core::types::{ + mcp::{McpRefreshClaim, McpSealedState}, + oauth::*, +}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BegunAuthorization { + pub flow_id: Uuid, + pub context: AuthorityContext, + pub authorization_url: String, + pub expires_at: DateTime, +} +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Callback { + pub context: AuthorityContext, + pub registration_id: String, + pub flow_id: Uuid, + pub state: String, + pub redirect_uri: String, + pub issuer: Option, + pub code: String, + pub actor_id: String, +} +#[derive(Serialize, Deserialize)] +struct FlowState { + require_issuer: bool, + v: u32, + id: Uuid, + state: String, + verifier: String, + context: AuthorityContext, + registration: ClientRegistration, + policies: BTreeMap, + client_secret: Option, + expected_version: Option, +} +#[derive(Serialize, Deserialize)] +pub(crate) struct Bundle { + v: u32, + pub context: AuthorityContext, + registration: ClientRegistration, + policies: BTreeMap, + client_secret: Option, + access_token: String, + refresh_token: Option, + expires_at: Option>, + provider_account: Option, + identity_source: String, +} +fn key(connection: &str, registration: &str) -> OAuthCredentialKey { + OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: connection.into(), + provider: registration.into(), + } +} +impl McpAuthority { + fn policies(&self, r: &ClientRegistration) -> Result> { + [ + &r.resource, + &r.resource_metadata, + &r.authorization_metadata, + &r.authorization, + &r.token, + ] + .into_iter() + .map(|d| { + let z = self.destination(d)?; + Ok(( + d.zone_id.clone(), + (z.policy_digest.clone(), z.policy_revision.clone()), + )) + }) + .collect() + } + fn registration(&self, id: &str) -> Result<&ClientRegistration> { + self.registry.registrations.get(id).ok_or(McpError::Denied) + } + fn check_registration( + &self, + r: &ClientRegistration, + policies: &BTreeMap, + ) -> Result<()> { + if self.registration(&r.id)? != r || &self.policies(r)? != policies { + return Err(McpError::Stale); + } + Ok(()) + } + async fn seal_state(&self, value: &T) -> Result { + let bytes = serde_json::to_vec(value).map_err(|_| McpError::Invalid)?; + let s = CredCipher::new(self.manager.kek.as_ref()) + .seal(&bytes) + .await + .map_err(|_| McpError::Crypto)?; + Ok(McpSealedState { + wrapped_dek: s.wrapped_dek, + nonce: s.nonce.to_vec(), + ciphertext: s.ciphertext, + key_id: s.key_id, + }) + } + async fn open_state(&self, s: McpSealedState) -> Result { + let bytes = CredCipher::new(self.manager.kek.as_ref()) + .open(&SealedCred { + wrapped_dek: s.wrapped_dek, + nonce: s.nonce.try_into().map_err(|_| McpError::Crypto)?, + ciphertext: s.ciphertext, + key_id: s.key_id, + }) + .await + .map_err(|_| McpError::Crypto)?; + serde_json::from_slice(&bytes).map_err(|_| McpError::Invalid) + } + pub async fn begin( + &self, + change: &ChangeContext, + registration_id: &str, + secret: Option<&CredentialReference>, + expected_version: Option, + actor: &str, + sharing_consent: bool, + ) -> Result { + let context = self.context(change)?; + if !sharing_consent { + return Err(McpError::Denied); + } + let r = self.registration(registration_id)?.clone(); + if r.resource != context.resource { + return Err(McpError::Denied); + } + let client_secret = match (r.confidential, secret) { + (false, None) => None, + (true, Some(CredentialReference::ClientSecret { reservation })) + if reservation.connection_id == context.connection_id + && reservation.role == McpSecretRole::ClientSecret => + { + let payload = self.open_secret(reservation).await?; + if payload.revoked { + return Err(McpError::Denied); + } + Some(reservation.clone()) + } + _ => return Err(McpError::Denied), + }; + let current = self + .manager + .meta + .get_oauth_credential(&key(&context.connection_id, registration_id)) + .await?; + if current.as_ref().map(|r| r.version) != expected_version { + return Err(McpError::Stale); + } + let resource = self + .oauth_io( + &context, + &r.resource_metadata, + actor, + WorkerOperation::Metadata { + endpoint: r.resource_metadata.endpoint.clone(), + }, + None, + None, + ) + .await?; + if resource["resource"] != r.resource.endpoint + || !resource["authorization_servers"] + .as_array() + .is_some_and(|a| a.iter().any(|v| v == &r.issuer)) + { + return Err(McpError::Denied); + } + let metadata = self + .oauth_io( + &context, + &r.authorization_metadata, + actor, + WorkerOperation::Metadata { + endpoint: r.authorization_metadata.endpoint.clone(), + }, + None, + None, + ) + .await?; + if metadata["issuer"] != r.issuer + || metadata["authorization_endpoint"] != r.authorization.endpoint + || metadata["token_endpoint"] != r.token.endpoint + || !metadata["code_challenge_methods_supported"] + .as_array() + .is_some_and(|a| a.iter().any(|v| v == "S256")) + { + return Err(McpError::Denied); + } + self.oauth_io( + &context, + &r.authorization, + actor, + WorkerOperation::ValidateDestination { + endpoint: r.authorization.endpoint.clone(), + }, + None, + None, + ) + .await?; + let now = self.manager.clock.now_utc(); + let expires_at = now + Duration::minutes(10); + let random = self.security.bytes()?; + let id = Uuid::from_bytes(random[..16].try_into().map_err(|_| McpError::Crypto)?); + let state = self.random()?; + let verifier = self.random()?; + let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes())); + let flow = FlowState { + require_issuer: r.issuer_parameter_required + || metadata["authorization_response_iss_parameter_supported"] == true, + v: 1, + id, + state: state.clone(), + verifier, + context: context.clone(), + policies: self.policies(&r)?, + registration: r.clone(), + client_secret, + expected_version, + }; + self.manager + .meta + .create_mcp_oauth_flow( + OAuthFlow { + id, + key: key(&context.connection_id, registration_id), + owner_replica: self.manager.replica.clone(), + lease_expires_at: expires_at, + expires_at, + status: OAuthFlowStatus::Pending, + error_code: None, + created_at: now, + updated_at: now, + }, + self.seal_state(&flow).await?, + ) + .await?; + let mut url = url::Url::parse(&r.authorization.endpoint).map_err(|_| McpError::Invalid)?; + url.query_pairs_mut() + .append_pair("response_type", "code") + .append_pair("client_id", &r.client_id) + .append_pair("redirect_uri", &r.redirect_uri) + .append_pair("resource", &r.resource.endpoint) + .append_pair( + "scope", + &r.scopes.iter().cloned().collect::>().join(" "), + ) + .append_pair("state", &state) + .append_pair("code_challenge", &challenge) + .append_pair("code_challenge_method", "S256"); + Ok(BegunAuthorization { + flow_id: id, + context, + authorization_url: url.into(), + expires_at, + }) + } + pub async fn cancel_authorization( + &self, + flow_id: Uuid, + connection_id: &str, + registration_id: &str, + ) -> Result<()> { + self.enabled()?; + self.manager + .meta + .cancel_mcp_oauth_flow(flow_id, &key(connection_id, registration_id)) + .await?; + Ok(()) + } + pub async fn callback(&self, callback: &Callback) -> Result { + self.enabled()?; + // The winner removes ciphertext atomically BEFORE any token request. + let flow: FlowState = self + .open_state( + self.manager + .meta + .consume_mcp_oauth_flow( + callback.flow_id, + &key(&callback.context.connection_id, &callback.registration_id), + ) + .await?, + ) + .await?; + let result = async { + let r = &flow.registration; + if flow.v != 1 + || flow.id != callback.flow_id + || flow.context != callback.context + || flow.state != callback.state + || r.id != callback.registration_id + || r.redirect_uri != callback.redirect_uri + || callback.issuer.as_ref().is_some_and(|i| i != &r.issuer) + || (flow.require_issuer && callback.issuer.is_none()) + || callback.code.is_empty() + || callback.code.len() > 8192 + { + return Err(McpError::Denied); + } + self.check_registration(r, &flow.policies)?; + let response = self + .oauth_io( + &flow.context, + &r.token, + &callback.actor_id, + WorkerOperation::Token { + endpoint: r.token.endpoint.clone(), + grant: TokenGrant::AuthorizationCode { + code: callback.code.clone(), + redirect_uri: r.redirect_uri.clone(), + code_verifier: flow.verifier, + client_id: r.client_id.clone(), + resource: r.resource.endpoint.clone(), + client_secret: None, + }, + }, + flow.client_secret.as_ref(), + None, + ) + .await?; + let grant = validate_token(&response, r, None, self.manager.clock.now_utc())?; + let bundle = Bundle { + v: 1, + context: flow.context, + registration: flow.registration, + policies: flow.policies, + client_secret: flow.client_secret, + access_token: grant.access_token, + refresh_token: grant.refresh_token, + expires_at: grant.expires_at, + identity_source: if grant.account.is_some() { + "provider_response" + } else { + "authorization_ceremony" + } + .into(), + provider_account: grant.account, + }; + let sealed = self.credential(&bundle).await?; + let row = self + .manager + .meta + .complete_mcp_oauth_flow(callback.flow_id, sealed, flow.expected_version) + .await?; + Ok(status(&bundle, &row)) + } + .await; + // Completion metadata cannot make a consumed flow replayable. + let _ = self + .manager + .meta + .finish_oauth_flow_unowned( + callback.flow_id, + if result.is_ok() { + OAuthFlowStatus::Succeeded + } else { + OAuthFlowStatus::Failed + }, + if result.is_err() { + Some("authorization_failed") + } else { + None + }, + ) + .await; + result + } + async fn credential(&self, bundle: &Bundle) -> Result { + let sealed = self.seal_state(bundle).await?; + Ok(NewSealedOAuthCredential { + key: key(&bundle.context.connection_id, &bundle.registration.id), + wrapped_dek: sealed.wrapped_dek, + nonce: sealed.nonce, + ciphertext: sealed.ciphertext, + key_id: sealed.key_id, + metadata: OAuthAccountMetadata { + account_id: bundle.context.credential_identity.clone(), + ..Default::default() + }, + expires_at: bundle.expires_at, + }) + } + pub(crate) async fn decode_bundle(&self, row: &SealedOAuthCredential) -> Result { + let bytes = self.manager.open(row).await.map_err(|_| McpError::Crypto)?; + let bundle: Bundle = serde_json::from_slice(&bytes).map_err(|_| McpError::Invalid)?; + if bundle.v != 1 + || row.key != key(&bundle.context.connection_id, &bundle.registration.id) + || row.metadata.account_id != bundle.context.credential_identity + { + return Err(McpError::Denied); + } + Ok(bundle) + } + pub(crate) async fn open_bundle(&self, row: &SealedOAuthCredential) -> Result { + let bundle = self.decode_bundle(row).await?; + self.check_registration(&bundle.registration, &bundle.policies)?; + if let Some(s) = &bundle.client_secret { + if self + .manager + .meta + .get_mcp_org_secret_reservation(&s.name) + .await? + .as_ref() + != Some(s) + { + return Err(McpError::Stale); + } + } + Ok(bundle) + } + pub(crate) async fn oauth_lease( + &self, + expected: &AuthorityContext, + registration_id: &str, + ) -> Result { + let key = key(&expected.connection_id, registration_id); + let mut row = self + .manager + .meta + .get_oauth_credential(&key) + .await? + .ok_or(McpError::Unavailable)?; + // Bounded follower wait. Never reacquire or retry the rotating exchange. + for _ in 0..20 { + if row.broken_reason.as_deref() != Some("refresh_pending") { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + row = self + .manager + .meta + .get_oauth_credential(&key) + .await? + .ok_or(McpError::Unavailable)?; + } + if row.revoked_at.is_some() || row.broken_at.is_some() { + return Err(McpError::Unavailable); + } + let mut bundle = self.open_bundle(&row).await?; + if &bundle.context != expected { + return Err(McpError::Stale); + } + if bundle + .expires_at + .is_some_and(|t| t <= self.manager.clock.now_utc()) + { + let claim = McpRefreshClaim { + id: self.manager.entropy.uuid(), + key: key.clone(), + version: row.version, + expires_at: self.manager.clock.now_utc() + Duration::seconds(25), + }; + let refresh_token = bundle.refresh_token.take().ok_or(McpError::Unavailable)?; + let response = self + .oauth_io( + &bundle.context, + &bundle.registration.token, + "credential-refresh", + WorkerOperation::Token { + endpoint: bundle.registration.token.endpoint.clone(), + grant: TokenGrant::RefreshToken { + refresh_token: refresh_token.clone(), + client_id: bundle.registration.client_id.clone(), + resource: bundle.registration.resource.endpoint.clone(), + client_secret: None, + }, + }, + bundle.client_secret.as_ref(), + Some(&claim), + ) + .await?; + let grant = validate_token( + &response, + &bundle.registration, + bundle.provider_account.as_deref(), + self.manager.clock.now_utc(), + )?; + bundle.access_token = grant.access_token; + bundle.refresh_token = grant.refresh_token.or(Some(refresh_token)); + bundle.expires_at = grant.expires_at; + if let Some(account) = grant.account { + bundle.provider_account = Some(account); + bundle.identity_source = "provider_response".into(); + } + self.manager + .meta + .complete_mcp_refresh(self.credential(&bundle).await?, &claim) + .await?; + } + if bundle + .expires_at + .is_some_and(|t| t <= self.manager.clock.now_utc()) + { + return Err(McpError::Unavailable); + } + Ok(ProviderCredential { + header: "authorization".into(), + value: format!("Bearer {}", bundle.access_token), + }) + } +} +fn status(bundle: &Bundle, row: &SealedOAuthCredential) -> CredentialStatus { + CredentialStatus { + context: bundle.context.clone(), + reference: CredentialReference::OAuth { + registration_id: bundle.registration.id.clone(), + }, + version: row.version, + status: if row.revoked_at.is_some() { + "revoked" + } else if row.broken_at.is_some() { + "broken" + } else { + "connected" + } + .into(), + } +} +struct ValidatedToken { + access_token: String, + refresh_token: Option, + expires_at: Option>, + account: Option, +} +fn token_text(v: &Value) -> Result { + let s = v.as_str().ok_or(McpError::Invalid)?; + if s.is_empty() || s.len() > 8192 || s.contains(['\r', '\n']) { + return Err(McpError::Invalid); + } + Ok(s.into()) +} +fn validate_token( + value: &Value, + r: &ClientRegistration, + account: Option<&str>, + now: DateTime, +) -> Result { + if !value.is_object() + || value.get("error").is_some() + || !value["token_type"] + .as_str() + .is_some_and(|s| s.eq_ignore_ascii_case("bearer")) + { + return Err(McpError::Denied); + } + for (field, expected) in [ + ("iss", &r.issuer), + ("issuer", &r.issuer), + ("resource", &r.resource.endpoint), + ("audience", &r.resource.endpoint), + ] { + if value.get(field).is_some_and(|v| v != expected) { + return Err(McpError::Denied); + } + } + if let Some(scope) = value.get("scope") { + let s = scope.as_str().ok_or(McpError::Invalid)?; + let scopes: std::collections::BTreeSet<_> = + s.split_whitespace().map(str::to_owned).collect(); + if scopes != r.scopes { + return Err(McpError::Denied); + } + } + let provider_account = r + .account_field + .as_ref() + .and_then(|f| value.get(f)) + .map(token_text) + .transpose()?; + if provider_account + .as_ref() + .is_some_and(|p| p.len() > 512 || account.is_some_and(|a| a != p)) + { + return Err(McpError::Denied); + } + let expires_at = value + .get("expires_in") + .map(|v| { + let n = v + .as_i64() + .filter(|n| *n > 0 && *n <= 31_536_000) + .ok_or(McpError::Invalid)?; + Ok::<_, McpError>(now + Duration::seconds(n)) + }) + .transpose()?; + Ok(ValidatedToken { + access_token: token_text(&value["access_token"])?, + refresh_token: value.get("refresh_token").map(token_text).transpose()?, + expires_at, + account: provider_account, + }) +} diff --git a/crates/engram-coordinator/src/mcp/operator.rs b/crates/engram-coordinator/src/mcp/operator.rs new file mode 100644 index 000000000..0eb394500 --- /dev/null +++ b/crates/engram-coordinator/src/mcp/operator.rs @@ -0,0 +1,311 @@ +//! Operator-only publication service and durable readiness driver. +use engram_core::{ + traits::MetadataStore, + types::{managed_boot::ManagedBootHostState, managed_deny::*, managed_operator::*}, +}; +use engram_protocol::mcp as wire; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeSet, sync::Arc, time::Duration}; +use tonic::{Request, Response, Status}; +use wire::operator_publication_service_server::{ + OperatorPublicationService, OperatorPublicationServiceServer, +}; + +pub struct OperatorTls { + pub certificate: Vec, + pub private_key: Vec, + pub client_ca: Vec, + pub fingerprints: Vec, +} +#[derive(Clone)] +struct PublicationService { + meta: Arc, + candidate: OperatorCandidate, + pins: Arc>, + admission: Arc, +} +fn valid_pin(s: &str) -> bool { + s.len() == 95 + && s.split(':').all(|p| { + p.len() == 2 + && p.bytes() + .all(|b| b.is_ascii_digit() || (b'A'..=b'F').contains(&b)) + }) +} +pub fn server( + meta: Arc, + candidate: OperatorCandidate, + tls: OperatorTls, + orchestrator_pins: &[String], +) -> Result { + candidate.identity().map_err(|_| super::McpError::Invalid)?; + if tls.fingerprints.is_empty() + || tls.fingerprints.len() > 8 + || tls + .fingerprints + .iter() + .any(|p| !valid_pin(p) || orchestrator_pins.contains(p)) + { + return Err(super::McpError::Invalid); + } + engram_tls::install_provider(); + let service = PublicationService { + meta, + candidate, + pins: Arc::new(tls.fingerprints.into_iter().collect()), + admission: Arc::new(tokio::sync::Semaphore::new(4)), + }; + Ok(tonic::transport::Server::builder() + .tls_config( + tonic::transport::ServerTlsConfig::new() + .identity(tonic::transport::Identity::from_pem( + tls.certificate, + tls.private_key, + )) + .client_ca_root(tonic::transport::Certificate::from_pem(tls.client_ca)), + ) + .map_err(|_| super::McpError::Invalid)? + .concurrency_limit_per_connection(4) + .max_concurrent_streams(4) + .timeout(Duration::from_secs(10)) + .add_service( + OperatorPublicationServiceServer::new(service) + .max_decoding_message_size(1024) + .max_encoding_message_size(3 * 1024 * 1024), + )) +} +impl PublicationService { + fn check( + &self, + req: &Request, + protocol: u32, + ) -> Result { + let certs = req + .peer_certs() + .ok_or_else(|| Status::unauthenticated("operator certificate required"))?; + let cert = certs + .first() + .ok_or_else(|| Status::unauthenticated("operator certificate required"))?; + let pin = Sha256::digest(cert.as_ref()) + .iter() + .map(|b| format!("{b:02X}")) + .collect::>() + .join(":"); + if !self.pins.contains(&pin) { + return Err(Status::permission_denied("operator certificate denied")); + } + if protocol != wire::PROTOCOL { + return Err(Status::invalid_argument("unsupported operator protocol")); + } + self.admission + .clone() + .try_acquire_owned() + .map_err(|_| Status::resource_exhausted("operator capacity exceeded")) + } +} +fn publication_bytes(value: OperatorPublication) -> Result, Status> { + serde_json::to_vec(&value).map_err(|_| Status::internal("publication encoding failed")) +} +#[tonic::async_trait] +impl OperatorPublicationService for PublicationService { + async fn stage( + &self, + req: Request, + ) -> Result, Status> { + let _permit = self.check(&req, req.get_ref().protocol)?; + let request = req.into_inner(); + if self + .candidate + .identity() + .map_err(|_| Status::unavailable("configured authority unavailable"))? + != request.configured_candidate_identity + { + return Err(Status::permission_denied("configured authority mismatch")); + } + Ok(Response::new( + wire::OperatorPublicationServiceStageResponse { + protocol: wire::PROTOCOL, + publication_json: publication_bytes( + self.meta + .stage_operator_publication( + request.expected_generation, + self.candidate.clone(), + ) + .await + .map_err(|_| Status::failed_precondition("publication changed"))?, + )?, + }, + )) + } + async fn read( + &self, + req: Request, + ) -> Result, Status> { + let _permit = self.check(&req, req.get_ref().protocol)?; + Ok(Response::new( + wire::OperatorPublicationServiceReadResponse { + protocol: wire::PROTOCOL, + publication_json: publication_bytes( + self.meta + .read_operator_publication() + .await + .map_err(|_| Status::unavailable("publication unavailable"))?, + )?, + }, + )) + } +} + +/// Called only after its retention transaction commits. Host evidence is fresh, +/// authenticated by HostClient, and then compared with the same durable snapshot. +async fn fence( + state: &crate::state::SharedState, + g: &OperatorGeneration, + p: &OperatorParticipant, +) -> Result { + if p.dormant() { + return Ok(OperatorReadiness::NeverAllocated); + } + // A live receipt has no durable host/proxy lifecycle authority. Policy-only + // legacy rows remain blockers; they cannot authorize even a host mutation. + let (request, _, prepared) = p.boot.as_ref().ok_or_else(conflict)?; + let key = &request.key; + let target = p.target()?.ok_or_else(conflict)?; + let mut observed_prepared = None; + match state.services.host.read_managed_boot(key.clone()).await? { + ManagedBootHostState::Retired { receipt } => { + return Ok(OperatorReadiness::Retired(receipt)) + } + ManagedBootHostState::Captured { authority } => { + return Ok(OperatorReadiness::Captured(Box::new(authority))) + } + ManagedBootHostState::Prepared { receipt } => { + receipt.validate()?; + if receipt.request != *request { + return Err(conflict().into()); + } + observed_prepared = Some(receipt); + } + ManagedBootHostState::Missing { .. } | ManagedBootHostState::Failed { .. } => { + return Err(conflict().into()) + } + _ => {} + } + let expected = state + .services + .host + .read_managed_policy(target.clone()) + .await?; + // Both a CLOSED bootstrap and a final live owner use the same exact CAS. + // A newer generation retires a managed owner; a rejected call proves nothing. + let context = ManagedPolicyContext { + target: target.clone(), + operator_generation: g.generation, + operator_identity: g.identity.clone(), + binding_hash: expected + .as_ref() + .and_then(|r| r.context.binding_hash.clone()), + }; + let guest_ip = expected + .as_ref() + .and_then(|r| r.network_proof.as_ref().map(|n| n.guest_ip)) + .or_else(|| observed_prepared.as_ref().map(|r| r.guest_ip)) + .or_else(|| prepared.as_ref().map(|r| r.guest_ip)) + .ok_or_else(conflict)?; + let egress = engram_core::types::egress::SessionEgressPolicy { + session_id: p.session_id, + sandbox_id: target.sandbox_id, + guest_ip, + network_allow_hosts: Vec::new(), + network_allow_host_patterns: Vec::new(), + allow_all: false, + secrets: Vec::new(), + injects: Vec::new(), + observes: Vec::new(), + guest_services: Vec::new(), + tunnels: Vec::new(), + secret_mode: engram_core::types::image::SecretMode::Broker, + apps: Vec::new(), + managed_deny: Some(p.policy.clone()), + }; + let _ = state + .services + .host + .apply_managed_policy(ManagedPolicyApply { + expected, + context, + egress, + }) + .await; + match state.services.host.read_managed_boot(key.clone()).await? { + ManagedBootHostState::Retired { receipt } => Ok(OperatorReadiness::Retired(receipt)), + ManagedBootHostState::Captured { authority } => { + Ok(OperatorReadiness::Captured(Box::new(authority))) + } + _ => Err(conflict().into()), + } +} +/// One bounded page per durable pass. No transaction is held across host I/O. +/// The final store CAS rechecks the complete participant set, not this page alone. +pub async fn run_once( + state: &crate::state::SharedState, + configured: &OperatorCandidate, + after: Option, +) -> Result, crate::error::ApiError> { + let publication = state.services.meta.read_operator_publication().await?; + let Some(g) = publication.pending else { + return Ok(None); + }; + if &g.candidate != configured || g.identity != configured.identity()? { + return Err(conflict().into()); + } + let ids = state + .services + .meta + .list_operator_participants(after, 64) + .await?; + for sid in &ids { + let p = state + .services + .meta + .retain_operator_participant(&g, *sid) + .await?; + match tokio::time::timeout(Duration::from_secs(5), fence(state, &g, &p)).await { + Ok(Ok(proof)) => { + state + .services + .meta + .record_operator_readiness(&g, &p, proof) + .await?; + } + _ => { + tracing::debug!("operator publication participant remains fenced from admission"); + } + } + } + if ids.len() == 64 { + return Ok(ids.last().copied()); + } + // A blocked/changed participant leaves the durable generation pending. + match state.services.meta.publish_operator_generation(&g).await { + Ok(_) => {} + Err(engram_core::MetaError::Conflict(_)) => {} + Err(e) => return Err(e.into()), + } + Ok(None) +} +pub async fn drive( + state: crate::state::SharedState, + candidate: OperatorCandidate, + mut shutdown: tokio::sync::watch::Receiver, +) { + let mut after = None; + loop { + tokio::select! { + changed=shutdown.changed()=>{ if changed.is_err() || *shutdown.borrow() { break; } }, + _=tokio::time::sleep(Duration::from_secs(1))=>{ + match run_once(&state,&candidate,after).await { Ok(next)=>after=next,Err(_)=>{after=None;tracing::warn!("operator publication remains closed");} } + } + } + } +} diff --git a/crates/engram-coordinator/src/mcp/policy.rs b/crates/engram-coordinator/src/mcp/policy.rs new file mode 100644 index 000000000..15f27ebcc --- /dev/null +++ b/crates/engram-coordinator/src/mcp/policy.rs @@ -0,0 +1,1018 @@ +//! Trusted policy service. It is never served on application gRPC. +use super::control::Control; +use crate::state::SharedState; +use engram_core::types::managed_create::{ + FrozenNativeManifest, ManagedCreateIdentity, ManagedCreation, +}; +use engram_core::{types::mcp_policy::*, SessionId}; +use engram_protocol::{app, mcp as wire}; +#[cfg(test)] +use prost::Message; +use tonic::{Request, Response, Status}; +use wire::session_policy_service_server::SessionPolicyService; +#[derive(Clone)] +pub(super) struct PolicyControl { + pub control: Control, + pub state: Option, +} +fn id(value: &str) -> Result { + value + .parse() + .map_err(|_| Status::invalid_argument("invalid session")) +} +fn err(_: engram_core::MetaError) -> Status { + Status::failed_precondition("session policy changed or unavailable") +} +fn required(value: Option) -> Result { + value.ok_or_else(|| Status::invalid_argument("required policy field missing")) +} +fn policy_ref(r: WritePolicyRef) -> app::McpWritePolicyRef { + app::McpWritePolicyRef { + session_id: r.session_id.to_string(), + policy_revision: r.policy_revision, + binding_epoch: r.binding_epoch, + } +} +fn from_ref(r: app::McpWritePolicyRef) -> Result { + if r.policy_revision < 0 || r.binding_epoch < 0 { + return Err(Status::invalid_argument("invalid policy reference")); + } + Ok(WritePolicyRef { + session_id: id(&r.session_id)?, + policy_revision: r.policy_revision, + binding_epoch: r.binding_epoch, + }) +} +fn policy(p: TrustedWritePolicy) -> app::McpTrustedWritePolicy { + app::McpTrustedWritePolicy { + r#ref: Some(policy_ref(p.reference)), + write_eligibility: p.write_eligibility.as_str().into(), + cause: p.cause, + accepted_input_id: p.accepted_input_id, + authenticated_actor_id: p.authenticated_actor_id, + prompt_id: p.prompt_id, + plan_tool_call_id: p.plan_tool_call_id, + plan_request_digest: p.plan_request_digest, + ancestor_policy_refs: p.ancestor_policy_refs.into_iter().map(policy_ref).collect(), + current: p.current, + initialized: p.initialized, + } +} +fn review(r: PlanReview) -> app::McpPlanReview { + app::McpPlanReview { + policy_ref: Some(policy_ref(r.policy_ref)), + tool_call_id: r.tool_call_id, + run_id: r.run_id, + event_idx: r.event_idx, + request_digest: r.request_digest, + args_json: r.args_json, + } +} +fn from_review(r: app::McpPlanReview) -> Result { + Ok(PlanReview { + policy_ref: from_ref(required(r.policy_ref)?)?, + tool_call_id: r.tool_call_id, + run_id: r.run_id, + event_idx: r.event_idx, + request_digest: r.request_digest, + args_json: r.args_json, + }) +} +fn eligibility(s: &str) -> Result { + match s { + "unknown" => Ok(WriteEligibility::Unknown), + "read_only" => Ok(WriteEligibility::ReadOnly), + "write_eligible" => Ok(WriteEligibility::WriteEligible), + _ => Err(Status::invalid_argument("invalid write eligibility")), + } +} +fn creation(request: &wire::CreatePolicySessionRequest) -> Result { + if request.actor_id.is_empty() + || request.actor_id.len() > 256 + || request.input_id.is_empty() + || request.input_id.len() > 256 + { + return Err(Status::invalid_argument("creation identity required")); + } + let create = request + .create + .as_ref() + .ok_or_else(|| Status::invalid_argument("creation required"))?; + let eligibility = match create.harness_mode.as_deref() { + None | Some("" | "default") => WriteEligibility::WriteEligible, + Some("plan") => WriteEligibility::ReadOnly, + _ => return Err(Status::invalid_argument("unsupported policy mode")), + }; + Ok(McpPolicyCreation { + actor_id: request.actor_id.clone(), + input_id: request.input_id.clone(), + parent_session_id: request.parent_session_id.as_deref().map(id).transpose()?, + eligibility, + }) +} +fn managed_identity(r: wire::ManagedCreateIdentity) -> Result { + let identity = ManagedCreateIdentity { + session_id: id(&r.session_id)?, + task_id: r.task_id, + input_digest: r.input_digest, + policy: McpPolicyCreation { + actor_id: r.actor_id, + input_id: r.input_id, + parent_session_id: r.parent_session_id.as_deref().map(id).transpose()?, + eligibility: eligibility(&r.write_eligibility)?, + }, + }; + identity.validate().map_err(err)?; + Ok(identity) +} +fn identity_wire(i: ManagedCreateIdentity) -> wire::ManagedCreateIdentity { + wire::ManagedCreateIdentity { + session_id: i.session_id.to_string(), + task_id: i.task_id, + input_id: i.policy.input_id, + actor_id: i.policy.actor_id, + parent_session_id: i.policy.parent_session_id.map(|p| p.to_string()), + write_eligibility: i.policy.eligibility.as_str().into(), + input_digest: i.input_digest, + } +} +fn managed_creation( + r: &wire::ReserveManagedCreateRequest, +) -> Result<(ManagedCreation, app::CreateSessionRequest), Status> { + if r.protocol != wire::PROTOCOL + || r.creation_request.len() > 3 * 1024 * 1024 + || r.input_digest != wire::managed_create_input_digest(r) + { + return Err(Status::invalid_argument("invalid managed creation input")); + } + let policy_request = r + .decode_creation() + .map_err(|_| Status::invalid_argument("invalid creation request"))?; + if policy_request.protocol != wire::PROTOCOL { + return Err(Status::invalid_argument("unsupported control protocol")); + } + let policy = creation(&policy_request)?; + let create = required(policy_request.create)?; + let session_id = id(create + .requested_session_id + .as_deref() + .ok_or_else(|| Status::invalid_argument("reserved session required"))?)?; + if !matches!(create.mode.as_str(), "" | "agent") + || !matches!(create.harness.as_deref(), Some("claude" | "codex")) + { + return Err(Status::invalid_argument( + "managed creation requires a supported agent harness", + )); + } + let bytes = create + .harness_env + .get("ENGRAM_TOOLS") + .ok_or_else(|| Status::invalid_argument("native manifest required"))? + .as_bytes() + .to_vec(); + let frozen = ManagedCreation { + identity: ManagedCreateIdentity { + session_id, + task_id: r.task_id.clone(), + policy, + input_digest: r.input_digest.clone(), + }, + native: FrozenNativeManifest { + version: r.native_version, + bytes, + digest: r.native_digest.clone(), + }, + }; + frozen.identity.validate().map_err(err)?; + frozen.native.validate().map_err(err)?; + Ok((frozen, create)) +} + +/// A missing creation may enter preparation; an existing creation must still be held. +async fn managed_creation_retry( + meta: &dyn engram_core::traits::MetadataStore, + frozen: &ManagedCreation, +) -> Result { + match meta.read_held_managed_creation(&frozen.identity).await { + Ok(existing) if existing == *frozen => Ok(true), + Ok(_) => Err(Status::failed_precondition("creation input changed")), + Err(engram_core::MetaError::NotFound) => Ok(false), + Err(e) => Err(err(e)), + } +} + +/// Read-only preflight. The create transaction repeats this under the generation lock. +async fn require_managed_create_admission( + meta: &dyn engram_core::traits::MetadataStore, +) -> Result<(), Status> { + let publication = meta.read_operator_publication().await.map_err(err)?; + if !publication.admission_open + || publication.pending.is_some() + || publication.published.is_none() + { + return Err(Status::unavailable( + "managed creation admission unavailable", + )); + } + Ok(()) +} + +#[tonic::async_trait] +impl SessionPolicyService for PolicyControl { + async fn read_published_operator( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + let publication = self + .control + .authority + .manager + .meta + .read_operator_publication() + .await + .map_err(err)?; + if !publication.admission_open + || publication.pending.is_some() + || publication.published.is_none() + { + return Err(Status::unavailable("operator publication unavailable")); + } + Ok(Response::new(wire::ReadPublishedOperatorResponse { + protocol: wire::PROTOCOL, + publication_json: serde_json::to_vec(&publication) + .map_err(|_| Status::internal("publication encoding failed"))?, + })) + } + + async fn read_managed_creation_state( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let r = req.into_inner(); + let q = bridge_query(r.identity, r.native_version, r.native_digest)?; + let state = self + .control + .authority + .manager + .meta + .read_managed_creation_state(&q) + .await + .map_err(bridge_err)?; + Ok(Response::new(wire::ReadManagedCreationStateResponse { + protocol: wire::PROTOCOL, + state_json: engram_protocol::managed_boot::encode(&state) + .map_err(|_| Status::internal("state encoding failed"))?, + })) + } + async fn prepare_managed_boot( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let r = req.into_inner(); + let q = bridge_query(r.identity, r.native_version, r.native_digest)?; + let state = self + .state + .as_ref() + .ok_or_else(|| Status::unavailable("managed boot unavailable"))?; + let view = crate::managed_bridge::prepare(state, q) + .await + .map_err(bridge_api_err)?; + Ok(Response::new(wire::PrepareManagedBootResponse { + protocol: wire::PROTOCOL, + state_json: engram_protocol::managed_boot::encode(&view) + .map_err(|_| Status::internal("state encoding failed"))?, + })) + } + async fn recover_managed_boot( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let r = req.into_inner(); + let query = bridge_query(r.identity, r.native_version, r.native_digest)?; + let source = engram_protocol::managed_boot::decode(&r.expected_source_key_json) + .map_err(|_| Status::invalid_argument("invalid recovery source"))?; + let input = engram_core::types::managed_bridge::ManagedRecoveryInput { + query, + expected_source_key: source, + recovery_id: r.recovery_id, + }; + input.validate().map_err(bridge_err)?; + let state = self + .state + .as_ref() + .ok_or_else(|| Status::unavailable("managed boot unavailable"))?; + let view = crate::managed_bridge::recover(state, input) + .await + .map_err(bridge_api_err)?; + Ok(Response::new(wire::RecoverManagedBootResponse { + protocol: wire::PROTOCOL, + state_json: engram_protocol::managed_boot::encode(&view) + .map_err(|_| Status::internal("state encoding failed"))?, + })) + } + async fn read_managed_boot( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + if req.get_ref().protocol != wire::PROTOCOL { + return Err(Status::invalid_argument("unsupported protocol")); + } + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let identity = managed_identity(required(req.into_inner().identity)?)?; + let row = self + .control + .authority + .manager + .meta + .get_managed_boot(&identity) + .await + .map_err(err)?; + Ok(Response::new(wire::ReadManagedBootResponse { + protocol: wire::PROTOCOL, + boot_json: boot_json(row)?, + })) + } + async fn finalize_managed_boot( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + if req.get_ref().protocol != wire::PROTOCOL { + return Err(Status::invalid_argument("unsupported protocol")); + } + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let input = engram_protocol::managed_boot::decode(&req.into_inner().input_json) + .map_err(|_| Status::invalid_argument("invalid managed finalization"))?; + let state = self + .state + .as_ref() + .ok_or_else(|| Status::unavailable("managed boot unavailable"))?; + let row = crate::managed_boot::finalize(state, input) + .await + .map_err(|_| { + Status::failed_precondition("managed boot authority changed or unavailable") + })?; + Ok(Response::new(wire::FinalizeManagedBootResponse { + protocol: wire::PROTOCOL, + boot_json: boot_json(Some(row))?, + })) + } + async fn reserve_managed_create( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let (frozen, create) = managed_creation(req.get_ref())?; + let state = self + .state + .as_ref() + .ok_or_else(|| Status::unavailable("creation unavailable"))?; + let identity = frozen.identity.clone(); + // The exact retry is independent of current image/profile resolution. + if !managed_creation_retry(self.control.authority.manager.meta.as_ref(), &frozen).await? { + require_managed_create_admission(self.control.authority.manager.meta.as_ref()).await?; + let identity_env = create.harness_env.clone().into_iter().collect(); + let mut api = crate::grpc_app::convert::create_request_from_proto(create) + .map_err(|_| Status::invalid_argument("invalid creation request"))?; + api.mcp_creation = Some(identity.policy.clone()); + api.managed_creation = Some(frozen); + crate::api::sessions::create_session_core(state, identity_env, api, None) + .await + .map_err(|_| Status::failed_precondition("session reservation unavailable"))?; + } + Ok(Response::new(wire::ReserveManagedCreateResponse { + protocol: wire::PROTOCOL, + identity: Some(identity_wire(identity)), + state: "held".into(), + })) + } + async fn read_native_manifest( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let identity = managed_identity(required(req.into_inner().identity)?)?; + let stored = self + .control + .authority + .manager + .meta + .read_managed_creation(&identity) + .await + .map_err(err)?; + Ok(Response::new(wire::ReadNativeManifestResponse { + protocol: wire::PROTOCOL, + identity: Some(identity_wire(stored.identity)), + native_version: stored.native.version, + native_bytes: stored.native.bytes, + native_digest: stored.native.digest, + })) + } + + async fn read_managed_policy( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let target: engram_core::types::managed_deny::ManagedPolicyTarget = + required(req.into_inner().target)? + .try_into() + .map_err(Status::invalid_argument)?; + let state = self + .state + .as_ref() + .ok_or_else(|| Status::unavailable("managed policy unavailable"))?; + read_managed_policy(state, &target).await + } + async fn apply_managed_policy( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let r = req.into_inner(); + let context: engram_core::types::managed_deny::ManagedPolicyContext = required(r.context)? + .try_into() + .map_err(Status::invalid_argument)?; + let policy: engram_core::types::managed_deny::ManagedDenyPolicy = required(r.policy)? + .try_into() + .map_err(Status::invalid_argument)?; + let expected = r + .expected + .map(TryInto::try_into) + .transpose() + .map_err(Status::invalid_argument)?; + let state = self + .state + .as_ref() + .ok_or_else(|| Status::unavailable("managed policy unavailable"))?; + apply_managed_policy(state, context, policy, expected).await + } + async fn create_policy_session( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let context = creation(req.get_ref())?; + let r = required(req.into_inner().create)?; + let identity_env = r.harness_env.clone().into_iter().collect(); + let mut api = crate::grpc_app::convert::create_request_from_proto(r) + .map_err(|_| Status::invalid_argument("invalid creation request"))?; + api.mcp_creation = Some(context); + let state = self + .state + .as_ref() + .ok_or_else(|| Status::unavailable("creation unavailable"))?; + let result = crate::api::sessions::create_session_core(state, identity_env, api, None) + .await + .map_err(|_| Status::failed_precondition("session creation unavailable"))?; + Ok(Response::new(wire::CreatePolicySessionResponse { + session: Some(app::CreateSessionResponse { + session_id: result.session_id.to_string(), + status: result.status.into(), + image_version: result.image_version, + kind: result.kind.into(), + }), + })) + } + async fn read_write_policy( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let r = req.into_inner(); + let sid = id(&r.session_id)?; + let meta = &self.control.authority.manager.meta; + let p = meta.read_mcp_write_policy(sid).await.map_err(err)?; + let reason = if !p.initialized { + "External writes are blocked because this session has no trusted initial policy. Start a new supported session." + } else { + match p.write_eligibility { + WriteEligibility::Unknown=>"External writes are blocked because this session or an ancestor has unknown policy.", + WriteEligibility::ReadOnly=>"External writes are blocked by this session or an ancestor. The owner or an administrator must approve the applicable current plan.", + WriteEligibility::WriteEligible=>"External writes still require tool-specific authorization.", + } + }; + let reviewed = match r.plan_tool_call_id { + Some(call) => { + let reviewed = meta.read_mcp_plan_review(sid, &call).await.map_err(err)?; + if reviewed.policy_ref != p.reference { + return Err(Status::failed_precondition( + "session policy changed; reload the plan", + )); + } + Some(review(reviewed)) + } + None => None, + }; + // Review includes its own revision. A restriction between these reads cannot relax policy. + Ok(Response::new(wire::ReadWritePolicyResponse { + state: Some(app::GetMcpWritePolicyResponse { + policy: Some(policy(p)), + plan_review: reviewed, + reason: reason.into(), + allowed_actions: Vec::new(), + }), + })) + } + async fn accept_plan_exit( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let r = req.into_inner(); + let input = PlanAcceptance { + review: from_review(required(r.expected)?)?, + actor_id: r.actor_id, + result_json: r.result_json, + }; + let sid = input.review.policy_ref.session_id; + let p = self + .control + .authority + .manager + .meta + .accept_mcp_plan_exit(&input) + .await + .map_err(err)?; + if let Some(state) = &self.state { + crate::outbox_delivery::enqueue_deliver_op(state, sid).await; + state.outbox_wake.notify_one(); + } + Ok(Response::new(wire::AcceptPlanExitResponse { + policy: Some(policy(p)), + })) + } + async fn validate_dispatch_fence( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let r = req.into_inner(); + let ctx = RequestPolicyContext { + session_id: id(&r.session_id)?, + run_id: r.run_id, + tool_call_id: r.tool_call_id, + accepted_policy_ref: from_ref(required(r.accepted_policy_ref)?)?, + accepted_write_eligibility: eligibility(&r.accepted_write_eligibility)?, + ancestor_policy_refs: r + .ancestor_policy_refs + .into_iter() + .map(from_ref) + .collect::>()?, + }; + if self + .control + .authority + .manager + .meta + .managed_boot_open(ctx.session_id) + .await + .map_err(err)? + == Some(false) + { + return Err(Status::failed_precondition("managed execution gate closed")); + } + let answer = self + .control + .authority + .manager + .meta + .validate_mcp_dispatch_fence( + &ctx, + &ExecutionRequest { + model_tool_name: r.model_tool_name, + original_arguments: r.original_arguments, + }, + ) + .await + .map_err(err)?; + Ok(Response::new(wire::ValidateDispatchFenceResponse { + protocol: wire::PROTOCOL, + status: match answer { + DispatchFence::Current => "current", + DispatchFence::Stale => "stale", + DispatchFence::Unknown => "unknown", + } + .into(), + })) + } + async fn validate_execution_identity( + &self, + req: Request, + ) -> Result, Status> { + let _admission = self.control.check(&req, req.get_ref().protocol)?; + self.control + .authority + .enabled() + .map_err(|_| Status::unavailable("policy unavailable"))?; + let r = req.into_inner(); + let ctx = RequestPolicyContext { + session_id: id(&r.session_id)?, + run_id: r.run_id, + tool_call_id: r.tool_call_id, + accepted_policy_ref: from_ref(required(r.accepted_policy_ref)?)?, + accepted_write_eligibility: eligibility(&r.accepted_write_eligibility)?, + ancestor_policy_refs: r + .ancestor_policy_refs + .into_iter() + .map(from_ref) + .collect::>()?, + }; + if self + .control + .authority + .manager + .meta + .managed_boot_open(ctx.session_id) + .await + .map_err(err)? + == Some(false) + { + return Err(Status::failed_precondition("managed execution gate closed")); + } + let answer = self + .control + .authority + .manager + .meta + .validate_mcp_execution_identity( + &ctx, + &ExecutionRequest { + model_tool_name: r.model_tool_name, + original_arguments: r.original_arguments, + }, + ) + .await + .map_err(err)?; + Ok(Response::new(wire::ValidateExecutionIdentityResponse { + protocol: wire::PROTOCOL, + status: match answer { + DispatchFence::Current => "current", + DispatchFence::Stale => "stale", + DispatchFence::Unknown => "unknown", + } + .into(), + })) + } +} + +async fn read_managed_policy( + state: &SharedState, + target: &engram_core::types::managed_deny::ManagedPolicyTarget, +) -> Result, Status> { + check_managed_target(state, target).await?; + let applied = state + .services + .host + .read_managed_policy(target.clone()) + .await + .map_err(|_| Status::failed_precondition("host policy unavailable"))?; + // Read after host I/O so a concurrent desired commit is retained. No SQL + // transaction crosses host I/O, and persisted bytes never become an ACK. + let retained = crate::api::sessions::load_session_policy(state, target.session_id) + .await + .map_err(|_| Status::failed_precondition("retained policy unavailable"))? + .and_then(|policy| policy.managed_deny); + check_managed_target(state, target).await?; + let response = wire::ReadManagedPolicyResponse { + protocol: wire::PROTOCOL, + applied: applied.map(Into::into), + retained_desired: Some(wire::ManagedRetainedDesired { + target: Some(target.clone().into()), + policy: retained.map(Into::into), + }), + }; + response + .validate_for(target) + .map_err(Status::failed_precondition)?; + Ok(Response::new(response)) +} + +async fn apply_managed_policy( + state: &SharedState, + context: engram_core::types::managed_deny::ManagedPolicyContext, + policy: engram_core::types::managed_deny::ManagedDenyPolicy, + expected: Option, +) -> Result, Status> { + let target = &context.target; + check_managed_target(state, target).await?; + // The desired union commits before any host call. A lost ACK cannot discard it on migration. + state + .services + .meta + .retain_managed_deny(target, &policy) + .await + .map_err(err)?; + let session = state + .services + .meta + .get_session(target.session_id) + .await + .map_err(err)?; + let boot = if state + .services + .meta + .managed_boot_open(target.session_id) + .await + .map_err(err)? + .is_some() + { + let boot = state + .services + .meta + .read_managed_boot_for_target(target) + .await + .map_err(err)?; + validate_prepared_policy_source(&boot, target)?; + Some(boot) + } else { + None + }; + let egress = if let Some(boot) = &boot { + let prepared = boot + .prepared + .as_ref() + .ok_or_else(|| Status::failed_precondition("managed guest identity unavailable"))?; + let policy = crate::api::sessions::load_session_policy(state, target.session_id) + .await + .map_err(|_| Status::failed_precondition("managed egress source unavailable"))?; + crate::api::sessions::build_egress_policy_for_guest( + state, + target.session_id, + target.sandbox_id, + &session.image.to_string(), + prepared.guest_ip, + policy, + ) + .await + .map_err(|_| Status::failed_precondition("managed egress source unavailable"))? + } else { + crate::api::sessions::build_resume_egress_policy( + state, + target.session_id, + target.sandbox_id, + &session.image.to_string(), + ) + .await + .map_err(|_| Status::failed_precondition("managed egress source unavailable"))? + .ok_or_else(|| Status::failed_precondition("managed guest identity unavailable"))? + }; + if !egress.tunnels.is_empty() { + return Err(Status::failed_precondition( + "managed policy does not support tunnels", + )); + } + if egress.managed_deny.as_ref() != Some(&policy) { + return Err(Status::failed_precondition("desired policy changed")); + } + check_managed_target(state, target).await?; + if let Some(boot) = &boot { + let current = state + .services + .meta + .read_managed_boot(&boot.request.key) + .await + .map_err(err)?; + validate_prepared_policy_source(¤t, target)?; + if current.request != boot.request || current.prepared != boot.prepared { + return Err(Status::failed_precondition( + "managed guest identity changed", + )); + } + } + let applied = state + .services + .host + .apply_managed_policy(engram_core::types::managed_deny::ManagedPolicyApply { + expected, + context: context.clone(), + egress, + }) + .await + .map_err(|_| Status::failed_precondition("host policy unavailable"))?; + check_managed_target(state, target).await?; + applied.validate().map_err(Status::failed_precondition)?; + if applied.context != context || applied.policy != policy { + return Err(Status::failed_precondition("host acknowledgement mismatch")); + } + Ok(Response::new(wire::ApplyManagedPolicyResponse { + protocol: wire::PROTOCOL, + applied: Some(applied.into()), + })) +} + +fn validate_prepared_policy_source( + boot: &engram_core::types::managed_boot::ManagedBootRecord, + target: &engram_core::types::managed_deny::ManagedPolicyTarget, +) -> Result<(), Status> { + use engram_core::types::managed_boot::ManagedBootPhase; + let invalid = || Status::failed_precondition("managed guest identity unavailable"); + let prepared = boot.prepared.as_ref().ok_or_else(invalid)?; + prepared.validate().map_err(|_| invalid())?; + if prepared.request != boot.request + || boot.request.key.context.target != *target + || matches!( + boot.phase, + ManagedBootPhase::Claimed | ManagedBootPhase::Interrupted + ) + { + return Err(invalid()); + } + Ok(()) +} + +async fn check_managed_target( + state: &SharedState, + target: &engram_core::types::managed_deny::ManagedPolicyTarget, +) -> Result<(), Status> { + if state + .services + .meta + .managed_policy_target(target.session_id) + .await + .map_err(err)? + != *target + { + return Err(Status::failed_precondition("physical placement changed")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn managed_creation_requires_exact_native_input_and_reserved_identity() { + let sid = "00000000-0000-0000-0000-000000000001"; + let make = |native: Option<&str>| { + let mut create = app::CreateSessionRequest { + requested_session_id: Some(sid.into()), + harness: Some("codex".into()), + ..Default::default() + }; + if let Some(native) = native { + create + .harness_env + .insert("ENGRAM_TOOLS".into(), native.into()); + } + let policy = wire::CreatePolicySessionRequest { + protocol: 1, + actor_id: "owner".into(), + input_id: "create-a".into(), + create: Some(create), + ..Default::default() + }; + let mut r = wire::ReserveManagedCreateRequest { + protocol: 1, + creation_request: policy.encode_to_vec(), + task_id: "task-a".into(), + native_version: 1, + native_digest: engram_core::types::managed_create::digest( + native.unwrap_or("").as_bytes(), + ), + input_digest: String::new(), + }; + r.input_digest = wire::managed_create_input_digest(&r); + r + }; + let original = make(Some(" [ ] ")); + let (frozen, _) = managed_creation(&original).unwrap(); + assert_eq!(frozen.native.bytes, b" [ ] "); + assert_eq!(frozen.identity.session_id.to_string(), sid); + assert!(managed_creation(&make(None)).is_err()); + let mut wrong = original.clone(); + wrong.task_id = "task-b".into(); + assert!(managed_creation(&wrong).is_err()); + let mut wrong = original.clone(); + wrong.native_version = 2; + wrong.input_digest = wire::managed_create_input_digest(&wrong); + assert!(managed_creation(&wrong).is_err()); + let mut wrong = original; + wrong.creation_request.push(0); + assert!(managed_creation(&wrong).is_err()); + } + + #[test] + fn creation_policy_comes_only_from_explicit_supported_creation_input() { + let mut r = wire::CreatePolicySessionRequest { + protocol: 1, + actor_id: "owner-a".into(), + input_id: "create-a".into(), + create: Some(app::CreateSessionRequest::default()), + ..Default::default() + }; + assert_eq!( + creation(&r).unwrap().eligibility, + WriteEligibility::WriteEligible + ); + r.create.as_mut().unwrap().harness_mode = Some("plan".into()); + assert_eq!( + creation(&r).unwrap().eligibility, + WriteEligibility::ReadOnly + ); + r.create.as_mut().unwrap().harness_mode = Some("custom".into()); + assert!(creation(&r).is_err()); + r.actor_id.clear(); + assert!(creation(&r).is_err()); + } +} + +#[cfg(test)] +#[path = "policy_tests.rs"] +mod recovery_tests; + +fn boot_json( + row: Option, +) -> Result>, Status> { + row.map(|mut r| { + r.sealed_agent = None; + engram_protocol::managed_boot::encode(&r) + .map_err(|_| Status::internal("managed boot encoding failed")) + }) + .transpose() +} + +fn bridge_query( + identity: Option, + native_version: u32, + native_digest: String, +) -> Result { + let q = engram_core::types::managed_bridge::ManagedCreationQuery { + identity: managed_identity(required(identity)?)?, + native_version, + native_digest, + }; + q.validate().map_err(bridge_err)?; + Ok(q) +} +fn bridge_err(e: engram_core::MetaError) -> Status { + match e { + engram_core::MetaError::Conflict(_) | engram_core::MetaError::NotFound => { + Status::failed_precondition("managed creation authority changed") + } + _ => Status::unavailable("managed creation state unavailable"), + } +} +fn bridge_api_err(e: crate::error::ApiError) -> Status { + match e { + crate::error::ApiError::Conflict(_) | crate::error::ApiError::NotFound(_) => { + Status::failed_precondition("managed creation authority changed") + } + _ => Status::unavailable("managed creation state unavailable"), + } +} diff --git a/crates/engram-coordinator/src/mcp/policy_tests.rs b/crates/engram-coordinator/src/mcp/policy_tests.rs new file mode 100644 index 000000000..097905031 --- /dev/null +++ b/crates/engram-coordinator/src/mcp/policy_tests.rs @@ -0,0 +1,592 @@ +//! Coordinator recovery through the actual apply/read functions; no guest is started. +use super::*; +use async_trait::async_trait; +use engram_core::traits::{Clock, Entropy, HostClient, MetadataStore, SessionFence}; +use engram_core::types::egress::SessionEgressPolicy; +use engram_core::types::managed_deny::*; +use engram_core::types::sandbox::{ + AgentSpec, ExecRequest as HostExecRequest, ExecStream, SandboxProbe, SandboxSpec, +}; +use engram_core::types::snapshot::SnapshotMetadata; +use engram_core::types::{host::*, SessionSpec}; +use engram_core::{SandboxError, SandboxId}; +use engram_sim::{ManualClock, MemBlobStorage, SimEntropy, SimMetadataStore}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, + }, + time::Duration, +}; + +struct PolicyHost { + prepared: Option, + guest_ip_calls: AtomicUsize, + applied: Mutex>, + entered: tokio::sync::Notify, + release: tokio::sync::Semaphore, +} +#[async_trait] +impl HostClient for PolicyHost { + async fn read_managed_policy( + &self, + _: ManagedPolicyTarget, + ) -> Result, SandboxError> { + Ok(self.applied.lock().unwrap().clone()) + } + async fn apply_managed_policy( + &self, + request: ManagedPolicyApply, + ) -> Result { + let mut current = self.applied.lock().unwrap(); + assert_eq!( + *current, request.expected, + "exact applied receipt must remain the CAS input" + ); + let proof = self.prepared.as_ref().map(|prepared| { + assert_eq!(request.context.target, prepared.request.key.context.target); + assert_eq!(request.egress.guest_ip, prepared.guest_ip); + ManagedNetworkProof { + target: request.context.target.clone(), + guest_ip: prepared.guest_ip, + network_identity: prepared.network_identity.clone(), + } + }); + let ack = ManagedAppliedState::new( + request.context, + request.egress.managed_deny.unwrap(), + "fixture-host".into(), + "fixture-proxy".into(), + proof, + ) + .unwrap(); + *current = Some(ack.clone()); + Ok(ack) + } + async fn create(&self, _spec: SandboxSpec) -> Result { + unreachable!("exec-core tests never create sandboxes") + } + + async fn destroy(&self, _id: SandboxId, _fence: SessionFence) -> Result<(), SandboxError> { + unreachable!("exec-core tests never destroy sandboxes") + } + + async fn list(&self) -> Result, SandboxError> { + unreachable!("exec-core tests never list sandboxes") + } + + async fn probe_sandbox(&self, _id: SandboxId) -> Result { + unreachable!("exec-core tests never probe sandboxes") + } + + async fn exec_stream( + &self, + id: SandboxId, + cmd: HostExecRequest, + ) -> Result { + let _ = (id, cmd); + unreachable!("readiness never executes commands") + } + + async fn snapshot( + &self, + _id: SandboxId, + _fence: SessionFence, + ) -> Result { + unreachable!("exec-core tests never snapshot sandboxes") + } + + async fn restore( + &self, + _metadata: SnapshotMetadata, + _fence: SessionFence, + ) -> Result { + unreachable!("exec-core tests never restore sandboxes") + } + + async fn start_agent( + &self, + _id: SandboxId, + _agent: AgentSpec, + _policy: SessionEgressPolicy, + _fence: SessionFence, + ) -> Result<(), SandboxError> { + unreachable!("policy control never starts guests") + } + + async fn guest_ip(&self, _id: SandboxId) -> Option { + self.guest_ip_calls.fetch_add(1, Ordering::SeqCst); + if self.prepared.is_some() { + return None; + } + self.entered.notify_one(); + self.release.acquire().await.unwrap().forget(); + Some("10.200.0.42".parse().unwrap()) + } + + async fn bind_session( + &self, + _session_id: SessionId, + _sandbox_id: SandboxId, + _binding_epoch: u64, + ) -> Result<(), engram_core::SandboxError> { + Err(SandboxError::Vm("bind rejected".into())) + } + + async fn unbind_session(&self, _session_id: SessionId) {} + + async fn send_prompt( + &self, + _sandbox_id: SandboxId, + _prompt_id: String, + _text: String, + _mode: Option, + ) -> Result<(), SandboxError> { + unreachable!("exec-core tests never send prompts") + } +} + +async fn retained_recovery(meta: Arc, clock: Arc) { + let entropy = Arc::new(SimEntropy::seeded(111)); + let sid = meta + .create_session(SessionSpec { + image: "test.invalid/policy:latest".into(), + mode: Default::default(), + }) + .await + .unwrap(); + let host_id = engram_core::HostId::from(entropy.uuid()); + let sandbox = SandboxId::from(entropy.uuid()); + meta.upsert_host(host_record(host_id, "fixture-host", clock.now_utc())) + .await + .unwrap(); + meta.assign_session_host(sid, Some(host_id)).await.unwrap(); + let binding_epoch = meta.transition_session_created(sid, sandbox).await.unwrap(); + let target = ManagedPolicyTarget { + session_id: sid, + sandbox_id: sandbox, + host_id, + binding_epoch, + }; + let context = ManagedPolicyContext { + target: target.clone(), + operator_generation: 1, + operator_identity: format!("sha256:{}", "a".repeat(64)), + binding_hash: None, + }; + let one = ManagedDenyPolicy::new(1, ["a.example".into()]).unwrap(); + meta.retain_managed_deny(&target, &one).await.unwrap(); + let initial_ack = ManagedAppliedState::new( + context.clone(), + one.clone(), + "fixture-host".into(), + "fixture-proxy".into(), + None, + ) + .unwrap(); + let host = Arc::new(PolicyHost { + prepared: None, + guest_ip_calls: AtomicUsize::new(0), + applied: Mutex::new(Some(initial_ack.clone())), + entered: Default::default(), + release: tokio::sync::Semaphore::new(0), + }); + let blob = Arc::new(MemBlobStorage::new()); + let services = crate::Services { + meta: meta.clone(), + host: host.clone(), + host_pool: Arc::new(engram_protocol::grpc_pool::GrpcHostPool::new()), + secrets: Arc::new(engram_secrets_dev::InMemorySecretStore::new()), + kek: Arc::new(engram_crypto::EnvVarKeyProvider::from_bytes( + [0u8; 32], "test:v1", + )), + oci: Arc::new(engram_oci::OciClient::new(Arc::new( + engram_oci::AnonymousResolver, + ))), + auth_resolver: Arc::new(engram_oci::AnonymousResolver), + blob: blob.clone(), + chunk_store: engram_chunk_store::ChunkStore::new(blob), + materialize_dir: None, + clock, + entropy, + }; + let state = Arc::new(crate::AppState::new( + crate::CoordinatorConfig::default(), + services, + )); + // Explicit wrapper + absent policy is authoritative absence for a fresh row. + let fresh = meta + .create_session(SessionSpec { + image: "test.invalid/fresh:latest".into(), + mode: Default::default(), + }) + .await + .unwrap(); + meta.assign_session_host(fresh, Some(host_id)) + .await + .unwrap(); + let fresh_sandbox = SandboxId::from(state.services.entropy.uuid()); + let fresh_epoch = meta + .transition_session_created(fresh, fresh_sandbox) + .await + .unwrap(); + let fresh_target = ManagedPolicyTarget { + session_id: fresh, + sandbox_id: fresh_sandbox, + host_id, + binding_epoch: fresh_epoch, + }; + *host.applied.lock().unwrap() = None; + let empty = read_managed_policy(&state, &fresh_target) + .await + .unwrap() + .into_inner(); + empty.validate_for(&fresh_target).unwrap(); + assert!(empty.applied.is_none()); + assert!(empty.retained_desired.unwrap().policy.is_none()); + *host.applied.lock().unwrap() = Some(initial_ack.clone()); + let three = ManagedDenyPolicy::new(3, ["a.example".into(), "orphan.example".into()]).unwrap(); + let pending = tokio::spawn({ + let state = state.clone(); + let context = context.clone(); + let three = three.clone(); + let expected = initial_ack.clone(); + async move { apply_managed_policy(&state, context, three, Some(expected)).await } + }); + tokio::time::timeout(Duration::from_secs(3), host.entered.notified()) + .await + .unwrap(); + // The real apply path is suspended at guest-IP host I/O after desired commit. + // A concurrent writer on the same row must complete while that I/O is held. + tokio::time::timeout( + Duration::from_secs(3), + meta.retain_managed_deny(&target, &three), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(*host.applied.lock().unwrap(), Some(initial_ack.clone())); + pending.abort(); + assert!(pending.await.unwrap_err().is_cancelled()); + for applied in [Some(initial_ack.clone()), None] { + *host.applied.lock().unwrap() = applied.clone(); + let response = read_managed_policy(&state, &target) + .await + .unwrap() + .into_inner(); + assert_eq!( + response + .applied + .map(ManagedAppliedState::try_from) + .transpose() + .unwrap(), + applied + ); + let retained = response + .retained_desired + .expect("complete read must expose the committed orphan union"); + assert_eq!( + ManagedPolicyTarget::try_from(retained.target.unwrap()).unwrap(), + target + ); + assert_eq!( + ManagedDenyPolicy::try_from(retained.policy.unwrap()).unwrap(), + three + ); + } + // A fresh consumer chooses max(retained, applied) + 1 and preserves both unions. + let read = read_managed_policy(&state, &target) + .await + .unwrap() + .into_inner(); + let retained: ManagedDenyPolicy = read + .retained_desired + .unwrap() + .policy + .unwrap() + .try_into() + .unwrap(); + let expected = read + .applied + .map(ManagedAppliedState::try_from) + .transpose() + .unwrap(); + let revision = retained + .revision + .max(expected.as_ref().map_or(0, |ack| ack.policy.revision)) + + 1; + let four = ManagedDenyPolicy::new( + revision, + retained.hosts.into_iter().chain(["next.example".into()]), + ) + .unwrap(); + host.release.add_permits(1); + let retry = apply_managed_policy(&state, context, four.clone(), expected) + .await + .unwrap() + .into_inner(); + let ack = ManagedAppliedState::try_from(retry.applied.unwrap()).unwrap(); + assert_eq!(ack.policy, four); + assert_eq!(ack.policy.revision, 4); + let response = read_managed_policy(&state, &target) + .await + .unwrap() + .into_inner(); + assert_eq!( + ManagedDenyPolicy::try_from(response.retained_desired.unwrap().policy.unwrap()).unwrap(), + four + ); +} + +#[tokio::test] +async fn retained_desired_recovers_cancelled_apply_sim() { + let clock = ManualClock::new(); + let meta = SimMetadataStore::new(clock.clone(), Arc::new(SimEntropy::seeded(222))); + retained_recovery(meta, clock).await; +} +#[tokio::test] +async fn retained_desired_recovers_cancelled_apply_pg() { + let Some(db) = engram_testkit::pg::fresh_db().await else { + return; + }; + let clock = ManualClock::new(); + let meta = Arc::new( + db.store + .clone() + .with_clock(clock.clone()) + .with_entropy(Arc::new(SimEntropy::seeded(222))), + ); + retained_recovery(meta, clock).await; +} +fn host_record( + id: engram_core::HostId, + name: &str, + now: chrono::DateTime, +) -> HostRecord { + HostRecord { + id, + hostname: name.to_string(), + cloud_metadata: Default::default(), + capacity: HostCapacity { + total_gb: 100, + used_gb: 0, + total_mib: 32_768, + used_mib: 0, + running_sandboxes: 0, + }, + utilization: Default::default(), + status: HostStatus::Ready, + last_heartbeat_at: now, + host_addr: None, + ready_images: Vec::new(), + current_bundles: Vec::new(), + sandbox_bundles: Vec::new(), + cordoned: false, + cordon_owner: None, + cordon_reason: None, + retire_requested_at: None, + retired_at: None, + total_vcpus: 16, + wire_version: 1, + stages_images: false, + capabilities: Default::default(), + lease_expires_at: None, + lease_state: Default::default(), + lease_epoch: 0, + } +} + +#[tokio::test] +async fn managed_retry_denies_requeued_nonzero_epoch() { + use engram_core::types::{ + managed_create::digest, runtime_spec::RuntimeSpec, BindingDisposition, SessionState, + }; + let meta = SimMetadataStore::new(ManualClock::new(), Arc::new(SimEntropy::seeded(333))); + let id = SessionId::from(uuid::Uuid::from_u128(333)); + let policy = McpPolicyCreation { + actor_id: "owner-a".into(), + input_id: "create-a".into(), + parent_session_id: None, + eligibility: WriteEligibility::ReadOnly, + }; + let frozen = ManagedCreation { + identity: ManagedCreateIdentity { + session_id: id, + task_id: "task-a".into(), + policy: policy.clone(), + input_digest: digest(b"original"), + }, + native: FrozenNativeManifest { + version: 1, + bytes: b"[]".to_vec(), + digest: digest(b"[]"), + }, + }; + assert!(!managed_creation_retry(meta.as_ref(), &frozen) + .await + .unwrap()); + let ws = engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: Some(frozen.clone()), + mcp_policy: Some(policy), + session_id: id, + spec: SessionSpec { + image: "test.invalid/managed:latest".into(), + mode: Default::default(), + }, + mem_budget_mib: 512, + cpu_budget_vcpus: 1, + sealed_secrets: None, + capabilities: Vec::new(), + integration_policy_json: None, + runtime_spec: RuntimeSpec::new(Vec::new(), Some("claude".into()), None, Vec::new()), + oauth_binding: None, + }; + engram_testkit::operator::publish_empty(meta.as_ref()).await; + meta.reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + assert!(managed_creation_retry(meta.as_ref(), &frozen) + .await + .unwrap()); + meta.transition_session( + id, + SessionState::Pending, + BindingDisposition::RequireUnbound, + ) + .await + .unwrap(); + assert_eq!( + meta.transition_session_created(id, SandboxId::from(uuid::Uuid::from_u128(334))) + .await + .unwrap(), + 1 + ); + meta.transition_session(id, SessionState::Idle, BindingDisposition::Detach) + .await + .unwrap(); + meta.transition_session(id, SessionState::Queued, BindingDisposition::RequireUnbound) + .await + .unwrap(); + let session = meta.get_session(id).await.unwrap(); + assert_eq!(session.status, SessionState::Queued); + assert!(session.host_id.is_none() && session.sandbox_id.is_none()); + assert_eq!( + meta.read_managed_creation(&frozen.identity).await.unwrap(), + frozen + ); + assert!(meta.reserve_and_persist_create(ws, &[], 0).await.is_err()); + assert_eq!( + managed_creation_retry(meta.as_ref(), &frozen) + .await + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); + assert_eq!( + { + let empty = + SimMetadataStore::new(ManualClock::new(), Arc::new(SimEntropy::seeded(334))); + require_managed_create_admission(empty.as_ref()) + .await + .unwrap_err() + .code() + }, + tonic::Code::Unavailable + ); +} + +#[tokio::test] +async fn managed_prepared_policy_uses_receipt_without_generic_guest_ip() { + let (mut state, meta, row) = crate::managed_boot::tests::prepared_policy_fixture().await; + let prepared = row.prepared.clone().unwrap(); + let mut context = row.request.key.context.clone(); + context.binding_hash = Some(engram_core::types::managed_create::digest(b"binding")); + let policy = + ManagedDenyPolicy::new(2, ["tools.example".into(), "next.example".into()]).unwrap(); + let host = Arc::new(PolicyHost { + prepared: Some(prepared.clone()), + guest_ip_calls: AtomicUsize::new(0), + applied: Mutex::new(None), + entered: Default::default(), + release: tokio::sync::Semaphore::new(0), + }); + Arc::get_mut(&mut state).unwrap().services.host = host.clone(); + assert_eq!( + meta.get_session(context.target.session_id) + .await + .unwrap() + .status, + engram_core::types::SessionState::Created + ); + assert!(state + .services + .host + .guest_ip(context.target.sandbox_id) + .await + .is_none()); + assert_eq!(host.guest_ip_calls.swap(0, Ordering::SeqCst), 1); + let response = apply_managed_policy(&state, context.clone(), policy.clone(), None) + .await + .unwrap() + .into_inner(); + let applied = ManagedAppliedState::try_from(response.applied.unwrap()).unwrap(); + assert_eq!(applied.context, context); + assert_eq!(applied.policy, policy); + assert_eq!(applied.network_proof.unwrap().guest_ip, prepared.guest_ip); + assert_eq!(host.guest_ip_calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn managed_prepared_policy_rejects_changed_receipt_or_owner() { + use engram_core::types::managed_boot::ManagedBootPhase; + for change in [ + "target", + "owner", + "request", + "receipt", + "interrupted", + "missing", + ] { + let (mut state, meta, row) = crate::managed_boot::tests::prepared_policy_fixture().await; + let mut context = row.request.key.context.clone(); + context.binding_hash = Some(engram_core::types::managed_create::digest(b"binding")); + let host = Arc::new(PolicyHost { + prepared: row.prepared.clone(), + guest_ip_calls: AtomicUsize::new(0), + applied: Mutex::new(None), + entered: Default::default(), + release: tokio::sync::Semaphore::new(0), + }); + Arc::get_mut(&mut state).unwrap().services.host = host.clone(); + meta.with_db_mut(|db| { + let boot = db + .managed_boots + .get_mut(&context.target.session_id) + .unwrap(); + match change { + "target" => boot.request.key.context.target.binding_epoch += 1, + "owner" => boot.request.key.creation.task_id = "other-owner".into(), + "request" => { + boot.prepared.as_mut().unwrap().request.key.boot_attempt = "stale".into() + } + "receipt" => { + boot.prepared.as_mut().unwrap().guest_ip = std::net::Ipv4Addr::UNSPECIFIED + } + "interrupted" => boot.phase = ManagedBootPhase::Interrupted, + "missing" => boot.prepared = None, + _ => unreachable!(), + } + }); + let result = + apply_managed_policy(&state, context, row.request.bootstrap_policy, None).await; + assert!(result.is_err(), "accepted {change} managed authority"); + assert!(host.applied.lock().unwrap().is_none()); + assert_eq!( + host.guest_ip_calls.load(Ordering::SeqCst), + 0, + "used generic identity for {change}" + ); + } +} diff --git a/crates/engram-coordinator/src/mcp/startup.rs b/crates/engram-coordinator/src/mcp/startup.rs new file mode 100644 index 000000000..7a05e0bf1 --- /dev/null +++ b/crates/engram-coordinator/src/mcp/startup.rs @@ -0,0 +1,193 @@ +//! Operator-owned files configure the dedicated listener and independent zones. +use super::{ + control::{self, ControlTls}, + model::Registry, + worker::{WorkerClient, WorkerTransport}, + McpAuthority, McpError, SystemSecurityEntropy, +}; +use serde::Deserialize; +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, + sync::Arc, +}; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct OperatorConfig { + registry: Registry, + listen_addr: std::net::SocketAddr, + server_certificate_file: PathBuf, + server_private_key_file: PathBuf, + orchestrator_ca_file: PathBuf, + orchestrator_fingerprints: Vec, + workers: BTreeMap, + #[serde(default)] + publication: Option, +} +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct WorkerFiles { + url: String, + server_ca_file: PathBuf, + client_identity_file: PathBuf, +} +async fn bounded_file(path: &Path) -> super::Result> { + if tokio::fs::metadata(path) + .await + .map_err(|_| McpError::Invalid)? + .len() + > 128 * 1024 + { + return Err(McpError::Invalid); + } + let bytes = tokio::fs::read(path).await.map_err(|_| McpError::Invalid)?; + if bytes.len() > 128 * 1024 { + return Err(McpError::Invalid); + } + Ok(bytes) +} +pub async fn start( + path: Option<&Path>, + manager: Arc, + shutdown: tokio::sync::watch::Receiver, +) -> super::Result>> { + start_with_policy(path, manager, shutdown, None).await +} +pub async fn start_with_policy( + path: Option<&Path>, + manager: Arc, + mut shutdown: tokio::sync::watch::Receiver, + state: Option, +) -> super::Result>> { + let Some(path) = path else { return Ok(None) }; + let config: OperatorConfig = + serde_json::from_slice(&bounded_file(path).await?).map_err(|_| McpError::Invalid)?; + if !config.registry.enabled { + return start_publication( + config.publication, + &config.orchestrator_fingerprints, + state, + shutdown, + ) + .await; + } + let operator_pins = config.orchestrator_fingerprints.clone(); + let mut workers = BTreeMap::new(); + for (id, files) in config.workers { + let client = WorkerClient::new( + &files.url, + &bounded_file(&files.server_ca_file).await?, + &bounded_file(&files.client_identity_file).await?, + )?; + workers.insert(id, Arc::new(client) as Arc); + } + let authority = Arc::new(McpAuthority::new( + manager, + Arc::new(SystemSecurityEntropy), + config.registry, + workers, + )?); + let tls = ControlTls { + certificate: bounded_file(&config.server_certificate_file).await?, + private_key: bounded_file(&config.server_private_key_file).await?, + client_ca: bounded_file(&config.orchestrator_ca_file).await?, + orchestrator_fingerprints: config.orchestrator_fingerprints, + }; + let router = control::server_with_policy(authority, tls, state.clone())?; + let listener = tokio::net::TcpListener::bind(config.listen_addr) + .await + .map_err(|_| McpError::Unavailable)?; + let publication = + start_publication(config.publication, &operator_pins, state, shutdown.clone()).await?; + Ok(Some(tokio::spawn(async move { + let stop = async move { + while !*shutdown.borrow_and_update() { + if shutdown.changed().await.is_err() { + break; + } + } + }; + if router + .serve_with_incoming_shutdown( + tonic::transport::server::TcpIncoming::from(listener), + stop, + ) + .await + .is_err() + { + tracing::error!("MCP control listener stopped"); + } + if let Some(publication) = publication { + publication.abort(); + let _ = publication.await; + } + }))) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct PublicationFiles { + listen_addr: std::net::SocketAddr, + server_certificate_file: PathBuf, + server_private_key_file: PathBuf, + operator_ca_file: PathBuf, + operator_fingerprints: Vec, + candidate_file: PathBuf, + ca_files: BTreeMap, +} +async fn start_publication( + files: Option, + orchestrator_pins: &[String], + state: Option, + shutdown: tokio::sync::watch::Receiver, +) -> super::Result>> { + let Some(files) = files else { return Ok(None) }; + let state = state.ok_or(McpError::Unavailable)?; + let candidate: engram_core::types::managed_operator::OperatorCandidate = + serde_json::from_slice(&bounded_file(&files.candidate_file).await?) + .map_err(|_| McpError::Invalid)?; + candidate.identity().map_err(|_| McpError::Invalid)?; + if files.ca_files.len() != candidate.ca_content_digests.len() { + return Err(McpError::Invalid); + } + for (id, path) in files.ca_files { + let actual = engram_core::types::managed_create::digest(&bounded_file(&path).await?); + if candidate.ca_content_digests.get(&id) != Some(&actual) { + return Err(McpError::Invalid); + } + } + let tls = super::operator::OperatorTls { + certificate: bounded_file(&files.server_certificate_file).await?, + private_key: bounded_file(&files.server_private_key_file).await?, + client_ca: bounded_file(&files.operator_ca_file).await?, + fingerprints: files.operator_fingerprints, + }; + let router = super::operator::server( + state.services.meta.clone(), + candidate.clone(), + tls, + orchestrator_pins, + )?; + let listener = tokio::net::TcpListener::bind(files.listen_addr) + .await + .map_err(|_| McpError::Unavailable)?; + Ok(Some(tokio::spawn(async move { + let mut stop = shutdown.clone(); + let serving = router.serve_with_incoming_shutdown( + tonic::transport::server::TcpIncoming::from(listener), + async move { + while !*stop.borrow_and_update() { + if stop.changed().await.is_err() { + break; + } + } + }, + ); + // If either producer exits, drop the other. No detached scanner survives it. + tokio::select! { + result=serving=>{if result.is_err(){tracing::error!("operator listener stopped");}}, + _=super::operator::drive(state,candidate,shutdown)=>{}, + } + }))) +} diff --git a/crates/engram-coordinator/src/mcp/worker.rs b/crates/engram-coordinator/src/mcp/worker.rs new file mode 100644 index 000000000..5d4887713 --- /dev/null +++ b/crates/engram-coordinator/src/mcp/worker.rs @@ -0,0 +1,275 @@ +//! Worker protocol 1. Operations are fixed; no provider HTTP proxy is exposed. +use super::{McpError, Result}; +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::time::Duration; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct WorkerIdentity { + pub protocol: u32, + pub boot_id: String, + pub zone_id: String, + pub policy_digest: String, + pub policy_revision: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase", deny_unknown_fields)] +pub enum WorkerScope { + #[serde(rename_all = "camelCase")] + Administration { + request_id: String, + actor_id: String, + }, + #[serde(rename_all = "camelCase")] + Execution { + session_id: String, + run_id: String, + tool_call_id: String, + binding_hash: String, + }, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct WorkerLease { + pub id: String, + pub deployment_id: String, + pub expires_at: i64, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct WorkerIntent { + pub id: String, + pub boot_id: String, + pub zone_id: String, + pub policy_digest: String, + pub connection_id: String, + pub connection_revision: i64, + pub credential_identity: String, + pub scope: WorkerScope, + pub operation: String, + pub destination: String, + pub request_digest: String, + pub operation_digest: String, + pub expires_at: i64, + pub lease: WorkerLease, +} +// These types intentionally have no Debug implementation: token fields are private. +#[derive(Serialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum WorkerOperation { + Metadata { endpoint: String }, + ValidateDestination { endpoint: String }, + Token { endpoint: String, grant: TokenGrant }, +} +#[derive(Serialize)] +#[serde(tag = "grant_type")] +pub enum TokenGrant { + #[serde(rename = "authorization_code")] + AuthorizationCode { + code: String, + redirect_uri: String, + code_verifier: String, + client_id: String, + resource: String, + #[serde(skip_serializing_if = "Option::is_none")] + client_secret: Option, + }, + #[serde(rename = "refresh_token")] + RefreshToken { + refresh_token: String, + client_id: String, + resource: String, + #[serde(skip_serializing_if = "Option::is_none")] + client_secret: Option, + }, +} +impl WorkerOperation { + pub fn kind(&self) -> &'static str { + match self { + Self::Metadata { .. } => "metadata", + Self::ValidateDestination { .. } => "validateDestination", + Self::Token { .. } => "token", + } + } + pub fn endpoint(&self) -> &str { + match self { + Self::Metadata { endpoint } + | Self::ValidateDestination { endpoint } + | Self::Token { endpoint, .. } => endpoint, + } + } +} + +/// This encoder is deliberately limited to the fixed coordinator wire: strings, +/// booleans, null and safe integers. No floating point enters these requests. +/// Keys follow JavaScript UTF-16 ordering, including non-BMP names in vectors. +pub fn canonical(value: &Value) -> Result { + Ok(match value { + Value::Null => "null".into(), + Value::Bool(v) => v.to_string(), + Value::String(v) => serde_json::to_string(v).map_err(|_| McpError::Invalid)?, + Value::Number(v) => { + let n = v.as_i64().ok_or(McpError::Invalid)?; + if !(-9_007_199_254_740_991..=9_007_199_254_740_991).contains(&n) { + return Err(McpError::Invalid); + } + n.to_string() + } + Value::Array(v) => format!( + "[{}]", + v.iter() + .map(canonical) + .collect::>>()? + .join(",") + ), + Value::Object(v) => { + let mut keys: Vec<_> = v.keys().collect(); + keys.sort_by(|a, b| a.encode_utf16().cmp(b.encode_utf16())); + let fields = keys + .into_iter() + .map(|k| { + Ok(format!( + "{}:{}", + serde_json::to_string(k).map_err(|_| McpError::Invalid)?, + canonical(&v[k])? + )) + }) + .collect::>>()?; + format!("{{{}}}", fields.join(",")) + } + }) +} +pub fn digest(value: &Value) -> Result { + Ok(hex::encode(Sha256::digest(canonical(value)?.as_bytes()))) +} + +#[async_trait] +pub trait WorkerTransport: Send + Sync { + async fn identity(&self) -> Result; + async fn rpc( + &self, + identity: &WorkerIdentity, + intent: &WorkerIntent, + operation: &WorkerOperation, + ) -> Result; + async fn check_identity(&self, expected: &WorkerIdentity) -> Result<()> { + if &self.identity().await? != expected { + return Err(McpError::Stale); + } + Ok(()) + } +} +pub struct WorkerClient { + client: reqwest::Client, + base: String, +} +impl WorkerClient { + pub fn new(base: &str, ca: &[u8], identity: &[u8]) -> Result { + let url = url::Url::parse(base).map_err(|_| McpError::Invalid)?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || url.path() != "/" + { + return Err(McpError::Invalid); + } + let certs = reqwest::Certificate::from_pem_bundle(ca).map_err(|_| McpError::Invalid)?; + if certs.is_empty() { + return Err(McpError::Invalid); + } + let client = engram_tls::client_builder() + .tls_certs_only(certs) + .identity(reqwest::Identity::from_pem(identity).map_err(|_| McpError::Invalid)?) + .tls_version_min(reqwest::tls::Version::TLS_1_3) + .https_only(true) + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()) + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(10)) + .tls_sslkeylogfile(false) + .build() + .map_err(|_| McpError::Transport)?; + Ok(Self { + client, + base: base.trim_end_matches('/').into(), + }) + } + async fn json(response: reqwest::Response) -> Result { + if !response.status().is_success() { + return Err(McpError::Transport); + } + let mut response = response; + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(|_| McpError::Transport)? { + if bytes.len() + chunk.len() > 2 * 1024 * 1024 { + return Err(McpError::Invalid); + } + bytes.extend_from_slice(&chunk); + } + serde_json::from_slice(&bytes).map_err(|_| McpError::Invalid) + } +} +#[async_trait] +impl WorkerTransport for WorkerClient { + async fn identity(&self) -> Result { + let v = Self::json( + self.client + .get(format!("{}/identity", self.base)) + .send() + .await + .map_err(|_| McpError::Transport)?, + ) + .await?; + let identity: WorkerIdentity = serde_json::from_value(v).map_err(|_| McpError::Invalid)?; + if identity.protocol != 1 || identity.boot_id.is_empty() { + return Err(McpError::Denied); + } + Ok(identity) + } + async fn rpc( + &self, + identity: &WorkerIdentity, + intent: &WorkerIntent, + operation: &WorkerOperation, + ) -> Result { + let op = serde_json::to_value(operation).map_err(|_| McpError::Invalid)?; + if intent.boot_id != identity.boot_id + || intent.zone_id != identity.zone_id + || intent.policy_digest != identity.policy_digest + || intent.operation != operation.kind() + || intent.destination != operation.endpoint() + || intent.operation_digest != digest(&op)? + { + return Err(McpError::Denied); + } + let body = serde_json::json!({"protocol":1,"intent":intent,"operation":op}); + let response = Self::json( + self.client + .post(format!("{}/rpc", self.base)) + .json(&body) + .send() + .await + .map_err(|_| McpError::Transport)?, + ) + .await?; + if response["ok"] != true + || response["identity"] + != serde_json::to_value(identity).map_err(|_| McpError::Invalid)? + || response["intent"] != body["intent"] + || response["intentId"] != intent.id + || response["requestDigest"] != intent.request_digest + || response["operationDigest"] != intent.operation_digest + || response["leaseId"] != intent.lease.id + { + return Err(McpError::Denied); + } + response.get("result").cloned().ok_or(McpError::Invalid) + } +} diff --git a/crates/engram-coordinator/src/oauth.rs b/crates/engram-coordinator/src/oauth.rs index 4d02b6a1a..03e45be75 100644 --- a/crates/engram-coordinator/src/oauth.rs +++ b/crates/engram-coordinator/src/oauth.rs @@ -316,15 +316,28 @@ impl OAuthManager { .await?) } - pub async fn fetch_session( + async fn guest_binding( &self, session_id: SessionId, - ) -> Result<(String, i64, Vec), OAuthServiceError> { + ) -> Result { let binding = self .meta .get_session_oauth_binding(session_id) .await? .ok_or(OAuthServiceError::NotFound)?; + if binding.key.subject_kind == OAuthSubjectKind::Mcp { + return Err(OAuthServiceError::BadRequest( + "MCP credentials are not available to guest sessions".into(), + )); + } + Ok(binding) + } + + pub async fn fetch_session( + &self, + session_id: SessionId, + ) -> Result<(String, i64, Vec), OAuthServiceError> { + let binding = self.guest_binding(session_id).await?; let row = self .meta .get_oauth_credential(&binding.key) @@ -344,11 +357,7 @@ impl OAuthManager { expected_version: i64, payload: &[u8], ) -> Result<(String, i64, Vec), OAuthServiceError> { - let binding = self - .meta - .get_session_oauth_binding(session_id) - .await? - .ok_or(OAuthServiceError::NotFound)?; + let binding = self.guest_binding(session_id).await?; let current = self .meta .get_oauth_credential(&binding.key) @@ -389,11 +398,7 @@ impl OAuthManager { expected_version: i64, reason: &str, ) -> Result<(), OAuthServiceError> { - let binding = self - .meta - .get_session_oauth_binding(session_id) - .await? - .ok_or(OAuthServiceError::NotFound)?; + let binding = self.guest_binding(session_id).await?; match self .meta .mark_oauth_credential_broken(&binding.key, expected_version, reason) @@ -645,6 +650,11 @@ impl From for OAuthServiceError { } pub(crate) fn validate_key(key: &OAuthCredentialKey) -> Result<(), OAuthServiceError> { + if key.subject_kind == OAuthSubjectKind::Mcp { + return Err(OAuthServiceError::BadRequest( + "MCP credentials require the dedicated control service".into(), + )); + } // Provider charset matches the connector-slug rule (`new_relic` carries // an underscore). if key.subject_id.trim().is_empty() @@ -1092,6 +1102,22 @@ async fn read_bounded_private(path: &Path) -> Result, OAuthDriverError> #[cfg(test)] mod tests { + #[test] + fn generic_oauth_acquisition_rejects_mcp_subjects() { + for kind in [ + OAuthSubjectKind::User, + OAuthSubjectKind::Connector, + OAuthSubjectKind::UserConnector, + OAuthSubjectKind::Mcp, + ] { + let key = OAuthCredentialKey { + subject_kind: kind, + subject_id: "connection-a".into(), + provider: "registered-a".into(), + }; + assert_eq!(validate_key(&key).is_ok(), kind != OAuthSubjectKind::Mcp); + } + } use super::*; #[cfg(unix)] diff --git a/crates/engram-coordinator/src/oauth_refresh.rs b/crates/engram-coordinator/src/oauth_refresh.rs index e1f491103..237db81e3 100644 --- a/crates/engram-coordinator/src/oauth_refresh.rs +++ b/crates/engram-coordinator/src/oauth_refresh.rs @@ -392,6 +392,11 @@ impl OAuthManager { &self, key: &OAuthCredentialKey, ) -> Result { + if key.subject_kind == OAuthSubjectKind::Mcp { + return Err(OAuthServiceError::BadRequest( + "MCP credentials require the dedicated control service".into(), + )); + } let row = self .meta .get_oauth_credential(key) diff --git a/crates/engram-coordinator/src/org_secrets.rs b/crates/engram-coordinator/src/org_secrets.rs index 46adb4b35..36d6c0029 100644 --- a/crates/engram-coordinator/src/org_secrets.rs +++ b/crates/engram-coordinator/src/org_secrets.rs @@ -39,18 +39,50 @@ impl OrgSecretBackend { #[async_trait] impl SecretStore for OrgSecretBackend { + async fn check_access(&self, name: &str, schema: &SecretSchema) -> Result<(), SecretError> { + for reference in [ + Some(name), + schema + .r#ref + .as_deref() + .filter(|reference| *reference != name), + ] + .into_iter() + .flatten() + { + if self + .meta + .get_mcp_org_secret_reservation(reference) + .await + .map_err(|e| SecretError::Backend(Box::new(e)))? + .is_some() + { + return Err(SecretError::Unauthorized( + "credential is reserved for MCP".into(), + )); + } + } + Ok(()) + } + async fn get( &self, _ctx: &SecretContext<'_>, name: &str, - _schema: &SecretSchema, + schema: &SecretSchema, ) -> Result, SecretError> { - let Some(sealed) = self + self.check_access(name, schema).await?; + let sealed = self .meta .get_org_secret_sealed(name) .await - .map_err(|e| SecretError::Backend(Box::new(e)))? - else { + .map_err(|e| SecretError::Backend(Box::new(e)))?; + // A fresh name can be reserved and published after the first check. + // Purpose is permanent, and these are current reads from the same + // store: any MCP payload observed above must have a reservation here. + // Check absent payloads too, before allowing a deployment fallback. + self.check_access(name, schema).await?; + let Some(sealed) = sealed else { return Ok(None); }; let cred = to_sealed_cred(&sealed)?; diff --git a/crates/engram-coordinator/src/outbox_delivery.rs b/crates/engram-coordinator/src/outbox_delivery.rs index 6704082d2..0a3df684e 100644 --- a/crates/engram-coordinator/src/outbox_delivery.rs +++ b/crates/engram-coordinator/src/outbox_delivery.rs @@ -42,6 +42,13 @@ const RESCAN_INTERVAL: Duration = Duration::from_secs(2); /// `pub` so the DST harness (engram-dst, ADR 0098 D6) drives outbox /// delivery like the spawn loop does. pub async fn enqueue_deliver_op(state: &SharedState, session_id: engram_core::SessionId) { + if crate::managed_boot::gate(state, session_id, true) + .await + .is_err() + { + return; + } + match state .services .meta diff --git a/crates/engram-coordinator/src/placement.rs b/crates/engram-coordinator/src/placement.rs index a0ffdfec4..486a31ed1 100644 --- a/crates/engram-coordinator/src/placement.rs +++ b/crates/engram-coordinator/src/placement.rs @@ -106,12 +106,9 @@ pub struct CapabilityRequirements { /// source blackout). pub needs_live_substrate: bool, /// The snapshot row's recorded capture-time FC snapshot version, - /// when known (`SnapshotRecord::fc_snapshot_version`). `Some(v)` - /// requires the candidate host's reported `fc_snapshot_version` to - /// equal `v` exactly — closes the cross-`SNAPSHOT_VERSION` restore- - /// corruption class at placement instead of at guest-boot failure. - /// `None` (pre-migration snapshot rows, VZ/Process) imposes no - /// constraint. + /// when known (`SnapshotRecord::fc_snapshot_version`). When both the + /// snapshot and candidate host report a version, they must match exactly. + /// Either side being `None` imposes no constraint. pub fc_snapshot_version: Option, } @@ -272,9 +269,9 @@ impl From for SandboxError { } } -/// ADR 0046/0047: ranked candidates for `reserve_placement` — the -/// snapshot-affinity hosts form the prefix (`hosts[..affinity_len]`), -/// the rest follow in row order. +/// Ranked candidates for `reserve_placement`, with bundle-covered hosts first +/// and row order preserved within each group. `affinity_len` is zero; +/// `pick_from` applies snapshot affinity after capacity checks. #[derive(Clone, Debug, Default)] pub struct RankedCandidates { pub hosts: Vec, @@ -474,20 +471,15 @@ pub fn exclusion_summary( .collect() } -/// Pure ranking: filter to schedulable candidates and put the -/// snapshot-affinity hosts first. Row order (PK-sorted from -/// `list_active_hosts`) breaks ties deterministically. +/// Filter to schedulable candidates and put bundle-covered hosts first. +/// Row order (PK-sorted from `list_active_hosts`) is preserved within each group. pub fn rank_hosts( hosts: &[HostRecord], ctx: &ScheduleContext<'_>, now: DateTime, ttl: Duration, ) -> RankedCandidates { - // ADR 0078: the authoritative `snapshot_host` tier-0 is decided in - // `pick_from` (it needs the reserved-budget map for the RAM/CPU + - // disk-headroom veto). `rank_hosts` just yields the schedulable - // candidate set in row order; there is no longer a capacity-blind - // affinity prefix baked in here (the dead `local_snapshots` tier). + // `pick_from` applies snapshot affinity after RAM, CPU, and disk checks. let mut ranked: Vec = Vec::new(); // ADR 0090: bundle-covered hosts first (soft, stable — row order // preserved within each tier). Uncovered hosts still rank; they just @@ -722,8 +714,8 @@ async fn hosts_and_reserved( /// that fits BOTH budgets? A HARD 2D check — unlike `pick_from`'s /// capacity-soft fallback — so a drain doesn't start a move that would /// strand the session when the fleet is genuinely full. An unmeasured -/// host (no allocatable / no reported vCPU) counts as fitting (same soft -/// posture `reserve_placement` takes for brand-new / dev hosts). +/// dimension (no allocatable RAM / no reported vCPU) counts as fitting; +/// each measured dimension must still fit. pub async fn placement_preview( meta: &dyn MetadataStore, ctx: &ScheduleContext<'_>, @@ -738,12 +730,11 @@ pub async fn placement_preview( return false; }; let alloc = h.utilization.allocatable_mib as i64; - if alloc <= 0 { - return true; // unmeasured → soft fallback fits - } - let free_mib = alloc - reserved.get(id).map(|r| r.mem_mib).unwrap_or(0); - if free_mib < mem_mib { - return false; + if alloc > 0 { + let free_mib = alloc - reserved.get(id).map(|r| r.mem_mib).unwrap_or(0); + if free_mib < mem_mib { + return false; + } } let cpu_budget = engram_core::types::host::host_cpu_budget(h.total_vcpus); if cpu_budget > 0 { @@ -1749,9 +1740,8 @@ mod tests { assert_eq!(ranked.hosts, vec![hid(4)]); } - /// ADR 0078 tier 0: an alive, headroom-OK `snapshot_host` is picked - /// authoritatively — even over a host with more free RAM (h1) that - /// best-fit would otherwise prefer. The bytes are on h2's NVMe. + /// An alive `snapshot_host` with headroom is picked authoritatively. + /// The bytes are on h2's NVMe; best-fit also prefers h2's smaller free RAM. #[test] fn authoritative_snapshot_host_wins_when_healthy() { let mut h1 = host(1); @@ -2023,15 +2013,10 @@ mod tests { #[test] fn wire_version_skewed_host_is_drained_from_scheduling() { - // Issue #229: a host reporting a NONZERO wire version that differs - // from the coordinator's is excluded from placement, so a - // non-atomic rolling deploy drains off stale hosts instead of - // hard-failing sessions on them. A version of 0 (not yet reported) - // and the coordinator's own version both stay schedulable. - let coord = engram_protocol::WIRE_VERSION; - + // Ordinary hosts remain eligible through the wire 38 release window. + // A managed protocol marker is not an ordinary placement version. let mut skewed = host(1); - skewed.wire_version = coord + 1; + skewed.wire_version = 43; assert!( !host_wire_version_ok(&skewed), "a nonzero mismatched version must be excluded", @@ -2047,7 +2032,7 @@ mod tests { assert!(host_is_schedulable(¬_reported, Utc::now(), TTL)); let mut matched = host(3); - matched.wire_version = coord; + matched.wire_version = 38; assert!(host_wire_version_ok(&matched)); assert!(host_is_schedulable(&matched, Utc::now(), TTL)); diff --git a/crates/engram-coordinator/src/session_boot.rs b/crates/engram-coordinator/src/session_boot.rs index b085393d5..59c0989b5 100644 --- a/crates/engram-coordinator/src/session_boot.rs +++ b/crates/engram-coordinator/src/session_boot.rs @@ -142,6 +142,7 @@ pub(crate) struct BootInputs { /// its response). Built by `sessions::prepare_from_request` (from a live /// request) or `sessions::prepare_from_row` (from a durable queued row). pub(crate) struct PreparedBoot { + pub managed_selection: Option, pub inputs: BootInputs, pub memory_mib: u32, pub cpu_budget_vcpus: u32, @@ -195,10 +196,14 @@ pub(crate) async fn boot_on_reserved_host( host_id: HostId, // ADR 0079: the caller's fencing epoch, stamped into every session- // scoped host RPC below. The create_boot verb threads its op fence; - // the direct (capacity-available) create pipeline is out-of-op and - // passes `SessionFence::unfenced()`. + // the direct (capacity-available) create pipeline preserves the + // session identity with epoch zero, without claiming an op epoch. fence: SessionFence, ) -> Result<(), BootError> { + crate::managed_boot::gate(state, inputs.session_id, false) + .await + .map_err(BootError::NotStarted)?; + let BootInputs { session_id, spec, @@ -363,6 +368,9 @@ pub(crate) async fn boot_on_reserved_host( let egress_secret_count = egress_secrets.len(); let inject_count = injects.len(); let resolved_policy = ResolvedEgressPolicy { + managed_deny: integration_policy + .as_ref() + .and_then(|p| p.managed_deny.clone()), secrets: egress_secrets, injects, observes, @@ -493,6 +501,15 @@ pub(crate) async fn boot_on_reserved_host( // the next thing that trips this is as invisible as the VZ boot race // was (a 2s guest-IP timeout that lost to a 2.5s guest boot; fixed in // `engram-sandbox-vz`'s `guest_endpoints`, which now waits). + if egress_policy.is_none() + && integration_policy + .as_ref() + .is_some_and(|p| p.managed_deny.is_some()) + { + return Err(BootError::NotStarted(ApiError::BadRequest( + "managed policy requires an enforceable guest identity".into(), + ))); + } let policy = egress_policy.unwrap_or_else(|| { tracing::warn!( %session_id, @@ -505,6 +522,7 @@ pub(crate) async fn boot_on_reserved_host( guest traffic as UnknownGuest and brokered credentials will not substitute", ); engram_core::types::egress::SessionEgressPolicy { + managed_deny: None, session_id, sandbox_id, guest_ip: std::net::Ipv4Addr::UNSPECIFIED, @@ -674,6 +692,7 @@ fn map_restore_error(e: engram_core::SandboxError) -> ApiError { /// IP (process backend / some VZ configs) — the caller synthesizes an /// unspecified-IP fallback. struct ResolvedEgressPolicy { + managed_deny: Option, secrets: Vec, injects: Vec, observes: Vec, @@ -692,6 +711,7 @@ async fn assemble_egress_policy( ) -> Option { let guest_ip = state.services.host.guest_ip(sandbox_id).await?; Some(engram_core::types::egress::SessionEgressPolicy { + managed_deny: resolved.managed_deny, session_id, sandbox_id, guest_ip, @@ -753,6 +773,7 @@ pub(crate) fn assemble_capture_egress_policy( return None; } Some(engram_core::types::egress::SessionEgressPolicy { + managed_deny: None, session_id, // Stamped by the host at registration (the capture VM doesn't // exist yet when this is assembled). @@ -878,15 +899,15 @@ pub(crate) async fn resolve_inject_entries( .get_session_capabilities(session_id) .await { - Ok(c) => c, + Ok(c) => Some(c), Err(e) => { tracing::warn!(%session_id, error = %e, - "capabilities lookup failed under a mint inject; minting unscoped"); - Vec::new() + "capabilities lookup failed; skipping mint injections"); + None } } } else { - Vec::new() + None }; let mut out = Vec::with_capacity(policy.injects.len()); for inj in &policy.injects { @@ -896,11 +917,14 @@ pub(crate) async fn resolve_inject_entries( continue; } let entry = if let Some(mint_source) = &inj.mint_source { + let Some(caps) = caps.as_deref() else { + continue; + }; // Minted: the GATING is policy-owned (this entry's hosts/methods/paths), // but the HEADER (name + rendered value) is the integration's — so the // auth scheme is the provider's, not hardcoded by the policy compiler. // The scoped credential never enters the guest. - match mint_inject_header(state, session_id, mint_source, &caps).await { + match mint_inject_header(state, session_id, mint_source, caps).await { Some((h, expires_at)) => { // Connection mints return a pre-rendered header (name + // full value; template collapses to "{}"). Connector @@ -1269,7 +1293,7 @@ pub(crate) async fn mint_remote_connection_credential( /// inject/refresh`) runs when the proxy's minted credential nears expiry. Fetches /// the session's bound capabilities (the same scope `resolve_inject_entries` uses /// at boot) and re-runs the mint. Returns the freshly rendered header + its new -/// `expires_at`, or `None` (logged in `mint_inject_header`) on any failure — the +/// `expires_at`, or `None` on any lookup or mint failure — the /// proxy then keeps the stale secret rather than failing the request. pub(crate) async fn refresh_inject_header( state: &SharedState, @@ -1279,12 +1303,19 @@ pub(crate) async fn refresh_inject_header( engram_core::traits::InjectHeader, chrono::DateTime, )> { - let caps = state + let caps = match state .services .meta .get_session_capabilities(session_id) .await - .unwrap_or_default(); + { + Ok(caps) => caps, + Err(e) => { + tracing::warn!(%session_id, error = %e, + "capabilities lookup failed; refusing mint refresh"); + return None; + } + }; mint_inject_header(state, session_id, source, &caps).await } @@ -1391,3 +1422,146 @@ mod capture_egress_tests { ); } } + +#[cfg(test)] +pub(crate) mod credential_tests { + use std::sync::Arc; + + use engram_core::traits::{Clock, CredentialHint, Integration, ScopedCredential}; + use engram_core::types::{Capability, IntegrationPolicy, Session, SessionMode, SessionState}; + use engram_sim::{SimClock, SimEntropy, SimMetadataStore}; + use parking_lot::Mutex; + + use super::*; + + pub(crate) fn fixture() -> ( + SharedState, + Arc, + Session, + tempfile::TempDir, + ) { + let clock = SimClock::new(); + let entropy = Arc::new(SimEntropy::seeded(71)); + let meta = SimMetadataStore::new(clock.clone(), entropy.clone()); + let session = Session { + id: SessionId::from(uuid::Uuid::from_u128(71)), + status: SessionState::Queued, + host_id: None, + sandbox_id: None, + image: "test/credential-scope:v1".into(), + mode: SessionMode::DevVm, + created_at: clock.now_utc(), + last_active_at: clock.now_utc(), + last_event_at: None, + park_rung: 0, + parked_at: None, + suggested_title: None, + }; + let (mut state, _, dir) = crate::state::tests::build_state_for_session(session.clone()); + let app = Arc::get_mut(&mut state).unwrap(); + app.services.meta = meta.clone(); + app.services.clock = clock.clone(); + app.services.entropy = entropy; + app.boot_bundles = Arc::new(crate::boot_bundle::BootBundleCache::new(clock)); + (state, meta, session, dir) + } + + #[derive(Default)] + struct RecordingMint(Mutex>>); + + #[async_trait::async_trait] + impl Integration for RecordingMint { + fn provider(&self) -> &str { + "github" + } + + async fn mint_credential( + &self, + caps: &[Capability], + _: &CredentialHint, + ) -> Result { + self.0.lock().push(caps.to_vec()); + Ok(ScopedCredential::Bearer { + token: "minted-token".into(), + expires_at: chrono::DateTime::from_timestamp(1_800_000_000, 0).unwrap(), + }) + } + } + + fn mint_source() -> CredentialMintSource { + CredentialMintSource::Connection { + connection_id: "test-connection".into(), + provider: "github".into(), + } + } + + fn policy() -> IntegrationPolicy { + serde_json::from_value(serde_json::json!({ + "injects": [ + { + "hosts": ["api.github.com"], "header_name": "Authorization", + "header_template": "Bearer {}", "mint_source": mint_source() + }, + { + "hosts": ["static.example.com"], "header_name": "X-Api-Key", + "header_template": "{}", "secret_ref": "static-key" + } + ] + })) + .unwrap() + } + + fn install_mint(state: &mut SharedState) -> Arc { + let mint = Arc::new(RecordingMint::default()); + let app = Arc::get_mut(state).unwrap(); + app.integrations = crate::integrations::IntegrationBroker::with(mint.clone()); + app.services.secrets = Arc::new(engram_secrets_dev::InMemorySecretStore::with_secrets([( + "static-key", + "static-token", + )])); + mint + } + + #[tokio::test(start_paused = true)] + async fn capability_read_failure_skips_mint_but_keeps_static_injection() { + let (mut state, meta, session, _dir) = fixture(); + let mint = install_mint(&mut state); + meta.set_outage(true); + let entries = + resolve_inject_entries(&state, session.id, Some(&policy()), &session.image).await; + assert!( + mint.0.lock().is_empty(), + "failed scope reads must never invoke the mint" + ); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].secret, "static-token"); + } + + #[tokio::test(start_paused = true)] + async fn capability_read_failure_prevents_refresh_mint() { + let (mut state, meta, session, _dir) = fixture(); + let mint = install_mint(&mut state); + meta.set_outage(true); + assert!(refresh_inject_header(&state, session.id, &mint_source()) + .await + .is_none()); + assert!(mint.0.lock().is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn successful_empty_capabilities_still_allow_boot_and_refresh_mints() { + let (mut state, _meta, session, _dir) = fixture(); + let mint = install_mint(&mut state); + let entries = + resolve_inject_entries(&state, session.id, Some(&policy()), &session.image).await; + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].secret, "Bearer minted-token"); + assert!(refresh_inject_header(&state, session.id, &mint_source()) + .await + .is_some()); + assert_eq!( + mint.0.lock().as_slice(), + &[Vec::::new(), Vec::new()] + ); + } +} diff --git a/crates/engram-coordinator/src/session_verbs.rs b/crates/engram-coordinator/src/session_verbs.rs index b3460b17b..a25f75111 100644 --- a/crates/engram-coordinator/src/session_verbs.rs +++ b/crates/engram-coordinator/src/session_verbs.rs @@ -14,6 +14,15 @@ use crate::session_ops::{OpCtx, OpOutcome}; use crate::state::{SessionEvent, SharedState}; pub async fn dispatch(ctx: &OpCtx<'_>) -> OpOutcome { + if ctx.op.kind != OpKind::Destroy { + if let Err(error) = + crate::managed_boot::gate(ctx.state, ctx.op.session_id, ctx.op.kind == OpKind::Deliver) + .await + { + return outcome_from_api_error(error); + } + } + match ctx.op.kind { OpKind::Resume => resume(ctx).await, OpKind::Evict => evict(ctx).await, @@ -1190,10 +1199,35 @@ async fn forward_outbox_row( let state = ctx.state; if retire_legacy_answer_row(state, row) .await - .map_err(ForwardDeferral::backoff)? + .map_err(|e| ForwardDeferral::backoff(e.to_string()))? { return Ok(()); } + crate::managed_boot::gate(state, row.session_id, true) + .await + .map_err(|e| ForwardDeferral::backoff(e.to_string()))?; + if row.kind == OutboxKind::ToolResult { + let call = row + .payload + .get("tool_call_id") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + if state + .services + .meta + .managed_tool_target_gone(row.session_id, call) + .await + .map_err(|e| ForwardDeferral::backoff(e.to_string()))? + { + state + .services + .meta + .outbox_ack(&row.prompt_id) + .await + .map_err(|e| ForwardDeferral::backoff(e.to_string()))?; + return Ok(()); + } + } let Some(sandbox_id) = state .resolve_sandbox(row.session_id) .instrument(tracing::info_span!("coord.deliver.resolve_sandbox")) diff --git a/crates/engram-coordinator/src/state.rs b/crates/engram-coordinator/src/state.rs index e009ac098..cf31ba112 100644 --- a/crates/engram-coordinator/src/state.rs +++ b/crates/engram-coordinator/src/state.rs @@ -200,9 +200,17 @@ pub enum SessionEvent { args_json: String, at: DateTime, }, + HarnessToolCallCancelRequested { + run_id: String, + tool_call_id: String, + name: String, + at: DateTime, + }, HarnessRunCompleted { run_id: String, ok: bool, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + deferred_tool_call_ids: Vec, at: DateTime, }, /// ADR 0030: the in-flight run was stopped by an operator interrupt @@ -531,6 +539,7 @@ impl SessionEvent { Self::HarnessToolCallCompleted { .. } => "tool_call_completed", Self::HarnessBrowserActivity { .. } => "browser_activity", Self::HarnessToolCallRequested { .. } => "tool_call_requested", + Self::HarnessToolCallCancelRequested { .. } => "tool_call_cancel_requested", Self::HarnessRunCompleted { .. } => "run_completed", Self::HarnessRunInterrupted { .. } => "run_interrupted", Self::HarnessRunContinued { .. } => "run_continued", @@ -634,9 +643,31 @@ impl SessionEvent { args_json, at, }, - HarnessEvent::RunCompleted { run_id, ok } => { - Self::HarnessRunCompleted { run_id, ok, at } - } + HarnessEvent::ToolCallCancelRequested { + run_id, + call_id, + name, + } => Self::HarnessToolCallCancelRequested { + run_id, + tool_call_id: call_id, + name, + at, + }, + HarnessEvent::RunCompleted { run_id, ok } => Self::HarnessRunCompleted { + run_id, + ok, + deferred_tool_call_ids: Vec::new(), + at, + }, + HarnessEvent::RunDeferred { + run_id, + deferred_tool_call_ids, + } => Self::HarnessRunCompleted { + run_id, + ok: true, + deferred_tool_call_ids, + at, + }, HarnessEvent::RunContinued { run_id } => Self::HarnessRunContinued { run_id, at }, HarnessEvent::RunInterrupted { run_id } => Self::HarnessRunInterrupted { run_id, @@ -822,13 +853,13 @@ impl PreviewConnLimiter { /// cap. The map entry is pruned when a session's last permit drops, so the /// map only ever holds sessions with live previews. pub fn try_acquire(self: &Arc, session: SessionId) -> Option { - let sem = self + let entry = self .per_session .entry(session) - .or_insert_with(|| Arc::new(tokio::sync::Semaphore::new(self.cap))) - .value() - .clone(); - let permit = sem.try_acquire_owned().ok()?; + .or_insert_with(|| Arc::new(tokio::sync::Semaphore::new(self.cap))); + // Hold the entry guard until acquisition so pruning cannot detach this semaphore. + let permit = entry.value().clone().try_acquire_owned().ok()?; + drop(entry); Some(PreviewPermit { limiter: Arc::clone(self), session, @@ -847,11 +878,8 @@ pub struct PreviewPermit { impl Drop for PreviewPermit { fn drop(&mut self) { - // Release the slot FIRST so `available_permits` reflects this drop, - // THEN prune the entry iff no live previews remain for the session. - // `remove_if` is atomic per key: a concurrent `try_acquire` either - // keeps the entry (it holds a permit → predicate false) or re-creates a - // fresh one after removal — never a lost or double-counted slot. + // Acquisition and pruning hold the same entry lock, so an acquired + // permit always keeps its semaphore in the map. drop(self.permit.take()); self.limiter.per_session.remove_if(&self.session, |_, sem| { sem.available_permits() >= self.limiter.cap @@ -1129,14 +1157,8 @@ impl AppState { .meta .append_session_event(session, kind, payload) .await?; - // ADR 0073: ack any outbox row a DIRECTLY-EMITTED confirming event - // retires. NOTE: the confirming events (`run_started`/`prompt_queued`/ - // `tool_call_completed`) are HARNESS events, and those ingest via - // `harness_event_sink` → `append_session_event`, NOT this `emit` — so - // they ack THERE (see the ack in `harness_event_sink`). This arm is the - // defensive catch for any confirming event authored/replayed straight - // through `emit`; a confirming event terminally retires the matching - // outbox row, and unknown / already-acked ids are no-ops (at-least-once). + // Directly emitted confirming events also retire their outbox row. + // The harness sink handles normal delivery acknowledgements. if let Some(ack_id) = outbox_ack_id(session, &event) { match self.services.meta.outbox_ack(&ack_id).await { Ok(true) => { @@ -1336,22 +1358,18 @@ pub type SharedState = Arc; pub async fn emit_harness_event( state: &SharedState, session_id: SessionId, - _sandbox_id: engram_core::SandboxId, + sandbox_id: engram_core::SandboxId, event: engram_harness_proto::HarnessEvent, _at: chrono::DateTime, delivery: Option, ) -> Result<(), crate::error::ApiError> { state .harness_event_writer - .emit(session_id, event, delivery) + .ingest(session_id, sandbox_id, event, delivery) .await } -/// Build the [`EventSink`] that forwards harness events into -/// `session_events` and triggers Track C.9 auto-checkpoints on -/// `Idle` / `RunCompleted` for Git sessions. Captures clones of -/// the bus + meta service + sandbox backend so the closure has no -/// cycles back into AppState. +/// Build an event sink with its own duplicate-event state for tests. #[cfg(test)] fn harness_event_sink( events: Arc, @@ -1395,31 +1413,97 @@ impl HarnessEventWriter { /// withholds the acknowledgement and drops the connection, so the /// harness replays the event (ADR 0123 C4). fn sink(self: Arc) -> EventSink { - Arc::new(move |session, _sandbox, event, delivery| { + Arc::new(move |session, sandbox, event, delivery| { let writer = self.clone(); Box::pin(async move { writer - .emit(session, event, delivery) + .ingest(session, sandbox, event, delivery) .await .map_err(|e| engram_host_agent::harness::SinkError(e.to_string())) }) }) } + async fn wake_waiting_harness(&self, session_id: SessionId, kind: &str) { + if matches!(kind, "harness_idle" | "harness_parked" | "harness_busy") { + let attach_wake = ingest_step_span!("coord.harness_event.attach_wake"); + // ADR 0108 A8: a harness that just announced Idle can + // take a prompt NOW — a row waiting out ACK_TIMEOUT + // from a forward into a dead link has no reason to + // keep waiting. Without this recall the wake below + // finds nothing due and no-ops; the row then sits out + // the full timeout after a fast reattach. + if let Err(e) = self + .meta + .outbox_make_due(session_id) + .instrument(attach_wake.clone()) + .await + { + tracing::debug!( + session_id = %session_id, + error = %e, + "outbox make-due on attach signal failed (ack timeout backstops)", + ); + } + if let Err(e) = self + .meta + .op_wake_queued_kind(session_id, engram_core::types::session_op::OpKind::Deliver) + .instrument(attach_wake) + .await + { + tracing::debug!( + session_id = %session_id, + error = %e, + "deliver wake on attach signal failed (poll backstops)", + ); + } + // The shim scan enqueues a Deliver op when rows are due + // and none is queued; a wake with nothing due is a + // no-op. + self.outbox_wake.notify_one(); + } + } + + async fn ingest( + &self, + session: SessionId, + sandbox: engram_core::SandboxId, + event: engram_harness_proto::HarnessEvent, + delivery: Option, + ) -> Result<(), crate::error::ApiError> { + let managed_target = if let Some(open) = self.meta.managed_boot_open(session).await? { + let target = self.meta.managed_policy_target(session).await?; + if !open + || target.sandbox_id != sandbox + || delivery.as_ref().map(|d| d.binding_epoch) != Some(target.binding_epoch) + { + return Err(engram_core::MetaError::Conflict( + "managed event authority is closed".into(), + ) + .into()); + } + Some(target) + } else { + None + }; + self.emit(session, event, delivery, managed_target).await + } + async fn emit( &self, session_id: SessionId, ev: engram_harness_proto::HarnessEvent, delivery: Option, + managed_target: Option, ) -> Result<(), crate::error::ApiError> { let Self { events, meta, clock, - outbox_wake, + outbox_wake: _, last_kind, } = self; - if let Some(delivery) = delivery { + if let Some(delivery) = delivery.filter(|_| managed_target.is_none()) { // ADR 0123 C5: the first event of a new generation settles the // open runs of older generations, except the run this event // itself continues (a re-attached harness keeps running it). @@ -1461,11 +1545,6 @@ impl HarnessEventWriter { ); } } - // Forward every harness event into session_events for live - // SSE / Web UI / Slackbot timeline. ADR 0005 retired the - // auto-checkpoint branch this used to trigger on Idle / - // RunCompleted; durability moved to hot+cold snapshots, - // not git checkpoints. let session_event = SessionEvent::from_harness(ev, clock.now_utc()); let kind = session_event.kind(); @@ -1486,17 +1565,9 @@ impl HarnessEventWriter { None }; - // ADR 0073 fix: harness events are the CONFIRMING events that retire - // the durable outbox row (`run_started{prompt_id}` / - // `prompt_queued{prompt_id}` / `tool_call_completed{tool_call_id}`), - // but they ingest through THIS sink — NOT `AppState::emit`, where the - // ack lived — so the ack never fired. An un-acked row is redelivered - // forever: the delivery driver re-resumes the session and re-runs the - // prompt on every idle cycle (acked_at NULL, attempts climbing; - // phantom re-runs + resume/evict churn + a duplicate-turn transcript - // the web can't render). Capture the ack id here (before - // `session_event` moves into the published frame) and retire the row - // after the append. Unknown / already-acked ids are no-ops. + // Managed events retire their outbox row in the append transaction. + // Ordinary events acknowledge after publication so the write does not + // delay the live frame. Unknown or acknowledged ids are no-ops. let ack_id = outbox_ack_id(session_id, &session_event); // Session titles: a harness-suggested title is materialized onto @@ -1512,50 +1583,10 @@ impl HarnessEventWriter { None }; - // ADR 0108 A3: an Idle/Parked/Busy announcement is the ATTACH - // signal — the harness (re)announced its waiting state on a - // live connection. Wake any backed-off Deliver op now: - // delivery keys on the attach, never on a boot milestone or - // a poll cadence (the 2026-07-31 50 s gap was a Deliver op - // racing the attach and then waiting out an inflated - // backoff). This MUST run before the back-to-back dedupe - // below: a re-announce after a reattach is exactly the - // signal the incident waited 41 s for, and it is exactly - // the shape the dedupe drops from the log. - if matches!(kind, "harness_idle" | "harness_parked" | "harness_busy") { - let attach_wake = ingest_step_span!("coord.harness_event.attach_wake"); - // ADR 0108 A8: a harness that just announced Idle can - // take a prompt NOW — a row waiting out ACK_TIMEOUT - // from a forward into a dead link has no reason to - // keep waiting. Without this recall the wake below - // finds nothing due and no-ops; the row then sits out - // the full timeout. - if let Err(e) = meta - .outbox_make_due(session_id) - .instrument(attach_wake.clone()) - .await - { - tracing::debug!( - session_id = %session_id, - error = %e, - "outbox make-due on attach signal failed (ack timeout backstops)", - ); - } - if let Err(e) = meta - .op_wake_queued_kind(session_id, engram_core::types::session_op::OpKind::Deliver) - .instrument(attach_wake) - .await - { - tracing::debug!( - session_id = %session_id, - error = %e, - "deliver wake on attach signal failed (poll backstops)", - ); - } - // The shim scan enqueues a Deliver op when rows are due - // and none is queued; a wake with nothing due is a - // no-op. - outbox_wake.notify_one(); + // Ordinary attach signals wake delivery before duplicate suppression. + // Managed signals must pass the atomic target check at append first. + if managed_target.is_none() { + self.wake_waiting_harness(session_id, kind).await; } // Drop a back-to-back duplicate waiting-state marker. The @@ -1628,28 +1659,46 @@ impl HarnessEventWriter { return Ok(()); } - let idx = async { - match delivery { - Some(delivery) => { - meta.append_session_event_idempotent( - session_id, - &format!( - "harness:{}:{}:{}", - delivery.binding_epoch, delivery.incarnation, delivery.seq - ), - kind, - payload, - ) - .await - } - None => meta - .append_session_event(session_id, kind, payload) - .await - .map(Some), + let managed = managed_target.is_some(); + let idx = if let Some(target) = managed_target { + let delivery = delivery.ok_or_else(|| { + engram_core::MetaError::Conflict("managed event sequence is missing".into()) + })?; + let accepted = meta + .append_managed_harness_event(&engram_core::types::event::ManagedHarnessEvent { + target, + incarnation: delivery.incarnation, + seq: delivery.seq, + kind: kind.into(), + payload, + ack_id: ack_id.clone(), + }) + .instrument(ingest_step_span!("coord.harness_event.append")) + .await?; + if accepted.outbox_acked { + metrics::counter!(crate::metrics::OUTBOX_ACKED_TOTAL).increment(1); } - } - .instrument(ingest_step_span!("coord.harness_event.append")) - .await?; + self.wake_waiting_harness(session_id, kind).await; + accepted.idx + } else if let Some(delivery) = delivery { + meta.append_session_event_idempotent( + session_id, + &format!( + "harness:{}:{}:{}", + delivery.binding_epoch, delivery.incarnation, delivery.seq + ), + kind, + payload, + ) + .instrument(ingest_step_span!("coord.harness_event.append")) + .await? + } else { + Some( + meta.append_session_event(session_id, kind, payload) + .instrument(ingest_step_span!("coord.harness_event.append")) + .await?, + ) + }; if let Some(idx) = idx { last_kind.insert(session_id, kind); @@ -1674,14 +1723,15 @@ impl HarnessEventWriter { // ADR 0073 fix: retire the durable outbox row this harness // event confirms. Placed AFTER `publish` so the ack's PG // write never sits in front of the live SSE frame (same - // rationale as the metric join below). Without this, the + // rationale as the metric join below). Managed sessions already + // acknowledged atomically with the accepted event. Without this, the // delivery driver never learns the prompt was consumed and // redelivers it on every resume forever. The ack runs for a // replayed duplicate too (idx None): the first delivery may // have stored the event and died before this ack, and the // replay is the only retry there is. A failed ack withholds the // harness ack so the event is replayed again. - if let Some(ack_id) = &ack_id { + if let Some(ack_id) = ack_id.as_ref().filter(|_| !managed) { match meta .outbox_ack(ack_id) .instrument(ingest_step_span!("coord.harness_event.outbox_ack")) @@ -1814,6 +1864,26 @@ fn strip_jsonb_nul(v: &mut serde_json::Value) -> bool { pub(crate) mod tests { use super::*; + #[test] + fn tool_cancel_requested_keeps_original_identity_without_completion() { + let event = SessionEvent::from_harness( + HarnessEvent::ToolCallCancelRequested { + run_id: "r1".into(), + call_id: "c1".into(), + name: "remote_inventory".into(), + }, + DateTime::from_timestamp(0, 0).unwrap(), + ); + assert_eq!(event.kind(), "tool_call_cancel_requested"); + assert_eq!(outbox_ack_id(SessionId::new(), &event), None); + let json = serde_json::to_value(event).unwrap(); + assert_eq!(json["run_id"], "r1"); + assert_eq!(json["tool_call_id"], "c1"); + assert_eq!(json["name"], "remote_inventory"); + assert!(json.get("ok").is_none()); + assert!(json.get("result_json").is_none()); + } + /// Incident 2026-07-10: PG jsonb rejects U+0000, so a /// `tool_call_completed` whose `result_summary` carried raw binary /// (an `xxd` of a corrupted file) failed the insert and silently @@ -2064,6 +2134,36 @@ pub(crate) mod tests { } } + #[test] + fn deferred_completion_keeps_exact_association_and_old_records_default_closed() { + let at = chrono::DateTime::from_timestamp(0, 0).unwrap(); + let event = SessionEvent::from_harness( + HarnessEvent::RunDeferred { + run_id: "r1".into(), + deferred_tool_call_ids: vec!["t1".into()], + }, + at, + ); + assert_eq!(event.kind(), "run_completed"); + let json = serde_json::to_value(&event).unwrap(); + assert_eq!(json["ok"], true); + assert_eq!(json["deferred_tool_call_ids"], serde_json::json!(["t1"])); + let old = SessionEvent::from_harness( + HarnessEvent::RunCompleted { + run_id: "r1".into(), + ok: true, + }, + at, + ); + let old_json = serde_json::to_value(&old).unwrap(); + assert!(old_json.get("deferred_tool_call_ids").is_none()); + assert!( + matches!(serde_json::from_value::(old_json).unwrap(), + SessionEvent::HarnessRunCompleted { deferred_tool_call_ids, .. } + if deferred_tool_call_ids.is_empty()) + ); + } + #[test] fn generic_tool_events_have_stable_kinds_and_opaque_payloads() { let args_json = r#" { "text": [1, true, null] } "#; @@ -2533,6 +2633,19 @@ pub(crate) mod tests { #[async_trait] impl MetadataStore for MiniMeta { + async fn managed_boot_open(&self, _: SessionId) -> Result, MetaError> { + Ok(None) + } + async fn managed_sandbox_open( + &self, + _: engram_core::SandboxId, + ) -> Result, MetaError> { + Ok(None) + } + async fn managed_tool_target_gone(&self, _: SessionId, _: &str) -> Result { + Ok(false) + } + /// ADR 0073: derive the idle-scan candidate from the mock's own /// session + event rows — the same "newest event" semantics the /// PG lateral implements, so idle_detector tests exercise real diff --git a/crates/engram-coordinator/src/teleport.rs b/crates/engram-coordinator/src/teleport.rs index 357a78e1c..0cd872944 100644 --- a/crates/engram-coordinator/src/teleport.rs +++ b/crates/engram-coordinator/src/teleport.rs @@ -194,6 +194,8 @@ pub(crate) async fn admit_for_rpc( id: SessionId, target: Option, ) -> Result { + crate::managed_boot::gate(state, id, false).await?; + let claim=crate::session_ops::OpClaim::try_acquire(state,id,OpKind::Teleport,serde_json::json!({"source_host":state.services.meta.get_session(id).await?.host_id,"reason":TeleportReason::Ui})).await? .ok_or_else(||ApiError::Conflict("busy_lane".into()))?; let admitted = steps::admit(&claim.as_ctx(), target, TeleportReason::Ui).await; @@ -746,6 +748,7 @@ mod steps { reason: TeleportReason, ) -> Result { let state = ctx.state; + crate::managed_boot::gate(state, ctx.op.session_id, false).await?; let session = state.services.meta.get_session(ctx.op.session_id).await?; if session.status != SessionState::Active { return Ok(TeleportAdmitOutcome::SessionNotActive(session.status)); diff --git a/crates/engram-coordinator/tests/binding_writer_inventory.rs b/crates/engram-coordinator/tests/binding_writer_inventory.rs index 58716340f..3d76eedda 100644 --- a/crates/engram-coordinator/tests/binding_writer_inventory.rs +++ b/crates/engram-coordinator/tests/binding_writer_inventory.rs @@ -54,6 +54,9 @@ const DECLARED: &[(&str, &str, usize, &str)] = &[ ("idle_evictor", "transition_session", 3, "evict/park-reaper exhaustion →HostLost (Retain/RequireUnbound) + Parked→Evicting descent (Retain)"), ("idle_evictor", "transition_with_fence", 4, "park ladder + unbound →HostLost + admin evict entry"), ("idle_evictor", "transition_with_fence_emitting", 1, "fused eviction completion detaches into Idle"), + ("managed_boot_tests", "transition_session_created", 1, "test fixture binds a policy-only lineage to verify that operator admission stays closed without host calls"), + ("mcp/policy_tests", "transition_session", 3, "test fixture enters Pending unbound, detaches into Idle, then requeues unbound to reject a retry with a nonzero binding epoch"), + ("mcp/policy_tests", "transition_session_created", 3, "test fixtures bind retained-policy recovery, fresh policy reads, and the requeued retry's nonzero epoch"), ("teleport", "teleport_commit", 1, "atomic teleport destination binding"), ("teleport", "transition_with_fence_emitting", 3, "attach->Active, rollback->Active, attach-failed->Created (Retain)"), ("teleport", "teleport_settle", 3, "failed settle: session Detach + source tombstone + failed row in one fenced txn"), diff --git a/crates/engram-coordinator/tests/fixtures/mcp-worker.mjs b/crates/engram-coordinator/tests/fixtures/mcp-worker.mjs new file mode 100644 index 000000000..486f99d28 --- /dev/null +++ b/crates/engram-coordinator/tests/fixtures/mcp-worker.mjs @@ -0,0 +1,107 @@ +// Actual worker service, with only fixture DNS resolution replaced. +import { createWorker } from "../../../../mcp-worker/src/service.ts"; +import { ZonePolicy } from "../../../../mcp-worker/src/network.ts"; +import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; +import { X509Certificate } from "node:crypto"; +const dir = mkdtempSync(join(tmpdir(), "mcp-coordinator-test-")); +const run = (args) => execFileSync("openssl", args, { stdio: "ignore" }); +run([ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + join(dir, "ca-key"), + "-out", + join(dir, "ca"), + "-days", + "1", + "-subj", + "/CN=fixture-ca", + "-addext", + "basicConstraints=critical,CA:TRUE", +]); +run([ + "req", + "-new", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + join(dir, "key"), + "-out", + join(dir, "csr"), + "-subj", + "/CN=localhost", +]); +writeFileSync( + join(dir, "ext"), + "basicConstraints=critical,CA:FALSE\nsubjectAltName=DNS:localhost\nextendedKeyUsage=serverAuth,clientAuth\n", +); +run([ + "x509", + "-req", + "-in", + join(dir, "csr"), + "-CA", + join(dir, "ca"), + "-CAkey", + join(dir, "ca-key"), + "-CAcreateserial", + "-out", + join(dir, "cert"), + "-days", + "1", + "-extfile", + join(dir, "ext"), +]); +const key = readFileSync(join(dir, "key"), "utf8"), + cert = readFileSync(join(dir, "cert"), "utf8"), + ca = readFileSync(join(dir, "ca"), "utf8"); +rmSync(dir, { recursive: true }); +const zone = new ZonePolicy({ + id: "public", + revision: "1", + mode: "public", + authorities: ["auth.example"], + aliases: [], + cidrs: [], + denyCidrs: [], +}); +const worker = createWorker({ + zone, + tls: { key, cert, ca }, + callers: [ + { + fingerprint: new X509Certificate(cert).fingerprint256, + role: "coordinator", + }, + ], + enabled: true, + network: { + resolve: async () => ["8.8.8.8"], + request: () => { + throw Error("no_provider_dial"); + }, + }, +}); +const port = await worker.listen(0, "127.0.0.1"); +console.log( + JSON.stringify({ + port, + key, + cert, + ca, + fingerprint: new X509Certificate(cert).fingerprint256, + identity: worker.identity, + }), +); +process.stdin.resume(); +process.stdin.on("end", async () => { + await worker.close(); + process.exit(0); +}); diff --git a/crates/engram-coordinator/tests/grpc_app.rs b/crates/engram-coordinator/tests/grpc_app.rs index 8906fef63..09e903bca 100644 --- a/crates/engram-coordinator/tests/grpc_app.rs +++ b/crates/engram-coordinator/tests/grpc_app.rs @@ -485,6 +485,7 @@ async fn complete_tool_call_appends_submitted_event_and_enqueues_outbox() { service .complete_tool_call(complete_tool_call_request(app::CompleteToolCallRequest { + expected_mcp_plan_review: None, session_id: session_id.to_string(), tool_call_id: "call_1".into(), result_json: r#" { "saved": true } "#.into(), @@ -508,6 +509,32 @@ async fn complete_tool_call_appends_submitted_event_and_enqueues_outbox() { assert_eq!(row.payload["result_json"], r#" { "saved": true } "#); } +#[tokio::test] +async fn complete_tool_call_app_transport_cannot_accept_policy_context() { + let (state, meta) = test_state(vec![TEST_TOKEN.into()]); + let session_id = meta + .create_session(SessionSpec { + image: "localhost:5001/demo:warm".into(), + mode: Default::default(), + }) + .await + .expect("seed session"); + let error = complete_tool_call_service(state) + .complete_tool_call(complete_tool_call_request(app::CompleteToolCallRequest { + expected_mcp_plan_review: Some(app::McpPlanReview::default()), + session_id: session_id.to_string(), + tool_call_id: "plan-call".into(), + result_json: r#"{"decision":"approve"}"#.into(), + })) + .await + .expect_err("application transport must reject policy authority"); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(outbox_rows(&meta).is_empty()); + assert!(!session_events(&meta, session_id) + .iter() + .any(|(kind, _)| kind == "tool_result_submitted")); +} + #[tokio::test] async fn complete_tool_call_rejects_empty_tool_call_id() { let (state, meta) = test_state(vec![TEST_TOKEN.into()]); @@ -522,6 +549,7 @@ async fn complete_tool_call_rejects_empty_tool_call_id() { let err = service .complete_tool_call(complete_tool_call_request(app::CompleteToolCallRequest { + expected_mcp_plan_review: None, session_id: session_id.to_string(), tool_call_id: String::new(), result_json: "{}".into(), @@ -555,6 +583,7 @@ async fn complete_tool_call_rejects_terminal_session() { let err = service .complete_tool_call(complete_tool_call_request(app::CompleteToolCallRequest { + expected_mcp_plan_review: None, session_id: session_id.to_string(), tool_call_id: "call_1".into(), result_json: "{}".into(), @@ -586,6 +615,7 @@ async fn complete_tool_call_is_idempotent_on_tool_call_id() { for _ in 0..2 { service .complete_tool_call(complete_tool_call_request(app::CompleteToolCallRequest { + expected_mcp_plan_review: None, session_id: session_id.to_string(), tool_call_id: "call_1".into(), result_json: r#"{"saved":true}"#.into(), @@ -625,6 +655,7 @@ async fn complete_tool_call_namespaces_outbox_identity_by_session() { for session_id in [first, second] { service .complete_tool_call(complete_tool_call_request(app::CompleteToolCallRequest { + expected_mcp_plan_review: None, session_id: session_id.to_string(), tool_call_id: "shared-call".into(), result_json: r#"{"saved":true}"#.into(), diff --git a/crates/engram-coordinator/tests/mcp_authority.rs b/crates/engram-coordinator/tests/mcp_authority.rs new file mode 100644 index 000000000..73ef37f97 --- /dev/null +++ b/crates/engram-coordinator/tests/mcp_authority.rs @@ -0,0 +1,304 @@ +use engram_coordinator::mcp::worker::{canonical, digest}; +use serde_json::json; + +#[test] +fn fixed_worker_wire_matches_node_utf16_and_integer_encoding() { + let value = json!({"\u{e000}":"last", "\u{10000}":"first", "line":"\n\u{000f}","n":9007199254740991i64}); + assert_eq!( + canonical(&value).unwrap(), + "{\"line\":\"\\n\\u000f\",\"n\":9007199254740991,\"𐀀\":\"first\",\"\":\"last\"}" + ); + assert_eq!( + digest(&json!({"kind":"metadata","endpoint":"https://auth.example/metadata"})).unwrap(), + "e776800d6d90b7a995cb193fbc359e5b248ed21f8c78519749d290599cef2b7d" + ); + assert!(canonical(&json!(9007199254740992i64)).is_err()); + assert!(canonical(&json!(1.1)).is_err()); +} + +#[tokio::test] +async fn actual_worker_mtls_identity_and_fixed_rpc() { + use engram_coordinator::mcp::worker::*; + use tokio::io::{AsyncBufReadExt, BufReader}; + let mut child = tokio::process::Command::new("node") + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/mcp-worker.mjs" + )) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let mut line = String::new(); + BufReader::new(child.stdout.take().unwrap()) + .read_line(&mut line) + .await + .unwrap(); + let fixture: serde_json::Value = serde_json::from_str(&line).unwrap(); + let url = format!("https://localhost:{}", fixture["port"]); + let cert = fixture["cert"].as_str().unwrap(); + let pem = format!("{}{}", cert, fixture["key"].as_str().unwrap()); + let client = WorkerClient::new( + &url, + fixture["ca"].as_str().unwrap().as_bytes(), + pem.as_bytes(), + ) + .unwrap(); + let identity = client.identity().await.unwrap(); + assert_eq!(identity.zone_id, "public"); + let no_cert = engram_tls::client_builder() + .tls_certs_only([reqwest::Certificate::from_pem( + fixture["ca"].as_str().unwrap().as_bytes(), + ) + .unwrap()]) + .no_proxy() + .build() + .unwrap(); + assert!(no_cert.get(format!("{url}/identity")).send().await.is_err()); + assert!(WorkerClient::new("http://localhost", cert.as_bytes(), pem.as_bytes()).is_err()); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + let operation = WorkerOperation::ValidateDestination { + endpoint: "https://auth.example/metadata".into(), + }; + let intent = WorkerIntent { + id: "intent-a".into(), + boot_id: identity.boot_id.clone(), + zone_id: identity.zone_id.clone(), + policy_digest: identity.policy_digest.clone(), + connection_id: "connection-a".into(), + connection_revision: 1, + credential_identity: "authority-a".into(), + scope: WorkerScope::Administration { + request_id: "request-a".into(), + actor_id: "actor-a".into(), + }, + operation: "validateDestination".into(), + destination: "https://auth.example/metadata".into(), + request_digest: "a".repeat(64), + operation_digest: digest(&serde_json::to_value(&operation).unwrap()).unwrap(), + expires_at: now + 10000, + lease: WorkerLease { + id: "lease-a".into(), + deployment_id: "deployment-a".into(), + expires_at: now + 10000, + }, + }; + client.rpc(&identity, &intent, &operation).await.unwrap(); + assert!(client.rpc(&identity, &intent, &operation).await.is_err()); + let mut wrong = identity.clone(); + wrong.boot_id = "wrong".into(); + assert!(client.check_identity(&wrong).await.is_err()); +} + +mod authority { + use engram_coordinator::{ + mcp::{model::*, worker::*, *}, + oauth::OAuthManager, + }; + use engram_core::{ + traits::{Clock, StaticSecretStore}, + types::org_secret::McpSecretRole, + }; + use engram_crypto::EnvVarKeyProvider; + use engram_sim::{ManualClock, SimEntropy, SimMetadataStore}; + use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + sync::{ + atomic::{AtomicU8, Ordering}, + Arc, Mutex, + }, + }; + struct DeterministicSecurity(AtomicU8); + impl SecurityEntropy for DeterministicSecurity { + fn bytes(&self) -> Result<[u8; 32]> { + Ok([self.0.fetch_add(1, Ordering::SeqCst); 32]) + } + } + struct FakeWorker(Mutex); + #[async_trait::async_trait] + impl WorkerTransport for FakeWorker { + async fn identity(&self) -> Result { + Ok(self.0.lock().unwrap().clone()) + } + async fn rpc( + &self, + _: &WorkerIdentity, + _: &WorkerIntent, + _: &WorkerOperation, + ) -> Result { + Err(McpError::Unavailable) + } + } + fn setup() -> ( + McpAuthority, + Arc, + WorkerIdentity, + Arc, + ) { + let clock = ManualClock::new(); + let entropy = Arc::new(SimEntropy::seeded(9)); + let meta = SimMetadataStore::new(clock.clone(), entropy.clone()); + let manager = OAuthManager::new( + meta, + Arc::new(EnvVarKeyProvider::from_bytes([8; 32], "test")), + clock.clone(), + entropy, + Arc::new(StaticSecretStore::new(HashMap::new())), + ); + let identity = WorkerIdentity { + protocol: 1, + boot_id: "boot-a".into(), + zone_id: "public".into(), + policy_digest: "a".repeat(64), + policy_revision: "1".into(), + }; + let registry = Registry { + enabled: true, + deployment_id: "deployment-a".into(), + zones: BTreeMap::from([( + "public".into(), + Zone { + policy_digest: identity.policy_digest.clone(), + policy_revision: "1".into(), + endpoints: BTreeSet::from(["https://resource.example/mcp".into()]), + }, + )]), + registrations: BTreeMap::new(), + }; + let worker = Arc::new(FakeWorker(Mutex::new(identity.clone()))); + ( + McpAuthority::new( + manager, + Arc::new(DeterministicSecurity(AtomicU8::new(1))), + registry, + BTreeMap::from([("public".into(), worker.clone() as Arc)]), + ) + .unwrap(), + clock, + identity, + worker, + ) + } + #[tokio::test] + async fn status_missing_authority_is_not_found() { + let (authority, _, _, _) = setup(); + assert!(matches!( + authority + .status( + "connection-a", + &CredentialReference::OAuth { + registration_id: "missing".into() + } + ) + .await, + Err(McpError::NotFound) + )); + } + #[tokio::test] + async fn api_rotation_fences_old_lease_even_before_orchestrator_publication() { + let (authority, clock, identity, worker) = setup(); + let change = ChangeContext { + connection_id: "connection-a".into(), + config_revision: 1, + previous_authorization_generation: 0, + resource: Destination { + zone_id: "public".into(), + endpoint: "https://resource.example/mcp".into(), + }, + }; + let first = authority + .put_secret( + &change, + McpSecretRole::ApiKey, + None, + "x-api-key", + "first-secret", + ) + .await + .unwrap(); + let lease = ResourceLease { + context: first.context.clone(), + reference: first.reference.clone(), + worker: identity, + intent_id: "intent-a".into(), + request_digest: "a".repeat(64), + operation: "discover".into(), + scope: WorkerScope::Administration { + request_id: "request-a".into(), + actor_id: "actor-a".into(), + }, + operation_expires_at: clock.now_utc().timestamp_millis() + 10000, + expires_at: clock.now_utc().timestamp_millis() + 10000, + admission: WorkerLease { + id: "lease-a".into(), + deployment_id: "deployment-a".into(), + expires_at: clock.now_utc().timestamp_millis() + 10000, + }, + }; + let mut operation = lease.clone(); + operation.operation_expires_at = clock.now_utc().timestamp_millis() + 60000; + operation.admission.expires_at = operation.operation_expires_at; + assert_eq!( + authority.lease(&operation).await.unwrap().value, + "first-secret" + ); + operation.admission.expires_at -= 1; + assert!(authority.lease(&operation).await.is_err()); + operation.operation_expires_at = clock.now_utc().timestamp_millis() + 300001; + operation.admission.expires_at = operation.operation_expires_at; + assert!(authority.lease(&operation).await.is_err()); + assert_eq!(authority.lease(&lease).await.unwrap().value, "first-secret"); + let mut next = change; + next.previous_authorization_generation = first.context.authorization_generation; + let second = authority + .put_secret( + &next, + McpSecretRole::ApiKey, + Some(&first.reference), + "x-api-key", + "second-secret", + ) + .await + .unwrap(); + assert_ne!( + first.context.credential_identity, + second.context.credential_identity + ); + assert!(authority.lease(&lease).await.is_err()); + let mut fresh = lease; + fresh.context = second.context; + fresh.reference = second.reference; + assert_eq!( + authority.lease(&fresh).await.unwrap().value, + "second-secret" + ); + let mut wrong = fresh.clone(); + wrong.context.config_revision += 1; + assert!(authority.lease(&wrong).await.is_err()); + wrong = fresh.clone(); + wrong.worker.boot_id = "old-boot".into(); + assert!(authority.lease(&wrong).await.is_err()); + // A new boot fences old intents, but preserves durable credential authority. + worker.0.lock().unwrap().boot_id = "boot-b".into(); + assert!(authority.lease(&fresh).await.is_err()); + fresh.worker.boot_id = "boot-b".into(); + assert_eq!( + authority.lease(&fresh).await.unwrap().value, + "second-secret" + ); + let current = authority + .status("connection-a", &fresh.reference) + .await + .unwrap(); + assert!(authority.revoke(&first).await.is_err()); + let revoked = authority.revoke(¤t).await.unwrap(); + assert_eq!(revoked.status, "revoked"); + assert!(authority.lease(&fresh).await.is_err()); + clock.advance(std::time::Duration::from_secs(10)); + assert!(authority.lease(&fresh).await.is_err()); + } +} diff --git a/crates/engram-coordinator/tests/mcp_control.rs b/crates/engram-coordinator/tests/mcp_control.rs new file mode 100644 index 000000000..eb9ed4698 --- /dev/null +++ b/crates/engram-coordinator/tests/mcp_control.rs @@ -0,0 +1,779 @@ +use engram_coordinator::{ + mcp::{control::*, model::Registry, *}, + oauth::OAuthManager, +}; +use engram_core::{ + traits::{Clock, MetadataStore, StaticSecretStore}, + types::{mcp::McpSealedState, oauth::*}, +}; +use engram_crypto::EnvVarKeyProvider; +use engram_protocol::mcp as wire; +use engram_sim::{ManualClock, SimEntropy, SimMetadataStore}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; +use tokio::io::{AsyncBufReadExt, BufReader}; + +#[tokio::test] +async fn control_listener_requires_tls_client_certificate_pin_and_protocol_one() { + let mut fixture = tokio::process::Command::new("node") + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/mcp-worker.mjs" + )) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let mut line = String::new(); + BufReader::new(fixture.stdout.take().unwrap()) + .read_line(&mut line) + .await + .unwrap(); + let f: serde_json::Value = serde_json::from_str(&line).unwrap(); + let clock = ManualClock::new(); + let entropy = Arc::new(SimEntropy::seeded(1)); + let meta = SimMetadataStore::new(clock.clone(), entropy.clone()); + let manager = OAuthManager::new( + meta.clone(), + Arc::new(EnvVarKeyProvider::from_bytes([7; 32], "test")), + clock.clone(), + entropy, + Arc::new(StaticSecretStore::new(HashMap::new())), + ); + let (_, shutdown) = tokio::sync::watch::channel(false); + assert!( + engram_coordinator::mcp::startup::start(None, manager.clone(), shutdown.clone()) + .await + .unwrap() + .is_none() + ); + assert!(engram_coordinator::mcp::startup::start( + Some(std::path::Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../deploy/mcp-control.example.json" + ))), + manager.clone(), + shutdown + ) + .await + .unwrap() + .is_none()); + let authority = Arc::new( + McpAuthority::new( + manager, + Arc::new(SystemSecurityEntropy), + Registry { + enabled: true, + deployment_id: "deployment-a".into(), + ..Default::default() + }, + BTreeMap::new(), + ) + .unwrap(), + ); + for pinned in [true, false] { + let tls = ControlTls { + certificate: f["cert"].as_str().unwrap().as_bytes().to_vec(), + private_key: f["key"].as_str().unwrap().as_bytes().to_vec(), + client_ca: f["ca"].as_str().unwrap().as_bytes().to_vec(), + orchestrator_fingerprints: vec![if pinned { + f["fingerprint"].as_str().unwrap().into() + } else { + "00:".repeat(31) + "00" + }], + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (stop, done) = tokio::sync::oneshot::channel::<()>(); + let task = tokio::spawn( + server(authority.clone(), tls) + .unwrap() + .serve_with_incoming_shutdown( + tonic::transport::server::TcpIncoming::from(listener), + async { + let _ = done.await; + }, + ), + ); + let endpoint = tonic::transport::Endpoint::from_shared(format!( + "https://localhost:{}", + address.port() + )) + .unwrap(); + let tls = tonic::transport::ClientTlsConfig::new() + .ca_certificate(tonic::transport::Certificate::from_pem( + f["ca"].as_str().unwrap(), + )) + .domain_name("localhost"); + let no_cert = endpoint + .clone() + .tls_config(tls.clone()) + .unwrap() + .connect() + .await; + if let Ok(channel) = no_cert { + assert!( + wire::credential_control_service_client::CredentialControlServiceClient::new( + channel + ) + .get_status(wire::GetStatusRequest::default()) + .await + .is_err() + ); + } + let channel = endpoint + .tls_config(tls.identity(tonic::transport::Identity::from_pem( + f["cert"].as_str().unwrap(), + f["key"].as_str().unwrap(), + ))) + .unwrap() + .connect() + .await + .unwrap(); + assert_eq!( + wire::operator_publication_service_client::OperatorPublicationServiceClient::new( + channel.clone() + ) + .stage(wire::OperatorPublicationServiceStageRequest { + protocol: 1, + expected_generation: None, + configured_candidate_identity: String::new() + }) + .await + .unwrap_err() + .code(), + tonic::Code::Unimplemented + ); + let mut policy_client = + wire::session_policy_service_client::SessionPolicyServiceClient::new(channel.clone()); + let sid = meta + .create_session(engram_core::types::SessionSpec { + image: "test.invalid/policy:latest".into(), + mode: Default::default(), + }) + .await + .unwrap(); + let policy = policy_client + .read_write_policy(wire::ReadWritePolicyRequest { + protocol: 1, + session_id: sid.to_string(), + plan_tool_call_id: None, + }) + .await; + if pinned { + assert_eq!( + policy + .unwrap() + .into_inner() + .state + .unwrap() + .policy + .unwrap() + .write_eligibility, + "unknown" + ); + assert_eq!( + policy_client + .read_write_policy(wire::ReadWritePolicyRequest { + protocol: 0, + session_id: sid.to_string(), + plan_tool_call_id: None + }) + .await + .unwrap_err() + .code(), + tonic::Code::InvalidArgument + ); + } else { + assert_eq!(policy.unwrap_err().code(), tonic::Code::PermissionDenied); + } + // Exact native input is available only to the pinned policy service. + // This fixture exercises persistence and transport, never backend execution. + use engram_core::types::{managed_create::*, mcp_policy::*, runtime_spec::RuntimeSpec}; + use prost::Message; + let native_sid = + engram_core::SessionId(uuid::Uuid::from_u128(if pinned { 1400 } else { 1401 })); + let policy = McpPolicyCreation { + actor_id: "owner-native".into(), + input_id: format!("create:{native_sid}"), + parent_session_id: None, + eligibility: WriteEligibility::ReadOnly, + }; + let mut create = engram_protocol::app::CreateSessionRequest { + requested_session_id: Some(native_sid.to_string()), + harness: Some("claude".into()), + harness_mode: Some("plan".into()), + ..Default::default() + }; + create + .harness_env + .insert("ENGRAM_TOOLS".into(), " [ ] ".into()); + let mut reserve = wire::ReserveManagedCreateRequest { + protocol: 1, + task_id: "task-native".into(), + native_version: 1, + native_digest: digest(b" [ ] "), + creation_request: wire::CreatePolicySessionRequest { + protocol: 1, + actor_id: policy.actor_id.clone(), + input_id: policy.input_id.clone(), + create: Some(create), + parent_session_id: None, + } + .encode_to_vec(), + input_digest: String::new(), + }; + reserve.input_digest = wire::managed_create_input_digest(&reserve); + assert_eq!( + policy_client + .reserve_managed_create(reserve.clone()) + .await + .unwrap_err() + .code(), + if pinned { + tonic::Code::Unavailable + } else { + tonic::Code::PermissionDenied + } + ); + assert!(meta.get_session(native_sid).await.is_err()); + let identity = ManagedCreateIdentity { + session_id: native_sid, + task_id: reserve.task_id.clone(), + policy: policy.clone(), + input_digest: reserve.input_digest.clone(), + }; + assert_eq!( + policy_client + .read_published_operator(wire::ReadPublishedOperatorRequest { protocol: 1 }) + .await + .unwrap_err() + .code(), + if pinned { + tonic::Code::Unavailable + } else { + tonic::Code::PermissionDenied + } + ); + engram_testkit::operator::publish_empty(meta.as_ref()).await; + let projection = policy_client + .read_published_operator(wire::ReadPublishedOperatorRequest { protocol: 1 }) + .await; + if pinned { + let projection: engram_core::types::managed_operator::OperatorPublication = + serde_json::from_slice(&projection.unwrap().into_inner().publication_json).unwrap(); + assert!(projection.admission_open); + assert_eq!( + projection.published.unwrap().candidate, + engram_testkit::operator::candidate() + ); + } else { + assert_eq!( + projection.unwrap_err().code(), + tonic::Code::PermissionDenied + ); + } + + meta.reserve_and_persist_create( + engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: Some(ManagedCreation { + identity: identity.clone(), + native: FrozenNativeManifest { + version: 1, + bytes: b" [ ] ".to_vec(), + digest: reserve.native_digest.clone(), + }, + }), + mcp_policy: Some(policy.clone()), + session_id: native_sid, + spec: engram_core::types::SessionSpec { + image: "test.invalid/native:latest".into(), + mode: Default::default(), + }, + mem_budget_mib: 512, + cpu_budget_vcpus: 1, + sealed_secrets: None, + capabilities: Vec::new(), + integration_policy_json: None, + runtime_spec: RuntimeSpec::new(Vec::new(), Some("claude".into()), None, Vec::new()), + oauth_binding: None, + }, + &[], + 0, + ) + .await + .unwrap(); + let native_identity = wire::ManagedCreateIdentity { + session_id: native_sid.to_string(), + task_id: identity.task_id, + input_id: policy.input_id, + actor_id: policy.actor_id, + parent_session_id: None, + write_eligibility: "read_only".into(), + input_digest: identity.input_digest, + }; + let boot = policy_client + .read_managed_boot(wire::ReadManagedBootRequest { + protocol: 1, + identity: Some(native_identity.clone()), + }) + .await; + if pinned { + assert!(boot.unwrap().into_inner().boot_json.is_none()); + } else { + assert_eq!(boot.unwrap_err().code(), tonic::Code::PermissionDenied); + } + for protocol in [0, 1] { + let code = policy_client + .finalize_managed_boot(wire::FinalizeManagedBootRequest { + protocol, + input_json: b"{}".to_vec(), + }) + .await + .unwrap_err() + .code(); + assert_eq!( + code, + if pinned { + tonic::Code::InvalidArgument + } else { + tonic::Code::PermissionDenied + } + ); + } + // A valid one-MiB manifest expands when encoded as JSON bytes. The + // service envelope must reach the bounded typed decoder above four MiB. + let large = policy_client + .finalize_managed_boot(wire::FinalizeManagedBootRequest { + protocol: 1, + input_json: vec![b' '; 5 * 1024 * 1024], + }) + .await + .unwrap_err() + .code(); + assert_eq!( + large, + if pinned { + tonic::Code::InvalidArgument + } else { + tonic::Code::PermissionDenied + } + ); + let bridge_request = wire::ReadManagedCreationStateRequest { + protocol: 1, + identity: Some(native_identity.clone()), + native_version: 1, + native_digest: reserve.native_digest.clone(), + }; + let observed = policy_client + .read_managed_creation_state(bridge_request.clone()) + .await; + if pinned { + let bytes = observed.unwrap().into_inner().state_json; + let view: engram_core::types::managed_bridge::ManagedCreationState = + serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + view.state, + engram_core::types::managed_bridge::ManagedCreationPhase::Held + ); + assert!(view.boot.is_none()); + assert!(view.operation.is_none()); + let mut absent = bridge_request.clone(); + absent.identity.as_mut().unwrap().session_id = uuid::Uuid::from_u128(1499).to_string(); + let absent = policy_client + .read_managed_creation_state(absent) + .await + .unwrap() + .into_inner(); + assert_eq!( + serde_json::from_slice::(&absent.state_json).unwrap()["state"], + "absent" + ); + let mut wrong = bridge_request.clone(); + wrong.native_digest = digest(b"changed native"); + assert_eq!( + policy_client + .read_managed_creation_state(wrong) + .await + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); + let mut wrong = bridge_request.clone(); + wrong.protocol = 0; + assert_eq!( + policy_client + .read_managed_creation_state(wrong) + .await + .unwrap_err() + .code(), + tonic::Code::InvalidArgument + ); + } else { + assert_eq!(observed.unwrap_err().code(), tonic::Code::PermissionDenied); + } + let prepare = policy_client + .prepare_managed_boot(wire::PrepareManagedBootRequest { + protocol: 1, + identity: Some(native_identity.clone()), + native_version: 1, + native_digest: reserve.native_digest.clone(), + }) + .await + .unwrap_err(); + assert_eq!( + prepare.code(), + if pinned { + tonic::Code::Unavailable + } else { + tonic::Code::PermissionDenied + } + ); // This transport fixture has no runtime AppState. + let recovery = policy_client + .recover_managed_boot(wire::RecoverManagedBootRequest { + protocol: 1, + identity: Some(native_identity.clone()), + native_version: 1, + native_digest: reserve.native_digest.clone(), + expected_source_key_json: br#"{"host_id":"caller-selected"}"#.to_vec(), + recovery_id: "recovery-one".into(), + }) + .await + .unwrap_err(); + assert_eq!( + recovery.code(), + if pinned { + tonic::Code::InvalidArgument + } else { + tonic::Code::PermissionDenied + } + ); + let request = wire::ReadNativeManifestRequest { + protocol: 1, + identity: Some(native_identity.clone()), + }; + let result = policy_client.read_native_manifest(request.clone()).await; + if pinned { + let native = result.unwrap().into_inner(); + assert_eq!(native.native_bytes, b" [ ] "); + assert_eq!(native.native_digest, reserve.native_digest); + assert_eq!(native.identity, Some(native_identity)); + assert_eq!(native.native_version, 1); + let mut wrong = request.clone(); + wrong.identity.as_mut().unwrap().task_id = "another-task".into(); + assert_eq!( + policy_client + .read_native_manifest(wrong) + .await + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); + let mut wrong = request.clone(); + wrong.identity.as_mut().unwrap().session_id = sid.to_string(); + assert_eq!( + policy_client + .read_native_manifest(wrong) + .await + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); + let mut wrong = request; + wrong.protocol = 0; + assert_eq!( + policy_client + .read_native_manifest(wrong) + .await + .unwrap_err() + .code(), + tonic::Code::InvalidArgument + ); + } else { + assert_eq!(result.unwrap_err().code(), tonic::Code::PermissionDenied); + } + let mut client = + wire::credential_control_service_client::CredentialControlServiceClient::new(channel); + if pinned { + use engram_core::types::mcp_policy::{McpPolicyCreation, WriteEligibility}; + let policy_sid = engram_core::SessionId(uuid::Uuid::from_u128(1200)); + meta.reserve_and_persist_create( + engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: Some(McpPolicyCreation { + actor_id: "owner-a".into(), + input_id: "create-a".into(), + parent_session_id: None, + eligibility: WriteEligibility::ReadOnly, + }), + session_id: policy_sid, + spec: engram_core::types::SessionSpec { + image: "test.invalid/policy:latest".into(), + mode: Default::default(), + }, + mem_budget_mib: 512, + cpu_budget_vcpus: 1, + sealed_secrets: None, + capabilities: Vec::new(), + integration_policy_json: None, + runtime_spec: engram_core::types::runtime_spec::RuntimeSpec::new( + Vec::new(), + Some("claude".into()), + None, + Vec::new(), + ), + oauth_binding: None, + }, + &[], + 0, + ) + .await + .unwrap(); + meta.append_session_event(policy_sid,"tool_call_requested",serde_json::json!({"name":"exit_plan_mode","tool_call_id":"plan-a","run_id":"run-a","args_json":"{}"})).await.unwrap(); + let reviewed = policy_client + .read_write_policy(wire::ReadWritePolicyRequest { + protocol: 1, + session_id: policy_sid.to_string(), + plan_tool_call_id: Some("plan-a".into()), + }) + .await + .unwrap() + .into_inner() + .state + .unwrap() + .plan_review + .unwrap(); + let approved = policy_client + .accept_plan_exit(wire::AcceptPlanExitRequest { + protocol: 1, + expected: Some(reviewed.clone()), + actor_id: "owner-a".into(), + result_json: "{\"decision\":\"approve\"}".into(), + }) + .await + .unwrap() + .into_inner() + .policy + .unwrap(); + assert_eq!(approved.write_eligibility, "write_eligible"); + assert_eq!(approved.authenticated_actor_id.as_deref(), Some("owner-a")); + let outbox = meta + .outbox_get(&engram_core::types::outbox::tool_result_outbox_id( + policy_sid, "plan-a", + )) + .await + .unwrap() + .unwrap(); + assert_eq!(outbox.payload["result_json"], "{\"decision\":\"approve\"}"); + for state in [ + engram_core::SessionState::Pending, + engram_core::SessionState::Created, + engram_core::SessionState::Active, + ] { + meta.transition_session(policy_sid, state, engram_core::BindingDisposition::Retain) + .await + .unwrap(); + } + meta.append_session_event( + policy_sid, + "run_started", + serde_json::json!({"run_id":"run-a"}), + ) + .await + .unwrap(); + meta.append_session_event(policy_sid, "tool_call_requested", serde_json::json!({"name":"mcp_docs_search", "tool_call_id":"read-a", "run_id":"run-a", "args_json":"{ \"x\": 1 }"})).await.unwrap(); + let exact = wire::ValidateExecutionIdentityRequest { + protocol: 1, + session_id: policy_sid.to_string(), + run_id: "run-a".into(), + tool_call_id: "read-a".into(), + accepted_policy_ref: approved.r#ref.clone(), + accepted_write_eligibility: "write_eligible".into(), + ancestor_policy_refs: Vec::new(), + model_tool_name: "mcp_docs_search".into(), + original_arguments: "{ \"x\": 1 }".into(), + }; + assert_eq!( + policy_client + .validate_execution_identity(exact.clone()) + .await + .unwrap() + .into_inner() + .status, + "current" + ); + assert_eq!( + policy_client + .validate_execution_identity(wire::ValidateExecutionIdentityRequest { + original_arguments: "{\"x\":1}".into(), + ..exact.clone() + }) + .await + .unwrap() + .into_inner() + .status, + "stale" + ); + assert_eq!( + policy_client + .validate_dispatch_fence(wire::ValidateDispatchFenceRequest { + protocol: exact.protocol, + session_id: exact.session_id, + run_id: exact.run_id, + tool_call_id: exact.tool_call_id, + accepted_policy_ref: exact.accepted_policy_ref, + accepted_write_eligibility: exact.accepted_write_eligibility, + ancestor_policy_refs: exact.ancestor_policy_refs, + model_tool_name: "wrong-name".into(), + original_arguments: exact.original_arguments, + }) + .await + .unwrap() + .into_inner() + .status, + "stale" + ); + let prior = policy_client + .validate_dispatch_fence(wire::ValidateDispatchFenceRequest { + protocol: 1, + session_id: policy_sid.to_string(), + run_id: "run-a".into(), + tool_call_id: "call-a".into(), + accepted_policy_ref: reviewed.policy_ref, + accepted_write_eligibility: "read_only".into(), + ancestor_policy_refs: Vec::new(), + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{}".into(), + }) + .await + .unwrap() + .into_inner(); + assert_ne!(prior.status, "current"); + assert_eq!( + policy_client + .create_policy_session(wire::CreatePolicySessionRequest { + protocol: 1, + ..Default::default() + }) + .await + .unwrap_err() + .code(), + tonic::Code::InvalidArgument + ); + let missing = client + .get_status(wire::GetStatusRequest { + protocol: 1, + connection_id: "connection-a".into(), + reference: Some(wire::CredentialReference { + kind: wire::CredentialKind::Oauth as i32, + registration_id: "absent".into(), + ..Default::default() + }), + }) + .await + .unwrap_err(); + assert_eq!(missing.code(), tonic::Code::NotFound); + let id = uuid::Uuid::from_u128(999); + let now = clock.now_utc(); + meta.create_mcp_oauth_flow( + OAuthFlow { + id, + key: OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: "connection-a".into(), + provider: "registered-a".into(), + }, + owner_replica: "replica-a".into(), + lease_expires_at: now + chrono::Duration::minutes(10), + expires_at: now + chrono::Duration::minutes(10), + status: OAuthFlowStatus::Pending, + error_code: None, + created_at: now, + updated_at: now, + }, + McpSealedState { + wrapped_dek: vec![], + nonce: vec![], + ciphertext: vec![], + key_id: "test".into(), + }, + ) + .await + .unwrap(); + let cancel = wire::CancelAuthorizationRequest { + protocol: 1, + flow_id: id.to_string(), + connection_id: "connection-a".into(), + registration_id: "registered-a".into(), + actor_id: "admin".into(), + }; + for _ in 0..2 { + let receipt = client + .cancel_authorization(cancel.clone()) + .await + .unwrap() + .into_inner(); + assert_eq!(receipt.flow_id, cancel.flow_id); + assert_eq!(receipt.connection_id, cancel.connection_id); + assert_eq!(receipt.registration_id, cancel.registration_id); + assert_eq!(receipt.protocol, 1); + } + assert_eq!( + meta.get_oauth_flow(id).await.unwrap().unwrap().status, + OAuthFlowStatus::Cancelled + ); + let mut wrong = cancel; + wrong.connection_id = "unrelated".into(); + assert!(client.cancel_authorization(wrong).await.is_err()); + } + let error = client + .get_status(wire::GetStatusRequest { + protocol: 2, + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!( + error.code(), + if pinned { + tonic::Code::InvalidArgument + } else { + tonic::Code::PermissionDenied + } + ); + let plaintext = + tonic::transport::Endpoint::from_shared(format!("http://localhost:{}", address.port())) + .unwrap() + .connect_timeout(std::time::Duration::from_secs(1)) + .connect() + .await; + if let Ok(channel) = plaintext { + assert!( + wire::credential_control_service_client::CredentialControlServiceClient::new( + channel + ) + .get_status(wire::GetStatusRequest::default()) + .await + .is_err() + ); + } + let _ = stop.send(()); + task.await.unwrap().unwrap(); + } + assert!(!Registry::default().enabled); + assert!(!format!( + "{:?}", + wire::PutSecretRequest { + value: "private-value".into(), + ..Default::default() + } + ) + .contains("private-value")); +} diff --git a/crates/engram-coordinator/tests/mcp_credentials.rs b/crates/engram-coordinator/tests/mcp_credentials.rs new file mode 100644 index 000000000..69218a5c3 --- /dev/null +++ b/crates/engram-coordinator/tests/mcp_credentials.rs @@ -0,0 +1,297 @@ +use engram_coordinator::org_secrets::{seal_org_secret, OrgSecretBackend}; +use engram_core::traits::{ + LayeredSecretStore, MetadataStore, SecretContext, SecretStore, StaticSecretStore, +}; +use engram_core::types::org_secret::McpSecretRole; +use engram_core::types::SecretSchema; +use engram_core::SecretError; +use engram_crypto::EnvVarKeyProvider; +use engram_sim::{ManualClock, SimEntropy, SimMetadataStore}; +use std::collections::HashMap; +use std::sync::Arc; + +#[tokio::test] +async fn reserved_refs_deny_aliases_fallbacks_and_overrides_before_value_lookup() { + let clock = ManualClock::new(); + let meta = SimMetadataStore::new(clock, Arc::new(SimEntropy::seeded(4))); + let kek = Arc::new(EnvVarKeyProvider::from_bytes([7; 32], "test")); + let r = meta + .reserve_mcp_org_secret("reserved", "connection-a", McpSecretRole::ApiKey) + .await + .unwrap(); + let store = LayeredSecretStore::new(vec![ + Arc::new(OrgSecretBackend::new(meta.clone(), kek.clone())), + Arc::new(StaticSecretStore::new(HashMap::from([( + "reserved".into(), + "fallback-value".into(), + )]))), + ]); + let ctx = SecretContext { + repo: "", + image_tag: "", + }; + for present in [false, true] { + if present { + meta.put_mcp_org_secret( + seal_org_secret(kek.as_ref(), "reserved", b"private") + .await + .unwrap(), + &r, + ) + .await + .unwrap(); + } + for (name, reference) in [("reserved", None), ("alias", Some("reserved"))] { + let schema = SecretSchema { + r#ref: reference.map(String::from), + ..Default::default() + }; + assert!(matches!( + store.get(&ctx, name, &schema).await, + Err(SecretError::Unauthorized(_)) + )); + assert!(matches!( + store + .resolve( + &ctx, + &HashMap::from([(name.into(), schema)]), + Some(&HashMap::from([(name.into(), "override".into())])) + ) + .await, + Err(SecretError::Unauthorized(_)) + )); + } + } +} + +#[tokio::test] +async fn legacy_mcp_bindings_cannot_fetch_or_update_guest_credentials() { + use engram_coordinator::oauth::{OAuthManager, OAuthServiceError}; + use engram_core::types::oauth::{ + NewSealedOAuthCredential, OAuthAccountMetadata, OAuthCredentialKey, OAuthSubjectKind, + SessionOAuthBinding, + }; + use engram_crypto::CredCipher; + let clock = ManualClock::new(); + let entropy = Arc::new(SimEntropy::seeded(4)); + let meta = SimMetadataStore::new(clock.clone(), entropy.clone()); + let kek = Arc::new(EnvVarKeyProvider::from_bytes([7; 32], "test")); + let key = OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: "connection-a".into(), + provider: "openai-codex".into(), + }; + let sealed = CredCipher::new(kek.as_ref()) + .seal(b"private-bundle") + .await + .unwrap(); + meta.put_oauth_credential( + NewSealedOAuthCredential { + key: key.clone(), + wrapped_dek: sealed.wrapped_dek, + nonce: sealed.nonce.to_vec(), + ciphertext: sealed.ciphertext, + key_id: sealed.key_id, + metadata: OAuthAccountMetadata::default(), + expires_at: None, + }, + None, + ) + .await + .unwrap(); + let sid = engram_core::SessionId::from(uuid::Uuid::from_u128(4)); + meta.with_db_mut(|db| { + db.session_oauth_bindings.insert( + sid, + SessionOAuthBinding { + session_id: sid, + key, + }, + ); + }); + let manager = OAuthManager::new( + meta, + kek, + clock, + entropy, + Arc::new(StaticSecretStore::new(HashMap::new())), + ); + assert!(matches!( + manager.fetch_session(sid).await, + Err(OAuthServiceError::BadRequest(_)) + )); + assert!(matches!( + manager.update_session(sid, 1, b"replacement").await, + Err(OAuthServiceError::BadRequest(_)) + )); + assert!(matches!( + manager.report_session_broken(sid, 1, "revoked").await, + Err(OAuthServiceError::BadRequest(_)) + )); +} + +mod publication_race { + use super::*; + use engram_core::traits::{ + CreateDisposition, DisableEnabledImageOutcome, SessionCreateWriteSet, + }; + use engram_core::types::event::{ArtifactRow, EventCursor, PersistedEvent}; + use engram_core::types::host::{HostHeartbeat, HostRecord, HostStatus}; + use engram_core::types::registry::{EnabledImage, RegistryCredential, SessionSecrets}; + use engram_core::types::session::{BindingDisposition, Session, SessionSpec, SessionState}; + use engram_core::types::snapshot::SnapshotRecord; + use engram_core::{HostId, MetaError, SandboxId, SessionId}; + use tokio::sync::Notify; + + // Forward to the real simulator. Only the payload-read boundary is paused; + // reservation, ciphertext publication, and all purpose reads keep real behavior. + struct PausedMeta { + inner: Arc, + read_started: Notify, + resume_read: Notify, + } + + macro_rules! delegate_required { + ($( $method:ident ( $( $arg:ident: $ty:ty ),* $(,)? ) -> $ret:ty; )*) => { + #[async_trait::async_trait] + impl MetadataStore for PausedMeta { + async fn get_mcp_org_secret_reservation(&self, name: &str) -> Result, MetaError> { + self.inner.get_mcp_org_secret_reservation(name).await + } + async fn get_org_secret_sealed(&self, name: &str) -> Result, MetaError> { + self.read_started.notify_one(); + self.resume_read.notified().await; + self.inner.get_org_secret_sealed(name).await + } + $(async fn $method(&self, $( $arg: $ty ),*) -> $ret { + self.inner.$method($( $arg ),*).await + })* + } + } + } + + delegate_required! { + create_session(spec: SessionSpec) -> Result; + transition_session_created(session_id: SessionId, sandbox_id: SandboxId,) -> Result; + get_session(id: SessionId) -> Result; + list_active_sessions() -> Result, MetaError>; + reserve_and_persist_create(ws: SessionCreateWriteSet, candidates: &[HostId], affinity_len: usize,) -> Result; + transition_session(id: SessionId, target: SessionState, disposition: BindingDisposition,) -> Result; + assign_session_host(id: SessionId, host_id: Option,) -> Result<(), MetaError>; + upsert_host(host: HostRecord) -> Result<(), MetaError>; + list_active_hosts() -> Result, MetaError>; + set_host_status(id: HostId, status: HostStatus) -> Result<(), MetaError>; + touch_host_heartbeat(id: HostId, hb: HostHeartbeat,) -> Result; + set_host_cordon(id: HostId, owner: Option, reason: Option<&str>,) -> Result<(), MetaError>; + mark_host_dead_if_lease_expired(host_id: HostId,) -> Result, MetaError>; + record_snapshot(snap: SnapshotRecord) -> Result; + list_snapshots_for_session(sid: SessionId,) -> Result, MetaError>; + latest_snapshot_for_session(sid: SessionId,) -> Result, MetaError>; + append_session_event(session_id: SessionId, kind: &str, payload: serde_json::Value,) -> Result; + list_session_events_window(session_id: SessionId, cursor: EventCursor, limit: i64, kinds: &[String], tool_names: &[String],) -> Result, MetaError>; + insert_artifact(id: uuid::Uuid, session_id: SessionId, blob_key: &str, media_type: &str, size_bytes: i64, caption: Option<&str>, file_name: Option<&str>,) -> Result<(), MetaError>; + get_artifact(session_id: SessionId, id: uuid::Uuid,) -> Result, MetaError>; + artifact_usage(session_id: SessionId) -> Result<(i64, i64), MetaError>; + upsert_registry_credential(cred: RegistryCredential) -> Result<(), MetaError>; + list_registry_credentials() -> Result, MetaError>; + registry_credential_for_host(registry_host: &str,) -> Result, MetaError>; + delete_registry_credential(registry_host: &str) -> Result<(), MetaError>; + upsert_enabled_image(image: EnabledImage) -> Result<(), MetaError>; + list_enabled_images() -> Result, MetaError>; + get_enabled_image(image_uri: &str) -> Result, MetaError>; + get_enabled_image_any(image_uri: &str,) -> Result, MetaError>; + soft_delete_enabled_image(image_uri: &str,) -> Result; + delete_enabled_image(image_uri: &str) -> Result<(), MetaError>; + get_session_secrets(session_id: SessionId,) -> Result, MetaError>; + delete_session_secrets(session_id: SessionId) -> Result<(), MetaError>; + } + + #[tokio::test] + async fn reservation_published_after_initial_check_blocks_payload_and_fallback() { + for publish_value in [true, false] { + let meta = SimMetadataStore::new(ManualClock::new(), Arc::new(SimEntropy::seeded(4))); + let paused = Arc::new(PausedMeta { + inner: meta.clone(), + read_started: Notify::new(), + resume_read: Notify::new(), + }); + let kek = Arc::new(EnvVarKeyProvider::from_bytes([7; 32], "test")); + let store = LayeredSecretStore::new(vec![ + Arc::new(OrgSecretBackend::new(paused.clone(), kek.clone())), + Arc::new(StaticSecretStore::new(HashMap::from([( + "fresh".into(), + "fallback-value".into(), + )]))), + ]); + let reader = tokio::spawn(async move { + store + .get( + &SecretContext { + repo: "", + image_tag: "", + }, + "fresh", + &SecretSchema::default(), + ) + .await + }); + tokio::time::timeout( + std::time::Duration::from_secs(5), + paused.read_started.notified(), + ) + .await + .unwrap(); + // The initial generic checks have returned no reservation. The read + // is now suspended before it can observe the newly published value. + let reserved = meta + .reserve_mcp_org_secret("fresh", "connection-a", McpSecretRole::ApiKey) + .await + .unwrap(); + if publish_value { + meta.put_mcp_org_secret( + seal_org_secret(kek.as_ref(), "fresh", b"private-value") + .await + .unwrap(), + &reserved, + ) + .await + .unwrap(); + } + paused.resume_read.notify_one(); + let result = tokio::time::timeout(std::time::Duration::from_secs(5), reader) + .await + .unwrap() + .unwrap(); + assert!( + matches!(result, Err(SecretError::Unauthorized(_))), + "a reservation published during a generic read must block plaintext and fallback" + ); + } + } +} + +#[tokio::test] +async fn generic_connector_resolver_rejects_mcp_before_loading_credentials() { + use engram_coordinator::oauth::{OAuthManager, OAuthServiceError}; + use engram_core::types::oauth::{OAuthCredentialKey, OAuthSubjectKind}; + let clock = ManualClock::new(); + let entropy = Arc::new(SimEntropy::seeded(4)); + let meta = SimMetadataStore::new(clock.clone(), entropy.clone()); + let manager = OAuthManager::new( + meta, + Arc::new(EnvVarKeyProvider::from_bytes([7; 32], "test")), + clock, + entropy, + Arc::new(StaticSecretStore::new(HashMap::new())), + ); + assert!(matches!( + manager + .resolve_connector_token(&OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: "connection-a".into(), + provider: "registered-a".into() + }) + .await, + Err(OAuthServiceError::BadRequest(_)) + )); +} diff --git a/crates/engram-coordinator/tests/mcp_oauth.rs b/crates/engram-coordinator/tests/mcp_oauth.rs new file mode 100644 index 000000000..951d586dc --- /dev/null +++ b/crates/engram-coordinator/tests/mcp_oauth.rs @@ -0,0 +1,741 @@ +use engram_coordinator::{ + mcp::{model::*, oauth::*, worker::*, *}, + oauth::OAuthManager, +}; +use engram_core::{ + traits::{Clock, StaticSecretStore}, + types::oauth::*, +}; +use engram_crypto::EnvVarKeyProvider; +use engram_sim::{ManualClock, SimEntropy, SimMetadataStore}; +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + sync::{ + atomic::{AtomicU8, Ordering}, + Arc, Mutex, + }, +}; +struct Security(AtomicU8); +impl SecurityEntropy for Security { + fn bytes(&self) -> Result<[u8; 32]> { + Ok([self.0.fetch_add(1, Ordering::SeqCst); 32]) + } +} +struct TokenPause { + entered: tokio::sync::oneshot::Sender<()>, + resume: tokio::sync::oneshot::Receiver<()>, +} +struct Provider { + identity: WorkerIdentity, + registration: ClientRegistration, + grants: Mutex>, + pause: Mutex>, + calls: Mutex>, +} +#[async_trait::async_trait] +impl WorkerTransport for Provider { + async fn identity(&self) -> Result { + Ok(self.identity.clone()) + } + async fn rpc( + &self, + _: &WorkerIdentity, + intent: &WorkerIntent, + op: &WorkerOperation, + ) -> Result { + let value = serde_json::to_value(op).unwrap(); + assert_eq!(intent.operation_digest, digest(&value).unwrap()); + self.calls.lock().unwrap().push(value); + match op { + WorkerOperation::Metadata { endpoint } + if endpoint == &self.registration.resource_metadata.endpoint => + { + Ok( + json!({"resource":self.registration.resource.endpoint,"authorization_servers":[self.registration.issuer]}), + ) + } + WorkerOperation::Metadata { .. } => Ok( + json!({"issuer":self.registration.issuer,"authorization_endpoint":self.registration.authorization.endpoint,"token_endpoint":self.registration.token.endpoint,"code_challenge_methods_supported":["S256"],"authorization_response_iss_parameter_supported":true}), + ), + WorkerOperation::ValidateDestination { .. } => Ok(json!({"valid":true})), + WorkerOperation::Token { .. } => { + let pause = self.pause.lock().unwrap().take(); + if let Some(pause) = pause { + pause.entered.send(()).unwrap(); + pause.resume.await.unwrap(); + } + self.grants.lock().unwrap().pop().ok_or(McpError::Transport) + } + } + } +} +struct Fixture { + authority: McpAuthority, + replica: McpAuthority, + clock: Arc, + meta: Arc, + provider: Arc, + change: ChangeContext, +} +fn fixture(grants: Vec) -> Fixture { + fixture_with_client(grants, false) +} +fn fixture_with_client(grants: Vec, confidential: bool) -> Fixture { + let clock = ManualClock::new(); + let entropy = Arc::new(SimEntropy::seeded(9)); + let meta = SimMetadataStore::new(clock.clone(), entropy.clone()); + let manager = OAuthManager::new( + meta.clone(), + Arc::new(EnvVarKeyProvider::from_bytes([8; 32], "test")), + clock.clone(), + entropy, + Arc::new(StaticSecretStore::new(HashMap::new())), + ); + let d = |path: &str| Destination { + zone_id: "public".into(), + endpoint: format!("https://auth.example/{path}"), + }; + let registration = ClientRegistration { + id: "registered-a".into(), + client_id: "public-client".into(), + confidential, + redirect_uri: "https://app.example/callback".into(), + issuer: "https://auth.example/".into(), + resource: d("mcp"), + resource_metadata: d("resource-metadata"), + authorization_metadata: d("metadata"), + authorization: d("authorize"), + token: d("token"), + scopes: BTreeSet::from(["tools.read".into()]), + issuer_parameter_required: false, + account_field: Some("account".into()), + }; + let identity = WorkerIdentity { + protocol: 1, + boot_id: "boot-a".into(), + zone_id: "public".into(), + policy_digest: "a".repeat(64), + policy_revision: "1".into(), + }; + let endpoints = [ + ®istration.resource, + ®istration.resource_metadata, + ®istration.authorization_metadata, + ®istration.authorization, + ®istration.token, + ] + .into_iter() + .map(|d| d.endpoint.clone()) + .collect(); + let registry = Registry { + enabled: true, + deployment_id: "deployment-a".into(), + zones: BTreeMap::from([( + "public".into(), + Zone { + policy_digest: identity.policy_digest.clone(), + policy_revision: "1".into(), + endpoints, + }, + )]), + registrations: BTreeMap::from([(registration.id.clone(), registration.clone())]), + }; + let provider = Arc::new(Provider { + identity, + registration: registration.clone(), + grants: Mutex::new(grants), + pause: Mutex::new(None), + calls: Mutex::new(vec![]), + }); + let security = Arc::new(Security(AtomicU8::new(1))); + let workers = BTreeMap::from([( + "public".into(), + provider.clone() as Arc, + )]); + let authority = McpAuthority::new( + manager.clone(), + security.clone(), + registry.clone(), + workers.clone(), + ) + .unwrap(); + let replica = McpAuthority::new(manager, security, registry, workers).unwrap(); + Fixture { + authority, + replica, + clock, + meta, + provider, + change: ChangeContext { + connection_id: "connection-a".into(), + config_revision: 1, + previous_authorization_generation: 0, + resource: registration.resource, + }, + } +} +fn grant(token: &str) -> Value { + json!({"access_token":token,"token_type":"Bearer","expires_in":60,"scope":"tools.read","refresh_token":"rotating-refresh"}) +} +async fn begin(f: &Fixture) -> BegunAuthorization { + f.authority + .begin(&f.change, "registered-a", None, None, "actor-a", true) + .await + .unwrap() +} +fn callback(b: &BegunAuthorization) -> Callback { + let u = url::Url::parse(&b.authorization_url).unwrap(); + Callback { + context: b.context.clone(), + registration_id: "registered-a".into(), + flow_id: b.flow_id, + state: u + .query_pairs() + .find(|(k, _)| k == "state") + .unwrap() + .1 + .into_owned(), + redirect_uri: "https://app.example/callback".into(), + issuer: Some("https://auth.example/".into()), + code: "authorization-code".into(), + actor_id: "actor-a".into(), + } +} +fn lease(f: &Fixture, status: &CredentialStatus) -> ResourceLease { + let now = f.clock.now_utc().timestamp_millis(); + ResourceLease { + context: status.context.clone(), + reference: status.reference.clone(), + worker: f.provider.identity.clone(), + intent_id: "intent-a".into(), + request_digest: "a".repeat(64), + operation: "discover".into(), + scope: WorkerScope::Administration { + request_id: "request-a".into(), + actor_id: "actor-a".into(), + }, + operation_expires_at: now + 10000, + expires_at: now + 10000, + admission: WorkerLease { + id: "lease-a".into(), + deployment_id: "deployment-a".into(), + expires_at: now + 10000, + }, + } +} +#[tokio::test] +async fn opaque_authorization_consumes_before_exchange_and_refresh_preserves_identity() { + let f = fixture(vec![grant("new-token"), grant("opaque-token")]); + let begun = begin(&f).await; + let cb = callback(&begun); + let (a, b) = tokio::join!(f.authority.callback(&cb), f.replica.callback(&cb)); + assert_eq!(usize::from(a.is_ok()) + usize::from(b.is_ok()), 1); + let status = a.or(b).unwrap(); + let first = lease(&f, &status); + assert_eq!( + f.authority.lease(&first).await.unwrap().value, + "Bearer opaque-token" + ); + f.clock.advance(std::time::Duration::from_secs(61)); + let next = lease(&f, &status); + assert_eq!( + f.replica.lease(&next).await.unwrap().value, + "Bearer new-token" + ); + assert_eq!(status.context, begun.context); + let calls = f.provider.calls.lock().unwrap(); + let tokens: Vec<_> = calls.iter().filter(|v| v["kind"] == "token").collect(); + assert_eq!(tokens.len(), 2); + assert!(tokens + .iter() + .all(|v| v["grant"].get("client_secret").is_none())); + assert_eq!(tokens[0]["grant"]["resource"], "https://auth.example/mcp"); + assert_eq!(calls.len(), 5, "no user-info or JWT probe"); +} +#[tokio::test] +async fn callback_rejects_state_issuer_redirect_expiry_and_scope() { + for defect in ["state", "issuer", "redirect", "expiry", "scope", "type"] { + let mut response = grant("opaque"); + if defect == "scope" { + response["scope"] = json!("tools.write"); + } + if defect == "type" { + response.as_object_mut().unwrap().remove("token_type"); + } + let f = fixture(vec![response]); + let b = begin(&f).await; + let mut cb = callback(&b); + match defect { + "state" => cb.state = "wrong".into(), + "issuer" => cb.issuer = None, + "redirect" => cb.redirect_uri = "https://app.example/other".into(), + "expiry" => f.clock.advance(std::time::Duration::from_secs(600)), + _ => {} + } + assert!(f.authority.callback(&cb).await.is_err(), "{defect}"); + use engram_core::traits::MetadataStore; + assert!(f + .meta + .get_oauth_credential(&OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: "connection-a".into(), + provider: "registered-a".into() + }) + .await + .unwrap() + .is_none()); + } +} +#[tokio::test] +async fn uncertain_refresh_cannot_reuse_rotated_token_after_claim_expiry() { + let f = fixture(vec![grant("opaque")]); + let b = begin(&f).await; + let status = f.authority.callback(&callback(&b)).await.unwrap(); + f.clock.advance(std::time::Duration::from_secs(61)); + assert!(f.authority.lease(&lease(&f, &status)).await.is_err()); + f.clock.advance(std::time::Duration::from_secs(31)); + assert!(f.authority.lease(&lease(&f, &status)).await.is_err()); + assert_eq!( + f.provider + .calls + .lock() + .unwrap() + .iter() + .filter(|v| v["kind"] == "token") + .count(), + 2 + ); +} + +#[tokio::test] +async fn reconnect_fences_old_binding_before_publication_and_revoke_is_cas() { + let f = fixture(vec![grant("replacement"), grant("original")]); + let first = f + .authority + .callback(&callback(&begin(&f).await)) + .await + .unwrap(); + let old_lease = lease(&f, &first); + let mut change = f.change.clone(); + change.previous_authorization_generation = first.context.authorization_generation; + let next = f + .authority + .begin( + &change, + "registered-a", + None, + Some(first.version), + "actor-a", + true, + ) + .await + .unwrap(); + let second = f.replica.callback(&callback(&next)).await.unwrap(); + assert_ne!( + first.context.credential_identity, + second.context.credential_identity + ); + assert_eq!( + second.context.authorization_generation, + first.context.authorization_generation + 1 + ); + assert!(f.authority.lease(&old_lease).await.is_err()); + assert_eq!( + f.authority.lease(&lease(&f, &second)).await.unwrap().value, + "Bearer replacement" + ); + assert!(f.authority.revoke(&first).await.is_err()); + assert_eq!(f.authority.revoke(&second).await.unwrap().status, "revoked"); + assert!(f.authority.lease(&lease(&f, &second)).await.is_err()); +} +#[tokio::test] +async fn provider_account_change_and_scope_expansion_fail_refresh() { + for field in ["account", "scope", "resource", "issuer"] { + let mut first = grant("original"); + first["account"] = json!("account-a"); + let mut refresh = grant("replacement"); + refresh[field] = json!("changed"); + let f = fixture(vec![refresh, first]); + let status = f + .authority + .callback(&callback(&begin(&f).await)) + .await + .unwrap(); + f.clock.advance(std::time::Duration::from_secs(61)); + assert!( + f.authority.lease(&lease(&f, &status)).await.is_err(), + "{field}" + ); + } +} +#[tokio::test] +async fn account_first_observed_during_refresh_remains_pinned() { + let mut known = grant("known-account"); + known["account"] = json!("account-a"); + let mut changed = grant("changed-account"); + changed["account"] = json!("account-b"); + let f = fixture(vec![ + changed, + grant("account-omitted"), + known, + grant("initial"), + ]); + let initial = f + .authority + .callback(&callback(&begin(&f).await)) + .await + .unwrap(); + for token in ["known-account", "account-omitted"] { + f.clock.advance(std::time::Duration::from_secs(61)); + assert_eq!( + f.replica.lease(&lease(&f, &initial)).await.unwrap().value, + format!("Bearer {token}") + ); + let current = f + .authority + .status(&f.change.connection_id, &initial.reference) + .await + .unwrap(); + assert_eq!(current.context, initial.context); + assert_eq!(current.reference, initial.reference); + } + f.clock.advance(std::time::Duration::from_secs(61)); + assert!(matches!( + f.authority.lease(&lease(&f, &initial)).await, + Err(McpError::Denied) + )); + assert!(matches!( + f.replica.lease(&lease(&f, &initial)).await, + Err(McpError::Unavailable) + )); + assert_eq!( + f.provider + .calls + .lock() + .unwrap() + .iter() + .filter(|call| call["kind"] == "token") + .count(), + 4, + "a rejected rotating exchange must not be retried" + ); +} + +#[tokio::test] +async fn confidential_secret_rotation_fences_oauth_even_without_orchestrator_update() { + use engram_core::types::org_secret::McpSecretRole; + let f = fixture_with_client(vec![grant("token")], true); + let secret = f + .authority + .put_secret( + &f.change, + McpSecretRole::ClientSecret, + None, + "authorization", + "client-private", + ) + .await + .unwrap(); + let b = f + .authority + .begin( + &f.change, + "registered-a", + Some(&secret.reference), + None, + "actor-a", + true, + ) + .await + .unwrap(); + let status = f.authority.callback(&callback(&b)).await.unwrap(); + { + let calls = f.provider.calls.lock().unwrap(); + let token = calls.iter().find(|v| v["kind"] == "token").unwrap(); + assert_eq!(token["endpoint"], "https://auth.example/token"); + assert_eq!(token["grant"]["client_secret"], "client-private"); + } + let mut change = f.change.clone(); + change.previous_authorization_generation = secret.context.authorization_generation; + f.authority + .put_secret( + &change, + McpSecretRole::ClientSecret, + Some(&secret.reference), + "authorization", + "rotated-private", + ) + .await + .unwrap(); + assert!(f.authority.lease(&lease(&f, &status)).await.is_err()); +} +#[tokio::test] +async fn encrypted_flow_swap_cannot_exchange_or_publish() { + let f = fixture(vec![grant("token")]); + let first = begin(&f).await; + let mut other = f.change.clone(); + other.connection_id = "connection-b".into(); + let second = f + .authority + .begin(&other, "registered-a", None, None, "actor-a", true) + .await + .unwrap(); + f.meta.with_db_mut(|db| { + let a = db.mcp_flow_states.remove(&first.flow_id).unwrap(); + let b = db.mcp_flow_states.remove(&second.flow_id).unwrap(); + db.mcp_flow_states.insert(first.flow_id, b); + db.mcp_flow_states.insert(second.flow_id, a); + }); + assert!(f.replica.callback(&callback(&first)).await.is_err()); + assert!(f.replica.callback(&callback(&second)).await.is_err()); + assert!(f + .provider + .calls + .lock() + .unwrap() + .iter() + .all(|v| v["kind"] != "token")); +} + +#[tokio::test] +async fn saturated_oauth_budget_does_not_claim_or_break_an_undispatched_refresh() { + use engram_core::{traits::MetadataStore, types::mcp::McpOAuthAdmission}; + let f = fixture(vec![grant("new"), grant("old")]); + let status = f + .authority + .callback(&callback(&begin(&f).await)) + .await + .unwrap(); + f.clock.advance(std::time::Duration::from_secs(61)); + let admissions: Vec<_> = (100..104) + .map(|n| McpOAuthAdmission { + id: uuid::Uuid::from_u128(n), + deployment_id: "deployment-a".into(), + connection_id: format!("busy-{n}"), + expires_at: f.clock.now_utc() + chrono::Duration::seconds(20), + }) + .collect(); + for a in &admissions { + assert!(f.meta.acquire_mcp_oauth_admission(a).await.unwrap()); + } + assert!(f.authority.lease(&lease(&f, &status)).await.is_err()); + let row = f + .meta + .get_oauth_credential(&OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: "connection-a".into(), + provider: "registered-a".into(), + }) + .await + .unwrap() + .unwrap(); + assert!( + row.broken_at.is_none(), + "admission denial must not consume the refresh claim" + ); + f.meta + .release_mcp_oauth_admission(&admissions[0]) + .await + .unwrap(); + assert_eq!( + f.authority.lease(&lease(&f, &status)).await.unwrap().value, + "Bearer new" + ); +} + +#[tokio::test] +async fn revoked_client_secret_cannot_start_a_new_authorization() { + use engram_core::types::org_secret::McpSecretRole; + let f = fixture_with_client(vec![grant("token")], true); + let secret = f + .authority + .put_secret( + &f.change, + McpSecretRole::ClientSecret, + None, + "authorization", + "client-private", + ) + .await + .unwrap(); + let revoked = f.authority.revoke(&secret).await.unwrap(); + let mut change = f.change.clone(); + change.previous_authorization_generation = revoked.context.authorization_generation; + assert!(f + .authority + .begin( + &change, + "registered-a", + Some(&revoked.reference), + None, + "actor-a", + true + ) + .await + .is_err()); + assert!(f.provider.calls.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn revoke_before_authority_switch_fences_old_leases_and_late_callbacks() { + use engram_core::types::org_secret::McpSecretRole; + let f = fixture(vec![ + grant("replacement"), + grant("late"), + grant("other"), + grant("original"), + ]); + let original = f + .authority + .callback(&callback(&begin(&f).await)) + .await + .unwrap(); + let original_lease = lease(&f, &original); + let mut other_change = f.change.clone(); + other_change.connection_id = "connection-b".into(); + let other_flow = f + .authority + .begin(&other_change, "registered-a", None, None, "actor-a", true) + .await + .unwrap(); + let other = f.authority.callback(&callback(&other_flow)).await.unwrap(); + let mut change = f.change.clone(); + change.previous_authorization_generation = original.context.authorization_generation; + let late = f + .authority + .begin( + &change, + "registered-a", + None, + Some(original.version), + "actor-a", + true, + ) + .await + .unwrap(); + let revoked = f.authority.revoke(&original).await.unwrap(); + assert_eq!(revoked.status, "revoked"); + assert_eq!(revoked.version, original.version + 1); + let replacement = f + .authority + .put_secret( + &change, + McpSecretRole::ApiKey, + None, + "x-api-key", + "replacement-key", + ) + .await + .unwrap(); + // No orchestrator publication occurs: the old frozen lease still fails. + assert!(f.authority.lease(&original_lease).await.is_err()); + assert!(f.replica.callback(&callback(&late)).await.is_err()); + assert_eq!( + f.authority + .status("connection-a", &original.reference) + .await + .unwrap() + .status, + "revoked" + ); + assert_eq!( + f.authority.lease(&lease(&f, &other)).await.unwrap().value, + "Bearer other" + ); + let old_key_lease = lease(&f, &replacement); + let key_receipt = f.authority.revoke(&replacement).await.unwrap(); + change.previous_authorization_generation = key_receipt.context.authorization_generation; + let flow = f + .authority + .begin( + &change, + "registered-a", + None, + Some(revoked.version), + "actor-a", + true, + ) + .await + .unwrap(); + f.authority.callback(&callback(&flow)).await.unwrap(); + assert!(f.authority.lease(&old_key_lease).await.is_err()); +} + +#[tokio::test] +async fn cancelled_attempt_cannot_exchange_or_block_replacement() { + for reconnect in [false, true] { + let f = fixture(vec![grant("new"), grant("old")]); + let expected = if reconnect { + let first = begin(&f).await; + let status = f.authority.callback(&callback(&first)).await.unwrap(); + Some(f.authority.revoke(&status).await.unwrap().version) + } else { + None + }; + let pending = f + .authority + .begin(&f.change, "registered-a", None, expected, "actor-a", true) + .await + .unwrap(); + let before = f.provider.calls.lock().unwrap().len(); + f.replica + .cancel_authorization(pending.flow_id, &f.change.connection_id, "registered-a") + .await + .unwrap(); + assert!(f.authority.callback(&callback(&pending)).await.is_err()); + assert_eq!(f.provider.calls.lock().unwrap().len(), before); + let replacement = f + .replica + .begin(&f.change, "registered-a", None, expected, "actor-a", true) + .await + .unwrap(); + f.authority + .cancel_authorization(pending.flow_id, &f.change.connection_id, "registered-a") + .await + .unwrap(); + let published = f.replica.callback(&callback(&replacement)).await.unwrap(); + assert!(f.authority.callback(&callback(&pending)).await.is_err()); + assert_eq!( + f.authority + .status(&f.change.connection_id, &published.reference) + .await + .unwrap(), + published + ); + } +} + +#[tokio::test] +async fn cancellation_fences_an_inflight_token_response() { + let f = fixture(vec![grant("late")]); + let begun = begin(&f).await; + let callback = callback(&begun); + let (entered, wait) = tokio::sync::oneshot::channel(); + let (resume, paused) = tokio::sync::oneshot::channel(); + *f.provider.pause.lock().unwrap() = Some(TokenPause { + entered, + resume: paused, + }); + let complete = f.authority.callback(&callback); + let cancel = async { + wait.await.unwrap(); + f.replica + .cancel_authorization(begun.flow_id, &f.change.connection_id, "registered-a") + .await + .unwrap(); + resume.send(()).unwrap(); + }; + let (result, ()) = tokio::join!(complete, cancel); + assert!(result.is_err()); + assert!(f + .authority + .status( + &f.change.connection_id, + &CredentialReference::OAuth { + registration_id: "registered-a".into() + } + ) + .await + .is_err()); +} diff --git a/crates/engram-coordinator/tests/mcp_operator.rs b/crates/engram-coordinator/tests/mcp_operator.rs new file mode 100644 index 000000000..9f94a3ab4 --- /dev/null +++ b/crates/engram-coordinator/tests/mcp_operator.rs @@ -0,0 +1,268 @@ +//! Actual loopback TLS tests. No worker, host, VM or deployment is started. +use engram_coordinator::mcp::operator::{server, OperatorTls}; +use engram_core::traits::MetadataStore; +use engram_protocol::mcp as wire; +use engram_sim::{ManualClock, SimEntropy, SimMetadataStore}; +use std::{path::Path, sync::Arc}; +struct Certs { + _dir: tempfile::TempDir, + ca: Vec, + server: Vec, + server_key: Vec, + operator: Vec, + operator_key: Vec, + other: Vec, + other_key: Vec, + pin: String, + other_pin: String, +} +fn certs() -> Certs { + let dir = tempfile::tempdir().unwrap(); + let run = |args: &[&str]| { + assert!(std::process::Command::new("openssl") + .args(args) + .current_dir(dir.path()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .unwrap() + .success()); + }; + run(&[ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + "ca.key", + "-out", + "ca.pem", + "-days", + "1", + "-subj", + "/CN=fixture-root", + "-addext", + "basicConstraints=critical,CA:TRUE", + ]); + std::fs::write(dir.path().join("ext"),"basicConstraints=critical,CA:FALSE\nsubjectAltName=DNS:localhost\nextendedKeyUsage=serverAuth,clientAuth\n").unwrap(); + for name in ["server", "operator", "other"] { + run(&[ + "req", + "-new", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + &format!("{name}.key"), + "-out", + &format!("{name}.csr"), + "-subj", + "/CN=localhost", + ]); + run(&[ + "x509", + "-req", + "-in", + &format!("{name}.csr"), + "-CA", + "ca.pem", + "-CAkey", + "ca.key", + "-CAcreateserial", + "-out", + &format!("{name}.pem"), + "-days", + "1", + "-extfile", + "ext", + ]); + } + fn read(dir: &Path, name: &str) -> Vec { + std::fs::read(dir.join(name)).unwrap() + } + let pin = |name: &str| { + let out = std::process::Command::new("openssl") + .args(["x509", "-in", name, "-noout", "-fingerprint", "-sha256"]) + .current_dir(dir.path()) + .output() + .unwrap(); + assert!(out.status.success()); + String::from_utf8(out.stdout) + .unwrap() + .trim() + .split_once('=') + .unwrap() + .1 + .to_owned() + }; + Certs { + ca: read(dir.path(), "ca.pem"), + server: read(dir.path(), "server.pem"), + server_key: read(dir.path(), "server.key"), + operator: read(dir.path(), "operator.pem"), + operator_key: read(dir.path(), "operator.key"), + other: read(dir.path(), "other.pem"), + other_key: read(dir.path(), "other.key"), + pin: pin("operator.pem"), + other_pin: pin("other.pem"), + _dir: dir, + } +} +#[tokio::test] +async fn operator_listener_authenticates_real_peer_and_configured_candidate() { + let f = certs(); + let meta = SimMetadataStore::new(ManualClock::new(), Arc::new(SimEntropy::seeded(831))); + let candidate = engram_testkit::operator::candidate(); + let tls = || OperatorTls { + certificate: f.server.clone(), + private_key: f.server_key.clone(), + client_ca: f.ca.clone(), + fingerprints: vec![f.pin.clone()], + }; + assert!( + server( + meta.clone(), + candidate.clone(), + tls(), + std::slice::from_ref(&f.pin) + ) + .is_err(), + "operator and orchestrator leaf identities must differ" + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (stop, done) = tokio::sync::oneshot::channel::<()>(); + let task = tokio::spawn( + server( + meta.clone(), + candidate.clone(), + tls(), + std::slice::from_ref(&f.other_pin), + ) + .unwrap() + .serve_with_incoming_shutdown( + tonic::transport::server::TcpIncoming::from(listener), + async { + let _ = done.await; + }, + ), + ); + let endpoint = + tonic::transport::Endpoint::from_shared(format!("https://localhost:{}", addr.port())) + .unwrap() + .connect_timeout(std::time::Duration::from_secs(2)) + .timeout(std::time::Duration::from_secs(2)); + let base = tonic::transport::ClientTlsConfig::new() + .ca_certificate(tonic::transport::Certificate::from_pem(&f.ca)) + .domain_name("localhost"); + if let Ok(channel) = endpoint + .clone() + .tls_config(base.clone()) + .unwrap() + .connect() + .await + { + assert!( + wire::operator_publication_service_client::OperatorPublicationServiceClient::new( + channel + ) + .read(wire::OperatorPublicationServiceReadRequest { protocol: 1 }) + .await + .is_err() + ); + } + let channel = endpoint + .clone() + .tls_config( + base.clone() + .identity(tonic::transport::Identity::from_pem(&f.other, &f.other_key)), + ) + .unwrap() + .connect() + .await + .unwrap(); + let mut denied = + wire::operator_publication_service_client::OperatorPublicationServiceClient::new(channel); + let mut forged = tonic::Request::new(wire::OperatorPublicationServiceStageRequest { + protocol: 1, + expected_generation: None, + configured_candidate_identity: candidate.identity().unwrap(), + }); + forged + .metadata_mut() + .insert("x-operator-identity", f.pin.parse().unwrap()); + assert_eq!( + denied.stage(forged).await.unwrap_err().code(), + tonic::Code::PermissionDenied + ); + assert!(meta + .read_operator_publication() + .await + .unwrap() + .pending + .is_none()); + let channel = endpoint + .tls_config(base.identity(tonic::transport::Identity::from_pem( + &f.operator, + &f.operator_key, + ))) + .unwrap() + .connect() + .await + .unwrap(); + let mut client = + wire::operator_publication_service_client::OperatorPublicationServiceClient::new(channel); + assert_eq!( + client + .read(wire::OperatorPublicationServiceReadRequest { protocol: 0 }) + .await + .unwrap_err() + .code(), + tonic::Code::InvalidArgument + ); + assert_eq!( + client + .stage(wire::OperatorPublicationServiceStageRequest { + protocol: 1, + expected_generation: None, + configured_candidate_identity: engram_core::types::managed_create::digest( + b"unconfigured input" + ) + }) + .await + .unwrap_err() + .code(), + tonic::Code::PermissionDenied + ); + let input = wire::OperatorPublicationServiceStageRequest { + protocol: 1, + expected_generation: None, + configured_candidate_identity: candidate.identity().unwrap(), + }; + let first = client.stage(input.clone()).await.unwrap().into_inner(); + let second = client.stage(input).await.unwrap().into_inner(); + assert_eq!(first.publication_json, second.publication_json); + let state: engram_core::types::managed_operator::OperatorPublication = + serde_json::from_slice(&first.publication_json).unwrap(); + assert!(!state.admission_open); + assert!(state.published.is_none()); + assert_eq!(state.pending.unwrap().candidate, candidate); + assert_eq!( + client + .read(wire::OperatorPublicationServiceReadRequest { protocol: 1 }) + .await + .unwrap() + .into_inner() + .publication_json, + first.publication_json + ); + drop(client); + drop(denied); + let _ = stop.send(()); + tokio::time::timeout(std::time::Duration::from_secs(5), task) + .await + .unwrap() + .unwrap() + .unwrap(); +} diff --git a/crates/engram-coordinator/tests/placement_reservation_live_pg.rs b/crates/engram-coordinator/tests/placement_reservation_live_pg.rs index 311c2cc45..acd43ff11 100644 --- a/crates/engram-coordinator/tests/placement_reservation_live_pg.rs +++ b/crates/engram-coordinator/tests/placement_reservation_live_pg.rs @@ -3,7 +3,7 @@ //! `reserve_placement`)'s `FOR UPDATE` transaction (burst-safe spread + //! reject), the `mem_budget_mib` ledger column, `fleet_free_mib` (Σ //! allocatable − reserved), and the one-transaction write-set's atomicity — -//! all against REAL Postgres. The api.rs create tests use a mock store whose +//! against REAL Postgres. The api.rs create tests use a mock store whose //! `reserve_and_persist_create` is a trivial in-memory stand-in, so this is //! the only coverage of the actual SQL: the transaction, the reserved-set //! aggregate, the pending/queued-row insert, the satellite writes (secrets, @@ -14,7 +14,9 @@ //! (issue #535) that the write-set commits as ONE transaction, not a chain //! of individually-failable writes. //! -//! `#[ignore]`'d by default; requires Postgres at `ENGRAM_TEST_DATABASE_URL`. +//! The Postgres tests are `#[ignore]`'d by default and require +//! `ENGRAM_TEST_DATABASE_URL`. The unignored preview regression uses the shared +//! fixtures with `SimMetadataStore` and needs no database service. //! Run: //! ```bash //! just db-up @@ -123,6 +125,10 @@ fn bare_write_set( cpu_budget_vcpus: i32, ) -> SessionCreateWriteSet { SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id, spec: spec(), mem_budget_mib, @@ -166,6 +172,71 @@ async fn reserve( } } +#[tokio::test] +async fn placement_preview_softens_only_unmeasured_dimensions() { + use engram_coordinator::placement::{placement_preview, ScheduleContext}; + + let meta: Arc = engram_sim::SimMetadataStore::new( + Arc::new(engram_core::traits::SystemClock::new()), + Arc::new(engram_sim::SimEntropy::seeded(48)), + ); + let host = seed_host(&meta, "preview-host", 0).await; + let reserved_cpu = i32::try_from(engram_core::types::host::host_cpu_budget(8)).unwrap(); + assert_eq!( + reserve(&meta, SessionId::new(), 0, reserved_cpu, &[host], 0).await, + Some(host) + ); + let context = ScheduleContext { + repo: "localhost:5001/placement-reservation", + image_version: "test", + snapshot_host: None, + memory_mib: Some(2), + cpu_budget_vcpus: Some(1), + required_image_digest: None, + exclude_host: None, + prefer_host: None, + caps: Default::default(), + prefer_bundles: &[], + }; + for (allocatable_mib, total_vcpus, need_cpu, fits) in [ + (0, 8, 1, false), + (0, 16, 1, true), + (0, 8, 0, true), + (0, 0, 1, true), + (1, 0, 1, false), + (2, 0, 1, true), + ] { + meta.touch_host_heartbeat( + host, + engram_core::types::host::HostHeartbeat { + status: HostStatus::Ready, + capacity: zero_capacity(), + utilization: HostUtilization { + allocatable_mib, + ..HostUtilization::default() + }, + ready_images: Vec::new(), + current_bundles: Vec::new(), + sandbox_bundles: Vec::new(), + total_vcpus, + wire_version: engram_protocol::WIRE_VERSION, + stages_images: false, + capabilities: Default::default(), + lease_renew_until: None, + }, + ) + .await + .unwrap(); + assert_eq!( + placement_preview(meta.as_ref(), &context, 2, need_cpu, Utc::now()) + .await + .unwrap(), + fits, + "allocatable_mib={allocatable_mib}, total_vcpus={total_vcpus}, need_cpu={need_cpu}" + ); + } +} + #[tokio::test] #[ignore = "requires live Postgres at ENGRAM_TEST_DATABASE_URL"] async fn burst_packs_one_host_then_overflows_then_rejects() { @@ -266,6 +337,10 @@ async fn reserve_and_persist_create_commits_the_full_write_set_together() { let session_id = SessionId::new(); let cap = Capability::parse("github:read@owner/repo").expect("valid capability"); let ws = SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id, spec: spec(), mem_budget_mib: 2048, diff --git a/crates/engram-coordinator/tests/queue_scanner_live_pg.rs b/crates/engram-coordinator/tests/queue_scanner_live_pg.rs index cee2d7e58..c972d6a64 100644 --- a/crates/engram-coordinator/tests/queue_scanner_live_pg.rs +++ b/crates/engram-coordinator/tests/queue_scanner_live_pg.rs @@ -69,6 +69,10 @@ async fn enqueue( cpu_budget_vcpus: i32, ) { let ws = engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id, spec, mem_budget_mib, @@ -110,6 +114,10 @@ async fn reserve( affinity_len: usize, ) -> Option { let ws = engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id, spec: spec.clone(), mem_budget_mib, diff --git a/crates/engram-core/src/error.rs b/crates/engram-core/src/error.rs index 021f0a0b0..f2f9e8919 100644 --- a/crates/engram-core/src/error.rs +++ b/crates/engram-core/src/error.rs @@ -181,6 +181,9 @@ pub enum SandboxError { /// gRPC boundary as a `failed_precondition` with a marker message /// (the `HarnessSpawn` precedent). MemoryImageUnusable(String), + /// The managed owner requires cold recovery. This error alone is not proof + /// of retirement; callers must read the exact authenticated boot receipt. + ManagedRecoveryRequired, } /// ADR 0116 B-D4: is a harness-spawn failure of this `kind` @@ -224,6 +227,7 @@ impl fmt::Display for SandboxError { write!(f, "harness_spawn: kind={kind} {message}") } Self::MemoryImageUnusable(msg) => write!(f, "memory_image_unusable: {msg}"), + Self::ManagedRecoveryRequired => write!(f, "managed sandbox requires cold recovery"), } } } diff --git a/crates/engram-core/src/traits/host_client.rs b/crates/engram-core/src/traits/host_client.rs index c9d51da41..07a711cca 100644 --- a/crates/engram-core/src/traits/host_client.rs +++ b/crates/engram-core/src/traits/host_client.rs @@ -1,24 +1,9 @@ //! The coord↔host boundary trait. //! -//! `SandboxBackend` is the VMM dimension (FC vs VZ vs Process — "what -//! kind of VM am I"). `HostClient` is the transport dimension ("where -//! am I"). Splitting the two means the coord never imports -//! `SandboxBackend` directly — it talks to a `HostClient`, which -//! either composes a local backend (in-process) or sends RPCs over the -//! WS (remote). Two impls today: -//! -//! - `LocalHostClient` in `engram-host-agent::host_client` — wraps an -//! `Arc` + an `Arc`. Used in -//! `--mode=all` (coord constructs one inline) and inside the -//! host-agent itself (what its WS server serves over the wire). -//! - `RemoteHostClient` in `engram-protocol::client` — wraps a -//! `ConnectedHost` (WS). Every method is a unary RPC. -//! -//! The trait surface unions what used to live separately on -//! `SandboxBackend`'s coord-facing methods and on `HarnessHub`'s -//! direct in-process API. `snapshot_path_for` and `set_harness_sink` -//! stay on `SandboxBackend` only — they're host-local concepts that -//! don't cross the wire. +//! `SandboxBackend` selects the VMM. `HostClient` selects its location: +//! `LocalHostClient` composes a backend and harness hub in this process; +//! `engram_protocol::grpc_client::GrpcHostClient` sends HTTP/2 RPCs to a host. +//! Host-local paths and callbacks stay on the backend. use async_trait::async_trait; @@ -386,6 +371,67 @@ pub trait HostClient: Send + Sync { )) } + /// Exact managed authority only. Unsupported hosts remain closed. + /// Read immutable disk, kernel and ordered bundle identities without creating a VM. + async fn read_managed_cold_assets( + &self, + _request: crate::types::managed_assets::ManagedColdAssetsRequest, + ) -> Result { + Err(SandboxError::Unsupported( + "managed cold assets unavailable".into(), + )) + } + async fn capture_managed_boot( + &self, + _request: crate::types::managed_boot::ManagedCaptureRequest, + ) -> Result { + Err(SandboxError::Unsupported( + "managed capture unavailable".into(), + )) + } + async fn prepare_managed_boot( + &self, + _request: crate::types::managed_boot::ManagedBootPrepare, + ) -> Result { + Err(SandboxError::Unsupported("managed boot unavailable".into())) + } + /// Exact managed authority only. Unsupported hosts remain closed. + async fn read_managed_boot( + &self, + _request: crate::types::managed_boot::ManagedBootKey, + ) -> Result { + Err(SandboxError::Unsupported("managed boot unavailable".into())) + } + /// Exact managed authority only. Unsupported hosts remain closed. + async fn finalize_managed_boot( + &self, + _request: crate::types::managed_boot::ManagedBootFinalize, + ) -> Result { + Err(SandboxError::Unsupported("managed boot unavailable".into())) + } + /// Exact managed authority only. Unsupported hosts remain closed. + async fn start_managed_boot( + &self, + _request: crate::types::managed_boot::ManagedBootStart, + ) -> Result { + Err(SandboxError::Unsupported("managed boot unavailable".into())) + } + async fn read_managed_policy( + &self, + _target: crate::types::managed_deny::ManagedPolicyTarget, + ) -> Result, SandboxError> { + Err(SandboxError::InvalidSpec( + "managed host mode unavailable".into(), + )) + } + async fn apply_managed_policy( + &self, + _request: crate::types::managed_deny::ManagedPolicyApply, + ) -> Result { + Err(SandboxError::InvalidSpec( + "managed host mode unavailable".into(), + )) + } /// ADR 0013: bundle the egress policy with the agent spawn so the /// host applies the policy to its egress proxy registry BEFORE /// starting the agent process. Atomic by construction — diff --git a/crates/engram-core/src/traits/metadata.rs b/crates/engram-core/src/traits/metadata.rs index 90bea4085..492e58c49 100644 --- a/crates/engram-core/src/traits/metadata.rs +++ b/crates/engram-core/src/traits/metadata.rs @@ -45,12 +45,47 @@ pub enum DisableEnabledImageOutcome { Blocked(Vec<(SessionId, String)>), } +#[derive(Clone, Debug)] +pub struct ManagedInitialPrompt { + pub row: crate::types::outbox::OutboxRow, + pub events: Vec<(String, serde_json::Value)>, +} + +impl ManagedInitialPrompt { + pub fn validate(&self, ws: &SessionCreateWriteSet) -> Result<(), MetaError> { + if ws.managed_creation.is_none() + || self.row.session_id != ws.session_id + || self.row.prompt_id != format!("create:{}", ws.session_id) + || self.row.kind != crate::types::outbox::OutboxKind::Prompt + || !self + .row + .payload + .get("text") + .and_then(|v| v.as_str()) + .is_some_and(|s| !s.is_empty() && s.len() <= 1024 * 1024) + || self.row.attempts != 0 + || self.row.delivered_at.is_some() + || self.row.acked_at.is_some() + { + return Err(MetaError::Conflict("invalid managed initial prompt".into())); + } + Ok(()) + } +} + /// Issue #535 (b): the durable write-set for a new session — one logical /// fact ("this session exists with these secrets/capabilities/policy/ /// harness/skills") passed to [`MetadataStore::reserve_and_persist_create`] /// to commit in a single transaction, before any host RPC. #[derive(Clone, Debug)] pub struct SessionCreateWriteSet { + /// Approved immutable cold input selection, committed with a managed reservation. + pub managed_boot_selection: Option, + /// Held managed reservation; ordinary placement must never consume it. + pub managed_creation: Option, + /// Initial prompt and visible events commit with a held managed creation. + pub managed_initial_prompt: Option, + pub mcp_policy: Option, pub session_id: SessionId, pub spec: SessionSpec, pub mem_budget_mib: i64, @@ -239,6 +274,222 @@ pub trait MetadataStore: Send + Sync { } // ---- sessions ---- + /// Read durable configured publication and pending barrier state. + async fn read_operator_publication( + &self, + ) -> Result { + unimplemented!("read_operator_publication") + } + async fn stage_operator_publication( + &self, + _expected: Option, + _candidate: crate::types::managed_operator::OperatorCandidate, + ) -> Result { + unimplemented!("stage_operator_publication") + } + async fn list_operator_participants( + &self, + _after: Option, + _limit: u32, + ) -> Result, MetaError> { + unimplemented!("list_operator_participants") + } + async fn retain_operator_participant( + &self, + _generation: &crate::types::managed_operator::OperatorGeneration, + _session: SessionId, + ) -> Result { + unimplemented!("retain_operator_participant") + } + async fn record_operator_readiness( + &self, + _generation: &crate::types::managed_operator::OperatorGeneration, + _participant: &crate::types::managed_operator::OperatorParticipant, + _proof: crate::types::managed_operator::OperatorReadiness, + ) -> Result<(), MetaError> { + unimplemented!("record_operator_readiness") + } + async fn publish_operator_generation( + &self, + _generation: &crate::types::managed_operator::OperatorGeneration, + ) -> Result { + unimplemented!("publish_operator_generation") + } + + /// Read sanitized state under the immutable creation and native association. + /// Explicit absence does not fence an in-flight reserve. Terminal rows remain readable. + async fn read_managed_creation_state( + &self, + _query: &crate::types::managed_bridge::ManagedCreationQuery, + ) -> Result { + panic!("read_managed_creation_state not implemented") + } + async fn read_managed_boot_selection( + &self, + _query: &crate::types::managed_bridge::ManagedCreationQuery, + ) -> Result { + panic!("read_managed_boot_selection not implemented") + } + async fn begin_managed_boot_intent( + &self, + _intent: crate::types::managed_bridge::ManagedBootIntent, + ) -> Result { + panic!("begin_managed_boot_intent not implemented") + } + async fn read_managed_boot_intent( + &self, + _query: &crate::types::managed_bridge::ManagedCreationQuery, + _operation: &str, + ) -> Result, MetaError> { + panic!("read_managed_boot_intent not implemented") + } + async fn advance_managed_boot_intent( + &self, + _intent_id: &str, + _revision: u64, + _change: crate::types::managed_bridge::ManagedIntentAdvance, + ) -> Result { + panic!("advance_managed_boot_intent not implemented") + } + async fn list_managed_boot_intents( + &self, + _after: Option, + _limit: u32, + ) -> Result, MetaError> { + panic!("list_managed_boot_intents not implemented") + } + + async fn get_managed_boot( + &self, + _identity: &crate::types::managed_create::ManagedCreateIdentity, + ) -> Result, MetaError> { + unimplemented!("get_managed_boot") + } + async fn reserve_managed_capture( + &self, + _request: &crate::types::managed_boot::ManagedCaptureRequest, + ) -> Result, MetaError> { + unimplemented!("reserve_managed_capture") + } + async fn record_managed_capture( + &self, + _authority: &crate::types::managed_boot::ManagedCaptureAuthority, + ) -> Result<(), MetaError> { + unimplemented!("record_managed_capture") + } + async fn managed_sandbox_open(&self, _sandbox: SandboxId) -> Result, MetaError> { + unimplemented!("managed_sandbox_open") + } + /// Claim the exact held row and physical epoch atomically before host I/O. + async fn claim_managed_boot( + &self, + _request: crate::types::managed_boot::ManagedBootPrepare, + ) -> Result { + unimplemented!("claim_managed_boot") + } + async fn read_managed_boot( + &self, + _key: &crate::types::managed_boot::ManagedBootKey, + ) -> Result { + unimplemented!("read_managed_boot") + } + /// Read the current complete boot authority for an exact physical target. + /// Missing, stale, and terminal placements fail closed. + async fn read_managed_boot_for_target( + &self, + _target: &crate::types::managed_deny::ManagedPolicyTarget, + ) -> Result { + unimplemented!("read_managed_boot_for_target") + } + async fn advance_managed_boot( + &self, + _key: &crate::types::managed_boot::ManagedBootKey, + _revision: u64, + _event: crate::types::managed_boot::ManagedBootAdvance, + ) -> Result { + unimplemented!("advance_managed_boot") + } + /// Stable session-ID pages prevent a closed earlier row from starving later boots. + async fn list_managed_boots( + &self, + _after: Option, + _limit: u32, + ) -> Result, MetaError> { + unimplemented!("list_managed_boots") + } + /// None is ordinary. Some(false) is managed and closed, including no claim. + async fn managed_boot_open(&self, _session_id: SessionId) -> Result, MetaError> { + unimplemented!("managed_boot_open") + } + async fn managed_tool_target_gone( + &self, + _session_id: SessionId, + _tool_call_id: &str, + ) -> Result { + unimplemented!("managed_tool_target_gone") + } + async fn managed_delivery_target_gone( + &self, + _session_id: SessionId, + _binding_epoch: u64, + ) -> Result { + unimplemented!("managed_delivery_target_gone") + } + async fn read_managed_creation( + &self, + _identity: &crate::types::managed_create::ManagedCreateIdentity, + ) -> Result { + unimplemented!("read_managed_creation") + } + + /// Read the original source and held state in one store snapshot. Exact + /// retries require queued, no host/sandbox and epoch zero. Only a missing + /// source returns NotFound; an existing non-held source returns Conflict. + async fn read_held_managed_creation( + &self, + _identity: &crate::types::managed_create::ManagedCreateIdentity, + ) -> Result { + unimplemented!("read_held_managed_creation") + } + + /// Read durable server policy and all current ancestors in one locked snapshot. + async fn read_mcp_write_policy( + &self, + _id: SessionId, + ) -> Result { + Err(MetaError::Conflict("policy unavailable".into())) + } + async fn read_mcp_plan_review( + &self, + _id: SessionId, + _call: &str, + ) -> Result { + Err(MetaError::Conflict("policy unavailable".into())) + } + /// Commit human decision, policy and original result delivery together. + async fn accept_mcp_plan_exit( + &self, + _input: &crate::types::mcp_policy::PlanAcceptance, + ) -> Result { + Err(MetaError::Conflict("policy unavailable".into())) + } + /// Call only after the external-write journal barrier. No grant is reusable. + async fn validate_mcp_dispatch_fence( + &self, + _context: &crate::types::mcp_policy::RequestPolicyContext, + _expected: &crate::types::mcp_policy::ExecutionRequest, + ) -> Result { + Err(MetaError::Conflict("policy unavailable".into())) + } + /// Exact execution identity only; the trusted caller must use the write fence for writes. + async fn validate_mcp_execution_identity( + &self, + _context: &crate::types::mcp_policy::RequestPolicyContext, + _expected: &crate::types::mcp_policy::ExecutionRequest, + ) -> Result { + Err(MetaError::Conflict("policy unavailable".into())) + } + async fn create_session(&self, spec: SessionSpec) -> Result; /// Atomically insert a session in `Created` status with `host_id` @@ -509,6 +760,36 @@ pub trait MetadataStore: Send + Sync { Ok(false) } + /// Reserve only a fresh ref. Ownership survives deletion of its value. + async fn reserve_mcp_org_secret( + &self, + _name: &str, + _connection_id: &str, + _role: crate::types::org_secret::McpSecretRole, + ) -> Result { + Err(MetaError::Conflict( + "MCP credentials are unavailable".into(), + )) + } + async fn get_mcp_org_secret_reservation( + &self, + _name: &str, + ) -> Result, MetaError> { + Err(MetaError::Conflict( + "MCP credential purpose is unavailable".into(), + )) + } + /// Store ciphertext only if ownership and generation still match. + async fn put_mcp_org_secret( + &self, + _sealed: crate::types::org_secret::SealedOrgSecret, + _expected: &crate::types::org_secret::McpSecretReservation, + ) -> Result { + Err(MetaError::Conflict( + "MCP credentials are unavailable".into(), + )) + } + // ---- ADR 0106: subject-scoped OAuth credentials and short-lived flows ---- /// Insert a credential when `expected_version` is `None`, or update it iff @@ -549,6 +830,72 @@ pub trait MetadataStore: Send + Sync { Err(MetaError::NotFound) } + /// Exclusive refresh dispatch. Marks the current token unavailable immediately; + /// an expired or uncertain claim cannot be reacquired without reconnect. + async fn claim_mcp_refresh( + &self, + _claim: &crate::types::mcp::McpRefreshClaim, + ) -> Result { + Err(MetaError::Conflict("MCP refresh unavailable".into())) + } + async fn complete_mcp_refresh( + &self, + _credential: crate::types::oauth::NewSealedOAuthCredential, + _claim: &crate::types::mcp::McpRefreshClaim, + ) -> Result { + Err(MetaError::Conflict("MCP refresh unavailable".into())) + } + /// Four deployment slots and one slot per connection, shared by replicas. + async fn acquire_mcp_oauth_admission( + &self, + _lease: &crate::types::mcp::McpOAuthAdmission, + ) -> Result { + Err(MetaError::Conflict("MCP admission unavailable".into())) + } + async fn release_mcp_oauth_admission( + &self, + _lease: &crate::types::mcp::McpOAuthAdmission, + ) -> Result<(), MetaError> { + Err(MetaError::Conflict("MCP admission unavailable".into())) + } + + /// Create encrypted MCP redirect state atomically with its metadata row. + async fn create_mcp_oauth_flow( + &self, + _flow: crate::types::oauth::OAuthFlow, + _state: crate::types::mcp::McpSealedState, + ) -> Result<(), MetaError> { + Err(MetaError::Conflict("MCP flow storage unavailable".into())) + } + /// Remove and return state exactly once, before any token exchange. + async fn consume_mcp_oauth_flow( + &self, + _id: uuid::Uuid, + _key: &crate::types::oauth::OAuthCredentialKey, + ) -> Result { + Err(MetaError::Conflict("MCP flow storage unavailable".into())) + } + + /// Fence this exact MCP attempt, including consumed state. Terminal attempts + /// are idempotent; a missing or differently scoped attempt is not evidence. + async fn cancel_mcp_oauth_flow( + &self, + _id: uuid::Uuid, + _key: &crate::types::oauth::OAuthCredentialKey, + ) -> Result<(), MetaError> { + Err(MetaError::Conflict("MCP flow storage unavailable".into())) + } + /// Atomically require a live consumed flow, publish by credential version CAS, + /// and finish that flow. Cancellation and publication lock the same flow row. + async fn complete_mcp_oauth_flow( + &self, + _id: uuid::Uuid, + _credential: crate::types::oauth::NewSealedOAuthCredential, + _expected_version: Option, + ) -> Result { + Err(MetaError::Conflict("MCP flow storage unavailable".into())) + } + async fn create_oauth_flow( &self, _flow: crate::types::oauth::OAuthFlow, @@ -2064,6 +2411,11 @@ pub trait MetadataStore: Send + Sync { Ok(Vec::new()) } + /// Cold inputs and capture reservations held by managed boots have no snapshot row. + async fn list_managed_disk_manifests(&self) -> Result, MetaError> { + Ok(Vec::new()) + } + // ---- session event log ---- /// Earliest `at` stamp CARRIED BY the given exec lifecycle event for @@ -2161,6 +2513,15 @@ pub trait MetadataStore: Send + Sync { )) } + /// Check exact managed physical authority, append, and acknowledge its outbox + /// row under the same session lock as interruption. Stale replays also fail. + async fn append_managed_harness_event( + &self, + _event: &crate::types::event::ManagedHarnessEvent, + ) -> Result { + unimplemented!("append_managed_harness_event") + } + /// Append once per session and key. A replay returns no index. async fn append_session_event_idempotent( &self, @@ -3214,6 +3575,25 @@ pub trait MetadataStore: Send + Sync { // injects without the orchestrator. One blob per session (upsert). Default // impls (no-op / None) so test doubles compile; `PostgresStore` is the // authority. + /// Exact current physical placement. Missing and unbound sessions fail closed. + async fn managed_policy_target( + &self, + _session_id: SessionId, + ) -> Result { + Err(MetaError::Conflict( + "managed policy producer unavailable".into(), + )) + } + /// Save the monotone desired union before host I/O, without changing placement. + async fn retain_managed_deny( + &self, + _target: &crate::types::managed_deny::ManagedPolicyTarget, + _policy: &crate::types::managed_deny::ManagedDenyPolicy, + ) -> Result<(), MetaError> { + Err(MetaError::Conflict( + "managed policy producer unavailable".into(), + )) + } async fn bind_session_integration_policy( &self, session_id: SessionId, diff --git a/crates/engram-core/src/traits/sandbox.rs b/crates/engram-core/src/traits/sandbox.rs index 82de37f53..9e78bc8ef 100644 --- a/crates/engram-core/src/traits/sandbox.rs +++ b/crates/engram-core/src/traits/sandbox.rs @@ -71,17 +71,6 @@ pub type ForgeSink = Arc; /// coord's loopback HTTP artifact endpoint). pub type UploadSink = Arc; -/// VM lifecycle seam. Production implementation: `engram-sandbox-firecracker` -/// (Firecracker over its HTTP-over-Unix-socket API). Dev implementation: -/// `engram-sandbox-process` (host subprocesses, no isolation). -/// -/// `exec_stream` is the primary exec method — it returns immediately -/// with a stream of [`ExecEvent`]s so callers can show output in real -/// time (and so multiple subscribers can observe one command running). -/// `exec` is provided with a default impl that drains the stream into -/// an in-memory [`ExecHandle`]; it's a convenience for short commands -/// and unit tests. Backend implementations only need to provide -/// `exec_stream`. /// ADR 0022 Option A: a point-in-time sample of guest memory across a /// backend's live sandboxes on one host, summed. `pss_bytes` (proportional /// set size) charges each shared clean page to a fraction of the sandboxes @@ -161,8 +150,19 @@ pub enum AgentRefresh { Restarted, } +/// Sandbox lifecycle and guest I/O. Firecracker provides production isolation; +/// VZ provides macOS development parity and Process runs without isolation. +/// `exec_stream` returns live events; `exec` buffers a short command's output. #[async_trait] pub trait SandboxBackend: Send + Sync { + /// Strict host-owned network enforcement for this exact placement. Unavailable + /// until the backend can prove all egress paths are controlled before execution. + async fn managed_network_proof( + &self, + _target: &crate::types::managed_deny::ManagedPolicyTarget, + ) -> Option { + None + } /// Boot a sandbox under `id`, which the caller mints. Every host /// resource the sandbox owns (volumes, jail, sockets) is named by /// `id` before the backend sees it. @@ -223,22 +223,19 @@ pub trait SandboxBackend: Send + Sync { /// endpoint instead). fn set_upload_sink(&self, _sink: UploadSink) {} - /// Push per-session egress policy to the backend. The - /// coordinator calls this after `create_for_session` returns, - /// once the sandbox's `guest_endpoints` is known and before - /// `start_agent` dispatches — so the harness can't make - /// network calls before the local proxy knows the policy. - /// WS-frame ordering between this notify and the subsequent - /// `start_agent` request is what guarantees the sequencing on - /// remote-host backends. + /// Install per-session egress policy in the local backend before + /// starting the agent. `HostClient::start_agent` carries the policy + /// and agent in one host RPC and applies the policy before spawn. /// - /// `RemoteSandboxBackend` forwards via the existing WS as a - /// `NotifyKind::SessionEgressPolicy`. Local backends apply the - /// policy in-process (used by `--mode=all`). The default rejects a - /// policy that needs the guest gateway because such a backend cannot - /// authenticate or intercept the guest caller. Other policies remain a - /// no-op for Process and in-test fixtures. ADR 0006 and ADR 0109. + /// The default rejects managed deny policies, guest services and tunnels + /// because they require a host proxy. Other policies are a no-op for + /// Process and test backends. async fn notify_session_policy(&self, policy: SessionEgressPolicy) -> Result<(), SandboxError> { + if policy.managed_deny.is_some() { + return Err(SandboxError::InvalidSpec( + "managed policy requires a host proxy".into(), + )); + } if !policy.guest_services.is_empty() || !policy.tunnels.is_empty() { return Err(SandboxError::InvalidSpec( "guest services and tunnels require a session-scoped guest gateway".to_string(), @@ -251,10 +248,8 @@ pub trait SandboxBackend: Send + Sync { /// chunks. A terminal backend result ends with one [`ExecEvent::Exit`] or /// [`ExecEvent::Refused`]; ending without either terminal means transport /// loss and must not be interpreted as a terminal status. Terminating the - /// stream early (dropping it) does NOT necessarily kill the - /// underlying process — backends are free to detach and let it run - /// to completion. Callers that need cancellation should hold the - /// stream until exit or use a separate kill API (future work). + /// stream early does not necessarily kill the underlying process. + /// Use `cancel_exec` to cancel its durable exec ticket. async fn exec_stream( &self, id: SandboxId, diff --git a/crates/engram-core/src/traits/secrets.rs b/crates/engram-core/src/traits/secrets.rs index 9cd0469ab..849484600 100644 --- a/crates/engram-core/src/traits/secrets.rs +++ b/crates/engram-core/src/traits/secrets.rs @@ -63,6 +63,12 @@ pub struct ResolvedSecret { #[async_trait] pub trait SecretStore: Send + Sync { + /// Check purpose before overrides, fallback, or reuse of a cached mint engine. + /// Stores with reserved references must reject generic use explicitly. + async fn check_access(&self, _name: &str, _schema: &SecretSchema) -> Result<(), SecretError> { + Ok(()) + } + /// Look up a single secret. Returns `Ok(None)` for "not present" /// (caller decides if that's fatal based on `schema.required`). /// `schema.ref` may be set, in which case the backend should @@ -94,6 +100,7 @@ pub trait SecretStore: Send + Sync { ) -> Result { let mut bundle = SecretBundle::default(); for (name, sch) in schema { + self.check_access(name, sch).await?; let override_value = overrides.and_then(|m| m.get(name)); let resolved = match override_value { Some(v) => Some(v.clone()), @@ -145,12 +152,20 @@ impl LayeredSecretStore { #[async_trait] impl SecretStore for LayeredSecretStore { + async fn check_access(&self, name: &str, schema: &SecretSchema) -> Result<(), SecretError> { + for backend in &self.backends { + backend.check_access(name, schema).await?; + } + Ok(()) + } + async fn get( &self, ctx: &SecretContext<'_>, name: &str, schema: &SecretSchema, ) -> Result, SecretError> { + self.check_access(name, schema).await?; for backend in &self.backends { if let Some(value) = backend.get(ctx, name, schema).await? { return Ok(Some(value)); diff --git a/crates/engram-core/src/types/egress.rs b/crates/engram-core/src/types/egress.rs index 026ff01d5..040f45601 100644 --- a/crates/engram-core/src/types/egress.rs +++ b/crates/engram-core/src/types/egress.rs @@ -1,5 +1,5 @@ //! Egress-policy types used by `SandboxBackend::notify_session_policy` -//! and the wire-protocol `NotifyKind::SessionEgressPolicy` frame. +//! and the policy bundled with `HostClient::start_agent`. //! //! These intentionally use flat `Vec` for host allow-lists //! rather than the `engram-egress-proxy::HostList` type — the proxy @@ -20,8 +20,8 @@ use crate::{SandboxId, SessionId}; /// Per-session egress policy the host-agent's proxy registers /// against a session's `guest_ip`. Built by the coordinator from -/// the image manifest + resolved secret bundle, shipped to the -/// owning host over the WS, applied locally by the host-agent. +/// the image manifest + resolved secret bundle and sent with the agent +/// in the host start RPC. The host-agent applies it before spawn. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct SessionEgressPolicy { pub session_id: SessionId, @@ -71,18 +71,19 @@ pub struct SessionEgressPolicy { /// straight back into the sibling's guest port instead of sending it out to /// the internet. /// - /// LAST on purpose. This type crosses the coord↔host wire as positional + /// This type crosses the coord↔host wire as positional /// bincode (ADR 0013), so field ORDER is the format: an inserted /// non-trailing field shifts every byte after it and desyncs a peer built - /// from an older tree across a roll. A trailing field is the only wire-safe - /// evolution — see `engram-protocol/tests/wire_golden.rs`, which pins these + /// from an older tree across a roll. Append fields and bump the wire version + /// on evolution — see `engram-protocol/tests/wire_golden.rs`, which pins these /// bytes and is what catches the mistake. /// - /// `#[serde(default)]` so policies serialized before this field decode with - /// none, and the short circuit simply never fires — the pre-ADR-0118 - /// behaviour. + /// The default applies to JSON only; positional bincode requires matching versions. #[serde(default)] pub apps: Vec, + /// Mandatory exact-host deny. Appended: bincode field order is the wire format. + #[serde(default)] + pub managed_deny: Option, } /// ADR 0118: one app's resolved public address, as the egress proxy needs it. diff --git a/crates/engram-core/src/types/event.rs b/crates/engram-core/src/types/event.rs index f4fdd9c69..bbb3934e6 100644 --- a/crates/engram-core/src/types/event.rs +++ b/crates/engram-core/src/types/event.rs @@ -132,3 +132,68 @@ pub struct ArtifactRow { pub file_name: Option, pub created_at: DateTime, } + +/// Sequenced managed event authority supplied by the authenticated harness route. +/// The replay key is derived from these fields; it is never parsed for authority. +#[derive(Clone, Debug)] +pub struct ManagedHarnessEvent { + pub target: super::managed_deny::ManagedPolicyTarget, + pub incarnation: u64, + pub seq: u64, + pub kind: String, + pub payload: serde_json::Value, + pub ack_id: Option, +} +impl ManagedHarnessEvent { + pub fn key(&self) -> String { + format!( + "harness:{}:{}:{}", + self.target.binding_epoch, self.incarnation, self.seq + ) + } + /// Ingestion assigns `at` anew on replay. All guest fields must still match. + pub fn matches_replay(&self, kind: &str, payload: &serde_json::Value) -> bool { + let mut stored = payload.clone(); + let mut incoming = self.payload.clone(); + for value in [&mut stored, &mut incoming] { + if let Some(object) = value.as_object_mut() { + object.remove("at"); + object.remove("idempotency_key"); + } + } + self.kind == kind && stored == incoming + } + pub fn validate(&self) -> Result<(), crate::MetaError> { + if self.target.binding_epoch == 0 + || self.target.binding_epoch > 9_007_199_254_740_991 + || !self.payload.is_object() + || self.payload.get("idempotency_key").is_some() + { + return Err(crate::MetaError::Conflict("invalid managed event".into())); + } + let expected = match self.kind.as_str() { + "run_started" | "prompt_steered" | "prompt_dequeued" => self + .payload + .get("prompt_id") + .and_then(|v| v.as_str()) + .map(str::to_owned), + "tool_call_completed" => self + .payload + .get("tool_call_id") + .and_then(|v| v.as_str()) + .map(|call| super::outbox::tool_result_outbox_id(self.target.session_id, call)), + _ => None, + }; + if self.ack_id != expected { + return Err(crate::MetaError::Conflict( + "managed event acknowledgement mismatch".into(), + )); + } + Ok(()) + } +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManagedHarnessEventAccepted { + pub idx: Option, + pub outbox_acked: bool, +} diff --git a/crates/engram-core/src/types/integration.rs b/crates/engram-core/src/types/integration.rs index 120371d8e..1c208213a 100644 --- a/crates/engram-core/src/types/integration.rs +++ b/crates/engram-core/src/types/integration.rs @@ -27,6 +27,9 @@ use serde::{Deserialize, Serialize}; /// policies persisted before 0057 still parse. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct IntegrationPolicy { + /// Mandatory exact-host denies. Retained across all policy merges. + #[serde(default)] + pub managed_deny: Option, /// Plane-B credential injections selected by the session's capabilities. #[serde(default)] pub injects: Vec, @@ -610,3 +613,23 @@ mod tests { assert_eq!(entry.mint_source, None); } } + +#[cfg(test)] +mod managed_deny_tests { + use super::*; + + #[test] + fn deny_only_policy_survives_json_and_harness_merge() { + let value = serde_json::json!({"managed_deny": crate::types::managed_deny::ManagedDenyPolicy::new(7, ["mcp.example".into()]).unwrap()}); + let policy: IntegrationPolicy = serde_json::from_value(value.clone()).unwrap(); + let merged = merge_harness_egress( + Some(policy), + &crate::types::harness::HarnessEgress { + allow_hosts: vec!["allowed.example".into()], + allow_host_patterns: vec![], + }, + ); + let encoded = serde_json::to_value(&merged).unwrap(); + assert_eq!(encoded.get("managed_deny"), value.get("managed_deny")); + } +} diff --git a/crates/engram-core/src/types/managed_assets.rs b/crates/engram-core/src/types/managed_assets.rs new file mode 100644 index 000000000..c91e8fe97 --- /dev/null +++ b/crates/engram-core/src/types/managed_assets.rs @@ -0,0 +1,137 @@ +//! Authenticated, read-only cold boot asset inspection. No VM or boot authority. +use super::{manifest::ManifestRef, sandbox::AuxRoDrive}; +use crate::{HostId, SessionId}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedColdAssetsRequest { + pub version: u32, + pub host_id: HostId, + pub session_id: SessionId, + pub intent_id: String, + pub image: String, + pub image_digest: String, + pub disk_manifest: ManifestRef, + pub aux_ro_drives: Vec, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ManagedColdBackend { + Firecracker, + Vz, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedColdAssetsReceipt { + pub request: ManagedColdAssetsRequest, + pub host_incarnation: String, + pub backend: ManagedColdBackend, + pub kernel_sha256: String, + pub disk_sha256: String, + pub disk_size_bytes: u64, +} + +impl ManagedColdAssetsRequest { + pub fn validate(&self) -> Result<(), crate::MetaError> { + let bounded = |s: &str| { + !s.is_empty() + && s.len() <= 128 + && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') + }; + if self.version != 1 + || self.host_id.as_uuid().is_nil() + || self.session_id.as_uuid().is_nil() + || !bounded(&self.intent_id) + || self.image.is_empty() + || self.image.len() > 256 + || !super::managed_deny::is_digest(&self.image_digest) + || self.disk_manifest.manifest_id.is_nil() + || self.disk_manifest.version == 0 + || self.aux_ro_drives.len() < 3 + || self.aux_ro_drives.len() > AuxRoDrive::RESERVED_SLOTS + || self.aux_ro_drives.iter().enumerate().any(|(i, d)| { + d.drive_id != AuxRoDrive::slot_drive_id(i) + || d.guest_mount != AuxRoDrive::slot_guest_mount(i) + || d.fs_type != "squashfs" + || !d + .sha256 + .as_deref() + .is_some_and(super::managed_cold::is_sha256) + }) + { + return Err(crate::MetaError::Conflict( + "invalid managed cold assets".into(), + )); + } + Ok(()) + } +} +impl ManagedColdAssetsReceipt { + /// Integrity and association only. Provenance comes from authenticated host transport. + pub fn validate_for(&self, request: &ManagedColdAssetsRequest) -> Result<(), crate::MetaError> { + request.validate()?; + if &self.request != request + || self.host_incarnation.is_empty() + || self.host_incarnation.len() > 128 + || self.disk_size_bytes == 0 + || !super::managed_cold::is_sha256(&self.disk_sha256) + || !super::managed_cold::is_sha256(&self.kernel_sha256) + { + return Err(crate::MetaError::Conflict( + "managed cold assets association mismatch".into(), + )); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn assets_reject_substitution_and_unresolved_drives() { + let r = ManagedColdAssetsRequest { + version: 1, + host_id: HostId::new(), + session_id: SessionId::new(), + intent_id: "intent-a".into(), + image: "test.invalid/image:v1".into(), + image_digest: format!("sha256:{}", "a".repeat(64)), + disk_manifest: ManifestRef::new(), + aux_ro_drives: (0..3) + .map(|i| AuxRoDrive { + drive_id: AuxRoDrive::slot_drive_id(i), + guest_mount: AuxRoDrive::slot_guest_mount(i), + fs_type: "squashfs".into(), + sha256: Some("b".repeat(64)), + }) + .collect(), + }; + let a = ManagedColdAssetsReceipt { + request: r.clone(), + host_incarnation: "incarnation-a".into(), + backend: ManagedColdBackend::Firecracker, + kernel_sha256: "c".repeat(64), + disk_sha256: "d".repeat(64), + disk_size_bytes: 4096, + }; + r.validate().unwrap(); + a.validate_for(&r).unwrap(); + let mut bad = r.clone(); + bad.aux_ro_drives.swap(0, 1); + assert!(bad.validate().is_err()); + let mut bad = r.clone(); + bad.disk_manifest = bad.disk_manifest.next_version(); + assert!(a.validate_for(&bad).is_err()); + let mut bad = a.clone(); + bad.host_incarnation.clear(); + assert!(bad.validate_for(&r).is_err()); + let mut bad = a.clone(); + bad.kernel_sha256 = "a".repeat(63); + assert!(bad.validate_for(&r).is_err()); + let mut value = serde_json::to_value(&r).unwrap(); + value["disk_path"] = "/untrusted".into(); + assert!(serde_json::from_value::(value).is_err()); + } +} diff --git a/crates/engram-core/src/types/managed_boot.rs b/crates/engram-core/src/types/managed_boot.rs new file mode 100644 index 000000000..d9438c788 --- /dev/null +++ b/crates/engram-core/src/types/managed_boot.rs @@ -0,0 +1,662 @@ +//! Durable coordinator authority and exact host commands for a closed boot. +use serde::{Deserialize, Serialize}; + +use super::{ + managed_cold::ManagedColdPrepare, + managed_create::{digest, FrozenNativeManifest, ManagedCreateIdentity}, + managed_deny::{is_digest, ManagedAppliedState, ManagedPolicyContext}, + sandbox::{AgentSpec, SandboxSpec}, +}; +use crate::MetaError; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootKey { + pub creation: ManagedCreateIdentity, + /// Registered physical placement. Binding hash is supplied only at finalization. + pub context: ManagedPolicyContext, + pub boot_attempt: String, + pub native_digest: String, +} +impl ManagedBootKey { + pub fn validate(&self) -> Result<(), MetaError> { + self.creation.validate()?; + self.context.validate().map_err(MetaError::Conflict)?; + if self.context.target.session_id != self.creation.session_id + || self.context.binding_hash.is_some() + || !is_digest(&self.native_digest) + || self.boot_attempt.is_empty() + || self.boot_attempt.len() > 128 + || !self + .boot_attempt + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') + { + return Err(conflict()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootPrepare { + pub key: ManagedBootKey, + pub cold: ManagedColdPrepare, + pub bootstrap_policy: super::managed_deny::ManagedDenyPolicy, + /// Exact persisted capture receipt. A fresh image has no capture authority. + pub capture_authority: Option, + /// Exact, non-secret SandboxSpec JSON. Decode through `spec`, never a profile. + pub sandbox_spec_json: String, +} +impl ManagedBootPrepare { + pub fn spec(&self) -> Result { + self.key.validate()?; + self.bootstrap_policy + .validate() + .map_err(MetaError::Conflict)?; + if self.sandbox_spec_json.len() > 1024 * 1024 { + return Err(conflict()); + } + let spec: SandboxSpec = + serde_json::from_str(&self.sandbox_spec_json).map_err(|_| conflict())?; + self.cold.validate(&spec).map_err(MetaError::Conflict)?; + if !spec.env.is_empty() + || self.cold.session_id != self.key.creation.session_id + || self.cold.sandbox_id != self.key.context.target.sandbox_id + || self.cold.boot_attempt != self.key.boot_attempt + { + return Err(conflict()); + } + match (&self.cold.disk.source, &self.capture_authority) { + (super::managed_cold::ManagedColdSource::FreshImage { .. }, None) => {} + (super::managed_cold::ManagedColdSource::Capture { .. }, Some(authority)) => { + authority.validate_recovery(self)?; + } + _ => return Err(conflict()), + } + Ok(spec) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootPreparedReceipt { + pub request: ManagedBootPrepare, + pub host_incarnation: String, + pub network_identity: String, + pub guest_ip: std::net::Ipv4Addr, + pub receipt_id: String, +} +impl ManagedBootPreparedReceipt { + pub fn validate(&self) -> Result<(), MetaError> { + self.request.spec()?; + if self.guest_ip.is_unspecified() + || self.guest_ip.is_loopback() + || self.host_incarnation.is_empty() + || self.host_incarnation.len() > 128 + || !is_digest(&self.network_identity) + || !is_digest(&self.receipt_id) + { + return Err(conflict()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootFinalization { + pub prepared: ManagedBootPreparedReceipt, + pub policy: ManagedAppliedState, + pub native_version: u32, + pub combined_manifest: Vec, + pub combined_manifest_digest: String, + /// Digest of the canonical full AgentSpec, including its environment. + pub agent_digest: String, +} +impl ManagedBootFinalization { + pub fn validate(&self, native: &FrozenNativeManifest) -> Result<(), MetaError> { + self.prepared.validate()?; + self.policy.validate().map_err(MetaError::Conflict)?; + let key = &self.prepared.request.key; + let mut context = key.context.clone(); + context.binding_hash = self.policy.context.binding_hash.clone(); + if context.binding_hash.is_none() + || context != self.policy.context + || self.policy.host_incarnation != self.prepared.host_incarnation + || self.policy.policy.revision < self.prepared.request.bootstrap_policy.revision + || !self + .policy + .policy + .contains(&self.prepared.request.bootstrap_policy) + || (self.policy.policy.revision == self.prepared.request.bootstrap_policy.revision + && self.policy.policy != self.prepared.request.bootstrap_policy) + || self + .policy + .network_proof + .as_ref() + .map(|p| &p.network_identity) + != Some(&self.prepared.network_identity) + || self.policy.network_proof.as_ref().map(|p| p.guest_ip) + != Some(self.prepared.guest_ip) + || native.version != self.native_version + || native.digest != key.native_digest + || self.combined_manifest_digest != digest(&self.combined_manifest) + || !is_digest(&self.agent_digest) + { + return Err(conflict()); + } + validate_combined_manifest(native, &self.combined_manifest) + } + pub fn digest(&self) -> Result { + serde_json::to_vec(self) + .map(|v| digest(&v)) + .map_err(|_| conflict()) + } +} + +/// Transport-only plaintext. The coordinator persists only its sealed form. +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootFinalize { + pub finalization: ManagedBootFinalization, + pub agent: AgentSpec, +} +impl ManagedBootFinalize { + pub fn validate_agent(&self) -> Result<(), MetaError> { + let f = &self.finalization; + let value = serde_json::to_value(&self.agent).map_err(|_| conflict())?; + let bytes = serde_json::to_vec(&value).map_err(|_| conflict())?; + if digest(&bytes) != f.agent_digest + || self.agent.binding_epoch != f.prepared.request.key.context.target.binding_epoch + || self.agent.env.get("ENGRAM_SANDBOX_ID") + != Some(&f.prepared.request.key.context.target.sandbox_id.to_string()) + || self.agent.env.get("ENGRAM_BINDING_EPOCH") + != Some( + &f.prepared + .request + .key + .context + .target + .binding_epoch + .to_string(), + ) + || self.agent.session_env.contains_key("ENGRAM_SANDBOX_ID") + || self.agent.session_env.contains_key("ENGRAM_BINDING_EPOCH") + || self.agent.argv.is_empty() + || self.agent.env.get("ENGRAM_TOOLS").map(String::as_bytes) + != Some(f.combined_manifest.as_slice()) + || self.agent.session_env.contains_key("ENGRAM_TOOLS") + || self + .agent + .env + .keys() + .chain(self.agent.session_env.keys()) + .any(|k| k.starts_with("ENGRAM_INITIAL_PROMPT")) + { + return Err(conflict()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootFinalReceipt { + pub key: ManagedBootKey, + pub prepared_receipt_id: String, + pub finalization_digest: String, + pub host_incarnation: String, + pub network_identity: String, + pub receipt_id: String, +} +impl ManagedBootFinalReceipt { + pub fn validate_for(&self, f: &ManagedBootFinalization) -> Result<(), MetaError> { + if self.key != f.prepared.request.key + || self.prepared_receipt_id != f.prepared.receipt_id + || self.finalization_digest != f.digest()? + || self.host_incarnation != f.prepared.host_incarnation + || self.network_identity != f.prepared.network_identity + || !is_digest(&self.receipt_id) + { + return Err(conflict()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootStart { + pub receipt: ManagedBootFinalReceipt, + pub start_id: String, +} +impl ManagedBootStart { + pub fn validate(&self) -> Result<(), MetaError> { + self.receipt.key.validate()?; + if !is_digest(&self.start_id) { + return Err(conflict()); + } + Ok(()) + } +} + +/// A read is bound to the complete key. Missing is never permission to start. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum ManagedBootHostState { + Missing { + key: ManagedBootKey, + }, + Prepared { + receipt: Box, + }, + Finalized { + receipt: ManagedBootFinalReceipt, + }, + StartPending { + intent: ManagedBootStart, + }, + Started { + intent: ManagedBootStart, + }, + Failed { + key: ManagedBootKey, + }, + Captured { + authority: ManagedCaptureAuthority, + }, + Retired { + receipt: Box, + }, +} + +pub fn validate_combined_manifest( + native: &FrozenNativeManifest, + bytes: &[u8], +) -> Result<(), MetaError> { + native.validate()?; + if bytes.is_empty() || bytes.len() > 1024 * 1024 { + return Err(conflict()); + } + let value: serde_json::Value = serde_json::from_slice(bytes).map_err(|_| conflict())?; + let entries = value.as_array().ok_or_else(conflict)?; + let original: Vec = + serde_json::from_slice(&native.bytes).map_err(|_| conflict())?; + if entries.len() > 128 || !entries.starts_with(&original) { + return Err(conflict()); + } + let mut names = std::collections::BTreeSet::new(); + for entry in entries { + let name = entry + .get("name") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .ok_or_else(conflict)?; + if !names.insert(name) + || !entry.get("description").is_some_and(|v| v.is_string()) + || !entry.get("inputSchema").is_some_and(|v| v.is_object()) + { + return Err(conflict()); + } + } + Ok(()) +} +pub fn conflict() -> MetaError { + MetaError::Conflict("managed boot authority changed or unavailable".into()) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ManagedBootPhase { + Claimed, + Prepared, + Finalizing, + Finalized, + StartIssued, + Started, + Interrupted, +} +impl ManagedBootPhase { + pub fn as_str(&self) -> &'static str { + match self { + Self::Claimed => "claimed", + Self::Prepared => "prepared", + Self::Finalizing => "finalizing", + Self::Finalized => "finalized", + Self::StartIssued => "start_issued", + Self::Started => "started", + Self::Interrupted => "interrupted", + } + } +} +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootSealedAgent { + pub wrapped_dek: Vec, + pub nonce: Vec, + pub ciphertext: Vec, + pub key_id: String, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootRecord { + pub request: ManagedBootPrepare, + pub revision: u64, + pub phase: ManagedBootPhase, + pub prepared: Option, + pub finalization: Option, + pub sealed_agent: Option, + pub write_policy: Option, + pub receipt: Option, + pub start: Option, + pub retired: Option, +} +/// Transactional events; no caller can assign arbitrary state or mutate a claim. +#[derive(Clone)] +pub enum ManagedBootAdvance { + Prepared(Box), + Finalize { + finalization: Box, + sealed_agent: ManagedBootSealedAgent, + write_policy: super::mcp_policy::TrustedWritePolicy, + }, + Finalized(ManagedBootFinalReceipt), + IssueStart(ManagedBootStart), + Started(ManagedBootStart), + Interrupt, + Retired(Box), +} +impl ManagedBootRecord { + pub fn claimed(request: ManagedBootPrepare) -> Self { + Self { + request, + revision: 1, + phase: ManagedBootPhase::Claimed, + prepared: None, + finalization: None, + sealed_agent: None, + write_policy: None, + receipt: None, + start: None, + retired: None, + } + } + pub fn advance( + &self, + event: ManagedBootAdvance, + native: &FrozenNativeManifest, + ) -> Result { + let mut next = self.clone(); + match event { + ManagedBootAdvance::Prepared(r) => { + r.validate()?; + if self.phase != ManagedBootPhase::Claimed || r.request != self.request { + return Err(conflict()); + } + next.prepared = Some(*r); + next.phase = ManagedBootPhase::Prepared; + } + ManagedBootAdvance::Finalize { + finalization: f, + sealed_agent, + write_policy, + } => { + f.validate(native)?; + if self.phase != ManagedBootPhase::Prepared + || Some(&f.prepared) != self.prepared.as_ref() + || sealed_agent.nonce.len() != 12 + || sealed_agent.wrapped_dek.is_empty() + || sealed_agent.ciphertext.is_empty() + || sealed_agent.key_id.is_empty() + { + return Err(conflict()); + } + next.write_policy = Some(write_policy); + next.finalization = Some(*f); + next.sealed_agent = Some(sealed_agent); + next.phase = ManagedBootPhase::Finalizing; + } + ManagedBootAdvance::Finalized(r) => { + if self.phase != ManagedBootPhase::Finalizing { + return Err(conflict()); + } + r.validate_for(self.finalization.as_ref().ok_or_else(conflict)?)?; + next.receipt = Some(r); + next.phase = ManagedBootPhase::Finalized; + } + ManagedBootAdvance::IssueStart(i) => { + i.validate()?; + if self.phase != ManagedBootPhase::Finalized + || Some(&i.receipt) != self.receipt.as_ref() + { + return Err(conflict()); + } + next.start = Some(i); + next.phase = ManagedBootPhase::StartIssued; + } + ManagedBootAdvance::Started(i) => { + if self.phase != ManagedBootPhase::StartIssued || Some(&i) != self.start.as_ref() { + return Err(conflict()); + } + next.phase = ManagedBootPhase::Started; + } + ManagedBootAdvance::Retired(receipt) => { + let prepared = self.prepared.as_ref().ok_or_else(conflict)?; + let minimum = self + .finalization + .as_ref() + .map(|f| &f.policy.policy) + .unwrap_or(&self.request.bootstrap_policy); + receipt.validate_for(prepared, minimum)?; + if self.retired.is_some() { + return Err(conflict()); + } + next.retired = Some(*receipt); + next.phase = ManagedBootPhase::Interrupted; + } + ManagedBootAdvance::Interrupt => { + if self.phase == ManagedBootPhase::Interrupted { + return Err(conflict()); + } + next.phase = ManagedBootPhase::Interrupted; + } + } + next.revision = self.revision.checked_add(1).ok_or_else(conflict)?; + Ok(next) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn combined_manifest_keeps_all_native_fields_and_order() { + let bytes=br#"[{"name":"native","description":"n","inputSchema":{},"execution":"sync","custom":{"x":1}}]"#; + let native = FrozenNativeManifest { + version: 1, + bytes: bytes.to_vec(), + digest: digest(bytes), + }; + let mut entries: Vec = serde_json::from_slice(bytes).unwrap(); + entries.push(serde_json::json!({"name":"remote","description":"r","inputSchema":{},"execution":"deferred","resultFormat":"mcp"})); + validate_combined_manifest(&native, &serde_json::to_vec(&entries).unwrap()).unwrap(); + let mut wrong = entries.clone(); + wrong[0].as_object_mut().unwrap().remove("custom"); + assert!(validate_combined_manifest(&native, &serde_json::to_vec(&wrong).unwrap()).is_err()); + let mut wrong = entries.clone(); + wrong.reverse(); + assert!(validate_combined_manifest(&native, &serde_json::to_vec(&wrong).unwrap()).is_err()); + let mut wrong = entries.clone(); + wrong[1]["name"] = "native".into(); + assert!(validate_combined_manifest(&native, &serde_json::to_vec(&wrong).unwrap()).is_err()); + assert!(validate_combined_manifest(&native, &vec![b' '; 1024 * 1024 + 1]).is_err()); + let mut many = entries; + for i in 2..129 { + many.push( + serde_json::json!({"name":format!("t{i}"),"description":"r","inputSchema":{}}), + ); + } + assert!(validate_combined_manifest(&native, &serde_json::to_vec(&many).unwrap()).is_err()); + } +} + +impl std::fmt::Debug for ManagedBootSealedAgent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ManagedBootSealedAgent([sealed])") + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedCaptureRequest { + pub key: ManagedBootKey, + pub capture_id: crate::SnapshotId, +} +impl ManagedCaptureRequest { + /// Pin this exact immutable artifact before the host starts the capture. + pub fn disk_manifest(&self) -> super::manifest::ManifestRef { + super::manifest::ManifestRef { + manifest_id: self.capture_id.as_uuid(), + version: 1, + } + } +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ManagedDiskConsistency { + CrashConsistentDiskCutV1, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ManagedCapturedSourceState { + Destroyed, +} +/// Returned only by the authenticated host capture owner. A digest is an integrity +/// check, not provenance. The coordinator persists this under the registered source. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedCaptureAuthority { + pub version: u32, + pub request: ManagedCaptureRequest, + pub host_incarnation: String, + pub disk_manifest: super::manifest::ManifestRef, + pub disk_sha256: String, + pub disk_size_bytes: u64, + pub image: String, + pub kernel_sha256: String, + pub aux_ro_drives: Vec, + pub consistency: ManagedDiskConsistency, + pub source_state: ManagedCapturedSourceState, + pub record_digest: String, +} +impl ManagedCaptureAuthority { + pub fn digest(&self) -> Result { + let mut value = self.clone(); + value.record_digest.clear(); + serde_json::to_vec(&value) + .map(|v| digest(&v)) + .map_err(|_| conflict()) + } + pub fn validate(&self) -> Result<(), MetaError> { + self.request.key.validate()?; + if self.version != 1 + || self.request.capture_id.as_uuid().is_nil() + || self.host_incarnation.is_empty() + || self.host_incarnation.len() > 128 + || self.disk_manifest != self.request.disk_manifest() + || self.disk_manifest.manifest_id.is_nil() + || self.disk_manifest.version == 0 + || self.disk_size_bytes == 0 + || !super::managed_cold::is_sha256(&self.disk_sha256) + || !super::managed_cold::is_sha256(&self.kernel_sha256) + || self.image.is_empty() + || self.image.len() > 256 + || self.record_digest != self.digest()? + { + return Err(conflict()); + } + Ok(()) + } + pub fn validate_recovery(&self, next: &ManagedBootPrepare) -> Result<(), MetaError> { + self.validate()?; + next.key.validate()?; + let old = &self.request.key; + let key = &next.key; + let cold = &next.cold; + if key.creation != old.creation + || key.native_digest != old.native_digest + || key.context.target.binding_epoch + != old + .context + .target + .binding_epoch + .checked_add(1) + .ok_or_else(conflict)? + || key.boot_attempt == old.boot_attempt + || key.context.target.sandbox_id == old.context.target.sandbox_id + || cold.image != self.image + || cold.disk.manifest != self.disk_manifest + || cold.disk.sha256 != self.disk_sha256 + || cold.disk.size_bytes != self.disk_size_bytes + || cold.kernel_sha256 != self.kernel_sha256 + || cold.aux_ro_drives != self.aux_ro_drives + || cold.disk.source + != (super::managed_cold::ManagedColdSource::Capture { + capture_id: self.request.capture_id, + session_id: old.creation.session_id, + sandbox_id: old.context.target.sandbox_id, + binding_epoch: old.context.target.binding_epoch, + record_digest: self.record_digest.clone(), + }) + { + return Err(conflict()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootFinalizeInput { + pub key: ManagedBootKey, + pub prepared_receipt_id: String, + pub policy: ManagedAppliedState, + pub native_version: u32, + pub combined_manifest: Vec, + pub combined_manifest_digest: String, +} + +/// Authenticated source retirement, not a capture or an applied-policy ACK. +/// The source VM and network are nonrunning and the exact private disk remains +/// owned for a later explicitly pinned capture. Missing state cannot produce this. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootRetiredReceipt { + pub prepared: ManagedBootPreparedReceipt, + pub retained_policy: super::managed_deny::ManagedDenyPolicy, + pub receipt_id: String, +} +impl ManagedBootRetiredReceipt { + pub fn validate_for( + &self, + prepared: &ManagedBootPreparedReceipt, + minimum: &super::managed_deny::ManagedDenyPolicy, + ) -> Result<(), MetaError> { + self.prepared.validate()?; + self.retained_policy + .validate() + .map_err(MetaError::Conflict)?; + if &self.prepared != prepared + || !is_digest(&self.receipt_id) + || self.retained_policy.revision < minimum.revision + || !self.retained_policy.contains(minimum) + || (self.retained_policy.revision == minimum.revision + && &self.retained_policy != minimum) + { + return Err(conflict()); + } + Ok(()) + } +} diff --git a/crates/engram-core/src/types/managed_bridge.rs b/crates/engram-core/src/types/managed_bridge.rs new file mode 100644 index 000000000..86b42bcbf --- /dev/null +++ b/crates/engram-core/src/types/managed_bridge.rs @@ -0,0 +1,359 @@ +//! Durable input selection and identity-only control of managed boot producers. +use super::{ + managed_assets::*, managed_boot::*, managed_create::*, manifest::ManifestRef, + sandbox::AuxRoDrive, +}; +use crate::{HostId, MetaError, SandboxId, SnapshotId}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedCreationQuery { + pub identity: ManagedCreateIdentity, + pub native_version: u32, + pub native_digest: String, +} +impl ManagedCreationQuery { + pub fn validate(&self) -> Result<(), MetaError> { + self.identity.validate()?; + if self.native_version != NATIVE_MANIFEST_VERSION + || !super::managed_deny::is_digest(&self.native_digest) + { + return Err(conflict()); + } + Ok(()) + } + pub fn matches(&self, c: &ManagedCreation) -> Result<(), MetaError> { + self.validate()?; + if self.identity != c.identity + || self.native_version != c.native.version + || self.native_digest != c.native.digest + { + return Err(conflict()); + } + Ok(()) + } +} +/// Trusted nonsecret launch metadata resolved together with dyn_0 at reservation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedHarnessLaunch { + pub descriptor: super::harness::HarnessDescriptor, + pub workdir: Option, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootSelection { + pub launch: ManagedHarnessLaunch, + pub image: String, + pub image_digest: String, + pub base_manifest: ManifestRef, + pub aux_ro_drives: Vec, + pub vcpus: u32, + pub memory_mib: u32, + pub disk_gib: u32, +} +impl ManagedBootSelection { + pub fn validate(&self) -> Result<(), MetaError> { + if !matches!(self.launch.descriptor.name.as_str(), "claude" | "codex") + || self.vcpus == 0 + || self.memory_mib == 0 + || self.disk_gib == 0 + || self.image.is_empty() + || self.image.len() > 256 + || !super::managed_deny::is_digest(&self.image_digest) + || self.base_manifest.manifest_id.is_nil() + || self.base_manifest.version == 0 + || self.aux_ro_drives.len() < 3 + || self.aux_ro_drives.len() > AuxRoDrive::RESERVED_SLOTS + || self.aux_ro_drives.iter().enumerate().any(|(i, d)| { + d.drive_id != AuxRoDrive::slot_drive_id(i) + || d.guest_mount != AuxRoDrive::slot_guest_mount(i) + || d.fs_type != "squashfs" + || !d + .sha256 + .as_deref() + .is_some_and(super::managed_cold::is_sha256) + }) + { + return Err(conflict()); + } + Ok(()) + } +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedRecoveryInput { + pub query: ManagedCreationQuery, + pub expected_source_key: ManagedBootKey, + pub recovery_id: String, +} +impl ManagedRecoveryInput { + pub fn validate(&self) -> Result<(), MetaError> { + self.query.validate()?; + self.expected_source_key.validate()?; + if self.expected_source_key.creation != self.query.identity + || self.expected_source_key.native_digest != self.query.native_digest + || !bounded_id(&self.recovery_id) + { + return Err(conflict()); + } + Ok(()) + } +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedRecoveryIntent { + pub recovery_id: String, + pub source: ManagedBootKey, + pub capture_id: SnapshotId, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootIntent { + pub query: ManagedCreationQuery, + pub intent_id: String, + pub revision: u64, + pub selection: ManagedBootSelection, + pub sandbox_id: SandboxId, + pub binding_epoch: u64, + pub disk_manifest: ManifestRef, + pub recovery: Option, + pub host_id: Option, + pub assets: Option, + pub completed: bool, +} +impl ManagedBootIntent { + pub fn operation_key(&self) -> String { + self.recovery + .as_ref() + .map(|r| format!("recover:{}", r.recovery_id)) + .unwrap_or_else(|| "initial".into()) + } + pub fn validate(&self) -> Result<(), MetaError> { + self.query.validate()?; + self.selection.validate()?; + if !bounded_id(&self.intent_id) + || self.revision == 0 + || self.sandbox_id.as_uuid().is_nil() + || self.binding_epoch == 0 + || self.binding_epoch > 9_007_199_254_740_991 + || self.disk_manifest.manifest_id.is_nil() + || self.disk_manifest.version != 1 + { + return Err(conflict()); + } + if let Some(r) = &self.recovery { + ManagedRecoveryInput { + query: self.query.clone(), + expected_source_key: r.source.clone(), + recovery_id: r.recovery_id.clone(), + } + .validate()?; + if r.capture_id.as_uuid().is_nil() + || self.binding_epoch + != r.source + .context + .target + .binding_epoch + .checked_add(1) + .ok_or_else(conflict)? + || self.disk_manifest.manifest_id != r.capture_id.as_uuid() + { + return Err(conflict()); + } + } else if self.binding_epoch != 1 + || self.disk_manifest.manifest_id == self.selection.base_manifest.manifest_id + { + return Err(conflict()); + } + if let Some(a) = &self.assets { + a.validate_for(&self.assets_request()?)?; + } + Ok(()) + } + pub fn assets_request(&self) -> Result { + Ok(ManagedColdAssetsRequest { + version: 1, + host_id: self.host_id.ok_or_else(conflict)?, + session_id: self.query.identity.session_id, + intent_id: self.intent_id.clone(), + image: self.selection.image.clone(), + image_digest: self.selection.image_digest.clone(), + disk_manifest: self.disk_manifest, + aux_ro_drives: self.selection.aux_ro_drives.clone(), + }) + } + pub fn view(&self) -> ManagedBootOperation { + ManagedBootOperation { + intent_id: self.intent_id.clone(), + revision: self.revision, + recovery_id: self.recovery.as_ref().map(|r| r.recovery_id.clone()), + source_key: self.recovery.as_ref().map(|r| r.source.clone()), + capture_id: self.recovery.as_ref().map(|r| r.capture_id), + completed: self.completed, + } + } + pub fn advance(&self, change: ManagedIntentAdvance) -> Result { + if self.completed { + return Err(conflict()); + } + let mut next = self.clone(); + match change { + ManagedIntentAdvance::SelectHost(h) + if self.host_id != Some(h) && !h.as_uuid().is_nil() => + { + next.host_id = Some(h); + next.assets = None; + } + ManagedIntentAdvance::Assets(a) if self.assets.is_none() => { + a.validate_for(&self.assets_request()?)?; + next.assets = Some(*a); + } + ManagedIntentAdvance::Complete if self.assets.is_some() => next.completed = true, + _ => return Err(conflict()), + } + next.revision = next.revision.checked_add(1).ok_or_else(conflict)?; + next.validate()?; + Ok(next) + } +} +#[derive(Clone, Debug)] +pub enum ManagedIntentAdvance { + SelectHost(HostId), + Assets(Box), + Complete, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedBootOperation { + pub intent_id: String, + pub revision: u64, + pub recovery_id: Option, + pub source_key: Option, + pub capture_id: Option, + pub completed: bool, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ManagedCreationPhase { + Absent, + Held, + Preparing, + Boot, + Recovering, + Terminal, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedCreationState { + pub query: ManagedCreationQuery, + pub state: ManagedCreationPhase, + pub boot: Option, + pub operation: Option, + pub terminal_reason: Option, +} +pub fn bounded_id(s: &str) -> bool { + !s.is_empty() && s.len() <= 128 && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') +} +pub fn conflict() -> MetaError { + MetaError::Conflict("managed creation authority changed".into()) +} + +impl ManagedCreationState { + pub fn from_parts( + query: ManagedCreationQuery, + status: Option, + creation: Option<&ManagedCreation>, + mut boot: Option, + intent: Option<&ManagedBootIntent>, + ) -> Result { + query.validate()?; + if let Some(c) = creation { + query.matches(c)?; + } else if status.is_some() || boot.is_some() || intent.is_some() { + return Err(conflict()); + } + if creation.is_some() && status.is_none() { + return Err(conflict()); + } + if let Some(b) = &mut boot { + if b.request.key.creation != query.identity + || b.request.key.native_digest != query.native_digest + { + return Err(conflict()); + } + b.sealed_agent = None; + } + if intent.is_some_and(|i| i.query != query) { + return Err(conflict()); + } + let terminal = status.filter(|s| { + matches!( + s, + super::session::SessionState::Dead + | super::session::SessionState::Failed + | super::session::SessionState::Completed + ) + }); + let state = if status.is_none() { + ManagedCreationPhase::Absent + } else if terminal.is_some() { + ManagedCreationPhase::Terminal + } else if intent.is_some_and(|i| !i.completed) { + if intent.is_some_and(|i| i.recovery.is_some()) { + ManagedCreationPhase::Recovering + } else { + ManagedCreationPhase::Preparing + } + } else if boot.is_some() { + ManagedCreationPhase::Boot + } else { + ManagedCreationPhase::Held + }; + Ok(Self { + query, + state, + boot, + operation: intent.map(|i| i.view()), + terminal_reason: terminal.map(|s| s.as_str().to_owned()), + }) + } +} + +impl ManagedBootIntent { + pub fn validate_claim(&self, r: &ManagedBootPrepare) -> Result<(), MetaError> { + self.validate()?; + let a = self.assets.as_ref().ok_or_else(conflict)?; + let spec = r.spec()?; + let t = &r.key.context.target; + if r.key.creation != self.query.identity + || r.key.native_digest != self.query.native_digest + || r.key.boot_attempt != self.intent_id + || t.sandbox_id != self.sandbox_id + || Some(t.host_id) != self.host_id + || t.binding_epoch != self.binding_epoch + || r.cold.image != self.selection.image + || r.cold.aux_ro_drives != self.selection.aux_ro_drives + || r.cold.disk.manifest != self.disk_manifest + || r.cold.disk.sha256 != a.disk_sha256 + || r.cold.disk.size_bytes != a.disk_size_bytes + || r.cold.kernel_sha256 != a.kernel_sha256 + || spec.cpu.vcpus != self.selection.vcpus + || spec.memory.max_mib != self.selection.memory_mib + || spec.disk.max_gib != self.selection.disk_gib + { + return Err(conflict()); + } + match (&self.recovery, &r.cold.disk.source) { + (None, super::managed_cold::ManagedColdSource::FreshImage { base_manifest }) + if *base_manifest == self.selection.base_manifest => {} + ( + Some(recovery), + super::managed_cold::ManagedColdSource::Capture { capture_id, .. }, + ) if *capture_id == recovery.capture_id => {} + _ => return Err(conflict()), + } + Ok(()) + } +} diff --git a/crates/engram-core/src/types/managed_cold.rs b/crates/engram-core/src/types/managed_cold.rs new file mode 100644 index 000000000..17331e102 --- /dev/null +++ b/crates/engram-core/src/types/managed_cold.rs @@ -0,0 +1,225 @@ +//! Inputs from the trusted disk-capture and boot-selection producers. +//! Structural validation is not a certificate of prior capture durability. +use serde::{Deserialize, Serialize}; + +use super::{ + ids::{SandboxId, SessionId, SnapshotId}, + manifest::ManifestRef, + sandbox::{AuxRoDrive, SandboxSpec}, +}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedColdPrepare { + pub version: u32, + pub session_id: SessionId, + pub sandbox_id: SandboxId, + pub boot_attempt: String, + pub image: String, + pub disk: ManagedColdDisk, + pub kernel_sha256: String, + /// Fully resolved, ordered boot drives. No current-stamp substitution. + pub aux_ro_drives: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedColdDisk { + /// The session's private writable lineage, selected at this exact version. + pub manifest: ManifestRef, + pub sha256: String, + pub size_bytes: u64, + pub source: ManagedColdSource, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ManagedColdSource { + /// The trusted producer forks this approved base before preparation. + FreshImage { base_manifest: ManifestRef }, + /// Only a durable, explicitly disk-only capture is eligible. The record + /// digest binds its consistency outcome; the backend cannot infer it. + Capture { + capture_id: SnapshotId, + session_id: SessionId, + sandbox_id: SandboxId, + binding_epoch: u64, + record_digest: String, + }, +} + +/// Prepared and quarantined only. This cannot be a final policy ACK and has no +/// physical epoch. The live owner must be checked again by the start producer. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedColdPrepared { + pub version: u32, + pub request: ManagedColdPrepare, + pub state: ManagedColdState, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ManagedColdState { + Closed, +} + +impl ManagedColdPrepare { + pub fn validate(&self, spec: &SandboxSpec) -> Result<(), String> { + let manifest_valid = |m: ManifestRef| !m.manifest_id.is_nil() && m.version > 0; + let bounded = |s: &str| { + !s.is_empty() + && s.len() <= 128 + && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') + }; + let source_valid = match &self.disk.source { + ManagedColdSource::FreshImage { base_manifest } => { + manifest_valid(*base_manifest) + && base_manifest.manifest_id != self.disk.manifest.manifest_id + } + ManagedColdSource::Capture { + capture_id, + session_id, + sandbox_id, + binding_epoch, + record_digest, + } => { + !capture_id.as_uuid().is_nil() + && *session_id == self.session_id + && !sandbox_id.as_uuid().is_nil() + && *binding_epoch > 0 + && *binding_epoch <= 9_007_199_254_740_991 + && super::managed_deny::is_digest(record_digest) + } + }; + let drives_valid = self.aux_ro_drives.len() >= 3 + && self.aux_ro_drives.len() <= AuxRoDrive::RESERVED_SLOTS + && self.aux_ro_drives.iter().enumerate().all(|(i, d)| { + d.drive_id == AuxRoDrive::slot_drive_id(i) + && d.guest_mount == AuxRoDrive::slot_guest_mount(i) + && d.fs_type == "squashfs" + && d.sha256.as_deref().is_some_and(is_sha256) + }); + if self.version != 1 + || self.session_id.as_uuid().is_nil() + || self.sandbox_id.as_uuid().is_nil() + || !bounded(&self.boot_attempt) + || self.image.is_empty() + || self.image.len() > 256 + || self.image != spec.image + || !manifest_valid(self.disk.manifest) + || spec.rootfs_manifest != Some(self.disk.manifest) + || spec.rootfs_source.is_none() + || self.disk.size_bytes == 0 + || !is_sha256(&self.disk.sha256) + || !is_sha256(&self.kernel_sha256) + || !source_valid + || !drives_valid + || self.aux_ro_drives != spec.aux_ro_drives + || spec.swap_source.is_some() + || spec.swap_manifest.is_some() + { + return Err("invalid managed cold preparation".into()); + } + Ok(()) + } +} + +/// Canonical content digest used by staged boot files. +pub fn is_sha256(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::sandbox::{CpuLimit, DiskLimit, MemoryLimit}; + fn inputs() -> (ManagedColdPrepare, SandboxSpec) { + let disk = ManifestRef::new(); + let drives: Vec<_> = (0..3) + .map(|i| AuxRoDrive { + drive_id: AuxRoDrive::slot_drive_id(i), + guest_mount: AuxRoDrive::slot_guest_mount(i), + fs_type: "squashfs".into(), + sha256: Some("a".repeat(64)), + }) + .collect(); + let spec = SandboxSpec { + image: "image-v1".into(), + rootfs_source: Some("disk.ext4".into()), + rootfs_manifest: Some(disk), + image_uri: None, + cpu: CpuLimit { vcpus: 1 }, + memory: MemoryLimit { max_mib: 128 }, + disk: DiskLimit { max_gib: 1 }, + ttl: None, + env: Default::default(), + workdir: None, + network: Default::default(), + aux_ro_drives: drives.clone(), + swap_mib: None, + swap_source: None, + swap_manifest: None, + }; + let request = ManagedColdPrepare { + version: 1, + session_id: SessionId::new(), + sandbox_id: SandboxId::new(), + boot_attempt: "boot-1".into(), + image: spec.image.clone(), + disk: ManagedColdDisk { + manifest: disk, + sha256: "b".repeat(64), + size_bytes: 4096, + source: ManagedColdSource::FreshImage { + base_manifest: ManifestRef::new(), + }, + }, + kernel_sha256: "c".repeat(64), + aux_ro_drives: drives, + }; + (request, spec) + } + #[test] + fn rejects_substituted_disk_and_unresolved_boot_inputs() { + let (request, mut spec) = inputs(); + assert!(request.validate(&spec).is_ok()); + spec.rootfs_manifest = Some(request.disk.manifest.next_version()); + assert!( + request.validate(&spec).is_err(), + "newer disk must not replace selected disk" + ); + spec.rootfs_manifest = Some(request.disk.manifest); + spec.aux_ro_drives[1].sha256 = None; + assert!(request.validate(&spec).is_err()); + } + #[test] + fn fresh_disk_must_have_private_lineage_and_capture_must_match_session() { + let (mut request, spec) = inputs(); + request.disk.source = ManagedColdSource::FreshImage { + base_manifest: request.disk.manifest, + }; + assert!( + request.validate(&spec).is_err(), + "shared base is not private lineage" + ); + request.disk.source = ManagedColdSource::Capture { + capture_id: SnapshotId::new(), + session_id: SessionId::new(), + sandbox_id: SandboxId::new(), + binding_epoch: 1, + record_digest: format!("sha256:{}", "d".repeat(64)), + }; + assert!(request.validate(&spec).is_err()); + } + #[test] + fn legacy_missing_source_and_unknown_mode_cannot_decode() { + let (request, _) = inputs(); + let mut encoded = serde_json::to_value(request).unwrap(); + encoded["disk"].as_object_mut().unwrap().remove("source"); + assert!(serde_json::from_value::(encoded.clone()).is_err()); + encoded["disk"]["source"] = serde_json::json!({"kind":"warm"}); + assert!(serde_json::from_value::(encoded).is_err()); + } +} diff --git a/crates/engram-core/src/types/managed_create.rs b/crates/engram-core/src/types/managed_create.rs new file mode 100644 index 000000000..ad661e44a --- /dev/null +++ b/crates/engram-core/src/types/managed_create.rs @@ -0,0 +1,176 @@ +//! Immutable native inputs for a held managed session. No boot authority. +use super::{managed_deny::is_digest, mcp_policy::McpPolicyCreation}; +use crate::{MetaError, SessionId}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +pub const NATIVE_MANIFEST_VERSION: u32 = 1; +pub const MAX_NATIVE_MANIFEST_BYTES: usize = 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManagedCreateIdentity { + pub session_id: SessionId, + pub task_id: String, + pub policy: McpPolicyCreation, + pub input_digest: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FrozenNativeManifest { + pub version: u32, + pub bytes: Vec, + pub digest: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManagedCreation { + pub identity: ManagedCreateIdentity, + pub native: FrozenNativeManifest, +} + +pub fn digest(bytes: &[u8]) -> String { + format!("sha256:{}", hex::encode(Sha256::digest(bytes))) +} + +impl FrozenNativeManifest { + pub fn validate(&self) -> Result<(), MetaError> { + let invalid = || MetaError::Conflict("invalid frozen native manifest".into()); + if self.version != NATIVE_MANIFEST_VERSION + || self.bytes.is_empty() + || self.bytes.len() > MAX_NATIVE_MANIFEST_BYTES + || self.digest != digest(&self.bytes) + { + return Err(invalid()); + } + let value: serde_json::Value = + serde_json::from_slice(&self.bytes).map_err(|_| invalid())?; + let entries = value.as_array().ok_or_else(invalid)?; + if entries.len() > 128 { + return Err(invalid()); + } + let mut names = std::collections::BTreeSet::new(); + for entry in entries { + let obj = entry.as_object().ok_or_else(invalid)?; + let name = obj + .get("name") + .and_then(|v| v.as_str()) + .filter(|v| !v.is_empty()) + .ok_or_else(invalid)?; + if !names.insert(name) + || !obj.get("description").is_some_and(|v| v.is_string()) + || !obj.get("inputSchema").is_some_and(|v| v.is_object()) + || !matches!( + obj.get("execution").and_then(|v| v.as_str()), + Some("sync" | "deferred") + ) + || obj.get("resultFormat").is_some_and(|v| v != "json") + { + return Err(invalid()); + } + if let Some(bindings) = obj.get("nativeBindings") { + let bindings = bindings.as_object().ok_or_else(invalid)?; + if bindings.values().any(|v| !v.is_null() && !v.is_string()) { + return Err(invalid()); + } + } + } + Ok(()) + } +} + +impl ManagedCreateIdentity { + pub fn validate(&self) -> Result<(), MetaError> { + if [ + self.task_id.as_str(), + self.policy.input_id.as_str(), + self.policy.actor_id.as_str(), + ] + .iter() + .any(|v| v.is_empty() || v.len() > 256) + || self.policy.parent_session_id == Some(self.session_id) + || !is_digest(&self.input_digest) + { + return Err(MetaError::Conflict( + "invalid managed creation identity".into(), + )); + } + Ok(()) + } +} + +impl ManagedCreation { + pub fn validate_for(&self, ws: &crate::traits::SessionCreateWriteSet) -> Result<(), MetaError> { + self.identity.validate()?; + self.native.validate()?; + if self.identity.session_id != ws.session_id + || ws.mcp_policy.as_ref() != Some(&self.identity.policy) + || ws.spec.mode != super::session::SessionMode::Agent + || !matches!( + ws.runtime_spec.selected_harness.as_deref(), + Some("claude" | "codex") + ) + || ws + .sealed_secrets + .as_ref() + .is_some_and(|s| s.session_id != ws.session_id) + || ws + .oauth_binding + .as_ref() + .is_some_and(|b| b.session_id != ws.session_id) + { + return Err(MetaError::Conflict( + "managed creation association mismatch".into(), + )); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn native(bytes: &[u8]) -> FrozenNativeManifest { + FrozenNativeManifest { + version: 1, + bytes: bytes.to_vec(), + digest: digest(bytes), + } + } + #[test] + fn native_creation_rejects_bad_bytes_and_preserves_full_native_contract() { + let bytes = br#" [ {"name":"mcp_native_tool","description":"native","inputSchema":{},"execution":"sync","nativeBindings":{"claude":"NativeTool","codex":"native_tool"},"extra":{"preserve":true}} ] "#; + let manifest = native(bytes); + manifest.validate().unwrap(); + assert_eq!(manifest.bytes, bytes); + native(b"[]").validate().unwrap(); + for bytes in [b"".as_slice(), b"{}", b"[{}]", br#"[{"name":"a","description":"a","inputSchema":{},"execution":"deferred","resultFormat":"mcp"}]"#] { + assert!(native(bytes).validate().is_err()); + } + let mut bad = manifest.clone(); + bad.digest = digest(b"[]"); + assert!(bad.validate().is_err()); + let mut bad = manifest; + bad.version = 2; + assert!(bad.validate().is_err()); + } + #[test] + fn native_creation_rejects_duplicates_and_limits() { + let tool = serde_json::json!({"name":"tool","description":"x","inputSchema":{},"execution":"deferred"}); + assert!(native(&serde_json::to_vec(&vec![tool.clone(); 2]).unwrap()) + .validate() + .is_err()); + let many: Vec<_> = (0..129) + .map(|i| { + let mut t = tool.clone(); + t["name"] = format!("tool_{i}").into(); + t + }) + .collect(); + assert!(native(&serde_json::to_vec(&many).unwrap()) + .validate() + .is_err()); + assert!(native(&vec![b' '; MAX_NATIVE_MANIFEST_BYTES + 1]) + .validate() + .is_err()); + } +} diff --git a/crates/engram-core/src/types/managed_deny.rs b/crates/engram-core/src/types/managed_deny.rs new file mode 100644 index 000000000..da60fc603 --- /dev/null +++ b/crates/engram-core/src/types/managed_deny.rs @@ -0,0 +1,317 @@ +//! Exact managed authority policy. The digest matches the canonical MCP envelope. +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManagedDenyPolicy { + pub version: u32, + pub revision: u64, + pub hosts: Vec, + pub digest: String, +} + +/// Accept DNS names only; normalize the same way for DNS, SNI and HTTP authority. +pub fn canonical_host(raw: &str) -> Option { + let host = raw.strip_suffix('.').unwrap_or(raw).to_ascii_lowercase(); + if host.len() > 253 + || !host.contains('.') + || host.parse::().is_ok() + || !host.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && label.as_bytes()[0].is_ascii_alphanumeric() + && label.as_bytes()[label.len() - 1].is_ascii_alphanumeric() + && label + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') + }) + { + return None; + } + Some(host) +} + +impl ManagedDenyPolicy { + pub fn new(revision: u64, hosts: impl IntoIterator) -> Result { + if revision > 9_007_199_254_740_991 { + return Err("invalid managed policy revision".into()); + } + let hosts: BTreeSet<_> = hosts + .into_iter() + .map(|h| canonical_host(&h).ok_or_else(|| "invalid managed host".to_string())) + .collect::>()?; + if hosts.len() > 4096 { + return Err("managed host limit exceeded".into()); + } + let mut result = Self { + version: 1, + revision, + hosts: hosts.into_iter().collect(), + digest: String::new(), + }; + // These are canonical key order and integer encoding in the O producer. + let bytes = format!( + "{{\"hosts\":{},\"revision\":{},\"version\":1}}", + serde_json::to_string(&result.hosts).map_err(|e| e.to_string())?, + revision + ); + result.digest = format!("sha256:{}", hex::encode(Sha256::digest(bytes.as_bytes()))); + Ok(result) + } + pub fn validate(&self) -> Result<(), String> { + if Self::new(self.revision, self.hosts.clone())? != *self { + return Err("invalid managed policy digest or canonical hosts".into()); + } + Ok(()) + } + pub fn denies(&self, host: &str) -> bool { + canonical_host(host).is_some_and(|h| self.hosts.binary_search(&h).is_ok()) + } + pub fn contains(&self, old: &Self) -> bool { + old.hosts + .iter() + .all(|h| self.hosts.binary_search(h).is_ok()) + } +} + +/// Physical placement and operator identity are supplied by trusted producers. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManagedPolicyTarget { + pub session_id: crate::SessionId, + pub sandbox_id: crate::SandboxId, + pub host_id: crate::HostId, + pub binding_epoch: u64, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManagedPolicyContext { + pub target: ManagedPolicyTarget, + pub operator_generation: u64, + pub operator_identity: String, + /// None is bootstrap, never a final binding acknowledgement. + pub binding_hash: Option, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManagedNetworkProof { + pub target: ManagedPolicyTarget, + pub guest_ip: std::net::Ipv4Addr, + pub network_identity: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManagedAppliedState { + pub context: ManagedPolicyContext, + pub policy: ManagedDenyPolicy, + pub host_incarnation: String, + pub proxy_incarnation: String, + pub state_id: String, + pub network_proof: Option, +} +impl ManagedAppliedState { + pub fn new( + context: ManagedPolicyContext, + policy: ManagedDenyPolicy, + host_incarnation: String, + proxy_incarnation: String, + network_proof: Option, + ) -> Result { + policy.validate()?; + context.validate()?; + if host_incarnation.is_empty() + || host_incarnation.len() > 128 + || proxy_incarnation.is_empty() + || proxy_incarnation.len() > 128 + { + return Err("missing managed service incarnation".into()); + } + if context.binding_hash.is_some() && network_proof.is_none() { + return Err("final policy requires strict network proof".into()); + } + if network_proof.as_ref().is_some_and(|p| { + p.target != context.target + || p.guest_ip.is_unspecified() + || !is_digest(&p.network_identity) + }) { + return Err("invalid strict network proof".into()); + } + let mut state = Self { + context, + policy, + host_incarnation, + proxy_incarnation, + state_id: String::new(), + network_proof, + }; + state.state_id = format!( + "sha256:{}", + hex::encode(Sha256::digest( + serde_json::to_vec(&state).map_err(|e| e.to_string())? + )) + ); + Ok(state) + } + pub fn validate(&self) -> Result<(), String> { + if Self::new( + self.context.clone(), + self.policy.clone(), + self.host_incarnation.clone(), + self.proxy_incarnation.clone(), + self.network_proof.clone(), + )? != *self + { + return Err("invalid managed receipt".into()); + } + Ok(()) + } +} +impl ManagedPolicyContext { + pub fn validate(&self) -> Result<(), String> { + let t = &self.target; + if t.session_id.as_uuid().is_nil() + || t.sandbox_id.as_uuid().is_nil() + || t.host_id.as_uuid().is_nil() + || t.binding_epoch == 0 + || t.binding_epoch > 9_007_199_254_740_991 + || (self.operator_generation == 0 || self.operator_generation > 9_007_199_254_740_991) + || !is_digest(&self.operator_identity) + || self.binding_hash.as_ref().is_some_and(|h| !is_digest(h)) + { + return Err("invalid managed context".into()); + } + Ok(()) + } +} +pub fn is_digest(value: &str) -> bool { + value.strip_prefix("sha256:").is_some_and(|s| { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManagedPolicyApply { + pub expected: Option, + pub context: ManagedPolicyContext, + pub egress: crate::types::egress::SessionEgressPolicy, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManagedProxyState { + pub incarnation: String, + pub session_id: crate::SessionId, + pub sandbox_id: crate::SandboxId, + pub guest_ip: std::net::Ipv4Addr, + pub policy: ManagedDenyPolicy, +} + +/// Retain the policy in its existing JSON document, including unrelated fields. +pub fn retain_in_policy_json( + old: Option<&str>, + next: &ManagedDenyPolicy, +) -> Result { + next.validate()?; + let mut value: serde_json::Value = match old { + Some(s) => serde_json::from_str(s).map_err(|e| e.to_string())?, + None => serde_json::json!({}), + }; + let object = value.as_object_mut().ok_or("invalid integration policy")?; + if let Some(old) = object.get("managed_deny").filter(|v| !v.is_null()) { + let old: ManagedDenyPolicy = + serde_json::from_value(old.clone()).map_err(|e| e.to_string())?; + old.validate()?; + if next.revision < old.revision + || !next.contains(&old) + || (next.revision == old.revision && next != &old) + { + return Err("managed policy regressed".into()); + } + } + object.insert( + "managed_deny".into(), + serde_json::to_value(next).map_err(|e| e.to_string())?, + ); + serde_json::to_string(&value).map_err(|e| e.to_string()) +} +/// A retirement observation can arrive after a newer desired policy was stored. +/// Keep that newer superset; the receipt proves source retirement, not policy ACK. +pub fn retain_retired_in_policy_json( + old: Option<&str>, + observed: &ManagedDenyPolicy, +) -> Result { + observed.validate()?; + if let Some(raw) = old { + let value: serde_json::Value = serde_json::from_str(raw).map_err(|e| e.to_string())?; + if let Some(prior) = value.get("managed_deny").filter(|v| !v.is_null()) { + let prior: ManagedDenyPolicy = + serde_json::from_value(prior.clone()).map_err(|e| e.to_string())?; + prior.validate()?; + if prior.revision > observed.revision && prior.contains(observed) { + return Ok(raw.to_owned()); + } + } + } + retain_in_policy_json(old, observed) +} + +pub fn validate_policy_replacement(old: Option<&str>, next: &str) -> Result<(), String> { + let old: serde_json::Value = old + .map(serde_json::from_str) + .transpose() + .map_err(|e| e.to_string())? + .unwrap_or(serde_json::Value::Null); + if old.get("managed_deny").is_some_and(|v| !v.is_null()) { + let new: serde_json::Value = serde_json::from_str(next).map_err(|e| e.to_string())?; + let policy = serde_json::from_value( + new.get("managed_deny") + .ok_or("managed policy cannot be removed")? + .clone(), + ) + .map_err(|e| e.to_string())?; + retain_in_policy_json(Some(&old.to_string()), &policy)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn canonical_digest_and_exact_authorities_match_the_control_contract() { + let policy = ManagedDenyPolicy::new( + 7, + ["MCP.EXAMPLE.", "alias.example", "mcp.example"].map(String::from), + ) + .unwrap(); + assert_eq!( + policy.digest, + "sha256:e8052caef31fb8d39292ff014cf58f711b253b326645e2066b5210b3bb49d265" + ); + assert_eq!(policy.hosts, ["alias.example", "mcp.example"]); + assert!(policy.denies("MCP.EXAMPLE.")); + for other in [ + "other.mcp.example", + "mcp.example.other", + "127.0.0.1", + "mcp.example:443", + ] { + assert!(!policy.denies(other)); + } + for invalid in [ + "127.0.0.1", + "::1", + "localhost", + "*.example", + "a..example", + "mcp.example..", + "mcp.example:443", + "mcp.example/", + "mcp.example\n", + ] { + assert!(ManagedDenyPolicy::new(1, [invalid.into()]).is_err()); + } + let mut reordered = policy.clone(); + reordered.hosts.reverse(); + assert!(reordered.validate().is_err()); + let mut changed = policy; + changed.revision += 1; + assert!(changed.validate().is_err()); + } +} diff --git a/crates/engram-core/src/types/managed_operator.rs b/crates/engram-core/src/types/managed_operator.rs new file mode 100644 index 000000000..6c5871289 --- /dev/null +++ b/crates/engram-core/src/types/managed_operator.rs @@ -0,0 +1,548 @@ +//! Configured operator authority and the durable all-session publication barrier. +use super::{ + managed_boot::{ + ManagedBootPrepare, ManagedBootPreparedReceipt, ManagedBootRetiredReceipt, + ManagedCaptureAuthority, + }, + managed_create::digest, + managed_deny::{canonical_host, is_digest, ManagedDenyPolicy, ManagedPolicyTarget}, +}; +use crate::{HostId, MetaError, SandboxId, SessionId}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperatorZone { + pub revision: String, + /// The worker's configuration digest is not a network or trust attestation. + pub worker_digest: String, + pub authorities: Vec, + pub aliases: Vec, + /// Exact operator configuration, including routes, CIDRs, limits and trust references. + pub configuration: serde_json::Value, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperatorCandidate { + pub version: u32, + pub zones: BTreeMap, + pub deployment_identity: String, + pub network_identity: String, + pub trust_identity: String, + /// Digests of actual CA contents supplied by the trusted projection producer. + pub ca_content_digests: BTreeMap, +} +impl OperatorCandidate { + pub fn hosts(&self) -> Result, MetaError> { + if self.version != 1 + || self.zones.is_empty() + || self.zones.len() > 128 + || ![ + &self.deployment_identity, + &self.network_identity, + &self.trust_identity, + ] + .into_iter() + .all(|s| is_digest(s)) + || self.ca_content_digests.is_empty() + || self + .ca_content_digests + .iter() + .any(|(k, v)| k.is_empty() || k.len() > 128 || !is_digest(v)) + { + return Err(conflict()); + } + let mut hosts = BTreeSet::new(); + for (id, zone) in &self.zones { + if id.is_empty() + || id.len() > 128 + || zone.revision.is_empty() + || zone.revision.len() > 128 + || !is_digest(&zone.worker_digest) + || !zone.configuration.is_object() + || zone.authorities.is_empty() + || !sorted(&zone.authorities) + || !sorted(&zone.aliases) + { + return Err(conflict()); + } + let config = &zone.configuration; + let allowed = [ + "id", + "revision", + "mode", + "authorities", + "aliases", + "cidrs", + "denyCidrs", + "ca", + "limits", + ]; + if config + .as_object() + .is_none_or(|o| o.keys().any(|k| !allowed.contains(&k.as_str()))) + || !matches!( + config.get("mode").and_then(|v| v.as_str()), + Some("public" | "internal") + ) + { + return Err(conflict()); + } + for field in ["cidrs", "denyCidrs"] { + let values = config + .get(field) + .and_then(|v| v.as_array()) + .ok_or_else(conflict)?; + for value in values { + let (ip, bits) = value + .as_str() + .and_then(|v| v.split_once('/')) + .ok_or_else(conflict)?; + let ip: std::net::IpAddr = ip.parse().map_err(|_| conflict())?; + let bits: u8 = bits.parse().map_err(|_| conflict())?; + if bits > if ip.is_ipv4() { 32 } else { 128 } { + return Err(conflict()); + } + } + } + if let Some(ca) = config.get("ca") { + let ca = ca.as_str().ok_or_else(conflict)?; + if !ca.starts_with("-----BEGIN CERTIFICATE-----") + || ca.contains("PRIVATE KEY") + || !self + .ca_content_digests + .values() + .any(|d| *d == digest(ca.as_bytes())) + { + return Err(conflict()); + } + } + if let Some(limits) = config.get("limits") { + let limits = limits.as_object().ok_or_else(conflict)?; + if limits.iter().any(|(k, v)| { + !["execute", "discover", "oauth", "schema", "toolMs"].contains(&k.as_str()) + || v.as_u64() + .is_none_or(|n| n == 0 || n > 9_007_199_254_740_991) + }) { + return Err(conflict()); + } + } + let projected = |field: &str| -> Option> { + let mut values: Vec = config + .get(field)? + .as_array()? + .iter() + .map(|v| v.as_str().map(str::to_owned)) + .collect::>()?; + values.sort(); + values.dedup(); + Some(values) + }; + if config.get("id").and_then(|v| v.as_str()) != Some(id.as_str()) + || config.get("revision").and_then(|v| v.as_str()) != Some(zone.revision.as_str()) + || projected("authorities").as_ref() != Some(&zone.authorities) + || projected("aliases").as_ref() != Some(&zone.aliases) + { + return Err(conflict()); + } + for authority in &zone.authorities { + let (host, port) = match authority.split_once(':') { + Some((h, p)) => (h, Some(p)), + None => (authority.as_str(), None), + }; + if canonical_host(host).as_deref() != Some(host) + || port.is_some_and(|p| { + p.parse::() + .ok() + .filter(|v| *v != 0 && *v != 443) + .is_none_or(|v| v.to_string() != p) + }) + { + return Err(conflict()); + } + hosts.insert(host.to_owned()); + } + for alias in &zone.aliases { + if canonical_host(alias).as_ref() != Some(alias) { + return Err(conflict()); + } + hosts.insert(alias.clone()); + } + } + if hosts.len() > 4096 + || serde_json::to_vec(self).map_err(|_| conflict())?.len() > 1024 * 1024 + { + return Err(conflict()); + } + Ok(hosts.into_iter().collect()) + } + pub fn identity(&self) -> Result { + self.hosts()?; + Ok(digest(&serde_json::to_vec(self).map_err(|_| conflict())?)) + } +} +fn sorted(v: &[String]) -> bool { + v.windows(2).all(|p| p[0] < p[1]) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperatorGeneration { + pub generation: u64, + pub expected_generation: Option, + pub identity: String, + pub candidate: OperatorCandidate, +} +impl OperatorGeneration { + pub fn new(expected: Option, candidate: OperatorCandidate) -> Result { + let generation = expected + .unwrap_or(0) + .checked_add(1) + .filter(|v| *v <= 9_007_199_254_740_991) + .ok_or_else(conflict)?; + let identity = candidate.identity()?; + Ok(Self { + generation, + expected_generation: expected, + identity, + candidate, + }) + } +} +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct OperatorPublication { + pub published: Option, + pub pending: Option, + pub admission_open: bool, +} +impl OperatorPublication { + /// A new legacy marker must include the published set before it can register. + pub fn validate_registration(&self, old: Option<&str>, next: &str) -> Result<(), MetaError> { + let managed = |raw: Option<&str>| -> Result { + Ok(raw + .map(serde_json::from_str::) + .transpose() + .map_err(|_| conflict())? + .is_some_and(|v| v.get("managed_deny").is_some_and(|p| !p.is_null()))) + }; + if !managed(old)? && managed(Some(next))? { + if self.pending.is_some() { + return Err(conflict()); + } + if let Some(published) = &self.published { + if !self.admission_open { + return Err(conflict()); + } + let (_, policy) = retain(Some(next), Vec::new())?; + if !published + .candidate + .hosts()? + .iter() + .all(|h| policy.hosts.contains(h)) + { + return Err(conflict()); + } + } + } + Ok(()) + } + + pub fn stage( + &self, + expected: Option, + candidate: OperatorCandidate, + ) -> Result { + let next = OperatorGeneration::new(expected, candidate)?; + if self.pending.as_ref() == Some(&next) + || (self.pending.is_none() && self.published.as_ref() == Some(&next)) + { + return Ok(self.clone()); + } + if self.pending.is_some() || self.published.as_ref().map(|p| p.generation) != expected { + return Err(conflict()); + } + Ok(Self { + published: self.published.clone(), + pending: Some(next), + admission_open: false, + }) + } + pub fn pending(&self, generation: &OperatorGeneration) -> Result<(), MetaError> { + if self.admission_open || self.pending.as_ref() != Some(generation) { + return Err(conflict()); + } + Ok(()) + } +} + +/// A snapshot of durable placement, registered attempt and next-boot policy. +/// Terminal UI status is included for CAS, never interpreted as destruction proof. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct OperatorParticipant { + pub session_id: SessionId, + pub status: String, + pub host_id: Option, + pub sandbox_id: Option, + pub binding_epoch: u64, + pub held_creation: bool, + pub boot: Option<(ManagedBootPrepare, u64, Option)>, + pub policy: ManagedDenyPolicy, +} +impl OperatorParticipant { + pub fn target(&self) -> Result, MetaError> { + match (self.host_id, self.sandbox_id) { + (Some(host_id), Some(sandbox_id)) => { + let target = ManagedPolicyTarget { + session_id: self.session_id, + host_id, + sandbox_id, + binding_epoch: self.binding_epoch, + }; + if self + .boot + .as_ref() + .is_some_and(|(k, _, _)| k.key.context.target != target) + { + return Err(conflict()); + } + Ok(Some(target)) + } + (host, None) => { + let Some((request, _, _)) = &self.boot else { + return if host.is_none() { + Ok(None) + } else { + Err(conflict()) + }; + }; + let target = &request.key.context.target; + if target.session_id != self.session_id + || target.binding_epoch != self.binding_epoch + || host.is_some_and(|h| h != target.host_id) + { + return Err(conflict()); + } + Ok(Some(target.clone())) + } + _ => Err(conflict()), + } + } + pub fn dormant(&self) -> bool { + self.held_creation + && self.boot.is_none() + && self.binding_epoch == 0 + && self.host_id.is_none() + && self.sandbox_id.is_none() + && self.status == "queued" + } +} +/// Publication accepts only never-allocated or permanent source evidence. +/// Live Applied receipts cannot represent durably invalidated service lifetimes. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum OperatorReadiness { + NeverAllocated, + Retired(Box), + Captured(Box), +} +impl OperatorReadiness { + pub fn validate( + &self, + generation: &OperatorGeneration, + p: &OperatorParticipant, + ) -> Result<(), MetaError> { + p.policy.validate().map_err(MetaError::Conflict)?; + if !generation + .candidate + .hosts()? + .iter() + .all(|h| p.policy.hosts.contains(h)) + { + return Err(conflict()); + } + match self { + Self::NeverAllocated if p.dormant() => Ok(()), + Self::Captured(a) => { + a.validate()?; + let (request, _, prepared) = p.boot.as_ref().ok_or_else(conflict)?; + let key = &request.key; + if a.request.key != *key + || a.image != request.cold.image + || a.kernel_sha256 != request.cold.kernel_sha256 + || a.aux_ro_drives != request.cold.aux_ro_drives + || Some(&key.context.target) != p.target()?.as_ref() + || prepared + .as_ref() + .is_some_and(|r| r.host_incarnation != a.host_incarnation) + { + return Err(conflict()); + } + Ok(()) + } + Self::Retired(r) => { + let (request, _, prepared) = p.boot.as_ref().ok_or_else(conflict)?; + let key = &request.key; + if r.prepared.request != *request + || Some(&key.context.target) != p.target()?.as_ref() + { + return Err(conflict()); + } + r.validate_for( + prepared.as_ref().unwrap_or(&r.prepared), + &request.bootstrap_policy, + )?; + if !p.policy.contains(&r.retained_policy) + || p.policy.revision < r.retained_policy.revision + { + return Err(conflict()); + } + Ok(()) + } + _ => Err(conflict()), + } + } +} +/// Union history in the existing IntegrationPolicy. Repeated scans keep a stable revision. +pub fn retain( + raw: Option<&str>, + hosts: Vec, +) -> Result<(String, ManagedDenyPolicy), MetaError> { + let value: serde_json::Value = raw + .map(serde_json::from_str) + .transpose() + .map_err(|_| conflict())? + .unwrap_or(serde_json::json!({})); + let old: Option = value + .get("managed_deny") + .filter(|v| !v.is_null()) + .map(|v| serde_json::from_value(v.clone())) + .transpose() + .map_err(|_| conflict())?; + if let Some(old) = &old { + old.validate().map_err(MetaError::Conflict)?; + } + let all = hosts + .into_iter() + .chain(old.iter().flat_map(|p| p.hosts.iter().cloned())); + let mut policy = ManagedDenyPolicy::new(old.as_ref().map_or(1, |p| p.revision), all) + .map_err(MetaError::Conflict)?; + if old.as_ref().is_some_and(|o| o.hosts != policy.hosts) { + policy = ManagedDenyPolicy::new( + policy.revision.checked_add(1).ok_or_else(conflict)?, + policy.hosts, + ) + .map_err(MetaError::Conflict)?; + } + Ok(( + super::managed_deny::retain_in_policy_json(raw, &policy).map_err(MetaError::Conflict)?, + policy, + )) +} +pub fn conflict() -> MetaError { + MetaError::Conflict("operator publication changed or unavailable".into()) +} + +/// A new operator candidate invalidates old receipts even if its host set is unchanged. +pub fn retain_generation( + raw: Option<&str>, + hosts: Vec, + first: bool, +) -> Result<(String, ManagedDenyPolicy), MetaError> { + let (mut json, mut policy) = retain(raw, hosts)?; + if first && raw.is_some() { + let (_, old) = retain(raw, Vec::new())?; + if policy.revision == old.revision { + policy = ManagedDenyPolicy::new( + policy.revision.checked_add(1).ok_or_else(conflict)?, + policy.hosts, + ) + .map_err(MetaError::Conflict)?; + json = super::managed_deny::retain_in_policy_json(raw, &policy) + .map_err(MetaError::Conflict)?; + } + } + Ok((json, policy)) +} + +#[cfg(test)] +mod tests { + use super::*; + fn candidate() -> OperatorCandidate { + let zone = OperatorZone { + revision: "1".into(), + worker_digest: digest(b"worker"), + authorities: vec!["unused.example:8443".into()], + aliases: vec!["alias.example".into()], + configuration: serde_json::json!({"id":"unused","revision":"1","mode":"public","authorities":["unused.example:8443"],"aliases":["alias.example"],"cidrs":[],"denyCidrs":[]}), + }; + OperatorCandidate { + version: 1, + zones: [("unused".into(), zone)].into(), + deployment_identity: digest(b"deployment"), + network_identity: digest(b"network"), + trust_identity: digest(b"trust"), + ca_content_digests: [("root".into(), digest(b"certificate"))].into(), + } + } + #[test] + fn complete_projection_and_non_host_changes_have_distinct_identity() { + let original = candidate(); + assert_eq!( + original.hosts().unwrap(), + ["alias.example", "unused.example"] + ); + for field in ["cidrs", "denyCidrs"] { + let mut changed = original.clone(); + changed.zones.get_mut("unused").unwrap().configuration[field] = + serde_json::json!(["192.0.2.0/24"]); + assert_eq!(changed.hosts().unwrap(), original.hosts().unwrap()); + assert_ne!(changed.identity().unwrap(), original.identity().unwrap()); + } + let mut changed = original.clone(); + changed.trust_identity = digest(b"next trust"); + assert_ne!(changed.identity().unwrap(), original.identity().unwrap()); + } + #[test] + fn candidate_rejects_credentials_invalid_origins_and_projection_drift() { + let original = candidate(); + for (field, value) in [ + ("credentials", serde_json::json!({"token":"fixture"})), + ("ca", serde_json::json!("-----BEGIN PRIVATE KEY-----")), + ("cidrs", serde_json::json!(["192.0.2.0/99"])), + ("limits", serde_json::json!({"token":"fixture"})), + ("authorities", serde_json::json!(["another.example"])), + ] { + let mut invalid = original.clone(); + invalid.zones.get_mut("unused").unwrap().configuration[field] = value; + assert!(invalid.identity().is_err(), "{field}"); + } + for authority in [ + "UPPER.example", + "https://api.example", + "api.example:443", + "api.example:0", + "api.example:0444", + "127.0.0.1", + ] { + let mut invalid = original.clone(); + let zone = invalid.zones.get_mut("unused").unwrap(); + zone.authorities = vec![authority.into()]; + zone.configuration["authorities"] = serde_json::json!([authority]); + assert!(invalid.identity().is_err(), "{authority}"); + } + let mut invalid = original; + invalid.ca_content_digests.clear(); + assert!(invalid.identity().is_err()); + } + #[test] + fn ca_identity_uses_actual_content_not_a_path() { + let mut value = candidate(); + let ca = "-----BEGIN CERTIFICATE-----\nfixture\n-----END CERTIFICATE-----"; + value.zones.get_mut("unused").unwrap().configuration["ca"] = ca.into(); + assert!(value.identity().is_err()); + value + .ca_content_digests + .insert("root".into(), digest(ca.as_bytes())); + assert!(value.identity().is_ok()); + } +} diff --git a/crates/engram-core/src/types/mcp.rs b/crates/engram-core/src/types/mcp.rs new file mode 100644 index 000000000..fef222648 --- /dev/null +++ b/crates/engram-core/src/types/mcp.rs @@ -0,0 +1,48 @@ +//! Coordinator-owned MCP persistence records. No plaintext credential fields. +use serde::{Deserialize, Serialize}; + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct McpSealedState { + pub wrapped_dek: Vec, + pub nonce: Vec, + pub ciphertext: Vec, + pub key_id: String, +} + +impl std::fmt::Debug for McpSealedState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("McpSealedState([redacted])") + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct McpRefreshClaim { + pub id: uuid::Uuid, + pub key: super::oauth::OAuthCredentialKey, + pub version: i64, + pub expires_at: chrono::DateTime, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct McpOAuthAdmission { + pub id: uuid::Uuid, + pub deployment_id: String, + pub connection_id: String, + pub expires_at: chrono::DateTime, +} + +#[cfg(test)] +mod tests { + use super::McpSealedState; + + #[test] + fn sealed_state_debug_redacts_all_envelope_fields() { + let state = McpSealedState { + wrapped_dek: vec![11; 32], + nonce: vec![22; 12], + ciphertext: vec![33; 64], + key_id: "test-key-id".into(), + }; + assert_eq!(format!("{state:?}"), "McpSealedState([redacted])"); + assert_eq!(format!("{state:#?}"), "McpSealedState([redacted])"); + } +} diff --git a/crates/engram-core/src/types/mcp_policy.rs b/crates/engram-core/src/types/mcp_policy.rs new file mode 100644 index 000000000..a75509585 --- /dev/null +++ b/crates/engram-core/src/types/mcp_policy.rs @@ -0,0 +1,394 @@ +//! Conservative server authorization, independent of guest mode reports. +use super::event::PersistedEvent; +use crate::{MetaError, SessionId}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WriteEligibility { + Unknown, + ReadOnly, + WriteEligible, +} +impl WriteEligibility { + pub fn as_str(self) -> &'static str { + match self { + Self::Unknown => "unknown", + Self::ReadOnly => "read_only", + Self::WriteEligible => "write_eligible", + } + } +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WritePolicyRef { + pub session_id: SessionId, + pub policy_revision: i64, + pub binding_epoch: i64, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct McpPolicyCreation { + pub actor_id: String, + pub input_id: String, + pub parent_session_id: Option, + pub eligibility: WriteEligibility, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StoredWritePolicy { + pub revision: i64, + pub eligibility: WriteEligibility, + pub initialized: bool, + pub cause: String, + pub accepted_input_id: String, + pub actor_id: Option, + pub prompt_id: Option, + pub plan_tool_call_id: Option, + pub plan_request_digest: Option, + /// Requests before the accepted restriction cannot approve that restriction. + pub event_floor: i64, +} +impl StoredWritePolicy { + pub fn initial(input: &McpPolicyCreation) -> Self { + Self { + revision: 1, + eligibility: input.eligibility, + initialized: input.eligibility != WriteEligibility::Unknown, + cause: "initial".into(), + accepted_input_id: input.input_id.clone(), + actor_id: Some(input.actor_id.clone()), + prompt_id: None, + plan_tool_call_id: None, + plan_request_digest: None, + event_floor: 0, + } + } + pub fn restrict( + previous: Option<&Self>, + input_id: &str, + event_floor: i64, + ) -> Result { + Ok(Self { + revision: previous + .map_or(0, |p| p.revision) + .checked_add(1) + .ok_or_else(|| conflict("policy revision exhausted"))?, + eligibility: WriteEligibility::ReadOnly, + initialized: previous.is_some_and(|p| p.initialized), + cause: "plan_accept".into(), + accepted_input_id: input_id.into(), + actor_id: None, + prompt_id: Some(input_id.into()), + plan_tool_call_id: None, + plan_request_digest: None, + event_floor, + }) + } +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TrustedWritePolicy { + #[serde(rename = "ref")] + pub reference: WritePolicyRef, + pub write_eligibility: WriteEligibility, + pub cause: String, + pub accepted_input_id: String, + pub authenticated_actor_id: Option, + pub prompt_id: Option, + pub plan_tool_call_id: Option, + pub plan_request_digest: Option, + pub ancestor_policy_refs: Vec, + pub current: bool, + pub initialized: bool, +} +/// Internal row view. The parent comes from persistence, never caller-supplied ancestry. +#[derive(Clone, Debug)] +pub struct PolicyNode { + pub id: SessionId, + pub binding_epoch: i64, + pub parent: Option, + pub policy: Option, + pub terminal: bool, + pub active: bool, +} +impl PolicyNode { + pub fn reference(&self) -> WritePolicyRef { + WritePolicyRef { + session_id: self.id, + policy_revision: self.policy.as_ref().map_or(0, |p| p.revision), + binding_epoch: self.binding_epoch, + } + } +} +pub fn compose_policy(nodes: &[PolicyNode]) -> Result { + let leaf = nodes.first().ok_or(MetaError::NotFound)?; + let mut eligibility = WriteEligibility::WriteEligible; + let mut current = true; + for node in nodes { + let e = node + .policy + .as_ref() + .map_or(WriteEligibility::Unknown, |p| p.eligibility); + current &= !node.terminal; + eligibility = match (eligibility, e) { + (WriteEligibility::Unknown, _) | (_, WriteEligibility::Unknown) => { + WriteEligibility::Unknown + } + (WriteEligibility::ReadOnly, _) | (_, WriteEligibility::ReadOnly) => { + WriteEligibility::ReadOnly + } + _ => WriteEligibility::WriteEligible, + }; + } + if !current { + eligibility = WriteEligibility::Unknown; + } + let p = leaf.policy.as_ref(); + Ok(TrustedWritePolicy { + reference: leaf.reference(), + write_eligibility: eligibility, + cause: p.map_or("unknown", |p| p.cause.as_str()).into(), + accepted_input_id: p.map_or("", |p| p.accepted_input_id.as_str()).into(), + authenticated_actor_id: p.and_then(|p| p.actor_id.clone()), + prompt_id: p.and_then(|p| p.prompt_id.clone()), + plan_tool_call_id: p.and_then(|p| p.plan_tool_call_id.clone()), + plan_request_digest: p.and_then(|p| p.plan_request_digest.clone()), + ancestor_policy_refs: nodes.iter().skip(1).map(PolicyNode::reference).collect(), + current, + initialized: p.is_some_and(|p| p.initialized), + }) +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PlanReview { + pub policy_ref: WritePolicyRef, + pub tool_call_id: String, + pub run_id: String, + pub event_idx: i64, + pub request_digest: String, + pub args_json: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PlanAcceptance { + pub review: PlanReview, + pub actor_id: String, + pub result_json: String, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RequestPolicyContext { + pub session_id: SessionId, + pub run_id: String, + pub tool_call_id: String, + pub accepted_policy_ref: WritePolicyRef, + pub accepted_write_eligibility: WriteEligibility, + pub ancestor_policy_refs: Vec, +} +/// Exact platform request bytes, before any argument normalization. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExecutionRequest { + pub model_tool_name: String, + pub original_arguments: String, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DispatchFence { + Current, + Stale, + Unknown, +} +pub fn conflict(message: &str) -> MetaError { + MetaError::Conflict(message.into()) +} +/// Canonical envelope preserves the exact submitted argument bytes. +fn request_digest(event: &PersistedEvent) -> String { + let p = &event.payload; + let bytes=serde_json::to_vec(&serde_json::json!({"run_id":p["run_id"],"tool_call_id":p["tool_call_id"],"name":p["name"],"args_json":p["args_json"]})).expect("JSON value serializes"); + hex::encode(Sha256::digest(bytes)) +} +pub fn pending_request<'a>( + events: &'a [PersistedEvent], + call: &str, +) -> Result<&'a PersistedEvent, MetaError> { + let relevant: Vec<_> = events + .iter() + .filter(|e| { + e.payload["tool_call_id"].as_str() == Some(call) + && matches!( + e.kind.as_str(), + "tool_call_requested" + | "tool_result_submitted" + | "tool_call_completed" + | "tool_call_cancel_requested" + ) + }) + .collect(); + if relevant.len() >= 1024 { + return Err(conflict("request history limit exceeded")); + } + let first = relevant + .iter() + .find(|e| e.kind == "tool_call_requested") + .ok_or_else(|| conflict("pending request missing"))?; + if first.rewound_at.is_some() + || relevant.iter().any(|e| { + matches!( + e.kind.as_str(), + "tool_result_submitted" | "tool_call_completed" | "tool_call_cancel_requested" + ) || (e.kind == "tool_call_requested" && request_digest(e) != request_digest(first)) + }) + { + return Err(conflict("pending request changed")); + } + Ok(first) +} +pub fn plan_review( + node: &PolicyNode, + events: &[PersistedEvent], + call: &str, +) -> Result { + let policy = node + .policy + .as_ref() + .ok_or_else(|| conflict("policy unknown"))?; + if node.terminal || !policy.initialized || policy.eligibility != WriteEligibility::ReadOnly { + return Err(conflict("known restrictive policy required")); + } + let event = pending_request(events, call)?; + if event.idx < policy.event_floor || event.payload["name"].as_str() != Some("exit_plan_mode") { + return Err(conflict("current plan request required")); + } + let run_id = event.payload["run_id"] + .as_str() + .filter(|s| !s.is_empty()) + .ok_or_else(|| conflict("run required"))?; + let args_json = event.payload["args_json"] + .as_str() + .ok_or_else(|| conflict("plan arguments required"))?; + Ok(PlanReview { + policy_ref: node.reference(), + tool_call_id: call.into(), + run_id: run_id.into(), + event_idx: event.idx, + request_digest: request_digest(event), + args_json: args_json.into(), + }) +} +pub fn accept_plan( + node: &PolicyNode, + events: &[PersistedEvent], + input: &PlanAcceptance, +) -> Result { + if input.actor_id.is_empty() + || input.actor_id.len() > 256 + || plan_review(node, events, &input.review.tool_call_id)? != input.review + { + return Err(conflict("plan review changed")); + } + let result: serde_json::Value = + serde_json::from_str(&input.result_json).map_err(|_| conflict("invalid plan result"))?; + if result["decision"] != "approve" { + return Err(conflict("only human approval can relax policy")); + } + let mut p = node + .policy + .clone() + .ok_or_else(|| conflict("policy unknown"))?; + p.revision = p + .revision + .checked_add(1) + .ok_or_else(|| conflict("policy revision exhausted"))?; + p.eligibility = WriteEligibility::WriteEligible; + p.cause = "owner_admin_plan_exit".into(); + p.accepted_input_id = super::outbox::tool_result_outbox_id(node.id, &input.review.tool_call_id); + p.actor_id = Some(input.actor_id.clone()); + p.plan_tool_call_id = Some(input.review.tool_call_id.clone()); + p.plan_request_digest = Some(input.review.request_digest.clone()); + Ok(p) +} +pub fn execution_identity_fence( + nodes: &[PolicyNode], + events: &[PersistedEvent], + ctx: &RequestPolicyContext, + expected: &ExecutionRequest, +) -> Result { + let p = compose_policy(nodes)?; + if !nodes[0].active || !p.current { + return Ok(DispatchFence::Stale); + } + if ctx.session_id != ctx.accepted_policy_ref.session_id + || p.reference != ctx.accepted_policy_ref + || p.ancestor_policy_refs != ctx.ancestor_policy_refs + { + return Ok(DispatchFence::Stale); + } + let request = match pending_request(events, &ctx.tool_call_id) { + Ok(r) => r, + Err(_) => return Ok(DispatchFence::Stale), + }; + let last_run = events.iter().rev().find(|e| { + e.rewound_at.is_none() + && matches!( + e.kind.as_str(), + "run_started" | "run_completed" | "run_interrupted" + ) + }); + let run_is_current = last_run.is_some_and(|e| { + e.payload["run_id"].as_str() == Some(&ctx.run_id) + && match e.kind.as_str() { + "run_started" => true, + "run_completed" => { + e.payload["ok"].as_bool() == Some(true) + && request.idx < e.idx + && e.payload["deferred_tool_call_ids"] + .as_array() + .is_some_and(|ids| { + let mut seen = std::collections::BTreeSet::new(); + ids.len() <= 128 + && ids.iter().all(|id| { + id.as_str().is_some_and(|id| { + !id.is_empty() && id.len() <= 256 && seen.insert(id) + }) + }) + && seen.contains(ctx.tool_call_id.as_str()) + }) + } + _ => false, + } + }); + if expected.model_tool_name.is_empty() + || expected.model_tool_name.len() > 256 + || expected.original_arguments.len() > 256 * 1024 + || request.payload["name"].as_str() != Some(&expected.model_tool_name) + || request.payload["args_json"].as_str() != Some(&expected.original_arguments) + || ctx.run_id.is_empty() + || request.payload["run_id"].as_str() != Some(&ctx.run_id) + || !run_is_current + { + return Ok(DispatchFence::Stale); + } + Ok(DispatchFence::Current) +} + +/// Write eligibility adds to the exact execution identity checks; it never creates identity. +pub fn dispatch_fence( + nodes: &[PolicyNode], + events: &[PersistedEvent], + ctx: &RequestPolicyContext, + expected: &ExecutionRequest, +) -> Result { + let p = compose_policy(nodes)?; + if ctx.accepted_write_eligibility != WriteEligibility::WriteEligible + || p.write_eligibility != WriteEligibility::WriteEligible + || nodes + .iter() + .any(|n| !n.policy.as_ref().is_some_and(|p| p.initialized)) + { + return Ok(DispatchFence::Unknown); + } + execution_identity_fence(nodes, events, ctx, expected) +} diff --git a/crates/engram-core/src/types/mod.rs b/crates/engram-core/src/types/mod.rs index 2d418d630..f72ce3110 100644 --- a/crates/engram-core/src/types/mod.rs +++ b/crates/engram-core/src/types/mod.rs @@ -16,6 +16,7 @@ pub mod image; pub mod integration; pub mod manifest; pub mod materialize; +pub mod mcp; pub mod oauth; pub mod org_secret; pub mod outbox; @@ -52,3 +53,18 @@ pub use snapshot::*; pub mod teleport; pub use teleport::*; + +pub mod mcp_policy; + +pub mod managed_deny; + +pub mod managed_cold; +pub mod managed_create; + +pub mod managed_boot; + +pub mod managed_operator; + +pub mod managed_assets; + +pub mod managed_bridge; diff --git a/crates/engram-core/src/types/oauth.rs b/crates/engram-core/src/types/oauth.rs index da2363d4d..5d928b1f7 100644 --- a/crates/engram-core/src/types/oauth.rs +++ b/crates/engram-core/src/types/oauth.rs @@ -68,7 +68,8 @@ pub struct SessionOAuthBinding { /// Provider-approved metadata safe to return to clients and logs. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct OAuthAccountMetadata { - /// Provider-stable identity used to reject account switching on refresh. + /// Authority identity used to reject account switching on refresh. MCP uses + /// an authorization-ceremony identity with sealed provider provenance. /// This value is not rendered by the UI. pub account_id: String, pub display_name: Option, diff --git a/crates/engram-core/src/types/org_secret.rs b/crates/engram-core/src/types/org_secret.rs index 5f1101336..423b8b91f 100644 --- a/crates/engram-core/src/types/org_secret.rs +++ b/crates/engram-core/src/types/org_secret.rs @@ -35,3 +35,28 @@ pub struct OrgSecret { pub created_at: DateTime, pub updated_at: DateTime, } + +/// The only two uses for reserved organization credentials. +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum McpSecretRole { + ApiKey, + ClientSecret, +} +impl McpSecretRole { + pub fn as_str(self) -> &'static str { + match self { + Self::ApiKey => "api_key", + Self::ClientSecret => "client_secret", + } + } +} + +/// Permanent ownership, with a fence that changes on every value mutation. +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct McpSecretReservation { + pub name: String, + pub connection_id: String, + pub role: McpSecretRole, + pub generation: i64, +} diff --git a/crates/engram-crypto/src/lib.rs b/crates/engram-crypto/src/lib.rs index d86c6c50e..28ce86273 100644 --- a/crates/engram-crypto/src/lib.rs +++ b/crates/engram-crypto/src/lib.rs @@ -165,6 +165,15 @@ impl std::fmt::Display for CryptoError { impl std::error::Error for CryptoError {} +/// Security tokens use OS entropy, never the reproducible simulation RNG. +pub fn security_random_bytes() -> Result<[u8; 32], CryptoError> { + let mut bytes = [0; 32]; + SysRng + .try_fill_bytes(&mut bytes) + .map_err(|_| CryptoError::Provider("security entropy unavailable".into()))?; + Ok(bytes) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/engram-dst-cosim/src/scheduler.rs b/crates/engram-dst-cosim/src/scheduler.rs index ce3cd7f8c..675f02833 100644 --- a/crates/engram-dst-cosim/src/scheduler.rs +++ b/crates/engram-dst-cosim/src/scheduler.rs @@ -120,6 +120,10 @@ impl Cosim { use engram_core::traits::Entropy as _; let session_id = SessionId::from(self.world.entropy.uuid()); let ws = SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id, spec: SessionSpec { image: COSIM_IMAGE.into(), diff --git a/crates/engram-dst/src/scheduler.rs b/crates/engram-dst/src/scheduler.rs index e8d59e630..e0198afb4 100644 --- a/crates/engram-dst/src/scheduler.rs +++ b/crates/engram-dst/src/scheduler.rs @@ -621,6 +621,10 @@ impl Sim { use engram_core::traits::Entropy as _; let session_id = SessionId::from(self.world.entropy.uuid()); let ws = SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id, spec: SessionSpec { image: SIM_IMAGE.into(), diff --git a/crates/engram-dst/tests/exactly_once.rs b/crates/engram-dst/tests/exactly_once.rs index 5683188e1..288f79414 100644 --- a/crates/engram-dst/tests/exactly_once.rs +++ b/crates/engram-dst/tests/exactly_once.rs @@ -44,6 +44,10 @@ async fn reserve_session(sim: &mut Sim) -> (engram_coordinator::state::SharedSta .expect("replica 0 is up"); let session_id = SessionId::from(sim.world.entropy.uuid()); let ws = SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id, spec: SessionSpec { image: SIM_IMAGE.into(), @@ -164,6 +168,7 @@ fn agent_spec_with_prompt(prompt_id: &str, text: &str) -> AgentSpec { fn egress_policy(session_id: SessionId, sandbox_id: SandboxId) -> SessionEgressPolicy { SessionEgressPolicy { + managed_deny: None, session_id, sandbox_id, guest_ip: std::net::Ipv4Addr::UNSPECIFIED, diff --git a/crates/engram-dst/tests/first_sim.rs b/crates/engram-dst/tests/first_sim.rs index 3c0e6fa77..4c90fac63 100644 --- a/crates/engram-dst/tests/first_sim.rs +++ b/crates/engram-dst/tests/first_sim.rs @@ -129,6 +129,18 @@ fn driver_coverage_is_declared() { "harness_desync", "not a timer driver, so no DriverKind: production runs run_once inside the host-heartbeat handler (api/host_http.rs), and the sim mirrors that exactly — Step::HostHeartbeats drives it with the world-derived running/attached sets right after touch_host_heartbeat (ADR 0108 A8; pinned in tests/ttft_attach.rs)", ), + ( + "managed_boot", + "the DST workload creates no managed boots and SimHostClient does not implement managed prepare/finalize/start; managed_boot::tests::managed_boot_lost_start_reply_is_read_only_reconciled tests the real scanner with a synthetic host, while meta_conformance pairs boot persistence cases on SimMeta and Postgres", + ), + ( + "managed_bridge", + "the DST workload creates no managed boot intents and SimHostClient does not implement managed asset inspection or capture; managed_boot::tests::managed_bridge_real_driver_claims_once_and_reconciles_prepared_reply tests the real bridge with a synthetic host, while meta_conformance pairs intent/claim/recovery persistence cases on SimMeta and Postgres", + ), + ( + "mcp/operator", + "the scanner needs an operator publication candidate and managed host retirement proof, neither modeled by the DST scheduler; managed_boot::tests::operator_scanner_requires_retirement_and_retains_before_host_io tests the real scanner with a synthetic host, while meta_conformance pairs publication/readiness persistence cases on SimMeta and Postgres", + ), ]; fn modules_for(kind: DriverKind) -> &'static [&'static str] { diff --git a/crates/engram-dst/tests/ttft_attach.rs b/crates/engram-dst/tests/ttft_attach.rs index ee21875c5..c4d694439 100644 --- a/crates/engram-dst/tests/ttft_attach.rs +++ b/crates/engram-dst/tests/ttft_attach.rs @@ -60,6 +60,10 @@ async fn boot_session( .expect("replica 0 is up"); let session_id = SessionId::from(sim.world.entropy.uuid()); let ws = SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id, spec: SessionSpec { image: SIM_IMAGE.into(), diff --git a/crates/engram-egress-proto/src/adopt.rs b/crates/engram-egress-proto/src/adopt.rs index 51345ae6d..135d9bf4f 100644 --- a/crates/engram-egress-proto/src/adopt.rs +++ b/crates/engram-egress-proto/src/adopt.rs @@ -161,6 +161,7 @@ mod tests { fn hello() -> HelloInfo { HelloInfo { + incarnation: "test-incarnation".into(), proto_version: crate::PROTO_VERSION, source_fingerprint: Some(FP.into()), proxy_port: 8443, diff --git a/crates/engram-egress-proto/src/lib.rs b/crates/engram-egress-proto/src/lib.rs index 8f316467e..1270ea5a7 100644 --- a/crates/engram-egress-proto/src/lib.rs +++ b/crates/engram-egress-proto/src/lib.rs @@ -36,6 +36,7 @@ pub mod adopt; pub mod manifest; +pub mod private; use engram_core::types::egress::SessionEgressPolicy; use engram_core::{SandboxId, SessionId}; @@ -44,7 +45,7 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; /// Bump on any wire-incompatible change. A mismatch in `Hello` /// position is never adopted — the caller kills and respawns. -pub const PROTO_VERSION: u32 = 1; +pub const PROTO_VERSION: u32 = 5; /// Frame size cap. `SyncPolicies` carries every live session's policy /// (resolved secrets included), so the cap is generous; both ends are @@ -85,6 +86,14 @@ pub const BUILD_ENV_SOURCE_FINGERPRINT: &str = "ENGRAM_EGRESS_PROXYD_FINGERPRINT /// Host-agent → proxyd. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub enum ToProxyd { + /// Upgrade this host-only connection. A bounded private Allocation follows. + PrivateNetwork, + HandoffClosedPrivate(Box), + /// Drain and remove only the expected private allocation and registration. + RetirePrivateNetwork(Box), + ListManaged, + ReadManaged(engram_core::SessionId), + QuarantineManaged(engram_core::SessionId), /// First frame on every connection. Hello { proto_version: u32, @@ -118,6 +127,8 @@ pub enum ToProxyd { /// proxyd → host-agent. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub enum FromProxyd { + ManagedSessions(Vec), + ManagedState(Option), HelloAck(HelloInfo), Ok, Err(String), @@ -141,6 +152,8 @@ pub struct GuestSummary { /// file, is the authority (the file can be stale or torn). #[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] pub struct HelloInfo { + #[serde(default)] + pub incarnation: String, pub proto_version: u32, /// The build-time source fingerprint, `None` for local builds. pub source_fingerprint: Option, @@ -303,6 +316,40 @@ pub fn recv_fd(sock: &std::os::unix::net::UnixStream) -> std::io::Result(&mut bytes.as_slice()) + .await + .unwrap(), + message + ); + for end in 0..bytes.len() { + assert!(read_frame::<_, ToProxyd>(&mut &bytes[..end]).await.is_err()); + } + let mut malformed = serde_json::to_value(&message).unwrap(); + malformed["RetirePrivateNetwork"]["attachment_generation"] = serde_json::json!("bad"); + assert!(serde_json::from_value::(malformed).is_err()); + } + #[tokio::test] async fn frames_round_trip_async() { let msgs = vec![ @@ -320,6 +367,7 @@ mod tests { assert_eq!(back, m); } let m = FromProxyd::HelloAck(HelloInfo { + incarnation: "test-incarnation".into(), proto_version: PROTO_VERSION, source_fingerprint: Some("abc123".into()), proxy_port: 8443, diff --git a/crates/engram-egress-proto/src/private.rs b/crates/engram-egress-proto/src/private.rs new file mode 100644 index 000000000..71e62fb0a --- /dev/null +++ b/crates/engram-egress-proto/src/private.rs @@ -0,0 +1,468 @@ +//! Host-private network transport. The connected stream is the capability. +//! No value in a guest packet selects an allocation or session. +use engram_core::{HostId, SandboxId, SessionId}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use std::{io, net::Ipv4Addr}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +pub const VERSION: u32 = 1; +pub const MAX_HEADER: usize = 2048; +pub const MAX_DNS: usize = 4096; +pub const MAX_FLOWS: usize = 64; +pub const MAX_DNS_FLOWS: usize = 8; +pub const HANDOFF_SECONDS: u64 = 5; + +/// Host-only registration, checked against the live proxy and policy. The +/// placement epoch is deliberately absent: registration is always CLOSED. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Allocation { + pub version: u32, + pub host_id: HostId, + pub host_incarnation: String, + pub proxy_incarnation: String, + pub session_id: SessionId, + pub sandbox_id: SandboxId, + pub boot_attempt: String, + pub attachment_generation: u64, + pub guest_ip: Ipv4Addr, + pub gateway_ip: Ipv4Addr, + pub policy_revision: u64, + pub policy_digest: String, +} +impl Allocation { + pub fn validate(&self) -> io::Result<()> { + let bounded = |s: &str| { + !s.is_empty() + && s.len() <= 128 + && s.bytes().all(|c| c.is_ascii_alphanumeric() || c == b'-') + }; + if self.version != VERSION + || self.host_id.as_uuid().is_nil() + || self.session_id.as_uuid().is_nil() + || self.sandbox_id.as_uuid().is_nil() + || !bounded(&self.host_incarnation) + || !bounded(&self.proxy_incarnation) + || !bounded(&self.boot_attempt) + || self.attachment_generation == 0 + || self.guest_ip == self.gateway_ip + || self.guest_ip.is_unspecified() + || self.guest_ip.is_multicast() + || self.gateway_ip.is_unspecified() + || self.gateway_ip.is_multicast() + || !engram_core::types::managed_deny::is_digest(&self.policy_digest) + { + return Err(invalid()); + } + Ok(()) + } +} + +/// Strict monotonic sequence on one allocation stream. Bind can occur once. +/// A TCP or DNS header is immediately followed by one SCM_RIGHTS carrier. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Request { + pub version: u32, + pub sequence: u64, + pub epoch: u64, + pub operation: Operation, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum Operation { + Bind, + Tcp { destination: Ipv4Addr, port: u16 }, + Dns { destination: Ipv4Addr, port: u16 }, + Close, +} +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Reply { + pub version: u32, + pub sequence: u64, + pub epoch: u64, + pub state: String, +} + +pub fn invalid() -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + "invalid private network transport", + ) +} +pub async fn read(r: &mut R) -> io::Result { + let mut prefix = [0; 4]; + r.read_exact(&mut prefix[..1]).await?; + tokio::time::timeout(std::time::Duration::from_secs(HANDOFF_SECONDS), async { + r.read_exact(&mut prefix[1..]).await?; + let len = u32::from_be_bytes(prefix) as usize; + if len == 0 || len > MAX_HEADER { + return Err(invalid()); + } + let mut body = vec![0; len]; + r.read_exact(&mut body).await?; + serde_json::from_slice(&body).map_err(|_| invalid()) + }) + .await + .map_err(|_| invalid())? +} +pub async fn write(w: &mut W, v: &T) -> io::Result<()> { + let body = serde_json::to_vec(v).map_err(|_| invalid())?; + if body.is_empty() || body.len() > MAX_HEADER { + return Err(invalid()); + } + tokio::time::timeout(std::time::Duration::from_secs(HANDOFF_SECONDS), async { + w.write_all(&(body.len() as u32).to_be_bytes()).await?; + w.write_all(&body).await?; + w.flush().await + }) + .await + .map_err(|_| invalid())? +} + +/// A host allocation and its registered reliable stream are inseparable until +/// the backend transfers ownership to its helper. No unchecked constructor. +pub struct RegisteredPrivateNetwork { + allocation: Allocation, + stream: tokio::net::UnixStream, +} +impl RegisteredPrivateNetwork { + pub fn allocation(&self) -> &Allocation { + &self.allocation + } + pub fn into_stream(self) -> tokio::net::UnixStream { + self.stream + } +} + +/// Register a fresh capability on a dedicated host control connection. Never +/// retry: loss of the reply makes this attempt unusable. The returned socket is +/// for one helper only and starts closed. It must not be cached across boots. +pub async fn register( + mut stream: tokio::net::UnixStream, + allocation: &Allocation, +) -> io::Result { + allocation.validate()?; + crate::write_frame(&mut stream, &crate::ToProxyd::PrivateNetwork).await?; + write(&mut stream, allocation).await?; + let reply: Reply = tokio::time::timeout( + std::time::Duration::from_secs(HANDOFF_SECONDS), + read(&mut stream), + ) + .await + .map_err(|_| invalid())??; + if reply + != (Reply { + version: VERSION, + sequence: 0, + epoch: 0, + state: "closed".into(), + }) + { + return Err(invalid()); + } + Ok(RegisteredPrivateNetwork { + allocation: allocation.clone(), + stream, + }) +} + +/// Receive exactly one connected Unix stream, with close-on-exec set before it +/// leaves this function. Close every installed descriptor on malformed input. +#[cfg(unix)] +pub async fn receive_stream(socket: &tokio::net::UnixStream) -> io::Result { + use nix::sys::socket::{self, ControlMessageOwned, MsgFlags}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + let fd = socket + .async_io(tokio::io::Interest::READABLE, || { + let mut carrier = [0u8; 1]; + let mut iov = [io::IoSliceMut::new(&mut carrier)]; + // Receive the OS maximum, then reject every count except one. + // XNU UIPC_MAX_CMSG_FD is 512; Linux SCM_MAX_FD is 253. + // A smaller buffer can leak unreported descriptors on Darwin: + // unp_externalize installs all rights before copyout truncation. + // https://github.com/apple-oss-distributions/xnu/blob/main/bsd/kern/uipc_usrreq.c + let mut control = nix::cmsg_space!([std::os::fd::RawFd; 512]); + #[cfg(target_os = "linux")] + let flags = MsgFlags::MSG_CMSG_CLOEXEC; + #[cfg(not(target_os = "linux"))] + let flags = MsgFlags::empty(); + let msg = + socket::recvmsg::<()>(socket.as_raw_fd(), &mut iov, Some(&mut control), flags) + .map_err(io::Error::from)?; + let mut fds = Vec::new(); + for message in msg.cmsgs().map_err(io::Error::from)? { + if let ControlMessageOwned::ScmRights(raw) = message { + for fd in raw { + // SAFETY: the kernel installed this descriptor. It is + // wrapped once and dropped even on a rejected message. + fds.push(unsafe { OwnedFd::from_raw_fd(fd) }); + } + } + } + let bad = msg.bytes != 1 + || msg + .flags + .intersects(MsgFlags::MSG_CTRUNC | MsgFlags::MSG_TRUNC); + if bad || carrier[0] != 0xED || fds.len() != 1 { + return Err(invalid()); + } + let fd = fds.pop().ok_or_else(invalid)?; + // SAFETY: fd is live and owned. F_SETFD changes only its descriptor flag. + if unsafe { + nix::libc::fcntl(fd.as_raw_fd(), nix::libc::F_SETFD, nix::libc::FD_CLOEXEC) + } == -1 + { + return Err(io::Error::last_os_error()); + } + if socket::getsockopt(&fd, socket::sockopt::SockType).map_err(io::Error::from)? + != socket::SockType::Stream + { + return Err(invalid()); + } + socket::getpeername::(fd.as_raw_fd()).map_err(io::Error::from)?; + Ok(fd) + }) + .await?; + let stream = std::os::unix::net::UnixStream::from(fd); + stream.set_nonblocking(true)?; + tokio::net::UnixStream::from_std(stream) +} + +/// Drain canceled headers and descriptor carriers until the helper confirms +/// cleanup by closing its reliable private stream. The caller bounds the wait. +#[cfg(unix)] +pub async fn drain_to_eof(socket: &tokio::net::UnixStream) -> io::Result<()> { + use nix::sys::socket::{self, ControlMessageOwned, MsgFlags}; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + loop { + let bytes = socket + .async_io(tokio::io::Interest::READABLE, || { + let mut bytes = [0; MAX_HEADER]; + let mut iov = [io::IoSliceMut::new(&mut bytes)]; + // Keep the same OS-maximum ancillary capacity as receive_stream. + let mut control = nix::cmsg_space!([std::os::fd::RawFd; 512]); + #[cfg(target_os = "linux")] + let flags = MsgFlags::MSG_CMSG_CLOEXEC; + #[cfg(not(target_os = "linux"))] + let flags = MsgFlags::empty(); + let message = + socket::recvmsg::<()>(socket.as_raw_fd(), &mut iov, Some(&mut control), flags) + .map_err(io::Error::from)?; + for message in message.cmsgs().map_err(io::Error::from)? { + if let ControlMessageOwned::ScmRights(fds) = message { + for fd in fds { + // SAFETY: each received descriptor is newly installed + // by the kernel and is closed here exactly once. + drop(unsafe { OwnedFd::from_raw_fd(fd) }); + } + } + } + if message + .flags + .intersects(MsgFlags::MSG_CTRUNC | MsgFlags::MSG_TRUNC) + { + return Err(invalid()); + } + Ok(message.bytes) + }) + .await?; + if bytes == 0 { + return Ok(()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::AsyncWriteExt; + #[tokio::test] + async fn registration_keeps_exact_allocation_and_rejects_wrong_closed_reply() { + let allocation = Allocation { + version: 1, + host_id: HostId::new(), + host_incarnation: "host-1".into(), + proxy_incarnation: "proxy-1".into(), + session_id: SessionId::new(), + sandbox_id: SandboxId::new(), + boot_attempt: "boot-1".into(), + attachment_generation: 1, + guest_ip: "192.0.2.2".parse().unwrap(), + gateway_ip: "192.0.2.1".parse().unwrap(), + policy_revision: 1, + policy_digest: format!("sha256:{}", "a".repeat(64)), + }; + for epoch in [0, 1] { + let (a, mut b) = tokio::net::UnixStream::pair().unwrap(); + let expected = allocation.clone(); + let peer = tokio::spawn(async move { + assert!(matches!( + crate::read_frame::<_, crate::ToProxyd>(&mut b) + .await + .unwrap(), + crate::ToProxyd::PrivateNetwork + )); + let sent: Allocation = read(&mut b).await.unwrap(); + assert_eq!(sent, expected); + write( + &mut b, + &Reply { + version: 1, + sequence: 0, + epoch, + state: "closed".into(), + }, + ) + .await + .unwrap(); + b + }); + let result = register(a, &allocation).await; + let mut peer = peer.await.unwrap(); + if epoch == 0 { + let registered = result.unwrap(); + assert_eq!(registered.allocation(), &allocation); + // The stream remains the exact capability, owned until consumed. + let mut capability = registered.into_stream(); + capability.write_all(b"x").await.unwrap(); + let mut byte = [0]; + peer.read_exact(&mut byte).await.unwrap(); + assert_eq!(byte, *b"x"); + } else { + assert!(result.is_err()); + assert_eq!(peer.read(&mut [0]).await.unwrap(), 0); + } + } + } + + #[tokio::test] + async fn oversized_private_header_is_rejected_before_body_read() { + let (mut a, mut b) = tokio::net::UnixStream::pair().unwrap(); + a.write_all(&2049u32.to_be_bytes()).await.unwrap(); + let result = tokio::time::timeout( + std::time::Duration::from_millis(100), + read::<_, Request>(&mut b), + ) + .await; + assert!(matches!(result, Ok(Err(e)) if e.kind() == io::ErrorKind::InvalidData)); + } + #[tokio::test] + async fn descriptor_handoff_rejects_wrong_types_and_closes_all_extra_fds() { + use nix::sys::socket::{sendmsg, ControlMessage, MsgFlags}; + use std::os::fd::{AsFd, AsRawFd}; + let (sender, receiver) = tokio::net::UnixStream::pair().unwrap(); + let file = tempfile::tempfile().unwrap(); + let raw = sender.as_fd().try_clone_to_owned().unwrap(); + crate::send_fd(&std::os::unix::net::UnixStream::from(raw), file.as_fd()).unwrap(); + assert!(receive_stream(&receiver).await.is_err()); + let (mut a, b) = tokio::net::UnixStream::pair().unwrap(); + let (mut c, d) = tokio::net::UnixStream::pair().unwrap(); + let fds = [b.as_raw_fd(), d.as_raw_fd()]; + let control = [ControlMessage::ScmRights(&fds)]; + sendmsg::<()>( + sender.as_raw_fd(), + &[io::IoSlice::new(&[0xED])], + &control, + MsgFlags::empty(), + None, + ) + .unwrap(); + drop(b); + drop(d); + assert!(receive_stream(&receiver).await.is_err()); + let mut byte = [0]; + assert_eq!(a.read(&mut byte).await.unwrap(), 0); + assert_eq!(c.read(&mut byte).await.unwrap(), 0); + } + #[tokio::test] + async fn oversized_rights_message_closes_every_installed_descriptor() { + use nix::sys::socket::{sendmsg, ControlMessage, MsgFlags}; + use std::os::fd::AsRawFd; + let (sender, receiver) = tokio::net::UnixStream::pair().unwrap(); + let mut peers = Vec::new(); + let mut passed = Vec::new(); + for _ in 0..32 { + let (a, b) = tokio::net::UnixStream::pair().unwrap(); + peers.push(a); + passed.push(b); + } + let fds: Vec<_> = passed.iter().map(AsRawFd::as_raw_fd).collect(); + sendmsg::<()>( + sender.as_raw_fd(), + &[io::IoSlice::new(&[0xED])], + &[ControlMessage::ScmRights(&fds)], + MsgFlags::empty(), + None, + ) + .unwrap(); + drop(passed); + assert!(receive_stream(&receiver).await.is_err()); + for (index, mut peer) in peers.into_iter().enumerate() { + let mut byte = [0]; + assert!( + matches!( + tokio::time::timeout( + std::time::Duration::from_millis(50), + peer.read(&mut byte) + ) + .await, + Ok(Ok(0)) + ), + "oversized SCM_RIGHTS must not leak an installed FD at {index}" + ); + } + } + #[tokio::test] + async fn cleanup_drain_closes_pending_rights_and_requires_reliable_eof() { + use std::os::fd::AsFd; + let (mut sender, receiver) = tokio::net::UnixStream::pair().unwrap(); + let (mut flow, passed) = tokio::net::UnixStream::pair().unwrap(); + sender.write_all(b"partial header").await.unwrap(); + let raw = sender.as_fd().try_clone_to_owned().unwrap(); + crate::send_fd(&std::os::unix::net::UnixStream::from(raw), passed.as_fd()).unwrap(); + drop(passed); + let drain = tokio::spawn(async move { drain_to_eof(&receiver).await }); + let mut byte = [0]; + assert_eq!( + tokio::time::timeout(std::time::Duration::from_secs(1), flow.read(&mut byte)) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!( + !drain.is_finished(), + "discarded bytes are not cleanup acknowledgement" + ); + drop(sender); + drain.await.unwrap().unwrap(); + } + #[test] + fn valid_bind_decodes() { + serde_json::from_str::( + r#"{"version":1,"sequence":1,"epoch":7,"operation":{"kind":"bind"}}"#, + ) + .unwrap(); + } + #[tokio::test] + async fn required_port_and_unknown_guest_identity_are_rejected() { + for body in [ + r#"{"version":1,"sequence":1,"epoch":1,"operation":{"kind":"tcp","destination":"192.0.2.9"}}"#, + r#"{"version":1,"sequence":1,"epoch":1,"operation":{"kind":"tcp","destination":"192.0.2.9","port":443,"session_id":"forged"}}"#, + ] { + let mut bytes = (body.len() as u32).to_be_bytes().to_vec(); + bytes.extend_from_slice(body.as_bytes()); + assert!(read::<_, Request>(&mut bytes.as_slice()).await.is_err()); + } + } +} + +/// Authenticated host control only. It cannot replace a bound allocation. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClosedPolicyHandoff { + pub allocation: Allocation, + pub policy: engram_core::types::egress::SessionEgressPolicy, +} diff --git a/crates/engram-egress-proto/tests/predecessor_hello.rs b/crates/engram-egress-proto/tests/predecessor_hello.rs new file mode 100644 index 000000000..bf08e76c9 --- /dev/null +++ b/crates/engram-egress-proto/tests/predecessor_hello.rs @@ -0,0 +1,115 @@ +#![cfg(unix)] + +use engram_egress_proto::{ + adopt::{decide_adopt, AdoptPlan, ExpectedProxyd, LiveEvidence}, + manifest::{ProcIdentity, ProxydManifest}, + read_frame, write_frame, FromProxyd, HelloInfo, +}; +use serde::{Deserialize, Serialize}; +use std::{path::PathBuf, time::Duration}; +use tokio::{io::AsyncWriteExt, net::UnixStream, time::timeout}; + +// Protocol 1 HelloAck from ec526661; the frame body is 180 bytes. +const PREDECESSOR_HEADER: &[u8] = b"\0\0\0\xb4"; +const PREDECESSOR_BODY: &[u8] = br#"{"HelloAck":{"proto_version":1,"source_fingerprint":"same-build","proxy_port":8443,"dns_port":5353,"gateway_port":13338,"ca_fingerprint":"test-ca","coord_url":"http://coord:8080"}}"#; + +#[tokio::test] +async fn predecessor_hello_selects_replacement_before_policy_replay() { + timeout(Duration::from_secs(5), async { + let (mut daemon, mut owner) = UnixStream::pair().unwrap(); + daemon.write_all(PREDECESSOR_HEADER).await.unwrap(); + daemon.write_all(PREDECESSOR_BODY).await.unwrap(); + let FromProxyd::HelloAck(hello) = read_frame(&mut owner).await.unwrap() else { + panic!("expected predecessor HelloAck"); + }; + assert!(hello.incarnation.is_empty()); + let manifest = ProxydManifest { + schema_version: 1, + pid: 100, + start_time_jiffies: 5, + comm: "engram-egress-p".into(), + source_fingerprint: Some("same-build".into()), + proxy_port: 8443, + dns_port: 5353, + gateway_port: 13338, + control_sock: PathBuf::from("/test/egress-proxyd.sock"), + }; + let evidence = LiveEvidence { + identity: Some(ProcIdentity { + pid: 100, + start_time_jiffies: 5, + comm: "engram-egress-p".into(), + }), + hello: Some(hello), + probe_ok: true, + }; + let expected = ExpectedProxyd { + source_fingerprint: Some("same-build"), + proxy_port: 8443, + dns_port: 5353, + gateway_port: 13338, + ca_fingerprint: "test-ca", + coord_url: "http://coord:8080", + }; + assert_eq!( + decide_adopt(Some(&manifest), &evidence, &expected), + AdoptPlan::KillStaleAndSpawn + ); + }) + .await + .expect("predecessor Hello exchange timed out"); +} + +// The HelloAck variant and HelloInfo fields from ec526661. +#[derive(Deserialize)] +enum PredecessorReply { + HelloAck(PredecessorHelloInfo), +} + +#[derive(Deserialize, Serialize)] +struct PredecessorHelloInfo { + proto_version: u32, + source_fingerprint: Option, + proxy_port: u16, + dns_port: u16, + gateway_port: u16, + ca_fingerprint: String, + coord_url: String, +} + +#[tokio::test] +async fn predecessor_owner_can_read_current_hello_protocol_mismatch() { + timeout(Duration::from_secs(5), async { + let (mut daemon, mut owner) = UnixStream::pair().unwrap(); + write_frame( + &mut daemon, + &FromProxyd::HelloAck(HelloInfo { + incarnation: "current-daemon".into(), + proto_version: 5, + source_fingerprint: Some("current-build".into()), + proxy_port: 8443, + dns_port: 5353, + gateway_port: 13338, + ca_fingerprint: "test-ca".into(), + coord_url: "http://coord:8080".into(), + }), + ) + .await + .unwrap(); + let PredecessorReply::HelloAck(hello) = read_frame(&mut owner).await.unwrap(); + assert_eq!( + serde_json::to_value(hello).unwrap(), + serde_json::json!({ + "proto_version": 5, + "source_fingerprint": "current-build", + "proxy_port": 8443, + "dns_port": 5353, + "gateway_port": 13338, + "ca_fingerprint": "test-ca", + "coord_url": "http://coord:8080", + }) + ); + }) + .await + .expect("current Hello exchange timed out"); +} diff --git a/crates/engram-egress-proxy/src/connections.rs b/crates/engram-egress-proxy/src/connections.rs new file mode 100644 index 000000000..23322afc8 --- /dev/null +++ b/crates/engram-egress-proxy/src/connections.rs @@ -0,0 +1,144 @@ +//! A stable session gate. All socket futures must remain owned by the lease. +use parking_lot::Mutex; +use std::{future::Future, sync::Arc}; +use tokio::sync::{watch, Notify}; + +pub(crate) struct Connections { + state: Mutex, + pub(crate) apply: tokio::sync::Mutex<()>, + drained: Notify, +} +struct State { + open: bool, + poisoned: bool, + active: usize, + cancel: watch::Sender, +} +impl Default for Connections { + fn default() -> Self { + Self { + state: Mutex::new(State { + open: true, + poisoned: false, + active: 0, + cancel: watch::channel(0).0, + }), + apply: tokio::sync::Mutex::new(()), + drained: Notify::new(), + } + } +} +impl Connections { + pub(crate) fn enroll(self: &Arc) -> Option { + let mut state = self.state.lock(); + if !state.open { + return None; + } + state.active += 1; + Some(ConnectionLease { + control: self.clone(), + cancel: state.cancel.subscribe(), + }) + } + pub(crate) fn is_open(&self) -> bool { + self.state.lock().open + } + pub(crate) fn close(&self) { + self.close_generation(); + } + pub(crate) fn close_generation(&self) -> u64 { + let mut state = self.state.lock(); + state.open = false; + state.cancel.send_modify(|v| *v = v.wrapping_add(1)); + let generation = *state.cancel.borrow(); + generation + } + pub(crate) async fn drain(&self) { + loop { + let notified = self.drained.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.state.lock().active == 0 { + return; + } + notified.await; + } + } + pub(crate) fn handoff_publish( + self: &Arc, + generation: u64, + publish: impl FnOnce(), + ) -> Option { + let mut state = self.state.lock(); + if state.poisoned || state.open || state.active != 0 || *state.cancel.borrow() != generation + { + return None; + } + publish(); + state.open = true; + state.active = 1; + Some(ConnectionLease { + control: self.clone(), + cancel: state.cancel.subscribe(), + }) + } + pub(crate) fn poison(&self) { + let mut state = self.state.lock(); + state.poisoned = true; + state.open = false; + state.cancel.send_modify(|v| *v = v.wrapping_add(1)); + } + pub(crate) fn is_poisoned(&self) -> bool { + self.state.lock().poisoned + } + pub(crate) fn open(&self) { + let mut state = self.state.lock(); + if !state.poisoned { + state.open = true; + } + } +} + +pub struct ConnectionLease { + control: Arc, + cancel: watch::Receiver, +} +impl ConnectionLease { + pub(crate) fn current_for(&self, control: &Arc) -> bool { + Arc::ptr_eq(&self.control, control) + && !self.cancel.has_changed().unwrap_or(true) + && control.is_open() + } + + /// A failed private cleanup barrier cannot be repaired by policy replay + /// in this proxy incarnation. Retain the closed session gate. + pub fn poison(&self) { + let mut state = self.control.state.lock(); + state.poisoned = true; + state.open = false; + state.cancel.send_modify(|v| *v = v.wrapping_add(1)); + } + + /// Allocation owners observe cancellation, close admission, then join all + /// child work before dropping this lease and releasing a policy drain. + pub async fn cancelled(&mut self) { + let _ = self.cancel.changed().await; + } + + /// Drop the complete socket future before releasing the drain guard. + pub async fn run(mut self, future: F) -> Option { + tokio::select! { biased; + _ = self.cancel.changed() => None, + result = future => Some(result), + } + } +} +impl Drop for ConnectionLease { + fn drop(&mut self) { + let mut state = self.control.state.lock(); + state.active -= 1; + if state.active == 0 { + self.control.drained.notify_waiters(); + } + } +} diff --git a/crates/engram-egress-proxy/src/dns.rs b/crates/engram-egress-proxy/src/dns.rs index 99738feeb..4c1ed1242 100644 --- a/crates/engram-egress-proxy/src/dns.rs +++ b/crates/engram-egress-proxy/src/dns.rs @@ -349,6 +349,9 @@ fn decide(registry: &Registry, guest_ip: Ipv4Addr, query: &Message) -> Decision /// /// A credential-exchange host is refused outright, matching `decide()`. fn name_allowed(state: &SessionState, qname: &str) -> bool { + if state.managed_deny.as_ref().is_some_and(|p| p.denies(qname)) { + return false; + } // ADR 0118: the session's OWN apps resolve unconditionally — before the // deny-list and before the allow-list. A sibling app is not an egress // destination at all: the TCP connection never leaves the host, so @@ -388,6 +391,43 @@ fn build_nxdomain(query: &Message) -> Message { resp } +/// Bounded private DNS exchange. Uses the ordinary name policy and codec; +/// no detached demux/reaper can deliver after the allocation lease ends. +pub async fn resolve_private( + bytes: &[u8], + state: &SessionState, + upstream: SocketAddr, +) -> std::io::Result> { + if bytes.len() > MAX_DGRAM { + return Err(std::io::Error::other("DNS query too large")); + } + let query = Message::from_bytes(bytes).map_err(std::io::Error::other)?; + if query.message_type() != MessageType::Query || query.queries().len() != 1 { + return Err(std::io::Error::other("invalid DNS query")); + } + let name = query.queries()[0] + .name() + .to_utf8() + .trim_end_matches('.') + .to_ascii_lowercase(); + if !name_allowed(state, &name) { + return build_nxdomain(&query) + .to_vec() + .map_err(std::io::Error::other); + } + let response = tokio::time::timeout(TCP_UPSTREAM_TIMEOUT, forward_tcp(bytes, upstream)) + .await + .map_err(|_| std::io::Error::other("DNS exchange timeout"))??; + let parsed = Message::from_bytes(&response).map_err(std::io::Error::other)?; + if parsed.id() != query.id() + || parsed.message_type() != MessageType::Response + || parsed.queries() != query.queries() + { + return Err(std::io::Error::other("invalid DNS reply")); + } + Ok(response) +} + #[cfg(test)] mod tests { use super::*; @@ -417,6 +457,7 @@ mod tests { #[test] fn allows_exact_match_in_network_allow() { let state = SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: Ipv4Addr::new(10, 200, 0, 2), @@ -452,6 +493,7 @@ mod tests { cred: crate::registry::RefreshableCred::new("token".into(), None), }; let state = SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: Ipv4Addr::new(10, 200, 0, 2), @@ -475,6 +517,7 @@ mod tests { #[test] fn refuses_to_resolve_a_credential_exchange_host() { let state = SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: Ipv4Addr::new(10, 200, 0, 2), @@ -503,6 +546,7 @@ mod tests { #[test] fn allows_wildcard_pattern() { let state = SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: Ipv4Addr::new(10, 200, 0, 2), @@ -536,6 +580,7 @@ mod tests { #[test] fn denies_name_not_in_allow_list() { let state = SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: Ipv4Addr::new(10, 200, 0, 2), @@ -576,6 +621,7 @@ mod tests { // network_allow) should still resolve, mirroring the TCP // path's `Intercept` decision. let state = SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: Ipv4Addr::new(10, 200, 0, 2), @@ -634,6 +680,7 @@ mod tests { panic!("client must be v4"); }; registry.register(SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: client_ip, @@ -675,6 +722,7 @@ mod tests { panic!("client must be v4"); }; registry.register(SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: client_ip, diff --git a/crates/engram-egress-proxy/src/guest_gateway.rs b/crates/engram-egress-proxy/src/guest_gateway.rs index 5b23d59e0..f8eed76a9 100644 --- a/crates/engram-egress-proxy/src/guest_gateway.rs +++ b/crates/engram-egress-proxy/src/guest_gateway.rs @@ -149,10 +149,10 @@ async fn serve_connection( gateway: &GuestGatewayRegistry, ) -> std::io::Result<()> { let session = match peer.ip() { - std::net::IpAddr::V4(ip) => registry.lookup(ip), + std::net::IpAddr::V4(ip) => registry.enroll(ip), std::net::IpAddr::V6(_) => None, }; - let Some(session) = session else { + let Some((session, lease)) = session else { // No registered session owns this guest address. write_response( &mut stream, @@ -165,6 +165,17 @@ async fn serve_connection( return stream.shutdown().await; }; + lease + .run(serve_enrolled(stream, peer, session, gateway)) + .await + .unwrap_or(Ok(())) +} +async fn serve_enrolled( + mut stream: TcpStream, + peer: SocketAddr, + session: Arc, + gateway: &GuestGatewayRegistry, +) -> std::io::Result<()> { let (request, initial_data) = read_request_header(&mut stream, REQUEST_HEADER_TIMEOUT).await?; let request = String::from_utf8_lossy(&request); let mut lines = request.lines(); @@ -665,6 +676,7 @@ mod tests { let registry = Registry::new(); let guest_ip = "10.200.0.2".parse().unwrap(); let state = |guest_services| SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip, @@ -692,6 +704,7 @@ mod tests { async fn socket_endpoint_returns_only_the_placeholder_to_an_enabled_session() { let registry = Arc::new(Registry::new()); registry.register(SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: std::net::Ipv4Addr::LOCALHOST, @@ -736,6 +749,7 @@ mod tests { let registry = Arc::new(Registry::new()); registry.register(SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: std::net::Ipv4Addr::LOCALHOST, @@ -799,6 +813,7 @@ mod tests { async fn connect_preserves_bytes_coalesced_after_the_request_header() { let registry = Arc::new(Registry::new()); registry.register(SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: std::net::Ipv4Addr::LOCALHOST, @@ -854,6 +869,7 @@ mod tests { async fn tunnel_connect_failure_returns_an_actionable_bad_gateway() { let registry = Arc::new(Registry::new()); registry.register(SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: std::net::Ipv4Addr::LOCALHOST, @@ -941,6 +957,7 @@ mod tests { use engram_core::types::integration::CredentialMintSource; let session = SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: std::net::Ipv4Addr::LOCALHOST, @@ -984,6 +1001,7 @@ mod tests { let registry = Arc::new(Registry::new()); registry.register(SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: std::net::Ipv4Addr::LOCALHOST, @@ -1051,3 +1069,96 @@ mod tests { .exists()); } } + +#[cfg(test)] +mod managed_tests { + use super::*; + struct LoopbackTunnel(SocketAddr); + #[async_trait] + impl TunnelUpstream for LoopbackTunnel { + fn kind(&self) -> &'static str { + "test.loopback" + } + async fn connect( + &self, + _: SessionId, + _: &SessionTunnel, + ) -> std::io::Result> { + Ok(Box::new(TcpStream::connect(self.0).await?)) + } + } + #[tokio::test] + async fn managed_transition_drains_gateway_and_refuses_opaque_tunnels() { + tokio::time::timeout(Duration::from_secs(3), async { + let registry = Arc::new(Registry::new()); + let old = SessionState { + managed_deny: None, + session_id: SessionId::new(), + sandbox_id: engram_core::SandboxId::new(), + guest_ip: std::net::Ipv4Addr::LOCALHOST, + network_allow: crate::HostList::empty(), + allow_all: false, + secrets: vec![], + injects: vec![], + observes: vec![], + guest_services: vec![], + apps: vec![], + tunnels: vec![SessionTunnel { + id: "echo".into(), + connector: "test.loopback".into(), + config_json: "{}".into(), + mint_source: None, + }], + }; + registry.register(old.clone()); + let retained = registry.lookup(old.guest_ip).unwrap(); + let upstream = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let gateway = GuestGatewayRegistry::new( + std::iter::empty::>(), + [Arc::new(LoopbackTunnel(upstream.local_addr().unwrap())) + as Arc], + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut guest = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (stream, peer) = listener.accept().await.unwrap(); + let run_registry = registry.clone(); + let task = tokio::spawn(async move { + serve_connection(stream, peer, &run_registry, &gateway).await + }); + guest + .write_all( + b"CONNECT /_engrams/v1/tunnels/echo HTTP/1.1\r\nEngram-Gateway: 1\r\n\r\n", + ) + .await + .unwrap(); + let mut response = [0; 256]; + let n = guest.read(&mut response).await.unwrap(); + assert!(response[..n].starts_with(b"HTTP/1.1 200")); + let (mut remote, _) = upstream.accept().await.unwrap(); + let mut next = old.clone(); + next.managed_deny = Some( + engram_core::types::managed_deny::ManagedDenyPolicy::new(1, ["mcp.example".into()]) + .unwrap(), + ); + assert_eq!( + registry.apply(next.clone()).await.unwrap_err(), + "managed policy does not support tunnels" + ); + next.tunnels.clear(); + registry.apply(next).await.unwrap(); + task.await.unwrap().unwrap(); + assert_eq!(guest.read(&mut [0]).await.unwrap(), 0); + assert_eq!(remote.read(&mut [0]).await.unwrap(), 0); + assert_eq!(retained.tunnels.len(), 1); + registry.unregister(old.session_id); + assert!( + registry.apply(old).await.is_err(), + "remove cannot erase the managed floor" + ); + }) + .await + .expect("bounded gateway drain test"); + } +} diff --git a/crates/engram-egress-proxy/src/intercept.rs b/crates/engram-egress-proxy/src/intercept.rs index 35d72a9ae..c99ce5e2a 100644 --- a/crates/engram-egress-proxy/src/intercept.rs +++ b/crates/engram-egress-proxy/src/intercept.rs @@ -259,6 +259,7 @@ impl rustls::server::ResolvesServerCert for SniResolver { /// protocol and miss the other. They used to carry two hand-copied versions of /// the gate, which is how HTTP/2 kept its own drift. pub struct StreamContext<'a> { + pub strict_authority: bool, pub sni: &'a str, pub port: u16, /// Secrets whose `allow` list covers this host, so their placeholders are @@ -296,6 +297,7 @@ where C: AsyncRead + AsyncWrite + Unpin + Send + 'static, { let StreamContext { + strict_authority, sni, port, secrets, @@ -351,6 +353,7 @@ where client_tls, upstream_tls, StreamContext { + strict_authority, sni, port, secrets, @@ -365,6 +368,7 @@ where .await; } let context = StreamContext { + strict_authority, sni, port, secrets, @@ -385,6 +389,16 @@ where let mut prefix = Vec::with_capacity(8192); let mut head_end = read_request_head(&mut client_tls, &mut prefix).await?; validate_header_block(&prefix[..head_end])?; + if strict_authority { + validate_managed_http1_authority(&prefix[..head_end], sni, port)?; + // Chunked/upgrade protocols cannot use the one-request adapter safely. + if matches!( + request_body_framing(&prefix[..head_end]), + RequestBodyFraming::Chunked + ) { + return Err(InterceptError::MalformedRequest); + } + } // Policy and upstream TLS identity are selected by SNI. Bind the cleartext // HTTP authority to that same name before any credential is attached. @@ -447,8 +461,17 @@ where placeholder: placeholder.to_string(), }); } - prefix = substitute_request(&mut client_tls, prefix, sni, secrets).await?; + prefix = if strict_authority { + substitute_managed_request(&mut client_tls, prefix, sni, secrets).await? + } else { + substitute_request(&mut client_tls, prefix, sni, secrets).await? + }; + let request_remaining = if strict_authority { + Some(bound_managed_request(&mut prefix)?) + } else { + None + }; if !firing.is_empty() { let sink = sink.expect("firing observes imply a sink"); // Force identity encoding + a single, close-delimited response so the @@ -463,6 +486,7 @@ where OBSERVE_RESPONSE_BUDGET, &response_redactions, head_request, + request_remaining, ) .await?; let parsed = observe::parse_response(&resp_buf); @@ -496,7 +520,7 @@ where // bytes-after-prefix (no further substitution — a request body mid-stream // is fine, a second request dies with the connection), upstream→client // is everything. - let (mut client_read, mut client_write) = tokio::io::split(client_tls); + let (client_read, mut client_write) = tokio::io::split(client_tls); let (mut upstream_read, mut upstream_write) = tokio::io::split(upstream_tls); // The gated request is already written and checked above. What remains is a // body that continues past the buffered head, and it is best-effort: we @@ -511,7 +535,11 @@ where // until it has the whole body would otherwise deadlock against a guest that // waits for the response before finishing its own write. let request = async { - let _ = tokio::io::copy(&mut client_read, &mut upstream_write).await; + let _ = tokio::io::copy( + &mut client_read.take(request_remaining.unwrap_or(u64::MAX)), + &mut upstream_write, + ) + .await; let _ = upstream_write.shutdown().await; }; // The response half keeps failing closed: it carries the redaction and the @@ -563,12 +591,13 @@ where .initial_connection_window_size(H2_CONNECTION_WINDOW) .handshake(upstream_tls) .await?; - let upstream = tokio::spawn(connection); + tokio::pin!(connection); let mut active = FuturesUnordered::new(); let mut accepting = true; while accepting || !active.is_empty() { tokio::select! { + result = &mut connection => { result?; break; } incoming = inbound.accept(), if accepting => { match incoming { Some(Ok((request, respond))) => { @@ -596,8 +625,6 @@ where } } - upstream.abort(); - let _ = upstream.await; Ok(()) } @@ -607,6 +634,24 @@ async fn process_h2_stream( mut outbound: h2::client::SendRequest, context: &StreamContext<'_>, ) -> Result<(), InterceptError> { + if context.strict_authority + && (request.method() == http::Method::CONNECT + || !request + .uri() + .authority() + .is_some_and(|a| managed_authority_matches(a.as_str(), context.sni, context.port)) + || request + .headers() + .get_all(http::header::HOST) + .iter() + .any(|h| { + !h.to_str() + .is_ok_and(|h| managed_authority_matches(h, context.sni, context.port)) + })) + { + deny_h2(&mut respond, http::StatusCode::FORBIDDEN)?; + return Err(InterceptError::MalformedRequest); + } let method = request.method().as_str().to_string(); let path = request .uri() @@ -1444,6 +1489,18 @@ where Ok(out) } +/// Bound the original framing before substitution changes Content-Length. +/// The limited reader also prevents body buffering from consuming a next request. +async fn substitute_managed_request( + client: &mut C, + mut prefix: Vec, + sni: &str, + secrets: &[&SecretEntry], +) -> Result, InterceptError> { + let remaining = bound_managed_request(&mut prefix)?; + substitute_request(&mut client.take(remaining), prefix, sni, secrets).await +} + /// Rewrite the request's `Content-Length` to `length`. The header block is /// already validated, so exactly one `Content-Length` is present. fn rewrite_content_length(head: &[u8], length: usize) -> Result, InterceptError> { @@ -2160,6 +2217,7 @@ async fn pump_and_observe( budget: usize, response_redactions: &[Vec], head_request: bool, + request_remaining: Option, ) -> Result, InterceptError> where C: AsyncRead + AsyncWrite + Unpin + Send + 'static, @@ -2168,16 +2226,19 @@ where let (client_rd, mut client_wr) = tokio::io::split(client_tls); let (mut up_rd, up_wr) = tokio::io::split(upstream_tls); - // Detached: drain any remaining request body to upstream. Aborted once the - // response is in — by then upstream has the full request (it responded). - let c2u = tokio::spawn(async move { - let mut client_rd = client_rd; + // Own both socket halves until the response completes or the session cancels. + let c2u = async move { let mut up_wr = up_wr; - let _ = tokio::io::copy(&mut client_rd, &mut up_wr).await; + let _ = tokio::io::copy( + &mut client_rd.take(request_remaining.unwrap_or(u64::MAX)), + &mut up_wr, + ) + .await; let _ = up_wr.shutdown().await; - }); + }; + tokio::pin!(c2u); - let response_result = async { + let response = async { let mut resp_buf = Vec::new(); let mut redactor = Http1ResponseRedactor::new(response_redactions, head_request); if !response_redactions.is_empty() { @@ -2235,11 +2296,66 @@ where let _ = client_wr.flush().await; let _ = client_wr.shutdown().await; Ok::<_, InterceptError>(resp_buf) + }; + tokio::pin!(response); + tokio::select! { + result = &mut response => result, + _ = &mut c2u => response.await, + } +} + +fn managed_authority_matches(raw: &str, sni: &str, port: u16) -> bool { + let Ok(authority) = raw.parse::() else { + return false; + }; + if raw.contains('@') || authority.port_u16().unwrap_or(443) != port { + return false; + } + let host = engram_core::types::managed_deny::canonical_host(authority.host()); + host.is_some() && host == engram_core::types::managed_deny::canonical_host(sni) +} +fn validate_managed_http1_authority( + head: &[u8], + sni: &str, + port: u16, +) -> Result<(), InterceptError> { + let text = std::str::from_utf8(head).map_err(|_| InterceptError::MalformedRequest)?; + let mut lines = text.split("\r\n"); + if lines.next().is_some_and(|l| l.starts_with("CONNECT ")) { + return Err(InterceptError::MalformedRequest); } - .await; - c2u.abort(); - let _ = c2u.await; - response_result + let mut hosts = 0; + for line in lines { + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("host") { + hosts += 1; + if !managed_authority_matches(value.trim(), sni, port) { + return Err(InterceptError::MalformedRequest); + } + } + if name.eq_ignore_ascii_case("upgrade") { + return Err(InterceptError::MalformedRequest); + } + } + } + if hosts != 1 { + return Err(InterceptError::MalformedRequest); + } + Ok(()) +} +/// Keep only this framed request; never forward a pipelined next request unchecked. +fn bound_managed_request(prefix: &mut Vec) -> Result { + let head_end = head_end_of(prefix).ok_or(InterceptError::MalformedRequest)?; + let body = match request_body_framing(&prefix[..head_end]) { + RequestBodyFraming::None => 0, + RequestBodyFraming::ContentLength(n) => n, + RequestBodyFraming::Chunked => return Err(InterceptError::MalformedRequest), + }; + let end = head_end + .checked_add(body) + .ok_or(InterceptError::MalformedRequest)?; + prefix.truncate(end); + Ok((end - prefix.len()) as u64) } #[cfg(test)] @@ -2523,6 +2639,41 @@ mod hardening_tests { assert_eq!(out, request); } + #[tokio::test] + async fn managed_substitution_does_not_fold_a_pipelined_request_into_the_body() { + let secret = placeholder_secret(); + let body = b"engram_ph_x"; + let head = format!( + "POST / HTTP/1.1\r\nHost: api.example\r\nContent-Length: {}\r\n\r\n", + body.len() + ) + .into_bytes(); + let next = b"GET /other HTTP/1.1\r\nHost: mcp.example\r\n\r\n"; + for buffered in [false, true] { + let mut prefix = head.clone(); + let mut unread = Vec::new(); + if buffered { + prefix.extend_from_slice(body); + prefix.extend_from_slice(next); + } else { + unread.extend_from_slice(body); + unread.extend_from_slice(next); + } + let out = substitute_managed_request( + &mut unread.as_slice(), + prefix, + "api.example", + &[&secret], + ) + .await + .unwrap(); + let text = String::from_utf8(out).unwrap(); + assert!(text.ends_with("sk-a-much-longer-real-value")); + assert!(!text.contains("/other") && !text.contains("mcp.example")); + assert!(text.contains("Content-Length: 27\r\n")); + } + } + #[test] fn binds_authority_to_sni_and_removes_duplicates() { let request = b"GET /v1 HTTP/1.1\r\nHost: allowed.example\r\nhOsT: attacker.example\r\nAccept: */*\r\n\r\n".to_vec(); @@ -2672,6 +2823,7 @@ mod hardening_tests { proxy_guest_io, proxy_upstream_io, StreamContext { + strict_authority: false, sni: "api.example", port: 443, secrets: &[&secret], diff --git a/crates/engram-egress-proxy/src/lib.rs b/crates/engram-egress-proxy/src/lib.rs index 915930270..14b259c33 100644 --- a/crates/engram-egress-proxy/src/lib.rs +++ b/crates/engram-egress-proxy/src/lib.rs @@ -60,3 +60,5 @@ pub use sni::{peek_sni, PeekError as SniPeekError}; pub use substitute::{scan_for_violation, substitute}; pub use tunnel_pool::{EndpointFactory, PoolConfig, PooledConn, TunnelEndpoint, TunnelPool}; pub use violation::first_match as scan_for_placeholder; + +mod connections; diff --git a/crates/engram-egress-proxy/src/proxy.rs b/crates/engram-egress-proxy/src/proxy.rs index 226b97ef5..abe4c0fc7 100644 --- a/crates/engram-egress-proxy/src/proxy.rs +++ b/crates/engram-egress-proxy/src/proxy.rs @@ -1,25 +1,19 @@ //! Accept loop + per-connection dispatcher. //! -//! Listens on `bind_addr` (TCP) for connections REDIRECTed by -//! iptables out of the FC TAPs. For each accepted connection: +//! Accepts redirected TCP connections and host-private streams. For each stream: //! -//! 1. Recover the original destination via `SO_ORIGINAL_DST`. (Linux -//! only — on macOS we just use the local addr; the proxy doesn't -//! actually run on macOS, that's compile-only.) -//! 2. Recover the source IP via `peer_addr`. Look up the matching -//! `SessionState` from the registry. No match → drop. -//! 3. Peek the SNI (TLS) or the HTTP `Host:` (plain HTTP — TODO). -//! For now, this proxy is HTTPS-only; iptables shouldn't -//! REDIRECT port 80 yet. +//! 1. Use the original destination supplied by the private ingress or +//! recovered with Linux `SO_ORIGINAL_DST` for redirected connections. +//! 2. Use the private ingress's session lease or enroll the source IP +//! in the registry. No session → drop. +//! 3. Peek the TLS SNI and preserve the original destination port. //! 4. Decide via `SessionState::decide(sni)`: //! - Reject → close. //! - Bypass → splice via `bypass::relay`. //! - Intercept → MITM via `intercept::run`. //! -//! Errors are logged but not propagated; the proxy is a best-effort -//! dispatcher. The most important failure mode is "violation" (a -//! placeholder appearing where it shouldn't) — those get a `WARN` -//! log so operators see them. +//! Redirected connection errors are logged; private dispatch returns errors +//! to its owner. Credential violations close the connection and emit a warning. use std::net::{IpAddr, SocketAddr}; use std::sync::Arc; @@ -37,6 +31,7 @@ const SNI_PEEK_BUDGET: Duration = Duration::from_secs(5); /// Max bytes we'll buffer of the ClientHello. const SNI_PEEK_BYTES: usize = 16 * 1024; +#[derive(Clone)] pub struct ProxyConfig { /// Where to listen for REDIRECTed traffic. Typically /// `0.0.0.0:9443` with iptables redirecting VM TAP traffic to @@ -48,12 +43,8 @@ pub struct ProxyConfig { /// `SystemResolver` (host's DNS). Tests pass a `StaticResolver` /// to point a fixed hostname at a loopback fixture. pub resolver: Arc, - /// Where the filtering DNS proxy listens (both UDP and TCP). - /// `None` disables the DNS path entirely — iptables would then - /// need to keep the unconditional `ACCEPT VM→1.1.1.1:53` rules - /// so the guest can resolve at all, accepting the DNS-exfil - /// channel. Production sets `Some(0.0.0.0:53)` and pairs it - /// with iptables `REDIRECT VM→{udp,tcp}/53 → :53`. + /// Where the filtering DNS proxy listens for redirected UDP and TCP. + /// `None` disables these listeners. Managed private DNS uses `resolve_private`. pub dns_bind_addr: Option, /// Where the session-scoped guest gateway listens. `None` disables it. pub guest_gateway_bind_addr: Option, @@ -65,10 +56,8 @@ pub struct ProxyConfig { /// The host-agent wires this to its coordinator bridge; tests pass a /// collecting closure. pub observe_sink: Option, - /// WS4: seam to re-mint a near-expiry inject credential via the coordinator. - /// `None` disables refresh (a minted inject then rides its boot token until - /// expiry — the pre-WS4 behaviour). The host-agent wires this to its coord - /// client; tests pass a stub. + /// Refresh near-expiry inject credentials through the coordinator. + /// `None` disables refresh. pub inject_refresher: Option>, /// ADR 0118: opens a byte stream to a port inside a session's guest, so a /// call to one of the session's OWN app hostnames is spliced back into the @@ -86,7 +75,7 @@ pub struct ProxyConfig { impl ProxyConfig { /// Convenience constructor: production wiring with the system /// DNS resolver and the bind addr / registry / mint the caller - /// provides. DNS filter on by default at port 53 with 1.1.1.1 + /// provides. DNS filter on by default at port 5353 with 1.1.1.1 /// upstream — disable by setting `dns_bind_addr = None`. pub fn new(bind_addr: SocketAddr, registry: Arc, mint: Arc) -> Self { Self { @@ -112,6 +101,7 @@ impl ProxyConfig { } } +#[derive(Clone)] pub struct Proxy { cfg: ProxyConfig, server_cfg: Arc, @@ -132,7 +122,43 @@ impl Proxy { } } - /// Bind the 443 listener and (when configured) the DNS sockets, + /// Dispatch a host-private stream with its mandatory original destination. + /// The caller must own a registry lease until this future has been dropped. + pub async fn dispatch_private( + &self, + stream: tokio::net::UnixStream, + session: Arc, + destination: std::net::SocketAddrV4, + ) -> std::io::Result<()> { + if destination.port() == 0 { + return Err(std::io::Error::other("missing original destination port")); + } + handle( + Some((IpAddr::V4(*destination.ip()), destination.port())), + stream, + session, + self.cfg.resolver.clone(), + self.server_cfg.clone(), + self.client_cfg.clone(), + self.cfg.observe_sink.clone(), + self.cfg.inject_refresher.clone(), + self.cfg.guest_port_dialer.clone(), + ) + .await + .map_err(std::io::Error::other) + } + + /// One controlled DNS request. The allocation owner holds the same lease + /// as TCP and owns cancellation through response delivery. + pub async fn resolve_private( + &self, + query: &[u8], + session: &crate::SessionState, + ) -> std::io::Result> { + dns::resolve_private(query, session, self.cfg.dns_upstream).await + } + + /// Bind the redirected TLS listener and (when configured) the DNS sockets, /// returning just the sockets for [`serve`](Self::serve) to run. /// /// Bind is split from serve so a bind failure is a value the caller @@ -224,19 +250,26 @@ impl Proxy { let observe_sink = self.cfg.observe_sink.clone(); let inject_refresher = self.cfg.inject_refresher.clone(); let guest_port_dialer = self.cfg.guest_port_dialer.clone(); + let SocketAddr::V4(peer_v4) = peer else { + continue; + }; + let Some((session, lease)) = registry.enroll(*peer_v4.ip()) else { + continue; + }; tokio::spawn(async move { - if let Err(e) = handle( - stream, - peer, - registry, - resolver, - server_cfg, - client_cfg, - observe_sink, - inject_refresher, - guest_port_dialer, - ) - .await + if let Some(Err(e)) = lease + .run(handle( + original_destination(&stream).ok(), + stream, + session, + resolver, + server_cfg, + client_cfg, + observe_sink, + inject_refresher, + guest_port_dialer, + )) + .await { tracing::debug!(peer = %peer, error = %e, "connection handler ended with error"); } @@ -246,7 +279,7 @@ impl Proxy { } /// The bound listeners handed from [`Proxy::bind`] to [`Proxy::serve`]: -/// the 443 TCP listener and, when the DNS filter is enabled, its +/// the redirected TCP listener and, when the DNS filter is enabled, its /// UDP+TCP sockets. An opaque token — all proxy state stays on `Proxy`. pub struct Listeners { tcp: TcpListener, @@ -255,10 +288,10 @@ pub struct Listeners { } #[allow(clippy::too_many_arguments)] -async fn handle( - mut stream: TcpStream, - peer: SocketAddr, - registry: Arc, +async fn handle( + original_dst: Option<(IpAddr, u16)>, + mut stream: C, + session: Arc, resolver: Arc, server_cfg: Arc, client_cfg: Arc, @@ -266,42 +299,15 @@ async fn handle( inject_refresher: Option>, guest_port_dialer: Option, ) -> Result<(), HandleError> { - let guest_ip = match peer.ip() { - IpAddr::V4(v4) => v4, - IpAddr::V6(_) => return Err(HandleError::NoSession), - }; - let session = registry.lookup(guest_ip).ok_or(HandleError::NoSession)?; - - // Pre-REDIRECT destination — useful only for logs. We don't - // dial it; the upstream is resolved fresh by SNI on the host. - // On non-Linux this errors (no SO_ORIGINAL_DST) — fine, we - // just log "unknown". - let original_dst = original_destination(&stream).ok(); - let (sni, peeked) = sni::peek_sni(&mut stream, SNI_PEEK_BYTES, SNI_PEEK_BUDGET) .await .map_err(HandleError::SniPeek)?; - // The proxy only handles port 443 today (iptables only REDIRECTs - // tcp/443). The port the upstream listens on is the same. + // Resolve the host by SNI and retain the original port. Ordinary redirected + // connections without a recovered destination use port 443. let port = original_dst.map(|(_, p)| p).unwrap_or(443); - // WS4 (campaign C2 audit — empty-body-looks-like-204 hazard): every reject - // path below closes the connection WITHOUT writing any HTTP bytes to the - // guest — `Reject` before any TLS at all, `RequestRejected`/`GraphqlRejected` - // after the guest-side TLS handshake but before a single response byte. The - // guest's HTTP client sees an unambiguous connection close (curl: "empty - // reply from server"; gh: transport error), NEVER a synthesized 2xx/204. And - // the intercept path streams the upstream response VERBATIM - // (`copy_bidirectional` / `pump_and_observe` — no status/body synthesis), so a - // real upstream 401 reaches the guest AS a 401. Therefore the campaign's - // "branch DELETE returned an empty body indistinguishable from 204 while the - // branch survived" was NOT produced here: the DELETE matched its inject policy - // (a permitted write), so it was forwarded with the boot-time injected token - // that had gone stale ~1h in — GitHub 401'd it, and `gh`'s own handling - // rendered that 401 as an empty body. The root cause is the stale token, fixed - // by the TTL-aware re-mint above (`InjectEntry::refresh_if_stale`), not a - // proxy-synthesized success. No proxy-side fix is warranted; do NOT add a path - // here that forwards or synthesizes an empty success-shaped reply. + // Rejection closes without an HTTP response. Interception preserves the + // upstream status and body; neither path synthesizes success. match session.decide(&sni) { Decision::Reject => { tracing::info!( @@ -372,6 +378,7 @@ async fn handle( peeked, resolver, intercept::StreamContext { + strict_authority: session.managed_deny.is_some(), sni: &sni, port, secrets: &secrets, @@ -438,7 +445,6 @@ async fn handle( #[derive(Debug)] enum HandleError { - NoSession, SniPeek(crate::sni::PeekError), Bypass(crate::bypass::BypassError), Intercept(crate::intercept::InterceptError), @@ -450,7 +456,6 @@ enum HandleError { impl std::fmt::Display for HandleError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::NoSession => write!(f, "no session for source IP"), Self::SniPeek(e) => write!(f, "sni peek: {e}"), Self::Bypass(e) => write!(f, "bypass: {e}"), Self::Intercept(e) => write!(f, "intercept: {e}"), @@ -464,9 +469,8 @@ impl std::error::Error for HandleError {} /// Recover the original destination of a REDIRECTed connection via /// `getsockopt(SOL_IP, SO_ORIGINAL_DST)`, exposed by socket2 as -/// `SockRef::original_dst()`. Linux-only; on macOS the call returns -/// `Unsupported` (the proxy doesn't actually run on macOS, but this -/// branch keeps `cargo check --workspace` clean). +/// `SockRef::original_dst_v4()`. Other platforms return `Unsupported`; +/// host-private ingress supplies its destination directly. fn original_destination(stream: &TcpStream) -> Result<(IpAddr, u16), HandleError> { #[cfg(target_os = "linux")] { diff --git a/crates/engram-egress-proxy/src/registry.rs b/crates/engram-egress-proxy/src/registry.rs index 5a15a5293..c16b3a8a8 100644 --- a/crates/engram-egress-proxy/src/registry.rs +++ b/crates/engram-egress-proxy/src/registry.rs @@ -27,6 +27,7 @@ use crate::policy::HostList; /// arriving from a given guest IP. #[derive(Clone, Debug)] pub struct SessionState { + pub managed_deny: Option, pub session_id: SessionId, /// The sandbox this session is bound to. Carried so the ADR 0118 short /// circuit can dial a guest port without a session→sandbox lookup — the @@ -474,19 +475,43 @@ pub struct Registry { struct RegistryInner { by_guest_ip: HashMap>, by_session: HashMap, + controls: HashMap>, + managed_floor: HashMap, } impl Registry { + pub fn managed_sessions(&self) -> Vec { + let state = self.inner.read(); + state + .by_session + .keys() + .filter(|id| state.managed_floor.contains_key(id)) + .copied() + .collect() + } pub fn new() -> Self { Self::default() } /// Replaces any prior registration for this `session_id`. pub fn register(&self, state: SessionState) { + // Synchronous registration is for an initial/ordinary policy. Managed + // replacement must use apply, which closes and drains old sockets. + let mut g = self.inner.write(); + if state.managed_deny.is_some() || g.managed_floor.contains_key(&state.session_id) { + return; + } + Self::publish(&mut g, state); + } + + fn publish(g: &mut RegistryInner, state: SessionState) { let guest_ip = state.guest_ip; let session_id = state.session_id; + if let Some(policy) = &state.managed_deny { + g.managed_floor.insert(session_id, policy.clone()); + } let arc = Arc::new(state); - let mut g = self.inner.write(); + g.controls.entry(session_id).or_default(); // Remove any stale guest_ip mapping if the session moved // (shouldn't happen — sandbox ids are stable — but defensive). if let Some(old_ip) = g.by_session.insert(session_id, guest_ip) { @@ -497,15 +522,256 @@ impl Registry { g.by_guest_ip.insert(guest_ip, arc); } + pub fn enroll( + &self, + guest_ip: Ipv4Addr, + ) -> Option<(Arc, crate::connections::ConnectionLease)> { + let g = self.inner.read(); + let state = g.by_guest_ip.get(&guest_ip)?.clone(); + let lease = g.controls.get(&state.session_id)?.enroll()?; + Some((state, lease)) + } + + /// Trusted host identity only. Enroll before cloning the policy snapshot. + /// Holding this lease makes replacement wait for allocation cleanup. + pub fn enroll_session( + &self, + session: SessionId, + ) -> Option<(Arc, crate::connections::ConnectionLease)> { + let g = self.inner.read(); + let lease = g.controls.get(&session)?.enroll()?; + let ip = g.by_session.get(&session)?; + let state = g.by_guest_ip.get(ip)?; + if state.session_id != session { + return None; + } + Some((state.clone(), lease)) + } + + /// Transfer a NEVER-BOUND private allocation lease. The ingress owns the + /// no-flow proof; all other old leases must drain before the new Arc exists. + pub async fn handoff_closed_private( + &self, + expected: &Arc, + lease: &mut Option, + state: SessionState, + ) -> Result, String> { + let invalid = || "closed private policy handoff changed".to_string(); + let new = state.managed_deny.as_ref().ok_or_else(invalid)?; + new.validate()?; + let old = expected.managed_deny.as_ref().ok_or_else(invalid)?; + if state.session_id != expected.session_id + || state.sandbox_id != expected.sandbox_id + || state.guest_ip != expected.guest_ip + || !state.tunnels.is_empty() + || new.revision < old.revision + || !new.contains(old) + || (new.revision == old.revision && new != old) + { + return Err(invalid()); + } + let control = self + .inner + .read() + .controls + .get(&state.session_id) + .cloned() + .ok_or_else(invalid)?; + let _apply = control.apply.lock().await; + { + let g = self.inner.read(); + if !g + .by_guest_ip + .get(&state.guest_ip) + .is_some_and(|s| Arc::ptr_eq(s, expected)) + || !lease.as_ref().is_some_and(|l| l.current_for(&control)) + { + return Err(invalid()); + } + } + struct PoisonOnDrop { + control: Arc, + armed: bool, + } + impl Drop for PoisonOnDrop { + fn drop(&mut self) { + if self.armed { + self.control.poison(); + } + } + } + let mut guard = PoisonOnDrop { + control: control.clone(), + armed: true, + }; + let generation = control.close_generation(); + drop(lease.take()); + control.drain().await; + let mut g = self.inner.write(); + let sid = state.session_id; + let ip = state.guest_ip; + let next = control + .handoff_publish(generation, || Self::publish(&mut g, state)) + .ok_or_else(invalid)?; + let current = g + .by_guest_ip + .get(&ip) + .filter(|s| s.session_id == sid) + .cloned() + .ok_or_else(invalid)?; + *lease = Some(next); + guard.armed = false; + Ok(current) + } + + pub async fn apply(&self, state: SessionState) -> Result<(), String> { + if let Some(policy) = &state.managed_deny { + policy.validate()?; + if !state.tunnels.is_empty() { + return Err("managed policy does not support tunnels".into()); + } + if state.guest_ip.is_unspecified() + || state.guest_ip.is_multicast() + || state.session_id.as_uuid().is_nil() + || state.sandbox_id.as_uuid().is_nil() + { + return Err("managed policy has no guest identity".into()); + } + } + let control = self + .inner + .write() + .controls + .entry(state.session_id) + .or_default() + .clone(); + let _apply = control.apply.lock().await; + { + let g = self.inner.read(); + if let Some(old) = g.managed_floor.get(&state.session_id) { + let new = state + .managed_deny + .as_ref() + .ok_or("managed policy cannot be removed")?; + if new.revision < old.revision + || !new.contains(old) + || (new.revision == old.revision && new != old) + { + return Err("managed policy regressed".into()); + } + } + } + if state.managed_deny.is_some() { + control.close(); + control.drain().await; + } + if control.is_poisoned() { + return Err("private network cleanup failed; replace proxy before replay".into()); + } + Self::publish(&mut self.inner.write(), state); + control.open(); + Ok(()) + } + + /// Compare even a CLOSED registration under the same lock as policy apply. + /// Keep that lock through drain and removal so a late old operation cannot + /// quarantine or unregister a replacement sandbox. + pub async fn retire_managed( + &self, + session: SessionId, + sandbox: engram_core::SandboxId, + guest: Ipv4Addr, + ) -> Result<(), String> { + let control = self + .inner + .write() + .controls + .entry(session) + .or_default() + .clone(); + let _apply = control.apply.lock().await; + let matches = |g: &RegistryInner| -> Result<(), String> { + if let Some(ip) = g.by_session.get(&session) { + if *ip != guest + || !g.by_guest_ip.get(ip).is_some_and(|state| { + state.session_id == session + && state.sandbox_id == sandbox + && state.managed_deny.is_some() + }) + { + return Err("private registration changed during retirement".into()); + } + } + Ok(()) + }; + matches(&self.inner.read())?; + control.close(); + control.drain().await; + if control.is_poisoned() { + return Err("private network cleanup failed; replace proxy before replay".into()); + } + let mut g = self.inner.write(); + matches(&g)?; + if let Some(ip) = g.by_session.remove(&session) { + g.by_guest_ip.remove(&ip); + } + Ok(()) + } + + pub async fn quarantine_and_drain(&self, session_id: SessionId) { + let control = self + .inner + .write() + .controls + .entry(session_id) + .or_default() + .clone(); + let _apply = control.apply.lock().await; + control.close(); + control.drain().await; + } + pub fn is_poisoned(&self, session_id: SessionId) -> bool { + self.inner + .read() + .controls + .get(&session_id) + .is_some_and(|control| control.is_poisoned()) + } + pub fn lookup_session(&self, session_id: SessionId) -> Option> { + let g = self.inner.read(); + let ip = g.by_session.get(&session_id)?; + g.controls + .get(&session_id)? + .is_open() + .then(|| g.by_guest_ip.get(ip).cloned()) + .flatten() + } + pub fn quarantine(&self, session_id: SessionId) { + self.inner + .write() + .controls + .entry(session_id) + .or_default() + .close(); + } + pub fn unregister(&self, session_id: SessionId) { let mut g = self.inner.write(); + if let Some(control) = g.controls.get(&session_id) { + control.close(); + } if let Some(ip) = g.by_session.remove(&session_id) { g.by_guest_ip.remove(&ip); } } pub fn lookup(&self, guest_ip: Ipv4Addr) -> Option> { - self.inner.read().by_guest_ip.get(&guest_ip).cloned() + let g = self.inner.read(); + let state = g.by_guest_ip.get(&guest_ip)?; + g.controls + .get(&state.session_id)? + .is_open() + .then(|| state.clone()) } pub fn live_count(&self) -> usize { @@ -533,6 +799,13 @@ impl SessionState { /// straight through — the guest could trade its session token for a /// credential this proxy no longer bounds. pub fn decide(&self, hostname: &str) -> Decision<'_> { + if self + .managed_deny + .as_ref() + .is_some_and(|p| p.denies(hostname)) + { + return Decision::Reject; + } // ADR 0118: checked FIRST, before the deny-list and the allow-list. A // sibling app is not an egress destination — the connection is spliced // back into this same sandbox and never leaves the host — so it is not @@ -573,7 +846,8 @@ impl SessionState { // spliced: the rule reads the request line, which only an // intercepted connection produces. Interception with an empty // policy relays the connection as before and lets the rule run. - if crate::google_denylist::requires_inspection(hostname) { + if self.managed_deny.is_some() || crate::google_denylist::requires_inspection(hostname) + { return Decision::Intercept { secrets: Vec::new(), injects: Vec::new(), @@ -649,8 +923,245 @@ mod tests { use super::*; use std::str::FromStr; + #[test] + fn managed_denies_win_over_every_existing_rule() { + let mut s = state(); + s.allow_all = true; + s.apps.push(AppEndpoint { + hostname: "api.openai.com".into(), + port: 8080, + }); + s.managed_deny = Some( + engram_core::types::managed_deny::ManagedDenyPolicy::new( + 1, + [ + "api.openai.com", + "api.datadoghq.com", + "api.github.com", + "registry.npmjs.org", + "alias.example", + ] + .map(String::from), + ) + .unwrap(), + ); + for host in [ + "API.OPENAI.COM.", + "api.datadoghq.com", + "api.github.com", + "registry.npmjs.org", + "alias.example", + ] { + assert!(matches!(s.decide(host), Decision::Reject), "{host}"); + } + assert!(matches!( + s.decide("near.alias.example"), + Decision::Intercept { .. } + )); + } + + #[tokio::test] + async fn managed_apply_drains_old_arc_sockets_before_return() { + use tokio::io::AsyncReadExt; + let registry = Arc::new(Registry::new()); + let initial = state(); + let ip = initial.guest_ip; + let mut next = initial.clone(); + registry.register(initial); + let retained = registry.lookup(ip).unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut peer = tokio::net::TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (mut socket, _) = listener.accept().await.unwrap(); + let (_, lease) = registry.enroll(ip).unwrap(); + let task = tokio::spawn(lease.run(async move { + let mut b = [0]; + let _ = socket.read(&mut b).await; + })); + next.managed_deny = Some( + engram_core::types::managed_deny::ManagedDenyPolicy::new(1, ["api.openai.com".into()]) + .unwrap(), + ); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + registry.apply(next).await.unwrap(); + task.await.unwrap(); + assert_eq!(peer.read(&mut [0]).await.unwrap(), 0); + }) + .await + .expect("socket drain is bounded"); + assert!(matches!( + retained.decide("api.openai.com"), + Decision::Intercept { .. } + )); + assert!(matches!( + registry.lookup(ip).unwrap().decide("api.openai.com"), + Decision::Reject + )); + } + + #[tokio::test] + async fn managed_drain_does_not_lock_other_sessions() { + tokio::time::timeout(std::time::Duration::from_secs(2), async { + let registry = Arc::new(Registry::new()); + let mut first = state(); + let mut second = state(); + second.guest_ip = "10.200.0.3".parse().unwrap(); + registry.register(first.clone()); + registry.register(second.clone()); + let (_, held) = registry.enroll(first.guest_ip).unwrap(); + first.managed_deny = Some( + engram_core::types::managed_deny::ManagedDenyPolicy::new(1, ["mcp.example".into()]) + .unwrap(), + ); + second.managed_deny = first.managed_deny.clone(); + let first_ip = first.guest_ip; + let other_ip = second.guest_ip; + let worker = registry.clone(); + let applying = tokio::spawn(async move { worker.apply(first).await }); + while registry.lookup(first_ip).is_some() { + tokio::task::yield_now().await; + } + registry.apply(second).await.unwrap(); + assert!(registry.lookup(other_ip).is_some()); + assert!( + !applying.is_finished(), + "first drain must still wait for its lease" + ); + drop(held); + applying.await.unwrap().unwrap(); + }) + .await + .expect("per-session drain has no global wait"); + } + + fn closed_state() -> SessionState { + let mut s = state(); + s.managed_deny = Some( + engram_core::types::managed_deny::ManagedDenyPolicy::new(1, ["mcp.example".into()]) + .unwrap(), + ); + s + } + + #[tokio::test] + async fn closed_handoff_drains_other_leases_and_reacquires_current_arc() { + tokio::time::timeout(std::time::Duration::from_secs(2), async { + let registry = Arc::new(Registry::new()); + let initial = closed_state(); + let ip = initial.guest_ip; + registry.apply(initial.clone()).await.unwrap(); + let (old, owned) = registry.enroll(ip).unwrap(); + let (_, mut other) = registry.enroll(ip).unwrap(); + let task_registry = registry.clone(); + let task = tokio::spawn(async move { + let mut lease = Some(owned); + let mut next = initial; + next.allow_all = true; + let current = task_registry + .handoff_closed_private(&old, &mut lease, next) + .await + .unwrap(); + assert!(!Arc::ptr_eq(&old, ¤t)); + (current, lease.unwrap()) + }); + other.cancelled().await; + assert!(registry.lookup(ip).is_none()); + assert!(!task.is_finished()); + drop(other); + let (current, lease) = task.await.unwrap(); + assert!(current.allow_all); + assert!(Arc::ptr_eq(¤t, ®istry.lookup(ip).unwrap())); + assert!(lease.current_for(®istry.inner.read().controls[¤t.session_id])); + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn closed_handoff_cancellation_poison_is_terminal() { + tokio::time::timeout(std::time::Duration::from_secs(2), async { + let registry = Arc::new(Registry::new()); + let initial = closed_state(); + registry.apply(initial.clone()).await.unwrap(); + let (old, owned) = registry.enroll(initial.guest_ip).unwrap(); + let (_, mut other) = registry.enroll(initial.guest_ip).unwrap(); + let worker = registry.clone(); + let candidate = initial.clone(); + let task = tokio::spawn(async move { + worker + .handoff_closed_private(&old, &mut Some(owned), candidate) + .await + }); + other.cancelled().await; + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + drop(other); + assert!(registry.is_poisoned(initial.session_id)); + assert!(registry.apply(initial.clone()).await.is_err()); + assert!(registry.lookup(initial.guest_ip).is_none()); + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn closed_handoff_rejects_unrelated_close_and_stale_arc() { + tokio::time::timeout(std::time::Duration::from_secs(2), async { + let registry = Arc::new(Registry::new()); + let initial = closed_state(); + registry.apply(initial.clone()).await.unwrap(); + let (old, owned) = registry.enroll(initial.guest_ip).unwrap(); + let (_, mut other) = registry.enroll(initial.guest_ip).unwrap(); + let worker = registry.clone(); + let candidate = initial.clone(); + let task = tokio::spawn(async move { + worker + .handoff_closed_private(&old, &mut Some(owned), candidate) + .await + }); + other.cancelled().await; + registry.inner.read().controls[&initial.session_id].close(); + drop(other); + assert!(task.await.unwrap().is_err()); + assert!(registry.is_poisoned(initial.session_id)); + assert!(registry.lookup(initial.guest_ip).is_none()); + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn ordinary_apply_after_closed_handoff_cancels_the_new_lease() { + tokio::time::timeout(std::time::Duration::from_secs(2), async { + let registry = Arc::new(Registry::new()); + let initial = closed_state(); + registry.apply(initial.clone()).await.unwrap(); + let (old, owned) = registry.enroll(initial.guest_ip).unwrap(); + let mut lease = Some(owned); + let current = registry + .handoff_closed_private(&old, &mut lease, initial.clone()) + .await + .unwrap(); + let worker = registry.clone(); + let apply = tokio::spawn(async move { worker.apply(initial).await }); + lease.as_mut().unwrap().cancelled().await; + assert!(!apply.is_finished()); + assert!(registry.lookup(current.guest_ip).is_none()); + drop(lease); + apply.await.unwrap().unwrap(); + assert!(!Arc::ptr_eq( + ¤t, + ®istry.lookup(current.guest_ip).unwrap() + )); + }) + .await + .unwrap(); + } + fn state() -> SessionState { SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: Ipv4Addr::from_str("10.200.0.2").unwrap(), diff --git a/crates/engram-egress-proxy/tests/intercept_e2e.rs b/crates/engram-egress-proxy/tests/intercept_e2e.rs index b80f29930..5641baf67 100644 --- a/crates/engram-egress-proxy/tests/intercept_e2e.rs +++ b/crates/engram-egress-proxy/tests/intercept_e2e.rs @@ -240,6 +240,7 @@ async fn substitutes_placeholder_in_intercept_path() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &secrets, @@ -335,6 +336,7 @@ async fn violation_returned_when_placeholder_targets_disallowed_host() { // this host disallow?" always got an empty answer. The detector could not // fire in production, and this test could not notice. let session = SessionState { + managed_deny: None, session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: "10.200.0.2".parse().unwrap(), @@ -382,6 +384,7 @@ async fn violation_returned_when_placeholder_targets_disallowed_host() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &secrets, @@ -462,6 +465,7 @@ async fn injects_header_on_allowed_request() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -537,6 +541,7 @@ async fn keep_alive_second_request_cannot_bypass_the_gate() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -662,6 +667,7 @@ async fn guest_upgrade_header_cannot_reopen_the_bypass() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -771,6 +777,7 @@ async fn rejects_request_shape_outside_policy() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -840,6 +847,7 @@ async fn reject_writes_no_response_to_the_guest() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -950,6 +958,7 @@ async fn near_expiry_inject_is_reminted_before_forwarding() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -1120,6 +1129,7 @@ async fn observes_response_and_emits_asset() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -1220,6 +1230,7 @@ async fn failed_status_emits_no_asset() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -1340,6 +1351,7 @@ async fn run_graphql_inject( Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -1574,6 +1586,7 @@ async fn run_graphql_observe( Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -1840,6 +1853,7 @@ async fn h2_intercept_injects_and_redacts_across_headers_body_and_trailers() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "fake-upstream", port: upstream_addr.port(), secrets: &[], @@ -1953,6 +1967,7 @@ async fn h2_intercept_denies_a_grpc_credential_method() { Vec::new(), resolver, intercept::StreamContext { + strict_authority: false, sni: "iam.googleapis.com", port: upstream_addr.port(), secrets: &[], @@ -2014,3 +2029,109 @@ async fn h2_intercept_denies_a_grpc_credential_method() { driver.abort(); let _ = tokio::time::timeout(std::time::Duration::from_secs(5), proxy_task).await; } + +#[tokio::test] +async fn managed_default_allow_rejects_host_override_without_injecting_credentials() { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + for (authority, succeeds) in [("mcp.example", false), ("API.EXAMPLE.", true)] { + let ca = ca(); + let captured = Arc::new(Mutex::new(Vec::new())); + let upstream_addr = fake_upstream(captured.clone()).await; + let mut session = SessionState { + managed_deny: Some(engram_core::types::managed_deny::ManagedDenyPolicy::new(1, ["mcp.example".into()]).unwrap()), + session_id: SessionId::new(), sandbox_id: engram_core::SandboxId::new(), guest_ip: "127.0.0.1".parse().unwrap(), + network_allow: HostList::from_manifest(&[], &[]).unwrap(), allow_all: true, + secrets: vec![], injects: vec![], observes: vec![], guest_services: vec![], tunnels: vec![], apps: vec![], + }; + let Decision::Intercept { secrets, injects, observes, .. } = session.decide("api.example") else { panic!("managed Literal/default-allow TLS must be inspected"); }; + assert!(secrets.is_empty() && injects.is_empty() && observes.is_empty()); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let guest = TcpStream::connect(listener.local_addr().unwrap()).await.unwrap(); + let (proxy, _) = listener.accept().await.unwrap(); + let server_cfg = build_server_config(Arc::new(CertMint::new(ca.clone()))); + let task = tokio::spawn(async move { + intercept::run(proxy, vec![], Arc::new(StaticResolver::new().with("api.example", upstream_addr)), intercept::StreamContext { + strict_authority: true, sni: "api.example", port: 443, secrets: &[], injects: &[], observes: &[], foreign_placeholders: &[], session_id: SessionId::new(), sink: None, refresher: None, + }, server_cfg, build_client_config()).await + }); + let mut roots = rustls::RootCertStore::empty(); + for cert in rustls_pemfile::certs(&mut ca.cert_pem.as_bytes()) { roots.add(cert.unwrap()).unwrap(); } + let connector = TlsConnector::from(Arc::new(rustls::ClientConfig::builder().with_root_certificates(roots).with_no_client_auth())); + let mut guest = connector.connect("api.example".try_into().unwrap(), guest).await.unwrap(); + guest.write_all(format!("GET / HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n\r\nGET /other HTTP/1.1\r\nHost: mcp.example\r\n\r\n").as_bytes()).await.unwrap(); + guest.shutdown().await.unwrap(); + let mut response = Vec::new(); let _ = guest.read_to_end(&mut response).await; + let result = task.await.unwrap(); + assert_eq!(result.is_ok(), succeeds); + let request = String::from_utf8_lossy(&captured.lock()).into_owned(); + if succeeds { + assert!(request.contains("Host: api.example\r\n")); + assert!(!request.contains("Authorization") && !request.contains("/other") && !request.contains("mcp.example")); + } else { assert!(request.is_empty()); } + session.managed_deny = None; + assert!(matches!(session.decide("api.example"), Decision::Bypass)); + } + }).await.expect("bounded managed HTTP authority test"); +} + +#[tokio::test] +async fn managed_h2_rejects_authority_override() { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + let ca = ca(); + let upstream = h2_upstream(Arc::new(Mutex::new(None))).await; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let guest = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (proxy, _) = listener.accept().await.unwrap(); + let server = build_server_config(Arc::new(CertMint::new(ca.clone()))); + let task = tokio::spawn(async move { + intercept::run( + proxy, + vec![], + Arc::new(StaticResolver::new().with("api.example", upstream)), + intercept::StreamContext { + strict_authority: true, + sni: "api.example", + port: 443, + secrets: &[], + injects: &[], + observes: &[], + foreign_placeholders: &[], + session_id: SessionId::new(), + sink: None, + refresher: None, + }, + server, + build_client_config(), + ) + .await + }); + let mut roots = rustls::RootCertStore::empty(); + for cert in rustls_pemfile::certs(&mut ca.cert_pem.as_bytes()) { + roots.add(cert.unwrap()).unwrap(); + } + let mut cfg = rustls::ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + cfg.alpn_protocols = vec![b"h2".to_vec()]; + let guest = TlsConnector::from(Arc::new(cfg)) + .connect("api.example".try_into().unwrap(), guest) + .await + .unwrap(); + let (mut client, connection) = h2::client::handshake(guest).await.unwrap(); + let connection = tokio::spawn(connection); + let request = http::Request::builder() + .uri("https://mcp.example/forbidden") + .body(()) + .unwrap(); + let (response, _) = client.send_request(request, true).unwrap(); + assert_eq!(response.await.unwrap().status(), 403); + drop(client); + connection.abort(); + let _ = connection.await; + let _ = task.await.unwrap(); + }) + .await + .expect("bounded managed H2 authority test"); +} diff --git a/crates/engram-egress-proxyd/src/control.rs b/crates/engram-egress-proxyd/src/control.rs index 0f4bc1bd1..4959f71ec 100644 --- a/crates/engram-egress-proxyd/src/control.rs +++ b/crates/engram-egress-proxyd/src/control.rs @@ -17,6 +17,9 @@ use engram_egress_proxy::{GuestGatewayRegistry, Listeners, Proxy, Registry}; pub(crate) struct ControlState { pub registry: Arc, + pub private: Arc, + pub proxy: Arc, + pub host_id: engram_core::HostId, pub gateway: Arc, pub hello: HelloInfo, /// Flipped by the `Shutdown` op; [`crate::run`] selects on it and @@ -57,6 +60,68 @@ async fn serve_conn( loop { let msg: ToProxyd = engram_egress_proto::read_frame(&mut stream).await?; let reply = match msg { + ToProxyd::PrivateNetwork => { + let allocation = engram_egress_proto::private::read(&mut stream).await?; + let proxy = state.proxy.clone(); + return state + .private + .serve( + stream, + allocation, + state.registry.clone(), + proxy, + state.host_id, + &state.hello.incarnation, + ) + .await; + } + ToProxyd::HandoffClosedPrivate(request) => { + match state.private.handoff_closed(*request).await { + Ok(()) => FromProxyd::Ok, + Err(error) => FromProxyd::Err(error), + } + } + ToProxyd::RetirePrivateNetwork(allocation) => { + // Managed registrations reject tunnels. Only ordinary removal + // needs the session-wide gateway callback. + match state + .private + .retire( + &allocation, + &state.registry, + state.host_id, + &state.hello.incarnation, + ) + .await + { + Ok(()) => FromProxyd::Ok, + Err(error) => FromProxyd::Err(error), + } + } + ToProxyd::ListManaged => FromProxyd::ManagedSessions(state.registry.managed_sessions()), + ToProxyd::QuarantineManaged(session) => { + state.registry.quarantine_and_drain(session).await; + if state.registry.is_poisoned(session) { + FromProxyd::Err( + "private network cleanup failed; replace proxy before replay".into(), + ) + } else { + FromProxyd::Ok + } + } + ToProxyd::ReadManaged(session) => { + FromProxyd::ManagedState(state.registry.lookup_session(session).and_then(|s| { + s.managed_deny.clone().map(|policy| { + engram_core::types::managed_deny::ManagedProxyState { + incarnation: state.hello.incarnation.clone(), + session_id: s.session_id, + sandbox_id: s.sandbox_id, + guest_ip: s.guest_ip, + policy, + } + }) + })) + } ToProxyd::Hello { proto_version } => { if proto_version != engram_egress_proto::PROTO_VERSION { tracing::warn!( @@ -70,14 +135,18 @@ async fn serve_conn( // on a mismatch. FromProxyd::HelloAck(state.hello.clone()) } - ToProxyd::SyncPolicies(policies) => sync_policies(&state, policies), - ToProxyd::ApplyPolicy(policy) => match register_policy(&state.registry, *policy) { - Ok(()) => FromProxyd::Ok, - Err(e) => FromProxyd::Err(format!("translate egress policy: {e}")), - }, + ToProxyd::SyncPolicies(policies) => sync_policies(&state, policies).await, + ToProxyd::ApplyPolicy(policy) => { + match register_policy(&state.registry, *policy).await { + Ok(()) => FromProxyd::Ok, + Err(e) => FromProxyd::Err(format!("translate egress policy: {e}")), + } + } ToProxyd::RemoveSession(session_id) => { - unregister_session(&state, session_id); - FromProxyd::Ok + match unregister_session(&state, session_id).await { + Ok(()) => FromProxyd::Ok, + Err(error) => FromProxyd::Err(error), + } } ToProxyd::Health => FromProxyd::HealthReport { sessions: state.registry.live_count(), @@ -102,6 +171,7 @@ async fn serve_conn( })) } ToProxyd::Shutdown => { + state.private.shutdown().await; tracing::info!("shutdown requested over control socket; exiting"); let _ = engram_egress_proto::write_frame(&mut stream, &FromProxyd::Ok).await; let _ = state.shutdown.send(true); @@ -115,9 +185,17 @@ async fn serve_conn( /// The single session-teardown entry point (the `HostEgress:: /// unregister_session` contract): drop the registration AND fan /// `session_closed` to every tunnel upstream, or pooled state leaks. -fn unregister_session(state: &ControlState, session_id: engram_core::SessionId) { +async fn unregister_session( + state: &ControlState, + session_id: engram_core::SessionId, +) -> Result<(), String> { + state.registry.quarantine_and_drain(session_id).await; + if state.registry.is_poisoned(session_id) { + return Err("private network cleanup failed; replace proxy before replay".into()); + } state.registry.unregister(session_id); state.gateway.session_closed(session_id); + Ok(()) } /// Full replace: register everything in the set, then drop any @@ -130,24 +208,27 @@ fn unregister_session(state: &ControlState, session_id: engram_core::SessionId) /// that translated, instead of being pruned into a total egress /// lockout. The per-apply path (`ApplyPolicy`) keeps erroring loudly /// — only the bulk replace shields survivors. -fn sync_policies( +async fn sync_policies( state: &ControlState, policies: Vec, ) -> FromProxyd { let mut keep = std::collections::HashSet::with_capacity(policies.len()); let mut failed = 0usize; + let mut managed_failed = false; let mut applied = 0usize; let total = policies.len(); for policy in policies { let session_id = policy.session_id; + let managed = policy.managed_deny.is_some(); // Shield the session from the prune below regardless of the // translate outcome — a defective UPDATE must not evict a // working registration. keep.insert(session_id); - match register_policy(&state.registry, policy) { + match register_policy(&state.registry, policy).await { Ok(()) => applied += 1, Err(e) => { failed += 1; + managed_failed |= managed || state.registry.is_poisoned(session_id); tracing::error!( %session_id, error = %e, @@ -164,11 +245,18 @@ fn sync_policies( .into_iter() .filter(|sid| !keep.contains(sid)) { - unregister_session(state, stale); - pruned += 1; + if unregister_session(state, stale).await.is_err() { + managed_failed = true; + } else { + pruned += 1; + } } tracing::info!(applied, failed, pruned, total, "policies synced"); - FromProxyd::Ok + if managed_failed { + FromProxyd::Err("managed policy unavailable; session quarantined".into()) + } else { + FromProxyd::Ok + } } /// Bind the proxy listeners, retrying briefly to ride a transient port @@ -232,19 +320,30 @@ fn graphql_match(op: &str, field: &str) -> Result Result<(), engram_egress_proxy::policy::ParseError> { +) -> Result<(), String> { + if policy.managed_deny.is_some() { + registry.quarantine(policy.session_id); + } + registry.apply(translate_policy(policy)?).await +} + +pub(crate) fn translate_policy( + policy: engram_core::types::egress::SessionEgressPolicy, +) -> Result { let network_allow = engram_egress_proxy::HostList::from_manifest( &policy.network_allow_hosts, &policy.network_allow_host_patterns, - )?; + ) + .map_err(|e| e.to_string())?; let allow_all = policy.allow_all; let mut secrets = Vec::with_capacity(policy.secrets.len()); for s in policy.secrets { let allow = - engram_egress_proxy::HostList::from_manifest(&s.allow_hosts, &s.allow_host_patterns)?; + engram_egress_proxy::HostList::from_manifest(&s.allow_hosts, &s.allow_host_patterns) + .map_err(|e| e.to_string())?; secrets.push(engram_egress_proxy::SecretEntry { placeholder: s.placeholder, real_value: s.real_value, @@ -257,7 +356,8 @@ pub(crate) fn register_policy( let mut injects = Vec::with_capacity(policy.injects.len()); for i in policy.injects { let allow = - engram_egress_proxy::HostList::from_manifest(&i.allow_hosts, &i.allow_host_patterns)?; + engram_egress_proxy::HostList::from_manifest(&i.allow_hosts, &i.allow_host_patterns) + .map_err(|e| e.to_string())?; // ADR 0059: a non-empty-but-unparseable graphql op is a corrupt entry — // skip it (fail-closed) so it never degrades into a permissive REST gate // on `/graphql`. @@ -292,7 +392,8 @@ pub(crate) fn register_policy( let mut observes = Vec::with_capacity(policy.observes.len()); for o in policy.observes { let allow = - engram_egress_proxy::HostList::from_manifest(&o.allow_hosts, &o.allow_host_patterns)?; + engram_egress_proxy::HostList::from_manifest(&o.allow_hosts, &o.allow_host_patterns) + .map_err(|e| e.to_string())?; let graphql = match graphql_match(&o.graphql_operation, &o.graphql_field) { Ok(g) => g, Err(()) => { @@ -332,7 +433,8 @@ pub(crate) fn register_policy( }), }); } - registry.register(engram_egress_proxy::SessionState { + Ok(engram_egress_proxy::SessionState { + managed_deny: policy.managed_deny, session_id: policy.session_id, sandbox_id: policy.sandbox_id, guest_ip: policy.guest_ip, @@ -345,8 +447,7 @@ pub(crate) fn register_policy( tunnels: policy.tunnels, // ADR 0118: the session's own app hostnames, for the SNI short circuit. apps: policy.apps, - }); - Ok(()) + }) } #[cfg(test)] @@ -359,6 +460,7 @@ mod tests { fn policy(session_id: SessionId, ip: Ipv4Addr) -> SessionEgressPolicy { SessionEgressPolicy { + managed_deny: None, session_id, sandbox_id: SandboxId::new(), guest_ip: ip, @@ -376,11 +478,23 @@ mod tests { } fn state() -> ControlState { + let registry = Arc::new(Registry::new()); + let dir = tempfile::tempdir().unwrap(); + let ca = Arc::new(engram_egress_proxy::Ca::load_or_generate(dir.path()).unwrap()); + let proxy = Arc::new(Proxy::new(engram_egress_proxy::ProxyConfig::new( + "127.0.0.1:0".parse().unwrap(), + registry.clone(), + Arc::new(engram_egress_proxy::CertMint::new(ca)), + ))); ControlState { - registry: Arc::new(Registry::new()), + private: Arc::new(crate::private::Ingress::default()), + proxy, + host_id: engram_core::HostId::new(), + registry, gateway: Arc::new(GuestGatewayRegistry::default()), shutdown: tokio::sync::watch::channel(false).0, hello: HelloInfo { + incarnation: "test-incarnation".into(), proto_version: engram_egress_proto::PROTO_VERSION, source_fingerprint: None, proxy_port: 8443, @@ -392,16 +506,108 @@ mod tests { } } - #[test] - fn sync_registers_and_prunes() { + #[tokio::test] + async fn late_managed_removal_cannot_retire_a_same_session_replacement() { + let state = Arc::new(state()); + let session = SessionId::new(); + let mut old = policy(session, "192.0.2.2".parse().unwrap()); + old.managed_deny = Some( + engram_core::types::managed_deny::ManagedDenyPolicy::new(1, ["mcp.example".into()]) + .unwrap(), + ); + register_policy(&state.registry, old.clone()).await.unwrap(); + let (_, mut lease) = state.registry.enroll(old.guest_ip).unwrap(); + let mut replacement = old.clone(); + replacement.sandbox_id = SandboxId::new(); + let registry = state.registry.clone(); + let next = replacement.clone(); + let apply = tokio::spawn(async move { register_policy(®istry, next).await }); + // Replacement owns the existing session lifecycle lock and waits for + // the old lease. The old removal will execute only after replacement. + lease.cancelled().await; + let (mut client, server) = tokio::net::UnixStream::pair().unwrap(); + let serving = state.clone(); + let removal = tokio::spawn(async move { serve_conn(server, serving).await }); + let allocation = engram_egress_proto::private::Allocation { + version: engram_egress_proto::private::VERSION, + host_id: state.host_id, + host_incarnation: "host-1".into(), + proxy_incarnation: state.hello.incarnation.clone(), + session_id: session, + sandbox_id: old.sandbox_id, + boot_attempt: "boot-1".into(), + attachment_generation: 1, + guest_ip: old.guest_ip, + gateway_ip: "192.0.2.1".parse().unwrap(), + policy_revision: 1, + policy_digest: old.managed_deny.as_ref().unwrap().digest.clone(), + }; + engram_egress_proto::write_frame( + &mut client, + &ToProxyd::RetirePrivateNetwork(Box::new(allocation)), + ) + .await + .unwrap(); + assert!(tokio::time::timeout( + std::time::Duration::from_millis(1), + engram_egress_proto::read_frame::<_, FromProxyd>(&mut client), + ) + .await + .is_err()); + drop(client); + drop(lease); + apply.await.unwrap().unwrap(); + let _ = removal.await.unwrap(); + let current = state + .registry + .lookup_session(session) + .expect("late removal must preserve replacement"); + assert_eq!(current.sandbox_id, replacement.sandbox_id); + assert_eq!(current.guest_ip, replacement.guest_ip); + } + + #[tokio::test] + async fn sync_cannot_acknowledge_a_poisoned_session_as_ordinary_or_removed() { + let state = state(); + let session = SessionId::new(); + let mut p = policy(session, "192.0.2.2".parse().unwrap()); + p.managed_deny = Some( + engram_core::types::managed_deny::ManagedDenyPolicy::new(1, ["mcp.example".into()]) + .unwrap(), + ); + register_policy(&state.registry, p.clone()).await.unwrap(); + let (_, lease) = state.registry.enroll_session(session).unwrap(); + lease.poison(); + drop(lease); + assert!(matches!( + sync_policies(&state, vec![p.clone()]).await, + FromProxyd::Err(_) + )); + p.managed_deny = None; + assert!(matches!( + sync_policies(&state, vec![p]).await, + FromProxyd::Err(_) + )); + assert!(matches!( + sync_policies(&state, vec![]).await, + FromProxyd::Err(_) + )); + assert!(unregister_session(&state, session).await.is_err()); + assert!(state.registry.is_poisoned(session)); + } + + #[tokio::test] + async fn sync_registers_and_prunes() { let state = state(); let kept = SessionId::new(); let stale = SessionId::new(); // A pre-existing registration the sync set does not carry. - register_policy(&state.registry, policy(stale, Ipv4Addr::new(10, 200, 0, 2))).unwrap(); + register_policy(&state.registry, policy(stale, Ipv4Addr::new(10, 200, 0, 2))) + .await + .unwrap(); assert_eq!(state.registry.live_count(), 1); - let reply = sync_policies(&state, vec![policy(kept, Ipv4Addr::new(10, 200, 0, 6))]); + let reply = sync_policies(&state, vec![policy(kept, Ipv4Addr::new(10, 200, 0, 6))]).await; assert_eq!(reply, FromProxyd::Ok); assert_eq!(state.registry.live_count(), 1); assert!(state @@ -414,8 +620,8 @@ mod tests { .is_none()); } - #[test] - fn sync_skips_untranslatable_policies_and_keeps_the_rest() { + #[tokio::test] + async fn sync_skips_untranslatable_policies_and_keeps_the_rest() { let state = state(); let good = SessionId::new(); let mut bad = policy(SessionId::new(), Ipv4Addr::new(10, 200, 0, 10)); @@ -424,7 +630,8 @@ mod tests { let reply = sync_policies( &state, vec![bad, policy(good, Ipv4Addr::new(10, 200, 0, 14))], - ); + ) + .await; assert_eq!(reply, FromProxyd::Ok); assert_eq!(state.registry.live_count(), 1); assert!(state @@ -438,17 +645,19 @@ mod tests { /// a total egress lockout by the sync's replace half. The failed /// update leaves the existing registration live under its previous /// policy (the ADR 0111 "reduce, never strand" posture). - #[test] - fn sync_keeps_a_live_session_whose_updated_policy_is_untranslatable() { + #[tokio::test] + async fn sync_keeps_a_live_session_whose_updated_policy_is_untranslatable() { let state = state(); let session = SessionId::new(); let ip = Ipv4Addr::new(10, 200, 0, 18); - register_policy(&state.registry, policy(session, ip)).expect("initial apply"); + register_policy(&state.registry, policy(session, ip)) + .await + .expect("initial apply"); assert!(state.registry.lookup(ip).is_some()); let mut update = policy(session, ip); update.network_allow_host_patterns = vec!["[".into()]; // untranslatable - let reply = sync_policies(&state, vec![update]); + let reply = sync_policies(&state, vec![update]).await; assert_eq!(reply, FromProxyd::Ok); let survivor = state .registry @@ -466,26 +675,30 @@ mod tests { /// posture-mapping tests live) with a scoped allowlist must /// translate + register cleanly. Moved from `pooled_backend.rs` /// with the translation itself (ADR 0121). - #[test] - fn capture_shaped_allowlist_policy_registers() { + #[tokio::test] + async fn capture_shaped_allowlist_policy_registers() { let mut p = policy(SessionId::new(), Ipv4Addr::new(169, 254, 0, 2)); p.network_allow_hosts = vec!["accounts.google.com".into()]; p.network_allow_host_patterns = vec!["*.auth0.com".into()]; let registry = Registry::new(); - register_policy(®istry, p).expect("proxy must accept the capture-egress allowlist"); + register_policy(®istry, p) + .await + .expect("proxy must accept the capture-egress allowlist"); } /// A capture-shaped allow-all policy must register, and the proxy /// must bypass an arbitrary host under it (the dev posture for an /// image whose warm boot needs unrestricted network). - #[test] - fn capture_shaped_allow_all_policy_bypasses() { + #[tokio::test] + async fn capture_shaped_allow_all_policy_bypasses() { let mut p = policy(SessionId::new(), Ipv4Addr::new(169, 254, 0, 3)); p.network_allow_hosts = Vec::new(); p.allow_all = true; let guest_ip = p.guest_ip; let registry = Registry::new(); - register_policy(®istry, p).expect("register allow-all"); + register_policy(®istry, p) + .await + .expect("register allow-all"); let state = registry.lookup(guest_ip).expect("registered"); assert!(matches!( state.decide("anything.example.com"), @@ -493,11 +706,76 @@ mod tests { )); } - #[test] - fn sha256_hex_is_stable() { + #[tokio::test] + async fn sha256_hex_is_stable() { assert_eq!( sha256_hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" ); } + #[tokio::test] + async fn private_upgrade_uses_dedicated_host_uds_and_never_returns_final_receipt() { + use engram_egress_proto::private::{ + self as wire, Allocation, Operation, Reply, Request, VERSION, + }; + let state = Arc::new(state()); + let session = SessionId::new(); + let mut p = policy(session, "192.0.2.2".parse().unwrap()); + let managed = + engram_core::types::managed_deny::ManagedDenyPolicy::new(1, ["mcp.example".into()]) + .unwrap(); + p.managed_deny = Some(managed.clone()); + let a = Allocation { + version: VERSION, + host_id: state.host_id, + host_incarnation: "host-1".into(), + proxy_incarnation: state.hello.incarnation.clone(), + session_id: session, + sandbox_id: p.sandbox_id, + boot_attempt: "boot-1".into(), + attachment_generation: 1, + guest_ip: p.guest_ip, + gateway_ip: "192.0.2.1".parse().unwrap(), + policy_revision: 1, + policy_digest: managed.digest, + }; + register_policy(&state.registry, p).await.unwrap(); + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("private.sock"); + let listener = tokio::net::UnixListener::bind(&path).unwrap(); + let task = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + serve_conn(stream, state).await + }); + let client = tokio::net::UnixStream::connect(path).await.unwrap(); + let mut capability = wire::register(client, &a).await.unwrap().into_stream(); + wire::write( + &mut capability, + &Request { + version: VERSION, + sequence: 1, + epoch: 7, + operation: Operation::Bind, + }, + ) + .await + .unwrap(); + let ready: Reply = wire::read(&mut capability).await.unwrap(); + assert_eq!(ready.state, "ready"); + wire::write( + &mut capability, + &Request { + version: VERSION, + sequence: 2, + epoch: 7, + operation: Operation::Close, + }, + ) + .await + .unwrap(); + let closed: Reply = wire::read(&mut capability).await.unwrap(); + assert_eq!(closed.state, "closed"); + drop(capability); + task.await.unwrap().unwrap(); + } } diff --git a/crates/engram-egress-proxyd/src/lib.rs b/crates/engram-egress-proxyd/src/lib.rs index 0a281d49f..2932136a8 100644 --- a/crates/engram-egress-proxyd/src/lib.rs +++ b/crates/engram-egress-proxyd/src/lib.rs @@ -16,6 +16,7 @@ mod control; mod coord; mod dialback; +mod private; use std::path::PathBuf; use std::sync::Arc; @@ -181,6 +182,17 @@ pub async fn run(args: DaemonArgs) -> i32 { } }; + // Registration grants a live descriptor capability. Only the daemon owner + // can use this host-local control socket. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if std::fs::set_permissions(&control_sock, std::fs::Permissions::from_mode(0o600)).is_err() + { + return 1; + } + } + let listeners = match control::bind_with_retry(&proxy).await { Ok(l) => l, Err(e) => { @@ -193,6 +205,7 @@ pub async fn run(args: DaemonArgs) -> i32 { // durable-at-the-point-it's-known rule, PR #1383 / ADR 0121). let identity = engram_egress_proto::manifest::self_identity(); let hello = engram_egress_proto::HelloInfo { + incarnation: engram_core::SessionId::new().to_string(), proto_version: engram_egress_proto::PROTO_VERSION, source_fingerprint: SOURCE_FINGERPRINT.map(str::to_string), proxy_port: args.proxy_port, @@ -227,7 +240,11 @@ pub async fn run(args: DaemonArgs) -> i32 { ); let (shutdown_tx, mut shutdown_rx) = tokio::sync::watch::channel(false); + let private = Arc::new(private::Ingress::default()); let control_state = Arc::new(control::ControlState { + private: private.clone(), + proxy: Arc::new(proxy.clone()), + host_id: args.host_id, registry, gateway: guest_gateway, hello, @@ -235,7 +252,7 @@ pub async fn run(args: DaemonArgs) -> i32 { }); tokio::spawn(control::serve(control_listener, control_state)); - tokio::select! { + let exit = tokio::select! { _ = proxy.serve(listeners) => { // `serve` loops forever on accept. If it returns, the data // plane is dead while iptables still REDIRECTs here: exit @@ -249,7 +266,9 @@ pub async fn run(args: DaemonArgs) -> i32 { tracing::info!("graceful shutdown (control-socket Shutdown op)"); 0 } - } + }; + private.shutdown().await; + exit } /// cgroup-v2 self-migration (the ADR 0044 K2 escape, self-service diff --git a/crates/engram-egress-proxyd/src/private.rs b/crates/engram-egress-proxyd/src/private.rs new file mode 100644 index 000000000..69f500c04 --- /dev/null +++ b/crates/engram-egress-proxyd/src/private.rs @@ -0,0 +1,1138 @@ +//! Allocation-owned private ingress. No guest listener or general proxy API. +use engram_core::{HostId, SessionId}; +use engram_egress_proto::private::{self as wire, Allocation}; +use engram_egress_proxy::{Proxy, Registry}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use tokio::{ + net::UnixStream, + sync::{watch, Mutex}, +}; + +type Slot = Arc>; +#[derive(Default)] +struct SlotState { + live: Option, + last: Option<(String, u64)>, + incarnations: HashSet, +} +struct Live { + cancel: watch::Sender, + done: watch::Receiver, + allocation: Allocation, + handoff: tokio::sync::mpsc::Sender, +} +struct Handoff { + request: wire::ClosedPolicyHandoff, + reply: tokio::sync::oneshot::Sender>, +} +#[derive(Default)] +pub(crate) struct Ingress { + stopped: std::sync::atomic::AtomicBool, + slots: Mutex>, + used: Mutex>, +} +impl Ingress { + pub(crate) async fn handoff_closed( + &self, + request: wire::ClosedPolicyHandoff, + ) -> Result<(), String> { + request + .allocation + .validate() + .map_err(|_| "invalid closed allocation")?; + let slot = self + .slots + .lock() + .await + .get(&request.allocation.session_id) + .cloned() + .ok_or("closed allocation missing")?; + let sender = { + let slot = slot.lock().await; + let live = slot.live.as_ref().ok_or("closed allocation missing")?; + if live.allocation != request.allocation || *live.done.borrow() || *live.cancel.borrow() + { + return Err("closed allocation changed".into()); + } + live.handoff.clone() + }; + let (reply, result) = tokio::sync::oneshot::channel(); + sender + .send(Handoff { request, reply }) + .await + .map_err(|_| "closed allocation stopped")?; + result.await.map_err(|_| "closed policy handoff stopped")? + } + + pub(crate) async fn retire( + &self, + allocation: &Allocation, + registry: &Registry, + host: HostId, + incarnation: &str, + ) -> Result<(), String> { + allocation + .validate() + .map_err(|_| "invalid private allocation")?; + if allocation.host_id != host || allocation.proxy_incarnation != incarnation { + return Err("private allocation owner changed".into()); + } + let slot = self + .slots + .lock() + .await + .entry(allocation.session_id) + .or_insert_with(|| Arc::new(Mutex::new(SlotState::default()))) + .clone(); + // Registration takes this same lock before it enrolls a new allocation. + // Keep it until the registry's session lock has drained and removed the + // expected sandbox. Completion never deletes replay history. + let slot = slot.lock().await; + if slot + .live + .as_ref() + .is_some_and(|live| live.allocation != *allocation) + { + return Err("private allocation changed during retirement".into()); + } + registry + .retire_managed( + allocation.session_id, + allocation.sandbox_id, + allocation.guest_ip, + ) + .await + } + + pub(crate) async fn shutdown(&self) { + self.stopped + .store(true, std::sync::atomic::Ordering::Release); + let slots: Vec<_> = self.slots.lock().await.values().cloned().collect(); + for slot in slots { + if let Some(mut live) = slot.lock().await.live.take() { + live.cancel.send_replace(true); + while !*live.done.borrow_and_update() { + if live.done.changed().await.is_err() { + break; + } + } + } + } + } + + pub(crate) async fn serve( + &self, + mut stream: UnixStream, + allocation: Allocation, + registry: Arc, + proxy: Arc, + host: HostId, + incarnation: &str, + ) -> std::io::Result<()> { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use wire::{Operation, Reply, Request, VERSION}; + allocation.validate()?; + if self.stopped.load(std::sync::atomic::Ordering::Acquire) { + return Err(wire::invalid()); + } + if allocation.host_id != host || allocation.proxy_incarnation != incarnation { + return Err(wire::invalid()); + } + { + // Bounded replay tombstones live for this proxy incarnation. Once + // exhausted, fail closed until an operator replaces the daemon. + let mut used = self.used.lock().await; + if used.len() >= 4096 + || !used.insert(( + allocation.host_incarnation.clone(), + allocation.boot_attempt.clone(), + allocation.attachment_generation, + )) + { + return Err(wire::invalid()); + } + } + let slot = self + .slots + .lock() + .await + .entry(allocation.session_id) + .or_insert_with(|| Arc::new(Mutex::new(SlotState::default()))) + .clone(); + let mut slot = slot.lock().await; + if let Some((incarnation, generation)) = &slot.last { + if (incarnation == &allocation.host_incarnation + && allocation.attachment_generation <= *generation) + || (incarnation != &allocation.host_incarnation + && slot.incarnations.contains(&allocation.host_incarnation)) + { + return Err(wire::invalid()); + } + } + if let Some(mut old) = slot.live.take() { + old.cancel.send_replace(true); + while !*old.done.borrow_and_update() { + if old.done.changed().await.is_err() { + break; + } + } + } + // The session lease precedes all policy reads. Policy replacement + // cancels this allocation and waits for every task below to be joined. + let (mut session, lease) = registry + .enroll_session(allocation.session_id) + .ok_or_else(wire::invalid)?; + let policy = session.managed_deny.as_ref().ok_or_else(wire::invalid)?; + if session.sandbox_id != allocation.sandbox_id + || session.guest_ip != allocation.guest_ip + || policy.revision != allocation.policy_revision + || policy.digest != allocation.policy_digest + { + return Err(wire::invalid()); + } + let mut lease = Some(lease); + let (handoff, mut handoffs) = tokio::sync::mpsc::channel::(1); + let (cancel, mut cancelled) = watch::channel(false); + let (done, done_rx) = watch::channel(false); + if self.stopped.load(std::sync::atomic::Ordering::Acquire) { + return Err(wire::invalid()); + } + slot.last = Some(( + allocation.host_incarnation.clone(), + allocation.attachment_generation, + )); + slot.incarnations + .insert(allocation.host_incarnation.clone()); + slot.live = Some(Live { + cancel, + done: done_rx, + allocation: allocation.clone(), + handoff, + }); + drop(slot); + let mut tasks = tokio::task::JoinSet::new(); + let dns_budget = Arc::new(tokio::sync::Semaphore::new(wire::MAX_DNS_FLOWS)); + let operation = async { + wire::write( + &mut stream, + &Reply { + version: VERSION, + sequence: 0, + epoch: 0, + state: "closed".into(), + }, + ) + .await?; + let mut sequence = 0u64; + let mut bound_epoch = None; + let mut final_policy_digest: Option = None; + loop { + // A frame read must not be canceled when an unrelated flow + // completes: it may already have consumed part of the header. + let request: Request = { + let read = wire::read::<_, Request>(&mut stream); + tokio::pin!(read); + loop { + let current_lease = lease.as_mut().ok_or_else(wire::invalid)?; + tokio::select! { biased; + _ = current_lease.cancelled() => return Ok(None), + _ = cancelled.changed() => return Ok(None), + command = handoffs.recv() => { + let command = command.ok_or_else(wire::invalid)?; + if bound_epoch.is_some() || !tasks.is_empty() || command.request.allocation != allocation { + let _ = command.reply.send(Err("allocation is not closed".into())); + continue; + } + let digest = engram_core::types::managed_create::digest(&serde_json::to_vec(&command.request).map_err(|_| wire::invalid())?); + if let Some(prior) = &final_policy_digest { + let _ = command.reply.send(if prior == &digest { Ok(()) } else { Err("closed handoff changed".into()) }); + continue; + } + let candidate = crate::control::translate_policy(command.request.policy).map_err(|_| wire::invalid())?; + let outcome = tokio::select! { biased; + _ = cancelled.changed() => return Ok(None), + result = registry.handoff_closed_private(&session, &mut lease, candidate) => result, + }; + match outcome { + Ok(current) => { + session = current; + final_policy_digest = Some(digest); + let _ = command.reply.send(Ok(())); + } + Err(_) => { + let _ = command.reply.send(Err("closed policy handoff failed".into())); + return Err(wire::invalid()); + } + } + } + request = &mut read => { + match request { + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), + other => break other?, + } + } + } + } + }; + while tasks.try_join_next().is_some() {} + if request.version != VERSION + || sequence.checked_add(1) != Some(request.sequence) + || request.epoch == 0 + || request.epoch > 9_007_199_254_740_991 + { + return Err(wire::invalid()); + } + sequence = request.sequence; + let state = match request.operation { + Operation::Bind if bound_epoch.is_none() => { + bound_epoch = Some(request.epoch); + "ready" + } + Operation::Close if bound_epoch == Some(request.epoch) => { + return Ok(Some(Reply { + version: VERSION, + sequence, + epoch: request.epoch, + state: "closed".into(), + })) + } + Operation::Tcp { destination, port } | Operation::Dns { destination, port } => { + if bound_epoch != Some(request.epoch) + || port == 0 + || destination.is_unspecified() + || destination.is_multicast() + || tasks.len() >= wire::MAX_FLOWS + { + return Err(wire::invalid()); + } + let dns = matches!(request.operation, Operation::Dns { .. }); + if dns && (destination != allocation.gateway_ip || port != 53) { + return Err(wire::invalid()); + } + // Gateway DNS must use the controlled decoder, including TCP. + if !dns && destination == allocation.gateway_ip && port == 53 { + return Err(wire::invalid()); + } + let permit = if dns { + Some( + dns_budget + .clone() + .try_acquire_owned() + .map_err(|_| wire::invalid())?, + ) + } else { + None + }; + let mut fd = tokio::time::timeout( + std::time::Duration::from_secs(wire::HANDOFF_SECONDS), + wire::receive_stream(&stream), + ) + .await + .map_err(|_| wire::invalid())??; + let proxy = proxy.clone(); + let session = session.clone(); + tasks.spawn(async move { + let _permit = permit; + if dns { + let _ = tokio::time::timeout( + std::time::Duration::from_secs(10), + async { + let len = fd.read_u16().await? as usize; + if len == 0 || len > wire::MAX_DNS { + return Err(wire::invalid()); + } + let mut query = vec![0; len]; + fd.read_exact(&mut query).await?; + let response = + proxy.resolve_private(&query, &session).await?; + if response.len() > wire::MAX_DNS { + return Err(wire::invalid()); + } + fd.write_u16(response.len() as u16).await?; + fd.write_all(&response).await + }, + ) + .await; + } else { + let _ = proxy + .dispatch_private( + fd, + session, + std::net::SocketAddrV4::new(destination, port), + ) + .await; + } + }); + "accepted" + } + _ => return Err(wire::invalid()), + }; + wire::write( + &mut stream, + &Reply { + version: VERSION, + sequence, + epoch: request.epoch, + state: state.into(), + }, + ) + .await?; + } + }; + let result = operation.await; + // No task outlives the stable session lease. This includes TLS peeks, + // HTTP/2 pumps, DNS upstream sockets, and response writes. + tasks.shutdown().await; + let closed_reply = if let Ok(Some(reply)) = &result { + wire::write(&mut stream, reply).await + } else { + Ok(()) + }; + let shutdown = stream.shutdown().await; + // EOF on this reliable stream is the helper's cleanup barrier. The + // helper closes it only after packet admission and all work stop. + // A missing barrier must not turn a policy apply into a success ACK. + let barrier = tokio::time::timeout( + std::time::Duration::from_secs(wire::HANDOFF_SECONDS), + wire::drain_to_eof(&stream), + ) + .await; + let barrier_failed = + !matches!(barrier, Ok(Ok(()))) || shutdown.is_err() || closed_reply.is_err(); + if barrier_failed { + if let Some(lease) = &lease { + lease.poison(); + } + } + drop(stream); + drop(lease); + done.send_replace(true); + if barrier_failed { + return Err(wire::invalid()); + } + result.map(|_| ()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use engram_core::{types::managed_deny::ManagedDenyPolicy, SandboxId}; + use engram_egress_proxy::{Ca, CertMint, HostList, ProxyConfig, SessionState}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use wire::{Operation, Reply, Request, VERSION}; + struct Fixture { + ingress: Arc, + registry: Arc, + proxy: Arc, + allocation: Allocation, + ca: Arc, + } + impl Fixture { + async fn new() -> Self { + let _ = rustls::crypto::ring::default_provider().install_default(); + let registry = Arc::new(Registry::new()); + let policy = ManagedDenyPolicy::new(1, ["mcp.example".into()]).unwrap(); + let allocation = Allocation { + version: VERSION, + host_id: HostId::new(), + host_incarnation: "host-1".into(), + proxy_incarnation: "proxy-1".into(), + session_id: SessionId::new(), + sandbox_id: SandboxId::new(), + boot_attempt: "boot-1".into(), + attachment_generation: 1, + guest_ip: "192.0.2.2".parse().unwrap(), + gateway_ip: "192.0.2.1".parse().unwrap(), + policy_revision: 1, + policy_digest: policy.digest.clone(), + }; + registry + .apply(SessionState { + managed_deny: Some(policy), + session_id: allocation.session_id, + sandbox_id: allocation.sandbox_id, + guest_ip: allocation.guest_ip, + network_allow: HostList::from_manifest(&[], &[]).unwrap(), + allow_all: true, + secrets: vec![], + injects: vec![], + observes: vec![], + guest_services: vec![], + tunnels: vec![], + apps: vec![], + }) + .await + .unwrap(); + let tmp = tempfile::tempdir().unwrap(); + let ca = Arc::new(Ca::load_or_generate(tmp.path()).unwrap()); + let proxy = Arc::new(Proxy::new(ProxyConfig::new( + "127.0.0.1:0".parse().unwrap(), + registry.clone(), + Arc::new(CertMint::new(ca.clone())), + ))); + Self { + ingress: Arc::new(Ingress::default()), + registry, + proxy, + allocation, + ca, + } + } + fn start( + &self, + a: Allocation, + ) -> (UnixStream, tokio::task::JoinHandle>) { + let (client, server) = UnixStream::pair().unwrap(); + let ingress = self.ingress.clone(); + let reg = self.registry.clone(); + let proxy = self.proxy.clone(); + let host = self.allocation.host_id; + ( + client, + tokio::spawn( + async move { ingress.serve(server, a, reg, proxy, host, "proxy-1").await }, + ), + ) + } + } + async fn reply(c: &mut UnixStream) -> Reply { + tokio::time::timeout(std::time::Duration::from_secs(2), wire::read(c)) + .await + .unwrap() + .unwrap() + } + async fn request(c: &mut UnixStream, seq: u64, op: Operation) { + wire::write( + c, + &Request { + version: VERSION, + sequence: seq, + epoch: 7, + operation: op, + }, + ) + .await + .unwrap(); + } + async fn handoff(c: &UnixStream) -> UnixStream { + use std::os::fd::AsFd; + let (a, b) = std::os::unix::net::UnixStream::pair().unwrap(); + let control = c.as_fd().try_clone_to_owned().unwrap(); + engram_egress_proto::send_fd(&std::os::unix::net::UnixStream::from(control), b.as_fd()) + .unwrap(); + a.set_nonblocking(true).unwrap(); + UnixStream::from_std(a).unwrap() + } + fn final_policy(f: &Fixture) -> wire::ClosedPolicyHandoff { + use engram_core::types::{egress::SessionEgressPolicy, image::SecretMode}; + wire::ClosedPolicyHandoff { + allocation: f.allocation.clone(), + policy: SessionEgressPolicy { + session_id: f.allocation.session_id, + sandbox_id: f.allocation.sandbox_id, + guest_ip: f.allocation.guest_ip, + allow_all: false, + network_allow_hosts: vec!["allowed.example".into()], + network_allow_host_patterns: vec![], + secrets: vec![], + injects: vec![], + observes: vec![], + guest_services: vec![], + tunnels: vec![], + secret_mode: SecretMode::Literal, + apps: vec![], + managed_deny: Some( + ManagedDenyPolicy::new(2, ["mcp.example".into(), "new.example".into()]) + .unwrap(), + ), + }, + } + } + + #[tokio::test] + async fn retirement_checks_exact_attachment_and_waits_for_cleanup_eof() { + let f = Fixture::new().await; + let mut allocation = f.allocation.clone(); + allocation.attachment_generation = 2; + let (mut client, serving) = f.start(allocation.clone()); + assert_eq!(reply(&mut client).await.state, "closed"); + assert!(f + .ingress + .retire(&f.allocation, &f.registry, allocation.host_id, "proxy-1") + .await + .is_err()); + assert!(f.registry.lookup_session(allocation.session_id).is_some()); + let ingress = f.ingress.clone(); + let registry = f.registry.clone(); + let expected = allocation.clone(); + let mut retirement = tokio::spawn(async move { + ingress + .retire(&expected, ®istry, expected.host_id, "proxy-1") + .await + }); + // The real private handler closes its output only after its flow joins. + let mut bytes = Vec::new(); + client.read_to_end(&mut bytes).await.unwrap(); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(1), &mut retirement) + .await + .is_err() + ); + assert!(f.registry.lookup_session(allocation.session_id).is_none()); + drop(client); + serving.await.unwrap().unwrap(); + retirement.await.unwrap().unwrap(); + assert!(!f.registry.session_ids().contains(&allocation.session_id)); + // Exact retirement is idempotent, but never deletes replay history. + f.ingress + .retire(&allocation, &f.registry, allocation.host_id, "proxy-1") + .await + .unwrap(); + let (mut replay, rejected) = f.start(allocation); + assert!(wire::read::<_, Reply>(&mut replay).await.is_err()); + assert!(rejected.await.unwrap().is_err()); + } + + #[tokio::test] + async fn closed_policy_handoff_keeps_socket_and_requires_never_bound_identity() { + let f = Fixture::new().await; + let (mut c, server) = f.start(f.allocation.clone()); + assert_eq!(reply(&mut c).await.state, "closed"); + let original = f.registry.lookup(f.allocation.guest_ip).unwrap(); + let candidate = final_policy(&f); + let mut wrong = candidate.clone(); + wrong.allocation.boot_attempt = "other".into(); + assert!(f.ingress.handoff_closed(wrong).await.is_err()); + f.ingress.handoff_closed(candidate.clone()).await.unwrap(); + f.ingress.handoff_closed(candidate.clone()).await.unwrap(); + let current = f.registry.lookup(f.allocation.guest_ip).unwrap(); + assert!(!Arc::ptr_eq(&original, ¤t)); + assert!(!current.allow_all); + assert_eq!( + current.managed_deny.as_ref(), + candidate.policy.managed_deny.as_ref() + ); + request(&mut c, 1, Operation::Bind).await; + assert_eq!(reply(&mut c).await.state, "ready"); + assert!(f.ingress.handoff_closed(candidate).await.is_err()); + drop(c); + server.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn closed_policy_handoff_shutdown_poison_prevents_ack_or_reopen() { + tokio::time::timeout(std::time::Duration::from_secs(2), async { + let f = Fixture::new().await; + let (mut c, server) = f.start(f.allocation.clone()); + reply(&mut c).await; + let (_, mut retained) = f.registry.enroll(f.allocation.guest_ip).unwrap(); + let ingress = f.ingress.clone(); + let candidate = final_policy(&f); + let handoff = tokio::spawn(async move { ingress.handoff_closed(candidate).await }); + retained.cancelled().await; + assert!(!handoff.is_finished()); + let ingress = f.ingress.clone(); + let shutdown = tokio::spawn(async move { ingress.shutdown().await }); + assert_eq!(c.read(&mut [0]).await.unwrap(), 0); + drop(c); + drop(retained); + shutdown.await.unwrap(); + assert!(handoff.await.unwrap().is_err()); + server.await.unwrap().unwrap(); + assert!(f.registry.is_poisoned(f.allocation.session_id)); + assert!(f.registry.lookup(f.allocation.guest_ip).is_none()); + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn private_replacement_drains_pending_tls_before_ack_and_rejects_replay() { + let f = Fixture::new().await; + let (mut c, task) = f.start(f.allocation.clone()); + assert_eq!(reply(&mut c).await.state, "closed"); + request(&mut c, 1, Operation::Bind).await; + assert_eq!(reply(&mut c).await.state, "ready"); + request( + &mut c, + 2, + Operation::Tcp { + destination: "198.51.100.7".parse().unwrap(), + port: 8443, + }, + ) + .await; + let mut flow = handoff(&c).await; + assert_eq!(reply(&mut c).await.state, "accepted"); + flow.write_all(&[22, 3]).await.unwrap(); + let mut next = f.allocation.clone(); + next.attachment_generation = 2; + let closed = tokio::spawn(async move { + assert!(wire::read::<_, Reply>(&mut c).await.is_err()); + drop(c); + }); + let (mut new, next_task) = f.start(next); + assert_eq!(reply(&mut new).await.state, "closed"); + let mut b = [0]; + assert_eq!(flow.read(&mut b).await.unwrap(), 0); + closed.await.unwrap(); + assert!(task.await.unwrap().is_ok()); + let (mut replay, old) = f.start(f.allocation.clone()); + assert!(wire::read::<_, Reply>(&mut replay).await.is_err()); + assert!(old.await.unwrap().is_err()); + drop(new); + next_task.await.unwrap().unwrap(); + } + #[tokio::test] + async fn completed_flow_does_not_cancel_a_partial_control_header() { + let f = Fixture::new().await; + let (mut c, task) = f.start(f.allocation.clone()); + reply(&mut c).await; + request(&mut c, 1, Operation::Bind).await; + reply(&mut c).await; + request( + &mut c, + 2, + Operation::Dns { + destination: f.allocation.gateway_ip, + port: 53, + }, + ) + .await; + let mut flow = handoff(&c).await; + reply(&mut c).await; + let body = serde_json::to_vec(&Request { + version: VERSION, + sequence: 3, + epoch: 7, + operation: Operation::Close, + }) + .unwrap(); + let frame = [(body.len() as u32).to_be_bytes().as_slice(), &body].concat(); + c.write_all(&frame[..2]).await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + // Zero length completes the owned DNS decoder without a TLS peek + // deadline or an upstream dial. Complete it after the partial header. + flow.write_u16(0).await.unwrap(); + flow.shutdown().await.unwrap(); + let mut byte = [0]; + assert!(matches!(flow.read(&mut byte).await, Ok(0) | Err(_))); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + c.write_all(&frame[2..]).await.unwrap(); + assert_eq!(reply(&mut c).await.state, "closed"); + drop(c); + task.await.unwrap().unwrap(); + } + #[tokio::test] + async fn policy_apply_waits_for_helper_cleanup_eof() { + let f = Fixture::new().await; + let state = (*f.registry.lookup_session(f.allocation.session_id).unwrap()).clone(); + let (mut c, task) = f.start(f.allocation.clone()); + reply(&mut c).await; + let registry = f.registry.clone(); + let apply = tokio::spawn(async move { registry.apply(state).await }); + let mut byte = [0]; + assert_eq!(c.read(&mut byte).await.unwrap(), 0); + tokio::time::sleep(std::time::Duration::from_millis(30)).await; + assert!( + !apply.is_finished(), + "write-half EOF is a stop request, not the helper cleanup barrier" + ); + assert!(f.registry.lookup_session(f.allocation.session_id).is_none()); + drop(c); + task.await.unwrap().unwrap(); + apply.await.unwrap().unwrap(); + assert!(f.registry.lookup_session(f.allocation.session_id).is_some()); + } + + #[tokio::test] + async fn missing_helper_barrier_poison_is_session_local_and_cannot_be_replayed() { + let f = Fixture::new().await; + let state = (*f.registry.lookup_session(f.allocation.session_id).unwrap()).clone(); + let (mut c, task) = f.start(f.allocation.clone()); + reply(&mut c).await; + let registry = f.registry.clone(); + let old = state.clone(); + let apply = tokio::spawn(async move { registry.apply(old).await }); + let mut byte = [0]; + assert_eq!(c.read(&mut byte).await.unwrap(), 0); + // Keep the helper write half open. Another session must remain usable + // during this allocation's bounded cleanup wait. + let mut other = state.clone(); + other.session_id = SessionId::new(); + other.sandbox_id = SandboxId::new(); + other.guest_ip = "192.0.2.8".parse().unwrap(); + f.registry.apply(other.clone()).await.unwrap(); + assert!(f.registry.enroll_session(other.session_id).is_some()); + assert!( + tokio::time::timeout(std::time::Duration::from_secs(6), apply) + .await + .unwrap() + .unwrap() + .is_err() + ); + assert!(task.await.unwrap().is_err()); + assert!(f.registry.is_poisoned(state.session_id)); + f.registry.unregister(state.session_id); + assert!(f.registry.apply(state.clone()).await.is_err()); + assert!(f.registry.enroll_session(state.session_id).is_none()); + let mut next = f.allocation.clone(); + next.attachment_generation += 1; + let (mut rejected, next_task) = f.start(next); + assert!(wire::read::<_, Reply>(&mut rejected).await.is_err()); + assert!(next_task.await.unwrap().is_err()); + assert!(f.registry.enroll_session(other.session_id).is_some()); + drop(c); + } + #[tokio::test] + async fn policy_quarantine_joins_allocation_and_dns_without_global_block() { + let f = Fixture::new().await; + let (mut c, task) = f.start(f.allocation.clone()); + reply(&mut c).await; + request(&mut c, 1, Operation::Bind).await; + reply(&mut c).await; + request( + &mut c, + 2, + Operation::Dns { + destination: f.allocation.gateway_ip, + port: 53, + }, + ) + .await; + let mut flow = handoff(&c).await; + reply(&mut c).await; + flow.write_all(&[0, 128, 0]).await.unwrap(); + let closed = tokio::spawn(async move { + assert!(wire::read::<_, Reply>(&mut c).await.is_err()); + drop(c); + }); + tokio::time::timeout( + std::time::Duration::from_secs(2), + f.registry.quarantine_and_drain(f.allocation.session_id), + ) + .await + .unwrap(); + closed.await.unwrap(); + let mut b = [0]; + assert_eq!(flow.read(&mut b).await.unwrap(), 0); + task.await.unwrap().unwrap(); + } + #[tokio::test] + async fn wrong_identity_epoch_replay_and_non_tls_fail_closed() { + let f = Fixture::new().await; + for kind in 0..5 { + let mut bad = f.allocation.clone(); + bad.attachment_generation += kind + 1; + match kind { + 0 => bad.host_id = HostId::new(), + 1 => bad.proxy_incarnation = "proxy-old".into(), + 2 => bad.sandbox_id = SandboxId::new(), + 3 => bad.policy_revision += 1, + _ => bad.session_id = SessionId::new(), + } + let (mut c, task) = f.start(bad); + assert!(wire::read::<_, Reply>(&mut c).await.is_err()); + assert!(task.await.unwrap().is_err()); + } + let mut a = f.allocation.clone(); + a.attachment_generation = 10; + let (mut c, task) = f.start(a); + reply(&mut c).await; + request(&mut c, 1, Operation::Bind).await; + reply(&mut c).await; + request( + &mut c, + 2, + Operation::Tcp { + destination: "198.51.100.7".parse().unwrap(), + port: 80, + }, + ) + .await; + let mut flow = handoff(&c).await; + reply(&mut c).await; + flow.write_all(b"GET / HTTP/1.1\r\nHost: mcp.example\r\n\r\n") + .await + .unwrap(); + let mut bytes = [0; 64]; + assert!(matches!(flow.read(&mut bytes).await, Ok(0) | Err(_))); + request(&mut c, 2, Operation::Bind).await; + assert!(wire::read::<_, Reply>(&mut c).await.is_err()); + drop(c); + assert!(task.await.unwrap().is_err()); + } + #[tokio::test] + async fn private_tls_uses_deny_first_and_original_port_authority_without_credentials() { + let exercise = async { + for port in [443, 8443] { + for (host, authority, allowed) in [ + ("mcp.example", "mcp.example", false), + ("api.example", "mcp.example", false), + ("api.example", "api.example", true), + ] { + let mut f = Fixture::new().await; + let key = rcgen::KeyPair::generate().unwrap(); + let cert = rcgen::CertificateParams::new(vec!["api.example".into()]) + .unwrap() + .self_signed(&key) + .unwrap(); + let mut roots = rustls::RootCertStore::empty(); + roots.add(cert.der().clone()).unwrap(); + let server = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![cert.der().clone()], + rustls::pki_types::PrivateKeyDer::try_from(key.serialize_der()) + .unwrap(), + ) + .unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut cfg = ProxyConfig::new( + "127.0.0.1:0".parse().unwrap(), + f.registry.clone(), + Arc::new(CertMint::new(f.ca.clone())), + ); + cfg.upstream_test_roots = Some(roots); + cfg.resolver = Arc::new( + engram_egress_proxy::StaticResolver::new() + .with("api.example", listener.local_addr().unwrap()), + ); + f.proxy = Arc::new(Proxy::new(cfg)); + let (captured, mut received) = tokio::sync::mpsc::channel(1); + let upstream = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut tls = tokio_rustls::TlsAcceptor::from(Arc::new(server)) + .accept(socket) + .await + .unwrap(); + let mut bytes = vec![]; + let mut byte = [0]; + while tls.read(&mut byte).await.unwrap_or(0) > 0 { + bytes.push(byte[0]); + if bytes.ends_with(b"\r\n\r\n") { + break; + } + } + captured.send(bytes).await.unwrap(); + let _ = tls.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok").await; + let _ = tls.shutdown().await; + }); + let (mut control, task) = f.start(f.allocation.clone()); + reply(&mut control).await; + request(&mut control, 1, Operation::Bind).await; + reply(&mut control).await; + request( + &mut control, + 2, + Operation::Tcp { + destination: "198.51.100.7".parse().unwrap(), + port, + }, + ) + .await; + let flow = handoff(&control).await; + reply(&mut control).await; + let mut trust = rustls::RootCertStore::empty(); + for der in rustls_pemfile::certs(&mut f.ca.cert_pem.as_bytes()) { + trust.add(der.unwrap()).unwrap(); + } + let connector = tokio_rustls::TlsConnector::from(Arc::new( + rustls::ClientConfig::builder() + .with_root_certificates(trust) + .with_no_client_auth(), + )); + let guest = connector + .connect(host.to_string().try_into().unwrap(), flow) + .await; + if host == "mcp.example" { + assert!(guest.is_err()); + } else { + let mut guest = guest.unwrap(); + let effective = if port == 443 { + authority.to_string() + } else { + format!("{authority}:{port}") + }; + guest.write_all(format!("GET / HTTP/1.1\r\nHost: {effective}\r\nConnection: close\r\n\r\n").as_bytes()).await.unwrap(); + let mut out = vec![]; + let _ = guest.read_to_end(&mut out).await; + assert_eq!(out.starts_with(b"HTTP/1.1 200"), allowed); + let bytes = received.recv().await.unwrap(); + if allowed { + let text = String::from_utf8(bytes).unwrap(); + assert!(text.contains(&format!("Host: {effective}\r\n"))); + assert!(!text.to_lowercase().contains("authorization")); + } else { + assert!(bytes.is_empty()); + } + } + drop(control); + task.await.unwrap().unwrap(); + upstream.abort(); + let _ = upstream.await; + } + } + }; + tokio::time::timeout(std::time::Duration::from_secs(10), exercise) + .await + .expect("bounded real private TLS fixture"); + } + + #[tokio::test] + async fn unused_older_attachment_generation_cannot_replace_newer() { + let f = Fixture::new().await; + let mut newer = f.allocation.clone(); + newer.attachment_generation = 5; + let (mut current, task) = f.start(newer); + reply(&mut current).await; + let (mut older, replay) = f.start(f.allocation.clone()); + assert!(wire::read::<_, Reply>(&mut older).await.is_err()); + assert!(replay.await.unwrap().is_err()); + request(&mut current, 1, Operation::Bind).await; + assert_eq!(reply(&mut current).await.state, "ready"); + drop(current); + task.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn private_dns_uses_controlled_resolver_and_joins_upstream_on_revocation() { + for return_answer in [true, false] { + let mut f = Fixture::new().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut cfg = ProxyConfig::new( + "127.0.0.1:0".parse().unwrap(), + f.registry.clone(), + Arc::new(CertMint::new(f.ca.clone())), + ); + cfg.dns_upstream = listener.local_addr().unwrap(); + f.proxy = Arc::new(Proxy::new(cfg)); + let query = b"\x00\x07\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03api\x07example\x00\x00\x01\x00\x01"; + let (seen, received) = tokio::sync::oneshot::channel(); + let upstream = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let length = socket.read_u16().await.unwrap(); + let mut data = vec![0; length as usize]; + socket.read_exact(&mut data).await.unwrap(); + assert_eq!(data, query); + seen.send(()).unwrap(); + if return_answer { + data[2] |= 0x80; + socket.write_u16(length).await.unwrap(); + socket.write_all(&data).await.unwrap(); + } else { + let mut byte = [0]; + assert_eq!( + socket.read(&mut byte).await.unwrap(), + 0, + "revocation must close the controlled upstream" + ); + } + }); + let (mut c, task) = f.start(f.allocation.clone()); + reply(&mut c).await; + request(&mut c, 1, Operation::Bind).await; + reply(&mut c).await; + request( + &mut c, + 2, + Operation::Dns { + destination: f.allocation.gateway_ip, + port: 53, + }, + ) + .await; + let mut dns = handoff(&c).await; + reply(&mut c).await; + dns.write_u16(query.len() as u16).await.unwrap(); + dns.write_all(query).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(1), received) + .await + .unwrap() + .unwrap(); + if return_answer { + let length = dns.read_u16().await.unwrap(); + let mut response = vec![0; length as usize]; + dns.read_exact(&mut response).await.unwrap(); + let mut expected = query.to_vec(); + expected[2] |= 0x80; + assert_eq!(response, expected); + } + let helper = tokio::spawn(async move { + assert!(wire::read::<_, Reply>(&mut c).await.is_err()); + drop(c); + }); + tokio::time::timeout( + std::time::Duration::from_secs(2), + f.registry.quarantine_and_drain(f.allocation.session_id), + ) + .await + .unwrap(); + helper.await.unwrap(); + upstream.await.unwrap(); + task.await.unwrap().unwrap(); + } + } + #[tokio::test] + async fn two_allocations_keep_dns_identity_and_revocation_is_local() { + let f = Fixture::new().await; + let mut other = (*f.registry.lookup_session(f.allocation.session_id).unwrap()).clone(); + other.session_id = SessionId::new(); + other.sandbox_id = SandboxId::new(); + other.guest_ip = "192.0.2.6".parse().unwrap(); + other.managed_deny = Some(ManagedDenyPolicy::new(1, ["api.example".into()]).unwrap()); + let mut a = f.allocation.clone(); + a.session_id = other.session_id; + a.sandbox_id = other.sandbox_id; + a.guest_ip = other.guest_ip; + a.gateway_ip = "192.0.2.5".parse().unwrap(); + a.boot_attempt = "boot-2".into(); + a.policy_digest = other.managed_deny.as_ref().unwrap().digest.clone(); + f.registry.apply(other).await.unwrap(); + let (mut first, one) = f.start(f.allocation.clone()); + reply(&mut first).await; + let (mut second, two) = f.start(a.clone()); + reply(&mut second).await; + request(&mut second, 1, Operation::Bind).await; + reply(&mut second).await; + let first_closed = tokio::spawn(async move { + assert!(wire::read::<_, Reply>(&mut first).await.is_err()); + drop(first); + }); + f.registry + .quarantine_and_drain(f.allocation.session_id) + .await; + first_closed.await.unwrap(); + one.await.unwrap().unwrap(); + request( + &mut second, + 2, + Operation::Dns { + destination: a.gateway_ip, + port: 53, + }, + ) + .await; + let mut dns = handoff(&second).await; + reply(&mut second).await; + let query=b"\x00\x07\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03api\x07example\x00\x00\x01\x00\x01"; + dns.write_u16(query.len() as u16).await.unwrap(); + dns.write_all(query).await.unwrap(); + let size = dns.read_u16().await.unwrap(); + let mut answer = vec![0; size as usize]; + dns.read_exact(&mut answer).await.unwrap(); + assert_eq!( + answer[3] & 15, + 3, + "the second allocation denies api.example independently" + ); + request(&mut second, 3, Operation::Close).await; + assert_eq!(reply(&mut second).await.state, "closed"); + drop(second); + two.await.unwrap().unwrap(); + } +} diff --git a/crates/engram-egress-proxyd/tests/spawn_serve.rs b/crates/engram-egress-proxyd/tests/spawn_serve.rs index 198c76888..0be967ade 100644 --- a/crates/engram-egress-proxyd/tests/spawn_serve.rs +++ b/crates/engram-egress-proxyd/tests/spawn_serve.rs @@ -224,6 +224,7 @@ fn loopback_policy( session_id: engram_core::SessionId, ) -> engram_core::types::egress::SessionEgressPolicy { engram_core::types::egress::SessionEgressPolicy { + managed_deny: None, session_id, sandbox_id: engram_core::SandboxId::new(), guest_ip: std::net::Ipv4Addr::LOCALHOST, diff --git a/crates/engram-harness-claude/examples/linux_mcp_supervisor.rs b/crates/engram-harness-claude/examples/linux_mcp_supervisor.rs new file mode 100644 index 000000000..af23fb1de --- /dev/null +++ b/crates/engram-harness-claude/examples/linux_mcp_supervisor.rs @@ -0,0 +1,238 @@ +//! Test-only supervisor for the Linux MCP entrypoint gate. Cargo examples are +//! not production binaries and are not selected by default nextest discovery. +#[cfg(not(target_os = "linux"))] +fn main() { + eprintln!("Linux MCP supervisor requires Linux"); + std::process::exit(1); +} + +#[cfg(target_os = "linux")] +mod linux { + use std::fs; + use std::os::unix::{fs::PermissionsExt, process::ExitStatusExt}; + use std::path::{Path, PathBuf}; + use std::process::{Child, Command, ExitStatus, Stdio}; + use std::thread::sleep; + use std::time::{Duration, Instant}; + + use nix::errno::Errno; + use nix::sys::prctl::set_child_subreaper; + use nix::sys::signal::{kill, Signal}; + use nix::sys::wait::{waitpid, WaitPidFlag, WaitStatus}; + use nix::unistd::Pid; + + type Result = std::result::Result>; + const CLEANUP: Duration = Duration::from_secs(10); + const TICK: Duration = Duration::from_millis(10); + + fn private_dir(path: &Path) -> Result<()> { + fs::create_dir(path)?; + fs::set_permissions(path, fs::Permissions::from_mode(0o700))?; + Ok(()) + } + + fn driver(root: &Path) -> Result { + let binary = PathBuf::from(std::env::var("ENGRAM_TEST_DRIVER_BIN")?); + if !binary.is_absolute() || !binary.is_file() { + return Err("current test driver is required".into()); + } + let mut command = Command::new(binary); + command + .env_clear() + .env("ENGRAM_TEST_STATE_ROOT", root) + .args(["--ignored", "--test-threads=1"]) + .stdin(Stdio::null()); + for name in [ + "ENGRAM_TEST_CLAUDE_BIN", + "ENGRAM_TEST_CODEX_BIN", + "ENGRAM_FIXTURE_BIN", + ] { + command.env( + name, + std::env::var_os(name).ok_or("required artifact missing")?, + ); + } + Ok(command) + } + + fn wait_until(child: &mut Child, deadline: Instant) -> Result> { + loop { + if let Some(status) = child.try_wait()? { + return Ok(Some(status)); + } + if Instant::now() >= deadline { + return Ok(None); + } + sleep(TICK); + } + } + + fn kill_driver(child: &mut Child) -> Result { + child.kill()?; + wait_until(child, Instant::now() + CLEANUP)?.ok_or_else(|| "driver did not exit".into()) + } + + /// This program is single-threaded and creates only the owned test driver. + /// After its death, Linux reparents orphan descendants to this subreaper. + /// Kill only our kernel-reported direct children; no PID file or name scan + /// supplies kill targets. Unreaped children reserve their PID until waitpid. + fn drain_children() -> Result<()> { + let deadline = Instant::now() + CLEANUP; + let children = format!("/proc/self/task/{}/children", std::process::id()); + loop { + for value in fs::read_to_string(&children)?.split_whitespace() { + let pid = Pid::from_raw(value.parse()?); + match kill(pid, Signal::SIGKILL) { + Ok(()) | Err(Errno::ESRCH) => {} + Err(error) => return Err(error.into()), + } + } + loop { + match waitpid(None, Some(WaitPidFlag::WNOHANG)) { + Err(Errno::ECHILD) => return Ok(()), + Ok(WaitStatus::StillAlive) => break, + Ok(_) | Err(Errno::EINTR) => {} + Err(error) => return Err(error.into()), + } + } + // /proc children can omit entries during concurrent exits. Only + // ECHILD, never an empty snapshot, confirms complete cleanup. + if Instant::now() >= deadline { + return Err("owned descendants did not exit".into()); + } + sleep(TICK); + } + } + + fn stat(pid: u32) -> Result> { + let value = fs::read_to_string(format!("/proc/{pid}/stat"))?; + Ok(value + .rsplit_once(") ") + .ok_or("invalid process stat")? + .1 + .split_whitespace() + .map(str::to_owned) + .collect()) + } + + fn forced_deadline(root: &Path) -> Result<()> { + private_dir(root)?; + let mut child = driver(root)? + .args(["supervisor_timeout_fixture", "--exact", "--nocapture"]) + .spawn()?; + let ready = root.join("cleanup-ready"); + let deadline = Instant::now() + Duration::from_secs(20); + while !ready.exists() { + if child.try_wait()?.is_some() { + return Err("cleanup fixture exited before readiness".into()); + } + if Instant::now() >= deadline { + return Err("cleanup fixture readiness deadline".into()); + } + sleep(TICK); + } + let pairs: Vec<[u32; 2]> = serde_json::from_slice(&fs::read(&ready)?)?; + if pairs.len() != 2 { + return Err("both held wrappers are required".into()); + } + let mut observed = Vec::new(); + for [wrapper, fixture] in pairs { + let wrapper_stat = stat(wrapper)?; + let fixture_stat = stat(fixture)?; + if wrapper_stat[1].parse::()? != child.id() + || wrapper_stat[2].parse::()? != wrapper + || fixture_stat[1].parse::()? != wrapper + || fixture_stat[2].parse::()? != wrapper + { + return Err("cleanup fixture ownership or separate process group mismatch".into()); + } + // starttime distinguishes the observed child from later PID reuse. + observed.push((wrapper, wrapper_stat[19].clone())); + observed.push((fixture, fixture_stat[19].clone())); + } + // Negative control: reproduce the old deadline that killed only the + // test process. SIGKILL cannot run Rust Drop. Its separate groups live. + if wait_until(&mut child, Instant::now() + Duration::from_millis(100))?.is_some() { + return Err("cleanup fixture did not reach its forced deadline".into()); + } + let status = kill_driver(&mut child)?; + if status.signal() != Some(9) { + return Err("test driver was not SIGKILLed".into()); + } + for (pid, started) in &observed { + let current = stat(*pid)?; + if ¤t[19] != started || current[0] == "Z" { + return Err("negative control did not leave the owned descendant alive".into()); + } + } + if !ready.exists() { + return Err("state removed before descendant cleanup".into()); + } + println!( + "forced deadline control: killed test; four owned descendants alive; state retained" + ); + drain_children()?; + for (pid, started) in observed { + if let Ok(current) = stat(pid) { + if current[19] == started { + return Err("observed descendant remains after cleanup".into()); + } + } + } + if !ready.exists() { + return Err("state removed before cleanup confirmation".into()); + } + fs::remove_dir_all(root)?; + println!("forced deadline cleanup: passed; all descendants reaped before state removal"); + Ok(()) + } + + fn run(root: &Path) -> Result<()> { + forced_deadline(&root.join("probe"))?; + let matrix = root.join("matrix"); + private_dir(&matrix)?; + let mut child = driver(&matrix)? + .args(["--skip", "supervisor_timeout_fixture"]) + .spawn()?; + let status = match wait_until(&mut child, Instant::now() + Duration::from_secs(100))? { + Some(status) => status, + None => { + kill_driver(&mut child)?; + return Err("Linux wrapper matrix deadline".into()); + } + }; + if !status.success() { + return Err("Linux wrapper matrix failed".into()); + } + Ok(()) + } + + pub fn main() -> Result<()> { + if std::env::args().skip(1).collect::>() != ["--check-cleanup-and-run"] { + return Err("explicit cleanup check and matrix mode required".into()); + } + // Establish ownership BEFORE the test driver can spawn any descendants. + set_child_subreaper(true)?; + let base = PathBuf::from(std::env::var("ENGRAM_TEST_STATE_ROOT")?); + if !base.is_absolute() || base.canonicalize()? != base { + return Err("owned canonical state root required".into()); + } + let root = base.join("runtime"); + private_dir(&root)?; + let result = run(&root); + // Also runs on fixture/assertion failure. If cleanup cannot be proved, + // leave state and no marker; the shell's hard deadline cannot erase it. + drain_children()?; + fs::remove_dir_all(&root)?; + fs::write(base.join("cleanup-complete"), b"complete\n")?; + result + } +} + +#[cfg(target_os = "linux")] +fn main() { + if let Err(error) = linux::main() { + eprintln!("Linux wrapper supervisor failed: {error}"); + std::process::exit(1); + } +} diff --git a/crates/engram-harness-claude/src/main.rs b/crates/engram-harness-claude/src/main.rs index e5c9c4fa4..fe53182be 100644 --- a/crates/engram-harness-claude/src/main.rs +++ b/crates/engram-harness-claude/src/main.rs @@ -1,42 +1,25 @@ //! Harness adapter for the `claude` CLI. //! -//! Lives at `/sbin/engram-harness-claude` inside the rootfs (or -//! cargo-target/.../engram-harness-claude on the dev-mode -//! ProcessBackend path). Exec'd by `engram-agentd`'s harness -//! supervisor after the host pushes a `SpawnHarness` request -//! describing this binary. +//! `engram-agentd` starts this binary from a `SpawnHarness` request. +//! One persistent `claude --print --input-format stream-json +//! --output-format stream-json --verbose` process serves each session. +//! The adapter keeps stdin open and sends a user record for each prompt. +//! Explicit result records delimit turns; EOF during a turn triggers +//! crash recovery. Generated PreToolUse settings route native tool +//! requests through the harness. //! -//! Strategy: a single **persistent streaming** `claude` process per -//! session. The wrapper spawns `claude --print --input-format -//! stream-json --output-format stream-json --verbose -//! --dangerously-skip-permissions [--resume ]` ONCE, holds -//! its stdin open, and writes one newline-delimited `user` message per -//! prompt. Turn boundaries come from claude's explicit `result` line on -//! the stream — NOT from process EOF — so a clean turn-end and a crash -//! are now unambiguous (EOF mid-turn = a real crash, auto-recovered by -//! respawning `--resume`). This is the root-cause fix for the -//! child-per-prompt EOF-inference wedge (ADR 0052). -//! -//! `--dangerously-skip-permissions` is mandatory: there's no human in -//! the VM to answer permission prompts, and `--print` mode aborts -//! with exit 1 the first time a tool needs approval otherwise. The -//! sandbox is the safety boundary, not Claude's per-tool consent. -//! -//! Each turn's `system`/`init` line carries Claude's session id, which -//! we stash in `/workspace/.engrams/claude-session-id` so a respawn -//! (idle-resume or crash recovery) can `--resume` into the same -//! conversation. Lost on cold death (Dead status); a forked session -//! gets a fresh Claude conversation. +//! The system/init record supplies Claude's conversation ID. The adapter +//! saves it in `/workspace/.engrams/claude-session-id` for `--resume`. // Cross-platform stub — vsock dialing is Linux-only, and the // adapter only ships inside FC rootfs / Linux ProcessBackend. -#[cfg(not(target_os = "linux"))] +#[cfg(all(not(target_os = "linux"), not(test)))] fn main() { eprintln!("engram-harness-claude is Linux-only"); std::process::exit(1); } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", test))] mod adapter { use std::collections::{HashMap, HashSet, VecDeque}; use std::os::unix::process::ExitStatusExt; @@ -47,6 +30,7 @@ mod adapter { use clap::Parser; use engram_core::SessionId; + use engram_harness_proto::tool_result::{McpToolResult, ResultFormat}; use engram_harness_proto::{ AgentRole, EditHunk, FileChange, HarnessCommand, HarnessEvent, MAX_FILE_CHANGE_BYTES, }; @@ -90,6 +74,8 @@ mod adapter { pub input_schema: Value, pub execution: ToolExecution, #[serde(default)] + pub result_format: ResultFormat, + #[serde(default)] pub native_bindings: NativeBindings, } @@ -134,6 +120,7 @@ mod adapter { description: browser_view::TOOL_DESCRIPTION.into(), input_schema: browser_view::input_schema(), execution: ToolExecution::Sync, + result_format: ResultFormat::Json, native_bindings: NativeBindings::default(), }); } @@ -320,12 +307,8 @@ mod adapter { #[arg(long, default_value_t = 600)] pub max_tool_call_secs: u64, - /// Whole-run wall-clock cap (seconds). After this the - /// adapter SIGTERMs `claude` and emits - /// `RunCompleted{ok:false}`. Default is 24h: an autonomous - /// agent in an isolated VM is expected to be able to grind - /// on a task for many hours without the wrapper killing it - /// out from under it. Tighten per-deployment if needed. + /// Whole-run wall-clock cap (seconds). On expiry, kill Claude + /// and report a failed completion. Default is 24 hours. #[arg(long, default_value_t = 86_400)] pub max_run_secs: u64, @@ -462,10 +445,9 @@ mod adapter { event_rx, reattach, } = engram_harness_sdk::Channels::new(); - // Issue #535 (d): no more env-seeded initial prompt — every prompt, - // first or follow-up, arrives as a `HarnessCommand::Prompt` frame - // over the same connection this loop dials. The engine starts with - // an empty pending queue and waits. + // The engine reads the create-time initial prompt from its spawn + // environment once. Later prompts arrive as `HarnessCommand::Prompt` + // frames; prompt-id deduplication also covers initial-prompt replays. let engine = tokio::spawn(run_engine( cli.clone(), @@ -819,18 +801,18 @@ mod adapter { // see `take_stashed_result`). HookVerdict::Allow } - Some(_) => { - // Claude #64389 can issue the same logical deferred - // call twice with distinct ids in one turn. Decide and - // insert under one lock so concurrent hook connections - // cannot both become host-visible. + Some(tool) => { + // External calls have distinct journal identities even + // when their names and arguments match. The same-name + // workaround applies only to internal deferred tools. let (first_request, duplicate) = { let mut calls = deferred_calls.lock().await; if calls.contains_key(&req.tool_use_id) { (false, false) - } else if calls - .values() - .any(|call| call.run_id == run_id && call.tool_name == name) + } else if tool.result_format != super::ResultFormat::Mcp + && calls + .values() + .any(|call| call.run_id == run_id && call.tool_name == name) { (false, true) } else { @@ -1267,14 +1249,13 @@ mod adapter { } } - /// ADR 0089 P2: disposable stdio MCP frontend. Claude owns this process's - /// lifetime, so it keeps no correlation state: initialize and tools/list - /// are answered from the inherited manifest, while each tools/call blocks - /// on the long-lived harness main process over a one-line unix-socket - /// request/response. + /// Stdio MCP frontend owned by Claude. It answers discovery from the + /// inherited manifest and tracks pending request IDs for cancellation. + /// Tool calls use the long-lived harness over a Unix socket. pub mod mcp_bridge { use super::{ - browser_view, injected_tools, manifest_from_env, BufReader, StateDir, ToolManifest, + browser_view, injected_tools, manifest_from_env, BufReader, McpToolResult, + ResultFormat, StateDir, ToolManifest, }; use serde::{Deserialize, Serialize}; use serde_json::Value; @@ -1301,27 +1282,86 @@ mod adapter { .filter(|value| !value.is_empty()) .map(PathBuf::from) .unwrap_or_else(|| StateDir::from_env().mcp_sock()); - let mut lines = BufReader::new(tokio::io::stdin()).lines(); - let mut stdout = tokio::io::stdout(); + serve_io( + BufReader::new(tokio::io::stdin()), + tokio::io::stdout(), + &manifest, + &socket_path, + ) + .await + } + + pub async fn serve_io( + reader: R, + mut stdout: W, + manifest: &ToolManifest, + socket_path: &Path, + ) -> std::process::ExitCode { + let mut lines = reader.lines(); + let mut calls = tokio::task::JoinSet::new(); + let mut pending: std::collections::HashMap = + std::collections::HashMap::new(); loop { - let line = match lines.next_line().await { - Ok(Some(line)) => line, - Ok(None) => return std::process::ExitCode::SUCCESS, - Err(_) => return std::process::ExitCode::from(1), - }; - let request: Value = match serde_json::from_str(&line) { - Ok(request) => request, - Err(_) => { - let response = json_rpc_error(Value::Null, -32700, "Parse error"); - if write_response(&mut stdout, &response).await.is_err() { - return std::process::ExitCode::from(1); + tokio::select! { + completed = calls.join_next_with_id(), if !calls.is_empty() => { + match completed { + Some(Ok((task_id, (key, response)))) => { + // Cancellation and request-id reuse can race a completed task. + if pending.get(&key).is_some_and(|handle| handle.id() == task_id) { + pending.remove(&key); + if let Some(response) = response { + if write_response(&mut stdout, &response).await.is_err() { + return std::process::ExitCode::FAILURE; + } + } + } + } + Some(Err(error)) => pending.retain(|_, handle| handle.id() != error.id()), + None => {} } - continue; } - }; - if let Some(response) = handle_request(request, &manifest, &socket_path).await { - if write_response(&mut stdout, &response).await.is_err() { - return std::process::ExitCode::from(1); + line = lines.next_line() => { + let line = match line { + Ok(Some(line)) => line, + Ok(None) => { + calls.shutdown().await; + return std::process::ExitCode::SUCCESS; + } + Err(_) => return std::process::ExitCode::FAILURE, + }; + let request: Value = match serde_json::from_str(&line) { + Ok(request) => request, + Err(_) => { + if write_response(&mut stdout, &json_rpc_error(Value::Null, -32700, "Parse error")).await.is_err() { + return std::process::ExitCode::FAILURE; + } + continue; + } + }; + if request.get("id").is_none() { + if request.get("method").and_then(Value::as_str) == Some("notifications/cancelled") { + if let Some(id) = request.pointer("/params/requestId") { + if let Some(handle) = pending.remove(&id.to_string()) { handle.abort(); } + } + } + continue; + } + let id = request["id"].clone(); + let key = id.to_string(); + if pending.contains_key(&key) || pending.len() >= 128 { + if write_response(&mut stdout, &json_rpc_error(id, -32600, "Duplicate request or too many pending calls.")).await.is_err() { + return std::process::ExitCode::FAILURE; + } + continue; + } + let manifest = manifest.clone(); + let socket_path = socket_path.to_path_buf(); + let request_key = key.clone(); + let handle = calls.spawn(async move { + let response = handle_request(request, &manifest, &socket_path).await; + (request_key, response) + }); + pending.insert(key, handle); } } } @@ -1424,6 +1464,12 @@ mod adapter { .pointer("/params/_meta/claudecode~1toolUseId") .and_then(Value::as_str); match round_trip(socket_path, name, args, tool_use_id).await { + Ok(result_json) if tool.result_format == ResultFormat::Mcp => { + Some(match McpToolResult::parse(&result_json) { + Ok(result) => result.claude_response(id), + Err(message) => json_rpc_error(id, -32603, message), + }) + } Ok(result_json) => Some(json_rpc_result( id, serde_json::json!({ @@ -1482,13 +1528,11 @@ mod adapter { } } - /// ADR 0089 P2: long-lived MCP server half owned by `run_engine`. Each - /// bridge connection represents one blocking MCP tools/call. A result - /// already in hand is consumed immediately (the deferred re-fire path); - /// otherwise the connection is parked in a oneshot table and the generic - /// request event is emitted exactly once. + /// Long-lived MCP server owned by `run_engine`. Pending connections park + /// in a oneshot table and emit requests. A new socket can replace the + /// same call's owner; a stashed result returns without a new request. pub mod mcp_server { - use super::{emit, hook_server, BufReader, HarnessEvent}; + use super::{emit, hook_server, BufReader, HarnessEvent, ResultFormat, ToolManifest}; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::HashMap; @@ -1498,12 +1542,20 @@ mod adapter { use tokio::sync::{mpsc, oneshot, Mutex}; pub type ResultsInHand = Arc>>; - pub type ParkedCalls = Arc>>>; + pub type ParkedCalls = Arc>>; + + pub struct PendingCall { + socket_owner: Arc<()>, + pub tool_name: String, + pub result_format: ResultFormat, + pub sender: Option>, + } #[derive(Clone)] pub struct State { pub results_in_hand: ResultsInHand, pub parked_calls: ParkedCalls, + pub manifest: Arc, pub deferred_calls: hook_server::DeferredCalls, pub current_run_id: hook_server::CurrentRunId, pub evt_tx: mpsc::Sender, @@ -1618,12 +1670,14 @@ mod adapter { let State { results_in_hand, parked_calls, + manifest, deferred_calls, current_run_id, evt_tx, } = state; let (r, mut w) = stream.into_split(); - let Some(line) = BufReader::new(r).lines().next_line().await.ok().flatten() else { + let mut lines = BufReader::new(r).lines(); + let Some(line) = lines.next_line().await.ok().flatten() else { return; }; let request: McpCallRequest = match serde_json::from_str(&line) { @@ -1638,6 +1692,7 @@ mod adapter { .filter(|id| !id.is_empty()) .unwrap_or_else(|| format!("toolu-engram-{}", uuid::Uuid::new_v4())); + let socket_owner = Arc::new(()); // Both this insertion and `route_tool_result` lock parked→results. // Holding the parked lock across the stash check closes the only // lost-wakeup window: a result can neither slip between the check @@ -1652,6 +1707,8 @@ mod adapter { calls.remove(&call_id); calls.remove(&stashed_id); drop(calls); + parked.remove(&call_id); + parked.remove(&stashed_id); drop(parked); if stashed_id != call_id { // The stash was keyed by an EARLIER call id (the @@ -1676,7 +1733,19 @@ mod adapter { return; } let (tx, rx) = oneshot::channel(); - parked.insert(call_id.clone(), tx); + parked.insert( + call_id.clone(), + PendingCall { + socket_owner: socket_owner.clone(), + tool_name: request.name.clone(), + result_format: manifest + .iter() + .find(|tool| tool.name == request.name) + .map(|tool| tool.result_format) + .unwrap_or_default(), + sender: Some(tx), + }, + ); rx }; @@ -1684,7 +1753,7 @@ mod adapter { emit( &evt_tx, HarnessEvent::ToolCallRequested { - run_id, + run_id: run_id.clone(), call_id: call_id.clone(), name: request.name.clone(), args_json: request.args.to_string(), @@ -1692,11 +1761,49 @@ mod adapter { ) .await; - match rx.await { - Ok(result_json) => { - write_result(&mut w, render_result_for_model(&request.name, result_json)).await + tokio::select! { + result = rx => match result { + Ok(result_json) => write_result(&mut w, render_result_for_model(&request.name, result_json)).await, + Err(_) => tracing::debug!(%call_id, "parked MCP call was canceled"), + }, + _ = lines.next_line() => { + cancel_disconnected_call(&parked_calls, &socket_owner, &evt_tx, + run_id, call_id, request.name).await; } - Err(_) => tracing::debug!(%call_id, "parked MCP call was cancelled"), + } + } + + async fn cancel_disconnected_call( + parked_calls: &ParkedCalls, + socket_owner: &Arc<()>, + evt_tx: &mpsc::Sender, + run_id: String, + call_id: String, + name: String, + ) { + let canceled = { + let mut calls = parked_calls.lock().await; + calls + .get_mut(&call_id) + .filter(|call| Arc::ptr_eq(&call.socket_owner, socket_owner)) + // Keep identity after disconnect for late host results. + .is_some_and(|call| { + let disconnected = call.sender.take().is_some(); + disconnected && call.result_format == ResultFormat::Mcp + }) + }; + // A re-fire can replace this ID while the old socket waits for + // the map lock. Only the socket that still owns it may cancel. + if canceled { + emit( + evt_tx, + HarnessEvent::ToolCallCancelRequested { + run_id, + call_id, + name, + }, + ) + .await; } } @@ -1712,8 +1819,9 @@ mod adapter { let _ = writer.flush().await; } - /// Route a host result to a currently parked bridge, or durably stash - /// it for a later id-stable re-fire. Returns true only when a live + /// Route a host result to a currently parked bridge, or stash it in + /// memory for a later id-stable re-fire. The host outbox owns durability. + /// Returns true only when a live /// parked connection accepted the result. pub async fn route_tool_result( call_id: &str, @@ -1722,9 +1830,12 @@ mod adapter { results_in_hand: &ResultsInHand, ) -> bool { let mut parked = parked_calls.lock().await; - if let Some(sender) = parked.remove(call_id) { + if let Some(sender) = parked.get_mut(call_id).and_then(|call| call.sender.take()) { match sender.send(result_json) { - Ok(()) => return true, + Ok(()) => { + parked.remove(call_id); + return true; + } Err(result_json) => { results_in_hand .lock() @@ -1740,6 +1851,95 @@ mod adapter { .insert(call_id.to_string(), result_json); false } + #[cfg(test)] + mod socket_ownership_tests { + use super::*; + + #[tokio::test] + async fn stale_socket_disconnect_cannot_cancel_replacement_call() { + let (evt_tx, mut events) = mpsc::channel(8); + let state = State { + results_in_hand: Arc::new(Mutex::new(HashMap::new())), + parked_calls: Arc::new(Mutex::new(HashMap::new())), + manifest: Arc::new(super::super::parse_tool_manifest(r#"[{"name":"remote_inventory","description":"Read","inputSchema":{"type":"object"},"execution":"sync","nativeBindings":{},"resultFormat":"mcp"}]"#).unwrap()), + deferred_calls: Arc::new(Mutex::new(HashMap::new())), + current_run_id: Arc::new(Mutex::new(Some("run-1".into()))), + evt_tx, + }; + let request = + b"{\"name\":\"remote_inventory\",\"tool_use_id\":\"same-call\",\"args\":{}}\n"; + let (socket_a, mut client_a) = UnixStream::pair().unwrap(); + client_a.write_all(request).await.unwrap(); + let mut a = std::pin::pin!(handle_one(socket_a, state.clone())); + tokio::select! { + () = &mut a => panic!("socket A must remain pending"), + event = events.recv() => assert!(matches!(event, Some(HarnessEvent::ToolCallRequested { .. }))), + } + let owner_a = state.parked_calls.lock().await["same-call"] + .socket_owner + .clone(); + + // Pause A before its EOF cleanup. B uses the same stable ID, + // then the exact production cleanup step for A runs late. + drop(client_a); + let (socket_b, mut client_b) = UnixStream::pair().unwrap(); + client_b.write_all(request).await.unwrap(); + let mut b = std::pin::pin!(handle_one(socket_b, state.clone())); + tokio::select! { + () = &mut b => panic!("socket B must remain pending"), + event = events.recv() => assert!(matches!(event, Some(HarnessEvent::ToolCallRequested { .. }))), + } + { + let calls = state.parked_calls.lock().await; + let replacement = &calls["same-call"]; + assert!(!Arc::ptr_eq(&replacement.socket_owner, &owner_a)); + assert_eq!(replacement.tool_name, "remote_inventory"); + assert_eq!(replacement.result_format, ResultFormat::Mcp); + } + cancel_disconnected_call( + &state.parked_calls, + &owner_a, + &state.evt_tx, + "run-1".into(), + "same-call".into(), + "remote_inventory".into(), + ) + .await; + tokio::time::timeout(std::time::Duration::from_secs(1), &mut a) + .await + .unwrap(); + let result = r#"{"outcome":"succeeded","delivery":"available","output":{"content":[{"type":"text","text":"replacement result"}]}}"#; + assert!( + route_tool_result( + "same-call", + result.into(), + &state.parked_calls, + &state.results_in_hand + ) + .await, + "the replacement socket must still accept its result" + ); + tokio::time::timeout(std::time::Duration::from_secs(1), &mut b) + .await + .unwrap(); + let response = BufReader::new(client_b) + .lines() + .next_line() + .await + .unwrap() + .unwrap(); + assert_eq!( + serde_json::from_str::(&response).unwrap()["result_json"], + result + ); + assert!( + matches!(events.try_recv(), Err(mpsc::error::TryRecvError::Empty)), + "stale cleanup must not emit cancellation for B" + ); + assert!(state.parked_calls.lock().await.is_empty()); + assert!(state.results_in_hand.lock().await.is_empty()); + } + } } /// Aborts the hook-socket accept task and unlinks the socket file when @@ -1809,24 +2009,8 @@ mod adapter { tokio::fs::write(tracker, format!("{}\n", cwd.display())).await } - /// Write claude's strict MCP config when the manifest contains at least - /// one tool that is not natively bound to Claude. Removing a stale file on - /// the empty path matters on ProcessBackend, where `/workspace` can - /// survive a harness restart with a different manifest. - async fn write_mcp_config( - path: &Path, - manifest: &ToolManifest, - self_exe: &Path, - ) -> std::io::Result { - if injected_tools(manifest).next().is_none() { - match tokio::fs::remove_file(path).await { - Ok(()) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(e), - } - return Ok(false); - } - + /// Always limit Claude to the trusted bridge, including an empty manifest. + async fn write_mcp_config(path: &Path, self_exe: &Path) -> std::io::Result<()> { if let Some(parent) = path.parent() { tokio::fs::create_dir_all(parent).await?; } @@ -1858,8 +2042,7 @@ mod adapter { path, serde_json::to_vec(&config).expect("MCP config contains only serializable values"), ) - .await?; - Ok(true) + .await } /// The long-lived run engine. Owns the persistent `claude` process @@ -1967,20 +2150,11 @@ mod adapter { let self_exe = std::env::current_exe().unwrap_or_else(|_| PathBuf::from("engram-harness-claude")); let mcp_config_file = state.claude_mcp_config(); - let mcp_config_path = match write_mcp_config( - &mcp_config_file, - &cli.tool_manifest, - &self_exe, - ) - .await - { - Ok(true) => Some(mcp_config_file.as_path()), - Ok(false) => None, - Err(e) => { - tracing::error!(error = %e, "couldn't write claude MCP config; exposing no injected tools"); - None - } - }; + if let Err(error) = write_mcp_config(&mcp_config_file, &self_exe).await { + tracing::error!(%error, "could not write strict Claude MCP config"); + return ExitCode::FAILURE; + } + let mcp_config_path = mcp_config_file.as_path(); let hook_sock_path = state.hook_sock(); let _ = tokio::fs::remove_file(&hook_sock_path).await; // clear a stale bind let _sock_guard = match tokio::net::UnixListener::bind(&hook_sock_path) { @@ -2017,6 +2191,7 @@ mod adapter { mcp_server::State { results_in_hand: results_in_hand.clone(), parked_calls: parked_mcp_calls.clone(), + manifest: tool_manifest.clone(), deferred_calls: deferred_calls.clone(), current_run_id: current_run_id.clone(), evt_tx: evt_tx.clone(), @@ -2200,6 +2375,28 @@ mod adapter { ] } + /// Deferred completion grants dispatch authority only to MCP-format calls. + fn unresolved_deferred_ids( + run_id: &str, + calls: &HashMap, + results: &HashMap, + manifest: &[ManifestTool], + ) -> Vec { + let mut ids: Vec<_> = calls + .iter() + .filter(|(id, call)| { + call.run_id == run_id + && !results.contains_key(*id) + && manifest.iter().any(|tool| { + tool.name == call.tool_name && tool.result_format == ResultFormat::Mcp + }) + }) + .map(|(id, _)| id.clone()) + .collect(); + ids.sort(); + ids + } + /// The single in-flight turn, owned by the session loop. `run_id` is /// minted on prompt-accept — BEFORE any claude output — so no event /// can be emitted without a valid `run_id`, and a bare-event desync @@ -2258,12 +2455,10 @@ mod adapter { /// these in memory and writes one to claude's stdin only at the /// consumption boundary (the running turn's `result`). `prompt_id` /// carries the client/coord-minted id used to correlate the queue - /// events and the eventual `RunStarted{prompt_id}` — issue #535 (d): - /// every prompt, first or follow-up, now arrives via `HarnessCommand:: - /// Prompt`, so this is always `Some` in practice (the historical `None` - /// case was the env-seeded initial prompt, deleted). Kept `Option` - /// rather than force-unwrapping at each read site — a defensive `None` - /// costs nothing here and this isn't a wire type. + /// events and the eventual `RunStarted{prompt_id}`. Both the create-time + /// prompt read from the spawn environment and host-delivered prompts + /// carry an id. The optional form allows callers to handle an absent id + /// without force-unwrapping it. struct QueuedPrompt { prompt_id: Option, text: String, @@ -2352,7 +2547,7 @@ mod adapter { // `run_engine` so it survives a respawn; populated at turn-end here. scrub_msg_ids: &Arc>>, duplicate_deferred_ids: &hook_server::DuplicateDeferredIds, - mcp_config_path: Option<&Path>, + mcp_config_path: &Path, ) -> SessionOutcome { // A socket call outside a turn must carry the wire-mandated empty // run_id, never the previous turn's id. @@ -2767,11 +2962,37 @@ mod adapter { interrupted_run = None; interrupt_deadline = None; } + // Only the hook ledger can establish a deferred request. + // The model's terminal marker and transcript are not authority. + let mut deferred_ids = Vec::new(); + if !abort_requested + && !marker.is_error + && (tool_deferred || narrated_past) + && pending.is_empty() + && bg_agents == 0 + { + let results = results_in_hand.lock().await; + if !is_delivery_resume || results.is_empty() { + let calls = deferred_calls.lock().await; + deferred_ids = unresolved_deferred_ids(&run_id, &calls, &results, &cli.tool_manifest); + } + } + let invalid_deferred_ids = deferred_ids.len() > 128 + || deferred_ids.iter().any(|id| id.is_empty() || id.len() > 256); if abort_requested && marker.subtype == "error_during_execution" { // The `control_request` abort landed. emit(evt_tx, HarnessEvent::RunInterrupted { run_id }).await; + } else if !deferred_ids.is_empty() && !invalid_deferred_ids { + emit( + evt_tx, + HarnessEvent::RunDeferred { + run_id, + deferred_tool_call_ids: deferred_ids, + }, + ) + .await; } else { // Either no abort was requested, or // the turn COMPLETED in the @@ -2785,7 +3006,7 @@ mod adapter { evt_tx, HarnessEvent::RunCompleted { run_id, - ok: !marker.is_error, + ok: !marker.is_error && !invalid_deferred_ids, }, ) .await; @@ -2814,6 +3035,7 @@ mod adapter { if stale_results.is_empty() { HashMap::new() } else { + let mut pending = parked_mcp_calls.lock().await; let mut deferred = deferred_calls.lock().await; stale_results .iter() @@ -2821,6 +3043,7 @@ mod adapter { let tool_name = deferred .remove(call_id) .map(|call| call.tool_name) + .or_else(|| pending.remove(call_id).map(|call| call.tool_name)) .unwrap_or_default(); (call_id.clone(), tool_name) }) @@ -2831,6 +3054,7 @@ mod adapter { let text = fallback_delivery_message( &stale_results, &stale_result_names, + &cli.tool_manifest, ); let ft = start_turn( evt_tx, s, cli, None, &text, current_run_id, @@ -2844,7 +3068,9 @@ mod adapter { // each drained result explicitly or // its coordinator outbox row will // redeliver forever. - for (tool_call_id, _) in stale_results { + for (tool_call_id, result_json) in stale_results { + let ok = external_fallback_result(&tool_call_id, &result_json, &stale_result_names, &cli.tool_manifest) + .is_none_or(|result| result.is_ok_and(|result| result.success())); emit( evt_tx, HarnessEvent::ToolCallCompleted { @@ -2854,7 +3080,7 @@ mod adapter { .cloned() .unwrap_or_default(), tool_call_id, - ok: true, + ok, duration_ms: 0, result_summary: None, }, @@ -3179,6 +3405,12 @@ mod adapter { .await .get(&call_id) .map(|call| call.tool_name.clone()); + let known_deferred = known_tool.is_some(); + let live_context = parked_mcp_calls.lock().await.get(&call_id) + .map(|call| (call.tool_name.clone(), call.result_format)); + let is_mcp = live_context.as_ref().is_some_and(|(_, format)| *format == ResultFormat::Mcp) + || known_tool.as_ref().is_some_and(|name| cli.tool_manifest.iter().any(|tool| tool.name == *name && tool.result_format == ResultFormat::Mcp)); + let known_tool = known_tool.or_else(|| live_context.map(|(name, _)| name)); // ADR 0107 (2026-08-03: injected exit_plan_mode). // BOTH plan decisions take the ordinary deferred // path: stash the result → respawn `--resume` → @@ -3194,8 +3426,8 @@ mod adapter { // `--permission-mode plan`; reject keeps plan for a // read-only revision turn. (The unknown-call_id arm // — post-eviction fresh harness — parses the shape.) - let is_plan_call = known_tool.as_deref() == Some("exit_plan_mode") - || known_tool.is_none(); + let is_plan_call = !is_mcp && (known_tool.as_deref() == Some("exit_plan_mode") + || known_tool.is_none()); if is_plan_call { if let Some(decision) = engram_harness_sdk::plan::parse_plan_decision(&result_json) @@ -3224,7 +3456,6 @@ mod adapter { // bridge until their id-stable resume re-fire, so // every other result is stashed before SIGINT and // consumed only after transcript scrub + --resume. - let known_deferred = known_tool.is_some(); let delivered = mcp_server::route_tool_result( &call_id, result_json, @@ -3626,9 +3857,18 @@ mod adapter { fn fallback_delivery_message( stale_results: &[(String, String)], names: &HashMap, + manifest: &[ManifestTool], ) -> String { let mut lines = vec!["Results for the deferred tool call(s) you made earlier:".to_string()]; for (call_id, result_json) in stale_results { + if let Some(result) = external_fallback_result(call_id, result_json, names, manifest) { + lines.push(format!("- {call_id}:")); + match result { + Ok(result) => lines.extend(result.text_items()), + Err(error) => lines.push(error.into()), + } + continue; + } let plan_decision = (names.get(call_id).map(String::as_str) == Some("exit_plan_mode")) .then(|| engram_harness_sdk::plan::parse_plan_decision(result_json)) .flatten(); @@ -3640,6 +3880,19 @@ mod adapter { lines.join("\n") } + fn external_fallback_result( + call_id: &str, + result_json: &str, + names: &HashMap, + manifest: &[ManifestTool], + ) -> Option> { + let name = names.get(call_id)?; + manifest + .iter() + .find(|tool| tool.name == *name && tool.result_format == ResultFormat::Mcp) + .map(|_| McpToolResult::parse(result_json)) + } + /// Write one prompt to claude's stdin as a stream-json `user` message. /// `serde_json` handles all escaping. async fn write_user_message( @@ -3736,7 +3989,7 @@ mod adapter { fn build_claude_argv( resume_id: &Option, append_system_prompt: Option<&str>, - mcp_config_path: Option<&Path>, + mcp_config_path: &Path, // ADR 0107: the latched session mode (from the workspace stamp, read // per spawn). `plan` maps to the CLI's native plan permission mode; // anything else adds no flag. Changing mode therefore = a clean @@ -3782,14 +4035,10 @@ mod adapter { argv.push("--resume".into()); argv.push(id.clone()); } - if let Some(path) = mcp_config_path { - argv.push("--mcp-config".into()); - argv.push(path.to_string_lossy().into_owned()); - // Without strict mode claude merges the guest user's own MCP - // configuration, exposing servers outside the orchestrator's - // per-session manifest (ADR 0089 spike, Claude 2.1.207). - argv.push("--strict-mcp-config".into()); - } + argv.push("--mcp-config".into()); + argv.push(mcp_config_path.to_string_lossy().into_owned()); + // Strict mode excludes guest MCP servers outside the session manifest. + argv.push("--strict-mcp-config".into()); // ADR 0060: an external trigger (e.g. Slack) flavors the agent's system // prompt via ENGRAM_APPEND_SYSTEM_PROMPT (carried through harness_env). // Empty = unset (skip the flag). @@ -4541,8 +4790,11 @@ mod adapter { // This budget bounds liveness only; event-order assertions stay exact. const SUBPROCESS_TRANSITION_TIMEOUT: Duration = Duration::from_secs(30); - /// The generated hook settings path an argv test doesn't care about - /// (production derives it from the state dir). + fn test_mcp_config() -> &'static Path { + Path::new("/tmp/mcp-config.json") + } + + /// The hook settings path for argv tests; production uses the state dir. fn test_settings() -> &'static Path { Path::new("/tmp/engram-test-state/claude-settings.json") } @@ -4555,7 +4807,7 @@ mod adapter { let argv = build_claude_argv( &None, Some("You were triggered from a Slack thread."), - None, + test_mcp_config(), "default", test_settings(), ); @@ -4572,7 +4824,8 @@ mod adapter { #[test] fn argv_omits_append_system_prompt_when_absent_or_empty() { for v in [None, Some("")] { - let argv = build_claude_argv(&None, v, None, "default", test_settings()); + let argv = + build_claude_argv(&None, v, test_mcp_config(), "default", test_settings()); assert!( !argv.iter().any(|a| a == "--append-system-prompt"), "flag must be absent for {v:?}", @@ -4641,7 +4894,7 @@ mod adapter { // permission mode; the mode comes from the workspace stamp per spawn. #[test] fn argv_carries_permission_mode_plan_only_for_plan() { - let argv = build_claude_argv(&None, None, None, "plan", test_settings()); + let argv = build_claude_argv(&None, None, test_mcp_config(), "plan", test_settings()); let pos = argv .iter() .position(|a| a == "--permission-mode") @@ -4649,7 +4902,7 @@ mod adapter { assert_eq!(argv.get(pos + 1).map(String::as_str), Some("plan")); for mode in ["default", "", "build"] { - let argv = build_claude_argv(&None, None, None, mode, test_settings()); + let argv = build_claude_argv(&None, None, test_mcp_config(), mode, test_settings()); assert!( !argv.iter().any(|a| a == "--permission-mode"), "no permission-mode flag for {mode:?}", @@ -4659,13 +4912,8 @@ mod adapter { #[test] fn argv_carries_strict_mcp_config_when_injected_tools_exist() { - let argv = build_claude_argv( - &None, - None, - Some(Path::new("/tmp/mcp-config.json")), - "default", - test_settings(), - ); + let argv = + build_claude_argv(&None, None, test_mcp_config(), "default", test_settings()); let pos = argv .iter() .position(|arg| arg == "--mcp-config") @@ -4681,10 +4929,13 @@ mod adapter { } #[test] - fn argv_omits_mcp_flags_without_injected_tools() { - let argv = build_claude_argv(&None, None, None, "default", test_settings()); - assert!(!argv.iter().any(|arg| arg == "--mcp-config")); - assert!(!argv.iter().any(|arg| arg == "--strict-mcp-config")); + fn argv_keeps_strict_mcp_flags_without_injected_tools() { + let argv = + build_claude_argv(&None, None, test_mcp_config(), "default", test_settings()); + assert!(argv + .windows(2) + .any(|args| args == ["--mcp-config", "/tmp/mcp-config.json"])); + assert!(argv.iter().any(|arg| arg == "--strict-mcp-config")); } #[tokio::test] @@ -4693,19 +4944,9 @@ mod adapter { .join(format!("engram-mcp-config-test-{}", uuid::Uuid::new_v4())); tokio::fs::create_dir_all(&dir).await.unwrap(); let path = dir.join("mcp-config.json"); - let manifest = parse_tool_manifest( - r#"[ - {"name":"save_memory","description":"Save it","inputSchema":{"type":"object"},"execution":"sync","nativeBindings":{}}, - {"name":"ask_user_question","description":"Ask","inputSchema":{"type":"object"},"execution":"deferred","nativeBindings":{"claude":"AskUserQuestion"}} - ]"#, - ) - .unwrap(); - - assert!( - write_mcp_config(&path, &manifest, Path::new("/sbin/engram-harness-claude")) - .await - .unwrap() - ); + write_mcp_config(&path, Path::new("/sbin/engram-harness-claude")) + .await + .unwrap(); let config: Value = serde_json::from_slice(&tokio::fs::read(&path).await.unwrap()).unwrap(); assert_eq!( @@ -4725,17 +4966,6 @@ mod adapter { }) ); - let native_only = parse_tool_manifest( - r#"[{"name":"ask_user_question","description":"Ask","inputSchema":{"type":"object"},"execution":"deferred","nativeBindings":{"claude":"AskUserQuestion"}}]"#, - ) - .unwrap(); - assert!( - !write_mcp_config(&path, &native_only, Path::new("/sbin/harness")) - .await - .unwrap(), - "a native-only manifest must not enable MCP" - ); - assert!(tokio::fs::metadata(&path).await.is_err()); let _ = tokio::fs::remove_dir_all(dir).await; } @@ -4842,7 +5072,7 @@ mod adapter { #[tokio::test] async fn mcp_tools_call_round_trips_through_main_process_socket() { - let sock = std::env::temp_dir() + let sock = test_socket_root() .join(format!( "engram-mcp-bridge-test-{}.sock", uuid::Uuid::new_v4() @@ -4889,6 +5119,58 @@ mod adapter { let _ = tokio::fs::remove_file(sock).await; } + #[tokio::test] + async fn external_mcp_bridge_preserves_text_structure_and_upstream_error() { + let dir = + test_socket_root().join(format!("engram-mcp-result-{}", uuid::Uuid::new_v4())); + tokio::fs::create_dir_all(&dir).await.unwrap(); + let sock = dir.join("bridge.sock"); + let listener = tokio::net::UnixListener::bind(&sock).unwrap(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let (r, mut w) = stream.into_split(); + let mut lines = BufReader::new(r).lines(); + let _: Value = + serde_json::from_str(&lines.next_line().await.unwrap().unwrap()).unwrap(); + let result = serde_json::json!({ + "outcome":"failed", "delivery":"available", "resultUnavailable":false, + "output": {"content":[{"type":"text","text":"first"},{"type":"text","text":"second"}], + "structuredContent":{"count":2}, "isError":true} + }); + let mut response = + serde_json::to_vec(&serde_json::json!({"result_json":result.to_string()})) + .unwrap(); + response.push(b'\n'); + w.write_all(&response).await.unwrap(); + }); + let manifest = parse_tool_manifest(r#"[{"name":"mcp_inventory_01234567890123456789","description":"Read inventory","inputSchema":{"type":"object"},"execution":"sync","nativeBindings":{},"resultFormat":"mcp"}]"#).unwrap(); + let response = mcp_bridge::handle_request( + serde_json::json!({"jsonrpc":"2.0","id":1,"method":"tools/call", + "params":{"name":"mcp_inventory_01234567890123456789","arguments":{}}}), + &manifest, + &sock, + ) + .await + .unwrap(); + assert_eq!( + response["result"]["content"], + serde_json::json!([ + {"type":"text","text":"first"}, {"type":"text","text":"second"} + ]) + ); + assert_eq!( + response["result"]["structuredContent"], + serde_json::json!({"count":2}) + ); + assert_eq!(response["result"]["isError"], true); + assert!( + response.get("error").is_none(), + "an upstream tool error is not a delivery error" + ); + server.await.unwrap(); + tokio::fs::remove_dir_all(dir).await.unwrap(); + } + async fn spawn_main_mcp_server() -> ( String, mcp_server::ResultsInHand, @@ -4896,7 +5178,7 @@ mod adapter { mpsc::Receiver, hook_server::DeferredCalls, ) { - let sock = std::env::temp_dir() + let sock = test_socket_root() .join(format!( "engram-mcp-main-test-{}.sock", uuid::Uuid::new_v4() @@ -4912,11 +5194,22 @@ mod adapter { let deferred: hook_server::DeferredCalls = Arc::new(tokio::sync::Mutex::new(HashMap::new())); let (evt_tx, evt_rx) = mpsc::channel(16); + let mut manifest = vec![ + generic_tool("save_memory", ToolExecution::Sync), + native_question_tool(), + generic_tool("exit_plan_mode", ToolExecution::Deferred), + ]; + for name in ["mcp_inventory_01234567890123456789", "remote_inventory"] { + let mut tool = generic_tool(name, ToolExecution::Sync); + tool.result_format = ResultFormat::Mcp; + manifest.push(tool); + } tokio::spawn(mcp_server::serve( listener, mcp_server::State { results_in_hand: results.clone(), parked_calls: parked.clone(), + manifest: Arc::new(manifest), deferred_calls: deferred.clone(), current_run_id: run_id.clone(), evt_tx, @@ -4995,6 +5288,165 @@ mod adapter { let _ = tokio::fs::remove_file(sock).await; } + #[tokio::test] + async fn ordinary_bridge_disconnect_preserves_predecessor_events() { + for name in [ + "ask_user_question", + "exit_plan_mode", + "save_memory", + "unknown_tool", + ] { + let (sock, results, parked, mut events, _) = spawn_main_mcp_server().await; + let mut stream = tokio::net::UnixStream::connect(&sock).await.unwrap(); + let request = + serde_json::json!({"name":name,"args":{},"tool_use_id":"ordinary-call"}); + stream + .write_all(format!("{request}\n").as_bytes()) + .await + .unwrap(); + assert!(matches!( + events.recv().await, + Some(HarnessEvent::ToolCallRequested { .. }) + )); + drop(stream); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + if parked + .lock() + .await + .get("ordinary-call") + .is_some_and(|call| call.sender.is_none()) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!( + events.try_recv().is_err(), + "ordinary {name} cannot emit a new event variant" + ); + assert!( + !mcp_server::route_tool_result( + "ordinary-call", + "late".into(), + &parked, + &results + ) + .await + ); + assert_eq!( + results + .lock() + .await + .get("ordinary-call") + .map(String::as_str), + Some("late") + ); + tokio::fs::remove_file(sock).await.unwrap(); + } + } + + #[tokio::test] + async fn main_mcp_disconnect_requests_cancel_with_original_identity() { + let (sock, results, parked, mut events, _deferred) = spawn_main_mcp_server().await; + let mut stream = tokio::net::UnixStream::connect(&sock).await.unwrap(); + stream.write_all(b"{\"name\":\"mcp_inventory_01234567890123456789\",\"args\":{},\"tool_use_id\":\"call-cancel\"}\n").await.unwrap(); + let requested = tokio::time::timeout(Duration::from_secs(1), events.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(requested.kind(), "tool_call_requested"); + drop(stream); + let canceled = tokio::time::timeout(Duration::from_secs(1), events.recv()) + .await + .expect("disconnect must request cancellation") + .unwrap(); + assert_eq!(canceled.kind(), "tool_call_cancel_requested"); + assert_eq!(canceled.run_id(), Some("run-mcp")); + assert_eq!(canceled.tool_call_id(), Some("call-cancel")); + assert_eq!( + serde_json::to_value(&canceled).unwrap()["ToolCallCancelRequested"]["name"], + "mcp_inventory_01234567890123456789" + ); + assert!(parked + .lock() + .await + .get("call-cancel") + .unwrap() + .sender + .is_none()); + assert_eq!( + parked.lock().await.get("call-cancel").unwrap().tool_name, + "mcp_inventory_01234567890123456789" + ); + // A late journal result is retained. Cancellation is not completion. + assert!( + !mcp_server::route_tool_result("call-cancel", "late".into(), &parked, &results) + .await + ); + assert_eq!( + results.lock().await.get("call-cancel").map(String::as_str), + Some("late") + ); + tokio::fs::remove_file(sock).await.unwrap(); + } + + #[tokio::test] + async fn external_mcp_native_cancel_is_processed_while_call_is_pending() { + let (sock, _results, _parked, mut events, _deferred) = spawn_main_mcp_server().await; + let manifest = parse_tool_manifest(r#"[{"name":"remote_inventory","description":"Read","inputSchema":{"type":"object"},"execution":"sync","nativeBindings":{},"resultFormat":"mcp"}]"#).unwrap(); + let (client, bridge) = tokio::io::duplex(8192); + let (client_read, mut client_write) = tokio::io::split(client); + let (bridge_read, bridge_write) = tokio::io::split(bridge); + let bridge_sock = sock.clone(); + let task = tokio::spawn(async move { + mcp_bridge::serve_io( + BufReader::new(bridge_read), + bridge_write, + &manifest, + Path::new(&bridge_sock), + ) + .await + }); + client_write.write_all(b"{\"jsonrpc\":\"2.0\",\"id\":17,\"method\":\"tools/call\",\"params\":{\"name\":\"remote_inventory\",\"arguments\":{},\"_meta\":{\"claudecode/toolUseId\":\"native-cancel\"}}}\n").await.unwrap(); + let requested = tokio::time::timeout(Duration::from_secs(1), events.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(requested.tool_call_id(), Some("native-cancel")); + client_write.write_all(b"{\"jsonrpc\":\"2.0\",\"method\":\"notifications/cancelled\",\"params\":{\"requestId\":17}}\n{\"jsonrpc\":\"2.0\",\"id\":18,\"method\":\"tools/list\"}\n").await.unwrap(); + let mut responses = BufReader::new(client_read).lines(); + let response = tokio::time::timeout(Duration::from_secs(1), responses.next_line()) + .await + .expect("pending call must not block cancellation or tools/list") + .unwrap() + .unwrap(); + let response: Value = serde_json::from_str(&response).unwrap(); + assert_eq!( + response["id"], 18, + "canceled request must not produce a late response" + ); + let canceled = tokio::time::timeout(Duration::from_secs(1), events.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(canceled.kind(), "tool_call_cancel_requested"); + assert_eq!(canceled.tool_call_id(), Some("native-cancel")); + assert_eq!(canceled.run_id(), Some("run-mcp")); + client_write.shutdown().await.unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(2), task) + .await + .unwrap() + .unwrap(), + std::process::ExitCode::SUCCESS + ); + tokio::fs::remove_file(sock).await.unwrap(); + } + #[tokio::test] async fn main_mcp_refire_consumes_result_in_hand_immediately() { let (sock, results, _parked, mut evt_rx, _deferred) = spawn_main_mcp_server().await; @@ -5306,7 +5758,7 @@ mod adapter { // starts clean instead of `--resume`-ing a conversation this fake // cannot replay. (State-dir isolation is `test_cli`'s job.) body.push_str("printf '%s\\n' '{\"type\":\"system\",\"subtype\":\"init\"}'\n"); - body.push_str("while IFS= read -r _line; do\n"); + body.push_str("while IFS= read -r _line; do\n :\n"); for l in per_turn { body.push_str(&format!(" printf '%s\\n' '{l}'\n")); } @@ -5772,9 +6224,16 @@ mod adapter { let _ = engine.await; } + // macOS TMPDIR can leave too little room for UUID-named socket paths. + fn test_socket_root() -> PathBuf { + let root = engram_core::socket::short_socket_dir(); + std::fs::create_dir_all(&root).expect("test socket root"); + root + } + fn test_cli(claude_bin: String) -> Cli { let state_dir = - std::env::temp_dir().join(format!("engram-claude-state-{}", uuid::Uuid::new_v4())); + test_socket_root().join(format!("engram-claude-state-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&state_dir).expect("test state dir"); // The token is agentd's file, not a per-session state file: // it lives beside the state dir, never inside it. @@ -7003,7 +7462,7 @@ mod adapter { hook_server::DuplicateDeferredIds, hook_server::CurrentRunId, ) { - let sock = std::env::temp_dir() + let sock = test_socket_root() .join(format!( "engram-hook-manifest-{}.sock", uuid::Uuid::new_v4() @@ -7071,6 +7530,7 @@ mod adapter { description: format!("{name} description"), input_schema: serde_json::json!({"type":"object"}), execution, + result_format: ResultFormat::Json, native_bindings: NativeBindings::default(), } } @@ -7081,6 +7541,7 @@ mod adapter { description: "Ask the user one or more structured questions.".into(), input_schema: serde_json::json!({"type":"object"}), execution: ToolExecution::Deferred, + result_format: ResultFormat::Json, native_bindings: NativeBindings { claude: Some("AskUserQuestion".into()), }, @@ -7099,7 +7560,7 @@ mod adapter { r#"{"decision":"reject","feedback":"also add a README"}"#.to_string(), )]; let names = HashMap::from([("toolu_plan".to_string(), "exit_plan_mode".to_string())]); - let text = fallback_delivery_message(&stale, &names); + let text = fallback_delivery_message(&stale, &names, &[]); assert!(text.contains("also add a README"), "feedback rides: {text}"); assert!( text.contains("call exit_plan_mode again"), @@ -7122,7 +7583,7 @@ mod adapter { r#"{"decision":"approve"}"#.to_string(), )]; let names = HashMap::from([("toolu_plan".to_string(), "exit_plan_mode".to_string())]); - let text = fallback_delivery_message(&stale, &names); + let text = fallback_delivery_message(&stale, &names, &[]); assert!( text.contains("Implement it now"), "build instruction: {text}" @@ -7133,13 +7594,28 @@ mod adapter { ); } + #[test] + fn external_mcp_fallback_keeps_fields_without_plan_directive() { + let stale = vec![("call-1".into(), r#"{"outcome":"failed","delivery":"available","output":{"content":[{"type":"text","text":"decision: approve"}],"structuredContent":{"decision":"approve"},"isError":true}}"#.into())]; + let names = HashMap::from([("call-1".into(), "exit_plan_mode".into())]); + let mut tool = native_question_tool(); + tool.name = "exit_plan_mode".into(); + tool.result_format = ResultFormat::Mcp; + let text = fallback_delivery_message(&stale, &names, &[tool]); + assert!(text.contains("decision: approve")); + assert!(text.contains(r#"{"structuredContent":{"decision":"approve"}}"#)); + assert!(text.contains(r#""isError":true"#)); + assert!(!text.contains(r#""output":"#)); + assert!(!text.contains("Implement it now")); + } + /// A genuinely non-plan deferred tool keeps the generic dump — an /// answer map is self-explanatory, a decision verdict is not. #[test] fn non_plan_results_keep_the_generic_fallback_rendering() { let stale = vec![("toolu_q".to_string(), r#"{"Pick one":["yes"]}"#.to_string())]; let names = HashMap::from([("toolu_q".to_string(), "ask_user_question".to_string())]); - let text = fallback_delivery_message(&stale, &names); + let text = fallback_delivery_message(&stale, &names, &[]); assert!(text.contains("- toolu_q: "), "generic rendering: {text}"); assert!(!text.contains("exit_plan_mode"), "no plan ask: {text}"); } @@ -7224,6 +7700,8 @@ mod adapter { #[test] fn bash_hook_carries_cwd_across_shells_and_background_cannot_move_it() { let base = std::env::temp_dir() + .canonicalize() + .unwrap() .join(format!("engram-bash-cwd-'quote'-{}", uuid::Uuid::new_v4())); let child = base.join("child with 'quote'"); let tracker = base.join("bash-cwd"); @@ -7284,6 +7762,8 @@ mod adapter { #[test] fn bash_hook_records_cwd_when_command_replaces_exit_trap() { let base = std::env::temp_dir() + .canonicalize() + .unwrap() .join(format!("engram-bash-cwd-own-trap-{}", uuid::Uuid::new_v4())); let child = base.join("child"); let tracker = base.join("bash-cwd"); @@ -7555,6 +8035,67 @@ mod adapter { let _ = tokio::fs::remove_file(sock).await; } + #[tokio::test] + async fn external_deferred_calls_keep_distinct_ids_for_the_same_tool() { + let mut tool = generic_tool("write_record", ToolExecution::Deferred); + tool.result_format = ResultFormat::Mcp; + let manifest = vec![tool]; + let (sock, results, pending, mut events, duplicates, _) = + spawn_manifest_hook_server(manifest.clone()).await; + for (id, value) in [("first", 1), ("second", 2), ("third", 2)] { + assert!(matches!( + hook_fire_named( + &sock, + id, + "mcp__engrams__write_record", + serde_json::json!({"value": value}), + ) + .await, + hook_server::HookVerdict::Defer + )); + assert!( + matches!( + events.try_recv(), + Ok(HarnessEvent::ToolCallRequested { call_id, args_json, .. }) + if call_id == id && args_json == format!("{{\"value\":{value}}}") + ), + "each distinct ID must reach the host" + ); + } + assert!(duplicates.lock().await.is_empty()); + assert_eq!( + unresolved_deferred_ids( + "run-x", + &*pending.lock().await, + &*results.lock().await, + &manifest, + ), + ["first", "second", "third"] + ); + assert!(matches!( + hook_fire_named( + &sock, + "second", + "mcp__engrams__write_record", + serde_json::json!({"value": 2}), + ) + .await, + hook_server::HookVerdict::Defer + )); + assert!( + events.try_recv().is_err(), + "an exact-ID replay stays deduplicated" + ); + for id in ["first", "second", "third"] { + results.lock().await.insert(id.into(), id.into()); + assert_eq!( + mcp_server::take_stashed_result(&results, &pending, id, "write_record").await, + Some((id.into(), id.into())) + ); + } + let _ = tokio::fs::remove_file(sock).await; + } + #[tokio::test] async fn hook_allows_manifest_sync_tool_without_emitting() { let manifest = vec![generic_tool("save_memory", ToolExecution::Sync)]; @@ -7739,6 +8280,108 @@ mod adapter { .unwrap_or_else(|_| panic!("fake claude never reached {n} invocations")); } + async fn assert_external_mcp_malformed_decision_keeps_plan(disconnect: bool) { + let script = write_persistent_fake_claude(&[]).await; + let mut cli = test_cli(script.clone()); + cli.tool_manifest = parse_tool_manifest(r#"[{"name":"remote_inventory","description":"Read","inputSchema":{"type":"object"},"execution":"sync","nativeBindings":{},"resultFormat":"mcp"}]"#).unwrap(); + let manifest = cli.tool_manifest.clone(); + let state = StateDir::new(&cli.state_dir); + let stamp = state.mode_stamp(); + engram_harness_sdk::mode_stamp::write_mode_stamp(&stamp, "plan").unwrap(); + let socket = state.mcp_sock(); + let (commands, command_rx) = mpsc::channel(8); + let (events, mut event_rx) = mpsc::channel(16); + let engine = tokio::spawn(run_engine(cli, command_rx, Arc::new(Notify::new()), events)); + assert!(matches!(event_rx.recv().await, Some(HarnessEvent::Idle))); + let bridge = tokio::spawn(async move { + mcp_bridge::handle_request(serde_json::json!({"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"remote_inventory","arguments":{},"_meta":{"claudecode/toolUseId":"live-call"}}}), &manifest, &socket).await.unwrap() + }); + assert!( + matches!(event_rx.recv().await, Some(HarnessEvent::ToolCallRequested { call_id, .. }) if call_id == "live-call") + ); + if disconnect { + bridge.abort(); + assert!( + matches!(event_rx.recv().await, Some(HarnessEvent::ToolCallCancelRequested { call_id, .. }) if call_id == "live-call") + ); + } + commands + .send(HarnessCommand::ToolResult { + call_id: "live-call".into(), + result_json: r#"{"decision":"approve"}"#.into(), + }) + .await + .unwrap(); + if !disconnect { + let response = tokio::time::timeout(Duration::from_secs(2), bridge) + .await + .unwrap() + .unwrap(); + assert_eq!(response["error"]["code"], -32603); + } + commands + .send(HarnessCommand::Shutdown { grace_secs: 1 }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), engine) + .await + .unwrap() + .unwrap(); + assert_eq!( + engram_harness_sdk::mode_stamp::read_mode_stamp(&stamp), + "plan" + ); + tokio::fs::remove_file(script).await.unwrap(); + } + + #[tokio::test] + async fn external_mcp_live_malformed_decision_cannot_relax_plan() { + assert_external_mcp_malformed_decision_keeps_plan(false).await; + } + + #[tokio::test] + async fn external_mcp_disconnected_malformed_decision_cannot_relax_plan() { + assert_external_mcp_malformed_decision_keeps_plan(true).await; + } + + #[tokio::test] + async fn external_mcp_zero_tools_fake_cli_still_receives_strict_bridge_config() { + let dir = + std::env::temp_dir().join(format!("engram-strict-zero-{}", uuid::Uuid::new_v4())); + tokio::fs::create_dir_all(&dir).await.unwrap(); + let invocations = dir.join("argv"); + let stdin_log = dir.join("stdin"); + let script = write_plan_flow_fake_claude(&invocations, &stdin_log).await; + let (mut cli, _, _) = deferred_engine_cli(script.clone()); + cli.tool_manifest.clear(); + let config_path = StateDir::new(&cli.state_dir).claude_mcp_config(); + let (commands, receiver) = mpsc::channel(8); + let (events, mut received) = mpsc::channel(16); + let engine = tokio::spawn(run_engine(cli, receiver, Arc::new(Notify::new()), events)); + assert!(matches!(received.recv().await, Some(HarnessEvent::Idle))); + wait_for_invocations(&invocations, 1).await; + let argv = tokio::fs::read_to_string(invocations).await.unwrap(); + assert!(argv.contains("--strict-mcp-config")); + assert!(argv.contains(&format!("--mcp-config {}", config_path.display()))); + let config: Value = + serde_json::from_slice(&tokio::fs::read(&config_path).await.unwrap()).unwrap(); + assert_eq!(config["mcpServers"].as_object().unwrap().len(), 1); + assert_eq!( + config["mcpServers"]["engrams"]["args"], + serde_json::json!(["mcp-bridge"]) + ); + commands + .send(HarnessCommand::Shutdown { grace_secs: 1 }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), engine) + .await + .unwrap() + .unwrap(); + tokio::fs::remove_file(script).await.unwrap(); + tokio::fs::remove_dir_all(dir).await.unwrap(); + } + /// `exit_plan_mode` as production ships it since PR #981: an INJECTED /// deferred tool (no claude native binding), reached as /// `mcp__engrams__exit_plan_mode`. The old `native_bindings.claude = @@ -7750,6 +8393,7 @@ mod adapter { description: "Present your finished implementation plan.".into(), input_schema: serde_json::json!({"type":"object"}), execution: ToolExecution::Deferred, + result_format: ResultFormat::Json, native_bindings: NativeBindings::default(), } } @@ -8197,6 +8841,236 @@ mod adapter { (cli, state.hook_sock(), state.mcp_sock()) } + #[test] + fn deferred_association_excludes_other_runs_and_received_results() { + let calls = [ + ("current", "run-a"), + ("received", "run-a"), + ("stale", "run-b"), + ] + .into_iter() + .map(|(id, run)| { + ( + id.to_string(), + hook_server::DeferredCall { + run_id: run.to_string(), + tool_name: "save_memory".to_string(), + }, + ) + }) + .collect(); + let results = [("received".to_string(), "{}".to_string())] + .into_iter() + .collect(); + let mut tool = generic_tool("save_memory", ToolExecution::Deferred); + tool.result_format = ResultFormat::Mcp; + assert_eq!( + unresolved_deferred_ids("run-a", &calls, &results, &[tool]), + ["current"] + ); + } + + #[tokio::test] + async fn deferred_event_contract_distinguishes_ordinary_and_mcp_calls() { + use std::os::unix::fs::PermissionsExt; + + let mut mismatches = Vec::new(); + for (case, hooks, expected_ids) in [ + ("question", vec![("question", "AskUserQuestion")], vec![]), + ("plan", vec![("plan", "ExitPlanMode")], vec![]), + ( + "generic", + vec![("generic", "mcp__engrams__save_memory")], + vec![], + ), + ( + "mcp", + vec![("managed", "mcp__engrams__remote_inventory")], + vec!["managed"], + ), + ( + "mixed", + vec![ + ("question", "AskUserQuestion"), + ("plan", "ExitPlanMode"), + ("managed-b", "mcp__engrams__remote_inventory"), + ("managed-a", "mcp__engrams__remote_write"), + ], + vec!["managed-a", "managed-b"], + ), + ( + "mixed-invalid-ordinary", + vec![ + ("", "AskUserQuestion"), + ("managed", "mcp__engrams__remote_inventory"), + ], + vec!["managed"], + ), + ] { + let base = + std::env::temp_dir().join(format!("event-contract-{}", uuid::Uuid::new_v4())); + let script = base.with_extension("sh"); + let gate = base.with_extension("gate"); + let body = format!( + "#!/bin/sh\nprintf '%s\\n' '{{\"type\":\"system\",\"subtype\":\"init\"}}'\nIFS= read -r _prompt\nwhile [ ! -f '{}' ]; do sleep 0.01; done\nprintf '%s\\n' '{{\"type\":\"result\",\"subtype\":\"success\",\"is_error\":false,\"terminal_reason\":\"tool_deferred\"}}'\nwhile IFS= read -r _line; do :; done\n", + gate.display(), + ); + tokio::fs::write(&script, body).await.unwrap(); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap(); + let (mut cli, hook, _) = deferred_engine_cli(script.to_string_lossy().into_owned()); + let mut plan = generic_tool("exit_plan_mode", ToolExecution::Deferred); + plan.native_bindings.claude = Some("ExitPlanMode".into()); + let mut external = generic_tool("remote_inventory", ToolExecution::Deferred); + external.result_format = ResultFormat::Mcp; + let mut write = external.clone(); + write.name = "remote_write".into(); + cli.tool_manifest + .extend([native_question_tool(), plan, external, write]); + let (cmd_tx, cmd_rx) = mpsc::channel(8); + let (evt_tx, mut events) = mpsc::channel(64); + let engine = tokio::spawn(run_engine(cli, cmd_rx, Arc::new(Notify::new()), evt_tx)); + assert!(matches!(events.recv().await, Some(HarnessEvent::Idle))); + cmd_tx + .send(HarnessCommand::Prompt { + prompt_id: case.into(), + text: "test".into(), + mode: None, + }) + .await + .unwrap(); + let run_id = expect_run_started(&mut events).await; + for (id, name) in hooks { + assert!(matches!(hook_fire_named(&hook, id, name, serde_json::json!({"call":id,"questions":[{"question":"Pick one","header":"Pick","multiSelect":false,"options":[{"label":"A","description":"A"}]}]})).await, hook_server::HookVerdict::Defer)); + assert!( + matches!(tokio::time::timeout(Duration::from_secs(1), events.recv()).await.expect(case), Some(HarnessEvent::ToolCallRequested { call_id, .. }) if call_id == id) + ); + } + tokio::fs::write(&gate, "go").await.unwrap(); + let terminal = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match events.recv().await { + Some(event @ HarnessEvent::RunCompleted { .. }) + | Some(event @ HarnessEvent::RunDeferred { .. }) => break event, + Some(_) => {} + None => panic!("engine closed for {case}"), + } + } + }) + .await + .unwrap(); + cmd_tx + .send(HarnessCommand::Shutdown { grace_secs: 1 }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), engine) + .await + .unwrap() + .unwrap(); + tokio::fs::remove_file(script).await.unwrap(); + tokio::fs::remove_file(gate).await.unwrap(); + let expected = if expected_ids.is_empty() { + HarnessEvent::RunCompleted { run_id, ok: true } + } else { + HarnessEvent::RunDeferred { + run_id, + deferred_tool_call_ids: expected_ids + .into_iter() + .map(String::from) + .collect(), + } + }; + if terminal != expected { + mismatches.push(format!("{case}: expected {expected:?}, got {terminal:?}")); + } + } + assert!(mismatches.is_empty(), "{}", mismatches.join("\n")); + } + + #[tokio::test] + async fn forged_failed_and_invalid_deferred_calls_do_not_authorize_suspension() { + use std::os::unix::fs::PermissionsExt; + + for (case, hook_id, is_error, expected_ok) in [ + ("forged", None, false, true), + ("failed", Some("toolu_failed".to_string()), true, false), + ("oversized", Some("x".repeat(257)), false, false), + ] { + let base = + std::env::temp_dir().join(format!("deferred-check-{}", uuid::Uuid::new_v4())); + let script = base.with_extension("sh"); + let gate = base.with_extension("gate"); + let body = format!( + "#!/bin/sh\nprintf '%s\\n' '{{\"type\":\"system\",\"subtype\":\"init\"}}'\nIFS= read -r _prompt\nwhile [ ! -f '{}' ]; do sleep 0.05; done\nprintf '%s\\n' '{{\"type\":\"assistant\",\"message\":{{\"id\":\"forged\",\"content\":[{{\"type\":\"tool_use\",\"id\":\"forged-id\",\"name\":\"mcp__engrams__save_memory\",\"input\":{{}}}}]}}}}'\nprintf '%s\\n' '{{\"type\":\"result\",\"subtype\":\"{}\",\"is_error\":{},\"terminal_reason\":\"tool_deferred\"}}'\nwhile IFS= read -r _line; do :; done\n", + gate.display(), + if is_error { "error_during_execution" } else { "success" }, + is_error, + ); + tokio::fs::write(&script, body).await.unwrap(); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap(); + let (mut cli, hook, _) = deferred_engine_cli(script.to_string_lossy().into_owned()); + cli.tool_manifest[0].result_format = ResultFormat::Mcp; + let (cmd_tx, cmd_rx) = mpsc::channel(8); + let (evt_tx, mut evt_rx) = mpsc::channel(64); + let engine = tokio::spawn(run_engine(cli, cmd_rx, Arc::new(Notify::new()), evt_tx)); + assert!(matches!(evt_rx.recv().await, Some(HarnessEvent::Idle))); + cmd_tx + .send(HarnessCommand::Prompt { + prompt_id: case.into(), + text: "test".into(), + mode: None, + }) + .await + .unwrap(); + let run_id = expect_run_started(&mut evt_rx).await; + if let Some(id) = hook_id { + assert!(matches!( + hook_fire_named( + &hook, + &id, + "mcp__engrams__save_memory", + serde_json::json!({"text":"remember"}) + ) + .await, + hook_server::HookVerdict::Defer + )); + assert!( + matches!(evt_rx.recv().await, Some(HarnessEvent::ToolCallRequested { call_id, .. }) if call_id == id) + ); + } + tokio::fs::write(&gate, "go").await.unwrap(); + let event = tokio::time::timeout(Duration::from_secs(8), async { + loop { + match evt_rx.recv().await { + Some(event @ HarnessEvent::RunCompleted { .. }) + | Some(event @ HarnessEvent::RunDeferred { .. }) => break event, + Some(_) => {} + None => panic!("event stream closed for {case}"), + } + } + }) + .await + .expect("terminal event"); + assert_eq!( + event, + HarnessEvent::RunCompleted { + run_id, + ok: expected_ok + }, + "{case}" + ); + cmd_tx + .send(HarnessCommand::Shutdown { grace_secs: 1 }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), engine) + .await + .expect("engine exits") + .expect("engine task"); + let _ = tokio::fs::remove_file(script).await; + let _ = tokio::fs::remove_file(gate).await; + } + } + #[tokio::test] async fn deferred_result_after_process_death_resumes_and_serves_refire() { let counter = @@ -8216,7 +9090,7 @@ mod adapter { }) .await .unwrap(); - let _ = expect_run_started(&mut evt_rx).await; + let deferred_run = expect_run_started(&mut evt_rx).await; assert!(matches!( hook_fire_named( &hook, @@ -8236,12 +9110,20 @@ mod adapter { // Drain the deferred turn's tool log/completion and both Parked // announcements: one at turn-end, one from the respawned process. let mut waiting = Vec::new(); + let mut terminal = None; tokio::time::timeout(Duration::from_secs(8), async { while waiting.len() < 2 { match evt_rx.recv().await { Some(HarnessEvent::Parked) => waiting.push("parked"), Some(HarnessEvent::Idle) => waiting.push("idle"), - Some(_) => {} + Some(event) => { + let value = serde_json::to_value(event).unwrap(); + if value.get("RunCompleted").is_some() + || value.get("RunDeferred").is_some() + { + terminal = Some(value); + } + } None => panic!("engine event channel closed before parked re-announce"), } } @@ -8249,6 +9131,12 @@ mod adapter { .await .expect("the dead first process should respawn"); assert_eq!(waiting, ["parked", "parked"]); + assert_eq!( + terminal, + Some(serde_json::json!({ + "RunCompleted": {"run_id": deferred_run, "ok": true} + })) + ); cmd_tx .send(HarnessCommand::ToolResult { @@ -8289,8 +9177,7 @@ mod adapter { let _ = tokio::fs::remove_file(counter).await; } - #[tokio::test] - async fn deferred_result_after_fresh_restore_resumes_and_serves_refire() { + async fn assert_fresh_restore_refire(external: bool) { let nonce = uuid::Uuid::new_v4(); let counter = std::env::temp_dir().join(format!("fake-claude-count-{nonce}")); let invocations = std::env::temp_dir().join(format!("fake-claude-invocations-{nonce}")); @@ -8304,7 +9191,21 @@ mod adapter { &session_id, ) .await; - let (cli, hook, mcp) = deferred_engine_cli(script.clone()); + let (mut cli, hook, mcp) = deferred_engine_cli(script.clone()); + if external { + cli.tool_manifest[0].result_format = ResultFormat::Mcp; + } + let manifest = cli.tool_manifest.clone(); + let mode_stamp = StateDir::new(&cli.state_dir).mode_stamp(); + engram_harness_sdk::mode_stamp::write_mode_stamp(&mode_stamp, "plan").unwrap(); + let result_json = if external { + serde_json::json!({"outcome":"succeeded","delivery":"available","output":{ + "content":[{"type":"text","text":"{\"decision\":\"approve\"}"},{"type":"text","text":"second"}], + "structuredContent":{"decision":"approve","count":2},"isError":false + },"resultUnavailable":false}).to_string() + } else { + r#"{"saved":true}"#.to_string() + }; let stash = StateDir::new(&cli.state_dir).claude_session_id(); let (cmd_tx, cmd_rx) = mpsc::channel(8); let (evt_tx, mut evt_rx) = mpsc::channel(64); @@ -8331,7 +9232,7 @@ mod adapter { cmd_tx .send(HarnessCommand::ToolResult { call_id: "toolu_restored".into(), - result_json: r#"{"saved":true}"#.into(), + result_json: result_json.clone(), }) .await .unwrap(); @@ -8371,10 +9272,32 @@ mod adapter { .await, hook_server::HookVerdict::Allow )); - assert_eq!( - fire_main_mcp_call(&mcp, "toolu_restored").await["result_json"], - r#"{"saved":true}"# - ); + let response = mcp_bridge::handle_request( + serde_json::json!({ + "jsonrpc":"2.0","id":1,"method":"tools/call","params":{ + "name":"save_memory","arguments":{"text":"remember"}, + "_meta":{"claudecode/toolUseId":"toolu_restored"} + } + }), + &manifest, + &mcp, + ) + .await + .unwrap(); + if external { + assert_eq!( + response["result"]["structuredContent"], + serde_json::json!({"decision":"approve","count":2}) + ); + assert_eq!(response["result"]["content"].as_array().unwrap().len(), 2); + assert_eq!(response["result"]["isError"], false); + assert_eq!( + engram_harness_sdk::mode_stamp::read_mode_stamp(&mode_stamp), + "plan" + ); + } else { + assert_eq!(response["result"]["content"][0]["text"], result_json); + } tokio::fs::write(&completion_gate, b"served").await.unwrap(); let mut continued = false; @@ -8415,6 +9338,16 @@ mod adapter { let _ = tokio::fs::remove_file(completion_gate).await; } + #[tokio::test] + async fn deferred_result_after_fresh_restore_resumes_and_serves_refire() { + assert_fresh_restore_refire(false).await; + } + + #[tokio::test] + async fn external_mcp_fresh_engine_refire_preserves_envelope_and_plan_latch() { + assert_fresh_restore_refire(true).await; + } + #[tokio::test] async fn deferred_result_with_live_process_resumes_and_serves_refire() { let counter = @@ -8651,7 +9584,7 @@ mod adapter { } } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", test))] #[tokio::main] async fn main() -> std::process::ExitCode { // ADR 0054: git/busybox argv dispatch — the SAME binary is the socket @@ -8667,7 +9600,7 @@ async fn main() -> std::process::ExitCode { adapter::entry().await } -#[cfg(all(test, target_os = "linux"))] +#[cfg(test)] mod tests { use super::adapter::*; use engram_harness_proto::{FileChange, HarnessEvent}; @@ -9209,22 +10142,22 @@ mod tests { /// Pins the OBSERVED `background_tasks_changed` wire shape (probed /// 2026-08-06 against the harness argv; the event is undocumented). - /// The lines are verbatim probe captures. If a future CLI changes the - /// shape, this fails loudly instead of the harness silently reverting + /// Synthetic fixtures preserve the observed shape. If a future CLI changes + /// the shape, this fails loudly instead of the harness silently reverting /// to announce-idle-immediately. #[test] fn background_agent_inventory_counts_local_agents_only() { - let agent = r#"{"type":"system","subtype":"background_tasks_changed","tasks":[{"task_id":"a9e33c94e91b3d1b5","task_type":"local_agent","description":"Sleep then output done"}],"uuid":"d3ae041e-47fd-4d0f-add6-0f13a50f6b2f","session_id":"45fff78d-5f35-497a-88e0-80eb21600a2e"}"#; + let agent = r#"{"type":"system","subtype":"background_tasks_changed","tasks":[{"task_id":"agent-fixture","task_type":"local_agent","description":"Sleep then output done"}],"uuid":"00000000-0000-4000-8000-000000000001","session_id":"00000000-0000-4000-8000-000000000010"}"#; assert_eq!(detect_background_agents(agent), Some(1)); - let drained = r#"{"type":"system","subtype":"background_tasks_changed","tasks":[],"uuid":"9b1482bc-3149-4371-8318-6f56b4397b81","session_id":"45fff78d-5f35-497a-88e0-80eb21600a2e"}"#; + let drained = r#"{"type":"system","subtype":"background_tasks_changed","tasks":[],"uuid":"00000000-0000-4000-8000-000000000002","session_id":"00000000-0000-4000-8000-000000000010"}"#; assert_eq!(detect_background_agents(drained), Some(0)); // A background SHELL never counts: it can be a never-exiting dev // server, and holding the idle announcement on it would pin the // session resident until the hard TTL. - let bash = r#"{"type":"system","subtype":"background_tasks_changed","tasks":[{"task_id":"bw9ttimtt","task_type":"local_bash","description":"Sleep 20 seconds in background"}],"uuid":"eb30b242-265e-4fec-8928-cf7368376864","session_id":"816f8b56-7307-4175-b4a5-89c453bbc577"}"#; + let bash = r#"{"type":"system","subtype":"background_tasks_changed","tasks":[{"task_id":"bash-fixture","task_type":"local_bash","description":"Sleep 20 seconds in background"}],"uuid":"00000000-0000-4000-8000-000000000003","session_id":"00000000-0000-4000-8000-000000000020"}"#; assert_eq!(detect_background_agents(bash), Some(0)); // Lifecycle lines are NOT inventory lines. - let started = r#"{"type":"system","subtype":"task_started","task_id":"a9e33c94e91b3d1b5","task_type":"local_agent"}"#; + let started = r#"{"type":"system","subtype":"task_started","task_id":"agent-fixture","task_type":"local_agent"}"#; assert_eq!(detect_background_agents(started), None); assert_eq!(detect_background_agents(r#"{"type":"assistant"}"#), None); } @@ -9241,9 +10174,9 @@ mod tests { assert!(is_main_loop_message( r#"{"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t1"}]},"parent_tool_use_id":null}"# )); - // Verbatim-shaped subagent line from the probe: tagged → never. + // Synthetic subagent line preserves the observed shape: tagged → never. assert!(!is_main_loop_message( - r#"{"type":"assistant","message":{"id":"m2","content":[]},"parent_tool_use_id":"toolu_01TUxcDgGE8cPe7wZ2B1sKbm"}"# + r#"{"type":"assistant","message":{"id":"m2","content":[]},"parent_tool_use_id":"toolu_fixture_parent"}"# )); assert!(!is_main_loop_message( r#"{"type":"system","subtype":"init"}"# @@ -9318,6 +10251,7 @@ mod tests { description: "Save a memory".into(), input_schema: serde_json::json!({"type":"object"}), execution: ToolExecution::Deferred, + result_format: engram_harness_proto::tool_result::ResultFormat::Json, native_bindings: NativeBindings::default(), }]; let deferred_names = deferred_tool_names(&manifest); @@ -9387,7 +10321,7 @@ mod tests { assert!(!ok("")); } - /// ADR 0107 / session 28216894: a natively-bound deferred tool arrives as + /// ADR 0107: a natively-bound deferred tool arrives as /// `ExitPlanMode`, NOT `mcp__engrams__exit_plan_mode`, and /// `injected_tools` excludes natively-bound tools by design. The parked /// call therefore never marked itself deferred, so the model's "the @@ -9401,6 +10335,7 @@ mod tests { description: "Present the plan".into(), input_schema: serde_json::json!({"type":"object"}), execution: ToolExecution::Deferred, + result_format: engram_harness_proto::tool_result::ResultFormat::Json, native_bindings: NativeBindings { claude: Some("ExitPlanMode".into()), }, diff --git a/crates/engram-harness-claude/tests/linux_mcp_entrypoint.rs b/crates/engram-harness-claude/tests/linux_mcp_entrypoint.rs new file mode 100644 index 000000000..d2d919826 --- /dev/null +++ b/crates/engram-harness-claude/tests/linux_mcp_entrypoint.rs @@ -0,0 +1,678 @@ +//! Linux production wrapper processes; only the model CLI is a fixture. +//! This is transport and presentation evidence, not VM isolation or server policy. +#![cfg(target_os = "linux")] + +use std::fs; +use std::os::unix::{fs::PermissionsExt, process::CommandExt}; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use engram_core::{SandboxId, SessionId}; +use engram_harness_proto::{ + read_msg, write_msg, HarnessAttach, HarnessAttachAck, HarnessCommand, HarnessEvent, + HarnessFrame, +}; +use nix::sys::signal::{kill, killpg, Signal}; +use nix::unistd::Pid; +use serde_json::{json, Value}; +use tokio::net::{tcp::OwnedWriteHalf, TcpListener}; +use tokio::sync::mpsc; +use tokio::time::{sleep, timeout}; + +const TOOL: &str = "remote_inventory"; +const CALL: &str = "fixture-call"; +const LIMIT: Duration = Duration::from_secs(15); + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Wrapper { + Claude, + Codex, +} +impl Wrapper { + fn name(self) -> &'static str { + match self { + Self::Claude => "claude", + Self::Codex => "codex", + } + } + fn binary(self) -> PathBuf { + artifact(match self { + Self::Claude => "ENGRAM_TEST_CLAUDE_BIN", + Self::Codex => "ENGRAM_TEST_CODEX_BIN", + }) + } +} + +fn artifact(variable: &str) -> PathBuf { + let path = PathBuf::from(std::env::var_os(variable).expect("required current Linux artifact")); + assert!(path.is_absolute()); + let path = path.canonicalize().unwrap(); + assert!(fs::metadata(&path).unwrap().permissions().mode() & 0o111 != 0); + let bytes = fs::read(&path).unwrap(); + assert_eq!( + &bytes[..4], + b"\x7fELF", + "a Linux production artifact is required" + ); + assert_eq!(bytes[4], 2, "64-bit ELF required"); + let machine = u16::from_le_bytes([bytes[18], bytes[19]]); + assert_eq!( + machine, + if cfg!(target_arch = "aarch64") { + 183 + } else { + 62 + } + ); + path +} + +struct Process { + child: Child, + root: PathBuf, +} +impl Drop for Process { + fn drop(&mut self) { + // The wrapper and all fixture hook/bridge descendants share this owned group. + let _ = killpg( + Pid::from_raw(self.child.id().try_into().unwrap()), + Signal::SIGKILL, + ); + let _ = self.child.wait(); + let _ = fs::remove_dir_all(&self.root); + } +} + +struct Harness { + process: Process, + writer: OwnedWriteHalf, + events: mpsc::Receiver, + reader: tokio::task::JoinHandle<()>, + trace: Vec, + fixture: PathBuf, +} +impl Drop for Harness { + fn drop(&mut self) { + self.reader.abort(); + } +} +impl Harness { + async fn start(wrapper: Wrapper, mode: &str, deferred: bool) -> Self { + Self::start_with_plan(wrapper, mode, deferred, None).await + } + + async fn start_with_plan( + wrapper: Wrapper, + mode: &str, + deferred: bool, + plan: Option<&Value>, + ) -> Self { + let base = + PathBuf::from(std::env::var_os("ENGRAM_TEST_STATE_ROOT").expect("owned state root")); + assert!(base.is_absolute()); + assert_eq!( + base.canonicalize().unwrap(), + base, + "state parent must be canonical" + ); + let root = base.join(format!("{}-{}", wrapper.name(), uuid::Uuid::new_v4())); + fs::create_dir(&root).unwrap(); + fs::set_permissions(&root, fs::Permissions::from_mode(0o700)).unwrap(); + let proof = root.join("proof"); + fs::create_dir(&proof).unwrap(); + fs::set_permissions(&proof, fs::Permissions::from_mode(0o700)).unwrap(); + let fixture = artifact("ENGRAM_FIXTURE_BIN"); + let version = timeout( + LIMIT, + tokio::process::Command::new(&fixture) + .arg("--fixture-version") + .env_clear() + .kill_on_drop(true) + .output(), + ) + .await + .expect("fixture version deadline") + .unwrap(); + assert!(version.status.success()); + assert_eq!(version.stdout, b"engram-mcp-model-fixture 1\n"); + assert!(version.stderr.is_empty()); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let session = SessionId::new(); + let sandbox = SandboxId::new(); + let manifest = if plan.is_some() { + json!([]) + } else { + json!([{"name":TOOL,"description":"Fixture inventory","inputSchema":{"type":"object","properties":{"fixture":{"type":"string"}},"required":["fixture"],"additionalProperties":false},"execution":if deferred {"deferred"} else {"sync"},"nativeBindings":{},"resultFormat":"mcp"}]) + }; + let mut command = Command::new(wrapper.binary()); + command + .env_clear() + .current_dir(&root) + .process_group(0) + .args([ + "--connect", + &listener.local_addr().unwrap().to_string(), + "--session-id", + &session.to_string(), + ]) + .arg("--state-dir") + .arg(root.join("state")) + .arg(format!("--{}-bin", wrapper.name())) + .arg(&fixture) + .env("HOME", &root) + .env("TMPDIR", &root) + .env("RUST_LOG", "off") + .env("ENGRAM_ATTACH_TOKEN_FILE", root.join("absent-token")) + .env("ENGRAM_SANDBOX_ID", sandbox.to_string()) + .env("ENGRAM_BINDING_EPOCH", "1") + .env("ENGRAM_TOOLS", manifest.to_string()) + .env("ENGRAM_FIXTURE_MODE", mode) + .env("ENGRAM_FIXTURE_STATE_DIR", &proof) + .env("ENGRAM_FIXTURE_PROOF", proof.join("launch-{pid}.json")) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + if let Some(plan) = plan { + command.env("ENGRAM_FIXTURE_CALL_PLAN_JSON", plan.to_string()); + } else { + command.env("ENGRAM_FIXTURE_TOOL", TOOL); + } + if wrapper == Wrapper::Codex { + command.env("CODEX_API_KEY", "fixture-placeholder"); + } + let process = Process { + child: command.spawn().unwrap(), + root, + }; + let (mut socket, _) = timeout(LIMIT, listener.accept()) + .await + .expect("wrapper attach deadline") + .unwrap(); + let attach: HarnessAttach = timeout(LIMIT, read_msg(&mut socket)) + .await + .unwrap() + .unwrap(); + assert_eq!(attach.session_id, session); + assert_eq!(attach.sandbox_id, sandbox); + assert_eq!(attach.binding_epoch, 1); + assert_eq!( + attach.harness_version, + format!( + "engram-harness-{}/{}", + wrapper.name(), + env!("CARGO_PKG_VERSION") + ) + ); + // HarnessAttach has no global wire-version negotiation. Current artifacts + // and decoding actual frames establish this test's protocol provenance. + write_msg( + &mut socket, + &HarnessAttachAck { + ok: true, + reject: None, + message: None, + }, + ) + .await + .unwrap(); + let (mut read, writer) = socket.into_split(); + let (send, events) = mpsc::channel(128); + let reader = tokio::spawn(async move { + let mut previous = 0; + let mut instance = None; + while let Ok(frame) = read_msg::<_, HarnessFrame>(&mut read).await { + let HarnessFrame::SeqEvent { + seq, + event, + binding_epoch, + incarnation, + } = frame + else { + panic!("expected sequenced event") + }; + assert_eq!(binding_epoch, 1); + assert_eq!(seq, previous + 1); + previous = seq; + if let Some(old) = instance { + assert_eq!(incarnation, old); + } + instance = Some(incarnation); + if send.send(event).await.is_err() { + return; + } + } + }); + let mut h = Self { + process, + writer, + events, + reader, + trace: vec![], + fixture, + }; + h.until(|e| matches!(e, HarnessEvent::Idle)).await; + h + } + async fn send(&mut self, command: HarnessCommand) { + timeout( + LIMIT, + write_msg(&mut self.writer, &HarnessFrame::Command(command)), + ) + .await + .unwrap() + .unwrap(); + } + async fn event(&mut self) -> HarnessEvent { + let event = timeout(LIMIT, self.events.recv()) + .await + .expect("harness event deadline") + .expect("wrapper event channel closed"); + self.trace.push(event.clone()); + write_msg( + &mut self.writer, + &HarnessFrame::EventAck { + seq: self.trace.len() as u64, + }, + ) + .await + .unwrap(); + event + } + async fn until(&mut self, predicate: impl Fn(&HarnessEvent) -> bool) -> HarnessEvent { + timeout(LIMIT, async { + loop { + let event = self.event().await; + if predicate(&event) { + return event; + } + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "expected event deadline; observed kinds: {:?}; launches: {}", + self.trace + .iter() + .map(HarnessEvent::kind) + .collect::>(), + self.proofs().len() + ) + }) + } + async fn prompt(&mut self) { + self.send(HarnessCommand::Prompt { + text: "fixture".into(), + prompt_id: "fixture-prompt".into(), + mode: None, + }) + .await; + } + async fn requested(&mut self, mode: &str) { + let event = self + .until(|e| matches!(e, HarnessEvent::ToolCallRequested { .. })) + .await; + let HarnessEvent::ToolCallRequested { + call_id, + name, + args_json, + run_id, + } = event + else { + unreachable!() + }; + assert_eq!(call_id, CALL); + assert_eq!(name, TOOL); + assert!(!run_id.is_empty()); + assert_eq!( + serde_json::from_str::(&args_json).unwrap(), + json!({"fixture":mode}) + ); + } + async fn result(&mut self, value: Value) { + self.send(HarnessCommand::ToolResult { + call_id: CALL.into(), + result_json: value.to_string(), + }) + .await; + } + fn proofs(&self) -> Vec { + let entries: Vec<_> = fs::read_dir(self.process.root.join("proof")) + .unwrap() + .map(|e| e.unwrap().path()) + .collect(); + assert!(entries.len() <= 4, "fixture launch budget exceeded"); + entries + } + async fn proof(&mut self) -> Value { + timeout(LIMIT, async { + loop { + for path in self.proofs() { + if let Ok(value) = serde_json::from_slice::(&fs::read(path).unwrap()) { return value; } + } + tokio::select! { + event = self.events.recv() => { + let event = event.expect("wrapper closed before proof"); + self.trace.push(event); + write_msg(&mut self.writer, &HarnessFrame::EventAck { seq:self.trace.len() as u64 }).await.unwrap(); + } + _ = sleep(Duration::from_millis(10)) => {} + } + } + }).await.expect("model CLI result proof deadline") + } + fn requests(&self) -> usize { + self.trace + .iter() + .filter(|e| matches!(e, HarnessEvent::ToolCallRequested { .. })) + .count() + } + async fn finish(mut self, expected_requests: usize) -> Vec { + self.send(HarnessCommand::Shutdown { grace_secs: 1 }).await; + // The SDK drains its event outbox before exit. Continue ACKing terminal + // events while waiting for the production wrapper to reap its CLI. + timeout(LIMIT, async { + let mut open = true; + loop { + if let Some(status) = self.process.child.try_wait().unwrap() { + assert!(status.success(), "wrapper shutdown failed"); + break; + } + tokio::select! { + event = self.events.recv(), if open => { + if let Some(event) = event { + self.trace.push(event); + let _ = write_msg(&mut self.writer, &HarnessFrame::EventAck { seq: self.trace.len() as u64 }).await; + } else { open = false; } + } + _ = sleep(Duration::from_millis(10)) => {} + } + } + }).await.expect("wrapper shutdown deadline"); + timeout(LIMIT, &mut self.reader) + .await + .expect("reader exit deadline") + .expect("sequence reader failed"); + while let Ok(event) = self.events.try_recv() { + self.trace.push(event); + } + assert_eq!(self.requests(), expected_requests); + self.trace.clone() + } + async fn kill_first_cli(&mut self) { + let paths = self.proofs(); + assert_eq!(paths.len(), 1, "one first fixture launch"); + let name = paths[0].file_stem().unwrap().to_str().unwrap(); + let pid: i32 = name.strip_prefix("launch-").unwrap().parse().unwrap(); + let proc = PathBuf::from(format!("/proc/{pid}")); + assert_eq!(fs::read_link(proc.join("exe")).unwrap(), self.fixture); + let stat = fs::read_to_string(proc.join("stat")).unwrap(); + let fields: Vec<_> = stat + .rsplit_once(") ") + .unwrap() + .1 + .split_whitespace() + .collect(); + assert_eq!( + fields[1].parse::().unwrap(), + self.process.child.id(), + "owned direct model child" + ); + assert_eq!( + fields[2].parse::().unwrap(), + self.process.child.id(), + "owned process group" + ); + kill(Pid::from_raw(pid), Signal::SIGKILL).unwrap(); + timeout(LIMIT, async { + while proc.exists() { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("first CLI reaped"); + timeout(LIMIT, async { + while self.proofs().len() < 2 { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("replacement CLI launch"); + assert!( + fs::read(&paths[0]).unwrap().is_empty(), + "dead CLI received no result" + ); + } +} + +fn output(error: bool) -> Value { + json!({"outcome":if error {"failed"} else {"succeeded"},"delivery":"available","output":{"content":[{"type":"text","text":"sentinel"},{"type":"text","text":"second sentinel"}],"structuredContent":{"fixture":true},"isError":error},"resultUnavailable":false}) +} +fn assert_receipt(proof: &Value, error: bool) { + assert_eq!(proof["callId"], CALL); + assert_eq!(proof["tool"], TOOL); + for field in [ + "resultReceived", + "textSentinel", + "textSequence", + "structuredSentinel", + ] { + assert_eq!(proof[field], true, "{field}"); + } + assert_eq!(proof["isError"], error); + assert_eq!(proof["outcome"], if error { "failed" } else { "succeeded" }); + assert_eq!(proof["delivery"], "available"); + assert_eq!(proof["resultUnavailable"], false); + if let Some(success) = proof.get("success") { + assert_eq!(success, &json!(!error)); + } +} + +async fn delivered(mode: &str, value: Value, check: fn(&Value), deferred: bool) { + for wrapper in [Wrapper::Claude, Wrapper::Codex] { + timeout(Duration::from_secs(45), async { + let mut h = Harness::start(wrapper, mode, deferred).await; + h.prompt().await; + h.requested(mode).await; + if deferred { + h.until(|e| matches!(e, HarnessEvent::Parked)).await; + } + h.result(value.clone()).await; + let proof = h.proof().await; + check(&proof); + assert_eq!(h.requests(), 1); + h.finish(1).await; + }) + .await + .expect("scenario deadline"); + } +} + +#[tokio::test] +#[ignore = "requires current Linux wrapper and fixture artifacts"] +async fn read_receipt_and_ordered_content() { + delivered("read", output(false), |v| assert_receipt(v, false), false).await; +} +#[tokio::test] +#[ignore = "requires current Linux wrapper and fixture artifacts"] +async fn upstream_error_preserves_output() { + delivered("read", output(true), |v| assert_receipt(v, true), false).await; +} +#[tokio::test] +#[ignore = "requires current Linux wrapper and fixture artifacts"] +async fn success_with_unavailable_delivery_is_distinct() { + delivered("read", json!({"outcome":"succeeded","delivery":"unavailable","reason":"unsupported_content","resultUnavailable":true}), |v| { + assert_eq!(v["outcome"], "succeeded"); assert_eq!(v["delivery"], "unavailable"); assert_eq!(v["resultUnavailable"], true); + assert_eq!(v["isError"], Value::Null); assert_eq!(v["textSentinel"], false); assert_eq!(v["structuredSentinel"], false); + if let Some(success) = v.get("success") { assert_eq!(success, &json!(false)); } + }, false).await; +} +#[tokio::test] +#[ignore = "requires current Linux wrapper and fixture artifacts"] +async fn denied_runtime_receipt_does_not_invent_an_outcome() { + delivered("denied", json!({"source":"runtime","delivery":"available","denial":"plan_mode","output":{"content":[{"type":"text","text":"denied"}],"isError":true},"resultUnavailable":false}), |v| { + assert_eq!(v["denialPlanMode"], true); assert_eq!(v["isError"], true); assert_eq!(v["outcome"], Value::Null); assert_eq!(v["delivery"], "available"); + }, false).await; +} +#[tokio::test] +#[ignore = "requires current Linux wrapper and fixture artifacts"] +async fn deferred_write_delivers_after_park() { + delivered("write", output(false), |v| assert_receipt(v, false), true).await; +} +#[tokio::test] +#[ignore = "requires current Linux wrapper and fixture artifacts"] +async fn omitted_forged_tool_is_rejected_by_wrapper() { + for wrapper in [Wrapper::Claude, Wrapper::Codex] { + let mut h = Harness::start(wrapper, "forged", false).await; + h.prompt().await; + assert_eq!( + h.proof().await, + json!({"callId":CALL,"tool":"fixture_forged","forgedRejected":true}) + ); + let trace = h.finish(0).await; + assert!(trace + .iter() + .any(|e| matches!(e, HarnessEvent::RunCompleted { .. }))); + } +} + +#[tokio::test] +#[ignore = "requires current Linux wrapper and fixture artifacts"] +async fn empty_manifest_absent_tool_returns_native_denial_proof() { + let plan = json!({"steps":[{"marker":"missing","tool":"mcp_read_counter","arguments":{},"callId":"grant-missing","resultFormat":"mcp","expectExposure":"absent"}]}); + let mut h = Harness::start_with_plan(Wrapper::Claude, "read", false, Some(&plan)).await; + h.send(HarnessCommand::Prompt { + text: "[[fixture:missing]]".into(), + prompt_id: "fixture-missing".into(), + mode: None, + }) + .await; + h.until(|event| matches!(event, HarnessEvent::RunCompleted { .. })) + .await; + let proofs: Vec<_> = h + .proofs() + .into_iter() + .filter(|path| { + path.file_name() + .unwrap() + .to_string_lossy() + .starts_with("launch-") + }) + .collect(); + assert_eq!(proofs.len(), 1); + let proof: Value = serde_json::from_slice(&fs::read(&proofs[0]).unwrap()).unwrap(); + assert_eq!( + proof, + json!({ + "callId":"grant-missing", + "tool":"mcp_read_counter", + "selectedToolName":"mcp_read_counter", + "planSha256":"sha256:8e9af5f9f44fe85c47bd540d29ca82f2084934aef94a5fb9ce9655439e06a3c5", + "exposedMcpToolNamesSha256":"sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "expectExposure":"absent", + "requestAttempted":true, + "protocolDenialCode":"unknown_tool", + "nativeDenialCode":-32602 + }) + ); + h.finish(0).await; +} +#[tokio::test] +#[ignore = "requires current Linux wrapper and fixture artifacts"] +async fn held_call_interrupt_emits_cancel_without_result_claim() { + for wrapper in [Wrapper::Claude, Wrapper::Codex] { + let mut h = Harness::start(wrapper, "held-call", false).await; + h.prompt().await; + h.requested("held-call").await; + h.send(HarnessCommand::Interrupt).await; + let mut canceled = false; + let mut interrupted = false; + timeout(LIMIT, async { + while !canceled || !interrupted { + match h.event().await { + HarnessEvent::ToolCallCancelRequested { call_id, name, .. } => { + assert_eq!(call_id, CALL); + assert_eq!(name, TOOL); + canceled = true; + } + HarnessEvent::RunInterrupted { .. } => interrupted = true, + HarnessEvent::ToolCallCompleted { .. } => panic!("held call has no result"), + _ => {} + } + } + }) + .await + .expect("cancel and interrupt deadline"); + assert!(h.proofs().iter().all(|p| fs::read(p).unwrap().is_empty())); + assert_eq!(h.requests(), 1); + // No late result is sent: this proves the cancellation signal only, + // never an external effect, rollback, or late-delivery guarantee. + h.finish(1).await; + } +} +#[tokio::test] +#[ignore = "requires current Linux wrapper and fixture artifacts"] +async fn stable_call_id_refires_after_model_cli_death() { + for wrapper in [Wrapper::Claude, Wrapper::Codex] { + timeout(Duration::from_secs(45), async { + let mut h = Harness::start(wrapper, "refire", true).await; + h.prompt().await; + h.requested("refire").await; + h.until(|e| matches!(e, HarnessEvent::Parked)).await; + h.kill_first_cli().await; + if wrapper == Wrapper::Codex { + // Codex resumes its thread, then a new model turn reissues the + // same native call ID. It does not promise Claude-style auto-refire. + h.until(|e| matches!(e, HarnessEvent::Idle)).await; + h.send(HarnessCommand::Prompt { + text: "fixture refire".into(), + prompt_id: "fixture-refire".into(), + mode: None, + }) + .await; + h.requested("refire").await; + } + h.result(output(false)).await; + assert_receipt(&h.proof().await, false); + assert_eq!(h.requests(), if wrapper == Wrapper::Claude { 1 } else { 2 }); + h.finish(if wrapper == Wrapper::Claude { 1 } else { 2 }) + .await; + }) + .await + .expect("refire scenario deadline"); + } +} + +/// Only the external supervisor invokes this case. It SIGKILLs this process +/// after both real wrappers are holding calls; ordinary suite selection skips it. +#[tokio::test] +#[ignore = "requires the external deadline supervisor"] +async fn supervisor_timeout_fixture() { + let mut held = Vec::new(); + let mut pairs = Vec::new(); + for wrapper in [Wrapper::Claude, Wrapper::Codex] { + let mut h = Harness::start(wrapper, "held-call", false).await; + h.prompt().await; + h.requested("held-call").await; + let proofs = h.proofs(); + assert_eq!(proofs.len(), 1); + let fixture: u32 = proofs[0] + .file_stem() + .unwrap() + .to_str() + .unwrap() + .strip_prefix("launch-") + .unwrap() + .parse() + .unwrap(); + pairs.push([h.process.child.id(), fixture]); + held.push(h); + } + let root = PathBuf::from(std::env::var_os("ENGRAM_TEST_STATE_ROOT").unwrap()); + let staged = root.join("cleanup-ready.tmp"); + fs::write(&staged, serde_json::to_vec(&pairs).unwrap()).unwrap(); + fs::rename(staged, root.join("cleanup-ready")).unwrap(); + std::future::pending::<()>().await; + drop(held); +} diff --git a/crates/engram-harness-codex/src/main.rs b/crates/engram-harness-codex/src/main.rs index dad0c4e00..14c854219 100644 --- a/crates/engram-harness-codex/src/main.rs +++ b/crates/engram-harness-codex/src/main.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use clap::Parser; use engram_core::SessionId; +use engram_harness_proto::tool_result::{McpToolResult, ResultFormat}; use engram_harness_proto::{ read_msg, write_msg, AgentRole, FileChange, ForgeOp, ForgeRequest, ForgeResponse, HarnessCommand, HarnessEvent, FORGE_VSOCK_PORT, @@ -31,6 +32,7 @@ struct ManifestTool { description: String, input_schema: Value, execution: ToolExecution, + result_format: ResultFormat, codex_native_binding: Option, } @@ -82,6 +84,12 @@ fn parse_tool_manifest(raw: &str) -> Result { .cloned() .ok_or("ENGRAM_TOOLS entry missing inputSchema")?, execution, + result_format: object + .get("resultFormat") + .map(|value| serde_json::from_value(value.clone())) + .transpose() + .map_err(|error| error.to_string())? + .unwrap_or_default(), codex_native_binding, }) }) @@ -113,6 +121,7 @@ fn with_browser_view(mut manifest: ToolManifest, enabled: bool) -> ToolManifest description: browser_view::TOOL_DESCRIPTION.into(), input_schema: browser_view::input_schema(), execution: ToolExecution::Sync, + result_format: ResultFormat::Json, codex_native_binding: None, }); } @@ -487,6 +496,7 @@ enum FollowUpCompletion { Tool { name: String, result_summary: String, + ok: bool, }, } @@ -788,6 +798,33 @@ impl AppServer { Ok(server) } + async fn accepted_follow_up(&mut self, client_id: &str) -> Result { + if self.persisted_prompts.contains_key(client_id) { + return Ok(true); + } + let response = tokio::time::timeout( + std::time::Duration::from_secs(5), + self.request_wait( + "thread/read", + json!({"threadId":self.thread_id,"includeTurns":true}), + ), + ) + .await + .map_err(|_| "Codex history query timed out".to_string())??; + if response + .pointer("/result/thread/id") + .and_then(Value::as_str) + != Some(self.thread_id.as_str()) + || !response + .pointer("/result/thread/turns") + .is_some_and(Value::is_array) + { + return Err("Codex history response is incomplete".into()); + } + self.persisted_prompts.extend(persisted_prompts(&response)); + Ok(self.persisted_prompts.contains_key(client_id)) + } + async fn send_request(&mut self, method: &str, params: Value) -> Result { let id = self.next_id; self.next_id += 1; @@ -1092,6 +1129,7 @@ async fn drive( summary: Some(engram_harness_sdk::truncate_utf8(&prompt.text, 1024)), }).await; queued.push_back(prompt); + request_tool_cancellations(parked, events, Some(&turn_id)).await; interrupt_turn(server, &mut pending, &mut interrupt, &turn_id, "mode change").await; } else if let Some(turn_id) = active.as_deref() { let id = server.send_request("turn/steer", json!({ @@ -1130,6 +1168,7 @@ async fn drive( } } Some(HarnessCommand::Interrupt) => { + request_tool_cancellations(parked, events, active.as_deref()).await; if let Some(turn_id) = active.clone() { interrupt_turn(server, &mut pending, &mut interrupt, &turn_id, "operator interrupt").await; } @@ -1150,7 +1189,10 @@ async fn drive( ) .await; } - Some(HarnessCommand::Shutdown { .. }) => return DriveOutcome::Shutdown, + Some(HarnessCommand::Shutdown { .. }) => { + request_tool_cancellations(parked, events, None).await; + return DriveOutcome::Shutdown; + }, Some(HarnessCommand::Checkpoint { .. }) => {} None => { if let Some(run_id) = active.take() { @@ -1408,7 +1450,7 @@ async fn handle_dynamic_tool_call( request_id, tool_name, server.generation, - json!({"execution":execution}), + json!({"execution":execution,"resultFormat":tool.result_format,"runId":params.get("turnId").and_then(Value::as_str).or(active.as_deref()).unwrap_or("")}), ); if let Err(error) = parked.record(call) { tracing::error!(%error, %call_id, "could not durably park Codex dynamic tool call"); @@ -1483,7 +1525,75 @@ impl InterruptWatchdog { } } -/// Ask codex to end `turn_id` and start the clock. The ONLY way to interrupt. +// Completion receipts contain identity and outcome only. They stay with the +// session because native completion does not acknowledge the host result outbox. +fn external_completion(call: &ParkedCall, ok: bool) -> HarnessEvent { + HarnessEvent::ToolCallCompleted { + run_id: call.context["runId"].as_str().unwrap_or_default().into(), + tool_call_id: call.tool_call_id.clone(), + tool_name: call.tool_name.clone(), + ok, + duration_ms: 0, + result_summary: None, + } +} + +fn record_external_completion( + parked: &mut ParkedCallStore, + mut call: ParkedCall, + ok: bool, +) -> std::io::Result<()> { + call.context = json!({ + "resultFormat": "mcp", + "runId": call.context["runId"], + "deliveryCompleted": ok, + }); + call.request_id = Value::Null; + call.request_generation = 0; + parked.record(call) +} + +/// A whole-turn interrupt is the only confirmed native cancellation signal. +/// Retain the parked record so a late journal result can still be delivered. +async fn request_tool_cancellations( + parked: &ParkedCallStore, + events: &mpsc::Sender, + run_id: Option<&str>, +) { + for call in parked.all() { + if call.context.get("resultFormat").and_then(Value::as_str) != Some("mcp") + || call + .context + .get("deliveryCompleted") + .and_then(Value::as_bool) + .is_some() + { + continue; + } + let Some(original_run) = call + .context + .get("runId") + .and_then(Value::as_str) + .filter(|run| !run.is_empty()) + else { + continue; + }; + if run_id.is_some_and(|run| run != original_run) { + continue; + } + emit( + events, + HarnessEvent::ToolCallCancelRequested { + run_id: original_run.into(), + call_id: call.tool_call_id, + name: call.tool_name, + }, + ) + .await; + } +} + +/// Ask Codex to end `turn_id` and start the interrupt clock. async fn interrupt_turn( server: &mut AppServer, pending: &mut HashMap, @@ -1528,11 +1638,60 @@ async fn route_tool_result( tracing::error!(%call_id, kind = ?call.kind, "ToolResult does not match a dynamic tool call"); return; } + let is_mcp = call.context.get("resultFormat").and_then(Value::as_str) == Some("mcp"); + if is_mcp { + if let Some(ok) = call + .context + .get("deliveryCompleted") + .and_then(Value::as_bool) + { + emit(events, external_completion(&call, ok)).await; + return; + } + if pending.values().any(|request| matches!(request, Pending::FollowUp { call_id: pending_id, .. } if pending_id == call_id)) + || (call.request_generation == server.generation + && !call.request_id.is_null() + && call.context.get("responseSent").and_then(Value::as_bool) == Some(true)) + { + return; + } + if let Some(client_id) = call.context.get("followUpId").and_then(Value::as_str) { + let Some(ok) = call.context.get("followUpOk").and_then(Value::as_bool) else { + tracing::error!(%call_id, "Codex follow-up intent has no outcome"); + return; + }; + match server.accepted_follow_up(client_id).await { + Ok(true) => { + if let Err(error) = record_external_completion(parked, call.clone(), ok) { + tracing::error!(%error, %call_id, "could not record recovered Codex result completion"); + return; + } + emit(events, external_completion(&call, ok)).await; + return; + } + Ok(false) => {} + Err(error) => { + tracing::error!(%error, %call_id, "could not reconcile Codex follow-up delivery"); + return; + } + } + } + } + let mcp_result = is_mcp.then(|| McpToolResult::parse(&result_json)); + let result_ok = call + .context + .get("followUpOk") + .and_then(Value::as_bool) + .unwrap_or_else(|| { + mcp_result + .as_ref() + .is_none_or(|result| result.as_ref().is_ok_and(McpToolResult::success)) + }); // ADR 0107: an approved plan flips the stamp FIRST, so every turn that // starts after this point (the live build turn below, or the // stale-generation follow-up) reads full access. A reject leaves the plan // stamp in place, so its revision turn stays read-only. - let plan_decision = (call.tool_name == "exit_plan_mode") + let plan_decision = (call.tool_name == "exit_plan_mode" && !is_mcp) .then(|| engram_harness_sdk::plan::parse_plan_decision(&result_json)) .flatten(); // ADR 0107: a REJECT ends the plan turn and starts a new one carrying the @@ -1592,8 +1751,10 @@ async fn route_tool_result( let completion = FollowUpCompletion::Tool { name: call.tool_name.clone(), result_summary: result_json, + ok: false, }; - if let Err(error) = send_follow_up(server, active, pending, call_id, text, completion).await + if let Err(error) = + send_follow_up(server, parked, active, pending, call_id, text, completion).await { tracing::error!(%error, %call_id, "could not deliver plan rejection as user message"); } @@ -1662,9 +1823,11 @@ async fn route_tool_result( let completion = FollowUpCompletion::Tool { name: call.tool_name.clone(), result_summary: result_json, + ok: true, }; if let Err(error) = send_follow_up( server, + parked, active, pending, call_id, @@ -1677,8 +1840,8 @@ async fn route_tool_result( } return; } - let native_question = - call.context.get("nativeBinding").and_then(Value::as_str) == Some("requestUserInput"); + let native_question = !is_mcp + && call.context.get("nativeBinding").and_then(Value::as_str) == Some("requestUserInput"); let answers = if native_question { match serde_json::from_str::(&result_json) { Ok(answers) => Some(answers), @@ -1691,7 +1854,16 @@ async fn route_tool_result( None }; if call.request_generation != server.generation || call.request_id.is_null() { - let message = if native_question { + let message = if let Some(result) = &mcp_result { + let text = match result { + Ok(result) => result.text_items().join("\n\n"), + Err(message) => (*message).to_string(), + }; + format!( + "Result of the earlier external tool call `{}` ({call_id}):\n{text}", + call.tool_name + ) + } else if native_question { format!( "Answer to the earlier question ({call_id}) after Codex restarted: {result_json}" ) @@ -1704,9 +1876,12 @@ async fn route_tool_result( let completion = FollowUpCompletion::Tool { name: call.tool_name.clone(), result_summary: result_json, + ok: result_ok, }; - if let Err(error) = - send_follow_up(server, active, pending, call_id, message, completion).await + if let Err(error) = send_follow_up( + server, parked, active, pending, call_id, message, completion, + ) + .await { tracing::error!(%error, %call_id, "could not deliver late ToolResult as user message"); } @@ -1715,7 +1890,7 @@ async fn route_tool_result( if native_question { let codex_answers = codex_answers(&call, answers.as_ref().expect("parsed above")); if let Err(error) = server - .respond(call.request_id, json!({"answers":codex_answers})) + .respond(call.request_id.clone(), json!({"answers":codex_answers})) .await { tracing::error!(%error, %call_id, "could not answer Codex requestUserInput"); @@ -1723,18 +1898,28 @@ async fn route_tool_result( } } else if let Err(error) = server .respond( - call.request_id, - json!({ - "success":true, - "contentItems":[{"type":"inputText","text":result_json}] - }), + call.request_id.clone(), + if let Some(result) = &mcp_result { + result + .as_ref() + .map(|result| result.codex_response()) + .unwrap_or_else(|_| engram_harness_proto::tool_result::invalid_codex_result()) + } else { + json!({"success":true,"contentItems":[{"type":"inputText","text":result_json}]}) + }, ) .await { tracing::error!(%error, %call_id, "could not answer Codex dynamic tool call"); return; } - if let Err(error) = parked.take(call_id) { + if is_mcp { + let mut delivered = call.clone(); + delivered.context["responseSent"] = Value::Bool(true); + if let Err(error) = parked.record(delivered) { + tracing::error!(%error, %call_id, "could not record Codex response delivery"); + } + } else if let Err(error) = parked.take(call_id) { tracing::error!(%error, %call_id, "could not retire answered Codex dynamic tool call"); } if native_question { @@ -1770,13 +1955,32 @@ fn codex_answers(call: &ParkedCall, answers: &Answers) -> serde_json::Map, pending: &mut HashMap, call_id: &str, text: String, completion: FollowUpCompletion, ) -> Result<(), String> { - let client_id = format!("codex-follow-up-{}", uuid::Uuid::new_v4()); + let client_id = if let Some(mut call) = parked + .get(call_id) + .cloned() + .filter(|call| call.context["resultFormat"] == "mcp") + { + if let Some(id) = call.context.get("followUpId").and_then(Value::as_str) { + id.to_owned() + } else { + let id = format!("codex-follow-up-{}", uuid::Uuid::new_v4()); + let FollowUpCompletion::Tool { ok, .. } = &completion; + call.context["followUpId"] = json!(id); + call.context["followUpOk"] = json!(ok); + // Native acceptance can precede its reply and the completion receipt. + parked.record(call).map_err(|error| error.to_string())?; + id + } + } else { + format!("codex-follow-up-{}", uuid::Uuid::new_v4()) + }; let request_id = if let Some(turn_id) = active.as_deref() { server .send_request( @@ -1950,6 +2154,20 @@ async fn handle_message( call_id, completion, } if value.get("error").is_none() => { + let external = tools + .parked + .get(&call_id) + .cloned() + .filter(|call| call.context["resultFormat"] == "mcp"); + if let Some(call) = &external { + let FollowUpCompletion::Tool { ok, .. } = &completion; + if let Err(error) = + record_external_completion(tools.parked, call.clone(), *ok) + { + tracing::error!(%error, %call_id, "could not record late Codex result completion"); + return; + } + } if let Some(turn_id) = value.pointer("/result/turn/id").and_then(Value::as_str) { *active = Some(turn_id.to_owned()); @@ -1963,6 +2181,11 @@ async fn handle_message( ) .await; } + if let Some(call) = external { + let FollowUpCompletion::Tool { ok, .. } = completion; + emit(events, external_completion(&call, ok)).await; + return; + } if let Err(error) = tools.parked.take(&call_id) { tracing::error!(%error, %call_id, "could not retire crash-degraded Codex call"); } @@ -1970,6 +2193,7 @@ async fn handle_message( FollowUpCompletion::Tool { name, result_summary, + ok, } => { emit( events, @@ -1977,7 +2201,7 @@ async fn handle_message( run_id: active.clone().unwrap_or_default(), tool_call_id: call_id, tool_name: name, - ok: true, + ok, duration_ms: 0, result_summary: Some(engram_harness_sdk::truncate_utf8( &result_summary, @@ -2030,6 +2254,47 @@ async fn handle_message( } "item/completed" => { if let Some(item) = params.get("item") { + if let Some(call) = item + .get("id") + .and_then(Value::as_str) + .and_then(|id| tools.parked.get(id)) + .cloned() + .filter(|call| { + call.context["resultFormat"] == "mcp" && item["type"] == "dynamicToolCall" + }) + { + if let Some(ok) = call + .context + .get("deliveryCompleted") + .and_then(Value::as_bool) + { + emit(events, external_completion(&call, ok)).await; + return; + } + // A native cancellation can finish the item before the + // journal result arrives. Deliver that result as a late + // follow-up instead of acknowledging an unseen result. + if call.context.get("responseSent").and_then(Value::as_bool) != Some(true) { + let mut stale = call; + stale.request_id = Value::Null; + stale.request_generation = 0; + if let Err(error) = tools.parked.record(stale) { + tracing::error!(%error, "could not retain completed native call for late delivery"); + } + return; + } + let status = item.get("status").and_then(Value::as_str); + let ok = item + .get("success") + .and_then(Value::as_bool) + .unwrap_or(status.is_none() || status == Some("completed")); + if let Err(error) = record_external_completion(tools.parked, call.clone(), ok) { + tracing::error!(%error, call_id = %call.tool_call_id, "could not record native Codex completion"); + return; + } + emit(events, external_completion(&call, ok)).await; + return; + } emit_item_completed(events, &run_id, item).await; } } @@ -2082,6 +2347,7 @@ async fn handle_message( .await; } if status == "interrupted" { + request_tool_cancellations(tools.parked, events, Some(&completed)).await; emit( events, HarnessEvent::RunInterrupted { @@ -3063,7 +3329,8 @@ done let manifest = r#"[ {"name":"save_memory","description":"Save a memory","inputSchema":{"type":"object","properties":{"text":{"type":"string"}}},"execution":"sync","nativeBindings":{}}, {"name":"ask_user_question","description":"Ask the user","inputSchema":{"type":"object"},"execution":"deferred","nativeBindings":{"codex":"requestUserInput"}}, - {"name":"claude_native","description":"Only native on Claude","inputSchema":{"type":"object"},"execution":"sync","nativeBindings":{"claude":"Example"}} + {"name":"claude_native","description":"Only native on Claude","inputSchema":{"type":"object"},"execution":"sync","nativeBindings":{"claude":"Example"}}, + {"name":"remote_inventory","description":"Inventory","inputSchema":{"type":"object","properties":{"code":{"type":"string","enum":["a","b"]}},"required":["code"],"additionalProperties":false},"execution":"sync","nativeBindings":{},"resultFormat":"mcp"} ]"#; std::env::set_var("ENGRAM_TOOLS", manifest); let (script, record) = write_fake_codex(&[]).await; @@ -3095,6 +3362,11 @@ done "name":"claude_native", "description":"Only native on Claude", "inputSchema":{"type":"object"} + }, + { + "name":"remote_inventory", + "description":"Inventory", + "inputSchema":{"type":"object","properties":{"code":{"type":"string","enum":["a","b"]}},"required":["code"],"additionalProperties":false} } ])) ); @@ -3520,6 +3792,328 @@ done engine.abort(); } + async fn assert_external_mcp_dynamic_result( + result_json: &str, + native_result: Value, + deferred: bool, + ) { + let tool_call = r#"{"id":77,"method":"item/tool/call","params":{"callId":"call-1","tool":"mcp_inventory_01234567890123456789","arguments":{"text":"remember this"},"threadId":"t1","turnId":"turn-1"}}"#; + let (script, record) = write_fake_codex(&[tool_call]).await; + let mut cli = test_cli(script); + let stamp = cli.state().mode_stamp(); + cli.tool_manifest = parse_tool_manifest( + r#"[{"name":"mcp_inventory_01234567890123456789","description":"Save a memory","inputSchema":{"type":"object"},"execution":"sync","nativeBindings":{},"resultFormat":"mcp"}]"#, + ) + .unwrap(); + if deferred { + cli.tool_manifest[0].execution = ToolExecution::Deferred; + } + let (command_tx, command_rx) = mpsc::channel(8); + let (event_tx, mut event_rx) = mpsc::channel(16); + let engine = tokio::spawn(run_engine( + cli, + command_rx, + Arc::new(Notify::new()), + event_tx, + )); + command_tx + .send(HarnessCommand::Prompt { + prompt_id: "prompt-1".into(), + text: "remember".into(), + mode: Some("plan".into()), + }) + .await + .unwrap(); + + let event = tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if let Some(HarnessEvent::ToolCallRequested { + run_id, + call_id, + name, + args_json, + }) = event_rx.recv().await + { + break (run_id, call_id, name, args_json); + } + } + }) + .await + .expect("dynamic tool request timed out"); + assert_eq!( + event, + ( + "turn-1".into(), + "call-1".into(), + "mcp_inventory_01234567890123456789".into(), + r#"{"text":"remember this"}"#.into(), + ) + ); + let parked = tokio::time::timeout(std::time::Duration::from_millis(100), async { + loop { + if matches!(event_rx.recv().await, Some(HarnessEvent::Parked)) { + break; + } + } + }) + .await; + assert_eq!(parked.is_ok(), deferred, "only deferred tools emit Parked"); + + command_tx + .send(HarnessCommand::ToolResult { + call_id: "call-1".into(), + result_json: result_json.into(), + }) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + let requests = recorded_requests(&record).await; + if requests.iter().any(|request| { + request + == &json!({ + "id":77, + "result":native_result + }) + }) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + }) + .await + .expect("dynamic tool result response timed out"); + assert_eq!( + engram_harness_sdk::mode_stamp::read_mode_stamp(&stamp), + "plan" + ); + engine.abort(); + } + + #[tokio::test] + async fn external_mcp_dynamic_result_preserves_fields_and_failure() { + assert_external_mcp_dynamic_result( + r#"{"outcome":"failed","delivery":"available","output":{"content":[{"type":"text","text":"first"},{"type":"text","text":"second"}],"structuredContent":{"count":2},"isError":true},"resultUnavailable":false}"#, + json!({"success":false,"contentItems":[ + {"type":"inputText","text":"first"}, + {"type":"inputText","text":"second"}, + {"type":"inputText","text":r#"{"structuredContent":{"count":2}}"#}, + {"type":"inputText","text":r#"{"delivery":"available","isError":true,"outcome":"failed","resultUnavailable":false}"#} + ]}), false, + ).await; + } + + #[tokio::test] + async fn external_mcp_structured_only_decision_cannot_relax_plan() { + assert_external_mcp_dynamic_result( + r#"{"outcome":"succeeded","delivery":"available","output":{"content":[],"structuredContent":{"decision":"approve"}},"resultUnavailable":false}"#, + json!({"success":true,"contentItems":[ + {"type":"inputText","text":r#"{"structuredContent":{"decision":"approve"}}"#}, + {"type":"inputText","text":r#"{"delivery":"available","outcome":"succeeded","resultUnavailable":false}"#} + ]}), false, + ).await; + } + + #[tokio::test] + async fn external_mcp_delivery_failure_keeps_established_success() { + assert_external_mcp_dynamic_result( + r#"{"outcome":"succeeded","delivery":"unavailable","reason":"unsupported_content","resultUnavailable":true}"#, + json!({"success":false,"contentItems":[ + {"type":"inputText","text":r#"{"delivery":"unavailable","outcome":"succeeded","reason":"unsupported_content","resultUnavailable":true}"#} + ]}), false, + ).await; + } + + #[tokio::test] + async fn external_mcp_deferred_write_denial_keeps_plan_mode() { + assert_external_mcp_dynamic_result( + r#"{"source":"runtime","denial":"write_not_allowed","delivery":"available","output":{"content":[{"type":"text","text":"External tool request denied."}],"isError":true}}"#, + json!({"success":false,"contentItems":[ + {"type":"inputText","text":"External tool request denied."}, + {"type":"inputText","text":r#"{"delivery":"available","denial":"write_not_allowed","isError":true,"source":"runtime"}"#} + ]}), true, + ).await; + } + + #[tokio::test] + async fn external_mcp_interrupt_requests_cancel_without_completing_parked_call() { + let tool_call = r#"{"id":77,"method":"item/tool/call","params":{"callId":"call-cancel","tool":"remote_inventory","arguments":{},"threadId":"t1","turnId":"turn-1"}}"#; + let (script, _record) = write_fake_codex(&[tool_call]).await; + let mut cli = test_cli(script); + cli.tool_manifest = parse_tool_manifest(r#"[{"name":"remote_inventory","description":"Inventory","inputSchema":{"type":"object"},"execution":"sync","nativeBindings":{},"resultFormat":"mcp"}]"#).unwrap(); + let parked_path = cli.parked_calls_file(); + let (commands, receiver) = mpsc::channel(8); + let (events, mut received) = mpsc::channel(16); + let engine = tokio::spawn(run_engine(cli, receiver, Arc::new(Notify::new()), events)); + commands + .send(HarnessCommand::Prompt { + prompt_id: "p-cancel".into(), + text: "read".into(), + mode: None, + }) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while let Some(event) = received.recv().await { + if event.kind() == "tool_call_requested" { + break; + } + } + }) + .await + .unwrap(); + commands.send(HarnessCommand::Interrupt).await.unwrap(); + let cancel = tokio::time::timeout(std::time::Duration::from_secs(3), async { + while let Some(event) = received.recv().await { + assert_ne!( + event.kind(), + "tool_call_completed", + "interrupt cannot claim completion" + ); + if event.kind() == "tool_call_cancel_requested" { + return event; + } + } + panic!("event stream ended"); + }) + .await + .expect("interrupt must request cancellation"); + assert_eq!(cancel.run_id(), Some("turn-1")); + assert_eq!(cancel.tool_call_id(), Some("call-cancel")); + assert!(ParkedCallStore::open(parked_path) + .unwrap() + .get("call-cancel") + .is_some()); + engine.abort(); + } + + // Explicit fixture gate: the caller builds the test-only CLI and supplies + // a private, empty state directory and a new proof path for each run. + #[tokio::test] + #[ignore = "requires the separately built MCP model fixture"] + async fn external_mcp_static_fixture_adapter_gate() { + let binary = PathBuf::from(std::env::var_os("ENGRAM_FIXTURE_BIN").expect("fixture binary")); + let mode = std::env::var("ENGRAM_FIXTURE_MODE").expect("fixture mode"); + assert!(["read", "held-call"].contains(&mode.as_str())); + assert_eq!( + std::env::var("ENGRAM_FIXTURE_TOOL").as_deref(), + Ok("remote_inventory") + ); + let state = + PathBuf::from(std::env::var_os("ENGRAM_FIXTURE_STATE_DIR").expect("fixture state")); + let proof = PathBuf::from(std::env::var_os("ENGRAM_FIXTURE_PROOF").expect("fixture proof")); + assert!(binary.is_absolute() && binary.is_file() && state.is_absolute()); + assert_eq!(proof.parent(), Some(state.as_path())); + assert!(!proof.exists()); + let version = tokio::time::timeout( + std::time::Duration::from_secs(3), + tokio::process::Command::new(&binary) + .arg("--fixture-version") + .kill_on_drop(true) + .output(), + ) + .await + .expect("fixture version timed out") + .expect("fixture version process"); + assert!(version.status.success()); + assert_eq!(version.stdout, b"engram-mcp-model-fixture 1\n"); + let mut cli = test_cli(binary); + cli.tool_manifest = parse_tool_manifest(r#"[{"name":"remote_inventory","description":"Fixture tool","inputSchema":{"type":"object"},"execution":"sync","nativeBindings":{},"resultFormat":"mcp"}]"#).unwrap(); + let harness_state = cli.state_dir.clone(); + let (commands, receiver) = mpsc::channel(8); + let (events, mut received) = mpsc::channel(16); + let engine = tokio::spawn(run_engine(cli, receiver, Arc::new(Notify::new()), events)); + commands + .send(HarnessCommand::Prompt { + prompt_id: "fixture-prompt".into(), + text: "fixture".into(), + mode: None, + }) + .await + .unwrap(); + let requested = tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let event = received.recv().await.expect("fixture event stream"); + if let HarnessEvent::ToolCallRequested { + run_id, + call_id, + name, + args_json, + } = event + { + break (run_id, call_id, name, args_json); + } + } + }) + .await + .expect("fixture call timed out"); + assert_eq!(requested.0, "fixture-turn"); + assert_eq!(requested.1, "fixture-call"); + assert_eq!(requested.2, "remote_inventory"); + assert_eq!( + serde_json::from_str::(&requested.3).unwrap(), + json!({"fixture":mode}) + ); + if mode == "held-call" { + commands.send(HarnessCommand::Interrupt).await.unwrap(); + let cancel = tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let event = received.recv().await.expect("fixture event stream"); + assert_ne!(event.kind(), "tool_call_completed"); + if event.kind() == "tool_call_cancel_requested" { + break event; + } + } + }) + .await + .expect("fixture cancellation timed out"); + assert_eq!(cancel.tool_call_id(), Some("fixture-call")); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let event = received.recv().await.expect("fixture event stream"); + assert_ne!(event.kind(), "tool_call_completed"); + if event.kind() == "run_interrupted" { + break; + } + } + }) + .await + .expect("fixture turn interruption timed out"); + assert!(tokio::fs::read(&proof).await.unwrap().is_empty()); + } else { + commands.send(HarnessCommand::ToolResult { call_id: "fixture-call".into(), result_json: r#"{"outcome":"succeeded","delivery":"available","output":{"content":[{"type":"text","text":"sentinel"},{"type":"text","text":"second sentinel"}],"structuredContent":{"fixture":true},"isError":false},"resultUnavailable":false}"#.into() }).await.unwrap(); + let observed = tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if let Ok(bytes) = tokio::fs::read(&proof).await { + if let Ok(value) = serde_json::from_slice::(&bytes) { + break value; + } + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + }) + .await + .expect("fixture receipt timed out"); + assert_eq!(observed["success"], true); + assert_eq!(observed["textSequence"], true); + assert_eq!(observed["structuredSentinel"], true); + assert_eq!(observed["outcome"], "succeeded"); + assert_eq!(observed["delivery"], "available"); + assert_eq!(observed["isError"], false); + assert_eq!(observed["resultUnavailable"], false); + } + commands + .send(HarnessCommand::Shutdown { grace_secs: 1 }) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), engine) + .await + .expect("fixture engine shutdown") + .unwrap(); + let _ = tokio::fs::remove_dir_all(&harness_state).await; + let _ = tokio::fs::remove_file(harness_state.with_extension("attach-token")).await; + } + #[tokio::test] async fn native_request_user_input_uses_generic_frames_and_is_parked_durably() { let question = r#"{"id":88,"method":"item/tool/requestUserInput","params":{"itemId":"question-1","threadId":"t1","turnId":"turn-1","questions":[{"id":"q1","question":"Deploy now?","header":"Deploy","multiSelect":false,"options":[{"label":"Yes","description":"Deploy it"}]}]}}"#; @@ -3768,6 +4362,465 @@ done assert_eq!(exit, ExitCode::SUCCESS); } + #[tokio::test] + async fn external_mcp_cold_reopen_preserves_failed_result_and_plan_latch() { + assert_external_mcp_cold_reopen(ResultCrashPoint::BeforeDelivery).await; + } + + #[tokio::test] + async fn external_mcp_accepted_response_survives_harness_death() { + assert_external_mcp_cold_reopen(ResultCrashPoint::AfterResponse).await; + } + + #[tokio::test] + async fn external_mcp_native_completion_receipt_survives_harness_death() { + assert_external_mcp_cold_reopen(ResultCrashPoint::AfterCompletion).await; + } + + #[tokio::test] + async fn external_mcp_native_cancellation_does_not_ack_an_unseen_result() { + assert_external_mcp_cold_reopen(ResultCrashPoint::CancelledBeforeResult).await; + } + + #[tokio::test] + async fn external_mcp_accepted_follow_up_survives_harness_death() { + assert_external_mcp_cold_reopen(ResultCrashPoint::AfterFollowUpAcceptance).await; + } + + #[tokio::test] + async fn external_mcp_follow_up_requires_complete_history_before_retry() { + for (history, can_retry) in [ + ( + r#""error":{"code":-32000,"message":"history unavailable"}"#, + false, + ), + (r#""result":{"thread":{"id":"t1"}}"#, false), + (r#""result":{"thread":{"id":"t1","turns":[]}}"#, true), + ] { + let (script, record) = write_fake_codex(&[]).await; + let source = tokio::fs::read_to_string(&script).await.unwrap(); + tokio::fs::write(&script, source.replace( + " turn/interrupt)", + &format!(" thread/read)\n printf '{{\"id\":%s,{history}}}\\n' \"$id\"\n ;;\n turn/interrupt)"), + )).await.unwrap(); + let cli = test_cli(script); + let mut server = AppServer::spawn(&cli, 1).await.unwrap(); + let mut parked = ParkedCallStore::open(cli.parked_calls_file()).unwrap(); + parked.record(ParkedCall::new("late-call", ParkedCallKind::DynamicTool, + Value::Null, "remote_inventory", 0, + json!({"resultFormat":"mcp","runId":"original-run","followUpId":"stable-input","followUpOk":false}), + )).unwrap(); + let (events, mut received) = mpsc::channel(8); + let mut pending = HashMap::new(); + let mut active = None; + let mut queued = VecDeque::new(); + let mut interrupt = InterruptWatchdog::default(); + for _ in 0..2 { + route_tool_result( + &mut server, + &mut parked, + RouteCtx { + pending: &mut pending, + active: &mut active, + queued: &mut queued, + interrupt: &mut interrupt, + }, + &events, + "late-call", + "{}".into(), + ) + .await; + } + server.request_wait("initialize", json!({})).await.unwrap(); + let requests = recorded_requests(&record).await; + let inputs: Vec<_> = requests + .iter() + .filter(|r| matches!(r["method"].as_str(), Some("turn/start" | "turn/steer"))) + .collect(); + assert_eq!(inputs.len(), usize::from(can_retry)); + if can_retry { + assert_eq!(inputs[0]["params"]["clientUserMessageId"], "stable-input"); + } + assert!( + received.try_recv().is_err(), + "a history miss is not delivery" + ); + assert_eq!( + parked.get("late-call").unwrap().context, + json!({"resultFormat":"mcp","runId":"original-run","followUpId":"stable-input","followUpOk":false}) + ); + assert!(requests + .iter() + .any(|r| r["method"] == "thread/read" && r["params"]["includeTurns"] == true)); + } + } + + #[derive(Clone, Copy, PartialEq, Eq)] + enum ResultCrashPoint { + BeforeDelivery, + AfterResponse, + AfterCompletion, + CancelledBeforeResult, + AfterFollowUpAcceptance, + } + + async fn assert_external_mcp_cold_reopen(crash_stage: ResultCrashPoint) { + let result_json = r#"{"outcome":"failed","delivery":"available","output":{"content":[{"type":"text","text":"denied"}],"structuredContent":{"decision":"approve"},"isError":true},"resultUnavailable":false}"#; + let tool_call = r#"{"id":77,"method":"item/tool/call","params":{"callId":"cold-call","tool":"remote_inventory","arguments":{},"threadId":"t1","turnId":"turn-1"}}"#; + let cancellation = r#"{"method":"item/completed","params":{"turnId":"turn-1","item":{"type":"dynamicToolCall","id":"cold-call","tool":"remote_inventory","success":false}}}"#; + let messages = if crash_stage == ResultCrashPoint::CancelledBeforeResult { + vec![tool_call, cancellation] + } else { + vec![tool_call] + }; + let (script, first_record) = write_fake_codex(&messages).await; + if crash_stage == ResultCrashPoint::AfterCompletion { + let source = tokio::fs::read_to_string(&script).await.unwrap(); + tokio::fs::write(&script, source.replace( + " case \"$method\" in", + r#" if [ "$id" = 77 ] && [ -z "$method" ]; then + printf '%s\n' '{"method":"item/completed","params":{"turnId":"turn-1","item":{"type":"dynamicToolCall","id":"cold-call","tool":"remote_inventory","success":false,"result":"must not persist this result"}}}' + fi + case "$method" in"#, + )).await.unwrap(); + } + let mut cli = test_cli(script); + cli.tool_manifest = parse_tool_manifest(r#"[{"name":"remote_inventory","description":"Read","inputSchema":{"type":"object"},"execution":"deferred","nativeBindings":{},"resultFormat":"mcp"}]"#).unwrap(); + let manifest = cli.tool_manifest.clone(); + let state_dir = cli.state_dir.clone(); + let parked_path = cli.parked_calls_file(); + let stamp = cli.state().mode_stamp(); + let (commands, receiver) = mpsc::channel(8); + let (events, mut received) = mpsc::channel(16); + let engine = tokio::spawn(run_engine(cli, receiver, Arc::new(Notify::new()), events)); + commands + .send(HarnessCommand::Prompt { + prompt_id: "cold-prompt".into(), + text: "read".into(), + mode: Some("plan".into()), + }) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while let Some(event) = received.recv().await { + if event.kind() == "tool_call_requested" { + return; + } + } + panic!("missing original call"); + }) + .await + .unwrap(); + if matches!( + crash_stage, + ResultCrashPoint::AfterResponse | ResultCrashPoint::AfterCompletion + ) { + commands + .send(HarnessCommand::ToolResult { + call_id: "cold-call".into(), + result_json: result_json.into(), + }) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if recorded_requests(&first_record) + .await + .iter() + .any(|request| request["id"] == 77 && request.get("result").is_some()) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("native response must be accepted before the crash"); + } + if crash_stage == ResultCrashPoint::AfterCompletion { + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while let Some(event) = received.recv().await { + if matches!(event, HarnessEvent::ToolCallCompleted { ok: false, .. }) { + return; + } + } + panic!("missing native completion"); + }) + .await + .unwrap(); + } + if crash_stage == ResultCrashPoint::CancelledBeforeResult { + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if ParkedCallStore::open(&parked_path) + .unwrap() + .get("cold-call") + .is_some_and(|call| call.request_id.is_null()) + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + while let Ok(event) = received.try_recv() { + assert!( + !matches!(event, HarnessEvent::ToolCallCompleted { .. }), + "an unseen result must stay unacknowledged" + ); + } + } + engine.abort(); + let _ = engine.await; + let persisted = ParkedCallStore::open(&parked_path).unwrap(); + let call = persisted.get("cold-call").unwrap(); + assert_eq!(call.tool_name, "remote_inventory"); + assert_eq!(call.context["resultFormat"], "mcp"); + assert_eq!(call.context["runId"], "turn-1"); + drop(persisted); + let accepted_follow_up = if crash_stage == ResultCrashPoint::AfterFollowUpAcceptance { + let (script, record) = write_fake_codex(&[]).await; + let source = tokio::fs::read_to_string(&script).await.unwrap(); + // The request log is the fake native durable history. Withhold its + // reply so the wrapper cannot persist a completion receipt. + tokio::fs::write(&script, source.replace( + r#" printf '{"id":%s,"result":{"turn":{"id":"turn-1","status":"inProgress"}}}\n' "$id""#, + " :", + )).await.unwrap(); + let mut restored = test_cli(script); + restored.state_dir = state_dir.clone(); + restored.tool_manifest = manifest.clone(); + let (commands, receiver) = mpsc::channel(8); + let (events, _received) = mpsc::channel(32); + let engine = tokio::spawn(run_engine( + restored, + receiver, + Arc::new(Notify::new()), + events, + )); + commands + .send(HarnessCommand::ToolResult { + call_id: "cold-call".into(), + result_json: result_json.into(), + }) + .await + .unwrap(); + let accepted = tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if !record.exists() { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + continue; + } + if let Some(request) = recorded_requests(&record) + .await + .into_iter() + .find(|request| request["method"] == "turn/start") + { + break request; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("native follow-up was not accepted"); + engine.abort(); + let _ = engine.await; + assert!(ParkedCallStore::open(&parked_path) + .unwrap() + .get("cold-call") + .unwrap() + .context + .get("deliveryCompleted") + .is_none()); + assert_eq!( + recorded_requests(&record) + .await + .iter() + .filter(|request| request["method"] == "turn/start") + .count(), + 1 + ); + let client_id = accepted["params"]["clientUserMessageId"].as_str().unwrap(); + let intent_store = ParkedCallStore::open(&parked_path).unwrap(); + let intent = intent_store.get("cold-call").unwrap(); + assert_eq!(intent.context["followUpId"], client_id); + assert_eq!(intent.context["followUpOk"], false); + Some(client_id.to_owned()) + } else { + None + }; + let (script, record) = write_fake_codex(&[]).await; + if let Some(client_id) = &accepted_follow_up { + let source = tokio::fs::read_to_string(&script).await.unwrap(); + let history = json!([{"id":"accepted-turn","status":"completed","items":[{ + "type":"userMessage","clientId":client_id + }]}]); + tokio::fs::write( + &script, + source.replace(r#""turns":[]"#, &format!(r#""turns":{history}"#)), + ) + .await + .unwrap(); + } + let mut restored = test_cli(script); + restored.state_dir = state_dir.clone(); + restored.tool_manifest = manifest.clone(); + let (commands, receiver) = mpsc::channel(8); + let (events, mut received) = mpsc::channel(32); + let engine = tokio::spawn(run_engine( + restored, + receiver, + Arc::new(Notify::new()), + events, + )); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while let Some(event) = received.recv().await { + if event.kind() == "harness_idle" { + return; + } + } + panic!("restored engine did not start"); + }) + .await + .unwrap(); + commands + .send(HarnessCommand::ToolResult { + call_id: "cold-call".into(), + result_json: result_json.into(), + }) + .await + .unwrap(); + let completed = tokio::time::timeout(std::time::Duration::from_secs(3), async { + while let Some(event) = received.recv().await { + if let HarnessEvent::ToolCallCompleted { + run_id, + tool_call_id, + tool_name, + ok, + result_summary, + .. + } = event + { + if tool_call_id == "cold-call" { + assert_eq!(run_id, "turn-1"); + assert_eq!(tool_name, "remote_inventory"); + assert!(result_summary.is_none()); + return ok; + } + } + } + panic!("no result projection"); + }) + .await + .unwrap(); + assert!( + !completed, + "delivery must not turn an upstream failure into success" + ); + let requests = recorded_requests(&record).await; + assert!(requests + .iter() + .any(|request| request["method"] == "thread/resume")); + if matches!( + crash_stage, + ResultCrashPoint::AfterCompletion | ResultCrashPoint::AfterFollowUpAcceptance + ) { + assert!(!requests.iter().any(|request| matches!( + request["method"].as_str(), + Some("turn/start" | "turn/steer") + ))); + } else { + let text = requests + .iter() + .find(|request| request["method"] == "turn/start") + .unwrap()["params"]["input"][0]["text"] + .as_str() + .unwrap() + .to_string(); + assert!(text.contains(r#"{"structuredContent":{"decision":"approve"}}"#)); + assert!(text.contains(r#""isError":true"#)); + assert!( + !text.contains(r#""output":"#), + "present fields separately instead of an opaque envelope" + ); + } + assert_eq!( + engram_harness_sdk::mode_stamp::read_mode_stamp(&stamp), + "plan" + ); + engine.abort(); + let _ = engine.await; + let receipt_store = ParkedCallStore::open(&parked_path).unwrap(); + let receipt = receipt_store.get("cold-call").unwrap(); + assert_eq!( + receipt.context, + json!({"resultFormat":"mcp","runId":"turn-1","deliveryCompleted":false}) + ); + assert!(receipt.request_id.is_null()); + let (cancel_events, mut cancellations) = mpsc::channel(8); + request_tool_cancellations(&receipt_store, &cancel_events, None).await; + assert!( + cancellations.try_recv().is_err(), + "completed receipts are not active calls" + ); + let (script, replay_record) = write_fake_codex(&[]).await; + let mut replay_cli = test_cli(script); + replay_cli.state_dir = state_dir; + replay_cli.tool_manifest = manifest; + let (commands, receiver) = mpsc::channel(8); + let (events, mut received) = mpsc::channel(32); + let engine = tokio::spawn(run_engine( + replay_cli, + receiver, + Arc::new(Notify::new()), + events, + )); + commands + .send(HarnessCommand::ToolResult { + call_id: "cold-call".into(), + result_json: "{}".into(), + }) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + while let Some(event) = received.recv().await { + if let HarnessEvent::ToolCallCompleted { + run_id, + tool_call_id, + tool_name, + ok, + result_summary, + .. + } = event + { + assert_eq!( + ( + run_id.as_str(), + tool_call_id.as_str(), + tool_name.as_str(), + ok + ), + ("turn-1", "cold-call", "remote_inventory", false) + ); + assert!(result_summary.is_none()); + return; + } + } + panic!("receipt was not replayed"); + }) + .await + .unwrap(); + assert!(!recorded_requests(&replay_record) + .await + .iter() + .any(|request| matches!( + request["method"].as_str(), + Some("turn/start" | "turn/steer") + ) || request["id"] == 77)); + engine.abort(); + } + #[tokio::test] async fn native_question_result_after_app_server_crash_becomes_follow_up_user_message() { let (script, record) = write_crashing_question_fake().await; diff --git a/crates/engram-harness-proto/src/lib.rs b/crates/engram-harness-proto/src/lib.rs index ac9f5c1f2..35c54aa72 100644 --- a/crates/engram-harness-proto/src/lib.rs +++ b/crates/engram-harness-proto/src/lib.rs @@ -33,6 +33,7 @@ //! image small (one codec). pub mod attach_token; +pub mod tool_result; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -469,6 +470,21 @@ pub enum HarnessEvent { }, /// The harness re-attached while this run was in flight. RunContinued { run_id: String }, + /// Best-effort cancellation signal for the original call. The trusted + /// runtime must validate session, run and pending tool identity. This + /// event neither completes a journal entry nor reverses an effect. + ToolCallCancelRequested { + run_id: String, + call_id: String, + name: String, + }, + /// Successful turn suspension on these exact, still-pending deferred calls. + /// Appended to preserve existing positional encodings. This is a lifecycle + /// association only; the coordinator must still authorize each request. + RunDeferred { + run_id: String, + deferred_tool_call_ids: Vec, + }, } /// Who emitted an [`HarnessEvent::AgentMessage`]. @@ -497,10 +513,12 @@ impl HarnessEvent { | Self::ToolCallStarted { run_id, .. } | Self::ToolCallCompleted { run_id, .. } | Self::RunCompleted { run_id, .. } + | Self::RunDeferred { run_id, .. } | Self::RunInterrupted { run_id } | Self::AgentMessageChunk { run_id, .. } | Self::FileChanged { run_id, .. } | Self::ToolCallRequested { run_id, .. } + | Self::ToolCallCancelRequested { run_id, .. } | Self::BrowserActivity { run_id, .. } | Self::Generation { run_id, .. } | Self::RunCost { run_id, .. } @@ -523,6 +541,7 @@ impl HarnessEvent { Self::ToolCallStarted { .. } => "tool_call_started", Self::ToolCallCompleted { .. } => "tool_call_completed", Self::RunCompleted { .. } => "run_completed", + Self::RunDeferred { .. } => "run_completed", Self::RunInterrupted { .. } => "run_interrupted", Self::RunContinued { .. } => "run_continued", Self::PromptQueued { .. } => "prompt_queued", @@ -533,6 +552,7 @@ impl HarnessEvent { Self::TitleSuggested { .. } => "title_suggested", Self::PromptSteered { .. } => "prompt_steered", Self::ToolCallRequested { .. } => "tool_call_requested", + Self::ToolCallCancelRequested { .. } => "tool_call_cancel_requested", Self::Idle => "harness_idle", Self::Parked => "harness_parked", Self::Busy => "harness_busy", @@ -551,7 +571,8 @@ impl HarnessEvent { | Self::ToolCallCompleted { tool_call_id, .. } | Self::FileChanged { tool_call_id, .. } | Self::BrowserActivity { tool_call_id, .. } => Some(tool_call_id.as_str()), - Self::ToolCallRequested { call_id, .. } => Some(call_id.as_str()), + Self::ToolCallRequested { call_id, .. } + | Self::ToolCallCancelRequested { call_id, .. } => Some(call_id.as_str()), _ => None, } } @@ -1208,6 +1229,14 @@ mod tests { HarnessEvent::RunInterrupted { run_id: "x".into() }.kind(), "run_interrupted" ); + assert_eq!( + HarnessEvent::RunDeferred { + run_id: "x".into(), + deferred_tool_call_ids: vec!["t1".into()], + } + .kind(), + "run_completed" + ); assert_eq!( HarnessEvent::PromptQueued { prompt_id: "p".into(), diff --git a/crates/engram-harness-proto/src/tool_result.rs b/crates/engram-harness-proto/src/tool_result.rs new file mode 100644 index 000000000..32b6f65ec --- /dev/null +++ b/crates/engram-harness-proto/src/tool_result.rs @@ -0,0 +1,267 @@ +//! Presentation of the trusted runtime's MCP completion envelope. +//! +//! The journal owns outcome and delivery. This module only validates and +//! presents those facts. It does not grant permission or prove an effect. + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Map, Value}; + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ResultFormat { + #[default] + Json, + Mcp, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct McpToolResult { + #[serde(skip_serializing_if = "Option::is_none")] + source: Option, + #[serde(skip_serializing_if = "Option::is_none")] + outcome: Option, + delivery: Delivery, + #[serde(skip_serializing_if = "Option::is_none")] + output: Option, + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + denial: Option, + #[serde(skip_serializing_if = "Option::is_none")] + result_unavailable: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum Source { + Runtime, + LocalValidation, +} +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum Outcome { + Succeeded, + Failed, + #[serde(rename = "outcome_unknown")] + Unknown, + Canceled, +} +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum Delivery { + Available, + Unavailable, +} +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum Reason { + UnsupportedContent, + ResultLimit, + OutputSchema, + UpstreamFailure, + ExecutionInterrupted, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Output { + content: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + structured_content: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + is_error: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +enum TextContent { + Text { text: String }, +} + +impl McpToolResult { + pub fn parse(raw: &str) -> Result { + // The normalized payload limit leaves room for the runtime receipt fields. + if raw.len() > 1024 * 1024 + 1024 { + return Err("External tool result exceeds the size limit."); + } + let document: Value = + serde_json::from_str(raw).map_err(|_| "Invalid external tool result.")?; + if document + .as_object() + .is_some_and(|object| object.values().any(Value::is_null)) + || document + .get("output") + .and_then(Value::as_object) + .is_some_and(|object| object.values().any(Value::is_null)) + { + return Err("Invalid external tool result."); + } + let result: Self = + serde_json::from_value(document).map_err(|_| "Invalid external tool result.")?; + let valid = match (result.delivery, result.output.as_ref()) { + (Delivery::Available, Some(output)) if result.reason.is_none() => { + match (result.source, result.outcome) { + (None, Some(Outcome::Succeeded)) => { + result.denial.is_none() && output.is_error != Some(true) + } + (None, Some(Outcome::Failed)) => { + result.denial.is_none() && output.is_error == Some(true) + } + (Some(Source::LocalValidation), None) => { + result.denial.is_none() && output.is_error == Some(true) + } + (Some(Source::Runtime), None) => { + result.denial.as_ref().is_some_and(|s| !s.is_empty()) + && output.is_error == Some(true) + } + _ => false, + } + } + (Delivery::Unavailable, None) => { + result.reason.is_some() + && result.denial.is_none() + && match (result.source, result.outcome) { + (None, Some(Outcome::Succeeded | Outcome::Failed | Outcome::Unknown)) => { + true + } + (Some(Source::Runtime), Some(Outcome::Canceled | Outcome::Unknown)) => { + result.reason == Some(Reason::ExecutionInterrupted) + } + _ => false, + } + } + _ => false, + }; + let unavailable_success = + result.outcome == Some(Outcome::Succeeded) && result.delivery == Delivery::Unavailable; + if !valid + || result + .result_unavailable + .is_some_and(|value| value != unavailable_success) + { + return Err("Invalid external tool result."); + } + Ok(result) + } + + pub fn success(&self) -> bool { + self.delivery == Delivery::Available && self.outcome == Some(Outcome::Succeeded) + } + + fn metadata(&self) -> Value { + let mut value = serde_json::to_value(self).expect("MCP result is JSON"); + let object = value.as_object_mut().expect("MCP result is an object"); + object.remove("output"); + if let Some(is_error) = self.output.as_ref().and_then(|output| output.is_error) { + object.insert("isError".into(), Value::Bool(is_error)); + } + value + } + + /// MCP has separate tool output and JSON-RPC delivery error surfaces. + pub fn claude_response(&self, id: Value) -> Value { + match &self.output { + Some(output) => { + let mut result = serde_json::to_value(output).expect("MCP output is JSON"); + result["_meta"] = json!({"engrams/toolResult": self.metadata()}); + json!({"jsonrpc":"2.0", "id":id, "result":result}) + } + None => json!({"jsonrpc":"2.0", "id":id, "error":{ + "code":-32603, "message":"External tool output is unavailable.", "data":self.metadata() + }}), + } + } + + /// Codex supports text items and one success flag. Keep each original text + /// item and present the other named fields as separate JSON text items. + pub fn text_items(&self) -> Vec { + let mut items = Vec::new(); + if let Some(output) = &self.output { + for TextContent::Text { text } in &output.content { + items.push(text.clone()); + } + if let Some(structured) = &output.structured_content { + items.push(json!({"structuredContent":structured}).to_string()); + } + } + items.push(self.metadata().to_string()); + items + } + + pub fn codex_response(&self) -> Value { + let items: Vec = self + .text_items() + .into_iter() + .map(|text| json!({"type":"inputText","text":text})) + .collect(); + json!({"success":self.success(), "contentItems":items}) + } +} + +pub fn invalid_codex_result() -> Value { + json!({"success":false,"contentItems":[{"type":"inputText", "text": + json!({"delivery":"unavailable","reason":"invalid_result"}).to_string()}]}) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_binary_resources_and_null_optional_fields() { + for output in [ + json!({"content":[{"type":"image","data":"AA==","mimeType":"image/png"}]}), + json!({"content":[{"type":"resource_link","uri":"https://example.test/x"}]}), + json!({"content":[{"type":"resource","resource":{"uri":"https://example.test/x","text":"x"}}]}), + json!({"content":[],"structuredContent":null}), + json!({"content":[],"structuredContent":[]}), + json!({"content":[],"isError":null}), + ] { + let envelope = json!({"outcome":"succeeded","delivery":"available","output":output}); + assert!( + McpToolResult::parse(&envelope.to_string()).is_err(), + "accepted {envelope}" + ); + } + } + + #[test] + fn delivery_error_keeps_outcome_and_never_becomes_tool_error() { + let result = McpToolResult::parse(r#"{"outcome":"succeeded","delivery":"unavailable","reason":"unsupported_content","resultUnavailable":true}"#).unwrap(); + let response = result.claude_response(json!(7)); + assert!(response.get("result").is_none()); + assert_eq!( + response["error"]["data"], + json!({"outcome":"succeeded","delivery":"unavailable","reason":"unsupported_content","resultUnavailable":true}) + ); + assert!(!result.success()); + } + + #[test] + fn local_denials_and_runtime_interruptions_do_not_claim_upstream_failure() { + for raw in [ + r#"{"source":"local_validation","delivery":"available","output":{"content":[{"type":"text","text":"invalid"}],"isError":true},"resultUnavailable":false}"#, + r#"{"source":"runtime","denial":"plan_mode","delivery":"available","output":{"content":[{"type":"text","text":"denied"}],"isError":true}}"#, + r#"{"source":"runtime","outcome":"canceled","delivery":"unavailable","reason":"execution_interrupted","resultUnavailable":false}"#, + ] { + let result = McpToolResult::parse(raw).unwrap(); + assert!(!result.success()); + let serialized = serde_json::to_value(&result).unwrap(); + assert_ne!(serialized["outcome"], "failed"); + } + } + + #[test] + fn rejects_inconsistent_outcome_and_delivery() { + for raw in [ + r#"{"outcome":"succeeded","delivery":"available","output":{"content":[],"isError":true}}"#, + r#"{"outcome":"failed","delivery":"available","output":{"content":[]}}"#, + r#"{"outcome":"outcome_unknown","delivery":"available","output":{"content":[]}}"#, + r#"{"outcome":"succeeded","delivery":"unavailable","reason":"result_limit","resultUnavailable":false}"#, + r#"{"outcome":"succeeded","delivery":"available","output":{"content":[]},"decision":"approve"}"#, + ] { + assert!(McpToolResult::parse(raw).is_err(), "accepted {raw}"); + } + } +} diff --git a/crates/engram-harness-proto/tests/decode_never_panics.rs b/crates/engram-harness-proto/tests/decode_never_panics.rs index d20cf0efd..67bf137b4 100644 --- a/crates/engram-harness-proto/tests/decode_never_panics.rs +++ b/crates/engram-harness-proto/tests/decode_never_panics.rs @@ -41,6 +41,9 @@ use proptest::prelude::*; /// them *panic*. A panic here unwinds out of the closure and fails the /// proptest case with the shrunk counterexample. fn decode_every_type(bytes: &[u8]) { + if let Ok(raw) = std::str::from_utf8(bytes) { + let _ = engram_harness_proto::tool_result::McpToolResult::parse(raw); + } let _ = serde_json::from_slice::(bytes); let _ = bincode::deserialize::(bytes); let _ = bincode::deserialize::(bytes); diff --git a/crates/engram-harness-proto/tests/golden/event_run_deferred.bin b/crates/engram-harness-proto/tests/golden/event_run_deferred.bin new file mode 100644 index 000000000..ff225b407 Binary files /dev/null and b/crates/engram-harness-proto/tests/golden/event_run_deferred.bin differ diff --git a/crates/engram-harness-proto/tests/golden/event_tool_call_cancel_requested.bin b/crates/engram-harness-proto/tests/golden/event_tool_call_cancel_requested.bin new file mode 100644 index 000000000..2e5714f7c Binary files /dev/null and b/crates/engram-harness-proto/tests/golden/event_tool_call_cancel_requested.bin differ diff --git a/crates/engram-harness-proto/tests/support/mod.rs b/crates/engram-harness-proto/tests/support/mod.rs index 1ee800910..ae7ff00f4 100644 --- a/crates/engram-harness-proto/tests/support/mod.rs +++ b/crates/engram-harness-proto/tests/support/mod.rs @@ -66,6 +66,13 @@ pub fn file_change() -> impl Strategy { pub fn harness_event() -> impl Strategy { prop_oneof![ + (s(), s(), s()).prop_map( + |(run_id, call_id, name)| HarnessEvent::ToolCallCancelRequested { + run_id, + call_id, + name + } + ), (s(), opt_s(), opt_s()).prop_map(|(run_id, prompt_summary, prompt_id)| { HarnessEvent::RunStarted { run_id, @@ -102,6 +109,12 @@ pub fn harness_event() -> impl Strategy { } ), (s(), any::()).prop_map(|(run_id, ok)| HarnessEvent::RunCompleted { run_id, ok }), + (s(), prop::collection::vec(s(), 0..5)).prop_map(|(run_id, deferred_tool_call_ids)| { + HarnessEvent::RunDeferred { + run_id, + deferred_tool_call_ids, + } + }), s().prop_map(|run_id| HarnessEvent::RunInterrupted { run_id }), s().prop_map(|run_id| HarnessEvent::RunContinued { run_id }), Just(HarnessEvent::Idle), @@ -378,6 +391,7 @@ fn _exhaustiveness_harness_event(e: &HarnessEvent) { HarnessEvent::ToolCallStarted { .. } => {} HarnessEvent::ToolCallCompleted { .. } => {} HarnessEvent::RunCompleted { .. } => {} + HarnessEvent::RunDeferred { .. } => {} HarnessEvent::RunInterrupted { .. } => {} HarnessEvent::RunContinued { .. } => {} HarnessEvent::Idle => {} @@ -389,6 +403,7 @@ fn _exhaustiveness_harness_event(e: &HarnessEvent) { HarnessEvent::TitleSuggested { .. } => {} HarnessEvent::PromptSteered { .. } => {} HarnessEvent::ToolCallRequested { .. } => {} + HarnessEvent::ToolCallCancelRequested { .. } => {} HarnessEvent::Parked => {} HarnessEvent::BrowserActivity { .. } => {} HarnessEvent::Busy => {} diff --git a/crates/engram-harness-proto/tests/wire_golden.rs b/crates/engram-harness-proto/tests/wire_golden.rs index 8db1a5b38..71770cd15 100644 --- a/crates/engram-harness-proto/tests/wire_golden.rs +++ b/crates/engram-harness-proto/tests/wire_golden.rs @@ -202,6 +202,12 @@ fn ev_run_completed() -> HarnessEvent { ok: true, } } +fn ev_run_deferred() -> HarnessEvent { + HarnessEvent::RunDeferred { + run_id: "r1".into(), + deferred_tool_call_ids: vec!["t1".into()], + } +} fn ev_run_continued() -> HarnessEvent { HarnessEvent::RunContinued { run_id: "r1".into(), @@ -322,6 +328,14 @@ fn cmd_tool_result() -> HarnessCommand { } } +fn ev_tool_call_cancel_requested() -> HarnessEvent { + HarnessEvent::ToolCallCancelRequested { + run_id: "run-1".into(), + call_id: "call-1".into(), + name: "remote_inventory".into(), + } +} + // ---- tests ------------------------------------------------------------- #[test] @@ -348,6 +362,17 @@ fn harness_frame_golden_and_variant_indices() { #[test] fn harness_event_golden_and_variant_indices() { + assert_golden("event_run_deferred", &ev_run_deferred()); + assert_variant_index(&ev_run_deferred(), 22, "HarnessEvent::RunDeferred"); + assert_golden( + "event_tool_call_cancel_requested", + &ev_tool_call_cancel_requested(), + ); + assert_variant_index( + &ev_tool_call_cancel_requested(), + 21, + "HarnessEvent::ToolCallCancelRequested", + ); assert_golden("event_run_continued", &ev_run_continued()); assert_variant_index(&ev_run_continued(), 20, "HarnessEvent::RunContinued"); assert_golden("event_run_started", &ev_run_started()); @@ -633,7 +658,12 @@ fn regen_golden() { ); write("frame_event_ack", &HarnessFrame::EventAck { seq: 42 }); + write( + "event_tool_call_cancel_requested", + &ev_tool_call_cancel_requested(), + ); write("event_run_continued", &ev_run_continued()); + write("event_run_deferred", &ev_run_deferred()); write("event_run_started", &ev_run_started()); write("event_agent_message", &ev_agent_message()); write("event_tool_call_started", &ev_tool_call_started()); diff --git a/crates/engram-harness-sdk/src/first_output.rs b/crates/engram-harness-sdk/src/first_output.rs index cb12f1497..3f194a089 100644 --- a/crates/engram-harness-sdk/src/first_output.rs +++ b/crates/engram-harness-sdk/src/first_output.rs @@ -95,6 +95,7 @@ fn output_run_id(event: &HarnessEvent) -> Option<&str> { HarnessEvent::RunStarted { .. } | HarnessEvent::ToolCallCompleted { .. } | HarnessEvent::RunCompleted { .. } + | HarnessEvent::RunDeferred { .. } | HarnessEvent::RunInterrupted { .. } | HarnessEvent::RunContinued { .. } | HarnessEvent::Idle @@ -107,7 +108,8 @@ fn output_run_id(event: &HarnessEvent) -> Option<&str> { | HarnessEvent::Parked | HarnessEvent::BrowserActivity { .. } | HarnessEvent::Busy - | HarnessEvent::RunCost { .. } => None, + | HarnessEvent::RunCost { .. } + | HarnessEvent::ToolCallCancelRequested { .. } => None, } } @@ -115,6 +117,17 @@ fn output_run_id(event: &HarnessEvent) -> Option<&str> { mod tests { use super::*; + #[test] + fn deferred_completion_is_lifecycle_not_model_output() { + assert_eq!( + output_run_id(&HarnessEvent::RunDeferred { + run_id: "r1".into(), + deferred_tool_call_ids: vec!["t1".into()], + }), + None + ); + } + fn run_started(run: &str) -> HarnessEvent { HarnessEvent::RunStarted { run_id: run.into(), diff --git a/crates/engram-harness-sdk/src/lib.rs b/crates/engram-harness-sdk/src/lib.rs index 8963dae16..1f38e37fe 100644 --- a/crates/engram-harness-sdk/src/lib.rs +++ b/crates/engram-harness-sdk/src/lib.rs @@ -81,6 +81,7 @@ impl EventOutbox { state.open_runs.push(run_id.clone()); } HarnessEvent::RunCompleted { run_id, .. } + | HarnessEvent::RunDeferred { run_id, .. } | HarnessEvent::RunInterrupted { run_id, .. } => { state.open_runs.retain(|r| r != run_id); } @@ -881,4 +882,26 @@ mod tests { } assert!(channels.event_tx.try_send(HarnessEvent::Idle).is_err()); } + + #[test] + fn deferred_completion_closes_the_reconnect_run_inventory() { + let outbox = EventOutbox::new(); + outbox.sequence( + 4, + HarnessEvent::RunStarted { + run_id: "r1".into(), + prompt_id: None, + prompt_summary: None, + }, + ); + assert_eq!(outbox.continued_runs(5), vec!["r1"]); + outbox.sequence( + 4, + HarnessEvent::RunDeferred { + run_id: "r1".into(), + deferred_tool_call_ids: vec!["t1".into()], + }, + ); + assert!(outbox.continued_runs(5).is_empty()); + } } diff --git a/crates/engram-host-agent/src/bindings.rs b/crates/engram-host-agent/src/bindings.rs index a85b55c3e..7045016cf 100644 --- a/crates/engram-host-agent/src/bindings.rs +++ b/crates/engram-host-agent/src/bindings.rs @@ -106,6 +106,72 @@ impl BindingStore { }) } + pub(crate) fn managed_required(&self, session: SessionId) -> bool { + if self + .dir + .join(format!("{session}.managed-boot")) + .try_exists() + .unwrap_or(true) + { + return true; + } + self.dir + .join(format!("{session}.managed-required")) + .try_exists() + .unwrap_or(true) + || self + .dir + .join(format!("{session}.managed")) + .try_exists() + .unwrap_or(true) + } + /// Check every durable managed association, including a record whose binding + /// has been removed or moved. An unreadable association cannot prove ordinary. + pub(crate) fn managed_sandbox_required(&self, sandbox: SandboxId) -> std::io::Result { + if self + .dir + .join(format!("{sandbox}.managed-sandbox")) + .try_exists()? + { + return Ok(true); + } + let mut sessions = std::collections::BTreeSet::new(); + for entry in std::fs::read_dir(&self.dir)? { + let path = entry?.path(); + if matches!( + path.extension().and_then(|v| v.to_str()), + Some("managed" | "managed-required") + ) { + let sid: SessionId = path + .file_stem() + .and_then(|v| v.to_str()) + .ok_or_else(|| std::io::Error::other("invalid managed filename"))? + .parse() + .map_err(std::io::Error::other)?; + sessions.insert(sid); + } + } + for sid in sessions { + if self + .read(sid)? + .is_some_and(|binding| binding.sandbox_id == sandbox) + { + return Ok(true); + } + // The desired record can retain the old physical target after unbind. + let record = crate::managed_policy::read_record(self, sid)? + .ok_or_else(|| std::io::Error::other("managed record missing"))?; + if record.egress.sandbox_id == sandbox { + return Ok(true); + } + } + Ok(false) + } + + pub(crate) fn directory(&self) -> &std::path::Path { + &self.dir + } + fn path_for(&self, session_id: SessionId) -> PathBuf { self.dir.join(format!("{session_id}.json")) } @@ -177,7 +243,7 @@ impl BindingStore { format!("binding record {}: {e}", path.display()), ) })?; - if record.schema_version != BINDING_SCHEMA_VERSION { + if record.schema_version != BINDING_SCHEMA_VERSION || record.session_id != session_id { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, format!( @@ -302,6 +368,23 @@ impl BindingStore { } } } + for entry in std::fs::read_dir(&self.dir)? { + let entry = entry?; + if entry.path().extension().and_then(|e| e.to_str()) != Some("managed-required") { + continue; + } + let sid: SessionId = entry + .path() + .file_stem() + .and_then(|s| s.to_str()) + .ok_or_else(|| std::io::Error::other("invalid managed filename"))? + .parse() + .map_err(std::io::Error::other)?; + let record = crate::managed_policy::read_record(self, sid)? + .ok_or_else(|| std::io::Error::other("managed record missing"))?; + out.retain(|p| p.session_id != sid); + out.push(record.egress); + } Ok(out) } @@ -402,6 +485,7 @@ mod tests { sandbox_id: SandboxId, ) -> engram_core::types::egress::SessionEgressPolicy { engram_core::types::egress::SessionEgressPolicy { + managed_deny: None, session_id, sandbox_id, guest_ip: std::net::Ipv4Addr::new(10, 200, 0, 2), diff --git a/crates/engram-host-agent/src/config.rs b/crates/engram-host-agent/src/config.rs index ea54b3879..74a9a1342 100644 --- a/crates/engram-host-agent/src/config.rs +++ b/crates/engram-host-agent/src/config.rs @@ -3,6 +3,7 @@ use std::time::Duration; #[derive(Clone, Debug)] pub struct HostAgentConfig { + pub managed_tls: Option, /// Where the SandboxBackend stores per-VM working directories, /// Firecracker sockets, and snapshot files. pub work_dir: PathBuf, @@ -65,6 +66,7 @@ pub const DEFAULT_DISK_HYDRATION_WINDOW_CHUNKS: usize = 12; impl Default for HostAgentConfig { fn default() -> Self { Self { + managed_tls: None, work_dir: PathBuf::from("./var/sandboxes"), heartbeat_interval: Duration::from_secs(5), local_snapshot_cap_bytes: 500 * 1024 * 1024 * 1024, // 500 GiB diff --git a/crates/engram-host-agent/src/grpc_server.rs b/crates/engram-host-agent/src/grpc_server.rs index a4546944b..d5295b277 100644 --- a/crates/engram-host-agent/src/grpc_server.rs +++ b/crates/engram-host-agent/src/grpc_server.rs @@ -54,6 +54,7 @@ use crate::session_epochs::{EpochError, SessionEpochStore}; /// the host's SandboxBackend + HarnessHub). pub struct HostServiceImpl { inner: Arc, + managed_tls: bool, /// ADR 0079: per-session fencing-epoch high-water, durable under /// work_dir. Gates every session-scoped lifecycle RPC. epochs: SessionEpochStore, @@ -68,24 +69,47 @@ impl HostServiceImpl { pub fn new(inner: Arc, epochs: SessionEpochStore) -> Self { Self { inner, + managed_tls: false, epochs, peer: None, } } + fn check_managed_protocol(&self, request: &Request) -> Result<(), Status> { + check_wire_version(request)?; + let exact = |key, expected: u32| { + request.metadata().get(key).and_then(|v| v.to_str().ok()) + == Some(expected.to_string().as_str()) + }; + if !self.managed_tls + || !exact( + engram_protocol::wire::WIRE_VERSION_METADATA_KEY, + engram_protocol::WIRE_VERSION, + ) + || !exact( + engram_protocol::wire::MANAGED_PROTOCOL_METADATA_KEY, + engram_protocol::wire::MANAGED_PROTOCOL_VERSION, + ) + { + return Err(Status::failed_precondition( + "managed RPC requires authenticated host mode and exact protocol", + )); + } + Ok(()) + } + pub fn with_peer_serve(mut self, peer: Arc) -> Self { self.peer = Some(peer); self } - /// ADR 0079: the fencing-epoch gate, called at the top of every - /// session-scoped lifecycle handler right after [`check_wire_version`] - /// (the same transport-level-gate move). Rejections are - /// `failed_precondition` carrying stored vs incoming so the - /// coordinator sees an explicit fence, never a decode error. + /// Check the session fence before a lifecycle operation reaches the backend. + /// Rejections carry the stored and incoming epochs as `failed_precondition`. /// /// Returns the decoded [`SessionFence`] so the handler can thread it /// to the inner `HostClient` without re-decoding. + /// Epoch zero preserves a supplied identity without accessing the + /// high-water store. Empty bytes retain the unfenced sentinel. /// /// `fencing_epoch == 0` (allow WITHOUT advancing the high-water; the /// store's `check_and_advance` holds the reject-zero semantics for @@ -110,10 +134,13 @@ impl HostServiceImpl { session_id: &[u8], fencing_epoch: u64, ) -> Result { - if fencing_epoch == 0 { + if fencing_epoch == 0 && session_id.is_empty() { return Ok(SessionFence::unfenced()); } let session_id = decode_session_id(session_id)?; + if fencing_epoch == 0 { + return Ok(SessionFence::new(session_id, 0)); + } self.epochs .check_and_advance(session_id, fencing_epoch) .map_err(|e| match e { @@ -152,32 +179,147 @@ pub fn bind( Ok(incoming.with_nodelay(Some(true))) } -/// Bind `listen_addr` and serve the `HostService` until the caller drops -/// the returned future. +/// A bound listener with its TLS identity and caller pins already validated. +pub struct PreparedHostServer { + incoming: tonic::transport::server::TcpIncoming, + server: tonic::transport::Server, + pins: Option>, +} + +impl PreparedHostServer { + pub fn bind( + listen_addr: std::net::SocketAddr, + tls: Option, + ) -> Result> { + Self::from_incoming(bind(listen_addr)?, tls) + } + + fn from_incoming( + incoming: tonic::transport::server::TcpIncoming, + tls: Option, + ) -> Result> { + let mut server = tonic::transport::Server::builder(); + let pins = match tls { + Some(config) => { + server = + server.tls_config(config.server_config().map_err(std::io::Error::other)?)?; + Some(config.coordinator_pins) + } + None => None, + }; + Ok(Self { + incoming, + server, + pins, + }) + } + + pub async fn serve( + self, + inner: Arc, + epochs: SessionEpochStore, + peer: Option>, + ) -> Result<(), tonic::transport::Error> { + serve_inner(self.incoming, inner, epochs, peer, self.server, self.pins).await + } +} + +/// Bind and serve the ordinary lifecycle listener. pub async fn boot( listen_addr: std::net::SocketAddr, inner: Arc, epochs: SessionEpochStore, peer: Option>, ) -> Result<(), Box> { - let incoming = bind(listen_addr)?; - Ok(serve(incoming, inner, epochs, peer).await?) + Ok(PreparedHostServer::bind(listen_addr, None)? + .serve(inner, epochs, peer) + .await?) } -/// Serve the `HostService` on a bound listener until the caller drops -/// the returned future. Ends with `Err` on a server error; a clean -/// Ctrl-C just drops the future, which the runtime collects. +/// Serve the ordinary lifecycle listener until the caller drops the future. pub async fn serve( incoming: tonic::transport::server::TcpIncoming, inner: Arc, epochs: SessionEpochStore, peer: Option>, +) -> Result<(), tonic::transport::Error> { + PreparedHostServer { + incoming, + server: tonic::transport::Server::builder(), + pins: None, + } + .serve(inner, epochs, peer) + .await +} + +/// Bind and serve the single lifecycle listener with optional managed TLS. +pub async fn boot_with_managed_tls( + listen_addr: std::net::SocketAddr, + inner: Arc, + epochs: SessionEpochStore, + peer: Option>, + tls: Option, +) -> Result<(), Box> { + Ok(PreparedHostServer::bind(listen_addr, tls)? + .serve(inner, epochs, peer) + .await?) +} + +/// Serve a bound lifecycle listener with optional managed TLS. +pub async fn serve_with_managed_tls( + incoming: tonic::transport::server::TcpIncoming, + inner: Arc, + epochs: SessionEpochStore, + peer: Option>, + tls: Option, +) -> Result<(), Box> { + Ok(PreparedHostServer::from_incoming(incoming, tls)? + .serve(inner, epochs, peer) + .await?) +} + +async fn serve_inner( + incoming: tonic::transport::server::TcpIncoming, + inner: Arc, + epochs: SessionEpochStore, + peer: Option>, + server: tonic::transport::Server, + pins: Option>, ) -> Result<(), tonic::transport::Error> { let mut svc = HostServiceImpl::new(inner, epochs); if let Some(p) = peer { svc = svc.with_peer_serve(p); } - tonic::transport::Server::builder() + svc.managed_tls = pins.is_some(); + let service = HostServiceServer::new(svc) + .max_encoding_message_size(32 * 1024 * 1024) + .max_decoding_message_size(9 * 1024 * 1024); + let authenticated = + tonic::service::interceptor::InterceptedService::new(service, move |req: Request<()>| { + if let Some(pins) = &pins { + let certificates = req + .peer_certs() + .ok_or_else(|| Status::unauthenticated("client certificate required"))?; + let leaf = certificates + .first() + .ok_or_else(|| Status::unauthenticated("client certificate required"))?; + if !pins.contains(&engram_protocol::host_tls::certificate_pin(leaf.as_ref())) { + return Err(Status::unauthenticated("client certificate not authorized")); + } + let version = req + .metadata() + .get(engram_protocol::wire::WIRE_VERSION_METADATA_KEY) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + if version != Some(engram_protocol::WIRE_VERSION) { + return Err(Status::failed_precondition( + "host protocol version required", + )); + } + } + Ok(req) + }); + server // ADR 0013: HTTP/2 multiplexes many concurrent streams over // one TCP connection per coord pod. 256 is well above the // running_sandboxes ceiling — we don't expect to hit it. @@ -191,7 +333,7 @@ pub async fn serve( .initial_stream_window_size(Some(16 * 1024 * 1024)) .initial_connection_window_size(Some(32 * 1024 * 1024)) .http2_adaptive_window(Some(true)) - .add_service(HostServiceServer::new(svc).max_encoding_message_size(32 * 1024 * 1024)) + .add_service(authenticated) .serve_with_incoming(incoming) .await } @@ -751,13 +893,7 @@ impl HostService for HostServiceImpl { .await } - /// ADR 0080 §C (wire v14): server-streaming, the exact two-task - /// shape of `build_base_snapshot` above — a dedicated task drives - /// the backend call while THIS task drains its progress channel - /// onto the outer gRPC stream, and only once that channel closes - /// (the backend call returned, all `Sender`s dropped) do we await - /// the result and emit exactly one terminal `done`/`failed` frame. - /// Every progress frame therefore strictly precedes the terminal. + /// Drain backend progress before sending its single terminal result. async fn materialize_image( &self, req: Request, @@ -1010,17 +1146,161 @@ impl HostService for HostServiceImpl { Ok(Response::new(Empty {})) } - /// ADR 0013: bundled policy + start. One trait call applies the - /// `SessionEgressPolicy` to the host's egress proxy registry - /// BEFORE spawning the agent process. Atomic by construction. - /// - /// This entry-point serves the **cold-create** path: coord - /// calls `StartAgent` after `Create` returns, the in-guest - /// bootstrap may not be accept()'ing yet, and the dial blocks - /// until kernel + engram-init + ext4 mount complete. The - /// warm-lease path doesn't reach here — it goes through - /// `WarmPool::launch` (which has its own histogram emission - /// labelled `kind="warm"`). + /// Read physical cold-boot inputs through the authenticated host boundary. + /// This operation cannot allocate a VM, policy lease or network owner. + async fn read_managed_cold_assets( + &self, + req: tonic::Request, + ) -> Result, tonic::Status> + { + self.check_managed_protocol(&req)?; + let request = engram_protocol::managed_boot::decode(&req.into_inner().payload_json) + .map_err(sandbox_to_status)?; + let result = self + .inner + .read_managed_cold_assets(request) + .await + .map_err(sandbox_to_status)?; + let payload_json = + engram_protocol::managed_boot::encode(&result).map_err(sandbox_to_status)?; + Ok(Response::new( + engram_protocol::grpc::ReadManagedColdAssetsResponse { payload_json }, + )) + } + async fn capture_managed_boot( + &self, + req: tonic::Request, + ) -> Result, tonic::Status> { + self.check_managed_protocol(&req)?; + let request = engram_protocol::managed_boot::decode(&req.into_inner().payload_json) + .map_err(sandbox_to_status)?; + let result = self + .inner + .capture_managed_boot(request) + .await + .map_err(sandbox_to_status)?; + let payload_json = + engram_protocol::managed_boot::encode(&result).map_err(sandbox_to_status)?; + Ok(Response::new(engram_protocol::grpc::ManagedBootResponse { + payload_json, + })) + } + async fn prepare_managed_boot( + &self, + req: tonic::Request, + ) -> Result, tonic::Status> { + self.check_managed_protocol(&req)?; + let request = engram_protocol::managed_boot::decode(&req.into_inner().payload_json) + .map_err(sandbox_to_status)?; + let result = self + .inner + .prepare_managed_boot(request) + .await + .map_err(sandbox_to_status)?; + let payload_json = + engram_protocol::managed_boot::encode(&result).map_err(sandbox_to_status)?; + Ok(Response::new(engram_protocol::grpc::ManagedBootResponse { + payload_json, + })) + } + async fn read_managed_boot( + &self, + req: tonic::Request, + ) -> Result, tonic::Status> { + self.check_managed_protocol(&req)?; + let request = engram_protocol::managed_boot::decode(&req.into_inner().payload_json) + .map_err(sandbox_to_status)?; + let result = self + .inner + .read_managed_boot(request) + .await + .map_err(sandbox_to_status)?; + let payload_json = + engram_protocol::managed_boot::encode(&result).map_err(sandbox_to_status)?; + Ok(Response::new(engram_protocol::grpc::ManagedBootResponse { + payload_json, + })) + } + async fn finalize_managed_boot( + &self, + req: tonic::Request, + ) -> Result, tonic::Status> { + self.check_managed_protocol(&req)?; + let request = engram_protocol::managed_boot::decode(&req.into_inner().payload_json) + .map_err(sandbox_to_status)?; + let result = self + .inner + .finalize_managed_boot(request) + .await + .map_err(sandbox_to_status)?; + let payload_json = + engram_protocol::managed_boot::encode(&result).map_err(sandbox_to_status)?; + Ok(Response::new(engram_protocol::grpc::ManagedBootResponse { + payload_json, + })) + } + async fn start_managed_boot( + &self, + req: tonic::Request, + ) -> Result, tonic::Status> { + self.check_managed_protocol(&req)?; + let request = engram_protocol::managed_boot::decode(&req.into_inner().payload_json) + .map_err(sandbox_to_status)?; + let result = self + .inner + .start_managed_boot(request) + .await + .map_err(sandbox_to_status)?; + let payload_json = + engram_protocol::managed_boot::encode(&result).map_err(sandbox_to_status)?; + Ok(Response::new(engram_protocol::grpc::ManagedBootResponse { + payload_json, + })) + } + async fn read_managed_policy( + &self, + req: Request, + ) -> Result, Status> { + self.check_managed_protocol(&req)?; + if req.get_ref().target_bincode.len() > 4096 { + return Err(Status::resource_exhausted("managed request too large")); + } + let target = engram_protocol::managed_policy::decode(&req.into_inner().target_bincode) + .map_err(|_| Status::invalid_argument("invalid managed target"))?; + let state = self + .inner + .read_managed_policy(target) + .await + .map_err(sandbox_to_status)?; + Ok(Response::new( + engram_protocol::grpc::ReadManagedPolicyResponse { + state_bincode: bincode::serialize(&state) + .map_err(|_| Status::internal("managed response encoding failed"))?, + }, + )) + } + async fn apply_managed_policy( + &self, + req: Request, + ) -> Result, Status> { + self.check_managed_protocol(&req)?; + if req.get_ref().request_bincode.len() > 4 * 1024 * 1024 { + return Err(Status::resource_exhausted("managed request too large")); + } + let request = engram_protocol::managed_policy::decode(&req.into_inner().request_bincode) + .map_err(|_| Status::invalid_argument("invalid managed request"))?; + let state = self + .inner + .apply_managed_policy(request) + .await + .map_err(sandbox_to_status)?; + Ok(Response::new( + engram_protocol::grpc::ApplyManagedPolicyResponse { + state_bincode: bincode::serialize(&state) + .map_err(|_| Status::internal("managed response encoding failed"))?, + }, + )) + } async fn start_agent( &self, req: Request, @@ -1033,10 +1313,11 @@ impl HostService for HostServiceImpl { check_wire_version(&req)?; async move { let r = req.into_inner(); - let fence = self.check_session_epoch(&r.session_id, r.fencing_epoch)?; let sandbox_id = decode_sandbox_id(&r.sandbox_id)?; let agent = decode_bincode(&r.agent_bincode, "AgentSpec")?; - let policy = decode_bincode(&r.policy_bincode, "SessionEgressPolicy")?; + let policy = engram_protocol::ordinary_policy::decode(&r.policy_bincode) + .map_err(sandbox_to_status)?; + let fence = self.check_session_epoch(&r.session_id, r.fencing_epoch)?; let phase_start = crate::time_source::metrics_now(); let result = self .inner @@ -1191,7 +1472,7 @@ impl HostService for HostServiceImpl { _ => { return Err(Status::invalid_argument( "first file frame must be metadata", - )) + )); } }; let sandbox_id = decode_sandbox_id(&metadata.sandbox_id)?; @@ -1245,13 +1526,12 @@ impl HostService for HostServiceImpl { use futures::StreamExt; let request = req.into_inner(); let sandbox_id = decode_sandbox_id(&request.sandbox_id)?; - let (metadata, mut bytes) = self + let (metadata, bytes) = self .inner .read_file(sandbox_id, request.path) .await .map_err(sandbox_to_status)?; - let (tx, rx) = mpsc::channel(8); - tx.send(Ok(HostReadFileResponse { + let metadata = HostReadFileResponse { frame: Some( engram_protocol::grpc::host_read_file_response::Frame::Metadata( HostReadFileMetadata { @@ -1261,28 +1541,21 @@ impl HostService for HostServiceImpl { }, ), ), - })) - .await - .map_err(|_| Status::cancelled("read client closed before metadata"))?; - tokio::spawn(async move { - while let Some(item) = bytes.next().await { - let frame = item - .map(|chunk| HostReadFileResponse { - frame: Some( - engram_protocol::grpc::host_read_file_response::Frame::Chunk( - chunk.to_vec(), - ), + }; + // The response owns the source, so downstream cancellation drops it even + // while its next chunk is pending. + let response = + futures::stream::once(async move { Ok(metadata) }).chain(bytes.map(|item| { + item.map(|chunk| HostReadFileResponse { + frame: Some( + engram_protocol::grpc::host_read_file_response::Frame::Chunk( + chunk.to_vec(), ), - }) - .map_err(sandbox_to_status); - if tx.send(frame).await.is_err() { - return; - } - } - }); - Ok(Response::new(Box::pin( - tokio_stream::wrappers::ReceiverStream::new(rx), - ))) + ), + }) + .map_err(sandbox_to_status) + })); + Ok(Response::new(Box::pin(response))) } /// ADR 0014 issue #6: bidi WS-frame tunnel. @@ -1599,21 +1872,8 @@ fn encode_bincode(value: &T, kind: &'static str) -> Result< bincode::serialize(value).map_err(|e| Status::internal(format!("bincode encode {kind}: {e}"))) } -/// Issue #229: reject a coord→host RPC whose stamped wire version differs -/// from ours BEFORE any bincode decode. The coordinator's -/// `TraceparentInjector` puts its `WIRE_VERSION` in the -/// [`engram_protocol::wire::WIRE_VERSION_METADATA_KEY`] metadata header -/// on every request; a mismatch means a mixed-version fleet mid rolling -/// deploy, where a bincode decode would otherwise EOF and surface to the -/// user as a misleading 400 "invalid sandbox spec". We return -/// `failed_precondition` with the skew marker so the coord maps it to a -/// retryable 503 (`SandboxError::WireSkew`). -/// -/// A request with NO version header (a coordinator that predates this -/// field) is tolerated — we can't compare what we didn't receive, and -/// the decode path is the same as before this fix. A header we can't -/// parse is likewise tolerated (fail-open) rather than blocking traffic -/// on a malformed value. +/// Reject an explicit incompatible version before decoding a payload. +/// An absent header retains the ordinary HTTP/2 predecessor contract. fn check_wire_version(req: &Request) -> Result<(), Status> { let Some(raw) = req .metadata() @@ -1622,7 +1882,7 @@ fn check_wire_version(req: &Request) -> Result<(), Status> { return Ok(()); }; let Some(coord) = raw.to_str().ok().and_then(|s| s.parse::().ok()) else { - return Ok(()); + return Err(Status::failed_precondition("invalid host protocol version")); }; let host = engram_protocol::WIRE_VERSION; if coord == host { @@ -1727,6 +1987,9 @@ fn sandbox_to_status(err: SandboxError) -> Status { ), // Same marker shape: the coordinator reads the typed variant and // recovers the session from its disk instead of retrying. + SandboxError::ManagedRecoveryRequired => { + Status::failed_precondition(engram_protocol::wire::MANAGED_RECOVERY_REQUIRED_STATUS) + } SandboxError::MemoryImageUnusable(message) => Status::failed_precondition( engram_protocol::wire::memory_image_unusable_message(&message), ), @@ -1741,6 +2004,69 @@ mod wire_version_tests { use engram_protocol::wire::WIRE_VERSION_METADATA_KEY; use futures::StreamExt; + #[test] + fn epoch_zero_preserves_session_identity_without_high_water_access() { + let work = tempfile::tempdir().unwrap(); + let backend = Arc::new(engram_sandbox_process::ProcessBackend::new(work.path())); + let host = Arc::new(crate::host_client::LocalHostClient::with_noop_hub(backend)); + let path = work.path().join("epochs"); + let epochs = SessionEpochStore::open(&path).unwrap(); + let service = HostServiceImpl::new(host, epochs.clone()); + let session = engram_core::SessionId::new(); + let uuid = session.as_uuid(); + let bytes = uuid.as_bytes(); + + assert_eq!( + service.check_session_epoch(&[], 0).unwrap(), + SessionFence::unfenced() + ); + assert_eq!( + service.check_session_epoch(&[0; 16], 0).unwrap(), + SessionFence::unfenced() + ); + assert_eq!( + service.check_session_epoch(bytes, 0).unwrap(), + SessionFence::new(session, 0) + ); + assert!(epochs.read(session).unwrap().is_none()); + assert_eq!( + service.check_session_epoch(bytes, 5).unwrap(), + SessionFence::new(session, 5) + ); + assert_eq!( + service.check_session_epoch(bytes, 0).unwrap(), + SessionFence::new(session, 0) + ); + assert_eq!(epochs.read(session).unwrap().unwrap().epoch, 5); + let stale = service.check_session_epoch(bytes, 4).unwrap_err(); + assert_eq!(stale.code(), tonic::Code::FailedPrecondition); + assert!(stale.message().contains("stored high-water 5, incoming 4")); + assert_eq!(epochs.read(session).unwrap().unwrap().epoch, 5); + + // A zero epoch must not read or repair even a corrupt high-water record. + let record = path.join(format!("{session}.json")); + std::fs::write(&record, b"corrupt").unwrap(); + assert_eq!( + service.check_session_epoch(bytes, 0).unwrap(), + SessionFence::new(session, 0) + ); + assert_eq!(std::fs::read(record).unwrap(), b"corrupt"); + } + + #[test] + fn epoch_zero_rejects_malformed_session_identity() { + let work = tempfile::tempdir().unwrap(); + let backend = Arc::new(engram_sandbox_process::ProcessBackend::new(work.path())); + let host = Arc::new(crate::host_client::LocalHostClient::with_noop_hub(backend)); + let service = HostServiceImpl::new(host, crate::session_epochs::ephemeral()); + for bytes in [&[1][..], &[1; 15][..], &[1; 17][..]] { + assert_eq!( + service.check_session_epoch(bytes, 0).unwrap_err().code(), + tonic::Code::InvalidArgument + ); + } + } + fn exec_events(events: I) -> ExecEventStream where I: IntoIterator, @@ -1867,6 +2193,64 @@ mod wire_version_tests { req } + #[tokio::test] + async fn cold_assets_route_requires_authenticated_mode_and_wire_match() { + let work = tempfile::tempdir().unwrap(); + let backend = Arc::new(engram_sandbox_process::ProcessBackend::new(work.path())); + let host = Arc::new(crate::host_client::LocalHostClient::with_noop_hub(backend)); + let mut service = HostServiceImpl::new(host, crate::session_epochs::ephemeral()); + let req = || { + let mut request = Request::new(engram_protocol::grpc::ReadManagedColdAssetsRequest { + payload_json: b"null".to_vec(), + }); + request.metadata_mut().insert( + WIRE_VERSION_METADATA_KEY, + engram_protocol::WIRE_VERSION.to_string().parse().unwrap(), + ); + request.metadata_mut().insert( + engram_protocol::wire::MANAGED_PROTOCOL_METADATA_KEY, + engram_protocol::wire::MANAGED_PROTOCOL_VERSION + .to_string() + .parse() + .unwrap(), + ); + request + }; + assert_eq!( + service + .read_managed_cold_assets(req()) + .await + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); + service.managed_tls = true; + let mut skew = req(); + skew.metadata_mut().insert( + WIRE_VERSION_METADATA_KEY, + (engram_protocol::WIRE_VERSION + 1) + .to_string() + .parse() + .unwrap(), + ); + assert_eq!( + service + .read_managed_cold_assets(skew) + .await + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); + assert_eq!( + service + .read_managed_cold_assets(req()) + .await + .unwrap_err() + .code(), + tonic::Code::InvalidArgument + ); + } + #[tokio::test] async fn migration_fetch_rejects_wire_skew_before_artifacts() { let work = tempfile::tempdir().unwrap(); @@ -1952,12 +2336,12 @@ mod wire_version_tests { } #[test] - fn unparseable_version_header_fails_open() { - // A malformed header value must not wedge traffic — fail open. + fn unparseable_version_header_is_rejected() { + // A declared version must be parseable before payload decoding. let mut req = Request::new(()); req.metadata_mut() .insert(WIRE_VERSION_METADATA_KEY, "not-a-number".parse().unwrap()); - assert!(check_wire_version(&req).is_ok()); + assert!(check_wire_version(&req).is_err()); } #[test] @@ -1979,6 +2363,16 @@ mod wire_version_tests { ); } + #[test] + fn managed_recovery_maps_to_exact_failed_precondition_marker() { + let status = sandbox_to_status(SandboxError::ManagedRecoveryRequired); + assert_eq!(status.code(), tonic::Code::FailedPrecondition); + assert_eq!( + status.message(), + engram_protocol::wire::MANAGED_RECOVERY_REQUIRED_STATUS + ); + } + #[test] fn memory_image_unusable_maps_to_failed_precondition_marker() { let status = sandbox_to_status(SandboxError::MemoryImageUnusable( diff --git a/crates/engram-host-agent/src/harness.rs b/crates/engram-host-agent/src/harness.rs index 38d06a541..ec081b358 100644 --- a/crates/engram-host-agent/src/harness.rs +++ b/crates/engram-host-agent/src/harness.rs @@ -100,6 +100,63 @@ pub struct HarnessHub { inner: Arc, } +/// Minted only after the host persists the exact StartPending intent. The +/// private backend receives a weak sink, so a stream cannot keep its owner alive. +/// Drop revokes admission and active registration under the registration mutex. +pub(crate) struct ManagedHarnessOwner { + hub: HarnessHub, + state: Arc, +} +struct ManagedAttachState { + intent: engram_core::types::managed_boot::ManagedBootStart, + revoked: tokio::sync::watch::Sender, +} +impl ManagedAttachState { + async fn revoked(&self) { + let mut receiver = self.revoked.subscribe(); + let _ = receiver.wait_for(|revoked| *revoked).await; + } + fn validate(&self, inner: &HubInner, attach: &HarnessAttach) -> bool { + let target = &self.intent.receipt.key.context.target; + !*self.revoked.borrow() + && attach.session_id == target.session_id + && attach.sandbox_id == target.sandbox_id + && attach.binding_epoch == target.binding_epoch + && validate_binding(inner, attach) + .is_ok_and(|record| record.sandbox_id == target.sandbox_id) + } +} +impl ManagedHarnessOwner { + pub(crate) fn sink(&self) -> engram_core::traits::HarnessSink { + let state = Arc::downgrade(&self.state); + let inner = self.hub.inner.clone(); + Arc::new(move |stream| { + let Some(state) = state.upgrade() else { + return; + }; + let inner = inner.clone(); + tokio::spawn(async move { + run_managed_connection(inner, state, stream).await; + }); + }) + } +} +impl Drop for ManagedHarnessOwner { + fn drop(&mut self) { + let mut connections = self.hub.inner.connections.lock(); + self.state.revoked.send_replace(true); + let id = self.state.intent.receipt.key.context.target.sandbox_id; + if connections.get(&id).is_some_and(|connection| { + connection + .managed_owner + .as_ref() + .is_some_and(|state| Arc::ptr_eq(state, &self.state)) + }) { + connections.remove(&id); + } + } +} + /// Locking (post-ADR 0073): ONE leaf mutex. The issue #217 "CANONICAL /// LOCK ORDER" era — eight mutexed maps, a documented acquisition /// order, and an ABBA wedge that stalled the whole harness plane while @@ -139,6 +196,7 @@ struct HubInner { } struct ConnectionHandle { + managed_owner: Option>, /// Ack writes retain a sender, so registration owns an explicit close signal. disconnected: Arc, /// Sender into the writer task; takes encoded `HarnessFrame`s. @@ -215,6 +273,10 @@ impl HarnessHub { /// record, so a restarted host-agent rebuilds its proxy registry /// from a local read. Called on the `start_agent` path AFTER the /// in-memory registration succeeds and BEFORE the ack returns. + pub(crate) fn binding_store(&self) -> crate::bindings::BindingStore { + self.inner.bindings.clone() + } + pub fn persist_egress_policy( &self, policy: &engram_core::types::egress::SessionEgressPolicy, @@ -252,6 +314,19 @@ impl HarnessHub { .map(|r| r.sandbox_id) } + pub(crate) fn managed_owner( + &self, + intent: engram_core::types::managed_boot::ManagedBootStart, + ) -> ManagedHarnessOwner { + ManagedHarnessOwner { + hub: self.clone(), + state: Arc::new(ManagedAttachState { + intent, + revoked: tokio::sync::watch::channel(false).0, + }), + } + } + /// Accept an anonymous connection from a harness — used by the /// TCP listener. Reads `HarnessAttach` from the stream, looks up /// `bind_session`'s map for the sandbox_id, then proceeds through @@ -590,6 +665,18 @@ fn register_generation_ordered( handle: ConnectionHandle, ) -> bool { let mut conns = inner.connections.lock(); + if let Some(owner) = &handle.managed_owner { + let target = &owner.intent.receipt.key.context.target; + let attach = HarnessAttach { + session_id: target.session_id, + sandbox_id: target.sandbox_id, + binding_epoch: target.binding_epoch, + harness_version: String::new(), + }; + if target.sandbox_id != sandbox_id || !owner.validate(inner, &attach) { + return false; + } + } match conns.get(&sandbox_id) { Some(existing) if existing.generation > handle.generation => false, _ => { @@ -640,6 +727,20 @@ async fn run_connection( return; } } + if inner + .bindings + .managed_sandbox_required(sandbox_id) + .unwrap_or(true) + { + reject_attach( + &mut writer, + &attach, + AttachReject::UnknownBinding, + "managed owner required", + ) + .await; + return; + } // ADR 0073: even with a transport-derived sandbox_id, the token is // validated against the durable record — the transport tells us // where the bytes came from, the record tells us which GENERATION @@ -660,7 +761,7 @@ async fn run_connection( "attach token/record sandbox differs from transport (live-move shape)", ); } - drive_attached(inner, sandbox_id, attach, reader, writer).await; + drive_attached(inner, sandbox_id, attach, reader, writer, None).await; } Err(reject) => { reject_attach(&mut writer, &attach, reject, reject_reason(reject)).await; @@ -687,7 +788,7 @@ where match validate_attach(&inner, &attach) { Ok(record) => { let sandbox_id = record.sandbox_id; - drive_attached(inner, sandbox_id, attach, reader, writer).await; + drive_attached(inner, sandbox_id, attach, reader, writer, None).await; } Err(reject) => { reject_attach(&mut writer, &attach, reject, reject_reason(reject)).await; @@ -695,8 +796,32 @@ where } } -/// ADR 0073 attach-token validation, shared by both connection paths. -/// The EPOCH is the entire fence: +async fn run_managed_connection(inner: Arc, owner: Arc, stream: S) +where + S: AsyncRead + AsyncWrite + Send + Unpin + 'static, +{ + let (mut reader, mut writer) = tokio::io::split(stream); + let attach = tokio::select! { + biased; + _ = owner.revoked() => return, + result = read_attach_bounded(&mut reader) => match result { Ok(attach) => attach, Err(_) => return }, + }; + if !owner.validate(&inner, &attach) { + tokio::select! { + biased; + _ = owner.revoked() => (), + _ = reject_attach(&mut writer, &attach, AttachReject::SessionMismatch, + "managed owner does not match attachment") => (), + } + return; + } + let sandbox_id = owner.intent.receipt.key.context.target.sandbox_id; + drive_attached(inner, sandbox_id, attach, reader, writer, Some(owner)).await; +} + +/// Ordinary connections use the binding epoch fence. Managed bindings require +/// the separate private owner capability and cannot use either generic path. +/// For ordinary bindings: /// - no record / unreadable record → `UnknownBinding` (transient); /// - presented epoch < record → `Superseded` (fatal); /// - presented epoch > record → `UnknownBinding` (the bind for @@ -707,6 +832,26 @@ where fn validate_attach( inner: &HubInner, attach: &HarnessAttach, +) -> Result { + let record = validate_binding(inner, attach)?; + if inner.bindings.managed_required(attach.session_id) + || inner + .bindings + .managed_sandbox_required(attach.sandbox_id) + .unwrap_or(true) + || inner + .bindings + .managed_sandbox_required(record.sandbox_id) + .unwrap_or(true) + { + return Err(AttachReject::UnknownBinding); + } + Ok(record) +} + +fn validate_binding( + inner: &HubInner, + attach: &HarnessAttach, ) -> Result { let record = match inner.bindings.read(attach.session_id) { Ok(Some(r)) => r, @@ -768,6 +913,7 @@ async fn drive_attached( attach: HarnessAttach, reader: R, mut writer: W, + managed_owner: Option>, ) where R: AsyncRead + Unpin + Send + 'static, W: AsyncWrite + Unpin + Send + 'static, @@ -788,6 +934,7 @@ async fn drive_attached( let disconnected = Arc::new(Notify::new()); let prompt_trace = Arc::new(Mutex::new(None)); let handle = ConnectionHandle { + managed_owner: managed_owner.clone(), disconnected: disconnected.clone(), cmd_tx: cmd_tx.clone(), pending_checkpoint: Mutex::new(None), @@ -818,7 +965,12 @@ async fn drive_attached( reject: None, message: None, }; - if let Err(e) = write_msg(&mut writer, &ack).await { + let ack_result = tokio::select! { + biased; + _ = managed_revoked(managed_owner.as_ref()) => Err(std::io::Error::other("managed owner revoked")), + result = write_msg(&mut writer, &ack) => result, + }; + if let Err(e) = ack_result { tracing::debug!(error = %e, "harness ack write failed"); // We registered but the peer can never learn it: undo, guarded // by our own generation (a newer connection may already own the @@ -854,6 +1006,7 @@ async fn drive_attached( // event acks the row) — host memory holds nothing durable. let reader_outcome = tokio::select! { biased; + _ = managed_revoked(managed_owner.as_ref()) => Err(HarnessError::WriterClosed), _ = disconnected.notified() => Err(HarnessError::WriterClosed), result = reader_loop(reader, &inner, attach.session_id, sandbox_id, attach.binding_epoch, &cmd_tx, &prompt_trace) => result, _ = &mut writer_task => Err(HarnessError::WriterClosed), @@ -887,6 +1040,13 @@ async fn drive_attached( } } +async fn managed_revoked(owner: Option<&Arc>) { + match owner { + Some(owner) => owner.revoked().await, + None => std::future::pending().await, + } +} + /// Agent-message chunks per run that get an event span. A reply streams /// one chunk per token batch; the first few show the time to first /// token, and the rest post without a span. @@ -1155,6 +1315,84 @@ mod tests { (sink, collected) } + #[tokio::test] + async fn managed_binding_rejects_generic_and_transport_claimed_streams() { + let (sink, events) = collecting_sink(); + let hub = test_hub(sink); + let session_id = SessionId::new(); + let sandbox_id = SandboxId::new(); + hub.bind_session(session_id, sandbox_id, 1).unwrap(); + std::fs::write( + hub.binding_store() + .directory() + .join(format!("{session_id}.managed-boot")), + b"1", + ) + .unwrap(); + std::fs::write( + hub.binding_store() + .directory() + .join(format!("{sandbox_id}.managed-sandbox")), + b"1", + ) + .unwrap(); + for transport in [None, Some(sandbox_id), Some(SandboxId::new())] { + let (host, mut guest) = duplex_pair(); + if let Some(transport) = transport { + hub.accept_connection(transport, Some(session_id), host); + } else { + hub.accept_via_session_lookup(host); + } + write_msg( + &mut guest, + &HarnessAttach { + session_id, + sandbox_id, + binding_epoch: 1, + harness_version: "test".into(), + }, + ) + .await + .unwrap(); + let ack: HarnessAttachAck = read_msg(&mut guest).await.unwrap(); + assert!( + !ack.ok, + "guest fields cannot grant managed transport authority" + ); + assert!(!hub.is_attached(sandbox_id)); + } + let ordinary_session = SessionId::new(); + let ordinary_sandbox = SandboxId::new(); + hub.bind_session(ordinary_session, ordinary_sandbox, 1) + .unwrap(); + for transport in [None, Some(sandbox_id)] { + let (host, mut guest) = duplex_pair(); + if let Some(transport) = transport { + hub.accept_connection(transport, Some(ordinary_session), host); + } else { + hub.accept_via_session_lookup(host); + } + write_msg( + &mut guest, + &HarnessAttach { + session_id: ordinary_session, + sandbox_id: ordinary_sandbox, + binding_epoch: 1, + harness_version: "test".into(), + }, + ) + .await + .unwrap(); + let ack: HarnessAttachAck = read_msg(&mut guest).await.unwrap(); + assert_eq!( + ack.ok, + transport.is_none(), + "ordinary lookup stays valid; managed transport cannot be relabeled ordinary" + ); + } + assert!(events.lock().is_empty()); + } + /// ADR 0073: hub over an ephemeral binding store. Tests exercise /// the same disk-validated attach path as production. fn test_hub(sink: EventSink) -> HarnessHub { @@ -1778,6 +2016,7 @@ mod tests { &hub.inner, sandbox_id, ConnectionHandle { + managed_owner: None, disconnected: Arc::new(Notify::new()), cmd_tx: tx_new, pending_checkpoint: Mutex::new(None), @@ -1790,6 +2029,7 @@ mod tests { &hub.inner, sandbox_id, ConnectionHandle { + managed_owner: None, disconnected: Arc::new(Notify::new()), cmd_tx: tx_old, pending_checkpoint: Mutex::new(None), diff --git a/crates/engram-host-agent/src/host_client.rs b/crates/engram-host-agent/src/host_client.rs index 539e6a549..59fb0940d 100644 --- a/crates/engram-host-agent/src/host_client.rs +++ b/crates/engram-host-agent/src/host_client.rs @@ -11,7 +11,7 @@ //! `engram-protocol::server` serves over the WS — so the coord on //! the other end of a `RemoteHostClient` is talking to one of these. //! -//! All sandbox methods are pure delegation to the inner `SandboxBackend`. +//! Sandbox methods delegate after the applicable host lifecycle guards. //! The harness methods (`bind_session`, `unbind_session`, //! `send_prompt`) delegate to the `HarnessHub`, mapping its //! `HarnessError` into `SandboxError` so the trait surface stays @@ -75,6 +75,8 @@ impl std::str::FromStr for IdleEvictionHarness { pub struct LocalHostClient { sandbox: Arc, harness_hub: Arc, + managed: Option>, + boot: Option>, idle_eviction_harness: IdleEvictionHarness, } @@ -87,10 +89,91 @@ impl LocalHostClient { Self { sandbox, harness_hub, + managed: None, + boot: None, idle_eviction_harness, } } + pub fn with_managed_boot(mut self, boot: Arc) -> Self { + self.boot = Some(boot); + self + } + /// The coordinator has disowned this exact sandbox through its heartbeat. + pub(crate) async fn destroy_tombstoned(&self, id: SandboxId) -> Result<(), SandboxError> { + if self + .harness_hub + .binding_store() + .managed_sandbox_required(id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + { + self.boot()?.destroy_tombstoned(id).await + } else { + self.sandbox.destroy(id).await + } + } + fn boot(&self) -> Result<&Arc, SandboxError> { + self.boot + .as_ref() + .ok_or_else(|| SandboxError::InvalidSpec("managed boot unavailable".into())) + } + fn ordinary_only(&self, id: SandboxId) -> Result<(), SandboxError> { + if self + .harness_hub + .binding_store() + .managed_sandbox_required(id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + { + return Err(SandboxError::InvalidSpec( + "managed session requires its current boot owner".into(), + )); + } + Ok(()) + } + async fn running_or_ordinary(&self, id: SandboxId) -> Result<(), SandboxError> { + if self + .harness_hub + .binding_store() + .managed_sandbox_required(id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + { + return self.boot()?.guard_running(id).await; + } + Ok(()) + } + pub fn with_managed_policy( + mut self, + host: Arc, + ) -> Self { + self.managed = Some(host); + self + } + + fn check_restore_gate( + &self, + fence: SessionFence, + sandbox: Option, + ) -> Result<(), SandboxError> { + let bindings = self.harness_hub.binding_store(); + let mut managed = bindings.managed_required(fence.session_id); + if let Some(sandbox) = sandbox { + managed |= bindings + .managed_sandbox_required(sandbox) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))?; + } + if managed { + return Err(SandboxError::InvalidSpec( + "managed restore quarantine unavailable".into(), + )); + } + if self.managed.is_some() && fence.session_id.as_uuid().is_nil() { + return Err(SandboxError::InvalidSpec( + "restore session identity required".into(), + )); + } + Ok(()) + } + /// Convenience constructor for callers that don't need to route /// harness events through a shared hub (tests, in-process glue /// where the harness path isn't exercised). Builds a `HarnessHub` @@ -117,6 +200,9 @@ impl LocalHostClient { #[async_trait] impl HostClient for LocalHostClient { async fn create(&self, spec: SandboxSpec) -> Result { + if let Some(boot) = &self.boot { + boot.ordinary_spec(&spec)?; + } let id = SandboxId::new(); self.sandbox.create(id, spec).await?; Ok(id) @@ -126,7 +212,16 @@ impl HostClient for LocalHostClient { // server's `check_session_epoch` gate BEFORE it delegates here — // the local composition is behind the fence, not the fence itself // (and in --mode=all there is no second writer to fence out). - async fn destroy(&self, id: SandboxId, _fence: SessionFence) -> Result<(), SandboxError> { + async fn destroy(&self, id: SandboxId, fence: SessionFence) -> Result<(), SandboxError> { + if self + .harness_hub + .binding_store() + .managed_sandbox_required(id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + { + return self.boot()?.destroy(id, fence).await; + } + self.ordinary_only(id)?; // ADR 0073 phase 4: no shell pin to clear — the pin is a PG // column stamped by the coordinator's relay and lapses by // itself (the issue #219 leak class is gone with the refcount). @@ -134,13 +229,28 @@ impl HostClient for LocalHostClient { } async fn list(&self) -> Result, SandboxError> { - self.sandbox.list().await + let mut ids = Vec::new(); + for id in self.sandbox.list().await? { + if !self + .harness_hub + .binding_store() + .managed_sandbox_required(id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + { + ids.push(id); + } + } + if let Some(boot) = &self.boot { + ids.extend(boot.list()?); + } + Ok(ids) } async fn probe_sandbox( &self, id: SandboxId, ) -> Result { + self.ordinary_only(id)?; self.sandbox.probe_sandbox(id).await } @@ -149,10 +259,28 @@ impl HostClient for LocalHostClient { id: SandboxId, cmd: ExecRequest, ) -> Result { + if self + .harness_hub + .binding_store() + .managed_sandbox_required(id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + { + return self.boot()?.exec_stream(id, cmd).await; + } + self.ordinary_only(id)?; self.sandbox.exec_stream(id, cmd).await } async fn cancel_exec(&self, id: SandboxId, exec_id: String) -> Result<(), SandboxError> { + if self + .harness_hub + .binding_store() + .managed_sandbox_required(id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + { + return self.boot()?.cancel_exec(id, exec_id).await; + } + self.ordinary_only(id)?; self.sandbox.cancel_exec(id, exec_id).await } @@ -162,6 +290,15 @@ impl HostClient for LocalHostClient { spec: SessionFileSpec, bytes: SessionFileStream, ) -> Result { + if self + .harness_hub + .binding_store() + .managed_sandbox_required(id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + { + return self.boot()?.write_file(id, spec, bytes).await; + } + self.ordinary_only(id)?; self.sandbox.write_file(id, spec, bytes).await } @@ -170,6 +307,15 @@ impl HostClient for LocalHostClient { id: SandboxId, path: String, ) -> Result<(SessionFileMetadata, SessionFileStream), SandboxError> { + if self + .harness_hub + .binding_store() + .managed_sandbox_required(id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + { + return self.boot()?.read_file(id, path).await; + } + self.ordinary_only(id)?; self.sandbox.read_file(id, path).await } @@ -178,6 +324,7 @@ impl HostClient for LocalHostClient { id: SandboxId, _fence: SessionFence, ) -> Result { + self.ordinary_only(id)?; self.sandbox.snapshot(id).await } @@ -186,6 +333,7 @@ impl HostClient for LocalHostClient { id: SandboxId, _fence: SessionFence, ) -> Result { + self.ordinary_only(id)?; self.sandbox.snapshot_hold(id, false).await } @@ -194,6 +342,7 @@ impl HostClient for LocalHostClient { id: SandboxId, _fence: SessionFence, ) -> Result { + self.ordinary_only(id)?; // The coordinator calls this only for an idle eviction (target // `Idle`); live teleport and checkpoints capture through other // methods and never drain. A drain is best-effort: a harness that @@ -218,6 +367,7 @@ impl HostClient for LocalHostClient { id: SandboxId, _fence: SessionFence, ) -> Result { + self.ordinary_only(id)?; self.sandbox.migration_presetup(id).await } @@ -227,6 +377,7 @@ impl HostClient for LocalHostClient { export_id: &str, _fence: SessionFence, ) -> Result { + self.ordinary_only(id)?; self.sandbox.migration_capture_postcopy(id, export_id).await } @@ -234,6 +385,7 @@ impl HostClient for LocalHostClient { &self, id: SandboxId, ) -> Result { + self.ordinary_only(id)?; self.sandbox.migration_drain_wait(id).await } @@ -257,6 +409,7 @@ impl HostClient for LocalHostClient { export_id: &str, _fence: SessionFence, ) -> Result<(), SandboxError> { + self.ordinary_only(id)?; self.sandbox.migration_commit(id, export_id).await } @@ -266,6 +419,7 @@ impl HostClient for LocalHostClient { export_id: &str, _fence: SessionFence, ) -> Result<(), SandboxError> { + self.ordinary_only(id)?; self.sandbox.migration_abort(id, export_id).await } @@ -275,6 +429,7 @@ impl HostClient for LocalHostClient { export_id: &str, _fence: SessionFence, ) -> Result { + self.ordinary_only(id)?; self.sandbox.migration_abort_unserved(id, export_id).await } @@ -283,6 +438,7 @@ impl HostClient for LocalHostClient { id: SandboxId, _fence: SessionFence, ) -> Result<(), SandboxError> { + self.ordinary_only(id)?; self.sandbox.commit_snapshot(id).await } @@ -291,14 +447,20 @@ impl HostClient for LocalHostClient { id: SandboxId, _fence: SessionFence, ) -> Result<(), SandboxError> { + self.ordinary_only(id)?; self.sandbox.abort_snapshot(id).await } async fn restore( &self, metadata: SnapshotMetadata, - _fence: SessionFence, + fence: SessionFence, ) -> Result { + self.check_restore_gate(fence, metadata.source_sandbox_id)?; + if let Some(boot) = &self.boot { + boot.ordinary_disk(metadata.disk_manifest, None)?; + boot.ordinary_disk(metadata.swap_manifest, None)?; + } let id = SandboxId::new(); self.sandbox.restore(id, metadata, None).await?; Ok(id) @@ -330,8 +492,13 @@ impl HostClient for LocalHostClient { metadata: SnapshotMetadata, session_env: std::collections::HashMap, selected_mounts: Vec, - _fence: SessionFence, + fence: SessionFence, ) -> Result { + self.check_restore_gate(fence, metadata.source_sandbox_id)?; + if let Some(boot) = &self.boot { + boot.ordinary_disk(metadata.disk_manifest, None)?; + boot.ordinary_disk(metadata.swap_manifest, None)?; + } let id = SandboxId::new(); self.sandbox .restore_base_for_session(id, metadata, session_env, selected_mounts) @@ -339,13 +506,111 @@ impl HostClient for LocalHostClient { Ok(id) } + async fn read_managed_cold_assets( + &self, + request: engram_core::types::managed_assets::ManagedColdAssetsRequest, + ) -> Result { + self.boot()?.read_cold_assets(request).await + } + async fn prepare_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootPrepare, + ) -> Result { + self.boot()?.prepare(request).await + } + async fn read_managed_boot( + &self, + key: engram_core::types::managed_boot::ManagedBootKey, + ) -> Result { + self.boot()?.read(key).await + } + async fn finalize_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootFinalize, + ) -> Result { + self.boot()?.finalize(request).await + } + async fn start_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootStart, + ) -> Result { + self.boot()?.start(request).await + } + async fn capture_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedCaptureRequest, + ) -> Result { + self.boot()?.capture(request).await + } + + async fn read_managed_policy( + &self, + target: engram_core::types::managed_deny::ManagedPolicyTarget, + ) -> Result, SandboxError> { + if self + .harness_hub + .binding_store() + .managed_sandbox_required(target.sandbox_id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + && self.boot.is_some() + { + return self.boot()?.read_policy(target).await; + } + self.managed + .as_ref() + .ok_or_else(|| SandboxError::InvalidSpec("managed host mode unavailable".into()))? + .read_applied(&target) + .await + } + async fn apply_managed_policy( + &self, + request: engram_core::types::managed_deny::ManagedPolicyApply, + ) -> Result { + if self + .harness_hub + .binding_store() + .managed_sandbox_required(request.context.target.sandbox_id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + && self.boot.is_some() + { + return self.boot()?.apply(request).await; + } + self.managed + .as_ref() + .ok_or_else(|| SandboxError::InvalidSpec("managed host mode unavailable".into()))? + .compare_and_apply(request) + .await + } async fn start_agent( &self, id: SandboxId, agent: AgentSpec, policy: SessionEgressPolicy, - _fence: SessionFence, + fence: SessionFence, ) -> Result<(), SandboxError> { + let bindings = self.harness_hub.binding_store(); + if bindings + .managed_sandbox_required(id) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))? + || bindings.managed_required(fence.session_id) + || policy.managed_deny.is_some() + || self + .harness_hub + .binding_store() + .managed_required(policy.session_id) + { + return Err(SandboxError::InvalidSpec( + "managed session requires acknowledged boot finalization".into(), + )); + } + // Legacy ordinary callers can still use the explicit unfenced sentinel. + // Every supplied identity must otherwise agree before any durable mutation. + if policy.sandbox_id != id + || policy.session_id.as_uuid().is_nil() + || (fence != SessionFence::unfenced() && fence.session_id != policy.session_id) + { + return Err(SandboxError::InvalidSpec("start identity mismatch".into())); + } // ADR 0073: persist the binding record BEFORE the harness can // dial (the spawn below), so the first attach validates instead // of bouncing UnknownBinding. Monotonic: a stale caller's bind @@ -385,6 +650,9 @@ impl HostClient for LocalHostClient { } async fn guest_ip(&self, id: SandboxId) -> Option { + if self.ordinary_only(id).is_err() { + return None; + } self.sandbox .guest_endpoints(id) .await @@ -397,12 +665,29 @@ impl HostClient for LocalHostClient { sandbox_id: SandboxId, binding_epoch: u64, ) -> Result<(), engram_core::SandboxError> { + self.ordinary_only(sandbox_id)?; + if self + .harness_hub + .binding_store() + .managed_required(session_id) + { + return Err(SandboxError::InvalidSpec( + "managed session requires its boot owner".into(), + )); + } self.harness_hub .bind_session(session_id, sandbox_id, binding_epoch) .map_err(|e| engram_core::SandboxError::Vm(format!("bind session: {e}").into())) } async fn unbind_session(&self, session_id: SessionId) { + if self + .harness_hub + .binding_store() + .managed_required(session_id) + { + return; + } self.harness_hub.unbind_session(session_id); } @@ -413,6 +698,7 @@ impl HostClient for LocalHostClient { text: String, mode: Option, ) -> Result<(), SandboxError> { + self.running_or_ordinary(sandbox_id).await?; self.harness_hub .send_prompt(sandbox_id, prompt_id, text, mode) .await @@ -425,6 +711,7 @@ impl HostClient for LocalHostClient { prompt_id: String, text: String, ) -> Result<(), SandboxError> { + self.running_or_ordinary(sandbox_id).await?; self.harness_hub .edit_queued_prompt(sandbox_id, prompt_id, text) .await @@ -436,6 +723,7 @@ impl HostClient for LocalHostClient { sandbox_id: SandboxId, prompt_id: String, ) -> Result<(), SandboxError> { + self.running_or_ordinary(sandbox_id).await?; self.harness_hub .dequeue_queued_prompt(sandbox_id, prompt_id) .await @@ -448,6 +736,7 @@ impl HostClient for LocalHostClient { tool_call_id: String, result_json: String, ) -> Result<(), SandboxError> { + self.running_or_ordinary(sandbox_id).await?; self.harness_hub .tool_result(sandbox_id, tool_call_id, result_json) .await @@ -455,6 +744,7 @@ impl HostClient for LocalHostClient { } async fn interrupt(&self, sandbox_id: SandboxId) -> Result<(), SandboxError> { + self.running_or_ordinary(sandbox_id).await?; self.harness_hub .interrupt(sandbox_id) .await @@ -464,14 +754,12 @@ impl HostClient for LocalHostClient { // ADR 0045 Phase F: freeze/unfreeze the microVM in place — pure // delegation to the inner backend (no harness involvement). async fn pause(&self, sandbox_id: SandboxId, _fence: SessionFence) -> Result<(), SandboxError> { + self.ordinary_only(sandbox_id)?; self.sandbox.pause(sandbox_id).await } - async fn resume( - &self, - sandbox_id: SandboxId, - _fence: SessionFence, - ) -> Result<(), SandboxError> { + async fn resume(&self, sandbox_id: SandboxId, fence: SessionFence) -> Result<(), SandboxError> { + self.check_restore_gate(fence, Some(sandbox_id))?; self.sandbox.resume(sandbox_id).await } @@ -479,6 +767,7 @@ impl HostClient for LocalHostClient { &self, sandbox_id: SandboxId, ) -> Result { + self.ordinary_only(sandbox_id)?; // ADR 0014 issue #6 + ADR 0066: bridge WS frames through a // ShellTunnel pair, reaching ttyd via the vsock relay when the // backend has one (FC; VZ after Phase 2) or a direct dial_ip @@ -526,6 +815,7 @@ impl HostClient for LocalHostClient { sandbox_id: SandboxId, port: u16, ) -> Result { + self.ordinary_only(sandbox_id)?; let (tunnel, ends) = engram_core::types::port::PortTunnel::pair(); // ADR 0066: FC (and, after its Phase 2 migration, VZ) reach the dev // server through the in-guest agentd relay, which dials the guest's own @@ -561,6 +851,7 @@ impl HostClient for LocalHostClient { &self, sandbox_id: SandboxId, ) -> Result { + self.ordinary_only(sandbox_id)?; // ADR 0065: bring up the in-guest browser stack (Xvfb + x11vnc + // headful chromium with the CDP debug port) and return the VNC port // (+ agentd's optional chromium-CDP liveness warning, issue #569). @@ -570,12 +861,14 @@ impl HostClient for LocalHostClient { } async fn stop_browser(&self, sandbox_id: SandboxId) -> Result<(), SandboxError> { + self.ordinary_only(sandbox_id)?; // ADR 0065: forward to the inner backend. Teardown is at the snapshot / // idle-eviction boundary (the browser is ephemeral, never snapshotted). self.sandbox.stop_browser(sandbox_id).await } async fn start_ide(&self, sandbox_id: SandboxId) -> Result { + self.ordinary_only(sandbox_id)?; // ADR 0085: bring up the in-guest IDE (code-server) and return its // loopback HTTP port. The orchestrator reaches it over the ADR-0066 // vsock port relay — the guest binds loopback, agentd dials it. @@ -583,6 +876,7 @@ impl HostClient for LocalHostClient { } async fn stop_ide(&self, sandbox_id: SandboxId) -> Result<(), SandboxError> { + self.ordinary_only(sandbox_id)?; // ADR 0085: forward to the inner backend. Teardown is at the snapshot / // idle-eviction boundary (code-server is ephemeral, never snapshotted). self.sandbox.stop_ide(sandbox_id).await diff --git a/crates/engram-host-agent/src/image_prefetch.rs b/crates/engram-host-agent/src/image_prefetch.rs index 4f1669e91..a26497881 100644 --- a/crates/engram-host-agent/src/image_prefetch.rs +++ b/crates/engram-host-agent/src/image_prefetch.rs @@ -587,8 +587,9 @@ async fn reconcile( // memfile must land — the SAME path a base session.create restore // reads (`snapshot_path_for(base_snapshot_id)/memory.bin`). `None` // when density is off. Recorded for disable-time reclaim. - let base_memfile = - base_memfile_dir.map(|resolver| resolver(image.base_snapshot_id).join("memory.bin")); + let base_memfile = base_memfile_dir + .filter(|_| image.base_snapshot_memory_manifest.is_some()) + .map(|resolver| resolver(image.base_snapshot_id).join("memory.bin")); if let Some(ref path) = base_memfile { memfiles .entry(image.manifest_digest.clone()) @@ -1955,6 +1956,47 @@ mod tests { assert!(readiness.contains(&img.manifest_digest), "still ready"); } + #[tokio::test] + async fn reconcile_tracks_base_memfile_only_with_memory_manifest() { + let (store, cache, dir, disk_ref, mem_ref, _mem_bytes) = seed().await; + let snapshot_root = dir.path().join("snapshots"); + let resolver: SnapshotDirResolver = Arc::new(move |id| snapshot_root.join(format!("{id}"))); + + for memory_manifest in [None, Some(mem_ref)] { + let img = image_ref(SnapshotId::new(), disk_ref, memory_manifest); + let mut memfiles = HashMap::new(); + reconcile( + std::slice::from_ref(&img), + ImageReadiness::new(), + parked(&store), + cache.clone(), + crate::peer_fill::PeerHealth::new(), + Some(&resolver), + &mut memfiles, + Arc::new(Mutex::new(HashMap::new())), + Arc::new(crate::ram_ledger::RamLedger::new()), + Arc::new(Mutex::new(HashMap::new())), + Arc::new(Mutex::new(HashSet::new())), + Arc::new(Mutex::new(HashSet::new())), + ) + .await; + + match memory_manifest { + None => assert!( + !memfiles.contains_key(&img.manifest_digest), + "a disk-only image must not require a base memfile", + ), + Some(_) => assert_eq!( + memfiles.get(&img.manifest_digest).unwrap().path, + dir.path() + .join("snapshots") + .join(img.base_snapshot_id.to_string()) + .join("memory.bin"), + ), + } + } + } + /// ADR 0092: a vanished base memfile (out-of-band rm — even under a /// live mlock, which silently pins the unlinked inode) flips the image /// unready and drops the stale `memfiles` entry so the re-prefetch diff --git a/crates/engram-host-agent/src/lib.rs b/crates/engram-host-agent/src/lib.rs index d5bd41b5a..4dce9b803 100644 --- a/crates/engram-host-agent/src/lib.rs +++ b/crates/engram-host-agent/src/lib.rs @@ -114,6 +114,8 @@ pub use config::{HostAgentConfig, DEFAULT_DISK_HYDRATION_WINDOW_CHUNKS}; pub struct HostAgent { pub cfg: HostAgentConfig, + managed_ownership: Option>, + managed_backend: Option, pub sandbox: Arc, pub cloud: Arc, /// Local egress proxy. The production binary refuses to start @@ -152,6 +154,8 @@ impl HostAgent { ) -> Self { Self { cfg, + managed_ownership: None, + managed_backend: None, sandbox, cloud, egress: None, @@ -218,9 +222,51 @@ impl HostAgent { self } + pub fn with_managed_backend(mut self, backend: managed_boot::ManagedBackend) -> Self { + self.managed_backend = Some(backend); + self + } + + pub fn with_managed_ownership( + mut self, + ownership: Option>, + ) -> Self { + self.managed_ownership = ownership; + self + } + /// Run until ctrl-c: the reattach pass, the background drivers, and /// the coordinator dialer when `coordinator_endpoint` is set. pub async fn run(self) -> Result<(), HostAgentError> { + let managed_ownership = match self.managed_ownership.clone() { + Some(owner) => Some(owner), + None => managed_policy::ManagedOwnership::for_mode( + &self.cfg.work_dir.join("bindings"), + self.cfg.managed_tls.is_some(), + )?, + }; + let managed_server = if let Some(tls) = &self.cfg.managed_tls { + if self.cfg.grpc_listen_addr.is_none() + || !self + .cfg + .grpc_advertise_addr + .as_ref() + .is_some_and(|a| a.starts_with("https://")) + || self.egress.is_none() + || self.host_id.is_none() + { + return Err(HostAgentError::Config("managed host mode requires an HTTPS lifecycle listener, stable host identity, and egress proxy".into())); + } + Some( + grpc_server::PreparedHostServer::bind( + self.cfg.grpc_listen_addr.expect("managed listener checked"), + Some(tls.clone()), + ) + .map_err(|error| HostAgentError::Config(error.to_string()))?, + ) + } else { + None + }; tracing::info!(?self.cfg.work_dir, "host-agent starting"); let _ = self.cloud.host_metadata().await; @@ -372,7 +418,10 @@ impl HostAgent { // (2026-07-13 incident: a survivor's evict capture ran // 40+ minutes). arc.rehydrate_chain_heads().await; - arc.reconcile_volumes().await; + let bindings = + crate::bindings::BindingStore::open(self.cfg.work_dir.join("bindings")) + .map_err(HostAgentError::Io)?; + arc.reconcile_volumes(&bindings).await; arc }; // ADR 0084 P1b: the capture-job executor + durable-record @@ -688,6 +737,29 @@ impl HostAgent { .expect("open binding store under work_dir (ADR 0073)"); let harness_hub = std::sync::Arc::new(crate::harness::HarnessHub::new(event_sink, bindings)); + let managed_policy = if self.cfg.managed_tls.is_some() { + use engram_core::traits::Entropy; + let managed = crate::managed_policy::ManagedPolicyHost::new( + host_id, + engram_core::traits::OsEntropy.uuid().to_string(), + harness_hub.binding_store(), + pooled.clone(), + self.egress.clone().expect("managed configuration checked"), + managed_ownership + .clone() + .expect("managed ownership acquired"), + )?; + Some(Arc::new(managed)) + } else { + None + }; + if let Some(managed) = &managed_policy { + managed + .recover_all() + .await + .map_err(|e| HostAgentError::Config(e.to_string()))?; + } + // Plumb the hub into the FC/VZ backend's vsock-accept // sink so inbound harness connections land on the local // hub's adapter loop. Without this the FC backend drops @@ -709,6 +781,9 @@ impl HostAgent { match harness_hub.persisted_egress_policies() { Ok(policies) => pooled.rebuild_egress_from_policies(policies).await, Err(e) => { + if self.cfg.managed_tls.is_some() { + return Err(HostAgentError::Io(e)); + } tracing::warn!(error = %e, "list persisted egress policies failed"); } } @@ -728,11 +803,19 @@ impl HostAgent { // persisted policies. Not armed until here because the // replay needs the pooled backend. let hub_for_replay = harness_hub.clone(); + let managed_for_replay = managed_policy.clone(); let pooled_for_replay = pooled.clone(); egress.spawn_supervisor(std::sync::Arc::new(move || { let hub = hub_for_replay.clone(); + let managed = managed_for_replay.clone(); let pooled = pooled_for_replay.clone(); Box::pin(async move { + if let Some(managed) = managed { + if let Err(error) = managed.recover_all().await { + tracing::error!(%error, "managed proxy recovery failed; admission remains closed"); + return; + } + } match hub.persisted_egress_policies() { Ok(policies) => pooled.rebuild_egress_from_policies(policies).await, Err(e) => { @@ -832,19 +915,48 @@ impl HostAgent { pooled.set_upload_sink(upload_sink); // ADR 0015 M5: warm-pool retired. LocalHostClient is just // the cold-create dispatch wrapper now. - let local_host: Arc = - Arc::new(crate::host_client::LocalHostClient::new( + let mut local = crate::host_client::LocalHostClient::new( + pooled.clone(), + harness_hub.clone(), + self.cfg.idle_eviction_harness, + ); + if let Some(managed) = managed_policy { + local = local.with_managed_policy(managed); + } + let mut managed_boot_host = None; + if self.cfg.managed_tls.is_some() { + use engram_core::traits::Entropy; + let backend = self.managed_backend.clone().ok_or_else(|| { + HostAgentError::Config("managed backend is not installed".into()) + })?; + let fill = self + .fill + .as_ref() + .ok_or_else(|| { + HostAgentError::Config("managed disk storage is not installed".into()) + })? + .clone(); + let boot = managed_boot::ManagedBootHost::new( + managed_boot::ManagedBootHostConfig { + host_id, + incarnation: engram_core::traits::OsEntropy.uuid().to_string(), + directory: self.cfg.work_dir.join("managed-boots"), + }, + backend, pooled.clone(), + self.egress.clone().expect("managed egress checked"), harness_hub.clone(), - self.cfg.idle_eviction_harness, - )); - // ADR 0009 §2: populate `running_sandboxes` from - // `backend.list()` on each heartbeat tick. The coord - // intersects this against expected-active sessions to - // detect divergence (missing sandbox → flip session per - // §3). On `list()` error, ship an empty list — the - // 3-strike grace window (15s) absorbs transient errors - // without flipping live sessions. + fill, + ) + .map_err(|e| HostAgentError::Config(e.to_string()))?; + let boot = Arc::new(boot); + managed_boot_host = Some(boot.clone()); + local = local.with_managed_boot(boot); + } + let tombstone_host = Arc::new(local); + let local_host: Arc = tombstone_host.clone(); + // The coordinator compares known inventory against active sessions. + // A failed read reports unknown inventory and cannot prove absence. // Seed capacity once at startup (/proc/meminfo on Linux, // hw.memsize on macOS). Used by the scheduler's fit check — // without this the coord sees `total_mib=0` and rejects @@ -902,13 +1014,16 @@ impl HostAgent { // the `grpc_self_connect` probe, and a probe that runs before // the bind fails, which keeps the host out of placement until // the next heartbeat. The server starts serving it below. - let grpc_incoming = self.cfg.grpc_listen_addr.and_then(|addr| { - grpc_server::bind(addr) - .inspect_err(|e| { - tracing::error!(addr = %addr, error = %e, "gRPC listener bind failed"); - }) - .ok() - }); + let host_server = match managed_server { + Some(server) => Some(server), + None => self.cfg.grpc_listen_addr.and_then(|addr| { + grpc_server::PreparedHostServer::bind(addr, None) + .inspect_err(|e| { + tracing::error!(addr = %addr, error = %e, "gRPC listener bind failed"); + }) + .ok() + }), + }; let mut registration_task: Option> = None; if let Some(advertise_addr) = self.cfg.grpc_advertise_addr.clone() { // hostname: prefer the env-supplied value (set by @@ -1027,7 +1142,7 @@ impl HostAgent { // via /api/hosts/register) to dispatch coord→host // RPCs. Required in production; mode=all leaves // `grpc_listen_addr=None`. - let grpc_task = grpc_incoming.map(|incoming| { + let grpc_task = host_server.map(|server| { let local_for_grpc = local_host.clone(); let peer_for_grpc = peer_serve.clone(); // ADR 0079: the per-session fencing-epoch high-water, @@ -1036,9 +1151,7 @@ impl HostAgent { session_epochs::SessionEpochStore::open(self.cfg.work_dir.join("epochs")) .expect("open session epoch store under work_dir (ADR 0079)"); tokio::spawn(async move { - if let Err(e) = - grpc_server::serve(incoming, local_for_grpc, epochs, peer_for_grpc).await - { + if let Err(e) = server.serve(local_for_grpc, epochs, peer_for_grpc).await { tracing::error!(error = %e, "gRPC server terminated with error"); } }) @@ -1223,6 +1336,7 @@ impl HostAgent { // heartbeat/registration path anyway.) const HEARTBEAT_FAILURES_BEFORE_ESCALATION: u32 = 6; // ~30s at 5s cadence let mut consecutive_heartbeat_failures: u32 = 0; + let managed_list_host = local_host.clone(); // The enabled-images channel starts empty, which is not an ack. // The first ack always notifies, even when the set is empty, so // the supervisors can tell "no images" from "not told yet". @@ -1246,7 +1360,7 @@ impl HostAgent { // coord skips reconcile for this heartbeat instead of // mistaking the empty set for a real running set. // Residents include running and tearing-down sandboxes. - let (running_sandboxes, running_sandboxes_known) = match pooled_for_heartbeat + let (running_sandboxes, running_sandboxes_known) = match managed_list_host .list() .await { @@ -1542,9 +1656,9 @@ impl HostAgent { // capture-assignment arm: its `None` case // `continue`s the loop. for &sandbox_id in &resp.tombstoned_sandboxes { - let backend = pooled_for_heartbeat.clone(); + let host = tombstone_host.clone(); tokio::spawn(async move { - match backend.destroy(sandbox_id).await { + match host.destroy_tombstoned(sandbox_id).await { Ok(()) => tracing::info!( %sandbox_id, "destroyed tombstoned sandbox (coordinator disowned it)", @@ -1657,6 +1771,9 @@ impl HostAgent { // exact post-bitmap failure this quiesce exists to prevent. // In-flight captures are drained (bounded) below instead. pooled.quiesce_captures_for_shutdown(); + if let Some(boot) = &managed_boot_host { + boot.shutdown().await; + } // ADR 0116 A-D2: declare the handoff BEFORE the heartbeat // task dies — the coordinator must learn "this exit is // planned; hold the lease" while our channel to it is still @@ -1974,3 +2091,112 @@ mod tests { assert!(adopt_handoff_marker(dir.path()).is_none()); } } + +pub mod managed_policy; + +pub mod managed_tls; + +mod managed_assets; +pub mod managed_boot; + +#[cfg(test)] +mod managed_startup_tests { + use super::*; + use std::sync::atomic::{AtomicBool, Ordering}; + + struct StartupBoundary(AtomicBool); + #[async_trait::async_trait] + impl CloudBackend for StartupBoundary { + async fn host_metadata( + &self, + ) -> Result { + self.0.store(true, Ordering::SeqCst); + std::future::pending().await + } + async fn provision_host( + &self, + _: engram_core::types::host::HostSpec, + ) -> Result { + unreachable!() + } + async fn deprovision_host( + &self, + _: engram_core::HostId, + ) -> Result<(), engram_core::BackendError> { + unreachable!() + } + } + + async fn rejects_before_startup(malformed: bool) { + let temp = tempfile::tempdir().unwrap(); + let ca = engram_egress_proxy::Ca::load_or_generate(&temp.path().join("ca")).unwrap(); + let certificate = temp.path().join("host.pem"); + let key = temp.path().join("host.key"); + std::fs::write( + &certificate, + if malformed { + "not a certificate" + } else { + &ca.cert_pem + }, + ) + .unwrap(); + std::fs::write(&key, ca.key_pair.serialize_pem()).unwrap(); + let occupied = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = occupied.local_addr().unwrap(); + // Malformed TLS must fail even with an available listener. + let occupied = if malformed { + drop(occupied); + None + } else { + Some(occupied) + }; + let cloud = Arc::new(StartupBoundary(AtomicBool::new(false))); + let config = HostAgentConfig { + managed_tls: Some(managed_tls::ManagedHostTls { + certificate: certificate.clone(), + key, + client_ca: certificate, + coordinator_pins: std::collections::BTreeSet::from(["a".repeat(64)]), + }), + work_dir: temp.path().join("host"), + grpc_listen_addr: Some(addr), + grpc_advertise_addr: Some(format!("https://{addr}")), + coordinator_endpoint: Some("http://127.0.0.1:1".into()), + ..HostAgentConfig::default() + }; + let agent = HostAgent::new( + config, + Arc::new(engram_sandbox_process::ProcessBackend::new( + temp.path().join("ordinary"), + )), + cloud.clone(), + ) + .with_host_id("00000000-0000-0000-0000-000000000001".parse().unwrap()) + .with_egress(Arc::new(egress::HostEgress::new( + Arc::new(proxyd_client::ProxydHandle::new( + temp.path().join("absent.sock"), + )), + ca.cert_pem, + None, + ))); + let outcome = tokio::time::timeout(std::time::Duration::from_secs(1), agent.run()).await; + drop(occupied); + assert!( + outcome.is_ok_and(|result| result.is_err()), + "invalid managed listener must fail startup" + ); + assert!( + !cloud.0.load(Ordering::SeqCst), + "failure must precede startup work and coordinator registration" + ); + } + #[tokio::test] + async fn malformed_managed_tls_cannot_reach_registration() { + rejects_before_startup(true).await; + } + #[tokio::test] + async fn occupied_managed_listener_cannot_reach_registration() { + rejects_before_startup(false).await; + } +} diff --git a/crates/engram-host-agent/src/main.rs b/crates/engram-host-agent/src/main.rs index 317d4aa73..e021c7307 100644 --- a/crates/engram-host-agent/src/main.rs +++ b/crates/engram-host-agent/src/main.rs @@ -338,6 +338,8 @@ async fn main() -> Result<(), HostAgentError> { let (migrate_peer_listen_addr, _) = parse_grpc_listen(&cli.migrate_peer_listen_addr); let cfg = HostAgentConfig { + managed_tls: engram_host_agent::managed_tls::ManagedHostTls::from_env() + .map_err(engram_host_agent::HostAgentError::Config)?, work_dir: cli.work_dir.clone(), coordinator_endpoint: cli.coordinator.clone(), coordinator_token: cli.coordinator_token.clone(), @@ -359,6 +361,11 @@ async fn main() -> Result<(), HostAgentError> { ..HostAgentConfig::default() }; + let managed_ownership = engram_host_agent::managed_policy::ManagedOwnership::for_mode( + &cli.work_dir.join("bindings"), + cfg.managed_tls.is_some(), + )?; + // ADR 0044 K2 (GAP 1): STABLE across restarts (persisted in work_dir, // node-name-seeded on K8s) so a DaemonSet pod restart keeps its host // identity and the coordinator's session→host binding survives. @@ -371,10 +378,8 @@ async fn main() -> Result<(), HostAgentError> { // sweep can never delete an enabled image's pre-warmed base file. let base_shm_protected = engram_host_agent::base_shm_gc::ProtectedPaths::new(); - // ADR 0007: blob backend + chunk store. Created before the backend - // match so the inner VZ backend can be wired with its own chunk - // store (snapshots chunk the rootfs clone and report the manifest - // ref). Misconfiguration fails closed at startup — better than + // ADR 0007: blob backend for the pooled disk and snapshot owner. + // Misconfiguration fails closed at startup — better than // pretending to be ready and failing every session create. Shares // the same `BlobStorage` the coordinator's chunk store (the // enable-time materializer) writes to (a shared GCS bucket in @@ -389,6 +394,7 @@ async fn main() -> Result<(), HostAgentError> { // for non-FC backends — the live-attach pass becomes a no-op // and the host-agent starts clean-slate. let fc_for_reattach: Option>; + let mut managed_backend = None; let sandbox: Arc = match cli.sandbox_backend { BackendChoice::Firecracker => { let kernel = cli.kernel_image_path.clone().ok_or_else(|| { @@ -540,10 +546,31 @@ async fn main() -> Result<(), HostAgentError> { cli.guest_gateway_port, ); fc_for_reattach = None; - Arc::new( + let vz = Arc::new( engram_sandbox_vz::VzBackend::new(cli.work_dir.clone(), vz_cfg) .map_err(|e| HostAgentError::Config(format!("vz backend: {e}")))?, - ) + ); + if cfg.managed_tls.is_some() { + let path = std::env::var_os("ENGRAM_MANAGED_NETSTACK_PATH") + .map(PathBuf::from) + .ok_or_else(|| { + HostAgentError::Config("managed network helper path required".into()) + })?; + let sha256 = std::env::var("ENGRAM_MANAGED_NETSTACK_SHA256").map_err(|_| { + HostAgentError::Config("managed network helper digest required".into()) + })?; + if !path.is_absolute() || !engram_core::types::managed_cold::is_sha256(&sha256) + { + return Err(HostAgentError::Config( + "invalid managed network helper artifact".into(), + )); + } + managed_backend = Some(engram_host_agent::managed_boot::ManagedBackend::Vz { + backend: vz.clone(), + helper: engram_sandbox_vz::ManagedHelperArtifact { path, sha256 }, + }); + } + vz } #[cfg(not(target_os = "macos"))] { @@ -610,6 +637,17 @@ async fn main() -> Result<(), HostAgentError> { Arc::new(disk) } }; + #[cfg(target_os = "linux")] + if cfg.managed_tls.is_some() { + if let Some(backend) = fc_for_reattach.as_ref() { + managed_backend = Some( + engram_host_agent::managed_boot::ManagedBackend::Firecracker { + backend: backend.clone(), + disk: session_disk.clone(), + }, + ); + } + } let cloud = Arc::new(StaticCloud::detect().map_err(HostAgentError::Backend)?); // ADR 0070: NVMe-backed chunk cache with a disk-derived absolute @@ -642,8 +680,7 @@ async fn main() -> Result<(), HostAgentError> { // (Spawned below once the chunk store exists; see substrate_spawn.) // ADR 0007: chunk store for the PooledBackend wrapper (materialize + - // base-snapshot residency). `blob` was created above the backend - // match so the inner VZ backend could share it; reuse it here. Wire the + // base-snapshot residency). Reuse the configured blob backend. Wire the // SAME local cache the UFFD handler + disk stage read from // (`work_dir/chunk-cache`) as a write-through tier, so a chunk this host // produces — e.g. an idle-eviction re-chunk of divergent memory — is @@ -711,12 +748,16 @@ async fn main() -> Result<(), HostAgentError> { let observe_token = cfg.coordinator_token.clone(); let mut agent = HostAgent::new(cfg, sandbox, cloud) + .with_managed_ownership(managed_ownership) .with_fill(fill) .with_chunk_cache(chunk_cache) .with_session_disk(session_disk) .with_image_cache(image_cache) .with_host_id(host_id) .with_base_shm_protected(base_shm_protected); + if let Some(backend) = managed_backend { + agent = agent.with_managed_backend(backend); + } if let Some(fc) = fc_for_reattach { agent = agent.with_fc_reattach(fc); } diff --git a/crates/engram-host-agent/src/managed_assets.rs b/crates/engram-host-agent/src/managed_assets.rs new file mode 100644 index 000000000..a56111616 --- /dev/null +++ b/crates/engram-host-agent/src/managed_assets.rs @@ -0,0 +1,285 @@ +//! Read-only physical inputs. Catalog approval and lineage belong to the coordinator. +use engram_chunk_store::{ChunkHash, ChunkStore, Manifest, ManifestKind}; +use engram_core::{types::sandbox::AuxRoDrive, SandboxError}; +use futures::StreamExt; +use sha2::{Digest, Sha256}; +use std::{path::Path, sync::Arc}; +use tokio::io::AsyncReadExt; + +const BUFFER_BYTES: usize = 64 * 1024; +const MAX_MANIFEST_BYTES: usize = 8 * 1024 * 1024; +// One inspection per host, including after cancellation of manifest decoding. +static READ_SLOT: tokio::sync::Semaphore = tokio::sync::Semaphore::const_new(1); + +fn invalid(_: impl std::fmt::Display) -> SandboxError { + SandboxError::InvalidSpec("managed cold asset verification failed".into()) +} + +pub(crate) async fn inspect( + chunks: &ChunkStore, + reference: engram_core::types::manifest::ManifestRef, + kernel: &Path, + bundle_dir: &Path, + drives: &[AuxRoDrive], +) -> Result<(String, String, u64), SandboxError> { + let permit = Arc::new(READ_SLOT.acquire().await.map_err(invalid)?); + // No spawned read worker survives cancellation. Tokio's file operations may + // finish one bounded read; manifest decoding retains the slot until it exits. + let work = async { + let storage = chunks.blob_storage(); + let mut body = storage + .get_streaming(&reference.storage_key()) + .await + .map_err(invalid)?; + let mut json = Vec::new(); + while let Some(bytes) = body.next().await { + let bytes = bytes.map_err(invalid)?; + if bytes.len() > MAX_MANIFEST_BYTES - json.len() { + return Err(invalid("manifest size")); + } + json.extend_from_slice(&bytes); + } + let decode_permit = permit.clone(); + let manifest = tokio::task::spawn_blocking(move || { + let _permit = decode_permit; + let manifest: Manifest = serde_json::from_slice(&json).map_err(invalid)?; + manifest.validate().map_err(invalid)?; + if manifest.kind != ManifestKind::Disk || manifest.total_bytes == 0 { + return Err(invalid("disk kind or size")); + } + Ok::<_, SandboxError>(manifest) + }) + .await + .map_err(invalid)??; + let mut disk_hash = Sha256::new(); + let mut offset = 0; + let zeros = vec![0; BUFFER_BYTES]; + for entry in &manifest.chunks { + hash_zeros(&mut disk_hash, entry.offset - offset, &zeros).await; + let expected = manifest + .chunk_size + .as_u64() + .min(manifest.total_bytes - entry.offset); + let mut body = storage + .get_streaming(&entry.hash.storage_key()) + .await + .map_err(invalid)?; + let mut chunk_hash = Sha256::new(); + let mut size = 0; + while let Some(bytes) = body.next().await { + let bytes = bytes.map_err(invalid)?; + if bytes.len() as u64 > expected - size { + return Err(invalid("chunk length")); + } + size += bytes.len() as u64; + for block in bytes.chunks(BUFFER_BYTES) { + chunk_hash.update(block); + disk_hash.update(block); + tokio::task::yield_now().await; + } + } + if size != expected || ChunkHash::from_bytes(chunk_hash.finalize().into()) != entry.hash + { + return Err(invalid("chunk content")); + } + offset = entry.offset + expected; + } + hash_zeros(&mut disk_hash, manifest.total_bytes - offset, &zeros).await; + let (kernel_hash, _) = hash_file(kernel).await?; + for drive in drives { + let expected = drive + .sha256 + .as_deref() + .ok_or_else(|| invalid("unresolved bundle"))?; + // The request validator checks slot order and digest syntax before + // deriving a path. There is no caller-supplied host path. + let (actual, _) = + hash_file(&bundle_dir.join(AuxRoDrive::staged_file_name(expected))).await?; + if actual != expected { + return Err(invalid("bundle content")); + } + } + Ok(( + kernel_hash, + hex::encode(disk_hash.finalize()), + manifest.total_bytes, + )) + }; + tokio::time::timeout(std::time::Duration::from_secs(300), work) + .await + .map_err(invalid)? +} + +async fn hash_zeros(hash: &mut Sha256, mut count: u64, zeros: &[u8]) { + while count > 0 { + let n = count.min(zeros.len() as u64) as usize; + hash.update(&zeros[..n]); + count -= n as u64; + tokio::task::yield_now().await; + } +} + +pub(crate) async fn hash_file(path: &Path) -> Result<(String, u64), SandboxError> { + use std::os::unix::fs::MetadataExt; + let mut file = tokio::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path) + .await + .map_err(invalid)?; + let before = file.metadata().await.map_err(invalid)?; + if !before.is_file() || before.len() == 0 { + return Err(invalid("artifact type")); + } + let mut hash = Sha256::new(); + let mut buffer = vec![0; BUFFER_BYTES]; + let mut size = 0; + loop { + let n = file.read(&mut buffer).await.map_err(invalid)?; + if n == 0 { + break; + } + size += n as u64; + if size > before.len() { + return Err(invalid("artifact grew")); + } + hash.update(&buffer[..n]); + tokio::task::yield_now().await; + } + let after = file.metadata().await.map_err(invalid)?; + let identity = |m: &std::fs::Metadata| { + ( + m.dev(), + m.ino(), + m.len(), + m.mtime(), + m.mtime_nsec(), + m.ctime(), + m.ctime_nsec(), + ) + }; + if size != before.len() || identity(&before) != identity(&after) { + return Err(invalid("artifact changed")); + } + Ok((hex::encode(hash.finalize()), size)) +} + +#[cfg(test)] +mod tests { + use super::*; + use engram_chunk_store::{ChunkRef, ChunkSize}; + use engram_core::types::manifest::ManifestRef; + + async fn fixture() -> (tempfile::TempDir, ChunkStore, ManifestRef) { + let temp = tempfile::tempdir().unwrap(); + let store = ChunkStore::new(Arc::new(engram_storage_local::LocalBlobStorage::new( + temp.path().join("blobs"), + ))); + tokio::fs::write(temp.path().join("kernel"), b"kernel") + .await + .unwrap(); + (temp, store, ManifestRef::new()) + } + + #[tokio::test] + async fn cold_assets_reject_corrupt_short_long_missing_and_non_disk_chunks() { + let (temp, store, reference) = fixture().await; + let mut manifest = Manifest::empty(ManifestKind::Disk, 12); + manifest.chunk_size = ChunkSize::bytes(4); + let hash = store.put_chunk(b"abcd").await.unwrap(); + manifest.chunks.push(ChunkRef { offset: 4, hash }); + store.put_manifest(reference, &manifest).await.unwrap(); + let kernel = temp.path().join("kernel"); + let read = || inspect(&store, reference, &kernel, temp.path(), &[]); + let (_, actual, size) = read().await.unwrap(); + assert_eq!(size, 12); + assert_eq!(actual, hex::encode(Sha256::digest(b"\0\0\0\0abcd\0\0\0\0"))); + let chunk = temp.path().join("blobs").join(hash.storage_key()); + for wrong in [b"wxyz".as_slice(), b"abc", b"abcde"] { + tokio::fs::write(&chunk, wrong).await.unwrap(); + assert!(read().await.is_err()); + } + tokio::fs::remove_file(&chunk).await.unwrap(); + assert!(read().await.is_err()); + // A correct hash is insufficient when the manifest gives a wrong length. + for wrong in [b"abc".as_slice(), b"abcde"] { + manifest.chunks[0].hash = store.put_chunk(wrong).await.unwrap(); + let next = ManifestRef::new(); + store.put_manifest(next, &manifest).await.unwrap(); + assert!(inspect(&store, next, &kernel, temp.path(), &[]) + .await + .is_err()); + } + manifest.kind = ManifestKind::Memory; + let next = ManifestRef::new(); + store.put_manifest(next, &manifest).await.unwrap(); + assert!(inspect(&store, next, &kernel, temp.path(), &[]) + .await + .is_err()); + assert!( + inspect(&store, ManifestRef::new(), &kernel, temp.path(), &[]) + .await + .is_err() + ); + } + + #[tokio::test] + async fn cold_assets_sparse_hash_yields_and_cancel_releases_read_slot() { + let (temp, store, reference) = fixture().await; + store + .put_manifest(reference, &Manifest::empty(ManifestKind::Disk, 1 << 40)) + .await + .unwrap(); + let kernel = temp.path().join("kernel"); + // The single-thread runtime must poll its timer while the sparse read + // hashes zeros. Cancellation must release the slot without a worker. + assert!(tokio::time::timeout( + std::time::Duration::from_millis(20), + inspect(&store, reference, &kernel, temp.path(), &[]) + ) + .await + .is_err()); + let small = ManifestRef::new(); + store + .put_manifest(small, &Manifest::empty(ManifestKind::Disk, 3)) + .await + .unwrap(); + let (_, hash, size) = tokio::time::timeout( + std::time::Duration::from_secs(1), + inspect(&store, small, &kernel, temp.path(), &[]), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(hash, hex::encode(Sha256::digest([0; 3]))); + assert_eq!(size, 3); + assert_eq!(std::fs::read_dir(temp.path()).unwrap().count(), 2); + } + + #[tokio::test] + async fn cold_assets_file_reader_rejects_symlinks_empty_and_special_files() { + let temp = tempfile::tempdir().unwrap(); + let file = temp.path().join("file"); + tokio::fs::write(&file, []).await.unwrap(); + assert!(hash_file(&file).await.is_err()); + tokio::fs::write(&file, b"abc").await.unwrap(); + let symlink = temp.path().join("link"); + std::os::unix::fs::symlink(&file, &symlink).unwrap(); + assert!(hash_file(&symlink).await.is_err()); + assert!(hash_file(temp.path()).await.is_err()); + let fifo = temp.path().join("fifo"); + nix::unistd::mkfifo(&fifo, nix::sys::stat::Mode::S_IRUSR).unwrap(); + assert!( + tokio::time::timeout(std::time::Duration::from_secs(1), hash_file(&fifo)) + .await + .unwrap() + .is_err() + ); + let (hash, size) = hash_file(&file).await.unwrap(); + assert_eq!( + hash, + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + assert_eq!(size, 3); + } +} diff --git a/crates/engram-host-agent/src/managed_boot.rs b/crates/engram-host-agent/src/managed_boot.rs new file mode 100644 index 000000000..5c493c9c7 --- /dev/null +++ b/crates/engram-host-agent/src/managed_boot.rs @@ -0,0 +1,4269 @@ +//! One host owner for each managed boot. No ordinary backend map can adopt it. +use crate::{ + bindings::BindingStore, egress::HostEgress, harness::HarnessHub, managed_assets::hash_file, +}; +use engram_core::{ + traits::{HarnessSink, SandboxBackend}, + types::{ + egress::SessionEgressPolicy, + image::SecretMode, + managed_boot::*, + managed_create::digest, + managed_deny::*, + manifest::ManifestRef, + sandbox::{ + AgentSpec, ExecRequest, ExecStream, SessionFileMetadata, SessionFileSpec, + SessionFileStream, + }, + }, + HostId, SandboxError, SandboxId, +}; +use engram_egress_proto::private::{Allocation, RegisteredPrivateNetwork}; +use serde::{Deserialize, Serialize}; +use std::{ + path::{Path, PathBuf}, + sync::Arc, +}; +use tokio::sync::Mutex; + +fn invalid(_: impl std::fmt::Display) -> SandboxError { + SandboxError::InvalidSpec("managed boot ownership changed or unavailable".into()) +} + +#[derive(Clone)] +pub enum ManagedBackend { + #[cfg(target_os = "linux")] + Firecracker { + backend: Arc, + disk: Arc, + }, + #[cfg(target_os = "macos")] + Vz { + backend: Arc, + helper: engram_sandbox_vz::ManagedHelperArtifact, + }, +} +impl ManagedBackend { + async fn prepare( + &self, + request: &ManagedBootPrepare, + spec: engram_core::types::sandbox::SandboxSpec, + network: RegisteredPrivateNetwork, + staging: tempfile::TempDir, + ) -> Result { + match self { + #[cfg(target_os = "linux")] + Self::Firecracker { backend, disk } => Ok(Owner::Firecracker(Box::new( + backend + .prepare_managed_cold( + request.cold.clone(), + spec, + network, + staging, + disk.clone(), + ) + .await?, + ))), + #[cfg(target_os = "macos")] + Self::Vz { backend, helper } => Ok(Owner::Vz(Box::new( + backend + .prepare_managed_cold( + request.cold.clone(), + spec, + network, + helper.clone(), + staging, + ) + .await?, + ))), + } + } +} +type FileRead = futures::future::BoxFuture< + 'static, + Result<(SessionFileMetadata, SessionFileStream), SandboxError>, +>; +type FileWrite = futures::future::BoxFuture<'static, Result>; + +enum Owner { + #[cfg(test)] + ExecTest(Box), + #[cfg(target_os = "linux")] + Firecracker(Box), + #[cfg(target_os = "macos")] + Vz(Box), +} +impl Owner { + async fn closed_identity(&mut self) -> Result { + let bytes = match self { + #[cfg(test)] + Self::ExecTest(_) => panic!("unexpected execution-test lifecycle call"), + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => serde_json::to_vec(&owner.closed_receipt().await?), + #[cfg(target_os = "macos")] + Self::Vz(owner) => serde_json::to_vec(&owner.closed_receipt().await?), + } + .map_err(invalid)?; + Ok(digest(&bytes)) + } + async fn release(&mut self, epoch: u64) -> Result<(), SandboxError> { + match self { + #[cfg(test)] + Self::ExecTest(_) => panic!("unexpected execution-test lifecycle call"), + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => owner.release(epoch).await, + #[cfg(target_os = "macos")] + Self::Vz(owner) => owner.release(epoch).await, + } + } + async fn ready(&mut self, epoch: u64) -> Result<(), SandboxError> { + match self { + #[cfg(test)] + Self::ExecTest(owner) => { + if owner.blocked { + futures::future::pending::<()>().await; + } + owner.ready(epoch) + } + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => owner.check_ready(epoch).await, + #[cfg(target_os = "macos")] + Self::Vz(owner) => owner.check_ready(epoch).await, + } + } + async fn start(&mut self, agent: AgentSpec, sink: HarnessSink) -> Result<(), SandboxError> { + match self { + #[cfg(test)] + Self::ExecTest(_) => panic!("unexpected execution-test lifecycle call"), + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => owner.start_agent(agent, sink).await, + #[cfg(target_os = "macos")] + Self::Vz(owner) => owner.start_agent(agent, sink).await, + } + } + fn owns(&self, request: &engram_core::types::managed_cold::ManagedColdPrepare) -> bool { + match self { + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => owner.owns(request), + #[cfg(target_os = "macos")] + Self::Vz(owner) => owner.owns(request), + #[cfg(test)] + Self::ExecTest(owner) => &owner.request == request, + } + } + async fn exec_stream( + &mut self, + epoch: u64, + cmd: ExecRequest, + ) -> Result { + match self { + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => owner.exec_stream(epoch, cmd).await, + #[cfg(target_os = "macos")] + Self::Vz(owner) => owner.exec_stream(epoch, cmd).await, + #[cfg(test)] + Self::ExecTest(owner) => owner.exec_stream(cmd), + } + } + async fn cancel_exec(&mut self, epoch: u64, exec_id: String) -> Result<(), SandboxError> { + match self { + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => owner.cancel_exec(epoch, exec_id).await, + #[cfg(target_os = "macos")] + Self::Vz(owner) => owner.cancel_exec(epoch, exec_id).await, + #[cfg(test)] + Self::ExecTest(owner) => owner.cancel_exec(exec_id), + } + } + async fn prepare_read_file( + &mut self, + epoch: u64, + path: String, + closed: tokio::sync::watch::Receiver<()>, + ) -> Result { + Ok(match self { + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => { + Box::pin(owner.prepare_read_file(epoch, path, closed).await?) + } + #[cfg(target_os = "macos")] + Self::Vz(owner) => Box::pin(owner.prepare_read_file(epoch, path, closed).await?), + #[cfg(test)] + Self::ExecTest(owner) => { + let result = owner.read_file(path, closed); + Box::pin(async move { result }) + } + }) + } + async fn prepare_write_file( + &mut self, + epoch: u64, + spec: SessionFileSpec, + bytes: SessionFileStream, + ) -> Result { + Ok(match self { + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => { + Box::pin(owner.prepare_write_file(epoch, spec, bytes).await?) + } + #[cfg(target_os = "macos")] + Self::Vz(owner) => Box::pin(owner.prepare_write_file(epoch, spec, bytes).await?), + #[cfg(test)] + Self::ExecTest(owner) => owner.prepare_write_file(spec, bytes), + }) + } + async fn retire_disk(&mut self, path: &Path) -> Result<(), SandboxError> { + match self { + #[cfg(test)] + Self::ExecTest(_) => panic!("unexpected execution-test lifecycle call"), + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => owner.retire_disk(path).await, + #[cfg(target_os = "macos")] + Self::Vz(owner) => owner.retire_disk(path).await, + } + } + fn capture_stored(&mut self) -> Result<(), SandboxError> { + match self { + #[cfg(test)] + Self::ExecTest(owner) => { + owner + .capture_acks + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => owner.capture_stored(), + #[cfg(target_os = "macos")] + Self::Vz(owner) => owner.capture_stored(), + } + } + async fn close(self, destroy: bool) -> Result<(), SandboxError> { + match self { + #[cfg(test)] + Self::ExecTest(mut owner) => { + owner.close_intents.lock().unwrap().push(destroy); + if let Some(wait) = owner.close_wait.take() { + let _ = wait.await; + } + if owner.close_fails { + Err(invalid("test close failed")) + } else { + Ok(()) + } + } + #[cfg(target_os = "linux")] + Self::Firecracker(owner) => { + if destroy { + (*owner).destroy().await + } else { + (*owner).close().await + } + } + #[cfg(target_os = "macos")] + Self::Vz(owner) => { + if destroy { + (*owner).destroy().await + } else { + (*owner).close().await + } + } + } + } +} + +#[derive(Clone, Serialize, Deserialize)] +struct CaptureProgress { + request: ManagedCaptureRequest, + manifest: ManifestRef, + /// Present only after retirement, durable file sync and exact hash read. + artifact: Option<(String, u64)>, + /// Published only after exact backend and proxy cleanup succeed. + authority: Option, +} +#[derive(Clone, Serialize, Deserialize)] +struct Record { + version: u32, + request: ManagedBootPrepare, + incarnation: String, + state: ManagedBootHostState, + allocation: Option, + prepared: Option, + applied: Option, + apply_digest: Option, + finalization: Option, + capture: Option, + /// Exact local disk retained after a live policy retirement, before upload. + retired: Option<(String, u64)>, +} +struct Entry { + /// Replaced with each prepared owner; close signals before native I/O. + transfer_stop: tokio::sync::watch::Sender<()>, + record: Record, + close_failed: bool, + attach_owner: Option, + owner: Option, + agent: Option, +} + +pub struct ManagedBootHostConfig { + pub host_id: HostId, + pub incarnation: String, + pub directory: PathBuf, +} + +/// Address slots and attachment identities have separate lifetimes. +struct NetworkSlots { + generation: u64, + slots: Vec>, +} +impl NetworkSlots { + fn new(capacity: usize) -> Self { + Self { + generation: 0, + slots: vec![None; capacity], + } + } + fn reserve(&mut self) -> Result<(usize, u64), SandboxError> { + let slot = self.slots.iter().position(Option::is_none).ok_or_else(|| { + SandboxError::Unavailable("managed network address capacity exhausted".into()) + })?; + self.generation = self + .generation + .checked_add(1) + .ok_or_else(|| invalid("attachment generation exhausted"))?; + self.slots[slot] = Some(self.generation); + Ok((slot, self.generation)) + } + fn contains(&self, generation: u64) -> bool { + self.slots.contains(&Some(generation)) + } + fn release(&mut self, generation: u64) { + if let Some(slot) = self + .slots + .iter_mut() + .find(|slot| **slot == Some(generation)) + { + *slot = None; + } + } +} + +pub struct ManagedBootHost { + host_id: HostId, + incarnation: String, + backend: ManagedBackend, + ordinary: Arc, + egress: Arc, + hub: Arc, + bindings: BindingStore, + directory: PathBuf, + fill: engram_fill::FillEngine, + networks: std::sync::Mutex, + binding_lock: std::sync::Mutex<()>, + entries: dashmap::DashMap>>, +} +impl ManagedBootHost { + pub fn new( + config: ManagedBootHostConfig, + backend: ManagedBackend, + ordinary: Arc, + egress: Arc, + hub: Arc, + fill: engram_fill::FillEngine, + ) -> Result { + let ManagedBootHostConfig { + host_id, + incarnation, + directory, + } = config; + use std::os::unix::fs::PermissionsExt; + std::fs::create_dir_all(&directory)?; + if !std::fs::symlink_metadata(&directory)?.is_dir() { + return Err(invalid("managed directory")); + } + std::fs::set_permissions(&directory, std::fs::Permissions::from_mode(0o700))?; + let directory = directory.canonicalize()?; + Ok(Self { + host_id, + incarnation, + backend, + ordinary, + egress, + bindings: hub.binding_store(), + hub, + directory, + fill, + networks: std::sync::Mutex::new(NetworkSlots::new(512)), + binding_lock: std::sync::Mutex::new(()), + entries: Default::default(), + }) + } + pub async fn read_cold_assets( + &self, + request: engram_core::types::managed_assets::ManagedColdAssetsRequest, + ) -> Result { + use engram_core::types::managed_assets::{ManagedColdAssetsReceipt, ManagedColdBackend}; + request.validate().map_err(invalid)?; + if request.host_id != self.host_id + || self.incarnation.is_empty() + || self.incarnation.len() > 128 + { + return Err(invalid("host identity")); + } + let (backend, kernel, bundles) = match &self.backend { + #[cfg(target_os = "linux")] + ManagedBackend::Firecracker { backend, .. } => ( + ManagedColdBackend::Firecracker, + backend.config().kernel_image_path.as_path(), + backend.bundle_dir(), + ), + #[cfg(target_os = "macos")] + ManagedBackend::Vz { backend, .. } => ( + ManagedColdBackend::Vz, + backend.managed_kernel_path(), + backend.bundle_dir(), + ), + }; + let (kernel_sha256, disk_sha256, disk_size_bytes) = crate::managed_assets::inspect( + self.fill.store(), + request.disk_manifest, + kernel, + bundles, + &request.aux_ro_drives, + ) + .await?; + let receipt = ManagedColdAssetsReceipt { + request, + host_incarnation: self.incarnation.clone(), + backend, + kernel_sha256, + disk_sha256, + disk_size_bytes, + }; + receipt.validate_for(&receipt.request).map_err(invalid)?; + Ok(receipt) + } + + fn path(&self, id: SandboxId) -> PathBuf { + self.directory.join(format!("{id}.boot")) + } + fn persist(&self, record: &Record) -> Result<(), SandboxError> { + let id = record.request.key.context.target.sandbox_id; + let bytes = serde_json::to_vec(record).map_err(invalid)?; + durable_write( + &self.path(id), + &crate::durable_envelope::seal( + &id.to_string(), + std::str::from_utf8(&bytes).map_err(invalid)?, + ), + )?; + Ok(()) + } + fn read_record(&self, id: SandboxId) -> Result, SandboxError> { + let bytes = match std::fs::read(self.path(id)) { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.into()), + }; + let bytes = crate::durable_envelope::open(&bytes, &id.to_string()).map_err(invalid)?; + let record: Record = serde_json::from_slice(&bytes).map_err(invalid)?; + record.request.spec().map_err(invalid)?; + if record.version != 1 + || record.request.key.context.target.sandbox_id != id + || record.incarnation.is_empty() + { + return Err(invalid("record")); + } + if let Some(prepared) = &record.prepared { + prepared.validate().map_err(invalid)?; + if prepared.request != record.request || prepared.host_incarnation != record.incarnation + { + return Err(invalid("prepared record")); + } + } + if let Some(capture) = &record.capture { + if capture.request.key != record.request.key + || capture.manifest != capture.request.disk_manifest() + { + return Err(invalid("capture record")); + } + if let Some(authority) = &capture.authority { + authority.validate().map_err(invalid)?; + if authority.request != capture.request + || authority.host_incarnation != record.incarnation + || capture.artifact.as_ref() + != Some(&(authority.disk_sha256.clone(), authority.disk_size_bytes)) + { + return Err(invalid("capture record")); + } + } + } + Ok(Some(record)) + } + fn entry(&self, key: &ManagedBootKey) -> Result>, SandboxError> { + key.validate().map_err(invalid)?; + if key.context.target.host_id != self.host_id { + return Err(invalid("host")); + } + self.entries + .get(&key.context.target.sandbox_id) + .map(|e| e.clone()) + .ok_or_else(|| invalid("owner absent")) + } + fn check_key(&self, entry: &Entry, key: &ManagedBootKey) -> Result<(), SandboxError> { + if &entry.record.request.key != key || entry.record.incarnation != self.incarnation { + return Err(invalid("key")); + } + Ok(()) + } + fn unbind_owned(&self, key: &ManagedBootKey) { + let Ok(_guard) = self.binding_lock.lock() else { + return; + }; + if self + .bindings + .read(key.creation.session_id) + .ok() + .flatten() + .is_some_and(|b| { + b.sandbox_id == key.context.target.sandbox_id + && b.binding_epoch == key.context.target.binding_epoch + }) + { + self.hub.unbind_session(key.creation.session_id); + } + } + async fn close_resources(&self, entry: &mut Entry, destroy: bool) -> Result<(), SandboxError> { + entry.transfer_stop.send_replace(()); + if entry.close_failed { + return Err(invalid("prior owner cleanup was uncertain")); + } + if let Some(owner) = entry.owner.take() { + // A retry must not treat a consumed, failed owner as a clean absence. + entry.close_failed = true; + owner.close(destroy).await?; + entry.close_failed = false; + } + let Some(allocation) = &entry.record.allocation else { + return Ok(()); + }; + // Recovery must still prove old proxy cleanup before capture. It must + // never release a current reservation with the same generation number. + let current_incarnation = allocation.host_incarnation == self.incarnation + && entry.record.incarnation == self.incarnation; + if current_incarnation + && !self + .networks + .lock() + .expect("network slots") + .contains(allocation.attachment_generation) + { + return Ok(()); + } + // Each operation owns its control stream and compares the allocation + // under the daemon's lifecycle locks. A late request cannot remove a + // replacement, and its late ACK cannot complete a different request. + tokio::time::timeout( + std::time::Duration::from_secs(2 * engram_egress_proto::private::HANDOFF_SECONDS), + self.egress.handle().retire_private_network(allocation), + ) + .await + .map_err(|_| invalid("proxy cleanup timed out"))? + .map_err(invalid)?; + if current_incarnation { + self.networks + .lock() + .expect("network slots") + .release(allocation.attachment_generation); + } + Ok(()) + } + async fn fail(&self, entry: &mut Entry) { + entry.attach_owner = None; + entry.record.state = ManagedBootHostState::Failed { + key: entry.record.request.key.clone(), + }; + entry.agent = None; + let _ = self.persist(&entry.record); + self.unbind_owned(&entry.record.request.key); + let _ = self.close_resources(entry, false).await; + } + pub async fn prepare( + self: &Arc, + request: ManagedBootPrepare, + ) -> Result { + let this = self.clone(); + let result = tokio::spawn(async move { + let key = request.key.clone(); + let had_owner = this.entries.contains_key(&key.context.target.sandbox_id); + let result = this.prepare_inner(request).await; + if result.is_ok() && !had_owner { + let host = Arc::downgrade(&this); + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + let Some(host) = host.upgrade() else { + return; + }; + match host.read(key.clone()).await { + Ok( + ManagedBootHostState::Failed { .. } + | ManagedBootHostState::Captured { .. } + | ManagedBootHostState::Retired { .. }, + ) + | Err(_) => return, + _ => (), + } + } + }); + } + result + }) + .await + .map_err(invalid)?; + result + } + async fn prepare_inner( + &self, + request: ManagedBootPrepare, + ) -> Result { + let mut spec = request.spec().map_err(invalid)?; + let key = &request.key; + if key.context.target.host_id != self.host_id { + return Err(invalid("host")); + } + let id = key.context.target.sandbox_id; + let entry = match self.entries.entry(id) { + dashmap::mapref::entry::Entry::Occupied(entry) => entry.get().clone(), + dashmap::mapref::entry::Entry::Vacant(entry) => { + if self.read_record(id)?.is_some() + || self + .bindings + .managed_sandbox_required(id) + .map_err(invalid)? + { + return Err(invalid("lost owner")); + } + let record = Record { + version: 1, + request: request.clone(), + incarnation: self.incarnation.clone(), + state: ManagedBootHostState::Failed { key: key.clone() }, + allocation: None, + prepared: None, + applied: None, + apply_digest: None, + finalization: None, + capture: None, + retired: None, + }; + // Both markers precede backend I/O; a partial record is never ordinary. + durable_write( + &self + .bindings + .directory() + .join(format!("{id}.managed-sandbox")), + b"1", + )?; + durable_write( + &self + .bindings + .directory() + .join(format!("{}.managed-boot", key.creation.session_id)), + id.to_string().as_bytes(), + )?; + self.persist(&record)?; + entry + .insert(Arc::new(Mutex::new(Entry { + transfer_stop: tokio::sync::watch::channel(()).0, + close_failed: false, + record, + attach_owner: None, + owner: None, + agent: None, + }))) + .clone() + } + }; + let mut entry = entry.lock().await; + self.check_key(&entry, key)?; + if entry.record.request != request { + return Err(invalid("retry changed")); + } + if let ManagedBootHostState::Prepared { receipt } = &entry.record.state { + let receipt = receipt.as_ref().clone(); + if entry.record.applied.is_some() { + entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .ready(key.context.target.binding_epoch) + .await?; + } else if entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .closed_identity() + .await? + != receipt.network_identity + { + return Err(invalid("network")); + } + return Ok(receipt); + } + // None is allowed only for the first attempt, before any allocation exists. + if entry.record.allocation.is_some() || entry.owner.is_some() { + return Err(invalid("attempt consumed")); + } + if self.ordinary.list().await?.contains(&id) + || self.hub.bound_sandbox(key.creation.session_id).is_some() + { + return Err(invalid("ordinary source")); + } + let result = async { + let staging = tempfile::Builder::new() + .prefix("staging-") + .tempdir_in(&self.directory)?; + let rootfs = staging.path().join("selected.ext4"); + let manifest = self + .fill + .store() + .get_manifest(request.cold.disk.manifest) + .await + .map_err(invalid)?; + self.fill + .store() + .materialize_to_file_from(&manifest, &rootfs, &self.fill) + .await + .map_err(invalid)?; + spec.rootfs_source = Some(rootfs); + let handle = self.egress.handle(); + let proxy_incarnation = handle.hello().await.map_err(invalid)?.incarnation; + let mut unavailable = Vec::new(); + let selected = loop { + let reserved = self.networks.lock().expect("network slots").reserve(); + let (slot, generation) = match reserved { + Ok(value) => value, + Err(error) => break Err(error), + }; + let second = 18 + (slot / 256) as u8; + let third = (slot % 256) as u8; + let guest = std::net::Ipv4Addr::new(198, second, third, 2); + unavailable.push(generation); + match handle.lookup_guest(guest).await.map_err(invalid) { + Ok(None) => { + unavailable.pop(); + break Ok(( + guest, + std::net::Ipv4Addr::new(198, second, third, 1), + generation, + )); + } + Ok(Some(_)) => (), + Err(error) => break Err(error), + } + }; + for generation in unavailable { + self.networks + .lock() + .expect("network slots") + .release(generation); + } + let (guest_ip, gateway_ip, generation) = selected?; + let allocation = Allocation { + version: 1, + host_id: self.host_id, + host_incarnation: self.incarnation.clone(), + proxy_incarnation, + session_id: key.creation.session_id, + sandbox_id: id, + boot_attempt: key.boot_attempt.clone(), + attachment_generation: generation, + guest_ip, + gateway_ip, + policy_revision: request.bootstrap_policy.revision, + policy_digest: request.bootstrap_policy.digest.clone(), + }; + entry.record.allocation = Some(allocation.clone()); + self.persist(&entry.record)?; + self.egress + .apply_policy(bootstrap(&allocation, request.bootstrap_policy.clone())) + .await + .map_err(invalid)?; + let network = handle + .register_private_network(&allocation) + .await + .map_err(invalid)?; + let mut owner = self + .backend + .prepare(&request, spec, network, staging) + .await?; + let network_identity = owner.closed_identity().await?; + let mut receipt = ManagedBootPreparedReceipt { + request: request.clone(), + host_incarnation: self.incarnation.clone(), + network_identity, + guest_ip, + receipt_id: String::new(), + }; + receipt.receipt_id = digest(&serde_json::to_vec(&receipt).map_err(invalid)?); + receipt.validate().map_err(invalid)?; + entry.transfer_stop = tokio::sync::watch::channel(()).0; + entry.owner = Some(owner); + entry.record.prepared = Some(receipt.clone()); + entry.record.state = ManagedBootHostState::Prepared { + receipt: Box::new(receipt.clone()), + }; + self.persist(&entry.record)?; + Ok(receipt) + } + .await; + if result.is_err() { + self.fail(&mut entry).await; + } + result + } + pub async fn read(&self, key: ManagedBootKey) -> Result { + key.validate().map_err(invalid)?; + if key.context.target.host_id != self.host_id { + return Err(invalid("host")); + } + if let Ok(entry) = self.entry(&key) { + let mut entry = entry.lock().await; + if entry.record.request.key != key { + return Err(invalid("key")); + } + if let ManagedBootHostState::Captured { authority } = &entry.record.state { + authority.validate().map_err(invalid)?; + return Ok(entry.record.state.clone()); + } + if matches!(entry.record.state, ManagedBootHostState::Retired { .. }) { + self.verify_retired(&entry.record).await?; + return Ok(entry.record.state.clone()); + } + self.check_key(&entry, &key)?; + let live = match &entry.record.state { + ManagedBootHostState::Prepared { .. } if entry.record.applied.is_none() => entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .closed_identity() + .await + .map(|_| ()), + ManagedBootHostState::Prepared { .. } + | ManagedBootHostState::Finalized { .. } + | ManagedBootHostState::Started { .. } => { + entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .ready(key.context.target.binding_epoch) + .await + } + _ => Ok(()), + }; + if live.is_err() { + self.fail(&mut entry).await; + } + return Ok(entry.record.state.clone()); + } + match self.read_record(key.context.target.sandbox_id)? { + Some(record) if record.request.key == key => { + if matches!(record.state, ManagedBootHostState::Retired { .. }) { + self.verify_retired(&record).await?; + return Ok(record.state); + } + if let Some(authority) = record.capture.and_then(|c| c.authority) { + authority.validate().map_err(invalid)?; + return Ok(ManagedBootHostState::Captured { authority }); + } + Ok(ManagedBootHostState::Failed { key }) + } + Some(_) => Err(invalid("key")), + None => Ok(ManagedBootHostState::Missing { key }), + } + } + pub async fn apply( + self: &Arc, + request: ManagedPolicyApply, + ) -> Result { + let this = self.clone(); + tokio::spawn(async move { this.apply_inner(request).await }) + .await + .map_err(invalid)? + } + async fn apply_inner( + &self, + request: ManagedPolicyApply, + ) -> Result { + let id = request.context.target.sandbox_id; + let entry = self + .entries + .get(&id) + .map(|e| e.clone()) + .ok_or_else(|| invalid("owner"))?; + let mut entry = entry.lock().await; + let key = entry.record.request.key.clone(); + self.check_key(&entry, &key)?; + request.context.validate().map_err(invalid)?; + if request.context.target != key.context.target { + return Err(invalid("context")); + } + // Retirement is permanent. A newer coordinator deny union must not + // erase this exact nonrunning proof or manufacture an applied ACK. + if matches!(entry.record.state, ManagedBootHostState::Retired { .. }) { + return Err(SandboxError::ManagedRecoveryRequired); + } + let context = request.context.clone(); + let allocation = entry + .record + .allocation + .clone() + .ok_or_else(|| invalid("allocation"))?; + let policy = request + .egress + .managed_deny + .as_ref() + .ok_or_else(|| invalid("policy"))?; + policy.validate().map_err(invalid)?; + if request.egress.session_id != allocation.session_id + || request.egress.sandbox_id != id + || request.egress.guest_ip != allocation.guest_ip + || !request.egress.tunnels.is_empty() + { + return Err(invalid("policy identity")); + } + if context.binding_hash.is_none() { + if !bootstrap_retirement_allowed(&entry.record, &request) { + return Err(invalid("bootstrap retirement context")); + } + if let Err(error) = self.retire_for_policy(&mut entry, policy.clone()).await { + self.fail(&mut entry).await; + return Err(error); + } + self.egress + .handle() + .quarantine_managed(allocation.session_id) + .await + .map_err(invalid)?; + return Err(SandboxError::ManagedRecoveryRequired); + } + let request_digest = digest(&serde_json::to_vec(&request).map_err(invalid)?); + if let Some(applied) = &entry.record.applied { + let applied = applied.clone(); + if entry.record.apply_digest.as_ref() != Some(&request_digest) { + if request.expected.as_ref() != Some(&applied) + || policy.revision <= applied.policy.revision + || !policy.contains(&applied.policy) + || context.operator_generation < applied.context.operator_generation + || (context.operator_generation == applied.context.operator_generation + && context.operator_identity != applied.context.operator_identity) + { + return Err(invalid("policy retry")); + } + // A bound allocation cannot transfer its lease. Retire the old + // physical context; recovery must claim a fresh closed owner. + if let Err(error) = self.retire_for_policy(&mut entry, policy.clone()).await { + self.fail(&mut entry).await; + return Err(error); + } + self.egress + .handle() + .quarantine_managed(allocation.session_id) + .await + .map_err(invalid)?; + return Err(SandboxError::ManagedRecoveryRequired); + } + entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .ready(key.context.target.binding_epoch) + .await?; + return Ok(applied); + } + let prepared = match &entry.record.state { + ManagedBootHostState::Prepared { receipt } => receipt.clone(), + _ => return Err(invalid("phase")), + }; + let mut bootstrap_context = key.context.clone(); + bootstrap_context.binding_hash = context.binding_hash.clone(); + if request.expected.is_some() || context != bootstrap_context { + return Err(invalid("expected policy")); + } + let result = async { + if entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .closed_identity() + .await? + != prepared.network_identity + { + return Err(invalid("network")); + } + self.egress + .handle() + .handoff_closed_private(engram_egress_proto::private::ClosedPolicyHandoff { + allocation: allocation.clone(), + policy: request.egress.clone(), + }) + .await + .map_err(invalid)?; + entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .release(key.context.target.binding_epoch) + .await?; + let current = self + .egress + .handle() + .read_managed(allocation.session_id) + .await + .map_err(invalid)? + .ok_or_else(|| invalid("policy absent"))?; + if current.incarnation != allocation.proxy_incarnation + || current.sandbox_id != id + || current.guest_ip != allocation.guest_ip + || current.policy != *policy + { + return Err(invalid("policy acknowledgement")); + } + let applied = ManagedAppliedState::new( + context, + policy.clone(), + self.incarnation.clone(), + current.incarnation, + Some(ManagedNetworkProof { + target: key.context.target.clone(), + guest_ip: allocation.guest_ip, + network_identity: prepared.network_identity, + }), + ) + .map_err(invalid)?; + entry.record.applied = Some(applied.clone()); + entry.record.apply_digest = Some(request_digest); + self.persist(&entry.record)?; + Ok(applied) + } + .await; + if result.is_err() { + self.fail(&mut entry).await; + } + result + } + async fn retire_for_policy( + &self, + entry: &mut Entry, + policy: ManagedDenyPolicy, + ) -> Result<(), SandboxError> { + if let ManagedBootHostState::Retired { receipt } = &entry.record.state { + if receipt.retained_policy == policy { + return Ok(()); + } + return Err(invalid("retirement policy changed")); + } + let prepared = entry + .record + .prepared + .clone() + .ok_or_else(|| invalid("prepared receipt absent"))?; + let key = entry.record.request.key.clone(); + entry.transfer_stop.send_replace(()); + entry.attach_owner = None; + entry.record.state = ManagedBootHostState::Failed { key: key.clone() }; + entry.agent = None; + // Persist the consumed run before stopping it. The owned disk is never + // selected by a generic pathname or resumed after this point. + self.persist(&entry.record)?; + self.unbind_owned(&key); + let path = self + .directory + .join(format!("{}.retired", key.context.target.sandbox_id)); + if path.try_exists()? { + return Err(invalid("retirement already attempted")); + } + entry + .owner + .as_mut() + .ok_or_else(|| invalid("retirement owner"))? + .retire_disk(&path) + .await?; + entry.record.retired = Some(hash_file(&path).await?); + std::fs::File::open(&self.directory)?.sync_all()?; + let mut receipt = ManagedBootRetiredReceipt { + prepared, + retained_policy: policy, + receipt_id: String::new(), + }; + receipt.receipt_id = digest(&serde_json::to_vec(&receipt).map_err(invalid)?); + receipt + .validate_for( + entry + .record + .prepared + .as_ref() + .ok_or_else(|| invalid("prepared receipt"))?, + &entry.record.request.bootstrap_policy, + ) + .map_err(invalid)?; + entry.record.state = ManagedBootHostState::Retired { + receipt: Box::new(receipt), + }; + self.persist(&entry.record)?; + entry + .owner + .as_mut() + .ok_or_else(|| invalid("retirement owner"))? + .capture_stored()?; + self.close_resources(entry, false).await?; + Ok(()) + } + async fn verify_retired(&self, record: &Record) -> Result<(), SandboxError> { + let ManagedBootHostState::Retired { receipt } = &record.state else { + return Err(invalid("retired state")); + }; + receipt + .validate_for( + record + .prepared + .as_ref() + .ok_or_else(|| invalid("retired preparation"))?, + &record.request.bootstrap_policy, + ) + .map_err(invalid)?; + let path = self.directory.join(format!( + "{}.retired", + record.request.key.context.target.sandbox_id + )); + if record.retired.as_ref() != Some(&hash_file(&path).await?) { + return Err(invalid("retired disk")); + } + Ok(()) + } + pub async fn read_policy( + &self, + target: ManagedPolicyTarget, + ) -> Result, SandboxError> { + let entry = self + .entries + .get(&target.sandbox_id) + .map(|e| e.clone()) + .ok_or_else(|| invalid("owner"))?; + let mut entry = entry.lock().await; + if entry.record.request.key.context.target != target { + return Err(invalid("target")); + } + if entry.record.applied.is_some() { + entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .ready(target.binding_epoch) + .await?; + } + Ok(entry.record.applied.clone()) + } + pub async fn finalize( + self: &Arc, + request: ManagedBootFinalize, + ) -> Result { + let this = self.clone(); + tokio::spawn(async move { this.finalize_inner(request).await }) + .await + .map_err(invalid)? + } + async fn finalize_inner( + &self, + request: ManagedBootFinalize, + ) -> Result { + request.validate_agent().map_err(invalid)?; + let f = &request.finalization; + f.prepared.validate().map_err(invalid)?; + f.policy.validate().map_err(invalid)?; + if f.native_version != 1 + || f.combined_manifest_digest != digest(&f.combined_manifest) + || f.combined_manifest.len() > 1024 * 1024 + { + return Err(invalid("manifest")); + } + let key = &f.prepared.request.key; + let entry = self.entry(key)?; + let mut entry = entry.lock().await; + self.check_key(&entry, key)?; + if entry.record.applied.as_ref() != Some(&f.policy) + || f.prepared.host_incarnation != self.incarnation + || f.policy.host_incarnation != self.incarnation + || f.policy.network_proof.as_ref().is_none_or(|p| { + p.target != key.context.target + || p.network_identity != f.prepared.network_identity + || p.guest_ip != f.prepared.guest_ip + }) + { + return Err(invalid("policy")); + } + if let ManagedBootHostState::Finalized { receipt } = &entry.record.state { + receipt.validate_for(f).map_err(invalid)?; + if entry.record.finalization.as_ref() != Some(&f.digest().map_err(invalid)?) { + return Err(invalid("final retry")); + } + let receipt = receipt.clone(); + entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .ready(key.context.target.binding_epoch) + .await?; + return Ok(receipt); + } + match &entry.record.state { + ManagedBootHostState::Prepared { receipt } if receipt.as_ref() == &f.prepared => (), + _ => return Err(invalid("phase")), + } + entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .ready(key.context.target.binding_epoch) + .await?; + let mut receipt = ManagedBootFinalReceipt { + key: key.clone(), + prepared_receipt_id: f.prepared.receipt_id.clone(), + finalization_digest: f.digest().map_err(invalid)?, + host_incarnation: self.incarnation.clone(), + network_identity: f.prepared.network_identity.clone(), + receipt_id: String::new(), + }; + receipt.receipt_id = digest(&serde_json::to_vec(&receipt).map_err(invalid)?); + entry.record.finalization = Some(f.digest().map_err(invalid)?); + entry.record.state = ManagedBootHostState::Finalized { + receipt: receipt.clone(), + }; + entry.agent = Some(request.agent); + self.persist(&entry.record)?; + Ok(receipt) + } + pub async fn start( + self: &Arc, + request: ManagedBootStart, + ) -> Result { + let this = self.clone(); + tokio::spawn(async move { this.start_inner(request).await }) + .await + .map_err(invalid)? + } + async fn start_inner( + &self, + request: ManagedBootStart, + ) -> Result { + request.validate().map_err(invalid)?; + let entry = self.entry(&request.receipt.key)?; + let mut entry = entry.lock().await; + self.check_key(&entry, &request.receipt.key)?; + if let ManagedBootHostState::Started { intent } = &entry.record.state { + if intent != &request { + return Err(invalid("start retry")); + } + entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .ready(request.receipt.key.context.target.binding_epoch) + .await?; + return Ok(entry.record.state.clone()); + } + match &entry.record.state { + ManagedBootHostState::Finalized { receipt } if receipt == &request.receipt => (), + _ => return Err(invalid("start consumed")), + } + let agent = entry.agent.take().ok_or_else(|| invalid("agent"))?; + entry.record.state = ManagedBootHostState::StartPending { + intent: request.clone(), + }; + self.persist(&entry.record)?; + let key = request.receipt.key.clone(); + let result = async { + { + let _guard = self.binding_lock.lock().map_err(invalid)?; + self.hub + .bind_session( + key.creation.session_id, + key.context.target.sandbox_id, + key.context.target.binding_epoch, + ) + .map_err(invalid)?; + } + let attach_owner = self.hub.managed_owner(request.clone()); + let sink = attach_owner.sink(); + entry.attach_owner = Some(attach_owner); + tokio::time::timeout( + std::time::Duration::from_secs(210), + entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .start(agent, sink), + ) + .await + .map_err(invalid)??; + entry.record.state = ManagedBootHostState::Started { intent: request }; + self.persist(&entry.record)?; + Ok(entry.record.state.clone()) + } + .await; + if result.is_err() { + self.fail(&mut entry).await; + } + result + } + pub async fn capture( + self: &Arc, + request: ManagedCaptureRequest, + ) -> Result { + let this = self.clone(); + tokio::spawn(async move { this.capture_inner(request).await }) + .await + .map_err(invalid)? + } + async fn capture_inner( + &self, + request: ManagedCaptureRequest, + ) -> Result { + request.key.validate().map_err(invalid)?; + if request.capture_id.as_uuid().is_nil() { + return Err(invalid("capture")); + } + let entry = match self.entry(&request.key) { + Ok(entry) => entry, + Err(_) => { + let record = self + .read_record(request.key.context.target.sandbox_id)? + .ok_or_else(|| invalid("capture record absent"))?; + if record.request.key != request.key + || record.request.key.context.target.host_id != self.host_id + || !(record.retired.is_some() + || record.capture.as_ref().is_some_and(|c| { + c.request == request && (c.artifact.is_some() || c.authority.is_some()) + })) + { + return Err(invalid("capture source lost")); + } + self.entries + .entry(request.key.context.target.sandbox_id) + .or_insert_with(|| { + Arc::new(Mutex::new(Entry { + transfer_stop: tokio::sync::watch::channel(()).0, + close_failed: false, + record, + attach_owner: None, + owner: None, + agent: None, + })) + }) + .clone() + } + }; + let mut entry = entry.lock().await; + if entry.record.request.key != request.key { + return Err(invalid("capture key")); + } + if let Some(progress) = &entry.record.capture { + if progress.request != request { + return Err(invalid("capture retry")); + } + if let Some(authority) = &progress.authority { + return Ok(authority.clone()); + } + } else { + if !matches!(entry.record.state, ManagedBootHostState::Started { .. }) + && entry.record.retired.is_none() + { + return Err(invalid("capture source")); + } + entry.record.capture = Some(CaptureProgress { + request: request.clone(), + manifest: request.disk_manifest(), + artifact: None, + authority: None, + }); + entry.attach_owner = None; + entry.record.state = ManagedBootHostState::Failed { + key: request.key.clone(), + }; + self.persist(&entry.record)?; + } + let path = self.directory.join(format!("{}.disk", request.capture_id)); + let result = async { + let mut progress = entry + .record + .capture + .clone() + .ok_or_else(|| invalid("capture"))?; + if progress.artifact.is_none() { + if path.try_exists()? { + return Err(invalid("incomplete capture artifact")); + } + self.unbind_owned(&request.key); + if let Some(retired) = &entry.record.retired { + let source = self + .directory + .join(format!("{}.retired", request.key.context.target.sandbox_id)); + if &hash_file(&source).await? != retired { + return Err(invalid("retired disk changed")); + } + let destination = path.clone(); + tokio::task::spawn_blocking(move || { + std::fs::copy(&source, &destination)?; + std::fs::File::open(&destination)?.sync_all() + }) + .await + .map_err(invalid)??; + } else { + entry.transfer_stop.send_replace(()); + entry + .owner + .as_mut() + .ok_or_else(|| invalid("source lost"))? + .retire_disk(&path) + .await?; + } + let (hash, size) = hash_file(&path).await?; + std::fs::File::open(&self.directory)?.sync_all()?; + progress.artifact = Some((hash, size)); + entry.record.capture = Some(progress.clone()); + self.persist(&entry.record)?; + } + let (hash, size) = progress + .artifact + .clone() + .ok_or_else(|| invalid("artifact"))?; + if hash_file(&path).await? != (hash.clone(), size) { + return Err(invalid("capture integrity")); + } + if entry.record.retired.is_none() { + if let Some(owner) = entry.owner.as_mut() { + owner.capture_stored()?; + } + } + let manifest = self + .fill + .store() + .chunk_file(&path, engram_chunk_store::ManifestKind::Disk, None) + .await + .map_err(invalid)?; + publish_capture_manifest(self.fill.store(), progress.manifest, &manifest).await?; + let cold = &entry.record.request.cold; + let mut authority = ManagedCaptureAuthority { + version: 1, + request, + host_incarnation: entry.record.incarnation.clone(), + disk_manifest: progress.manifest, + disk_sha256: hash, + disk_size_bytes: size, + image: cold.image.clone(), + kernel_sha256: cold.kernel_sha256.clone(), + aux_ro_drives: cold.aux_ro_drives.clone(), + consistency: ManagedDiskConsistency::CrashConsistentDiskCutV1, + source_state: ManagedCapturedSourceState::Destroyed, + record_digest: String::new(), + }; + authority.record_digest = authority.digest().map_err(invalid)?; + authority.validate().map_err(invalid)?; + // Replacement authority is durable only after exact owner and + // proxy cleanup succeed. A failed attempt leaves an artifact to retry. + self.close_resources(&mut entry, false).await?; + progress.authority = Some(authority.clone()); + entry.record.capture = Some(progress); + entry.record.state = ManagedBootHostState::Captured { + authority: authority.clone(), + }; + self.persist(&entry.record)?; + Ok(authority) + } + .await; + if result.is_err() { + self.fail(&mut entry).await; + } + result + } + pub async fn destroy( + self: &Arc, + id: SandboxId, + fence: engram_core::traits::SessionFence, + ) -> Result<(), SandboxError> { + let this = self.clone(); + tokio::spawn(async move { + let entry = this + .entries + .get(&id) + .map(|e| e.clone()) + .ok_or_else(|| invalid("owner"))?; + let mut entry = entry.lock().await; + let target = entry.record.request.key.context.target.clone(); + if target.session_id != fence.session_id || fence.epoch == 0 { + return Err(invalid("destroy target")); + } + this.destroy_entry(&mut entry).await + }) + .await + .map_err(invalid)? + } + /// Heartbeat tombstones name the sandbox, independently of a session fence. + pub(crate) async fn destroy_tombstoned( + self: &Arc, + id: SandboxId, + ) -> Result<(), SandboxError> { + let this = self.clone(); + tokio::spawn(async move { + let entry = this + .entries + .get(&id) + .map(|e| e.clone()) + .ok_or_else(|| invalid("tombstone owner absent"))?; + let mut entry = entry.lock().await; + let key = &entry.record.request.key; + this.check_key(&entry, key)?; + if key.context.target.host_id != this.host_id + || key.context.target.sandbox_id != id + || entry + .owner + .as_ref() + .is_some_and(|owner| !owner.owns(&entry.record.request.cold)) + { + return Err(invalid("tombstone owner identity")); + } + this.destroy_entry(&mut entry).await + }) + .await + .map_err(invalid)? + } + async fn destroy_entry(&self, entry: &mut Entry) -> Result<(), SandboxError> { + if entry.owner.is_none() + && matches!( + entry.record.state, + ManagedBootHostState::Retired { .. } | ManagedBootHostState::Captured { .. } + ) + { + // The physical source is already gone. Preserve its durable proof. + return self.close_resources(entry, true).await; + } + entry.attach_owner = None; + entry.record.state = ManagedBootHostState::Failed { + key: entry.record.request.key.clone(), + }; + self.persist(&entry.record)?; + entry.agent = None; + self.unbind_owned(&entry.record.request.key); + self.close_resources(entry, true).await + } + pub fn ordinary_spec( + &self, + spec: &engram_core::types::sandbox::SandboxSpec, + ) -> Result<(), SandboxError> { + self.ordinary_disk(spec.rootfs_manifest, spec.rootfs_source.as_deref())?; + self.ordinary_disk(spec.swap_manifest, spec.swap_source.as_deref()) + } + pub fn ordinary_disk( + &self, + manifest: Option, + source: Option<&Path>, + ) -> Result<(), SandboxError> { + for entry in std::fs::read_dir(&self.directory)? { + let path = entry?.path(); + if path.extension().and_then(|e| e.to_str()) != Some("boot") { + continue; + } + let id: SandboxId = path + .file_stem() + .and_then(|s| s.to_str()) + .ok_or_else(|| invalid("record name"))? + .parse() + .map_err(invalid)?; + let record = self + .read_record(id)? + .ok_or_else(|| invalid("record absent"))?; + let key = &record.request.key; + let fc_name = format!( + "m-{}", + &digest( + format!("{}\0{}", key.context.target.sandbox_id, key.boot_attempt).as_bytes() + )[7..31] + ); + let vz_name = format!( + "managed-{}-{}", + key.context.target.sandbox_id, key.boot_attempt + ); + let private_path = source.is_some_and(|path| { + let path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); + path.starts_with(&self.directory) + || path.components().any(|c| { + c.as_os_str() == std::ffi::OsStr::new(&fc_name) + || c.as_os_str() == std::ffi::OsStr::new(&vz_name) + }) + }); + if manifest == Some(record.request.cold.disk.manifest) + || record + .capture + .as_ref() + .is_some_and(|c| manifest == Some(c.manifest)) + || private_path + { + return Err(invalid("managed disk cannot use ordinary create")); + } + } + Ok(()) + } + /// Execution is authorized by the exact live start and durable binding. + /// Keep the lifecycle lock through submission, with bounded guest I/O. + async fn execution_owner<'a>( + &self, + id: SandboxId, + entry: &'a mut Entry, + ) -> Result<(u64, &'a mut Owner), SandboxError> { + let key = &entry.record.request.key; + self.check_key(entry, key)?; + let target = &key.context.target; + if target.host_id != self.host_id + || target.sandbox_id != id + || !matches!(&entry.record.state, ManagedBootHostState::Started { intent } if intent.receipt.key == *key) + || entry.attach_owner.is_none() + || !self + .bindings + .read(target.session_id) + .map_err(invalid)? + .is_some_and(|binding| { + binding.sandbox_id == id && binding.binding_epoch == target.binding_epoch + }) + { + return Err(invalid("execution owner")); + } + let epoch = target.binding_epoch; + let owner = entry.owner.as_mut().ok_or_else(|| invalid("owner"))?; + if !owner.owns(&entry.record.request.cold) { + return Err(invalid("execution owner identity")); + } + owner.ready(epoch).await?; + Ok((epoch, owner)) + } + pub async fn exec_stream( + &self, + id: SandboxId, + cmd: ExecRequest, + ) -> Result { + let entry = self + .entries + .get(&id) + .map(|e| e.clone()) + .ok_or_else(|| invalid("owner"))?; + let mut entry = entry.lock().await; + tokio::time::timeout(std::time::Duration::from_secs(30), async { + let (epoch, owner) = self.execution_owner(id, &mut entry).await?; + owner.exec_stream(epoch, cmd).await + }) + .await + .map_err(|_| { + SandboxError::Unavailable( + "managed execution submission timed out; retry with the same execution ticket" + .into(), + ) + })? + } + pub async fn cancel_exec(&self, id: SandboxId, exec_id: String) -> Result<(), SandboxError> { + let entry = self + .entries + .get(&id) + .map(|e| e.clone()) + .ok_or_else(|| invalid("owner"))?; + let mut entry = entry.lock().await; + tokio::time::timeout(std::time::Duration::from_secs(30), async { + let (epoch, owner) = self.execution_owner(id, &mut entry).await?; + owner.cancel_exec(epoch, exec_id).await + }) + .await + .map_err(|_| SandboxError::Unavailable("managed execution cancellation timed out".into()))? + } + async fn transfer( + mut closed: tokio::sync::watch::Receiver<()>, + transfer: impl std::future::Future>, + ) -> Result { + tokio::select! { + biased; + _ = closed.changed() => Err(SandboxError::Unavailable("managed file owner closed".into())), + result = transfer => result, + } + } + pub async fn read_file( + &self, + id: SandboxId, + path: String, + ) -> Result<(SessionFileMetadata, SessionFileStream), SandboxError> { + let (closed, transfer) = { + let entry = self + .entries + .get(&id) + .map(|e| e.clone()) + .ok_or_else(|| invalid("owner"))?; + let mut entry = entry.lock().await; + let closed = entry.transfer_stop.subscribe(); + let transfer = tokio::time::timeout(std::time::Duration::from_secs(30), async { + let (epoch, owner) = self.execution_owner(id, &mut entry).await?; + owner.prepare_read_file(epoch, path, closed.clone()).await + }) + .await + .map_err(|_| { + SandboxError::Unavailable("managed file read submission timed out".into()) + })??; + (closed, transfer) + }; + Self::transfer(closed, transfer).await + } + pub async fn write_file( + &self, + id: SandboxId, + spec: SessionFileSpec, + bytes: SessionFileStream, + ) -> Result { + let (closed, transfer) = { + let entry = self + .entries + .get(&id) + .map(|e| e.clone()) + .ok_or_else(|| invalid("owner"))?; + let mut entry = entry.lock().await; + let closed = entry.transfer_stop.subscribe(); + let transfer = tokio::time::timeout(std::time::Duration::from_secs(30), async { + let (epoch, owner) = self.execution_owner(id, &mut entry).await?; + owner.prepare_write_file(epoch, spec, bytes).await + }) + .await + .map_err(|_| { + SandboxError::Unavailable("managed file write submission timed out".into()) + })??; + (closed, transfer) + }; + Self::transfer(closed, transfer).await + } + pub async fn guard_running(&self, id: SandboxId) -> Result<(), SandboxError> { + let entry = self + .entries + .get(&id) + .map(|e| e.clone()) + .ok_or_else(|| invalid("owner"))?; + let mut entry = entry.lock().await; + if !matches!(entry.record.state, ManagedBootHostState::Started { .. }) { + return Err(invalid("not started")); + } + let epoch = entry.record.request.key.context.target.binding_epoch; + entry + .owner + .as_mut() + .ok_or_else(|| invalid("owner"))? + .ready(epoch) + .await + } + pub async fn shutdown(&self) { + let entries: Vec<_> = self.entries.iter().map(|e| e.value().clone()).collect(); + for entry in entries { + let mut entry = entry.lock().await; + if matches!(entry.record.state, ManagedBootHostState::Started { .. }) { + if let Some(policy) = entry.record.applied.as_ref().map(|a| a.policy.clone()) { + if self.retire_for_policy(&mut entry, policy).await.is_ok() { + continue; + } + } + } + if entry.owner.is_some() { + self.fail(&mut entry).await; + } + } + } + /// Diagnostic residency only. A busy lifecycle is unknown, never absence. + /// Do not wait for native I/O or reuse this inventory as start/policy proof. + pub fn list(&self) -> Result, SandboxError> { + let entries: Vec<_> = self + .entries + .iter() + .map(|e| (*e.key(), e.value().clone())) + .collect(); + let mut residents = Vec::new(); + for (id, entry) in entries { + let entry = entry.try_lock().map_err(|_| invalid("inventory busy"))?; + let network_reserved = entry.record.allocation.as_ref().is_some_and(|allocation| { + allocation.host_incarnation == self.incarnation + && entry.record.incarnation == self.incarnation + && self + .networks + .lock() + .expect("network slots") + .contains(allocation.attachment_generation) + }); + if entry.owner.is_some() + || entry.close_failed + || network_reserved + || matches!( + entry.record.state, + ManagedBootHostState::Prepared { .. } + | ManagedBootHostState::Finalized { .. } + | ManagedBootHostState::StartPending { .. } + | ManagedBootHostState::Started { .. } + ) + { + residents.push(id); + } + } + Ok(residents) + } +} + +/// Called only while holding the same owner lock as finalize and start. +fn bootstrap_retirement_allowed(record: &Record, request: &ManagedPolicyApply) -> bool { + let key = &record.request.key; + let Some(policy) = request.egress.managed_deny.as_ref() else { + return false; + }; + matches!(&record.state, ManagedBootHostState::Prepared { receipt } + if receipt.request == record.request && receipt.host_incarnation == record.incarnation) + && record.applied.is_none() + && request.expected.is_none() + && request.context.target == key.context.target + && request.context.binding_hash.is_none() + && request.context.operator_generation > key.context.operator_generation + && policy.revision >= record.request.bootstrap_policy.revision + && policy.contains(&record.request.bootstrap_policy) + && (policy.revision != record.request.bootstrap_policy.revision + || policy == &record.request.bootstrap_policy) +} + +fn bootstrap(allocation: &Allocation, policy: ManagedDenyPolicy) -> SessionEgressPolicy { + SessionEgressPolicy { + session_id: allocation.session_id, + sandbox_id: allocation.sandbox_id, + guest_ip: allocation.guest_ip, + network_allow_hosts: vec![], + network_allow_host_patterns: vec![], + allow_all: false, + secrets: vec![], + injects: vec![], + observes: vec![], + guest_services: vec![], + tunnels: vec![], + secret_mode: SecretMode::Literal, + apps: vec![], + managed_deny: Some(policy), + } +} +fn durable_write(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + let temporary = path.with_extension("partial"); + let mut file = std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(bytes)?; + file.sync_all()?; + std::fs::rename(&temporary, path)?; + std::fs::File::open( + path.parent() + .ok_or_else(|| std::io::Error::other("missing parent"))?, + )? + .sync_all() +} + +async fn publish_capture_manifest( + chunks: &engram_chunk_store::ChunkStore, + reference: ManifestRef, + manifest: &engram_chunk_store::Manifest, +) -> Result<(), SandboxError> { + if let Err(error) = chunks.put_manifest(reference, manifest).await { + // The immutable write may have completed before its reply was lost. + // Only the same reserved name and exact chunk list can complete a retry. + if chunks.get_manifest(reference).await.map_err(invalid)? != *manifest { + return Err(invalid(error)); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use engram_core::{ + traits::HostClient, + types::{ + managed_cold::*, + managed_create::ManagedCreateIdentity, + mcp_policy::{McpPolicyCreation, WriteEligibility}, + sandbox::AuxRoDrive, + }, + }; + + struct TestFile { + metadata: SessionFileMetadata, + chunks: Vec, + } + pub(super) struct ExecOwner { + pub(super) close_fails: bool, + pub(super) close_intents: Arc>>, + pub(super) capture_acks: Arc, + pub(super) close_wait: Option>, + epoch: u64, + live: bool, + pub(super) blocked: bool, + id: SandboxId, + pub(super) request: ManagedColdPrepare, + file: Arc>>, + file_read_fails: bool, + file_read_stalls: bool, + } + impl ExecOwner { + pub(super) fn exec_stream(&self, cmd: ExecRequest) -> Result { + Ok(ExecStream { + sandbox_id: self.id, + exec_id: cmd.exec_id.clone().unwrap(), + events: Box::pin(futures::stream::iter([ + engram_core::types::sandbox::ExecEvent::Stdout(bytes::Bytes::from( + serde_json::to_vec(&cmd).unwrap(), + )), + ])), + }) + } + pub(super) fn cancel_exec(&self, exec_id: String) -> Result<(), SandboxError> { + if exec_id == "ticket-1" { + Ok(()) + } else { + Err(invalid("wrong cancel ticket")) + } + } + pub(super) fn prepare_write_file( + &self, + spec: SessionFileSpec, + bytes: SessionFileStream, + ) -> FileWrite { + let file = self.file.clone(); + Box::pin(async move { + use futures::TryStreamExt; + let chunks = bytes.try_collect::>().await?; + let metadata = SessionFileMetadata { + path: spec.path, + size_bytes: spec.size_bytes, + sha256: spec.sha256, + }; + *file.lock().unwrap() = Some(TestFile { + metadata: metadata.clone(), + chunks, + }); + Ok(metadata) + }) + } + pub(super) fn read_file( + &self, + path: String, + closed: tokio::sync::watch::Receiver<()>, + ) -> Result<(SessionFileMetadata, SessionFileStream), SandboxError> { + let file = self.file.lock().unwrap(); + let TestFile { metadata, chunks } = file.as_ref().ok_or(SandboxError::NotFound)?; + if path != metadata.path { + return Err(SandboxError::NotFound); + } + let metadata = metadata.clone(); + let mut events = chunks.iter().cloned().map(Ok).collect::>(); + if self.file_read_fails { + events.push(Err(SandboxError::Unavailable("owner stream closed".into()))); + } + let events: SessionFileStream = if self.file_read_stalls { + Box::pin(futures::stream::pending()) + } else { + Box::pin(futures::stream::iter(events)) + }; + let stream = futures::stream::unfold(Some((events, closed)), |state| async move { + use futures::StreamExt; + let (mut events, mut closed) = state?; + tokio::select! { + biased; + _ = closed.changed() => Some((Err(SandboxError::Unavailable("owner stream closed".into())), None)), + item = events.next() => item.map(|item| (item, Some((events, closed)))), + } + }); + Ok((metadata, Box::pin(stream))) + } + pub(super) fn ready(&self, epoch: u64) -> Result<(), SandboxError> { + if self.live && self.epoch == epoch { + Ok(()) + } else { + Err(invalid("test owner closed or epoch changed")) + } + } + } + + fn install_exec_owner( + host: &Arc, + request: &ManagedBootPrepare, + ) -> Arc> { + let key = &request.key; + let id = key.context.target.sandbox_id; + durable_write( + &host + .bindings + .directory() + .join(format!("{id}.managed-sandbox")), + b"1", + ) + .unwrap(); + host.hub + .bind_session(key.creation.session_id, id, 1) + .unwrap(); + let intent = issued_start(request); + let mut record = failed(request.clone()); + record.incarnation = host.incarnation.clone(); + record.state = ManagedBootHostState::Started { + intent: intent.clone(), + }; + let entry = Arc::new(Mutex::new(Entry { + transfer_stop: tokio::sync::watch::channel(()).0, + close_failed: false, + record, + attach_owner: Some(host.hub.managed_owner(intent)), + owner: Some(Owner::ExecTest(Box::new(ExecOwner { + close_fails: false, + close_intents: Arc::new(std::sync::Mutex::new(Vec::new())), + capture_acks: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + close_wait: None, + epoch: 1, + live: true, + blocked: false, + id, + request: request.cold.clone(), + file: Arc::new(std::sync::Mutex::new(None)), + file_read_fails: false, + file_read_stalls: false, + }))), + agent: None, + })); + host.entries.insert(id, entry.clone()); + entry + } + fn exec_request() -> ExecRequest { + ExecRequest { + command: vec!["printf".into(), "ready".into()], + stdin: Some(b"input".to_vec()), + env: [("CHECK".into(), "value".into())].into(), + workdir: Some("/workspace".into()), + timeout: Some(std::time::Duration::from_secs(3)), + exec_id: Some("ticket-1".into()), + stdout_offset: Some(7), + stderr_offset: Some(2), + wake: Some(false), + } + } + #[tokio::test] + async fn managed_exec_uses_current_live_owner() { + use futures::StreamExt; + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + install_exec_owner(&host, &request); + let local = crate::host_client::LocalHostClient::new( + host.ordinary.clone(), + host.hub.clone(), + crate::IdleEvictionHarness::default(), + ) + .with_managed_boot(host); + let mut stream = local + .exec_stream(id, exec_request()) + .await + .expect("marked managed execution uses its live owner"); + assert_eq!(stream.sandbox_id, id); + assert_eq!(stream.exec_id, "ticket-1"); + let Some(engram_core::types::sandbox::ExecEvent::Stdout(bytes)) = + stream.events.next().await + else { + panic!("owner output missing") + }; + let routed: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + routed, + serde_json::json!({ + "command": ["printf", "ready"], "stdin": [105, 110, 112, 117, 116], + "env": {"CHECK": "value"}, "workdir": "/workspace", "timeout": {"secs": 3, "nanos": 0}, + "exec_id": "ticket-1", "stdout_offset": 7, "stderr_offset": 2, "wake": false, + }) + ); + local.cancel_exec(id, "ticket-1".into()).await.unwrap(); + assert!(local.cancel_exec(id, "wrong-ticket".into()).await.is_err()); + assert!(matches!( + local.probe_sandbox(id).await, + Err(SandboxError::InvalidSpec(_)) + )); + assert!(matches!( + local.exec_stream(SandboxId::new(), exec_request()).await, + Err(SandboxError::NotFound) + )); + assert!(matches!( + local.cancel_exec(SandboxId::new(), "ticket-1".into()).await, + Err(SandboxError::NotFound) + )); + local + .destroy( + id, + engram_core::traits::SessionFence::new(request.key.creation.session_id, 1), + ) + .await + .unwrap(); + assert!(matches!( + local.exec_stream(id, exec_request()).await, + Err(SandboxError::InvalidSpec(_)) + )); + assert!(matches!( + local.cancel_exec(id, "ticket-1".into()).await, + Err(SandboxError::InvalidSpec(_)) + )); + } + #[tokio::test] + async fn managed_exec_denies_stale_closed_and_recovered_owners() { + for case in 0..11 { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + let entry = install_exec_owner(&host, &request); + { + let mut entry = entry.lock().await; + match case { + 0 => { + if let Some(Owner::ExecTest(owner)) = &mut entry.owner { + owner.live = false; + } + } + 1 => { + if let Some(Owner::ExecTest(owner)) = &mut entry.owner { + owner.epoch = 2; + } + } + 2 => entry.owner = None, + 3 => entry.record.incarnation = "old-host".into(), + 4 => { + entry.record.state = ManagedBootHostState::Failed { + key: request.key.clone(), + } + } + 5 => host + .hub + .bind_session(request.key.creation.session_id, id, 2) + .unwrap(), + 6 => { + if let ManagedBootHostState::Started { intent } = &mut entry.record.state { + intent.receipt.key.boot_attempt = "other-boot".into(); + } + } + 7 => entry.attach_owner = None, + 8 => entry.record.request.key.context.target.host_id = HostId::new(), + 9 => entry.record.request.key.context.target.sandbox_id = SandboxId::new(), + _ => { + if let Some(Owner::ExecTest(owner)) = &mut entry.owner { + owner.request.boot_attempt = "other-owner".into(); + } + } + } + } + let local = crate::host_client::LocalHostClient::new( + host.ordinary.clone(), + host.hub.clone(), + crate::IdleEvictionHarness::default(), + ) + .with_managed_boot(host.clone()); + assert!( + matches!( + local.exec_stream(id, exec_request()).await, + Err(SandboxError::InvalidSpec(_)) + ), + "case {case}" + ); + assert!( + matches!( + local.cancel_exec(id, "ticket-1".into()).await, + Err(SandboxError::InvalidSpec(_)) + ), + "case {case}" + ); + host.entries.remove(&id); + assert!( + matches!( + local.exec_stream(id, exec_request()).await, + Err(SandboxError::InvalidSpec(_)) + ), + "lost owner {case}" + ); + } + } + #[tokio::test(start_paused = true)] + async fn managed_exec_submission_timeout_releases_lifecycle_lock() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + let entry = install_exec_owner(&host, &request); + if let Some(Owner::ExecTest(owner)) = &mut entry.lock().await.owner { + owner.blocked = true; + } + let local = crate::host_client::LocalHostClient::new( + host.ordinary.clone(), + host.hub.clone(), + crate::IdleEvictionHarness::default(), + ) + .with_managed_boot(host); + assert!(matches!( + local.exec_stream(id, exec_request()).await, + Err(SandboxError::Unavailable(_)) + )); + assert!(entry.try_lock().is_ok()); + assert!(matches!( + local.cancel_exec(id, "ticket-1".into()).await, + Err(SandboxError::Unavailable(_)) + )); + assert!(entry.try_lock().is_ok()); + } + + #[derive(Default)] + struct FileBackendCalls(std::sync::atomic::AtomicUsize, Vec); + #[async_trait::async_trait] + impl SandboxBackend for FileBackendCalls { + fn captures_guest_memory(&self) -> bool { + false + } + async fn exec_stream( + &self, + _: SandboxId, + _: ExecRequest, + ) -> Result { + panic!("unexpected ordinary execution") + } + fn snapshot_path_for(&self, _: engram_core::SnapshotId) -> PathBuf { + panic!("unexpected ordinary snapshot path") + } + async fn create( + &self, + _: SandboxId, + _: engram_core::types::sandbox::SandboxSpec, + ) -> Result<(), SandboxError> { + panic!("unexpected ordinary create") + } + async fn snapshot( + &self, + _: SandboxId, + ) -> Result { + panic!("unexpected ordinary snapshot") + } + async fn restore( + &self, + _: SandboxId, + _: engram_core::types::snapshot::SnapshotMetadata, + _: Option, + ) -> Result<(), SandboxError> { + panic!("unexpected ordinary restore") + } + async fn destroy(&self, _: SandboxId) -> Result<(), SandboxError> { + panic!("unexpected ordinary destroy") + } + async fn list(&self) -> Result, SandboxError> { + Ok(self.1.clone()) + } + async fn read_file( + &self, + _: SandboxId, + _: String, + ) -> Result<(SessionFileMetadata, SessionFileStream), SandboxError> { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Err(SandboxError::NotFound) + } + async fn write_file( + &self, + _: SandboxId, + _: SessionFileSpec, + _: SessionFileStream, + ) -> Result { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Err(SandboxError::NotFound) + } + } + fn file_spec() -> SessionFileSpec { + SessionFileSpec { + path: "/tmp/uploads/file.bin".into(), + size_bytes: 11, + sha256: "a".repeat(64), + mode: Some(0o640), + } + } + fn file_bytes() -> SessionFileStream { + Box::pin(futures::stream::iter([Ok(bytes::Bytes::from_static( + b"owner-bytes", + ))])) + } + fn file_client( + host: &Arc, + ordinary: &Arc, + ) -> crate::host_client::LocalHostClient { + crate::host_client::LocalHostClient::new( + ordinary.clone(), + host.hub.clone(), + crate::IdleEvictionHarness::default(), + ) + .with_managed_boot(host.clone()) + } + #[tokio::test] + async fn inventory_propagates_unknown_managed_binding_state() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let managed = request.key.context.target.sandbox_id; + install_exec_owner(&host, &request); + let ordinary = SandboxId::new(); + let backend = Arc::new(FileBackendCalls( + Default::default(), + vec![ordinary, managed], + )); + let local = file_client(&host, &backend); + // The managed ID comes from its live owner, once, after ordinary IDs. + assert_eq!(local.list().await.unwrap(), vec![ordinary, managed]); + let corrupt = host.bindings.directory().join("invalid.managed-required"); + std::fs::write(&corrupt, b"1").unwrap(); + assert!(matches!( + local.list().await, + Err(SandboxError::InvalidSpec(_)) + )); + std::fs::remove_file(corrupt).unwrap(); + assert_eq!(local.list().await.unwrap(), vec![ordinary, managed]); + } + #[tokio::test] + async fn tombstone_retries_proxy_cleanup_before_inventory_ack() { + let temp = tempfile::tempdir_in("/tmp").unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.cold.sandbox_id; + let entry = install_exec_owner(&host, &request); + let generation = reserve_test_network(&host, &mut *entry.lock().await); + let local = file_client(&host, &Arc::new(FileBackendCalls::default())); + let retained = host.directory.join(format!("{id}.retired")); + durable_write(&retained, b"retained disk").unwrap(); + let replacement = SandboxId::new(); + host.hub + .bind_session(request.cold.session_id, replacement, 2) + .unwrap(); + let (server, mut removals, _) = cleanup_proxy(temp.path()); + for succeeds in [false, true] { + let cleanup = local.destroy_tombstoned(id); + let ack = async { + let reply = removals.recv().await.unwrap(); + assert!(host.networks.lock().unwrap().contains(generation)); + reply + .send(if succeeds { + engram_egress_proto::FromProxyd::Ok + } else { + engram_egress_proto::FromProxyd::Err("still draining".into()) + }) + .unwrap(); + }; + let (result, ()) = tokio::join!(cleanup, ack); + assert_eq!(result.is_ok(), succeeds); + assert_eq!(local.list().await.unwrap().contains(&id), !succeeds); + assert_eq!( + host.networks.lock().unwrap().contains(generation), + !succeeds + ); + let entry = entry.lock().await; + assert!(entry.owner.is_none()); + assert!(entry.attach_owner.is_none()); + assert!(matches!( + entry.record.state, + ManagedBootHostState::Failed { .. } + )); + } + local.destroy_tombstoned(id).await.unwrap(); + assert_eq!( + host.hub.bound_sandbox(request.cold.session_id), + Some(replacement) + ); + assert_eq!(std::fs::read(retained).unwrap(), b"retained disk"); + assert!(host.bindings.managed_sandbox_required(id).unwrap()); + assert!(host.read_record(id).unwrap().is_some()); + server.abort(); + let _ = server.await; + } + + #[tokio::test] + async fn started_destroy_grants_disk_release_but_failure_cleanup_does_not() { + for destroy in [true, false] { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let entry = install_exec_owner(&host, &request); + let intents = match &entry.lock().await.owner { + Some(Owner::ExecTest(owner)) => owner.close_intents.clone(), + _ => panic!("execution owner missing"), + }; + if destroy { + host.destroy_tombstoned(request.cold.sandbox_id) + .await + .unwrap(); + } else { + host.fail(&mut *entry.lock().await).await; + } + assert_eq!(*intents.lock().unwrap(), vec![destroy]); + assert!(!entry.lock().await.close_failed); + } + } + + #[tokio::test] + async fn caller_cancellation_does_not_drop_an_authorized_destroy_owner() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let entry = install_exec_owner(&host, &request); + let (finish, wait) = tokio::sync::oneshot::channel(); + let intents = match entry.lock().await.owner.as_mut() { + Some(Owner::ExecTest(owner)) => { + owner.close_wait = Some(wait); + owner.close_intents.clone() + } + _ => panic!("execution owner missing"), + }; + let call = tokio::spawn({ + let host = host.clone(); + async move { host.destroy_tombstoned(request.cold.sandbox_id).await } + }); + while intents.lock().unwrap().is_empty() { + tokio::task::yield_now().await; + } + call.abort(); + let _ = call.await; + finish.send(()).unwrap(); + let entry = entry.lock().await; + assert!(entry.owner.is_none()); + assert!(!entry.close_failed); + assert_eq!(*intents.lock().unwrap(), vec![true]); + } + + #[tokio::test] + async fn tombstone_uncertain_native_close_remains_in_inventory() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.cold.sandbox_id; + let entry = install_exec_owner(&host, &request); + if let Some(Owner::ExecTest(owner)) = &mut entry.lock().await.owner { + owner.close_fails = true; + } + let local = file_client(&host, &Arc::new(FileBackendCalls::default())); + // Generic callers still need a session fence, even with a live owner. + assert!(local + .destroy(id, engram_core::traits::SessionFence::unfenced()) + .await + .is_err()); + assert!(entry.lock().await.owner.is_some()); + for _ in 0..2 { + assert!(local.destroy_tombstoned(id).await.is_err()); + assert_eq!(local.list().await.unwrap(), vec![id]); + assert!(entry.lock().await.close_failed); + } + } + + #[tokio::test] + async fn tombstone_rejects_unknown_or_changed_managed_owner() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.cold.sandbox_id; + let entry = install_exec_owner(&host, &request); + let local = file_client(&host, &Arc::new(FileBackendCalls::default())); + entry.lock().await.record.incarnation = "previous-host".into(); + assert!(local.destroy_tombstoned(id).await.is_err()); + assert!(entry.lock().await.owner.is_some()); + entry.lock().await.record.incarnation = host.incarnation.clone(); + if let Some(Owner::ExecTest(owner)) = &mut entry.lock().await.owner { + owner.request.boot_attempt = "different-owner".into(); + } + assert!(local.destroy_tombstoned(id).await.is_err()); + assert!(entry.lock().await.owner.is_some()); + host.entries.remove(&id); + assert!(local.destroy_tombstoned(id).await.is_err()); + assert!(host.bindings.managed_sandbox_required(id).unwrap()); + let corrupt = host.bindings.directory().join("invalid.managed-required"); + std::fs::write(&corrupt, b"1").unwrap(); + assert!(local.destroy_tombstoned(SandboxId::new()).await.is_err()); + assert_eq!(std::fs::read(corrupt).unwrap(), b"1"); + } + #[tokio::test] + async fn managed_file_io_uses_current_owner() { + use futures::TryStreamExt; + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + install_exec_owner(&host, &request); + let ordinary = Arc::new(FileBackendCalls::default()); + let local = file_client(&host, &ordinary); + let metadata = local + .write_file(id, file_spec(), file_bytes()) + .await + .expect("managed upload uses its current owner"); + assert_eq!(metadata.path, "/tmp/uploads/file.bin"); + assert_eq!(metadata.size_bytes, 11); + assert_eq!(metadata.sha256, "a".repeat(64)); + let (download, stream) = local + .read_file(id, metadata.path.clone()) + .await + .expect("managed download uses its current owner"); + assert_eq!(download, metadata); + assert_eq!( + stream.try_collect::>().await.unwrap(), + [bytes::Bytes::from_static(b"owner-bytes")] + ); + assert_eq!(ordinary.0.load(std::sync::atomic::Ordering::SeqCst), 0); + } + #[tokio::test(start_paused = true)] + async fn managed_file_io_rejects_stale_or_closed_owner() { + for case in 0..13 { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + let entry = install_exec_owner(&host, &request); + { + let mut entry = entry.lock().await; + match case { + 0 => { + if let Some(Owner::ExecTest(owner)) = &mut entry.owner { + owner.live = false; + } + } + 1 => { + if let Some(Owner::ExecTest(owner)) = &mut entry.owner { + owner.epoch = 2; + } + } + 2 => entry.owner = None, + 3 => entry.record.incarnation = "old-host".into(), + 4 => { + entry.record.state = ManagedBootHostState::Failed { + key: request.key.clone(), + } + } + 5 => host + .hub + .bind_session(request.key.creation.session_id, id, 2) + .unwrap(), + 6 => { + if let ManagedBootHostState::Started { intent } = &mut entry.record.state { + intent.receipt.key.boot_attempt = "other-boot".into(); + } + } + 7 => entry.attach_owner = None, + 8 => entry.record.request.key.context.target.host_id = HostId::new(), + 9 => entry.record.request.key.context.target.sandbox_id = SandboxId::new(), + 10 => { + if let Some(Owner::ExecTest(owner)) = &mut entry.owner { + owner.request.boot_attempt = "replacement".into(); + } + } + 11 => { + if let Some(Owner::ExecTest(owner)) = &mut entry.owner { + owner.blocked = true; + } + } + _ => { + host.entries.remove(&id); + } + } + } + let ordinary = Arc::new(FileBackendCalls::default()); + let local = file_client(&host, &ordinary); + let read = local + .read_file(id, file_spec().path) + .await + .err() + .expect("stale read denied"); + let write = local + .write_file(id, file_spec(), file_bytes()) + .await + .unwrap_err(); + if case == 11 { + assert!( + matches!(read, SandboxError::Unavailable(_)), + "blocked read readiness is bounded" + ); + assert!( + matches!(write, SandboxError::Unavailable(_)), + "blocked write readiness is bounded" + ); + } else { + assert!( + matches!(read, SandboxError::InvalidSpec(_)), + "read case {case}" + ); + assert!( + matches!(write, SandboxError::InvalidSpec(_)), + "write case {case}" + ); + } + assert!( + entry.try_lock().is_ok(), + "submission must release the lifecycle lock" + ); + assert_eq!( + ordinary.0.load(std::sync::atomic::Ordering::SeqCst), + 0, + "case {case}" + ); + } + } + #[tokio::test] + async fn managed_file_io_never_uses_ordinary_backend() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + let entry = install_exec_owner(&host, &request); + entry.lock().await.owner = None; + let ordinary = Arc::new(FileBackendCalls::default()); + let local = file_client(&host, &ordinary); + assert!(matches!( + local.read_file(id, file_spec().path).await, + Err(SandboxError::InvalidSpec(_)) + )); + assert!(matches!( + local.write_file(id, file_spec(), file_bytes()).await, + Err(SandboxError::InvalidSpec(_)) + )); + let no_boot = crate::host_client::LocalHostClient::new( + ordinary.clone(), + host.hub.clone(), + crate::IdleEvictionHarness::default(), + ); + assert!(matches!( + no_boot.read_file(id, file_spec().path).await, + Err(SandboxError::InvalidSpec(_)) + )); + assert!(matches!( + no_boot.write_file(id, file_spec(), file_bytes()).await, + Err(SandboxError::InvalidSpec(_)) + )); + assert_eq!(ordinary.0.load(std::sync::atomic::Ordering::SeqCst), 0); + let ordinary_id = SandboxId::new(); + assert!(matches!( + local.read_file(ordinary_id, file_spec().path).await, + Err(SandboxError::NotFound) + )); + assert!(matches!( + local + .write_file(ordinary_id, file_spec(), file_bytes()) + .await, + Err(SandboxError::NotFound) + )); + assert_eq!( + ordinary.0.load(std::sync::atomic::Ordering::SeqCst), + 2, + "ordinary IDs still use the ordinary backend" + ); + } + + #[tokio::test] + async fn managed_file_io_cancellation_and_replacement_keep_stream_owner() { + use futures::TryStreamExt; + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + let entry = install_exec_owner(&host, &request); + let ordinary = Arc::new(FileBackendCalls::default()); + let local = file_client(&host, &ordinary); + local + .write_file(id, file_spec(), file_bytes()) + .await + .unwrap(); + let (_, opened) = local.read_file(id, file_spec().path).await.unwrap(); + if let Some(Owner::ExecTest(owner)) = &mut entry.lock().await.owner { + owner.file_read_fails = true; + } + let (_, mut failed_stream) = local.read_file(id, file_spec().path).await.unwrap(); + let mut upload = + Box::pin(local.write_file(id, file_spec(), Box::pin(futures::stream::pending()))); + tokio::select! { + result = &mut upload => panic!("upload must wait for its source: {result:?}"), + _ = tokio::task::yield_now() => {}, + } + assert!( + entry.try_lock().is_ok(), + "body transfer releases the lifecycle lock" + ); + drop(upload); + assert!( + entry.try_lock().is_ok(), + "cancelled upload releases its lifecycle lock" + ); + let replacement = install_exec_owner(&host, &request); + local + .write_file( + id, + file_spec(), + Box::pin(futures::stream::iter([Ok(bytes::Bytes::from_static( + b"other-bytes", + ))])), + ) + .await + .unwrap(); + assert_eq!( + opened.try_collect::>().await.unwrap(), + [bytes::Bytes::from_static(b"owner-bytes")], + "opened download never reads the replacement owner" + ); + use futures::StreamExt; + assert_eq!( + failed_stream.next().await.unwrap().unwrap(), + bytes::Bytes::from_static(b"owner-bytes") + ); + assert!(failed_stream.next().await.unwrap().is_err()); + assert!( + failed_stream.next().await.is_none(), + "failed stream must end without reading its replacement" + ); + let (_, current) = local.read_file(id, file_spec().path).await.unwrap(); + assert_eq!( + current.try_collect::>().await.unwrap(), + [bytes::Bytes::from_static(b"other-bytes")] + ); + assert!(local + .write_file( + id, + file_spec(), + Box::pin(futures::stream::iter([Err(SandboxError::Unavailable( + "source stopped".into() + ))])) + ) + .await + .is_err()); + assert!(replacement.try_lock().is_ok()); + assert_eq!(ordinary.0.load(std::sync::atomic::Ordering::SeqCst), 0); + } + + #[tokio::test(start_paused = true)] + async fn managed_file_progressing_upload_can_exceed_submission_timeout() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + install_exec_owner(&host, &request); + let ordinary = Arc::new(FileBackendCalls::default()); + let local = file_client(&host, &ordinary); + let source = futures::stream::unfold(0, |n| async move { + if n == 4 { + return None; + } + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + Some((Ok(bytes::Bytes::from_static(b"a")), n + 1)) + }); + let mut spec = file_spec(); + spec.size_bytes = 4; + let result = local + .write_file(id, spec, Box::pin(source)) + .await + .expect("progressing body has no fixed submission deadline"); + assert_eq!(result.size_bytes, 4); + assert_eq!(ordinary.0.load(std::sync::atomic::Ordering::SeqCst), 0); + } + + #[tokio::test(start_paused = true)] + async fn managed_file_close_cancels_stalled_upload_outside_lifecycle_lock() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + let entry = install_exec_owner(&host, &request); + let ordinary = Arc::new(FileBackendCalls::default()); + let local = file_client(&host, &ordinary); + let mut upload = + Box::pin(local.write_file(id, file_spec(), Box::pin(futures::stream::pending()))); + tokio::select! { + result = &mut upload => panic!("stalled upload must wait for source: {result:?}"), + _ = tokio::task::yield_now() => {}, + } + let mut entry = entry + .try_lock() + .expect("body transfer must release lifecycle lock"); + let (release_close, native_close) = tokio::sync::oneshot::channel(); + if let Some(Owner::ExecTest(owner)) = &mut entry.owner { + owner.close_wait = Some(native_close); + } + let mut close = Box::pin(host.close_resources(&mut entry, false)); + tokio::select! { + result = &mut close => panic!("native close must wait: {result:?}"), + _ = tokio::task::yield_now() => {}, + } + assert!(matches!( + tokio::time::timeout(std::time::Duration::from_secs(1), &mut upload).await, + Ok(Err(SandboxError::Unavailable(_))) + )); + release_close.send(()).unwrap(); + close.await.unwrap(); + assert!(entry.owner.is_none()); + drop(entry); + let replacement = install_exec_owner(&host, &request); + assert!(replacement.try_lock().is_ok()); + local + .write_file(id, file_spec(), file_bytes()) + .await + .unwrap(); + assert_eq!(ordinary.0.load(std::sync::atomic::Ordering::SeqCst), 0); + } + + #[tokio::test(start_paused = true)] + async fn managed_file_close_stalled_read_before_native_close_completes() { + use futures::StreamExt; + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + let entry = install_exec_owner(&host, &request); + let ordinary = Arc::new(FileBackendCalls::default()); + let local = file_client(&host, &ordinary); + local + .write_file(id, file_spec(), file_bytes()) + .await + .unwrap(); + let (release_close, native_close) = tokio::sync::oneshot::channel(); + if let Some(Owner::ExecTest(owner)) = &mut entry.lock().await.owner { + owner.file_read_stalls = true; + owner.close_wait = Some(native_close); + } + let (_, mut stream) = local.read_file(id, file_spec().path).await.unwrap(); + let mut entry = entry.lock().await; + let mut close = Box::pin(host.close_resources(&mut entry, false)); + tokio::select! { + result = &mut close => panic!("native close must wait: {result:?}"), + _ = tokio::task::yield_now() => {}, + } + assert!( + matches!( + tokio::time::timeout(std::time::Duration::from_secs(1), stream.next()).await, + Ok(Some(Err(SandboxError::Unavailable(_)))) + ), + "old owner close must end the returned read before native close completes" + ); + release_close.send(()).unwrap(); + close.await.unwrap(); + drop(entry); + install_exec_owner(&host, &request); + local + .write_file(id, file_spec(), file_bytes()) + .await + .unwrap(); + assert!( + stream.next().await.is_none(), + "closed read cannot adopt replacement bytes" + ); + let (_, mut current) = local.read_file(id, file_spec().path).await.unwrap(); + assert_eq!( + current.next().await.unwrap().unwrap(), + bytes::Bytes::from_static(b"owner-bytes") + ); + assert_eq!(ordinary.0.load(std::sync::atomic::Ordering::SeqCst), 0); + } + + fn fixture(path: &Path) -> (Arc, ManagedBootPrepare) { + let host_id = HostId::new(); + let session_id = engram_core::SessionId::new(); + let sandbox_id = SandboxId::new(); + let manifest = ManifestRef::new(); + let drives: Vec<_> = (0..3) + .map(|i| AuxRoDrive { + drive_id: AuxRoDrive::slot_drive_id(i), + guest_mount: AuxRoDrive::slot_guest_mount(i), + fs_type: "squashfs".into(), + sha256: Some("a".repeat(64)), + }) + .collect(); + let spec = serde_json::json!({"image":"image-v1", "rootfs_source":path.join("selected"), "rootfs_manifest":manifest, "cpu":{"vcpus":1}, "memory":{"max_mib":128}, "disk":{"max_gib":1}, "env":{}, "aux_ro_drives":drives}); + let request = ManagedBootPrepare { + key: ManagedBootKey { + creation: ManagedCreateIdentity { + session_id, + task_id: "task".into(), + policy: McpPolicyCreation { + actor_id: "actor".into(), + input_id: "input".into(), + parent_session_id: None, + eligibility: WriteEligibility::ReadOnly, + }, + input_digest: digest(b"input"), + }, + context: ManagedPolicyContext { + target: ManagedPolicyTarget { + session_id, + sandbox_id, + host_id, + binding_epoch: 1, + }, + operator_generation: 1, + operator_identity: digest(b"operator"), + binding_hash: None, + }, + boot_attempt: "boot-1".into(), + native_digest: digest(b"[]"), + }, + cold: ManagedColdPrepare { + version: 1, + session_id, + sandbox_id, + boot_attempt: "boot-1".into(), + image: "image-v1".into(), + disk: ManagedColdDisk { + manifest, + sha256: "b".repeat(64), + size_bytes: 4, + source: ManagedColdSource::FreshImage { + base_manifest: ManifestRef::new(), + }, + }, + kernel_sha256: "c".repeat(64), + aux_ro_drives: drives, + }, + bootstrap_policy: ManagedDenyPolicy::new(1, ["mcp.example".into()]).unwrap(), + capture_authority: None, + sandbox_spec_json: spec.to_string(), + }; + request.spec().unwrap(); + std::fs::write(path.join("kernel"), b"kernel").unwrap(); + #[cfg(target_os = "macos")] + let backend = ManagedBackend::Vz { + backend: Arc::new( + engram_sandbox_vz::VzBackend::new( + path.join("native"), + engram_sandbox_vz::VzConfig::with_kernel(path.join("kernel")) + .with_bundle_dir(path), + ) + .unwrap(), + ), + helper: engram_sandbox_vz::ManagedHelperArtifact { + path: path.join("absent-helper"), + sha256: "a".repeat(64), + }, + }; + #[cfg(target_os = "linux")] + let backend = ManagedBackend::Firecracker { + backend: Arc::new(engram_sandbox_firecracker::FirecrackerBackend::new( + path.join("native"), + { + let mut config = engram_sandbox_firecracker::FirecrackerConfig::with_kernel( + path.join("kernel"), + ); + config.bundle_dir = path.to_owned(); + config + }, + )), + disk: Arc::new(engram_host_core::FileDisk::new(path.join("volumes"), 4096).unwrap()), + }; + let ordinary = Arc::new(engram_sandbox_process::ProcessBackend::new( + path.join("ordinary"), + )); + let bindings = BindingStore::open(path.join("bindings")).unwrap(); + let hub = Arc::new(HarnessHub::new( + crate::harness::event_sink_to(|_, _, _, _| async { Ok(()) }), + bindings, + )); + let egress = Arc::new(HostEgress::new( + Arc::new(crate::proxyd_client::ProxydHandle::new( + path.join("absent.sock"), + )), + String::new(), + None, + )); + let chunks = engram_chunk_store::ChunkStore::new(Arc::new( + engram_storage_local::LocalBlobStorage::new(path.join("blobs")), + )); + let host = Arc::new( + ManagedBootHost::new( + ManagedBootHostConfig { + host_id, + incarnation: "current-host".into(), + directory: path.join("records"), + }, + backend, + ordinary, + egress, + hub, + engram_fill::FillEngine::new(chunks, engram_fill::Config::default()), + ) + .unwrap(), + ); + (host, request) + } + #[test] + fn address_slots_reuse_retired_capacity_without_reusing_attachment_identity() { + let mut slots = NetworkSlots::new(2); + let live = slots.reserve().unwrap(); + let mut previous = live.1; + for _ in 0..8 { + let current = slots.reserve().expect("retired capacity is reusable"); + assert_ne!(current.0, live.0); + assert!(current.1 > previous); + assert!(slots.reserve().is_err(), "live slots must remain exclusive"); + if previous != live.1 { + slots.release(previous); + } + assert!( + slots.contains(current.1), + "stale release must not retire a replacement" + ); + slots.release(current.1); + previous = current.1; + } + } + + #[test] + fn concurrent_address_reservations_are_unique() { + let slots = Arc::new(std::sync::Mutex::new(NetworkSlots::new(8))); + let owners: Vec<_> = (0..8) + .map(|_| { + let slots = slots.clone(); + std::thread::spawn(move || slots.lock().unwrap().reserve().unwrap()) + }) + .collect(); + let reservations: std::collections::HashSet<_> = owners + .into_iter() + .map(|owner| owner.join().unwrap().0) + .collect(); + assert_eq!(reservations.len(), 8); + assert!(slots.lock().unwrap().reserve().is_err()); + } + + fn reserve_test_network(host: &ManagedBootHost, entry: &mut Entry) -> u64 { + let (slot, generation) = host.networks.lock().unwrap().reserve().unwrap(); + let request = &entry.record.request; + entry.record.allocation = Some(Allocation { + version: 1, + host_id: host.host_id, + host_incarnation: host.incarnation.clone(), + proxy_incarnation: "proxy-1".into(), + session_id: request.key.creation.session_id, + sandbox_id: request.cold.sandbox_id, + boot_attempt: request.cold.boot_attempt.clone(), + attachment_generation: generation, + guest_ip: std::net::Ipv4Addr::new(198, 18, slot as u8, 2), + gateway_ip: std::net::Ipv4Addr::new(198, 18, slot as u8, 1), + policy_revision: request.bootstrap_policy.revision, + policy_digest: request.bootstrap_policy.digest.clone(), + }); + generation + } + + fn cleanup_proxy( + path: &Path, + ) -> ( + tokio::task::JoinHandle<()>, + tokio::sync::mpsc::Receiver>, + tokio::sync::mpsc::Receiver, + ) { + use engram_egress_proto::{FromProxyd, ToProxyd}; + let listener = tokio::net::UnixListener::bind(path.join("absent.sock")).unwrap(); + let (pending, removals) = + tokio::sync::mpsc::channel::>(1); + let (registered, registrations) = tokio::sync::mpsc::channel(1); + let server = tokio::spawn(async move { + let mut tasks = tokio::task::JoinSet::new(); + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + let pending = pending.clone(); + let registered = registered.clone(); + tasks.spawn(async move { + while let Ok(message) = + engram_egress_proto::read_frame::<_, ToProxyd>(&mut stream).await + { + let reply = match message { + ToProxyd::Hello { .. } => { + FromProxyd::HelloAck(engram_egress_proto::HelloInfo { + incarnation: "proxy-1".into(), + proto_version: engram_egress_proto::PROTO_VERSION, + source_fingerprint: None, + proxy_port: 1, + dns_port: 2, + gateway_port: 3, + ca_fingerprint: "test".into(), + coord_url: "test".into(), + }) + } + ToProxyd::LookupGuest(_) => FromProxyd::Guest(None), + ToProxyd::ApplyPolicy(_) => FromProxyd::Ok, + ToProxyd::PrivateNetwork => { + use engram_egress_proto::private; + let allocation: Allocation = + private::read(&mut stream).await.unwrap(); + private::write( + &mut stream, + &private::Reply { + version: 1, + sequence: 0, + epoch: 0, + state: "closed".into(), + }, + ) + .await + .unwrap(); + registered.send(allocation).await.unwrap(); + private::drain_to_eof(&stream).await.unwrap(); + return; + } + ToProxyd::ReadManaged(_) => FromProxyd::ManagedState(None), + ToProxyd::RetirePrivateNetwork(_) => { + let (reply, wait) = tokio::sync::oneshot::channel(); + pending.send(reply).await.unwrap(); + wait.await.unwrap() + } + _ => panic!("unexpected cleanup request"), + }; + if engram_egress_proto::write_frame(&mut stream, &reply) + .await + .is_err() + { + return; + } + } + }); + } + }); + (server, removals, registrations) + } + + #[tokio::test] + async fn address_release_waits_for_successful_proxy_retirement() { + use engram_egress_proto::FromProxyd; + let temp = tempfile::tempdir_in("/tmp").unwrap(); + let (host, request) = fixture(temp.path()); + let entry = install_exec_owner(&host, &request); + let generation = reserve_test_network(&host, &mut *entry.lock().await); + let (server, mut removals, mut registrations) = cleanup_proxy(temp.path()); + // Once a written removal times out, its late ACK must never complete + // the next removal, whose drain is still held by the daemon. + let h = host.clone(); + let e = entry.clone(); + let first = + tokio::spawn(async move { h.close_resources(&mut *e.lock().await, false).await }); + let late = removals.recv().await.unwrap(); + tokio::time::pause(); + tokio::time::advance(std::time::Duration::from_secs(11)).await; + assert!(first.await.unwrap().is_err()); + tokio::time::resume(); + assert!(host.networks.lock().unwrap().contains(generation)); + late.send(FromProxyd::Ok).unwrap(); + let h = host.clone(); + let e = entry.clone(); + let mut second = + tokio::spawn(async move { h.close_resources(&mut *e.lock().await, false).await }); + let held = removals.recv().await.unwrap(); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(1), &mut second) + .await + .is_err(), + "a late ACK must not complete the next removal" + ); + assert!(host.networks.lock().unwrap().contains(generation)); + held.send(FromProxyd::Err("still draining".into())).unwrap(); + assert!(second.await.unwrap().is_err()); + for succeed in [false, true] { + let h = host.clone(); + let e = entry.clone(); + let close = + tokio::spawn(async move { h.close_resources(&mut *e.lock().await, false).await }); + let reply = tokio::time::timeout(std::time::Duration::from_secs(5), removals.recv()) + .await + .unwrap() + .unwrap(); + assert!( + host.networks.lock().unwrap().contains(generation), + "no address reuse before cleanup ACK" + ); + reply + .send(if succeed { + FromProxyd::Ok + } else { + FromProxyd::Err("cleanup uncertain".into()) + }) + .unwrap(); + assert_eq!(close.await.unwrap().is_ok(), succeed); + assert_eq!(host.networks.lock().unwrap().contains(generation), !succeed); + } + // Real host prepares reach the actual backend pin and fail verification. + // With one live address and capacity two, retries must reuse only the + // other slot, even after more attempts than the capacity. + host.networks.lock().unwrap().slots.truncate(2); + let held = host.networks.lock().unwrap().reserve().unwrap(); + host.fill + .store() + .put_manifest( + request.cold.disk.manifest, + &engram_chunk_store::Manifest::empty(engram_chunk_store::ManifestKind::Disk, 4), + ) + .await + .unwrap(); + let mut previous = held.1; + for n in 0..6 { + let mut next = request.clone(); + let session = engram_core::SessionId::new(); + let sandbox = SandboxId::new(); + next.key.creation.session_id = session; + next.key.context.target.session_id = session; + next.key.context.target.sandbox_id = sandbox; + next.cold.session_id = session; + next.cold.sandbox_id = sandbox; + next.key.boot_attempt = format!("failed-{n}"); + next.cold.boot_attempt = next.key.boot_attempt.clone(); + let h = host.clone(); + let prepared = tokio::spawn(async move { h.prepare(next).await }); + let allocation = + tokio::time::timeout(std::time::Duration::from_secs(5), registrations.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(allocation.guest_ip, std::net::Ipv4Addr::new(198, 18, 1, 2)); + assert!(allocation.attachment_generation > previous); + previous = allocation.attachment_generation; + let reply = tokio::time::timeout(std::time::Duration::from_secs(5), removals.recv()) + .await + .unwrap() + .unwrap(); + assert!(host.networks.lock().unwrap().contains(previous)); + reply.send(FromProxyd::Ok).unwrap(); + assert!(prepared.await.unwrap().is_err()); + assert!(!host.networks.lock().unwrap().contains(previous)); + assert!(host.networks.lock().unwrap().contains(held.1)); + assert!(std::fs::read_dir(&host.directory) + .unwrap() + .all(|entry| !entry.unwrap().file_type().unwrap().is_dir())); + } + server.abort(); + let _ = server.await; + } + + #[tokio::test] + async fn capture_retry_waits_for_cleanup_after_a_timed_out_removal() { + use engram_egress_proto::FromProxyd; + let temp = tempfile::tempdir_in("/tmp").unwrap(); + let (host, request) = fixture(temp.path()); + let entry = install_exec_owner(&host, &request); + let capture_acks = match &entry.lock().await.owner { + Some(Owner::ExecTest(owner)) => owner.capture_acks.clone(), + _ => panic!("missing owner"), + }; + let capture = ManagedCaptureRequest { + key: request.key.clone(), + capture_id: engram_core::SnapshotId::new(), + }; + let path = host.directory.join(format!("{}.disk", capture.capture_id)); + durable_write(&path, b"disk cut").unwrap(); + let artifact = hash_file(&path).await.unwrap(); + let generation = { + let mut entry = entry.lock().await; + entry.record.capture = Some(CaptureProgress { + request: capture.clone(), + manifest: capture.disk_manifest(), + artifact: Some(artifact), + authority: None, + }); + reserve_test_network(&host, &mut entry) + }; + let (server, mut removals, _) = cleanup_proxy(temp.path()); + let h = host.clone(); + let c = capture.clone(); + let first = tokio::spawn(async move { h.capture(c).await }); + let late = removals.recv().await.unwrap(); + assert_eq!(capture_acks.load(std::sync::atomic::Ordering::SeqCst), 1); + tokio::time::pause(); + tokio::time::advance(std::time::Duration::from_secs(11)).await; + tokio::time::resume(); + let _ = late.send(FromProxyd::Ok); + // The failure path retries cleanup, but that also cannot prove drain. + let failure_cleanup = removals.recv().await.unwrap(); + failure_cleanup + .send(FromProxyd::Err("still draining".into())) + .unwrap(); + assert!(first.await.unwrap().is_err()); + assert!(host.networks.lock().unwrap().contains(generation)); + let stored = host.read_record(request.cold.sandbox_id).unwrap().unwrap(); + assert!( + stored.capture.unwrap().authority.is_none(), + "replacement authority must not precede cleanup" + ); + assert!(!matches!( + host.read(request.key.clone()).await.unwrap(), + ManagedBootHostState::Captured { .. } + )); + let h = host.clone(); + let c = capture.clone(); + let mut retry = tokio::spawn(async move { h.capture(c).await }); + let current = removals.recv().await.unwrap(); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(1), &mut retry) + .await + .is_err() + ); + assert!(host.networks.lock().unwrap().contains(generation)); + current.send(FromProxyd::Ok).unwrap(); + let authority = retry.await.unwrap().unwrap(); + assert_eq!(authority.request, capture); + assert!(!host.networks.lock().unwrap().contains(generation)); + assert_eq!(host.capture(capture).await.unwrap(), authority); + server.abort(); + let _ = server.await; + } + + #[tokio::test] + async fn uncertain_destroy_never_releases_an_address_on_retry() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let entry = install_exec_owner(&host, &request); + let mut owned = entry.lock().await; + let generation = reserve_test_network(&host, &mut owned); + if let Some(Owner::ExecTest(owner)) = &mut owned.owner { + owner.close_fails = true; + } + drop(owned); + for _ in 0..2 { + assert!(host + .destroy( + request.cold.sandbox_id, + engram_core::traits::SessionFence { + session_id: request.key.creation.session_id, + epoch: 1, + } + ) + .await + .is_err()); + assert!(host.networks.lock().unwrap().contains(generation)); + } + } + + #[tokio::test] + async fn recovered_record_cannot_release_current_incarnation_slot() { + let temp = tempfile::tempdir_in("/tmp").unwrap(); + let (host, request) = fixture(temp.path()); + let entry = install_exec_owner(&host, &request); + let mut entry = entry.lock().await; + let generation = reserve_test_network(&host, &mut entry); + entry.owner = None; + entry.record.incarnation = "old-host".into(); + entry.record.allocation.as_mut().unwrap().host_incarnation = "old-host".into(); + assert!( + host.close_resources(&mut entry, false).await.is_err(), + "recovery cannot assume missing cleanup proof" + ); + assert!(host.networks.lock().unwrap().contains(generation)); + let (server, mut removals, _) = cleanup_proxy(temp.path()); + let cleanup = host.close_resources(&mut entry, false); + let ack = async { + removals + .recv() + .await + .unwrap() + .send(engram_egress_proto::FromProxyd::Ok) + .unwrap(); + }; + let (result, ()) = tokio::join!(cleanup, ack); + result.unwrap(); + assert!(host.networks.lock().unwrap().contains(generation)); + server.abort(); + let _ = server.await; + } + + #[tokio::test] + async fn failed_materialization_removes_disposable_staging() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + assert!(host.prepare(request).await.is_err()); + assert!( + !host.directory.join(id.to_string()).exists(), + "failed prepare retained old staging" + ); + assert!( + std::fs::read_dir(&host.directory) + .unwrap() + .all(|entry| !entry.unwrap().file_type().unwrap().is_dir()), + "failed prepare retained staging" + ); + } + + #[tokio::test] + async fn cold_assets_hash_exact_sparse_disk_and_configured_files_without_owner() { + use engram_chunk_store::{ChunkRef, ChunkSize, Manifest, ManifestKind}; + use sha2::{Digest, Sha256}; + let temp = tempfile::tempdir().unwrap(); + let (host, boot) = fixture(temp.path()); + let mut drives = boot.cold.aux_ro_drives.clone(); + for (i, drive) in drives.iter_mut().enumerate() { + let bytes = vec![i as u8 + 1; 17]; + let hash = hex::encode(Sha256::digest(&bytes)); + std::fs::write(temp.path().join(AuxRoDrive::staged_file_name(&hash)), bytes).unwrap(); + drive.sha256 = Some(hash); + } + let mut manifest = Manifest::empty(ManifestKind::Disk, 15); + manifest.chunk_size = ChunkSize::bytes(4); + manifest.chunks = vec![ + ChunkRef { + offset: 4, + hash: host.fill.store().put_chunk(b"abcd").await.unwrap(), + }, + ChunkRef { + offset: 12, + hash: host.fill.store().put_chunk(b"xyz").await.unwrap(), + }, + ]; + host.fill + .store() + .put_manifest(boot.cold.disk.manifest, &manifest) + .await + .unwrap(); + let request = engram_core::types::managed_assets::ManagedColdAssetsRequest { + version: 1, + host_id: host.host_id, + session_id: boot.cold.session_id, + intent_id: "intent-a".into(), + image: boot.cold.image, + image_digest: format!("sha256:{}", "e".repeat(64)), + disk_manifest: boot.cold.disk.manifest, + aux_ro_drives: drives, + }; + let client = crate::host_client::LocalHostClient::new( + host.ordinary.clone(), + host.hub.clone(), + crate::IdleEvictionHarness::default(), + ) + .with_managed_boot(host.clone()); + let receipt = client + .read_managed_cold_assets(request.clone()) + .await + .unwrap(); + receipt.validate_for(&request).unwrap(); + assert_eq!( + receipt.disk_sha256, + hex::encode(Sha256::digest(b"\0\0\0\0abcd\0\0\0\0xyz")) + ); + assert_eq!(receipt.disk_size_bytes, 15); + assert_eq!( + receipt.kernel_sha256, + hex::encode(Sha256::digest(b"kernel")) + ); + assert_eq!(receipt.host_incarnation, "current-host"); + #[cfg(target_os = "macos")] + assert_eq!( + receipt.backend, + engram_core::types::managed_assets::ManagedColdBackend::Vz + ); + #[cfg(target_os = "linux")] + assert_eq!( + receipt.backend, + engram_core::types::managed_assets::ManagedColdBackend::Firecracker + ); + assert!(host.entries.is_empty()); + assert_eq!(host.networks.lock().unwrap().generation, 0); + assert!(host.ordinary.list().await.unwrap().is_empty()); + assert_eq!(std::fs::read_dir(&host.directory).unwrap().count(), 0); + std::fs::write(temp.path().join("kernel"), b"changed kernel").unwrap(); + let changed = client + .read_managed_cold_assets(request.clone()) + .await + .unwrap(); + assert_ne!(changed.kernel_sha256, receipt.kernel_sha256); + let mut wrong_host = request.clone(); + wrong_host.host_id = HostId::new(); + assert!(client.read_managed_cold_assets(wrong_host).await.is_err()); + let mut reordered = request.clone(); + reordered.aux_ro_drives.swap(0, 1); + assert!(client.read_managed_cold_assets(reordered).await.is_err()); + for field in 0..6 { + let mut invalid = request.clone(); + match field { + 0 => invalid.version = 2, + 1 => invalid.intent_id = "i".repeat(129), + 2 => invalid.image = "i".repeat(257), + 3 => invalid.aux_ro_drives.pop().map(|_| ()).unwrap(), + 4 => invalid.aux_ro_drives[0].sha256 = Some("../kernel".into()), + _ => invalid.disk_manifest.version = 0, + } + assert!(client.read_managed_cold_assets(invalid).await.is_err()); + } + let bundle = temp.path().join(AuxRoDrive::staged_file_name( + request.aux_ro_drives[1].sha256.as_ref().unwrap(), + )); + std::fs::write(&bundle, b"corrupt").unwrap(); + assert!(client + .read_managed_cold_assets(request.clone()) + .await + .is_err()); + std::fs::remove_file(bundle).unwrap(); + assert!(client.read_managed_cold_assets(request).await.is_err()); + } + + fn failed(request: ManagedBootPrepare) -> Record { + Record { + version: 1, + state: ManagedBootHostState::Failed { + key: request.key.clone(), + }, + request, + incarnation: "retired-host".into(), + allocation: None, + prepared: None, + applied: None, + apply_digest: None, + finalization: None, + capture: None, + retired: None, + } + } + + fn issued_start(request: &ManagedBootPrepare) -> ManagedBootStart { + ManagedBootStart { + receipt: ManagedBootFinalReceipt { + key: request.key.clone(), + prepared_receipt_id: digest(b"prepared"), + finalization_digest: digest(b"final"), + host_incarnation: "current-host".into(), + network_identity: digest(b"network"), + receipt_id: digest(b"receipt"), + }, + start_id: "issued-start".into(), + } + } + + #[tokio::test] + async fn managed_attach_requires_exact_private_owner_and_revokes_pending_and_live_streams() { + use engram_harness_proto::{read_msg, write_msg, HarnessAttach, HarnessAttachAck}; + use tokio::io::AsyncReadExt; + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let target = &request.key.context.target; + host.hub + .bind_session(target.session_id, target.sandbox_id, target.binding_epoch) + .unwrap(); + durable_write( + &host + .bindings + .directory() + .join(format!("{}.managed-boot", target.session_id)), + b"1", + ) + .unwrap(); + durable_write( + &host + .bindings + .directory() + .join(format!("{}.managed-sandbox", target.sandbox_id)), + b"1", + ) + .unwrap(); + let intent = issued_start(&request); + let mut record = failed(request.clone()); + record.state = ManagedBootHostState::StartPending { + intent: intent.clone(), + }; + host.persist(&record).unwrap(); + let entry = Arc::new(Mutex::new(Entry { + transfer_stop: tokio::sync::watch::channel(()).0, + close_failed: false, + record, + attach_owner: Some(host.hub.managed_owner(intent)), + owner: None, + agent: None, + })); + host.entries.insert(target.sandbox_id, entry.clone()); + let mut guard = entry.lock().await; + let sink = guard.attach_owner.as_ref().unwrap().sink(); + let exact = HarnessAttach { + session_id: target.session_id, + sandbox_id: target.sandbox_id, + binding_epoch: target.binding_epoch, + harness_version: "test".into(), + }; + // A valid token from another private source still cannot enter this owner. + let other_session = engram_core::SessionId::new(); + let other_sandbox = SandboxId::new(); + host.hub + .bind_session(other_session, other_sandbox, target.binding_epoch) + .unwrap(); + for wrong in [ + HarnessAttach { + session_id: other_session, + sandbox_id: other_sandbox, + ..exact.clone() + }, + HarnessAttach { + sandbox_id: other_sandbox, + ..exact.clone() + }, + HarnessAttach { + binding_epoch: target.binding_epoch + 1, + ..exact.clone() + }, + ] { + let (transport, mut guest) = tokio::io::duplex(4096); + sink(Box::pin(transport)); + write_msg(&mut guest, &wrong).await.unwrap(); + let ack: HarnessAttachAck = read_msg(&mut guest).await.unwrap(); + assert!(!ack.ok); + assert!(!host.hub.is_attached(target.sandbox_id)); + assert!(!host.hub.is_attached(other_sandbox)); + } + let (transport, mut live) = tokio::io::duplex(4096); + sink(Box::pin(transport)); + write_msg(&mut live, &exact).await.unwrap(); + let ack: HarnessAttachAck = + tokio::time::timeout(std::time::Duration::from_millis(100), read_msg(&mut live)) + .await + .unwrap() + .unwrap(); + assert!( + ack.ok, + "first attach must work while issued start holds its lifecycle lock" + ); + assert!(matches!( + guard.record.state, + ManagedBootHostState::StartPending { .. } + )); + assert!(host.hub.is_attached(target.sandbox_id)); + // A second accepted transport is waiting for Hello when the owner is lost. + let (transport, mut waiting) = tokio::io::duplex(4096); + sink(Box::pin(transport)); + tokio::task::yield_now().await; + host.fail(&mut guard).await; + assert!(!host.hub.is_attached(target.sandbox_id)); + for guest in [&mut live, &mut waiting] { + assert_eq!( + tokio::time::timeout( + std::time::Duration::from_millis(100), + guest.read(&mut [0u8; 1]) + ) + .await + .unwrap() + .unwrap(), + 0 + ); + } + // The sink outlives its owner but cannot issue another capability. + let (transport, mut late) = tokio::io::duplex(4096); + sink(Box::pin(transport)); + assert_eq!(late.read(&mut [0u8; 1]).await.unwrap(), 0); + } + + #[tokio::test] + async fn managed_attach_revocation_serializes_registration_and_preserves_replacement() { + use engram_harness_proto::{read_msg, write_msg, HarnessAttach, HarnessAttachAck}; + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let target = &request.key.context.target; + host.hub + .bind_session(target.session_id, target.sandbox_id, target.binding_epoch) + .unwrap(); + let intent = issued_start(&request); + let old = host.hub.managed_owner(intent.clone()); + let old_sink = old.sink(); + let (transport, mut old_guest) = tokio::io::duplex(4096); + old_sink(Box::pin(transport)); + let exact = HarnessAttach { + session_id: target.session_id, + sandbox_id: target.sandbox_id, + binding_epoch: target.binding_epoch, + harness_version: "test".into(), + }; + write_msg(&mut old_guest, &exact).await.unwrap(); + assert!( + read_msg::<_, HarnessAttachAck>(&mut old_guest) + .await + .unwrap() + .ok + ); + let replacement = host.hub.managed_owner(intent.clone()); + let (transport, mut new_guest) = tokio::io::duplex(4096); + replacement.sink()(Box::pin(transport)); + write_msg(&mut new_guest, &exact).await.unwrap(); + assert!( + read_msg::<_, HarnessAttachAck>(&mut new_guest) + .await + .unwrap() + .ok + ); + drop(old); + tokio::task::yield_now().await; + assert!( + host.hub.is_attached(target.sandbox_id), + "stale owner cleanup cannot remove replacement registration" + ); + drop(replacement); + assert!(!host.hub.is_attached(target.sandbox_id)); + // The ACK is blocked after registration. Revocation must remove the + // entry immediately and cut the pending write, not wait for the peer. + let pending = host.hub.managed_owner(intent); + let (transport, mut pending_guest) = tokio::io::duplex(1); + pending.sink()(Box::pin(transport)); + write_msg(&mut pending_guest, &exact).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_millis(100), async { + while !host.hub.is_attached(target.sandbox_id) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + drop(pending); + assert!(!host.hub.is_attached(target.sandbox_id)); + assert!(tokio::time::timeout( + std::time::Duration::from_millis(100), + read_msg::<_, HarnessAttachAck>(&mut pending_guest) + ) + .await + .unwrap() + .is_err()); + } + + #[tokio::test] + async fn managed_inventory_busy_is_unknown_without_waiting_for_lifecycle() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + let mut record = failed(request.clone()); + record.state = ManagedBootHostState::Prepared { + receipt: Box::new(ManagedBootPreparedReceipt { + request, + host_incarnation: host.incarnation.clone(), + network_identity: digest(b"network"), + guest_ip: "192.0.2.2".parse().unwrap(), + receipt_id: digest(b"prepared"), + }), + }; + let entry = Arc::new(Mutex::new(Entry { + transfer_stop: tokio::sync::watch::channel(()).0, + close_failed: false, + record, + attach_owner: None, + owner: None, + agent: None, + })); + host.entries.insert(id, entry.clone()); + let local = crate::host_client::LocalHostClient::new( + host.ordinary.clone(), + host.hub.clone(), + crate::IdleEvictionHarness::default(), + ) + .with_managed_boot(host.clone()); + let guard = entry.lock().await; + let inventory = tokio::time::timeout(std::time::Duration::from_millis(100), local.list()) + .await + .expect("heartbeat inventory must not wait for lifecycle work"); + assert!( + inventory.is_err(), + "busy is unknown, never an empty known inventory" + ); + drop(guard); + assert!(local.list().await.unwrap().contains(&id)); + } + + #[tokio::test] + async fn managed_disk_reservation_covers_swap_create_and_both_restores() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let capture = ManagedCaptureRequest { + key: request.key.clone(), + capture_id: engram_core::SnapshotId::new(), + }; + let mut record = failed(request.clone()); + record.capture = Some(CaptureProgress { + request: capture.clone(), + manifest: capture.disk_manifest(), + artifact: None, + authority: None, + }); + host.persist(&record).unwrap(); + let local = crate::host_client::LocalHostClient::new( + host.ordinary.clone(), + host.hub.clone(), + crate::IdleEvictionHarness::default(), + ) + .with_managed_boot(host.clone()); + let mut ordinary: engram_core::types::sandbox::SandboxSpec = + serde_json::from_str(&request.sandbox_spec_json).unwrap(); + ordinary.rootfs_manifest = Some(ManifestRef::new()); + ordinary.rootfs_source = Some(temp.path().join("ordinary.ext4")); + host.ordinary_spec(&ordinary).unwrap(); + let rejected = |result: Result| { + assert!( + matches!(result, Err(SandboxError::InvalidSpec(message)) if message == "managed boot ownership changed or unavailable") + ); + }; + for protected in [request.cold.disk.manifest, capture.disk_manifest()] { + let mut spec = ordinary.clone(); + spec.swap_manifest = Some(protected); + rejected(local.create(spec).await); + let metadata: engram_core::types::snapshot::SnapshotMetadata = + serde_json::from_value(serde_json::json!({ + "id": engram_core::SnapshotId::new(), "size_bytes": 4, + "created_at": "2026-01-01T00:00:00Z", "image_version": "image-v1", + "disk_manifest": ordinary.rootfs_manifest, "swap_manifest": protected + })) + .unwrap(); + let fence = engram_core::traits::SessionFence { + session_id: engram_core::SessionId::new(), + epoch: 1, + }; + rejected(local.restore(metadata.clone(), fence).await); + rejected( + local + .restore_base_for_session(metadata, Default::default(), vec![], fence) + .await, + ); + } + for path in [ + host.directory.join("private.ext4"), + temp.path().join(format!( + "managed-{}-{}/rootfs.ext4", + request.key.context.target.sandbox_id, request.key.boot_attempt + )), + temp.path().join(format!( + "m-{}/rootfs.ext4", + &digest( + format!( + "{}\0{}", + request.key.context.target.sandbox_id, request.key.boot_attempt + ) + .as_bytes() + )[7..31] + )), + ] { + let mut spec = ordinary.clone(); + spec.swap_source = Some(path); + rejected(local.create(spec).await); + } + } + + #[tokio::test] + async fn durable_managed_identity_blocks_generic_paths_after_owner_loss() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let key = request.key.clone(); + let id = key.context.target.sandbox_id; + host.persist(&failed(request.clone())).unwrap(); + assert!(host + .ordinary_disk(Some(request.cold.disk.manifest), None) + .is_err()); + assert!(host + .ordinary_disk(None, Some(&host.directory.join("private.ext4"))) + .is_err()); + durable_write( + &host + .bindings + .directory() + .join(format!("{id}.managed-sandbox")), + b"1", + ) + .unwrap(); + durable_write( + &host + .bindings + .directory() + .join(format!("{}.managed-boot", key.creation.session_id)), + id.to_string().as_bytes(), + ) + .unwrap(); + let local = crate::host_client::LocalHostClient::new( + host.ordinary.clone(), + host.hub.clone(), + crate::IdleEvictionHarness::default(), + ) + .with_managed_boot(host.clone()); + assert!(matches!( + host.read(key.clone()).await.unwrap(), + ManagedBootHostState::Failed { .. } + )); + assert!(host.prepare(request).await.is_err()); + assert!(local + .pause( + id, + engram_core::traits::SessionFence { + session_id: key.creation.session_id, + epoch: 1 + } + ) + .await + .is_err()); + assert!(local + .resume( + id, + engram_core::traits::SessionFence { + session_id: key.creation.session_id, + epoch: 1 + } + ) + .await + .is_err()); + assert!(local + .snapshot( + id, + engram_core::traits::SessionFence { + session_id: key.creation.session_id, + epoch: 1 + } + ) + .await + .is_err()); + assert!(local + .bind_session(key.creation.session_id, id, 1) + .await + .is_err()); + assert!(local.interrupt(id).await.is_err()); + let error = local + .migration_abort_unserved( + id, + "unserved-export", + engram_core::traits::SessionFence::new(key.creation.session_id, 1), + ) + .await + .unwrap_err(); + assert!(matches!(error, SandboxError::InvalidSpec(message) + if message == "managed session requires its current boot owner")); + assert!(local.list().await.unwrap().is_empty()); + let mut wrong = key; + wrong.boot_attempt = "other".into(); + assert!(host.read(wrong).await.is_err()); + } + + #[tokio::test] + async fn durable_capture_retries_exact_artifact_after_lost_manifest_reply() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let capture = ManagedCaptureRequest { + key: request.key.clone(), + capture_id: engram_core::SnapshotId::new(), + }; + let path = host.directory.join(format!("{}.disk", capture.capture_id)); + durable_write(&path, b"retired disk cut").unwrap(); + let artifact = hash_file(&path).await.unwrap(); + let mut record = failed(request); + record.capture = Some(CaptureProgress { + request: capture.clone(), + manifest: capture.disk_manifest(), + artifact: Some(artifact.clone()), + authority: None, + }); + host.persist(&record).unwrap(); + let manifest = host + .fill + .store() + .chunk_file(&path, engram_chunk_store::ManifestKind::Disk, None) + .await + .unwrap(); + // The prior call wrote the manifest, then lost its reply before receipt persistence. + host.fill + .store() + .put_manifest(capture.disk_manifest(), &manifest) + .await + .unwrap(); + let authority = host.capture(capture.clone()).await.unwrap(); + assert_eq!( + (authority.disk_sha256.clone(), authority.disk_size_bytes), + artifact + ); + assert_eq!(authority.host_incarnation, "retired-host"); + assert_eq!(host.capture(capture.clone()).await.unwrap(), authority); + assert_eq!( + host.read(capture.key.clone()).await.unwrap(), + ManagedBootHostState::Captured { + authority: authority.clone() + } + ); + let mut changed = capture.clone(); + changed.capture_id = engram_core::SnapshotId::new(); + assert!(host.capture(changed).await.is_err()); + let mut spec = record.request.spec().unwrap(); + spec.rootfs_manifest = Some(capture.disk_manifest()); + assert!(host.ordinary_spec(&spec).is_err()); + } + + #[tokio::test] + async fn capture_never_adopts_unrecorded_or_mutated_disk_path() { + for recorded in [false, true] { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let capture = ManagedCaptureRequest { + key: request.key.clone(), + capture_id: engram_core::SnapshotId::new(), + }; + let path = host.directory.join(format!("{}.disk", capture.capture_id)); + durable_write(&path, b"old").unwrap(); + let mut record = failed(request); + record.capture = Some(CaptureProgress { + request: capture.clone(), + manifest: capture.disk_manifest(), + artifact: if recorded { + Some(hash_file(&path).await.unwrap()) + } else { + None + }, + authority: None, + }); + host.persist(&record).unwrap(); + durable_write(&path, b"new").unwrap(); + assert!(host.capture(capture.clone()).await.is_err()); + assert!(host + .fill + .store() + .get_manifest(capture.disk_manifest()) + .await + .is_err()); + } + } + + #[tokio::test] + async fn retired_source_survives_restart_and_pinned_capture_without_old_attach() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let key = request.key.clone(); + let id = key.context.target.sandbox_id; + let mut record = failed(request.clone()); + let prepared = ManagedBootPreparedReceipt { + request, + host_incarnation: record.incarnation.clone(), + network_identity: digest(b"network"), + guest_ip: "192.0.2.2".parse().unwrap(), + receipt_id: digest(b"prepared"), + }; + let retired = ManagedBootRetiredReceipt { + prepared: prepared.clone(), + retained_policy: ManagedDenyPolicy::new( + 2, + ["mcp.example".into(), "new.example".into()], + ) + .unwrap(), + receipt_id: digest(b"retirement"), + }; + record.prepared = Some(prepared); + record.state = ManagedBootHostState::Retired { + receipt: Box::new(retired.clone()), + }; + let path = host.directory.join(format!("{id}.retired")); + durable_write(&path, b"retained source disk").unwrap(); + record.retired = Some(hash_file(&path).await.unwrap()); + host.persist(&record).unwrap(); + assert_eq!(host.read(key.clone()).await.unwrap(), record.state); + assert!(host.guard_running(id).await.is_err()); + assert!(host.prepare(record.request.clone()).await.is_err()); + let next = SandboxId::new(); + host.hub + .bind_session(key.creation.session_id, next, 2) + .unwrap(); + host.unbind_owned(&key); + assert_eq!(host.hub.bound_sandbox(key.creation.session_id), Some(next)); + let capture = ManagedCaptureRequest { + key, + capture_id: engram_core::SnapshotId::new(), + }; + let authority = host.capture(capture.clone()).await.unwrap(); + assert_eq!(authority.disk_sha256, record.retired.unwrap().0); + assert_eq!(host.capture(capture).await.unwrap(), authority); + assert!( + path.is_file(), + "retirement disk is retained until explicit cleanup" + ); + } + + #[tokio::test] + async fn bootstrap_retirement_cannot_cross_a_start_intent_or_same_generation() { + let temp = tempfile::tempdir().unwrap(); + let (host, prepare) = fixture(temp.path()); + let key = prepare.key.clone(); + let id = key.context.target.sandbox_id; + let mut record = failed(prepare.clone()); + record.incarnation = host.incarnation.clone(); + let receipt = ManagedBootPreparedReceipt { + request: prepare, + host_incarnation: host.incarnation.clone(), + network_identity: digest(b"network"), + guest_ip: "192.0.2.2".parse().unwrap(), + receipt_id: digest(b"prepared"), + }; + let allocation = Allocation { + version: 1, + host_id: host.host_id, + host_incarnation: host.incarnation.clone(), + proxy_incarnation: "proxy".into(), + session_id: key.creation.session_id, + sandbox_id: id, + boot_attempt: key.boot_attempt.clone(), + attachment_generation: 1, + guest_ip: receipt.guest_ip, + gateway_ip: "192.0.2.1".parse().unwrap(), + policy_revision: 1, + policy_digest: record.request.bootstrap_policy.digest.clone(), + }; + record.prepared = Some(receipt.clone()); + record.allocation = Some(allocation.clone()); + record.state = ManagedBootHostState::Prepared { + receipt: Box::new(receipt.clone()), + }; + let mut request = ManagedPolicyApply { + expected: None, + context: key.context.clone(), + egress: bootstrap(&allocation, record.request.bootstrap_policy.clone()), + }; + assert!(!bootstrap_retirement_allowed(&record, &request)); + request.context.operator_generation += 1; + assert!( + bootstrap_retirement_allowed(&record, &request), + "a later generation may republish the same identity" + ); + let mut changed = request.clone(); + changed.context.binding_hash = Some(digest(b"binding")); + assert!(!bootstrap_retirement_allowed(&record, &changed)); + changed = request.clone(); + changed.egress.managed_deny = + Some(ManagedDenyPolicy::new(2, ["other.example".into()]).unwrap()); + assert!(!bootstrap_retirement_allowed(&record, &changed)); + let entry = Arc::new(Mutex::new(Entry { + transfer_stop: tokio::sync::watch::channel(()).0, + close_failed: false, + record, + attach_owner: None, + owner: None, + agent: None, + })); + host.entries.insert(id, entry.clone()); + let mut guard = entry.lock().await; + let pending = ManagedBootHostState::StartPending { + intent: ManagedBootStart { + receipt: ManagedBootFinalReceipt { + key: key.clone(), + prepared_receipt_id: receipt.receipt_id, + finalization_digest: digest(b"final"), + host_incarnation: host.incarnation.clone(), + network_identity: receipt.network_identity, + receipt_id: digest(b"final receipt"), + }, + start_id: "start".into(), + }, + }; + let caller = host.clone(); + let race = tokio::spawn(async move { caller.apply(request).await }); + tokio::task::yield_now().await; + // The actual start method uses this same lock before persisting intent. + guard.record.state = pending.clone(); + drop(guard); + assert!(race.await.unwrap().is_err()); + assert_eq!(entry.lock().await.record.state, pending); + assert!(!host.directory.join(format!("{id}.retired")).exists()); + } + + #[tokio::test] + async fn later_policy_apply_preserves_exact_retired_proof() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let key = request.key.clone(); + let id = key.context.target.sandbox_id; + let mut record = failed(request.clone()); + record.incarnation = host.incarnation.clone(); + let prepared = ManagedBootPreparedReceipt { + request, + host_incarnation: host.incarnation.clone(), + network_identity: digest(b"network"), + guest_ip: "192.0.2.2".parse().unwrap(), + receipt_id: digest(b"prepared"), + }; + let allocation = Allocation { + version: 1, + host_id: host.host_id, + host_incarnation: host.incarnation.clone(), + proxy_incarnation: "proxy".into(), + session_id: key.creation.session_id, + sandbox_id: id, + boot_attempt: key.boot_attempt.clone(), + attachment_generation: 1, + guest_ip: prepared.guest_ip, + gateway_ip: "192.0.2.1".parse().unwrap(), + policy_revision: 1, + policy_digest: record.request.bootstrap_policy.digest.clone(), + }; + let mut context = key.context.clone(); + context.binding_hash = Some(digest(b"binding")); + let applied = ManagedAppliedState::new( + context.clone(), + record.request.bootstrap_policy.clone(), + host.incarnation.clone(), + "proxy".into(), + Some(ManagedNetworkProof { + target: key.context.target.clone(), + guest_ip: prepared.guest_ip, + network_identity: prepared.network_identity.clone(), + }), + ) + .unwrap(); + let retained = + ManagedDenyPolicy::new(2, ["mcp.example".into(), "new.example".into()]).unwrap(); + record.state = ManagedBootHostState::Retired { + receipt: Box::new(ManagedBootRetiredReceipt { + prepared: prepared.clone(), + retained_policy: retained, + receipt_id: digest(b"retired"), + }), + }; + record.prepared = Some(prepared); + record.allocation = Some(allocation.clone()); + record.applied = Some(applied.clone()); + let path = host.directory.join(format!("{id}.retired")); + durable_write(&path, b"exact retained disk").unwrap(); + record.retired = Some(hash_file(&path).await.unwrap()); + let expected = record.state.clone(); + host.persist(&record).unwrap(); + host.entries.insert( + id, + Arc::new(Mutex::new(Entry { + transfer_stop: tokio::sync::watch::channel(()).0, + close_failed: false, + record, + attach_owner: None, + owner: None, + agent: None, + })), + ); + for revision in [2, 3] { + let request = ManagedPolicyApply { + expected: Some(applied.clone()), + context: context.clone(), + egress: bootstrap( + &allocation, + ManagedDenyPolicy::new( + revision, + [ + "mcp.example".into(), + "new.example".into(), + "later.example".into(), + ], + ) + .unwrap(), + ), + }; + assert!(matches!( + host.apply(request).await, + Err(SandboxError::ManagedRecoveryRequired) + )); + assert_eq!(host.read(key.clone()).await.unwrap(), expected); + assert!(host.read_policy(key.context.target.clone()).await.is_err()); + assert_eq!(std::fs::read(&path).unwrap(), b"exact retained disk"); + } + host.destroy( + id, + engram_core::traits::SessionFence::new(key.creation.session_id, 1), + ) + .await + .unwrap(); + host.destroy_tombstoned(id).await.unwrap(); + assert_eq!(host.read(key).await.unwrap(), expected); + } + + #[test] + fn torn_host_records_never_authorize_an_ordinary_fallback() { + let temp = tempfile::tempdir().unwrap(); + let (host, request) = fixture(temp.path()); + let id = request.key.context.target.sandbox_id; + host.persist(&failed(request)).unwrap(); + let path = host.path(id); + let bytes = std::fs::read(&path).unwrap(); + for end in 0..bytes.len() { + std::fs::write(&path, &bytes[..end]).unwrap(); + assert!(host.read_record(id).is_err(), "offset {end}"); + } + std::fs::write(&path, &bytes).unwrap(); + std::fs::write(path.with_extension("partial"), b"garbage sibling").unwrap(); + assert!(host.read_record(id).unwrap().is_some()); + } +} diff --git a/crates/engram-host-agent/src/managed_policy.rs b/crates/engram-host-agent/src/managed_policy.rs new file mode 100644 index 000000000..3f00dc676 --- /dev/null +++ b/crates/engram-host-agent/src/managed_policy.rs @@ -0,0 +1,916 @@ +//! Durable managed policy application. It does not start a guest. + +use crate::{bindings::BindingStore, egress::HostEgress}; +use engram_core::{ + traits::SandboxBackend, types::managed_deny::*, HostId, SandboxError, SessionId, +}; +use serde::{Deserialize, Serialize}; +use std::{io::Write, sync::Arc}; + +#[derive(Clone, Serialize, Deserialize)] +pub(crate) struct ManagedRecord { + version: u32, + context: ManagedPolicyContext, + pub(crate) egress: engram_core::types::egress::SessionEgressPolicy, + applied: Option, + expected: Option, + accepted_host_incarnation: String, + accepted_proxy_incarnation: String, +} +fn invalid(e: impl std::fmt::Display) -> SandboxError { + SandboxError::InvalidSpec(e.to_string()) +} +fn io_invalid(e: impl std::fmt::Display) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()) +} +fn write_atomic( + store: &BindingStore, + sid: SessionId, + suffix: &str, + bytes: &[u8], +) -> std::io::Result<()> { + let path = store.directory().join(format!("{sid}.{suffix}")); + let temp = path.with_extension(format!("{suffix}.partial")); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let mut file = opts.open(&temp)?; + file.write_all(bytes)?; + file.sync_all()?; + std::fs::rename(temp, path)?; + std::fs::File::open(store.directory())?.sync_all() +} +fn persist(store: &BindingStore, record: &ManagedRecord) -> std::io::Result<()> { + let sid = record.context.target.session_id; + // This completeness marker makes a missing record distinguishable from no policy. + write_atomic(store, sid, "managed-required", b"1")?; + let body = serde_json::to_string(record).map_err(io_invalid)?; + write_atomic( + store, + sid, + "managed", + &crate::durable_envelope::seal(&sid.to_string(), &body), + ) +} +pub(crate) fn read_record( + store: &BindingStore, + sid: SessionId, +) -> std::io::Result> { + let path = store.directory().join(format!("{sid}.managed")); + let bytes = match std::fs::read(path) { + Ok(v) => v, + Err(e) + if e.kind() == std::io::ErrorKind::NotFound + && !store + .directory() + .join(format!("{sid}.managed-required")) + .exists() => + { + return Ok(None) + } + Err(e) => return Err(e), + }; + if std::fs::read(store.directory().join(format!("{sid}.managed-required")))? != b"1" { + return Err(io_invalid("invalid managed completeness marker")); + } + let body = crate::durable_envelope::open(&bytes, &sid.to_string()).map_err(io_invalid)?; + let record: ManagedRecord = serde_json::from_slice(&body).map_err(io_invalid)?; + record.context.validate().map_err(io_invalid)?; + if record.version != 1 + || record.context.target.session_id != sid + || record.egress.session_id != sid + || record.egress.sandbox_id != record.context.target.sandbox_id + { + return Err(io_invalid("invalid managed record identity")); + } + record + .egress + .managed_deny + .as_ref() + .ok_or_else(|| io_invalid("missing managed deny"))? + .validate() + .map_err(io_invalid)?; + if let Some(applied) = &record.applied { + applied.validate().map_err(io_invalid)?; + if applied.context != record.context + || Some(&applied.policy) != record.egress.managed_deny.as_ref() + { + return Err(io_invalid( + "managed applied marker differs from desired policy", + )); + } + } + Ok(Some(record)) +} + +/// Exclusive ownership of this directory. The stable lock file is also the +/// durable marker that prevents a later plaintext host from reopening it. +pub struct ManagedOwnership { + _file: std::fs::File, + directory: std::path::PathBuf, +} +impl ManagedOwnership { + pub fn for_mode( + directory: &std::path::Path, + managed: bool, + ) -> std::io::Result>> { + let path = directory.join("managed-owner.lock"); + if !managed { + if path.exists() { + return Err(io_invalid( + "managed host directory requires authenticated mode", + )); + } + return Ok(None); + } + std::fs::create_dir_all(directory)?; + let mut options = std::fs::OpenOptions::new(); + options.read(true).write(true).create(true).truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_CLOEXEC); + } + let file = options.open(path)?; + file.try_lock().map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "managed host directory is already owned", + ) + })?; + file.sync_all()?; + std::fs::File::open(directory)?.sync_all()?; + Ok(Some(Arc::new(Self { + _file: file, + directory: directory.canonicalize()?, + }))) + } +} + +pub struct ManagedPolicyHost { + _ownership: Arc, + host_id: HostId, + incarnation: String, + bindings: BindingStore, + backend: Arc, + egress: Arc, + locks: dashmap::DashMap>>, +} +impl ManagedPolicyHost { + pub fn new( + host_id: HostId, + incarnation: String, + bindings: BindingStore, + backend: Arc, + egress: Arc, + ownership: Arc, + ) -> std::io::Result { + if ownership.directory != bindings.directory().canonicalize()? { + return Err(io_invalid("managed ownership directory mismatch")); + } + Ok(Self { + _ownership: ownership, + host_id, + incarnation, + bindings, + backend, + egress, + locks: dashmap::DashMap::new(), + }) + } + /// Quarantine trusted proxy registrations before reading any durable policy. + /// Ordinary replay must run only after this succeeds. + pub async fn recover_all(&self) -> Result<(), SandboxError> { + let handle = self.egress.handle(); + let known: std::collections::BTreeSet<_> = handle + .managed_sessions() + .await + .map_err(invalid)? + .into_iter() + .collect(); + for sid in &known { + handle.quarantine_managed(*sid).await.map_err(invalid)?; + } + let mut sessions = known.clone(); + for entry in std::fs::read_dir(self.bindings.directory()).map_err(invalid)? { + let path = entry.map_err(invalid)?.path(); + if matches!( + path.extension().and_then(|s| s.to_str()), + Some("managed" | "managed-required" | "managed-boot") + ) { + sessions.insert( + path.file_stem() + .and_then(|s| s.to_str()) + .ok_or_else(|| invalid("invalid managed filename"))? + .parse::() + .map_err(invalid)?, + ); + } + } + for sid in &sessions { + handle.quarantine_managed(*sid).await.map_err(invalid)?; + } + // Validate the complete set before reopening any session. + let mut records = Vec::new(); + for sid in &sessions { + let sid = *sid; + // A boot owner never survives a host incarnation. Keep its proxy + // quarantined; typed boot recovery validates retained disk records. + if self + .bindings + .directory() + .join(format!("{sid}.managed-boot")) + .try_exists() + .map_err(invalid)? + { + continue; + } + let record = read_record(&self.bindings, sid) + .map_err(invalid)? + .ok_or_else(|| invalid("managed record missing"))?; + if !known.contains(&sid) && self.bindings.read(sid).map_err(invalid)?.is_none() { + continue; + } + self.check_record(&record).await?; + records.push(record); + } + for record in records { + if let Err(error) = self.read_applied(&record.context.target).await { + for sid in &sessions { + handle.quarantine_managed(*sid).await.map_err(invalid)?; + } + return Err(error); + } + } + Ok(()) + } + async fn check_target(&self, target: &ManagedPolicyTarget) -> Result<(), SandboxError> { + if target.host_id != self.host_id { + return Err(invalid("managed host identity mismatch")); + } + let binding = self + .bindings + .read(target.session_id) + .map_err(invalid)? + .ok_or_else(|| invalid("managed session binding absent"))?; + if binding.sandbox_id != target.sandbox_id || binding.binding_epoch != target.binding_epoch + { + return Err(invalid("managed physical binding changed")); + } + let endpoint = self + .backend + .guest_endpoints(target.sandbox_id) + .await + .ok_or_else(|| invalid("managed guest identity absent"))?; + if endpoint.egress_identity.is_unspecified() { + return Err(invalid("managed guest identity absent")); + } + Ok(()) + } + async fn check_record( + &self, + record: &ManagedRecord, + ) -> Result, SandboxError> { + self.check_target(&record.context.target).await?; + let endpoint = self + .backend + .guest_endpoints(record.context.target.sandbox_id) + .await + .ok_or_else(|| invalid("managed guest identity absent"))?; + if endpoint.egress_identity != record.egress.guest_ip { + return Err(invalid("managed guest identity mismatch")); + } + if !record.egress.tunnels.is_empty() { + return Err(invalid("managed policy does not support tunnels")); + } + let proof = self + .backend + .managed_network_proof(&record.context.target) + .await; + if record.context.binding_hash.is_some() && proof.is_none() { + return Err(invalid("strict managed network capability unavailable")); + } + if proof.as_ref().is_some_and(|p| { + p.target != record.context.target + || p.guest_ip != record.egress.guest_ip + || !is_digest(&p.network_identity) + }) { + return Err(invalid("strict managed network identity mismatch")); + } + Ok(proof) + } + async fn replay(&self, mut record: ManagedRecord) -> Result { + let proof = self.check_record(&record).await?; + let sid = record.context.target.session_id; + let proxy = self + .egress + .handle() + .read_managed(sid) + .await + .map_err(invalid)?; + let matches = proxy.as_ref().is_some_and(|p| { + p.sandbox_id == record.egress.sandbox_id + && p.guest_ip == record.egress.guest_ip + && Some(&p.policy) == record.egress.managed_deny.as_ref() + }); + if let Some(applied) = &record.applied { + if matches + && applied.network_proof == proof + && applied.host_incarnation == self.incarnation + && proxy + .as_ref() + .is_some_and(|p| p.incarnation == applied.proxy_incarnation) + { + return Ok(applied.clone()); + } + } + self.backend + .notify_session_policy(record.egress.clone()) + .await?; + let actual = self + .egress + .handle() + .read_managed(sid) + .await + .map_err(invalid)? + .ok_or_else(|| invalid("managed policy not installed"))?; + if actual.sandbox_id != record.egress.sandbox_id + || actual.guest_ip != record.egress.guest_ip + || Some(&actual.policy) != record.egress.managed_deny.as_ref() + { + return Err(invalid("managed proxy acknowledgement mismatch")); + } + self.check_target(&record.context.target).await?; + let applied = ManagedAppliedState::new( + record.context.clone(), + actual.policy, + self.incarnation.clone(), + actual.incarnation, + proof, + ) + .map_err(invalid)?; + record.applied = Some(applied.clone()); + persist(&self.bindings, &record).map_err(invalid)?; + Ok(applied) + } + async fn read_inner( + &self, + target: &ManagedPolicyTarget, + ) -> Result, SandboxError> { + self.check_target(target).await?; + match read_record(&self.bindings, target.session_id).map_err(invalid)? { + Some(record) => { + if &record.context.target != target { + return Err(invalid("managed policy belongs to another placement")); + } + self.replay(record).await.map(Some) + } + None => Ok(None), + } + } + pub async fn read_applied( + &self, + target: &ManagedPolicyTarget, + ) -> Result, SandboxError> { + let lock = self.locks.entry(target.session_id).or_default().clone(); + let _guard = lock.lock().await; + let result = self.read_inner(target).await; + if result.is_err() { + let _ = self + .egress + .handle() + .quarantine_managed(target.session_id) + .await; + } + result + } + pub async fn compare_and_apply( + &self, + request: ManagedPolicyApply, + ) -> Result { + request.context.validate().map_err(invalid)?; + if !request.egress.tunnels.is_empty() { + return Err(invalid("managed policy does not support tunnels")); + } + let target = request.context.target.clone(); + let lock = self.locks.entry(target.session_id).or_default().clone(); + let _guard = lock.lock().await; + self.check_target(&target).await?; + let policy = request + .egress + .managed_deny + .as_ref() + .ok_or_else(|| invalid("managed policy missing"))?; + policy.validate().map_err(invalid)?; + let endpoint = self + .backend + .guest_endpoints(target.sandbox_id) + .await + .ok_or_else(|| invalid("managed guest identity absent"))?; + if request.egress.session_id != target.session_id + || request.egress.sandbox_id != target.sandbox_id + || request.egress.guest_ip != endpoint.egress_identity + { + return Err(invalid("managed guest identity mismatch")); + } + if let Some(expected) = &request.expected { + expected.validate().map_err(invalid)?; + } + let current = match self.read_inner(&target).await { + Ok(value) => value, + Err(error) => { + let _ = self + .egress + .handle() + .quarantine_managed(target.session_id) + .await; + return Err(error); + } + }; + if let Some(state) = ¤t { + // An exact lost-response retry has already committed this candidate. + if state.context == request.context && &state.policy == policy { + let record = read_record(&self.bindings, target.session_id) + .map_err(invalid)? + .ok_or_else(|| invalid("managed record absent"))?; + if request.expected == current + || (request.expected == record.expected + && state.host_incarnation == record.accepted_host_incarnation + && state.proxy_incarnation == record.accepted_proxy_incarnation) + { + return Ok(state.clone()); + } + return Err(invalid("managed expected state changed")); + } + } + if current != request.expected { + return Err(invalid("managed expected state changed")); + } + if let Some(old) = ¤t { + if policy.revision <= old.policy.revision + || !policy.contains(&old.policy) + || (request.context.operator_generation < old.context.operator_generation + || (request.context.operator_generation == old.context.operator_generation + && request.context.operator_identity != old.context.operator_identity)) + { + return Err(invalid("managed policy regressed")); + } + } + let proxy_incarnation = self + .egress + .handle() + .hello() + .await + .map_err(invalid)? + .incarnation; + let record = ManagedRecord { + accepted_host_incarnation: self.incarnation.clone(), + accepted_proxy_incarnation: proxy_incarnation, + version: 1, + context: request.context, + egress: request.egress, + applied: None, + expected: request.expected, + }; + persist(&self.bindings, &record).map_err(invalid)?; + match self.replay(record).await { + Ok(state) => Ok(state), + Err(error) => { + let _ = self + .egress + .handle() + .quarantine_managed(target.session_id) + .await; + Err(error) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn torn_managed_record_never_becomes_absent() { + let dir = tempfile::tempdir().unwrap(); + let sid = engram_core::SessionId::new(); + let store = crate::bindings::BindingStore::open(dir.path()).unwrap(); + let target = ManagedPolicyTarget { + session_id: sid, + sandbox_id: engram_core::SandboxId::new(), + host_id: HostId::new(), + binding_epoch: 1, + }; + let record = ManagedRecord { + version: 1, + context: ManagedPolicyContext { + target: target.clone(), + operator_generation: 1, + operator_identity: format!("sha256:{}", "a".repeat(64)), + binding_hash: None, + }, + egress: super::live_tests::egress(&target, 1, &["mcp.example"]), + applied: None, + expected: None, + accepted_host_incarnation: "host-a".into(), + accepted_proxy_incarnation: "proxy-a".into(), + }; + let bytes = crate::durable_envelope::seal( + &sid.to_string(), + &serde_json::to_string(&record).unwrap(), + ); + let marker = dir.path().join(format!("{sid}.managed-required")); + let path = dir.path().join(format!("{sid}.managed")); + std::fs::write(&marker, b"1").unwrap(); + assert!(read_record(&store, sid).is_err()); + for end in 0..bytes.len() { + std::fs::write(&path, &bytes[..end]).unwrap(); + assert!(read_record(&store, sid).is_err(), "offset {end}"); + } + std::fs::write(&path, &bytes).unwrap(); + assert!(read_record(&store, sid).unwrap().is_some()); + std::fs::write( + dir.path().join(format!("{sid}.managed.partial")), + b"garbage", + ) + .unwrap(); + assert!( + read_record(&store, sid).unwrap().is_some(), + "incomplete sibling cannot replace committed policy" + ); + std::fs::remove_file(&marker).unwrap(); + assert!( + read_record(&store, sid).is_err(), + "missing completeness marker cannot be accepted" + ); + } +} + +#[cfg(test)] +mod live_tests { + use super::*; + use engram_core::{ + types::{ + egress::SessionEgressPolicy, + endpoints::GuestEndpoints, + image::SecretMode, + sandbox::{ExecRequest, ExecStream, SandboxSpec}, + snapshot::SnapshotMetadata, + }, + SandboxId, SnapshotId, + }; + struct EndpointFixture { + egress: Arc, + } + #[async_trait::async_trait] + impl SandboxBackend for EndpointFixture { + fn captures_guest_memory(&self) -> bool { + false + } + async fn create(&self, _: SandboxId, _: SandboxSpec) -> Result<(), SandboxError> { + Err(invalid("unused fixture")) + } + async fn exec_stream( + &self, + _: SandboxId, + _: ExecRequest, + ) -> Result { + Err(invalid("unused fixture")) + } + async fn snapshot(&self, _: SandboxId) -> Result { + Err(invalid("unused fixture")) + } + fn snapshot_path_for(&self, _: SnapshotId) -> std::path::PathBuf { + std::path::PathBuf::new() + } + async fn restore( + &self, + _: SandboxId, + _: SnapshotMetadata, + _: Option, + ) -> Result<(), SandboxError> { + Err(invalid("unused fixture")) + } + async fn destroy(&self, _: SandboxId) -> Result<(), SandboxError> { + Ok(()) + } + async fn list(&self) -> Result, SandboxError> { + Ok(Vec::new()) + } + async fn guest_endpoints(&self, _: SandboxId) -> Option { + Some(GuestEndpoints { + egress_identity: "10.200.0.42".parse().unwrap(), + dial_ip: "10.200.0.42".parse().unwrap(), + netns: None, + vsock_uds: None, + }) + } + async fn notify_session_policy( + &self, + policy: SessionEgressPolicy, + ) -> Result<(), SandboxError> { + self.egress.apply_policy(policy).await.map_err(invalid) + } + } + pub(super) fn egress( + target: &ManagedPolicyTarget, + revision: u64, + hosts: &[&str], + ) -> SessionEgressPolicy { + SessionEgressPolicy { + session_id: target.session_id, + sandbox_id: target.sandbox_id, + guest_ip: "10.200.0.42".parse().unwrap(), + network_allow_hosts: vec![], + network_allow_host_patterns: vec![], + allow_all: true, + secrets: vec![], + injects: vec![], + observes: vec![], + guest_services: vec![], + tunnels: vec![], + secret_mode: SecretMode::Literal, + apps: vec![], + managed_deny: Some( + ManagedDenyPolicy::new(revision, hosts.iter().map(|s| s.to_string())).unwrap(), + ), + } + } + #[tokio::test] + async fn durable_cas_replays_pending_union_and_fences_receipts() { + tokio::time::timeout(std::time::Duration::from_secs(10), async { + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path(); + let ca = engram_egress_proxy::Ca::load_or_generate(&dir.join("ca")).unwrap(); + let host_id = HostId::new(); + let mut args = engram_egress_proxyd::DaemonArgs::new( + dir.to_path_buf(), + ca.cert_pem.clone(), + ca.key_pair.serialize_pem(), + "http://127.0.0.1:1".into(), + None, + host_id, + ); + [args.proxy_port, args.dns_port, args.gateway_port] = + engram_testkit::net::free_tcp_udp_ports(); + let daemon = tokio::spawn(engram_egress_proxyd::run(args)); + let handle = Arc::new(crate::proxyd_client::ProxydHandle::new( + dir.join(engram_egress_proto::CONTROL_SOCK_NAME), + )); + loop { + if handle.hello().await.is_ok() { + break; + } + assert!(!daemon.is_finished(), "daemon stopped"); + tokio::task::yield_now().await; + } + let egress_host = Arc::new(HostEgress::new(handle.clone(), ca.cert_pem, None)); + let backend = Arc::new(EndpointFixture { + egress: egress_host.clone(), + }); + let bindings = BindingStore::open(dir.join("bindings")).unwrap(); + let ownership = ManagedOwnership::for_mode(bindings.directory(), true) + .unwrap() + .unwrap(); + let target = ManagedPolicyTarget { + session_id: SessionId::new(), + sandbox_id: SandboxId::new(), + host_id, + binding_epoch: 1, + }; + bindings + .bind(target.session_id, target.sandbox_id, 1) + .unwrap(); + let host = ManagedPolicyHost::new( + host_id, + "host-a".into(), + bindings.clone(), + backend.clone(), + egress_host.clone(), + ownership.clone(), + ) + .unwrap(); + let context = ManagedPolicyContext { + target: target.clone(), + operator_generation: 1, + operator_identity: format!("sha256:{}", "a".repeat(64)), + binding_hash: None, + }; + assert!(host.read_applied(&target).await.unwrap().is_none()); + let first = ManagedPolicyApply { + expected: None, + context: context.clone(), + egress: egress(&target, 1, &["mcp.example"]), + }; + let ack = host.compare_and_apply(first.clone()).await.unwrap(); + assert_eq!(host.compare_and_apply(first.clone()).await.unwrap(), ack); + let mut bad = first.clone(); + bad.expected = Some( + ManagedAppliedState::new( + context.clone(), + ack.policy.clone(), + "foreign".into(), + "foreign".into(), + None, + ) + .unwrap(), + ); + assert!(host.compare_and_apply(bad).await.is_err()); + assert_eq!( + handle + .decide(first.egress.guest_ip, "MCP.EXAMPLE.") + .await + .unwrap() + .as_deref(), + Some("reject") + ); + let second = ManagedPolicyApply { + expected: Some(ack.clone()), + context: context.clone(), + egress: egress(&target, 2, &["mcp.example", "alias.example"]), + }; + let ack2 = host.compare_and_apply(second.clone()).await.unwrap(); + assert!(host.compare_and_apply(first).await.is_err()); + let mut shrink = second.clone(); + shrink.expected = Some(ack2.clone()); + shrink.egress = egress(&target, 3, &["mcp.example"]); + assert!(host.compare_and_apply(shrink).await.is_err()); + // Construct the crash state: desired rename completed, proxy apply/ACK did not. + let pending = ManagedRecord { + version: 1, + context: context.clone(), + egress: egress(&target, 3, &["mcp.example", "alias.example", "new.example"]), + applied: None, + expected: Some(ack2.clone()), + accepted_host_incarnation: "host-a".into(), + accepted_proxy_incarnation: ack2.proxy_incarnation.clone(), + }; + persist(&bindings, &pending).unwrap(); + let successor = ManagedPolicyHost::new( + host_id, + "host-b".into(), + bindings.clone(), + backend, + egress_host, + ownership, + ) + .unwrap(); + successor.recover_all().await.unwrap(); + let recovered = successor.read_applied(&target).await.unwrap().unwrap(); + assert_eq!(recovered.policy.revision, 3); + assert_eq!(recovered.host_incarnation, "host-b"); + assert_eq!( + handle + .decide(pending.egress.guest_ip, "new.example") + .await + .unwrap() + .as_deref(), + Some("reject") + ); + let stale = ManagedPolicyApply { + expected: Some(ack2), + context: context.clone(), + egress: pending.egress.clone(), + }; + assert!(successor.compare_and_apply(stale).await.is_err()); + let current = ManagedPolicyApply { + expected: Some(recovered.clone()), + context, + egress: pending.egress, + }; + assert_eq!( + successor.compare_and_apply(current.clone()).await.unwrap(), + recovered + ); + let record_path = bindings + .directory() + .join(format!("{}.managed", target.session_id)); + let valid_bytes = std::fs::read(&record_path).unwrap(); + std::fs::remove_file(&record_path).unwrap(); + assert!(successor.recover_all().await.is_err()); + assert!(handle + .read_managed(target.session_id) + .await + .unwrap() + .is_none()); + std::fs::write(&record_path, b"corrupt").unwrap(); + assert!(successor.recover_all().await.is_err()); + assert!(handle + .read_managed(target.session_id) + .await + .unwrap() + .is_none()); + std::fs::write(&record_path, valid_bytes).unwrap(); + successor.recover_all().await.unwrap(); + let mut final_request = current.clone(); + final_request.context.binding_hash = Some(format!("sha256:{}", "b".repeat(64))); + final_request.egress = + egress(&target, 4, &["mcp.example", "alias.example", "new.example"]); + assert!(successor.compare_and_apply(final_request).await.is_err()); + assert!(successor.read_applied(&target).await.is_err()); + bindings + .bind(target.session_id, target.sandbox_id, 2) + .unwrap(); + assert!(successor.read_applied(&target).await.is_err()); + assert!(handle + .read_managed(target.session_id) + .await + .unwrap() + .is_none()); + std::fs::write( + bindings + .directory() + .join(format!("{}.managed", target.session_id)), + b"corrupt", + ) + .unwrap(); + let mut next = target.clone(); + next.binding_epoch = 2; + assert!(successor.read_applied(&next).await.is_err()); + // A new boot marker stays quarantined without an ordinary policy + // replay record; restart never adopts its missing native owner. + std::fs::remove_file(&record_path).unwrap(); + std::fs::write( + bindings + .directory() + .join(format!("{}.managed-boot", target.session_id)), + target.sandbox_id.to_string(), + ) + .unwrap(); + successor.recover_all().await.unwrap(); + assert!(handle + .read_managed(target.session_id) + .await + .unwrap() + .is_none()); + let mut socket = + tokio::net::UnixStream::connect(dir.join(engram_egress_proto::CONTROL_SOCK_NAME)) + .await + .unwrap(); + engram_egress_proto::write_frame(&mut socket, &engram_egress_proto::ToProxyd::Shutdown) + .await + .unwrap(); + assert_eq!(daemon.await.unwrap(), 0); + }) + .await + .expect("bounded host policy test"); + } +} + +#[cfg(test)] +mod ownership_tests { + // Tests drive subprocesses; OS time is a bounded test input (ADR 0098 D1). + #![allow(clippy::disallowed_methods)] + use super::*; + #[test] + fn ownership_child() { + let Some(dir) = std::env::var_os("ENGRAM_MANAGED_OWNERSHIP_TEST_DIR") else { + return; + }; + let path = std::path::PathBuf::from(dir); + let _owner = ManagedOwnership::for_mode(&path, true).unwrap().unwrap(); + std::fs::write(path.join("test-ready"), b"ready").unwrap(); + loop { + std::thread::park(); + } + } + #[test] + fn subprocess_overlap_rejected_and_crash_releases_ownership() { + let temp = tempfile::tempdir().unwrap(); + let mut child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "managed_policy::ownership_tests::ownership_child", + "--nocapture", + ]) + .env("ENGRAM_MANAGED_OWNERSHIP_TEST_DIR", temp.path()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !temp.path().join("test-ready").exists() { + if std::time::Instant::now() >= deadline || child.try_wait().unwrap().is_some() { + let _ = child.kill(); + let _ = child.wait(); + panic!("ownership child did not become ready"); + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + let rejected = ManagedOwnership::for_mode(temp.path(), true).is_err(); + child.kill().unwrap(); + child.wait().unwrap(); + assert!(rejected); + let _successor = ManagedOwnership::for_mode(temp.path(), true) + .unwrap() + .unwrap(); + assert!( + ManagedOwnership::for_mode(temp.path(), false).is_err(), + "plaintext mode cannot reopen a managed directory" + ); + } +} diff --git a/crates/engram-host-agent/src/managed_tls.rs b/crates/engram-host-agent/src/managed_tls.rs new file mode 100644 index 000000000..e3114646a --- /dev/null +++ b/crates/engram-host-agent/src/managed_tls.rs @@ -0,0 +1,35 @@ +//! Managed host mode authenticates the single lifecycle listener. +use serde::Deserialize; +use std::{collections::BTreeSet, path::PathBuf}; +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ManagedHostTls { + pub certificate: PathBuf, + pub key: PathBuf, + pub client_ca: PathBuf, + pub coordinator_pins: BTreeSet, +} +impl ManagedHostTls { + pub fn from_env() -> Result, String> { + let Some(path) = std::env::var_os("ENGRAM_HOST_MANAGED_TLS_CONFIG") else { + return Ok(None); + }; + let bytes = + std::fs::read(path).map_err(|_| "cannot read managed host TLS configuration")?; + let value: Self = + serde_json::from_slice(&bytes).map_err(|_| "invalid managed host TLS configuration")?; + engram_protocol::host_tls::validate_pins(&value.coordinator_pins)?; + Ok(Some(value)) + } + pub fn server_config(&self) -> Result { + engram_protocol::host_tls::validate_pins(&self.coordinator_pins)?; + engram_tls::install_provider(); + let cert = + std::fs::read(&self.certificate).map_err(|_| "cannot read managed host certificate")?; + let key = std::fs::read(&self.key).map_err(|_| "cannot read managed host key")?; + let ca = std::fs::read(&self.client_ca).map_err(|_| "cannot read coordinator CA")?; + Ok(tonic::transport::ServerTlsConfig::new() + .identity(tonic::transport::Identity::from_pem(cert, key)) + .client_ca_root(tonic::transport::Certificate::from_pem(ca))) + } +} diff --git a/crates/engram-host-agent/src/pooled_backend.rs b/crates/engram-host-agent/src/pooled_backend.rs index 5ca75b66f..a43f098d7 100644 --- a/crates/engram-host-agent/src/pooled_backend.rs +++ b/crates/engram-host-agent/src/pooled_backend.rs @@ -60,13 +60,13 @@ impl ItemAssembler { return Err(SandboxError::Snapshot(format!( "migration fetch: item {} arrived twice", frame.item_idx - ))) + ))); } None => { return Err(SandboxError::Snapshot(format!( "migration fetch: item {} was not requested", frame.item_idx - ))) + ))); } } if self.item.is_some_and(|i| i != frame.item_idx) || frame.offset != self.bytes.len() as u64 @@ -2610,17 +2610,39 @@ impl PooledBackend { /// volumes of every sandbox that is neither alive nor awaiting an /// eviction finalize. Runs once at startup, after the reattach pass /// fixes the live set and before anything can provision a sandbox. - pub async fn reconcile_volumes(&self) { + pub async fn reconcile_volumes(&self, bindings: &crate::bindings::BindingStore) { let Some(volumes) = &self.volumes else { return; }; - let live = match self.inner.list().await { + let mut live = match self.inner.list().await { Ok(ids) => ids, Err(error) => { tracing::warn!(%error, "volume reconcile skipped: cannot list sandboxes"); return; } }; + // Managed owners are absent from the ordinary backend map. Durable + // markers reserve their live and transient volumes after a restart. + // This does not adopt them or prove that their former VM was reaped. + let protected = async { + let mut managed = Vec::new(); + for name in volumes.disk().list().await? { + if let Some(parsed) = engram_host_core::session_volumes::parse(&name) { + if bindings.managed_sandbox_required(parsed.sandbox)? { + managed.push(parsed.sandbox); + } + } + } + Ok::<_, std::io::Error>(managed) + } + .await; + let protected = match protected { + Ok(ids) => ids, + Err(error) => { + tracing::warn!(%error, "volume reconcile skipped: managed ownership unavailable"); + return; + } + }; match volumes.reattach_all().await { Ok(reattached) if reattached.is_empty() => {} Ok(reattached) => { @@ -2632,6 +2654,7 @@ impl PooledBackend { for id in live.iter().copied() { self.settle_disks(id); } + live.extend(protected); let finalizer = self.eviction_finalizer(); match crate::eviction_finalize::reconcile_volumes(volumes, finalizer.as_ref(), live).await { Ok(report) => { @@ -5579,7 +5602,7 @@ impl SandboxBackend for PooledBackend { std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut ) => { - continue + continue; } Err(e) => return Err(format!("control frame: {e}")), }; @@ -6858,6 +6881,12 @@ impl SandboxBackend for PooledBackend { self.inner.set_upload_sink(sink); } + async fn managed_network_proof( + &self, + target: &engram_core::types::managed_deny::ManagedPolicyTarget, + ) -> Option { + self.inner.managed_network_proof(target).await + } async fn notify_session_policy(&self, policy: SessionEgressPolicy) -> Result<(), SandboxError> { let sandbox_id = policy.sandbox_id; let session_id = policy.session_id; @@ -6867,7 +6896,10 @@ impl SandboxBackend for PooledBackend { self.session_bindings.insert(sandbox_id, session_id); let Some(egress) = self.egress.as_ref() else { - if !policy.guest_services.is_empty() || !policy.tunnels.is_empty() { + if policy.managed_deny.is_some() + || !policy.guest_services.is_empty() + || !policy.tunnels.is_empty() + { return Err(SandboxError::InvalidSpec( "guest services and tunnels require a host egress proxy".into(), )); @@ -7107,6 +7139,7 @@ mod tests { for (guest_services, tunnels) in cases { let error = pooled .notify_session_policy(SessionEgressPolicy { + managed_deny: None, session_id: SessionId::new(), sandbox_id: SandboxId::new(), guest_ip: std::net::Ipv4Addr::LOCALHOST, @@ -10900,6 +10933,7 @@ mod tests { allow: &[&str], ) -> SessionEgressPolicy { SessionEgressPolicy { + managed_deny: None, session_id, sandbox_id, guest_ip, @@ -13235,3 +13269,67 @@ mod tests { assert_eq!(inner.inner.destroy_calls.lock().as_slice(), &[id]); } } + +#[cfg(test)] +mod managed_volume_reconcile_tests { + // tests drive a live system; wall clock/OS entropy here is input, not a decision source (ADR 0098 D1) + #![allow(clippy::disallowed_methods)] + use super::*; + use engram_host_core::{ + disk::SessionDisk, + session_volumes::{live_name, transient_name}, + FileDisk, + }; + + #[tokio::test] + async fn startup_keeps_managed_volumes_and_sweeps_unowned_volumes() { + let dir = tempfile::tempdir().unwrap(); + let bindings = crate::bindings::BindingStore::open(dir.path().join("bindings")).unwrap(); + let managed = SandboxId::new(); + let ordinary = SandboxId::new(); + std::fs::write( + bindings + .directory() + .join(format!("{managed}.managed-sandbox")), + b"1", + ) + .unwrap(); + let disk = Arc::new(FileDisk::new(dir.path().join("volumes"), 4096).unwrap()); + let managed_root = live_name(managed, DiskRole::Root); + let managed_cut = transient_name(managed, DiskRole::Root); + let dead_root = live_name(ordinary, DiskRole::Root); + for name in [&managed_root, &managed_cut, &dead_root] { + disk.create(name, 4096).await.unwrap(); + } + let volumes = Arc::new(SessionVolumes::new( + disk.clone(), + engram_fill::FillEngine::new( + engram_chunk_store::ChunkStore::new(Arc::new( + engram_storage_local::LocalBlobStorage::new(dir.path().join("blob")), + )), + engram_fill::Config::default(), + ), + )); + let backend = PooledBackend::new(Arc::new(engram_sandbox_process::ProcessBackend::new( + dir.path().join("ordinary"), + ))) + .with_session_volumes(volumes); + backend.reconcile_volumes(&bindings).await; + assert!(!backend.disk_settles.contains_key(&managed)); + assert!(disk.open(&managed_root).await.is_ok()); + assert!(disk.open(&managed_cut).await.is_ok()); + assert!(disk.open(&dead_root).await.is_err()); + + // A corrupt association cannot authorize deleting any unowned disk. + disk.create(&dead_root, 4096).await.unwrap(); + std::fs::write( + bindings.directory().join("invalid.managed-required"), + b"invalid", + ) + .unwrap(); + backend.reconcile_volumes(&bindings).await; + assert!(disk.open(&dead_root).await.is_ok()); + assert!(disk.open(&managed_root).await.is_ok()); + assert!(disk.open(&managed_cut).await.is_ok()); + } +} diff --git a/crates/engram-host-agent/src/proxyd_client.rs b/crates/engram-host-agent/src/proxyd_client.rs index 28f231122..2af955fca 100644 --- a/crates/engram-host-agent/src/proxyd_client.rs +++ b/crates/engram-host-agent/src/proxyd_client.rs @@ -121,11 +121,31 @@ impl ProxydHandle { } } - /// One request/response round-trip. Serialized behind the mutex — - /// the control plane is low-rate (policy applies and removals). - /// On any transport error the cached connection is dropped so the - /// next call redials (the respawn recovery path). + /// Managed requests own separate streams so a session drain cannot block + /// another session. Ordinary requests share the cached stream; an error + /// drops that connection so the next call redials. No request is retried. async fn request(&self, msg: &ToProxyd) -> Result { + let managed = matches!( + msg, + ToProxyd::ListManaged + | ToProxyd::ReadManaged(_) + | ToProxyd::QuarantineManaged(_) + | ToProxyd::RetirePrivateNetwork(_) + | ToProxyd::HandoffClosedPrivate(_) + | ToProxyd::Hello { .. } + ) || matches!(msg, ToProxyd::ApplyPolicy(policy) if policy.managed_deny.is_some()); + if managed { + // A session drain must not occupy another session's control stream. + let mut stream = tokio::net::UnixStream::connect(&self.control_sock) + .await + .map_err(|e| ProxydError(format!("managed control connect: {e}")))?; + engram_egress_proto::write_frame(&mut stream, msg) + .await + .map_err(|e| ProxydError(format!("managed control write: {e}")))?; + return engram_egress_proto::read_frame(&mut stream) + .await + .map_err(|e| ProxydError(format!("managed control read: {e}"))); + } let mut guard = self.conn.lock().await; if guard.is_none() { let stream = tokio::net::UnixStream::connect(&self.control_sock) @@ -158,6 +178,44 @@ impl ProxydHandle { } } + /// Dedicated allocation capability, initially CLOSED. This is not a final + /// policy receipt. A new boot or channel loss requires a fresh allocation. + pub async fn register_private_network( + &self, + allocation: &engram_egress_proto::private::Allocation, + ) -> Result { + let stream = tokio::net::UnixStream::connect(&self.control_sock) + .await + .map_err(|e| ProxydError(format!("private network connect: {e}")))?; + engram_egress_proto::private::register(stream, allocation) + .await + .map_err(|e| ProxydError(format!("private network register: {e}"))) + } + + /// The response is for this exact allocation on a disposable stream. + /// Cancellation drops that stream; the daemon can finish only this retirement. + pub async fn retire_private_network( + &self, + allocation: &engram_egress_proto::private::Allocation, + ) -> Result<(), ProxydError> { + self.expect_ok( + &ToProxyd::RetirePrivateNetwork(Box::new(allocation.clone())), + "retire private network", + ) + .await + } + + pub async fn handoff_closed_private( + &self, + request: engram_egress_proto::private::ClosedPolicyHandoff, + ) -> Result<(), ProxydError> { + self.expect_ok( + &ToProxyd::HandoffClosedPrivate(Box::new(request)), + "closed policy handoff", + ) + .await + } + pub async fn hello(&self) -> Result { match self .request(&ToProxyd::Hello { @@ -170,6 +228,31 @@ impl ProxydHandle { } } + pub async fn managed_sessions(&self) -> Result, ProxydError> { + match self.request(&ToProxyd::ListManaged).await? { + FromProxyd::ManagedSessions(sessions) => Ok(sessions), + _ => Err(ProxydError("invalid managed enumeration response".into())), + } + } + pub async fn read_managed( + &self, + session: engram_core::SessionId, + ) -> Result, ProxydError> { + match self.request(&ToProxyd::ReadManaged(session)).await? { + FromProxyd::ManagedState(state) => Ok(state), + _ => Err(ProxydError("invalid managed policy response".into())), + } + } + pub async fn quarantine_managed( + &self, + session: engram_core::SessionId, + ) -> Result<(), ProxydError> { + self.expect_ok( + &ToProxyd::QuarantineManaged(session), + "quarantine managed session", + ) + .await + } pub async fn apply_policy( &self, policy: engram_core::types::egress::SessionEgressPolicy, @@ -567,3 +650,103 @@ async fn await_exit(process: &mut ProxydProcess) { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn closed_handoff_does_not_block_independent_control() { + use engram_core::types::{ + egress::SessionEgressPolicy, image::SecretMode, managed_deny::ManagedDenyPolicy, + }; + use engram_egress_proto::private::{Allocation, ClosedPolicyHandoff}; + let temp = tempfile::tempdir().unwrap(); + let socket = temp.path().join("control.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let (received, waiting) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let mut received = Some(received); + let mut tasks = tokio::task::JoinSet::new(); + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + let received = received.take(); + tasks.spawn(async move { + while let Ok(message) = + engram_egress_proto::read_frame::<_, ToProxyd>(&mut stream).await + { + let reply = match message { + ToProxyd::HandoffClosedPrivate(_) => { + received.unwrap().send(()).unwrap(); + std::future::pending::<()>().await; + return; + } + ToProxyd::Health => FromProxyd::HealthReport { sessions: 0 }, + ToProxyd::LookupGuest(_) => FromProxyd::Guest(None), + _ => panic!("unexpected control operation"), + }; + engram_egress_proto::write_frame(&mut stream, &reply) + .await + .unwrap(); + } + }); + } + }); + let handle = Arc::new(ProxydHandle::new(socket)); + let policy = ManagedDenyPolicy::new(1, ["mcp.example".into()]).unwrap(); + let allocation = Allocation { + version: 1, + host_id: "00000000-0000-0000-0000-000000000001".parse().unwrap(), + host_incarnation: "host-a".into(), + proxy_incarnation: "proxy-a".into(), + session_id: "00000000-0000-0000-0000-000000000002".parse().unwrap(), + sandbox_id: "00000000-0000-0000-0000-000000000003".parse().unwrap(), + boot_attempt: "boot-a".into(), + attachment_generation: 1, + guest_ip: "198.18.0.2".parse().unwrap(), + gateway_ip: "198.18.0.1".parse().unwrap(), + policy_revision: policy.revision, + policy_digest: policy.digest.clone(), + }; + let request = ClosedPolicyHandoff { + policy: SessionEgressPolicy { + session_id: allocation.session_id, + sandbox_id: allocation.sandbox_id, + guest_ip: allocation.guest_ip, + network_allow_hosts: vec![], + network_allow_host_patterns: vec![], + allow_all: false, + secrets: vec![], + injects: vec![], + observes: vec![], + guest_services: vec![], + tunnels: vec![], + secret_mode: SecretMode::Literal, + apps: vec![], + managed_deny: Some(policy), + }, + allocation, + }; + let handoff_handle = handle.clone(); + let handoff = + tokio::spawn(async move { handoff_handle.handoff_closed_private(request).await }); + tokio::time::timeout(Duration::from_secs(1), waiting) + .await + .unwrap() + .unwrap(); + let independent = tokio::time::timeout(Duration::from_secs(1), async { + assert_eq!(handle.health().await.unwrap(), 0); + assert!(handle + .lookup_guest("198.18.1.2".parse().unwrap()) + .await + .unwrap() + .is_none()); + }) + .await; + handoff.abort(); + server.abort(); + let _ = handoff.await; + let _ = server.await; + independent.expect("a held handoff must not block another session or ordinary control"); + } +} diff --git a/crates/engram-host-agent/tests/e2e_harness.rs b/crates/engram-host-agent/tests/e2e_harness.rs index 2ce32473b..9af6ea088 100644 --- a/crates/engram-host-agent/tests/e2e_harness.rs +++ b/crates/engram-host-agent/tests/e2e_harness.rs @@ -768,6 +768,7 @@ async fn e2e_harness_cold_via_pooled_backend() { let allow_list: Vec = ALLOW_HOSTS.iter().map(|s| s.to_string()).collect(); let network_allow = engram_egress_proxy::HostList::from_manifest(&allow_list, &[]).unwrap(); registry.register(engram_egress_proxy::SessionState { + managed_deny: None, session_id, guest_ip, network_allow, @@ -884,6 +885,7 @@ async fn e2e_harness_warm_via_pooled_backend() { let allow_list: Vec = ALLOW_HOSTS.iter().map(|s| s.to_string()).collect(); let network_allow = engram_egress_proxy::HostList::from_manifest(&allow_list, &[]).unwrap(); registry.register(engram_egress_proxy::SessionState { + managed_deny: None, session_id, guest_ip, network_allow, diff --git a/crates/engram-host-agent/tests/egress_stream_survives_roll.rs b/crates/engram-host-agent/tests/egress_stream_survives_roll.rs index 1da5e332c..fc208d3aa 100644 --- a/crates/engram-host-agent/tests/egress_stream_survives_roll.rs +++ b/crates/engram-host-agent/tests/egress_stream_survives_roll.rs @@ -306,6 +306,7 @@ async fn established_guest_stream_survives_a_host_agent_roll() { // The production apply path: policy to the daemon + the ADR 0111 // persist the successor replays from. let policy = engram_core::types::egress::SessionEgressPolicy { + managed_deny: None, session_id: engram_core::SessionId::new(), sandbox_id: sandbox, guest_ip, diff --git a/crates/engram-host-agent/tests/egress_survives_roll.rs b/crates/engram-host-agent/tests/egress_survives_roll.rs index 66e73fd9b..b818d81f6 100644 --- a/crates/engram-host-agent/tests/egress_survives_roll.rs +++ b/crates/engram-host-agent/tests/egress_survives_roll.rs @@ -185,6 +185,7 @@ async fn survivor_keeps_egress_across_a_host_agent_roll() { let session_id = engram_core::SessionId::new(); let guest_ip = std::net::Ipv4Addr::new(10, 200, 0, 2); let policy = engram_core::types::egress::SessionEgressPolicy { + managed_deny: None, session_id, sandbox_id: sandbox, guest_ip, diff --git a/crates/engram-host-agent/tests/grpc_pause_resume.rs b/crates/engram-host-agent/tests/grpc_pause_resume.rs index 4951aaf9d..da32b3063 100644 --- a/crates/engram-host-agent/tests/grpc_pause_resume.rs +++ b/crates/engram-host-agent/tests/grpc_pause_resume.rs @@ -1,12 +1,4 @@ -//! gRPC-layer wiring test for ADR 0045 Phase F pause / resume. -//! -//! Pause/resume are thin passthroughs (proto `PauseSandbox`/`ResumeSandbox` -//! → host gRPC server → `HostClient::pause`/`resume` → the backend). The -//! risk in that plumbing is a wiring mismatch — the RPC bound to the wrong -//! handler, or pause/resume swapped. This stands up the real gRPC server -//! with a stub host that records which method it was asked to run for which -//! sandbox, drives it through the real `GrpcHostClient`, and asserts each -//! verb reaches the host with the right sandbox id, in order. +//! Real gRPC lifecycle, cancellation, and predecessor wire 38 compatibility tests. // tests drive a live system; wall clock/OS entropy here is input, not a decision source (ADR 0098 D1) #![allow(clippy::disallowed_methods)] @@ -24,12 +16,19 @@ use engram_core::types::snapshot::SnapshotMetadata; use engram_core::{SandboxId, SessionId}; use parking_lot::Mutex; -/// Records the (verb, sandbox_id) of every pause/resume the gRPC server -/// forwarded. Every other trait method is `unreachable!()` — a loud -/// failure if the test starts touching surface it doesn't intend to cover. +/// Records lifecycle calls and supplies a stalled file source for cancellation. #[derive(Default)] struct FakeHost { calls: Mutex>, + starts: Mutex>, + file_drop: Mutex>>, +} + +struct StartCall { + sandbox: SandboxId, + agent: AgentSpec, + policy: SessionEgressPolicy, + fence: SessionFence, } #[async_trait] @@ -43,6 +42,27 @@ impl HostClient for FakeHost { Ok(()) } + async fn read_file( + &self, + _: SandboxId, + path: String, + ) -> Result< + ( + engram_core::types::sandbox::SessionFileMetadata, + engram_core::types::sandbox::SessionFileStream, + ), + SandboxError, + > { + Ok(( + engram_core::types::sandbox::SessionFileMetadata { + path, + size_bytes: 1, + sha256: "a".repeat(64), + }, + Box::pin(StalledFile(self.file_drop.lock().take())), + )) + } + async fn create(&self, _: SandboxSpec) -> Result { unreachable!("pause/resume test path doesn't call create") } @@ -77,12 +97,18 @@ impl HostClient for FakeHost { } async fn start_agent( &self, - _: SandboxId, - _: AgentSpec, - _: SessionEgressPolicy, - _: SessionFence, + sandbox: SandboxId, + agent: AgentSpec, + policy: SessionEgressPolicy, + fence: SessionFence, ) -> Result<(), SandboxError> { - unreachable!() + self.starts.lock().push(StartCall { + sandbox, + agent, + policy, + fence, + }); + Ok(()) } async fn guest_ip(&self, _: SandboxId) -> Option { None @@ -182,3 +208,156 @@ async fn pause_then_resume_round_trip_to_the_host_with_the_right_sandbox() { "pause/resume did not round-trip in order with the right sandbox id", ); } + +struct StalledFile(Option>); +impl futures::Stream for StalledFile { + type Item = Result; + fn poll_next( + self: std::pin::Pin<&mut Self>, + _: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Pending + } +} +impl Drop for StalledFile { + fn drop(&mut self) { + if let Some(signal) = self.0.take() { + let _ = signal.send(()); + } + } +} + +#[tokio::test] +async fn dropping_stalled_file_response_drops_host_source() { + use engram_protocol::grpc::host_service_server::HostService; + use futures::StreamExt; + let (signal, dropped) = tokio::sync::oneshot::channel(); + let host = Arc::new(FakeHost::default()); + *host.file_drop.lock() = Some(signal); + let service = engram_host_agent::grpc_server::HostServiceImpl::new( + host, + engram_host_agent::session_epochs::ephemeral(), + ); + let mut response = service + .read_file(tonic::Request::new( + engram_protocol::grpc::HostReadFileRequest { + sandbox_id: SandboxId::new().as_uuid().as_bytes().to_vec(), + path: "/tmp/stalled.bin".into(), + }, + )) + .await + .unwrap() + .into_inner(); + assert!(matches!( + response.next().await.unwrap().unwrap().frame, + Some(engram_protocol::grpc::host_read_file_response::Frame::Metadata(_)) + )); + tokio::task::yield_now().await; + drop(response); + tokio::time::timeout(Duration::from_millis(100), dropped) + .await + .expect("downstream cancellation must drop the stalled host source") + .unwrap(); +} + +// These bytes come directly from ec526661, before managed authority was added. +const OLD_POLICY: &[u8] = include_bytes!( + "../../engram-protocol/tests/golden/ordinary_v38/session_egress_policy_minted.bin" +); +const OLD_AGENT: &[u8] = + include_bytes!("../../engram-protocol/tests/golden/ordinary_v38/agent_spec.bin"); + +#[tokio::test] +async fn predecessor_38_caller_reaches_new_host_with_exact_policy_and_fence() { + use engram_protocol::grpc::{host_service_client::HostServiceClient, StartAgentRequest}; + let host = Arc::new(FakeHost::default()); + let addr = boot_grpc_server(host.clone()).await; + let mut client = HostServiceClient::connect(format!("http://{addr}")) + .await + .unwrap(); + let expected: SessionEgressPolicy = bincode::deserialize(include_bytes!( + "../../engram-protocol/tests/golden/session_egress_policy_minted.bin" + )) + .unwrap(); + let message = StartAgentRequest { + sandbox_id: expected.sandbox_id.as_uuid().as_bytes().to_vec(), + agent_bincode: OLD_AGENT.to_vec(), + policy_bincode: OLD_POLICY.to_vec(), + session_id: expected.session_id.as_uuid().as_bytes().to_vec(), + fencing_epoch: 7, + }; + let request = |message: StartAgentRequest, version: &str| { + let mut req = tonic::Request::new(message); + req.set_timeout(Duration::from_secs(3)); + req.metadata_mut() + .insert("x-engram-wire-version", version.parse().unwrap()); + req + }; + client + .start_agent(request(message.clone(), "38")) + .await + .unwrap(); + { + let starts = host.starts.lock(); + assert_eq!(starts.len(), 1); + assert_eq!(starts[0].sandbox, expected.sandbox_id); + assert_eq!(starts[0].policy, expected); + assert_eq!(bincode::serialize(&starts[0].agent).unwrap(), OLD_AGENT); + assert_eq!(starts[0].fence.session_id, expected.session_id); + assert_eq!(starts[0].fence.epoch, 7); + } + for version in ["0", "43", "unknown"] { + assert_eq!( + client + .start_agent(request(message.clone(), version)) + .await + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); + } + for bad_bytes in [ + OLD_POLICY[..OLD_POLICY.len() - 1].to_vec(), + [OLD_POLICY, &[0]].concat(), + include_bytes!("../../engram-protocol/tests/golden/session_egress_policy_managed.bin") + .to_vec(), + ] { + let mut bad = message.clone(); + bad.policy_bincode = bad_bytes; + bad.fencing_epoch = 8; + assert_eq!( + client + .start_agent(request(bad, "38")) + .await + .unwrap_err() + .code(), + tonic::Code::InvalidArgument + ); + } + let mut stale = message; + stale.fencing_epoch = 6; + assert_eq!( + client + .start_agent(request(stale, "38")) + .await + .unwrap_err() + .code(), + tonic::Code::FailedPrecondition + ); + assert_eq!( + host.starts.lock().len(), + 1, + "refusals must not reach the backend" + ); + let mut pause = tonic::Request::new(engram_protocol::grpc::FencedSandboxRequest { + uuid: expected.sandbox_id.as_uuid().as_bytes().to_vec(), + session_id: expected.session_id.as_uuid().as_bytes().to_vec(), + fencing_epoch: 7, + }); + pause.set_timeout(Duration::from_secs(3)); + pause + .metadata_mut() + .insert("x-engram-wire-version", "38".parse().unwrap()); + client.pause_sandbox(pause).await.unwrap(); + assert_eq!(*host.calls.lock(), vec![("pause", expected.sandbox_id)]); +} diff --git a/crates/engram-host-agent/tests/managed_cold_assets_tls.rs b/crates/engram-host-agent/tests/managed_cold_assets_tls.rs new file mode 100644 index 000000000..2c2119150 --- /dev/null +++ b/crates/engram-host-agent/tests/managed_cold_assets_tls.rs @@ -0,0 +1,323 @@ +//! Physical file and authenticated transport proof. No VM or proxy is started. +use engram_core::{ + traits::HostClient, + types::{managed_assets::ManagedColdAssetsRequest, manifest::ManifestRef, sandbox::AuxRoDrive}, + HostId, SessionId, +}; +use engram_host_agent::{ + bindings::BindingStore, + harness::HarnessHub, + managed_boot::{ManagedBackend, ManagedBootHost, ManagedBootHostConfig}, +}; +use engram_protocol::{ + grpc_client::GrpcHostClient, + host_tls::{HostClientTls, HostPeer}, +}; +use futures::FutureExt; +use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::Path, + sync::Arc, + time::Duration, +}; + +fn openssl(dir: &Path, args: &[&str]) { + let output = std::process::Command::new("openssl") + .current_dir(dir) + .args(args) + .output() + .unwrap(); + assert!( + output.status.success(), + "certificate fixture generation failed" + ); +} +fn certificates(dir: &Path) { + openssl( + dir, + &[ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + "ca.key", + "-out", + "ca.pem", + "-days", + "1", + "-subj", + "/CN=test-ca", + ], + ); + std::fs::write( + dir.join("extensions"), + "subjectAltName=DNS:localhost\nextendedKeyUsage=serverAuth,clientAuth\n", + ) + .unwrap(); + for name in ["host", "coord", "other"] { + openssl( + dir, + &[ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + &format!("{name}.key"), + "-out", + &format!("{name}.csr"), + "-subj", + "/CN=localhost", + ], + ); + openssl( + dir, + &[ + "x509", + "-req", + "-in", + &format!("{name}.csr"), + "-CA", + "ca.pem", + "-CAkey", + "ca.key", + "-CAcreateserial", + "-out", + &format!("{name}.pem"), + "-days", + "1", + "-extfile", + "extensions", + ], + ); + openssl( + dir, + &[ + "x509", + "-in", + &format!("{name}.pem"), + "-outform", + "DER", + "-out", + &format!("{name}.der"), + ], + ); + } +} + +#[tokio::test] +async fn cold_assets_actual_files_require_pinned_mutual_tls() { + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path(); + certificates(dir); + std::fs::write(dir.join("kernel"), b"selected kernel").unwrap(); + let host_id = HostId::new(); + let session_id = SessionId::new(); + let ordinary = Arc::new(engram_sandbox_process::ProcessBackend::new( + dir.join("ordinary"), + )); + #[cfg(target_os = "macos")] + let backend = ManagedBackend::Vz { + backend: Arc::new( + engram_sandbox_vz::VzBackend::new( + dir.join("native"), + engram_sandbox_vz::VzConfig::with_kernel(dir.join("kernel")).with_bundle_dir(dir), + ) + .unwrap(), + ), + helper: engram_sandbox_vz::ManagedHelperArtifact { + path: dir.join("absent-helper"), + sha256: "a".repeat(64), + }, + }; + #[cfg(target_os = "linux")] + let backend = { + let mut config = + engram_sandbox_firecracker::FirecrackerConfig::with_kernel(dir.join("kernel")); + config.bundle_dir = dir.to_owned(); + ManagedBackend::Firecracker { + backend: Arc::new(engram_sandbox_firecracker::FirecrackerBackend::new( + dir.join("native"), + config, + )), + disk: Arc::new(engram_host_core::FileDisk::new(dir.join("volumes"), 4096).unwrap()), + } + }; + let chunks = engram_chunk_store::ChunkStore::new(Arc::new( + engram_storage_local::LocalBlobStorage::new(dir.join("blob")), + )); + let manifest = ManifestRef::new(); + chunks + .put_manifest( + manifest, + &engram_chunk_store::Manifest::empty(engram_chunk_store::ManifestKind::Disk, 9), + ) + .await + .unwrap(); + let drives = (0..3) + .map(|i| { + let bytes = [i as u8 + 1; 4]; + let hash = hex::encode(Sha256::digest(bytes)); + std::fs::write(dir.join(AuxRoDrive::staged_file_name(&hash)), bytes).unwrap(); + AuxRoDrive { + drive_id: AuxRoDrive::slot_drive_id(i), + guest_mount: AuxRoDrive::slot_guest_mount(i), + fs_type: "squashfs".into(), + sha256: Some(hash), + } + }) + .collect(); + let request = ManagedColdAssetsRequest { + version: 1, + host_id, + session_id, + intent_id: "intent-a".into(), + image: "test.invalid/image:v1".into(), + image_digest: format!("sha256:{}", "a".repeat(64)), + disk_manifest: manifest, + aux_ro_drives: drives, + }; + let hub = Arc::new(HarnessHub::new( + engram_host_agent::harness::event_sink_to(|_, _, _, _| async { Ok(()) }), + BindingStore::open(dir.join("bindings")).unwrap(), + )); + let boot = Arc::new( + ManagedBootHost::new( + ManagedBootHostConfig { + host_id, + incarnation: "tls-host".into(), + directory: dir.join("records"), + }, + backend, + ordinary.clone(), + Arc::new(engram_host_agent::egress::HostEgress::new( + Arc::new(engram_host_agent::proxyd_client::ProxydHandle::new( + dir.join("absent-proxy.sock"), + )), + String::new(), + None, + )), + hub.clone(), + engram_fill::FillEngine::new(chunks, engram_fill::Config::default()), + ) + .unwrap(), + ); + let local = Arc::new( + engram_host_agent::host_client::LocalHostClient::new( + ordinary, + hub, + engram_host_agent::IdleEvictionHarness::default(), + ) + .with_managed_boot(boot), + ); + let expected = local + .read_managed_cold_assets(request.clone()) + .await + .unwrap(); + let pins = |name: &str| { + BTreeSet::from([engram_protocol::host_tls::certificate_pin( + &std::fs::read(dir.join(format!("{name}.der"))).unwrap(), + )]) + }; + let socket = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = socket.local_addr().unwrap(); + drop(socket); + let mut server = tokio::spawn(engram_host_agent::grpc_server::boot_with_managed_tls( + addr, + local, + engram_host_agent::session_epochs::SessionEpochStore::open(dir.join("epochs")).unwrap(), + None, + Some(engram_host_agent::managed_tls::ManagedHostTls { + certificate: dir.join("host.pem"), + key: dir.join("host.key"), + client_ca: dir.join("ca.pem"), + coordinator_pins: pins("coord"), + }), + )); + let address = format!("https://{addr}"); + let mut tls = HostClientTls { + client_certificate: dir.join("coord.pem"), + client_key: dir.join("coord.key"), + hosts: BTreeMap::from([( + host_id, + HostPeer { + endpoint: address.clone(), + server_name: "localhost".into(), + server_ca: dir.join("ca.pem"), + server_pins: pins("host"), + }, + )]), + }; + let connect = |config: &HostClientTls| { + let pool = engram_protocol::grpc_pool::GrpcHostPool::new().with_tls(config.clone()); + let address = address.clone(); + async move { pool.get_or_warm(host_id, address).await.unwrap() } + }; + let outcome = tokio::time::timeout( + Duration::from_secs(10), + std::panic::AssertUnwindSafe(async { + loop { + if connect(&tls).await.ping().await.is_ok() { + break; + } + assert!(!server.is_finished()); + tokio::time::sleep(Duration::from_millis(10)).await; + } + let receipt = connect(&tls) + .await + .read_managed_cold_assets(request.clone()) + .await + .unwrap(); + assert_eq!(receipt, expected); + assert_eq!(receipt.disk_sha256, hex::encode(Sha256::digest([0; 9]))); + assert_eq!( + receipt.kernel_sha256, + hex::encode(Sha256::digest(b"selected kernel")) + ); + assert_eq!(receipt.host_incarnation, "tls-host"); + let mut changed = request.clone(); + changed.host_id = HostId::new(); + assert!(connect(&tls) + .await + .read_managed_cold_assets(changed) + .await + .is_err()); + tls.client_certificate = dir.join("other.pem"); + tls.client_key = dir.join("other.key"); + assert!(connect(&tls) + .await + .read_managed_cold_assets(request.clone()) + .await + .is_err()); + let no_certificate = tonic::transport::Endpoint::from_shared(address.clone()) + .unwrap() + .tls_config( + tonic::transport::ClientTlsConfig::new() + .domain_name("localhost") + .ca_certificate(tonic::transport::Certificate::from_pem( + std::fs::read(dir.join("ca.pem")).unwrap(), + )), + ) + .unwrap() + .connect_lazy(); + assert!(GrpcHostClient::new(no_certificate) + .read_managed_cold_assets(request) + .await + .is_err()); + assert_eq!(std::fs::read_dir(dir.join("records")).unwrap().count(), 0); + assert!(!dir.join("native").exists()); + assert!(!dir.join("absent-proxy.sock").exists()); + }) + .catch_unwind(), + ) + .await; + // The service's public runner has no graceful stop handle. Drop its exact + // task, join it, then let the owned certificate and asset directory drop. + server.abort(); + let stopped = tokio::time::timeout(Duration::from_secs(2), &mut server).await; + assert!(stopped.is_ok()); + outcome.unwrap().unwrap(); +} diff --git a/crates/engram-host-agent/tests/managed_tls.rs b/crates/engram-host-agent/tests/managed_tls.rs new file mode 100644 index 000000000..91a46b34c --- /dev/null +++ b/crates/engram-host-agent/tests/managed_tls.rs @@ -0,0 +1,803 @@ +//! Local TLS process proof; no guest or deployment is involved. +use engram_protocol::{ + grpc_client::GrpcHostClient, + host_tls::{HostClientTls, HostPeer}, +}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; +fn openssl(dir: &std::path::Path, args: &[&str]) { + let output = std::process::Command::new("openssl") + .current_dir(dir) + .args(args) + .output() + .unwrap(); + assert!(output.status.success(), "openssl fixture generation failed"); +} +#[tokio::test] +async fn managed_host_requires_pinned_mutual_tls() { + let temp = tempfile::tempdir().unwrap(); + let dir = temp.path(); + openssl( + dir, + &[ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + "ca.key", + "-out", + "ca.pem", + "-days", + "1", + "-subj", + "/CN=test-ca", + ], + ); + std::fs::write( + dir.join("extensions"), + "subjectAltName=DNS:localhost\nextendedKeyUsage=serverAuth,clientAuth\n", + ) + .unwrap(); + for name in ["host", "coord", "other"] { + openssl( + dir, + &[ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + &format!("{name}.key"), + "-out", + &format!("{name}.csr"), + "-subj", + "/CN=localhost", + ], + ); + openssl( + dir, + &[ + "x509", + "-req", + "-in", + &format!("{name}.csr"), + "-CA", + "ca.pem", + "-CAkey", + "ca.key", + "-CAcreateserial", + "-out", + &format!("{name}.pem"), + "-days", + "1", + "-extfile", + "extensions", + ], + ); + openssl( + dir, + &[ + "x509", + "-in", + &format!("{name}.pem"), + "-outform", + "DER", + "-out", + &format!("{name}.der"), + ], + ); + } + let host_id = engram_core::HostId::new(); + use engram_core::{traits::HostClient, types::managed_deny::*}; + let ca = engram_egress_proxy::Ca::load_or_generate(&dir.join("proxy-ca")).unwrap(); + let mut args = engram_egress_proxyd::DaemonArgs::new( + dir.to_path_buf(), + ca.cert_pem.clone(), + ca.key_pair.serialize_pem(), + "http://127.0.0.1:1".into(), + None, + host_id, + ); + [args.proxy_port, args.dns_port, args.gateway_port] = engram_testkit::net::free_tcp_udp_ports(); + let daemon = tokio::spawn(engram_egress_proxyd::run(args)); + let handle = Arc::new(engram_host_agent::proxyd_client::ProxydHandle::new( + dir.join(engram_egress_proto::CONTROL_SOCK_NAME), + )); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if handle.hello().await.is_ok() { + break; + } + assert!(!daemon.is_finished()); + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let egress = Arc::new(engram_host_agent::egress::HostEgress::new( + handle, + ca.cert_pem, + None, + )); + let backend = Arc::new(EndpointFixture { + egress: egress.clone(), + calls: Default::default(), + }); + let bindings = engram_host_agent::bindings::BindingStore::open(dir.join("bindings")).unwrap(); + let target = ManagedPolicyTarget { + session_id: engram_core::SessionId::new(), + sandbox_id: engram_core::SandboxId::new(), + host_id, + binding_epoch: 1, + }; + bindings + .bind(target.session_id, target.sandbox_id, target.binding_epoch) + .unwrap(); + let owner = + engram_host_agent::managed_policy::ManagedOwnership::for_mode(&dir.join("bindings"), true) + .unwrap() + .unwrap(); + let managed = Arc::new( + engram_host_agent::managed_policy::ManagedPolicyHost::new( + host_id, + "tls-test-host".into(), + bindings.clone(), + backend.clone(), + egress, + owner, + ) + .unwrap(), + ); + let hub = Arc::new(engram_host_agent::harness::HarnessHub::new( + engram_host_agent::harness::event_sink_to(|_, _, _, _| async { Ok(()) }), + bindings.clone(), + )); + let local = Arc::new( + engram_host_agent::host_client::LocalHostClient::new( + backend.clone(), + hub, + engram_host_agent::IdleEvictionHarness::default(), + ) + .with_managed_policy(managed), + ); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + drop(listener); + let pins = |name: &str| { + BTreeSet::from([engram_protocol::host_tls::certificate_pin( + &std::fs::read(dir.join(format!("{name}.der"))).unwrap(), + )]) + }; + let prepared = engram_host_agent::grpc_server::PreparedHostServer::bind( + addr, + Some(engram_host_agent::managed_tls::ManagedHostTls { + certificate: dir.join("host.pem"), + key: dir.join("host.key"), + client_ca: dir.join("ca.pem"), + coordinator_pins: pins("coord"), + }), + ) + .unwrap(); + // The serving task owns the parsed identity and does not reopen key files. + std::fs::remove_file(dir.join("host.pem")).unwrap(); + std::fs::remove_file(dir.join("host.key")).unwrap(); + let server = tokio::spawn(prepared.serve( + local.clone(), + engram_host_agent::session_epochs::SessionEpochStore::open(dir.join("epochs")).unwrap(), + None, + )); + let address = format!("https://{addr}"); + let mut client = HostClientTls { + client_certificate: dir.join("coord.pem"), + client_key: dir.join("coord.key"), + hosts: BTreeMap::from([( + host_id, + HostPeer { + endpoint: address.clone(), + server_name: "localhost".into(), + server_ca: dir.join("ca.pem"), + server_pins: pins("host"), + }, + )]), + }; + let connect = |config: &HostClientTls| { + config + .endpoint( + host_id, + &address, + tonic::transport::Endpoint::from_shared(address.clone()).unwrap(), + ) + .unwrap() + .connect_lazy() + }; + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if GrpcHostClient::new(connect(&client)).ping().await.is_ok() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(matches!( + GrpcHostClient::new(connect(&client)) + .read_managed_policy(target.clone()) + .await, + Err(engram_core::SandboxError::Unsupported(_)) + )); + let calls_before = backend.calls.load(std::sync::atomic::Ordering::SeqCst); + for marker in [None, Some("38"), Some("44"), Some("043"), Some("invalid")] { + let mut raw = + engram_protocol::grpc::host_service_client::HostServiceClient::new(connect(&client)); + macro_rules! refused { + ($method:ident, $message:expr) => {{ + let mut request = tonic::Request::new($message); + request + .metadata_mut() + .insert("x-engram-wire-version", "38".parse().unwrap()); + if let Some(marker) = marker { + request + .metadata_mut() + .insert("x-engram-managed-protocol", marker.parse().unwrap()); + } + assert_eq!( + raw.$method(request).await.unwrap_err().code(), + tonic::Code::FailedPrecondition, + stringify!($method) + ); + }}; + } + use engram_protocol::grpc::{ + ApplyManagedPolicyRequest, ManagedBootRequest, ReadManagedColdAssetsRequest, + ReadManagedPolicyRequest, + }; + refused!( + read_managed_policy, + ReadManagedPolicyRequest { + target_bincode: bincode::serialize(&target).unwrap() + } + ); + refused!( + apply_managed_policy, + ApplyManagedPolicyRequest { + request_bincode: vec![255] + } + ); + refused!( + read_managed_cold_assets, + ReadManagedColdAssetsRequest { + payload_json: vec![255] + } + ); + refused!( + capture_managed_boot, + ManagedBootRequest { + payload_json: vec![255] + } + ); + refused!( + prepare_managed_boot, + ManagedBootRequest { + payload_json: vec![255] + } + ); + refused!( + read_managed_boot, + ManagedBootRequest { + payload_json: vec![255] + } + ); + refused!( + finalize_managed_boot, + ManagedBootRequest { + payload_json: vec![255] + } + ); + refused!( + start_managed_boot, + ManagedBootRequest { + payload_json: vec![255] + } + ); + } + assert_eq!( + backend.calls.load(std::sync::atomic::Ordering::SeqCst), + calls_before + ); + let remote = engram_protocol::grpc_pool::GrpcHostPool::new() + .with_tls(client.clone()) + .get_or_warm(host_id, address.clone()) + .await + .unwrap(); + assert!(remote + .read_managed_policy(target.clone()) + .await + .unwrap() + .is_none()); + let capture: engram_core::types::managed_boot::ManagedCaptureRequest = serde_json::from_slice( + include_bytes!("../../engram-protocol/tests/golden/managed_capture.json"), + ) + .unwrap(); + assert!(matches!( + remote.capture_managed_boot(capture.clone()).await, + Err(engram_core::SandboxError::InvalidSpec(message)) if message.contains("managed boot unavailable") + )); + assert!(matches!( + remote.read_managed_boot(capture.key).await, + Err(engram_core::SandboxError::InvalidSpec(message)) if message.contains("managed boot unavailable") + )); + let context = ManagedPolicyContext { + target: target.clone(), + operator_generation: 1, + operator_identity: format!("sha256:{}", "a".repeat(64)), + binding_hash: None, + }; + let policy = engram_core::types::egress::SessionEgressPolicy { + session_id: target.session_id, + sandbox_id: target.sandbox_id, + guest_ip: "10.200.0.42".parse().unwrap(), + network_allow_hosts: vec![], + network_allow_host_patterns: vec![], + allow_all: true, + secrets: vec![], + injects: vec![], + observes: vec![], + guest_services: vec![], + tunnels: vec![], + secret_mode: engram_core::types::image::SecretMode::Literal, + apps: vec![], + managed_deny: Some(ManagedDenyPolicy::new(1, ["mcp.example".into()]).unwrap()), + }; + let apply = ManagedPolicyApply { + context, + expected: None, + egress: policy.clone(), + }; + let ack = remote.apply_managed_policy(apply.clone()).await.unwrap(); + assert_eq!( + remote.read_managed_policy(target.clone()).await.unwrap(), + Some(ack.clone()) + ); + assert_eq!(remote.apply_managed_policy(apply).await.unwrap(), ack); + assert!( + ack.context.binding_hash.is_none() && ack.network_proof.is_none(), + "bootstrap is not final network proof" + ); + // Every generic start must validate durable sandbox identity before bind or policy I/O. + // An ordinary duplicate binding must not hide the managed association. + let alias = engram_core::SessionId::new(); + bindings.bind(alias, target.sandbox_id, 1).unwrap(); + let agent = engram_core::types::sandbox::AgentSpec { + binding_epoch: 1, + argv: vec!["fixture".into()], + env: Default::default(), + session_env: Default::default(), + host_ca_pem: None, + }; + let binding_bytes = || { + std::fs::read_dir(dir.join("bindings")) + .unwrap() + .map(|entry| { + let path = entry.unwrap().path(); + ( + path.file_name().unwrap().to_owned(), + std::fs::read(path).unwrap(), + ) + }) + .collect::>() + }; + for host in [&*local as &dyn HostClient, &remote as &dyn HostClient] { + let mut ordinary = policy.clone(); + ordinary.managed_deny = None; + ordinary.session_id = engram_core::SessionId::new(); + let fence = engram_core::traits::SessionFence::new(ordinary.session_id, 1); + let before = binding_bytes(); + let calls = backend.calls.load(std::sync::atomic::Ordering::SeqCst); + for supplied_fence in [fence, engram_core::traits::SessionFence::unfenced()] { + let error = host + .start_agent( + target.sandbox_id, + agent.clone(), + ordinary.clone(), + supplied_fence, + ) + .await + .unwrap_err(); + assert!( + error + .to_string() + .contains("managed session requires acknowledged boot finalization"), + "{error}" + ); + } + assert_eq!( + binding_bytes(), + before, + "rejected start must not bind or persist policy" + ); + assert_eq!( + backend.calls.load(std::sync::atomic::Ordering::SeqCst), + calls + ); + assert_eq!( + host.read_managed_policy(target.clone()).await.unwrap(), + Some(ack.clone()) + ); + ordinary.sandbox_id = engram_core::SandboxId::new(); + for (sandbox, supplied_fence) in [ + (engram_core::SandboxId::new(), fence), + ( + ordinary.sandbox_id, + engram_core::traits::SessionFence::new(engram_core::SessionId::new(), 1), + ), + ( + ordinary.sandbox_id, + engram_core::traits::SessionFence::new(engram_core::SessionId::new(), 0), + ), + ] { + let error = host + .start_agent(sandbox, agent.clone(), ordinary.clone(), supplied_fence) + .await + .unwrap_err(); + assert!( + error.to_string().contains("start identity mismatch"), + "{error}" + ); + } + assert_eq!(binding_bytes(), before); + assert_eq!( + backend.calls.load(std::sync::atomic::Ordering::SeqCst), + calls + ); + ordinary.guest_ip = "10.200.0.43".parse().unwrap(); + host.start_agent(ordinary.sandbox_id, agent.clone(), ordinary.clone(), fence) + .await + .unwrap(); + assert_eq!( + backend.calls.load(std::sync::atomic::Ordering::SeqCst), + calls + 2 + ); + assert_eq!( + bindings + .read(ordinary.session_id) + .unwrap() + .unwrap() + .sandbox_id, + ordinary.sandbox_id + ); + host.start_agent( + ordinary.sandbox_id, + agent.clone(), + ordinary, + engram_core::traits::SessionFence::unfenced(), + ) + .await + .unwrap(); + assert_eq!( + backend.calls.load(std::sync::atomic::Ordering::SeqCst), + calls + 4 + ); + } + // The retained managed record still names the sandbox after the binding is removed. + let binding_path = dir + .join("bindings") + .join(format!("{}.json", target.session_id)); + let binding = std::fs::read(&binding_path).unwrap(); + std::fs::remove_file(&binding_path).unwrap(); + let mut disguised = policy.clone(); + disguised.managed_deny = None; + disguised.session_id = engram_core::SessionId::new(); + for host in [&*local as &dyn HostClient, &remote as &dyn HostClient] { + let before = binding_bytes(); + let calls = backend.calls.load(std::sync::atomic::Ordering::SeqCst); + assert!(host + .start_agent( + target.sandbox_id, + agent.clone(), + disguised.clone(), + engram_core::traits::SessionFence::unfenced() + ) + .await + .unwrap_err() + .to_string() + .contains("managed session requires acknowledged boot finalization")); + assert_eq!(binding_bytes(), before); + assert_eq!( + backend.calls.load(std::sync::atomic::Ordering::SeqCst), + calls + ); + } + std::fs::write(binding_path, binding).unwrap(); + let snapshot: engram_core::types::snapshot::SnapshotMetadata = + serde_json::from_value(serde_json::json!({ + "id": engram_core::SnapshotId::new(), "size_bytes": 0, + "created_at": "2026-01-01T00:00:00Z", "image_version": "fixture", + })) + .unwrap(); + let managed_fence = engram_core::traits::SessionFence::new(target.session_id, 1); + let ordinary_session = engram_core::SessionId::new(); + let mut managed_source = snapshot.clone(); + managed_source.source_sandbox_id = Some(target.sandbox_id); + for host in [&*local as &dyn HostClient, &remote as &dyn HostClient] { + for epoch in [1, 0] { + let ordinary_fence = engram_core::traits::SessionFence::new(ordinary_session, epoch); + let managed_fence = engram_core::traits::SessionFence::new(target.session_id, epoch); + let calls = backend.calls.load(std::sync::atomic::Ordering::SeqCst); + host.restore(snapshot.clone(), ordinary_fence) + .await + .expect("ordinary restore preserves session identity"); + host.restore_base_for_session( + snapshot.clone(), + Default::default(), + vec![], + ordinary_fence, + ) + .await + .expect("ordinary base restore preserves session identity"); + host.resume(engram_core::SandboxId::new(), ordinary_fence) + .await + .expect("ordinary resume preserves session identity"); + assert_eq!( + backend.calls.load(std::sync::atomic::Ordering::SeqCst), + calls + 3 + ); + for error in [ + host.resume(target.sandbox_id, ordinary_fence) + .await + .unwrap_err(), + host.restore(managed_source.clone(), ordinary_fence) + .await + .unwrap_err(), + host.restore_base_for_session( + managed_source.clone(), + Default::default(), + vec![], + ordinary_fence, + ) + .await + .unwrap_err(), + host.restore(snapshot.clone(), managed_fence) + .await + .unwrap_err(), + host.restore_base_for_session( + snapshot.clone(), + Default::default(), + vec![], + managed_fence, + ) + .await + .unwrap_err(), + host.resume(target.sandbox_id, managed_fence) + .await + .unwrap_err(), + ] { + assert!( + error + .to_string() + .contains("managed restore quarantine unavailable"), + "{error}" + ); + } + assert_eq!( + backend.calls.load(std::sync::atomic::Ordering::SeqCst), + calls + 3 + ); + } + let calls = backend.calls.load(std::sync::atomic::Ordering::SeqCst); + let nil = engram_core::traits::SessionFence::unfenced(); + for error in [ + host.restore(snapshot.clone(), nil).await.unwrap_err(), + host.restore_base_for_session(snapshot.clone(), Default::default(), vec![], nil) + .await + .unwrap_err(), + host.resume(engram_core::SandboxId::new(), nil) + .await + .unwrap_err(), + ] { + assert!( + error + .to_string() + .contains("restore session identity required"), + "{error}" + ); + } + assert_eq!( + backend.calls.load(std::sync::atomic::Ordering::SeqCst), + calls + ); + } + // Missing desired state and a corrupt marker still identify a known managed session. + std::fs::remove_file( + dir.join("bindings") + .join(format!("{}.managed", target.session_id)), + ) + .unwrap(); + std::fs::write( + dir.join("bindings") + .join(format!("{}.managed-required", target.session_id)), + b"corrupt", + ) + .unwrap(); + assert!(remote + .restore(snapshot, managed_fence) + .await + .unwrap_err() + .to_string() + .contains("managed restore quarantine unavailable")); + for host in [&*local as &dyn HostClient, &remote as &dyn HostClient] { + let before = binding_bytes(); + let calls = backend.calls.load(std::sync::atomic::Ordering::SeqCst); + assert!(host + .start_agent( + target.sandbox_id, + agent.clone(), + disguised.clone(), + engram_core::traits::SessionFence::unfenced() + ) + .await + .unwrap_err() + .to_string() + .contains("managed session requires acknowledged boot finalization")); + assert_eq!(binding_bytes(), before); + assert_eq!( + backend.calls.load(std::sync::atomic::Ordering::SeqCst), + calls + ); + } + client.client_certificate = dir.join("other.pem"); + client.client_key = dir.join("other.key"); + assert!( + GrpcHostClient::new(connect(&client)).ping().await.is_err(), + "same CA does not replace caller pins" + ); + client.client_certificate = dir.join("coord.pem"); + client.client_key = dir.join("coord.key"); + client.hosts.get_mut(&host_id).unwrap().server_pins = pins("other"); + assert!( + GrpcHostClient::new(connect(&client)).ping().await.is_err(), + "server pin required as well as CA/name" + ); + assert!(client + .endpoint( + host_id, + &format!("http://{addr}"), + tonic::transport::Endpoint::from_shared(format!("http://{addr}")).unwrap() + ) + .is_err()); + let plaintext = tonic::transport::Endpoint::from_shared(format!("http://{addr}")) + .unwrap() + .connect_lazy(); + assert!(GrpcHostClient::new(plaintext).ping().await.is_err()); + client.hosts.get_mut(&host_id).unwrap().server_pins = pins("host"); + client.hosts.get_mut(&host_id).unwrap().server_name = "wrong.example".into(); + assert!(GrpcHostClient::new(connect(&client)).ping().await.is_err()); + client.hosts.get_mut(&host_id).unwrap().server_name = "localhost".into(); + client.hosts.get_mut(&host_id).unwrap().server_ca = dir.join("coord.pem"); + assert!(GrpcHostClient::new(connect(&client)).ping().await.is_err()); + client.hosts.get_mut(&host_id).unwrap().server_ca = dir.join("ca.pem"); + let mut raw = + engram_protocol::grpc::host_service_client::HostServiceClient::new(connect(&client)); + assert!( + raw.ping(engram_protocol::grpc::Empty {}).await.is_err(), + "missing protocol version rejected" + ); + let mut old = tonic::Request::new(engram_protocol::grpc::Empty {}); + old.metadata_mut().insert( + engram_protocol::wire::WIRE_VERSION_METADATA_KEY, + "1".parse().unwrap(), + ); + assert!(raw.ping(old).await.is_err(), "old protocol rejected"); + let no_identity = tonic::transport::Endpoint::from_shared(address.clone()) + .unwrap() + .tls_config( + tonic::transport::ClientTlsConfig::new() + .domain_name("localhost") + .ca_certificate(tonic::transport::Certificate::from_pem( + std::fs::read(dir.join("ca.pem")).unwrap(), + )), + ) + .unwrap() + .connect_lazy(); + assert!(GrpcHostClient::new(no_identity).ping().await.is_err()); + daemon.abort(); + let _ = daemon.await; + server.abort(); + let _ = server.await; +} + +struct EndpointFixture { + egress: Arc, + calls: std::sync::atomic::AtomicUsize, +} +#[async_trait::async_trait] +impl engram_core::traits::SandboxBackend for EndpointFixture { + fn captures_guest_memory(&self) -> bool { + false + } + async fn start_agent( + &self, + _: engram_core::SandboxId, + _: engram_core::types::sandbox::AgentSpec, + ) -> Result<(), engram_core::SandboxError> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + async fn create( + &self, + _: engram_core::SandboxId, + _: engram_core::types::sandbox::SandboxSpec, + ) -> Result<(), engram_core::SandboxError> { + unimplemented!("fixture does not create guests") + } + async fn exec_stream( + &self, + _: engram_core::SandboxId, + _: engram_core::types::sandbox::ExecRequest, + ) -> Result { + unimplemented!("fixture does not execute") + } + async fn snapshot( + &self, + _: engram_core::SandboxId, + ) -> Result { + unimplemented!("fixture does not snapshot") + } + fn snapshot_path_for(&self, _: engram_core::SnapshotId) -> std::path::PathBuf { + std::path::PathBuf::new() + } + async fn restore( + &self, + _: engram_core::SandboxId, + _: engram_core::types::snapshot::SnapshotMetadata, + _: Option, + ) -> Result<(), engram_core::SandboxError> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + async fn restore_base_for_session( + &self, + _: engram_core::SandboxId, + _: engram_core::types::snapshot::SnapshotMetadata, + _: std::collections::HashMap, + _: Vec, + ) -> Result<(), engram_core::SandboxError> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + async fn resume(&self, _: engram_core::SandboxId) -> Result<(), engram_core::SandboxError> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + async fn destroy(&self, _: engram_core::SandboxId) -> Result<(), engram_core::SandboxError> { + Ok(()) + } + async fn list(&self) -> Result, engram_core::SandboxError> { + Ok(vec![]) + } + async fn guest_endpoints( + &self, + _: engram_core::SandboxId, + ) -> Option { + Some(engram_core::types::endpoints::GuestEndpoints { + egress_identity: "10.200.0.42".parse().unwrap(), + dial_ip: "10.200.0.42".parse().unwrap(), + netns: None, + vsock_uds: None, + }) + } + async fn notify_session_policy( + &self, + policy: engram_core::types::egress::SessionEgressPolicy, + ) -> Result<(), engram_core::SandboxError> { + self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.egress + .apply_policy(policy) + .await + .map_err(|e| engram_core::SandboxError::InvalidSpec(e.to_string())) + } +} diff --git a/crates/engram-host-agent/tests/session_disk.rs b/crates/engram-host-agent/tests/session_disk.rs index cdf2f1608..2834d8088 100644 --- a/crates/engram-host-agent/tests/session_disk.rs +++ b/crates/engram-host-agent/tests/session_disk.rs @@ -216,7 +216,9 @@ async fn a_host_agent_restart_keeps_the_guest_and_every_write() { .with_checkpoint_dir(rig.work.path().join("checkpoints")), ); generation_b.set_self_ref(&generation_b); - generation_b.reconcile_volumes().await; + let bindings = + engram_host_agent::bindings::BindingStore::open(rig.work.path().join("bindings")).unwrap(); + generation_b.reconcile_volumes(&bindings).await; assert_eq!( counted.writes.load(Ordering::Relaxed), disk_writes_before, diff --git a/crates/engram-host-core/src/session_volumes.rs b/crates/engram-host-core/src/session_volumes.rs index 8f25d279c..f6e07dcb9 100644 --- a/crates/engram-host-core/src/session_volumes.rs +++ b/crates/engram-host-core/src/session_volumes.rs @@ -163,7 +163,7 @@ impl StagePath { } } -fn transient_name(sandbox: SandboxId, role: DiskRole) -> VolumeName { +pub fn transient_name(sandbox: SandboxId, role: DiskRole) -> VolumeName { VolumeName::new(format!( "{TRANSIENT_PREFIX}{}-{}", sandbox.as_uuid().simple(), diff --git a/crates/engram-postgres/src/lib.rs b/crates/engram-postgres/src/lib.rs index 56e3d41c4..456eb9414 100644 --- a/crates/engram-postgres/src/lib.rs +++ b/crates/engram-postgres/src/lib.rs @@ -34,6 +34,13 @@ use sqlx::postgres::{PgPool, PgPoolOptions}; use sqlx::Row; use uuid::Uuid; +mod managed_boot; +mod managed_bridge; +mod managed_create; +mod managed_operator; +mod managed_operator_gate; +mod mcp_policy; +mod outbox_accept; mod row; #[derive(Clone)] @@ -588,8 +595,7 @@ fn classify_no_fit( } else if free_vcpus < budget_vcpus { "cpu_full" } else { - // Fits at read time — the pick ran earlier under FOR - // UPDATE and lost a race; honest label beats a lie. + // Capacity changed after the locked placement attempt. "fits_now" }; engram_core::traits::PlacementNoFit { @@ -635,20 +641,33 @@ fn best_fit_measured( best.map(|(_, h)| h) } -/// ADR 0046/0048/0081: the shared `FOR UPDATE` 2D pick every reserving -/// placer runs — session create (`reserve_and_persist_create`), the -/// queue scanner (`place_queued_session`), and capture reservation -/// (`place_capture_job` / `reassign_capture_job`). Locks the candidate host rows in PK order -/// so all placers (any replica) serialize on the overlap without -/// deadlock, builds the fit map from host-measured `allocatable_mib` -/// (0 = unmeasured → soft) + the vCPU budget, subtracts the reserved -/// SUM — memory-reserving sessions UNION capturing enable jobs (ADR -/// 0081: sessions and captures are mutually visible) — and best-fit -/// chooses via [`choose_placement_host`]. -/// -/// `Ok(None)` = nothing fits (or no candidates); the caller decides -/// queue-vs-reject and owns the commit/rollback — the lock is held for -/// the REST of the caller's transaction. +/// Placement locks hosts before sessions, including exact claim retries. +async fn lock_placement_host( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + host: HostId, +) -> Result { + Ok(!lock_placement_hosts(tx, &[host.as_uuid()]) + .await? + .is_empty()) +} + +/// Capacity writers serialize without blocking foreign-key KEY SHARE locks. +async fn lock_placement_hosts( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + hosts: &[Uuid], +) -> Result, MetaError> { + sqlx::query_scalar("SELECT id FROM hosts WHERE id=ANY($1) ORDER BY id FOR NO KEY UPDATE") + .bind(hosts) + .fetch_all(&mut **tx) + .await + .map_err(db_err) +} + +/// Locks candidate hosts in ID order, then chooses the best two-dimensional fit. +/// Host-measured memory and CPU budgets include reservations by sessions, +/// captures and teleports. NO KEY UPDATE serializes capacity writers while +/// allowing foreign-key checks from session rebinds. Locks last until commit; +/// the caller decides whether no fit means queueing or refusal. async fn pick_host_2d( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, cand: &[uuid::Uuid], @@ -664,14 +683,9 @@ async fn pick_host_2d( SELECT id, allocatable_mib, total_vcpus FROM hosts WHERE id = ANY($1) AND status IN ('ready','draining') AND NOT cordoned - -- ORDER BY id BEFORE `FOR UPDATE`: every placer (any replica) - -- locks the overlapping host rows in the SAME (PK) order, so a - -- burst can't lock {A,B} vs {B,A} and deadlock. The LockRows - -- executor node sits atop the sort, so rows are locked in id - -- order. (Load test: `deadlock detected` under concurrent - -- creates before this.) + -- Overlapping placers lock rows in the same primary-key order. ORDER BY id - FOR UPDATE + FOR NO KEY UPDATE "#, ) .bind(cand) @@ -1198,7 +1212,17 @@ impl MetadataStore for PostgresStore { &self, req: TeleportAdmitRequest, ) -> Result { + let mut candidates: Vec = req + .pinned_dest + .map(|h| vec![h]) + .unwrap_or(req.candidates) + .into_iter() + .map(|h| h.as_uuid()) + .collect(); let mut tx = self.pool.begin().await.map_err(db_err)?; + // Policy-child creation locks candidate hosts before its parent session. + let locked = lock_placement_hosts(&mut tx, &candidates).await?; + candidates.retain(|host| locked.contains(host)); let Some(session) = sqlx::query("SELECT status, host_id, sandbox_id FROM sessions WHERE id=$1 AND current_epoch=$2 FOR UPDATE") .bind(req.session_id.as_uuid()).bind(req.epoch).fetch_optional(&mut *tx).await.map_err(db_err)? else { return Ok(TeleportAdmitOutcome::Fenced) }; let open: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM session_teleports WHERE session_id=$1 AND phase NOT IN ('done','aborted','failed'))") @@ -1221,20 +1245,7 @@ impl MetadataStore for PostgresStore { "active session has no source binding".into(), )); }; - let mut candidates: Vec = req - .pinned_dest - .map(|h| vec![h]) - .unwrap_or(req.candidates) - .into_iter() - .map(|h| h.as_uuid()) - .filter(|h| *h != source) - .collect(); - // Lock before counting open moves. Every placement writer uses this order. - sqlx::query("SELECT id FROM hosts WHERE id=ANY($1) ORDER BY id FOR UPDATE") - .bind(&candidates) - .fetch_all(&mut *tx) - .await - .map_err(db_err)?; + candidates.retain(|host| *host != source); let capped: Vec = sqlx::query_scalar("SELECT dest_host_id FROM session_teleports WHERE dest_host_id=ANY($1) AND phase NOT IN ('done','aborted','failed') GROUP BY dest_host_id HAVING COUNT(*) >= $2") .bind(&candidates).bind(i64::from(req.max_open_per_dest)).fetch_all(&mut *tx).await.map_err(db_err)?; candidates.retain(|h| req.max_open_per_dest > 0 && !capped.contains(h)); @@ -1444,6 +1455,40 @@ impl MetadataStore for PostgresStore { .map_err(db_err) } + async fn read_mcp_write_policy( + &self, + id: SessionId, + ) -> Result { + self.policy_read(id).await + } + async fn read_mcp_plan_review( + &self, + id: SessionId, + call: &str, + ) -> Result { + self.policy_review(id, call).await + } + async fn accept_mcp_plan_exit( + &self, + input: &engram_core::types::mcp_policy::PlanAcceptance, + ) -> Result { + self.policy_accept(input).await + } + async fn validate_mcp_dispatch_fence( + &self, + ctx: &engram_core::types::mcp_policy::RequestPolicyContext, + expected: &engram_core::types::mcp_policy::ExecutionRequest, + ) -> Result { + self.policy_fence(ctx, expected, true).await + } + async fn validate_mcp_execution_identity( + &self, + ctx: &engram_core::types::mcp_policy::RequestPolicyContext, + expected: &engram_core::types::mcp_policy::ExecutionRequest, + ) -> Result { + self.policy_fence(ctx, expected, false).await + } + async fn create_session(&self, spec: SessionSpec) -> Result { let id = self.entropy.uuid(); let now = self.clock.now_utc(); @@ -1502,27 +1547,248 @@ impl MetadataStore for PostgresStore { Ok(row.try_get::(0).map_err(db_err)? as u64) } + async fn read_operator_publication( + &self, + ) -> Result { + self.operator_read().await + } + async fn stage_operator_publication( + &self, + expected: Option, + candidate: engram_core::types::managed_operator::OperatorCandidate, + ) -> Result { + self.operator_stage(expected, candidate).await + } + async fn list_operator_participants( + &self, + after: Option, + limit: u32, + ) -> Result, MetaError> { + self.operator_list(after, limit).await + } + async fn retain_operator_participant( + &self, + generation: &engram_core::types::managed_operator::OperatorGeneration, + session: SessionId, + ) -> Result { + self.operator_retain(generation, session).await + } + async fn record_operator_readiness( + &self, + generation: &engram_core::types::managed_operator::OperatorGeneration, + participant: &engram_core::types::managed_operator::OperatorParticipant, + proof: engram_core::types::managed_operator::OperatorReadiness, + ) -> Result<(), MetaError> { + self.operator_ready(generation, participant, proof).await + } + async fn publish_operator_generation( + &self, + generation: &engram_core::types::managed_operator::OperatorGeneration, + ) -> Result { + self.operator_publish(generation).await + } + + async fn read_managed_creation_state( + &self, + q: &engram_core::types::managed_bridge::ManagedCreationQuery, + ) -> Result { + self.bridge_state(q).await + } + async fn read_managed_boot_selection( + &self, + q: &engram_core::types::managed_bridge::ManagedCreationQuery, + ) -> Result { + self.bridge_selection(q).await + } + + async fn begin_managed_boot_intent( + &self, + i: engram_core::types::managed_bridge::ManagedBootIntent, + ) -> Result { + self.bridge_begin(i).await + } + async fn read_managed_boot_intent( + &self, + q: &engram_core::types::managed_bridge::ManagedCreationQuery, + op: &str, + ) -> Result, MetaError> { + self.bridge_intent(q, op).await + } + async fn advance_managed_boot_intent( + &self, + id: &str, + r: u64, + c: engram_core::types::managed_bridge::ManagedIntentAdvance, + ) -> Result { + self.bridge_advance(id, r, c).await + } + async fn list_managed_boot_intents( + &self, + a: Option, + l: u32, + ) -> Result, MetaError> { + self.bridge_list(a, l).await + } + async fn get_managed_boot( + &self, + identity: &engram_core::types::managed_create::ManagedCreateIdentity, + ) -> Result, MetaError> { + self.boot_get(identity).await + } + async fn reserve_managed_capture( + &self, + request: &engram_core::types::managed_boot::ManagedCaptureRequest, + ) -> Result, MetaError> { + self.capture_reserve(request).await + } + async fn record_managed_capture( + &self, + authority: &engram_core::types::managed_boot::ManagedCaptureAuthority, + ) -> Result<(), MetaError> { + self.capture_record(authority).await + } + async fn managed_sandbox_open(&self, sandbox: SandboxId) -> Result, MetaError> { + self.boot_sandbox_open(sandbox).await + } + async fn claim_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootPrepare, + ) -> Result { + self.boot_claim(request).await + } + async fn read_managed_boot( + &self, + key: &engram_core::types::managed_boot::ManagedBootKey, + ) -> Result { + self.boot_read(key).await + } + async fn read_managed_boot_for_target( + &self, + target: &engram_core::types::managed_deny::ManagedPolicyTarget, + ) -> Result { + self.boot_read_target(target).await + } + async fn advance_managed_boot( + &self, + key: &engram_core::types::managed_boot::ManagedBootKey, + revision: u64, + event: engram_core::types::managed_boot::ManagedBootAdvance, + ) -> Result { + self.boot_advance(key, revision, event).await + } + async fn list_managed_boots( + &self, + after: Option, + limit: u32, + ) -> Result, MetaError> { + self.boot_list(after, limit).await + } + async fn managed_boot_open(&self, sid: SessionId) -> Result, MetaError> { + self.boot_open(sid).await + } + async fn managed_tool_target_gone( + &self, + sid: SessionId, + call: &str, + ) -> Result { + self.boot_tool_target_gone(sid, call).await + } + async fn managed_delivery_target_gone( + &self, + sid: SessionId, + epoch: u64, + ) -> Result { + self.boot_target_gone(sid, epoch).await + } + async fn read_managed_creation( + &self, + identity: &engram_core::types::managed_create::ManagedCreateIdentity, + ) -> Result { + self.managed_creation(identity, false).await + } + + async fn read_held_managed_creation( + &self, + identity: &engram_core::types::managed_create::ManagedCreateIdentity, + ) -> Result { + self.managed_creation(identity, true).await + } + async fn reserve_and_persist_create( &self, - ws: SessionCreateWriteSet, + mut ws: SessionCreateWriteSet, candidates: &[HostId], affinity_len: usize, ) -> Result { + if ws.oauth_binding.as_ref().is_some_and(|binding| { + binding.key.subject_kind == engram_core::types::oauth::OAuthSubjectKind::Mcp + }) { + return Err(MetaError::Conflict( + "MCP credentials cannot be bound to a guest session".into(), + )); + } + + if let Some(creation) = &ws.managed_creation { + creation.validate_for(&ws)?; + } + if let Some(initial) = &ws.managed_initial_prompt { + initial.validate(&ws)?; + } + if let Some(selection) = &ws.managed_boot_selection { + selection.validate()?; + if ws.managed_creation.is_none() + || selection.image != ws.spec.image + || i64::from(selection.memory_mib) > ws.mem_budget_mib + || i64::from(selection.vcpus) > i64::from(ws.cpu_budget_vcpus) + { + return Err(engram_core::types::managed_bridge::conflict()); + } + } + let now = self.clock.now_utc(); let mut tx = self.pool.begin().await.map_err(db_err)?; + managed_operator::prepare_creation(&mut tx, &mut ws).await?; - // -------- pick a host (ADR 0046/0048 best-fit 2D), if any candidate -------- - // Issue #535 (b): `pick_host_2d` is `reserve_placement`'s FOR-UPDATE - // pick (now shared with `place_queued_session` and ADR 0084's - // `place_capture_job` / `reassign_capture_job`) — extended below so the SAME transaction - // also writes the satellites instead of stopping at the bare row - // insert. The candidate host-row locks are held for the REST of the - // transaction, not just the pick + insert — `tx.commit()` is at the - // bottom of this function, after the sealed-secrets insert, the - // per-capability insert loop, and the integration-policy upsert all - // run on the same `tx`. A many-capability create serializes - // concurrent placers on that whole multi-round-trip critical - // section, not a sub-ms window. + if let Some(creation) = &ws.managed_creation { + if !managed_create::reserve(&mut tx, creation).await? { + let selection: Option = sqlx::query_scalar( + "SELECT selection FROM session_managed_boot_selections WHERE session_id=$1", + ) + .bind(ws.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let expected = ws + .managed_boot_selection + .as_ref() + .map(serde_json::to_value) + .transpose() + .map_err(|_| engram_core::types::managed_bridge::conflict())?; + if selection != expected { + return Err(engram_core::types::managed_bridge::conflict()); + } + let prior: Option = sqlx::query_scalar("SELECT payload FROM session_outbox WHERE prompt_id=$1 AND session_id=$2 AND kind='prompt'") + .bind(format!("create:{}",ws.session_id)).bind(ws.session_id.as_uuid()).fetch_optional(&mut *tx).await.map_err(db_err)?; + if prior.as_ref() != ws.managed_initial_prompt.as_ref().map(|p| &p.row.payload) { + return Err(MetaError::Conflict("managed initial prompt changed".into())); + } + tx.commit().await.map_err(db_err)?; + return Ok(CreateDisposition::Queued); + } + } + let candidates = if ws.managed_creation.is_some() { + &[][..] + } else { + candidates + }; + let affinity_len = if ws.managed_creation.is_some() { + 0 + } else { + affinity_len + }; + + // Creation satellites and the capacity reservation share one transaction. + // The host lock lasts until all satellites are written and committed. let cand: Vec = candidates.iter().map(|h| h.as_uuid()).collect(); let picked: Option = pick_host_2d( &mut tx, @@ -1593,6 +1859,32 @@ impl MetadataStore for PostgresStore { } }; + if let Some(policy) = &ws.mcp_policy { + if policy.actor_id.is_empty() + || policy.input_id.is_empty() + || policy.parent_session_id == Some(ws.session_id) + { + return Err(MetaError::Conflict("invalid policy creation".into())); + } + if let Some(parent) = policy.parent_session_id { + mcp_policy::nodes(&mut tx, parent).await?; + } + sqlx::query( + "UPDATE sessions SET mcp_write_policy=$2,mcp_policy_parent_id=$3 WHERE id=$1", + ) + .bind(ws.session_id.as_uuid()) + .bind( + serde_json::to_value(engram_core::types::mcp_policy::StoredWritePolicy::initial( + policy, + )) + .map_err(|_| MetaError::Conflict("policy encode failed".into()))?, + ) + .bind(policy.parent_session_id.map(|id| id.as_uuid())) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + // -------- satellites, in the SAME transaction (issue #535 (b)) -------- // `harness` already rode the row INSERT above (mirrored from the // RuntimeSpec); the remaining satellites keep their own tables (FK'd @@ -1701,6 +1993,27 @@ impl MetadataStore for PostgresStore { } } + if let Some(selection) = &ws.managed_boot_selection { + sqlx::query( + "INSERT INTO session_managed_boot_selections(session_id,selection) VALUES($1,$2)", + ) + .bind(ws.session_id.as_uuid()) + .bind( + serde_json::to_value(selection) + .map_err(|_| engram_core::types::managed_bridge::conflict())?, + ) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query("UPDATE chunk_generation SET generation=generation+1 WHERE id=TRUE") + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + if let Some(initial) = &ws.managed_initial_prompt { + outbox_accept::accept(&mut tx, now, ws.session_id, &initial.events, &initial.row) + .await?; + } tx.commit().await.map_err(db_err)?; if matches!(disposition, CreateDisposition::Queued) { // Closes the race where capacity frees between the failed @@ -1781,6 +2094,22 @@ impl MetadataStore for PostgresStore { &self, sealed: engram_core::types::org_secret::SealedOrgSecret, ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + sqlx::query("INSERT INTO org_secret_purposes (name) VALUES ($1) ON CONFLICT DO NOTHING") + .bind(&sealed.name) + .execute(&mut *tx) + .await + .map_err(db_err)?; + let owner: Option = sqlx::query_scalar( + "SELECT connection_id FROM org_secret_purposes WHERE name=$1 FOR UPDATE", + ) + .bind(&sealed.name) + .fetch_one(&mut *tx) + .await + .map_err(db_err)?; + if owner.is_some() { + return Err(MetaError::Conflict("credential is reserved for MCP".into())); + } let (name, key_id, created_at, updated_at): ( String, String, @@ -1788,8 +2117,8 @@ impl MetadataStore for PostgresStore { chrono::DateTime, ) = sqlx::query_as( r#" - INSERT INTO org_secrets (name, wrapped_dek, nonce, ciphertext, key_id, updated_at) - VALUES ($1, $2, $3, $4, $5, $6) + INSERT INTO org_secrets (name, wrapped_dek, nonce, ciphertext, key_id, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $6) ON CONFLICT (name) DO UPDATE SET wrapped_dek = EXCLUDED.wrapped_dek, nonce = EXCLUDED.nonce, @@ -1805,9 +2134,10 @@ impl MetadataStore for PostgresStore { .bind(&sealed.ciphertext) .bind(&sealed.key_id) .bind(self.clock.now_utc()) - .fetch_one(&self.pool) + .fetch_one(&mut *tx) .await .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; // Wake every replica's mint broker so a rotated cred rebuilds its // engine (ADR 0057 C2). Best-effort: the value is already committed, // so a notify failure must not fail the write. @@ -1873,12 +2203,23 @@ impl MetadataStore for PostgresStore { } async fn delete_org_secret(&self, name: &str) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + sqlx::query("SELECT name FROM org_secret_purposes WHERE name=$1 FOR UPDATE") + .bind(name) + .execute(&mut *tx) + .await + .map_err(db_err)?; let res = sqlx::query("DELETE FROM org_secrets WHERE name = $1") .bind(name) - .execute(&self.pool) + .execute(&mut *tx) .await .map_err(db_err)?; let deleted = res.rows_affected() > 0; + if deleted { + sqlx::query("UPDATE org_secret_purposes SET generation=generation+1 WHERE name=$1 AND connection_id IS NOT NULL") + .bind(name).execute(&mut *tx).await.map_err(db_err)?; + } + tx.commit().await.map_err(db_err)?; if deleted { let _ = sqlx::query("SELECT pg_notify('org_secret_changed', $1)") .bind(name) @@ -1888,78 +2229,104 @@ impl MetadataStore for PostgresStore { Ok(deleted) } + async fn reserve_mcp_org_secret( + &self, + name: &str, + connection_id: &str, + role: engram_core::types::org_secret::McpSecretRole, + ) -> Result { + if name.is_empty() || connection_id.is_empty() { + return Err(MetaError::Conflict("invalid MCP reservation".into())); + } + let mut tx = self.pool.begin().await.map_err(db_err)?; + sqlx::query("INSERT INTO org_secret_purposes (name, connection_id, role) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING") + .bind(name).bind(connection_id).bind(role.as_str()).execute(&mut *tx).await.map_err(db_err)?; + let (owner, stored_role, generation): (Option, Option, i64) = sqlx::query_as("SELECT connection_id, role, generation FROM org_secret_purposes WHERE name=$1 FOR UPDATE") + .bind(name).fetch_one(&mut *tx).await.map_err(db_err)?; + if owner.as_deref() != Some(connection_id) || stored_role.as_deref() != Some(role.as_str()) + { + return Err(MetaError::Conflict("credential purpose conflict".into())); + } + sqlx::query("SELECT pg_notify('org_secret_changed', $1)") + .bind(name) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(engram_core::types::org_secret::McpSecretReservation { + name: name.into(), + connection_id: connection_id.into(), + role, + generation, + }) + } + + async fn get_mcp_org_secret_reservation( + &self, + name: &str, + ) -> Result, MetaError> { + use engram_core::types::org_secret::{McpSecretReservation, McpSecretRole}; + let row: Option<(String,String,i64)> = sqlx::query_as("SELECT connection_id, role, generation FROM org_secret_purposes WHERE name=$1 AND connection_id IS NOT NULL") + .bind(name).fetch_optional(&self.pool).await.map_err(db_err)?; + row.map(|(connection_id, role, generation)| { + Ok(McpSecretReservation { + name: name.into(), + connection_id, + role: match role.as_str() { + "api_key" => McpSecretRole::ApiKey, + "client_secret" => McpSecretRole::ClientSecret, + _ => return Err(MetaError::Conflict("invalid credential role".into())), + }, + generation, + }) + }) + .transpose() + } + + async fn put_mcp_org_secret( + &self, + sealed: engram_core::types::org_secret::SealedOrgSecret, + expected: &engram_core::types::org_secret::McpSecretReservation, + ) -> Result { + if sealed.name != expected.name { + return Err(MetaError::Conflict("credential reservation changed".into())); + } + let mut tx = self.pool.begin().await.map_err(db_err)?; + let generation: Option = sqlx::query_scalar("UPDATE org_secret_purposes SET generation=generation+1 WHERE name=$1 AND connection_id=$2 AND role=$3 AND generation=$4 RETURNING generation") + .bind(&sealed.name).bind(&expected.connection_id).bind(expected.role.as_str()).bind(expected.generation) + .fetch_optional(&mut *tx).await.map_err(db_err)?; + let generation = generation + .ok_or_else(|| MetaError::Conflict("credential reservation changed".into()))?; + sqlx::query("INSERT INTO org_secrets (name,wrapped_dek,nonce,ciphertext,key_id,created_at,updated_at) VALUES ($1,$2,$3,$4,$5,$6,$6) ON CONFLICT (name) DO UPDATE SET wrapped_dek=$2,nonce=$3,ciphertext=$4,key_id=$5,updated_at=$6") + .bind(&sealed.name).bind(&sealed.wrapped_dek).bind(&sealed.nonce).bind(&sealed.ciphertext).bind(&sealed.key_id).bind(self.clock.now_utc()) + .execute(&mut *tx).await.map_err(db_err)?; + sqlx::query("SELECT pg_notify('org_secret_changed', $1)") + .bind(&sealed.name) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(engram_core::types::org_secret::McpSecretReservation { + generation, + ..expected.clone() + }) + } + // ---- ADR 0106: subject-scoped OAuth credentials and flow state ---- async fn put_oauth_credential( &self, credential: engram_core::types::oauth::NewSealedOAuthCredential, - expected_version: Option, - ) -> Result { - let now = self.clock.now_utc(); - let metadata = serde_json::to_value(&credential.metadata) - .map_err(|e| MetaError::Serialization(e.to_string()))?; - let row = if let Some(expected) = expected_version { - sqlx::query( - r#" - UPDATE oauth_credentials SET - wrapped_dek = $4, nonce = $5, ciphertext = $6, key_id = $7, - account_metadata = $8, version = version + 1, - updated_at = $9, revoked_at = NULL, expires_at = $11, - refresh_claim_until = NULL, broken_at = NULL, broken_reason = NULL - WHERE subject_kind = $1 AND subject_id = $2 AND provider = $3 - AND version = $10 - RETURNING * - "#, - ) - .bind(credential.key.subject_kind.as_str()) - .bind(&credential.key.subject_id) - .bind(&credential.key.provider) - .bind(&credential.wrapped_dek) - .bind(&credential.nonce) - .bind(&credential.ciphertext) - .bind(&credential.key_id) - .bind(metadata) - .bind(now) - .bind(expected) - .bind(credential.expires_at) - .fetch_optional(&self.pool) - .await - .map_err(db_err)? - } else { - sqlx::query( - r#" - INSERT INTO oauth_credentials ( - subject_kind, subject_id, provider, wrapped_dek, nonce, - ciphertext, key_id, account_metadata, version, created_at, - updated_at, revoked_at, expires_at - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,1,$9,$9,NULL,$10) - ON CONFLICT (subject_kind, subject_id, provider) DO NOTHING - RETURNING * - "#, - ) - .bind(credential.key.subject_kind.as_str()) - .bind(&credential.key.subject_id) - .bind(&credential.key.provider) - .bind(&credential.wrapped_dek) - .bind(&credential.nonce) - .bind(&credential.ciphertext) - .bind(&credential.key_id) - .bind(metadata) - .bind(now) - .bind(credential.expires_at) - .fetch_optional(&self.pool) - .await - .map_err(db_err)? - }; - match row { - Some(row) => oauth_credential_from_pg(&row), - None => Err(MetaError::Conflict(format!( - "OAuth credential CAS rejected for {}/{}/{}", - credential.key.subject_kind.as_str(), - credential.key.subject_id, - credential.key.provider - ))), - } + expected_version: Option, + ) -> Result { + let mut conn = self.pool.acquire().await.map_err(db_err)?; + put_oauth_credential_on( + &mut conn, + self.clock.now_utc(), + credential, + expected_version, + ) + .await } async fn get_oauth_credential( @@ -2044,6 +2411,160 @@ impl MetadataStore for PostgresStore { } } + async fn claim_mcp_refresh( + &self, + claim: &engram_core::types::mcp::McpRefreshClaim, + ) -> Result { + let now = self.clock.now_utc(); + if claim.key.subject_kind != engram_core::types::oauth::OAuthSubjectKind::Mcp + || claim.expires_at <= now + || claim.expires_at > now + chrono::Duration::seconds(30) + { + return Err(MetaError::Conflict("invalid refresh claim".into())); + } + let r = sqlx::query("UPDATE oauth_credentials SET mcp_refresh_claim=$4, refresh_claim_until=$5, broken_at=$6, broken_reason='refresh_pending' WHERE subject_kind='mcp' AND subject_id=$1 AND provider=$2 AND version=$3 AND revoked_at IS NULL AND broken_at IS NULL") + .bind(&claim.key.subject_id).bind(&claim.key.provider).bind(claim.version).bind(claim.id).bind(claim.expires_at).bind(now).execute(&self.pool).await.map_err(db_err)?; + Ok(r.rows_affected() == 1) + } + async fn complete_mcp_refresh( + &self, + credential: engram_core::types::oauth::NewSealedOAuthCredential, + claim: &engram_core::types::mcp::McpRefreshClaim, + ) -> Result { + if credential.key != claim.key + || claim.key.subject_kind != engram_core::types::oauth::OAuthSubjectKind::Mcp + { + return Err(MetaError::Conflict("refresh claim lost".into())); + } + let metadata = serde_json::to_value(&credential.metadata) + .map_err(|e| MetaError::Serialization(e.to_string()))?; + let row = sqlx::query("UPDATE oauth_credentials SET wrapped_dek=$6,nonce=$7,ciphertext=$8,key_id=$9,account_metadata=$10,expires_at=$11,version=version+1,updated_at=$12,broken_at=NULL,broken_reason=NULL,refresh_claim_until=NULL,mcp_refresh_claim=NULL WHERE subject_kind='mcp' AND subject_id=$1 AND provider=$2 AND version=$3 AND mcp_refresh_claim=$4 AND refresh_claim_until=$5 AND refresh_claim_until>$12 AND revoked_at IS NULL AND broken_reason='refresh_pending' RETURNING *") + .bind(&claim.key.subject_id).bind(&claim.key.provider).bind(claim.version).bind(claim.id).bind(claim.expires_at) + .bind(credential.wrapped_dek).bind(credential.nonce).bind(credential.ciphertext).bind(credential.key_id).bind(metadata).bind(credential.expires_at).bind(self.clock.now_utc()).fetch_optional(&self.pool).await.map_err(db_err)?.ok_or_else(|| MetaError::Conflict("refresh claim lost".into()))?; + oauth_credential_from_pg(&row) + } + async fn acquire_mcp_oauth_admission( + &self, + lease: &engram_core::types::mcp::McpOAuthAdmission, + ) -> Result { + let now = self.clock.now_utc(); + if lease.deployment_id.is_empty() + || lease.connection_id.is_empty() + || lease.expires_at <= now + || lease.expires_at > now + chrono::Duration::seconds(30) + { + return Err(MetaError::Conflict("invalid OAuth admission".into())); + } + sqlx::query("DELETE FROM mcp_oauth_admission WHERE expires_at<=$1") + .bind(now) + .execute(&self.pool) + .await + .map_err(db_err)?; + // The fixed slot primary key and connection uniqueness bound concurrent replicas. + for slot in 0..4i16 { + let r = sqlx::query("INSERT INTO mcp_oauth_admission (deployment_id,slot,id,connection_id,expires_at) VALUES ($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING") + .bind(&lease.deployment_id).bind(slot).bind(lease.id).bind(&lease.connection_id).bind(lease.expires_at).execute(&self.pool).await.map_err(db_err)?; + if r.rows_affected() == 1 { + return Ok(true); + } + } + Ok(false) + } + async fn release_mcp_oauth_admission( + &self, + lease: &engram_core::types::mcp::McpOAuthAdmission, + ) -> Result<(), MetaError> { + sqlx::query("DELETE FROM mcp_oauth_admission WHERE id=$1 AND deployment_id=$2 AND connection_id=$3 AND expires_at=$4") + .bind(lease.id).bind(&lease.deployment_id).bind(&lease.connection_id).bind(lease.expires_at).execute(&self.pool).await.map_err(db_err)?; + Ok(()) + } + + async fn create_mcp_oauth_flow( + &self, + flow: engram_core::types::oauth::OAuthFlow, + state: engram_core::types::mcp::McpSealedState, + ) -> Result<(), MetaError> { + use engram_core::types::oauth::{OAuthFlowStatus, OAuthSubjectKind}; + if flow.key.subject_kind != OAuthSubjectKind::Mcp + || flow.status != OAuthFlowStatus::Pending + || flow.expires_at <= self.clock.now_utc() + { + return Err(MetaError::Conflict("MCP flow creation rejected".into())); + } + let state = + serde_json::to_value(state).map_err(|e| MetaError::Serialization(e.to_string()))?; + let result = sqlx::query("INSERT INTO oauth_flows (id,subject_kind,subject_id,provider,owner_replica,lease_expires_at,expires_at,status,created_at,updated_at,mcp_state) VALUES ($1,'mcp',$2,$3,$4,$5,$6,'pending',$7,$8,$9) ON CONFLICT DO NOTHING") + .bind(flow.id).bind(&flow.key.subject_id).bind(&flow.key.provider).bind(&flow.owner_replica).bind(flow.lease_expires_at).bind(flow.expires_at).bind(flow.created_at).bind(flow.updated_at).bind(state).execute(&self.pool).await.map_err(db_err)?; + if result.rows_affected() != 1 { + return Err(MetaError::Conflict("MCP flow creation rejected".into())); + } + Ok(()) + } + async fn consume_mcp_oauth_flow( + &self, + id: uuid::Uuid, + key: &engram_core::types::oauth::OAuthCredentialKey, + ) -> Result { + if key.subject_kind != engram_core::types::oauth::OAuthSubjectKind::Mcp { + return Err(MetaError::Conflict("MCP state unavailable".into())); + } + let mut tx = self.pool.begin().await.map_err(db_err)?; + let state: Option = sqlx::query_scalar("SELECT mcp_state FROM oauth_flows WHERE id=$1 AND subject_kind='mcp' AND subject_id=$2 AND provider=$3 AND status='pending' AND expires_at>$4 AND mcp_state IS NOT NULL FOR UPDATE") + .bind(id).bind(&key.subject_id).bind(&key.provider).bind(self.clock.now_utc()).fetch_optional(&mut *tx).await.map_err(db_err)?; + let state = state.ok_or_else(|| MetaError::Conflict("MCP state unavailable".into()))?; + sqlx::query("UPDATE oauth_flows SET mcp_state=NULL WHERE id=$1") + .bind(id) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + serde_json::from_value(state).map_err(|e| MetaError::Serialization(e.to_string())) + } + + async fn cancel_mcp_oauth_flow( + &self, + id: uuid::Uuid, + key: &engram_core::types::oauth::OAuthCredentialKey, + ) -> Result<(), MetaError> { + if key.subject_kind != engram_core::types::oauth::OAuthSubjectKind::Mcp { + return Err(MetaError::Conflict("MCP flow cancellation rejected".into())); + } + let result = sqlx::query("UPDATE oauth_flows SET status=CASE WHEN status='pending' THEN 'cancelled' ELSE status END, mcp_state=NULL, updated_at=$4 WHERE id=$1 AND subject_kind='mcp' AND subject_id=$2 AND provider=$3") + .bind(id).bind(&key.subject_id).bind(&key.provider).bind(self.clock.now_utc()) + .execute(&self.pool).await.map_err(db_err)?; + if result.rows_affected() != 1 { + return Err(MetaError::Conflict("MCP flow cancellation rejected".into())); + } + Ok(()) + } + async fn complete_mcp_oauth_flow( + &self, + id: uuid::Uuid, + credential: engram_core::types::oauth::NewSealedOAuthCredential, + expected_version: Option, + ) -> Result { + if credential.key.subject_kind != engram_core::types::oauth::OAuthSubjectKind::Mcp { + return Err(MetaError::Conflict("MCP flow publication rejected".into())); + } + let mut tx = self.pool.begin().await.map_err(db_err)?; + let now = self.clock.now_utc(); + let locked: Option = sqlx::query_scalar("SELECT id FROM oauth_flows WHERE id=$1 AND subject_kind='mcp' AND subject_id=$2 AND provider=$3 AND status='pending' AND mcp_state IS NULL AND expires_at>$4 FOR UPDATE") + .bind(id).bind(&credential.key.subject_id).bind(&credential.key.provider).bind(now) + .fetch_optional(&mut *tx).await.map_err(db_err)?; + if locked.is_none() { + return Err(MetaError::Conflict("MCP flow publication rejected".into())); + } + let row = put_oauth_credential_on(&mut tx, now, credential, expected_version).await?; + sqlx::query("UPDATE oauth_flows SET status='succeeded', updated_at=$2 WHERE id=$1") + .bind(id) + .bind(now) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(row) + } + async fn create_oauth_flow( &self, flow: engram_core::types::oauth::OAuthFlow, @@ -2366,6 +2887,8 @@ impl MetadataStore for PostgresStore { now: chrono::DateTime, delete_before: chrono::DateTime, ) -> Result { + sqlx::query("UPDATE oauth_flows SET mcp_state=NULL WHERE mcp_state IS NOT NULL AND (status<>'pending' OR expires_at<=$1 OR lease_expires_at<=$1)") + .bind(now).execute(&self.pool).await.map_err(db_err)?; let mut tx = self.pool.begin().await.map_err(db_err)?; let changed = sqlx::query( r#" @@ -2489,7 +3012,7 @@ impl MetadataStore for PostgresStore { COALESCE(cpu_budget_vcpus, 0) AS cpu_budget_vcpus, queue_origin, queued_at FROM sessions - WHERE status = 'queued' + WHERE status = 'queued' AND NOT EXISTS (SELECT 1 FROM session_managed_creations c WHERE c.session_id=sessions.id) ORDER BY queued_at ASC "#, ) @@ -2513,8 +3036,7 @@ impl MetadataStore for PostgresStore { let cand: Vec = candidates.iter().map(|h| h.as_uuid()).collect(); let now = self.clock.now_utc(); let mut tx = self.pool.begin().await.map_err(db_err)?; - // Same FOR UPDATE serialization + 2D fit as every reserving placer - // (shared `pick_host_2d`, ADR 0046/0048/0081). + // Capacity accounting uses the same row lock as every placement writer. let Some(picked) = pick_host_2d( &mut tx, &cand, @@ -2533,7 +3055,7 @@ impl MetadataStore for PostgresStore { r#" UPDATE sessions SET status = 'pending', host_id = $2, last_active_at = $3 - WHERE id = $1 AND status = 'queued' + WHERE id = $1 AND status = 'queued' AND NOT EXISTS (SELECT 1 FROM session_managed_creations c WHERE c.session_id=sessions.id) "#, ) .bind(id.as_uuid()) @@ -2706,11 +3228,8 @@ impl MetadataStore for PostgresStore { Ok(row.map(SessionId)) } - /// Diagnostic twin of `pick_host_2d`, without the `FOR UPDATE`: same - /// fit-map construction (same status/cordon predicate, same reserved - /// SUM incl. capture jobs — R3 #722: a `pending` reserves - /// unconditionally, no crash-orphan gate), then the pure - /// [`classify_no_fit`]. Runs only on the no-capacity path. + /// Builds the placement fit map without row locks, then classifies refusal. + /// Runs only on the no-capacity path. async fn placement_no_fit_details( &self, candidates: &[HostId], @@ -3482,6 +4001,13 @@ impl MetadataStore for PostgresStore { } let mut tx = self.pool.begin().await.map_err(db_err)?; + sqlx::query("SELECT id FROM sessions WHERE id=$1 FOR UPDATE") + .bind(session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + crate::managed_boot::require_tool_result_target(&mut tx, outbox).await?; let inserted = sqlx::query( r#" INSERT INTO session_outbox @@ -3577,121 +4103,12 @@ impl MetadataStore for PostgresStore { events: &[(String, serde_json::Value)], outbox: &engram_core::types::outbox::OutboxRow, ) -> Result>, MetaError> { - if outbox.session_id != session_id { - return Err(MetaError::Serialization( - "event/outbox session ids do not match".into(), - )); - } let mut tx = self.pool.begin().await.map_err(db_err)?; - let inserted = sqlx::query( - r#" - INSERT INTO session_outbox - (prompt_id, session_id, kind, payload, created_at, not_before) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (prompt_id) DO NOTHING - RETURNING prompt_id - "#, - ) - .bind(&outbox.prompt_id) - .bind(session_id.as_uuid()) - .bind(outbox.kind.as_str()) - .bind(&outbox.payload) - .bind(outbox.created_at) - .bind(outbox.not_before) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)? - .is_some(); - - if !inserted { - // A retry of the SAME command is the designed no-op; a - // prompt_id claimed by a DIFFERENT command — including the - // same kind with different text/mode (review finding on - // #993: payload must be part of the identity, as it is in - // `append_session_event_and_outbox`) — is a Conflict, never - // a silent drop. - let existing = sqlx::query( - "SELECT session_id, kind, payload FROM session_outbox WHERE prompt_id = $1", - ) - .bind(&outbox.prompt_id) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)? - .ok_or_else(|| { - MetaError::Conflict(format!( - "outbox id {} disappeared during accept", - outbox.prompt_id - )) - })?; - let existing_session: uuid::Uuid = existing.try_get("session_id").map_err(db_err)?; - let existing_kind: String = existing.try_get("kind").map_err(db_err)?; - let existing_payload: serde_json::Value = - existing.try_get("payload").map_err(db_err)?; - if existing_session != session_id.as_uuid() - || existing_kind != outbox.kind.as_str() - || existing_payload != outbox.payload - { - return Err(MetaError::Conflict(format!( - "outbox id {} belongs to another command", - outbox.prompt_id - ))); - } - tx.rollback().await.map_err(db_err)?; - return Ok(None); - } - - let mut idxs = Vec::with_capacity(events.len()); - for (kind, payload) in events { - let event = sqlx::query( - r#" - WITH next AS ( - UPDATE sessions - SET next_event_idx = next_event_idx + 1, - updated_at = $4, - last_event_at = $4 - WHERE id = $1 - RETURNING next_event_idx - 1 AS allocated_idx, recovery_epoch - ), - inserted AS ( - INSERT INTO session_events (session_id, idx, kind, payload, recovery_epoch, created_at) - SELECT $1, allocated_idx, $2, $3, recovery_epoch, $4 FROM next - RETURNING idx - ) - SELECT idx FROM inserted - "#, - ) - .bind(session_id.as_uuid()) - .bind(kind) - .bind(payload) - .bind(self.clock.now_utc()) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)? - .ok_or(MetaError::NotFound)?; - idxs.push(event.try_get::("idx").map_err(db_err)?); - } - - // Notifications fire at commit: replicas see the events and the - // outbox row together, so delivery can never observe the row - // without its echo (the ADR 0052 echo-before-run_started - // ordering, now transactional instead of sequenced). - for idx in &idxs { - sqlx::query("SELECT pg_notify('session_events', $1)") - .bind( - serde_json::json!({ "session_id": session_id.to_string(), "idx": idx }) - .to_string(), - ) - .execute(&mut *tx) - .await - .map_err(db_err)?; - } - sqlx::query("SELECT pg_notify('session_outbox', $1)") - .bind(session_id.to_string()) - .execute(&mut *tx) - .await - .map_err(db_err)?; + let result = + outbox_accept::accept(&mut tx, self.clock.now_utc(), session_id, events, outbox) + .await?; tx.commit().await.map_err(db_err)?; - Ok(Some(idxs)) + Ok(result) } async fn outbox_due_sessions(&self) -> Result, MetaError> { @@ -5301,6 +5718,13 @@ impl MetadataStore for PostgresStore { Ok(Some(settled)) } + async fn append_managed_harness_event( + &self, + event: &engram_core::types::event::ManagedHarnessEvent, + ) -> Result { + self.managed_event_append(event).await + } + async fn append_session_event_idempotent( &self, session: SessionId, @@ -7643,24 +8067,83 @@ impl MetadataStore for PostgresStore { // ---- session integration policy (ADR 0056 B′) ---- + async fn managed_policy_target( + &self, + session_id: SessionId, + ) -> Result { + use sqlx::Row; + let row = sqlx::query("SELECT host_id, sandbox_id, binding_epoch FROM sessions WHERE id=$1 AND host_id IS NOT NULL AND sandbox_id IS NOT NULL AND binding_epoch > 0 AND status NOT IN ('dead','failed','completed')") + .bind(session_id.as_uuid()).fetch_optional(&self.pool).await.map_err(db_err)?.ok_or(MetaError::NotFound)?; + Ok(engram_core::types::managed_deny::ManagedPolicyTarget { + session_id, + host_id: HostId(row.try_get("host_id").map_err(db_err)?), + sandbox_id: SandboxId(row.try_get("sandbox_id").map_err(db_err)?), + binding_epoch: u64::try_from(row.try_get::("binding_epoch").map_err(db_err)?) + .map_err(|e| MetaError::Serialization(e.to_string()))?, + }) + } + async fn retain_managed_deny( + &self, + target: &engram_core::types::managed_deny::ManagedPolicyTarget, + policy: &engram_core::types::managed_deny::ManagedDenyPolicy, + ) -> Result<(), MetaError> { + use sqlx::Row; + let mut tx = self.pool.begin().await.map_err(db_err)?; + let pending = managed_operator::policy_registration_lock(&mut tx).await?; + let row = sqlx::query("SELECT host_id, sandbox_id, binding_epoch FROM sessions WHERE id=$1 AND status NOT IN ('dead','failed','completed') FOR UPDATE") + .bind(target.session_id.as_uuid()).fetch_optional(&mut *tx).await.map_err(db_err)?.ok_or(MetaError::NotFound)?; + if row + .try_get::, _>("host_id") + .map_err(db_err)? + != Some(target.host_id.as_uuid()) + || row + .try_get::, _>("sandbox_id") + .map_err(db_err)? + != Some(target.sandbox_id.as_uuid()) + || row.try_get::("binding_epoch").map_err(db_err)? + != i64::try_from(target.binding_epoch) + .map_err(|e| MetaError::Serialization(e.to_string()))? + { + return Err(MetaError::Conflict("managed placement changed".into())); + } + let old: Option = sqlx::query_scalar( + "SELECT policy_json FROM session_integration_policy WHERE session_id=$1", + ) + .bind(target.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let next = engram_core::types::managed_deny::retain_in_policy_json(old.as_deref(), policy) + .map_err(MetaError::Conflict)?; + managed_operator::check_registration(pending, old.as_deref(), &next)?; + sqlx::query("INSERT INTO session_integration_policy (session_id,policy_json) VALUES ($1,$2) ON CONFLICT (session_id) DO UPDATE SET policy_json=EXCLUDED.policy_json").bind(target.session_id.as_uuid()).bind(next).execute(&mut *tx).await.map_err(db_err)?; + tx.commit().await.map_err(db_err) + } async fn bind_session_integration_policy( &self, session_id: SessionId, policy_json: &str, ) -> Result<(), MetaError> { - sqlx::query( - r#" - INSERT INTO session_integration_policy (session_id, policy_json) - VALUES ($1, $2) - ON CONFLICT (session_id) DO UPDATE SET policy_json = EXCLUDED.policy_json - "#, + let mut tx = self.pool.begin().await.map_err(db_err)?; + let pending = managed_operator::policy_registration_lock(&mut tx).await?; + sqlx::query("SELECT id FROM sessions WHERE id=$1 FOR UPDATE") + .bind(session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + let old: Option = sqlx::query_scalar( + "SELECT policy_json FROM session_integration_policy WHERE session_id=$1", ) .bind(session_id.as_uuid()) - .bind(policy_json) - .execute(&self.pool) + .fetch_optional(&mut *tx) .await .map_err(db_err)?; - Ok(()) + managed_operator::check_registration(pending, old.as_deref(), policy_json)?; + engram_core::types::managed_deny::validate_policy_replacement(old.as_deref(), policy_json) + .map_err(MetaError::Conflict)?; + sqlx::query("INSERT INTO session_integration_policy (session_id,policy_json) VALUES ($1,$2) ON CONFLICT (session_id) DO UPDATE SET policy_json=EXCLUDED.policy_json").bind(session_id.as_uuid()).bind(policy_json).execute(&mut *tx).await.map_err(db_err)?; + tx.commit().await.map_err(db_err) } async fn get_session_integration_policy( @@ -8553,6 +9036,37 @@ impl MetadataStore for PostgresStore { Ok(()) } + /// Managed cold inputs and capture reservations are roots without snapshot rows. + async fn list_managed_disk_manifests( + &self, + ) -> Result, MetaError> { + let rows = sqlx::query_as::<_, (Uuid, i64)>( + "SELECT (record->'request'->'cold'->'disk'->'manifest'->>'manifest_id')::UUID, + (record->'request'->'cold'->'disk'->'manifest'->>'version')::BIGINT + FROM session_managed_boots + UNION SELECT capture_id, 1 FROM session_managed_captures + UNION SELECT (selection->'base_manifest'->>'manifest_id')::UUID, + (selection->'base_manifest'->>'version')::BIGINT + FROM session_managed_boot_selections + UNION SELECT (record->'disk_manifest'->>'manifest_id')::UUID, + (record->'disk_manifest'->>'version')::BIGINT + FROM session_managed_boot_intents", + ) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + rows.into_iter() + .map(|(manifest_id, version)| { + Ok(engram_core::types::manifest::ManifestRef { + manifest_id, + version: u64::try_from(version).map_err(|_| { + MetaError::Serialization("invalid managed manifest version".into()) + })?, + }) + }) + .collect() + } + /// Pin-set sources #3 and #8: root and swap refs from recoverable /// snapshots. Return each distinct pair, including swap-only rows. async fn list_recoverable_snapshot_disk_manifests( @@ -8995,7 +9509,9 @@ impl MetadataStore for PostgresStore { UNION SELECT b->>'drive_id', b->>'sha256' FROM hosts h, jsonb_array_elements(h.current_bundles) AS b - WHERE h.status IN ('ready','draining')", + WHERE h.status IN ('ready','draining') + UNION SELECT b->>'drive_id',b->>'sha256' FROM session_managed_boot_selections,jsonb_array_elements(selection->'aux_ro_drives') AS b + UNION SELECT b->>'drive_id',b->>'sha256' FROM session_managed_boot_intents,jsonb_array_elements(record->'selection'->'aux_ro_drives') AS b", ) .fetch_all(&self.pool) .await @@ -9543,3 +10059,75 @@ async fn append_teleport_finished_tx( } Ok(()) } + +async fn put_oauth_credential_on( + conn: &mut sqlx::PgConnection, + now: chrono::DateTime, + credential: engram_core::types::oauth::NewSealedOAuthCredential, + expected_version: Option, +) -> Result { + let metadata = serde_json::to_value(&credential.metadata) + .map_err(|e| MetaError::Serialization(e.to_string()))?; + let row = if let Some(expected) = expected_version { + sqlx::query( + r#" + UPDATE oauth_credentials SET + wrapped_dek = $4, nonce = $5, ciphertext = $6, key_id = $7, + account_metadata = $8, version = version + 1, + updated_at = $9, revoked_at = NULL, expires_at = $11, + refresh_claim_until = NULL, broken_at = NULL, broken_reason = NULL + WHERE subject_kind = $1 AND subject_id = $2 AND provider = $3 + AND version = $10 + RETURNING * + "#, + ) + .bind(credential.key.subject_kind.as_str()) + .bind(&credential.key.subject_id) + .bind(&credential.key.provider) + .bind(&credential.wrapped_dek) + .bind(&credential.nonce) + .bind(&credential.ciphertext) + .bind(&credential.key_id) + .bind(metadata) + .bind(now) + .bind(expected) + .bind(credential.expires_at) + .fetch_optional(&mut *conn) + .await + .map_err(db_err)? + } else { + sqlx::query( + r#" + INSERT INTO oauth_credentials ( + subject_kind, subject_id, provider, wrapped_dek, nonce, + ciphertext, key_id, account_metadata, version, created_at, + updated_at, revoked_at, expires_at + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,1,$9,$9,NULL,$10) + ON CONFLICT (subject_kind, subject_id, provider) DO NOTHING + RETURNING * + "#, + ) + .bind(credential.key.subject_kind.as_str()) + .bind(&credential.key.subject_id) + .bind(&credential.key.provider) + .bind(&credential.wrapped_dek) + .bind(&credential.nonce) + .bind(&credential.ciphertext) + .bind(&credential.key_id) + .bind(metadata) + .bind(now) + .bind(credential.expires_at) + .fetch_optional(&mut *conn) + .await + .map_err(db_err)? + }; + match row { + Some(row) => oauth_credential_from_pg(&row), + None => Err(MetaError::Conflict(format!( + "OAuth credential CAS rejected for {}/{}/{}", + credential.key.subject_kind.as_str(), + credential.key.subject_id, + credential.key.provider + ))), + } +} diff --git a/crates/engram-postgres/src/managed_boot.rs b/crates/engram-postgres/src/managed_boot.rs new file mode 100644 index 000000000..ffa0a92a3 --- /dev/null +++ b/crates/engram-postgres/src/managed_boot.rs @@ -0,0 +1,610 @@ +//! One session-row lock per transition. Host calls never occur in a transaction. +use crate::{db_err, managed_create, pick_host_2d, PostgresStore}; +use engram_core::{ + types::{managed_boot::*, managed_cold::ManagedColdSource}, + MetaError, SessionId, +}; +use sqlx::{PgConnection, Row}; + +pub(crate) async fn read( + conn: &mut PgConnection, + key: &ManagedBootKey, +) -> Result { + key.validate()?; + managed_create::read(conn, &key.creation, false).await?; + let r = sqlx::query("SELECT b.record,s.sandbox_id AS current_sandbox FROM session_managed_boots b JOIN sessions s ON s.id=b.session_id WHERE b.session_id=$1 AND s.host_id=b.host_id AND (s.sandbox_id=b.sandbox_id OR s.sandbox_id IS NULL) AND s.binding_epoch=b.binding_epoch AND s.status NOT IN ('dead','failed','completed')") + .bind(key.creation.session_id.as_uuid()).fetch_optional(&mut *conn).await.map_err(db_err)?.ok_or_else(conflict)?; + let record: ManagedBootRecord = + serde_json::from_value(r.try_get("record").map_err(db_err)?).map_err(|_| conflict())?; + if &record.request.key != key { + return Err(conflict()); + } + if r.try_get::, _>("current_sandbox") + .map_err(db_err)? + .is_none() + { + if record.phase != ManagedBootPhase::Interrupted { + return Err(conflict()); + } + record.retired.as_ref().ok_or_else(conflict)?.validate_for( + record.prepared.as_ref().ok_or_else(conflict)?, + &record.request.bootstrap_policy, + )?; + } + Ok(record) +} +impl PostgresStore { + pub(crate) async fn boot_claim( + &self, + request: ManagedBootPrepare, + ) -> Result { + let spec = request.spec()?; + let k = &request.key; + let t = &k.context.target; + let mut tx = self.pool.begin().await.map_err(db_err)?; + crate::managed_operator_gate::require(&mut tx, k).await?; + let host_present = crate::lock_placement_host(&mut tx, t.host_id).await?; + let s = sqlx::query("SELECT status,host_id,sandbox_id,binding_epoch,image_uri,mem_budget_mib,cpu_budget_vcpus FROM sessions WHERE id=$1 FOR UPDATE") + .bind(t.session_id.as_uuid()).fetch_optional(&mut *tx).await.map_err(db_err)?.ok_or(MetaError::NotFound)?; + if spec.image != s.try_get::("image_uri").map_err(db_err)? + || spec.cpu.vcpus == 0 + || i64::from(spec.cpu.vcpus) + > i64::from(s.try_get::("cpu_budget_vcpus").map_err(db_err)?) + || spec.memory.max_mib == 0 + || i64::from(spec.memory.max_mib) + > s.try_get::("mem_budget_mib").map_err(db_err)? + { + return Err(conflict()); + } + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM session_managed_boots WHERE session_id=$1)", + ) + .bind(t.session_id.as_uuid()) + .fetch_one(&mut *tx) + .await + .map_err(db_err)?; + let intent: Option = sqlx::query_scalar( + "SELECT record FROM session_managed_boot_intents WHERE session_id=$1 AND intent_id=$2", + ) + .bind(t.session_id.as_uuid()) + .bind(&k.boot_attempt) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + if let Some(v) = intent { + let i: engram_core::types::managed_bridge::ManagedBootIntent = + serde_json::from_value(v).map_err(|_| conflict())?; + i.validate_claim(&request)?; + } else { + let selected: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM session_managed_boot_selections WHERE session_id=$1)", + ) + .bind(t.session_id.as_uuid()) + .fetch_one(&mut *tx) + .await + .map_err(db_err)?; + if selected { + return Err(conflict()); + } + } + let old = if exists { + let value: serde_json::Value = + sqlx::query_scalar("SELECT record FROM session_managed_boots WHERE session_id=$1") + .bind(t.session_id.as_uuid()) + .fetch_one(&mut *tx) + .await + .map_err(db_err)?; + let old: ManagedBootRecord = serde_json::from_value(value).map_err(|_| conflict())?; + let old = read(&mut tx, &old.request.key).await?; + if old.request == request && old.phase != ManagedBootPhase::Interrupted { + tx.commit().await.map_err(db_err)?; + return Ok(old); + } + if old.phase != ManagedBootPhase::Interrupted { + return Err(conflict()); + } + let ManagedColdSource::Capture { capture_id, .. } = request.cold.disk.source else { + return Err(conflict()); + }; + let value: Option=sqlx::query_scalar("SELECT authority FROM session_managed_captures WHERE capture_id=$1 AND session_id=$2 AND binding_epoch=$3").bind(capture_id.as_uuid()).bind(t.session_id.as_uuid()).bind(old.request.key.context.target.binding_epoch as i64).fetch_optional(&mut *tx).await.map_err(db_err)?.flatten(); + let capture: ManagedCaptureAuthority = + serde_json::from_value(value.ok_or_else(conflict)?).map_err(|_| conflict())?; + capture.validate_recovery(&request)?; + if request.capture_authority.as_ref() != Some(&capture) { + return Err(MetaError::Conflict("capture authority mismatch".into())); + } + Some(old) + } else { + None + }; + let source = managed_create::read(&mut tx, &k.creation, old.is_none()).await?; + if source.native.digest != k.native_digest + || (old.is_none() + && (t.binding_epoch != 1 + || !matches!( + request.cold.disk.source, + ManagedColdSource::FreshImage { .. } + ))) + { + return Err(conflict()); + } + let prior: Option = sqlx::query_scalar( + "SELECT policy_json FROM session_integration_policy WHERE session_id=$1", + ) + .bind(t.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let retained = engram_core::types::managed_deny::retain_in_policy_json( + prior.as_deref(), + &request.bootstrap_policy, + ) + .map_err(MetaError::Conflict)?; + if !host_present { + return Err(conflict()); + } + let picked = pick_host_2d( + &mut tx, + &[t.host_id.as_uuid()], + 0, + s.try_get("mem_budget_mib").map_err(db_err)?, + i64::from(s.try_get::("cpu_budget_vcpus").map_err(db_err)?), + ) + .await?; + if picked != Some(t.host_id.as_uuid()) { + return Err(conflict()); + } + sqlx::query("UPDATE sessions SET status='created',host_id=$2,sandbox_id=$3,binding_epoch=$4,queued_at=NULL,queue_origin=NULL,updated_at=$5,last_active_at=$5 WHERE id=$1") + .bind(t.session_id.as_uuid()).bind(t.host_id.as_uuid()).bind(t.sandbox_id.as_uuid()).bind(t.binding_epoch as i64).bind(self.clock.now_utc()).execute(&mut *tx).await.map_err(db_err)?; + sqlx::query("UPDATE chunk_generation SET generation=generation+1 WHERE id=TRUE") + .execute(&mut *tx) + .await + .map_err(db_err)?; + let record = ManagedBootRecord::claimed(request.clone()); + sqlx::query("INSERT INTO session_managed_boots(session_id,host_id,sandbox_id,binding_epoch,boot_attempt,revision,phase,record,updated_at) VALUES($1,$2,$3,$4,$5,1,'claimed',$6,$7) ON CONFLICT(session_id) DO UPDATE SET host_id=EXCLUDED.host_id,sandbox_id=EXCLUDED.sandbox_id,binding_epoch=EXCLUDED.binding_epoch,boot_attempt=EXCLUDED.boot_attempt,revision=1,phase='claimed',record=EXCLUDED.record,updated_at=EXCLUDED.updated_at") + .bind(t.session_id.as_uuid()).bind(t.host_id.as_uuid()).bind(t.sandbox_id.as_uuid()).bind(t.binding_epoch as i64).bind(&k.boot_attempt).bind(serde_json::to_value(&record).map_err(|_| conflict())?).bind(self.clock.now_utc()).execute(&mut *tx).await.map_err(db_err)?; + sqlx::query("INSERT INTO session_integration_policy(session_id,policy_json) VALUES($1,$2) ON CONFLICT(session_id) DO UPDATE SET policy_json=EXCLUDED.policy_json").bind(t.session_id.as_uuid()).bind(retained).execute(&mut *tx).await.map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(record) + } + pub(crate) async fn boot_read( + &self, + key: &ManagedBootKey, + ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + sqlx::query("SELECT id FROM sessions WHERE id=$1 FOR SHARE") + .bind(key.creation.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + let record = read(&mut tx, key).await?; + tx.commit().await.map_err(db_err)?; + Ok(record) + } + pub(crate) async fn boot_advance( + &self, + key: &ManagedBootKey, + revision: u64, + event: ManagedBootAdvance, + ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + if matches!( + &event, + ManagedBootAdvance::Finalize { .. } + | ManagedBootAdvance::IssueStart(_) + | ManagedBootAdvance::Started(_) + ) { + crate::managed_operator_gate::require(&mut tx, key).await?; + } + let sid = key.creation.session_id; + let policy = engram_core::types::mcp_policy::compose_policy( + &crate::mcp_policy::nodes(&mut tx, sid).await?, + )?; + let old = read(&mut tx, key).await?; + if old.revision != revision { + return Err(conflict()); + } + if let ManagedBootAdvance::Prepared(receipt) = &event { + let v:Option=sqlx::query_scalar("SELECT record FROM session_managed_boot_intents WHERE intent_id=$1 AND session_id=$2").bind(&key.boot_attempt).bind(sid.as_uuid()).fetch_optional(&mut *tx).await.map_err(db_err)?; + if let Some(v) = v { + let i: engram_core::types::managed_bridge::ManagedBootIntent = + serde_json::from_value(v).map_err(|_| conflict())?; + i.validate_claim(&receipt.request)?; + if i.assets.as_ref().map(|a| &a.host_incarnation) != Some(&receipt.host_incarnation) + { + return Err(conflict()); + } + } + } + let source = managed_create::read(&mut tx, &key.creation, false).await?; + match &event { + ManagedBootAdvance::Finalize { write_policy, .. } if write_policy != &policy => { + return Err(conflict()) + } + ManagedBootAdvance::IssueStart(_) | ManagedBootAdvance::Started(_) + if old.write_policy.as_ref() != Some(&policy) => + { + return Err(conflict()) + } + _ => {} + } + let next = old.advance(event, &source.native)?; + if let Some(retired) = &next.retired { + let prior: Option = sqlx::query_scalar( + "SELECT policy_json FROM session_integration_policy WHERE session_id=$1", + ) + .bind(sid.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let retained = engram_core::types::managed_deny::retain_retired_in_policy_json( + prior.as_deref(), + &retired.retained_policy, + ) + .map_err(MetaError::Conflict)?; + sqlx::query("INSERT INTO session_integration_policy(session_id,policy_json) VALUES($1,$2) ON CONFLICT(session_id) DO UPDATE SET policy_json=EXCLUDED.policy_json").bind(sid.as_uuid()).bind(retained).execute(&mut *tx).await.map_err(db_err)?; + } + if next.phase == ManagedBootPhase::Interrupted && old.phase != ManagedBootPhase::Interrupted + { + sqlx::query("INSERT INTO session_managed_interrupted_targets(session_id,binding_epoch,boot_attempt,interrupted_at,event_floor) SELECT $1,$2,$3,$4,next_event_idx FROM sessions WHERE id=$1 ON CONFLICT DO NOTHING").bind(sid.as_uuid()).bind(key.context.target.binding_epoch as i64).bind(&key.boot_attempt).bind(self.clock.now_utc()).execute(&mut *tx).await.map_err(db_err)?; + sqlx::query("UPDATE session_outbox SET acked_at=$2 WHERE session_id=$1 AND kind='tool_result' AND acked_at IS NULL").bind(sid.as_uuid()).bind(self.clock.now_utc()).execute(&mut *tx).await.map_err(db_err)?; + let node = &crate::mcp_policy::nodes(&mut tx, sid).await?[0]; + let floor: i64 = sqlx::query_scalar("SELECT next_event_idx FROM sessions WHERE id=$1") + .bind(sid.as_uuid()) + .fetch_one(&mut *tx) + .await + .map_err(db_err)?; + let policy = engram_core::types::mcp_policy::StoredWritePolicy::restrict( + node.policy.as_ref(), + &format!("cold:{}", key.boot_attempt), + floor, + )?; + sqlx::query("UPDATE sessions SET mcp_write_policy=$2 WHERE id=$1") + .bind(sid.as_uuid()) + .bind(serde_json::to_value(policy).map_err(|_| conflict())?) + .execute(&mut *tx) + .await + .map_err(db_err)?; + let latest=sqlx::query("SELECT kind,payload FROM session_events WHERE session_id=$1 AND rewound_at IS NULL AND kind IN ('run_started','run_completed','run_interrupted') ORDER BY idx DESC LIMIT 1").bind(sid.as_uuid()).fetch_optional(&mut *tx).await.map_err(db_err)?; + if let Some(event) = latest { + let payload: serde_json::Value = event.try_get("payload").map_err(db_err)?; + if event.try_get::("kind").map_err(db_err)? == "run_started" { + if let Some(run) = payload.get("run_id").and_then(|v| v.as_str()) { + crate::append_event_idempotent_tx( + &mut tx, sid, &format!("managed-interrupt:{}:{run}", key.boot_attempt), + "run_interrupted", + serde_json::json!({"type":"harness_run_interrupted","run_id":run,"at":self.clock.now_utc()}), + self.clock.now_utc(), + ).await?; + } + } + } + } + if next.phase == ManagedBootPhase::Started { + sqlx::query( + "UPDATE sessions SET status='active',last_active_at=$2,updated_at=$2 WHERE id=$1", + ) + .bind(sid.as_uuid()) + .bind(self.clock.now_utc()) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + sqlx::query("UPDATE session_managed_boots SET revision=$2,phase=$3,record=$4,updated_at=$5 WHERE session_id=$1").bind(sid.as_uuid()).bind(i64::try_from(next.revision).map_err(|_|conflict())?).bind(next.phase.as_str()).bind(serde_json::to_value(&next).map_err(|_|conflict())?).bind(self.clock.now_utc()).execute(&mut *tx).await.map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(next) + } + pub(crate) async fn boot_list( + &self, + after: Option, + limit: u32, + ) -> Result, MetaError> { + let rows: Vec = sqlx::query_scalar("SELECT record FROM session_managed_boots WHERE phase IN ('claimed','prepared','finalizing','finalized','start_issued','started') AND ($2::UUID IS NULL OR session_id > $2) ORDER BY session_id LIMIT $1").bind(i64::from(limit.min(128))).bind(after.map(|id|id.as_uuid())).fetch_all(&self.pool).await.map_err(db_err)?; + rows.into_iter() + .map(|v| { + serde_json::from_value::(v) + .map(|r| r.request.key) + .map_err(|_| conflict()) + }) + .collect() + } + pub(crate) async fn boot_open(&self, sid: SessionId) -> Result, MetaError> { + sqlx::query_scalar("SELECT COALESCE(b.phase='started' AND EXISTS(SELECT 1 FROM managed_operator_generation_gate g WHERE g.singleton AND g.admission_open AND g.published_generation=(b.record->'request'->'key'->'context'->>'operator_generation')::BIGINT AND g.published_identity=b.record->'request'->'key'->'context'->>'operator_identity') AND s.status='active' AND s.host_id=b.host_id AND s.sandbox_id=b.sandbox_id AND s.binding_epoch=b.binding_epoch AND s.mcp_policy_parent_id IS NOT DISTINCT FROM c.parent_session_id,false) FROM session_managed_creations c JOIN sessions s ON s.id=c.session_id LEFT JOIN session_managed_boots b ON b.session_id=c.session_id WHERE c.session_id=$1").bind(sid.as_uuid()).fetch_optional(&self.pool).await.map_err(db_err) + } + pub(crate) async fn boot_target_gone( + &self, + sid: SessionId, + epoch: u64, + ) -> Result { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM session_managed_interrupted_targets WHERE session_id=$1 AND binding_epoch=$2)").bind(sid.as_uuid()).bind(i64::try_from(epoch).map_err(|_|conflict())?).fetch_one(&self.pool).await.map_err(db_err) + } +} + +impl PostgresStore { + pub(crate) async fn boot_read_target( + &self, + target: &engram_core::types::managed_deny::ManagedPolicyTarget, + ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + let sandbox: Option = + sqlx::query_scalar("SELECT sandbox_id FROM sessions WHERE id=$1 FOR SHARE") + .bind(target.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + if sandbox != Some(target.sandbox_id.as_uuid()) { + return Err(conflict()); + } + let value: serde_json::Value = + sqlx::query_scalar("SELECT record FROM session_managed_boots WHERE session_id=$1") + .bind(target.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or_else(conflict)?; + let row: ManagedBootRecord = serde_json::from_value(value).map_err(|_| conflict())?; + if row.request.key.context.target != *target { + return Err(conflict()); + } + let result = read(&mut tx, &row.request.key).await?; + tx.commit().await.map_err(db_err)?; + Ok(result) + } + pub(crate) async fn boot_get( + &self, + identity: &engram_core::types::managed_create::ManagedCreateIdentity, + ) -> Result, MetaError> { + let mut tx = self.pool.begin().await.map_err(db_err)?; + sqlx::query("SELECT id FROM sessions WHERE id=$1 FOR SHARE") + .bind(identity.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + managed_create::read(&mut tx, identity, false).await?; + let value: Option = + sqlx::query_scalar("SELECT record FROM session_managed_boots WHERE session_id=$1") + .bind(identity.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let Some(value) = value else { + return Ok(None); + }; + let row: ManagedBootRecord = serde_json::from_value(value).map_err(|_| conflict())?; + let result = read(&mut tx, &row.request.key).await?; + tx.commit().await.map_err(db_err)?; + Ok(Some(result)) + } + pub(crate) async fn capture_reserve( + &self, + request: &ManagedCaptureRequest, + ) -> Result, MetaError> { + request.key.validate()?; + if request.capture_id.as_uuid().is_nil() { + return Err(conflict()); + } + let mut tx = self.pool.begin().await.map_err(db_err)?; + let sid = request.key.creation.session_id; + sqlx::query("SELECT id FROM sessions WHERE id=$1 FOR UPDATE") + .bind(sid.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + if let Some(prior) = sqlx::query("SELECT request,authority FROM session_managed_captures WHERE capture_id=$1 AND authority IS NOT NULL") + .bind(request.capture_id.as_uuid()).fetch_optional(&mut *tx).await.map_err(db_err)? { + let original:ManagedCaptureRequest=serde_json::from_value(prior.try_get("request").map_err(db_err)?).map_err(|_|conflict())?; + if &original != request { return Err(conflict()); } + managed_create::read(&mut tx,&request.key.creation,false).await?; + let authority=serde_json::from_value(prior.try_get("authority").map_err(db_err)?).map_err(|_|conflict())?; + tx.commit().await.map_err(db_err)?; return Ok(Some(authority)); + } + let boot = read(&mut tx, &request.key).await?; + if boot.phase != ManagedBootPhase::Interrupted || boot.prepared.is_none() { + return Err(conflict()); + } + sqlx::query("INSERT INTO session_managed_captures(capture_id,session_id,binding_epoch,source_sandbox_id,request,created_at) VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT DO NOTHING").bind(request.capture_id.as_uuid()).bind(sid.as_uuid()).bind(request.key.context.target.binding_epoch as i64).bind(request.key.context.target.sandbox_id.as_uuid()).bind(serde_json::to_value(request).map_err(|_|conflict())?).bind(self.clock.now_utc()).execute(&mut *tx).await.map_err(db_err)?; + sqlx::query("UPDATE chunk_generation SET generation=generation+1 WHERE id=TRUE") + .execute(&mut *tx) + .await + .map_err(db_err)?; + let row = sqlx::query( + "SELECT request,authority FROM session_managed_captures WHERE capture_id=$1", + ) + .bind(request.capture_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or_else(conflict)?; + let original: ManagedCaptureRequest = + serde_json::from_value(row.try_get("request").map_err(db_err)?) + .map_err(|_| conflict())?; + if original != *request { + return Err(conflict()); + } + let authority: Option = row.try_get("authority").map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + authority + .map(|v| serde_json::from_value(v).map_err(|_| conflict())) + .transpose() + } + pub(crate) async fn capture_record( + &self, + authority: &ManagedCaptureAuthority, + ) -> Result<(), MetaError> { + authority.validate()?; + let request = &authority.request; + let sid = request.key.creation.session_id; + let mut tx = self.pool.begin().await.map_err(db_err)?; + sqlx::query("SELECT id FROM sessions WHERE id=$1 FOR UPDATE") + .bind(sid.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + let boot = read(&mut tx, &request.key).await?; + if boot.phase != ManagedBootPhase::Interrupted + || boot.prepared.as_ref().map(|r| &r.host_incarnation) + != Some(&authority.host_incarnation) + || boot.request.cold.image != authority.image + || boot.request.cold.kernel_sha256 != authority.kernel_sha256 + || boot.request.cold.aux_ro_drives != authority.aux_ro_drives + { + return Err(conflict()); + } + let row = sqlx::query( + "SELECT request,authority FROM session_managed_captures WHERE capture_id=$1 FOR UPDATE", + ) + .bind(request.capture_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or_else(conflict)?; + let original: ManagedCaptureRequest = + serde_json::from_value(row.try_get("request").map_err(db_err)?) + .map_err(|_| conflict())?; + let prior: Option = row.try_get("authority").map_err(db_err)?; + let value = serde_json::to_value(authority).map_err(|_| conflict())?; + if original != *request || prior.as_ref().is_some_and(|p| p != &value) { + return Err(conflict()); + } + sqlx::query( + "UPDATE session_managed_captures SET authority=$2,record_digest=$3 WHERE capture_id=$1", + ) + .bind(request.capture_id.as_uuid()) + .bind(value) + .bind(&authority.record_digest) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query( + "UPDATE sessions SET status='idle',last_active_at=$2,updated_at=$2 WHERE id=$1", + ) + .bind(sid.as_uuid()) + .bind(self.clock.now_utc()) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(()) + } + pub(crate) async fn boot_sandbox_open( + &self, + sandbox: engram_core::SandboxId, + ) -> Result, MetaError> { + let sid: Option = + sqlx::query_scalar("SELECT session_id FROM session_managed_boots WHERE sandbox_id=$1") + .bind(sandbox.as_uuid()) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + if let Some(sid) = sid { + return self.boot_open(sid.into()).await; + } + let old: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM session_managed_captures WHERE source_sandbox_id=$1)", + ) + .bind(sandbox.as_uuid()) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + Ok(old.then_some(false)) + } +} + +impl PostgresStore { + pub(crate) async fn boot_tool_target_gone( + &self, + sid: SessionId, + call: &str, + ) -> Result { + sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM session_managed_creations WHERE session_id=$1) AND COALESCE((SELECT min(e.idx) FROM session_events e WHERE e.session_id=$1 AND e.kind='tool_call_requested' AND e.payload->>'tool_call_id'=$2),-1) < COALESCE((SELECT max(event_floor) FROM session_managed_interrupted_targets WHERE session_id=$1),0)").bind(sid.as_uuid()).bind(call).fetch_one(&self.pool).await.map_err(db_err) + } +} + +/// Called while holding the session lock, before any durable result write. +pub(crate) async fn require_tool_result_target( + conn: &mut PgConnection, + row: &engram_core::types::outbox::OutboxRow, +) -> Result<(), MetaError> { + if row.kind != engram_core::types::outbox::OutboxKind::ToolResult { + return Ok(()); + } + let call = row + .payload + .get("tool_call_id") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + let gone:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM session_managed_creations WHERE session_id=$1) AND COALESCE((SELECT min(e.idx) FROM session_events e WHERE e.session_id=$1 AND e.kind='tool_call_requested' AND e.payload->>'tool_call_id'=$2),-1) < COALESCE((SELECT max(event_floor) FROM session_managed_interrupted_targets WHERE session_id=$1),0)").bind(row.session_id.as_uuid()).bind(call).fetch_one(conn).await.map_err(db_err)?; + if gone { + return Err(MetaError::Conflict("delivery target gone".into())); + } + Ok(()) +} + +impl PostgresStore { + pub(crate) async fn managed_event_append( + &self, + event: &engram_core::types::event::ManagedHarnessEvent, + ) -> Result { + event.validate()?; + let sid = event.target.session_id; + let mut tx = self.pool.begin().await.map_err(db_err)?; + // Read only to select the generation lock. Authority is checked again + // after the session lock, which also serializes interruption and append. + let raw: serde_json::Value = + sqlx::query_scalar("SELECT record FROM session_managed_boots WHERE session_id=$1") + .bind(sid.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or_else(conflict)?; + let candidate: ManagedBootRecord = serde_json::from_value(raw).map_err(|_| conflict())?; + crate::managed_operator_gate::require(&mut tx, &candidate.request.key).await?; + let active: bool = + sqlx::query_scalar("SELECT status='active' FROM sessions WHERE id=$1 FOR UPDATE") + .bind(sid.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or_else(conflict)?; + let current = read(&mut tx, &candidate.request.key).await?; + let interrupted: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM session_managed_interrupted_targets WHERE session_id=$1 AND binding_epoch=$2)") + .bind(sid.as_uuid()).bind(event.target.binding_epoch as i64).fetch_one(&mut *tx).await.map_err(db_err)?; + if !active + || interrupted + || current.phase != ManagedBootPhase::Started + || current.request.key.context.target != event.target + { + return Err(conflict()); + } + let key = event.key(); + if let Some(previous) = sqlx::query("SELECT kind,payload FROM session_events WHERE session_id=$1 AND payload->>'idempotency_key'=$2") + .bind(sid.as_uuid()).bind(&key).fetch_optional(&mut *tx).await.map_err(db_err)? { + if !event.matches_replay(&previous.try_get::("kind").map_err(db_err)?, &previous.try_get("payload").map_err(db_err)?) { + return Err(conflict()); + } + } + let now = self.clock.now_utc(); + let idx = crate::append_event_idempotent_tx( + &mut tx, + sid, + &key, + &event.kind, + event.payload.clone(), + now, + ) + .await?; + let outbox_acked = if let Some(ack) = &event.ack_id { + sqlx::query("UPDATE session_outbox SET acked_at=$3 WHERE session_id=$1 AND prompt_id=$2 AND acked_at IS NULL") + .bind(sid.as_uuid()).bind(ack).bind(now).execute(&mut *tx).await.map_err(db_err)?.rows_affected() > 0 + } else { + false + }; + tx.commit().await.map_err(db_err)?; + Ok(engram_core::types::event::ManagedHarnessEventAccepted { idx, outbox_acked }) + } +} diff --git a/crates/engram-postgres/src/managed_bridge.rs b/crates/engram-postgres/src/managed_bridge.rs new file mode 100644 index 000000000..2ea5f9b6d --- /dev/null +++ b/crates/engram-postgres/src/managed_bridge.rs @@ -0,0 +1,284 @@ +//! Held-input and intent CAS. Gate, host, then session locks; no external I/O. +use crate::{db_err, managed_create, PostgresStore}; +use engram_core::{ + types::{managed_boot::ManagedBootRecord, managed_bridge::*, session::SessionState}, + MetaError, SessionId, +}; +use sqlx::{PgConnection, Row}; +fn decode(v: serde_json::Value) -> Result { + serde_json::from_value(v).map_err(|_| conflict()) +} +async fn gate(c: &mut PgConnection) -> Result<(), MetaError> { + let row=sqlx::query("SELECT admission_open,published_generation,published_identity FROM managed_operator_generation_gate WHERE singleton FOR SHARE").fetch_one(c).await.map_err(db_err)?; + if !row.try_get::("admission_open").map_err(db_err)? + || row + .try_get::, _>("published_generation") + .map_err(db_err)? + .is_none() + || row + .try_get::, _>("published_identity") + .map_err(db_err)? + .is_none() + { + return Err(conflict()); + } + Ok(()) +} +async fn lock(c: &mut PgConnection, sid: SessionId) -> Result<(), MetaError> { + sqlx::query("SELECT id FROM sessions WHERE id=$1 FOR UPDATE") + .bind(sid.as_uuid()) + .fetch_optional(c) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + Ok(()) +} + +async fn selection( + c: &mut PgConnection, + q: &ManagedCreationQuery, +) -> Result { + q.matches(&managed_create::read_with_terminal(c, &q.identity, false, true).await?)?; + let v = sqlx::query_scalar( + "SELECT selection FROM session_managed_boot_selections WHERE session_id=$1", + ) + .bind(q.identity.session_id.as_uuid()) + .fetch_optional(c) + .await + .map_err(db_err)? + .ok_or_else(conflict)?; + let s: ManagedBootSelection = decode(v)?; + s.validate()?; + Ok(s) +} +async fn intent( + c: &mut PgConnection, + q: &ManagedCreationQuery, + op: &str, +) -> Result, MetaError> { + q.matches(&managed_create::read_with_terminal(c, &q.identity, false, true).await?)?; + let v: Option = sqlx::query_scalar( + "SELECT record FROM session_managed_boot_intents WHERE session_id=$1 AND operation_key=$2", + ) + .bind(q.identity.session_id.as_uuid()) + .bind(op) + .fetch_optional(c) + .await + .map_err(db_err)?; + v.map(decode).transpose() +} +impl PostgresStore { + pub(crate) async fn bridge_state( + &self, + q: &ManagedCreationQuery, + ) -> Result { + q.validate()?; + let mut tx = self.pool.begin().await.map_err(db_err)?; + let status: Option = + sqlx::query_scalar("SELECT status FROM sessions WHERE id=$1 FOR SHARE") + .bind(q.identity.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM session_managed_creations WHERE session_id=$1)", + ) + .bind(q.identity.session_id.as_uuid()) + .fetch_one(&mut *tx) + .await + .map_err(db_err)?; + let creation = if exists { + Some(managed_create::read_with_terminal(&mut tx, &q.identity, false, true).await?) + } else { + None + }; + let b: Option = + sqlx::query_scalar("SELECT record FROM session_managed_boots WHERE session_id=$1") + .bind(q.identity.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let i:Option=sqlx::query_scalar("SELECT record FROM session_managed_boot_intents WHERE session_id=$1 ORDER BY binding_epoch DESC LIMIT 1").bind(q.identity.session_id.as_uuid()).fetch_optional(&mut *tx).await.map_err(db_err)?; + let i: Option = i.map(decode).transpose()?; + let status = status + .map(|s| decode::(s.into())) + .transpose()?; + let b: Option = b.map(decode).transpose()?; + if status.is_some_and(|s| !s.is_terminal()) { + if let Some(b) = &b { + crate::managed_boot::read(&mut tx, &b.request.key).await?; + } + } + let state = + ManagedCreationState::from_parts(q.clone(), status, creation.as_ref(), b, i.as_ref())?; + tx.commit().await.map_err(db_err)?; + Ok(state) + } + pub(crate) async fn bridge_selection( + &self, + q: &ManagedCreationQuery, + ) -> Result { + selection(&mut *self.pool.acquire().await.map_err(db_err)?, q).await + } + + pub(crate) async fn bridge_intent( + &self, + q: &ManagedCreationQuery, + op: &str, + ) -> Result, MetaError> { + intent(&mut *self.pool.acquire().await.map_err(db_err)?, q, op).await + } + pub(crate) async fn bridge_begin( + &self, + i: ManagedBootIntent, + ) -> Result { + i.validate()?; + if i.revision != 1 || i.host_id.is_some() || i.assets.is_some() || i.completed { + return Err(conflict()); + } + let mut tx = self.pool.begin().await.map_err(db_err)?; + gate(&mut tx).await?; + lock(&mut tx, i.query.identity.session_id).await?; + let original = selection(&mut tx, &i.query).await?; + if let Some(old) = intent(&mut tx, &i.query, &i.operation_key()).await? { + if old.recovery.as_ref().map(|r| (&r.recovery_id, &r.source)) + != i.recovery.as_ref().map(|r| (&r.recovery_id, &r.source)) + { + return Err(conflict()); + } + tx.commit().await.map_err(db_err)?; + return Ok(old); + } + managed_create::read(&mut tx, &i.query.identity, false).await?; + let collision: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM session_managed_boot_intents WHERE intent_id=$1 OR (session_id=$2 AND (binding_epoch=$3 OR NOT completed)))") + .bind(&i.intent_id).bind(i.query.identity.session_id.as_uuid()).bind(i.binding_epoch as i64) + .fetch_one(&mut *tx).await.map_err(db_err)?; + if collision { + return Err(conflict()); + } + let b: Option = + sqlx::query_scalar("SELECT record FROM session_managed_boots WHERE session_id=$1") + .bind(i.query.identity.session_id.as_uuid()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let b: Option = b.map(decode).transpose()?; + if let Some(r) = &i.recovery { + let old = b.as_ref().ok_or_else(conflict)?; + if old.request.key != r.source + || old.prepared.is_none() + || i.selection.image != old.request.cold.image + || i.selection.aux_ro_drives != old.request.cold.aux_ro_drives + || i.selection.launch != original.launch + || i.selection.image_digest != original.image_digest + || i.selection.base_manifest != original.base_manifest + || i.selection.vcpus != old.request.spec()?.cpu.vcpus + || i.selection.memory_mib != old.request.spec()?.memory.max_mib + || i.selection.disk_gib != old.request.spec()?.disk.max_gib + { + return Err(conflict()); + } + } else { + managed_create::read(&mut tx, &i.query.identity, true).await?; + if b.is_some() || i.selection != original { + return Err(conflict()); + } + } + sqlx::query("INSERT INTO session_managed_boot_intents(intent_id,session_id,operation_key,binding_epoch,revision,record,created_at,updated_at) VALUES($1,$2,$3,$4,1,$5,$6,$6)").bind(&i.intent_id).bind(i.query.identity.session_id.as_uuid()).bind(i.operation_key()).bind(i.binding_epoch as i64).bind(serde_json::to_value(&i).map_err(|_|conflict())?).bind(self.clock.now_utc()).execute(&mut *tx).await.map_err(db_err)?; + sqlx::query("UPDATE chunk_generation SET generation=generation+1 WHERE id=TRUE") + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(i) + } + pub(crate) async fn bridge_advance( + &self, + id: &str, + revision: u64, + c: ManagedIntentAdvance, + ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + gate(&mut tx).await?; + let sid: uuid::Uuid = sqlx::query_scalar( + "SELECT session_id FROM session_managed_boot_intents WHERE intent_id=$1", + ) + .bind(id) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or_else(conflict)?; + let placement_host = match &c { + ManagedIntentAdvance::SelectHost(host) => { + Some((*host, crate::lock_placement_host(&mut tx, *host).await?)) + } + _ => None, + }; + lock(&mut tx, sid.into()).await?; + let v = sqlx::query_scalar( + "SELECT record FROM session_managed_boot_intents WHERE intent_id=$1", + ) + .bind(id) + .fetch_one(&mut *tx) + .await + .map_err(db_err)?; + let old: ManagedBootIntent = decode(v)?; + old.query + .matches(&managed_create::read(&mut tx, &old.query.identity, false).await?)?; + if old.revision != revision { + return Err(conflict()); + } + if matches!(c, ManagedIntentAdvance::Complete) { + let v = + sqlx::query_scalar("SELECT record FROM session_managed_boots WHERE session_id=$1") + .bind(sid) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + .ok_or_else(conflict)?; + let b: ManagedBootRecord = decode(v)?; + old.validate_claim(&b.request)?; + } + if !matches!(c, ManagedIntentAdvance::Complete) { + let v: Option = + sqlx::query_scalar("SELECT record FROM session_managed_boots WHERE session_id=$1") + .bind(sid) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let boot: Option = v.map(decode).transpose()?; + if boot.as_ref().map(|b| &b.request.key) != old.recovery.as_ref().map(|r| &r.source) { + return Err(conflict()); + } + } + if let Some((host, present)) = placement_host { + if !present { + return Err(conflict()); + } + if crate::pick_host_2d( + &mut tx, + &[host.as_uuid()], + 0, + i64::from(old.selection.memory_mib), + i64::from(old.selection.vcpus), + ) + .await? + != Some(host.as_uuid()) + { + return Err(conflict()); + } + } + let next = old.advance(c)?; + sqlx::query("UPDATE session_managed_boot_intents SET revision=$2,completed=$3,record=$4,updated_at=$5 WHERE intent_id=$1").bind(id).bind(next.revision as i64).bind(next.completed).bind(serde_json::to_value(&next).map_err(|_|conflict())?).bind(self.clock.now_utc()).execute(&mut *tx).await.map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(next) + } + pub(crate) async fn bridge_list( + &self, + after: Option, + limit: u32, + ) -> Result, MetaError> { + let rows:Vec=sqlx::query_scalar("SELECT i.record FROM session_managed_boot_intents i JOIN sessions s ON s.id=i.session_id WHERE NOT i.completed AND s.status NOT IN ('dead','failed','completed') AND ($1::UUID IS NULL OR i.session_id>$1) ORDER BY i.session_id LIMIT $2").bind(after.map(|x|x.as_uuid())).bind(i64::from(limit.min(128))).fetch_all(&self.pool).await.map_err(db_err)?; + rows.into_iter().map(decode).collect() + } +} diff --git a/crates/engram-postgres/src/managed_create.rs b/crates/engram-postgres/src/managed_create.rs new file mode 100644 index 000000000..6c4c7508c --- /dev/null +++ b/crates/engram-postgres/src/managed_create.rs @@ -0,0 +1,125 @@ +//! Atomic immutable authority, in the existing session creation transaction. +use crate::{db_err, PostgresStore}; +use engram_core::{ + types::{managed_create::*, mcp_policy::*}, + MetaError, +}; +use sqlx::{PgConnection, Row}; + +fn conflict() -> MetaError { + MetaError::Conflict("managed creation association changed or unavailable".into()) +} + +pub(crate) async fn read( + conn: &mut PgConnection, + identity: &ManagedCreateIdentity, + require_held: bool, +) -> Result { + read_with_terminal(conn, identity, require_held, false).await +} +pub(crate) async fn read_with_terminal( + conn: &mut PgConnection, + identity: &ManagedCreateIdentity, + require_held: bool, + allow_terminal: bool, +) -> Result { + identity.validate()?; + let row = sqlx::query( + "SELECT c.*, s.status AS session_status, + s.mcp_policy_parent_id IS NOT DISTINCT FROM c.parent_session_id AS parent_matches, + (c.state='held' AND s.status='queued' AND s.host_id IS NULL + AND s.sandbox_id IS NULL AND s.binding_epoch=0) AS held + FROM session_managed_creations c JOIN sessions s ON s.id=c.session_id + WHERE c.session_id=$1", + ) + .bind(identity.session_id.as_uuid()) + .fetch_optional(conn) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + if require_held && !row.try_get::("held").map_err(db_err)? { + return Err(conflict()); + } + if !row.try_get::("parent_matches").map_err(db_err)? + || (!allow_terminal + && matches!( + row.try_get::<&str, _>("session_status").map_err(db_err)?, + "dead" | "completed" | "failed" + )) + { + return Err(if require_held { + conflict() + } else { + MetaError::NotFound + }); + } + let policy = McpPolicyCreation { + actor_id: row.try_get("actor_id").map_err(db_err)?, + input_id: row.try_get("input_id").map_err(db_err)?, + parent_session_id: row + .try_get::, _>("parent_session_id") + .map_err(db_err)? + .map(Into::into), + eligibility: match row.try_get::<&str, _>("eligibility").map_err(db_err)? { + "unknown" => WriteEligibility::Unknown, + "read_only" => WriteEligibility::ReadOnly, + "write_eligible" => WriteEligibility::WriteEligible, + _ => return Err(conflict()), + }, + }; + let stored = ManagedCreation { + identity: ManagedCreateIdentity { + session_id: identity.session_id, + task_id: row.try_get("task_id").map_err(db_err)?, + policy, + input_digest: row.try_get("input_digest").map_err(db_err)?, + }, + native: FrozenNativeManifest { + version: u32::try_from(row.try_get::("native_version").map_err(db_err)?) + .map_err(|_| conflict())?, + bytes: row.try_get("native_bytes").map_err(db_err)?, + digest: row.try_get("native_digest").map_err(db_err)?, + }, + }; + if stored.identity != *identity { + return Err(conflict()); + } + stored.native.validate()?; + Ok(stored) +} + +/// true: new reservation; false: exact retry. Deferred FK commits with the session row. +pub(crate) async fn reserve( + conn: &mut PgConnection, + creation: &ManagedCreation, +) -> Result { + let i = &creation.identity; + let n = sqlx::query("INSERT INTO session_managed_creations (session_id, task_id, input_id, actor_id, parent_session_id, eligibility, input_digest, native_version, native_bytes, native_digest) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT DO NOTHING") + .bind(i.session_id.as_uuid()).bind(&i.task_id).bind(&i.policy.input_id).bind(&i.policy.actor_id) + .bind(i.policy.parent_session_id.map(|p| p.as_uuid())).bind(i.policy.eligibility.as_str()) + .bind(&i.input_digest).bind(i32::try_from(creation.native.version).map_err(|_| conflict())?) + .bind(&creation.native.bytes).bind(&creation.native.digest) + .execute(&mut *conn).await.map_err(db_err)?.rows_affected(); + if n == 1 { + return Ok(true); + } + if read(conn, i, true).await? != *creation { + return Err(conflict()); + } + Ok(false) +} + +impl PostgresStore { + pub(crate) async fn managed_creation( + &self, + identity: &ManagedCreateIdentity, + require_held: bool, + ) -> Result { + read( + &mut *self.pool.acquire().await.map_err(db_err)?, + identity, + require_held, + ) + .await + } +} diff --git a/crates/engram-postgres/src/managed_operator.rs b/crates/engram-postgres/src/managed_operator.rs new file mode 100644 index 000000000..861d24481 --- /dev/null +++ b/crates/engram-postgres/src/managed_operator.rs @@ -0,0 +1,314 @@ +//! Publication transactions are short. None performs host I/O. +use crate::{db_err, PostgresStore}; +use engram_core::{ + types::{managed_boot::ManagedBootRecord, managed_operator::*}, + MetaError, SessionId, +}; +use sqlx::{PgConnection, Row}; + +const IDS: &str = "SELECT session_id FROM session_managed_creations UNION SELECT session_id FROM session_managed_boots UNION SELECT session_id FROM session_integration_policy WHERE policy_json::jsonb->'managed_deny' IS NOT NULL AND policy_json::jsonb->'managed_deny' <> 'null'::jsonb"; +async fn gate(conn: &mut PgConnection) -> Result<(), MetaError> { + sqlx::query( + "SELECT singleton FROM managed_operator_generation_gate WHERE singleton FOR UPDATE", + ) + .fetch_one(conn) + .await + .map_err(db_err)?; + Ok(()) +} +async fn state(conn: &mut PgConnection) -> Result { + let row=sqlx::query("SELECT published_generation,published_identity,admission_open FROM managed_operator_generation_gate WHERE singleton").fetch_one(&mut *conn).await.map_err(db_err)?; + let id: Option = row.try_get("published_generation").map_err(db_err)?; + let mut result = OperatorPublication { + admission_open: row.try_get("admission_open").map_err(db_err)?, + ..Default::default() + }; + for row in sqlx::query("SELECT record,published FROM managed_operator_publications WHERE NOT published OR generation=$1").bind(id).fetch_all(conn).await.map_err(db_err)? { + let value:serde_json::Value=row.try_get("record").map_err(db_err)?; + let generation:OperatorGeneration=serde_json::from_value(value).map_err(|_| conflict())?; + if row.try_get::("published").map_err(db_err)? { result.published=Some(generation); } else { result.pending=Some(generation); } + } + let identity: Option = row.try_get("published_identity").map_err(db_err)?; + // A seeded or old gate is never a configured published projection. + if result.published.as_ref().map(|p| p.generation) != id.and_then(|v| u64::try_from(v).ok()) + || result.published.as_ref().is_none_or(|p| { + Some(&p.identity) != identity.as_ref() + || p.candidate.identity().ok().as_ref() != Some(&p.identity) + }) + || result.pending.is_some() + { + result.admission_open = false; + } + Ok(result) +} +async fn snapshot( + conn: &mut PgConnection, + sid: SessionId, + hosts: Vec, + write: bool, + first: bool, +) -> Result { + let row=sqlx::query("SELECT s.status,s.host_id,s.sandbox_id,s.binding_epoch, c.state='held' AS held,b.record,p.policy_json FROM sessions s LEFT JOIN session_managed_creations c ON c.session_id=s.id LEFT JOIN session_managed_boots b ON b.session_id=s.id LEFT JOIN session_integration_policy p ON p.session_id=s.id WHERE s.id=$1") + .bind(sid.as_uuid()).fetch_optional(&mut *conn).await.map_err(db_err)?.ok_or(MetaError::NotFound)?; + let raw: Option = row.try_get("policy_json").map_err(db_err)?; + let (retained, policy) = retain_generation(raw.as_deref(), hosts, first)?; + if write { + sqlx::query("INSERT INTO session_integration_policy(session_id,policy_json) VALUES($1,$2) ON CONFLICT(session_id) DO UPDATE SET policy_json=EXCLUDED.policy_json").bind(sid.as_uuid()).bind(retained).execute(&mut *conn).await.map_err(db_err)?; + } else if raw + .as_deref() + .is_none_or(|r| retain(Some(r), Vec::new()).is_err()) + { + return Err(conflict()); + } + let boot: Option = row.try_get("record").map_err(db_err)?; + let boot: Option = boot + .map(serde_json::from_value) + .transpose() + .map_err(|_| conflict())?; + Ok(OperatorParticipant { + session_id: sid, + status: row.try_get("status").map_err(db_err)?, + host_id: row + .try_get::, _>("host_id") + .map_err(db_err)? + .map(Into::into), + sandbox_id: row + .try_get::, _>("sandbox_id") + .map_err(db_err)? + .map(Into::into), + binding_epoch: u64::try_from(row.try_get::("binding_epoch").map_err(db_err)?) + .map_err(|_| conflict())?, + held_creation: row + .try_get::, _>("held") + .map_err(db_err)? + .unwrap_or(false), + boot: boot.map(|b| (b.request, b.revision, b.prepared)), + policy, + }) +} +async fn lock_session(conn: &mut PgConnection, sid: SessionId) -> Result<(), MetaError> { + sqlx::query("SELECT id FROM sessions WHERE id=$1 FOR UPDATE") + .bind(sid.as_uuid()) + .fetch_optional(conn) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + Ok(()) +} +impl PostgresStore { + pub(crate) async fn operator_read(&self) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + // The same snapshot includes the public gate and its immutable candidate. + sqlx::query( + "SELECT singleton FROM managed_operator_generation_gate WHERE singleton FOR SHARE", + ) + .fetch_one(&mut *tx) + .await + .map_err(db_err)?; + let result = state(&mut tx).await?; + tx.commit().await.map_err(db_err)?; + Ok(result) + } + pub(crate) async fn operator_stage( + &self, + expected: Option, + candidate: OperatorCandidate, + ) -> Result { + candidate.identity()?; + let mut tx = self.pool.begin().await.map_err(db_err)?; + gate(&mut tx).await?; + let old = state(&mut tx).await?; + let next = old.stage(expected, candidate)?; + if next != old { + let pending = next.pending.as_ref().ok_or_else(conflict)?; + sqlx::query( + "UPDATE managed_operator_generation_gate SET admission_open=FALSE WHERE singleton", + ) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query("INSERT INTO managed_operator_publications(generation,record,updated_at) VALUES($1,$2,$3)").bind(pending.generation as i64).bind(serde_json::to_value(pending).map_err(|_|conflict())?).bind(self.clock.now_utc()).execute(&mut *tx).await.map_err(db_err)?; + } + tx.commit().await.map_err(db_err)?; + Ok(next) + } + pub(crate) async fn operator_list( + &self, + after: Option, + limit: u32, + ) -> Result, MetaError> { + if !(1..=512).contains(&limit) { + return Err(conflict()); + } + let query=format!("SELECT session_id FROM ({IDS}) p WHERE ($1::uuid IS NULL OR session_id>$1) ORDER BY session_id LIMIT $2"); + Ok(sqlx::query_scalar::<_, uuid::Uuid>(&query) + .bind(after.map(|s| s.as_uuid())) + .bind(i64::from(limit)) + .fetch_all(&self.pool) + .await + .map_err(db_err)? + .into_iter() + .map(Into::into) + .collect()) + } + pub(crate) async fn operator_retain( + &self, + g: &OperatorGeneration, + sid: SessionId, + ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + gate(&mut tx).await?; + state(&mut tx).await?.pending(g)?; + lock_session(&mut tx, sid).await?; + let query = format!("SELECT EXISTS(SELECT 1 FROM ({IDS}) p WHERE session_id=$1)"); + if !sqlx::query_scalar::<_, bool>(&query) + .bind(sid.as_uuid()) + .fetch_one(&mut *tx) + .await + .map_err(db_err)? + { + return Err(conflict()); + } + let exists:bool=sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM managed_operator_readiness WHERE generation=$1 AND session_id=$2)").bind(g.generation as i64).bind(sid.as_uuid()).fetch_one(&mut *tx).await.map_err(db_err)?; + let p = snapshot(&mut tx, sid, g.candidate.hosts()?, true, !exists).await?; + // Persist the requirement before host I/O. Reobserving clears transient proof. + sqlx::query("INSERT INTO managed_operator_readiness(generation,session_id,participant,proof) VALUES($1,$2,$3,NULL) ON CONFLICT(generation,session_id) DO UPDATE SET participant=EXCLUDED.participant,proof=NULL").bind(g.generation as i64).bind(sid.as_uuid()).bind(serde_json::to_value(&p).map_err(|_|conflict())?).execute(&mut *tx).await.map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(p) + } + pub(crate) async fn operator_ready( + &self, + g: &OperatorGeneration, + p: &OperatorParticipant, + proof: OperatorReadiness, + ) -> Result<(), MetaError> { + proof.validate(g, p)?; + let mut tx = self.pool.begin().await.map_err(db_err)?; + gate(&mut tx).await?; + state(&mut tx).await?.pending(g)?; + lock_session(&mut tx, p.session_id).await?; + if snapshot(&mut tx, p.session_id, Vec::new(), false, false).await? != *p { + return Err(conflict()); + } + let changed=sqlx::query("UPDATE managed_operator_readiness SET proof=$4 WHERE generation=$1 AND session_id=$2 AND participant=$3").bind(g.generation as i64).bind(p.session_id.as_uuid()).bind(serde_json::to_value(p).map_err(|_|conflict())?).bind(serde_json::to_value(proof).map_err(|_|conflict())?).execute(&mut *tx).await.map_err(db_err)?.rows_affected(); + if changed != 1 { + return Err(conflict()); + } + tx.commit().await.map_err(db_err)?; + Ok(()) + } + pub(crate) async fn operator_publish( + &self, + g: &OperatorGeneration, + ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + gate(&mut tx).await?; + let old = state(&mut tx).await?; + if old.pending.is_none() && old.published.as_ref() == Some(g) && old.admission_open { + return Ok(old); + } + old.pending(g)?; + // Lock every participant in stable order. New reservations/claims take the gate first. + let query=format!("SELECT s.id FROM sessions s JOIN ({IDS}) p ON p.session_id=s.id ORDER BY s.id FOR UPDATE OF s"); + let ids = sqlx::query_scalar::<_, uuid::Uuid>(&query) + .fetch_all(&mut *tx) + .await + .map_err(db_err)?; + for id in ids { + let p = snapshot(&mut tx, id.into(), Vec::new(), false, false).await?; + let row=sqlx::query("SELECT participant,proof FROM managed_operator_readiness WHERE generation=$1 AND session_id=$2").bind(g.generation as i64).bind(id).fetch_optional(&mut *tx).await.map_err(db_err)?.ok_or_else(conflict)?; + let recorded: OperatorParticipant = + serde_json::from_value(row.try_get("participant").map_err(db_err)?) + .map_err(|_| conflict())?; + let proof: OperatorReadiness = serde_json::from_value( + row.try_get::, _>("proof") + .map_err(db_err)? + .ok_or_else(conflict)?, + ) + .map_err(|_| conflict())?; + if p != recorded { + return Err(conflict()); + } + proof.validate(g, &p)?; + } + sqlx::query("UPDATE managed_operator_publications SET published=TRUE,updated_at=$2 WHERE generation=$1").bind(g.generation as i64).bind(self.clock.now_utc()).execute(&mut *tx).await.map_err(db_err)?; + sqlx::query("UPDATE managed_operator_generation_gate SET published_generation=$1,published_identity=$2,admission_open=TRUE WHERE singleton").bind(g.generation as i64).bind(&g.identity).execute(&mut *tx).await.map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(OperatorPublication { + published: Some(g.clone()), + pending: None, + admission_open: true, + }) + } +} + +/// New managed creations register under the same generation lock as staging. +/// Exact existing reservations are checked by their immutable create path instead. +pub(crate) async fn prepare_creation( + conn: &mut PgConnection, + ws: &mut engram_core::traits::SessionCreateWriteSet, +) -> Result<(), MetaError> { + let managed_policy = has_managed_policy(ws.integration_policy_json.as_deref())?; + if ws.managed_creation.is_none() && !managed_policy { + return Ok(()); + } + sqlx::query("SELECT singleton FROM managed_operator_generation_gate WHERE singleton FOR SHARE") + .fetch_one(&mut *conn) + .await + .map_err(db_err)?; + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM session_managed_creations WHERE session_id=$1)", + ) + .bind(ws.session_id.as_uuid()) + .fetch_one(&mut *conn) + .await + .map_err(db_err)?; + if exists { + return Ok(()); + } + let publication = state(conn).await?; + if ws.managed_creation.is_none() + && publication.published.is_none() + && publication.pending.is_none() + { + return Ok(()); + } + if !publication.admission_open || publication.pending.is_some() { + return Err(conflict()); + } + let published = publication.published.ok_or_else(conflict)?; + ws.integration_policy_json = Some( + retain( + ws.integration_policy_json.as_deref(), + published.candidate.hosts()?, + )? + .0, + ); + Ok(()) +} +/// A newly introduced legacy managed marker cannot appear behind a pending scan. +/// Existing policies can still grow monotonically while publication is closed. +pub(crate) async fn policy_registration_lock( + conn: &mut PgConnection, +) -> Result { + sqlx::query("SELECT singleton FROM managed_operator_generation_gate WHERE singleton FOR SHARE") + .fetch_one(&mut *conn) + .await + .map_err(db_err)?; + state(conn).await +} +pub(crate) fn check_registration( + publication: OperatorPublication, + old: Option<&str>, + next: &str, +) -> Result<(), MetaError> { + publication.validate_registration(old, next) +} + +fn has_managed_policy(raw: Option<&str>) -> Result { + Ok(raw + .map(serde_json::from_str::) + .transpose() + .map_err(|_| conflict())? + .is_some_and(|v| v.get("managed_deny").is_some_and(|p| !p.is_null()))) +} diff --git a/crates/engram-postgres/src/managed_operator_gate.rs b/crates/engram-postgres/src/managed_operator_gate.rs new file mode 100644 index 000000000..923121dd0 --- /dev/null +++ b/crates/engram-postgres/src/managed_operator_gate.rs @@ -0,0 +1,30 @@ +//! Short transaction lock shared with the authenticated operator publisher. +use engram_core::{types::managed_boot::ManagedBootKey, MetaError}; +use sqlx::{PgConnection, Row}; + +/// Acquire this before session/ancestor locks. The publisher uses FOR UPDATE, +/// closes admission and invalidates readiness before it releases its lock. +/// No host I/O is permitted while either lock is held. +pub(crate) async fn require( + conn: &mut PgConnection, + key: &ManagedBootKey, +) -> Result<(), MetaError> { + let row = sqlx::query("SELECT published_generation,published_identity,admission_open FROM managed_operator_generation_gate WHERE singleton FOR SHARE") + .fetch_optional(conn).await.map_err(crate::db_err)?; + let Some(row) = row else { + return Err(closed()); + }; + let generation: Option = row.try_get("published_generation").map_err(crate::db_err)?; + let identity: Option = row.try_get("published_identity").map_err(crate::db_err)?; + let open: bool = row.try_get("admission_open").map_err(crate::db_err)?; + if !open + || generation != i64::try_from(key.context.operator_generation).ok() + || identity.as_ref() != Some(&key.context.operator_identity) + { + return Err(closed()); + } + Ok(()) +} +fn closed() -> MetaError { + MetaError::Conflict("managed operator generation is closed".into()) +} diff --git a/crates/engram-postgres/src/mcp_policy.rs b/crates/engram-postgres/src/mcp_policy.rs new file mode 100644 index 000000000..96bbfb315 --- /dev/null +++ b/crates/engram-postgres/src/mcp_policy.rs @@ -0,0 +1,182 @@ +use super::{db_err, PostgresStore}; +use engram_core::{ + types::{ + event::PersistedEvent, + mcp_policy::*, + outbox::{tool_result_outbox_id, OutboxKind}, + }, + MetaError, SessionId, +}; +use sqlx::{Postgres, Row, Transaction}; +use std::collections::BTreeSet; + +pub(super) async fn nodes( + tx: &mut Transaction<'_, Postgres>, + id: SessionId, +) -> Result, MetaError> { + // Parent links never change. Read the chain, then lock every row in ID order. + let mut chain = Vec::new(); + let mut seen = BTreeSet::new(); + let mut next = Some(id); + while let Some(id) = next { + if chain.len() >= 32 || !seen.insert(id) { + return Err(conflict("policy ancestry unavailable")); + } + let row = sqlx::query("SELECT mcp_policy_parent_id FROM sessions WHERE id=$1") + .bind(id.as_uuid()) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + next = row + .try_get::, _>("mcp_policy_parent_id") + .map_err(db_err)? + .map(SessionId); + chain.push(id); + } + let ids: Vec<_> = seen.iter().map(|id| id.as_uuid()).collect(); + let rows=sqlx::query("SELECT id, binding_epoch, mcp_policy_parent_id, mcp_write_policy, status FROM sessions WHERE id=ANY($1) ORDER BY id FOR UPDATE") + .bind(ids).fetch_all(&mut **tx).await.map_err(db_err)?; + if rows.len() != chain.len() { + return Err(conflict("policy ancestry unavailable")); + } + let mut result = Vec::new(); + for id in chain { + let row = rows + .iter() + .find(|r| r.get::("id") == id.as_uuid()) + .ok_or(MetaError::NotFound)?; + let raw: Option = row.try_get("mcp_write_policy").map_err(db_err)?; + let policy = raw + .map(serde_json::from_value) + .transpose() + .map_err(|_| conflict("policy unavailable"))?; + let status: String = row.try_get("status").map_err(db_err)?; + result.push(PolicyNode { + id, + binding_epoch: row.try_get("binding_epoch").map_err(db_err)?, + parent: row + .try_get::, _>("mcp_policy_parent_id") + .map_err(db_err)? + .map(SessionId), + policy, + terminal: matches!(status.as_str(), "dead" | "completed" | "failed"), + active: status == "active", + }); + } + Ok(result) +} +async fn events( + tx: &mut Transaction<'_, Postgres>, + id: SessionId, + call: &str, +) -> Result, MetaError> { + // At most one run event accompanies call history. A truncated history always + // contains at least 1024 call events, which the shared validator rejects. + let rows=sqlx::query("SELECT idx,kind,payload,created_at,recovery_epoch::bigint AS recovery_epoch,rewound_at FROM session_events WHERE session_id=$1 AND ((payload->>'tool_call_id'=$2 AND kind IN ('tool_call_requested','tool_result_submitted','tool_call_completed','tool_call_cancel_requested')) OR idx=(SELECT max(idx) FROM session_events WHERE session_id=$1 AND rewound_at IS NULL AND kind IN ('run_started','run_completed','run_interrupted'))) ORDER BY idx LIMIT 1025") + .bind(id.as_uuid()).bind(call).fetch_all(&mut **tx).await.map_err(db_err)?; + rows.into_iter() + .map(|r| { + Ok(PersistedEvent { + idx: r.try_get("idx").map_err(db_err)?, + kind: r.try_get("kind").map_err(db_err)?, + payload: r.try_get("payload").map_err(db_err)?, + created_at: r.try_get("created_at").map_err(db_err)?, + recovery_epoch: r.try_get("recovery_epoch").map_err(db_err)?, + rewound_at: r.try_get("rewound_at").map_err(db_err)?, + }) + }) + .collect() +} +impl PostgresStore { + pub(super) async fn policy_read(&self, id: SessionId) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + let p = compose_policy(&nodes(&mut tx, id).await?)?; + tx.commit().await.map_err(db_err)?; + Ok(p) + } + pub(super) async fn policy_review( + &self, + id: SessionId, + call: &str, + ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + let ns = nodes(&mut tx, id).await?; + let p = plan_review(&ns[0], &events(&mut tx, id, call).await?, call)?; + tx.commit().await.map_err(db_err)?; + Ok(p) + } + pub(super) async fn policy_fence( + &self, + ctx: &RequestPolicyContext, + expected: &ExecutionRequest, + write: bool, + ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + let ns = nodes(&mut tx, ctx.session_id).await?; + let check = if write { + dispatch_fence + } else { + execution_identity_fence + }; + let answer = check( + &ns, + &events(&mut tx, ctx.session_id, &ctx.tool_call_id).await?, + ctx, + expected, + )?; + tx.commit().await.map_err(db_err)?; + Ok(answer) + } + pub(super) async fn policy_accept( + &self, + input: &PlanAcceptance, + ) -> Result { + let id = input.review.policy_ref.session_id; + let now = self.clock.now_utc(); + let mut tx = self.pool.begin().await.map_err(db_err)?; + let mut ns = nodes(&mut tx, id).await?; + let outbox_id = tool_result_outbox_id(id, &input.review.tool_call_id); + let payload = serde_json::json!({"tool_call_id":input.review.tool_call_id,"result_json":input.result_json,"mcp_policy_acceptance":input}); + if let Some(row) = sqlx::query( + "SELECT payload FROM session_outbox WHERE prompt_id=$1 AND session_id=$2 AND kind=$3", + ) + .bind(&outbox_id) + .bind(id.as_uuid()) + .bind(OutboxKind::ToolResult.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)? + { + if row + .try_get::("payload") + .map_err(db_err)? + != payload + { + return Err(conflict("plan result already accepted")); + } + let p = compose_policy(&ns)?; + tx.commit().await.map_err(db_err)?; + return Ok(p); + } + let updated = accept_plan( + &ns[0], + &events(&mut tx, id, &input.review.tool_call_id).await?, + input, + )?; + sqlx::query("INSERT INTO session_outbox(prompt_id,session_id,kind,payload,created_at,not_before) VALUES($1,$2,$3,$4,$5,$5)").bind(&outbox_id).bind(id.as_uuid()).bind(OutboxKind::ToolResult.as_str()).bind(payload).bind(now).execute(&mut *tx).await.map_err(db_err)?; + let idx:i64=sqlx::query_scalar("UPDATE sessions SET mcp_write_policy=$2,next_event_idx=next_event_idx+1,last_event_at=$3,updated_at=$3 WHERE id=$1 RETURNING next_event_idx-1").bind(id.as_uuid()).bind(serde_json::to_value(&updated).map_err(|_|conflict("policy encode failed"))?).bind(now).fetch_one(&mut *tx).await.map_err(db_err)?; + let event = serde_json::json!({"tool_call_id":input.review.tool_call_id,"result_json":input.result_json,"at":now}); + sqlx::query("INSERT INTO session_events(session_id,idx,kind,payload,recovery_epoch,created_at) SELECT id,$2,'tool_result_submitted',$3,recovery_epoch,$4 FROM sessions WHERE id=$1").bind(id.as_uuid()).bind(idx).bind(event).bind(now).execute(&mut *tx).await.map_err(db_err)?; + sqlx::query("SELECT pg_notify('session_events',$1),pg_notify('session_outbox',$2)") + .bind(serde_json::json!({"session_id":id,"idx":idx}).to_string()) + .bind(id.to_string()) + .execute(&mut *tx) + .await + .map_err(db_err)?; + ns[0].policy = Some(updated); + let p = compose_policy(&ns)?; + tx.commit().await.map_err(db_err)?; + Ok(p) + } +} diff --git a/crates/engram-postgres/src/outbox_accept.rs b/crates/engram-postgres/src/outbox_accept.rs new file mode 100644 index 000000000..598c303c3 --- /dev/null +++ b/crates/engram-postgres/src/outbox_accept.rs @@ -0,0 +1,156 @@ +use crate::db_err; +use engram_core::{MetaError, SessionId}; +use sqlx::Row; +pub(crate) async fn accept( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + now: chrono::DateTime, + session_id: SessionId, + events: &[(String, serde_json::Value)], + outbox: &engram_core::types::outbox::OutboxRow, +) -> Result>, MetaError> { + if outbox.session_id != session_id { + return Err(MetaError::Serialization( + "event/outbox session ids do not match".into(), + )); + } + sqlx::query("SELECT id FROM sessions WHERE id=$1 FOR UPDATE") + .bind(session_id.as_uuid()) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + crate::managed_boot::require_tool_result_target(tx, outbox).await?; + let inserted = sqlx::query( + r#" + INSERT INTO session_outbox + (prompt_id, session_id, kind, payload, created_at, not_before) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (prompt_id) DO NOTHING + RETURNING prompt_id + "#, + ) + .bind(&outbox.prompt_id) + .bind(session_id.as_uuid()) + .bind(outbox.kind.as_str()) + .bind(&outbox.payload) + .bind(outbox.created_at) + .bind(outbox.not_before) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)? + .is_some(); + + if !inserted { + // A retry of the SAME command is the designed no-op; a + // prompt_id claimed by a DIFFERENT command — including the + // same kind with different text/mode (review finding on + // #993: payload must be part of the identity, as it is in + // `append_session_event_and_outbox`) — is a Conflict, never + // a silent drop. + let existing = sqlx::query( + "SELECT session_id, kind, payload FROM session_outbox WHERE prompt_id = $1", + ) + .bind(&outbox.prompt_id) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)? + .ok_or_else(|| { + MetaError::Conflict(format!( + "outbox id {} disappeared during accept", + outbox.prompt_id + )) + })?; + let existing_session: uuid::Uuid = existing.try_get("session_id").map_err(db_err)?; + let existing_kind: String = existing.try_get("kind").map_err(db_err)?; + let existing_payload: serde_json::Value = existing.try_get("payload").map_err(db_err)?; + if existing_session != session_id.as_uuid() + || existing_kind != outbox.kind.as_str() + || existing_payload != outbox.payload + { + return Err(MetaError::Conflict(format!( + "outbox id {} belongs to another command", + outbox.prompt_id + ))); + } + return Ok(None); + } + + if outbox.kind == engram_core::types::outbox::OutboxKind::Prompt + && outbox.payload["mode"] == "plan" + { + let row = sqlx::query("SELECT mcp_write_policy,next_event_idx FROM sessions WHERE id=$1") + .bind(session_id.as_uuid()) + .fetch_one(&mut **tx) + .await + .map_err(db_err)?; + let old: Option = row.try_get("mcp_write_policy").map_err(db_err)?; + let old = old + .map(serde_json::from_value) + .transpose() + .map_err(|_| MetaError::Conflict("policy unavailable".into()))?; + let policy = engram_core::types::mcp_policy::StoredWritePolicy::restrict( + old.as_ref(), + &outbox.prompt_id, + row.try_get("next_event_idx").map_err(db_err)?, + )?; + sqlx::query("UPDATE sessions SET mcp_write_policy=$2 WHERE id=$1") + .bind(session_id.as_uuid()) + .bind( + serde_json::to_value(policy) + .map_err(|_| MetaError::Conflict("policy encode failed".into()))?, + ) + .execute(&mut **tx) + .await + .map_err(db_err)?; + } + let mut idxs = Vec::with_capacity(events.len()); + for (kind, payload) in events { + let event = sqlx::query( + r#" + WITH next AS ( + UPDATE sessions + SET next_event_idx = next_event_idx + 1, + updated_at = $4, + last_event_at = $4 + WHERE id = $1 + RETURNING next_event_idx - 1 AS allocated_idx, recovery_epoch + ), + inserted AS ( + INSERT INTO session_events (session_id, idx, kind, payload, recovery_epoch, created_at) + SELECT $1, allocated_idx, $2, $3, recovery_epoch, $4 FROM next + RETURNING idx + ) + SELECT idx FROM inserted + "#, + ) + .bind(session_id.as_uuid()) + .bind(kind) + .bind(payload) + .bind(now) + .fetch_optional(&mut **tx) + .await + .map_err(db_err)? + .ok_or(MetaError::NotFound)?; + idxs.push(event.try_get::("idx").map_err(db_err)?); + } + + // Notifications fire at commit: replicas see the events and the + // outbox row together, so delivery can never observe the row + // without its echo (the ADR 0052 echo-before-run_started + // ordering, now transactional instead of sequenced). + for idx in &idxs { + sqlx::query("SELECT pg_notify('session_events', $1)") + .bind( + serde_json::json!({ "session_id": session_id.to_string(), "idx": idx }).to_string(), + ) + .execute(&mut **tx) + .await + .map_err(db_err)?; + } + sqlx::query("SELECT pg_notify('session_outbox', $1)") + .bind(session_id.to_string()) + .execute(&mut **tx) + .await + .map_err(db_err)?; + Ok(Some(idxs)) +} diff --git a/crates/engram-postgres/tests/migrations.rs b/crates/engram-postgres/tests/migrations.rs new file mode 100644 index 000000000..692ef14db --- /dev/null +++ b/crates/engram-postgres/tests/migrations.rs @@ -0,0 +1,13 @@ +#[test] +fn migration_versions_are_unique_and_increasing() { + let migrations = sqlx::migrate!("../../deploy/migrations"); + let mut previous = None; + for migration in migrations.iter() { + assert!( + previous.is_none_or(|version| migration.version > version), + "migration version {} is duplicated or out of order", + migration.version, + ); + previous = Some(migration.version); + } +} diff --git a/crates/engram-protocol/Cargo.toml b/crates/engram-protocol/Cargo.toml index e0cc7061e..962b00480 100644 --- a/crates/engram-protocol/Cargo.toml +++ b/crates/engram-protocol/Cargo.toml @@ -10,6 +10,11 @@ description = "Wire types for Engram coordinator<->host communication." workspace = true [dependencies] +engram-tls.workspace = true +rustls.workspace = true +rustls-pemfile.workspace = true +sha2.workspace = true +hex.workspace = true engram-core.workspace = true engram-telemetry.workspace = true async-stream.workspace = true diff --git a/crates/engram-protocol/build.rs b/crates/engram-protocol/build.rs index 4fa276dcd..dbb9057bf 100644 --- a/crates/engram-protocol/build.rs +++ b/crates/engram-protocol/build.rs @@ -41,6 +41,30 @@ fn main() -> Result<(), Box> { .build_server(true) .build_client(true) .file_descriptor_set_path(&descriptor_path) + .skip_debug([".engram.host.v1.ManagedBootRequest"]) .compile_protos(&protos, &includes)?; + let mcp_out = std::path::PathBuf::from(std::env::var("OUT_DIR")?).join("mcp"); + std::fs::create_dir_all(&mcp_out)?; + tonic_prost_build::configure() + .out_dir(mcp_out) + .build_server(true) + .build_client(true) + .extern_path(".engram.app.v1", "crate::app") + .skip_debug([ + ".engram.mcp.v1.PutSecretRequest", + ".engram.mcp.v1.ReserveManagedCreateRequest", + ".engram.mcp.v1.CreatePolicySessionRequest", + ".engram.mcp.v1.CompleteAuthorizationRequest", + ".engram.mcp.v1.BeginAuthorizationResponse", + ".engram.mcp.v1.LeaseResponse", + ]) + .compile_protos( + &[ + "proto/engram/mcp/v1/control.proto", + "proto/engram/mcp/v1/operator.proto", + "proto/engram/mcp/v1/session_policy.proto", + ], + &includes, + )?; Ok(()) } diff --git a/crates/engram-protocol/proto/engram/app/v1/integration.proto b/crates/engram-protocol/proto/engram/app/v1/integration.proto index 6387547a7..b77493696 100644 --- a/crates/engram-protocol/proto/engram/app/v1/integration.proto +++ b/crates/engram-protocol/proto/engram/app/v1/integration.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package engram.app.v1; +import "engram/app/v1/session.proto"; + // ADR 0057 C3: orchestrator-native connector catalog CRUD (Plane B). Connectors // live in the orchestrator DB (the `connector` table, C1); the built-ins // (github/datadog) are read-only file seeds surfaced alongside custom ones. @@ -52,6 +54,23 @@ service IntegrationService { rpc TestConnection(TestConnectionRequest) returns (TestConnectionResponse); rpc SetConnectionEnabled(SetConnectionEnabledRequest) returns (SetConnectionEnabledResponse); rpc GetGoogleCloudSetup(GetGoogleCloudSetupRequest) returns (GetGoogleCloudSetupResponse); + rpc GetMcpCatalogs(GetMcpCatalogsRequest) returns (GetMcpCatalogsResponse); + rpc DiscoverMcp(DiscoverMcpRequest) returns (DiscoverMcpResponse); + rpc PublishMcp(PublishMcpRequest) returns (PublishMcpResponse); + rpc PutMcpCredential(PutMcpCredentialRequest) returns (PutMcpCredentialResponse); + rpc GetMcpSetup(GetMcpSetupRequest) returns (GetMcpSetupResponse); + rpc CompleteMcpAuthorization(CompleteMcpAuthorizationRequest) returns (CompleteMcpAuthorizationResponse); + rpc RecoverMcpCredential(RecoverMcpCredentialRequest) returns (RecoverMcpCredentialResponse); + rpc AuthorizeMcp(AuthorizeMcpRequest) returns (AuthorizeMcpResponse); + rpc DisconnectMcp(DisconnectMcpRequest) returns (DisconnectMcpResponse); + rpc GetSessionMcpState(GetSessionMcpStateRequest) returns (GetSessionMcpStateResponse); + rpc GetMcpExecutionResult(GetMcpExecutionResultRequest) returns (GetMcpExecutionResultResponse); + rpc ListMcpApprovals(ListMcpApprovalsRequest) returns (ListMcpApprovalsResponse); + rpc DecideMcpApproval(DecideMcpApprovalRequest) returns (DecideMcpApprovalResponse); + rpc CancelMcpCall(CancelMcpCallRequest) returns (CancelMcpCallResponse); + rpc ResolveMcpOutcome(ResolveMcpOutcomeRequest) returns (ResolveMcpOutcomeResponse); + rpc PrepareMcpRebind(PrepareMcpRebindRequest) returns (PrepareMcpRebindResponse); + rpc RebindMcp(RebindMcpRequest) returns (RebindMcpResponse); } message GoogleCloudConnectionConfig { @@ -78,37 +97,72 @@ message IntegrationConnection { string updated_at = 8; GoogleCloudConnectionConfig google_cloud = 9; bool is_default = 10; + McpConnectionConfig mcp = 11; + uint32 config_revision = 12; + uint32 auth_generation = 13; + string credential_identity = 14; + uint32 catalog_revision = 15; + bool revoked = 16; } message ListConnectionsRequest {} -message ListConnectionsResponse { repeated IntegrationConnection connections = 1; } +message ListConnectionsResponse { + repeated IntegrationConnection connections = 1; +} message CreateConnectionRequest { string alias = 1; string provider = 2; string display_name = 3; GoogleCloudConnectionConfig google_cloud = 4; + McpConnectionConfig mcp = 5; +} +message CreateConnectionResponse { + IntegrationConnection connection = 1; } -message CreateConnectionResponse { IntegrationConnection connection = 1; } message UpdateConnectionRequest { + uint32 expected_generation = 7; + uint32 expected_revision = 6; string id = 1; string alias = 2; string display_name = 3; GoogleCloudConnectionConfig google_cloud = 4; + McpConnectionConfig mcp = 5; +} +message UpdateConnectionResponse { + IntegrationConnection connection = 1; } -message UpdateConnectionResponse { IntegrationConnection connection = 1; } -message DeleteConnectionRequest { string id = 1; } -message DeleteConnectionResponse { bool deleted = 1; } +message DeleteConnectionRequest { + uint32 expected_revision = 2; + uint32 expected_generation = 3; + string id = 1; +} +message DeleteConnectionResponse { + bool deleted = 1; +} -message TestConnectionRequest { string id = 1; } -message TestConnectionResponse { bool ok = 1; string message = 2; } +message TestConnectionRequest { + string id = 1; +} +message TestConnectionResponse { + bool ok = 1; + string message = 2; +} -message SetConnectionEnabledRequest { string id = 1; bool enabled = 2; } -message SetConnectionEnabledResponse { IntegrationConnection connection = 1; } +message SetConnectionEnabledRequest { + string id = 1; + bool enabled = 2; + uint32 expected_revision = 3; +} +message SetConnectionEnabledResponse { + IntegrationConnection connection = 1; +} -message GetGoogleCloudSetupRequest { string id = 1; } +message GetGoogleCloudSetupRequest { + string id = 1; +} message GetGoogleCloudSetupResponse { string issuer = 1; string audience = 2; @@ -184,9 +238,9 @@ message ConnectorDisplay { // One grantable power, derived for display (the slug stays canonical). message CatalogCapability { - string action = 1; // e.g. "issues:write" - string access = 2; // "read" | "write" (derived from the op's HTTP method) - string asset = 3; // asset kind this op surfaces, or "" when none + string action = 1; // e.g. "issues:write" + string access = 2; // "read" | "write" (derived from the op's HTTP method) + string asset = 3; // asset kind this op surfaces, or "" when none // ADR 0109 seam: operator-facing name. A named-connection provider serves // its operation list here so the web renders from the SAME table the // orchestrator compiles policy from — two hand-maintained copies drift, and @@ -239,7 +293,7 @@ message GetIntegrationCatalogResponse { message SetMintCredentialRequest { string provider = 1; // e.g. "github" - string kind = 2; // e.g. "github_app" + string kind = 2; // e.g. "github_app" // field name → value (e.g. app_id, private_key_pem). Each non-empty value is // sealed as the org secret `.`; values are never returned. map values = 3; @@ -291,3 +345,390 @@ message SetConnectorSettingsRequest { message SetConnectorSettingsResponse { Connector connector = 1; } + +// Organization credential references only. Secret values never enter this contract. +message McpConnectionConfig { + string endpoint = 1; + string zone = 2; + string auth_kind = 3; + string api_key_secret_ref = 4; + string oauth_client_registration_id = 5; + bool share_with_org_sessions = 6; +} +message McpTool { + string upstream_name = 1; + string description = 2; + string input_schema_json = 3; + string output_schema_json = 4; + // Empty in discovery drafts. Publication requires explicit read or write. + string classification = 5; + bool enabled = 6; + string fingerprint = 7; +} +message McpCatalog { + string id = 1; + string connection_id = 2; + string kind = 3; + uint32 revision = 4; + uint32 config_revision = 5; + uint32 auth_generation = 6; + repeated McpTool tools = 7; + string fingerprint = 8; +} +message GetMcpCatalogsRequest { + string connection_id = 1; +} +message GetMcpCatalogsResponse { + uint32 config_revision = 1; + uint32 catalog_revision = 2; + McpCatalog draft = 3; + McpCatalog published = 4; +} +message DiscoverMcpRequest { + string connection_id = 1; + uint32 expected_revision = 2; +} +message DiscoverMcpResponse { + McpCatalog catalog = 1; +} +message PublishMcpRequest { + string connection_id = 1; + string draft_id = 2; + uint32 expected_revision = 3; + repeated McpTool tools = 4; +} +message PublishMcpResponse { + McpCatalog catalog = 1; +} +message AuthorizeMcpRequest { + uint32 expected_revision = 4; + string connection_id = 1; + uint32 expected_generation = 2; + bool share_with_org_sessions = 3; +} +message AuthorizeMcpResponse { + uint32 config_revision = 4; + uint32 auth_generation = 5; + string flow_id = 2; + string expires_at = 3; + string authorization_url = 1; +} +message DisconnectMcpRequest { + uint32 expected_revision = 3; + string connection_id = 1; + uint32 expected_generation = 2; +} +message DisconnectMcpResponse { + IntegrationConnection connection = 1; +} +message McpApproval { + string execution_id = 1; + string session_id = 2; + string run_id = 3; + string tool_call_id = 4; + string connection_id = 5; + string upstream_name = 6; + string request_digest = 7; + string argument_hash = 8; + string expires_at = 9; + string original_arguments = 10; + McpTool reviewed_tool = 11; + repeated string argument_constraint_documents_json = 12; + McpApprovalReviewContext expected_review_context = 13; +} +message ListMcpApprovalsRequest { + string session_id = 1; + string after_execution_id = 2; +} +message ListMcpApprovalsResponse { + repeated McpApproval approvals = 1; + string next_execution_id = 2; +} +message DecideMcpApprovalRequest { + string execution_id = 1; + bool approve = 2; + McpApprovalReviewContext expected_review_context = 3; +} +message DecideMcpApprovalResponse { + string state = 1; + McpApprovalReceipt receipt = 2; +} +message CancelMcpCallRequest { + string execution_id = 1; + uint32 expected_journal_revision = 2; +} +message CancelMcpCallResponse { + string state = 1; + McpCancellationReceipt receipt = 2; +} +message ResolveMcpOutcomeRequest { + string execution_id = 1; + string outcome = 2; + McpUnknownContext expected_unknown_context = 3; +} +message ResolveMcpOutcomeResponse { + string outcome = 1; + McpResolutionReceipt receipt = 2; +} +message PrepareMcpRebindRequest { + string session_id = 1; + string expected_binding_hash = 2; +} +message PrepareMcpRebindResponse { + string binding_hash = 1; + string binding_epoch = 2; + repeated McpBoundToolSummary binding_tools = 3; + repeated string deny_hosts = 4; + string deny_revision = 5; + string deny_digest = 6; + repeated McpRebindConnection connections = 7; + McpRebindDrain drain = 8; +} +enum McpRebindDrainReason { + MCP_REBIND_DRAIN_REASON_UNSPECIFIED = 0; + MCP_REBIND_DRAIN_REASON_DISPATCH_IN_PROGRESS = 1; + MCP_REBIND_DRAIN_REASON_UNKNOWN_INTENT = 2; + MCP_REBIND_DRAIN_REASON_LEGACY_INTENT = 3; +} +message McpRebindDrain { + McpRebindDrainReason reason = 1; + string retry_after = 2; +} +message McpRebindConnection { + string connection_id = 1; + string alias = 2; + string endpoint = 3; + string zone = 4; + string credential_identity = 5; + uint32 config_revision = 6; + uint32 auth_generation = 7; + uint32 catalog_revision = 8; +} +message RebindMcpRequest { + string session_id = 1; + string expected_binding_hash = 2; + string expected_candidate_hash = 3; +} +message RebindMcpResponse { + string binding_hash = 1; +} + +// Entered values are request-only; responses contain metadata and references. +message PutMcpCredentialRequest { + string connection_id = 1; + uint32 expected_revision = 2; + uint32 expected_generation = 3; + string kind = 4; + string header = 5; + string value = 6; +} +message McpCredentialStatus { + string status = 1; + string reference = 2; + string kind = 3; + string credential_identity = 4; + int64 version = 5; + int64 reservation_generation = 6; +} +message PutMcpCredentialResponse { + IntegrationConnection connection = 1; + McpCredentialStatus credential = 2; +} +message CompleteMcpAuthorizationResponse { + IntegrationConnection connection = 1; + McpCredentialStatus credential = 2; +} +message RecoverMcpCredentialResponse { + IntegrationConnection connection = 1; + McpCredentialStatus credential = 2; +} +message McpCredentialState { + IntegrationConnection connection = 1; + McpCredentialStatus credential = 2; +} +message McpApprovedDestination { + string zone_id = 1; + string endpoint = 2; +} +message McpClientRegistration { + string id = 1; + string client_id = 2; + bool confidential = 3; + string redirect_uri = 4; + string issuer = 5; + McpApprovedDestination resource = 6; + repeated string scopes = 7; +} +message GetMcpSetupRequest { + string connection_id = 1; +} +message GetMcpSetupResponse { + bool enabled = 1; + repeated McpClientRegistration registrations = 2; + repeated McpApprovedDestination destinations = 3; + McpCredentialState current = 4; + bool transition_pending = 5; + string flow_id = 6; + string flow_expires_at = 7; + bool client_secret_configured = 8; + bool discovery_available = 9; + // Credential reconciliation and disconnect, independent of new dispatch. + bool recovery_available = 10; +} +message CompleteMcpAuthorizationRequest { + string connection_id = 1; + uint32 expected_revision = 2; + uint32 expected_generation = 3; + string flow_id = 4; + string state = 5; + string code = 6; + optional string issuer = 7; + string redirect_uri = 8; +} +message RecoverMcpCredentialRequest { + string connection_id = 1; + uint32 expected_revision = 2; + uint32 expected_generation = 3; +} + +// Exact server review. Clients must return it unchanged when they decide. +message McpApprovalReviewContext { + string execution_id = 1; + string session_id = 2; + string run_id = 3; + string tool_call_id = 4; + string request_digest = 5; + string argument_hash = 6; + string binding_hash = 7; + string connection_id = 8; + string catalog_id = 9; + uint32 catalog_revision = 10; + uint32 config_revision = 11; + uint32 auth_generation = 12; + string credential_identity = 13; + string upstream_name = 14; + string tool_fingerprint = 15; + string context_digest = 16; + uint32 journal_revision = 17; + string expires_at = 18; + string policy_revision = 19; + string binding_epoch = 20; +} +message McpApprovalReceipt { + string execution_id = 1; + string decision = 2; + string actor_id = 3; + string decided_at = 4; + uint32 journal_revision = 5; + McpApprovalReviewContext reviewed_context = 6; +} +message McpExecutionReview { + string execution_id = 1; + string connection_id = 2; + string state = 3; + uint32 journal_revision = 4; + bool result_unavailable = 5; + McpApproval review = 6; + repeated string allowed_actions = 7; + string reason = 8; + string tool_call_id = 9; + string run_id = 10; + string request_digest = 11; + string upstream_name = 12; + string approval_decision = 13; + string original_arguments = 14; + McpTool reviewed_tool = 15; + repeated string argument_constraint_documents_json = 16; + string binding_hash = 17; + McpUnknownContext expected_unknown_context = 18; + McpCancellationReceipt cancellation_receipt = 19; +} +message McpUnknownContext { + string execution_id = 1; + uint32 journal_revision = 2; + string request_digest = 3; + string binding_hash = 4; + string intent_id = 5; +} +message McpCancellationReceipt { + string execution_id = 1; + string actor_id = 2; + string cause = 3; + uint32 expected_journal_revision = 4; + uint32 journal_revision = 5; + string state = 6; + string created_at = 7; +} +message McpResolutionReceipt { + string execution_id = 1; + string actor_id = 2; + string outcome = 3; + string created_at = 4; + McpUnknownContext expected_context = 5; +} +message McpBindingReceipt { + string binding_hash = 1; + string binding_epoch = 2; + bool current = 3; + repeated string stale_reasons = 4; + repeated string allowed_actions = 5; +} +message GetSessionMcpStateRequest { + string session_id = 1; + string after_execution_id = 2; + uint32 page_size = 3; + optional string plan_tool_call_id = 4; + string after_binding_tool_name = 5; + uint32 binding_page_size = 6; + bool omit_bindings = 7; + bool omit_executions = 8; +} +message GetSessionMcpStateResponse { + McpBindingReceipt binding_receipt = 1; + repeated McpExecutionReview executions = 2; + repeated McpApproval approvals = 3; + repeated string unknown_write_blocks = 4; + repeated McpResolutionReceipt resolution_receipts = 5; + repeated string allowed_actions = 6; + string availability = 7; + string next_execution_id = 8; + GetMcpWritePolicyResponse write_policy = 9; + repeated McpBoundToolSummary binding_tools = 10; + string next_binding_tool_name = 11; + bool has_more_binding_tools = 12; + bool has_more_executions = 13; +} + +message GetMcpExecutionResultRequest { + string session_id = 1; + string execution_id = 2; +} + +// Settlement does not establish guest delivery or an upstream outcome. +enum McpCompletionSettlement { + MCP_COMPLETION_SETTLEMENT_UNSPECIFIED = 0; + MCP_COMPLETION_SETTLEMENT_PENDING = 1; + MCP_COMPLETION_SETTLEMENT_SETTLED_OR_TARGET_GONE = 2; + MCP_COMPLETION_SETTLEMENT_NOT_APPLICABLE = 3; +} +message GetMcpExecutionResultResponse { + string state = 1; + bool result_unavailable = 2; + optional string result_json = 3; + McpCompletionSettlement settlement = 4; +} + +// Immutable session grant. Schemas remain independent documents with their own roots. +message McpBoundToolSummary { + string connection_id = 1; + string connection_alias = 2; + string model_tool_name = 3; + McpTool reviewed_tool = 4; + repeated string argument_constraint_documents_json = 5; + bool automation_write = 6; + bool unrestricted = 7; + string catalog_id = 8; + uint32 catalog_revision = 9; + uint32 config_revision = 10; + uint32 auth_generation = 11; +} diff --git a/crates/engram-protocol/proto/engram/app/v1/profile.proto b/crates/engram-protocol/proto/engram/app/v1/profile.proto index 7ac6c19f7..39a8ca378 100644 --- a/crates/engram-protocol/proto/engram/app/v1/profile.proto +++ b/crates/engram-protocol/proto/engram/app/v1/profile.proto @@ -84,6 +84,8 @@ message Profile { repeated SessionApp apps = 24; // When true, only automations, API keys, and admins can start sessions. bool programmatic_only = 25; + // Revision for explicit MCP grant publication; zero means no publication. + int64 mcp_grant_revision = 26; } // ADR 0118: one service a session hosts, published at @@ -144,6 +146,7 @@ message ProfileIntegrationGrant { // user's personal credential (human launches then require it; programmatic // sessions always use the org credential). string credential_scope = 4; + McpToolGrant mcp = 5; } // ADR 0057: a profile's egress network allow-list. `default` is the posture for @@ -230,6 +233,8 @@ message CreateProfileRequest { repeated SessionApp apps = 20; // When true, only automations, API keys, and admins can start sessions. bool programmatic_only = 21; + // Presence requests explicit MCP grant publication. First publication expects zero. + optional int64 expected_mcp_grant_revision = 22; } message CreateProfileResponse { @@ -265,6 +270,8 @@ message UpdateProfileRequest { // When true, only automations, API keys, and admins can start sessions. // Omit to preserve the current setting. optional bool programmatic_only = 22; + // Presence requests publication or republication of the exact MCP grants. + optional int64 expected_mcp_grant_revision = 23; } message UpdateProfileResponse { @@ -276,3 +283,14 @@ message DeleteProfileRequest { } message DeleteProfileResponse {} + +message McpToolGrant { + string upstream_name = 1; + bool automation_write = 2; + reserved 3; + reserved "argument_constraints_json"; + bool unrestricted = 4; + // Every document must pass in its own dialect and resource scope. + // An empty list requires an explicit unrestricted choice. + repeated string argument_constraint_documents_json = 5; +} diff --git a/crates/engram-protocol/proto/engram/app/v1/session.proto b/crates/engram-protocol/proto/engram/app/v1/session.proto index 7b964e6ef..1b384ca93 100644 --- a/crates/engram-protocol/proto/engram/app/v1/session.proto +++ b/crates/engram-protocol/proto/engram/app/v1/session.proto @@ -25,6 +25,7 @@ service SessionService { // `tool_result_submitted` event at RPC time, enqueues a durable ToolResult // outbox row, and resumes an idle session exactly as SendPrompt does. rpc CompleteToolCall(CompleteToolCallRequest) returns (CompleteToolCallResponse); + rpc GetMcpWritePolicy(GetMcpWritePolicyRequest) returns (GetMcpWritePolicyResponse); // Streams the event envelope itself, not a wrapped Response message. // buf:lint:ignore RPC_RESPONSE_STANDARD_NAME rpc StreamEvents(StreamEventsRequest) returns (stream SessionEvent); @@ -316,6 +317,42 @@ message DequeueQueuedPromptResponse { string note = 2; } +message McpWritePolicyRef { + string session_id = 1; + int64 policy_revision = 2; + int64 binding_epoch = 3; +} +message McpTrustedWritePolicy { + McpWritePolicyRef ref = 1; + string write_eligibility = 2; + string cause = 3; + string accepted_input_id = 4; + optional string authenticated_actor_id = 5; + optional string prompt_id = 6; + optional string plan_tool_call_id = 7; + optional string plan_request_digest = 8; + repeated McpWritePolicyRef ancestor_policy_refs = 9; + bool current = 10; + bool initialized = 11; +} +message McpPlanReview { + McpWritePolicyRef policy_ref = 1; + string tool_call_id = 2; + string run_id = 3; + int64 event_idx = 4; + string request_digest = 5; + string args_json = 6; +} +message GetMcpWritePolicyRequest { + string session_id = 1; + optional string plan_tool_call_id = 2; +} +message GetMcpWritePolicyResponse { + McpTrustedWritePolicy policy = 1; + McpPlanReview plan_review = 2; + string reason = 3; + repeated string allowed_actions = 4; +} message CompleteToolCallRequest { string session_id = 1; // The call_id carried on the `tool_call_requested` event — the @@ -324,6 +361,8 @@ message CompleteToolCallRequest { // Opaque JSON result. Validated upstream (orchestrator) against the // tool's output schema; the coordinator never inspects it. string result_json = 3; + // Optional exact server review. Only the authenticated human handler consumes it. + McpPlanReview expected_mcp_plan_review = 4; } message CompleteToolCallResponse { diff --git a/crates/engram-protocol/proto/engram/mcp/v1/control.proto b/crates/engram-protocol/proto/engram/mcp/v1/control.proto new file mode 100644 index 000000000..b1ef74ee4 --- /dev/null +++ b/crates/engram-protocol/proto/engram/mcp/v1/control.proto @@ -0,0 +1,180 @@ +syntax = "proto3"; +package engram.mcp.v1; + +// Internal protocol 1. Served only on the separate mTLS listener. +service CredentialControlService { + rpc PutSecret(PutSecretRequest) returns (PutSecretResponse); + rpc BeginAuthorization(BeginAuthorizationRequest) returns (BeginAuthorizationResponse); + rpc CancelAuthorization(CancelAuthorizationRequest) returns (CancelAuthorizationResponse); + rpc CompleteAuthorization(CompleteAuthorizationRequest) returns (CompleteAuthorizationResponse); + rpc GetStatus(GetStatusRequest) returns (GetStatusResponse); + rpc Revoke(RevokeRequest) returns (RevokeResponse); + rpc Lease(LeaseRequest) returns (LeaseResponse); +} +message Destination { + string zone_id = 1; + string endpoint = 2; +} +message ChangeContext { + string connection_id = 1; + int64 config_revision = 2; + int64 previous_authorization_generation = 3; + Destination resource = 4; +} +message AuthorityContext { + string connection_id = 1; + int64 config_revision = 2; + int64 authorization_generation = 3; + string credential_identity = 4; + Destination resource = 5; + string policy_digest = 6; + string policy_revision = 7; +} +enum CredentialKind { + CREDENTIAL_KIND_UNSPECIFIED = 0; + CREDENTIAL_KIND_API_KEY = 1; + CREDENTIAL_KIND_CLIENT_SECRET = 2; + CREDENTIAL_KIND_OAUTH = 3; +} +message CredentialReference { + CredentialKind kind = 1; + string name = 2; + string connection_id = 3; + int64 generation = 4; + string registration_id = 5; +} +message CredentialStatus { + uint32 protocol = 1; + AuthorityContext context = 2; + CredentialReference reference = 3; + int64 version = 4; + string status = 5; +} +message PutSecretRequest { + uint32 protocol = 1; + ChangeContext context = 2; + CredentialKind kind = 3; + CredentialReference expected = 4; + string header = 5; + string value = 6; + string actor_id = 7; +} +message BeginAuthorizationRequest { + uint32 protocol = 1; + ChangeContext context = 2; + string registration_id = 3; + CredentialReference client_secret = 4; + optional int64 expected_version = 5; + string actor_id = 6; + bool sharing_consent = 7; +} +message BeginAuthorizationResponse { + uint32 protocol = 1; + string flow_id = 2; + AuthorityContext context = 3; + string authorization_url = 4; + int64 expires_at = 5; +} +message CompleteAuthorizationRequest { + uint32 protocol = 1; + AuthorityContext context = 2; + string registration_id = 3; + string flow_id = 4; + string state = 5; + string redirect_uri = 6; + optional string issuer = 7; + string code = 8; + string actor_id = 9; +} +message GetStatusRequest { + uint32 protocol = 1; + string connection_id = 2; + CredentialReference reference = 3; +} +message RevokeRequest { + uint32 protocol = 1; + CredentialStatus expected = 2; + string actor_id = 3; +} +message WorkerIdentity { + uint32 protocol = 1; + string boot_id = 2; + string zone_id = 3; + string policy_digest = 4; + string policy_revision = 5; +} +message Admission { + string id = 1; + string deployment_id = 2; + int64 expires_at = 3; +} +message AdministrationScope { + string request_id = 1; + string actor_id = 2; +} +message ExecutionScope { + string session_id = 1; + string run_id = 2; + string tool_call_id = 3; + string binding_hash = 4; +} +message LeaseRequest { + uint32 protocol = 1; + AuthorityContext context = 2; + CredentialReference reference = 3; + WorkerIdentity worker = 4; + string intent_id = 5; + string request_digest = 6; + string operation = 7; + // Short credential delivery deadline, at most ten seconds. + int64 expires_at = 8; + // Frozen worker operation deadline, at most five minutes. + int64 operation_expires_at = 12; + Admission admission = 9; + oneof scope { + AdministrationScope administration = 10; + ExecutionScope execution = 11; + } +} + +// Transient operation memory only. Never queue, cache, persist, log, or return +// this message to a browser or guest. Worker dispatch consumes the bound intent. +message LeaseResponse { + uint32 protocol = 1; + string header = 2; + string value = 3; + int64 expires_at = 4; + string intent_id = 5; + int64 operation_expires_at = 6; +} + +// Exact attempt only. Cancellation persists in the existing flow row. +message CancelAuthorizationRequest { + uint32 protocol = 1; + string flow_id = 2; + string connection_id = 3; + string registration_id = 4; + string actor_id = 5; +} +message CancelAuthorizationResponse { + uint32 protocol = 1; + string flow_id = 2; + string connection_id = 3; + string registration_id = 4; +} + +message PutSecretResponse { + CredentialStatus status = 1; +} + +message CompleteAuthorizationResponse { + CredentialStatus status = 1; +} + +message GetStatusResponse { + CredentialStatus status = 1; +} + +message RevokeResponse { + CredentialStatus status = 1; +} diff --git a/crates/engram-protocol/proto/engram/mcp/v1/operator.proto b/crates/engram-protocol/proto/engram/mcp/v1/operator.proto new file mode 100644 index 000000000..6076803da --- /dev/null +++ b/crates/engram-protocol/proto/engram/mcp/v1/operator.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; +package engram.mcp.v1; + +// Only the dedicated operator mTLS listener registers this mutation service. +service OperatorPublicationService { + rpc Stage(OperatorPublicationServiceStageRequest) returns (OperatorPublicationServiceStageResponse); + rpc Read(OperatorPublicationServiceReadRequest) returns (OperatorPublicationServiceReadResponse); +} +message OperatorPublicationServiceStageRequest { + uint32 protocol = 1; + optional uint64 expected_generation = 2; + string configured_candidate_identity = 3; +} +message OperatorPublicationServiceReadRequest { + uint32 protocol = 1; +} +message OperatorPublicationServiceStageResponse { + uint32 protocol = 1; + bytes publication_json = 2; +} +message OperatorPublicationServiceReadResponse { + uint32 protocol = 1; + bytes publication_json = 2; +} diff --git a/crates/engram-protocol/proto/engram/mcp/v1/session_policy.proto b/crates/engram-protocol/proto/engram/mcp/v1/session_policy.proto new file mode 100644 index 000000000..53887954c --- /dev/null +++ b/crates/engram-protocol/proto/engram/mcp/v1/session_policy.proto @@ -0,0 +1,223 @@ +syntax = "proto3"; +package engram.mcp.v1; + +import "engram/app/v1/session.proto"; + +// Dedicated pinned mTLS only. Create reuses the ordinary creation core. +service SessionPolicyService { + rpc ReadManagedCreationState(ReadManagedCreationStateRequest) returns (ReadManagedCreationStateResponse); + rpc PrepareManagedBoot(PrepareManagedBootRequest) returns (PrepareManagedBootResponse); + rpc RecoverManagedBoot(RecoverManagedBootRequest) returns (RecoverManagedBootResponse); + rpc ReadPublishedOperator(ReadPublishedOperatorRequest) returns (ReadPublishedOperatorResponse); + rpc ReadManagedBoot(ReadManagedBootRequest) returns (ReadManagedBootResponse); + rpc FinalizeManagedBoot(FinalizeManagedBootRequest) returns (FinalizeManagedBootResponse); + rpc ReserveManagedCreate(ReserveManagedCreateRequest) returns (ReserveManagedCreateResponse); + rpc ReadNativeManifest(ReadNativeManifestRequest) returns (ReadNativeManifestResponse); + rpc ReadManagedPolicy(ReadManagedPolicyRequest) returns (ReadManagedPolicyResponse); + rpc ApplyManagedPolicy(ApplyManagedPolicyRequest) returns (ApplyManagedPolicyResponse); + rpc CreatePolicySession(CreatePolicySessionRequest) returns (CreatePolicySessionResponse); + rpc ReadWritePolicy(ReadWritePolicyRequest) returns (ReadWritePolicyResponse); + rpc AcceptPlanExit(AcceptPlanExitRequest) returns (AcceptPlanExitResponse); + rpc ValidateExecutionIdentity(ValidateExecutionIdentityRequest) returns (ValidateExecutionIdentityResponse); + rpc ValidateDispatchFence(ValidateDispatchFenceRequest) returns (ValidateDispatchFenceResponse); +} +message CreatePolicySessionRequest { + uint32 protocol = 1; + engram.app.v1.CreateSessionRequest create = 2; + string actor_id = 3; + string input_id = 4; + optional string parent_session_id = 5; +} +message ReadWritePolicyRequest { + uint32 protocol = 1; + string session_id = 2; + optional string plan_tool_call_id = 3; +} +message AcceptPlanExitRequest { + uint32 protocol = 1; + engram.app.v1.McpPlanReview expected = 2; + string actor_id = 3; + string result_json = 4; +} +message ValidateDispatchFenceRequest { + uint32 protocol = 1; + string session_id = 2; + string run_id = 3; + string tool_call_id = 4; + engram.app.v1.McpWritePolicyRef accepted_policy_ref = 5; + string accepted_write_eligibility = 6; + repeated engram.app.v1.McpWritePolicyRef ancestor_policy_refs = 7; + string model_tool_name = 8; + string original_arguments = 9; +} +message ValidateDispatchFenceResponse { + uint32 protocol = 1; + string status = 2; +} + +message CreatePolicySessionResponse { + engram.app.v1.CreateSessionResponse session = 1; +} +message ReadWritePolicyResponse { + engram.app.v1.GetMcpWritePolicyResponse state = 1; +} +message AcceptPlanExitResponse { + engram.app.v1.McpTrustedWritePolicy policy = 1; +} + +message ValidateExecutionIdentityRequest { + uint32 protocol = 1; + string session_id = 2; + string run_id = 3; + string tool_call_id = 4; + engram.app.v1.McpWritePolicyRef accepted_policy_ref = 5; + string accepted_write_eligibility = 6; + repeated engram.app.v1.McpWritePolicyRef ancestor_policy_refs = 7; + string model_tool_name = 8; + string original_arguments = 9; +} +message ValidateExecutionIdentityResponse { + uint32 protocol = 1; + string status = 2; +} + +// These messages describe a host receipt, not permission to expose guest tools. +enum ManagedPolicyPhase { + MANAGED_POLICY_PHASE_UNSPECIFIED = 0; + MANAGED_POLICY_PHASE_BOOTSTRAP = 1; + MANAGED_POLICY_PHASE_FINAL = 2; +} +message ManagedPolicyTarget { + string session_id = 1; + string sandbox_id = 2; + string host_id = 3; + uint64 binding_epoch = 4; +} +message ManagedPolicyContext { + ManagedPolicyTarget target = 1; + uint64 operator_generation = 2; + string operator_identity = 3; + ManagedPolicyPhase phase = 4; + optional string binding_hash = 5; +} +message ManagedDenyPolicy { + uint32 version = 1; + uint64 revision = 2; + repeated string hosts = 3; + string digest = 4; +} +message ManagedAppliedState { + ManagedPolicyContext context = 1; + ManagedDenyPolicy policy = 2; + string host_incarnation = 3; + string proxy_incarnation = 4; + string state_id = 5; + optional ManagedNetworkProof network_proof = 6; +} +message ReadManagedPolicyRequest { + uint32 protocol = 1; + ManagedPolicyTarget target = 2; +} +message ReadManagedPolicyResponse { + uint32 protocol = 1; + optional ManagedAppliedState applied = 2; + // Required presence: absence means an unsupported read contract, not no policy. + ManagedRetainedDesired retained_desired = 3; +} +// Coordinator persistence at the checked read target. This is never an applied ACK. +message ManagedRetainedDesired { + ManagedPolicyTarget target = 1; + // Absence inside a present wrapper means no retained policy at this read. + optional ManagedDenyPolicy policy = 2; +} +message ApplyManagedPolicyRequest { + uint32 protocol = 1; + ManagedPolicyContext context = 2; + ManagedDenyPolicy policy = 3; + optional ManagedAppliedState expected = 4; +} +message ApplyManagedPolicyResponse { + uint32 protocol = 1; + ManagedAppliedState applied = 2; +} + +message ManagedNetworkProof { + ManagedPolicyTarget target = 1; + string guest_ip = 2; + string network_identity = 3; +} + +// Exact bytes of CreatePolicySessionRequest, retained by the caller for retries. +// The coordinator stores only the digest of this payload, which can hold secrets. +message ReserveManagedCreateRequest { + uint32 protocol = 1; + bytes creation_request = 2; + string task_id = 3; + uint32 native_version = 4; + string native_digest = 5; + string input_digest = 6; +} +message ManagedCreateIdentity { + string session_id = 1; + string task_id = 2; + string input_id = 3; + string actor_id = 4; + optional string parent_session_id = 5; + string write_eligibility = 6; + string input_digest = 7; +} +message ReserveManagedCreateResponse { + uint32 protocol = 1; + ManagedCreateIdentity identity = 2; + // A held queued reservation is not permission to start a sandbox. + string state = 3; +} +message ReadNativeManifestRequest { + uint32 protocol = 1; + ManagedCreateIdentity identity = 2; +} +message ReadNativeManifestResponse { + uint32 protocol = 1; + ManagedCreateIdentity identity = 2; + uint32 native_version = 3; + bytes native_bytes = 4; + string native_digest = 5; +} + +message ReadManagedBootRequest { uint32 protocol = 1; ManagedCreateIdentity identity = 2; } +message ReadManagedBootResponse { uint32 protocol = 1; optional bytes boot_json = 2; } +// Exact typed ManagedBootFinalizeInput JSON. No guest or profile input is read. +message FinalizeManagedBootRequest { uint32 protocol = 1; bytes input_json = 2; } + +message FinalizeManagedBootResponse { uint32 protocol = 1; optional bytes boot_json = 2; } +message ReadPublishedOperatorRequest { + uint32 protocol = 1; +} +message ReadPublishedOperatorResponse { + uint32 protocol = 1; + bytes publication_json = 2; +} + +message ReadManagedCreationStateRequest { + uint32 protocol = 1; + ManagedCreateIdentity identity = 2; + uint32 native_version = 3; + string native_digest = 4; +} +message ReadManagedCreationStateResponse { uint32 protocol = 1; bytes state_json = 2; } +message PrepareManagedBootRequest { + uint32 protocol = 1; + ManagedCreateIdentity identity = 2; + uint32 native_version = 3; + string native_digest = 4; +} +message PrepareManagedBootResponse { uint32 protocol = 1; bytes state_json = 2; } +message RecoverManagedBootRequest { + uint32 protocol = 1; + ManagedCreateIdentity identity = 2; + uint32 native_version = 3; + string native_digest = 4; + bytes expected_source_key_json = 5; + string recovery_id = 6; +} +message RecoverManagedBootResponse { uint32 protocol = 1; bytes state_json = 2; } diff --git a/crates/engram-protocol/proto/host_service.proto b/crates/engram-protocol/proto/host_service.proto index 99a49a933..77656a819 100644 --- a/crates/engram-protocol/proto/host_service.proto +++ b/crates/engram-protocol/proto/host_service.proto @@ -8,9 +8,8 @@ syntax = "proto3"; // SnapshotMetadata, SessionEgressPolicy, AgentSpec, and ExecRequest's base) // stay bincode-encoded inside a `bytes` field. That carves a small // proto surface while keeping the existing Rust types -// authoritative — coord+host always deploy at the same OSS tag so -// bincode drift can't happen (same atomic-deploy property the WS -// path relies on today). +// authoritative. The wire-version gate rejects incompatible peers +// before it decodes positional payloads. // // UUIDs ride as 16-byte `bytes` fields, not as strings. Strings // would double the wire size for no schema benefit. @@ -18,13 +17,21 @@ syntax = "proto3"; package engram.host.v1; service HostService { + rpc CaptureManagedBoot(ManagedBootRequest) returns (ManagedBootResponse); + rpc ReadManagedColdAssets(ReadManagedColdAssetsRequest) returns (ReadManagedColdAssetsResponse); + rpc PrepareManagedBoot(ManagedBootRequest) returns (ManagedBootResponse); + rpc ReadManagedBoot(ManagedBootRequest) returns (ManagedBootResponse); + rpc FinalizeManagedBoot(ManagedBootRequest) returns (ManagedBootResponse); + rpc StartManagedBoot(ManagedBootRequest) returns (ManagedBootResponse); + rpc ReadManagedPolicy(ReadManagedPolicyRequest) returns (ReadManagedPolicyResponse); + rpc ApplyManagedPolicy(ApplyManagedPolicyRequest) returns (ApplyManagedPolicyResponse); // ---- sandbox lifecycle ---- // // ADR 0079 (wire v12): every session-scoped lifecycle RPC carries a // (session_id, fencing_epoch) stamp — see `FencedSandboxRequest` — // checked by the host's `check_session_epoch` gate right after the // wire-version gate. Sandbox-scoped-but-not-lifecycle RPCs (harness - // routing, proxies, COW diagnostics, migration internals) are NOT + // routing, proxies, migration internals) are NOT // fenced: they never make the writes the op executor serializes. rpc CreateSandbox(CreateSandboxRequest) returns (CreateSandboxResponse); rpc DestroySandbox(FencedSandboxRequest) returns (Empty); @@ -798,3 +805,16 @@ message ProxyPortData { // side say "I'm done writing" explicitly when it has no trailing data. message ProxyPortClose {} +// These operations are served only in mandatory managed mTLS mode. +message ReadManagedPolicyRequest { bytes target_bincode = 1; } +message ReadManagedPolicyResponse { bytes state_bincode = 1; } +message ApplyManagedPolicyRequest { bytes request_bincode = 1; } +message ApplyManagedPolicyResponse { bytes state_bincode = 1; } + +// Versioned, bounded JSON for the exact typed managed boot commands. +// Finalize contains plaintext environment only on authenticated host transport. +message ManagedBootRequest { bytes payload_json = 1; } +message ManagedBootResponse { bytes payload_json = 1; } + +message ReadManagedColdAssetsRequest { bytes payload_json = 1; } +message ReadManagedColdAssetsResponse { bytes payload_json = 1; } diff --git a/crates/engram-protocol/src/grpc_client.rs b/crates/engram-protocol/src/grpc_client.rs index a6838feaa..4fbc88670 100644 --- a/crates/engram-protocol/src/grpc_client.rs +++ b/crates/engram-protocol/src/grpc_client.rs @@ -117,22 +117,10 @@ fn fenced_request(id: SandboxId, fence: SessionFence) -> FencedSandboxRequest { } } -/// Deadline applied (via the gRPC `grpc-timeout` header) to the -/// snapshot-restore RPCs — `restore` (resume), -/// `restore_base_for_session` (cold-create-via-restore), and -/// `start_agent` (the harness (re)attach leg of the same resume -/// pipeline, which shares the wedge modes). Without it a -/// host that wedges mid-restore leaves the coord caller hung -/// indefinitely (observed as a ~6-minute dead-host stall); the -/// keepalive pings only catch a *silent* connection, not a peer that -/// ACKs but never completes the call. Set generously above the -/// slowest legitimate restore (cold boot ~15-30s, rechunk-heavy -/// resumes up to a couple of minutes) so it never aborts a real -/// restore, while still bounding the pathological hang. Because it -/// rides the gRPC deadline header, the *host* side observes it too and -/// can abort its own work. Override via -/// `ENGRAM_RESTORE_RPC_TIMEOUT_SECS`. -fn restore_rpc_timeout() -> Duration { +/// Deadline for restore, harness start and managed lifecycle calls. +/// The host receives the same deadline through the gRPC timeout header. +/// `ENGRAM_RESTORE_RPC_TIMEOUT_SECS` overrides the 240-second default. +pub fn restore_rpc_timeout() -> Duration { let secs = std::env::var("ENGRAM_RESTORE_RPC_TIMEOUT_SECS") .ok() .and_then(|s| s.parse::().ok()) @@ -141,36 +129,72 @@ fn restore_rpc_timeout() -> Duration { Duration::from_secs(secs) } +fn managed_request(message: T) -> tonic::Request { + let mut request = tonic::Request::new(message); + request.set_timeout(restore_rpc_timeout()); + request.metadata_mut().insert( + crate::wire::MANAGED_PROTOCOL_METADATA_KEY, + crate::wire::MANAGED_PROTOCOL_VERSION + .to_string() + .parse() + .expect("protocol number is ASCII"), + ); + request +} + /// Coord-side client wrapping a `tonic::transport::Channel` to one /// host. Cheap to clone (tonic clients hold an `Arc` /// internally); one instance per host is the steady-state shape. #[derive(Clone)] pub struct GrpcHostClient { inner: HostServiceClient>, + managed_transport: bool, } impl GrpcHostClient { /// Build a client over an existing tonic `Channel`. The channel /// is HTTP/2-multiplexing — many concurrent RPCs share one /// TCP+H2 connection — so we want exactly one instance per - /// host. Callers go through `GrpcHostPool::dispatch` which + /// host. Callers go through `GrpcHostPool::get_or_warm` which /// hands out a clone of the pool's entry. /// /// Every RPC carries the caller's `traceparent` via /// [`TraceparentInjector`] (ADR 0019 distributed tracing). /// - /// The inbound decode cap is 32 MiB on every client (was an opt-in - /// for `ReapMaterializeDir`'s 16 MiB reply; ADR 0095's - /// `PeerChunkGet` frames — 4 MiB data + overhead — need past - /// tonic's 4 MiB default too, and the cap is just a limit check, - /// no allocation change). + /// The 32 MiB decode cap permits peer chunk frames with 4 MiB of data + /// plus framing overhead. It does not allocate memory before a reply. pub fn new(channel: Channel) -> Self { Self { inner: HostServiceClient::with_interceptor(channel, TraceparentInjector) .max_decoding_message_size(32 * 1024 * 1024), + managed_transport: false, } } + /// Only the pool can construct a client with pinned mutual TLS authority. + pub(crate) fn with_pinned_tls( + host: engram_core::HostId, + address: &str, + endpoint: tonic::transport::Endpoint, + tls: &crate::host_tls::HostClientTls, + ) -> Result { + let endpoint = tls + .endpoint(host, address, endpoint) + .map_err(SandboxError::InvalidSpec)?; + let mut client = Self::new(endpoint.connect_lazy()); + client.managed_transport = true; + Ok(client) + } + + fn require_managed_transport(&self) -> Result<(), SandboxError> { + if !self.managed_transport { + return Err(SandboxError::Unsupported( + "managed RPC requires pinned mutual TLS".into(), + )); + } + Ok(()) + } + /// Fire a no-op `Ping` to force TCP+H2 handshake on a freshly /// built `Channel::connect_lazy`. Used by `GrpcHostPool::warm` /// so the next real RPC doesn't pay cold-dial latency. @@ -908,7 +932,7 @@ impl GrpcHostClient { let mut req = tonic::Request::new(StartAgentRequest { sandbox_id: sandbox_id.as_uuid().as_bytes().to_vec(), agent_bincode: encode_bincode(&agent, "AgentSpec")?, - policy_bincode: encode_bincode(&policy, "SessionEgressPolicy")?, + policy_bincode: crate::ordinary_policy::encode(&policy)?, session_id: fence.session_id.as_uuid().as_bytes().to_vec(), fencing_epoch: fence.epoch, }); @@ -1417,13 +1441,7 @@ fn proxy_body_to_shell_frame( }) } -// ---- helpers ---- -// -// Use `bincode::serialize` / `bincode::deserialize` with default -// config to match the existing WS codec (`src/codec.rs:49,71`) so -// a SandboxSpec encoded on the gRPC wire decodes identically to one -// on the WS wire. Both transports use the same Rust types during -// the ADR 0013 transition. +// General payloads use bincode defaults. Ordinary policy has its own strict codec. fn encode_bincode( value: &T, @@ -1656,6 +1674,150 @@ impl HostClient for GrpcHostClient { Self::restore_base_for_session(self, metadata, session_env, selected_mounts, fence).await } + async fn read_managed_cold_assets( + &self, + request: engram_core::types::managed_assets::ManagedColdAssetsRequest, + ) -> Result { + self.require_managed_transport()?; + request + .validate() + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))?; + let payload_json = crate::managed_boot::encode(&request)?; + let response = self + .inner + .clone() + .read_managed_cold_assets(managed_request(crate::grpc::ReadManagedColdAssetsRequest { + payload_json, + })) + .await + .map_err(managed_grpc_error)? + .into_inner(); + let receipt: engram_core::types::managed_assets::ManagedColdAssetsReceipt = + crate::managed_boot::decode(&response.payload_json)?; + receipt + .validate_for(&request) + .map_err(|e| SandboxError::InvalidSpec(e.to_string()))?; + Ok(receipt) + } + async fn capture_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedCaptureRequest, + ) -> Result { + self.require_managed_transport()?; + let payload_json = crate::managed_boot::encode(&request)?; + let response = self + .inner + .clone() + .capture_managed_boot(managed_request(crate::grpc::ManagedBootRequest { + payload_json, + })) + .await + .map_err(managed_grpc_error)? + .into_inner(); + crate::managed_boot::decode(&response.payload_json) + } + async fn prepare_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootPrepare, + ) -> Result { + self.require_managed_transport()?; + let payload_json = crate::managed_boot::encode(&request)?; + let response = self + .inner + .clone() + .prepare_managed_boot(managed_request(crate::grpc::ManagedBootRequest { + payload_json, + })) + .await + .map_err(managed_grpc_error)? + .into_inner(); + crate::managed_boot::decode(&response.payload_json) + } + async fn read_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootKey, + ) -> Result { + self.require_managed_transport()?; + let payload_json = crate::managed_boot::encode(&request)?; + let response = self + .inner + .clone() + .read_managed_boot(managed_request(crate::grpc::ManagedBootRequest { + payload_json, + })) + .await + .map_err(managed_grpc_error)? + .into_inner(); + crate::managed_boot::decode(&response.payload_json) + } + async fn finalize_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootFinalize, + ) -> Result { + self.require_managed_transport()?; + let payload_json = crate::managed_boot::encode(&request)?; + let response = self + .inner + .clone() + .finalize_managed_boot(managed_request(crate::grpc::ManagedBootRequest { + payload_json, + })) + .await + .map_err(managed_grpc_error)? + .into_inner(); + crate::managed_boot::decode(&response.payload_json) + } + async fn start_managed_boot( + &self, + request: engram_core::types::managed_boot::ManagedBootStart, + ) -> Result { + self.require_managed_transport()?; + let payload_json = crate::managed_boot::encode(&request)?; + let response = self + .inner + .clone() + .start_managed_boot(managed_request(crate::grpc::ManagedBootRequest { + payload_json, + })) + .await + .map_err(managed_grpc_error)? + .into_inner(); + crate::managed_boot::decode(&response.payload_json) + } + async fn read_managed_policy( + &self, + target: engram_core::types::managed_deny::ManagedPolicyTarget, + ) -> Result, SandboxError> { + self.require_managed_transport()?; + let response = self + .inner + .clone() + .read_managed_policy(managed_request(crate::grpc::ReadManagedPolicyRequest { + target_bincode: encode_bincode(&target, "managed target")?, + })) + .await + .map_err(managed_grpc_error)? + .into_inner(); + crate::managed_policy::decode(&response.state_bincode) + .map_err(|_| SandboxError::InvalidSpec("invalid managed state".into())) + } + async fn apply_managed_policy( + &self, + request: engram_core::types::managed_deny::ManagedPolicyApply, + ) -> Result { + self.require_managed_transport()?; + let response = self + .inner + .clone() + .apply_managed_policy(managed_request(crate::grpc::ApplyManagedPolicyRequest { + request_bincode: encode_bincode(&request, "managed apply")?, + })) + .await + .map_err(managed_grpc_error)? + .into_inner(); + crate::managed_policy::decode(&response.state_bincode) + .map_err(|_| SandboxError::InvalidSpec("invalid managed state".into())) + } async fn start_agent( &self, id: SandboxId, @@ -1776,10 +1938,7 @@ impl HostClient for GrpcHostClient { // doesn't carry static capability or in-proc sink wiring. } -/// Map a tonic Status into the same `SandboxError` shape today's WS -/// path produces. The bincode wire used `RemoteError::into_sandbox`; -/// tonic gives us a status code + message, so we do the equivalent -/// mapping per code. +/// Map gRPC status codes and exact refusal markers to typed sandbox errors. fn grpc_to_sandbox_err(status: tonic::Status) -> SandboxError { use tonic::Code; match status.code() { @@ -1810,7 +1969,11 @@ fn grpc_to_sandbox_err(status: tonic::Status) -> SandboxError { // io::ErrorKind for the resume re-plan. A `failed_precondition` // with NEITHER marker falls through to the generic mapping. Code::FailedPrecondition => { - if let Some((host, coord)) = crate::wire::parse_wire_skew_message(status.message()) { + if status.message() == crate::wire::MANAGED_RECOVERY_REQUIRED_STATUS { + SandboxError::ManagedRecoveryRequired + } else if let Some((host, coord)) = + crate::wire::parse_wire_skew_message(status.message()) + { SandboxError::WireSkew { host, coord } } else if let Some((kind, message)) = crate::wire::parse_harness_spawn_message(status.message()) @@ -1828,10 +1991,147 @@ fn grpc_to_sandbox_err(status: tonic::Status) -> SandboxError { } } +fn managed_grpc_error(status: tonic::Status) -> SandboxError { + if status.code() == tonic::Code::Unimplemented { + SandboxError::Unsupported(status.message().to_string()) + } else { + grpc_to_sandbox_err(status) + } +} + #[cfg(test)] mod grpc_err_tests { use super::*; + #[derive(Clone)] + struct CountingHost(std::sync::Arc); + + impl tonic::server::NamedService for CountingHost { + const NAME: &'static str = "engram.host.v1.HostService"; + } + + impl tonic::codegen::Service> for CountingHost { + type Response = tonic::codegen::http::Response; + type Error = std::convert::Infallible; + type Future = std::future::Ready>; + + fn poll_ready( + &mut self, + _: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn call( + &mut self, + request: tonic::codegen::http::Request, + ) -> Self::Future { + let _ = request; + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let status = tonic::Status::unimplemented("unexpected ordinary transport call"); + std::future::ready(Ok(status.into_http())) + } + } + + #[tokio::test] + async fn ordinary_client_refuses_managed_calls_before_network() { + use engram_core::types::{managed_assets::ManagedColdAssetsRequest, manifest::ManifestRef}; + use futures::FutureExt; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let incoming = futures::stream::unfold(listener, |listener| async move { + let next = listener.accept().await.map(|(stream, _)| stream); + Some((next, listener)) + }); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let mut server = tokio::spawn( + tonic::transport::Server::builder() + .add_service(CountingHost(calls.clone())) + .serve_with_incoming_shutdown(incoming, async { + let _ = stopped.await; + }), + ); + let exercise = async { + let channel = tonic::transport::Endpoint::from_shared(format!("http://{address}")) + .unwrap() + .connect() + .await + .unwrap(); + let client = GrpcHostClient::new(channel); + let request = ManagedColdAssetsRequest { + version: 1, + host_id: engram_core::HostId::new(), + session_id: SessionId::new(), + intent_id: "intent-a".into(), + image: "test.invalid/image:v1".into(), + image_digest: format!("sha256:{}", "a".repeat(64)), + disk_manifest: ManifestRef::new(), + aux_ro_drives: (0..3) + .map(|i| AuxRoDrive { + drive_id: AuxRoDrive::slot_drive_id(i), + guest_mount: AuxRoDrive::slot_guest_mount(i), + fs_type: "squashfs".into(), + sha256: Some("b".repeat(64)), + }) + .collect(), + }; + let error = client.read_managed_cold_assets(request).await.unwrap_err(); + assert!(matches!(error, SandboxError::Unsupported(_)), "{error:?}"); + assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 0); + // The legacy snapshot fallback must retain its existing mapping. + assert!( + matches!(grpc_to_sandbox_err(tonic::Status::unimplemented("legacy")), SandboxError::InvalidSpec(message) if message == "legacy") + ); + }; + let outcome = tokio::time::timeout( + Duration::from_secs(3), + std::panic::AssertUnwindSafe(exercise).catch_unwind(), + ) + .await; + let _ = stop.send(()); + let joined = tokio::time::timeout(Duration::from_secs(2), &mut server).await; + if joined.is_err() { + server.abort(); + let _ = server.await; + } + joined.unwrap().unwrap().unwrap(); + outcome.unwrap().unwrap(); + } + + #[test] + fn managed_request_retains_both_contract_markers_and_deadline() { + let request = TraceparentInjector.call(managed_request(())).unwrap(); + assert_eq!( + request + .metadata() + .get(crate::wire::WIRE_VERSION_METADATA_KEY) + .unwrap(), + "38" + ); + assert_eq!( + request + .metadata() + .get(crate::wire::MANAGED_PROTOCOL_METADATA_KEY) + .unwrap(), + "43" + ); + assert!(request.metadata().contains_key("grpc-timeout")); + } + + #[test] + fn managed_errors_preserve_unsupported_and_current_transport_failures() { + assert!( + matches!(managed_grpc_error(tonic::Status::unimplemented("absent")), SandboxError::Unsupported(message) if message == "absent") + ); + assert!( + matches!(managed_grpc_error(tonic::Status::invalid_argument("invalid")), SandboxError::InvalidSpec(message) if message == "invalid") + ); + assert!( + matches!(managed_grpc_error(tonic::Status::unavailable("down")), SandboxError::Unavailable(message) if message == "down") + ); + } + #[test] fn migration_client_interceptor_stamps_wire_version() { let req = TraceparentInjector.call(tonic::Request::new(())).unwrap(); @@ -1904,6 +2204,26 @@ mod grpc_err_tests { assert!(matches!(err, SandboxError::Vm(_))); } + #[test] + fn managed_recovery_requires_exact_status_marker() { + let marker = crate::wire::MANAGED_RECOVERY_REQUIRED_STATUS; + assert!(matches!( + grpc_to_sandbox_err(tonic::Status::failed_precondition(marker)), + SandboxError::ManagedRecoveryRequired + )); + for status in [ + tonic::Status::failed_precondition(format!("{marker}: extra")), + tonic::Status::failed_precondition("managed sandbox requires cold recovery"), + tonic::Status::invalid_argument(marker), + tonic::Status::unavailable(marker), + ] { + assert!(!matches!( + grpc_to_sandbox_err(status), + SandboxError::ManagedRecoveryRequired + )); + } + } + #[test] fn harness_spawn_failed_precondition_maps_to_typed_variant() { // ADR 0116 B-D4: the host's spawn-rejection marker maps back to diff --git a/crates/engram-protocol/src/grpc_pool.rs b/crates/engram-protocol/src/grpc_pool.rs index d55c0bafa..9c14a1ac4 100644 --- a/crates/engram-protocol/src/grpc_pool.rs +++ b/crates/engram-protocol/src/grpc_pool.rs @@ -30,6 +30,7 @@ use crate::grpc_client::GrpcHostClient; /// coordinator; each coord pod gets its own. pub struct GrpcHostPool { entries: DashMap, + tls: Option, /// HTTP/2 keepalive interval. Default 15s — short enough that /// idle connections stay healthy through L4-LB / NAT timeouts, /// long enough that the chatter is invisible in production. @@ -60,21 +61,27 @@ impl GrpcHostPool { pub fn new() -> Self { Self { entries: DashMap::new(), + tls: None, keepalive_interval: Duration::from_secs(15), keepalive_timeout: Duration::from_secs(5), connect_timeout: Duration::from_secs(2), } } + pub fn with_tls(mut self, tls: crate::host_tls::HostClientTls) -> Self { + self.tls = Some(tls); + self + } + /// Idempotent warm-up: ensure a `Channel` exists for `host_id` at /// `host_addr` and start the TCP+H2 handshake. A different address /// replaces the entry. Callers: host registration, the heartbeat /// handler, coordinator startup, and the host registry's read-through /// of the `hosts` row. /// - /// `host_addr` example: `http://10.10.0.42:9101`. Plain HTTP/2 - /// (h2c) — TLS off, bearer-token auth happens per-RPC via tonic - /// interceptors that the call sites attach. + /// Managed mode requires the configured HTTPS endpoint, CA, server name, + /// leaf pins, and client identity. Ordinary development can use HTTP/2 + /// without TLS. An HTTPS address never falls back to plaintext. pub async fn warm(&self, host_id: HostId, host_addr: String) -> Result<(), SandboxError> { // If we already have an entry for this addr, leave it alone. // If the addr changed (host re-registered with a fresh IP @@ -96,11 +103,15 @@ impl GrpcHostPool { self.keepalive_interval, self.keepalive_timeout, )?; - // `connect_lazy` returns immediately; the first RPC pays the - // TCP+H2 handshake. We follow up with `ping` below to force - // the handshake before the first real RPC. - let channel = endpoint.connect_lazy(); - let client = GrpcHostClient::new(channel); + let client = match &self.tls { + Some(tls) => GrpcHostClient::with_pinned_tls(host_id, &host_addr, endpoint, tls)?, + None if host_addr.starts_with("https://") => { + return Err(SandboxError::InvalidSpec( + "host TLS identity is not configured".into(), + )); + } + None => GrpcHostClient::new(endpoint.connect_lazy()), + }; self.entries.insert( host_id, diff --git a/crates/engram-protocol/src/host_tls.rs b/crates/engram-protocol/src/host_tls.rs new file mode 100644 index 000000000..a7eb25ab4 --- /dev/null +++ b/crates/engram-protocol/src/host_tls.rs @@ -0,0 +1,141 @@ +//! Operator-owned, host-specific mTLS configuration. No plaintext fallback. +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use serde::Deserialize; +use sha2::{Digest, Sha256}; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::PathBuf, + sync::Arc, +}; + +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HostPeer { + pub endpoint: String, + pub server_name: String, + pub server_ca: PathBuf, + pub server_pins: BTreeSet, +} +#[derive(Clone, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct HostClientTls { + pub client_certificate: PathBuf, + pub client_key: PathBuf, + pub hosts: BTreeMap, +} +pub fn certificate_pin(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} +pub fn validate_pins(pins: &BTreeSet) -> Result<(), String> { + if pins.is_empty() + || pins.len() > 8 + || pins.iter().any(|p| { + p.len() != 64 + || !p + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) + { + return Err("invalid host certificate pins".into()); + } + Ok(()) +} +#[derive(Debug)] +struct PinnedVerifier { + chain: Arc, + pins: BTreeSet, +} +impl ServerCertVerifier for PinnedVerifier { + fn verify_server_cert( + &self, + end: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + name: &ServerName<'_>, + ocsp: &[u8], + now: UnixTime, + ) -> Result { + let verified = self + .chain + .verify_server_cert(end, intermediates, name, ocsp, now)?; + if !self.pins.contains(&certificate_pin(end.as_ref())) { + return Err(rustls::Error::General( + "host certificate pin mismatch".into(), + )); + } + Ok(verified) + } + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + sig: &rustls::DigitallySignedStruct, + ) -> Result { + self.chain.verify_tls12_signature(message, cert, sig) + } + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + sig: &rustls::DigitallySignedStruct, + ) -> Result { + self.chain.verify_tls13_signature(message, cert, sig) + } + fn supported_verify_schemes(&self) -> Vec { + self.chain.supported_verify_schemes() + } +} +impl HostClientTls { + pub fn from_env() -> Result, String> { + let Some(path) = std::env::var_os("ENGRAM_HOST_CLIENT_TLS_CONFIG") else { + return Ok(None); + }; + let bytes = std::fs::read(path).map_err(|_| "cannot read host TLS configuration")?; + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|_| "invalid host TLS configuration".into()) + } + pub fn endpoint( + &self, + host: engram_core::HostId, + address: &str, + endpoint: tonic::transport::Endpoint, + ) -> Result { + engram_tls::install_provider(); + let peer = self + .hosts + .get(&host) + .ok_or("host TLS identity is not configured")?; + if peer.endpoint != address + || !address.starts_with("https://") + || peer.server_name.is_empty() + { + return Err("host TLS endpoint mismatch".into()); + } + validate_pins(&peer.server_pins)?; + let ca = std::fs::read(&peer.server_ca).map_err(|_| "cannot read host CA")?; + let mut roots = rustls::RootCertStore::empty(); + for cert in rustls_pemfile::certs(&mut &ca[..]) { + roots + .add(cert.map_err(|_| "invalid host CA")?) + .map_err(|_| "invalid host CA")?; + } + let chain = rustls::client::WebPkiServerVerifier::builder(Arc::new(roots)) + .build() + .map_err(|_| "invalid host CA")?; + let certificate = std::fs::read(&self.client_certificate) + .map_err(|_| "cannot read coordinator certificate")?; + let key = std::fs::read(&self.client_key).map_err(|_| "cannot read coordinator TLS key")?; + endpoint + .tls_config_with_verifier( + tonic::transport::ClientTlsConfig::new() + .domain_name(&peer.server_name) + .identity(tonic::transport::Identity::from_pem(certificate, key)), + Arc::new(PinnedVerifier { + chain, + pins: peer.server_pins.clone(), + }), + ) + .map_err(|_| "invalid host TLS configuration".into()) + } +} diff --git a/crates/engram-protocol/src/lib.rs b/crates/engram-protocol/src/lib.rs index d9141da09..6d78c0f1f 100644 --- a/crates/engram-protocol/src/lib.rs +++ b/crates/engram-protocol/src/lib.rs @@ -9,16 +9,17 @@ //! idle-eviction-candidates, register) is plain HTTP/JSON — the //! payload types ([`heartbeat::HostCapacityReport`] etc.) are //! defined here for sharing across both sides. -//! -//! Pre-ADR-0013 this crate held a full bincode-over-WebSocket -//! frame protocol; that's been retired and only the shared -//! payload shapes remain. pub mod app; pub mod grpc; pub mod grpc_client; pub mod grpc_pool; pub mod heartbeat; +pub mod host_tls; +pub mod managed_boot; +pub mod managed_policy; +pub mod mcp; +pub mod ordinary_policy; pub mod wire; pub use heartbeat::*; diff --git a/crates/engram-protocol/src/managed_boot.rs b/crates/engram-protocol/src/managed_boot.rs new file mode 100644 index 000000000..64091168a --- /dev/null +++ b/crates/engram-protocol/src/managed_boot.rs @@ -0,0 +1,37 @@ +//! Bounded exact managed commands. Decoding cannot grant host authority. +use engram_core::SandboxError; +use serde::{de::DeserializeOwned, Serialize}; +const MAX_BYTES: usize = 8 * 1024 * 1024; +pub fn encode(value: &T) -> Result, SandboxError> { + let bytes = serde_json::to_vec(value).map_err(|_| invalid())?; + if bytes.len() > MAX_BYTES { + return Err(invalid()); + } + Ok(bytes) +} +pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() > MAX_BYTES { + return Err(invalid()); + } + serde_json::from_slice(bytes).map_err(|_| invalid()) +} +fn invalid() -> SandboxError { + SandboxError::InvalidSpec("invalid managed boot command".into()) +} +#[cfg(test)] +mod tests { + use super::*; + use engram_core::types::managed_boot::ManagedBootHostState; + #[test] + fn managed_boot_decode_is_bounded_and_rejects_unknown_state() { + assert!(decode::(br#"{"state":"warm"}"#).is_err()); + assert!(decode::(&vec![b' '; MAX_BYTES + 1]).is_err()); + assert!(decode::(b"null").is_err()); + } + proptest::proptest! { + #[test] + fn managed_boot_decode_never_panics(bytes in proptest::collection::vec(proptest::num::u8::ANY,0..8192)) { + let _=decode::(&bytes); + } + } +} diff --git a/crates/engram-protocol/src/managed_policy.rs b/crates/engram-protocol/src/managed_policy.rs new file mode 100644 index 000000000..d1e7530d5 --- /dev/null +++ b/crates/engram-protocol/src/managed_policy.rs @@ -0,0 +1,204 @@ +//! Strict conversions for the dedicated managed policy control service. +use crate::mcp as wire; +use engram_core::types::managed_deny as core; +fn missing() -> String { + "required managed policy field absent".into() +} +impl TryFrom for core::ManagedPolicyTarget { + type Error = String; + fn try_from(v: wire::ManagedPolicyTarget) -> Result { + let target = Self { + session_id: v + .session_id + .parse() + .map_err(|_| "invalid session identity")?, + sandbox_id: v + .sandbox_id + .parse() + .map_err(|_| "invalid sandbox identity")?, + host_id: v.host_id.parse().map_err(|_| "invalid host identity")?, + binding_epoch: v.binding_epoch, + }; + if target.session_id.as_uuid().is_nil() + || target.sandbox_id.as_uuid().is_nil() + || target.host_id.as_uuid().is_nil() + || target.binding_epoch == 0 + || target.binding_epoch > 9_007_199_254_740_991 + { + return Err("invalid physical placement".into()); + } + Ok(target) + } +} +impl From for wire::ManagedPolicyTarget { + fn from(v: core::ManagedPolicyTarget) -> Self { + Self { + session_id: v.session_id.to_string(), + sandbox_id: v.sandbox_id.to_string(), + host_id: v.host_id.to_string(), + binding_epoch: v.binding_epoch, + } + } +} +impl TryFrom for core::ManagedPolicyContext { + type Error = String; + fn try_from(v: wire::ManagedPolicyContext) -> Result { + match (wire::ManagedPolicyPhase::try_from(v.phase), &v.binding_hash) { + (Ok(wire::ManagedPolicyPhase::Bootstrap), None) + | (Ok(wire::ManagedPolicyPhase::Final), Some(_)) => {} + _ => return Err("managed policy phase and binding hash disagree".into()), + } + let value = Self { + target: v.target.ok_or_else(missing)?.try_into()?, + operator_generation: v.operator_generation, + operator_identity: v.operator_identity, + binding_hash: v.binding_hash, + }; + value.validate()?; + Ok(value) + } +} +impl From for wire::ManagedPolicyContext { + fn from(v: core::ManagedPolicyContext) -> Self { + Self { + phase: if v.binding_hash.is_some() { + wire::ManagedPolicyPhase::Final as i32 + } else { + wire::ManagedPolicyPhase::Bootstrap as i32 + }, + target: Some(v.target.into()), + operator_generation: v.operator_generation, + operator_identity: v.operator_identity, + binding_hash: v.binding_hash, + } + } +} +impl TryFrom for core::ManagedDenyPolicy { + type Error = String; + fn try_from(v: wire::ManagedDenyPolicy) -> Result { + let value = Self { + version: v.version, + revision: v.revision, + hosts: v.hosts, + digest: v.digest, + }; + value.validate()?; + Ok(value) + } +} +impl From for wire::ManagedDenyPolicy { + fn from(v: core::ManagedDenyPolicy) -> Self { + Self { + version: v.version, + revision: v.revision, + hosts: v.hosts, + digest: v.digest, + } + } +} +impl TryFrom for core::ManagedAppliedState { + type Error = String; + fn try_from(v: wire::ManagedAppliedState) -> Result { + let value = Self { + context: v.context.ok_or_else(missing)?.try_into()?, + policy: v.policy.ok_or_else(missing)?.try_into()?, + host_incarnation: v.host_incarnation, + proxy_incarnation: v.proxy_incarnation, + state_id: v.state_id, + network_proof: v.network_proof.map(TryInto::try_into).transpose()?, + }; + value.validate()?; + Ok(value) + } +} +impl From for wire::ManagedAppliedState { + fn from(v: core::ManagedAppliedState) -> Self { + Self { + context: Some(v.context.into()), + policy: Some(v.policy.into()), + host_incarnation: v.host_incarnation, + proxy_incarnation: v.proxy_incarnation, + state_id: v.state_id, + network_proof: v.network_proof.map(Into::into), + } + } +} +impl TryFrom for core::ManagedNetworkProof { + type Error = String; + fn try_from(v: wire::ManagedNetworkProof) -> Result { + Ok(Self { + target: v.target.ok_or_else(missing)?.try_into()?, + guest_ip: v.guest_ip.parse().map_err(|_| "invalid guest identity")?, + network_identity: v.network_identity, + }) + } +} +impl From for wire::ManagedNetworkProof { + fn from(v: core::ManagedNetworkProof) -> Self { + Self { + target: Some(v.target.into()), + guest_ip: v.guest_ip.to_string(), + network_identity: v.network_identity, + } + } +} + +impl wire::ReadManagedPolicyResponse { + /// Validate the complete trusted read, including explicit authoritative absence. + /// Retained desired state is a revision/union floor, never proof of application. + pub fn validate_for(&self, target: &core::ManagedPolicyTarget) -> Result<(), String> { + if self.protocol != wire::PROTOCOL { + return Err("unsupported managed read protocol".into()); + } + let retained = self.retained_desired.as_ref().ok_or_else(missing)?; + let actual: core::ManagedPolicyTarget = + retained.target.clone().ok_or_else(missing)?.try_into()?; + if actual != *target { + return Err("retained policy identity changed".into()); + } + if let Some(policy) = &retained.policy { + core::ManagedDenyPolicy::try_from(policy.clone())?; + } + if let Some(applied) = &self.applied { + let applied: core::ManagedAppliedState = applied.clone().try_into()?; + if applied.context.target != *target { + return Err("host policy identity changed".into()); + } + } + Ok(()) + } +} + +/// The managed bincode surface is bounded and rejects unknown trailing fields. +pub fn decode(bytes: &[u8]) -> Result { + use bincode::Options; + bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(4 * 1024 * 1024) + .reject_trailing_bytes() + .deserialize(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn unknown_phase_and_false_final_ack_are_closed() { + for (phase, binding_hash) in [ + (0, None), + (999, None), + (2, None), + (1, Some("sha256:invalid".into())), + (2, Some(String::new())), + ] { + assert!( + core::ManagedPolicyContext::try_from(wire::ManagedPolicyContext { + phase, + binding_hash, + ..Default::default() + }) + .is_err() + ); + } + } +} diff --git a/crates/engram-protocol/src/mcp.rs b/crates/engram-protocol/src/mcp.rs new file mode 100644 index 000000000..10cf31a5d --- /dev/null +++ b/crates/engram-protocol/src/mcp.rs @@ -0,0 +1,92 @@ +//! Internal MCP credential control protocol 1; no cleartext app registration. +include!(concat!(env!("OUT_DIR"), "/mcp/engram.mcp.v1.rs")); +pub const PROTOCOL: u32 = 1; +macro_rules! redacted_debug { + ($($ty:ty),*) => {$ (impl std::fmt::Debug for $ty { + fn fmt(&self,f:&mut std::fmt::Formatter<'_>)->std::fmt::Result {f.write_str("McpControlMessage([redacted])")} + })*}; +} +redacted_debug!( + PutSecretRequest, + ReserveManagedCreateRequest, + CreatePolicySessionRequest, + CompleteAuthorizationRequest, + BeginAuthorizationResponse, + LeaseResponse +); + +/// SHA-256 of exact creation wire bytes, task context, and native descriptor +/// version/digest. Length prefixes prevent ambiguous concatenation. Callers +/// retain the original wire bytes; they must not re-encode maps for a retry. +pub fn managed_create_input_digest(request: &ReserveManagedCreateRequest) -> String { + use sha2::{Digest, Sha256}; + let mut hash = Sha256::new(); + hash.update(b"engram-managed-create-v1\0"); + hash.update(request.protocol.to_be_bytes()); + hash.update(request.native_version.to_be_bytes()); + for bytes in [ + request.task_id.as_bytes(), + request.native_digest.as_bytes(), + request.creation_request.as_slice(), + ] { + hash.update((bytes.len() as u64).to_be_bytes()); + hash.update(bytes); + } + format!("sha256:{}", hex::encode(hash.finalize())) +} + +impl ReserveManagedCreateRequest { + pub fn decode_creation(&self) -> Result { + use prost::Message; + if self.creation_request.len() > 3 * 1024 * 1024 { + return Err("creation request exceeds limit"); + } + CreatePolicySessionRequest::decode(self.creation_request.as_slice()) + .map_err(|_| "invalid creation request") + } +} + +#[cfg(test)] +mod managed_creation_tests { + use super::*; + use prost::Message; + #[test] + fn exact_input_digest_and_wire_contract() { + let request = ReserveManagedCreateRequest { + protocol: 1, + creation_request: vec![1, 2, 3], + task_id: "task-a".into(), + native_version: 1, + native_digest: format!("sha256:{}", "a".repeat(64)), + input_digest: String::new(), + }; + assert_eq!( + managed_create_input_digest(&request), + "sha256:fa03d5903a5d81b1127d47cc331d0d833eb68f5fe42012397d86958bf5b995b5" + ); + assert_eq!( + ReserveManagedCreateRequest::decode(request.encode_to_vec().as_slice()).unwrap(), + request + ); + assert!(request.decode_creation().is_err()); + let response = ReadNativeManifestResponse { + protocol: 1, + identity: Some(ManagedCreateIdentity { + session_id: "s-a".into(), + task_id: "task-a".into(), + input_id: "create-a".into(), + actor_id: "owner".into(), + parent_session_id: Some("parent".into()), + write_eligibility: "read_only".into(), + input_digest: "digest".into(), + }), + native_version: 1, + native_bytes: b" [ ] ".to_vec(), + native_digest: "digest".into(), + }; + assert_eq!( + ReadNativeManifestResponse::decode(response.encode_to_vec().as_slice()).unwrap(), + response + ); + } +} diff --git a/crates/engram-protocol/src/ordinary_policy.rs b/crates/engram-protocol/src/ordinary_policy.rs new file mode 100644 index 000000000..64b960b79 --- /dev/null +++ b/crates/engram-protocol/src/ordinary_policy.rs @@ -0,0 +1,82 @@ +//! Ordinary wire 38 policy boundary. Managed authority has a separate RPC contract. +use bincode::Options; +use engram_core::types::{ + egress::{ + AppEndpoint, EgressInjectEntry, EgressObserveEntry, EgressSecretEntry, SessionEgressPolicy, + }, + image::SecretMode, + integration::{GuestService, SessionTunnel}, +}; +use engram_core::{SandboxError, SandboxId, SessionId}; +use serde::{Deserialize, Serialize}; +use std::net::Ipv4Addr; + +// Field order is pinned to the ordinary wire 38 contract. +#[derive(Serialize, Deserialize)] +struct OrdinaryPolicy { + session_id: SessionId, + sandbox_id: SandboxId, + guest_ip: Ipv4Addr, + network_allow_hosts: Vec, + network_allow_host_patterns: Vec, + allow_all: bool, + secrets: Vec, + injects: Vec, + observes: Vec, + guest_services: Vec, + tunnels: Vec, + secret_mode: SecretMode, + apps: Vec, +} + +/// Encode ordinary policy without discarding managed authority. +pub fn encode(policy: &SessionEgressPolicy) -> Result, SandboxError> { + if policy.managed_deny.is_some() { + return Err(SandboxError::InvalidSpec( + "managed policy requires the managed protocol".into(), + )); + } + let ordinary = OrdinaryPolicy { + session_id: policy.session_id, + sandbox_id: policy.sandbox_id, + guest_ip: policy.guest_ip, + network_allow_hosts: policy.network_allow_hosts.clone(), + network_allow_host_patterns: policy.network_allow_host_patterns.clone(), + allow_all: policy.allow_all, + secrets: policy.secrets.clone(), + injects: policy.injects.clone(), + observes: policy.observes.clone(), + guest_services: policy.guest_services.clone(), + tunnels: policy.tunnels.clone(), + secret_mode: policy.secret_mode, + apps: policy.apps.clone(), + }; + bincode::serialize(&ordinary) + .map_err(|e| SandboxError::InvalidSpec(format!("encode ordinary policy: {e}"))) +} + +/// Decode exactly one complete ordinary policy, without managed authority. +pub fn decode(bytes: &[u8]) -> Result { + let policy: OrdinaryPolicy = bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(bytes.len() as u64) + .reject_trailing_bytes() + .deserialize(bytes) + .map_err(|e| SandboxError::InvalidSpec(format!("decode ordinary policy: {e}")))?; + Ok(SessionEgressPolicy { + session_id: policy.session_id, + sandbox_id: policy.sandbox_id, + guest_ip: policy.guest_ip, + network_allow_hosts: policy.network_allow_hosts, + network_allow_host_patterns: policy.network_allow_host_patterns, + allow_all: policy.allow_all, + secrets: policy.secrets, + injects: policy.injects, + observes: policy.observes, + guest_services: policy.guest_services, + tunnels: policy.tunnels, + secret_mode: policy.secret_mode, + apps: policy.apps, + managed_deny: None, + }) +} diff --git a/crates/engram-protocol/src/wire.rs b/crates/engram-protocol/src/wire.rs index e13e5a11b..146a9e215 100644 --- a/crates/engram-protocol/src/wire.rs +++ b/crates/engram-protocol/src/wire.rs @@ -1,182 +1,21 @@ -//! Shared bincode-encoded payload types crossing the coord ↔ host -//! gRPC boundary (ADR 0013). -//! -//! The gRPC service in `host_service.proto` carries these inside -//! `bytes` fields — bincode round-trips Rust types without needing -//! a proto mirror for every nested struct (SandboxSpec, -//! SnapshotMetadata, SessionEgressPolicy, AgentSpec, etc. stay -//! authoritative). Coord + host deploy together (OSS image tag → -//! engrams-internal Helm pin) so bincode skew can't happen across -//! versions. -//! -//! Pre-0013 this file held the full `Frame`/`RequestKind`/`ResponseKind`/ -//! `NotifyKind` enum surface for the bincode-over-WS protocol. ADR 0013 -//! retired that protocol; what's left here is just the shared shapes. +//! Payloads and protocol markers for the coordinator-to-host gRPC boundary. +//! Ordinary policy has an explicit wire type in [`crate::ordinary_policy`]. +//! Managed RPCs use their own marker and authenticated transport. use std::collections::HashMap; use std::time::Duration; use serde::{Deserialize, Serialize}; -/// Version stamp for the bincode payloads crossing the coord ↔ host -/// gRPC boundary. Bumped on every bincode-payload-incompatible change -/// (a new field on `SandboxSpec`, etc.). -/// -/// Issue #229: this is NO LONGER advisory. A Helm rollout is not -/// atomic — coord pods finish in ~1 min while a 40-node host DaemonSet -/// rolls over ~20 min — so a mixed-version fleet is a *normal*, -/// transient state, not a misconfiguration. During that window the -/// version is enforced on both ends: -/// - The coordinator stamps every coord→host gRPC request with this -/// value in the [`WIRE_VERSION_METADATA_KEY`] metadata header -/// (via `TraceparentInjector`). -/// - The host-agent's gRPC server rejects a request whose stamped -/// version differs from its own with `failed_precondition` BEFORE -/// any bincode decode, so the caller sees an explicit, retryable -/// `WireSkew` (→ HTTP 503) instead of a misleading 400 "invalid -/// sandbox spec" decode error. -/// - The host reports its version on every heartbeat; the scheduler -/// excludes version-mismatched hosts so a rolling deploy becomes a -/// graceful drain rather than a stream of hard failures. -// v3 (ADR 0056 Phase 3b): added `injects` to `SessionEgressPolicy` (coord→host). -// v4 (ADR 0056 Phase 4b): added `observes`. A mixed-version fleet fails fast on -// the version gate above rather than misparsing the positional bincode payload. -// v5 (ADR 0059): added `graphql_operation`/`graphql_field` to the inject + observe -// entries (+ `success_no_graphql_errors` on observe) for GraphQL operation gating. -// v6: added `capture_env_bincode` to `BuildBaseSnapshotRequest` — the resolved -// capture-time env injected into the `[warm]` hook at base-snapshot capture. -// v7: capture-VM egress for the `[warm]` hook — `WarmConfig` gains a -// `network: Option` (the `warm_bincode` field of -// `BuildBaseSnapshotRequest`) and `SessionEgressPolicy` gains `allow_all`. The -// host registers a matching egress policy (allow-all or allowlist) for the -// capture VM's guest IP so the warm boot can reach the network. -// v8 (issue #539): `BuildBaseSnapshot` becomes server-streaming -// (`BuildBaseSnapshotEvent` — `progress`/`done`/`failed`) instead of unary, -// carrying the `[warm]`-hook progress protocol's `CaptureProgress` events -// and a structured `CaptureFailed` terminal frame. Clean break — coord+host -// roll together, no dual-decode ladder. -// v9 (issue #529): `SnapshotMetadata` gains `paused_at: Option>` — -// the host's exact pause instant, carried over the coord↔host RPC boundary -// (the eviction/snapshot response) so the composed eviction path can resolve -// the `session_events` coherence cursor from it instead of coord wall-clock -// `now` sampled after the capture returns. -// v10 (ADR 0073 / epic #542): bind_session carries binding_epoch; AgentSpec -// carries binding_epoch; the shell-pin RPCs (AcquireShell/ReleaseShell/ -// RenewShell) and RehandshakeHarness are deleted; heartbeat gains -// harness_attached. Lockstep coord+host roll, no fallback ladder. -// v11 (issue #548 / ADR 0078): the never-populated `local_snapshots` -// heartbeat mirror is retired end-to-end — the `Heartbeat.local_snapshots` -// wire field, `HostRecord`/`HostHeartbeat` fields, the `hosts.local_snapshots` -// PG column (migration 0091), and the fleet-view proto count (reserved 7). -// Clean break — coord+host roll together; skewed hosts drain off scheduling -// via `host_wire_version_ok` until the host MIG rolls. -// v12 (ADR 0079 / #543): fencing_epoch on session-scoped host RPCs — the -// SandboxIdMessage-shaped lifecycle RPCs (destroy / snapshot family / -// pause / resume) migrate to `FencedSandboxRequest{uuid, fencing_epoch, -// session_id}`; StartAgent / Restore / RestoreBaseForSession gain the -// same (session_id, fencing_epoch) pair. Lockstep coord+host roll. -// v13 (ADR 0080 Phase 2a): `BuildBaseSnapshotRequest` gains -// `capture_egress_bincode` — the capture VM's `[warm]`-hook egress policy -// is assembled COORDINATOR-side (one `SessionEgressPolicy` builder for -// sessions and captures alike) and shipped ready-to-register; the host's -// own `capture_egress_policy` builder (from `WarmConfig.network`) is -// retired, and the host no longer interprets `warm.env`/`warm.network` -// out of `warm_bincode`. Lockstep coord+host roll. -// v14 (ADR 0080 Phase 3b): new server-streaming `MaterializeImage` RPC — -// enable-time host-side materialization of a STANDARD docker/OCI image -// (pull → flatten → pack → chunk) replacing the coordinator's -// engram-artifact pull (`fetch_and_seal_artifact`/`materialize_disk_chunks` -// retired). Carries bincode `Option` coord→host and -// bincode `ManifestRef`/`OciRuntimeDefaults` host→coord. A proto RPC -// ADDITION is protobuf-compatible, but the bump makes the deploy posture -// explicit: an enable driven by a v14 coord must never land on a v13 host -// (which would answer `Unimplemented`), so skewed hosts drain off -// scheduling until the MIG rolls. Lockstep coord+host roll. -// v15 (ADR 0084): capture_jobs heartbeat dispatch/reporting (#546) — -// `Heartbeat.capture_job_reports`, `HeartbeatAck.capture_assignments`/ -// `acked_capture_jobs`. BuildBaseSnapshot RPC deletion rides this bump -// (removed in the cutover commit). -// v16 (ADR 0095): `SnapshotMetadata.peer_hints` — peer-fill seed addrs -// for the restore destination (bincode field addition). The standing -// `PeerChunkGet` RPC + the heartbeat-ack `warm_peers` field ride this -// bump too (both are independently mixed-roll-safe — proto addition / -// serde-default JSON — but the bump makes the deploy posture explicit: -// a v16 coord never dispatches a peer-hinted restore to a v15 host, -// whose bincode decode would fail loudly). Lockstep coord+host roll. -// v17 (ADR 0100): `WriteFiles` coord↔host RPC and its bincode request / -// response mirrors. Lockstep coord+host roll. -// v18 (ADR 0093 addendum): `MaterializeImageRequest.min_disk_gib` — the -// image's `suggested_disk_gib` now floors the packed ext4 size. Proto -// field addition (mixed-roll-safe: an old host ignores it and packs -// content-sized), bumped so the deploy posture is explicit — an enable -// on a v17 host silently loses the floor. -// v19 (ADR 0103): durable ExecRequest ticket/resume fields and the -// CancelExec coord↔host RPC. Lockstep coord+host roll. -// v20 (ADR 0103 review hardening): ExecFrame carries terminal refusals as -// their own oneof variant instead of collapsing them into Exit(None). -// v21 (ADR 0109): `SessionEgressPolicy.google_adc` tells the host whether -// to expose the session-local Google metadata endpoint. Trailing bincode -// field addition. Lockstep coordinator and host roll. -// v22 (issue: engrams-review outage 2026-08-01): `CredentialMintSource` -// becomes externally tagged. The internally-tagged form (#931) bincode- -// ENCODED as a map but could never DECODE (`deserialize_any`), so every -// `SessionEgressPolicy` carrying a minted inject failed host-side at -// boot — no peer ever decoded the old `Some(mint_source)` bytes. The -// bump makes the mixed-fleet posture explicit. Lockstep coord+host roll. -// v23 (ADR 0109 seam): `SessionEgressPolicy.google_adc` became -// `metadata_flavor: Option`. The boolean conflated "does -// this session need a metadata endpoint?" with "is it Google's?", so a -// second cloud would have needed a second boolean and the proxy would -// have had to decide which one wins. The enum makes it one value, and a -// wildcard-free match makes a new variant a compile error. Bincode field -// REPLACEMENT (not an addition), so the roll is lockstep: a v22 host -// cannot decode a v23 policy. -// v24 (ADR 0106 addendum): `CredentialMintSource` gains the trailing -// `OauthConnector` variant (connector OAuth tokens resolved from the sealed -// credential store) on `SessionEgressPolicy` inject entries and the -// host→coord inject/refresh route. Trailing-variant addition: every existing -// encoding is unchanged, but a v23 host cannot decode a policy carrying the -// new variant, so the roll is lockstep. -// v25 (ADR 0112): `SandboxSpec.swap_mib` — the guest's ephemeral swap -// device size, opt-in per image. Trailing bincode field addition on the -// create/spec wire (and the JSON sidecar, which is serde-defaulted), so -// the roll is lockstep: a v24 host cannot decode a v25 create spec. -// v26 (ADR 0109 Cloud SQL addendum): the metadata listener becomes a -// session-scoped guest gateway. `metadata_flavor` becomes the extensible -// `guest_services` set, and `SessionEgressPolicy` gains provider-neutral -// tunnels selected by an opaque id and a registered host connector. A -// credential purpose becomes a provider-owned string, and tunnel mint -// authority is optional. Bincode field replacement and addition; coordinator -// and host roll in lockstep. -// v27 (ADR 0113): HostService gains streamed session-file upload and read -// operations. The payload itself is protobuf, but an older host cannot serve -// the new methods. Fence mixed fleets so the coordinator retries after the -// host roll instead of accepting a prompt whose required file was not copied. -// v28 (ADR 0115): `CredentialMintSource` gains the trailing `OauthUser` -// variant (a user's personal connector credential resolved from the sealed -// store) on `SessionEgressPolicy` inject entries and the host→coord -// inject/refresh route. Trailing-variant addition: every existing encoding -// is unchanged, but a v27 host cannot decode a policy carrying the new -// variant, so the roll is lockstep. -// v29 (ADR 0123): SnapshotHold keeps a source paused through teleport commit. -// Older hosts cannot provide this capture contract. Roll coordinator and hosts together. -// v30 removes stop-and-copy capture and its artifact kinds. -// v31 tags migration disk items with their device role. -// v32 appends SnapshotMetadata.swap_manifest; coordinator and hosts roll together. -// v33 appends the chunked swap source and manifest to SandboxSpec. -// v34 carries the swap base lineage in live migration presetup. -// v35 types the memory image refusal (`memory_image_unusable:` marker). The -// payloads do not change. The bump fences mixed fleets: an older host refuses -// the same snapshot with an untyped error, which the resume verb counts toward -// the Dead budget instead of recovering from disk. -// v36 gates page-channel v6 fault windows for a lockstep host roll. -// v37 appends HostCapabilities.page_idle and the optional MigrationItem::HotHint. -// v38 moves disk changes in 64 KiB blocks and carries the presetup's -// snapshot before the pause: MigrationItem::DiskBlockAt replaces -// DiskChunkAt at the same index, DiskPrecopyInfo is appended, and the -// capture reports sealed and precopy block counts. +/// Ordinary coordinator-to-host payload contract. Hosts advertise this version +/// for registration and placement. Managed RPCs also require their own marker. pub const WIRE_VERSION: u32 = 38; +/// Exact contract for the additive managed host RPCs. +pub const MANAGED_PROTOCOL_VERSION: u32 = 43; +/// Managed contract marker, independent of the ordinary wire version. +pub const MANAGED_PROTOCOL_METADATA_KEY: &str = "x-engram-managed-protocol"; + /// Whether a host-agent build can adopt the microVMs the previous build left /// running (ADR 0126). Two builds with the same number can reattach each /// other's VMs; raise it in any change that stops a new build from adopting @@ -190,11 +29,8 @@ pub const WIRE_VERSION: u32 = 38; /// which an earlier build cannot serve again after a restart. pub const REATTACH_GENERATION: u32 = 3; -/// gRPC metadata (header) key carrying the caller's [`WIRE_VERSION`] on -/// every coord→host request (issue #229). ASCII, lowercase — tonic -/// rejects non-lowercase ASCII metadata keys. Absent on requests from a -/// coordinator that predates this field, which the host-agent tolerates -/// (it can't compare what it didn't receive). +/// Ordinary wire version on coordinator-to-host requests. Ordinary HTTP/2 +/// tolerates an absent header; authenticated managed requests require it. pub const WIRE_VERSION_METADATA_KEY: &str = "x-engram-wire-version"; /// Marker prefix for the host-agent's wire-version-skew rejection @@ -255,6 +91,9 @@ pub fn parse_harness_spawn_message(msg: &str) -> Option<(String, String)> { Some((kind.to_string(), message.trim().to_string())) } +/// Exact failed-precondition marker. It is an outcome, not retirement proof. +pub const MANAGED_RECOVERY_REQUIRED_STATUS: &str = "managed_recovery_required"; + /// Marker prefix for the host-agent's "memory image unusable" restore /// refusal ([`engram_core::SandboxError::MemoryImageUnusable`], the /// [`HARNESS_SPAWN_STATUS_PREFIX`] precedent). The host returns a diff --git a/crates/engram-protocol/tests/decode_never_panics.rs b/crates/engram-protocol/tests/decode_never_panics.rs index 17a5ad8ec..58b566fa8 100644 --- a/crates/engram-protocol/tests/decode_never_panics.rs +++ b/crates/engram-protocol/tests/decode_never_panics.rs @@ -1,20 +1,5 @@ -//! ADR 0099 H4: decode-never-panics property suite for the coord ↔ host -//! bincode payloads (`engram_protocol::wire`). -//! -//! **Lowest priority** of the H4 targets by threat model: this wire is -//! version-fenced and trusted — coord and host deploy together and a -//! `WIRE_VERSION` skew is rejected with `failed_precondition` BEFORE any -//! bincode decode (see `wire.rs`), so an attacker never places bytes here. -//! Included because it is cheap: the crate's own payload types decode -//! through the same positional bincode, and a never-panic guard costs -//! almost nothing. -//! -//! These payloads ride inside gRPC `bytes` fields — there is no -//! length-prefix framing layer in this crate, so the decode entry point is -//! `bincode::deserialize` directly. Shapes (a) arbitrary bytes and (b) -//! mutations-of-valid-frames apply; the frame-cap half of (c) lives in the -//! framed crates (harness/agentd/substrate/migrate), so here (c) is just the -//! bincode-slice allocation bound. +//! Arbitrary and mutated bytes must not panic in host payload decoders. +//! Ordinary policy decoding also rejects incomplete and trailing bytes. use engram_core::types::egress::SessionEgressPolicy; use engram_core::types::manifest::ManifestRef; @@ -31,6 +16,18 @@ use support::wire_exec_request; // ---- helpers ----------------------------------------------------------- fn decode_every_type(bytes: &[u8]) { + use prost::Message; + let _ = engram_protocol::mcp::ReserveManagedCreateRequest::decode(bytes); + let _ = engram_protocol::mcp::ReadNativeManifestRequest::decode(bytes); + let _ = engram_protocol::mcp::ReadNativeManifestResponse::decode(bytes); + + let _ = engram_protocol::managed_policy::decode::< + engram_core::types::managed_deny::ManagedPolicyApply, + >(bytes); + let _ = engram_protocol::managed_policy::decode::< + Option, + >(bytes); + let _ = engram_protocol::ordinary_policy::decode(bytes); // Crate-local payloads. let _ = bincode::deserialize::(bytes); // A representative slice of the shared engram-core payloads that cross @@ -72,7 +69,13 @@ fn weighted_blob() -> impl Strategy> { // ---- shape (a): arbitrary bytes --------------------------------------- proptest! { - #![proptest_config(ProptestConfig::with_cases(128))] + #![proptest_config(ProptestConfig { + cases: 128, + failure_persistence: Some(Box::new(proptest::test_runner::FileFailurePersistence::Direct( + concat!(env!("CARGO_MANIFEST_DIR"), "/proptest-regressions/decode_never_panics.txt") + ))), + ..ProptestConfig::default() + })] #[test] fn arbitrary_bytes_never_panic(bytes in weighted_blob()) { @@ -83,7 +86,13 @@ proptest! { // ---- shape (b): mutations of valid frames ------------------------------ proptest! { - #![proptest_config(ProptestConfig::with_cases(128))] + #![proptest_config(ProptestConfig { + cases: 128, + failure_persistence: Some(Box::new(proptest::test_runner::FileFailurePersistence::Direct( + concat!(env!("CARGO_MANIFEST_DIR"), "/proptest-regressions/decode_never_panics.txt") + ))), + ..ProptestConfig::default() + })] #[test] fn mutated_exec_request_never_panic( diff --git a/crates/engram-protocol/tests/golden/managed_capture.json b/crates/engram-protocol/tests/golden/managed_capture.json new file mode 100644 index 000000000..7bf7d2ca9 --- /dev/null +++ b/crates/engram-protocol/tests/golden/managed_capture.json @@ -0,0 +1 @@ +{"key":{"creation":{"session_id":"00000000-0000-0000-0000-000000000001","task_id":"task","policy":{"actorId":"owner","inputId":"create","parentSessionId":null,"eligibility":"read_only"},"input_digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"context":{"target":{"session_id":"00000000-0000-0000-0000-000000000001","sandbox_id":"00000000-0000-0000-0000-000000000002","host_id":"00000000-0000-0000-0000-000000000003","binding_epoch":1},"operator_generation":1,"operator_identity":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","binding_hash":null},"boot_attempt":"boot-one","native_digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"capture_id":"00000000-0000-0000-0000-000000000004"} \ No newline at end of file diff --git a/crates/engram-protocol/tests/golden/ordinary_v38/README.md b/crates/engram-protocol/tests/golden/ordinary_v38/README.md new file mode 100644 index 000000000..cc8042f78 --- /dev/null +++ b/crates/engram-protocol/tests/golden/ordinary_v38/README.md @@ -0,0 +1,6 @@ +# Ordinary wire 38 fixtures + +Each binary is copied without modification from `ec526661` at +`crates/engram-protocol/tests/golden/.bin` using `git show`. +Do not regenerate these source fixtures with the current codec. +The predecessor policy ends with `apps`; it has no managed deny field. diff --git a/crates/engram-protocol/tests/golden/ordinary_v38/agent_spec.bin b/crates/engram-protocol/tests/golden/ordinary_v38/agent_spec.bin new file mode 100644 index 000000000..91c1ed7fc Binary files /dev/null and b/crates/engram-protocol/tests/golden/ordinary_v38/agent_spec.bin differ diff --git a/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy.bin b/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy.bin new file mode 100644 index 000000000..5509a6b83 Binary files /dev/null and b/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy.bin differ diff --git a/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_google.bin b/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_google.bin new file mode 100644 index 000000000..f82cf9282 Binary files /dev/null and b/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_google.bin differ diff --git a/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_minted.bin b/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_minted.bin new file mode 100644 index 000000000..ade0428eb Binary files /dev/null and b/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_minted.bin differ diff --git a/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_oauth.bin b/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_oauth.bin new file mode 100644 index 000000000..69dcfd2cc Binary files /dev/null and b/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_oauth.bin differ diff --git a/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_oauth_user.bin b/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_oauth_user.bin new file mode 100644 index 000000000..236d58114 Binary files /dev/null and b/crates/engram-protocol/tests/golden/ordinary_v38/session_egress_policy_oauth_user.bin differ diff --git a/crates/engram-protocol/tests/golden/ordinary_v38/snapshot_metadata.bin b/crates/engram-protocol/tests/golden/ordinary_v38/snapshot_metadata.bin new file mode 100644 index 000000000..4290c09d9 Binary files /dev/null and b/crates/engram-protocol/tests/golden/ordinary_v38/snapshot_metadata.bin differ diff --git a/crates/engram-protocol/tests/golden/session_egress_policy.bin b/crates/engram-protocol/tests/golden/session_egress_policy.bin index 5509a6b83..15156f081 100644 Binary files a/crates/engram-protocol/tests/golden/session_egress_policy.bin and b/crates/engram-protocol/tests/golden/session_egress_policy.bin differ diff --git a/crates/engram-protocol/tests/golden/session_egress_policy_google.bin b/crates/engram-protocol/tests/golden/session_egress_policy_google.bin index f82cf9282..b52799ccb 100644 Binary files a/crates/engram-protocol/tests/golden/session_egress_policy_google.bin and b/crates/engram-protocol/tests/golden/session_egress_policy_google.bin differ diff --git a/crates/engram-protocol/tests/golden/session_egress_policy_managed.bin b/crates/engram-protocol/tests/golden/session_egress_policy_managed.bin new file mode 100644 index 000000000..348e9cadd Binary files /dev/null and b/crates/engram-protocol/tests/golden/session_egress_policy_managed.bin differ diff --git a/crates/engram-protocol/tests/golden/session_egress_policy_minted.bin b/crates/engram-protocol/tests/golden/session_egress_policy_minted.bin index ade0428eb..e536215c4 100644 Binary files a/crates/engram-protocol/tests/golden/session_egress_policy_minted.bin and b/crates/engram-protocol/tests/golden/session_egress_policy_minted.bin differ diff --git a/crates/engram-protocol/tests/golden/session_egress_policy_oauth.bin b/crates/engram-protocol/tests/golden/session_egress_policy_oauth.bin index 69dcfd2cc..5b1025382 100644 Binary files a/crates/engram-protocol/tests/golden/session_egress_policy_oauth.bin and b/crates/engram-protocol/tests/golden/session_egress_policy_oauth.bin differ diff --git a/crates/engram-protocol/tests/golden/session_egress_policy_oauth_user.bin b/crates/engram-protocol/tests/golden/session_egress_policy_oauth_user.bin index 236d58114..38088e451 100644 Binary files a/crates/engram-protocol/tests/golden/session_egress_policy_oauth_user.bin and b/crates/engram-protocol/tests/golden/session_egress_policy_oauth_user.bin differ diff --git a/crates/engram-protocol/tests/mcp_policy_wire.rs b/crates/engram-protocol/tests/mcp_policy_wire.rs new file mode 100644 index 000000000..7a4a9ffdd --- /dev/null +++ b/crates/engram-protocol/tests/mcp_policy_wire.rs @@ -0,0 +1,236 @@ +use engram_protocol::{app, mcp}; +use proptest::{prelude::*, test_runner::FileFailurePersistence}; +use prost::Message; + +#[test] +fn app_reflection_excludes_internal_policy_and_credential_services() { + let bytes = app::FILE_DESCRIPTOR_SET; + for private in [ + "engram/mcp/v1/session_policy.proto", + "SessionPolicyService", + "CredentialControlService", + "OperatorPublicationService", + "engram/mcp/v1/operator.proto", + ] { + assert!(!bytes + .windows(private.len()) + .any(|w| w == private.as_bytes())); + } +} +proptest! { + #![proptest_config(ProptestConfig { cases: 64, failure_persistence: Some(Box::new(FileFailurePersistence::Direct("proptest-regressions/mcp_policy_wire.txt"))), ..ProptestConfig::default() })] + #[test] + fn managed_boot_response_envelopes_are_wire_compatible( + protocol in any::(), + boot_json in proptest::option::of(proptest::collection::vec(any::(),0..=4096)), + ) { + let read = mcp::ReadManagedBootResponse { protocol, boot_json: boot_json.clone() }; + let finalized = mcp::FinalizeManagedBootResponse { protocol, boot_json }; + prop_assert_eq!(read.encode_to_vec(), finalized.encode_to_vec()); + prop_assert_eq!(mcp::ReadManagedBootResponse::decode(finalized.encode_to_vec().as_slice()).unwrap(), read.clone()); + prop_assert_eq!(mcp::FinalizeManagedBootResponse::decode(read.encode_to_vec().as_slice()).unwrap(), finalized); + } + #[test] + fn policy_decode_never_panics(bytes in proptest::collection::vec(any::(),0..=4096)) { + let _=mcp::CreatePolicySessionRequest::decode(bytes.as_slice()); + let _=mcp::ReadManagedPolicyResponse::decode(bytes.as_slice()); + let _=mcp::ManagedRetainedDesired::decode(bytes.as_slice()); + let _=mcp::ReadWritePolicyRequest::decode(bytes.as_slice()); + let _=mcp::AcceptPlanExitRequest::decode(bytes.as_slice()); + let _=mcp::ValidateDispatchFenceRequest::decode(bytes.as_slice()); + let _=app::CompleteToolCallRequest::decode(bytes.as_slice()); + let _=app::McpPlanReview::decode(bytes.as_slice()); + let _=mcp::PutSecretResponse::decode(bytes.as_slice()); + let _=mcp::CompleteAuthorizationResponse::decode(bytes.as_slice()); + let _=mcp::GetStatusResponse::decode(bytes.as_slice()); + let _=mcp::RevokeResponse::decode(bytes.as_slice()); + let _=mcp::ReadManagedBootRequest::decode(bytes.as_slice()); + let _=mcp::ReadManagedBootResponse::decode(bytes.as_slice()); + let _=mcp::FinalizeManagedBootRequest::decode(bytes.as_slice()); + let _=mcp::FinalizeManagedBootResponse::decode(bytes.as_slice()); + let _=mcp::OperatorPublicationServiceStageRequest::decode(bytes.as_slice()); + let _=mcp::OperatorPublicationServiceReadRequest::decode(bytes.as_slice()); + let _=mcp::OperatorPublicationServiceReadResponse::decode(bytes.as_slice()); + let _=mcp::OperatorPublicationServiceStageResponse::decode(bytes.as_slice()); + let _=mcp::ReadPublishedOperatorRequest::decode(bytes.as_slice()); + let _=mcp::ReadPublishedOperatorResponse::decode(bytes.as_slice()); + let _=mcp::ReadManagedCreationStateRequest::decode(bytes.as_slice()); + let _=mcp::ReadManagedCreationStateResponse::decode(bytes.as_slice()); + let _=mcp::PrepareManagedBootRequest::decode(bytes.as_slice()); + let _=mcp::PrepareManagedBootResponse::decode(bytes.as_slice()); + let _=mcp::RecoverManagedBootRequest::decode(bytes.as_slice()); + let _=mcp::RecoverManagedBootResponse::decode(bytes.as_slice()); + let _=serde_json::from_slice::(&bytes); + let _=serde_json::from_slice::(&bytes); + let _=serde_json::from_slice::(&bytes); + let _=serde_json::from_slice::(&bytes); + } +} + +#[test] +fn managed_read_requires_explicit_canonical_retention_and_exact_target() { + use engram_core::{types::managed_deny::*, HostId, SandboxId, SessionId}; + let target = ManagedPolicyTarget { + session_id: SessionId::new(), + sandbox_id: SandboxId::new(), + host_id: HostId::new(), + binding_epoch: 1, + }; + let mut response = mcp::ReadManagedPolicyResponse { + protocol: mcp::PROTOCOL, + applied: None, + retained_desired: None, + }; + assert!( + response.validate_for(&target).is_err(), + "old server omission must fail closed" + ); + response.retained_desired = Some(mcp::ManagedRetainedDesired { + target: Some(target.clone().into()), + policy: None, + }); + response.validate_for(&target).unwrap(); // Explicit absence, with no fabricated ACK. + let policy = ManagedDenyPolicy::new(3, ["orphan.example".into()]).unwrap(); + response.retained_desired.as_mut().unwrap().policy = Some(policy.clone().into()); + let roundtrip = + mcp::ReadManagedPolicyResponse::decode(response.encode_to_vec().as_slice()).unwrap(); + roundtrip.validate_for(&target).unwrap(); + assert!(roundtrip.applied.is_none()); + let mut wrong = target.clone(); + wrong.binding_epoch += 1; + assert!(response.validate_for(&wrong).is_err()); + response + .retained_desired + .as_mut() + .unwrap() + .policy + .as_mut() + .unwrap() + .hosts + .push("UPPER.EXAMPLE".into()); + assert!( + response.validate_for(&target).is_err(), + "noncanonical retained bytes rejected" + ); + response.retained_desired.as_mut().unwrap().policy = Some(policy.clone().into()); + let context = ManagedPolicyContext { + target: wrong, + operator_generation: 1, + operator_identity: format!("sha256:{}", "a".repeat(64)), + binding_hash: None, + }; + response.applied = Some( + ManagedAppliedState::new(context, policy, "host".into(), "proxy".into(), None) + .unwrap() + .into(), + ); + assert!( + response.validate_for(&target).is_err(), + "applied target must agree independently" + ); +} + +#[test] +fn managed_boot_pinned_wire_tags_are_stable() { + let request = mcp::FinalizeManagedBootRequest { + protocol: 1, + input_json: b"{}".to_vec(), + }; + assert_eq!(request.encode_to_vec(), [8, 1, 18, 2, 123, 125]); + let response = mcp::ReadManagedBootResponse { + protocol: 1, + boot_json: Some(b"{}".to_vec()), + }; + assert_eq!(response.encode_to_vec(), [8, 1, 18, 2, 123, 125]); + for (boot_json, bytes) in [ + (None, vec![8, 1]), + (Some(Vec::new()), vec![8, 1, 18, 0]), + (Some(b"{}".to_vec()), vec![8, 1, 18, 2, 123, 125]), + ] { + let finalized = mcp::FinalizeManagedBootResponse { + protocol: 1, + boot_json, + }; + assert_eq!(finalized.encode_to_vec(), bytes); + assert_eq!( + mcp::FinalizeManagedBootResponse::decode(bytes.as_slice()).unwrap(), + finalized + ); + } + assert_eq!( + mcp::ReadManagedBootResponse::decode([8u8, 1].as_slice()) + .unwrap() + .boot_json, + None + ); +} +#[test] +fn managed_capture_json_golden_roundtrip() { + use engram_core::types::managed_boot::ManagedCaptureRequest; + let bytes = include_bytes!("golden/managed_capture.json"); + let request: ManagedCaptureRequest = engram_protocol::managed_boot::decode(bytes).unwrap(); + request.key.validate().unwrap(); + assert_eq!( + engram_protocol::managed_boot::encode(&request).unwrap(), + bytes + ); + assert_eq!( + request.disk_manifest().manifest_id, + request.capture_id.as_uuid() + ); + assert_eq!(request.disk_manifest().version, 1); +} + +#[test] +fn managed_creation_bridge_envelopes_preserve_identity_and_operation() { + let identity = mcp::ManagedCreateIdentity { + session_id: "11111111-1111-4111-8111-111111111111".into(), + task_id: "task-one".into(), + input_id: "input-one".into(), + actor_id: "actor-one".into(), + parent_session_id: None, + write_eligibility: "read_only".into(), + input_digest: format!("sha256:{}", "a".repeat(64)), + }; + let read = mcp::ReadManagedCreationStateRequest { + protocol: mcp::PROTOCOL, + identity: Some(identity.clone()), + native_version: 1, + native_digest: format!("sha256:{}", "b".repeat(64)), + }; + let prepare = mcp::PrepareManagedBootRequest { + protocol: read.protocol, + identity: read.identity.clone(), + native_version: read.native_version, + native_digest: read.native_digest.clone(), + }; + assert_eq!(read.encode_to_vec(), prepare.encode_to_vec()); + let recovery = mcp::RecoverManagedBootRequest { + protocol: read.protocol, + identity: Some(identity), + native_version: read.native_version, + native_digest: read.native_digest, + expected_source_key_json: b"exact-source".to_vec(), + recovery_id: "recovery-one".into(), + }; + assert_eq!( + mcp::RecoverManagedBootRequest::decode(recovery.encode_to_vec().as_slice()).unwrap(), + recovery + ); + let response = mcp::ReadManagedCreationStateResponse { + protocol: mcp::PROTOCOL, + state_json: b"explicit-state".to_vec(), + }; + assert_eq!( + mcp::PrepareManagedBootResponse::decode(response.encode_to_vec().as_slice()) + .unwrap() + .state_json, + response.state_json + ); + assert_eq!( + mcp::RecoverManagedBootResponse::decode(response.encode_to_vec().as_slice()) + .unwrap() + .state_json, + response.state_json + ); +} diff --git a/crates/engram-protocol/tests/ordinary_transport.rs b/crates/engram-protocol/tests/ordinary_transport.rs new file mode 100644 index 000000000..dbd77db71 --- /dev/null +++ b/crates/engram-protocol/tests/ordinary_transport.rs @@ -0,0 +1,153 @@ +//! Real HTTP/2 exchange with the strict ordinary wire 38 source contract. +use engram_core::{traits::SessionFence, types::egress::SessionEgressPolicy}; +use engram_protocol::{grpc, grpc_client::GrpcHostClient}; +use std::{ + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; +use tonic::{Request, Response, Status}; + +const POLICY: &[u8] = include_bytes!("golden/ordinary_v38/session_egress_policy_minted.bin"); +const AGENT: &[u8] = include_bytes!("golden/ordinary_v38/agent_spec.bin"); +const SNAPSHOT: &[u8] = include_bytes!("golden/ordinary_v38/snapshot_metadata.bin"); +type Reply = + std::pin::Pin, Status>> + Send>>; + +#[derive(Clone)] +struct Predecessor(Arc); +impl tonic::server::NamedService for Predecessor { + const NAME: &'static str = "engram.host.v1.HostService"; +} +impl tonic::server::UnaryService for Predecessor { + type Response = grpc::Empty; + type Future = Reply; + fn call(&mut self, request: Request) -> Self::Future { + assert!(request.metadata().contains_key("grpc-timeout")); + let body = request.into_inner(); + assert_eq!(body.policy_bincode, POLICY); + assert_eq!(body.agent_bincode, AGENT); + assert_eq!(body.session_id, [0x30; 16]); + assert_eq!(body.sandbox_id, [0x31; 16]); + assert_eq!(body.fencing_epoch, 7); + self.0.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(Response::new(grpc::Empty {})) }) + } +} +impl tonic::server::UnaryService for Predecessor { + type Response = grpc::SnapshotResponse; + type Future = Reply; + fn call(&mut self, request: Request) -> Self::Future { + let body = request.into_inner(); + assert_eq!(body.uuid, [0x31; 16]); + assert_eq!(body.session_id, [0x30; 16]); + assert_eq!(body.fencing_epoch, 7); + self.0.fetch_add(1, Ordering::SeqCst); + Box::pin(async { + Ok(Response::new(grpc::SnapshotResponse { + metadata_bincode: SNAPSHOT.to_vec(), + })) + }) + } +} +impl tonic::codegen::Service> for Predecessor { + type Response = tonic::codegen::http::Response; + type Error = std::convert::Infallible; + type Future = std::pin::Pin< + Box> + Send>, + >; + fn poll_ready( + &mut self, + _: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + fn call(&mut self, request: tonic::codegen::http::Request) -> Self::Future { + let service = self.clone(); + Box::pin(async move { + if request + .headers() + .get("x-engram-wire-version") + .and_then(|v| v.to_str().ok()) + != Some("38") + { + return Ok(Status::failed_precondition("ordinary wire 38 required").into_http()); + } + assert!(!request.headers().contains_key("x-engram-managed-protocol")); + match request.uri().path() { + "/engram.host.v1.HostService/StartAgent" => { + Ok(tonic::server::Grpc::new(tonic_prost::ProstCodec::< + grpc::Empty, + grpc::StartAgentRequest, + >::default()) + .unary(service, request) + .await) + } + "/engram.host.v1.HostService/Snapshot" => { + Ok(tonic::server::Grpc::new(tonic_prost::ProstCodec::< + grpc::SnapshotResponse, + grpc::FencedSandboxRequest, + >::default()) + .unary(service, request) + .await) + } + _ => Ok(Status::unimplemented("predecessor method").into_http()), + } + }) + } +} + +#[tokio::test] +async fn new_client_sends_predecessor_policy_and_reads_predecessor_snapshot() { + use futures::FutureExt; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let incoming = futures::stream::unfold(listener, |listener| async move { + Some((listener.accept().await.map(|(socket, _)| socket), listener)) + }); + let calls = Arc::new(AtomicUsize::new(0)); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let mut server = tokio::spawn( + tonic::transport::Server::builder() + .add_service(Predecessor(calls.clone())) + .serve_with_incoming_shutdown(incoming, async { + let _ = stopped.await; + }), + ); + let result = tokio::time::timeout( + Duration::from_secs(5), + std::panic::AssertUnwindSafe(async { + let channel = tonic::transport::Endpoint::from_shared(format!("http://{address}")) + .unwrap() + .connect() + .await + .unwrap(); + let client = GrpcHostClient::new(channel); + let policy: SessionEgressPolicy = + bincode::deserialize(include_bytes!("golden/session_egress_policy_minted.bin")) + .unwrap(); + let id = policy.sandbox_id; + let fence = SessionFence::new(policy.session_id, 7); + client + .start_agent(id, bincode::deserialize(AGENT).unwrap(), policy, fence) + .await + .unwrap(); + let snapshot = client.snapshot(id, fence).await.unwrap(); + assert_eq!(bincode::serialize(&snapshot).unwrap(), SNAPSHOT); + assert_eq!(calls.load(Ordering::SeqCst), 2); + }) + .catch_unwind(), + ) + .await; + let _ = stop.send(()); + if tokio::time::timeout(Duration::from_secs(2), &mut server) + .await + .is_err() + { + server.abort(); + let _ = server.await; + } + result.unwrap().unwrap(); +} diff --git a/crates/engram-protocol/tests/support/mod.rs b/crates/engram-protocol/tests/support/mod.rs index 18e7eae9e..6bdc3ea23 100644 --- a/crates/engram-protocol/tests/support/mod.rs +++ b/crates/engram-protocol/tests/support/mod.rs @@ -310,6 +310,13 @@ pub fn session_egress_policy() -> impl Strategy { (secrets, injects, observes), (guest_services, secret_mode, apps), )| SessionEgressPolicy { + managed_deny: allow_all.then(|| { + engram_core::types::managed_deny::ManagedDenyPolicy::new( + (session % 1000) as u64, + ["mcp.example".into()], + ) + .unwrap() + }), session_id: SessionId::from(uuid::Uuid::from_u128(session)), sandbox_id: SandboxId::from(uuid::Uuid::from_u128(sandbox)), guest_ip: std::net::Ipv4Addr::from(ip), diff --git a/crates/engram-protocol/tests/wire_golden.rs b/crates/engram-protocol/tests/wire_golden.rs index 4d509d322..fa8af5ceb 100644 --- a/crates/engram-protocol/tests/wire_golden.rs +++ b/crates/engram-protocol/tests/wire_golden.rs @@ -1,48 +1,13 @@ -//! Byte-level golden + variant-index pins for the coord ↔ host-agent -//! gRPC `bytes`-payload wire types. +//! Byte-level pins for host payloads and their nested enum indices. //! -//! ## Why this exists +//! `golden/ordinary_v38` holds unchanged predecessor bytes. Ordinary policy +//! tests use the transport codec; the other policy goldens pin the full internal +//! type, including managed authority. Never regenerate the predecessor corpus. //! -//! ADR 0013's host service carries structurally-complex payloads as -//! bincode blobs inside proto `bytes` fields -//! (`engram-protocol/src/grpc_client.rs::{encode,decode}_bincode`): -//! `SandboxSpec`, `AgentSpec`, `SnapshotMetadata`, `ManifestRef`, -//! `SessionEgressPolicy`. These are the AUTHORITATIVE in-process -//! engram-core types — they cross the wire as-is, with no proto mirror -//! per nested struct. -//! -//! `bincode` (1.x) is positional: enums encode by `u32` variant *index*, -//! structs by field *order*. Neither is self-describing. The existing -//! round-trip tests on these types cannot catch a format break — both -//! ends recompile together in CI, so a reordered field/variant or an -//! inserted (non-trailing) field round-trips green and then desyncs -//! against a peer built from an older tree. Coord and host roll on -//! independent (if close) schedules, so the skew window is real. -//! -//! This test pins the exact bytes (`golden/.bin`) plus, for every -//! enum, the `u32` variant index in `bytes[0..4]`. Reordering a variant -//! or adding a non-trailing field fails here with a message naming the -//! evolution rule, turning a silent wire desync into a CI failure. -//! -//! Also covered: the `wire.rs` mirror `WireExecRequest` — the pattern -//! ADR 0013 says SHOULD eventually wrap every cross-boundary shape. -//! -//! ## Regenerating the corpus (only when you INTENTIONALLY evolve a type) -//! -//! Adding a *trailing* enum variant or a *trailing* struct field is the -//! only wire-safe evolution. After such a change, regenerate: -//! -//! ```text -//! cargo test -p engram-protocol --test wire_golden -- --ignored regen_golden -//! ``` -//! -//! then `git add` the changed `golden/*.bin` and review the diff: an -//! EXISTING golden file changing bytes is a RED FLAG (you broke the wire -//! for a peer on the other side of a roll); only NEW files are expected. -//! Persisted bincode is forbidden; use JSON for durable structured payloads. -//! -//! NOTE: every sample uses EMPTY or SINGLE-entry `HashMap`s and FIXED -//! UUIDs / timestamps so the encoding is deterministic. +//! The ignored `regen_golden` test writes only the current internal corpus. +//! Field additions are not compatible merely because they are trailing: a +//! positional bincode reader must agree on the complete payload shape. +//! Samples use empty or single-entry maps and fixed IDs for deterministic bytes. use std::collections::HashMap; use std::net::Ipv4Addr; @@ -91,9 +56,7 @@ where bincode is POSITIONAL — a reordered field/variant or an inserted \ (non-trailing) field breaks the coord↔host gRPC payload for a \ peer on the other side of a deploy roll.\n\ - If this is an intentional, wire-SAFE evolution (a TRAILING \ - field/variant only), regenerate the corpus per the module header \ - and bump WIRE_VERSION." + Review the transport contract before changing this golden." ); let decoded: T = bincode::deserialize(&golden).expect("bincode decode golden"); assert_eq!( @@ -221,6 +184,7 @@ fn snapshot_metadata() -> SnapshotMetadata { fn session_egress_policy() -> SessionEgressPolicy { SessionEgressPolicy { + managed_deny: None, session_id: SessionId::from(fixed_uuid(0x30)), sandbox_id: SandboxId::from(fixed_uuid(0x31)), guest_ip: Ipv4Addr::new(169, 254, 0, 21), @@ -250,7 +214,7 @@ fn session_egress_policy() -> SessionEgressPolicy { } } -/// The engrams-review outage (2026-08-01): every field of the inject/observe +/// A populated minted policy: every field of the inject/observe /// ENTRY types populated — most importantly `mint_source: Some(..)`, which the /// empty-`injects` golden above can never exercise. `CredentialMintSource` was /// internally tagged (#931); bincode ENCODED that as a map but could never @@ -260,6 +224,7 @@ fn session_egress_policy() -> SessionEgressPolicy { /// entry types can't silently drop off the corpus again. fn session_egress_policy_minted() -> SessionEgressPolicy { SessionEgressPolicy { + managed_deny: None, injects: vec![EgressInjectEntry { secret: "ghs_minted".into(), header_name: "authorization".into(), @@ -312,6 +277,7 @@ fn session_egress_policy_minted() -> SessionEgressPolicy { /// that #931 fixed stays pinned for a second provider. fn session_egress_policy_google() -> SessionEgressPolicy { SessionEgressPolicy { + managed_deny: None, network_allow_hosts: vec!["compute.googleapis.com".into()], injects: vec![EgressInjectEntry { secret: "ya29.minted".into(), @@ -353,6 +319,7 @@ fn session_egress_policy_google() -> SessionEgressPolicy { /// variant's encoding (index 1) inside a fully-populated entry. fn session_egress_policy_oauth() -> SessionEgressPolicy { SessionEgressPolicy { + managed_deny: None, injects: vec![EgressInjectEntry { secret: "linear-access-token".into(), header_name: "Authorization".into(), @@ -380,6 +347,7 @@ fn session_egress_policy_oauth() -> SessionEgressPolicy { /// fully-populated entry. fn session_egress_policy_oauth_user() -> SessionEgressPolicy { SessionEgressPolicy { + managed_deny: None, injects: vec![EgressInjectEntry { secret: "personal-access-token".into(), header_name: "Authorization".into(), @@ -482,8 +450,7 @@ fn struct_payloads_golden() { sha256: "d".repeat(64), }, ); - // SessionEgressPolicy doesn't derive `PartialEq`; pin its encoded - // bytes (field-order regression catch). + // Pin the policy field order, including the optional managed deny. assert_golden_no_eq("session_egress_policy", &session_egress_policy()); assert_golden_no_eq( "session_egress_policy_minted", @@ -501,8 +468,7 @@ fn struct_payloads_golden() { "session_egress_policy_oauth_user", &session_egress_policy_oauth_user(), ); - // `Vec` — the `CaptureProgress.warm_stages_bincode` - // payload (issue #539). `WarmStageRecord` derives `PartialEq`. + // Pin the persisted warm-stage shape. assert_golden("warm_stages", &warm_stages()); // ADR 0080 phase 3b (wire v14) — the MaterializeImage payloads. @@ -581,8 +547,7 @@ fn nested_enum_variant_indices() { #[test] fn wire_mirrors_golden() { - // The wire.rs mirror shapes (ADR 0013 — the pattern the rest should - // eventually adopt). WireExecRequest isn't `PartialEq`; pin bytes. + // Pin the exec request field order. assert_golden_no_eq( "wire_exec_request", &WireExecRequest { @@ -601,111 +566,8 @@ fn wire_mirrors_golden() { #[test] fn wire_version_pinned() { - // The version stamp coord/host exchange. Bumping it is the deliberate - // signal that a payload shape changed; pin it so a payload change - // without a bump (or vice-versa) is a conscious decision. - // - // 9 -> 10: ADR 0073 (epic #542) — bind_session + AgentSpec carry - // binding_epoch, the shell-pin/rehandshake RPCs are deleted, heartbeat - // gains harness_attached. Goldens regenerated in the same change. - // 10 -> 11: ADR 0078 phase 2 (epic #548) — the heartbeat drops the - // never-populated `local_snapshots` advert (placement affinity reads - // PG `snapshots.host_id` instead). Goldens regenerated. - // 11 -> 12: ADR 0079 (#543) — fencing_epoch on session-scoped host - // RPCs (FencedSandboxRequest + the StartAgent/Restore pairs). - // Proto-native fields only; bincode goldens unchanged. - // 12 -> 13: ADR 0080 phase 2a — BuildBaseSnapshotRequest gains - // capture_egress_bincode (coordinator-assembled capture egress); - // WarmConfig (inside warm_bincode) gains `env`. Neither type is in - // the golden corpus (SessionEgressPolicy itself is unchanged), so - // bincode goldens are unchanged. - // 13 -> 14: ADR 0080 phase 3b — the new server-streaming - // `MaterializeImage` RPC (host-side materialization of standard - // docker images). NEW bincode payloads only (`ResolvedRegistryAuth` - // coord→host; `ManifestRef` — already pinned — and - // `OciRuntimeDefaults` host→coord), goldens ADDED for the new - // shapes; every existing golden is byte-identical. - // 14 -> 15: ADR 0084 (#546) — capture_jobs heartbeat dispatch/ - // reporting. `Heartbeat.capture_job_reports` / - // `HeartbeatAck.capture_assignments`/`acked_capture_jobs` ride the - // JSON heartbeat/ack, NOT the gRPC bincode `bytes` payloads this - // corpus pins — no new golden entries here. BuildBaseSnapshot RPC - // deletion rides this bump too (removed in the cutover commit). - // 15 -> 16: ADR 0095 — `SnapshotMetadata` gains the TRAILING - // `peer_hints` field (peer-fill seed addrs for the restore - // destination). Goldens regenerated. The `PeerChunkGet` RPC and the - // JSON-ack `warm_peers` field ride this bump too (both are - // independently roll-safe; the bump pins the deploy posture — a v16 - // coord never dispatches a peer-hinted restore to a v15 host). - // 16 -> 17: ADR 0100 — the coord↔host WriteFiles RPC carries new - // WireWriteFilesRequest/WireWriteFilesResponse bincode mirrors. - // 17 -> 18: ADR 0093 addendum — `MaterializeImageRequest.min_disk_gib` - // (the image's `suggested_disk_gib` now floors the packed ext4 size). - // Proto-native scalar only; no bincode payload changed, so every - // golden is byte-identical. - // 19 -> 20: ADR 0103 review hardening — ExecFrame gains the proto-native - // Refused terminal. No bincode payload changed. - // 20 -> 21: ADR 0109 — `SessionEgressPolicy.google_adc` is a trailing - // bincode field. The session-egress-policy golden was regenerated. - // 21 -> 22: `CredentialMintSource` becomes externally tagged — the - // internally-tagged form (#931) could never bincode-DECODE, so every - // minted-inject policy failed host-side at boot (the engrams-review - // outage, 2026-08-01). No existing golden changes bytes (the broken - // `Some` shape never had one); the minted-policy golden is ADDED. - // 22 -> 23: ADR 0109 seam — `SessionEgressPolicy.google_adc` became - // `metadata_flavor: Option`. A field REPLACEMENT, not an - // addition, so a v22 host cannot decode a v23 policy: the roll is - // lockstep. All three session-egress-policy goldens were regenerated, and - // the `session_egress_policy_google` fixture pinned the `Some(Gce)` - // encoding beside the `None` case the other two carry. - // 23 -> 24: ADR 0106 addendum — `CredentialMintSource` gains the TRAILING - // `OauthConnector` variant (connector OAuth on the inject rail). Existing - // goldens keep their bytes (trailing-variant addition); the oauth-policy - // golden is ADDED and the variant index is pinned at 1. - // 24 -> 25: ADR 0112 — `SandboxSpec` gains the TRAILING `swap_mib` - // field (ephemeral guest swap size). Only `sandbox_spec.bin` changed - // bytes (the fixture pins `Some(6144)`); every other golden embeds no - // spec and kept its bytes. Lockstep coord+host roll. - // 25 -> 26: ADR 0109 Cloud SQL addendum — the metadata listener becomes a - // generic guest gateway. `metadata_flavor` becomes registered - // `guest_services`, and `SessionEgressPolicy` gains generic tunnel entries. - // All session-policy goldens were regenerated; the Google fixture pins a - // service and a populated tunnel. - // 27 -> 28: ADR 0115 — `CredentialMintSource` gains the TRAILING - // `OauthUser` variant (a user's personal connector credential on the - // inject rail). Existing goldens keep their bytes (trailing-variant - // addition); the oauth-user-policy golden is ADDED and the variant index - // is pinned at 2. - // 28 -> 29: SnapshotHold adds an RPC; bincode payloads are unchanged. - // 29 -> 30: ADR 0045 C1 retired — `MigrationCapture`, `SnapshotWait` - // and the `Chunk`/`Sidecar`/`DiskManifest` export items are deleted; - // `MigrationSourceInfo` loses its C1-only fields (no golden embeds - // it: `snapshot_metadata.bin` pins `migration_source: None`). Lockstep - // coord+host roll. - // 30 -> 31: migration disk items carry the device role. - // 31 -> 32: SnapshotMetadata appends swap_manifest. Serde default does - // not decode old positional bincode; the exact-match wire gate makes the roll safe. - // snapshot_metadata.bin gains one trailing zero byte for None. - // Checkpoint adverts also carry the optional swap manifest. - // 32 -> 33: SandboxSpec appends swap_source and swap_manifest. - // HostUtilization also drops the committed-swap reservation. - // 33 -> 34: MigrationPresetup advertises the swap base lineage. - // Existing bincode goldens do not embed presetup; their bytes do not change. - // 34 -> 35: the memory image refusal is typed. No payload changes; the - // bump fences old hosts that refuse untyped. - // 35 -> 36: page-channel v6 fault windows require a lockstep host roll. - // Coordinator bincode payload bytes are unchanged. - // 36 -> 37: HostCapabilities appends page_idle. - // MigrationItem appends HotHint (bincode 3, protobuf 6). Existing item - // indices stay fixed; the optional hint is separate from immutable disk seals. - // Existing goldens embed neither payload and keep their bytes. - // 37 -> 38: MigrationItem::DiskBlockAt replaces DiskChunkAt at bincode - // index 2 (protobuf 7) and DiskPrecopyInfo is appended (bincode 4, - // protobuf 8); the capture reports block counts. No golden embeds either. - assert_eq!( - WIRE_VERSION, 38, - "WIRE_VERSION changed — confirm payload goldens were regenerated too" - ); + assert_eq!(WIRE_VERSION, 38); + assert_eq!(engram_protocol::wire::MANAGED_PROTOCOL_VERSION, 43); } /// Writer for the golden corpus. `#[ignore]`d so a normal `cargo test` @@ -740,6 +602,7 @@ fn regen_golden() { }, ); write("session_egress_policy", &session_egress_policy()); + write("session_egress_policy_managed", &managed_egress_policy()); write( "session_egress_policy_minted", &session_egress_policy_minted(), @@ -798,30 +661,17 @@ fn regen_golden() { ); } -/// The engrams-review outage (2026-08-01): the internally-tagged -/// `CredentialMintSource` (#931) bincode-ENCODED fine but could never DECODE -/// (`deserialize_any`), so the encode-only golden pin above was green while -/// every minted-inject policy failed host-side. This test pins the DECODE leg: -/// a populated `Some(mint_source)` + `Some(expires_at)` must round-trip. -/// `SessionEgressPolicy` has no `PartialEq`, so assert the fields that carry -/// the regression. +/// Populated minted credentials must retain every field through internal encoding. #[test] fn minted_inject_policy_bincode_round_trips() { let policy = session_egress_policy_minted(); let bytes = bincode::serialize(&policy).expect("bincode encode minted policy"); let back: SessionEgressPolicy = bincode::deserialize(&bytes).expect("bincode decode minted policy"); - assert_eq!(back.injects.len(), 1); - assert_eq!(back.injects[0].mint_source, policy.injects[0].mint_source); - assert_eq!(back.injects[0].expires_at, policy.injects[0].expires_at); - assert_eq!(back.observes.len(), 1); - assert_eq!( - back.observes[0].url_fallback, - policy.observes[0].url_fallback - ); + assert_eq!(back, policy); } -/// ADR 0080 prod regression (dev-brain enable): `WarmConfig.env` holds +/// ADR 0080 capture contract: `WarmConfig.env` holds /// `CaptureEnvValue`, an internally-tagged serde enum — bincode cannot /// DECODE that representation (`deserialize_any`), so a non-empty /// `[[warm.env]]` crossing the wire fails host-side at capture. The @@ -863,3 +713,93 @@ fn warm_config_wire_shape_round_trips_only_when_stripped() { wire-strip in enabled_images.rs and this pin together" ); } + +fn managed_egress_policy() -> SessionEgressPolicy { + let mut policy = session_egress_policy(); + policy.managed_deny = Some( + engram_core::types::managed_deny::ManagedDenyPolicy::new( + 9, + ["mcp.example".into(), "alias.example".into()], + ) + .unwrap(), + ); + policy +} +#[test] +fn managed_policy_internal_encoding_retains_authority() { + assert_golden("session_egress_policy_managed", &managed_egress_policy()); +} + +#[test] +fn ordinary_v38_source_bytes_are_unchanged() { + for (name, policy) in [ + ("session_egress_policy", session_egress_policy()), + ( + "session_egress_policy_minted", + session_egress_policy_minted(), + ), + ( + "session_egress_policy_google", + session_egress_policy_google(), + ), + ("session_egress_policy_oauth", session_egress_policy_oauth()), + ( + "session_egress_policy_oauth_user", + session_egress_policy_oauth_user(), + ), + ] { + let old = std::fs::read( + golden_path(name) + .parent() + .unwrap() + .join("ordinary_v38") + .join(format!("{name}.bin")), + ) + .unwrap(); + assert_eq!( + engram_protocol::ordinary_policy::encode(&policy).unwrap(), + old, + "{name}" + ); + assert_eq!( + engram_protocol::ordinary_policy::decode(&old).unwrap(), + policy + ); + for cut in 0..old.len() { + assert!(engram_protocol::ordinary_policy::decode(&old[..cut]).is_err()); + } + let mut trailing = old; + trailing.push(0); + assert!(engram_protocol::ordinary_policy::decode(&trailing).is_err()); + } +} + +#[test] +fn ordinary_policy_rejects_managed_authority_even_when_empty() { + for hosts in [vec![], vec!["mcp.example".to_string()]] { + let mut policy = session_egress_policy(); + policy.managed_deny = + Some(engram_core::types::managed_deny::ManagedDenyPolicy::new(1, hosts).unwrap()); + assert!(engram_protocol::ordinary_policy::encode(&policy).is_err()); + } +} + +#[test] +fn ordinary_policy_preserves_apps_and_secret_modes() { + for secret_mode in [SecretMode::Literal, SecretMode::Broker] { + let mut policy = session_egress_policy_google(); + policy.apps.push(engram_core::types::egress::AppEndpoint { + hostname: "app.example".into(), + port: 8080, + }); + policy.secret_mode = secret_mode; + policy.allow_all = true; + assert_eq!( + engram_protocol::ordinary_policy::decode( + &engram_protocol::ordinary_policy::encode(&policy).unwrap() + ) + .unwrap(), + policy + ); + } +} diff --git a/crates/engram-rootfs-materializer/src/inject.rs b/crates/engram-rootfs-materializer/src/inject.rs index a76373134..f31f8b05f 100644 --- a/crates/engram-rootfs-materializer/src/inject.rs +++ b/crates/engram-rootfs-materializer/src/inject.rs @@ -164,7 +164,24 @@ mark fs_mounts_done # timeout:2/attempts:2 bounds the residual worst case. Skip if # /etc/resolv.conf already exists (operator override). mkdir -p /etc -if [ ! -s /etc/resolv.conf ]; then +managed_dns="" +if [ -r /proc/cmdline ]; then + read -r boot_line < /proc/cmdline || true + for boot_arg in $boot_line; do + case "$boot_arg" in + engram_managed_dns=*) + [ -z "$managed_dns" ] || exit 1 + managed_dns=${boot_arg#engram_managed_dns=} + case "$managed_dns" in ""|*[!0-9.]*) exit 1 ;; esac + ;; + esac + done +fi +if [ -n "$managed_dns" ]; then + # The host selects this gateway before boot. Override image resolver data. + rm -f /etc/resolv.conf + printf 'nameserver %s\noptions timeout:2 attempts:2\n' "$managed_dns" > /etc/resolv.conf +elif [ ! -s /etc/resolv.conf ]; then # Shell-pure VZ detection (no ip/grep dependency — the shim only # assumes /bin/sh): the guest's local addresses appear in # /proc/net/fib_trie; a 192.168.64.x entry means Apple's VZ NAT. @@ -493,6 +510,44 @@ async fn chmod_executable(path: &Path) -> Result<(), std::io::Error> { mod tests { use super::*; + #[test] + fn managed_dns_uses_exact_boot_gateway_over_existing_resolver() { + let dir = tempfile::tempdir().unwrap(); + let resolver = dir.path().join("resolv.conf"); + let cmdline = dir.path().join("cmdline"); + std::fs::write(&resolver, "nameserver 203.0.113.53\n").unwrap(); + std::fs::write( + &cmdline, + "root=/dev/vda engram_managed_dns=192.0.2.1 ip=192.0.2.2::192.0.2.1", + ) + .unwrap(); + let dns = DEFAULT_INIT_SHIM + .split("# DNS for userspace.") + .nth(1) + .unwrap() + .split("# /etc/hosts:") + .next() + .unwrap(); + let script = format!("# DNS for userspace.{}", dns) + .replace("/etc/resolv.conf", resolver.to_str().unwrap()) + .replace("/proc/cmdline", cmdline.to_str().unwrap()) + .replace("mkdir -p /etc", "true"); + let output = std::process::Command::new("/bin/sh") + .arg("-ec") + .arg(script) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + std::fs::read_to_string(resolver).unwrap(), + "nameserver 192.0.2.1\noptions timeout:2 attempts:2\n" + ); + } + /// issue #240: the FC branch of the init shim's resolv.conf writer /// must NOT hand the guest a public recursive resolver. With the /// mandatory egress proxy, the host REDIRECTs guest :53 to the diff --git a/crates/engram-rootfs-materializer/src/memo.rs b/crates/engram-rootfs-materializer/src/memo.rs index fbbc79778..94e3df955 100644 --- a/crates/engram-rootfs-materializer/src/memo.rs +++ b/crates/engram-rootfs-materializer/src/memo.rs @@ -18,7 +18,7 @@ use crate::pull::{LayerPlan, Platform}; /// hit returns the output of the earlier layout. /// The pinned fixture manifest in `tests/materialize.rs` fails when the /// output changes. -pub const LAYOUT_VERSION: u32 = 1; +pub const LAYOUT_VERSION: u32 = 2; const PREFIX: &str = "materialize-memo/"; diff --git a/crates/engram-rootfs-materializer/tests/materialize.rs b/crates/engram-rootfs-materializer/tests/materialize.rs index 4167b4d56..a1d5d18a2 100644 --- a/crates/engram-rootfs-materializer/tests/materialize.rs +++ b/crates/engram-rootfs-materializer/tests/materialize.rs @@ -453,7 +453,7 @@ async fn materialize_is_deterministic_and_scrubs_scratch() { /// The disk manifest of the arm64 fixture. A memo hit returns the /// output of an earlier materialize, so a change to the packed bytes /// must come with a new `memo::LAYOUT_VERSION`. -const PINNED_ARM64_DISK_MANIFEST: &str = "e23f6cc8-e0aa-8a55-814c-b7cf9b2d7fa7@v1"; +const PINNED_ARM64_DISK_MANIFEST: &str = "8724d704-ec4e-87fb-bf7c-670134aacb2e@v1"; /// A second materialize of unchanged layers into the same store reads /// the memo: the same result, and no layer blob is downloaded. diff --git a/crates/engram-sandbox-firecracker/Cargo.toml b/crates/engram-sandbox-firecracker/Cargo.toml index 627435077..3627a45d9 100644 --- a/crates/engram-sandbox-firecracker/Cargo.toml +++ b/crates/engram-sandbox-firecracker/Cargo.toml @@ -16,8 +16,13 @@ description = "Production SandboxBackend driving Firecracker microVMs over its H unsafe_code = "allow" [dependencies] +tempfile = "3" engram-chunk-store.workspace = true engram-core.workspace = true +engram-host-core.workspace = true +engram-egress-proto.workspace = true +sha2.workspace = true +hex.workspace = true engram-telemetry.workspace = true engram-agentd.workspace = true engram-harness-proto.workspace = true @@ -43,8 +48,6 @@ workspace-hack = { version = "0.1", path = "../../workspace-hack" } rtnetlink.workspace = true [dev-dependencies] -sha2.workspace = true -hex.workspace = true engram-egress-proxy.workspace = true engram-storage-local.workspace = true # ADR 0080 phase 3b: `materialize_boot` runs the real materializer @@ -60,4 +63,3 @@ rcgen.workspace = true rustls.workspace = true rustls-pemfile.workspace = true tokio-rustls.workspace = true -tempfile = "3" diff --git a/crates/engram-sandbox-firecracker/src/client.rs b/crates/engram-sandbox-firecracker/src/client.rs index 055b3be66..0168c7b69 100644 --- a/crates/engram-sandbox-firecracker/src/client.rs +++ b/crates/engram-sandbox-firecracker/src/client.rs @@ -451,6 +451,13 @@ impl FirecrackerClient { self.patch_vm_state(VmState::Resumed).await } + /// Read back the cold VM state and configuration for the managed owner. + #[cfg(target_os = "linux")] + pub(super) async fn managed_get(&self, path: &str) -> Result { + let bytes = self.request_with_body::<()>("GET", path, None).await?; + serde_json::from_slice(&bytes).map_err(|_| vm_err("invalid managed VM readback")) + } + // ---- transport -------------------------------------------------- /// One-shot PUT helper for endpoints that just acknowledge with 204. diff --git a/crates/engram-sandbox-firecracker/src/lib.rs b/crates/engram-sandbox-firecracker/src/lib.rs index e9621c1d1..5457b6c76 100644 --- a/crates/engram-sandbox-firecracker/src/lib.rs +++ b/crates/engram-sandbox-firecracker/src/lib.rs @@ -104,6 +104,7 @@ use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; pub mod client; +pub mod managed; pub mod net; pub mod paths; pub mod pidfd; @@ -2573,9 +2574,7 @@ impl FirecrackerBackend { // ADR 0026: artifact-upload bridge accept loop. self.spawn_upload_listener(sandbox_id, &vsock_uds_path) .await?; - let agent_ready_rx = self - .spawn_agent_ready_listener(sandbox_id, &vsock_uds_path) - .await?; + let agent_ready_rx = Self::spawn_agent_ready_listener(sandbox_id, &vsock_uds_path).await?; api.put_action(ActionType::InstanceStart).await?; @@ -2670,7 +2669,6 @@ impl FirecrackerBackend { /// successful dial (subsequent dials are no-ops — agentd only /// signals once per process lifetime). async fn spawn_agent_ready_listener( - &self, sandbox_id: SandboxId, vsock_uds_path: &Path, ) -> Result { @@ -9080,7 +9078,9 @@ Pss: 8200000 kB // Keep the UDS below macOS's 104-byte path limit. let sockets = tempfile::tempdir_in("/tmp").unwrap(); let vsock = sockets.path().join("vsock.sock"); - let readiness = be.spawn_agent_ready_listener(id, &vsock).await.unwrap(); + let readiness = FirecrackerBackend::spawn_agent_ready_listener(id, &vsock) + .await + .unwrap(); let listener_abort = readiness.listener.as_ref().unwrap().abort_handle(); let mut child = tokio::process::Command::new("sleep") .arg("30") diff --git a/crates/engram-sandbox-firecracker/src/managed.rs b/crates/engram-sandbox-firecracker/src/managed.rs new file mode 100644 index 000000000..2d4f0e272 --- /dev/null +++ b/crates/engram-sandbox-firecracker/src/managed.rs @@ -0,0 +1,673 @@ +//! Exact managed Firecracker ownership from CLOSED preparation to retirement. +#[cfg(any(target_os = "linux", test))] +mod adapter; +#[cfg(any(target_os = "linux", test))] +mod child; +#[cfg(any(target_os = "linux", test))] +mod configure; +#[cfg(any(target_os = "linux", test))] +mod disk; +#[cfg(any(target_os = "linux", test))] +mod files; +mod network; +#[cfg(target_os = "linux")] +mod process; + +#[cfg(target_os = "linux")] +use engram_core::types::sandbox::{SessionFileMetadata, SessionFileSpec, SessionFileStream}; +use engram_core::{ + types::{ + managed_cold::{ManagedColdPrepare, ManagedColdPrepared}, + sandbox::SandboxSpec, + }, + SandboxError, +}; +use engram_egress_proto::private::{Allocation, RegisteredPrivateNetwork}; + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub struct ManagedFcClosedReceipt { + pub prepared: ManagedColdPrepared, + pub allocation: Allocation, + pub shape: network::NetworkShape, + pub namespace_device: u64, + pub namespace_inode: u64, + pub tap_ifindex: u32, + pub rules_sha256: String, + pub configuration_sha256: String, +} +/// Exact private process, boot files and network ownership through retirement. +/// It cannot enter the ordinary sandbox map or expose a generic resume permit. +pub struct PreparedManagedFc { + #[cfg(not(target_os = "linux"))] + _unavailable: std::convert::Infallible, + #[cfg(target_os = "linux")] + inner: Prepared, +} +#[cfg(target_os = "linux")] +struct Prepared { + process: process::Process, + adapter: adapter::Adapter, + namespace: std::sync::Arc, + api: crate::FirecrackerClient, + config: serde_json::Value, + receipt: ManagedFcClosedReceipt, + failed: bool, + files: std::sync::Arc, + released_epoch: Option, + started: bool, + streams: tokio::task::JoinSet<()>, +} +#[cfg(any(target_os = "linux", test))] +fn normalize_config(mut config: serde_json::Value) -> std::io::Result { + let drives = config + .get_mut("drives") + .and_then(serde_json::Value::as_array_mut) + .ok_or_else(|| std::io::Error::other("managed VM drives are invalid"))?; + // Preboot uses the configured drive order; runtime iterates a device map. + let mut ordered = std::collections::BTreeMap::new(); + for drive in std::mem::take(drives) { + let id = drive + .get("drive_id") + .and_then(serde_json::Value::as_str) + .filter(|id| !id.is_empty()) + .ok_or_else(|| std::io::Error::other("managed VM drive identity is invalid"))? + .to_owned(); + if ordered.insert(id, drive).is_some() { + return Err(std::io::Error::other( + "managed VM drive identity is duplicated", + )); + } + } + *drives = ordered.into_values().collect(); + Ok(config) +} +impl crate::FirecrackerBackend { + /// Transfer disposable staging to the blocking pin operation. Cancellation + /// can close ingress while that operation still owns its input files. + pub async fn prepare_managed_cold( + &self, + request: ManagedColdPrepare, + spec: SandboxSpec, + network: RegisteredPrivateNetwork, + staging: tempfile::TempDir, + session_disk: std::sync::Arc, + ) -> Result { + request.validate(&spec).map_err(SandboxError::InvalidSpec)?; + let allocation = network.allocation(); + if allocation.session_id != request.session_id + || allocation.sandbox_id != request.sandbox_id + || allocation.boot_attempt != request.boot_attempt + { + return Err(SandboxError::InvalidSpec( + "managed allocation does not match cold request".into(), + )); + } + let shape = network::NetworkShape::new(allocation.guest_ip, allocation.gateway_ip)?; + #[cfg(target_os = "linux")] + { + prepare(self, request, spec, network, shape, staging, session_disk).await + } + #[cfg(not(target_os = "linux"))] + { + let _ = (shape, staging, session_disk); + Err(SandboxError::InvalidSpec( + "managed Firecracker preparation requires Linux".into(), + )) + } + } +} +impl PreparedManagedFc { + pub async fn closed_receipt(&mut self) -> Result { + #[cfg(target_os = "linux")] + { + let result = self.inner.check().await; + if result.is_err() { + self.inner.failed = true; + let _ = self.inner.adapter.close().await; + let _ = self.inner.process.close().await; + } + result + } + #[cfg(not(target_os = "linux"))] + { + Err(SandboxError::InvalidSpec( + "managed Firecracker preparation requires Linux".into(), + )) + } + } + #[cfg(target_os = "linux")] + pub async fn release(&mut self, epoch: u64) -> Result<(), SandboxError> { + self.closed_receipt().await?; + self.inner.failed = true; + self.inner.adapter.release(epoch).await?; + let pid = self.inner.process.live_pid()?; + let namespace = self.inner.namespace.clone(); + tokio::task::spawn_blocking(move || namespace.release(pid)) + .await + .map_err(std::io::Error::other)??; + self.inner.adapter.check_ready(epoch)?; + self.inner.released_epoch = Some(epoch); + self.inner.failed = false; + Ok(()) + } + #[cfg(target_os = "linux")] + pub async fn check_ready(&mut self, epoch: u64) -> Result<(), SandboxError> { + if self.inner.failed || self.inner.released_epoch != Some(epoch) { + return Err(std::io::Error::other("managed owner is not ready").into()); + } + self.inner.adapter.check_ready(epoch)?; + let pid = self.inner.process.live_pid()?; + let namespace = self.inner.namespace.clone(); + tokio::task::spawn_blocking(move || namespace.check(Some(pid))) + .await + .map_err(std::io::Error::other)??; + if normalize_config(self.inner.api.managed_get("/vm/config").await?)? != self.inner.config + || self.inner.api.managed_get("/").await?["state"] + != if self.inner.started { + "Running" + } else { + "Not started" + } + { + return Err(std::io::Error::other("managed VM configuration changed").into()); + } + Ok(()) + } + #[cfg(target_os = "linux")] + pub async fn start_agent( + &mut self, + agent: engram_core::types::sandbox::AgentSpec, + sink: engram_core::traits::HarnessSink, + ) -> Result<(), SandboxError> { + self.check_ready(agent.binding_epoch).await?; + if self.inner.started { + return Err(std::io::Error::other("managed start already issued").into()); + } + self.inner.started = true; + // An issued start can change the disk before its reply. Preserve the + // exact private lineage through cancellation, Drop and failure cleanup. + self.inner + .files + .retain + .store(true, std::sync::atomic::Ordering::Release); + self.inner.failed = true; + let vsock = self.inner.files.path.join("guest.vsock"); + // Drain the cold-boot hint before agentd starts serving RPCs. This + // listener has no authority over the private network's release state. + let _agent_ready = crate::FirecrackerBackend::spawn_agent_ready_listener( + self.inner.receipt.prepared.request.sandbox_id, + &vsock, + ) + .await?; + let listener = tokio::net::UnixListener::bind(crate::harness_uds_for(&vsock))?; + self.inner.streams.spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + sink(Box::pin(stream)); + } + }); + self.inner + .api + .put_action(crate::client::ActionType::InstanceStart) + .await?; + let mut conn = tokio::time::timeout(std::time::Duration::from_secs(180), async { + loop { + self.inner.process.live_pid()?; + self.inner.adapter.check_ready(agent.binding_epoch)?; + if let Ok(conn) = + crate::FirecrackerBackend::connect_fc_vsock_once(&vsock, 1024).await + { + return Ok::<_, SandboxError>(conn); + } + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + }) + .await + .map_err(|_| std::io::Error::other("managed agent connection timed out"))??; + let request = + engram_agentd::WireRequest::SpawnHarness(engram_agentd::SpawnHarnessRequest { + argv: agent.argv, + env: agent.env.into_iter().collect(), + session_env: agent.session_env.into_iter().collect(), + host_ca_pem: agent.host_ca_pem, + }); + // Never retry the write or unknown response in this fresh guest context. + engram_agentd::write_msg(&mut conn, &request).await?; + let response: engram_agentd::WireResponse = engram_agentd::read_msg(&mut conn).await?; + if !matches!(response, engram_agentd::WireResponse::HarnessSpawned { .. }) { + return Err(std::io::Error::other("managed harness start failed").into()); + } + self.inner.failed = false; + Ok(()) + } + #[cfg(target_os = "linux")] + pub fn owns(&self, request: &engram_core::types::managed_cold::ManagedColdPrepare) -> bool { + &self.inner.receipt.prepared.request == request + } + #[cfg(target_os = "linux")] + async fn exec_connection( + &mut self, + epoch: u64, + ) -> Result { + self.check_ready(epoch).await?; + if !self.inner.started { + return Err(std::io::Error::other("managed execution requires a started owner").into()); + } + crate::FirecrackerBackend::connect_fc_vsock_once( + &self.inner.files.path.join("guest.vsock"), + 1024, + ) + .await + } + #[cfg(target_os = "linux")] + pub async fn exec_stream( + &mut self, + epoch: u64, + cmd: engram_core::types::sandbox::ExecRequest, + ) -> Result { + let probe = self.exec_connection(epoch).await?; + let durable = crate::probe_durable_exec(Box::new(probe)).await?; + let connection = self.exec_connection(epoch).await?; + // Never hand a reconnect capability out of this live owner. On EOF, + // the caller retries the same journal ticket through current ownership. + crate::drive_exec_protocol( + self.inner.receipt.prepared.request.sandbox_id, + Box::new(connection), + cmd, + durable, + None, + ) + .await + } + #[cfg(target_os = "linux")] + pub async fn prepare_read_file( + &mut self, + epoch: u64, + path: String, + mut closed: tokio::sync::watch::Receiver<()>, + ) -> Result< + impl std::future::Future< + Output = Result<(SessionFileMetadata, SessionFileStream), SandboxError>, + > + Send + + 'static, + SandboxError, + > { + let connection = self.exec_connection(epoch).await?; + // This future owns the exact connection and has no reconnect capability. + Ok(engram_agentd::file_transfer::read_file_until_closed( + connection, + path, + async move { + let _ = closed.changed().await; + }, + )) + } + #[cfg(target_os = "linux")] + pub async fn prepare_write_file( + &mut self, + epoch: u64, + spec: SessionFileSpec, + bytes: SessionFileStream, + ) -> Result< + impl std::future::Future> + Send + 'static, + SandboxError, + > { + let connection = self.exec_connection(epoch).await?; + Ok(engram_agentd::file_transfer::write_file( + connection, spec, bytes, + )) + } + #[cfg(target_os = "linux")] + pub async fn cancel_exec(&mut self, epoch: u64, exec_id: String) -> Result<(), SandboxError> { + let connection = self.exec_connection(epoch).await?; + crate::cancel_exec_roundtrip(Box::new(connection), &exec_id).await + } + #[cfg(target_os = "linux")] + pub async fn retire_disk(&mut self, destination: &std::path::Path) -> Result<(), SandboxError> { + if self.inner.failed { + return Err(std::io::Error::other("managed capture source unavailable").into()); + } + self.inner + .files + .retain + .store(true, std::sync::atomic::Ordering::Release); + self.inner + .files + .disks + .as_ref() + .ok_or_else(|| std::io::Error::other("managed disk owner missing"))? + .retain_source(); + if !self.inner.started { + if let Some(epoch) = self.inner.released_epoch { + self.check_ready(epoch).await?; + } else { + self.closed_receipt().await?; + } + } + self.inner.failed = true; + if self.inner.started { + self.inner.api.pause().await?; + } + self.inner.streams.shutdown().await; + self.inner.adapter.close().await?; + self.inner.process.close().await?; + self.inner.files.export_capture(destination).await?; + Ok(()) + } + /// The host has durably recorded the exported file's exact hash and size. + #[cfg(target_os = "linux")] + pub fn capture_stored(&mut self) -> Result<(), SandboxError> { + self.inner.files.capture_stored()?; + Ok(()) + } + /// Wait for adapter and VM cleanup. Drop initiates cleanup only. + pub async fn close(self) -> Result<(), SandboxError> { + self.close_inner(false).await + } + /// An authenticated delete may discard a known live source after reaping. + /// It cannot discard an uncertain start or an incomplete capture. + pub async fn destroy(self) -> Result<(), SandboxError> { + self.close_inner(true).await + } + async fn close_inner(mut self, destroy: bool) -> Result<(), SandboxError> { + #[cfg(target_os = "linux")] + { + let uncertain = self.inner.failed; + self.inner.failed = true; + self.inner.streams.shutdown().await; + let network = self.inner.adapter.close().await; + let process = self.inner.process.close().await; + network?; + process?; + self.inner.files.close(destroy, uncertain).await?; + Ok(()) + } + #[cfg(not(target_os = "linux"))] + { + let _ = (&mut self, destroy); + Ok(()) + } + } +} +#[cfg(target_os = "linux")] +impl Prepared { + async fn check(&mut self) -> Result { + if self.failed { + return Err(std::io::Error::other("managed preparation failed").into()); + } + self.adapter.check_closed()?; + let pid = self.process.live_pid()?; + let ns = self.namespace.clone(); + tokio::task::spawn_blocking(move || ns.check(Some(pid))) + .await + .map_err(|_| std::io::Error::other("namespace check stopped"))??; + if self.api.managed_get("/").await?["state"] != "Not started" + || normalize_config(self.api.managed_get("/vm/config").await?)? != self.config + { + return Err(std::io::Error::other("managed VM state changed").into()); + } + self.adapter.check_closed()?; + Ok(self.receipt.clone()) + } +} +#[cfg(target_os = "linux")] +async fn prepare( + backend: &crate::FirecrackerBackend, + request: ManagedColdPrepare, + spec: SandboxSpec, + registered: RegisteredPrivateNetwork, + shape: network::NetworkShape, + staging: tempfile::TempDir, + session_disk: std::sync::Arc, +) -> Result { + use std::{io, sync::Arc}; + let allocation = registered.allocation().clone(); + let private = registered.into_stream(); + let files = { + let work = backend.work_dir.clone(); + let cfg = backend.config.clone(); + let req = request.clone(); + let spec = spec.clone(); + // Own creation through completion: cancellation cannot abandon an ioctl + // after it creates a volume but before the owner records its name. + let pin = tokio::spawn(async move { + let pin_request = req.clone(); + let pin_spec = spec.clone(); + let mut files = tokio::task::spawn_blocking(move || { + files::PinnedFiles::prepare(&work, &cfg, &pin_request, &pin_spec, staging) + }) + .await + .map_err(io::Error::other)??; + files.disks = Some(disk::Disks::new(session_disk, req.sandbox_id)); + files + .disks + .as_ref() + .expect("disk owner installed") + .install( + &files.path, + &req.disk.sha256, + req.disk.size_bytes, + spec.swap_mib, + ) + .await?; + Ok::<_, io::Error>(files) + }); + tokio::select! { biased; + _ = wait_for_stop(&private) => return Err(io::Error::other("private allocation stopped").into()), + files = pin => Arc::new(files.map_err(|_| io::Error::other("file preparation stopped"))??), + } + }; + let (namespace, tcp, dns) = tokio::select! { biased; + _ = wait_for_stop(&private) => return Err(io::Error::other("private allocation stopped").into()), + network = network::Namespace::create(shape.clone()) => network?, + }; + let path = files.path.clone(); + let mut process = process::Process::reserve(namespace.clone(), files.clone())?; + let adapter = adapter::Adapter::spawn( + private, + tokio::net::TcpListener::from_std(tcp)?, + tokio::net::UdpSocket::from_std(dns)?, + shape.guest, + shape.gateway, + Some(process.stop_signal()), + )?; + process.spawn(&backend.config.firecracker_bin)?; + let api = crate::FirecrackerClient::new(path.join("firecracker.sock")); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + process.live_pid()?; + adapter.check_closed()?; + if tokio::net::UnixStream::connect(api.socket()).await.is_ok() { + return Ok::<_, io::Error>(()); + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .map_err(|_| io::Error::other("managed VM API timed out"))??; + let cid = backend + .next_cid + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if cid < 3 || cid == u32::MAX { + return Err(io::Error::other("invalid vsock identity").into()); + } + configure::configure(&api, &backend.config, &request, &spec, &path, &shape, cid).await?; + let config = normalize_config(api.managed_get("/vm/config").await?)?; + let devices = config["network-interfaces"] + .as_array() + .ok_or_else(|| io::Error::other("missing VM network readback"))?; + if devices.len() != 1 + || devices[0]["host_dev_name"] != network::TAP + || devices[0]["guest_mac"] != network::GUEST_MAC + { + return Err(io::Error::other("wrong VM network attachment").into()); + } + let identity = namespace.identity.clone(); + use sha2::{Digest, Sha256}; + let configuration_sha256 = hex::encode(Sha256::digest( + serde_json::to_vec(&config).map_err(io::Error::other)?, + )); + let mut prepared = PreparedManagedFc { + inner: Prepared { + process, + adapter, + namespace, + api, + config, + failed: false, + files, + released_epoch: None, + started: false, + streams: tokio::task::JoinSet::new(), + receipt: ManagedFcClosedReceipt { + prepared: ManagedColdPrepared { + version: 1, + request, + state: engram_core::types::managed_cold::ManagedColdState::Closed, + }, + allocation, + shape, + namespace_device: identity.namespace_device, + namespace_inode: identity.namespace_inode, + tap_ifindex: identity.tap_ifindex, + rules_sha256: identity.rules_sha256, + configuration_sha256, + }, + }, + }; + prepared.closed_receipt().await?; + Ok(prepared) +} +#[cfg(any(target_os = "linux", test))] +async fn wait_for_stop(stream: &tokio::net::UnixStream) { + loop { + match stream.try_read(&mut [0]) { + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + if stream.readable().await.is_err() { + return; + } + } + _ => return, + } + } +} + +#[cfg(test)] +mod tests { + #[test] + fn configuration_comparison_ignores_only_drive_order() { + let expected = serde_json::json!({ + "drives": [ + {"drive_id":"rootfs","path_on_host":"/owned/root","is_root_device":true,"is_read_only":false}, + {"drive_id":"agentd","path_on_host":"/owned/agentd","is_root_device":false,"is_read_only":true} + ], + "network-interfaces":[{"iface_id":"a"},{"iface_id":"b"}] + }); + let mut reordered = expected.clone(); + reordered["drives"].as_array_mut().unwrap().reverse(); + assert_eq!( + super::normalize_config(expected.clone()).unwrap(), + super::normalize_config(reordered).unwrap() + ); + for (field, value) in [ + ("path_on_host", serde_json::json!("/other/root")), + ("is_root_device", serde_json::json!(false)), + ("is_read_only", serde_json::json!(true)), + ] { + let mut changed = expected.clone(); + changed["drives"][0][field] = value; + assert_ne!( + super::normalize_config(expected.clone()).unwrap(), + super::normalize_config(changed).unwrap(), + "drive field {field} must remain authoritative" + ); + } + let mut changed = expected.clone(); + changed["network-interfaces"] + .as_array_mut() + .unwrap() + .reverse(); + assert_ne!( + super::normalize_config(expected).unwrap(), + super::normalize_config(changed).unwrap(), + "other arrays must retain their order" + ); + } + + #[test] + fn configuration_comparison_rejects_invalid_drive_identities() { + for config in [ + serde_json::json!({}), + serde_json::json!({"drives":{}}), + serde_json::json!({"drives":[null]}), + serde_json::json!({"drives":[{}]}), + serde_json::json!({"drives":[{"drive_id":""}]}), + serde_json::json!({"drives":[{"drive_id":1}]}), + serde_json::json!({"drives":[{"drive_id":"same"},{"drive_id":"same"}]}), + ] { + assert!(super::normalize_config(config.clone()).is_err(), "{config}"); + } + } + + #[tokio::test] + async fn cold_boot_ready_hint_is_drained_and_pending_listener_is_owned() { + let sockets = tempfile::tempdir_in("/tmp").unwrap(); + let vsock = sockets.path().join("guest.vsock"); + let id = engram_core::SandboxId::new(); + let readiness = crate::FirecrackerBackend::spawn_agent_ready_listener(id, &vsock) + .await + .unwrap(); + let ready_path = crate::per_port_uds(&vsock, engram_agentd::ENGRAM_AGENTD_READY_PORT); + // A guest can close an early connection before it writes its frame. + drop(tokio::net::UnixStream::connect(&ready_path).await.unwrap()); + let mut guest = tokio::net::UnixStream::connect(&ready_path).await.unwrap(); + engram_agentd::write_msg( + &mut guest, + &engram_agentd::AgentReady { + agent_version: "fixture".into(), + }, + ) + .await + .unwrap(); + let mut receiver = readiness.receiver.clone(); + tokio::time::timeout(std::time::Duration::from_secs(1), receiver.wait_for(|v| *v)) + .await + .unwrap() + .unwrap(); + drop(readiness); + + let pending = crate::FirecrackerBackend::spawn_agent_ready_listener( + id, + &sockets.path().join("pending.vsock"), + ) + .await + .unwrap(); + let listener = pending.listener.as_ref().unwrap().abort_handle(); + drop(pending); + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while !listener.is_finished() { + tokio::task::yield_now().await; + } + }) + .await + .expect("dropping the owner must stop the ready listener"); + } + + #[tokio::test] + async fn proxy_shutdown_and_unexpected_data_stop_preparation_before_copy_completes() { + use tokio::io::AsyncWriteExt; + for send_data in [false, true] { + let (private, mut peer) = tokio::net::UnixStream::pair().unwrap(); + if send_data { + peer.write_all(b"x").await.unwrap(); + } else { + peer.shutdown().await.unwrap(); + } + tokio::time::timeout( + std::time::Duration::from_millis(100), + super::wait_for_stop(&private), + ) + .await + .unwrap(); + } + } +} diff --git a/crates/engram-sandbox-firecracker/src/managed/adapter.rs b/crates/engram-sandbox-firecracker/src/managed/adapter.rs new file mode 100644 index 000000000..fc6127555 --- /dev/null +++ b/crates/engram-sandbox-firecracker/src/managed/adapter.rs @@ -0,0 +1,1007 @@ +//! Bounded TCP/DNS streams reach only the registered private policy ingress. +use engram_egress_proto::private::{self as wire, Operation, Reply, Request}; +use std::{ + io, + net::{Ipv4Addr, SocketAddrV4}, + sync::{ + atomic::{AtomicU8, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream, UdpSocket, UnixStream}, + sync::{mpsc, oneshot, watch, Mutex}, + task::JoinSet, +}; + +const CLOSED: u8 = 0; +const READY: u8 = 1; +const DEAD: u8 = 2; + +pub(super) struct Adapter { + cancel: watch::Sender, + // The exact host-owned epoch consumes this sender once. Until then the + // worker cannot admit packet flows. + _release: Option>, + released_epoch: Option, + state: Arc, + done: oneshot::Receiver<()>, + result: Option>, + probe: Arc>>, +} +impl Adapter { + pub fn spawn( + private: UnixStream, + tcp: TcpListener, + dns: UdpSocket, + guest: Ipv4Addr, + gateway: Ipv4Addr, + terminal: Option>, + ) -> io::Result { + use std::os::fd::AsFd; + let probe = Arc::new(std::sync::Mutex::new(Some( + std::os::unix::net::UnixStream::from(private.as_fd().try_clone_to_owned()?), + ))); + let worker_probe = probe.clone(); + let (cancel, mut stopped) = watch::channel(false); + let (release, released) = oneshot::channel(); + let (done_tx, done) = oneshot::channel(); + let state = Arc::new(AtomicU8::new(CLOSED)); + let state_worker = state.clone(); + let lifetime = Lifetime { + state: state.clone(), + probe: worker_probe, + terminal, + reader: None, + }; + tokio::spawn(async move { + let mut lifetime = lifetime; + let (read, write) = private.into_split(); + let (reply_tx, replies) = mpsc::channel(1); + let (dead_tx, mut dead) = watch::channel(false); + let fail = dead_tx.clone(); + let reader_state = state_worker.clone(); + let mut reader = tokio::spawn(async move { + let mut read = read; + loop { + // Any bytes/EOF while CLOSED are terminal, including a + // partial header. Do not spend a handoff timeout parsing it. + if read.as_ref().readable().await.is_err() { + break; + } + if reader_state.load(Ordering::Acquire) == CLOSED { + break; + } + let Ok(reply) = wire::read::<_, Reply>(&mut read).await else { + break; + }; + if reply_tx.send(reply).await.is_err() { + break; + } + } + dead_tx.send_replace(true); + }); + lifetime.reader = Some(reader.abort_handle()); + let proxy = Arc::new(Proxy { + call: Mutex::new(Call { + write, + replies, + sequence: 0, + epoch: 0, + failed: false, + }), + fail, + }); + let mut flows = JoinSet::new(); + let result = tokio::select! { biased; + _ = stopped.changed() => Ok(()), + _ = dead.changed() => Err(wire::invalid()), + result = async { + let epoch = released.await.map_err(|_| wire::invalid())?; + if epoch == 0 || epoch > 9_007_199_254_740_991 { return Err(wire::invalid()); } + state_worker.store(3, Ordering::Release); + proxy.call.lock().await.epoch = epoch; + proxy.request(Operation::Bind, None).await?; + state_worker.store(READY, Ordering::Release); + serve(&tcp, &dns, guest, gateway, proxy.clone(), &mut flows).await + } => result, + }; + let _ = result; + state_worker.store(DEAD, Ordering::Release); + if let Some(signal) = &lifetime.terminal { + let _ = signal.send(super::child::Control::Stop); + } + // Listeners and every accepted guest/Unix socket close before the + // private write half returns cleanup EOF to the policy lease. + drop(tcp); + drop(dns); + flows.shutdown().await; + reader.abort(); + let _ = (&mut reader).await; + lifetime.probe.lock().expect("probe lock").take(); + drop(proxy); + let _ = done_tx.send(()); + }); + Ok(Self { + cancel, + _release: Some(release), + released_epoch: None, + state, + done, + result: None, + probe, + }) + } + /// Only READY after the exact private Bind reply can release native fences. + pub async fn release(&mut self, epoch: u64) -> io::Result<()> { + if self.released_epoch == Some(epoch) && self.state.load(Ordering::Acquire) == READY { + return self.check_ready(epoch); + } + self.check_closed()?; + if self.released_epoch.is_some() || epoch == 0 || epoch > 9_007_199_254_740_991 { + return Err(wire::invalid()); + } + struct ReleaseGuard { + cancel: watch::Sender, + state: Arc, + armed: bool, + } + impl Drop for ReleaseGuard { + fn drop(&mut self) { + if self.armed { + self.state.store(DEAD, Ordering::Release); + self.cancel.send_replace(true); + } + } + } + let mut guard = ReleaseGuard { + cancel: self.cancel.clone(), + state: self.state.clone(), + armed: true, + }; + self.released_epoch = Some(epoch); + self._release + .take() + .ok_or_else(wire::invalid)? + .send(epoch) + .map_err(|_| wire::invalid())?; + let result = tokio::time::timeout(Duration::from_secs(wire::HANDOFF_SECONDS), async { + loop { + match self.state.load(Ordering::Acquire) { + READY => return Ok(()), + DEAD => return Err(wire::invalid()), + _ => tokio::time::sleep(Duration::from_millis(1)).await, + } + } + }) + .await + .map_err(|_| wire::invalid()) + .and_then(|r| r); + if result.is_err() { + self.cancel.send_replace(true); + } + if result.is_ok() { + guard.armed = false; + } + result + } + pub fn check_ready(&self, epoch: u64) -> io::Result<()> { + if self.released_epoch != Some(epoch) + || self.state.load(Ordering::Acquire) != READY + || *self.cancel.borrow() + { + return Err(wire::invalid()); + } + Ok(()) + } + pub fn check_closed(&self) -> io::Result<()> { + use std::os::fd::AsRawFd; + if self.state.load(Ordering::Acquire) != CLOSED || *self.cancel.borrow() { + return Err(io::Error::other("private network is no longer closed")); + } + let guard = self.probe.lock().map_err(|_| wire::invalid())?; + let socket = guard.as_ref().ok_or_else(wire::invalid)?; + let mut byte = [0u8]; + // SAFETY: live owned stream and one writable byte. PEEK cannot consume + // a reply; no reply is legal before the private release command. + let count = unsafe { + libc::recv( + socket.as_raw_fd(), + byte.as_mut_ptr().cast(), + 1, + libc::MSG_PEEK | libc::MSG_DONTWAIT, + ) + }; + if count < 0 && io::Error::last_os_error().kind() == io::ErrorKind::WouldBlock { + return Ok(()); + } + self.state.store(DEAD, Ordering::Release); + self.cancel.send_replace(true); + Err(io::Error::other("private network control stopped")) + } + pub async fn close(&mut self) -> io::Result<()> { + self.cancel.send_replace(true); + if let Some(result) = &self.result { + return result.clone().map_err(io::Error::other); + } + let result = + tokio::time::timeout(Duration::from_secs(wire::HANDOFF_SECONDS), &mut self.done) + .await + .map_err(|_| io::Error::other("network cleanup timed out")) + .and_then(|r| r.map_err(|_| io::Error::other("network cleanup stopped"))); + self.result = Some(result.as_ref().map(|_| ()).map_err(ToString::to_string)); + result + } +} +impl Drop for Adapter { + fn drop(&mut self) { + self.cancel.send_replace(true); + } +} +// This guard is constructed before spawn. Runtime cancellation, including an +// unpolled task, cannot leave a live-looking closed owner or a private probe FD. +struct Lifetime { + state: Arc, + probe: Arc>>, + terminal: Option>, + reader: Option, +} +impl Drop for Lifetime { + fn drop(&mut self) { + self.state.store(DEAD, Ordering::Release); + if let Some(probe) = self.probe.lock().expect("probe lock").take() { + // Terminal shutdown applies to all duplicates of this control + // socket, even if another task has not yet dropped its half. + let _ = probe.shutdown(std::net::Shutdown::Both); + } + if let Some(reader) = &self.reader { + reader.abort(); + } + if let Some(signal) = &self.terminal { + let _ = signal.send(super::child::Control::Stop); + } + } +} + +struct Call { + write: tokio::net::unix::OwnedWriteHalf, + replies: mpsc::Receiver, + sequence: u64, + epoch: u64, + failed: bool, +} +struct Proxy { + call: Mutex, + fail: watch::Sender, +} +// This guard exists only while the caller owns the serialized transaction. +// Drop poisons a partial write/FD/ACK exchange before the mutex is released. +struct InFlightHandoff<'a> { + call: &'a mut Call, + fail: &'a watch::Sender, + acknowledged: bool, +} +impl Drop for InFlightHandoff<'_> { + fn drop(&mut self) { + if !self.acknowledged { + self.call.failed = true; + self.fail.send_replace(true); + } + } +} +impl Proxy { + async fn request(&self, operation: Operation, fd: Option<&UnixStream>) -> io::Result<()> { + use std::os::fd::AsFd; + let mut call = self.call.lock().await; + if call.failed { + return Err(wire::invalid()); + } + call.sequence = call.sequence.checked_add(1).ok_or_else(wire::invalid)?; + let expected = Reply { + version: wire::VERSION, + sequence: call.sequence, + epoch: call.epoch, + state: if matches!(operation, Operation::Bind) { + "ready" + } else { + "accepted" + } + .into(), + }; + let request = Request { + version: wire::VERSION, + sequence: call.sequence, + epoch: call.epoch, + operation, + }; + let mut handoff = InFlightHandoff { + call: &mut call, + fail: &self.fail, + acknowledged: false, + }; + let result = tokio::time::timeout(Duration::from_secs(wire::HANDOFF_SECONDS), async { + wire::write(&mut handoff.call.write, &request).await?; + if let Some(fd) = fd { + let socket = handoff.call.write.as_ref(); + let carrier = + std::os::unix::net::UnixStream::from(socket.as_fd().try_clone_to_owned()?); + socket + .async_io(tokio::io::Interest::WRITABLE, || { + engram_egress_proto::send_fd(&carrier, fd.as_fd()) + }) + .await?; + } + if handoff.call.replies.recv().await != Some(expected) { + return Err(wire::invalid()); + } + Ok(()) + }) + .await + .map_err(|_| wire::invalid()) + .and_then(|r| r); + handoff.acknowledged = result.is_ok(); + result + } + async fn flow(&self, operation: Operation) -> io::Result { + let (local, remote) = UnixStream::pair()?; + self.request(operation, Some(&remote)).await?; + Ok(local) + } +} + +async fn serve( + tcp: &TcpListener, + dns: &UdpSocket, + guest: Ipv4Addr, + gateway: Ipv4Addr, + proxy: Arc, + flows: &mut JoinSet>, +) -> io::Result<()> { + let dns_budget = Arc::new(tokio::sync::Semaphore::new(wire::MAX_DNS_FLOWS)); + let mut bytes = [0; wire::MAX_DNS + 1]; + loop { + tokio::select! { biased; + Some(result) = flows.join_next(), if !flows.is_empty() => { result.map_err(|_| wire::invalid())??; } + connection = tcp.accept() => { + let (stream, peer) = connection?; + if peer.ip() != guest || flows.len() >= wire::MAX_FLOWS { continue; } + let destination = original_destination(&stream)?; + let is_dns = *destination.ip() == gateway && destination.port() == 53; + let permit = if is_dns { match dns_budget.clone().try_acquire_owned() { Ok(p) => Some(p), Err(_) => continue } } else { None }; + let proxy = proxy.clone(); + flows.spawn(async move { + let _permit = permit; + tcp_flow(stream, destination, gateway, proxy).await + }); + } + datagram = dns.recv_from(&mut bytes) => { + let (len, peer) = datagram?; + if peer.ip() != guest || len == 0 || len > wire::MAX_DNS || flows.len() >= wire::MAX_FLOWS { continue; } + let Ok(permit) = dns_budget.clone().try_acquire_owned() else { continue; }; + let query = bytes[..len].to_vec(); + let proxy = proxy.clone(); + // A duplicate of the same namespace-bound socket, never a new + // root-namespace socket or a dial to an upstream destination. + use std::os::fd::AsFd; + let socket = UdpSocket::from_std(std::net::UdpSocket::from(dns.as_fd().try_clone_to_owned()?))?; + flows.spawn(async move { + let _permit = permit; + match dns_exchange(query, gateway, proxy).await { + Ok(reply) => { socket.send_to(&reply, peer).await?; }, + Err(_) => return Ok(()), + } + Ok(()) + }); + } + } + } +} +async fn dns_exchange(query: Vec, gateway: Ipv4Addr, proxy: Arc) -> io::Result> { + tokio::time::timeout(Duration::from_secs(10), async { + if query.is_empty() || query.len() > wire::MAX_DNS { + return Err(wire::invalid()); + } + let mut stream = proxy + .flow(Operation::Dns { + destination: gateway, + port: 53, + }) + .await?; + stream.write_u16(query.len() as u16).await?; + stream.write_all(&query).await?; + let len = stream.read_u16().await? as usize; + if len == 0 || len > wire::MAX_DNS { + return Err(wire::invalid()); + } + let mut reply = vec![0; len]; + stream.read_exact(&mut reply).await?; + Ok(reply) + }) + .await + .map_err(|_| wire::invalid())? +} +async fn tcp_flow( + mut stream: TcpStream, + destination: SocketAddrV4, + gateway: Ipv4Addr, + proxy: Arc, +) -> io::Result<()> { + if *destination.ip() == gateway && destination.port() == 53 { + // One bounded DNS request per TCP connection. The reviewed proxy owns + // DNS decoding and policy; no raw TCP tunnel reaches its resolver. + let result = tokio::time::timeout(Duration::from_secs(10), async { + let len = stream.read_u16().await? as usize; + if len == 0 || len > wire::MAX_DNS { + return Err(wire::invalid()); + } + let mut query = vec![0; len]; + stream.read_exact(&mut query).await?; + let reply = dns_exchange(query, gateway, proxy).await?; + stream.write_u16(reply.len() as u16).await?; + stream.write_all(&reply).await + }) + .await; + let _ = result; + return Ok(()); + } + let mut upstream = proxy + .flow(Operation::Tcp { + destination: *destination.ip(), + port: destination.port(), + }) + .await?; + let _ = tokio::io::copy_bidirectional(&mut stream, &mut upstream).await; + Ok(()) +} +fn checked_destination(address: Ipv4Addr, port: u16) -> io::Result { + if port == 0 || address.is_unspecified() || address.is_multicast() { + return Err(wire::invalid()); + } + Ok(SocketAddrV4::new(address, port)) +} +#[cfg(target_os = "linux")] +fn original_destination(stream: &TcpStream) -> io::Result { + use std::os::fd::AsRawFd; + let mut address: libc::sockaddr_in = unsafe { std::mem::zeroed() }; + let mut len = std::mem::size_of_val(&address) as libc::socklen_t; + // SAFETY: a live socket, correct sockaddr size, and writable output. The + // kernel tuple is mandatory; no local address, SNI port or 443 fallback. + if unsafe { + libc::getsockopt( + stream.as_raw_fd(), + libc::SOL_IP, + 80, + (&mut address as *mut libc::sockaddr_in).cast(), + &mut len, + ) + } != 0 + { + return Err(io::Error::last_os_error()); + } + if len as usize != std::mem::size_of_val(&address) + || address.sin_family != libc::AF_INET as libc::sa_family_t + { + return Err(wire::invalid()); + } + checked_destination( + Ipv4Addr::from(address.sin_addr.s_addr.to_ne_bytes()), + u16::from_be(address.sin_port), + ) +} +#[cfg(not(target_os = "linux"))] +fn original_destination(_stream: &TcpStream) -> io::Result { + Err(io::Error::other("original destination requires Linux")) +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test] + async fn release_requires_exact_epoch_ack_and_never_reuses_unknown_start() { + for wrong_epoch in [false, true] { + let (private, mut peer) = UnixStream::pair().unwrap(); + let tcp = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dns = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let mut adapter = Adapter::spawn( + private, + tcp, + dns, + Ipv4Addr::LOCALHOST, + Ipv4Addr::LOCALHOST, + None, + ) + .unwrap(); + let server = tokio::spawn(async move { + let bind: Request = wire::read(&mut peer).await.unwrap(); + assert_eq!(bind.epoch, 7); + assert_eq!(bind.sequence, 1); + assert!(matches!(bind.operation, Operation::Bind)); + wire::write( + &mut peer, + &Reply { + version: 1, + sequence: 1, + epoch: if wrong_epoch { 8 } else { 7 }, + state: "ready".into(), + }, + ) + .await + .unwrap(); + peer + }); + let result = adapter.release(7).await; + let mut peer = server.await.unwrap(); + if wrong_epoch { + assert!(result.is_err()); + } else { + result.unwrap(); + adapter.check_ready(7).unwrap(); + adapter.release(7).await.unwrap(); + assert!(adapter.release(8).await.is_err()); + assert!(adapter.check_closed().is_err()); + } + adapter.close().await.unwrap(); + assert_eq!(peer.read(&mut [0]).await.unwrap(), 0); + assert!(adapter.check_ready(7).is_err()); + } + } + + #[tokio::test] + async fn cancelled_release_cannot_reissue_bind() { + let (private, mut peer) = UnixStream::pair().unwrap(); + let tcp = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dns = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let mut adapter = Adapter::spawn( + private, + tcp, + dns, + Ipv4Addr::LOCALHOST, + Ipv4Addr::LOCALHOST, + None, + ) + .unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(20), adapter.release(7)) + .await + .is_err() + ); + let bind: Request = wire::read(&mut peer).await.unwrap(); + assert!(matches!(bind.operation, Operation::Bind)); + assert!(adapter.release(7).await.is_err()); + adapter.close().await.unwrap(); + assert_eq!(peer.read(&mut [0]).await.unwrap(), 0); + } + #[tokio::test] + async fn closed_adapter_emits_nothing_and_proxy_loss_is_terminal() { + let (private, mut peer) = UnixStream::pair().unwrap(); + let tcp = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = tcp.local_addr().unwrap(); + let dns = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let mut adapter = Adapter::spawn( + private, + tcp, + dns, + Ipv4Addr::LOCALHOST, + Ipv4Addr::LOCALHOST, + None, + ) + .unwrap(); + let mut client = TcpStream::connect(address).await.unwrap(); + client.write_all(b"untrusted").await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(30), peer.read_u8()) + .await + .is_err() + ); + peer.shutdown().await.unwrap(); + tokio::time::timeout(Duration::from_millis(200), async { + loop { + if adapter.check_closed().is_err() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(peer.read(&mut [0]).await.unwrap(), 0); + assert!(adapter.check_closed().is_err()); + // The worker has already completed; close can still join that result. + adapter.close().await.unwrap(); + } + #[tokio::test] + async fn repeated_close_preserves_cleanup_result() { + let (private, _peer) = UnixStream::pair().unwrap(); + let tcp = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dns = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let mut adapter = Adapter::spawn( + private, + tcp, + dns, + Ipv4Addr::LOCALHOST, + Ipv4Addr::LOCALHOST, + None, + ) + .unwrap(); + adapter.close().await.unwrap(); + adapter.close().await.unwrap(); + assert!(adapter.check_closed().is_err()); + } + #[tokio::test] + async fn dns_budget_and_stop_join_all_owned_flows_before_proxy_eof() { + let (private, mut peer) = UnixStream::pair().unwrap(); + let tcp = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dns = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let address = dns.local_addr().unwrap(); + let mut adapter = Adapter::spawn( + private, + tcp, + dns, + Ipv4Addr::LOCALHOST, + Ipv4Addr::LOCALHOST, + None, + ) + .unwrap(); + adapter._release.take().unwrap().send(7).unwrap(); + let bind: Request = wire::read(&mut peer).await.unwrap(); + assert!(matches!(bind.operation, Operation::Bind)); + wire::write( + &mut peer, + &Reply { + version: 1, + sequence: 1, + epoch: 7, + state: "ready".into(), + }, + ) + .await + .unwrap(); + let guest = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let mut handed = Vec::new(); + for i in 0..wire::MAX_DNS_FLOWS { + guest.send_to(b"query", address).await.unwrap(); + let request: Request = wire::read(&mut peer).await.unwrap(); + assert_eq!(request.sequence, i as u64 + 2); + let mut stream = wire::receive_stream(&peer).await.unwrap(); + wire::write( + &mut peer, + &Reply { + version: 1, + sequence: request.sequence, + epoch: 7, + state: "accepted".into(), + }, + ) + .await + .unwrap(); + assert_eq!(stream.read_u16().await.unwrap(), 5); + let mut query = [0; 5]; + stream.read_exact(&mut query).await.unwrap(); + assert_eq!(&query, b"query"); + handed.push(stream); + } + guest.send_to(b"ninth", address).await.unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(30), peer.read_u8()) + .await + .is_err() + ); + peer.shutdown().await.unwrap(); + tokio::time::timeout(Duration::from_millis(200), async { + assert_eq!(peer.read(&mut [0]).await.unwrap(), 0); + for mut stream in handed { + assert_eq!(stream.read(&mut [0]).await.unwrap(), 0); + } + }) + .await + .unwrap(); + adapter.close().await.unwrap(); + } + #[test] + fn runtime_loss_invalidates_even_unpolled_owners_and_closes_private_probe() { + use std::io::Read; + for poll_once in [false, true] { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (terminal, stopped) = std::sync::mpsc::channel(); + let (adapter, mut peer, retained_duplicate) = runtime.block_on(async { + let (private, peer) = UnixStream::pair().unwrap(); + use std::os::fd::AsFd; + // Hold a duplicate past runtime destruction. Terminal control + // must not depend on the order of independent FD destructors. + let duplicate = private.as_fd().try_clone_to_owned().unwrap(); + let tcp = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dns = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let adapter = Adapter::spawn( + private, + tcp, + dns, + Ipv4Addr::LOCALHOST, + Ipv4Addr::LOCALHOST, + Some(terminal), + ) + .unwrap(); + if poll_once { + tokio::task::yield_now().await; + } + (adapter, peer.into_std().unwrap(), duplicate) + }); + drop(runtime); + assert!(adapter.check_closed().is_err()); + stopped.recv_timeout(Duration::from_millis(100)).unwrap(); + assert_eq!(peer.read(&mut [0]).unwrap(), 0); + drop(retained_duplicate); + } + } + #[tokio::test] + async fn partial_header_is_terminal_before_frame_timeout() { + let (private, mut peer) = UnixStream::pair().unwrap(); + let tcp = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dns = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let adapter = Adapter::spawn( + private, + tcp, + dns, + Ipv4Addr::LOCALHOST, + Ipv4Addr::LOCALHOST, + None, + ) + .unwrap(); + tokio::task::yield_now().await; + peer.write_all(&[0]).await.unwrap(); + tokio::time::timeout(Duration::from_millis(100), async { + loop { + if adapter.check_closed().is_err() { + break; + } + tokio::task::yield_now().await; + } + assert_eq!(peer.read(&mut [0]).await.unwrap(), 0); + }) + .await + .unwrap(); + } + fn fixture_proxy(stream: UnixStream) -> (Arc, tokio::task::JoinHandle<()>) { + let (mut read, write) = stream.into_split(); + let (tx, replies) = mpsc::channel(1); + let (fail, _) = watch::channel(false); + let reader = tokio::spawn(async move { + while let Ok(reply) = wire::read::<_, Reply>(&mut read).await { + if tx.send(reply).await.is_err() { + break; + } + } + }); + ( + Arc::new(Proxy { + call: Mutex::new(Call { + write, + replies, + sequence: 0, + epoch: 7, + failed: false, + }), + fail, + }), + reader, + ) + } + #[tokio::test] + async fn handoff_cancellation_after_fd_is_terminal_and_cannot_reuse_stream() { + let (private, mut peer) = UnixStream::pair().unwrap(); + let (proxy, reader) = fixture_proxy(private); + let operation = || Operation::Dns { + destination: "192.0.2.1".parse().unwrap(), + port: 53, + }; + let pending = tokio::spawn({ + let proxy = proxy.clone(); + async move { proxy.flow(operation()).await } + }); + let request: Request = wire::read(&mut peer).await.unwrap(); + let mut handed = wire::receive_stream(&peer).await.unwrap(); + // The peer owns the FD, but has not acknowledged the transaction. + // Aborting drops the same future that an outer DNS deadline drops. + pending.abort(); + assert!(pending.await.unwrap_err().is_cancelled()); + assert!( + *proxy.fail.borrow(), + "canceled handoff must signal terminal cleanup" + ); + assert!(proxy.call.lock().await.failed); + assert_eq!(handed.read(&mut [0]).await.unwrap(), 0); + // A late exact ACK cannot recover an abandoned transaction. + wire::write( + &mut peer, + &Reply { + version: wire::VERSION, + sequence: request.sequence, + epoch: request.epoch, + state: "accepted".into(), + }, + ) + .await + .unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(100), proxy.flow(operation())) + .await + .unwrap() + .is_err() + ); + assert_eq!(proxy.call.lock().await.sequence, request.sequence); + assert_eq!( + peer.try_read(&mut [0]).unwrap_err().kind(), + io::ErrorKind::WouldBlock + ); + reader.abort(); + } + #[tokio::test] + async fn handoff_cancellation_while_waiting_for_mutex_does_not_poison_stream() { + use std::{future::Future, task::Poll}; + let (private, mut peer) = UnixStream::pair().unwrap(); + let (proxy, reader) = fixture_proxy(private); + let operation = || Operation::Dns { + destination: "192.0.2.1".parse().unwrap(), + port: 53, + }; + let first = tokio::spawn({ + let proxy = proxy.clone(); + async move { proxy.flow(operation()).await } + }); + let request: Request = wire::read(&mut peer).await.unwrap(); + let handed = wire::receive_stream(&peer).await.unwrap(); + let mut waiting = Box::pin(proxy.flow(operation())); + std::future::poll_fn(|cx| { + assert!(waiting.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + drop(waiting); + assert!(!*proxy.fail.borrow()); + wire::write( + &mut peer, + &Reply { + version: wire::VERSION, + sequence: request.sequence, + epoch: request.epoch, + state: "accepted".into(), + }, + ) + .await + .unwrap(); + let first_stream = first.await.unwrap().unwrap(); + assert!(!proxy.call.lock().await.failed); + assert!(!*proxy.fail.borrow()); + let next = tokio::spawn({ + let proxy = proxy.clone(); + async move { proxy.flow(operation()).await } + }); + let next_request: Request = wire::read(&mut peer).await.unwrap(); + assert_eq!(next_request.sequence, request.sequence + 1); + let next_handed = wire::receive_stream(&peer).await.unwrap(); + wire::write( + &mut peer, + &Reply { + version: wire::VERSION, + sequence: next_request.sequence, + epoch: next_request.epoch, + state: "accepted".into(), + }, + ) + .await + .unwrap(); + let next_stream = next.await.unwrap().unwrap(); + assert!(!*proxy.fail.borrow()); + drop((first_stream, handed, next_stream, next_handed)); + reader.abort(); + } + #[tokio::test] + async fn tcp_bridge_preserves_original_nonstandard_port_and_transfers_only_private_stream() { + let (private, mut peer) = UnixStream::pair().unwrap(); + let (proxy, reader) = fixture_proxy(private); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut guest = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (stream, _) = listener.accept().await.unwrap(); + let bridge = tokio::spawn(tcp_flow( + stream, + "198.51.100.9:8443".parse().unwrap(), + "192.0.2.1".parse().unwrap(), + proxy, + )); + let request: Request = wire::read(&mut peer).await.unwrap(); + assert_eq!((request.sequence, request.epoch), (1, 7)); + assert!( + matches!(request.operation, Operation::Tcp { destination, port: 8443 } if destination.to_string() == "198.51.100.9") + ); + let mut handed = wire::receive_stream(&peer).await.unwrap(); + wire::write( + &mut peer, + &Reply { + version: 1, + sequence: 1, + epoch: 7, + state: "accepted".into(), + }, + ) + .await + .unwrap(); + guest.write_all(b"guest-bytes").await.unwrap(); + let mut bytes = [0; 11]; + handed.read_exact(&mut bytes).await.unwrap(); + assert_eq!(&bytes, b"guest-bytes"); + handed.write_all(b"reply").await.unwrap(); + let mut bytes = [0; 5]; + guest.read_exact(&mut bytes).await.unwrap(); + assert_eq!(&bytes, b"reply"); + guest.shutdown().await.unwrap(); + handed.shutdown().await.unwrap(); + bridge.await.unwrap().unwrap(); + reader.abort(); + } + #[tokio::test] + async fn dns_is_length_bounded_and_handoff_mismatch_is_terminal() { + let (private, mut peer) = UnixStream::pair().unwrap(); + let (proxy, reader) = fixture_proxy(private); + assert!(dns_exchange( + vec![0; wire::MAX_DNS + 1], + "192.0.2.1".parse().unwrap(), + proxy.clone() + ) + .await + .is_err()); + assert!( + tokio::time::timeout(Duration::from_millis(20), peer.read_u8()) + .await + .is_err() + ); + let flow = tokio::spawn({ + let proxy = proxy.clone(); + async move { dns_exchange(b"query".to_vec(), "192.0.2.1".parse().unwrap(), proxy).await } + }); + let request: Request = wire::read(&mut peer).await.unwrap(); + assert!( + matches!(request.operation, Operation::Dns { destination, port: 53 } if destination.to_string() == "192.0.2.1") + ); + let mut handed = wire::receive_stream(&peer).await.unwrap(); + // The wrong epoch must poison this transport, even for a DNS failure. + wire::write( + &mut peer, + &Reply { + version: 1, + sequence: 1, + epoch: 8, + state: "accepted".into(), + }, + ) + .await + .unwrap(); + assert!(flow.await.unwrap().is_err()); + assert!(*proxy.fail.borrow()); + assert!(proxy + .flow(Operation::Dns { + destination: "192.0.2.1".parse().unwrap(), + port: 53 + }) + .await + .is_err()); + assert_eq!(handed.read(&mut [0]).await.unwrap(), 0); + reader.abort(); + } + #[test] + fn mandatory_destination_never_defaults_the_port() { + assert!(checked_destination(Ipv4Addr::UNSPECIFIED, 443).is_err()); + assert!(checked_destination("198.51.100.9".parse().unwrap(), 0).is_err()); + assert_eq!( + checked_destination("198.51.100.9".parse().unwrap(), 8443) + .unwrap() + .port(), + 8443 + ); + } +} diff --git a/crates/engram-sandbox-firecracker/src/managed/child.rs b/crates/engram-sandbox-firecracker/src/managed/child.rs new file mode 100644 index 000000000..cf4e265c4 --- /dev/null +++ b/crates/engram-sandbox-firecracker/src/managed/child.rs @@ -0,0 +1,323 @@ +//! Process cleanup runs on a reserved native thread, including retained files. +use std::{ + io, + process::{Child, Command}, + sync::{mpsc, Arc, Mutex}, +}; +pub(super) enum Control { + Spawn(Box, mpsc::SyncSender>), + Stop, +} + +struct State { + child: Option, + closed: bool, +} +pub(super) struct ChildOwner { + state: Arc>, + stop: mpsc::Sender, + done: tokio::sync::oneshot::Receiver>, + result: Option>, +} +impl ChildOwner { + pub fn reserve(retained: impl Send + 'static) -> io::Result { + let state = Arc::new(Mutex::new(State { + child: None, + closed: false, + })); + let worker = state.clone(); + let (stop, rx) = mpsc::channel::(); + let (tx, done) = tokio::sync::oneshot::channel(); + std::thread::Builder::new() + .name("managed-fc-cleanup".into()) + .spawn(move || { + while let Ok(Control::Spawn(mut command, reply)) = rx.recv() { + let mut state = worker.lock().expect("child lock"); + let result = if state.closed || state.child.is_some() { + Err(io::Error::other("child owner already used")) + } else { + // This reserved thread stays alive through exact-child wait. + // Linux ties PDEATHSIG to the spawning thread, not its process. + #[cfg(target_os = "linux")] + unsafe { + use std::os::unix::process::CommandExt; + let parent = libc::getpid(); + // Append to the namespace hook. Only syscalls and errno + // reads are allowed between fork and exec. + command.pre_exec(move || { + if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) != 0 { + return Err(io::Error::last_os_error()); + } + if libc::getppid() != parent { + return Err(io::Error::from_raw_os_error(libc::ECHILD)); + } + Ok(()) + }); + } + command.spawn().map(|child| state.child = Some(child)) + }; + let _ = reply.send(result); + } + // Reject queued spawns before potentially slow retained-file cleanup. + drop(rx); + let mut child = { + let mut state = worker.lock().expect("child lock"); + state.closed = true; + state.child.take() + }; + let result = if let Some(child) = &mut child { + match child.try_wait() { + Ok(Some(_)) => Ok(()), + Ok(None) => { + let kill = child.kill(); + if kill + .as_ref() + .is_err_and(|e| e.raw_os_error() != Some(libc::ESRCH)) + { + kill + } else { + child.wait().map(|_| ()) + } + } + Err(e) => Err(e), + } + } else { + Ok(()) + }; + if result.is_err() { + // A failed cleanup cannot release files/namespace or authorize + // reuse. Preserve the ambiguous owner and report failure. + std::mem::forget((child, retained)); + tracing::error!("managed VM cleanup failed"); + } else { + drop(child); + drop(retained); + } + let _ = tx.send(result); + })?; + Ok(Self { + state, + stop, + done, + result: None, + }) + } + pub fn spawn(&mut self, command: Command) -> io::Result<()> { + let (reply, result) = mpsc::sync_channel(1); + self.stop + .send(Control::Spawn(Box::new(command), reply)) + .map_err(|_| io::Error::other("child owner closed"))?; + result + .recv() + .map_err(|_| io::Error::other("child owner failed"))? + } + pub fn live_pid(&self) -> io::Result { + let mut state = self + .state + .lock() + .map_err(|_| io::Error::other("child owner failed"))?; + let child = state + .child + .as_mut() + .ok_or_else(|| io::Error::other("VM closed"))?; + if child.try_wait()?.is_some() { + return Err(io::Error::other("VM exited")); + } + // The sole Child owner is the only reaper. An exit after this read + // leaves a zombie, whose PID cannot be reused by another process. + Ok(child.id()) + } + pub fn stop_signal(&self) -> mpsc::Sender { + self.stop.clone() + } + pub async fn close(&mut self) -> io::Result<()> { + let _ = self.stop.send(Control::Stop); + if let Some(result) = &self.result { + return result.clone().map_err(io::Error::other); + } + let result = tokio::time::timeout(std::time::Duration::from_secs(5), &mut self.done) + .await + .map_err(|_| io::Error::other("VM cleanup timed out")) + .and_then(|r| r.map_err(|_| io::Error::other("VM cleanup owner lost"))) + .and_then(|r| r); + self.result = Some(result.as_ref().map(|_| ()).map_err(ToString::to_string)); + result + } +} +impl Drop for ChildOwner { + fn drop(&mut self) { + let _ = self.stop.send(Control::Stop); + } +} + +#[cfg(test)] +mod tests { + use super::*; + /// Uses the real owner in a separate process so SIGKILL cannot run Drop. + #[cfg(target_os = "linux")] + #[test] + fn abrupt_parent_death_stops_exact_child_after_spawning_thread_exits() { + use std::{ + os::fd::{AsRawFd, FromRawFd, OwnedFd}, + time::{Duration, Instant}, + }; + const HELPER: &str = "ENGRAM_CHILD_OWNER_TEST"; + if let Some(path) = std::env::var_os("ENGRAM_CHILD_OWNER_LEAF") { + use std::io::Read; + std::io::stdin().read_exact(&mut [0]).unwrap(); + std::fs::write(path, b"ready").unwrap(); + loop { + std::thread::park(); + } + } + if let Some(path) = std::env::var_os(HELPER) { + let leaf = std::path::PathBuf::from(&path).with_extension("leaf"); + let proof = leaf.clone(); + let owner = std::thread::spawn(move || { + let mut owner = ChildOwner::reserve(()).unwrap(); + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", "managed::child::tests::abrupt_parent_death_stops_exact_child_after_spawning_thread_exits"]) + .env("ENGRAM_CHILD_OWNER_LEAF", proof) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()); + owner.spawn(command).unwrap(); + owner + }).join().unwrap(); + // A parent-death signal tied to the caller's short-lived thread + // would stop the child here while the actual owner is still live. + use std::io::Write; + owner + .state + .lock() + .unwrap() + .child + .as_mut() + .unwrap() + .stdin + .as_mut() + .unwrap() + .write_all(b"?") + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !leaf.exists() { + owner.live_pid().unwrap(); + assert!( + Instant::now() < deadline, + "child did not answer after caller thread exit" + ); + std::thread::sleep(Duration::from_millis(10)); + } + std::fs::write(path, owner.live_pid().unwrap().to_string()).unwrap(); + loop { + std::thread::park(); + } + } + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("child"); + let mut parent = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "managed::child::tests::abrupt_parent_death_stops_exact_child_after_spawning_thread_exits", "--nocapture"]) + .env(HELPER, &path).spawn().unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !path.exists() { + assert!( + parent.try_wait().unwrap().is_none(), + "parent exited before child readiness" + ); + if Instant::now() >= deadline { + parent.kill().unwrap(); + parent.wait().unwrap(); + panic!("child readiness timed out"); + } + std::thread::sleep(Duration::from_millis(10)); + } + let pid: i32 = std::fs::read_to_string(path).unwrap().parse().unwrap(); + // The parent owns the unreaped child while we acquire its exact handle. + let fd = unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }; + assert!(fd >= 0, "pidfd_open: {}", io::Error::last_os_error()); + let child = unsafe { OwnedFd::from_raw_fd(fd as i32) }; + let mut event = libc::pollfd { + fd: child.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + assert_eq!( + unsafe { libc::poll(&mut event, 1, 0) }, + 0, + "child must survive caller thread exit" + ); + parent.kill().unwrap(); + use std::os::unix::process::ExitStatusExt; + assert_eq!(parent.wait().unwrap().signal(), Some(libc::SIGKILL)); + let exited = unsafe { libc::poll(&mut event, 1, 2000) }; + // Clean the negative control through the exact handle, never a PID scan. + if exited != 1 { + unsafe { + libc::syscall( + libc::SYS_pidfd_send_signal, + child.as_raw_fd(), + libc::SIGKILL, + std::ptr::null::(), + 0, + ); + } + } + assert_eq!(exited, 1, "child survived abrupt owner death"); + } + + #[tokio::test(flavor = "current_thread")] + async fn retained_cleanup_does_not_block_unrelated_private_stop_or_cancel_ownership() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + struct Held { + wait: mpsc::Receiver<()>, + entered: mpsc::Sender<()>, + } + impl Drop for Held { + fn drop(&mut self) { + let _ = self.entered.send(()); + let _ = self.wait.recv(); + } + } + let (release, wait) = mpsc::channel(); + let (entered, entry) = mpsc::channel(); + let mut owner = ChildOwner::reserve(Held { wait, entered }).unwrap(); + let mut command = Command::new("/bin/cat"); + command + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::null()); + owner.spawn(command).unwrap(); + owner.live_pid().unwrap(); + let stop = owner.stop_signal(); + stop.send(Control::Stop).unwrap(); + tokio::time::timeout(std::time::Duration::from_millis(200), async { + loop { + if entry.try_recv().is_ok() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(owner.live_pid().is_err()); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), owner.close()) + .await + .is_err() + ); + let (private, mut peer) = tokio::net::UnixStream::pair().unwrap(); + peer.shutdown().await.unwrap(); + tokio::time::timeout( + std::time::Duration::from_millis(100), + super::super::wait_for_stop(&private), + ) + .await + .unwrap(); + drop(private); + assert_eq!(peer.read(&mut [0]).await.unwrap(), 0); + release.send(()).unwrap(); + owner.close().await.unwrap(); + owner.close().await.unwrap(); + assert!(owner.spawn(Command::new("/bin/cat")).is_err()); + } +} diff --git a/crates/engram-sandbox-firecracker/src/managed/configure.rs b/crates/engram-sandbox-firecracker/src/managed/configure.rs new file mode 100644 index 000000000..50274bd3a --- /dev/null +++ b/crates/engram-sandbox-firecracker/src/managed/configure.rs @@ -0,0 +1,193 @@ +use super::network::{NetworkShape, GUEST_MAC, TAP}; +use crate::{FirecrackerClient, FirecrackerConfig}; +use engram_core::{ + types::{managed_cold::ManagedColdPrepare, sandbox::SandboxSpec}, + SandboxError, +}; +use std::path::Path; + +pub(super) async fn configure( + api: &FirecrackerClient, + cfg: &FirecrackerConfig, + request: &ManagedColdPrepare, + spec: &SandboxSpec, + path: &Path, + shape: &NetworkShape, + cid: u32, +) -> Result<(), SandboxError> { + use crate::{BootSource, DriveConfig, MachineConfig, NetworkInterface, VsockConfig}; + api.put_machine_config(&MachineConfig { + vcpu_count: spec + .cpu + .vcpus + .try_into() + .map_err(|_| SandboxError::InvalidSpec("invalid CPU count".into()))?, + mem_size_mib: spec.memory.max_mib, + smt: false, + track_dirty_pages: cfg.track_dirty_pages, + cpu_template: cfg.cpu_template.clone(), + }) + .await?; + // Reject a second kernel networking declaration. Guest configuration gives + // function; the namespace and nft rules provide enforcement. + if cfg.default_boot_args.split_whitespace().any(|arg| { + arg.starts_with("ip=") + || arg.starts_with("engram_managed_dns=") + || arg == "engram_base_capture=1" + }) { + return Err(SandboxError::InvalidSpec( + "managed boot arguments conflict with selected network".into(), + )); + } + let boot_args = format!( + "{} ip={}::{}:255.255.255.0::eth0:off:{} engram_managed_dns={}", + cfg.default_boot_args.trim_end(), + shape.guest, + shape.gateway, + shape.gateway, + shape.gateway + ); + api.put_boot_source(&BootSource { + kernel_image_path: path.join("kernel").to_string_lossy().into_owned(), + boot_args, + initrd_path: None, + }) + .await?; + api.put_drive(&DriveConfig { + drive_id: "rootfs".into(), + path_on_host: path.join("rootfs.ext4").to_string_lossy().into_owned(), + is_root_device: true, + is_read_only: false, + }) + .await?; + for drive in &request.aux_ro_drives { + let hash = drive + .sha256 + .as_ref() + .ok_or_else(|| SandboxError::InvalidSpec("unresolved boot drive".into()))?; + api.put_drive(&DriveConfig { + drive_id: drive.drive_id.clone(), + path_on_host: path + .join(format!("{hash}.squashfs")) + .to_string_lossy() + .into_owned(), + is_root_device: false, + is_read_only: true, + }) + .await?; + } + if spec.swap_mib.is_some_and(|n| n > 0) { + api.put_drive(&DriveConfig { + drive_id: "swap".into(), + path_on_host: path.join("swap.img").to_string_lossy().into_owned(), + is_root_device: false, + is_read_only: false, + }) + .await?; + } + if cfg.balloon { + api.put_balloon(&crate::client::BalloonConfig { + amount_mib: 0, + deflate_on_oom: true, + stats_polling_interval_s: 1, + }) + .await?; + } + api.put_network_interface(&NetworkInterface { + iface_id: "eth0".into(), + host_dev_name: TAP.into(), + guest_mac: Some(GUEST_MAC.into()), + }) + .await?; + api.put_vsock(&VsockConfig { + guest_cid: cid, + uds_path: path.join("guest.vsock").to_string_lossy().into_owned(), + }) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + #[tokio::test] + async fn cold_configuration_never_starts_or_loads_and_uses_exact_private_files() { + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, spec) = super::super::files::tests::inputs(dir.path()); + let socket = dir.path().join("api.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let (tx, mut rx) = tokio::sync::mpsc::channel(16); + let server = tokio::spawn(async move { + loop { + let Ok((mut connection, _)) = listener.accept().await else { + break; + }; + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(connection.read_u8().await.unwrap()); + } + let head = String::from_utf8(head).unwrap(); + let length: usize = head + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length:") + .map(|s| s.trim().parse().unwrap()) + }) + .unwrap_or(0); + let mut body = vec![0; length]; + connection.read_exact(&mut body).await.unwrap(); + tx.send(( + head.lines().next().unwrap().to_string(), + serde_json::from_slice::(&body).unwrap(), + )) + .await + .unwrap(); + connection + .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n") + .await + .unwrap(); + } + }); + let shape = + NetworkShape::new("192.0.2.2".parse().unwrap(), "192.0.2.1".parse().unwrap()).unwrap(); + configure( + &FirecrackerClient::new(socket), + &cfg, + &request, + &spec, + dir.path(), + &shape, + 42, + ) + .await + .unwrap(); + let mut sent = Vec::new(); + while let Ok(item) = rx.try_recv() { + sent.push(item); + } + server.abort(); + assert_eq!( + sent.len(), + 8, + "cold prepare must configure all devices without start/load" + ); + assert!(sent.iter().all(|(line, _)| !line.contains("/actions") + && !line.contains("/snapshot") + && !line.contains("PATCH"))); + assert_eq!( + sent[1].1["kernel_image_path"], + dir.path().join("kernel").to_str().unwrap() + ); + assert_eq!( + sent[2].1["path_on_host"], + dir.path().join("rootfs.ext4").to_str().unwrap() + ); + assert_eq!(sent[6].1["guest_mac"], "02:00:00:00:00:02"); + assert!(sent[1].1["boot_args"] + .as_str() + .unwrap() + .contains("ip=192.0.2.2::192.0.2.1:255.255.255.0::eth0:off:192.0.2.1")); + } +} diff --git a/crates/engram-sandbox-firecracker/src/managed/disk.rs b/crates/engram-sandbox-firecracker/src/managed/disk.rs new file mode 100644 index 000000000..e68508a81 --- /dev/null +++ b/crates/engram-sandbox-firecracker/src/managed/disk.rs @@ -0,0 +1,462 @@ +//! Session volumes retained by the managed child owner until the VM is reaped. +use std::{io, path::Path, sync::Arc}; + +use engram_core::{types::snapshot::DiskRole, SandboxId}; +use engram_host_core::{ + disk::{ChunkIndex, SessionDisk, Volume, VolumeName}, + session_volumes::{live_name, transient_name}, +}; +use sha2::{Digest, Sha256}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +pub(super) struct Disks { + disk: Arc, + sandbox: SandboxId, + owned: tokio::sync::Mutex>, + runtime: tokio::runtime::Handle, + retain: std::sync::atomic::AtomicBool, +} +impl Disks { + pub fn new(disk: Arc, sandbox: SandboxId) -> Self { + Self { + disk, + sandbox, + owned: tokio::sync::Mutex::new(Vec::new()), + runtime: tokio::runtime::Handle::current(), + retain: std::sync::atomic::AtomicBool::new(false), + } + } + + pub fn retain_source(&self) { + self.retain + .store(true, std::sync::atomic::Ordering::Release); + } + + async fn create(&self, role: DiskRole, bytes: u64) -> io::Result { + use std::os::unix::fs::FileTypeExt; + let name = live_name(self.sandbox, role); + let volume = self.disk.create(&name, bytes).await?; + self.owned.lock().await.push(name); + if !std::fs::metadata(&volume.path)? + .file_type() + .is_block_device() + { + return Err(io::Error::other( + "managed Firecracker requires a block session volume", + )); + } + Ok(volume) + } + + pub async fn install( + &self, + directory: &Path, + sha256: &str, + bytes: u64, + swap_mib: Option, + ) -> io::Result<()> { + let result = async { + let root = self.create(DiskRole::Root, bytes).await?; + import(&root, &directory.join("selected.ext4"), sha256).await?; + std::os::unix::fs::symlink(&root.path, directory.join("rootfs.ext4"))?; + if let Some(mib) = swap_mib.filter(|n| *n > 0) { + let swap = self + .create(DiskRole::Swap, u64::from(mib) * 1024 * 1024) + .await?; + swap.sync().await?; + std::os::unix::fs::symlink(&swap.path, directory.join("swap.img"))?; + } + std::fs::remove_file(directory.join("selected.ext4"))?; + Ok(()) + } + .await; + if result.is_err() { + self.release().await?; + } + result + } + + /// The caller has stopped and reaped the VM before taking this disk cut. + pub async fn export(&self, destination: &Path) -> io::Result<()> { + self.retain + .store(true, std::sync::atomic::Ordering::Release); + let root = self + .disk + .open(&live_name(self.sandbox, DiskRole::Root)) + .await?; + root.sync().await?; + let name = transient_name(self.sandbox, DiskRole::Root); + let snapshot = self.disk.snapshot(&root, &name).await?; + self.owned.lock().await.push(name.clone()); + export(&snapshot, destination).await?; + drop(snapshot); + self.disk.delete(&name).await?; + self.owned.lock().await.retain(|n| n != &name); + self.retain + .store(false, std::sync::atomic::Ordering::Release); + Ok(()) + } + + pub async fn release(&self) -> io::Result<()> { + if self.retain.load(std::sync::atomic::Ordering::Acquire) { + return Err(io::Error::other( + "managed disk retained after incomplete capture", + )); + } + let mut owned = self.owned.lock().await; + while let Some(name) = owned.last() { + self.disk.delete(name).await?; + owned.pop(); + } + Ok(()) + } +} +impl Drop for Disks { + fn drop(&mut self) { + if self.retain.load(std::sync::atomic::Ordering::Acquire) { + return; + } + let names = std::mem::take(self.owned.get_mut()); + let disk = self.disk.clone(); + self.runtime.spawn(async move { + for name in names.into_iter().rev() { + if let Err(error) = disk.delete(&name).await { + tracing::warn!(volume = %name, %error, "managed disk cleanup incomplete"); + } + } + }); + } +} + +async fn import(volume: &Volume, source: &Path, expected_hash: &str) -> io::Result<()> { + let mut source = tokio::fs::File::open(source).await?; + let metadata = source.metadata().await?; + if !metadata.is_file() || metadata.len() != volume.size_bytes { + return Err(io::Error::other("managed disk source size or type changed")); + } + let mut hash = Sha256::new(); + for index in (0..volume.chunk_count()).map(ChunkIndex) { + let mut bytes = vec![0; volume.chunk_len(index)?]; + source.read_exact(&mut bytes).await?; + hash.update(&bytes); + if bytes.iter().any(|b| *b != 0) { + volume.write_chunk(index, bytes).await?; + } + } + if source.read(&mut [0]).await? != 0 || hex::encode(hash.finalize()) != expected_hash { + return Err(io::Error::other("managed disk source digest changed")); + } + volume.sync().await +} + +async fn export(volume: &Volume, destination: &Path) -> io::Result<()> { + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(destination) + .await?; + for index in (0..volume.chunk_count()).map(ChunkIndex) { + file.write_all(&volume.read_chunk(index).await?).await?; + } + file.sync_all().await?; + tokio::fs::File::open( + destination + .parent() + .ok_or_else(|| io::Error::other("capture parent missing"))?, + ) + .await? + .sync_all() + .await +} + +#[cfg(test)] +mod tests { + use super::*; + use engram_host_core::FileDisk; + use std::os::unix::fs::PermissionsExt; + + struct BusyDisk { + inner: FileDisk, + busy: std::sync::atomic::AtomicBool, + pause_delete: Option>, + } + #[async_trait::async_trait] + impl SessionDisk for BusyDisk { + async fn create(&self, name: &VolumeName, bytes: u64) -> io::Result { + self.inner.create(name, bytes).await + } + async fn snapshot(&self, origin: &Volume, name: &VolumeName) -> io::Result { + self.inner.snapshot(origin, name).await + } + async fn open(&self, name: &VolumeName) -> io::Result { + self.inner.open(name).await + } + async fn delete(&self, name: &VolumeName) -> io::Result<()> { + if let Some(pause) = &self.pause_delete { + pause.notified().await; + } + if self.busy.load(std::sync::atomic::Ordering::Acquire) { + Err(io::ErrorKind::ResourceBusy.into()) + } else { + self.inner.delete(name).await + } + } + async fn list(&self) -> io::Result> { + self.inner.list().await + } + async fn rename(&self, from: &VolumeName, to: &VolumeName) -> io::Result<()> { + self.inner.rename(from, to).await + } + async fn capacity(&self) -> io::Result { + self.inner.capacity().await + } + } + + #[tokio::test] + async fn started_normal_destroy_releases_disks_but_uncertain_capture_does_not() { + for (destroy, uncertain, capture_failed, busy, released) in [ + (true, false, false, false, true), + (false, false, false, false, false), + (true, true, false, false, false), + (true, false, true, false, false), + (true, false, false, true, false), + ] { + let dir = tempfile::tempdir().unwrap(); + let disk = Arc::new(BusyDisk { + inner: FileDisk::new(dir.path().join("volumes"), 4096).unwrap(), + busy: std::sync::atomic::AtomicBool::new(busy), + pause_delete: None, + }); + let sandbox = SandboxId::new(); + let root = live_name(sandbox, DiskRole::Root); + disk.create(&root, 4096).await.unwrap(); + let disks = Disks::new(disk.clone(), sandbox); + disks.owned.lock().await.push(root.clone()); + if capture_failed { + disks.retain_source(); + } + let files = super::super::files::PinnedFiles { + path: dir.path().join("private"), + retain: std::sync::atomic::AtomicBool::new(true), + capture_ready: std::sync::atomic::AtomicBool::new(false), + disks: Some(disks), + }; + assert_eq!(files.close(destroy, uncertain).await.is_ok(), released); + drop(files); + assert_eq!(disk.open(&root).await.is_err(), released); + } + } + + #[tokio::test] + async fn exported_capture_keeps_source_until_host_records_its_hash_and_size() { + let dir = tempfile::tempdir().unwrap(); + let disk = Arc::new(FileDisk::new(dir.path().join("volumes"), 4096).unwrap()); + let sandbox = SandboxId::new(); + let root = live_name(sandbox, DiskRole::Root); + disk.create(&root, 4096).await.unwrap(); + let disks = Disks::new(disk.clone(), sandbox); + disks.owned.lock().await.push(root.clone()); + let files = super::super::files::PinnedFiles { + path: dir.path().join("private"), + retain: std::sync::atomic::AtomicBool::new(true), + capture_ready: std::sync::atomic::AtomicBool::new(false), + disks: Some(disks), + }; + assert!(files.capture_stored().is_err()); + files + .export_capture(&dir.path().join("capture")) + .await + .unwrap(); + assert!( + files.close(false, true).await.is_err(), + "unrecorded export is not release authority" + ); + assert!(disk.open(&root).await.is_ok()); + files.capture_stored().unwrap(); + files.close(false, true).await.unwrap(); + assert!(disk.open(&root).await.is_err()); + } + + #[tokio::test(start_paused = true)] + async fn cancelled_destroy_preserves_the_started_source() { + let dir = tempfile::tempdir().unwrap(); + let disk = Arc::new(BusyDisk { + inner: FileDisk::new(dir.path().join("volumes"), 4096).unwrap(), + busy: std::sync::atomic::AtomicBool::new(false), + pause_delete: Some(Arc::new(tokio::sync::Notify::new())), + }); + let sandbox = SandboxId::new(); + let root = live_name(sandbox, DiskRole::Root); + disk.create(&root, 4096).await.unwrap(); + let disks = Disks::new(disk.clone(), sandbox); + disks.owned.lock().await.push(root.clone()); + let files = super::super::files::PinnedFiles { + path: dir.path().join("private"), + retain: std::sync::atomic::AtomicBool::new(true), + capture_ready: std::sync::atomic::AtomicBool::new(false), + disks: Some(disks), + }; + assert!( + tokio::time::timeout(std::time::Duration::from_secs(1), files.close(true, false)) + .await + .is_err() + ); + drop(files); + assert!(disk.open(&root).await.is_ok()); + } + + #[tokio::test] + async fn busy_volume_fails_release_without_losing_ownership() { + let dir = tempfile::tempdir().unwrap(); + let disk = Arc::new(BusyDisk { + inner: FileDisk::new(dir.path().join("volumes"), 4096).unwrap(), + busy: std::sync::atomic::AtomicBool::new(true), + pause_delete: None, + }); + let sandbox = SandboxId::new(); + let root = live_name(sandbox, DiskRole::Root); + disk.create(&root, 4096).await.unwrap(); + let owner = Disks::new(disk.clone(), sandbox); + owner.owned.lock().await.push(root.clone()); + assert_eq!( + owner.release().await.unwrap_err().kind(), + io::ErrorKind::ResourceBusy + ); + assert!(disk.open(&root).await.is_ok()); + disk.busy.store(false, std::sync::atomic::Ordering::Release); + owner.release().await.unwrap(); + assert!(disk.open(&root).await.is_err()); + } + + #[tokio::test] + async fn import_and_snapshot_export_preserve_bytes_and_reject_changed_sources() { + let dir = tempfile::tempdir().unwrap(); + let disk = FileDisk::new(dir.path().join("volumes"), 4096).unwrap(); + let volume = disk + .create(&VolumeName::new("source").unwrap(), 8192) + .await + .unwrap(); + let mut bytes = vec![0; 8192]; + bytes[4096..].fill(23); + let source = dir.path().join("source"); + std::fs::write(&source, &bytes).unwrap(); + let hash = hex::encode(Sha256::digest(&bytes)); + import(&volume, &source, &hash).await.unwrap(); + let snap = disk + .snapshot(&volume, &VolumeName::new("cut").unwrap()) + .await + .unwrap(); + volume + .write_chunk(ChunkIndex(1), vec![42; 4096]) + .await + .unwrap(); + let output = dir.path().join("capture"); + export(&snap, &output).await.unwrap(); + assert_eq!( + std::fs::metadata(&output).unwrap().permissions().mode() & 0o777, + 0o600, + "capture must be private at creation" + ); + assert_eq!(std::fs::read(&output).unwrap(), bytes); + assert!( + export(&snap, &output).await.is_err(), + "capture must not overwrite evidence" + ); + bytes[0] = 1; + std::fs::write(&source, &bytes).unwrap(); + assert!(import(&volume, &source, &hash).await.is_err()); + std::fs::write(&source, b"short").unwrap(); + assert!(import(&volume, &source, &hash).await.is_err()); + } + + #[tokio::test] + async fn export_releases_its_snapshot_and_close_releases_only_owned_volumes() { + let dir = tempfile::tempdir().unwrap(); + let disk = Arc::new(FileDisk::new(dir.path().join("volumes"), 4096).unwrap()); + let sandbox = SandboxId::new(); + let root = live_name(sandbox, DiskRole::Root); + let foreign = VolumeName::new("foreign").unwrap(); + disk.create(&foreign, 4096).await.unwrap(); + let volume = disk.create(&root, 4096).await.unwrap(); + volume + .write_chunk(ChunkIndex(0), vec![7; 4096]) + .await + .unwrap(); + drop(volume); + let owner = Disks::new(disk.clone(), sandbox); + owner.owned.lock().await.push(root.clone()); + let output = dir.path().join("capture"); + owner.export(&output).await.unwrap(); + assert_eq!(std::fs::read(output).unwrap(), vec![7; 4096]); + assert!(disk + .open(&transient_name(sandbox, DiskRole::Root)) + .await + .is_err()); + assert!(disk.open(&root).await.is_ok()); + owner.release().await.unwrap(); + assert_eq!(disk.list().await.unwrap(), vec![foreign]); + } + + #[tokio::test] + async fn failed_export_keeps_source_and_snapshot_and_refuses_release() { + let dir = tempfile::tempdir().unwrap(); + let disk = Arc::new(FileDisk::new(dir.path().join("volumes"), 4096).unwrap()); + let sandbox = SandboxId::new(); + let root = live_name(sandbox, DiskRole::Root); + disk.create(&root, 4096).await.unwrap(); + let owner = Disks::new(disk.clone(), sandbox); + owner.owned.lock().await.push(root.clone()); + assert!(owner + .export(&dir.path().join("missing/capture")) + .await + .is_err()); + assert!(owner.release().await.is_err()); + drop(owner); + assert!(disk.open(&root).await.is_ok()); + assert!(disk + .open(&transient_name(sandbox, DiskRole::Root)) + .await + .is_ok()); + } + + #[tokio::test] + async fn file_disk_is_rejected_and_new_volume_is_removed() { + let dir = tempfile::tempdir().unwrap(); + let disk = Arc::new(FileDisk::new(dir.path().join("volumes"), 4096).unwrap()); + let owner = Disks::new(disk.clone(), SandboxId::new()); + assert!(owner + .install(dir.path(), &"a".repeat(64), 8192, None) + .await + .is_err()); + assert!(disk.list().await.unwrap().is_empty()); + } + + #[tokio::test] + async fn existing_volume_is_never_adopted_or_deleted_on_create_failure() { + let dir = tempfile::tempdir().unwrap(); + let disk = Arc::new(FileDisk::new(dir.path().join("volumes"), 4096).unwrap()); + let sandbox = SandboxId::new(); + let name = live_name(sandbox, DiskRole::Root); + let volume = disk.create(&name, 4096).await.unwrap(); + volume + .write_chunk(ChunkIndex(0), vec![19; 4096]) + .await + .unwrap(); + let owner = Disks::new(disk.clone(), sandbox); + assert!(owner + .install(dir.path(), &"a".repeat(64), 4096, None) + .await + .is_err()); + owner.release().await.unwrap(); + assert_eq!( + disk.open(&name) + .await + .unwrap() + .read_chunk(ChunkIndex(0)) + .await + .unwrap(), + vec![19; 4096] + ); + } +} diff --git a/crates/engram-sandbox-firecracker/src/managed/files.rs b/crates/engram-sandbox-firecracker/src/managed/files.rs new file mode 100644 index 000000000..9e2507b7e --- /dev/null +++ b/crates/engram-sandbox-firecracker/src/managed/files.rs @@ -0,0 +1,322 @@ +use std::{io, path::Path}; + +fn verify_file(path: &Path, digest: &str, size: Option) -> io::Result<()> { + use sha2::{Digest, Sha256}; + use std::io::Read; + let mut file = std::fs::File::open(path)?; + let metadata = file.metadata()?; + if !metadata.is_file() || size.is_some_and(|n| n != metadata.len()) { + return Err(io::Error::other("invalid managed file size or type")); + } + let mut hash = Sha256::new(); + let mut buffer = [0; 65536]; + loop { + let n = file.read(&mut buffer)?; + if n == 0 { + break; + } + hash.update(&buffer[..n]); + } + if hex::encode(hash.finalize()) != digest { + return Err(io::Error::other("managed artifact digest mismatch")); + } + Ok(()) +} + +pub(super) struct PinnedFiles { + pub(super) retain: std::sync::atomic::AtomicBool, + pub(super) capture_ready: std::sync::atomic::AtomicBool, + pub path: std::path::PathBuf, + pub(super) disks: Option, +} +impl PinnedFiles { + pub fn prepare( + work: &Path, + cfg: &crate::FirecrackerConfig, + request: &engram_core::types::managed_cold::ManagedColdPrepare, + spec: &engram_core::types::sandbox::SandboxSpec, + _staging: tempfile::TempDir, + ) -> io::Result { + use std::os::unix::fs::DirBuilderExt; + std::fs::create_dir_all(work)?; + use sha2::{Digest, Sha256}; + let name = hex::encode(Sha256::digest( + format!("{}\0{}", request.sandbox_id, request.boot_attempt).as_bytes(), + )); + // Keep the control-socket path short. This digest is only a filename; + // exclusive creation rejects a collision, never adopts another owner. + let path = work.join(format!("m-{}", &name[..24])); + std::fs::DirBuilder::new().mode(0o700).create(&path)?; + let files = Self { + path, + disks: None, + capture_ready: std::sync::atomic::AtomicBool::new(false), + retain: std::sync::atomic::AtomicBool::new(false), + }; + files.pin( + spec.rootfs_source + .as_deref() + .ok_or_else(|| io::Error::other("missing selected disk"))?, + "selected.ext4", + &request.disk.sha256, + Some(request.disk.size_bytes), + )?; + files.pin( + &cfg.kernel_image_path, + "kernel", + &request.kernel_sha256, + None, + )?; + for drive in &request.aux_ro_drives { + let hash = drive + .sha256 + .as_deref() + .ok_or_else(|| io::Error::other("unresolved drive"))?; + let name = format!("{hash}.squashfs"); + if !files.path.join(&name).exists() { + files.pin(&cfg.bundle_dir.join(&name), &name, hash, None)?; + } + } + Ok(files) + } + pub async fn export_capture(&self, destination: &Path) -> io::Result<()> { + self.disks + .as_ref() + .ok_or_else(|| io::Error::other("managed disk owner missing"))? + .export(destination) + .await?; + self.capture_ready + .store(true, std::sync::atomic::Ordering::Release); + Ok(()) + } + pub fn capture_stored(&self) -> io::Result<()> { + if !self + .capture_ready + .load(std::sync::atomic::Ordering::Acquire) + { + return Err(io::Error::other("managed capture export incomplete")); + } + self.retain + .store(false, std::sync::atomic::Ordering::Release); + Ok(()) + } + + pub async fn close(&self, destroy: bool, uncertain: bool) -> io::Result<()> { + if self.retain.load(std::sync::atomic::Ordering::Acquire) && (!destroy || uncertain) { + return Err(io::Error::other("managed capture source retained")); + } + // Disk retention independently protects an incomplete capture, including + // a failure before the snapshot operation starts. + self.disks + .as_ref() + .ok_or_else(|| io::Error::other("managed disk owner missing"))? + .release() + .await?; + self.retain + .store(false, std::sync::atomic::Ordering::Release); + Ok(()) + } + + fn pin(&self, source: &Path, name: &str, hash: &str, size: Option) -> io::Result<()> { + // File input only. A pooled block device must be materialized at the + // exact selected version before entering this cold constructor. + if !std::fs::metadata(source)?.is_file() { + return Err(io::Error::other("managed input is not a file")); + } + let destination = self.path.join(name); + std::fs::copy(source, &destination)?; + verify_file(&destination, hash, size) + } +} +impl Drop for PinnedFiles { + fn drop(&mut self) { + if self.retain.load(std::sync::atomic::Ordering::Acquire) { + if let Some(disks) = &self.disks { + disks.retain_source(); + } + return; + } + let _ = std::fs::remove_dir_all(&self.path); + } +} + +#[cfg(test)] +pub(super) mod tests { + use super::*; + use sha2::{Digest, Sha256}; + pub(in crate::managed) fn inputs( + dir: &Path, + ) -> ( + crate::FirecrackerConfig, + engram_core::types::managed_cold::ManagedColdPrepare, + engram_core::types::sandbox::SandboxSpec, + ) { + use engram_core::types::{ + managed_cold::{ManagedColdDisk, ManagedColdPrepare, ManagedColdSource}, + manifest::ManifestRef, + sandbox::AuxRoDrive, + }; + let hash = |bytes: &[u8]| hex::encode(Sha256::digest(bytes)); + std::fs::write(dir.join("kernel"), b"kernel-a").unwrap(); + std::fs::write(dir.join("disk"), b"disk-a").unwrap(); + let bundle = hash(b"bundle"); + std::fs::write(dir.join(format!("{bundle}.squashfs")), b"bundle").unwrap(); + let drives: Vec<_> = (0..3) + .map(|i| AuxRoDrive { + drive_id: AuxRoDrive::slot_drive_id(i), + guest_mount: AuxRoDrive::slot_guest_mount(i), + fs_type: "squashfs".into(), + sha256: Some(bundle.clone()), + }) + .collect(); + let manifest = ManifestRef::new(); + let spec: engram_core::types::sandbox::SandboxSpec = serde_json::from_value(serde_json::json!({"image":"image-v1", "rootfs_source":dir.join("disk"), "rootfs_manifest":manifest, + "cpu":{"vcpus":1}, "memory":{"max_mib":128}, "disk":{"max_gib":1}, "env":{}, "aux_ro_drives":drives})).unwrap(); + let request = ManagedColdPrepare { + version: 1, + session_id: engram_core::SessionId::new(), + sandbox_id: engram_core::SandboxId::new(), + boot_attempt: "boot-1".into(), + image: spec.image.clone(), + disk: ManagedColdDisk { + manifest, + sha256: hash(b"disk-a"), + size_bytes: 6, + source: ManagedColdSource::FreshImage { + base_manifest: ManifestRef::new(), + }, + }, + kernel_sha256: hash(b"kernel-a"), + aux_ro_drives: spec.aux_ro_drives.clone(), + }; + let mut cfg = crate::FirecrackerConfig::with_kernel(dir.join("kernel")); + cfg.bundle_dir = dir.to_owned(); + cfg.balloon = false; + (cfg, request, spec) + } + #[test] + fn staging_is_removed_after_pin_success_or_verification_failure() { + for valid in [true, false] { + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, mut spec) = inputs(dir.path()); + let staging = tempfile::tempdir_in(dir.path()).unwrap(); + let path = staging.path().to_owned(); + let disk = path.join("selected.ext4"); + std::fs::write(&disk, if valid { b"disk-a" } else { b"bad-aa" }).unwrap(); + spec.rootfs_source = Some(disk); + let result = + PinnedFiles::prepare(&dir.path().join("owned"), &cfg, &request, &spec, staging); + assert!(!path.exists(), "staging must end with the pin operation"); + if valid { + let files = result.unwrap(); + assert_eq!( + std::fs::read(files.path.join("selected.ext4")).unwrap(), + b"disk-a" + ); + } else { + assert!(result.is_err()); + } + } + } + + #[test] + fn managed_root_disk_is_not_a_regular_file() { + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, spec) = inputs(dir.path()); + let files = PinnedFiles::prepare( + &dir.path().join("owned"), + &cfg, + &request, + &spec, + tempfile::tempdir().unwrap(), + ) + .unwrap(); + assert!( + !files.path.join("rootfs.ext4").exists(), + "managed Firecracker must attach a session volume" + ); + } + + #[test] + fn retirement_failure_retains_exact_private_source_disk() { + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, spec) = inputs(dir.path()); + let files = PinnedFiles::prepare( + &dir.path().join("owned"), + &cfg, + &request, + &spec, + tempfile::tempdir().unwrap(), + ) + .unwrap(); + let path = files.path.clone(); + files + .retain + .store(true, std::sync::atomic::Ordering::Release); + drop(files); + assert_eq!( + std::fs::read(path.join("selected.ext4")).unwrap(), + b"disk-a" + ); + } + #[test] + fn uuid_boot_attempt_fits_native_control_socket_path() { + let dir = tempfile::tempdir().unwrap(); + let (cfg, mut request, spec) = inputs(dir.path()); + request.boot_attempt = "12345678-1234-1234-1234-123456789abc".into(); + let files = PinnedFiles::prepare( + &dir.path().join("owned"), + &cfg, + &request, + &spec, + tempfile::tempdir().unwrap(), + ) + .unwrap(); + let runtime_path = Path::new("/var/lib/engrams") + .join(files.path.file_name().unwrap()) + .join("firecracker.sock"); + assert!( + std::os::unix::net::SocketAddr::from_pathname(runtime_path).is_ok(), + "ordinary UUID boot attempts must fit a Unix control socket" + ); + } + #[test] + fn pinning_is_private_and_collision_does_not_remove_previous_attempt() { + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, spec) = inputs(dir.path()); + let work = dir.path().join("owned"); + let files = + PinnedFiles::prepare(&work, &cfg, &request, &spec, tempfile::tempdir().unwrap()) + .unwrap(); + let owned = files.path.clone(); + std::fs::write(dir.path().join("disk"), b"disk-b").unwrap(); + assert_eq!( + std::fs::read(owned.join("selected.ext4")).unwrap(), + b"disk-a" + ); + assert!( + PinnedFiles::prepare(&work, &cfg, &request, &spec, tempfile::tempdir().unwrap()) + .is_err() + ); + assert!(owned.exists()); + drop(files); + assert!(!owned.exists()); + assert!( + PinnedFiles::prepare(&work, &cfg, &request, &spec, tempfile::tempdir().unwrap()) + .is_err() + ); + assert!(!owned.exists()); + } + #[test] + fn selected_bytes_and_length_must_match_after_copy() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("disk"); + std::fs::write(&path, b"disk-a").unwrap(); + let digest = hex::encode(Sha256::digest(b"disk-a")); + verify_file(&path, &digest, Some(6)).unwrap(); + std::fs::write(&path, b"disk-b").unwrap(); + assert!(verify_file(&path, &digest, Some(6)).is_err()); + std::fs::write(&path, b"disk-a").unwrap(); + assert!(verify_file(&path, &digest, Some(7)).is_err()); + } +} diff --git a/crates/engram-sandbox-firecracker/src/managed/network.rs b/crates/engram-sandbox-firecracker/src/managed/network.rs new file mode 100644 index 000000000..bd9a6c786 --- /dev/null +++ b/crates/engram-sandbox-firecracker/src/managed/network.rs @@ -0,0 +1,256 @@ +//! A private namespace owns its TAP, rules, listeners and conntrack table. +use serde::{Deserialize, Serialize}; +use std::{io, net::Ipv4Addr}; + +#[cfg(any(target_os = "linux", test))] +pub(super) const TAP: &str = "managed0"; +pub(super) const GUEST_MAC: &str = "02:00:00:00:00:02"; +pub(super) const GATEWAY_MAC: &str = "02:00:00:00:00:01"; +#[cfg(target_os = "linux")] +pub(super) const PROXY_PORT: u16 = 15000; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NetworkShape { + pub guest: Ipv4Addr, + pub gateway: Ipv4Addr, + pub guest_mac: String, + pub gateway_mac: String, + pub mtu: u32, +} +impl NetworkShape { + pub(super) fn new(guest: Ipv4Addr, gateway: Ipv4Addr) -> io::Result { + let valid = |ip: Ipv4Addr| { + !ip.is_loopback() + && !ip.is_unspecified() + && !ip.is_multicast() + && !ip.is_link_local() + && ip.octets()[0] != 0 + && ip.octets()[0] < 224 + && !matches!(ip.octets()[3], 0 | 255) + }; + if guest == gateway + || !valid(guest) + || !valid(gateway) + || guest.octets()[..3] != gateway.octets()[..3] + { + return Err(io::Error::other("invalid managed subnet")); + } + Ok(Self { + guest, + gateway, + guest_mac: GUEST_MAC.into(), + gateway_mac: GATEWAY_MAC.into(), + mtu: 1500, + }) + } +} + +#[cfg(target_os = "linux")] +pub(super) fn rules(shape: &NetworkShape) -> String { + let guest = shape.guest; + let gateway = shape.gateway; + // CLOSED precedes even ARP and tracked packets. The second netdev table + // retains the source/destination checks after a later exact release. + format!( + r#"table netdev closed {{ + chain ingress {{ type filter hook ingress device "{TAP}" priority -500; policy drop; }} + chain egress {{ type filter hook egress device "{TAP}" priority -500; policy drop; }} +}} +table netdev identity {{ + chain ingress {{ type filter hook ingress device "{TAP}" priority -450; policy drop; + ether saddr {GUEST_MAC} ether type arp arp operation request arp saddr ether {GUEST_MAC} arp saddr ip {guest} arp daddr ip {gateway} accept + ether saddr {GUEST_MAC} ether daddr {GATEWAY_MAC} ether type arp arp operation reply arp saddr ether {GUEST_MAC} arp saddr ip {guest} arp daddr ether {GATEWAY_MAC} arp daddr ip {gateway} accept + ether saddr {GUEST_MAC} ether daddr {GATEWAY_MAC} ether type ip ip saddr {guest} accept + }} + chain egress {{ type filter hook egress device "{TAP}" priority -450; policy drop; + ether saddr {GATEWAY_MAC} ether daddr ff:ff:ff:ff:ff:ff ether type arp arp operation request arp saddr ether {GATEWAY_MAC} arp saddr ip {gateway} arp daddr ip {guest} accept + ether saddr {GATEWAY_MAC} ether daddr {GUEST_MAC} ether type arp arp operation reply arp saddr ether {GATEWAY_MAC} arp saddr ip {gateway} arp daddr ip {guest} accept + ether saddr {GATEWAY_MAC} ether daddr {GUEST_MAC} ether type ip ip daddr {guest} accept + }} +}} +table inet closed {{ + chain input {{ type filter hook input priority -200; policy drop; }} + chain output {{ type filter hook output priority -200; policy drop; }} + chain forward {{ type filter hook forward priority -200; policy drop; }} +}} +table inet strict {{ + chain classify {{ type filter hook prerouting priority -150; policy drop; + iifname "{TAP}" ip saddr {guest} ip daddr {gateway} udp dport 53 accept + iifname "{TAP}" ip saddr {guest} ip daddr != {gateway} tcp dport 53 drop + iifname "{TAP}" ip saddr {guest} meta l4proto tcp accept + }} + chain proxy_redirect {{ type nat hook prerouting priority -110; policy accept; + iifname "{TAP}" ip saddr {guest} meta l4proto tcp redirect to :{PROXY_PORT} + }} + chain input {{ type filter hook input priority -100; policy drop; + iifname "{TAP}" ip saddr {guest} ct status dnat tcp dport {PROXY_PORT} accept + iifname "{TAP}" ip saddr {guest} ip daddr {gateway} udp dport 53 accept + }} + chain output {{ type filter hook output priority -100; policy drop; + oifname "{TAP}" ip daddr {guest} ct state established tcp sport {PROXY_PORT} accept + oifname "{TAP}" ip saddr {gateway} ip daddr {guest} udp sport 53 accept + }} + chain forward {{ type filter hook forward priority -100; policy drop; }} +}} +"# + ) +} + +#[cfg(target_os = "linux")] +mod linux; +#[cfg(target_os = "linux")] +pub(super) use linux::Namespace; + +#[cfg(any(target_os = "linux", test))] +use serde_json::Value; +#[cfg(any(target_os = "linux", test))] +fn verify_links(links: &Value, shape: &NetworkShape) -> io::Result { + let links = links + .as_array() + .ok_or_else(|| io::Error::other("invalid links"))?; + if links.len() != 2 || !links.iter().any(|v| v["ifname"] == "lo") { + return Err(io::Error::other("unexpected namespace links")); + } + let tap = links + .iter() + .find(|v| v["ifname"] == TAP) + .ok_or_else(|| io::Error::other("missing TAP"))?; + if tap["address"] != shape.gateway_mac + || tap["mtu"] != shape.mtu + || tap["linkinfo"]["info_kind"] != "tun" + || tap["linkinfo"]["info_data"]["type"] != "tap" + { + return Err(io::Error::other("wrong TAP shape")); + } + tap["ifindex"] + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .filter(|n| *n > 0) + .ok_or_else(|| io::Error::other("invalid TAP index")) +} +#[cfg(any(target_os = "linux", test))] +fn verify_closed_tables(rules: &Value) -> io::Result<()> { + let values = rules["nftables"] + .as_array() + .ok_or_else(|| io::Error::other("invalid nft rules"))?; + if values.iter().any(|v| v["rule"]["table"] == "closed") { + return Err(io::Error::other("closed fence contains rules")); + } + for (family, hook, priority) in [ + ("netdev", "ingress", -500), + ("netdev", "egress", -500), + ("inet", "input", -200), + ("inet", "output", -200), + ("inet", "forward", -200), + ] { + if !values.iter().any(|v| { + v["chain"]["family"] == family + && v["chain"]["table"] == "closed" + && v["chain"]["hook"] == hook + && v["chain"]["policy"] == "drop" + && v["chain"]["prio"] == priority + && (family != "netdev" || v["chain"]["dev"] == TAP) + }) { + return Err(io::Error::other("closed fence absent")); + } + } + Ok(()) +} + +#[cfg(any(target_os = "linux", test))] +fn stable_link_shape(value: Value) -> io::Result { + let links = value + .as_array() + .ok_or_else(|| io::Error::other("invalid links"))?; + Ok(Value::Array(links.iter().map(|v| serde_json::json!({ + "ifindex": v["ifindex"], "ifname": v["ifname"], "address": v["address"], "mtu": v["mtu"], + "link_type": v["link_type"], "master": v["master"], + "up": v["flags"].as_array().is_some_and(|flags| flags.iter().any(|f| f == "UP")), + "linkinfo": {"info_kind": v["linkinfo"]["info_kind"], "info_data": {"type": v["linkinfo"]["info_data"]["type"]}} + })).collect())) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn readback_rejects_accept_rules_or_wrong_hook_priority_in_closed_fence() { + let mut objects: Vec = [("netdev", "ingress", -500), ("netdev", "egress", -500), ("inet", "input", -200), ("inet", "output", -200), ("inet", "forward", -200)].into_iter().map(|(family, hook, priority)| serde_json::json!({"chain":{"family":family,"table":"closed","hook":hook,"policy":"drop", "prio": priority,"dev":"managed0"}})).collect(); + let good = serde_json::json!({"nftables":objects}); + verify_closed_tables(&good).unwrap(); + objects[0]["chain"]["prio"] = 0.into(); + assert!( + verify_closed_tables(&serde_json::json!({"nftables":objects})).is_err(), + "late ingress fence is not the approved closed boundary" + ); + let mut changed = good.clone(); + changed["nftables"].as_array_mut().unwrap().push(serde_json::json!({"rule":{"family":"netdev","table":"closed","chain":"ingress","expr":[{"accept":null}]}})); + assert!( + verify_closed_tables(&changed).is_err(), + "an early accept in a default-drop chain is not closed" + ); + } + #[test] + fn live_link_readback_requires_private_tap_type_and_exact_selected_shape() { + let shape = + NetworkShape::new("192.0.2.2".parse().unwrap(), "192.0.2.1".parse().unwrap()).unwrap(); + let good = serde_json::json!([{"ifname":"lo","ifindex":1}, {"ifname":"managed0","ifindex":2,"address":"02:00:00:00:00:01","mtu":1500,"linkinfo":{"info_kind":"tun","info_data":{"type":"tap"}}}]); + assert_eq!(verify_links(&good, &shape).unwrap(), 2); + for (key, replacement) in [ + ("address", serde_json::json!("02:00:00:00:00:03")), + ("mtu", serde_json::json!(1400)), + ("ifindex", serde_json::json!(0)), + ] { + let mut bad = good.clone(); + bad[1][key] = replacement; + assert!(verify_links(&bad, &shape).is_err()); + } + let mut bad = good.clone(); + bad[1]["linkinfo"]["info_data"]["type"] = "tun".into(); + assert!(verify_links(&bad, &shape).is_err()); + let mut bad = good; + bad.as_array_mut() + .unwrap() + .push(serde_json::json!({"ifname":"escape0","ifindex":3})); + assert!(verify_links(&bad, &shape).is_err()); + } + #[test] + fn opening_same_tap_changes_carrier_and_queue_count_not_attachment_identity() { + let before = serde_json::json!([{"ifname":"managed0","ifindex":2,"address":"02:00:00:00:00:01","mtu":1500,"flags":["BROADCAST","UP"],"operstate":"DOWN", "linkinfo":{"info_kind":"tun","info_data":{"type":"tap","num_queues":0}}}]); + let mut after = before.clone(); + after[0]["flags"] = serde_json::json!(["BROADCAST", "UP", "LOWER_UP"]); + after[0]["operstate"] = "UP".into(); + after[0]["linkinfo"]["info_data"]["num_queues"] = 1.into(); + assert_eq!( + stable_link_shape(before.clone()).unwrap(), + stable_link_shape(after.clone()).unwrap() + ); + after[0]["ifindex"] = 3.into(); + assert_ne!( + stable_link_shape(before.clone()).unwrap(), + stable_link_shape(after.clone()).unwrap() + ); + after[0]["ifindex"] = 2.into(); + after[0]["flags"] = serde_json::json!(["BROADCAST"]); + assert_ne!( + stable_link_shape(before).unwrap(), + stable_link_shape(after).unwrap() + ); + } + #[test] + fn rejects_subnet_escape_and_non_unicast_addresses() { + for (guest, gateway) in [ + ("192.0.2.2", "198.51.100.1"), + ("127.0.0.2", "127.0.0.1"), + ("224.0.0.2", "224.0.0.1"), + ("192.0.2.0", "192.0.2.1"), + ("192.0.2.2", "192.0.2.255"), + ("192.0.2.2", "192.0.2.2"), + ] { + assert!( + NetworkShape::new(guest.parse().unwrap(), gateway.parse().unwrap()).is_err(), + "{guest} {gateway}" + ); + } + } +} diff --git a/crates/engram-sandbox-firecracker/src/managed/network/linux.rs b/crates/engram-sandbox-firecracker/src/managed/network/linux.rs new file mode 100644 index 000000000..5c534a8e9 --- /dev/null +++ b/crates/engram-sandbox-firecracker/src/managed/network/linux.rs @@ -0,0 +1,363 @@ +use super::*; +use serde_json::Value; +use std::{ + fs::File, + os::{ + fd::AsRawFd, + unix::{fs::MetadataExt, process::CommandExt}, + }, + process::{Command, Stdio}, + sync::Arc, +}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct LiveIdentity { + pub namespace_device: u64, + pub namespace_inode: u64, + pub tap_ifindex: u32, + pub rules_sha256: String, +} + +pub(in crate::managed) struct Namespace { + pub fd: File, + pub shape: NetworkShape, + pub identity: LiveIdentity, + links: Value, + addresses: Value, + routes: Value, + rules: Value, + released: std::sync::atomic::AtomicBool, +} +impl Namespace { + pub async fn create( + shape: NetworkShape, + ) -> io::Result<(Arc, std::net::TcpListener, std::net::UdpSocket)> { + let (tx, rx) = tokio::sync::oneshot::channel(); + // Never change a runtime worker's namespace. This thread cannot return + // to a pool, and only inherited child processes enter the new namespace. + std::thread::Builder::new() + .name("managed-net-create".into()) + .spawn(move || { + let _ = tx.send(Self::create_inner(shape)); + })?; + rx.await + .map_err(|_| io::Error::other("network creation worker stopped"))? + } + fn create_inner( + shape: NetworkShape, + ) -> io::Result<(Arc, std::net::TcpListener, std::net::UdpSocket)> { + // SAFETY: this dedicated native thread has no shared network activity. + if unsafe { libc::unshare(libc::CLONE_NEWNET) } != 0 { + return Err(io::Error::last_os_error()); + } + let fd = File::open("/proc/thread-self/ns/net")?; + let metadata = fd.metadata()?; + let mut owner = Self { + fd, + shape, + identity: LiveIdentity { + namespace_device: metadata.dev(), + namespace_inode: metadata.ino(), + tap_ifindex: 0, + rules_sha256: String::new(), + }, + links: Value::Null, + addresses: Value::Null, + routes: Value::Null, + rules: Value::Null, + released: std::sync::atomic::AtomicBool::new(false), + }; + owner.run( + "sysctl", + &[ + "-q", + "-w", + "net.ipv6.conf.all.disable_ipv6=1", + "net.ipv6.conf.default.disable_ipv6=1", + ], + None, + )?; + owner.run("ip", &["tuntap", "add", "dev", TAP, "mode", "tap"], None)?; + owner.run( + "ip", + &[ + "link", + "set", + "dev", + TAP, + "address", + GATEWAY_MAC, + "mtu", + "1500", + ], + None, + )?; + // No link is up and no FC process exists at this point. + owner.run("nft", &["-f", "-"], Some(rules(&owner.shape).as_bytes()))?; + owner.rules = owner.json("nft", &["-j", "-s", "list", "ruleset"])?; + verify_closed_tables(&owner.rules)?; + owner.run( + "ip", + &[ + "address", + "add", + &format!("{}/24", owner.shape.gateway), + "dev", + TAP, + ], + None, + )?; + let before_up = owner.link_shape()?; + let tap_ifindex = verify_links(&before_up, &owner.shape)?; + if before_up + .as_array() + .is_some_and(|links| links.iter().any(|v| v["up"] == true)) + { + return Err(io::Error::other("namespace link was already up")); + } + owner.run("ip", &["link", "set", "dev", TAP, "up"], None)?; + owner.links = owner.link_shape()?; + owner.addresses = owner.json("ip", &["-j", "address", "show"])?; + normalize_addresses(&mut owner.addresses); + owner.routes = owner.route_shape()?; + owner.identity.tap_ifindex = verify_links(&owner.links, &owner.shape)?; + if owner.identity.tap_ifindex != tap_ifindex { + return Err(io::Error::other("TAP changed during setup")); + } + if !owner + .links + .as_array() + .is_some_and(|links| links.iter().any(|v| v["ifname"] == TAP && v["up"] == true)) + { + return Err(io::Error::other("TAP is not up")); + } + use sha2::{Digest, Sha256}; + owner.identity.rules_sha256 = + hex::encode(Sha256::digest(serde_json::to_vec(&owner.rules)?)); + // Binding happens in this native namespace thread. No port is public. + let tcp = std::net::TcpListener::bind((owner.shape.gateway, PROXY_PORT))?; + let dns = std::net::UdpSocket::bind((owner.shape.gateway, 53))?; + tcp.set_nonblocking(true)?; + dns.set_nonblocking(true)?; + owner.check(None)?; + Ok((Arc::new(owner), tcp, dns)) + } + pub fn command(&self, program: &std::ffi::OsStr) -> io::Result { + let mut cmd = Command::new(program); + let fd = self.fd.as_raw_fd(); + // SAFETY: the owner retains fd through synchronous spawn. setns is the + // only child-side change; no name or numeric PID selects this namespace. + unsafe { + cmd.pre_exec(move || { + if libc::setns(fd, libc::CLONE_NEWNET) != 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + }); + } + Ok(cmd) + } + fn run(&self, program: &str, args: &[&str], input: Option<&[u8]>) -> io::Result> { + use std::io::{Read, Write}; + let mut child = self + .command(program.as_ref())? + .args(args) + .stdin(if input.is_some() { + Stdio::piped() + } else { + Stdio::null() + }) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn()?; + let result = (|| { + if let Some(bytes) = input { + child + .stdin + .take() + .ok_or_else(|| io::Error::other("missing command input"))? + .write_all(bytes)?; + } + let mut stdout = child + .stdout + .take() + .ok_or_else(|| io::Error::other("missing command output"))?; + // SAFETY: the owned pipe remains open. Preserve all existing flags. + let flags = unsafe { libc::fcntl(stdout.as_raw_fd(), libc::F_GETFL) }; + if flags < 0 + || unsafe { + libc::fcntl(stdout.as_raw_fd(), libc::F_SETFL, flags | libc::O_NONBLOCK) + } < 0 + { + return Err(io::Error::last_os_error()); + } + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let mut bytes = Vec::new(); + let mut buffer = [0; 8192]; + loop { + match stdout.read(&mut buffer) { + Ok(n) => { + bytes.extend_from_slice(&buffer[..n]); + if n == 0 { + if let Some(status) = child.try_wait()? { + return if status.success() { + Ok(bytes) + } else { + Err(io::Error::other("managed network command failed")) + }; + } + } + } + Err(e) + if e.kind() == io::ErrorKind::WouldBlock + || e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + if bytes.len() >= 1024 * 1024 || std::time::Instant::now() >= deadline { + return Err(io::Error::other( + "managed network command exceeded its limit", + )); + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + })(); + if result.is_err() { + let _ = child.kill(); + let _ = child.wait(); + } + result + } + fn json(&self, command: &str, args: &[&str]) -> io::Result { + serde_json::from_slice(&self.run(command, args, None)?).map_err(io::Error::other) + } + fn link_shape(&self) -> io::Result { + stable_link_shape(self.json("ip", &["-j", "-d", "link", "show"])?) + } + fn route_shape(&self) -> io::Result { + let mut routes = self.json("ip", &["-j", "route", "show", "table", "all"])?; + if let Some(items) = routes.as_array_mut() { + for item in items { + if let Some(flags) = item["flags"].as_array_mut() { + flags.retain(|v| v != "linkdown"); + } + } + } + Ok(routes) + } + /// Remove only the two retained CLOSED tables after the adapter's exact ACK. + pub fn release(&self, child_pid: u32) -> io::Result<()> { + self.check(Some(child_pid))?; + if self.released.load(std::sync::atomic::Ordering::Acquire) { + return Ok(()); + } + self.run( + "nft", + &["-f", "-"], + Some(b"delete table netdev closed\ndelete table inet closed\n"), + )?; + if self.json("nft", &["-j", "-s", "list", "ruleset"])? != self.released_rules() { + return Err(io::Error::other("released network rules changed")); + } + self.released + .store(true, std::sync::atomic::Ordering::Release); + self.check(Some(child_pid)) + } + fn released_rules(&self) -> Value { + let mut rules = self.rules.clone(); + if let Some(items) = rules["nftables"].as_array_mut() { + items.retain(|item| { + item["table"]["name"] != "closed" + && item["chain"]["table"] != "closed" + && item["rule"]["table"] != "closed" + }); + } + rules + } + pub fn check(&self, child_pid: Option) -> io::Result<()> { + let metadata = self.fd.metadata()?; + if metadata.dev() != self.identity.namespace_device + || metadata.ino() != self.identity.namespace_inode + { + return Err(io::Error::other("namespace changed")); + } + if let Some(pid) = child_pid { + let child = std::fs::metadata(format!("/proc/{pid}/ns/net"))?; + if child.dev() != metadata.dev() || child.ino() != metadata.ino() { + return Err(io::Error::other("VM namespace changed")); + } + } + let mut addresses = self.json("ip", &["-j", "address", "show"])?; + normalize_addresses(&mut addresses); + if self.link_shape()? != self.links + || addresses != self.addresses + || self.route_shape()? != self.routes + || self.json("nft", &["-j", "-s", "list", "ruleset"])? + != if self.released.load(std::sync::atomic::Ordering::Acquire) { + self.released_rules() + } else { + self.rules.clone() + } + { + return Err(io::Error::other("managed network shape changed")); + } + Ok(()) + } +} +fn normalize_addresses(value: &mut Value) { + if let Some(items) = value.as_array_mut() { + for item in items { + if let Some(object) = item.as_object_mut() { + for key in ["flags", "operstate", "linkmode", "qdisc"] { + object.remove(key); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + /// Task 8 must invoke this explicitly on an isolated Linux root worker. + /// This checks installed state; VM and packet acceptance are separate tests. + #[tokio::test] + #[ignore = "requires isolated Linux root, iproute2, nftables and netdev egress support"] + async fn actual_closed_namespace_readback_detects_route_tampering() { + // SAFETY: read-only process identity query. + assert_eq!( + unsafe { libc::geteuid() }, + 0, + "this explicit acceptance check needs root" + ); + let root_before = File::open("/proc/thread-self/ns/net") + .unwrap() + .metadata() + .unwrap(); + let shape = + NetworkShape::new("192.0.2.2".parse().unwrap(), "192.0.2.1".parse().unwrap()).unwrap(); + let (namespace, tcp, dns) = Namespace::create(shape).await.unwrap(); + assert_eq!(tcp.local_addr().unwrap().port(), PROXY_PORT); + assert_eq!(dns.local_addr().unwrap().port(), 53); + let checked = namespace.clone(); + tokio::task::spawn_blocking(move || { + checked.check(None).unwrap(); + checked + .run("ip", &["route", "add", "blackhole", "default"], None) + .unwrap(); + assert!(checked.check(None).is_err()); + }) + .await + .unwrap(); + drop(tcp); + drop(dns); + drop(namespace); + let root_after = File::open("/proc/thread-self/ns/net") + .unwrap() + .metadata() + .unwrap(); + assert_eq!( + (root_before.dev(), root_before.ino()), + (root_after.dev(), root_after.ino()) + ); + } +} diff --git a/crates/engram-sandbox-firecracker/src/managed/process.rs b/crates/engram-sandbox-firecracker/src/managed/process.rs new file mode 100644 index 000000000..7791c3e7d --- /dev/null +++ b/crates/engram-sandbox-firecracker/src/managed/process.rs @@ -0,0 +1,47 @@ +use super::{ + child::{ChildOwner, Control}, + files::PinnedFiles, + network::Namespace, +}; +use std::{ + io, + sync::{mpsc, Arc}, +}; + +pub(super) struct Process { + child: ChildOwner, + namespace: Arc, + path: std::path::PathBuf, +} +impl Process { + pub fn reserve(namespace: Arc, files: Arc) -> io::Result { + let path = files.path.clone(); + let child = ChildOwner::reserve((namespace.clone(), files))?; + Ok(Self { + child, + namespace, + path, + }) + } + pub fn spawn(&mut self, binary: &std::path::Path) -> io::Result<()> { + use std::process::Stdio; + let log = std::fs::File::create_new(self.path.join("firecracker.log"))?; + let mut command = self.namespace.command(binary.as_os_str())?; + command + .arg("--api-sock") + .arg(self.path.join("firecracker.sock")) + .stdin(Stdio::null()) + .stdout(log.try_clone()?) + .stderr(log); + self.child.spawn(command) + } + pub fn live_pid(&self) -> io::Result { + self.child.live_pid() + } + pub fn stop_signal(&self) -> mpsc::Sender { + self.child.stop_signal() + } + pub async fn close(&mut self) -> io::Result<()> { + self.child.close().await + } +} diff --git a/crates/engram-sandbox-firecracker/src/net.rs b/crates/engram-sandbox-firecracker/src/net.rs index d81b740d1..f000504d9 100644 --- a/crates/engram-sandbox-firecracker/src/net.rs +++ b/crates/engram-sandbox-firecracker/src/net.rs @@ -650,6 +650,30 @@ async fn run_iptables(line: &str) -> Result<(), NetError> { run_cmd("iptables", &argv).await } +#[cfg(any(target_os = "linux", test))] +async fn ensure_sysctl_enabled(path: &str) -> Result<(), NetError> { + let value = tokio::fs::read_to_string(path) + .await + .map_err(|e| NetError::Spawn(format!("read {path}"), e))?; + if value.trim() == "1" { + return Ok(()); + } + tokio::fs::write(path, b"1") + .await + .map_err(|e| NetError::Spawn(format!("write {path}"), e))?; + let value = tokio::fs::read_to_string(path) + .await + .map_err(|e| NetError::Spawn(format!("read {path}"), e))?; + if value.trim() != "1" { + return Err(NetError::Failed { + cmd: format!("read {path}"), + status: -1, + stderr: "expected sysctl value 1 after write".into(), + }); + } + Ok(()) +} + /// Apply the static ruleset. Idempotent — each rule has a unique /// `--comment` tag and we check-then-insert so a coord restart /// doesn't double up. @@ -660,12 +684,7 @@ pub async fn host_startup( guest_gateway_port: Option, guest_otel_port: Option, ) -> Result<(), NetError> { - if let Err(e) = tokio::fs::write("/proc/sys/net/ipv4/ip_forward", b"1").await { - return Err(NetError::Spawn( - "write /proc/sys/net/ipv4/ip_forward".into(), - e, - )); - } + ensure_sysctl_enabled("/proc/sys/net/ipv4/ip_forward").await?; // Required when proxy mode is on: iptables PREROUTING REDIRECT // rewrites a guest-bound 1.2.3.4 destination to 127.0.0.1 (the // proxy's listener). Without `route_localnet`, the kernel marks @@ -674,12 +693,7 @@ pub async fn host_startup( // Setting `all` covers all current and future TAPs without // having to set it per-interface. if proxy_port.is_some() { - if let Err(e) = tokio::fs::write("/proc/sys/net/ipv4/conf/all/route_localnet", b"1").await { - return Err(NetError::Spawn( - "write /proc/sys/net/ipv4/conf/all/route_localnet".into(), - e, - )); - } + ensure_sysctl_enabled("/proc/sys/net/ipv4/conf/all/route_localnet").await?; } // Clear any prior engram proxy/dns/redirect/otlp rules before // re-applying. The `-C`-then-insert loop below is history-BLIND: if a @@ -1496,6 +1510,102 @@ mod tests { use super::*; use std::str::FromStr; + #[tokio::test] + async fn sysctl_accepts_preconfigured_read_only_value_without_write() { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + let file = tempfile::NamedTempFile::new().unwrap(); + let path = file.path().to_str().unwrap(); + std::fs::write(path, b" \t1\n").unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o444)).unwrap(); + if std::fs::OpenOptions::new().write(true).open(path).is_ok() { + eprintln!("skipped permission assertions: this user can write read-only files"); + return; + } + let before = std::fs::metadata(path).unwrap(); + assert_eq!( + std::fs::write(path, b"1").unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied, + "this regression test needs a user that cannot write the read-only fixture" + ); + + ensure_sysctl_enabled(path).await.unwrap(); + + assert_eq!(std::fs::read(path).unwrap(), b" \t1\n"); + let after = std::fs::metadata(path).unwrap(); + assert_eq!(after.mode(), before.mode()); + assert_eq!(after.len(), before.len()); + assert_eq!(after.modified().unwrap(), before.modified().unwrap()); + assert_eq!( + (after.ctime(), after.ctime_nsec()), + (before.ctime(), before.ctime_nsec()) + ); + } + + #[tokio::test] + async fn sysctl_repairs_disabled_and_malformed_values() { + let file = tempfile::NamedTempFile::new().unwrap(); + let path = file.path().to_str().unwrap(); + for value in ["0\n", "01\n", "true\n", "", "11\n"] { + std::fs::write(path, value).unwrap(); + ensure_sysctl_enabled(path).await.unwrap(); + assert_eq!(std::fs::read(path).unwrap(), b"1", "input: {value:?}"); + } + } + + #[tokio::test] + async fn sysctl_rejects_read_only_values_that_are_not_exactly_one() { + use std::os::unix::fs::PermissionsExt; + + for value in ["0\n", "01\n", "true\n", "", "11\n"] { + let file = tempfile::NamedTempFile::new().unwrap(); + let path = file.path().to_str().unwrap(); + std::fs::write(path, value).unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o444)).unwrap(); + if std::fs::OpenOptions::new().write(true).open(path).is_ok() { + eprintln!("skipped permission assertions: this user can write read-only files"); + return; + } + assert!(matches!( + ensure_sysctl_enabled(path).await, + Err(NetError::Spawn(op, e)) + if op == format!("write {path}") + && e.kind() == std::io::ErrorKind::PermissionDenied + )); + assert_eq!(std::fs::read_to_string(path).unwrap(), value); + } + } + + #[tokio::test] + async fn sysctl_rejects_missing_and_unreadable_files() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let missing = dir.path().join("missing"); + let path = missing.to_str().unwrap(); + assert!(matches!( + ensure_sysctl_enabled(path).await, + Err(NetError::Spawn(op, e)) + if op == format!("read {path}") && e.kind() == std::io::ErrorKind::NotFound + )); + assert!(!missing.exists()); + + let file = tempfile::NamedTempFile::new().unwrap(); + let path = file.path().to_str().unwrap(); + std::fs::write(path, b"1\n").unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::read(path).is_ok() { + eprintln!("skipped permission assertions: this user can read unreadable files"); + return; + } + assert!(matches!( + ensure_sysctl_enabled(path).await, + Err(NetError::Spawn(op, e)) + if op == format!("read {path}") + && e.kind() == std::io::ErrorKind::PermissionDenied + )); + } + fn id() -> SandboxId { SandboxId::new() } diff --git a/crates/engram-sandbox-firecracker/tests/proxy_e2e.rs b/crates/engram-sandbox-firecracker/tests/proxy_e2e.rs index 2e7ce4b77..bae3ea0b8 100644 --- a/crates/engram-sandbox-firecracker/tests/proxy_e2e.rs +++ b/crates/engram-sandbox-firecracker/tests/proxy_e2e.rs @@ -440,6 +440,7 @@ async fn proxy_substitutes_real_value_into_outbound_https() { let network_allow = engram_egress_proxy::HostList::from_manifest(&[TEST_HOST.into()], &[]).unwrap(); registry.register(engram_egress_proxy::SessionState { + managed_deny: None, session_id, sandbox_id, guest_ip, diff --git a/crates/engram-sandbox-firecracker/tests/warm_proxy_chain.rs b/crates/engram-sandbox-firecracker/tests/warm_proxy_chain.rs index aff411795..8da059d71 100644 --- a/crates/engram-sandbox-firecracker/tests/warm_proxy_chain.rs +++ b/crates/engram-sandbox-firecracker/tests/warm_proxy_chain.rs @@ -241,6 +241,7 @@ async fn warm_path_redirects_through_proxy_with_correct_source_lookup() { allow: engram_egress_proxy::HostList::from_manifest(&[TEST_HOST.into()], &[]).unwrap(), }; registry.register(engram_egress_proxy::SessionState { + managed_deny: None, session_id, sandbox_id, guest_ip: snat_ip, diff --git a/crates/engram-sandbox-process/src/lib.rs b/crates/engram-sandbox-process/src/lib.rs index 89698368d..9ff5aca9e 100644 --- a/crates/engram-sandbox-process/src/lib.rs +++ b/crates/engram-sandbox-process/src/lib.rs @@ -1980,6 +1980,7 @@ mod tests { let (backend, _dir) = backend(); let error = backend .notify_session_policy(SessionEgressPolicy { + managed_deny: None, session_id: SessionId::new(), sandbox_id: SandboxId::new(), guest_ip: std::net::Ipv4Addr::LOCALHOST, diff --git a/crates/engram-sandbox-vz/Cargo.toml b/crates/engram-sandbox-vz/Cargo.toml index 6fdefc68b..eefe67412 100644 --- a/crates/engram-sandbox-vz/Cargo.toml +++ b/crates/engram-sandbox-vz/Cargo.toml @@ -22,7 +22,11 @@ description = "macOS Apple Silicon SandboxBackend driving Apple's Virtualization unsafe_code = "allow" [dependencies] +tempfile = "3" engram-core.workspace = true +engram-egress-proto.workspace = true +sha2.workspace = true +hex.workspace = true engram-agentd.workspace = true engram-harness-proto.workspace = true async-trait.workspace = true @@ -50,5 +54,4 @@ libc = "0.2" [dev-dependencies] futures.workspace = true -tempfile = "3" tracing-subscriber.workspace = true diff --git a/crates/engram-sandbox-vz/src/backend.rs b/crates/engram-sandbox-vz/src/backend.rs index 4d88ce1a0..bb797886d 100644 --- a/crates/engram-sandbox-vz/src/backend.rs +++ b/crates/engram-sandbox-vz/src/backend.rs @@ -148,6 +148,11 @@ pub struct VzBackend { } impl VzBackend { + /// The same configured kernel that managed preparation copies and verifies. + pub fn managed_kernel_path(&self) -> &Path { + &self.cfg.kernel_path + } + pub fn new(work_dir: impl Into, cfg: VzConfig) -> Result { let work_dir = work_dir.into(); // Validate the kernel exists up-front so misconfiguration @@ -171,6 +176,29 @@ impl VzBackend { }) } + /// Prepare exact disk-only managed inputs with closed host networking. + /// The returned owner is outside ordinary lifecycles. The blocking pin + /// operation owns disposable staging until the input copy is complete. + pub async fn prepare_managed_cold( + &self, + request: engram_core::types::managed_cold::ManagedColdPrepare, + spec: SandboxSpec, + network: engram_egress_proto::private::RegisteredPrivateNetwork, + helper: crate::managed::ManagedHelperArtifact, + staging: tempfile::TempDir, + ) -> Result { + crate::managed::prepare( + &self.work_dir, + &self.cfg, + request, + spec, + network, + helper, + staging, + ) + .await + } + fn vsock_uds_path_for(&self, id: SandboxId) -> Result { let dir = engram_core::socket::short_socket_dir(); std::fs::create_dir_all(&dir)?; @@ -397,6 +425,13 @@ impl VzBackend { let Some(w) = manifest.warm.clone() else { return Err("manifest has no warm block (the save failed)".into()); }; + if w.configuration_revision != crate::snapshot::CONFIGURATION_REVISION { + return Err(format!( + "configuration revision differs (saved {}, current {})", + w.configuration_revision, + crate::snapshot::CONFIGURATION_REVISION + )); + } if !src.join(crate::snapshot::MACHINE_STATE_FILENAME).exists() { return Err("machine.vzs missing from the snapshot dir".into()); } @@ -575,7 +610,7 @@ impl VzBackend { /// Run the agentd exec protocol over a connected byte stream. The /// Firecracker backend carries the same function; only the connection /// setup differs. -async fn drive_exec_protocol( +pub(crate) async fn drive_exec_protocol( sandbox_id: SandboxId, mut reader: R, mut writer: W, @@ -695,7 +730,9 @@ where }) } -async fn probe_durable_exec(mut stream: UnixStream) -> Result { +pub(crate) async fn probe_durable_exec( + mut stream: S, +) -> Result { write_msg( &mut stream, &WireRequest::CancelExec { @@ -1085,6 +1122,7 @@ impl SandboxBackend for VzBackend { let state_path = dest.join(crate::snapshot::MACHINE_STATE_FILENAME); match vm.save(&state_path).await { Ok(()) => Some(crate::snapshot::WarmMachineState { + configuration_revision: crate::snapshot::CONFIGURATION_REVISION, machine_identifier: machine_identifier.clone(), mac_address: mac_address.clone(), kernel_cmdline: self.current_kernel_cmdline(), @@ -1574,6 +1612,66 @@ mod tests { assert!(stream.next().await.is_none()); } + #[test] + fn warm_gate_requires_current_configuration_revision() { + let work = tempfile::tempdir().unwrap(); + let kernel = tempfile::NamedTempFile::new().unwrap(); + let backend = + VzBackend::new(work.path().join("sb"), VzConfig::with_kernel(kernel.path())).unwrap(); + std::fs::write( + work.path().join(crate::snapshot::MACHINE_STATE_FILENAME), + [], + ) + .unwrap(); + let mut value = serde_json::json!({ + "configuration_revision": 1, + "machine_identifier": [1, 2, 3], + "mac_address": "02:00:00:00:00:02", + "kernel_cmdline": backend.current_kernel_cmdline(), + "aux_ro_drives": [], + "memory_mib": 1024, + "vcpus": 2, + "swap_mib": null + }); + let manifest_for = |value| { + crate::snapshot::VzSnapshotManifest::new( + SandboxId::new(), + SandboxSpec { + image: "warm-test".into(), + rootfs_source: None, + image_uri: None, + rootfs_manifest: None, + cpu: engram_core::types::sandbox::CpuLimit { vcpus: 2 }, + memory: engram_core::types::sandbox::MemoryLimit { max_mib: 1024 }, + disk: engram_core::types::sandbox::DiskLimit { max_gib: 1 }, + ttl: None, + env: HashMap::new(), + workdir: None, + network: Default::default(), + aux_ro_drives: vec![], + swap_mib: None, + swap_source: None, + swap_manifest: None, + }, + Some(serde_json::from_value(value).unwrap()), + ) + }; + let current = manifest_for(value.clone()); + assert!(backend.warm_gate(¤t, work.path()).is_ok()); + for revision in [0, u32::MAX] { + value["configuration_revision"] = revision.into(); + let manifest = manifest_for(value.clone()); + let error = backend.warm_gate(&manifest, work.path()).unwrap_err(); + assert!(error.contains("configuration revision"), "{error}"); + } + value + .as_object_mut() + .unwrap() + .remove("configuration_revision"); + let legacy = manifest_for(value); + assert!(backend.warm_gate(&legacy, work.path()).is_err()); + } + #[test] fn new_rejects_missing_kernel_with_actionable_error() { let tmp = std::env::temp_dir(); diff --git a/crates/engram-sandbox-vz/src/lib.rs b/crates/engram-sandbox-vz/src/lib.rs index a6ca58f44..b1d09d715 100644 --- a/crates/engram-sandbox-vz/src/lib.rs +++ b/crates/engram-sandbox-vz/src/lib.rs @@ -36,6 +36,9 @@ #[cfg(target_os = "macos")] mod backend; +#[cfg(target_os = "macos")] +mod managed; + #[cfg(target_os = "macos")] mod vm; @@ -47,6 +50,9 @@ mod snapshot; #[cfg(target_os = "macos")] pub use backend::{VzBackend, VzConfig}; +#[cfg(target_os = "macos")] +pub use managed::{ManagedHelperArtifact, ManagedVzClosedReceipt, PreparedManagedVz}; + #[cfg(not(target_os = "macos"))] mod stub { use std::path::PathBuf; diff --git a/crates/engram-sandbox-vz/src/managed.rs b/crates/engram-sandbox-vz/src/managed.rs new file mode 100644 index 000000000..2bac2a1ec --- /dev/null +++ b/crates/engram-sandbox-vz/src/managed.rs @@ -0,0 +1,1068 @@ +//! Exact managed VZ ownership from CLOSED preparation through source retirement. +use engram_core::{ + types::sandbox::{SessionFileMetadata, SessionFileSpec, SessionFileStream}, + SandboxError, +}; +use std::{io, path::Path}; + +fn verify_file(path: &Path, digest: &str, size: Option) -> io::Result<()> { + use sha2::{Digest, Sha256}; + use std::io::Read; + let mut file = std::fs::File::open(path)?; + let metadata = file.metadata()?; + if !metadata.is_file() + || size.is_some_and(|s| s != metadata.len()) + || !engram_core::types::managed_cold::is_sha256(digest) + { + return Err(io::Error::other("invalid managed artifact")); + } + let mut hash = Sha256::new(); + let mut buf = [0; 65536]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hash.update(&buf[..n]); + } + if hex::encode(hash.finalize()) != digest { + return Err(io::Error::other("managed artifact mismatch")); + } + Ok(()) +} + +/// The selected helper artifact is copied and verified before execution. The +/// digest must come from the trusted installed host artifact set. +#[derive(Clone, Debug)] +pub struct ManagedHelperArtifact { + pub path: std::path::PathBuf, + pub sha256: String, +} + +/// A private VM and network owner. It never enters the ordinary sandbox map. +pub struct PreparedManagedVz { + // Drop the native machine before files or the attachment owner. + vm: crate::vm::VzVm, + helper: helper::Helper, + _files: PinnedFiles, + receipt: ManagedVzClosedReceipt, + failed: bool, + released_epoch: Option, + started: bool, + bridge: Option, + connector: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub struct ManagedVzClosedReceipt { + pub prepared: engram_core::types::managed_cold::ManagedColdPrepared, + pub allocation: engram_egress_proto::private::Allocation, + pub mac: String, + pub mtu: u32, + pub helper_sha256: String, +} + +impl PreparedManagedVz { + /// A fresh liveness check, never a final enforcement or start receipt. + pub async fn closed_receipt( + &mut self, + ) -> Result { + let native_closed = tokio::time::timeout( + std::time::Duration::from_secs(engram_egress_proto::private::HANDOFF_SECONDS), + self.vm.managed_attachment_closed(), + ) + .await + .unwrap_or(false); + if self.failed || !native_closed || self.helper.check_closed().is_err() { + self.failed = true; + let _ = self.helper.close().await; + return Err(io::Error::other("managed preparation is no longer live").into()); + } + Ok(self.receipt.clone()) + } + /// Keep the fresh VM stopped until the helper acknowledges the exact epoch. + pub async fn release(&mut self, epoch: u64) -> Result<(), engram_core::SandboxError> { + self.closed_receipt().await?; + self.failed = true; + self.helper.release(epoch).await?; + if !self.vm.managed_attachment_closed().await { + let _ = self.helper.close().await; + return Err(io::Error::other("managed attachment changed before release").into()); + } + self.released_epoch = Some(epoch); + self.failed = false; + Ok(()) + } + pub async fn check_ready(&mut self, epoch: u64) -> Result<(), engram_core::SandboxError> { + if self.failed || self.released_epoch != Some(epoch) || self.vm.is_dead() { + return Err(io::Error::other("managed owner is not ready").into()); + } + self.helper.check_ready(epoch)?; + if !self.vm.managed_attachment_current(self.started).await { + return Err(io::Error::other("managed attachment changed").into()); + } + Ok(()) + } + /// A single SpawnHarness write into a fresh kernel context. No reattach retry. + pub async fn start_agent( + &mut self, + agent: engram_core::types::sandbox::AgentSpec, + sink: engram_core::traits::HarnessSink, + ) -> Result<(), engram_core::SandboxError> { + let epoch = agent.binding_epoch; + self.check_ready(epoch).await?; + if self.started { + return Err(io::Error::other("managed start already issued").into()); + } + self.started = true; + // Preserve the exact source through an unknown start, Drop or close. + self._files.retain_source(); + self.failed = true; + self.vm.start().await?; + let dir = engram_core::socket::short_socket_dir(); + std::fs::create_dir_all(&dir)?; + let path = dir.join(format!( + "{}.managed", + self.receipt.prepared.request.sandbox_id + )); + let (bridge, connector) = crate::vsock_bridge::VsockBridge::start( + self.vm.raw_clone(), + self.vm.queue_clone(), + path, + Some(sink), + None, + None, + ) + .await + .map_err(io::Error::other)?; + self.bridge = Some(bridge); + self.connector = Some(connector.clone()); + let mut conn = connector.connect_stream(1024).await?; + let request = + engram_agentd::WireRequest::SpawnHarness(engram_agentd::SpawnHarnessRequest { + argv: agent.argv, + env: agent.env.into_iter().collect(), + session_env: agent.session_env.into_iter().collect(), + host_ca_pem: agent.host_ca_pem, + }); + engram_agentd::write_msg(&mut conn, &request).await?; + let response: engram_agentd::WireResponse = engram_agentd::read_msg(&mut conn).await?; + if !matches!(response, engram_agentd::WireResponse::HarnessSpawned { .. }) { + return Err(io::Error::other("managed harness start failed").into()); + } + self.helper.check_ready(epoch)?; + self.failed = false; + Ok(()) + } + pub fn owns(&self, request: &engram_core::types::managed_cold::ManagedColdPrepare) -> bool { + &self.receipt.prepared.request == request + } + async fn exec_connection( + &mut self, + epoch: u64, + ) -> Result { + self.check_ready(epoch).await?; + if !self.started { + return Err(io::Error::other("managed execution requires a started owner").into()); + } + Ok(self + .connector + .as_ref() + .ok_or_else(|| io::Error::other("managed connector unavailable"))? + .connect_stream(1024) + .await?) + } + pub async fn exec_stream( + &mut self, + epoch: u64, + cmd: engram_core::types::sandbox::ExecRequest, + ) -> Result { + let probe = self.exec_connection(epoch).await?; + let durable = crate::backend::probe_durable_exec(probe).await?; + let connection = self.exec_connection(epoch).await?; + let (reader, writer) = tokio::io::split(connection); + crate::backend::drive_exec_protocol( + self.receipt.prepared.request.sandbox_id, + reader, + writer, + cmd, + durable, + ) + .await + } + pub async fn prepare_read_file( + &mut self, + epoch: u64, + path: String, + mut closed: tokio::sync::watch::Receiver<()>, + ) -> Result< + impl std::future::Future< + Output = Result<(SessionFileMetadata, SessionFileStream), SandboxError>, + > + Send + + 'static, + SandboxError, + > { + let connection = self.exec_connection(epoch).await?; + // This future owns the exact connection and has no reconnect capability. + Ok(engram_agentd::file_transfer::read_file_until_closed( + connection, + path, + async move { + let _ = closed.changed().await; + }, + )) + } + pub async fn prepare_write_file( + &mut self, + epoch: u64, + spec: SessionFileSpec, + bytes: SessionFileStream, + ) -> Result< + impl std::future::Future> + Send + 'static, + SandboxError, + > { + let connection = self.exec_connection(epoch).await?; + Ok(engram_agentd::file_transfer::write_file( + connection, spec, bytes, + )) + } + pub async fn cancel_exec( + &mut self, + epoch: u64, + exec_id: String, + ) -> Result<(), engram_core::SandboxError> { + let mut connection = self.exec_connection(epoch).await?; + engram_agentd::write_msg( + &mut connection, + &engram_agentd::WireRequest::CancelExec { exec_id }, + ) + .await?; + match engram_agentd::read_msg::<_, engram_agentd::WireResponse>(&mut connection).await? { + engram_agentd::WireResponse::ExecCancelled => Ok(()), + _ => Err(io::Error::other("managed execution cancellation failed").into()), + } + } + /// Pause, retire the source, sync its disk, then copy the exact durable cut. + /// This makes no claim about unflushed guest page-cache data. + pub async fn retire_disk( + &mut self, + destination: &Path, + ) -> Result<(), engram_core::SandboxError> { + if self.failed { + return Err(io::Error::other("managed capture source unavailable").into()); + } + self._files.retain_source(); + if !self.started { + if let Some(epoch) = self.released_epoch { + self.check_ready(epoch).await?; + } else { + self.closed_receipt().await?; + } + } + self.failed = true; + let paused = if self.started { + self.vm.pause().await.map_err(SandboxError::from) + } else { + Ok(()) + }; + let cleanup = self.stop_resources().await; + paused?; + cleanup?; + let source = self._files.path.join("rootfs.ext4"); + let destination = destination.to_owned(); + tokio::task::spawn_blocking(move || { + std::fs::File::open(&source)?.sync_all()?; + std::fs::copy(&source, &destination)?; + std::fs::File::open(&destination)?.sync_all() + }) + .await + .map_err(io::Error::other)??; + self._files.capture_exported(); + Ok(()) + } + /// The host has durably recorded the exported file's exact hash and size. + pub fn capture_stored(&mut self) -> Result<(), SandboxError> { + self._files.capture_stored()?; + Ok(()) + } + + async fn stop_resources(&mut self) -> Result<(), SandboxError> { + let stopped = async { + if !self.vm.managed_attachment_closed().await { + self.vm.stop().await?; + if !self.vm.managed_attachment_closed().await { + return Err(io::Error::other("managed VM stop was not confirmed").into()); + } + } + self.started = false; + Ok::<_, SandboxError>(()) + } + .await; + if let Some(mut bridge) = self.bridge.take() { + bridge.stop().await; + } + self.connector = None; + let network = self.helper.close().await; + stopped?; + network?; + Ok(()) + } + + /// Await owned process cleanup without blocking the executor. Drop starts + /// cleanup but does not wait for its result. Success does not acknowledge the proxy + /// policy barrier. Only that proxy can release its lease or poison its gate. + pub async fn close(self) -> Result<(), engram_core::SandboxError> { + self.close_inner(false).await + } + /// An authenticated delete can release a known source after native cleanup. + pub async fn destroy(self) -> Result<(), engram_core::SandboxError> { + self.close_inner(true).await + } + async fn close_inner(mut self, destroy: bool) -> Result<(), SandboxError> { + let release = self._files.begin_close(destroy, self.failed); + self.failed = true; + self.stop_resources().await?; + if !release { + return Err(io::Error::other("managed capture source retained").into()); + } + self._files.release().await?; + Ok(()) + } +} + +pub(crate) async fn prepare( + work_dir: &Path, + cfg: &crate::backend::VzConfig, + request: engram_core::types::managed_cold::ManagedColdPrepare, + spec: engram_core::types::sandbox::SandboxSpec, + network: engram_egress_proto::private::RegisteredPrivateNetwork, + artifact: ManagedHelperArtifact, + staging: tempfile::TempDir, +) -> Result { + use engram_core::{ + types::managed_cold::{ManagedColdPrepared, ManagedColdState}, + SandboxError, + }; + request.validate(&spec).map_err(SandboxError::InvalidSpec)?; + let allocation = network.allocation().clone(); + if allocation.session_id != request.session_id + || allocation.sandbox_id != request.sandbox_id + || allocation.boot_attempt != request.boot_attempt + { + return Err(SandboxError::InvalidSpec( + "managed allocation does not match cold request".into(), + )); + } + let private_stream = network.into_stream(); + let pin = { + let request = request.clone(); + let spec = spec.clone(); + let cfg = cfg.clone(); + let work_dir = work_dir.to_owned(); + let artifact = artifact.clone(); + tokio::task::spawn_blocking(move || { + PinnedFiles::prepare(&work_dir, &cfg, &request, &spec, &artifact, staging) + }) + }; + // Before a helper exists, this stream owns no packet or flow work. Return + // cleanup EOF at once on proxy stop, even if the disk copy is still running. + // The blocking task owns its files and removes them when its result drops. + let files = tokio::select! { + biased; + _ = wait_for_private_stop(&private_stream) => { + drop(private_stream); + return Err(io::Error::other("private allocation closed during preparation").into()); + } + result = pin => result.map_err(|_| io::Error::other("artifact preparation failed"))??, + }; + let (packet, peer) = std::os::unix::net::UnixDatagram::pair()?; + configure_packet(&packet)?; + configure_packet(&peer)?; + // A point-to-point attachment has no shared Ethernet segment. Its fixed + // unicast source is selected here and shared with the exact helper config. + let mac = "02:00:00:00:00:02".to_owned(); + let vm_cfg = crate::vm::VmConfig::new( + files.path.join("kernel"), + files.path.join("rootfs.ext4"), + if spec.memory.max_mib > 0 { + spec.memory.max_mib + } else { + cfg.default_memory_mib + }, + if spec.cpu.vcpus > 0 { + spec.cpu.vcpus + } else { + cfg.default_vcpus + }, + ) + .with_aux_ro_drives(request.aux_ro_drives.clone(), files.path.clone()) + .with_swap_path( + spec.swap_mib + .filter(|m| *m > 0) + .map(|_| files.path.join("swap.img")), + ) + .with_machine_identifier(crate::vm::fresh_machine_identifier()) + .with_managed_network(crate::vm::ManagedVmNetwork { + packet: std::sync::Arc::new(packet), + guest: allocation.guest_ip, + gateway: allocation.gateway_ip, + mac: mac.clone(), + })?; + let helper = helper::Helper::spawn( + &files.path.join("netstack"), + peer, + private_stream, + &helper::Config { + version: 1, + guest: allocation.guest_ip, + gateway: allocation.gateway_ip, + mac: mac.clone(), + }, + ) + .await?; + // New kernel/device construction only. Neither start nor restore is called. + let vm = crate::vm::VzVm::new(vm_cfg)?; + let mut prepared = PreparedManagedVz { + vm, + helper, + _files: files, + failed: false, + released_epoch: None, + started: false, + bridge: None, + connector: None, + receipt: ManagedVzClosedReceipt { + prepared: ManagedColdPrepared { + version: 1, + request, + state: ManagedColdState::Closed, + }, + allocation, + mac, + mtu: 1500, + helper_sha256: artifact.sha256, + }, + }; + prepared.closed_receipt().await?; + Ok(prepared) +} + +async fn wait_for_private_stop(stream: &tokio::net::UnixStream) { + loop { + // No reply is valid here: registration has already consumed CLOSED and + // the unbound allocation has no operation in flight. Data also fails closed. + match stream.try_read(&mut [0]) { + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + if stream.readable().await.is_err() { + return; + } + } + _ => return, + } + } +} + +fn configure_packet(socket: &std::os::unix::net::UnixDatagram) -> io::Result<()> { + use std::os::fd::AsRawFd; + // Apple's attachment requires receive >= 2 * send. Bound both queues. + for (option, size) in [(libc::SO_SNDBUF, 16_384_i32), (libc::SO_RCVBUF, 65_536_i32)] { + // SAFETY: socket owns this FD, and the integer pointer and size agree. + let result = unsafe { + libc::setsockopt( + socket.as_raw_fd(), + libc::SOL_SOCKET, + option, + (&size as *const i32).cast(), + std::mem::size_of_val(&size) as libc::socklen_t, + ) + }; + if result != 0 { + return Err(io::Error::last_os_error()); + } + } + Ok(()) +} + +#[derive(Clone, Copy)] +enum Retention { + Disposable, + Source, + Exported, + Stored, +} + +struct PinnedFiles { + retention: Retention, + path: std::path::PathBuf, +} +impl PinnedFiles { + fn retain_source(&mut self) { + self.retention = Retention::Source; + } + fn capture_exported(&mut self) { + self.retention = Retention::Exported; + } + fn capture_stored(&mut self) -> io::Result<()> { + match self.retention { + Retention::Exported | Retention::Stored => { + self.retention = Retention::Stored; + Ok(()) + } + _ => Err(io::Error::other("managed capture export incomplete")), + } + } + fn begin_close(&mut self, destroy: bool, uncertain: bool) -> bool { + let release = matches!(self.retention, Retention::Disposable | Retention::Stored) + || (destroy && !uncertain); + // Cancellation or a failed native stop must retain even a prepared disk. + self.retain_source(); + release + } + async fn release(&mut self) -> io::Result<()> { + tokio::fs::remove_dir_all(&self.path).await?; + self.retention = Retention::Disposable; + Ok(()) + } + + fn prepare( + work_dir: &Path, + cfg: &crate::backend::VzConfig, + request: &engram_core::types::managed_cold::ManagedColdPrepare, + spec: &engram_core::types::sandbox::SandboxSpec, + artifact: &ManagedHelperArtifact, + _staging: tempfile::TempDir, + ) -> Result { + use std::os::unix::fs::DirBuilderExt; + std::fs::create_dir_all(work_dir)?; + let path = work_dir.join(format!( + "managed-{}-{}", + request.sandbox_id, request.boot_attempt + )); + // No overwrite or adoption of an earlier attempt's directory. + std::fs::DirBuilder::new().mode(0o700).create(&path)?; + let files = Self { + path, + retention: Retention::Disposable, + }; + files.pin( + spec.rootfs_source + .as_deref() + .ok_or_else(|| io::Error::other("missing disk"))?, + "rootfs.ext4", + &request.disk.sha256, + Some(request.disk.size_bytes), + )?; + files.pin(&cfg.kernel_path, "kernel", &request.kernel_sha256, None)?; + for drive in &request.aux_ro_drives { + let hash = drive + .sha256 + .as_deref() + .ok_or_else(|| io::Error::other("unresolved boot drive"))?; + let filename = format!("{hash}.squashfs"); + // Several slots can intentionally have the same immutable content. + if !files.path.join(&filename).exists() { + files.pin(&cfg.bundle_dir.join(&filename), &filename, hash, None)?; + } + } + use std::os::unix::fs::MetadataExt; + let helper_metadata = std::fs::symlink_metadata(&artifact.path)?; + // SAFETY: geteuid only reads this process identity. + let owner = unsafe { libc::geteuid() }; + if !artifact.path.is_absolute() + || !helper_metadata.is_file() + || helper_metadata.mode() & 0o022 != 0 + || (helper_metadata.uid() != 0 && helper_metadata.uid() != owner) + { + return Err(io::Error::other("managed helper is not operator owned").into()); + } + files.pin(&artifact.path, "netstack", &artifact.sha256, None)?; + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + files.path.join("netstack"), + std::fs::Permissions::from_mode(0o500), + )?; + if let Some(mib) = spec.swap_mib.filter(|m| *m > 0) { + let swap = std::fs::File::create_new(files.path.join("swap.img"))?; + swap.set_len(u64::from(mib) * 1024 * 1024)?; + } + Ok(files) + } + fn pin( + &self, + source: &Path, + name: &str, + digest: &str, + size: Option, + ) -> Result<(), engram_core::SandboxError> { + let dest = self.path.join(name); + std::fs::copy(source, &dest)?; + if !std::fs::symlink_metadata(&dest)?.is_file() { + return Err(io::Error::other("managed artifact is not a regular file").into()); + } + verify_file(&dest, digest, size)?; + Ok(()) + } +} +impl Drop for PinnedFiles { + fn drop(&mut self) { + if !matches!(self.retention, Retention::Disposable) { + return; + } + let _ = std::fs::remove_dir_all(&self.path); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sha2::{Digest, Sha256}; + + #[tokio::test] + async fn approved_cleanup_releases_only_recorded_capture_or_known_destroy() { + for (destroy, uncertain, exported, recorded, released) in [ + (true, false, false, false, true), + (false, false, false, false, false), + (true, true, false, false, false), + (true, true, true, false, false), + (false, true, true, true, true), + ] { + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, spec, artifact) = inputs(dir.path()); + let mut files = PinnedFiles::prepare( + &dir.path().join("owned"), + &cfg, + &request, + &spec, + &artifact, + tempfile::tempdir().unwrap(), + ) + .unwrap(); + let source = files.path.clone(); + files.retain_source(); + assert!(files.capture_stored().is_err()); + if exported { + files.capture_exported(); + } + if recorded { + files.capture_stored().unwrap(); + } + let authorized = files.begin_close(destroy, uncertain); + assert_eq!(authorized, released); + if authorized { + files.release().await.unwrap(); + } + drop(files); + assert_eq!(!source.exists(), released); + assert_eq!(std::fs::read(dir.path().join("disk")).unwrap(), b"disk-a"); + } + } + + #[tokio::test] + async fn unfinished_cleanup_and_file_removal_error_preserve_source() { + for removal_error in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, spec, artifact) = inputs(dir.path()); + let mut files = PinnedFiles::prepare( + &dir.path().join("owned"), + &cfg, + &request, + &spec, + &artifact, + tempfile::tempdir().unwrap(), + ) + .unwrap(); + let source = files.path.clone(); + files.retain_source(); + assert!(files.begin_close(true, false)); + if removal_error { + std::fs::remove_dir_all(&source).unwrap(); + std::fs::write(&source, b"retained").unwrap(); + assert_eq!( + files.release().await.unwrap_err().kind(), + io::ErrorKind::NotADirectory + ); + } + // Native cleanup failure or cancellation must not grant file release. + drop(files); + assert!(source.exists()); + } + } + #[tokio::test] + async fn proxy_stop_is_observed_before_any_helper_or_packet_work() { + use tokio::io::AsyncWriteExt; + let (client, mut proxy) = tokio::net::UnixStream::pair().unwrap(); + proxy.shutdown().await.unwrap(); + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(100), + wait_for_private_stop(&client) + ) + .await + .is_ok(), + "proxy drain must not wait for disk pinning" + ); + } + + fn inputs( + dir: &Path, + ) -> ( + crate::backend::VzConfig, + engram_core::types::managed_cold::ManagedColdPrepare, + engram_core::types::sandbox::SandboxSpec, + ManagedHelperArtifact, + ) { + use engram_core::types::{ + managed_cold::{ManagedColdDisk, ManagedColdPrepare, ManagedColdSource}, + manifest::ManifestRef, + sandbox::AuxRoDrive, + }; + let hash = |s: &[u8]| hex::encode(Sha256::digest(s)); + for (name, bytes) in [ + ("kernel", b"kernel-a".as_slice()), + ("disk", b"disk-a"), + ("helper", b"helper-a"), + ] { + std::fs::write(dir.join(name), bytes).unwrap(); + } + let bundle = hash(b"bundle-a"); + std::fs::write(dir.join(format!("{bundle}.squashfs")), b"bundle-a").unwrap(); + // The managed path must not resolve the current host's symbolic stamp. + std::fs::write(dir.join("current.json"), b"invalid-current-stamp").unwrap(); + let drives: Vec<_> = (0..3) + .map(|i| AuxRoDrive { + drive_id: AuxRoDrive::slot_drive_id(i), + guest_mount: AuxRoDrive::slot_guest_mount(i), + fs_type: "squashfs".into(), + sha256: Some(bundle.clone()), + }) + .collect(); + let manifest = ManifestRef::new(); + let spec: engram_core::types::sandbox::SandboxSpec = + serde_json::from_value(serde_json::json!({ + "image": "image-v1", "rootfs_source": dir.join("disk"), "rootfs_manifest": manifest, + "cpu": {"vcpus": 1}, "memory": {"max_mib": 128}, "disk": {"max_gib": 1}, + "env": {}, "aux_ro_drives": drives, + })) + .unwrap(); + let request = ManagedColdPrepare { + version: 1, + session_id: engram_core::SessionId::new(), + sandbox_id: engram_core::SandboxId::new(), + boot_attempt: "boot-1".into(), + image: "image-v1".into(), + disk: ManagedColdDisk { + manifest, + sha256: hash(b"disk-a"), + size_bytes: 6, + source: ManagedColdSource::FreshImage { + base_manifest: ManifestRef::new(), + }, + }, + kernel_sha256: hash(b"kernel-a"), + aux_ro_drives: spec.aux_ro_drives.clone(), + }; + ( + crate::backend::VzConfig::with_kernel(dir.join("kernel")).with_bundle_dir(dir), + request, + spec, + ManagedHelperArtifact { + path: dir.join("helper"), + sha256: hash(b"helper-a"), + }, + ) + } + + #[test] + fn staging_is_removed_after_pin_success_or_verification_failure() { + for valid in [true, false] { + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, mut spec, artifact) = inputs(dir.path()); + let staging = tempfile::tempdir_in(dir.path()).unwrap(); + let path = staging.path().to_owned(); + let disk = path.join("selected.ext4"); + std::fs::write(&disk, if valid { b"disk-a" } else { b"bad-aa" }).unwrap(); + spec.rootfs_source = Some(disk); + let result = PinnedFiles::prepare( + &dir.path().join("owned"), + &cfg, + &request, + &spec, + &artifact, + staging, + ); + assert!(!path.exists(), "staging must end with the pin operation"); + if valid { + let files = result.unwrap(); + assert_eq!( + std::fs::read(files.path.join("rootfs.ext4")).unwrap(), + b"disk-a" + ); + } else { + assert!(result.is_err()); + } + } + } + + #[test] + fn retirement_failure_retains_source_and_untrusted_helper_mode_is_rejected() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, spec, artifact) = inputs(dir.path()); + let mut files = PinnedFiles::prepare( + &dir.path().join("owned"), + &cfg, + &request, + &spec, + &artifact, + tempfile::tempdir().unwrap(), + ) + .unwrap(); + let path = files.path.clone(); + files.retain_source(); + drop(files); + assert_eq!(std::fs::read(path.join("rootfs.ext4")).unwrap(), b"disk-a"); + std::fs::set_permissions(&artifact.path, std::fs::Permissions::from_mode(0o666)).unwrap(); + assert!(PinnedFiles::prepare( + &dir.path().join("other"), + &cfg, + &request, + &spec, + &artifact, + tempfile::tempdir().unwrap() + ) + .is_err()); + } + #[test] + fn all_selected_boot_files_are_private_and_failed_attempts_leave_no_files() { + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, spec, artifact) = inputs(dir.path()); + request.validate(&spec).unwrap(); + let work = dir.path().join("work"); + let files = PinnedFiles::prepare( + &work, + &cfg, + &request, + &spec, + &artifact, + tempfile::tempdir().unwrap(), + ) + .unwrap(); + let path = files.path.clone(); + assert_eq!(std::fs::read(path.join("kernel")).unwrap(), b"kernel-a"); + assert_eq!(std::fs::read(path.join("netstack")).unwrap(), b"helper-a"); + let name = format!( + "{}.squashfs", + request.aux_ro_drives[1].sha256.as_ref().unwrap() + ); + assert_eq!(std::fs::read(path.join(name)).unwrap(), b"bundle-a"); + assert!(PinnedFiles::prepare( + &work, + &cfg, + &request, + &spec, + &artifact, + tempfile::tempdir().unwrap() + ) + .is_err()); + assert!( + path.join("rootfs.ext4").exists(), + "collision must not remove the first owner" + ); + drop(files); + assert!(!path.exists()); + std::fs::write(&cfg.kernel_path, b"kernel-b").unwrap(); + assert!(PinnedFiles::prepare( + &work, + &cfg, + &request, + &spec, + &artifact, + tempfile::tempdir().unwrap() + ) + .is_err()); + assert!( + !path.exists(), + "failed pin must remove the private partial copy" + ); + assert_eq!(std::fs::read(&cfg.kernel_path).unwrap(), b"kernel-b"); + } + + #[tokio::test] + async fn allocation_mismatch_stops_before_artifacts_or_helper() { + use engram_egress_proto::private::{self, Allocation, Reply}; + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, mut spec, artifact) = inputs(dir.path()); + let staging = tempfile::tempdir_in(dir.path()).unwrap(); + let staging_path = staging.path().to_owned(); + let staged_disk = staging_path.join("selected.ext4"); + std::fs::copy(spec.rootfs_source.as_ref().unwrap(), &staged_disk).unwrap(); + spec.rootfs_source = Some(staged_disk); + let allocation = Allocation { + version: 1, + host_id: engram_core::HostId::new(), + host_incarnation: "host-1".into(), + proxy_incarnation: "proxy-1".into(), + session_id: request.session_id, + sandbox_id: engram_core::SandboxId::new(), + boot_attempt: request.boot_attempt.clone(), + attachment_generation: 1, + guest_ip: "192.0.2.2".parse().unwrap(), + gateway_ip: "192.0.2.1".parse().unwrap(), + policy_revision: 1, + policy_digest: format!("sha256:{}", "a".repeat(64)), + }; + let (client, mut server) = tokio::net::UnixStream::pair().unwrap(); + let peer = tokio::spawn(async move { + let _: engram_egress_proto::ToProxyd = + engram_egress_proto::read_frame(&mut server).await.unwrap(); + let _: Allocation = private::read(&mut server).await.unwrap(); + private::write( + &mut server, + &Reply { + version: 1, + sequence: 0, + epoch: 0, + state: "closed".into(), + }, + ) + .await + .unwrap(); + server + }); + let capability = private::register(client, &allocation).await.unwrap(); + let mut peer = peer.await.unwrap(); + let work = dir.path().join("work"); + let result = prepare(&work, &cfg, request, spec, capability, artifact, staging).await; + assert!(matches!( + result, + Err(engram_core::SandboxError::InvalidSpec(_)) + )); + assert!(!work.exists()); + assert!( + !staging_path.exists(), + "backend rejection must remove staging" + ); + use tokio::io::AsyncReadExt; + assert_eq!( + peer.read(&mut [0]).await.unwrap(), + 0, + "rejected capability must close" + ); + } + + #[test] + fn cancelled_prepare_keeps_staging_until_queued_pin_finishes() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap(); + runtime.block_on(async { + use engram_egress_proto::private::{self, Allocation, Reply}; + let dir = tempfile::tempdir().unwrap(); + let (cfg, request, mut spec, artifact) = inputs(dir.path()); + let staging = tempfile::tempdir_in(dir.path()).unwrap(); + let staging_path = staging.path().to_owned(); + let disk = staging_path.join("selected.ext4"); + std::fs::write(&disk, b"disk-a").unwrap(); + spec.rootfs_source = Some(disk); + let allocation = Allocation { + version: 1, + host_id: engram_core::HostId::new(), + host_incarnation: "host-1".into(), + proxy_incarnation: "proxy-1".into(), + session_id: request.session_id, + sandbox_id: request.sandbox_id, + boot_attempt: request.boot_attempt.clone(), + attachment_generation: 1, + guest_ip: "192.0.2.2".parse().unwrap(), + gateway_ip: "192.0.2.1".parse().unwrap(), + policy_revision: 1, + policy_digest: format!("sha256:{}", "a".repeat(64)), + }; + let (client, mut server) = tokio::net::UnixStream::pair().unwrap(); + let peer = tokio::spawn(async move { + let _: engram_egress_proto::ToProxyd = + engram_egress_proto::read_frame(&mut server).await.unwrap(); + let _: Allocation = private::read(&mut server).await.unwrap(); + private::write( + &mut server, + &Reply { + version: 1, + sequence: 0, + epoch: 0, + state: "closed".into(), + }, + ) + .await + .unwrap(); + server + }); + let capability = private::register(client, &allocation).await.unwrap(); + let peer = peer.await.unwrap(); + let (release, held) = std::sync::mpsc::channel(); + let (entered, ready) = tokio::sync::oneshot::channel(); + let blocker = tokio::task::spawn_blocking(move || { + entered.send(()).unwrap(); + held.recv().unwrap(); + }); + ready.await.unwrap(); + let work = dir.path().join("work"); + let task = tokio::spawn(async move { + prepare(&work, &cfg, request, spec, capability, artifact, staging).await + }); + tokio::task::yield_now().await; + drop(peer); + let result = task.await.unwrap(); + let retained_while_queued = staging_path.join("selected.ext4").is_file(); + release.send(()).unwrap(); + blocker.await.unwrap(); + // The queued pin runs before this queue barrier can complete. + tokio::task::spawn_blocking(|| ()).await.unwrap(); + assert!(result.is_err()); + assert!( + retained_while_queued, + "cancel must not unlink a pending pin input" + ); + assert!(!staging_path.exists(), "finished pin must remove staging"); + }); + } + + #[test] + fn pinned_disk_survives_source_replacement_and_rejects_changed_bytes() { + let dir = tempfile::tempdir().unwrap(); + let source = dir.path().join("source"); + std::fs::write(&source, b"disk-a").unwrap(); + let owned = dir.path().join("private"); + std::fs::create_dir(&owned).unwrap(); + let files = PinnedFiles { + retention: Retention::Disposable, + path: owned.clone(), + }; + let digest = hex::encode(Sha256::digest(b"disk-a")); + files.pin(&source, "rootfs.ext4", &digest, Some(6)).unwrap(); + std::fs::write(&source, b"disk-b").unwrap(); + assert_eq!(std::fs::read(owned.join("rootfs.ext4")).unwrap(), b"disk-a"); + assert!(files.pin(&source, "changed", &digest, Some(6)).is_err()); + drop(files); + assert!(!owned.exists()); + assert_eq!(std::fs::read(source).unwrap(), b"disk-b"); + } + + #[test] + fn materialized_bytes_must_match_selected_disk_even_at_same_size() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("disk"); + std::fs::write(&path, b"disk-a").unwrap(); + let digest = hex::encode(Sha256::digest(b"disk-a")); + assert!(verify_file(&path, &digest, Some(6)).is_ok()); + std::fs::write(&path, b"disk-b").unwrap(); + assert!( + verify_file(&path, &digest, Some(6)).is_err(), + "same size is not byte identity" + ); + assert!(verify_file(&path, &digest, Some(7)).is_err()); + } +} + +mod helper; diff --git a/crates/engram-sandbox-vz/src/managed/helper.rs b/crates/engram-sandbox-vz/src/managed/helper.rs new file mode 100644 index 000000000..cadc365c5 --- /dev/null +++ b/crates/engram-sandbox-vz/src/managed/helper.rs @@ -0,0 +1,891 @@ +//! The native spawn flag closes even unrelated inheritable host descriptors. +use engram_egress_proto::private::{self, Reply}; +use std::{ + ffi::CString, + io, + os::{ + fd::{AsRawFd, FromRawFd, OwnedFd}, + unix::{ffi::OsStrExt, net::UnixDatagram}, + }, + path::Path, + time::{Duration, Instant}, +}; +use tokio::net::UnixStream; + +const DEADLINE: Duration = Duration::from_secs(private::HANDOFF_SECONDS); + +#[derive(serde::Serialize)] +pub(super) struct Config { + pub version: u32, + pub guest: std::net::Ipv4Addr, + pub gateway: std::net::Ipv4Addr, + pub mac: String, +} + +async fn read_closed(control: &mut UnixStream) -> io::Result<()> { + let reply: Reply = tokio::time::timeout(DEADLINE, private::read(control)) + .await + .map_err(|_| private::invalid())??; + if reply + != (Reply { + version: 1, + sequence: 0, + epoch: 0, + state: "closed".into(), + }) + { + return Err(private::invalid()); + } + Ok(()) +} + +pub(super) struct Helper { + control: UnixStream, + child: Option, + cleanup: Cleanup, + cleanup_complete: bool, + cleanup_failed: bool, + released_epoch: Option, + release_pending: bool, +} +impl Helper { + pub async fn spawn( + path: &Path, + packet: UnixDatagram, + proxy: UnixStream, + config: &Config, + ) -> io::Result { + let (control, child_control) = UnixStream::pair()?; + // Reserve the cleanup thread before creating a process. Drop then needs + // neither a running Tokio runtime nor a new thread allocation. + let cleanup = Cleanup::new()?; + let child = Child::spawn( + path, + [ + packet.as_raw_fd(), + child_control.as_raw_fd(), + proxy.as_raw_fd(), + ], + )?; + // Only the helper retains the child endpoints after spawn. + drop((packet, child_control, proxy)); + let mut helper = Self { + control, + child: Some(child), + cleanup, + cleanup_complete: false, + cleanup_failed: false, + released_epoch: None, + release_pending: false, + }; + // Establish the Drop owner before the first cancellable handshake write. + private::write(&mut helper.control, config).await?; + read_closed(&mut helper.control).await?; + helper.check_closed()?; + Ok(helper) + } + /// Reliable-channel and process checks. Datagram EOF is never evidence. + pub fn check_closed(&mut self) -> io::Result<()> { + if self.released_epoch.is_some() || self.release_pending { + return Err(private::invalid()); + } + self.check_live() + } + pub fn check_ready(&mut self, epoch: u64) -> io::Result<()> { + if self.released_epoch != Some(epoch) || self.release_pending { + return Err(private::invalid()); + } + self.check_live() + } + /// One exact epoch, acknowledged by the helper after its private proxy bind. + /// A canceled handshake cannot be sent again on this owner. + pub async fn release(&mut self, epoch: u64) -> io::Result<()> { + if self.released_epoch == Some(epoch) && !self.release_pending { + return self.check_ready(epoch); + } + self.check_closed()?; + if epoch == 0 || epoch > 9_007_199_254_740_991 { + return Err(private::invalid()); + } + use std::os::fd::AsFd; + struct ReleaseGuard { + socket: std::os::unix::net::UnixStream, + armed: bool, + } + impl Drop for ReleaseGuard { + fn drop(&mut self) { + if self.armed { + let _ = self.socket.shutdown(std::net::Shutdown::Both); + } + } + } + let mut guard = ReleaseGuard { + socket: std::os::unix::net::UnixStream::from( + self.control.as_fd().try_clone_to_owned()?, + ), + armed: true, + }; + self.release_pending = true; + let result = tokio::time::timeout(DEADLINE, async { + private::write( + &mut self.control, + &serde_json::json!({"version":1,"kind":"open","epoch":epoch}), + ) + .await?; + let reply: Reply = private::read(&mut self.control).await?; + if reply + != (Reply { + version: 1, + sequence: 1, + epoch, + state: "ready".into(), + }) + { + return Err(private::invalid()); + } + Ok(()) + }) + .await + .map_err(|_| private::invalid()) + .and_then(|r| r); + if result.is_err() { + self.begin_close(); + return result; + } + guard.armed = false; + self.released_epoch = Some(epoch); + self.release_pending = false; + self.check_ready(epoch) + } + fn check_live(&mut self) -> io::Result<()> { + if self.cleanup_failed + || self + .child + .as_mut() + .ok_or_else(private::invalid)? + .poll()? + .is_some() + { + return Err(private::invalid()); + } + let mut byte = [0]; + // SAFETY: a live nonblocking socket, borrowed buffer, no FD transfer. + let n = unsafe { + libc::recv( + self.control.as_raw_fd(), + byte.as_mut_ptr().cast(), + 1, + libc::MSG_PEEK | libc::MSG_DONTWAIT, + ) + }; + if n < 0 && io::Error::last_os_error().kind() == io::ErrorKind::WouldBlock { + Ok(()) + } else { + Err(private::invalid()) + } + } + fn begin_close(&mut self) { + // SAFETY: control owns this live socket. Shutdown changes no FD ownership. + unsafe { + libc::shutdown(self.control.as_raw_fd(), libc::SHUT_RDWR); + } + if let Some(child) = self.child.take() { + // The reserved worker waits only on this channel before receiving + // the child; it cannot exit while its sender is still owned here. + self.cleanup + .sender + .take() + .expect("cleanup sender") + .send(child) + .expect("reserved cleanup worker"); + } + } + pub async fn close(&mut self) -> io::Result<()> { + self.begin_close(); + if self.cleanup_failed { + return Err(io::Error::other("network helper cleanup remains failed")); + } + if self.cleanup_complete { + return Ok(()); + } + // Keep the receiver in the owner. Canceling this await neither cancels + // the worker nor consumes its result; Drop does not wait for it. + let result = (&mut self.cleanup.result) + .await + .unwrap_or_else(|_| Err(io::Error::other("network helper cleanup worker failed"))); + self.cleanup_complete = true; + self.cleanup_failed = result.is_err(); + result + } +} +impl Drop for Helper { + fn drop(&mut self) { + self.begin_close(); + } +} + +/// A reserved thread owns the child from cleanup handoff through final reap. +/// Its result is local cleanup evidence only, never a proxy barrier ACK. +struct Cleanup { + sender: Option>, + result: tokio::sync::oneshot::Receiver>, +} +impl Cleanup { + fn new() -> io::Result { + let (sender, receiver) = std::sync::mpsc::channel::(); + let (result_tx, result) = tokio::sync::oneshot::channel(); + std::thread::Builder::new() + .name("managed-network-cleanup".into()) + .spawn(move || { + if let Ok(mut child) = receiver.recv() { + let result = child.reap_bounded(); + // Complete all fallback cleanup before publishing a result. + drop(child); + if let Err(error) = &result { + tracing::error!(%error, "network helper cleanup failed"); + } + let _ = result_tx.send(result); + } + })?; + Ok(Self { + sender: Some(sender), + result, + }) + } +} + +#[derive(Debug)] +struct Child { + pid: libc::pid_t, + status: Option, +} +impl Child { + fn spawn(path: &Path, descriptors: [i32; 3]) -> io::Result { + let path = CString::new(path.as_os_str().as_bytes()).map_err(|_| private::invalid())?; + // Duplicate above the standard slots. This also handles a host whose + // stdin was closed before socket allocation; dup2 actions cannot alias. + let mut owned = Vec::new(); + for fd in descriptors { + // SAFETY: input descriptors are borrowed from live owned sockets. + let copy = unsafe { libc::fcntl(fd, libc::F_DUPFD_CLOEXEC, 3) }; + if copy < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: fcntl returned a new descriptor owned here. + owned.push(unsafe { OwnedFd::from_raw_fd(copy) }); + } + let check = |code| { + if code == 0 { + Ok(()) + } else { + Err(io::Error::from_raw_os_error(code)) + } + }; + // SAFETY: initialized POSIX objects are destroyed on every path below. + // argv has only argv[0], envp is empty. All pointers live through spawn. + unsafe { + let mut attr = std::mem::zeroed(); + check(libc::posix_spawnattr_init(&mut attr))?; + let mut actions = std::mem::zeroed(); + let init = libc::posix_spawn_file_actions_init(&mut actions); + if init != 0 { + libc::posix_spawnattr_destroy(&mut attr); + return Err(io::Error::from_raw_os_error(init)); + } + let result = (|| { + check(libc::posix_spawnattr_setflags( + &mut attr, + libc::POSIX_SPAWN_CLOEXEC_DEFAULT as i16, + ))?; + for (slot, fd) in owned.iter().enumerate() { + check(libc::posix_spawn_file_actions_adddup2( + &mut actions, + fd.as_raw_fd(), + slot as i32, + ))?; + } + for fd in &owned { + check(libc::posix_spawn_file_actions_addclose( + &mut actions, + fd.as_raw_fd(), + ))?; + } + let argv = [path.as_ptr().cast_mut(), std::ptr::null_mut()]; + let envp = [std::ptr::null_mut()]; + let mut pid = 0; + check(libc::posix_spawn( + &mut pid, + path.as_ptr(), + &actions, + &attr, + argv.as_ptr(), + envp.as_ptr(), + ))?; + Ok(Self { pid, status: None }) + })(); + libc::posix_spawn_file_actions_destroy(&mut actions); + libc::posix_spawnattr_destroy(&mut attr); + result + } + } + fn poll(&mut self) -> io::Result> { + if self.status.is_some() { + return Ok(self.status); + } + let mut status = 0; + // SAFETY: only this owner waits on the unreaped child PID. + let result = unsafe { libc::waitpid(self.pid, &mut status, libc::WNOHANG) }; + if result == self.pid { + self.status = Some(status); + } else if result < 0 { + let err = io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::ECHILD) { + self.status = Some(-1); + } + if err.kind() != io::ErrorKind::Interrupted { + return Err(err); + } + } + Ok(self.status) + } + fn reap_bounded(&mut self) -> io::Result<()> { + let deadline = Instant::now() + DEADLINE; + while Instant::now() < deadline { + if self.poll()?.is_some() { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(10)); + } + // Killing an unresponsive child is cleanup, never a successful proxy + // drain. The proxy owns the stable poison decision and final ACK. + self.kill_and_reap(); + Err(io::Error::new( + io::ErrorKind::TimedOut, + "network helper cleanup failed", + )) + } + fn kill_and_reap(&mut self) { + if self.status.is_some() { + return; + } + // SAFETY: an unreaped owned child PID cannot be reused by another process. + unsafe { + libc::kill(self.pid, libc::SIGKILL); + } + loop { + let mut status = 0; + // SAFETY: this is the sole owner of this child. + let result = unsafe { libc::waitpid(self.pid, &mut status, 0) }; + if result == self.pid { + self.status = Some(status); + break; + } + if result < 0 && io::Error::last_os_error().kind() != io::ErrorKind::Interrupted { + break; + } + } + } +} +impl Drop for Child { + fn drop(&mut self) { + self.kill_and_reap(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[tokio::test(flavor = "current_thread")] + async fn pending_helper_cleanup_does_not_delay_an_unrelated_private_stop() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for await_close in [false, true] { + let (input, keep_input_open) = std::os::unix::net::UnixStream::pair().unwrap(); + let (control, output) = UnixStream::pair().unwrap(); + let (errors, _read_errors) = UnixStream::pair().unwrap(); + let cleanup = Cleanup::new().unwrap(); + let child = Child::spawn( + Path::new("/bin/cat"), + [input.as_raw_fd(), output.as_raw_fd(), errors.as_raw_fd()], + ) + .unwrap(); + let pid = child.pid; + drop((input, output, errors)); + let mut helper = Helper { + control, + child: Some(child), + cleanup, + cleanup_complete: false, + cleanup_failed: false, + released_epoch: None, + release_pending: false, + }; + let (allocation, mut proxy) = UnixStream::pair().unwrap(); + let started = Instant::now(); + let cleanup = tokio::spawn(async move { + if await_close { + helper.close().await.unwrap(); + } else { + drop(helper); + } + }); + // First poll must transfer cleanup, not wait for the held child. + tokio::task::yield_now().await; + proxy.shutdown().await.unwrap(); + let watcher = tokio::spawn(async move { + super::super::wait_for_private_stop(&allocation).await; + drop(allocation); + }); + assert_eq!( + tokio::time::timeout(Duration::from_millis(250), proxy.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert!( + started.elapsed() < Duration::from_millis(500), + "helper cleanup blocked the unrelated allocation stop" + ); + // SAFETY: signal zero only observes the owned fixture; it does not reap. + assert_eq!( + unsafe { libc::kill(pid, 0) }, + 0, + "child must still be held during unrelated cleanup" + ); + watcher.await.unwrap(); + if await_close { + assert!(!cleanup.is_finished(), "close must wait for the held child"); + // Cancellation drops the owner but cannot cancel the reserved reaper. + cleanup.abort(); + assert!(cleanup.await.unwrap_err().is_cancelled()); + } else { + cleanup.await.unwrap(); + } + drop(keep_input_open); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + // SAFETY: signal zero does not change the child or its ownership. + if unsafe { libc::kill(pid, 0) } < 0 { + assert_eq!(io::Error::last_os_error().raw_os_error(), Some(libc::ESRCH)); + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + } + } + + #[tokio::test] + async fn forced_cleanup_failure_is_terminal_and_child_is_reaped() { + let (input, _keep_input_open) = std::os::unix::net::UnixStream::pair().unwrap(); + let (control, output) = UnixStream::pair().unwrap(); + let (errors, _read_errors) = UnixStream::pair().unwrap(); + // cat blocks on input and does not implement the reliable close barrier. + let cleanup = Cleanup::new().unwrap(); + let child = Child::spawn( + Path::new("/bin/cat"), + [input.as_raw_fd(), output.as_raw_fd(), errors.as_raw_fd()], + ) + .unwrap(); + let pid = child.pid; + drop((input, output, errors)); + let mut helper = Helper { + control, + child: Some(child), + cleanup, + cleanup_complete: false, + cleanup_failed: false, + released_epoch: None, + release_pending: false, + }; + assert!( + tokio::time::timeout(Duration::from_millis(10), helper.close()) + .await + .is_err() + ); + assert!( + helper.check_closed().is_err(), + "cleanup cannot become ready again" + ); + assert!( + helper.close().await.is_err(), + "unresponsive helper must fail cleanup" + ); + assert!( + helper.close().await.is_err(), + "reaping cannot erase cleanup failure" + ); + // SAFETY: the owner must have reaped its child after the bounded kill. + assert_eq!( + unsafe { libc::waitpid(pid, std::ptr::null_mut(), libc::WNOHANG) }, + -1 + ); + assert_eq!( + io::Error::last_os_error().raw_os_error(), + Some(libc::ECHILD) + ); + } + + #[tokio::test] + async fn helper_exit_before_ready_closes_private_channel() { + use tokio::io::AsyncReadExt; + let (packet, _peer) = UnixDatagram::pair().unwrap(); + let (private, mut proxy) = UnixStream::pair().unwrap(); + let config = Config { + version: 1, + guest: "192.0.2.2".parse().unwrap(), + gateway: "192.0.2.1".parse().unwrap(), + mac: "02:00:00:00:00:02".into(), + }; + assert!( + Helper::spawn(Path::new("/usr/bin/true"), packet, private, &config) + .await + .is_err() + ); + assert_eq!( + tokio::time::timeout(DEADLINE, proxy.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + } + + #[tokio::test] + #[ignore = "requires the verified packaged native helper"] + async fn actual_helper_release_ack_and_active_dns_cleanup() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let path = std::path::PathBuf::from( + std::env::var_os("ENGRAM_SANDBOX_NETSTACK_TEST_BIN").expect("packaged helper"), + ); + let config = Config { + version: 1, + guest: "192.0.2.2".parse().unwrap(), + gateway: "192.0.2.1".parse().unwrap(), + mac: "02:00:00:00:00:02".into(), + }; + let (packet, peer) = UnixDatagram::pair().unwrap(); + let (private, mut proxy) = UnixStream::pair().unwrap(); + let mut helper = Helper::spawn(&path, packet, private, &config) + .await + .unwrap(); + let pid = helper.child.as_ref().unwrap().pid; + use futures::FutureExt; + let outcome = tokio::time::timeout( + Duration::from_secs(20), + std::panic::AssertUnwindSafe(async { + let ack = tokio::spawn(async move { + let bind: private::Request = private::read(&mut proxy).await.unwrap(); + assert_eq!(bind.epoch, 7); + assert_eq!(bind.sequence, 1); + assert!(matches!(bind.operation, private::Operation::Bind)); + private::write( + &mut proxy, + &Reply { + version: 1, + sequence: 1, + epoch: 7, + state: "ready".into(), + }, + ) + .await + .unwrap(); + proxy + }); + helper.release(7).await.unwrap(); + let mut proxy = ack.await.unwrap(); + helper.check_ready(7).unwrap(); + helper.release(7).await.unwrap(); + assert!(helper.release(8).await.is_err()); + assert!(helper.check_closed().is_err()); + // One UDP DNS datagram, entirely on the anonymous packet pair. The + // proxy holds its private stream; there is no host network socket. + let query = [ + 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, b'a', 7, b'e', b'x', b'a', b'm', b'p', + b'l', b'e', 0, 0, 1, 0, 1, + ]; + let mut frame = vec![0; 14 + 20 + 8 + query.len()]; + frame[..14].copy_from_slice(&[2, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2, 8, 0]); + let len = (20 + 8 + query.len()) as u16; + frame[14..34].copy_from_slice(&[ + 0x45, + 0, + (len >> 8) as u8, + len as u8, + 0, + 1, + 0, + 0, + 64, + 17, + 0, + 0, + 192, + 0, + 2, + 2, + 192, + 0, + 2, + 1, + ]); + let mut sum: u32 = frame[14..34] + .as_chunks::<2>() + .0 + .iter() + .map(|b| u32::from(u16::from_be_bytes([b[0], b[1]]))) + .sum(); + while sum > 65535 { + sum = (sum & 65535) + (sum >> 16); + } + frame[24..26].copy_from_slice(&(!(sum as u16)).to_be_bytes()); + let udp = (8 + query.len()) as u16; + frame[34..42].copy_from_slice(&[ + 0x12, + 0x34, + 0, + 53, + (udp >> 8) as u8, + udp as u8, + 0, + 0, + ]); + frame[42..].copy_from_slice(&query); + peer.send(&frame).unwrap(); + let flow: private::Request = + tokio::time::timeout(DEADLINE, private::read(&mut proxy)) + .await + .unwrap() + .unwrap(); + assert!(matches!( + flow.operation, + private::Operation::Dns { port: 53, .. } + )); + let mut dns = private::receive_stream(&proxy).await.unwrap(); + private::write( + &mut proxy, + &Reply { + version: 1, + sequence: flow.sequence, + epoch: 7, + state: "accepted".into(), + }, + ) + .await + .unwrap(); + assert_eq!(dns.read_u16().await.unwrap() as usize, query.len()); + let mut received = vec![0; query.len()]; + dns.read_exact(&mut received).await.unwrap(); + proxy.shutdown().await.unwrap(); + assert_eq!( + tokio::time::timeout(DEADLINE, proxy.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + assert_eq!( + tokio::time::timeout(DEADLINE, dns.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + }) + .catch_unwind(), + ) + .await; + helper.close().await.unwrap(); + outcome + .expect("bounded helper exercise") + .expect("helper assertions"); + assert!(helper.check_ready(7).is_err()); + // SAFETY: this test observes only its exact already-reaped child. + assert_eq!( + unsafe { libc::waitpid(pid, std::ptr::null_mut(), libc::WNOHANG) }, + -1 + ); + assert_eq!( + io::Error::last_os_error().raw_os_error(), + Some(libc::ECHILD) + ); + } + + #[tokio::test] + #[ignore = "requires the verified packaged native helper"] + async fn actual_helper_rejects_wrong_epoch_and_canceled_release() { + use futures::FutureExt; + let path = std::path::PathBuf::from( + std::env::var_os("ENGRAM_SANDBOX_NETSTACK_TEST_BIN").expect("packaged helper"), + ); + let config = Config { + version: 1, + guest: "192.0.2.2".parse().unwrap(), + gateway: "192.0.2.1".parse().unwrap(), + mac: "02:00:00:00:00:02".into(), + }; + for cancel in [false, true] { + let (packet, _peer) = UnixDatagram::pair().unwrap(); + let (private, mut proxy) = UnixStream::pair().unwrap(); + let mut helper = Helper::spawn(&path, packet, private, &config) + .await + .unwrap(); + let pid = helper.child.as_ref().unwrap().pid; + let outcome = tokio::time::timeout( + Duration::from_secs(15), + std::panic::AssertUnwindSafe(async { + if cancel { + assert!( + tokio::time::timeout(Duration::from_millis(20), helper.release(7)) + .await + .is_err() + ); + assert!(helper.release(7).await.is_err()); + } else { + let ack = tokio::spawn(async move { + let _: private::Request = private::read(&mut proxy).await.unwrap(); + private::write( + &mut proxy, + &Reply { + version: 1, + sequence: 1, + epoch: 8, + state: "ready".into(), + }, + ) + .await + .unwrap(); + proxy + }); + assert!(helper.release(7).await.is_err()); + drop(ack.await.unwrap()); + } + assert!(helper.check_ready(7).is_err()); + }) + .catch_unwind(), + ) + .await; + let _ = helper.close().await; + outcome + .expect("bounded invalid ACK exercise") + .expect("invalid ACK assertions"); + // SAFETY: observe only the exact child owned and reaped above. + assert_eq!( + unsafe { libc::waitpid(pid, std::ptr::null_mut(), libc::WNOHANG) }, + -1 + ); + assert_eq!( + io::Error::last_os_error().raw_os_error(), + Some(libc::ECHILD) + ); + } + } + + // Task 8 supplies the packaged helper and invokes this explicit target. + // A missing artifact fails; it never turns into a passing skip. + #[tokio::test] + #[ignore = "requires ENGRAM_SANDBOX_NETSTACK_TEST_BIN from the pinned helper build"] + async fn actual_helper_closed_lifetime_and_descriptor_isolation() { + use tokio::io::AsyncReadExt; + let path = std::path::PathBuf::from( + std::env::var_os("ENGRAM_SANDBOX_NETSTACK_TEST_BIN").expect("packaged helper"), + ); + let config = Config { + version: 1, + guest: "192.0.2.2".parse().unwrap(), + gateway: "192.0.2.1".parse().unwrap(), + mac: "02:00:00:00:00:02".into(), + }; + for lose_proxy in [false, true] { + let (packet, peer) = UnixDatagram::pair().unwrap(); + let (private, mut proxy) = UnixStream::pair().unwrap(); + let (extra, mut extra_peer) = UnixStream::pair().unwrap(); + // SAFETY: this fixture deliberately makes an unrelated FD inheritable. + assert_eq!( + unsafe { libc::fcntl(extra.as_raw_fd(), libc::F_SETFD, 0) }, + 0 + ); + let mut helper = Helper::spawn(&path, packet, private, &config) + .await + .unwrap(); + let pid = helper.child.as_ref().unwrap().pid; + drop(extra); + assert_eq!( + tokio::time::timeout(DEADLINE, extra_peer.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0, + "helper inherited unrelated host descriptor" + ); + helper.check_closed().unwrap(); + peer.send(&[0; 60]).unwrap(); + if lose_proxy { + drop(proxy); + tokio::time::timeout(DEADLINE, async { + while helper.check_closed().is_ok() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + helper.close().await.unwrap(); + drop(helper); + } else { + helper.close().await.unwrap(); + assert_eq!( + tokio::time::timeout(DEADLINE, proxy.read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); + drop(helper); + } + // SAFETY: the helper owner must already have reaped its own child. + assert_eq!( + unsafe { libc::waitpid(pid, std::ptr::null_mut(), libc::WNOHANG) }, + -1 + ); + assert_eq!( + io::Error::last_os_error().raw_os_error(), + Some(libc::ECHILD) + ); + peer.set_nonblocking(true).unwrap(); + let result = peer.recv(&mut [0; 1600]); + assert!( + matches!(&result, Err(e) if matches!(e.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::ConnectionReset)), + "unexpected packet result: {result:?}" + ); + } + } + + #[tokio::test] + async fn transport_readiness_must_be_exact_closed_epoch_zero() { + for (version, sequence, epoch, state) in [ + (1, 0, 0, "ready"), + (1, 0, 1, "closed"), + (1, 1, 0, "closed"), + (2, 0, 0, "closed"), + ] { + let (mut a, mut b) = UnixStream::pair().unwrap(); + private::write( + &mut b, + &Reply { + version, + sequence, + epoch, + state: state.into(), + }, + ) + .await + .unwrap(); + assert!( + read_closed(&mut a).await.is_err(), + "invalid CLOSED handshake accepted" + ); + } + } +} diff --git a/crates/engram-sandbox-vz/src/snapshot.rs b/crates/engram-sandbox-vz/src/snapshot.rs index bd1beb8ff..67d3301d2 100644 --- a/crates/engram-sandbox-vz/src/snapshot.rs +++ b/crates/engram-sandbox-vz/src/snapshot.rs @@ -44,10 +44,17 @@ pub(crate) struct VzSnapshotManifest { pub(crate) warm: Option, } +/// Default VZ device layout. Increase for incompatible device changes. +/// Revision 1 adds the virtio entropy device. +pub(crate) const CONFIGURATION_REVISION: u32 = 1; + /// The identity and effective config of the saved VM. One all-or-nothing /// block: a partial identity is not warm-eligible. #[derive(Clone, Debug, Serialize, Deserialize)] pub(crate) struct WarmMachineState { + /// Missing metadata is revision 0 and cannot use current warm state. + #[serde(default)] + pub(crate) configuration_revision: u32, /// The restoring VM's identifier and MAC must match the saved ones. pub(crate) machine_identifier: Vec, pub(crate) mac_address: String, @@ -121,9 +128,6 @@ pub(crate) async fn build_metadata( base_memory_manifest: None, base_disk_manifest: None, migration_source: None, - // ADR 0014: VZ stays portable-snapshot-agnostic. The wrap - // layer (PooledBackend) is FC-only on Linux; macOS dev paths - // don't go through BlobStorage upload yet. source_sandbox_id: None, state_blob_key: None, sidecar_blob_key: None, @@ -200,9 +204,25 @@ mod tests { ); } + #[test] + fn legacy_warm_configuration_revision_defaults_to_zero() { + let value = serde_json::json!({ + "machine_identifier": [1, 2, 3], + "mac_address": "02:00:00:00:00:02", + "kernel_cmdline": "console=hvc0", + "aux_ro_drives": [], + "memory_mib": 1024, + "vcpus": 2 + }); + let warm: WarmMachineState = serde_json::from_value(value).unwrap(); + let encoded = serde_json::to_value(warm).unwrap(); + assert_eq!(encoded["configuration_revision"], 0); + } + #[test] fn warm_block_round_trips() { let warm = WarmMachineState { + configuration_revision: CONFIGURATION_REVISION, machine_identifier: vec![1, 2, 3], mac_address: "0a:00:00:00:00:01".into(), kernel_cmdline: "console=hvc0 ENGRAM_EGRESS=1:2:3".into(), @@ -215,6 +235,7 @@ mod tests { let bytes = serde_json::to_vec(&m).unwrap(); let parsed: VzSnapshotManifest = serde_json::from_slice(&bytes).unwrap(); let w = parsed.warm.expect("warm survives"); + assert_eq!(w.configuration_revision, CONFIGURATION_REVISION); assert_eq!(w.machine_identifier, vec![1, 2, 3]); assert_eq!(w.mac_address, "0a:00:00:00:00:01"); assert_eq!(w.memory_mib, 1024); diff --git a/crates/engram-sandbox-vz/src/vm.rs b/crates/engram-sandbox-vz/src/vm.rs index 53c67ad7d..6ecbca6a2 100644 --- a/crates/engram-sandbox-vz/src/vm.rs +++ b/crates/engram-sandbox-vz/src/vm.rs @@ -71,6 +71,58 @@ pub(crate) struct VmConfig { /// state pins the whole effective device config, not just the /// machine identifier). `None` keeps the random default. pub mac_address: Option, + pub managed_network: Option, +} + +#[derive(Clone, Debug)] +pub(crate) struct ManagedVmNetwork { + pub packet: Arc, + pub guest: std::net::Ipv4Addr, + pub gateway: std::net::Ipv4Addr, + pub mac: String, +} + +impl ManagedVmNetwork { + fn validate(&self) -> Result<(), VzError> { + let parts: Vec<_> = self.mac.split(':').collect(); + let bytes: Option> = parts + .iter() + .map(|p| { + if p.len() != 2 { + return None; + } + u8::from_str_radix(p, 16).ok() + }) + .collect(); + let mac_valid = bytes.is_some_and(|m| { + m.len() == 6 && m[0] & 1 == 0 && m != [0; 6] && m != [2, 0, 0, 0, 0, 1] + }); + let valid_ip = |ip: std::net::Ipv4Addr| { + !ip.is_unspecified() + && !ip.is_multicast() + && !ip.is_loopback() + && !ip.is_broadcast() + && !ip.is_link_local() + && ip.octets()[0] != 0 + }; + if !mac_valid + || !valid_ip(self.guest) + || !valid_ip(self.gateway) + || self.guest == self.gateway + || self.guest.octets()[..3] != self.gateway.octets()[..3] + || [self.guest, self.gateway] + .iter() + .any(|ip| matches!(ip.octets()[3], 0 | 255)) + { + return Err(VzError::ConfigInvalid( + "invalid managed network shape".into(), + )); + } + self.packet + .peer_addr() + .map_err(|e| VzError::AttachmentFailed(e.to_string()))?; + Ok(()) + } } impl VmConfig { @@ -120,9 +172,20 @@ impl VmConfig { bundle_dir: std::path::PathBuf::new(), machine_identifier: None, mac_address: None, + managed_network: None, } } + pub fn with_managed_network(mut self, network: ManagedVmNetwork) -> Result { + network.validate()?; + // IP configuration is a host-selected boot input. No DHCP or legacy + // guest DNAT is used on this attachment. Enforcement stays in the host. + self.kernel_cmdline = format!("console=hvc0 tsc=reliable panic=0 root=/dev/vda rw quiet init=/sbin/engram-init ip={}::{}:255.255.255.0::eth0:off:{} engram_managed_dns={}", network.guest, network.gateway, network.gateway, network.gateway); + self.mac_address = Some(network.mac.clone()); + self.managed_network = Some(network); + Ok(self) + } + /// ADR 0096 spike: pin the platform machine identifier (see the /// field docs). pub fn with_machine_identifier(mut self, bytes: Vec) -> Self { @@ -339,6 +402,8 @@ pub(crate) struct VzVm { /// delegates): dropping this deallocates the delegate and the stop /// callbacks silently never fire. _delegate: Retained, + managed_attachment: Option>, + managed_attachment_identity: Option<(i32, (u64, u64))>, } // SAFETY: see `SendableVm` for the full argument. The `VzVm` itself @@ -380,6 +445,35 @@ impl VzVm { // becomes "live" when handed to VZVirtualMachine::init. let vz_cfg = build_configuration(&cfg)?; + // Retain the exact attachment object so replacement or FD reuse cannot + // satisfy a later liveness read. This is still a local configuration. + let managed_attachment = if cfg.managed_network.is_some() { + // SAFETY: freshly built configuration, no concurrent VM access. + let attachment = unsafe { vz_cfg.networkDevices().objectAtIndex(0).attachment() } + .and_then(|a| { + a.downcast::() + .ok() + }) + .ok_or_else(|| VzError::ConfigInvalid("managed attachment missing".into()))?; + Some(attachment) + } else { + None + }; + + let managed_attachment_identity = match &managed_attachment { + Some(attachment) => { + // SAFETY: the fresh retained attachment owns this descriptor. + let fd = unsafe { attachment.fileHandle().fileDescriptor() }; + Some(( + fd, + descriptor_identity(fd).ok_or_else(|| { + VzError::ConfigInvalid("managed descriptor missing".into()) + })?, + )) + } + None => None, + }; + // Validate. Returns `Result<(), Retained>`. // SAFETY: vz_cfg is freshly built and live for the call. let validation = unsafe { vz_cfg.validateWithError() }; @@ -440,9 +534,58 @@ impl VzVm { queue, dead, _delegate: delegate, + managed_attachment, + managed_attachment_identity, }) } + /// Check the actual connected device on the VM queue, not a guest report. + pub async fn managed_attachment_closed(&self) -> bool { + self.managed_attachment_current(false).await + } + + pub(crate) async fn managed_attachment_current(&self, running: bool) -> bool { + let (tx, rx) = oneshot::channel(); + let vm = Sendable(self.vm.clone()); + let expected = self.managed_attachment.clone().map(Sendable); + let identity = self.managed_attachment_identity; + self.queue.exec_async(move || { + let vm = vm; + let expected = expected; + // SAFETY: all live VM property access is on its dispatch queue. + let closed = unsafe { + let devices = vm.0.networkDevices(); + vm.0.state() + == if running { + VZVirtualMachineState::Running + } else { + VZVirtualMachineState::Stopped + } + && devices.count() == 1 + && devices + .objectAtIndex(0) + .attachment() + .and_then(|a| { + a.downcast::( + ) + .ok() + }) + .is_some_and(|a| { + expected.as_ref().is_some_and(|e| { + Retained::as_ptr(&a) == Retained::as_ptr(&e.0) + && a.maximumTransmissionUnit() == 1500 + && identity.as_ref().is_some_and(|(fd, stamp)| { + a.fileHandle().fileDescriptor() == *fd + && descriptor_identity(*fd).as_ref() == Some(stamp) + }) + }) + }) + }; + let _ = tx.send(closed); + }); + rx.await.unwrap_or(false) + } + /// ADR 0096: true once the stop delegate has fired — the guest /// stopped the VM or VZ stopped it with an error. The backend's /// `list()`/`probe_sandbox` key off this. @@ -671,10 +814,86 @@ pub(crate) fn aux_drives_in_slot_order(drives: &[AuxRoDrive]) -> Vec<&AuxRoDrive ordered } +fn descriptor_identity(fd: i32) -> Option<(u64, u64)> { + use std::os::{fd::BorrowedFd, unix::fs::MetadataExt}; + if fd < 0 { + return None; + } + // SAFETY: the retained native file handle owns this descriptor throughout + // the queue operation. The duplicate is used only for a metadata read. + let duplicate = unsafe { BorrowedFd::borrow_raw(fd) } + .try_clone_to_owned() + .ok()?; + let metadata = std::fs::File::from(duplicate).metadata().ok()?; + Some((metadata.dev(), metadata.ino())) +} + +fn build_network(cfg: &VmConfig) -> Result, VzError> { + // SAFETY: these fresh configuration objects are not attached to a running VM. + unsafe { + let net_dev = VZVirtioNetworkDeviceConfiguration::new(); + let attachment: Retained = + if let Some(network) = &cfg.managed_network { + use objc2_virtualization::VZFileHandleNetworkDeviceAttachment; + use std::os::fd::{AsRawFd, IntoRawFd}; + network.validate()?; + if cfg.mac_address.as_ref() != Some(&network.mac) { + return Err(VzError::ConfigInvalid( + "managed network MAC mismatch".into(), + )); + } + // The native attachment owns this duplicate. The source endpoint + // stays owned by the preparation object until VM destruction. + let fd = network + .packet + .try_clone() + .map_err(|e| VzError::AttachmentFailed(e.to_string()))? + .into_raw_fd(); + let handle = NSFileHandle::initWithFileDescriptor_closeOnDealloc( + NSFileHandle::alloc(), + fd, + true, + ); + let attachment = VZFileHandleNetworkDeviceAttachment::initWithFileHandle( + VZFileHandleNetworkDeviceAttachment::alloc(), + &handle, + ); + attachment.setMaximumTransmissionUnit(1500); + if attachment.maximumTransmissionUnit() != 1500 + || attachment.fileHandle().fileDescriptor() != fd + || network.packet.as_raw_fd() == fd + { + return Err(VzError::ConfigInvalid( + "managed network attachment mismatch".into(), + )); + } + Retained::cast_unchecked(attachment) + } else { + Retained::cast_unchecked(VZNATNetworkDeviceAttachment::new()) + }; + net_dev.setAttachment(Some(&attachment)); + // ADR 0096 spike round 2: pin the MAC when asked — the default + // is a fresh random VZMACAddress per configuration, and a + // machine-state restore requires the restoring config's MAC to + // MATCH the saved VM's (mismatch = generic VZErrorRestore=12). + if let Some(mac) = &cfg.mac_address { + let mac_ns = NSString::from_str(mac); + let mac_addr = objc2_virtualization::VZMACAddress::initWithString( + objc2_virtualization::VZMACAddress::alloc(), + &mac_ns, + ) + .ok_or_else(|| VzError::ConfigInvalid(format!("mac_address {mac:?} did not parse")))?; + net_dev.setMACAddress(&mac_addr); + } + Ok(net_dev) + } +} + /// Build a fully-configured `VZVirtualMachineConfiguration` for /// `cfg`. Linux boot, virtio-block rootfs, a `VZVirtioSocketDevice` /// (real virtio-vsock) for the host↔guest control/harness/relay -/// channels, virtio-net NAT, and a separate single-port virtio-console +/// channels, virtio-net NAT or a managed file-handle attachment, and a +/// separate single-port virtio-console /// wired to host stderr for the kernel boot log. /// /// ADR 0066 Phase 2: the transport swapped from a multi-port @@ -830,26 +1049,7 @@ fn build_configuration(cfg: &VmConfig) -> Result = - Retained::cast_unchecked(nat); - net_dev.setAttachment(Some(&nat_super)); - // A machine-state restore requires the restoring config's MAC to - // equal the saved VM's; the default is a fresh random one. - if let Some(mac) = &cfg.mac_address { - let mac_ns = NSString::from_str(mac); - let mac_addr = objc2_virtualization::VZMACAddress::initWithString( - objc2_virtualization::VZMACAddress::alloc(), - &mac_ns, - ) - .ok_or_else(|| VzError::ConfigInvalid(format!("mac_address {mac:?} did not parse")))?; - net_dev.setMACAddress(&mac_addr); - } + let net_dev = build_network(cfg)?; let net_dev_super: Retained = Retained::cast_unchecked(net_dev); let network_array: Retained> = @@ -976,6 +1176,59 @@ fn ns_error_message(err: &NSError) -> String { mod tests { use super::*; + #[test] + fn managed_attachment_is_file_handle_with_exact_mac_and_mtu() { + let (packet, peer) = std::os::unix::net::UnixDatagram::pair().unwrap(); + let mut cfg = VmConfig::new("kernel", "disk", 128, 1); + cfg.mac_address = Some("02:00:00:00:00:02".into()); + cfg.managed_network = Some(ManagedVmNetwork { + packet: Arc::new(packet), + guest: "192.0.2.2".parse().unwrap(), + gateway: "192.0.2.1".parse().unwrap(), + mac: "02:00:00:00:00:02".into(), + }); + let device = build_network(&cfg).unwrap(); + // SAFETY: fresh local configuration, no VM or queue access. + unsafe { + let attachment = device + .attachment() + .unwrap() + .downcast::() + .expect("managed network must never use NAT"); + assert_eq!(attachment.maximumTransmissionUnit(), 1500); + let byte = b"x"; + assert_eq!( + libc::send( + attachment.fileHandle().fileDescriptor(), + byte.as_ptr().cast(), + 1, + 0 + ), + 1 + ); + let mut received = [0]; + peer.set_read_timeout(Some(std::time::Duration::from_secs(1))) + .unwrap(); + assert_eq!(peer.recv(&mut received).unwrap(), 1); + assert_eq!(received, *b"x"); + assert_eq!( + device.MACAddress().string().to_string(), + "02:00:00:00:00:02" + ); + } + cfg.managed_network.as_mut().unwrap().gateway = "198.51.100.1".parse().unwrap(); + assert!( + build_network(&cfg).is_err(), + "gateway must be reachable in selected subnet" + ); + cfg.managed_network.as_mut().unwrap().gateway = "192.0.2.1".parse().unwrap(); + cfg.mac_address = Some("02:00:00:00:00:03".into()); + assert!( + build_network(&cfg).is_err(), + "backend MAC differs from helper" + ); + } + /// Without the `com.apple.security.virtualization` entitlement, /// `validateWithError` returns NSError code 7 (or a similar /// "configuration invalid" code). This test confirms that path @@ -1018,6 +1271,21 @@ mod tests { } } + #[test] + fn configuration_has_one_virtio_entropy_device() { + let kernel = tempfile::NamedTempFile::new().unwrap(); + let rootfs = tempfile::NamedTempFile::new().unwrap(); + let cfg = VmConfig::new(kernel.path(), rootfs.path(), 512, 1); + let configuration = build_configuration(&cfg).unwrap(); + // SAFETY: fresh local configuration, no VM or queue access. + let devices = unsafe { configuration.entropyDevices() }; + assert_eq!(devices.len(), 1); + assert!(devices + .objectAtIndex(0) + .downcast::() + .is_ok()); + } + #[test] fn staged_bundle_path_is_content_keyed() { let p = super::staged_bundle_path(std::path::Path::new("/var/shared"), "abc123"); diff --git a/crates/engram-sandbox-vz/tests/e2e_vz.rs b/crates/engram-sandbox-vz/tests/e2e_vz.rs index b6cff4824..0c1e15932 100644 --- a/crates/engram-sandbox-vz/tests/e2e_vz.rs +++ b/crates/engram-sandbox-vz/tests/e2e_vz.rs @@ -38,8 +38,7 @@ use engram_core::types::sandbox::{ AuxRoDrive, CpuLimit, DiskLimit, ExecEvent, ExecRequest, MemoryLimit, SandboxSpec, SessionDisks, }; use engram_core::types::snapshot::SnapshotMetadata; -use engram_core::SandboxError; -use engram_core::SandboxId; +use engram_core::{SandboxError, SandboxId}; use engram_harness_proto::{ read_msg, write_msg, ForgeOp, ForgeRequest, ForgeResponse, RelayAck, RelayConnect, PROXY_PORT_VSOCK_PORT, @@ -242,16 +241,16 @@ fn spec_with_skill(rootfs: &Path, sha: &str) -> SandboxSpec { /// Run one command to completion, returning (stdout, exit_code). async fn exec(backend: &VzBackend, id: SandboxId, sh: &str) -> (String, Option) { - try_exec(backend, id, sh).await.expect("exec_stream") + let (out, exit) = try_exec(backend, id, sh).await.expect("exec_stream"); + (out, exit.flatten()) } -/// `exec` for a guest that may refuse the dial: a paused VM fails fast -/// or hangs, and both are the outcome under test. +/// Preserve submission errors and distinguish stream loss from any Exit event. async fn try_exec( backend: &VzBackend, id: SandboxId, sh: &str, -) -> Result<(String, Option), SandboxError> { +) -> Result<(String, Option>), SandboxError> { let req = ExecRequest { command: vec!["/bin/sh".into(), "-c".into(), sh.into()], stdin: None, @@ -271,7 +270,7 @@ async fn try_exec( ExecEvent::Stdout(b) => out.push_str(&String::from_utf8_lossy(&b)), ExecEvent::Stderr(b) => out.push_str(&String::from_utf8_lossy(&b)), ExecEvent::Exit(c) => { - code = c; + code = Some(c); break; } ExecEvent::Refused(reason) => panic!("exec refused: {reason}"), @@ -280,6 +279,21 @@ async fn try_exec( Ok((out, code)) } +fn exec_has_terminal_exit(result: &Result<(String, Option>), SandboxError>) -> bool { + matches!(result, Ok((_, Some(_)))) +} + +#[test] +fn parked_exec_rejects_every_terminal_exit() { + assert!(!exec_has_terminal_exit(&Err(SandboxError::Unavailable( + "transport closed before submission".into(), + )))); + assert!(!exec_has_terminal_exit(&Ok((String::new(), None)))); + for code in [None, Some(0), Some(1)] { + assert!(exec_has_terminal_exit(&Ok((String::new(), Some(code))))); + } +} + /// Poll `guest_endpoints` until the in-VM agentd answers (proves boot + agentd up). async fn await_agent(backend: &VzBackend, id: SandboxId) { let start = std::time::Instant::now(); @@ -787,10 +801,9 @@ async fn e2e_vz_pause_freezes_and_resume_revives() { let (out, code) = exec(&backend, id, "echo pre-pause").await; assert_eq!(code, Some(0), "pre-pause exec; out={out}"); - // Park. A frozen guest can't serve a fresh exec: the vsock dial - // hangs (timeout), fails fast (`Err`), or the stream ends with no - // Exit event (`code=None`). Only a completed `Some(0)` exec proves - // the guest is still running. + // Park. A frozen guest can't serve a fresh exec: either the vsock + // dial hangs (timeout), submission fails, or the stream ends with NO + // Exit event. Any Exit event proves the guest made progress. backend.pause(id).await.expect("pause"); backend.pause(id).await.expect("pause is idempotent"); let frozen = tokio::time::timeout( @@ -799,7 +812,7 @@ async fn e2e_vz_pause_freezes_and_resume_revives() { ) .await; assert!( - !matches!(&frozen, Ok(Ok((_, Some(0))))), + !frozen.as_ref().is_ok_and(exec_has_terminal_exit), "exec completed against a paused VM — pause didn't freeze the guest: {frozen:?}", ); @@ -812,7 +825,7 @@ async fn e2e_vz_pause_freezes_and_resume_revives() { ) .await; assert!( - !matches!(&still_frozen, Ok(Ok((_, Some(0))))), + !still_frozen.as_ref().is_ok_and(exec_has_terminal_exit), "snapshot un-parked the VM as a side effect: {still_frozen:?}", ); diff --git a/crates/engram-sim/src/meta/managed_operator.rs b/crates/engram-sim/src/meta/managed_operator.rs new file mode 100644 index 000000000..4f30abd11 --- /dev/null +++ b/crates/engram-sim/src/meta/managed_operator.rs @@ -0,0 +1,51 @@ +use super::SimDb; +use engram_core::{types::managed_operator::*, MetaError, SessionId}; +use std::collections::BTreeSet; + +pub(super) fn ids(db: &SimDb) -> BTreeSet { + db.managed_creations + .keys() + .chain(db.managed_boots.keys()) + .copied() + .chain( + db.session_integration_policy + .iter() + .filter_map(|(id, raw)| { + let value: serde_json::Value = serde_json::from_str(raw).ok()?; + value + .get("managed_deny") + .filter(|v| !v.is_null()) + .map(|_| *id) + }), + ) + .collect() +} +pub(super) fn snapshot( + db: &SimDb, + sid: SessionId, + hosts: Vec, + first: bool, +) -> Result<(String, OperatorParticipant), MetaError> { + let s = db.sessions.get(&sid).ok_or(MetaError::NotFound)?; + let (raw, policy) = retain_generation( + db.session_integration_policy.get(&sid).map(String::as_str), + hosts, + first, + )?; + Ok(( + raw, + OperatorParticipant { + session_id: sid, + status: s.session.status.as_str().into(), + host_id: s.session.host_id, + sandbox_id: s.session.sandbox_id, + binding_epoch: u64::try_from(s.binding_epoch).map_err(|_| conflict())?, + held_creation: db.managed_creations.contains_key(&sid), + boot: db + .managed_boots + .get(&sid) + .map(|b| (b.request.clone(), b.revision, b.prepared.clone())), + policy, + }, + )) +} diff --git a/crates/engram-sim/src/meta/mcp_policy.rs b/crates/engram-sim/src/meta/mcp_policy.rs new file mode 100644 index 000000000..9288f7d71 --- /dev/null +++ b/crates/engram-sim/src/meta/mcp_policy.rs @@ -0,0 +1,150 @@ +use super::{SimDb, SimMetadataStore}; +use chrono::{DateTime, Utc}; +use engram_core::{ + types::{event::PersistedEvent, mcp_policy::*, outbox::*}, + MetaError, SessionId, +}; +use std::collections::BTreeSet; +pub(super) fn nodes(db: &SimDb, id: SessionId) -> Result, MetaError> { + let mut result = Vec::new(); + let mut seen = BTreeSet::new(); + let mut next = Some(id); + while let Some(id) = next { + if result.len() >= 32 || !seen.insert(id) { + return Err(conflict("policy ancestry unavailable")); + } + let row = db.sessions.get(&id).ok_or(MetaError::NotFound)?; + result.push(PolicyNode { + id, + binding_epoch: row.binding_epoch, + parent: row.mcp_policy_parent_id, + policy: row.mcp_write_policy.clone(), + terminal: row.session.status.is_terminal(), + active: row.session.status == engram_core::types::session::SessionState::Active, + }); + next = row.mcp_policy_parent_id; + } + Ok(result) +} +pub(super) fn append( + db: &mut SimDb, + id: SessionId, + kind: &str, + payload: serde_json::Value, + now: DateTime, +) -> Result { + let row = db.sessions.get_mut(&id).ok_or(MetaError::NotFound)?; + let idx = row.next_event_idx; + row.next_event_idx += 1; + row.session.last_event_at = Some(now); + row.updated_at = now; + let recovery_epoch = row.recovery_epoch; + db.session_events + .entry(id) + .or_default() + .push(PersistedEvent { + idx, + kind: kind.into(), + payload, + created_at: now, + recovery_epoch, + rewound_at: None, + }); + Ok(idx) +} +impl SimMetadataStore { + pub(super) fn policy_read(&self, id: SessionId) -> Result { + self.gate()?; + compose_policy(&nodes(&self.db.lock(), id)?) + } + pub(super) fn policy_review(&self, id: SessionId, call: &str) -> Result { + self.gate()?; + let db = self.db.lock(); + let ns = nodes(&db, id)?; + plan_review( + &ns[0], + db.session_events.get(&id).map_or(&[], Vec::as_slice), + call, + ) + } + pub(super) fn policy_fence( + &self, + ctx: &RequestPolicyContext, + expected: &ExecutionRequest, + write: bool, + ) -> Result { + self.gate()?; + let db = self.db.lock(); + let check = if write { + dispatch_fence + } else { + execution_identity_fence + }; + check( + &nodes(&db, ctx.session_id)?, + db.session_events + .get(&ctx.session_id) + .map_or(&[], Vec::as_slice), + ctx, + expected, + ) + } + pub(super) fn policy_accept( + &self, + input: &PlanAcceptance, + ) -> Result { + self.gate()?; + let id = input.review.policy_ref.session_id; + let now = self.now(); + let mut db = self.db.lock(); + let mut ns = nodes(&db, id)?; + let outbox_id = tool_result_outbox_id(id, &input.review.tool_call_id); + let payload = serde_json::json!({"tool_call_id":input.review.tool_call_id,"result_json":input.result_json,"mcp_policy_acceptance":input}); + if let Some(row) = db.outbox.get(&outbox_id) { + if row.session_id != id || row.kind != OutboxKind::ToolResult || row.payload != payload + { + return Err(conflict("plan result already accepted")); + } + return compose_policy(&ns); + } + let updated = accept_plan( + &ns[0], + db.session_events.get(&id).map_or(&[], Vec::as_slice), + input, + )?; + db.sessions + .get_mut(&id) + .ok_or(MetaError::NotFound)? + .mcp_write_policy = Some(updated.clone()); + db.outbox.insert( + outbox_id.clone(), + OutboxRow { + prompt_id: outbox_id, + session_id: id, + kind: OutboxKind::ToolResult, + payload, + created_at: now, + attempts: 0, + not_before: now, + delivered_at: None, + acked_at: None, + }, + ); + let idx = append( + &mut db, + id, + "tool_result_submitted", + serde_json::json!({"tool_call_id":input.review.tool_call_id,"result_json":input.result_json,"at":now}), + now, + )?; + ns[0].policy = Some(updated); + let p = compose_policy(&ns)?; + drop(db); + self.notify( + "session_events", + serde_json::json!({"session_id":id,"idx":idx}).to_string(), + ); + self.notify("session_outbox", id.to_string()); + Ok(p) + } +} diff --git a/crates/engram-sim/src/meta/mod.rs b/crates/engram-sim/src/meta/mod.rs index a16f1fd0b..8fb7893f4 100644 --- a/crates/engram-sim/src/meta/mod.rs +++ b/crates/engram-sim/src/meta/mod.rs @@ -42,6 +42,8 @@ use engram_core::traits::{Clock, Entropy}; use engram_core::MetaError; use parking_lot::Mutex; +mod managed_operator; +mod mcp_policy; mod store_impl; /// A would-be `pg_notify` recorded instead of fired. The D5 scheduler @@ -138,6 +140,8 @@ use engram_core::{HostId, SessionId, SnapshotId}; /// keeps that the domain type doesn't carry. #[derive(Clone, Debug)] pub struct SessRow { + pub mcp_write_policy: Option, + pub mcp_policy_parent_id: Option, pub session: Session, pub mem_budget_mib: i64, pub cpu_budget_vcpus: i32, @@ -221,6 +225,17 @@ pub struct SimDb { /// `session_broker_tokens`, PK `session_id`). pub broker_tokens: std::collections::BTreeMap, + pub org_secret_purposes: std::collections::BTreeMap< + String, + Option, + >, + pub org_secrets: std::collections::BTreeMap< + String, + ( + engram_core::types::org_secret::SealedOrgSecret, + engram_core::types::org_secret::OrgSecret, + ), + >, /// ADR 0106: sealed payloads keyed by opaque subject + provider. pub oauth_credentials: std::collections::BTreeMap< engram_core::types::oauth::OAuthCredentialKey, @@ -231,6 +246,14 @@ pub struct SimDb { /// state, not part of the domain credential type. pub oauth_refresh_claims: std::collections::BTreeMap>, + pub mcp_refresh_claims: std::collections::BTreeMap< + engram_core::types::oauth::OAuthCredentialKey, + engram_core::types::mcp::McpRefreshClaim, + >, + pub mcp_oauth_admission: + std::collections::BTreeMap, + pub mcp_flow_states: + std::collections::BTreeMap, pub oauth_flows: std::collections::BTreeMap, pub session_oauth_bindings: std::collections::BTreeMap, @@ -260,6 +283,35 @@ pub struct SimDb { std::collections::BTreeMap, pub session_capabilities: std::collections::BTreeMap>, + pub managed_captures: std::collections::BTreeMap< + SnapshotId, + ( + engram_core::types::managed_boot::ManagedCaptureRequest, + Option, + ), + >, + pub managed_boots: + std::collections::BTreeMap, + /// Actual published gate state; only explicit test fixtures seed this before a publisher exists. + pub managed_operator_gate: Option<(u64, String, bool)>, + pub operator_publication: engram_core::types::managed_operator::OperatorPublication, + pub operator_readiness: std::collections::BTreeMap< + (u64, SessionId), + ( + engram_core::types::managed_operator::OperatorParticipant, + Option, + ), + >, + pub managed_interrupted_floors: std::collections::BTreeMap, + pub managed_interrupted: std::collections::BTreeSet<(SessionId, u64)>, + pub managed_boot_selections: std::collections::BTreeMap< + SessionId, + engram_core::types::managed_bridge::ManagedBootSelection, + >, + pub managed_boot_intents: + std::collections::BTreeMap, + pub managed_creations: + std::collections::BTreeMap, pub session_integration_policy: std::collections::BTreeMap, /// `cold_bases`: content-keyed base snapshots, also pinned against GC. pub cold_bases: diff --git a/crates/engram-sim/src/meta/store_impl.rs b/crates/engram-sim/src/meta/store_impl.rs index edbd945d3..bb1401609 100644 --- a/crates/engram-sim/src/meta/store_impl.rs +++ b/crates/engram-sim/src/meta/store_impl.rs @@ -1,7 +1,7 @@ //! The `MetadataStore` impl. One lock per method; SQL semantics mirrored //! from `PostgresStore` (see each method's comment for the statement it //! replicates). Grouped by table family; unimplemented PG-semantic -//! methods panic via `sim_unimplemented!` at the bottom. +//! methods have explicit panic bodies. use engram_core::types::host::DeleteHostOutcome; use engram_core::types::host::{ @@ -26,6 +26,11 @@ use engram_core::types::capture_job::{ use engram_core::types::event::{ArtifactRow, EventCursor, PersistedEvent}; use engram_core::types::host::{HostRecord, HostStatus, ReservedBudget}; use engram_core::types::ids::CaptureJobId; +use engram_core::types::managed_boot::*; +use engram_core::types::managed_bridge::{ + ManagedBootIntent, ManagedBootSelection, ManagedCreationQuery, ManagedCreationState, + ManagedIntentAdvance, +}; use engram_core::types::manifest::ManifestRef; use engram_core::types::registry::{EnableJob, EnableJobState}; use engram_core::types::registry::{EnabledImage, RegistryCredential, SessionSecrets}; @@ -287,6 +292,82 @@ impl SimMetadataStore { } } +fn managed_boot_read_locked( + db: &SimDb, + key: &ManagedBootKey, +) -> Result { + key.validate()?; + managed_creation_source(db, &key.creation, false)?; + let r = db + .managed_boots + .get(&key.creation.session_id) + .ok_or_else(conflict)?; + let s = db + .sessions + .get(&key.creation.session_id) + .ok_or(MetaError::NotFound)?; + let t = &key.context.target; + if r.request.key != *key + || s.session.host_id != Some(t.host_id) + || s.session.sandbox_id.is_some_and(|id| id != t.sandbox_id) + || s.binding_epoch != t.binding_epoch as i64 + { + return Err(conflict()); + } + if s.session.sandbox_id.is_none() { + if r.phase != ManagedBootPhase::Interrupted { + return Err(conflict()); + } + r.retired.as_ref().ok_or_else(conflict)?.validate_for( + r.prepared.as_ref().ok_or_else(conflict)?, + &r.request.bootstrap_policy, + )?; + } + Ok(r.clone()) +} + +fn managed_creation_source( + db: &SimDb, + identity: &engram_core::types::managed_create::ManagedCreateIdentity, + require_held: bool, +) -> Result { + identity.validate()?; + let session = db + .sessions + .get(&identity.session_id) + .ok_or(MetaError::NotFound)?; + let stored = db + .managed_creations + .get(&identity.session_id) + .ok_or(MetaError::NotFound)?; + let conflict = + || MetaError::Conflict("managed creation association changed or unavailable".into()); + if require_held + && (session.session.status != SessionState::Queued + || session.session.host_id.is_some() + || session.session.sandbox_id.is_some() + || session.binding_epoch != 0) + { + return Err(conflict()); + } + if matches!( + session.session.status, + SessionState::Dead | SessionState::Completed | SessionState::Failed + ) || session.mcp_policy_parent_id != stored.identity.policy.parent_session_id + { + return Err(if require_held { + conflict() + } else { + MetaError::NotFound + }); + } + if stored.identity != *identity { + return Err(conflict()); + } + stored.native.validate()?; + Ok(stored.clone()) +} + #[async_trait] impl MetadataStore for SimMetadataStore { async fn session_binding_generations(&self, id: SessionId) -> Result<(u64, u64), MetaError> { @@ -643,6 +724,40 @@ impl MetadataStore for SimMetadataStore { // ================= sessions ================= + async fn read_mcp_write_policy( + &self, + id: SessionId, + ) -> Result { + self.policy_read(id) + } + async fn read_mcp_plan_review( + &self, + id: SessionId, + call: &str, + ) -> Result { + self.policy_review(id, call) + } + async fn accept_mcp_plan_exit( + &self, + input: &engram_core::types::mcp_policy::PlanAcceptance, + ) -> Result { + self.policy_accept(input) + } + async fn validate_mcp_dispatch_fence( + &self, + ctx: &engram_core::types::mcp_policy::RequestPolicyContext, + expected: &engram_core::types::mcp_policy::ExecutionRequest, + ) -> Result { + self.policy_fence(ctx, expected, true) + } + async fn validate_mcp_execution_identity( + &self, + ctx: &engram_core::types::mcp_policy::RequestPolicyContext, + expected: &engram_core::types::mcp_policy::ExecutionRequest, + ) -> Result { + self.policy_fence(ctx, expected, false) + } + /// `INSERT INTO sessions (id, status='pending', image_uri, mode, /// created_at, last_active_at) VALUES (...)`. async fn create_session(&self, spec: SessionSpec) -> Result { @@ -653,6 +768,8 @@ impl MetadataStore for SimMetadataStore { db.sessions.insert( id, SessRow { + mcp_write_policy: None, + mcp_policy_parent_id: None, session: Session { id, status: SessionState::Pending, @@ -741,17 +858,890 @@ impl MetadataStore for SimMetadataStore { .collect()) } - /// One transaction: pick_host_2d, then INSERT the session row as - /// pending-on-host (Placed) or queued (Queued) + satellites. + async fn read_operator_publication( + &self, + ) -> Result { + self.gate()?; + Ok(self.db.lock().operator_publication.clone()) + } + async fn stage_operator_publication( + &self, + expected: Option, + candidate: engram_core::types::managed_operator::OperatorCandidate, + ) -> Result { + self.gate()?; + let mut db = self.db.lock(); + let next = db.operator_publication.stage(expected, candidate)?; + if !next.admission_open { + if let Some(g) = &mut db.managed_operator_gate { + g.2 = false; + } + } + db.operator_publication = next.clone(); + Ok(next) + } + async fn list_operator_participants( + &self, + after: Option, + limit: u32, + ) -> Result, MetaError> { + self.gate()?; + if !(1..=512).contains(&limit) { + return Err(engram_core::types::managed_operator::conflict()); + } + Ok(super::managed_operator::ids(&self.db.lock()) + .into_iter() + .filter(|s| after.is_none_or(|a| *s > a)) + .take(limit as usize) + .collect()) + } + async fn retain_operator_participant( + &self, + g: &engram_core::types::managed_operator::OperatorGeneration, + sid: SessionId, + ) -> Result { + self.gate()?; + let mut db = self.db.lock(); + db.operator_publication.pending(g)?; + if !super::managed_operator::ids(&db).contains(&sid) { + return Err(engram_core::types::managed_operator::conflict()); + } + let (raw, p) = super::managed_operator::snapshot( + &db, + sid, + g.candidate.hosts()?, + !db.operator_readiness.contains_key(&(g.generation, sid)), + )?; + db.session_integration_policy.insert(sid, raw); + db.operator_readiness + .insert((g.generation, sid), (p.clone(), None)); + Ok(p) + } + async fn record_operator_readiness( + &self, + g: &engram_core::types::managed_operator::OperatorGeneration, + p: &engram_core::types::managed_operator::OperatorParticipant, + proof: engram_core::types::managed_operator::OperatorReadiness, + ) -> Result<(), MetaError> { + self.gate()?; + proof.validate(g, p)?; + let mut db = self.db.lock(); + db.operator_publication.pending(g)?; + if super::managed_operator::snapshot(&db, p.session_id, Vec::new(), false)?.1 != *p { + return Err(engram_core::types::managed_operator::conflict()); + } + if db + .operator_readiness + .get(&(g.generation, p.session_id)) + .is_none_or(|(stored, _)| stored != p) + { + return Err(engram_core::types::managed_operator::conflict()); + } + db.operator_readiness + .insert((g.generation, p.session_id), (p.clone(), Some(proof))); + Ok(()) + } + async fn publish_operator_generation( + &self, + g: &engram_core::types::managed_operator::OperatorGeneration, + ) -> Result { + self.gate()?; + let mut db = self.db.lock(); + let old = &db.operator_publication; + if old.pending.is_none() && old.published.as_ref() == Some(g) && old.admission_open { + return Ok(old.clone()); + } + old.pending(g)?; + for sid in super::managed_operator::ids(&db) { + let p = super::managed_operator::snapshot(&db, sid, Vec::new(), false)?.1; + let (recorded, proof) = db + .operator_readiness + .get(&(g.generation, sid)) + .ok_or_else(engram_core::types::managed_operator::conflict)?; + if &p != recorded { + return Err(engram_core::types::managed_operator::conflict()); + } + proof + .as_ref() + .ok_or_else(engram_core::types::managed_operator::conflict)? + .validate(g, &p)?; + } + db.managed_operator_gate = Some((g.generation, g.identity.clone(), true)); + db.operator_publication = engram_core::types::managed_operator::OperatorPublication { + published: Some(g.clone()), + pending: None, + admission_open: true, + }; + Ok(db.operator_publication.clone()) + } + + async fn read_managed_creation_state( + &self, + q: &ManagedCreationQuery, + ) -> Result { + self.gate()?; + let db = self.db.lock(); + let session = db.sessions.get(&q.identity.session_id); + let c = db.managed_creations.get(&q.identity.session_id); + if session + .zip(c) + .is_some_and(|(s, c)| s.mcp_policy_parent_id != c.identity.policy.parent_session_id) + { + return Err(conflict()); + } + let i = db + .managed_boot_intents + .values() + .filter(|i| i.query.identity.session_id == q.identity.session_id) + .max_by_key(|i| i.binding_epoch); + if session.is_some_and(|s| !s.session.status.is_terminal()) { + if let Some(b) = db.managed_boots.get(&q.identity.session_id) { + managed_boot_read_locked(&db, &b.request.key)?; + } + } + ManagedCreationState::from_parts( + q.clone(), + session.map(|r| r.session.status), + c, + db.managed_boots.get(&q.identity.session_id).cloned(), + i, + ) + } + async fn read_managed_boot_selection( + &self, + q: &ManagedCreationQuery, + ) -> Result { + self.gate()?; + let db = self.db.lock(); + q.matches( + db.managed_creations + .get(&q.identity.session_id) + .ok_or(MetaError::NotFound)?, + )?; + let s = db + .managed_boot_selections + .get(&q.identity.session_id) + .ok_or_else(conflict)? + .clone(); + s.validate()?; + Ok(s) + } + + async fn read_managed_boot_intent( + &self, + q: &ManagedCreationQuery, + op: &str, + ) -> Result, MetaError> { + self.gate()?; + let db = self.db.lock(); + q.matches( + db.managed_creations + .get(&q.identity.session_id) + .ok_or(MetaError::NotFound)?, + )?; + Ok(db + .managed_boot_intents + .values() + .find(|i| { + i.query.identity.session_id == q.identity.session_id && i.operation_key() == op + }) + .cloned()) + } + async fn begin_managed_boot_intent( + &self, + i: ManagedBootIntent, + ) -> Result { + self.gate()?; + i.validate()?; + if i.revision != 1 || i.host_id.is_some() || i.assets.is_some() || i.completed { + return Err(conflict()); + } + let mut db = self.db.lock(); + bridge_gate(&db)?; + q_match(&db, &i.query)?; + let original = db + .managed_boot_selections + .get(&i.query.identity.session_id) + .ok_or_else(conflict)?; + if let Some(old) = db.managed_boot_intents.values().find(|o| { + o.query.identity.session_id == i.query.identity.session_id + && o.operation_key() == i.operation_key() + }) { + if old.recovery.as_ref().map(|r| (&r.recovery_id, &r.source)) + != i.recovery.as_ref().map(|r| (&r.recovery_id, &r.source)) + { + return Err(conflict()); + } + return Ok(old.clone()); + } + managed_creation_source(&db, &i.query.identity, false)?; + if db + .managed_boot_intents + .values() + .any(|o| o.query.identity.session_id == i.query.identity.session_id && !o.completed) + { + return Err(conflict()); + } + let b = db.managed_boots.get(&i.query.identity.session_id); + if let Some(r) = &i.recovery { + let old = b.ok_or_else(conflict)?; + if old.request.key != r.source + || old.prepared.is_none() + || i.selection.image != old.request.cold.image + || i.selection.aux_ro_drives != old.request.cold.aux_ro_drives + || i.selection.launch != original.launch + || i.selection.image_digest != original.image_digest + || i.selection.base_manifest != original.base_manifest + || i.selection.vcpus != old.request.spec()?.cpu.vcpus + || i.selection.memory_mib != old.request.spec()?.memory.max_mib + || i.selection.disk_gib != old.request.spec()?.disk.max_gib + { + return Err(conflict()); + } + } else { + managed_creation_source(&db, &i.query.identity, true)?; + if b.is_some() || &i.selection != original { + return Err(conflict()); + } + } + if db.managed_boot_intents.contains_key(&i.intent_id) + || db.managed_boot_intents.values().any(|o| { + o.query.identity.session_id == i.query.identity.session_id + && o.binding_epoch == i.binding_epoch + }) + { + return Err(conflict()); + } + db.managed_boot_intents + .insert(i.intent_id.clone(), i.clone()); + db.chunk_generation += 1; + Ok(i) + } + async fn advance_managed_boot_intent( + &self, + id: &str, + r: u64, + c: ManagedIntentAdvance, + ) -> Result { + self.gate()?; + let mut db = self.db.lock(); + bridge_gate(&db)?; + let old = db + .managed_boot_intents + .get(id) + .ok_or_else(conflict)? + .clone(); + old.query + .matches(&managed_creation_source(&db, &old.query.identity, false)?)?; + if old.revision != r { + return Err(conflict()); + } + if matches!(c, ManagedIntentAdvance::Complete) { + let b = db + .managed_boots + .get(&old.query.identity.session_id) + .ok_or_else(conflict)?; + old.validate_claim(&b.request)?; + } + if !matches!(c, ManagedIntentAdvance::Complete) + && db + .managed_boots + .get(&old.query.identity.session_id) + .map(|b| &b.request.key) + != old.recovery.as_ref().map(|r| &r.source) + { + return Err(conflict()); + } + if let ManagedIntentAdvance::SelectHost(host) = &c { + if Self::pick_host_2d( + &db, + &[*host], + 0, + i64::from(old.selection.memory_mib), + i64::from(old.selection.vcpus), + ) != Some(*host) + { + return Err(conflict()); + } + } + let next = old.advance(c)?; + db.managed_boot_intents.insert(id.into(), next.clone()); + Ok(next) + } + async fn list_managed_boot_intents( + &self, + after: Option, + limit: u32, + ) -> Result, MetaError> { + self.gate()?; + let db = self.db.lock(); + let mut rows: Vec<_> = db + .managed_boot_intents + .values() + .filter(|i| { + !i.completed + && after.is_none_or(|a| i.query.identity.session_id > a) + && db + .sessions + .get(&i.query.identity.session_id) + .is_some_and(|s| { + !matches!( + s.session.status, + SessionState::Dead | SessionState::Failed | SessionState::Completed + ) + }) + }) + .cloned() + .collect(); + rows.sort_by_key(|i| i.query.identity.session_id); + rows.truncate(limit.min(128) as usize); + Ok(rows) + } + async fn get_managed_boot( + &self, + identity: &engram_core::types::managed_create::ManagedCreateIdentity, + ) -> Result, MetaError> { + self.gate()?; + let db = self.db.lock(); + managed_creation_source(&db, identity, false)?; + db.managed_boots + .get(&identity.session_id) + .map(|r| managed_boot_read_locked(&db, &r.request.key)) + .transpose() + } + async fn reserve_managed_capture( + &self, + request: &ManagedCaptureRequest, + ) -> Result, MetaError> { + self.gate()?; + request.key.validate()?; + if request.capture_id.as_uuid().is_nil() { + return Err(conflict()); + } + let mut db = self.db.lock(); + if let Some((original, Some(authority))) = db.managed_captures.get(&request.capture_id) { + if original != request { + return Err(conflict()); + } + managed_creation_source(&db, &request.key.creation, false)?; + return Ok(Some(authority.clone())); + } + let boot = managed_boot_read_locked(&db, &request.key)?; + if boot.phase != ManagedBootPhase::Interrupted || boot.prepared.is_none() { + return Err(conflict()); + } + if let Some((old, authority)) = db.managed_captures.get(&request.capture_id) { + if old != request { + return Err(conflict()); + } + return Ok(authority.clone()); + } + if db.managed_captures.values().any(|(r, _)| { + r.key.creation.session_id == request.key.creation.session_id + && r.key.context.target.binding_epoch == request.key.context.target.binding_epoch + }) { + return Err(conflict()); + } + db.managed_captures + .insert(request.capture_id, (request.clone(), None)); + db.chunk_generation += 1; + Ok(None) + } + async fn record_managed_capture( + &self, + authority: &ManagedCaptureAuthority, + ) -> Result<(), MetaError> { + self.gate()?; + authority.validate()?; + let mut db = self.db.lock(); + let request = &authority.request; + let boot = managed_boot_read_locked(&db, &request.key)?; + if boot.phase != ManagedBootPhase::Interrupted + || boot.prepared.as_ref().map(|r| &r.host_incarnation) + != Some(&authority.host_incarnation) + || boot.request.cold.image != authority.image + || boot.request.cold.kernel_sha256 != authority.kernel_sha256 + || boot.request.cold.aux_ro_drives != authority.aux_ro_drives + { + return Err(conflict()); + } + let (original, prior) = db + .managed_captures + .get_mut(&request.capture_id) + .ok_or_else(conflict)?; + if original != request || prior.as_ref().is_some_and(|p| p != authority) { + return Err(conflict()); + } + *prior = Some(authority.clone()); + let s = db + .sessions + .get_mut(&request.key.creation.session_id) + .ok_or(MetaError::NotFound)?; + s.session.status = SessionState::Idle; + s.session.last_active_at = self.now(); + s.updated_at = self.now(); + Ok(()) + } + async fn managed_sandbox_open(&self, sandbox: SandboxId) -> Result, MetaError> { + self.gate()?; + let db = self.db.lock(); + if let Some(r) = db + .managed_boots + .values() + .find(|r| r.request.key.context.target.sandbox_id == sandbox) + { + return Ok(Some( + r.phase == ManagedBootPhase::Started + && managed_operator_gate(&db, &r.request.key).is_ok() + && db + .sessions + .get(&r.request.key.creation.session_id) + .is_some_and(|s| s.session.status == SessionState::Active) + && managed_boot_read_locked(&db, &r.request.key).is_ok(), + )); + } + Ok(db + .managed_captures + .values() + .any(|(r, _)| r.key.context.target.sandbox_id == sandbox) + .then_some(false)) + } + async fn claim_managed_boot( + &self, + request: ManagedBootPrepare, + ) -> Result { + self.gate()?; + let spec = request.spec()?; + let mut db = self.db.lock(); + managed_operator_gate(&db, &request.key)?; + if let Some(i) = db.managed_boot_intents.get(&request.key.boot_attempt) { + i.validate_claim(&request)?; + } else if db + .managed_boot_selections + .contains_key(&request.key.creation.session_id) + { + return Err(conflict()); + } + let k = &request.key; + let t = &k.context.target; + let session = db.sessions.get(&t.session_id).ok_or(MetaError::NotFound)?; + if spec.image != session.session.image + || spec.cpu.vcpus == 0 + || i64::from(spec.cpu.vcpus) > i64::from(session.cpu_budget_vcpus) + || spec.memory.max_mib == 0 + || i64::from(spec.memory.max_mib) > session.mem_budget_mib + { + return Err(conflict()); + } + let old = if let Some(prior) = db.managed_boots.get(&t.session_id) { + let prior = managed_boot_read_locked(&db, &prior.request.key)?; + if prior.request == request && prior.phase != ManagedBootPhase::Interrupted { + return Ok(prior); + } + if prior.phase != ManagedBootPhase::Interrupted { + return Err(conflict()); + } + let engram_core::types::managed_cold::ManagedColdSource::Capture { capture_id, .. } = + request.cold.disk.source + else { + return Err(conflict()); + }; + let (_, capture) = db.managed_captures.get(&capture_id).ok_or_else(conflict)?; + let capture = capture.as_ref().ok_or_else(conflict)?; + if capture.request.key != prior.request.key { + return Err(conflict()); + } + capture.validate_recovery(&request)?; + if request.capture_authority.as_ref() != Some(capture) { + return Err(MetaError::Conflict("capture authority mismatch".into())); + } + Some(prior) + } else { + None + }; + let source = managed_creation_source(&db, &k.creation, old.is_none())?; + if source.native.digest != k.native_digest + || (old.is_none() + && (t.binding_epoch != 1 + || !matches!( + request.cold.disk.source, + engram_core::types::managed_cold::ManagedColdSource::FreshImage { .. } + ))) + { + return Err(conflict()); + } + let retained = engram_core::types::managed_deny::retain_in_policy_json( + db.session_integration_policy + .get(&t.session_id) + .map(String::as_str), + &request.bootstrap_policy, + ) + .map_err(MetaError::Conflict)?; + let s = db.sessions.get(&t.session_id).ok_or(MetaError::NotFound)?; + if Self::pick_host_2d( + &db, + &[t.host_id], + 0, + s.mem_budget_mib, + i64::from(s.cpu_budget_vcpus), + ) != Some(t.host_id) + || db.managed_boots.values().any(|r| { + r.request.key.boot_attempt == k.boot_attempt + || r.request.key.context.target.sandbox_id == t.sandbox_id + }) + { + return Err(conflict()); + } + let s = db + .sessions + .get_mut(&t.session_id) + .ok_or(MetaError::NotFound)?; + s.session.status = SessionState::Created; + s.session.host_id = Some(t.host_id); + s.session.sandbox_id = Some(t.sandbox_id); + s.binding_epoch = t.binding_epoch as i64; + s.queued_at = None; + s.queue_origin = None; + s.updated_at = self.now(); + s.session.last_active_at = self.now(); + db.session_integration_policy.insert(t.session_id, retained); + db.chunk_generation += 1; + let record = ManagedBootRecord::claimed(request); + db.managed_boots + .insert(record.request.key.creation.session_id, record.clone()); + Ok(record) + } + async fn read_managed_boot( + &self, + key: &ManagedBootKey, + ) -> Result { + self.gate()?; + managed_boot_read_locked(&self.db.lock(), key) + } + async fn read_managed_boot_for_target( + &self, + target: &engram_core::types::managed_deny::ManagedPolicyTarget, + ) -> Result { + self.gate()?; + let db = self.db.lock(); + let session = db + .sessions + .get(&target.session_id) + .ok_or(MetaError::NotFound)?; + if session.session.sandbox_id != Some(target.sandbox_id) { + return Err(conflict()); + } + let row = db + .managed_boots + .get(&target.session_id) + .ok_or_else(conflict)?; + if row.request.key.context.target != *target { + return Err(conflict()); + } + managed_boot_read_locked(&db, &row.request.key) + } + async fn advance_managed_boot( + &self, + key: &ManagedBootKey, + revision: u64, + event: ManagedBootAdvance, + ) -> Result { + self.gate()?; + let mut db = self.db.lock(); + if matches!( + &event, + ManagedBootAdvance::Finalize { .. } + | ManagedBootAdvance::IssueStart(_) + | ManagedBootAdvance::Started(_) + ) { + managed_operator_gate(&db, key)?; + } + let old = managed_boot_read_locked(&db, key)?; + if old.revision != revision { + return Err(conflict()); + } + let source = managed_creation_source(&db, &key.creation, false)?; + let policy = engram_core::types::mcp_policy::compose_policy(&super::mcp_policy::nodes( + &db, + key.creation.session_id, + )?)?; + match &event { + ManagedBootAdvance::Finalize { write_policy, .. } if write_policy != &policy => { + return Err(conflict()) + } + ManagedBootAdvance::IssueStart(_) | ManagedBootAdvance::Started(_) + if old.write_policy.as_ref() != Some(&policy) => + { + return Err(conflict()) + } + _ => {} + } + if let ManagedBootAdvance::Prepared(receipt) = &event { + if let Some(i) = db.managed_boot_intents.get(&key.boot_attempt) { + i.validate_claim(&receipt.request)?; + if i.assets.as_ref().map(|a| &a.host_incarnation) != Some(&receipt.host_incarnation) + { + return Err(conflict()); + } + } + } + let next = old.advance(event, &source.native)?; + let sid = key.creation.session_id; + if let Some(retired) = &next.retired { + let retained = engram_core::types::managed_deny::retain_retired_in_policy_json( + db.session_integration_policy.get(&sid).map(String::as_str), + &retired.retained_policy, + ) + .map_err(MetaError::Conflict)?; + db.session_integration_policy.insert(sid, retained); + } + let mut interruption_idx = None; + if next.phase == ManagedBootPhase::Interrupted && old.phase != ManagedBootPhase::Interrupted + { + let session = db.sessions.get_mut(&sid).ok_or(MetaError::NotFound)?; + let floor = session.next_event_idx; + session.mcp_write_policy = + Some(engram_core::types::mcp_policy::StoredWritePolicy::restrict( + session.mcp_write_policy.as_ref(), + &format!("cold:{}", key.boot_attempt), + floor, + )?); + db.managed_interrupted_floors.insert(sid, floor); + let run = db + .session_events + .get(&sid) + .and_then(|events| { + events.iter().rev().find(|e| { + e.rewound_at.is_none() + && matches!( + e.kind.as_str(), + "run_started" | "run_completed" | "run_interrupted" + ) + }) + }) + .filter(|e| e.kind == "run_started") + .and_then(|e| e.payload.get("run_id").and_then(|v| v.as_str())) + .map(str::to_string); + if let Some(run) = run { + interruption_idx = append_event_idempotent_locked( + &mut db, + sid, + &format!("managed-interrupt:{}:{run}", key.boot_attempt), + "run_interrupted", + serde_json::json!({"type":"harness_run_interrupted","run_id":run,"at":self.now()}), + self.now(), + )?; + } + db.managed_interrupted + .insert((sid, key.context.target.binding_epoch)); + for row in db.outbox.values_mut().filter(|r| { + r.session_id == sid + && r.kind == engram_core::types::outbox::OutboxKind::ToolResult + && r.acked_at.is_none() + }) { + row.acked_at = Some(self.now()); + } + } + if next.phase == ManagedBootPhase::Started { + let s = db.sessions.get_mut(&sid).ok_or(MetaError::NotFound)?; + s.session.status = SessionState::Active; + s.session.last_active_at = self.now(); + s.updated_at = self.now(); + } + db.managed_boots.insert(sid, next.clone()); + drop(db); + if let Some(idx) = interruption_idx { + self.notify( + "session_events", + serde_json::json!({"session_id":sid,"idx":idx}).to_string(), + ); + } + Ok(next) + } + async fn list_managed_boots( + &self, + after: Option, + limit: u32, + ) -> Result, MetaError> { + self.gate()?; + Ok(self + .db + .lock() + .managed_boots + .values() + .filter(|r| after.is_none_or(|id| r.request.key.creation.session_id > id)) + .filter(|r| { + matches!( + r.phase, + ManagedBootPhase::Claimed + | ManagedBootPhase::Prepared + | ManagedBootPhase::Finalizing + | ManagedBootPhase::Finalized + | ManagedBootPhase::StartIssued + | ManagedBootPhase::Started + ) + }) + .take(limit.min(128) as usize) + .map(|r| r.request.key.clone()) + .collect()) + } + async fn managed_boot_open(&self, sid: SessionId) -> Result, MetaError> { + self.gate()?; + let db = self.db.lock(); + if !db.managed_creations.contains_key(&sid) { + return Ok(None); + } + Ok(Some(db.managed_boots.get(&sid).is_some_and(|r| { + r.phase == ManagedBootPhase::Started + && managed_operator_gate(&db, &r.request.key).is_ok() + && db + .sessions + .get(&sid) + .is_some_and(|s| s.session.status == SessionState::Active) + && managed_boot_read_locked(&db, &r.request.key).is_ok() + }))) + } + async fn managed_tool_target_gone( + &self, + sid: SessionId, + call: &str, + ) -> Result { + self.gate()?; + let db = self.db.lock(); + let floor = db + .managed_interrupted_floors + .get(&sid) + .copied() + .unwrap_or(0); + Ok(db.managed_creations.contains_key(&sid) + && !db.session_events.get(&sid).is_some_and(|events| { + events + .iter() + .filter(|e| { + e.kind == "tool_call_requested" && e.payload["tool_call_id"] == call + }) + .map(|e| e.idx) + .min() + .is_some_and(|idx| idx >= floor) + })) + } + async fn managed_delivery_target_gone( + &self, + sid: SessionId, + epoch: u64, + ) -> Result { + self.gate()?; + Ok(self.db.lock().managed_interrupted.contains(&(sid, epoch))) + } + + async fn read_managed_creation( + &self, + identity: &engram_core::types::managed_create::ManagedCreateIdentity, + ) -> Result { + self.gate()?; + managed_creation_source(&self.db.lock(), identity, false) + } + + async fn read_held_managed_creation( + &self, + identity: &engram_core::types::managed_create::ManagedCreateIdentity, + ) -> Result { + self.gate()?; + managed_creation_source(&self.db.lock(), identity, true) + } + + /// One transaction: session row, held authority and all satellites. async fn reserve_and_persist_create( &self, - ws: SessionCreateWriteSet, + mut ws: SessionCreateWriteSet, candidates: &[HostId], affinity_len: usize, ) -> Result { + if ws.oauth_binding.as_ref().is_some_and(|binding| { + binding.key.subject_kind == engram_core::types::oauth::OAuthSubjectKind::Mcp + }) { + return Err(MetaError::Conflict( + "MCP credentials cannot be bound to a guest session".into(), + )); + } + + if let Some(creation) = &ws.managed_creation { + creation.validate_for(&ws)?; + } + if let Some(selection) = &ws.managed_boot_selection { + selection.validate()?; + if ws.managed_creation.is_none() + || selection.image != ws.spec.image + || i64::from(selection.memory_mib) > ws.mem_budget_mib + || i64::from(selection.vcpus) > i64::from(ws.cpu_budget_vcpus) + { + return Err(conflict()); + } + } self.gate()?; + if let Some(initial) = &ws.managed_initial_prompt { + initial.validate(&ws)?; + } let now = self.now(); let mut db = self.db.lock(); + if let Some(creation) = &ws.managed_creation { + if db.managed_creations.contains_key(&ws.session_id) { + if managed_creation_source(&db, &creation.identity, true)? != *creation { + return Err(MetaError::Conflict( + "managed creation association changed".into(), + )); + } + if db.managed_boot_selections.get(&ws.session_id) + != ws.managed_boot_selection.as_ref() + { + return Err(conflict()); + } + let prior = db.outbox.get(&format!("create:{}", ws.session_id)); + if prior.map(|p| &p.payload) + != ws.managed_initial_prompt.as_ref().map(|p| &p.row.payload) + { + return Err(MetaError::Conflict("managed initial prompt changed".into())); + } + return Ok(CreateDisposition::Queued); + } + if db + .managed_creations + .values() + .any(|c| c.identity.policy.input_id == creation.identity.policy.input_id) + { + return Err(MetaError::Conflict( + "managed creation input already reserved".into(), + )); + } + } + let managed_policy = ws + .integration_policy_json + .as_deref() + .map(serde_json::from_str::) + .transpose() + .map_err(|_| engram_core::types::managed_operator::conflict())? + .is_some_and(|v| v.get("managed_deny").is_some_and(|p| !p.is_null())); + if ws.managed_creation.is_some() + || (managed_policy + && (db.operator_publication.published.is_some() + || db.operator_publication.pending.is_some())) + { + let publication = &db.operator_publication; + if !publication.admission_open || publication.pending.is_some() { + return Err(engram_core::types::managed_operator::conflict()); + } + let published = publication + .published + .as_ref() + .ok_or_else(engram_core::types::managed_operator::conflict)?; + ws.integration_policy_json = Some( + engram_core::types::managed_operator::retain( + ws.integration_policy_json.as_deref(), + published.candidate.hosts()?, + )? + .0, + ); + } if ws.oauth_binding.as_ref().is_some_and(|binding| { db.oauth_credentials .get(&binding.key) @@ -761,6 +1751,30 @@ impl MetadataStore for SimMetadataStore { "session OAuth binding requires a live credential".into(), )); } + if db.sessions.contains_key(&ws.session_id) { + return Err(MetaError::Conflict("session already exists".into())); + } + if let Some(policy) = &ws.mcp_policy { + if policy.actor_id.is_empty() + || policy.input_id.is_empty() + || policy.parent_session_id == Some(ws.session_id) + { + return Err(MetaError::Conflict("invalid policy creation".into())); + } + if let Some(parent) = policy.parent_session_id { + super::mcp_policy::nodes(&db, parent)?; + } + } + let candidates = if ws.managed_creation.is_some() { + &[][..] + } else { + candidates + }; + let affinity_len = if ws.managed_creation.is_some() { + 0 + } else { + affinity_len + }; let picked = Self::pick_host_2d( &db, candidates, @@ -780,6 +1794,11 @@ impl MetadataStore for SimMetadataStore { db.sessions.insert( ws.session_id, SessRow { + mcp_write_policy: ws + .mcp_policy + .as_ref() + .map(engram_core::types::mcp_policy::StoredWritePolicy::initial), + mcp_policy_parent_id: ws.mcp_policy.as_ref().and_then(|p| p.parent_session_id), session: Session { id: ws.session_id, status, @@ -810,6 +1829,9 @@ impl MetadataStore for SimMetadataStore { updated_at: now, }, ); + if let Some(creation) = ws.managed_creation { + db.managed_creations.insert(ws.session_id, creation); + } db.runtime_specs.insert(ws.session_id, ws.runtime_spec); if let Some(secrets) = ws.sealed_secrets { db.session_secrets.insert(ws.session_id, secrets); @@ -825,6 +1847,28 @@ impl MetadataStore for SimMetadataStore { if let Some(binding) = ws.oauth_binding { db.session_oauth_bindings.insert(ws.session_id, binding); } + if let Some(selection) = ws.managed_boot_selection { + db.managed_boot_selections.insert(ws.session_id, selection); + db.chunk_generation += 1; + } + if let Some(initial) = ws.managed_initial_prompt { + if initial.row.payload["mode"] == "plan" { + let session = db + .sessions + .get_mut(&ws.session_id) + .ok_or(MetaError::NotFound)?; + session.mcp_write_policy = + Some(engram_core::types::mcp_policy::StoredWritePolicy::restrict( + session.mcp_write_policy.as_ref(), + &initial.row.prompt_id, + session.next_event_idx, + )?); + } + db.outbox.insert(initial.row.prompt_id.clone(), initial.row); + for (kind, payload) in initial.events { + super::mcp_policy::append(&mut db, ws.session_id, &kind, payload, now)?; + } + } drop(db); match picked { Some(h) => Ok(CreateDisposition::Placed(h)), @@ -844,6 +1888,16 @@ impl MetadataStore for SimMetadataStore { r.session.status == SessionState::Pending && r.session.sandbox_id.is_none() }); if remove { + // The immutable ancestry FK prevents orphaning a child policy. + if db + .sessions + .values() + .any(|row| row.mcp_policy_parent_id == Some(session_id)) + { + return Err(MetaError::Db(Box::new(std::io::Error::other( + "policy parent is referenced", + )))); + } db.sessions.remove(&session_id); } Ok(()) @@ -974,7 +2028,10 @@ impl MetadataStore for SimMetadataStore { let mut rows: Vec<&SessRow> = db .sessions .values() - .filter(|r| r.session.status == SessionState::Queued) + .filter(|r| { + r.session.status == SessionState::Queued + && !db.managed_creations.contains_key(&r.session.id) + }) .collect(); rows.sort_by_key(|r| (r.queued_at, r.session.id)); Ok(rows @@ -1005,7 +2062,7 @@ impl MetadataStore for SimMetadataStore { } let now = self.now(); let mut db = self.db.lock(); - if !db.sessions.contains_key(&id) { + if db.managed_creations.contains_key(&id) || !db.sessions.contains_key(&id) { return Ok(None); } let Some(host) = Self::pick_host_2d( @@ -1033,7 +2090,7 @@ impl MetadataStore for SimMetadataStore { Ok(Some(host)) } - /// Reserving-state sessions summed per host (pending fresh-only). + /// Reservations from sessions, capture jobs, and open teleport endpoints. async fn per_host_reserved( &self, ) -> Result, MetaError> { @@ -1116,7 +2173,7 @@ impl MetadataStore for SimMetadataStore { Ok(()) } - /// `WHERE status IN ('ready','draining') ORDER BY id`. + /// Exact host lookup, including dead and retired hosts. async fn get_host(&self, id: HostId) -> Result, MetaError> { self.gate()?; Ok(self.db.lock().hosts.get(&id).cloned()) @@ -1899,6 +2956,65 @@ impl MetadataStore for SimMetadataStore { Ok(Some(settled)) } + async fn append_managed_harness_event( + &self, + event: &engram_core::types::event::ManagedHarnessEvent, + ) -> Result { + self.gate()?; + event.validate()?; + let sid = event.target.session_id; + let mut db = self.db.lock(); + let candidate = db.managed_boots.get(&sid).ok_or_else(conflict)?; + managed_operator_gate(&db, &candidate.request.key)?; + let current = managed_boot_read_locked(&db, &candidate.request.key)?; + if current.phase != ManagedBootPhase::Started + || current.request.key.context.target != event.target + || db.sessions.get(&sid).ok_or_else(conflict)?.session.status != SessionState::Active + || db + .managed_interrupted + .contains(&(sid, event.target.binding_epoch)) + { + return Err(conflict()); + } + let key = event.key(); + if let Some(previous) = db + .session_events + .get(&sid) + .into_iter() + .flatten() + .find(|row| { + row.payload.get("idempotency_key").and_then(|v| v.as_str()) == Some(key.as_str()) + }) + { + if !event.matches_replay(&previous.kind, &previous.payload) { + return Err(conflict()); + } + } + let idx = append_event_idempotent_locked( + &mut db, + sid, + &key, + &event.kind, + event.payload.clone(), + self.now(), + )?; + let mut outbox_acked = false; + if let Some(row) = event.ack_id.as_ref().and_then(|ack| db.outbox.get_mut(ack)) { + if row.session_id == sid && row.acked_at.is_none() { + row.acked_at = Some(self.now()); + outbox_acked = true; + } + } + drop(db); + if let Some(idx) = idx { + self.notify( + "session_events", + serde_json::json!({"session_id":sid,"idx":idx}).to_string(), + ); + } + Ok(engram_core::types::event::ManagedHarnessEventAccepted { idx, outbox_acked }) + } + async fn append_session_event_idempotent( &self, session: SessionId, @@ -3786,8 +4902,6 @@ impl MetadataStore for SimMetadataStore { self.gate()?; let now = self.now(); let mut db = self.db.lock(); - // (PG stamps `updated_at`, a column the in-memory `Session` - // doesn't carry — nothing to mirror here.) let row_recoverable = db .snapshots .get(&snapshot_id) @@ -3940,26 +5054,30 @@ impl MetadataStore for SimMetadataStore { "outbox row session_id mismatch".into(), )); } - { - let mut db = self.db.lock(); - match db.outbox.get(&row.prompt_id) { - None => { - db.outbox.insert(row.prompt_id.clone(), row.clone()); - } - Some(existing) => { - if existing.session_id != row.session_id - || existing.kind != row.kind - || existing.payload != row.payload - { - return Err(MetaError::Conflict(format!( - "outbox id {} belongs to another command", - row.prompt_id - ))); - } - } + let mut db = self.db.lock(); + require_tool_result_target(&db, row)?; + if !db.sessions.contains_key(&session_id) { + return Err(MetaError::NotFound); + } + if let Some(existing) = db.outbox.get(&row.prompt_id) { + if existing.session_id != row.session_id + || existing.kind != row.kind + || existing.payload != row.payload + { + return Err(MetaError::Conflict( + "outbox belongs to another command".into(), + )); } } - let idx = self.append_session_event(session_id, kind, payload).await?; + db.outbox + .entry(row.prompt_id.clone()) + .or_insert_with(|| row.clone()); + let idx = super::mcp_policy::append(&mut db, session_id, kind, payload, self.now())?; + drop(db); + self.notify( + "session_events", + serde_json::json!({"session_id":session_id,"idx":idx}).to_string(), + ); self.notify("session_outbox", session_id.to_string()); Ok(idx) } @@ -3970,44 +5088,59 @@ impl MetadataStore for SimMetadataStore { events: &[(String, serde_json::Value)], row: &engram_core::types::outbox::OutboxRow, ) -> Result>, MetaError> { - // Faithful mirror of the PG impl: a retry of the SAME command - // (existing prompt_id, matching identity) appends NOTHING and - // returns Ok(None); a prompt_id claimed by a DIFFERENT command - // is Conflict. Only a fresh insert appends the events. self.gate()?; if row.session_id != session_id { return Err(MetaError::Serialization( "outbox row session_id mismatch".into(), )); } - { - let mut db = self.db.lock(); - match db.outbox.get(&row.prompt_id) { - None => { - db.outbox.insert(row.prompt_id.clone(), row.clone()); - } - Some(existing) => { - // Payload is part of the command identity (review - // finding on #993, PG parity): a reused prompt_id - // with different text/mode conflicts, never drops. - if existing.session_id != row.session_id - || existing.kind != row.kind - || existing.payload != row.payload - { - return Err(MetaError::Conflict(format!( - "outbox id {} belongs to another command", - row.prompt_id - ))); - } - return Ok(None); - } + let mut db = self.db.lock(); + require_tool_result_target(&db, row)?; + let session = db.sessions.get(&session_id).ok_or(MetaError::NotFound)?; + if let Some(existing) = db.outbox.get(&row.prompt_id) { + if existing.session_id != row.session_id + || existing.kind != row.kind + || existing.payload != row.payload + { + return Err(MetaError::Conflict( + "outbox belongs to another command".into(), + )); } + return Ok(None); } + let restriction = if row.kind == engram_core::types::outbox::OutboxKind::Prompt + && row.payload["mode"] == "plan" + { + Some(engram_core::types::mcp_policy::StoredWritePolicy::restrict( + session.mcp_write_policy.as_ref(), + &row.prompt_id, + session.next_event_idx, + )?) + } else { + None + }; + if let Some(policy) = restriction { + db.sessions + .get_mut(&session_id) + .ok_or(MetaError::NotFound)? + .mcp_write_policy = Some(policy); + } + db.outbox.insert(row.prompt_id.clone(), row.clone()); let mut idxs = Vec::with_capacity(events.len()); for (kind, payload) in events { - idxs.push( - self.append_session_event(session_id, kind, payload.clone()) - .await?, + idxs.push(super::mcp_policy::append( + &mut db, + session_id, + kind, + payload.clone(), + self.now(), + )?); + } + drop(db); + for idx in &idxs { + self.notify( + "session_events", + serde_json::json!({"session_id":session_id,"idx":idx}).to_string(), ); } self.notify("session_outbox", session_id.to_string()); @@ -4082,30 +5215,99 @@ impl MetadataStore for SimMetadataStore { } Ok(()) } - - async fn get_session_capabilities( - &self, - session_id: SessionId, - ) -> Result, MetaError> { - self.gate()?; - Ok(self - .db - .lock() - .session_capabilities - .get(&session_id) - .cloned() - .unwrap_or_default()) - } - + + async fn get_session_capabilities( + &self, + session_id: SessionId, + ) -> Result, MetaError> { + self.gate()?; + Ok(self + .db + .lock() + .session_capabilities + .get(&session_id) + .cloned() + .unwrap_or_default()) + } + + async fn managed_policy_target( + &self, + session_id: SessionId, + ) -> Result { + self.gate()?; + let db = self.db.lock(); + let row = db.sessions.get(&session_id).ok_or(MetaError::NotFound)?; + if row.session.status.is_terminal() || row.binding_epoch <= 0 { + return Err(MetaError::NotFound); + } + Ok(engram_core::types::managed_deny::ManagedPolicyTarget { + session_id, + host_id: row.session.host_id.ok_or(MetaError::NotFound)?, + sandbox_id: row.session.sandbox_id.ok_or(MetaError::NotFound)?, + binding_epoch: row.binding_epoch as u64, + }) + } + async fn retain_managed_deny( + &self, + target: &engram_core::types::managed_deny::ManagedPolicyTarget, + policy: &engram_core::types::managed_deny::ManagedDenyPolicy, + ) -> Result<(), MetaError> { + self.gate()?; + let mut db = self.db.lock(); + let row = db + .sessions + .get(&target.session_id) + .ok_or(MetaError::NotFound)?; + if row.session.status.is_terminal() { + return Err(MetaError::NotFound); + } + if row.session.host_id != Some(target.host_id) + || row.session.sandbox_id != Some(target.sandbox_id) + || u64::try_from(row.binding_epoch).ok() != Some(target.binding_epoch) + { + return Err(MetaError::Conflict("managed placement changed".into())); + } + let next = engram_core::types::managed_deny::retain_in_policy_json( + db.session_integration_policy + .get(&target.session_id) + .map(String::as_str), + policy, + ) + .map_err(MetaError::Conflict)?; + db.operator_publication.validate_registration( + db.session_integration_policy + .get(&target.session_id) + .map(String::as_str), + &next, + )?; + db.session_integration_policy + .insert(target.session_id, next); + Ok(()) + } async fn bind_session_integration_policy( &self, session_id: SessionId, policy_json: &str, ) -> Result<(), MetaError> { self.gate()?; - self.db - .lock() - .session_integration_policy + let mut db = self.db.lock(); + if !db.sessions.contains_key(&session_id) { + return Err(MetaError::NotFound); + } + db.operator_publication.validate_registration( + db.session_integration_policy + .get(&session_id) + .map(String::as_str), + policy_json, + )?; + engram_core::types::managed_deny::validate_policy_replacement( + db.session_integration_policy + .get(&session_id) + .map(String::as_str), + policy_json, + ) + .map_err(MetaError::Conflict)?; + db.session_integration_policy .insert(session_id, policy_json.to_string()); Ok(()) } @@ -4197,24 +5399,45 @@ impl MetadataStore for SimMetadataStore { push(b); } } + for d in db + .managed_boot_selections + .values() + .flat_map(|s| s.aux_ro_drives.iter()) + .chain( + db.managed_boot_intents + .values() + .flat_map(|i| i.selection.aux_ro_drives.iter()), + ) + { + if let Some(sha256) = &d.sha256 { + push(&engram_core::types::sandbox::AuxBundleRef { + drive_id: d.drive_id.clone(), + sha256: sha256.clone(), + }); + } + } drop(db); out.sort_by(|a, b| (&a.drive_id, &a.sha256).cmp(&(&b.drive_id, &b.sha256))); Ok(out) } - /// `SELECT id, epoch FROM capture_jobs WHERE host_id=$1 AND stage NOT - /// IN ('done','failed')` — this host's live capture assignments, - /// echoed on heartbeat. - /// - /// DIVERGENCE (documented, same family as `pick_host_2d` / - /// `live_enable_work_by_host`): the sim has no `capture_jobs` table, - /// so this is always empty. Conformance covers only that case. + /// This host's nonterminal capture jobs and their current epochs. async fn capture_assignments_for_host( &self, - _host: HostId, + host: HostId, ) -> Result, MetaError> { self.gate()?; - Ok(Vec::new()) + Ok(self + .db + .lock() + .capture_jobs + .values() + .filter(|job| job.host_id == Some(host) && !job.stage.is_terminal()) + .map(|job| CaptureJobAssignment { + job_id: job.id, + epoch: job.epoch, + }) + .collect()) } /// The distinct disk manifests of the non-terminal capture jobs. A @@ -4404,8 +5627,20 @@ impl MetadataStore for SimMetadataStore { Ok(DeleteHostOutcome::Deleted) } - async fn delete_org_secret(&self, _name: &str) -> Result { - panic!("SimMeta: delete_org_secret not implemented — add it plus a conformance case (ADR 0098 D4)") + async fn delete_org_secret(&self, name: &str) -> Result { + self.gate()?; + let mut db = self.db.lock(); + let deleted = db.org_secrets.remove(name).is_some(); + if deleted { + if let Some(Some(reservation)) = db.org_secret_purposes.get_mut(name) { + reservation.generation += 1; + } + } + drop(db); + if deleted { + self.notify("org_secret_changed", name.to_string()); + } + Ok(deleted) } async fn expire_capture_job_stages( @@ -4476,9 +5711,15 @@ impl MetadataStore for SimMetadataStore { async fn get_org_secret_sealed( &self, - _name: &str, + name: &str, ) -> Result, MetaError> { - panic!("SimMeta: get_org_secret_sealed not implemented — add it plus a conformance case (ADR 0098 D4)") + self.gate()?; + Ok(self + .db + .lock() + .org_secrets + .get(name) + .map(|(sealed, _)| sealed.clone())) } async fn get_oauth_credential( @@ -4633,7 +5874,7 @@ impl MetadataStore for SimMetadataStore { .collect()) } - /// The reconcile pass query: Active + bound sessions on `host_id`. + /// Bound memory-reserving sessions on this host. async fn list_resident_sandbox_assignments_on_host( &self, host_id: HostId, @@ -4693,20 +5934,46 @@ impl MetadataStore for SimMetadataStore { async fn list_org_secrets( &self, ) -> Result, MetaError> { - panic!("SimMeta: list_org_secrets not implemented — add it plus a conformance case (ADR 0098 D4)") + self.gate()?; + Ok(self + .db + .lock() + .org_secrets + .values() + .map(|(_, meta)| meta.clone()) + .collect()) } /// `SELECT prestage_ref FROM enable_jobs WHERE state='prestaging'` — /// the images a host should prefetch, echoed to it on heartbeat. /// - /// DIVERGENCE (documented, same family as `pick_host_2d` / - /// `live_enable_work_by_host`): the sim has no `enable_jobs` table, so - /// this is always the empty set. Conformance covers only that case. + /// The simulator does not model prestage references. async fn list_prestaging_refs(&self) -> Result, MetaError> { self.gate()?; Ok(Vec::new()) } + async fn list_managed_disk_manifests(&self) -> Result, MetaError> { + self.gate()?; + let db = self.db.lock(); + let refs: std::collections::BTreeSet<_> = db + .managed_boots + .values() + .map(|r| r.request.cold.disk.manifest) + .chain(db.managed_boot_selections.values().map(|s| s.base_manifest)) + .chain(db.managed_boot_intents.values().map(|i| i.disk_manifest)) + .chain(db.managed_captures.values().map(|(r, _)| r.disk_manifest())) + .map(|m| (m.manifest_id, m.version)) + .collect(); + Ok(refs + .into_iter() + .map(|(manifest_id, version)| ManifestRef { + manifest_id, + version, + }) + .collect()) + } + async fn list_recoverable_snapshot_disk_manifests( &self, ) -> Result, Option)>, MetaError> { @@ -5366,46 +6633,123 @@ impl MetadataStore for SimMetadataStore { async fn upsert_org_secret( &self, - _sealed: engram_core::types::org_secret::SealedOrgSecret, + sealed: engram_core::types::org_secret::SealedOrgSecret, ) -> Result { - panic!("SimMeta: upsert_org_secret not implemented — add it plus a conformance case (ADR 0098 D4)") + self.gate()?; + let now = self.now(); + let mut db = self.db.lock(); + if db + .org_secret_purposes + .get(&sealed.name) + .is_some_and(Option::is_some) + { + return Err(MetaError::Conflict("credential is reserved for MCP".into())); + } + db.org_secret_purposes.insert(sealed.name.clone(), None); + let meta = engram_core::types::org_secret::OrgSecret { + name: sealed.name.clone(), + key_id: sealed.key_id.clone(), + created_at: db + .org_secrets + .get(&sealed.name) + .map_or(now, |(_, row)| row.created_at), + updated_at: now, + }; + db.org_secrets + .insert(sealed.name.clone(), (sealed, meta.clone())); + drop(db); + self.notify("org_secret_changed", meta.name.clone()); + Ok(meta) } - async fn put_oauth_credential( + async fn reserve_mcp_org_secret( &self, - credential: engram_core::types::oauth::NewSealedOAuthCredential, - expected_version: Option, - ) -> Result { + name: &str, + connection_id: &str, + role: engram_core::types::org_secret::McpSecretRole, + ) -> Result { self.gate()?; - let now = self.now(); let mut db = self.db.lock(); - let (version, created_at) = match db.oauth_credentials.get(&credential.key) { - None if expected_version.is_none() => (1, now), - Some(current) if expected_version == Some(current.version) => { - (current.version + 1, current.created_at) - } - _ => return Err(MetaError::Conflict("OAuth credential CAS rejected".into())), + if name.is_empty() || connection_id.is_empty() { + return Err(MetaError::Conflict("invalid MCP reservation".into())); + } + if let Some(current) = db.org_secret_purposes.get(name) { + return match current { + Some(r) if r.connection_id == connection_id && r.role == role => Ok(r.clone()), + _ => Err(MetaError::Conflict("credential purpose conflict".into())), + }; + } + let reservation = engram_core::types::org_secret::McpSecretReservation { + name: name.into(), + connection_id: connection_id.into(), + role, + generation: 0, }; - let row = engram_core::types::oauth::SealedOAuthCredential { - key: credential.key.clone(), - wrapped_dek: credential.wrapped_dek, - nonce: credential.nonce, - ciphertext: credential.ciphertext, - key_id: credential.key_id, - metadata: credential.metadata, - version, - created_at, + db.org_secret_purposes + .insert(name.into(), Some(reservation.clone())); + drop(db); + self.notify("org_secret_changed", name.to_string()); + Ok(reservation) + } + + async fn get_mcp_org_secret_reservation( + &self, + name: &str, + ) -> Result, MetaError> { + self.gate()?; + Ok(self + .db + .lock() + .org_secret_purposes + .get(name) + .cloned() + .flatten()) + } + + async fn put_mcp_org_secret( + &self, + sealed: engram_core::types::org_secret::SealedOrgSecret, + expected: &engram_core::types::org_secret::McpSecretReservation, + ) -> Result { + self.gate()?; + let now = self.now(); + let mut db = self.db.lock(); + if sealed.name != expected.name + || db.org_secret_purposes.get(&sealed.name) != Some(&Some(expected.clone())) + { + return Err(MetaError::Conflict("credential reservation changed".into())); + } + let mut next = expected.clone(); + next.generation += 1; + let meta = engram_core::types::org_secret::OrgSecret { + name: sealed.name.clone(), + key_id: sealed.key_id.clone(), + created_at: db + .org_secrets + .get(&sealed.name) + .map_or(now, |(_, row)| row.created_at), updated_at: now, - revoked_at: None, - expires_at: credential.expires_at, - broken_at: None, - broken_reason: None, }; - // Publishing a validated bundle IS the repair: broken/claim state - // clears on every successful write (PG mirrors this in the UPDATE). - db.oauth_refresh_claims.remove(&credential.key); - db.oauth_credentials.insert(credential.key, row.clone()); - Ok(row) + db.org_secrets.insert(sealed.name.clone(), (sealed, meta)); + db.org_secret_purposes + .insert(next.name.clone(), Some(next.clone())); + drop(db); + self.notify("org_secret_changed", next.name.clone()); + Ok(next) + } + + async fn put_oauth_credential( + &self, + credential: engram_core::types::oauth::NewSealedOAuthCredential, + expected_version: Option, + ) -> Result { + self.gate()?; + put_oauth_credential_in( + &mut self.db.lock(), + self.now(), + credential, + expected_version, + ) } async fn list_oauth_credentials_due_for_refresh( @@ -5512,6 +6856,211 @@ impl MetadataStore for SimMetadataStore { Ok(row.clone()) } + async fn claim_mcp_refresh( + &self, + claim: &engram_core::types::mcp::McpRefreshClaim, + ) -> Result { + self.gate()?; + let now = self.now(); + if claim.key.subject_kind != engram_core::types::oauth::OAuthSubjectKind::Mcp + || claim.expires_at <= now + || claim.expires_at > now + chrono::Duration::seconds(30) + { + return Err(MetaError::Conflict("invalid refresh claim".into())); + } + let mut db = self.db.lock(); + let Some(row) = db.oauth_credentials.get_mut(&claim.key) else { + return Ok(false); + }; + if row.version != claim.version || row.revoked_at.is_some() || row.broken_at.is_some() { + return Ok(false); + } + row.broken_at = Some(now); + row.broken_reason = Some("refresh_pending".into()); + db.mcp_refresh_claims + .insert(claim.key.clone(), claim.clone()); + Ok(true) + } + async fn complete_mcp_refresh( + &self, + credential: engram_core::types::oauth::NewSealedOAuthCredential, + claim: &engram_core::types::mcp::McpRefreshClaim, + ) -> Result { + self.gate()?; + let now = self.now(); + let mut db = self.db.lock(); + if credential.key != claim.key + || db.mcp_refresh_claims.get(&claim.key) != Some(claim) + || claim.expires_at <= now + { + return Err(MetaError::Conflict("refresh claim lost".into())); + } + let row = db + .oauth_credentials + .get_mut(&claim.key) + .ok_or(MetaError::NotFound)?; + if row.version != claim.version + || row.revoked_at.is_some() + || row.broken_reason.as_deref() != Some("refresh_pending") + { + return Err(MetaError::Conflict("refresh claim lost".into())); + } + row.wrapped_dek = credential.wrapped_dek; + row.nonce = credential.nonce; + row.ciphertext = credential.ciphertext; + row.key_id = credential.key_id; + row.metadata = credential.metadata; + row.expires_at = credential.expires_at; + row.version += 1; + row.updated_at = now; + row.broken_at = None; + row.broken_reason = None; + let result = row.clone(); + db.mcp_refresh_claims.remove(&claim.key); + db.oauth_refresh_claims.remove(&claim.key); + Ok(result) + } + async fn acquire_mcp_oauth_admission( + &self, + lease: &engram_core::types::mcp::McpOAuthAdmission, + ) -> Result { + self.gate()?; + let now = self.now(); + if lease.deployment_id.is_empty() + || lease.connection_id.is_empty() + || lease.expires_at <= now + || lease.expires_at > now + chrono::Duration::seconds(30) + { + return Err(MetaError::Conflict("invalid OAuth admission".into())); + } + let mut db = self.db.lock(); + db.mcp_oauth_admission.retain(|_, l| l.expires_at > now); + if db.mcp_oauth_admission.contains_key(&lease.id) + || db.mcp_oauth_admission.values().any(|l| { + l.deployment_id == lease.deployment_id && l.connection_id == lease.connection_id + }) + || db + .mcp_oauth_admission + .values() + .filter(|l| l.deployment_id == lease.deployment_id) + .count() + >= 4 + { + return Ok(false); + } + db.mcp_oauth_admission.insert(lease.id, lease.clone()); + Ok(true) + } + async fn release_mcp_oauth_admission( + &self, + lease: &engram_core::types::mcp::McpOAuthAdmission, + ) -> Result<(), MetaError> { + self.gate()?; + let mut db = self.db.lock(); + if db.mcp_oauth_admission.get(&lease.id) == Some(lease) { + db.mcp_oauth_admission.remove(&lease.id); + } + Ok(()) + } + + async fn create_mcp_oauth_flow( + &self, + flow: engram_core::types::oauth::OAuthFlow, + state: engram_core::types::mcp::McpSealedState, + ) -> Result<(), MetaError> { + use engram_core::types::oauth::{OAuthFlowStatus, OAuthSubjectKind}; + self.gate()?; + let mut db = self.db.lock(); + if flow.key.subject_kind != OAuthSubjectKind::Mcp + || flow.status != OAuthFlowStatus::Pending + || flow.expires_at <= self.now() + || db.oauth_flows.contains_key(&flow.id) + || db + .oauth_flows + .values() + .any(|f| f.key == flow.key && f.status == OAuthFlowStatus::Pending) + { + return Err(MetaError::Conflict("MCP flow creation rejected".into())); + } + db.mcp_flow_states.insert(flow.id, state); + db.oauth_flows.insert(flow.id, flow); + Ok(()) + } + async fn consume_mcp_oauth_flow( + &self, + id: uuid::Uuid, + key: &engram_core::types::oauth::OAuthCredentialKey, + ) -> Result { + use engram_core::types::oauth::{OAuthFlowStatus, OAuthSubjectKind}; + self.gate()?; + let mut db = self.db.lock(); + let flow = db + .oauth_flows + .get(&id) + .ok_or_else(|| MetaError::Conflict("MCP state unavailable".into()))?; + if key.subject_kind != OAuthSubjectKind::Mcp + || &flow.key != key + || flow.status != OAuthFlowStatus::Pending + || flow.expires_at <= self.now() + { + return Err(MetaError::Conflict("MCP state unavailable".into())); + } + db.mcp_flow_states + .remove(&id) + .ok_or_else(|| MetaError::Conflict("MCP state consumed".into())) + } + + async fn cancel_mcp_oauth_flow( + &self, + id: uuid::Uuid, + key: &engram_core::types::oauth::OAuthCredentialKey, + ) -> Result<(), MetaError> { + use engram_core::types::oauth::{OAuthFlowStatus, OAuthSubjectKind}; + self.gate()?; + let mut db = self.db.lock(); + let flow = db + .oauth_flows + .get_mut(&id) + .ok_or_else(|| MetaError::Conflict("MCP flow cancellation rejected".into()))?; + if key.subject_kind != OAuthSubjectKind::Mcp || &flow.key != key { + return Err(MetaError::Conflict("MCP flow cancellation rejected".into())); + } + if flow.status == OAuthFlowStatus::Pending { + flow.status = OAuthFlowStatus::Cancelled; + } + flow.updated_at = self.now(); + db.mcp_flow_states.remove(&id); + Ok(()) + } + async fn complete_mcp_oauth_flow( + &self, + id: uuid::Uuid, + credential: engram_core::types::oauth::NewSealedOAuthCredential, + expected_version: Option, + ) -> Result { + use engram_core::types::oauth::{OAuthFlowStatus, OAuthSubjectKind}; + self.gate()?; + let mut db = self.db.lock(); + let now = self.now(); + let flow = db + .oauth_flows + .get(&id) + .ok_or_else(|| MetaError::Conflict("MCP flow publication rejected".into()))?; + if credential.key.subject_kind != OAuthSubjectKind::Mcp + || flow.key != credential.key + || flow.status != OAuthFlowStatus::Pending + || flow.expires_at <= now + || db.mcp_flow_states.contains_key(&id) + { + return Err(MetaError::Conflict("MCP flow publication rejected".into())); + } + let row = put_oauth_credential_in(&mut db, now, credential, expected_version)?; + let flow = db.oauth_flows.get_mut(&id).expect("locked flow"); + flow.status = OAuthFlowStatus::Succeeded; + flow.updated_at = now; + Ok(row) + } + async fn create_oauth_flow( &self, flow: engram_core::types::oauth::OAuthFlow, @@ -5668,6 +7217,13 @@ impl MetadataStore for SimMetadataStore { let before = db.oauth_flows.len(); db.oauth_flows .retain(|_, flow| !flow.status.is_terminal() || flow.updated_at >= delete_before); + let pending: std::collections::BTreeSet<_> = db + .oauth_flows + .values() + .filter(|f| !f.status.is_terminal()) + .map(|f| f.id) + .collect(); + db.mcp_flow_states.retain(|id, _| pending.contains(id)); Ok(changed + (before - db.oauth_flows.len()) as u64) } } @@ -5732,3 +7288,112 @@ fn append_event_idempotent_locked( }); Ok(Some(idx)) } + +fn put_oauth_credential_in( + db: &mut SimDb, + now: DateTime, + credential: engram_core::types::oauth::NewSealedOAuthCredential, + expected_version: Option, +) -> Result { + let (version, created_at) = match db.oauth_credentials.get(&credential.key) { + None if expected_version.is_none() => (1, now), + Some(current) if expected_version == Some(current.version) => { + (current.version + 1, current.created_at) + } + _ => return Err(MetaError::Conflict("OAuth credential CAS rejected".into())), + }; + let row = engram_core::types::oauth::SealedOAuthCredential { + key: credential.key.clone(), + wrapped_dek: credential.wrapped_dek, + nonce: credential.nonce, + ciphertext: credential.ciphertext, + key_id: credential.key_id, + metadata: credential.metadata, + version, + created_at, + updated_at: now, + revoked_at: None, + expires_at: credential.expires_at, + broken_at: None, + broken_reason: None, + }; + // Publishing a validated bundle IS the repair: broken/claim state + // clears on every successful write (PG mirrors this in the UPDATE). + db.oauth_refresh_claims.remove(&credential.key); + db.oauth_credentials.insert(credential.key, row.clone()); + Ok(row) +} + +fn managed_operator_gate(db: &super::SimDb, key: &ManagedBootKey) -> Result<(), MetaError> { + if db + .managed_operator_gate + .as_ref() + .is_some_and(|(generation, identity, open)| { + *open + && *generation == key.context.operator_generation + && identity == &key.context.operator_identity + }) + { + Ok(()) + } else { + Err(MetaError::Conflict( + "managed operator generation is closed".into(), + )) + } +} + +fn require_tool_result_target( + db: &SimDb, + row: &engram_core::types::outbox::OutboxRow, +) -> Result<(), MetaError> { + if row.kind == engram_core::types::outbox::OutboxKind::ToolResult + && db.managed_creations.contains_key(&row.session_id) + { + let call = row + .payload + .get("tool_call_id") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + let idx = db + .session_events + .get(&row.session_id) + .and_then(|events| { + events + .iter() + .filter(|e| { + e.kind == "tool_call_requested" && e.payload["tool_call_id"] == call + }) + .map(|e| e.idx) + .min() + }) + .unwrap_or(-1); + if idx + < db.managed_interrupted_floors + .get(&row.session_id) + .copied() + .unwrap_or(0) + { + return Err(MetaError::Conflict("delivery target gone".into())); + } + } + Ok(()) +} + +fn bridge_gate(db: &SimDb) -> Result<(), MetaError> { + if db + .managed_operator_gate + .as_ref() + .is_some_and(|(_, _, open)| *open) + { + Ok(()) + } else { + Err(conflict()) + } +} +fn q_match(db: &SimDb, q: &ManagedCreationQuery) -> Result<(), MetaError> { + q.matches( + db.managed_creations + .get(&q.identity.session_id) + .ok_or(MetaError::NotFound)?, + ) +} diff --git a/crates/engram-sim/tests/meta_conformance.rs b/crates/engram-sim/tests/meta_conformance.rs index e0cfe3dd8..e27bd0e96 100644 --- a/crates/engram-sim/tests/meta_conformance.rs +++ b/crates/engram-sim/tests/meta_conformance.rs @@ -745,6 +745,81 @@ async fn capture_job_scan_boundary(ctx: &Ctx) { expired[0].id, placed.id, "only placed over-budget rows expire" ); + + let placed = ctx + .meta + .reassign_capture_job(placed.id, placed.epoch, &[host]) + .await + .unwrap() + .unwrap(); + assert_eq!(placed.epoch, 2); + let other_host = HostId::from(uuid::Uuid::from_u128(0xc002)); + ctx.meta + .upsert_host(host_record( + other_host, + "capture-other-host", + ctx.clock.now_utc(), + )) + .await + .unwrap(); + let other_parent = ctx + .meta + .create_or_get_enable_job("conf:capture:other", None, &config) + .await + .unwrap() + .id; + let other = ctx + .meta + .insert_capture_job(new_capture(other_parent)) + .await + .unwrap(); + let other = ctx + .meta + .place_capture_job(other.id, &[other_host]) + .await + .unwrap() + .unwrap(); + assert_eq!(other.host_id, Some(other_host)); + for (assigned_host, expected) in [ + (host, vec![(placed.id, 2)]), + (other_host, vec![(other.id, 1)]), + ] { + let assignments = ctx + .meta + .capture_assignments_for_host(assigned_host) + .await + .unwrap(); + assert_eq!( + assignments + .into_iter() + .map(|assignment| (assignment.job_id, assignment.epoch)) + .collect::>(), + expected, + "only this host's nonterminal jobs are assigned, at their current epoch" + ); + } + assert!(ctx + .meta + .record_capture_job_report(&CaptureJobReport { + job_id: placed.id, + epoch: placed.epoch, + stage: CaptureJobStage::Failed, + progress: None, + fc_snapshot_version: None, + terminal: Some(CaptureTerminalReport::Failed { + error: "capture failed".into(), + error_stage: "assigned".into(), + retryable: false, + }), + }) + .await + .unwrap()); + assert!(ctx + .meta + .capture_assignments_for_host(host) + .await + .unwrap() + .is_empty()); } /// Transition legality: illegal edges are Conflict, transitions return @@ -1017,16 +1092,13 @@ async fn host_fc_snapshot_version(ctx: &Ctx) { /// FIFO by queued_at; the queue drains oldest-first. Sessions enter the /// queue via the REAL enqueue path (`reserve_and_persist_create` with no /// candidates), which stamps `queue_origin` + `queued_at`. -/// -/// FINDING (surfaced by this suite): a bare -/// `transition_session(_, Queued)` — legal per the FSM table — leaves -/// both columns NULL in PostgresStore, and `list_queued_sessions_fifo` -/// then fails to DECODE the row (`queue_origin: UnexpectedNullError`), -/// which would break the queue scanner. Production never enters Queued -/// that way today; flagged for a follow-up guard. async fn queue_fifo(ctx: &Ctx) { let meta = &ctx.meta; let enqueue = |sid: SessionId| engram_core::traits::metadata::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id: sid, spec: spec("conf:q"), mem_budget_mib: 1024, @@ -4183,6 +4255,10 @@ async fn stale_pending_reservation(ctx: &Ctx) { // A placed pending with a queued create_boot op. let with_op = SessionId::new(); let ws = |sid: SessionId| engram_core::traits::metadata::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id: sid, spec: spec("conf:722"), mem_budget_mib: 2048, @@ -4387,9 +4463,7 @@ async fn capturing_disk_manifests_in_flight(ctx: &Ctx) { /// `live_enable_work_by_host` is the empty map, while /// `list_prestaging_refs`, `capturing_disk_manifests` and /// `capture_assignments_for_host` are empty — and registering or -/// heartbeating a host fabricates none of them. (The sim models no -/// enable_jobs/capture_jobs table; a fresh PG DB has no such rows — both -/// stores agree on the empty case, the only one conformance exercises.) +/// heartbeating a host fabricates none of them. async fn live_enable_work_empty(ctx: &Ctx) { let meta = &ctx.meta; assert!(meta @@ -5923,6 +5997,10 @@ async fn teleport_reservation_arm_counts_open_rows(ctx: &Ctx) { let mut t = teleport_fixture(ctx, source, h).await; let pending = SessionId::new(); let ws = engram_core::traits::metadata::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id: pending, spec: spec("conf:placement"), mem_budget_mib: 1, @@ -6401,6 +6479,10 @@ async fn teleport_admission_fixture( } let sid = SessionId::new(); let ws = engram_core::traits::metadata::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, session_id: sid, spec: spec("conf:teleport-machine"), mem_budget_mib: 4096, @@ -7294,3 +7376,5418 @@ conformance!( t_capture_job_epoch_fences_reassignment_and_redrive, super::capture_job_epoch_fences_reassignment_and_redrive ); + +async fn mcp_org_secret_clock(ctx: &Ctx) { + use engram_core::types::org_secret::SealedOrgSecret; + let value = SealedOrgSecret { + name: "fresh-key".into(), + wrapped_dek: vec![1], + nonce: vec![0; 12], + ciphertext: vec![2], + key_id: "test".into(), + }; + let row = ctx.meta.upsert_org_secret(value.clone()).await.unwrap(); + assert_eq!(row.created_at, ctx.clock.now_utc()); + ctx.clock.advance(Duration::from_secs(5)); + let rotated = ctx.meta.upsert_org_secret(value).await.unwrap(); + assert_eq!(rotated.created_at, row.created_at); + assert_eq!(rotated.updated_at, ctx.clock.now_utc()); + assert_eq!(ctx.meta.list_org_secrets().await.unwrap().len(), 1); + assert_eq!( + ctx.meta + .get_org_secret_sealed("fresh-key") + .await + .unwrap() + .unwrap() + .ciphertext, + vec![2] + ); + assert!(ctx.meta.delete_org_secret("fresh-key").await.unwrap()); + assert!(!ctx.meta.delete_org_secret("fresh-key").await.unwrap()); + assert!(ctx + .meta + .get_org_secret_sealed("fresh-key") + .await + .unwrap() + .is_none()); +} +conformance!(t_mcp_org_secret_clock, super::mcp_org_secret_clock); + +async fn mcp_org_reservation(ctx: &Ctx) { + use engram_core::types::org_secret::{McpSecretRole, SealedOrgSecret}; + let sealed = |name: &str| SealedOrgSecret { + name: name.into(), + wrapped_dek: vec![1], + nonce: vec![0; 12], + ciphertext: vec![2], + key_id: "test".into(), + }; + let reserved = ctx + .meta + .reserve_mcp_org_secret("reserved", "connection-a", McpSecretRole::ApiKey) + .await + .unwrap(); + assert_eq!(reserved.generation, 0); + assert_eq!( + ctx.meta + .reserve_mcp_org_secret("reserved", "connection-a", McpSecretRole::ApiKey) + .await + .unwrap(), + reserved + ); + assert!(ctx + .meta + .reserve_mcp_org_secret("reserved", "connection-b", McpSecretRole::ApiKey) + .await + .is_err()); + assert!(ctx + .meta + .reserve_mcp_org_secret("reserved", "connection-a", McpSecretRole::ClientSecret) + .await + .is_err()); + assert!(ctx + .meta + .upsert_org_secret(sealed("reserved")) + .await + .is_err()); + let first = ctx + .meta + .put_mcp_org_secret(sealed("reserved"), &reserved) + .await + .unwrap(); + assert_eq!(first.generation, 1); + assert!(ctx + .meta + .put_mcp_org_secret(sealed("reserved"), &reserved) + .await + .is_err()); + assert!(ctx.meta.delete_org_secret("reserved").await.unwrap()); + let deleted = ctx + .meta + .get_mcp_org_secret_reservation("reserved") + .await + .unwrap() + .unwrap(); + assert_eq!(deleted.generation, 2); + assert!(ctx + .meta + .get_org_secret_sealed("reserved") + .await + .unwrap() + .is_none()); + assert!(ctx + .meta + .put_mcp_org_secret(sealed("reserved"), &first) + .await + .is_err()); + assert_eq!( + ctx.meta + .put_mcp_org_secret(sealed("reserved"), &deleted) + .await + .unwrap() + .generation, + 3 + ); + let concurrent = ctx + .meta + .reserve_mcp_org_secret("concurrent", "connection-a", McpSecretRole::ApiKey) + .await + .unwrap(); + let (a, b) = tokio::join!( + ctx.meta + .put_mcp_org_secret(sealed("concurrent"), &concurrent), + ctx.meta + .put_mcp_org_secret(sealed("concurrent"), &concurrent), + ); + assert_eq!(usize::from(a.is_ok()) + usize::from(b.is_ok()), 1); + assert_eq!( + ctx.meta + .get_mcp_org_secret_reservation("concurrent") + .await + .unwrap() + .unwrap() + .generation, + 1 + ); + ctx.meta.upsert_org_secret(sealed("exposed")).await.unwrap(); + ctx.meta.delete_org_secret("exposed").await.unwrap(); + assert!(ctx + .meta + .reserve_mcp_org_secret("exposed", "connection-a", McpSecretRole::ApiKey) + .await + .is_err()); +} +conformance!(t_mcp_org_reservation, super::mcp_org_reservation); + +async fn mcp_guest_binding_rejected(ctx: &Ctx) { + use engram_core::types::oauth::{ + NewSealedOAuthCredential, OAuthAccountMetadata, OAuthCredentialKey, OAuthSubjectKind, + SessionOAuthBinding, + }; + let key = OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: "connection-a".into(), + provider: "mcp".into(), + }; + ctx.meta + .put_oauth_credential( + NewSealedOAuthCredential { + key: key.clone(), + wrapped_dek: vec![1], + nonce: vec![0; 12], + ciphertext: vec![2], + key_id: "test".into(), + metadata: OAuthAccountMetadata::default(), + expires_at: None, + }, + None, + ) + .await + .unwrap(); + let sid = SessionId::from(uuid::Uuid::from_u128(4)); + let ws = engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: None, + session_id: sid, + spec: spec("test:mcp"), + mem_budget_mib: 1024, + cpu_budget_vcpus: 1, + sealed_secrets: None, + capabilities: vec![], + integration_policy_json: None, + runtime_spec: engram_core::types::runtime_spec::RuntimeSpec::new( + vec![], + None, + None, + vec![], + ), + oauth_binding: Some(SessionOAuthBinding { + session_id: sid, + key, + }), + }; + assert!(matches!( + ctx.meta.reserve_and_persist_create(ws, &[], 0).await, + Err(MetaError::Conflict(_)) + )); + assert!(ctx + .meta + .get_session_oauth_binding(sid) + .await + .unwrap() + .is_none()); + assert!(matches!( + ctx.meta.get_session(sid).await, + Err(MetaError::NotFound) + )); +} +conformance!( + t_mcp_guest_binding_rejected, + super::mcp_guest_binding_rejected +); + +// Consuming a redirect must remove its encrypted state before any exchange. +async fn mcp_flow_consume(ctx: &Ctx) { + use engram_core::types::{mcp::*, oauth::*}; + let now = ctx.clock.now_utc(); + let key = OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: "connection-a".into(), + provider: "registered-a".into(), + }; + let id = uuid::Uuid::from_u128(810); + let flow = OAuthFlow { + id, + key: key.clone(), + owner_replica: "replica-a".into(), + lease_expires_at: now + chrono::Duration::minutes(10), + expires_at: now + chrono::Duration::minutes(10), + status: OAuthFlowStatus::Pending, + error_code: None, + created_at: now, + updated_at: now, + }; + let state = McpSealedState { + wrapped_dek: vec![1; 32], + nonce: vec![2; 12], + ciphertext: vec![3; 64], + key_id: "test".into(), + }; + assert!(matches!( + ctx.meta.consume_mcp_oauth_flow(id, &key).await, + Err(MetaError::Conflict(_)) + )); + ctx.meta + .create_mcp_oauth_flow(flow.clone(), state.clone()) + .await + .unwrap(); + let mut wrong = key.clone(); + wrong.subject_id = "connection-b".into(); + assert!(ctx.meta.consume_mcp_oauth_flow(id, &wrong).await.is_err()); + let (a, b) = tokio::join!( + ctx.meta.consume_mcp_oauth_flow(id, &key), + ctx.meta.consume_mcp_oauth_flow(id, &key) + ); + assert_eq!(usize::from(a.is_ok()) + usize::from(b.is_ok()), 1); + assert_eq!(a.or(b).unwrap(), state); + assert!(ctx.meta.consume_mcp_oauth_flow(id, &key).await.is_err()); + ctx.meta + .finish_oauth_flow_unowned(id, OAuthFlowStatus::Succeeded, None) + .await + .unwrap(); + let mut expired = flow; + expired.id = uuid::Uuid::from_u128(811); + ctx.meta + .create_mcp_oauth_flow(expired, state) + .await + .unwrap(); + ctx.clock.advance(Duration::from_secs(600)); + assert!(ctx + .meta + .consume_mcp_oauth_flow(uuid::Uuid::from_u128(811), &key) + .await + .is_err()); +} +conformance!(t_mcp_flow_consume, mcp_flow_consume); + +async fn mcp_admission_and_refresh(ctx: &Ctx) { + use engram_core::types::{mcp::*, oauth::*}; + let now = ctx.clock.now_utc(); + let key = OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: "connection-a".into(), + provider: "registered-a".into(), + }; + let candidate = NewSealedOAuthCredential { + key: key.clone(), + wrapped_dek: vec![1; 32], + nonce: vec![2; 12], + ciphertext: vec![3; 64], + key_id: "test".into(), + metadata: OAuthAccountMetadata::default(), + expires_at: Some(now), + }; + let row = ctx + .meta + .put_oauth_credential(candidate.clone(), None) + .await + .unwrap(); + let claim = McpRefreshClaim { + id: uuid::Uuid::from_u128(901), + key: key.clone(), + version: row.version, + expires_at: now + chrono::Duration::seconds(20), + }; + let (a, b) = tokio::join!( + ctx.meta.claim_mcp_refresh(&claim), + ctx.meta.claim_mcp_refresh(&claim) + ); + assert_eq!(usize::from(a.unwrap()) + usize::from(b.unwrap()), 1); + assert!(ctx + .meta + .get_oauth_credential(&key) + .await + .unwrap() + .unwrap() + .broken_at + .is_some()); + let mut wrong = claim.clone(); + wrong.id = uuid::Uuid::from_u128(902); + assert!(ctx + .meta + .complete_mcp_refresh(candidate.clone(), &wrong) + .await + .is_err()); + let refreshed = ctx + .meta + .complete_mcp_refresh(candidate.clone(), &claim) + .await + .unwrap(); + assert_eq!(refreshed.version, row.version + 1); + assert!(refreshed.broken_at.is_none()); + let mut uncertain = claim; + uncertain.version = refreshed.version; + assert!(ctx.meta.claim_mcp_refresh(&uncertain).await.unwrap()); + ctx.clock.advance(Duration::from_secs(21)); + assert!(ctx + .meta + .complete_mcp_refresh(candidate, &uncertain) + .await + .is_err()); + uncertain.expires_at = ctx.clock.now_utc() + chrono::Duration::seconds(20); + assert!( + !ctx.meta.claim_mcp_refresh(&uncertain).await.unwrap(), + "uncertain rotation must require reconnect" + ); + let lease = |n| McpOAuthAdmission { + id: uuid::Uuid::from_u128(n), + deployment_id: "deployment-a".into(), + connection_id: format!("connection-{n}"), + expires_at: ctx.clock.now_utc() + chrono::Duration::seconds(15), + }; + let leases: Vec<_> = (1..=5).map(lease).collect(); + for item in &leases[..4] { + assert!(ctx.meta.acquire_mcp_oauth_admission(item).await.unwrap()); + } + assert!(!ctx + .meta + .acquire_mcp_oauth_admission(&leases[4]) + .await + .unwrap()); + ctx.meta + .release_mcp_oauth_admission(&leases[0]) + .await + .unwrap(); + assert!(ctx + .meta + .acquire_mcp_oauth_admission(&leases[4]) + .await + .unwrap()); + let mut same_connection = leases[4].clone(); + same_connection.id = uuid::Uuid::from_u128(99); + assert!(!ctx + .meta + .acquire_mcp_oauth_admission(&same_connection) + .await + .unwrap()); + ctx.clock.advance(Duration::from_secs(15)); + same_connection.expires_at = ctx.clock.now_utc() + chrono::Duration::seconds(15); + assert!(ctx + .meta + .acquire_mcp_oauth_admission(&same_connection) + .await + .unwrap()); + let contenders: Vec<_> = (201..=205) + .map(|n| McpOAuthAdmission { + id: uuid::Uuid::from_u128(n), + deployment_id: "deployment-race".into(), + connection_id: format!("connection-{n}"), + expires_at: ctx.clock.now_utc() + chrono::Duration::seconds(15), + }) + .collect(); + let (a, b, c, d, e) = tokio::join!( + ctx.meta.acquire_mcp_oauth_admission(&contenders[0]), + ctx.meta.acquire_mcp_oauth_admission(&contenders[1]), + ctx.meta.acquire_mcp_oauth_admission(&contenders[2]), + ctx.meta.acquire_mcp_oauth_admission(&contenders[3]), + ctx.meta.acquire_mcp_oauth_admission(&contenders[4]), + ); + assert_eq!( + [a, b, c, d, e] + .into_iter() + .filter(|r| *r.as_ref().unwrap()) + .count(), + 4 + ); + // Revocation fences both a dispatched refresh and a late reconnect CAS. + let mut replacement = NewSealedOAuthCredential { + key: key.clone(), + wrapped_dek: vec![1; 32], + nonce: vec![2; 12], + ciphertext: vec![3; 64], + key_id: "test".into(), + metadata: OAuthAccountMetadata::default(), + expires_at: None, + }; + replacement.key.subject_id = "connection-revoked".into(); + let row = ctx + .meta + .put_oauth_credential(replacement.clone(), None) + .await + .unwrap(); + let claim = McpRefreshClaim { + id: uuid::Uuid::from_u128(903), + key: replacement.key.clone(), + version: row.version, + expires_at: ctx.clock.now_utc() + chrono::Duration::seconds(20), + }; + assert!(ctx.meta.claim_mcp_refresh(&claim).await.unwrap()); + let receipt = ctx + .meta + .revoke_oauth_credential(&replacement.key, row.version) + .await + .unwrap(); + assert_eq!(receipt.version, row.version + 1); + assert!(ctx + .meta + .complete_mcp_refresh(replacement.clone(), &claim) + .await + .is_err()); + assert!(ctx + .meta + .put_oauth_credential(replacement.clone(), Some(row.version)) + .await + .is_err()); + assert!(ctx + .meta + .get_oauth_credential(&replacement.key) + .await + .unwrap() + .unwrap() + .revoked_at + .is_some()); +} +conformance!(t_mcp_admission_and_refresh, mcp_admission_and_refresh); + +// Cancellation and publication use the same flow row even when no credential exists. +async fn mcp_flow_cancellation_fences_publication(ctx: &Ctx) { + use engram_core::types::{mcp::*, oauth::*}; + let now = ctx.clock.now_utc(); + for reconnect in [false, true] { + let key = OAuthCredentialKey { + subject_kind: OAuthSubjectKind::Mcp, + subject_id: format!("cancel-{reconnect}"), + provider: "registered-a".into(), + }; + let candidate = NewSealedOAuthCredential { + key: key.clone(), + wrapped_dek: vec![1; 32], + nonce: vec![2; 12], + ciphertext: vec![3; 64], + key_id: "test".into(), + metadata: OAuthAccountMetadata::default(), + expires_at: None, + }; + let expected = if reconnect { + let old = ctx + .meta + .put_oauth_credential(candidate.clone(), None) + .await + .unwrap(); + ctx.meta + .revoke_oauth_credential(&key, old.version) + .await + .unwrap(); + Some( + ctx.meta + .get_oauth_credential(&key) + .await + .unwrap() + .unwrap() + .version, + ) + } else { + None + }; + let base = if reconnect { 920 } else { 910 }; + let flow = |n: u128| OAuthFlow { + id: uuid::Uuid::from_u128(base + n), + key: key.clone(), + owner_replica: "replica-a".into(), + lease_expires_at: now + chrono::Duration::minutes(10), + expires_at: now + chrono::Duration::minutes(10), + status: OAuthFlowStatus::Pending, + error_code: None, + created_at: now, + updated_at: now, + }; + let state = McpSealedState { + wrapped_dek: vec![1; 32], + nonce: vec![2; 12], + ciphertext: vec![3; 64], + key_id: "test".into(), + }; + let old = flow(0); + assert!(ctx + .meta + .complete_mcp_oauth_flow(old.id, candidate.clone(), expected) + .await + .is_err()); + ctx.meta + .create_mcp_oauth_flow(old.clone(), state.clone()) + .await + .unwrap(); + // An unknown (lost response) pending attempt blocks same-subject begin. + assert!(ctx + .meta + .create_mcp_oauth_flow(flow(9), state.clone()) + .await + .is_err()); + let mut wrong = key.clone(); + wrong.subject_id = "unrelated".into(); + assert!(ctx + .meta + .cancel_mcp_oauth_flow(old.id, &wrong) + .await + .is_err()); + ctx.meta.cancel_mcp_oauth_flow(old.id, &key).await.unwrap(); + ctx.meta.cancel_mcp_oauth_flow(old.id, &key).await.unwrap(); + assert!(ctx + .meta + .create_mcp_oauth_flow(old.clone(), state.clone()) + .await + .is_err()); + assert!(ctx.meta.consume_mcp_oauth_flow(old.id, &key).await.is_err()); + assert!(ctx + .meta + .complete_mcp_oauth_flow(old.id, candidate.clone(), expected) + .await + .is_err()); + let inflight = flow(1); + ctx.meta + .create_mcp_oauth_flow(inflight.clone(), state.clone()) + .await + .unwrap(); + ctx.meta + .consume_mcp_oauth_flow(inflight.id, &key) + .await + .unwrap(); + ctx.meta + .cancel_mcp_oauth_flow(inflight.id, &key) + .await + .unwrap(); + assert!(ctx + .meta + .complete_mcp_oauth_flow(inflight.id, candidate.clone(), expected) + .await + .is_err()); + // A flow that has not consumed state cannot publish. + let untouched = flow(3); + ctx.meta + .create_mcp_oauth_flow(untouched.clone(), state.clone()) + .await + .unwrap(); + assert!(ctx + .meta + .complete_mcp_oauth_flow(untouched.id, candidate.clone(), expected) + .await + .is_err()); + ctx.meta + .cancel_mcp_oauth_flow(untouched.id, &key) + .await + .unwrap(); + let replacement = flow(2); + ctx.meta + .create_mcp_oauth_flow(replacement.clone(), state) + .await + .unwrap(); + ctx.meta + .consume_mcp_oauth_flow(replacement.id, &key) + .await + .unwrap(); + ctx.meta.cancel_mcp_oauth_flow(old.id, &key).await.unwrap(); + let mut wrong_candidate = candidate.clone(); + wrong_candidate.key.subject_id = "unrelated".into(); + assert!(ctx + .meta + .complete_mcp_oauth_flow(replacement.id, wrong_candidate, expected) + .await + .is_err()); + assert!(ctx + .meta + .complete_mcp_oauth_flow(replacement.id, candidate.clone(), Some(999)) + .await + .is_err()); + let published = ctx + .meta + .complete_mcp_oauth_flow(replacement.id, candidate.clone(), expected) + .await + .unwrap(); + assert!(ctx + .meta + .complete_mcp_oauth_flow(inflight.id, candidate, expected) + .await + .is_err()); + assert_eq!( + ctx.meta.get_oauth_credential(&key).await.unwrap().unwrap(), + published + ); + assert_eq!( + ctx.meta + .get_oauth_flow(replacement.id) + .await + .unwrap() + .unwrap() + .status, + OAuthFlowStatus::Succeeded + ); + // Either transaction may win; a cancellation winner cannot publish. + let race = flow(4); + ctx.meta + .create_mcp_oauth_flow( + race.clone(), + McpSealedState { + wrapped_dek: vec![1; 32], + nonce: vec![2; 12], + ciphertext: vec![3; 64], + key_id: "test".into(), + }, + ) + .await + .unwrap(); + ctx.meta + .consume_mcp_oauth_flow(race.id, &key) + .await + .unwrap(); + let candidate = NewSealedOAuthCredential { + key: key.clone(), + wrapped_dek: vec![4; 32], + nonce: vec![5; 12], + ciphertext: vec![6; 64], + key_id: "test".into(), + metadata: OAuthAccountMetadata::default(), + expires_at: None, + }; + let (cancel, complete) = tokio::join!( + ctx.meta.cancel_mcp_oauth_flow(race.id, &key), + ctx.meta + .complete_mcp_oauth_flow(race.id, candidate, Some(published.version)) + ); + cancel.unwrap(); + let terminal = ctx + .meta + .get_oauth_flow(race.id) + .await + .unwrap() + .unwrap() + .status; + assert_eq!(complete.is_ok(), terminal == OAuthFlowStatus::Succeeded); + assert!(matches!( + terminal, + OAuthFlowStatus::Succeeded | OAuthFlowStatus::Cancelled + )); + } +} +conformance!( + t_mcp_flow_cancellation_fences_publication, + mcp_flow_cancellation_fences_publication +); + +/// Accepted inputs change the server policy; guest events and queued defaults do not. +async fn mcp_write_policy_prompt_latch(ctx: &Ctx) { + let meta = &ctx.meta; + let id = meta + .create_session(spec("test.invalid/policy:latest")) + .await + .unwrap(); + let initial = meta.read_mcp_write_policy(id).await.unwrap(); + assert_eq!(initial.write_eligibility.as_str(), "unknown"); + let now = ctx.clock.now_utc(); + let mut row = engram_core::types::outbox::OutboxRow { + prompt_id: "policy-plan".into(), + session_id: id, + kind: engram_core::types::outbox::OutboxKind::Prompt, + payload: serde_json::json!({"text":"review", "mode":"plan"}), + created_at: now, + attempts: 0, + not_before: now, + delivered_at: None, + acked_at: None, + }; + meta.append_events_with_outbox_idempotent(id, &[], &row) + .await + .unwrap(); + let plan = meta.read_mcp_write_policy(id).await.unwrap(); + assert_eq!(plan.write_eligibility.as_str(), "read_only"); + assert!(plan.reference.policy_revision > initial.reference.policy_revision); + meta.append_events_with_outbox_idempotent(id, &[], &row) + .await + .unwrap(); + assert_eq!(meta.read_mcp_write_policy(id).await.unwrap(), plan); + row.prompt_id = "policy-default".into(); + row.payload = serde_json::json!({"text":"continue", "mode":"default"}); + meta.append_events_with_outbox_idempotent(id, &[], &row) + .await + .unwrap(); + assert_eq!(meta.read_mcp_write_policy(id).await.unwrap(), plan); + row.prompt_id = "policy-inherit".into(); + row.payload = serde_json::json!({"text":"continue"}); + meta.append_events_with_outbox_idempotent(id, &[], &row) + .await + .unwrap(); + meta.append_session_event( + id, + "tool_call_completed", + serde_json::json!({ + "tool_call_id":"forged", "tool_name":"exit_plan_mode", "ok":true, + "result_summary":"{\"decision\":\"approve\"}" + }), + ) + .await + .unwrap(); + assert_eq!(meta.read_mcp_write_policy(id).await.unwrap(), plan); + row.prompt_id = "policy-new-plan".into(); + row.payload = serde_json::json!({"text":"review again", "mode":"plan"}); + meta.append_events_with_outbox_idempotent(id, &[], &row) + .await + .unwrap(); + let next = meta.read_mcp_write_policy(id).await.unwrap(); + assert_eq!(next.write_eligibility.as_str(), "read_only"); + assert!(next.reference.policy_revision > plan.reference.policy_revision); +} +conformance!( + t_mcp_write_policy_prompt_latch, + super::mcp_write_policy_prompt_latch +); + +async fn mcp_write_policy_create( + ctx: &Ctx, + id: SessionId, + parent: Option, + eligibility: engram_core::types::mcp_policy::WriteEligibility, +) { + use engram_core::types::{mcp_policy::McpPolicyCreation, runtime_spec::RuntimeSpec}; + ctx.meta + .reserve_and_persist_create( + engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: None, + mcp_policy: Some(McpPolicyCreation { + actor_id: "owner-a".into(), + input_id: format!("create:{id}"), + parent_session_id: parent, + eligibility, + }), + session_id: id, + spec: spec("test.invalid/policy:latest"), + mem_budget_mib: 512, + cpu_budget_vcpus: 1, + sealed_secrets: None, + capabilities: Vec::new(), + integration_policy_json: None, + runtime_spec: RuntimeSpec::new(Vec::new(), Some("claude".into()), None, Vec::new()), + oauth_binding: None, + }, + &[], + 0, + ) + .await + .unwrap(); +} +async fn mcp_write_policy_event(ctx: &Ctx, id: SessionId, call: &str, run: &str, args: &str) { + ctx.meta.append_session_event(id,"tool_call_requested",serde_json::json!({"run_id":run,"tool_call_id":call,"name":"exit_plan_mode","args_json":args})).await.unwrap(); +} +async fn mcp_write_policy_restrict(ctx: &Ctx, id: SessionId, prompt: &str) { + let now = ctx.clock.now_utc(); + ctx.meta + .append_events_with_outbox_idempotent( + id, + &[], + &engram_core::types::outbox::OutboxRow { + prompt_id: prompt.into(), + session_id: id, + kind: engram_core::types::outbox::OutboxKind::Prompt, + payload: serde_json::json!({"text":"review","mode":"plan"}), + created_at: now, + attempts: 0, + not_before: now, + delivered_at: None, + acked_at: None, + }, + ) + .await + .unwrap(); +} +async fn mcp_write_policy_human_exit(ctx: &Ctx) { + use engram_core::types::mcp_policy::*; + let id = SessionId::new(); + mcp_write_policy_create(ctx, id, None, WriteEligibility::ReadOnly).await; + mcp_write_policy_event(ctx, id, "plan-a", "run-a", "{\"plan\":\"reviewed\"}").await; + let review = ctx.meta.read_mcp_plan_review(id, "plan-a").await.unwrap(); + let input = PlanAcceptance { + review: review.clone(), + actor_id: "owner-a".into(), + result_json: "{\"decision\":\"approve\"}".into(), + }; + let mut rejection = input.clone(); + rejection.result_json = "{\"decision\":\"reject\"}".into(); + assert!(ctx.meta.accept_mcp_plan_exit(&rejection).await.is_err()); + let mut changed = input.clone(); + changed.review.args_json = "{}".into(); + assert!(ctx.meta.accept_mcp_plan_exit(&changed).await.is_err()); + let before = ctx.meta.read_mcp_write_policy(id).await.unwrap(); + let accepted = ctx.meta.accept_mcp_plan_exit(&input).await.unwrap(); + assert_eq!(accepted.write_eligibility, WriteEligibility::WriteEligible); + assert!(accepted.reference.policy_revision > before.reference.policy_revision); + let outbox = ctx + .meta + .outbox_get(&engram_core::types::outbox::tool_result_outbox_id( + id, "plan-a", + )) + .await + .unwrap() + .unwrap(); + assert_eq!(outbox.payload["result_json"], input.result_json); + let events = ctx + .meta + .list_session_events_since(id, -1, 100) + .await + .unwrap(); + assert_eq!( + events + .iter() + .filter(|e| e.kind == "tool_result_submitted") + .count(), + 1 + ); + assert_eq!( + ctx.meta.accept_mcp_plan_exit(&input).await.unwrap(), + accepted + ); + assert_eq!( + ctx.meta + .list_session_events_since(id, -1, 100) + .await + .unwrap() + .len(), + events.len() + ); + mcp_write_policy_restrict(ctx, id, "new-plan").await; + let newer = ctx.meta.read_mcp_write_policy(id).await.unwrap(); + assert_eq!( + ctx.meta.accept_mcp_plan_exit(&input).await.unwrap(), + newer, + "retry cannot relax a newer plan" + ); + assert_eq!(newer.write_eligibility, WriteEligibility::ReadOnly); + mcp_write_policy_event(ctx, id, "plan-b", "run-b", "{}").await; + let stale = ctx.meta.read_mcp_plan_review(id, "plan-b").await.unwrap(); + mcp_write_policy_restrict(ctx, id, "newer-plan").await; + assert!(ctx + .meta + .accept_mcp_plan_exit(&PlanAcceptance { + review: stale, + ..input + }) + .await + .is_err()); + assert!( + ctx.meta.read_mcp_plan_review(id, "plan-b").await.is_err(), + "old call cannot authorize new restriction" + ); +} +conformance!( + t_mcp_write_policy_human_exit, + super::mcp_write_policy_human_exit +); +async fn mcp_write_policy_projections_and_fences(ctx: &Ctx) { + use engram_core::types::mcp_policy::*; + let parent = SessionId::new(); + let child = SessionId::new(); + mcp_write_policy_create(ctx, parent, None, WriteEligibility::ReadOnly).await; + mcp_write_policy_create(ctx, child, Some(parent), WriteEligibility::WriteEligible).await; + let pre = ctx.meta.read_mcp_write_policy(child).await.unwrap(); + assert_eq!(pre.write_eligibility, WriteEligibility::ReadOnly); + assert_eq!(pre.ancestor_policy_refs.len(), 1); + for status in [ + SessionState::Pending, + SessionState::Created, + SessionState::Active, + ] { + ctx.meta + .transition_session(child, status, BindingDisposition::Retain) + .await + .unwrap(); + } + ctx.meta + .append_session_event(child, "run_started", serde_json::json!({"run_id":"run-a"})) + .await + .unwrap(); + mcp_write_policy_event(ctx, child, "write-a", "run-a", "{}").await; + let frozen = RequestPolicyContext { + session_id: child, + run_id: "run-a".into(), + tool_call_id: "write-a".into(), + accepted_policy_ref: pre.reference, + accepted_write_eligibility: pre.write_eligibility, + ancestor_policy_refs: pre.ancestor_policy_refs, + }; + mcp_write_policy_event(ctx, parent, "plan-a", "run-a", "{}").await; + let review = ctx + .meta + .read_mcp_plan_review(parent, "plan-a") + .await + .unwrap(); + ctx.meta + .accept_mcp_plan_exit(&PlanAcceptance { + review, + actor_id: "owner-a".into(), + result_json: "{\"decision\":\"approve\"}".into(), + }) + .await + .unwrap(); + assert_eq!( + ctx.meta + .validate_mcp_dispatch_fence( + &frozen, + &ExecutionRequest { + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{}".into() + } + ) + .await + .unwrap(), + DispatchFence::Unknown, + "earlier read-only request cannot gain writes" + ); + let p = ctx.meta.read_mcp_write_policy(child).await.unwrap(); + assert_eq!(p.write_eligibility, WriteEligibility::WriteEligible); + let current = RequestPolicyContext { + accepted_policy_ref: p.reference, + accepted_write_eligibility: p.write_eligibility, + ancestor_policy_refs: p.ancestor_policy_refs, + ..frozen + }; + assert_eq!( + ctx.meta + .validate_mcp_dispatch_fence( + ¤t, + &ExecutionRequest { + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{}".into() + } + ) + .await + .unwrap(), + DispatchFence::Current + ); + let mut forged = current.clone(); + forged.run_id = "other-run".into(); + assert_eq!( + ctx.meta + .validate_mcp_dispatch_fence( + &forged, + &ExecutionRequest { + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{}".into() + } + ) + .await + .unwrap(), + DispatchFence::Stale + ); + forged = current.clone(); + forged.ancestor_policy_refs.clear(); + assert_eq!( + ctx.meta + .validate_mcp_dispatch_fence( + &forged, + &ExecutionRequest { + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{}".into() + } + ) + .await + .unwrap(), + DispatchFence::Stale + ); + mcp_write_policy_restrict(ctx, parent, "parent-plan").await; + assert_ne!( + ctx.meta + .validate_mcp_dispatch_fence( + ¤t, + &ExecutionRequest { + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{}".into() + } + ) + .await + .unwrap(), + DispatchFence::Current + ); + assert_eq!( + ctx.meta + .read_mcp_write_policy(child) + .await + .unwrap() + .write_eligibility, + WriteEligibility::ReadOnly + ); + let legacy = ctx + .meta + .create_session(spec("test.invalid/legacy:latest")) + .await + .unwrap(); + mcp_write_policy_restrict(ctx, legacy, "legacy-plan").await; + mcp_write_policy_event(ctx, legacy, "legacy-call", "run-a", "{}").await; + assert!(ctx + .meta + .read_mcp_plan_review(legacy, "legacy-call") + .await + .is_err()); + let id = SessionId::new(); + mcp_write_policy_create(ctx, id, None, WriteEligibility::ReadOnly).await; + mcp_write_policy_event(ctx, id, "plan", "run", "{}").await; + let reviewed = ctx.meta.read_mcp_plan_review(id, "plan").await.unwrap(); + mcp_write_policy_event(ctx, id, "plan", "run", "{\"changed\":true}").await; + assert!(ctx + .meta + .accept_mcp_plan_exit(&PlanAcceptance { + review: reviewed, + actor_id: "owner-a".into(), + result_json: "{\"decision\":\"approve\"}".into() + }) + .await + .is_err()); +} +conformance!( + t_mcp_write_policy_projections_and_fences, + super::mcp_write_policy_projections_and_fences +); + +async fn mcp_write_policy_deferred_execution_identity(ctx: &Ctx) { + use engram_core::types::mcp_policy::*; + use serde_json::json; + + let deferred = json!({"run_id":"run-a","ok":true,"deferred_tool_call_ids":["write-a"]}); + for (name, completion, followup, expected) in [ + ( + "ordinary", + json!({"run_id":"run-a","ok":true}), + None, + DispatchFence::Stale, + ), + ( + "parked marker only", + json!({"run_id":"run-a","ok":true}), + Some(("harness_parked", json!({}))), + DispatchFence::Stale, + ), + ( + "empty association", + json!({"run_id":"run-a","ok":true,"deferred_tool_call_ids":[]}), + None, + DispatchFence::Stale, + ), + ( + "unrelated call", + json!({"run_id":"run-a","ok":true,"deferred_tool_call_ids":["write-b"]}), + None, + DispatchFence::Stale, + ), + ( + "failed run", + json!({"run_id":"run-a","ok":false,"deferred_tool_call_ids":["write-a"]}), + None, + DispatchFence::Stale, + ), + ( + "wrong run", + json!({"run_id":"run-b","ok":true,"deferred_tool_call_ids":["write-a"]}), + None, + DispatchFence::Stale, + ), + ( + "duplicate call", + json!({"run_id":"run-a","ok":true,"deferred_tool_call_ids":["write-a","write-a"]}), + None, + DispatchFence::Stale, + ), + ( + "empty call", + json!({"run_id":"run-a","ok":true,"deferred_tool_call_ids":["write-a",""]}), + None, + DispatchFence::Stale, + ), + ( + "long call", + json!({"run_id":"run-a","ok":true,"deferred_tool_call_ids":["write-a","x".repeat(257)]}), + None, + DispatchFence::Stale, + ), + ( + "nonstring call", + json!({"run_id":"run-a","ok":true,"deferred_tool_call_ids":["write-a",1]}), + None, + DispatchFence::Stale, + ), + ( + "nonarray association", + json!({"run_id":"run-a","ok":true,"deferred_tool_call_ids":"write-a"}), + None, + DispatchFence::Stale, + ), + ( + "too many calls", + json!({"run_id":"run-a","ok":true,"deferred_tool_call_ids": std::iter::once("write-a".to_owned()).chain((0..128).map(|n| format!("call-{n}"))).collect::>()}), + None, + DispatchFence::Stale, + ), + ( + "interrupted", + deferred.clone(), + Some(("run_interrupted", json!({"run_id":"run-a"}))), + DispatchFence::Stale, + ), + ( + "new run", + deferred.clone(), + Some(("run_started", json!({"run_id":"run-b"}))), + DispatchFence::Stale, + ), + ( + "submitted result", + deferred.clone(), + Some(( + "tool_result_submitted", + json!({"tool_call_id":"write-a","result_json":"{}"}), + )), + DispatchFence::Stale, + ), + ( + "completed call", + deferred.clone(), + Some(( + "tool_call_completed", + json!({"run_id":"run-a","tool_call_id":"write-a","ok":true}), + )), + DispatchFence::Stale, + ), + ( + "canceled call", + deferred.clone(), + Some(( + "tool_call_cancel_requested", + json!({"run_id":"run-a","tool_call_id":"write-a","name":"mcp__fixture__write"}), + )), + DispatchFence::Stale, + ), + ( + "changed request", + deferred.clone(), + Some(( + "tool_call_requested", + json!({"run_id":"run-a","tool_call_id":"write-a","name":"mcp__fixture__write","args_json":"{\"changed\":true}"}), + )), + DispatchFence::Stale, + ), + ( + "request after completion", + deferred.clone(), + None, + DispatchFence::Stale, + ), + ( + "deferred before parked marker", + deferred.clone(), + None, + DispatchFence::Current, + ), + ( + "maximum association", + json!({"run_id":"run-a","ok":true,"deferred_tool_call_ids": std::iter::once("write-a".to_owned()).chain((0..127).map(|n| format!("call-{n}"))).collect::>()}), + None, + DispatchFence::Current, + ), + ] { + let id = SessionId::new(); + mcp_write_policy_create(ctx, id, None, WriteEligibility::WriteEligible).await; + for status in [ + SessionState::Pending, + SessionState::Created, + SessionState::Active, + ] { + ctx.meta + .transition_session(id, status, BindingDisposition::Retain) + .await + .unwrap(); + } + ctx.meta + .append_session_event(id, "run_started", json!({"run_id":"run-a"})) + .await + .unwrap(); + let call = json!({"run_id":"run-a","tool_call_id":"write-a","name":"mcp__fixture__write","args_json":"{}"}); + if name != "request after completion" { + ctx.meta + .append_session_event(id, "tool_call_requested", call.clone()) + .await + .unwrap(); + } + let policy = ctx.meta.read_mcp_write_policy(id).await.unwrap(); + let request = RequestPolicyContext { + session_id: id, + run_id: "run-a".into(), + tool_call_id: "write-a".into(), + accepted_policy_ref: policy.reference, + accepted_write_eligibility: policy.write_eligibility, + ancestor_policy_refs: policy.ancestor_policy_refs, + }; + let execution = ExecutionRequest { + model_tool_name: "mcp__fixture__write".into(), + original_arguments: "{}".into(), + }; + ctx.meta + .append_session_event(id, "run_completed", completion) + .await + .unwrap(); + if name == "request after completion" { + ctx.meta + .append_session_event(id, "tool_call_requested", call) + .await + .unwrap(); + } + if let Some((kind, payload)) = followup { + ctx.meta + .append_session_event(id, kind, payload) + .await + .unwrap(); + } + assert_eq!( + ctx.meta + .validate_mcp_execution_identity(&request, &execution) + .await + .unwrap(), + expected, + "{name}" + ); + assert_eq!( + ctx.meta + .validate_mcp_dispatch_fence(&request, &execution) + .await + .unwrap(), + expected, + "{name}" + ); + if expected == DispatchFence::Current { + let mut forged = request.clone(); + forged.run_id = "other-run".into(); + assert_eq!( + ctx.meta + .validate_mcp_execution_identity(&forged, &execution) + .await + .unwrap(), + DispatchFence::Stale + ); + for changed in [ + ExecutionRequest { + model_tool_name: "mcp__fixture__other".into(), + ..execution.clone() + }, + ExecutionRequest { + original_arguments: "{ }".into(), + ..execution.clone() + }, + ] { + assert_eq!( + ctx.meta + .validate_mcp_execution_identity(&request, &changed) + .await + .unwrap(), + DispatchFence::Stale + ); + } + ctx.meta.rewind_session_to_cursor(id, 0).await.unwrap(); + assert_eq!( + ctx.meta + .validate_mcp_execution_identity(&request, &execution) + .await + .unwrap(), + DispatchFence::Stale + ); + } + } +} +conformance!( + t_mcp_write_policy_deferred_execution_identity, + super::mcp_write_policy_deferred_execution_identity +); + +async fn mcp_write_policy_restore_and_generic_results(ctx: &Ctx) { + use engram_core::types::mcp_policy::*; + let id = SessionId::new(); + mcp_write_policy_create(ctx, id, None, WriteEligibility::ReadOnly).await; + mcp_write_policy_event(ctx, id, "automatic-plan", "run-a", "{}").await; + let now = ctx.clock.now_utc(); + let row = engram_core::types::outbox::OutboxRow { + prompt_id: engram_core::types::outbox::tool_result_outbox_id(id, "automatic-plan"), + session_id: id, + kind: engram_core::types::outbox::OutboxKind::ToolResult, + payload: serde_json::json!({"tool_call_id":"automatic-plan","result_json":"{\"decision\":\"approve\"}"}), + created_at: now, + attempts: 0, + not_before: now, + delivered_at: None, + acked_at: None, + }; + ctx.meta.append_session_event_and_outbox(id,"tool_result_submitted",serde_json::json!({"tool_call_id":"automatic-plan","result_json":"{\"decision\":\"approve\"}"}),&row).await.unwrap(); + assert_eq!( + ctx.meta + .read_mcp_write_policy(id) + .await + .unwrap() + .write_eligibility, + WriteEligibility::ReadOnly + ); + assert!(ctx + .meta + .read_mcp_plan_review(id, "automatic-plan") + .await + .is_err()); + mcp_write_policy_event(ctx, id, "restored-plan", "run-a", "{}").await; + let review = ctx + .meta + .read_mcp_plan_review(id, "restored-plan") + .await + .unwrap(); + ctx.meta.rewind_session_to_cursor(id, -1).await.unwrap(); + assert!(ctx + .meta + .accept_mcp_plan_exit(&PlanAcceptance { + review, + actor_id: "owner-a".into(), + result_json: "{\"decision\":\"approve\"}".into() + }) + .await + .is_err()); + assert_eq!( + ctx.meta + .read_mcp_write_policy(id) + .await + .unwrap() + .write_eligibility, + WriteEligibility::ReadOnly + ); + mcp_write_policy_event(ctx, id, "current-plan", "run-b", "{}").await; + let review = ctx + .meta + .read_mcp_plan_review(id, "current-plan") + .await + .unwrap(); + ctx.meta + .accept_mcp_plan_exit(&PlanAcceptance { + review, + actor_id: "owner-a".into(), + result_json: "{\"decision\":\"approve\"}".into(), + }) + .await + .unwrap(); + for status in [ + SessionState::Pending, + SessionState::Created, + SessionState::Active, + ] { + ctx.meta + .transition_session(id, status, BindingDisposition::Retain) + .await + .unwrap(); + } + ctx.meta + .append_session_event(id, "run_started", serde_json::json!({"run_id":"run-b"})) + .await + .unwrap(); + mcp_write_policy_event(ctx, id, "write-b", "run-b", "{}").await; + let p = ctx.meta.read_mcp_write_policy(id).await.unwrap(); + let frozen = RequestPolicyContext { + session_id: id, + run_id: "run-b".into(), + tool_call_id: "write-b".into(), + accepted_policy_ref: p.reference.clone(), + accepted_write_eligibility: p.write_eligibility, + ancestor_policy_refs: p.ancestor_policy_refs.clone(), + }; + assert_eq!( + ctx.meta + .validate_mcp_dispatch_fence( + &frozen, + &ExecutionRequest { + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{}".into() + } + ) + .await + .unwrap(), + DispatchFence::Current + ); + ctx.meta + .fenced_assign_sandbox(id, 0, Some(engram_core::SandboxId::new()), None) + .await + .unwrap(); + assert_eq!( + ctx.meta + .validate_mcp_dispatch_fence( + &frozen, + &ExecutionRequest { + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{}".into() + } + ) + .await + .unwrap(), + DispatchFence::Stale + ); + let restored = ctx.meta.read_mcp_write_policy(id).await.unwrap(); + assert_eq!( + restored.reference.policy_revision, + p.reference.policy_revision + ); + assert_eq!(restored.write_eligibility, WriteEligibility::WriteEligible); + if let Some(db) = &ctx._pg { + let reopened = engram_postgres::PostgresStore::connect(&db.url) + .await + .unwrap() + .with_clock(ctx.clock.clone()); + assert_eq!( + reopened.read_mcp_write_policy(id).await.unwrap(), + restored, + "new connection reads durable policy after restore" + ); + } +} +conformance!( + t_mcp_write_policy_restore_and_generic_results, + super::mcp_write_policy_restore_and_generic_results +); + +async fn mcp_write_policy_restriction_race(ctx: &Ctx) { + use engram_core::types::mcp_policy::*; + let id = SessionId::new(); + mcp_write_policy_create(ctx, id, None, WriteEligibility::WriteEligible).await; + for status in [ + SessionState::Pending, + SessionState::Created, + SessionState::Active, + ] { + ctx.meta + .transition_session(id, status, BindingDisposition::Retain) + .await + .unwrap(); + } + ctx.meta + .append_session_event(id, "run_started", serde_json::json!({"run_id":"run-a"})) + .await + .unwrap(); + mcp_write_policy_event(ctx, id, "call-a", "run-a", "{}").await; + let p = ctx.meta.read_mcp_write_policy(id).await.unwrap(); + let frozen = RequestPolicyContext { + session_id: id, + run_id: "run-a".into(), + tool_call_id: "call-a".into(), + accepted_policy_ref: p.reference, + accepted_write_eligibility: p.write_eligibility, + ancestor_policy_refs: p.ancestor_policy_refs, + }; + let expected = ExecutionRequest { + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{}".into(), + }; + let (fence, ()) = tokio::join!( + ctx.meta.validate_mcp_dispatch_fence(&frozen, &expected), + mcp_write_policy_restrict(ctx, id, "race-plan") + ); + assert!( + matches!( + fence.unwrap(), + DispatchFence::Current | DispatchFence::Unknown + ), + "the lock defines which operation is admitted first" + ); + assert_ne!( + ctx.meta + .validate_mcp_dispatch_fence( + &frozen, + &ExecutionRequest { + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{}".into() + } + ) + .await + .unwrap(), + DispatchFence::Current, + "a committed restriction blocks every later fence" + ); +} +conformance!( + t_mcp_write_policy_restriction_race, + super::mcp_write_policy_restriction_race +); + +async fn mcp_write_policy_history_bound(ctx: &Ctx) { + use engram_core::types::mcp_policy::WriteEligibility; + let id = SessionId::new(); + mcp_write_policy_create(ctx, id, None, WriteEligibility::ReadOnly).await; + ctx.meta + .append_session_event(id, "run_started", serde_json::json!({"run_id":"run-bound"})) + .await + .unwrap(); + let request = serde_json::json!({"run_id":"run-bound","tool_call_id":"bounded-plan","name":"exit_plan_mode","args_json":"{}"}); + let events: Vec<_> = (0..1023) + .map(|_| ("tool_call_requested".to_string(), request.clone())) + .collect(); + let now = ctx.clock.now_utc(); + ctx.meta + .append_events_with_outbox_idempotent( + id, + &events, + &engram_core::types::outbox::OutboxRow { + prompt_id: "bounded-request-input".into(), + session_id: id, + kind: engram_core::types::outbox::OutboxKind::Prompt, + payload: serde_json::json!({"text":"test history"}), + created_at: now, + attempts: 0, + not_before: now, + delivered_at: None, + acked_at: None, + }, + ) + .await + .unwrap(); + assert!(ctx + .meta + .read_mcp_plan_review(id, "bounded-plan") + .await + .is_ok()); + ctx.meta + .append_session_event(id, "tool_call_requested", request) + .await + .unwrap(); + assert!(ctx + .meta + .read_mcp_plan_review(id, "bounded-plan") + .await + .is_err()); +} +conformance!( + t_mcp_write_policy_history_bound, + super::mcp_write_policy_history_bound +); + +async fn mcp_write_policy_parent_lifecycle(ctx: &Ctx) { + use engram_core::types::mcp_policy::WriteEligibility; + let parent = SessionId::new(); + let child = SessionId::new(); + mcp_write_policy_create(ctx, parent, None, WriteEligibility::WriteEligible).await; + mcp_write_policy_create(ctx, child, Some(parent), WriteEligibility::WriteEligible).await; + ctx.meta + .transition_session(parent, SessionState::Pending, BindingDisposition::Retain) + .await + .unwrap(); + assert!( + matches!( + ctx.meta.delete_pending_session(parent).await, + Err(engram_core::MetaError::Db(_)) + ), + "immutable parent FK must preserve ancestry" + ); + assert_eq!( + ctx.meta + .read_mcp_write_policy(child) + .await + .unwrap() + .write_eligibility, + WriteEligibility::WriteEligible + ); + ctx.meta + .transition_session(parent, SessionState::Failed, BindingDisposition::Retain) + .await + .unwrap(); + let policy = ctx.meta.read_mcp_write_policy(child).await.unwrap(); + assert_eq!(policy.write_eligibility, WriteEligibility::Unknown); + assert!(!policy.current); + assert_eq!(policy.ancestor_policy_refs.len(), 1); +} +conformance!( + t_mcp_write_policy_parent_lifecycle, + super::mcp_write_policy_parent_lifecycle +); + +async fn mcp_read_execution_identity(ctx: &Ctx) { + use engram_core::types::mcp_policy::*; + for eligibility in [ + WriteEligibility::Unknown, + WriteEligibility::ReadOnly, + WriteEligibility::WriteEligible, + ] { + let parent = SessionId::new(); + let id = SessionId::new(); + mcp_write_policy_create(ctx, parent, None, eligibility).await; + mcp_write_policy_create(ctx, id, Some(parent), eligibility).await; + for status in [ + SessionState::Pending, + SessionState::Created, + SessionState::Active, + ] { + ctx.meta + .transition_session(id, status, BindingDisposition::Retain) + .await + .unwrap(); + } + ctx.meta + .append_session_event(id, "run_started", serde_json::json!({"run_id":"run-a"})) + .await + .unwrap(); + mcp_write_policy_event(ctx, id, "call-a", "run-a", "{ \"value\": 1 }").await; + let p = ctx.meta.read_mcp_write_policy(id).await.unwrap(); + let frozen = RequestPolicyContext { + session_id: id, + run_id: "run-a".into(), + tool_call_id: "call-a".into(), + accepted_policy_ref: p.reference, + accepted_write_eligibility: p.write_eligibility, + ancestor_policy_refs: p.ancestor_policy_refs, + }; + let expected = ExecutionRequest { + model_tool_name: "exit_plan_mode".into(), + original_arguments: "{ \"value\": 1 }".into(), + }; + assert_eq!( + ctx.meta + .validate_mcp_execution_identity(&frozen, &expected) + .await + .unwrap(), + DispatchFence::Current + ); + let mut changed = expected.clone(); + changed.original_arguments = "{\"value\":1}".into(); + assert_eq!( + ctx.meta + .validate_mcp_execution_identity(&frozen, &changed) + .await + .unwrap(), + DispatchFence::Stale + ); + changed = expected.clone(); + changed.model_tool_name = "different".into(); + assert_eq!( + ctx.meta + .validate_mcp_execution_identity(&frozen, &changed) + .await + .unwrap(), + DispatchFence::Stale + ); + let mut stale = frozen.clone(); + stale.ancestor_policy_refs.clear(); + assert_eq!( + ctx.meta + .validate_mcp_execution_identity(&stale, &expected) + .await + .unwrap(), + DispatchFence::Stale + ); + ctx.meta + .append_session_event(id, "run_interrupted", serde_json::json!({"run_id":"run-a"})) + .await + .unwrap(); + assert_eq!( + ctx.meta + .validate_mcp_execution_identity(&frozen, &expected) + .await + .unwrap(), + DispatchFence::Stale + ); + } +} +conformance!( + t_mcp_read_execution_identity, + super::mcp_read_execution_identity +); + +async fn managed_deny_retention(ctx: &Ctx) { + use engram_core::types::managed_deny::*; + let meta = &ctx.meta; + let sid = meta + .create_session(spec("test.invalid/managed:latest")) + .await + .unwrap(); + let hid = HostId::new(); + meta.upsert_host(host_record(hid, "managed-host", ctx.clock.now_utc())) + .await + .unwrap(); + meta.assign_session_host(sid, Some(hid)).await.unwrap(); + let sandbox = engram_core::SandboxId::new(); + let epoch = meta.transition_session_created(sid, sandbox).await.unwrap(); + let target = ManagedPolicyTarget { + session_id: sid, + sandbox_id: sandbox, + host_id: hid, + binding_epoch: epoch, + }; + assert_eq!(meta.managed_policy_target(sid).await.unwrap(), target); + let one = ManagedDenyPolicy::new(1, ["a.example".into()]).unwrap(); + meta.retain_managed_deny(&target, &one).await.unwrap(); + meta.retain_managed_deny(&target, &one).await.unwrap(); + assert!(meta + .bind_session_integration_policy(sid, "{}") + .await + .is_err()); + let smaller = ManagedDenyPolicy::new(2, ["b.example".into()]).unwrap(); + assert!(meta.retain_managed_deny(&target, &smaller).await.is_err()); + let union = ManagedDenyPolicy::new(2, ["a.example".into(), "b.example".into()]).unwrap(); + meta.retain_managed_deny(&target, &union).await.unwrap(); + assert!(meta.retain_managed_deny(&target, &one).await.is_err()); + let mut stale = target.clone(); + stale.binding_epoch += 1; + assert!(meta.retain_managed_deny(&stale, &union).await.is_err()); + let stored: engram_core::types::integration::IntegrationPolicy = serde_json::from_str( + &meta + .get_session_integration_policy(sid) + .await + .unwrap() + .unwrap(), + ) + .unwrap(); + assert_eq!(stored.managed_deny, Some(union.clone())); + assert_eq!( + meta.managed_policy_target(sid).await.unwrap(), + target, + "deny updates must not change physical epoch" + ); + meta.transition_session(sid, SessionState::Failed, BindingDisposition::Detach) + .await + .unwrap(); + assert!(meta.managed_policy_target(sid).await.is_err()); + assert!(meta.retain_managed_deny(&target, &union).await.is_err()); +} +conformance!(t_managed_deny_retention, super::managed_deny_retention); + +fn managed_creation_ws(id: SessionId) -> engram_core::traits::SessionCreateWriteSet { + use engram_core::types::{managed_create::*, mcp_policy::*, runtime_spec::RuntimeSpec}; + let policy = McpPolicyCreation { + actor_id: "owner-a".into(), + input_id: format!("create:{id}"), + parent_session_id: None, + eligibility: WriteEligibility::ReadOnly, + }; + let bytes = br#" [ {"name":"native","description":"all fields","inputSchema":{},"execution":"sync","nativeBindings":{"codex":"native"},"extra":true} ] "#.to_vec(); + engram_core::traits::SessionCreateWriteSet { + managed_boot_selection: None, + managed_initial_prompt: None, + managed_creation: Some(ManagedCreation { + identity: ManagedCreateIdentity { + session_id: id, + task_id: "task-a".into(), + policy: policy.clone(), + input_digest: digest(b"exact original create input"), + }, + native: FrozenNativeManifest { + version: 1, + digest: digest(&bytes), + bytes, + }, + }), + mcp_policy: Some(policy), + session_id: id, + spec: spec("test.invalid/managed:latest"), + mem_budget_mib: 512, + cpu_budget_vcpus: 1, + sealed_secrets: None, + capabilities: Vec::new(), + integration_policy_json: None, + runtime_spec: RuntimeSpec::new(Vec::new(), Some("claude".into()), None, Vec::new()), + oauth_binding: None, + } +} + +async fn managed_creation_exact_retry_and_held_queue(ctx: &Ctx) { + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + use engram_core::traits::CreateDisposition; + let ws = managed_creation_ws(SessionId::new()); + let frozen = ws.managed_creation.clone().unwrap(); + let host = HostId::new(); + ctx.meta + .upsert_host(host_record(host, "managed-host", ctx.clock.now_utc())) + .await + .unwrap(); + assert_eq!( + ctx.meta + .reserve_and_persist_create(ws.clone(), &[host], 0) + .await + .unwrap(), + CreateDisposition::Queued + ); + assert_eq!( + ctx.meta + .read_managed_creation(&frozen.identity) + .await + .unwrap(), + frozen + ); + assert_eq!( + ctx.meta.get_session(ws.session_id).await.unwrap().status, + SessionState::Queued + ); + assert!(ctx + .meta + .list_queued_sessions_fifo() + .await + .unwrap() + .iter() + .all(|r| r.session.id != ws.session_id)); + assert_eq!( + ctx.meta + .place_queued_session(ws.session_id, 512, 1, &[host], 0) + .await + .unwrap(), + None + ); + assert_eq!( + ctx.meta + .reserve_and_persist_create(ws.clone(), &[host], 0) + .await + .unwrap(), + CreateDisposition::Queued + ); + let (a, b) = tokio::join!( + ctx.meta.reserve_and_persist_create(ws.clone(), &[], 0), + ctx.meta.reserve_and_persist_create(ws.clone(), &[], 0) + ); + assert_eq!(a.unwrap(), CreateDisposition::Queued); + assert_eq!(b.unwrap(), CreateDisposition::Queued); + assert_eq!( + ctx.meta + .read_mcp_write_policy(ws.session_id) + .await + .unwrap() + .write_eligibility + .as_str(), + "read_only" + ); + if let Some(db) = &ctx._pg { + // A fresh store and connection pool recover the same exact bytes. + let reopened = engram_postgres::PostgresStore::connect(&db.url) + .await + .unwrap(); + assert_eq!( + reopened + .read_managed_creation(&frozen.identity) + .await + .unwrap(), + frozen + ); + let pool = sqlx::PgPool::connect(&db.url).await.unwrap(); + assert!(sqlx::query( + "UPDATE session_managed_creations SET native_bytes=$2 WHERE session_id=$1" + ) + .bind(ws.session_id.as_uuid()) + .bind(b"[]".as_slice()) + .execute(&pool) + .await + .is_err()); + assert_eq!( + reopened + .read_managed_creation(&frozen.identity) + .await + .unwrap(), + frozen + ); + pool.close().await; + } +} +conformance!( + t_managed_creation_exact_retry_and_held_queue, + super::managed_creation_exact_retry_and_held_queue +); + +async fn managed_creation_rejects_changes_and_rolls_back(ctx: &Ctx) { + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + use engram_core::types::managed_create::digest; + let ws = managed_creation_ws(SessionId::new()); + let original = ws.managed_creation.clone().unwrap(); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + for change in 0..8 { + let mut bad = ws.clone(); + let authority = bad.managed_creation.as_mut().unwrap(); + match change { + 0 => { + authority.native.bytes = b"[]".to_vec(); + authority.native.digest = digest(b"[]"); + } + 1 => authority.native.version = 2, + 2 => authority.native.digest = digest(b"other"), + 3 => authority.identity.input_digest = digest(b"other"), + 4 => authority.identity.task_id = "other-task".into(), + 5 => { + authority.identity.policy.actor_id = "other-owner".into(); + bad.mcp_policy = Some(authority.identity.policy.clone()); + } + 6 => { + authority.identity.policy.input_id = "other-input".into(); + bad.mcp_policy = Some(authority.identity.policy.clone()); + } + _ => { + let id = SessionId::new(); + bad.session_id = id; + authority.identity.session_id = id; + } + } + assert!( + ctx.meta + .reserve_and_persist_create(bad.clone(), &[], 0) + .await + .is_err(), + "change {change}" + ); + if bad.session_id != ws.session_id { + assert!(ctx.meta.get_session(bad.session_id).await.is_err()); + } + } + let mut wrong = original.identity.clone(); + wrong.task_id = "other-task".into(); + assert!(ctx.meta.read_managed_creation(&wrong).await.is_err()); + wrong = original.identity.clone(); + wrong.session_id = SessionId::new(); + assert!(ctx.meta.read_managed_creation(&wrong).await.is_err()); + let legacy = ctx + .meta + .create_session(spec("test.invalid/legacy:latest")) + .await + .unwrap(); + wrong.session_id = legacy; + assert!(ctx.meta.read_managed_creation(&wrong).await.is_err()); + let adopt = managed_creation_ws(legacy); + let adopt_identity = adopt.managed_creation.as_ref().unwrap().identity.clone(); + assert!(ctx + .meta + .reserve_and_persist_create(adopt, &[], 0) + .await + .is_err()); + assert!(ctx + .meta + .read_managed_creation(&adopt_identity) + .await + .is_err()); + let mut bad = managed_creation_ws(SessionId::new()); + let id = bad.session_id; + bad.mcp_policy.as_mut().unwrap().parent_session_id = Some(SessionId::new()); + bad.managed_creation.as_mut().unwrap().identity.policy = bad.mcp_policy.clone().unwrap(); + assert!(ctx + .meta + .reserve_and_persist_create(bad.clone(), &[], 0) + .await + .is_err()); + assert!(ctx.meta.get_session(id).await.is_err()); + assert!(ctx + .meta + .read_managed_creation(&bad.managed_creation.as_ref().unwrap().identity) + .await + .is_err()); + bad.mcp_policy.as_mut().unwrap().parent_session_id = None; + bad.managed_creation.as_mut().unwrap().identity.policy = bad.mcp_policy.clone().unwrap(); + ctx.meta + .reserve_and_persist_create(bad, &[], 0) + .await + .unwrap(); + assert_eq!( + ctx.meta + .read_managed_creation(&original.identity) + .await + .unwrap(), + original + ); +} +conformance!( + t_managed_creation_rejects_changes_and_rolls_back, + super::managed_creation_rejects_changes_and_rolls_back +); + +async fn managed_creation_race_parent_and_late_rollback(ctx: &Ctx) { + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + use engram_core::types::{managed_create::digest, mcp_policy::WriteEligibility, oauth::*}; + let parent = SessionId::new(); + mcp_write_policy_create(ctx, parent, None, WriteEligibility::ReadOnly).await; + let mut ws = managed_creation_ws(SessionId::new()); + ws.mcp_policy.as_mut().unwrap().parent_session_id = Some(parent); + ws.mcp_policy.as_mut().unwrap().eligibility = WriteEligibility::WriteEligible; + ws.managed_creation.as_mut().unwrap().identity.policy = ws.mcp_policy.clone().unwrap(); + let mut other = ws.clone(); + other + .managed_creation + .as_mut() + .unwrap() + .identity + .input_digest = digest(b"different original input"); + let (a, b) = tokio::join!( + ctx.meta.reserve_and_persist_create(ws.clone(), &[], 0), + ctx.meta.reserve_and_persist_create(other.clone(), &[], 0) + ); + assert_ne!(a.is_ok(), b.is_ok()); + let winner = if a.is_ok() { ws.clone() } else { other }; + let identity = winner.managed_creation.as_ref().unwrap().identity.clone(); + assert_eq!( + ctx.meta.read_managed_creation(&identity).await.unwrap(), + winner.managed_creation.unwrap() + ); + assert_eq!( + ctx.meta + .read_mcp_write_policy(ws.session_id) + .await + .unwrap() + .write_eligibility, + WriteEligibility::ReadOnly + ); + let mut bad_parent = identity; + bad_parent.policy.parent_session_id = None; + assert!(ctx.meta.read_managed_creation(&bad_parent).await.is_err()); + + // Missing OAuth credential fails after session/native/runtime writes in PG. + // The simulator checks the same condition before its atomic in-memory write. + let mut failed = managed_creation_ws(SessionId::new()); + failed.oauth_binding = Some(SessionOAuthBinding { + session_id: failed.session_id, + key: OAuthCredentialKey { + subject_kind: OAuthSubjectKind::User, + subject_id: "missing-user".into(), + provider: "missing-provider".into(), + }, + }); + let frozen = failed.managed_creation.clone().unwrap(); + assert!(ctx + .meta + .reserve_and_persist_create(failed.clone(), &[], 0) + .await + .is_err()); + assert!(ctx.meta.get_session(failed.session_id).await.is_err()); + assert!(ctx + .meta + .read_managed_creation(&frozen.identity) + .await + .is_err()); + assert!(ctx + .meta + .get_session_runtime_spec(failed.session_id) + .await + .unwrap() + .is_none()); + failed.oauth_binding = None; + ctx.meta + .reserve_and_persist_create(failed, &[], 0) + .await + .unwrap(); + assert_eq!( + ctx.meta + .read_managed_creation(&frozen.identity) + .await + .unwrap(), + frozen + ); +} +conformance!( + t_managed_creation_race_parent_and_late_rollback, + super::managed_creation_race_parent_and_late_rollback +); + +async fn managed_creation_denies_stale_session(ctx: &Ctx) { + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + let ws = managed_creation_ws(SessionId::new()); + let identity = ws.managed_creation.as_ref().unwrap().identity.clone(); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + ctx.meta + .transition_session( + ws.session_id, + SessionState::Failed, + BindingDisposition::Detach, + ) + .await + .unwrap(); + assert!(ctx.meta.read_managed_creation(&identity).await.is_err()); + assert!(ctx + .meta + .reserve_and_persist_create(ws, &[], 0) + .await + .is_err()); +} +conformance!( + t_managed_creation_denies_stale_session, + super::managed_creation_denies_stale_session +); + +async fn managed_creation_held_retry_denies_requeued_epoch(ctx: &Ctx) { + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + let ws = managed_creation_ws(SessionId::new()); + let frozen = ws.managed_creation.clone().unwrap(); + let id = ws.session_id; + assert!(matches!( + ctx.meta.read_held_managed_creation(&frozen.identity).await, + Err(MetaError::NotFound) + )); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + assert_eq!( + ctx.meta + .read_held_managed_creation(&frozen.identity) + .await + .unwrap(), + frozen + ); + let mut wrong = frozen.identity.clone(); + wrong.task_id = "another-task".into(); + assert!(matches!( + ctx.meta.read_held_managed_creation(&wrong).await, + Err(MetaError::Conflict(_)) + )); + ctx.meta + .transition_session( + id, + SessionState::Pending, + BindingDisposition::RequireUnbound, + ) + .await + .unwrap(); + assert_eq!( + ctx.meta + .transition_session_created(id, engram_core::SandboxId::new()) + .await + .unwrap(), + 1 + ); + ctx.meta + .transition_session(id, SessionState::Idle, BindingDisposition::Detach) + .await + .unwrap(); + ctx.meta + .transition_session(id, SessionState::Queued, BindingDisposition::RequireUnbound) + .await + .unwrap(); + let session = ctx.meta.get_session(id).await.unwrap(); + assert_eq!(session.status, SessionState::Queued); + assert!(session.host_id.is_none() && session.sandbox_id.is_none()); + assert_eq!( + ctx.meta + .read_managed_creation(&frozen.identity) + .await + .unwrap(), + frozen + ); + assert!(matches!( + ctx.meta.read_held_managed_creation(&frozen.identity).await, + Err(MetaError::Conflict(_)) + )); + assert!(matches!( + ctx.meta.reserve_and_persist_create(ws, &[], 0).await, + Err(MetaError::Conflict(_)) + )); +} +conformance!( + t_managed_creation_held_retry_denies_requeued_epoch, + super::managed_creation_held_retry_denies_requeued_epoch +); + +fn managed_boot_request( + ws: &engram_core::traits::SessionCreateWriteSet, + host: HostId, +) -> engram_core::types::managed_boot::ManagedBootPrepare { + use engram_core::types::{managed_boot::*, managed_cold::*, managed_deny::*, sandbox::*}; + let creation = ws.managed_creation.as_ref().unwrap(); + let sandbox_id = engram_core::SandboxId::new(); + let disk = engram_core::types::manifest::ManifestRef::new(); + let drives: Vec<_> = (0..3) + .map(|i| AuxRoDrive { + drive_id: AuxRoDrive::slot_drive_id(i), + guest_mount: AuxRoDrive::slot_guest_mount(i), + fs_type: "squashfs".into(), + sha256: Some("a".repeat(64)), + }) + .collect(); + let spec = SandboxSpec { + image: ws.spec.image.clone(), + rootfs_source: Some("disk.ext4".into()), + image_uri: None, + rootfs_manifest: Some(disk), + cpu: CpuLimit { vcpus: 1 }, + memory: MemoryLimit { max_mib: 512 }, + disk: DiskLimit { max_gib: 1 }, + ttl: None, + env: Default::default(), + workdir: None, + network: Default::default(), + aux_ro_drives: drives.clone(), + swap_mib: None, + swap_source: None, + swap_manifest: None, + }; + ManagedBootPrepare { + capture_authority: None, + bootstrap_policy: ManagedDenyPolicy::new(1, ["tools.example".into()]).unwrap(), + key: ManagedBootKey { + creation: creation.identity.clone(), + context: ManagedPolicyContext { + target: ManagedPolicyTarget { + session_id: ws.session_id, + host_id: host, + sandbox_id, + binding_epoch: 1, + }, + operator_generation: 1, + operator_identity: engram_testkit::operator::candidate().identity().unwrap(), + binding_hash: None, + }, + boot_attempt: format!("boot-{}", ws.session_id), + native_digest: creation.native.digest.clone(), + }, + cold: ManagedColdPrepare { + version: 1, + session_id: ws.session_id, + sandbox_id, + boot_attempt: format!("boot-{}", ws.session_id), + image: spec.image.clone(), + disk: ManagedColdDisk { + manifest: disk, + sha256: "b".repeat(64), + size_bytes: 4096, + source: ManagedColdSource::FreshImage { + base_manifest: engram_core::types::manifest::ManifestRef::new(), + }, + }, + kernel_sha256: "c".repeat(64), + aux_ro_drives: drives, + }, + sandbox_spec_json: serde_json::to_string(&spec).unwrap(), + } +} +async fn managed_boot_claim_is_atomic_and_exact(ctx: &Ctx) { + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + use engram_core::types::managed_boot::*; + let ws = managed_creation_ws(SessionId::new()); + let host = HostId::new(); + ctx.meta + .upsert_host(host_record(host, "managed-host", ctx.clock.now_utc())) + .await + .unwrap(); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + assert_eq!( + ctx.meta.managed_boot_open(ws.session_id).await.unwrap(), + Some(false) + ); + let request = managed_boot_request(&ws, host); + managed_operator_gate_fixture(ctx, &request.key, false).await; + assert!(ctx.meta.claim_managed_boot(request.clone()).await.is_err()); + managed_operator_gate_fixture(ctx, &request.key, true).await; + for field in ["image", "cpu", "memory"] { + let mut invalid = request.clone(); + let mut spec = invalid.spec().unwrap(); + match field { + "image" => { + spec.image = "test.invalid/another:latest".into(); + invalid.cold.image = spec.image.clone(); + } + "cpu" => spec.cpu.vcpus = u32::try_from(ws.cpu_budget_vcpus).unwrap() + 1, + _ => spec.memory.max_mib = u32::try_from(ws.mem_budget_mib).unwrap() + 1, + } + invalid.sandbox_spec_json = serde_json::to_string(&spec).unwrap(); + assert!(ctx.meta.claim_managed_boot(invalid).await.is_err()); + assert!(ctx + .meta + .get_managed_boot(&request.key.creation) + .await + .unwrap() + .is_none()); + } + let (a, b) = tokio::join!( + ctx.meta.claim_managed_boot(request.clone()), + ctx.meta.claim_managed_boot(request.clone()) + ); + assert_eq!(a.unwrap().revision, 1); + assert_eq!(b.unwrap().revision, 1); + let row = ctx.meta.read_managed_boot(&request.key).await.unwrap(); + assert_eq!(row.phase, ManagedBootPhase::Claimed); + assert_eq!( + ctx.meta.managed_policy_target(ws.session_id).await.unwrap(), + request.key.context.target + ); + assert!(ctx + .meta + .read_held_managed_creation(&request.key.creation) + .await + .is_err()); + assert_eq!( + ctx.meta.list_managed_boots(None, 10).await.unwrap(), + vec![request.key.clone()] + ); + for n in 0..8 { + let mut wrong = request.clone(); + match n { + 0 => wrong.key.boot_attempt.push('x'), + 1 => wrong.key.context.target.binding_epoch += 1, + 2 => wrong.key.context.operator_generation += 1, + 3 => wrong.key.native_digest = engram_core::types::managed_create::digest(b"wrong"), + 4 => wrong.key.creation.task_id.push('x'), + 5 => wrong.key.context.target.host_id = HostId::new(), + 6 => wrong.key.context.target.sandbox_id = engram_core::SandboxId::new(), + _ => wrong.key.creation.policy.parent_session_id = Some(SessionId::new()), + } + assert!(ctx.meta.claim_managed_boot(wrong.clone()).await.is_err()); + assert!(ctx.meta.read_managed_boot(&wrong.key).await.is_err()); + } + assert_eq!( + ctx.meta + .read_managed_boot(&request.key) + .await + .unwrap() + .revision, + 1 + ); +} +conformance!( + t_managed_boot_claim_is_atomic_and_exact, + super::managed_boot_claim_is_atomic_and_exact +); + +// Fixture only: this exercises the real transactional predicate, not the +// authenticated publisher or public admission (which remain closed). +async fn managed_operator_gate_fixture( + ctx: &Ctx, + key: &engram_core::types::managed_boot::ManagedBootKey, + open: bool, +) { + if let Some(sim) = &ctx.sim { + sim.with_db_mut(|db| { + db.managed_operator_gate = Some(( + key.context.operator_generation, + key.context.operator_identity.clone(), + open, + )) + }); + } else { + sqlx::query("UPDATE managed_operator_generation_gate SET published_generation=$1,published_identity=$2,admission_open=$3 WHERE singleton") + .bind(key.context.operator_generation as i64).bind(&key.context.operator_identity).bind(open) + .execute(ctx._pg.as_ref().unwrap().store.pool()).await.unwrap(); + } +} + +async fn managed_boot_prepared_fixture( + ctx: &Ctx, +) -> ( + engram_core::traits::SessionCreateWriteSet, + engram_core::types::managed_boot::ManagedBootRecord, +) { + use engram_core::types::{managed_boot::*, managed_create::digest}; + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + let ws = managed_creation_ws(SessionId::new()); + let host = HostId::new(); + ctx.meta + .upsert_host(host_record(host, "managed-host", ctx.clock.now_utc())) + .await + .unwrap(); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + let request = managed_boot_request(&ws, host); + managed_operator_gate_fixture(ctx, &request.key, true).await; + let row = ctx.meta.claim_managed_boot(request.clone()).await.unwrap(); + let prepared = ManagedBootPreparedReceipt { + request, + host_incarnation: "host-incarnation".into(), + network_identity: digest(b"network"), + guest_ip: "192.0.2.2".parse().unwrap(), + receipt_id: digest(b"prepared"), + }; + let row = ctx + .meta + .advance_managed_boot( + &row.request.key, + row.revision, + ManagedBootAdvance::Prepared(Box::new(prepared)), + ) + .await + .unwrap(); + (ws, row) +} +async fn managed_boot_target_read_is_exact(ctx: &Ctx) { + use engram_core::types::managed_deny::ManagedPolicyTarget; + let (_ws, row) = managed_boot_prepared_fixture(ctx).await; + let target = row.request.key.context.target.clone(); + let read = ctx + .meta + .read_managed_boot_for_target(&target) + .await + .unwrap(); + assert_eq!( + serde_json::to_value(&read).unwrap(), + serde_json::to_value(&row).unwrap() + ); + for field in ["session", "host", "sandbox", "epoch"] { + let mut changed = target.clone(); + match field { + "session" => changed.session_id = SessionId::new(), + "host" => changed.host_id = HostId::new(), + "sandbox" => changed.sandbox_id = engram_core::SandboxId::new(), + "epoch" => changed.binding_epoch += 1, + _ => unreachable!(), + } + assert!( + ctx.meta + .read_managed_boot_for_target(&changed) + .await + .is_err(), + "accepted changed {field}" + ); + } + // A physical target from an ordinary session supplies no managed boot authority. + let ordinary = ctx + .meta + .create_session(spec("test.invalid/ordinary:latest")) + .await + .unwrap(); + ctx.meta + .assign_session_host(ordinary, Some(target.host_id)) + .await + .unwrap(); + let sandbox = engram_core::SandboxId::new(); + let epoch = ctx + .meta + .transition_session_created(ordinary, sandbox) + .await + .unwrap(); + assert!(ctx + .meta + .read_managed_boot_for_target(&ManagedPolicyTarget { + session_id: ordinary, + host_id: target.host_id, + sandbox_id: sandbox, + binding_epoch: epoch, + }) + .await + .is_err()); + ctx.meta + .advance_managed_boot( + &row.request.key, + row.revision, + engram_core::types::managed_boot::ManagedBootAdvance::Interrupt, + ) + .await + .unwrap(); + // Interruption remains readable as state, never permission to apply or start. + assert_eq!( + ctx.meta + .read_managed_boot_for_target(&target) + .await + .unwrap() + .phase, + engram_core::types::managed_boot::ManagedBootPhase::Interrupted + ); +} +conformance!( + t_managed_boot_target_read_is_exact, + super::managed_boot_target_read_is_exact +); + +async fn managed_boot_target_read_finalized_and_stale(ctx: &Ctx) { + let row = managed_boot_finalized_fixture(ctx).await; + let target = row.request.key.context.target.clone(); + assert_eq!( + ctx.meta + .read_managed_boot_for_target(&target) + .await + .unwrap() + .phase, + engram_core::types::managed_boot::ManagedBootPhase::Finalized + ); + // A later physical placement makes the retained old receipt unusable. + let sandbox = engram_core::SandboxId::new(); + if let Some(sim) = &ctx.sim { + sim.with_db_mut(|db| { + db.sessions + .get_mut(&target.session_id) + .unwrap() + .session + .sandbox_id = Some(sandbox); + }); + } else { + sqlx::query("UPDATE sessions SET sandbox_id=$2 WHERE id=$1") + .bind(target.session_id.as_uuid()) + .bind(sandbox.as_uuid()) + .execute(ctx._pg.as_ref().unwrap().store.pool()) + .await + .unwrap(); + } + assert!(ctx + .meta + .read_managed_boot_for_target(&target) + .await + .is_err()); +} +conformance!( + t_managed_boot_target_read_finalized_and_stale, + super::managed_boot_target_read_finalized_and_stale +); + +async fn managed_boot_finalized_fixture( + ctx: &Ctx, +) -> engram_core::types::managed_boot::ManagedBootRecord { + use engram_core::types::{managed_boot::*, managed_create::digest, managed_deny::*}; + let (ws, row) = managed_boot_prepared_fixture(ctx).await; + let prepared = row.prepared.clone().unwrap(); + let mut context = row.request.key.context.clone(); + context.binding_hash = Some(digest(b"binding")); + let policy = ManagedAppliedState::new( + context.clone(), + row.request.bootstrap_policy.clone(), + prepared.host_incarnation.clone(), + "proxy-incarnation".into(), + Some(ManagedNetworkProof { + target: context.target, + guest_ip: "192.0.2.2".parse().unwrap(), + network_identity: prepared.network_identity.clone(), + }), + ) + .unwrap(); + let bytes = ws.managed_creation.unwrap().native.bytes; + let f = ManagedBootFinalization { + prepared, + policy, + native_version: 1, + combined_manifest_digest: digest(&bytes), + combined_manifest: bytes, + agent_digest: digest(b"sealed fixture agent"), + }; + let sealed_agent = ManagedBootSealedAgent { + wrapped_dek: vec![1], + nonce: vec![2; 12], + ciphertext: vec![3], + key_id: "fixture".into(), + }; + let write_policy = ctx.meta.read_mcp_write_policy(ws.session_id).await.unwrap(); + for bad_policy in [ + ManagedDenyPolicy::new(2, []).unwrap(), + ManagedDenyPolicy::new(0, ["tools.example".into()]).unwrap(), + ] { + let mut wrong = f.clone(); + wrong.policy = ManagedAppliedState::new( + wrong.policy.context.clone(), + bad_policy, + wrong.policy.host_incarnation.clone(), + wrong.policy.proxy_incarnation.clone(), + wrong.policy.network_proof.clone(), + ) + .unwrap(); + assert!(ctx + .meta + .advance_managed_boot( + &row.request.key, + row.revision, + ManagedBootAdvance::Finalize { + finalization: Box::new(wrong), + sealed_agent: sealed_agent.clone(), + write_policy: write_policy.clone(), + } + ) + .await + .is_err()); + } + let command = ManagedBootAdvance::Finalize { + finalization: Box::new(f.clone()), + sealed_agent, + write_policy, + }; + managed_operator_gate_fixture(ctx, &row.request.key, false).await; + assert!(ctx + .meta + .advance_managed_boot(&row.request.key, row.revision, command.clone()) + .await + .is_err()); + managed_operator_gate_fixture(ctx, &row.request.key, true).await; + let finalized = ctx + .meta + .advance_managed_boot(&row.request.key, row.revision, command) + .await + .unwrap(); + let receipt = ManagedBootFinalReceipt { + key: row.request.key.clone(), + prepared_receipt_id: f.prepared.receipt_id.clone(), + finalization_digest: f.digest().unwrap(), + host_incarnation: f.prepared.host_incarnation, + network_identity: f.prepared.network_identity, + receipt_id: digest(b"final"), + }; + let mut wrong = receipt.clone(); + wrong.finalization_digest = digest(b"wrong"); + assert!(ctx + .meta + .advance_managed_boot( + &row.request.key, + finalized.revision, + ManagedBootAdvance::Finalized(wrong) + ) + .await + .is_err()); + ctx.meta + .advance_managed_boot( + &row.request.key, + finalized.revision, + ManagedBootAdvance::Finalized(receipt), + ) + .await + .unwrap() +} +async fn managed_boot_start_is_single_cas_and_generation_gated(ctx: &Ctx) { + use engram_core::types::{managed_boot::*, managed_create::digest}; + let row = managed_boot_finalized_fixture(ctx).await; + let key = &row.request.key; + let start = ManagedBootStart { + receipt: row.receipt.clone().unwrap(), + start_id: digest(b"start"), + }; + managed_operator_gate_fixture(ctx, key, false).await; + assert!(ctx + .meta + .advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::IssueStart(start.clone()) + ) + .await + .is_err()); + managed_operator_gate_fixture(ctx, key, true).await; + let (a, b) = tokio::join!( + ctx.meta.advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::IssueStart(start.clone()) + ), + ctx.meta.advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::IssueStart(start.clone()) + ) + ); + assert_eq!(usize::from(a.is_ok()) + usize::from(b.is_ok()), 1); + let issued = ctx.meta.read_managed_boot(key).await.unwrap(); + assert_eq!(issued.phase, ManagedBootPhase::StartIssued); + assert_eq!( + ctx.meta + .managed_boot_open(key.creation.session_id) + .await + .unwrap(), + Some(false) + ); + let mut wrong = start.clone(); + wrong.start_id = digest(b"other start"); + assert!(ctx + .meta + .advance_managed_boot(key, issued.revision, ManagedBootAdvance::Started(wrong)) + .await + .is_err()); + managed_operator_gate_fixture(ctx, key, false).await; + assert!(ctx + .meta + .advance_managed_boot( + key, + issued.revision, + ManagedBootAdvance::Started(start.clone()) + ) + .await + .is_err()); + managed_operator_gate_fixture(ctx, key, true).await; + let started = ctx + .meta + .advance_managed_boot(key, issued.revision, ManagedBootAdvance::Started(start)) + .await + .unwrap(); + assert_eq!(started.phase, ManagedBootPhase::Started); + assert_eq!( + ctx.meta + .managed_boot_open(key.creation.session_id) + .await + .unwrap(), + Some(true) + ); + assert_eq!( + ctx.meta + .managed_sandbox_open(key.context.target.sandbox_id) + .await + .unwrap(), + Some(true) + ); + managed_operator_gate_fixture(ctx, key, false).await; + assert_eq!( + ctx.meta + .managed_boot_open(key.creation.session_id) + .await + .unwrap(), + Some(false) + ); + assert_eq!( + ctx.meta + .managed_sandbox_open(key.context.target.sandbox_id) + .await + .unwrap(), + Some(false) + ); + assert_eq!( + ctx.meta.list_managed_boots(None, 128).await.unwrap(), + vec![key.clone()] + ); + assert!(ctx + .meta + .list_managed_boots(Some(key.creation.session_id), 128) + .await + .unwrap() + .is_empty()); +} +conformance!( + t_managed_boot_start_is_single_cas_and_generation_gated, + super::managed_boot_start_is_single_cas_and_generation_gated +); + +async fn managed_boot_interrupt_capture_and_recovery(ctx: &Ctx) { + use engram_core::types::{managed_boot::*, managed_cold::ManagedColdSource}; + let (_, row) = managed_boot_prepared_fixture(ctx).await; + let key = &row.request.key; + let sid = key.creation.session_id; + let request = ManagedCaptureRequest { + key: key.clone(), + capture_id: SnapshotId::new(), + }; + assert!(ctx.meta.reserve_managed_capture(&request).await.is_err()); + ctx.meta + .append_session_event(sid, "run_started", serde_json::json!({"run_id":"run-old"})) + .await + .unwrap(); + ctx.meta + .append_session_event( + sid, + "tool_call_requested", + serde_json::json!({"tool_call_id":"call-old","run_id":"run-old"}), + ) + .await + .unwrap(); + assert!(!ctx + .meta + .managed_tool_target_gone(sid, "call-old") + .await + .unwrap()); + let now = ctx.clock.now_utc(); + let result = engram_core::types::outbox::OutboxRow { + prompt_id: format!("tool_result:{sid}:call-old"), + session_id: sid, + kind: engram_core::types::outbox::OutboxKind::ToolResult, + payload: serde_json::json!({"tool_call_id":"call-old","result_json":"{}"}), + created_at: now, + not_before: now, + attempts: 0, + delivered_at: None, + acked_at: None, + }; + ctx.meta + .append_session_event_and_outbox( + sid, + "tool_result_submitted", + serde_json::json!({"tool_call_id":"call-old"}), + &result, + ) + .await + .unwrap(); + let mut pending = result.clone(); + pending.prompt_id = format!("pending:{sid}"); + pending.kind = engram_core::types::outbox::OutboxKind::Prompt; + pending.payload = serde_json::json!({"text":"next"}); + ctx.meta.outbox_enqueue(&pending).await.unwrap(); + ctx.meta + .advance_managed_boot(key, row.revision, ManagedBootAdvance::Interrupt) + .await + .unwrap(); + assert!(ctx + .meta + .outbox_get(&result.prompt_id) + .await + .unwrap() + .unwrap() + .acked_at + .is_some()); + assert!(ctx + .meta + .outbox_get(&pending.prompt_id) + .await + .unwrap() + .unwrap() + .acked_at + .is_none()); + assert!(ctx + .meta + .append_session_event_and_outbox( + sid, + "tool_result_submitted", + serde_json::json!({"tool_call_id":"call-old"}), + &result + ) + .await + .is_err()); + let events = ctx + .meta + .list_session_events_since(sid, -1, 20) + .await + .unwrap(); + assert!(events + .iter() + .any(|e| e.kind == "run_interrupted" && e.payload["run_id"] == "run-old")); + assert_eq!( + events + .iter() + .find(|e| e.kind == "run_interrupted") + .unwrap() + .payload["type"], + "harness_run_interrupted" + ); + assert!(ctx.meta.managed_delivery_target_gone(sid, 1).await.unwrap()); + assert!(ctx + .meta + .managed_tool_target_gone(sid, "call-old") + .await + .unwrap()); + assert!(!ctx.meta.managed_delivery_target_gone(sid, 2).await.unwrap()); + assert!(ctx + .meta + .reserve_managed_capture(&request) + .await + .unwrap() + .is_none()); + assert!(ctx + .meta + .list_managed_disk_manifests() + .await + .unwrap() + .contains(&request.disk_manifest())); + let mut competing = request.clone(); + competing.capture_id = SnapshotId::new(); + assert!(ctx.meta.reserve_managed_capture(&competing).await.is_err()); + let cold = &row.request.cold; + let mut authority = ManagedCaptureAuthority { + version: 1, + request: request.clone(), + host_incarnation: row.prepared.unwrap().host_incarnation, + disk_manifest: request.disk_manifest(), + disk_sha256: "d".repeat(64), + disk_size_bytes: 4096, + image: cold.image.clone(), + kernel_sha256: cold.kernel_sha256.clone(), + aux_ro_drives: cold.aux_ro_drives.clone(), + consistency: ManagedDiskConsistency::CrashConsistentDiskCutV1, + source_state: ManagedCapturedSourceState::Destroyed, + record_digest: String::new(), + }; + authority.record_digest = authority.digest().unwrap(); + let mut wrong = authority.clone(); + wrong.host_incarnation = "unknown".into(); + wrong.record_digest = wrong.digest().unwrap(); + assert!(ctx.meta.record_managed_capture(&wrong).await.is_err()); + ctx.meta.record_managed_capture(&authority).await.unwrap(); + ctx.meta.record_managed_capture(&authority).await.unwrap(); + assert_eq!( + ctx.meta.reserve_managed_capture(&request).await.unwrap(), + Some(authority.clone()) + ); + let mut next = row.request.clone(); + next.key.context.target.binding_epoch += 1; + next.key.context.target.sandbox_id = engram_core::SandboxId::new(); + next.key.boot_attempt.push_str("-next"); + next.cold.boot_attempt = next.key.boot_attempt.clone(); + next.cold.sandbox_id = next.key.context.target.sandbox_id; + next.cold.disk.manifest = authority.disk_manifest; + next.cold.disk.sha256 = authority.disk_sha256.clone(); + next.cold.disk.source = ManagedColdSource::Capture { + capture_id: request.capture_id, + session_id: sid, + sandbox_id: key.context.target.sandbox_id, + binding_epoch: 1, + record_digest: authority.record_digest.clone(), + }; + let mut spec = next.spec().err(); + assert!( + spec.take().is_some(), + "capture-shaped input without authority is rejected" + ); + let mut sandbox: engram_core::types::sandbox::SandboxSpec = + serde_json::from_str(&next.sandbox_spec_json).unwrap(); + sandbox.rootfs_manifest = Some(authority.disk_manifest); + next.sandbox_spec_json = serde_json::to_string(&sandbox).unwrap(); + next.capture_authority = Some(authority.clone()); + next.spec().unwrap(); + let recovered = ctx.meta.claim_managed_boot(next.clone()).await.unwrap(); + assert_eq!(recovered.phase, ManagedBootPhase::Claimed); + assert_eq!(recovered.request.key.context.target.binding_epoch, 2); + assert_eq!( + ctx.meta.reserve_managed_capture(&request).await.unwrap(), + Some(authority) + ); + ctx.meta + .append_session_event( + sid, + "tool_call_requested", + serde_json::json!({"tool_call_id":"call-old","run_id":"run-new"}), + ) + .await + .unwrap(); + assert!(ctx + .meta + .managed_tool_target_gone(sid, "call-old") + .await + .unwrap()); + assert!(ctx.meta.read_managed_boot(key).await.is_err()); + assert_eq!( + ctx.meta + .managed_sandbox_open(key.context.target.sandbox_id) + .await + .unwrap(), + Some(false) + ); + assert!(ctx + .meta + .managed_tool_target_gone(sid, "call-old") + .await + .unwrap()); +} +conformance!( + t_managed_boot_interrupt_capture_and_recovery, + super::managed_boot_interrupt_capture_and_recovery +); + +async fn managed_boot_initial_prompt_is_atomic(ctx: &Ctx) { + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + use engram_core::{ + traits::metadata::ManagedInitialPrompt, + types::outbox::{OutboxKind, OutboxRow}, + }; + let mut ws = managed_creation_ws(SessionId::new()); + let now = ctx.clock.now_utc(); + let prompt_id = format!("create:{}", ws.session_id); + ws.managed_initial_prompt = Some(ManagedInitialPrompt { + row: OutboxRow { + session_id: ws.session_id, + prompt_id: prompt_id.clone(), + kind: OutboxKind::Prompt, + payload: serde_json::json!({"text":"first prompt","mode":"plan"}), + created_at: now, + not_before: now, + attempts: 0, + delivered_at: None, + acked_at: None, + }, + events: vec![( + "prompt_received".into(), + serde_json::json!({"prompt_id":prompt_id}), + )], + }); + let mut bad = ws.clone(); + bad.managed_initial_prompt.as_mut().unwrap().row.session_id = SessionId::new(); + assert!(ctx + .meta + .reserve_and_persist_create(bad, &[], 0) + .await + .is_err()); + assert!(ctx.meta.get_session(ws.session_id).await.is_err()); + assert!(ctx.meta.outbox_get(&prompt_id).await.unwrap().is_none()); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + let row = ctx.meta.outbox_get(&prompt_id).await.unwrap().unwrap(); + assert_eq!(row.payload["text"], "first prompt"); + assert!(row.acked_at.is_none() && row.delivered_at.is_none()); + assert_eq!( + ctx.meta + .list_session_events_since(ws.session_id, -1, 10) + .await + .unwrap() + .len(), + 1 + ); + let mut changed = ws.clone(); + changed.managed_initial_prompt.as_mut().unwrap().row.payload["text"] = + serde_json::json!("changed"); + assert!(ctx + .meta + .reserve_and_persist_create(changed, &[], 0) + .await + .is_err()); + let mut absent = ws.clone(); + absent.managed_initial_prompt = None; + assert!(ctx + .meta + .reserve_and_persist_create(absent, &[], 0) + .await + .is_err()); + assert_eq!( + ctx.meta.managed_boot_open(ws.session_id).await.unwrap(), + Some(false) + ); +} +conformance!( + t_managed_boot_initial_prompt_is_atomic, + super::managed_boot_initial_prompt_is_atomic +); + +async fn managed_boot_retirement_preserves_source_and_next_deny(ctx: &Ctx) { + use engram_core::types::{ + managed_boot::*, managed_create::digest, managed_deny::ManagedDenyPolicy, + }; + for newer_desired in [false, true] { + let (_, row) = managed_boot_prepared_fixture(ctx).await; + let key = &row.request.key; + let receipt = ManagedBootRetiredReceipt { + prepared: row.prepared.clone().unwrap(), + retained_policy: ManagedDenyPolicy::new( + 2, + ["tools.example".into(), "next.example".into()], + ) + .unwrap(), + receipt_id: digest(b"retired"), + }; + let mut wrong = receipt.clone(); + wrong.prepared.host_incarnation = "different".into(); + assert!(ctx + .meta + .advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::Retired(Box::new(wrong)) + ) + .await + .is_err()); + let expected_policy = if newer_desired { + let newer = ManagedDenyPolicy::new( + 3, + [ + "tools.example".into(), + "next.example".into(), + "later.example".into(), + ], + ) + .unwrap(); + ctx.meta + .bind_session_integration_policy( + key.creation.session_id, + &serde_json::json!({"managed_deny":newer}).to_string(), + ) + .await + .unwrap(); + newer + } else { + receipt.retained_policy.clone() + }; + let closed = ctx + .meta + .advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::Retired(Box::new(receipt.clone())), + ) + .await + .unwrap(); + assert_eq!(closed.phase, ManagedBootPhase::Interrupted); + assert_eq!(closed.retired, Some(receipt.clone())); + let retained: serde_json::Value = serde_json::from_str( + &ctx.meta + .get_session_integration_policy(key.creation.session_id) + .await + .unwrap() + .unwrap(), + ) + .unwrap(); + assert_eq!( + retained["managed_deny"], + serde_json::to_value(expected_policy).unwrap() + ); + assert!(ctx + .meta + .reserve_managed_capture(&ManagedCaptureRequest { + key: key.clone(), + capture_id: SnapshotId::new() + }) + .await + .unwrap() + .is_none()); + assert_eq!( + ctx.meta + .managed_boot_open(key.creation.session_id) + .await + .unwrap(), + Some(false) + ); + } +} +conformance!( + t_managed_boot_retirement_preserves_source_and_next_deny, + super::managed_boot_retirement_preserves_source_and_next_deny +); + +async fn managed_boot_ingestion_is_serialized_with_interruption(ctx: &Ctx) { + use engram_core::types::{event::ManagedHarnessEvent, managed_boot::*, managed_create::digest}; + let row = managed_boot_finalized_fixture(ctx).await; + let key = row.request.key.clone(); + let start = ManagedBootStart { + receipt: row.receipt.unwrap(), + start_id: digest(b"event-start"), + }; + let row = ctx + .meta + .advance_managed_boot( + &key, + row.revision, + ManagedBootAdvance::IssueStart(start.clone()), + ) + .await + .unwrap(); + let row = ctx + .meta + .advance_managed_boot(&key, row.revision, ManagedBootAdvance::Started(start)) + .await + .unwrap(); + let sid = key.creation.session_id; + let now = ctx.clock.now_utc(); + let pending = engram_core::types::outbox::OutboxRow { + prompt_id: format!("paused:{sid}"), + session_id: sid, + kind: engram_core::types::outbox::OutboxKind::Prompt, + payload: serde_json::json!({"text":"pending"}), + created_at: now, + not_before: now, + attempts: 0, + delivered_at: None, + acked_at: None, + }; + ctx.meta.outbox_enqueue(&pending).await.unwrap(); + let original = ManagedHarnessEvent { + target: key.context.target.clone(), + incarnation: 1, + seq: 1, + kind: "run_started".into(), + payload: serde_json::json!({"type":"harness_run_started","run_id":"old-run","prompt_id":null,"at":now}), + ack_id: None, + }; + let first = ctx + .meta + .append_managed_harness_event(&original) + .await + .unwrap(); + assert!(first.idx.is_some()); + assert!(!first.outbox_acked); + assert!(ctx + .meta + .append_managed_harness_event(&original) + .await + .unwrap() + .idx + .is_none()); + let mut mismatch = original.clone(); + mismatch.target.sandbox_id = engram_core::SandboxId::new(); + assert!(ctx + .meta + .append_managed_harness_event(&mismatch) + .await + .is_err()); + let mut changed = original.clone(); + changed.payload["prompt_id"] = pending.prompt_id.clone().into(); + changed.ack_id = Some(pending.prompt_id.clone()); + assert!(ctx + .meta + .append_managed_harness_event(&changed) + .await + .is_err()); + + let mut consumed = pending.clone(); + consumed.prompt_id = format!("consumed:{sid}"); + ctx.meta.outbox_enqueue(&consumed).await.unwrap(); + let acknowledged = ManagedHarnessEvent { + target: key.context.target.clone(), + incarnation: 1, + seq: 4, + kind: "prompt_steered".into(), + payload: serde_json::json!({"type":"harness_prompt_steered","prompt_id":consumed.prompt_id,"at":now}), + ack_id: Some(consumed.prompt_id.clone()), + }; + let accepted = ctx + .meta + .append_managed_harness_event(&acknowledged) + .await + .unwrap(); + assert!(accepted.idx.is_some() && accepted.outbox_acked); + assert!(ctx + .meta + .outbox_get(&consumed.prompt_id) + .await + .unwrap() + .unwrap() + .acked_at + .is_some()); + let mut replay = acknowledged.clone(); + replay.payload["at"] = serde_json::to_value(now + chrono::Duration::seconds(1)).unwrap(); + let accepted = ctx + .meta + .append_managed_harness_event(&replay) + .await + .unwrap(); + assert!(accepted.idx.is_none() && !accepted.outbox_acked); + + // Both ingestions have observed Started before interruption. The barriers + // pause them at the transaction boundary without sleeps or scheduler guesses. + let paused = Arc::new(tokio::sync::Barrier::new(3)); + let resume = Arc::new(tokio::sync::Barrier::new(3)); + let mut tasks = Vec::new(); + for (seq, kind, payload, ack_id) in [ + ( + 2, + "tool_call_requested", + serde_json::json!({"type":"harness_tool_call_requested","tool_call_id":"late-call","run_id":"old-run","at":now}), + None, + ), + ( + 3, + "run_started", + serde_json::json!({"type":"harness_run_started","run_id":"late-run","prompt_id":pending.prompt_id,"at":now}), + Some(pending.prompt_id.clone()), + ), + ] { + let meta = ctx.meta.clone(); + let paused = paused.clone(); + let resume = resume.clone(); + let target = key.context.target.clone(); + tasks.push(tokio::spawn(async move { + assert_eq!(meta.managed_boot_open(sid).await.unwrap(), Some(true)); + assert_eq!(meta.managed_policy_target(sid).await.unwrap(), target); + paused.wait().await; + resume.wait().await; + meta.append_managed_harness_event(&ManagedHarnessEvent { + target, + incarnation: 1, + seq, + kind: kind.into(), + payload, + ack_id, + }) + .await + })); + } + paused.wait().await; + ctx.meta + .advance_managed_boot(&key, row.revision, ManagedBootAdvance::Interrupt) + .await + .unwrap(); + resume.wait().await; + for task in tasks { + assert!(task.await.unwrap().is_err()); + } + assert!( + ctx.meta + .append_managed_harness_event(&original) + .await + .is_err(), + "stale replay must not acknowledge after interruption" + ); + let completed = ManagedHarnessEvent { + target: key.context.target.clone(), + incarnation: 1, + seq: 5, + kind: "tool_call_completed".into(), + payload: serde_json::json!({"type":"harness_tool_call_completed","tool_call_id":"late-call","at":now}), + ack_id: Some(engram_core::types::outbox::tool_result_outbox_id( + sid, + "late-call", + )), + }; + assert!(ctx + .meta + .append_managed_harness_event(&completed) + .await + .is_err()); + let events = ctx + .meta + .list_session_events_since(sid, -1, 100) + .await + .unwrap(); + assert!(!events + .iter() + .any(|e| e.payload["tool_call_id"] == "late-call" || e.payload["run_id"] == "late-run")); + assert!(ctx + .meta + .outbox_get(&pending.prompt_id) + .await + .unwrap() + .unwrap() + .acked_at + .is_none()); + assert!(ctx + .meta + .managed_tool_target_gone(sid, "late-call") + .await + .unwrap()); + let interrupted = events.iter().find(|e| e.kind == "run_interrupted").unwrap(); + assert_eq!(interrupted.payload["type"], "harness_run_interrupted"); +} +conformance!( + t_managed_boot_ingestion_is_serialized_with_interruption, + super::managed_boot_ingestion_is_serialized_with_interruption +); + +fn operator_candidate() -> engram_core::types::managed_operator::OperatorCandidate { + use engram_core::types::managed_operator::*; + let d = engram_core::types::managed_create::digest(b"operator fixture"); + OperatorCandidate { version:1, deployment_identity:d.clone(),network_identity:d.clone(),trust_identity:d.clone(),ca_content_digests:[("fixture-root".into(),d.clone())].into(),zones:[("unused-zone".into(),OperatorZone { revision:"1".into(),worker_digest:d,authorities:vec!["api.example".into(),"unused.example:8443".into()],aliases:vec!["alias.example".into()],configuration:serde_json::json!({"id":"unused-zone","revision":"1","authorities":["api.example","unused.example:8443"],"aliases":["alias.example"],"mode":"public","cidrs":[],"denyCidrs":["127.0.0.0/8"]}) })].into() } +} +async fn operator_stage_retry_publish(ctx: &Ctx) { + let candidate = operator_candidate(); + assert!( + !ctx.meta + .read_operator_publication() + .await + .unwrap() + .admission_open + ); + let staged = ctx + .meta + .stage_operator_publication(None, candidate.clone()) + .await + .unwrap(); + assert!(!staged.admission_open); + assert_eq!( + staged, + ctx.meta + .stage_operator_publication(None, candidate.clone()) + .await + .unwrap() + ); + let mut wrong = candidate.clone(); + wrong.network_identity = engram_core::types::managed_create::digest(b"different network"); + assert!(ctx + .meta + .stage_operator_publication(None, wrong) + .await + .is_err()); + assert!(ctx + .meta + .stage_operator_publication(Some(1), candidate.clone()) + .await + .is_err()); + assert_eq!(staged, ctx.meta.read_operator_publication().await.unwrap()); + let g = staged.pending.unwrap(); + assert_eq!( + g.candidate.hosts().unwrap(), + ["alias.example", "api.example", "unused.example"] + ); + let published = ctx.meta.publish_operator_generation(&g).await.unwrap(); + assert!(published.admission_open); + assert_eq!( + published, + ctx.meta + .stage_operator_publication(None, candidate) + .await + .unwrap() + ); + assert_eq!( + published, + ctx.meta.publish_operator_generation(&g).await.unwrap() + ); +} +conformance!( + t_operator_stage_retry_publish, + super::operator_stage_retry_publish +); + +async fn operator_all_sessions_retired_and_dormant(ctx: &Ctx) { + use engram_core::types::{managed_boot::*, managed_create::digest, managed_operator::*}; + let (owned, row) = managed_boot_prepared_fixture(ctx).await; + let held = managed_creation_ws(SessionId::new()); + ctx.meta + .reserve_and_persist_create(held.clone(), &[], 0) + .await + .unwrap(); + let next = ctx + .meta + .stage_operator_publication(Some(1), operator_candidate()) + .await + .unwrap(); + let g = next.pending.unwrap(); + assert!( + !ctx.meta + .read_operator_publication() + .await + .unwrap() + .admission_open + ); + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); + assert!(ctx + .meta + .reserve_and_persist_create(managed_creation_ws(SessionId::new()), &[], 0) + .await + .is_err()); + // Lost reservation replies still reconcile while fresh admission is closed. + ctx.meta + .reserve_and_persist_create(held.clone(), &[], 0) + .await + .unwrap(); + let mut ids = vec![held.session_id, owned.session_id]; + ids.sort(); + assert_eq!( + ctx.meta.list_operator_participants(None, 1).await.unwrap(), + ids[..1] + ); + assert_eq!( + ctx.meta + .list_operator_participants(Some(ids[0]), 1) + .await + .unwrap(), + ids[1..] + ); + assert_eq!( + ctx.meta.list_operator_participants(None, 64).await.unwrap(), + ids + ); + let dormant = ctx + .meta + .retain_operator_participant(&g, held.session_id) + .await + .unwrap(); + assert!(dormant.dormant()); + assert!(dormant.policy.denies("unused.example")); + assert!(dormant.policy.denies("tools.example")); + ctx.meta + .record_operator_readiness(&g, &dormant, OperatorReadiness::NeverAllocated) + .await + .unwrap(); + let p = ctx + .meta + .retain_operator_participant(&g, owned.session_id) + .await + .unwrap(); + assert!(ctx + .meta + .record_operator_readiness(&g, &p, OperatorReadiness::NeverAllocated) + .await + .is_err()); + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); + // A permanent retirement can predate the candidate expansion. It proves + // nonrunning only; the next-boot union is the newer durable policy above. + let receipt = ManagedBootRetiredReceipt { + prepared: row.prepared.unwrap(), + retained_policy: row.request.bootstrap_policy.clone(), + receipt_id: digest(b"retired source"), + }; + // Even before the prepared reply is stored, the full registered source is immutable. + let mut claimed = p.clone(); + claimed.boot.as_mut().unwrap().2 = None; + let mut different_source = receipt.clone(); + different_source + .prepared + .request + .sandbox_spec_json + .push(' '); + different_source.prepared.validate().unwrap(); + assert!(OperatorReadiness::Retired(Box::new(different_source)) + .validate(&g, &claimed) + .is_err()); + let mut wrong = receipt.clone(); + wrong.prepared.host_incarnation = "other-incarnation".into(); + assert!(ctx + .meta + .record_operator_readiness(&g, &p, OperatorReadiness::Retired(Box::new(wrong))) + .await + .is_err()); + ctx.meta + .record_operator_readiness( + &g, + &p, + OperatorReadiness::Retired(Box::new(receipt.clone())), + ) + .await + .unwrap(); + // A boot transition after the observation invalidates the exact snapshot. + ctx.meta + .advance_managed_boot( + &row.request.key, + row.revision, + ManagedBootAdvance::Retired(Box::new(receipt.clone())), + ) + .await + .unwrap(); + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); + let fresh = ctx + .meta + .retain_operator_participant(&g, owned.session_id) + .await + .unwrap(); + ctx.meta + .record_operator_readiness(&g, &fresh, OperatorReadiness::Retired(Box::new(receipt))) + .await + .unwrap(); + let published = ctx.meta.publish_operator_generation(&g).await.unwrap(); + assert!(published.admission_open); + // Publication does not authorize the retired old-generation guest to restart. + assert_eq!( + ctx.meta.managed_boot_open(owned.session_id).await.unwrap(), + Some(false) + ); + assert!(ctx.meta.claim_managed_boot(row.request).await.is_err()); + let retained: serde_json::Value = serde_json::from_str( + &ctx.meta + .get_session_integration_policy(owned.session_id) + .await + .unwrap() + .unwrap(), + ) + .unwrap(); + assert!(retained["managed_deny"]["hosts"] + .as_array() + .unwrap() + .contains(&serde_json::json!("tools.example"))); +} +conformance!( + t_operator_all_sessions_retired_and_dormant, + super::operator_all_sessions_retired_and_dormant +); + +async fn operator_stage_races_registered_claim(ctx: &Ctx) { + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + let ws = managed_creation_ws(SessionId::new()); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + let host = HostId::new(); + ctx.meta + .upsert_host(host_record(host, "claim-race", ctx.clock.now_utc())) + .await + .unwrap(); + let request = managed_boot_request(&ws, host); + let (staged, claim) = tokio::join!( + ctx.meta + .stage_operator_publication(Some(1), operator_candidate()), + ctx.meta.claim_managed_boot(request.clone()) + ); + let g = staged.unwrap().pending.unwrap(); + assert_eq!( + ctx.meta.list_operator_participants(None, 64).await.unwrap(), + vec![ws.session_id] + ); + let p = ctx + .meta + .retain_operator_participant(&g, ws.session_id) + .await + .unwrap(); + assert_eq!(p.boot.is_some(), claim.is_ok()); + assert_eq!(p.dormant(), claim.is_err()); + assert!(ctx.meta.claim_managed_boot(request).await.is_err()); + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); +} +conformance!( + t_operator_stage_races_registered_claim, + super::operator_stage_races_registered_claim +); + +async fn operator_stage_races_start_and_terminal_owner(ctx: &Ctx) { + use engram_core::types::{managed_boot::*, managed_create::digest, managed_operator::*}; + let row = managed_boot_finalized_fixture(ctx).await; + let key = &row.request.key; + let intent = ManagedBootStart { + receipt: row.receipt.unwrap(), + start_id: digest(b"operator race start"), + }; + let (staged, issued) = tokio::join!( + ctx.meta + .stage_operator_publication(Some(1), operator_candidate()), + ctx.meta.advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::IssueStart(intent.clone()) + ) + ); + let g = staged.unwrap().pending.unwrap(); + if let Ok(issued) = issued { + assert!(ctx + .meta + .advance_managed_boot(key, issued.revision, ManagedBootAdvance::Started(intent)) + .await + .is_err()); + } + assert_eq!( + ctx.meta + .managed_boot_open(key.creation.session_id) + .await + .unwrap(), + Some(false) + ); + // A terminal SQL label cannot remove the physical owner from the scan. + ctx.meta + .transition_session( + key.creation.session_id, + SessionState::Failed, + BindingDisposition::Detach, + ) + .await + .unwrap(); + assert_eq!( + ctx.meta.list_operator_participants(None, 64).await.unwrap(), + vec![key.creation.session_id] + ); + let p = ctx + .meta + .retain_operator_participant(&g, key.creation.session_id) + .await + .unwrap(); + assert!(!p.dormant()); + assert!(ctx + .meta + .record_operator_readiness(&g, &p, OperatorReadiness::NeverAllocated) + .await + .is_err()); + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); +} +conformance!( + t_operator_stage_races_start_and_terminal_owner, + super::operator_stage_races_start_and_terminal_owner +); + +async fn operator_stage_races_started(ctx: &Ctx) { + use engram_core::types::{managed_boot::*, managed_create::digest}; + let row = managed_boot_finalized_fixture(ctx).await; + let key = &row.request.key; + let start = ManagedBootStart { + receipt: row.receipt.unwrap(), + start_id: digest(b"delayed started"), + }; + let issued = ctx + .meta + .advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::IssueStart(start.clone()), + ) + .await + .unwrap(); + let (stage, started) = tokio::join!( + ctx.meta + .stage_operator_publication(Some(1), operator_candidate()), + ctx.meta.advance_managed_boot( + key, + issued.revision, + ManagedBootAdvance::Started(start.clone()) + ) + ); + let g = stage.unwrap().pending.unwrap(); + let current = ctx.meta.read_managed_boot(key).await.unwrap(); + assert_eq!( + current.phase, + if started.is_ok() { + ManagedBootPhase::Started + } else { + ManagedBootPhase::StartIssued + } + ); + assert_eq!( + ctx.meta.list_operator_participants(None, 64).await.unwrap(), + vec![key.creation.session_id] + ); + assert_eq!( + ctx.meta + .managed_boot_open(key.creation.session_id) + .await + .unwrap(), + Some(false) + ); + assert!(ctx + .meta + .advance_managed_boot(key, current.revision, ManagedBootAdvance::Started(start)) + .await + .is_err()); + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); +} +conformance!( + t_operator_stage_races_started, + super::operator_stage_races_started +); + +async fn operator_readiness_invalidates_policy_and_recovers_restart(ctx: &Ctx) { + use engram_core::types::{ + managed_boot::*, managed_create::digest, managed_deny::*, managed_operator::*, + }; + let (_, row) = managed_boot_prepared_fixture(ctx).await; + let mut candidate = engram_testkit::operator::candidate(); + candidate.network_identity = digest(b"same hosts new route"); + let g = ctx + .meta + .stage_operator_publication(Some(1), candidate.clone()) + .await + .unwrap() + .pending + .unwrap(); + let sid = row.request.key.creation.session_id; + let p = ctx.meta.retain_operator_participant(&g, sid).await.unwrap(); + assert_eq!(p.policy.revision, 2); + assert_eq!( + ctx.meta + .retain_operator_participant(&g, sid) + .await + .unwrap() + .policy, + p.policy, + "restart/retry keeps one durable required revision" + ); + let retired = OperatorReadiness::Retired(Box::new(ManagedBootRetiredReceipt { + prepared: row.prepared.unwrap(), + retained_policy: row.request.bootstrap_policy, + receipt_id: digest(b"permanent retirement"), + })); + ctx.meta + .record_operator_readiness(&g, &p, retired.clone()) + .await + .unwrap(); + let expanded = + ManagedDenyPolicy::new(3, ["tools.example".into(), "orphan.example".into()]).unwrap(); + let raw = serde_json::json!({"managed_deny":expanded,"observes":[]}).to_string(); + tokio::time::timeout( + Duration::from_secs(2), + ctx.meta.bind_session_integration_policy(sid, &raw), + ) + .await + .unwrap() + .unwrap(); + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); + let fresh = ctx.meta.retain_operator_participant(&g, sid).await.unwrap(); + assert_eq!(fresh.policy, expanded); + assert!(ctx + .meta + .record_operator_readiness(&g, &p, retired.clone()) + .await + .is_err()); + ctx.meta + .record_operator_readiness(&g, &fresh, retired) + .await + .unwrap(); + if let Some(db) = &ctx._pg { + let reopened = engram_postgres::PostgresStore::connect(&db.url) + .await + .unwrap(); + assert_eq!( + reopened.read_operator_publication().await.unwrap().pending, + Some(g.clone()) + ); + assert!( + reopened + .publish_operator_generation(&g) + .await + .unwrap() + .admission_open + ); + } else { + assert!( + ctx.meta + .publish_operator_generation(&g) + .await + .unwrap() + .admission_open + ); + } + assert!( + ctx.meta + .stage_operator_publication(Some(1), candidate) + .await + .unwrap() + .admission_open, + "exact retry after restart returns original publication" + ); +} +conformance!( + t_operator_readiness_invalidates_policy_and_recovers_restart, + super::operator_readiness_invalidates_policy_and_recovers_restart +); + +async fn operator_new_legacy_registration_requires_complete_published_set(ctx: &Ctx) { + use engram_core::types::managed_deny::ManagedDenyPolicy; + let candidate = operator_candidate(); + let staged = ctx + .meta + .stage_operator_publication(None, candidate.clone()) + .await + .unwrap(); + ctx.meta + .publish_operator_generation(&staged.pending.unwrap()) + .await + .unwrap(); + let sid = ctx + .meta + .create_session(spec("test.invalid/operator:latest")) + .await + .unwrap(); + let incomplete = ManagedDenyPolicy::new(1, ["api.example".into()]).unwrap(); + assert!(ctx + .meta + .bind_session_integration_policy( + sid, + &serde_json::json!({"managed_deny":incomplete}).to_string() + ) + .await + .is_err()); + assert!(ctx + .meta + .list_operator_participants(None, 64) + .await + .unwrap() + .is_empty()); + let complete = ManagedDenyPolicy::new(1, candidate.hosts().unwrap()).unwrap(); + let raw = serde_json::json!({"managed_deny":complete}).to_string(); + ctx.meta + .bind_session_integration_policy(sid, &raw) + .await + .unwrap(); + let second = ctx + .meta + .create_session(spec("test.invalid/operator:latest")) + .await + .unwrap(); + ctx.meta + .stage_operator_publication(Some(1), candidate) + .await + .unwrap(); + assert!(ctx + .meta + .bind_session_integration_policy(second, &raw) + .await + .is_err()); + assert_eq!( + ctx.meta.list_operator_participants(None, 64).await.unwrap(), + vec![sid] + ); +} +conformance!( + t_operator_new_legacy_registration_requires_complete_published_set, + super::operator_new_legacy_registration_requires_complete_published_set +); + +async fn operator_placement_change_invalidates_readiness(ctx: &Ctx) { + use engram_core::types::{managed_boot::*, managed_create::digest, managed_operator::*}; + let (_, row) = managed_boot_prepared_fixture(ctx).await; + let sid = row.request.key.creation.session_id; + let g = ctx + .meta + .stage_operator_publication(Some(1), operator_candidate()) + .await + .unwrap() + .pending + .unwrap(); + let p = ctx.meta.retain_operator_participant(&g, sid).await.unwrap(); + let proof = OperatorReadiness::Retired(Box::new(ManagedBootRetiredReceipt { + prepared: row.prepared.unwrap(), + retained_policy: row.request.bootstrap_policy, + receipt_id: digest(b"permanent source"), + })); + ctx.meta + .record_operator_readiness(&g, &p, proof.clone()) + .await + .unwrap(); + ctx.meta + .transition_session(sid, SessionState::Failed, BindingDisposition::Detach) + .await + .unwrap(); + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); + assert!(ctx + .meta + .record_operator_readiness(&g, &p, proof.clone()) + .await + .is_err()); + let next = ctx.meta.retain_operator_participant(&g, sid).await.unwrap(); + assert_eq!(next.target().unwrap(), p.target().unwrap()); + assert_ne!(next, p); + ctx.meta + .record_operator_readiness(&g, &next, proof) + .await + .unwrap(); + assert!( + ctx.meta + .publish_operator_generation(&g) + .await + .unwrap() + .admission_open + ); +} +conformance!( + t_operator_placement_change_invalidates_readiness, + super::operator_placement_change_invalidates_readiness +); + +async fn operator_captured_source_requires_exact_destroyed_owner(ctx: &Ctx) { + use engram_core::types::{managed_boot::*, managed_operator::*}; + let (_, row) = managed_boot_prepared_fixture(ctx).await; + let sid = row.request.key.creation.session_id; + let g = ctx + .meta + .stage_operator_publication(Some(1), operator_candidate()) + .await + .unwrap() + .pending + .unwrap(); + let p = ctx.meta.retain_operator_participant(&g, sid).await.unwrap(); + let request = ManagedCaptureRequest { + key: row.request.key.clone(), + capture_id: SnapshotId::new(), + }; + let mut authority = ManagedCaptureAuthority { + version: 1, + request: request.clone(), + host_incarnation: row.prepared.unwrap().host_incarnation, + disk_manifest: request.disk_manifest(), + disk_sha256: "d".repeat(64), + disk_size_bytes: 4096, + image: row.request.cold.image, + kernel_sha256: row.request.cold.kernel_sha256, + aux_ro_drives: row.request.cold.aux_ro_drives, + consistency: ManagedDiskConsistency::CrashConsistentDiskCutV1, + source_state: ManagedCapturedSourceState::Destroyed, + record_digest: String::new(), + }; + authority.record_digest = authority.digest().unwrap(); + let mut wrong = authority.clone(); + wrong.host_incarnation = "other-owner".into(); + wrong.record_digest = wrong.digest().unwrap(); + assert!(ctx + .meta + .record_operator_readiness(&g, &p, OperatorReadiness::Captured(Box::new(wrong))) + .await + .is_err()); + ctx.meta + .record_operator_readiness(&g, &p, OperatorReadiness::Captured(Box::new(authority))) + .await + .unwrap(); + assert!( + ctx.meta + .publish_operator_generation(&g) + .await + .unwrap() + .admission_open + ); + // Publication consumes destruction evidence; it does not invent a pinned recovery artifact. + assert!(ctx.meta.reserve_managed_capture(&request).await.is_err()); +} +conformance!( + t_operator_captured_source_requires_exact_destroyed_owner, + super::operator_captured_source_requires_exact_destroyed_owner +); + +async fn operator_policy_only_lineage_blocks_stale_and_fresh_live_receipts(ctx: &Ctx) { + use engram_core::types::{managed_deny::*, managed_operator::*}; + let sid = ctx + .meta + .create_session(spec("test.invalid/legacy:latest")) + .await + .unwrap(); + let hid = HostId::new(); + ctx.meta + .upsert_host(host_record(hid, "legacy-host", ctx.clock.now_utc())) + .await + .unwrap(); + ctx.meta.assign_session_host(sid, Some(hid)).await.unwrap(); + let sandbox = engram_core::SandboxId::new(); + let epoch = ctx + .meta + .transition_session_created(sid, sandbox) + .await + .unwrap(); + ctx.meta + .transition_session(sid, SessionState::Idle, BindingDisposition::Retain) + .await + .unwrap(); + let target = ManagedPolicyTarget { + session_id: sid, + host_id: hid, + sandbox_id: sandbox, + binding_epoch: epoch, + }; + ctx.meta + .retain_managed_deny( + &target, + &ManagedDenyPolicy::new(1, ["history.example".into()]).unwrap(), + ) + .await + .unwrap(); + let g = ctx + .meta + .stage_operator_publication(None, operator_candidate()) + .await + .unwrap() + .pending + .unwrap(); + let p = ctx.meta.retain_operator_participant(&g, sid).await.unwrap(); + assert_eq!(p.status, "idle"); + assert!(p.boot.is_none()); + let receipt = ManagedAppliedState::new( + ManagedPolicyContext { + target: target.clone(), + operator_generation: g.generation, + operator_identity: g.identity.clone(), + binding_hash: None, + }, + p.policy.clone(), + "host-incarnation".into(), + "proxy-incarnation".into(), + Some(ManagedNetworkProof { + target, + guest_ip: "192.0.2.2".parse().unwrap(), + network_identity: engram_core::types::managed_create::digest(b"actual network proof"), + }), + ) + .unwrap(); + let original_policy = ctx.meta.get_session_integration_policy(sid).await.unwrap(); + for (host, proxy) in [ + ("host-incarnation", "proxy-incarnation"), + ("next-host", "proxy-incarnation"), + ("host-incarnation", "next-proxy"), + ("next-host", "next-proxy"), + ] { + let live = ManagedAppliedState::new( + receipt.context.clone(), + receipt.policy.clone(), + host.into(), + proxy.into(), + receipt.network_proof.clone(), + ) + .unwrap(); + live.validate().unwrap(); + let saved = serde_json::json!({"Applied":live}); + // Neither an old receipt nor a fresh receipt has durable lifetime authority. + assert!(serde_json::from_value::(saved.clone()).is_err()); + assert!(ctx + .meta + .record_operator_readiness(&g, &p, OperatorReadiness::NeverAllocated) + .await + .is_err()); + if let Some(db) = &ctx._pg { + // Model an existing pending proof written by the earlier implementation. + // Sim cannot construct the removed enum variant; both stores reject it + // at their typed/JSON boundary and keep this participant unready. + sqlx::query("UPDATE managed_operator_readiness SET proof=$3 WHERE generation=$1 AND session_id=$2") + .bind(g.generation as i64).bind(sid.as_uuid()).bind(saved).execute(db.store.pool()).await.unwrap(); + } + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); + assert!( + !ctx.meta + .read_operator_publication() + .await + .unwrap() + .admission_open + ); + assert_eq!( + ctx.meta.managed_policy_target(sid).await.unwrap(), + receipt.context.target + ); + assert_eq!( + ctx.meta.get_session_integration_policy(sid).await.unwrap(), + original_policy + ); + } + // Terminal labels do not remove unsupported lineage from the barrier. + ctx.meta + .transition_session(sid, SessionState::Completed, BindingDisposition::Detach) + .await + .unwrap(); + assert_eq!( + ctx.meta.list_operator_participants(None, 64).await.unwrap(), + vec![sid] + ); + let terminal = ctx.meta.retain_operator_participant(&g, sid).await.unwrap(); + assert!(terminal.policy.denies("history.example")); + assert!(terminal.policy.denies("unused.example")); + assert!(!terminal.dormant()); + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); +} +conformance!( + t_operator_policy_only_lineage_blocks_stale_and_fresh_live_receipts, + super::operator_policy_only_lineage_blocks_stale_and_fresh_live_receipts +); + +async fn operator_started_owner_progresses_after_permanent_retirement(ctx: &Ctx) { + use engram_core::types::{managed_boot::*, managed_create::digest, managed_operator::*}; + let row = managed_boot_finalized_fixture(ctx).await; + let key = &row.request.key; + let intent = ManagedBootStart { + receipt: row.receipt.clone().unwrap(), + start_id: digest(b"started owner"), + }; + let issued = ctx + .meta + .advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::IssueStart(intent.clone()), + ) + .await + .unwrap(); + ctx.meta + .advance_managed_boot(key, issued.revision, ManagedBootAdvance::Started(intent)) + .await + .unwrap(); + let g = ctx + .meta + .stage_operator_publication(Some(1), operator_candidate()) + .await + .unwrap() + .pending + .unwrap(); + let p = ctx + .meta + .retain_operator_participant(&g, key.creation.session_id) + .await + .unwrap(); + assert_eq!(p.status, "active"); + assert!(ctx.meta.publish_operator_generation(&g).await.is_err()); + let proof = OperatorReadiness::Retired(Box::new(ManagedBootRetiredReceipt { + prepared: row.prepared.unwrap(), + retained_policy: row.request.bootstrap_policy.clone(), + receipt_id: digest(b"permanent started owner retirement"), + })); + ctx.meta + .record_operator_readiness(&g, &p, proof) + .await + .unwrap(); + assert!( + ctx.meta + .publish_operator_generation(&g) + .await + .unwrap() + .admission_open + ); + assert_eq!( + ctx.meta + .managed_boot_open(key.creation.session_id) + .await + .unwrap(), + Some(false) + ); +} +conformance!( + t_operator_started_owner_progresses_after_permanent_retirement, + super::operator_started_owner_progresses_after_permanent_retirement +); + +async fn managed_bridge_identity_state(ctx: &Ctx) { + use engram_core::types::managed_bridge::*; + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + let ws = managed_creation_ws(SessionId::new()); + let c = ws.managed_creation.clone().unwrap(); + let q = ManagedCreationQuery { + identity: c.identity, + native_version: c.native.version, + native_digest: c.native.digest, + }; + assert_eq!( + ctx.meta + .read_managed_creation_state(&q) + .await + .unwrap() + .state, + ManagedCreationPhase::Absent + ); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + assert_eq!( + ctx.meta + .read_managed_creation_state(&q) + .await + .unwrap() + .state, + ManagedCreationPhase::Held + ); + let mut wrong = q.clone(); + wrong.native_digest = engram_core::types::managed_create::digest(b"wrong"); + assert!(matches!( + ctx.meta.read_managed_creation_state(&wrong).await, + Err(MetaError::Conflict(_)) + )); + ctx.meta + .transition_session( + ws.session_id, + SessionState::Failed, + engram_core::types::BindingDisposition::Detach, + ) + .await + .unwrap(); + let terminal = ctx.meta.read_managed_creation_state(&q).await.unwrap(); + assert_eq!(terminal.state, ManagedCreationPhase::Terminal); + assert_eq!(terminal.terminal_reason.as_deref(), Some("failed")); + let finalized = managed_boot_finalized_fixture(ctx).await; + assert!(finalized.sealed_agent.is_some()); + let query = ManagedCreationQuery { + identity: finalized.request.key.creation.clone(), + native_version: 1, + native_digest: finalized.request.key.native_digest.clone(), + }; + let view = ctx.meta.read_managed_creation_state(&query).await.unwrap(); + assert!(view.boot.as_ref().unwrap().sealed_agent.is_none()); + assert_eq!( + view.boot + .as_ref() + .unwrap() + .finalization + .as_ref() + .unwrap() + .combined_manifest_digest, + finalized.finalization.unwrap().combined_manifest_digest + ); + assert!(!serde_json::to_string(&view).unwrap().contains("ciphertext")); +} +conformance!( + t_managed_bridge_identity_state, + managed_bridge_identity_state +); + +fn bridge_intent( + ws: &engram_core::traits::SessionCreateWriteSet, +) -> engram_core::types::managed_bridge::ManagedBootIntent { + use engram_core::types::{managed_bridge::*, managed_create::digest}; + let c = ws.managed_creation.as_ref().unwrap(); + let request = managed_boot_request(ws, HostId::new()); + let base = match request.cold.disk.source { + engram_core::types::managed_cold::ManagedColdSource::FreshImage { base_manifest } => { + base_manifest + } + _ => unreachable!(), + }; + ManagedBootIntent { + query: ManagedCreationQuery { + identity: c.identity.clone(), + native_version: c.native.version, + native_digest: c.native.digest.clone(), + }, + intent_id: uuid::Uuid::new_v4().to_string(), + revision: 1, + selection: ManagedBootSelection { + launch: ManagedHarnessLaunch { + descriptor: engram_core::types::harness::HarnessDescriptor { + name: "claude".into(), + ..Default::default() + }, + workdir: None, + }, + image: ws.spec.image.clone(), + image_digest: digest(b"approved image"), + base_manifest: base, + aux_ro_drives: request.cold.aux_ro_drives, + vcpus: 1, + memory_mib: 128, + disk_gib: 1, + }, + sandbox_id: engram_core::SandboxId::new(), + binding_epoch: 1, + disk_manifest: engram_core::types::ManifestRef::new(), + recovery: None, + host_id: None, + assets: None, + completed: false, + } +} +async fn managed_bridge_intent_cas_pins_and_gate(ctx: &Ctx) { + use engram_core::types::managed_bridge::*; + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + let mut ws = managed_creation_ws(SessionId::new()); + let i = bridge_intent(&ws); + ws.managed_boot_selection = Some(i.selection.clone()); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + let mut wrong_query = i.query.clone(); + wrong_query.native_digest = engram_core::types::managed_create::digest(b"other native"); + assert!(ctx + .meta + .read_managed_boot_selection(&wrong_query) + .await + .is_err()); + assert_eq!( + ctx.meta + .read_managed_boot_selection(&i.query) + .await + .unwrap(), + i.selection + ); + // Lost reserve reply uses the immutable selected inputs; a new catalog value cannot replace them. + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + let mut changed_selection = ws.clone(); + changed_selection + .managed_boot_selection + .as_mut() + .unwrap() + .base_manifest = engram_core::types::ManifestRef::new(); + assert!(ctx + .meta + .reserve_and_persist_create(changed_selection, &[], 0) + .await + .is_err()); + let mut changed = i.clone(); + changed.intent_id = uuid::Uuid::new_v4().to_string(); + changed.disk_manifest = engram_core::types::ManifestRef::new(); + changed.sandbox_id = engram_core::SandboxId::new(); + let (a, b) = tokio::join!( + ctx.meta.begin_managed_boot_intent(i.clone()), + ctx.meta.begin_managed_boot_intent(changed) + ); + let a = a.unwrap(); + assert_eq!(a, b.unwrap()); + assert_eq!( + ctx.meta + .read_managed_boot_intent(&i.query, "initial") + .await + .unwrap(), + Some(a.clone()) + ); + assert_eq!( + ctx.meta.list_managed_boot_intents(None, 10).await.unwrap(), + vec![a.clone()] + ); + let refs = ctx.meta.list_managed_disk_manifests().await.unwrap(); + assert!(refs.contains(&a.selection.base_manifest)); + assert!(refs.contains(&a.disk_manifest)); + let pins = ctx.meta.bundle_pin_set().await.unwrap(); + for d in &a.selection.aux_ro_drives { + assert!(pins + .iter() + .any(|p| p.drive_id == d.drive_id && Some(&p.sha256) == d.sha256.as_ref())); + } + assert_eq!( + ctx.meta + .read_managed_creation_state(&i.query) + .await + .unwrap() + .state, + ManagedCreationPhase::Preparing + ); + assert!(ctx + .meta + .advance_managed_boot_intent(&a.intent_id, a.revision, ManagedIntentAdvance::Complete) + .await + .is_err()); + let host = HostId::new(); + let full = HostId::new(); + for (id, capacity) in [(full, 64), (host, 1024)] { + ctx.meta + .upsert_host(host_record(id, "bridge-candidate", ctx.clock.now_utc())) + .await + .unwrap(); + let mut heartbeat = heartbeat_fixture(); + heartbeat.utilization.allocatable_mib = capacity; + ctx.meta.touch_host_heartbeat(id, heartbeat).await.unwrap(); + } + assert!( + ctx.meta + .advance_managed_boot_intent( + &a.intent_id, + a.revision, + ManagedIntentAdvance::SelectHost(full) + ) + .await + .is_err(), + "full candidate must not become pinned" + ); + let cordoned = HostId::new(); + ctx.meta + .upsert_host(host_record(cordoned, "cordoned-host", ctx.clock.now_utc())) + .await + .unwrap(); + let mut heartbeat = heartbeat_fixture(); + heartbeat.utilization.allocatable_mib = 1024; + ctx.meta + .touch_host_heartbeat(cordoned, heartbeat) + .await + .unwrap(); + ctx.meta + .set_host_cordon( + cordoned, + Some(engram_core::types::host::CordonOwner::Admin), + Some("capacity fixture"), + ) + .await + .unwrap(); + assert!( + ctx.meta + .advance_managed_boot_intent( + &a.intent_id, + a.revision, + ManagedIntentAdvance::SelectHost(cordoned) + ) + .await + .is_err(), + "held creation must reject a cordoned host" + ); + let mut heartbeat = heartbeat_fixture(); + heartbeat.utilization.allocatable_mib = 1024; + ctx.meta + .touch_host_heartbeat(host, heartbeat) + .await + .unwrap(); + let (x, y) = tokio::join!( + ctx.meta.advance_managed_boot_intent( + &a.intent_id, + a.revision, + ManagedIntentAdvance::SelectHost(host) + ), + ctx.meta.advance_managed_boot_intent( + &a.intent_id, + a.revision, + ManagedIntentAdvance::SelectHost(host) + ) + ); + assert_eq!(usize::from(x.is_ok()) + usize::from(y.is_ok()), 1); + let mut next = ctx + .meta + .read_managed_boot_intent(&i.query, "initial") + .await + .unwrap() + .unwrap(); + let req = next.assets_request().unwrap(); + let receipt = engram_core::types::managed_assets::ManagedColdAssetsReceipt { + request: req.clone(), + host_incarnation: "fixture-incarnation".into(), + backend: engram_core::types::managed_assets::ManagedColdBackend::Firecracker, + kernel_sha256: "c".repeat(64), + disk_sha256: "d".repeat(64), + disk_size_bytes: 4096, + }; + let mut wrong = receipt.clone(); + wrong.request.disk_manifest = engram_core::types::ManifestRef::new(); + assert!(ctx + .meta + .advance_managed_boot_intent( + &next.intent_id, + next.revision, + ManagedIntentAdvance::Assets(Box::new(wrong)) + ) + .await + .is_err()); + next = ctx + .meta + .advance_managed_boot_intent( + &next.intent_id, + next.revision, + ManagedIntentAdvance::Assets(Box::new(receipt.clone())), + ) + .await + .unwrap(); + ctx.meta + .set_host_cordon( + host, + Some(engram_core::types::host::CordonOwner::Admin), + Some("fixture"), + ) + .await + .unwrap(); + let mut heartbeat = heartbeat_fixture(); + heartbeat.utilization.allocatable_mib = 1024; + ctx.meta + .touch_host_heartbeat(full, heartbeat) + .await + .unwrap(); + let stale = next.clone(); + next = ctx + .meta + .advance_managed_boot_intent( + &next.intent_id, + next.revision, + ManagedIntentAdvance::SelectHost(full), + ) + .await + .unwrap(); + assert_eq!(next.sandbox_id, a.sandbox_id); + assert_eq!(next.disk_manifest, a.disk_manifest); + assert!(next.assets.is_none()); + assert!(ctx + .meta + .advance_managed_boot_intent( + &next.intent_id, + stale.revision, + ManagedIntentAdvance::Assets(Box::new(receipt.clone())) + ) + .await + .is_err()); + assert!(ctx + .meta + .advance_managed_boot_intent( + &next.intent_id, + next.revision, + ManagedIntentAdvance::Assets(Box::new(receipt)) + ) + .await + .is_err()); + // Actual publication stage closes the shared gate before any further intent CAS. + let p = ctx.meta.read_operator_publication().await.unwrap(); + let g = p.published.unwrap(); + ctx.meta + .stage_operator_publication(Some(g.generation), operator_candidate()) + .await + .unwrap(); + assert!(ctx + .meta + .advance_managed_boot_intent( + &next.intent_id, + next.revision, + ManagedIntentAdvance::Complete + ) + .await + .is_err()); + let mut another = bridge_intent(&ws); + another.query = i.query.clone(); + assert!(ctx.meta.begin_managed_boot_intent(another).await.is_err()); +} +conformance!( + t_managed_bridge_intent_cas_pins_and_gate, + managed_bridge_intent_cas_pins_and_gate +); + +async fn managed_bridge_detached_retired_source_can_capture(ctx: &Ctx) { + use engram_core::types::{managed_boot::*, managed_create::digest}; + let (_, row) = managed_boot_prepared_fixture(ctx).await; + let key = &row.request.key; + let receipt = ManagedBootRetiredReceipt { + prepared: row.prepared.clone().unwrap(), + retained_policy: row.request.bootstrap_policy.clone(), + receipt_id: digest(b"retained source"), + }; + ctx.meta + .advance_managed_boot( + key, + row.revision, + ManagedBootAdvance::Retired(Box::new(receipt)), + ) + .await + .unwrap(); + ctx.meta + .transition_session( + key.creation.session_id, + SessionState::Idle, + BindingDisposition::Detach, + ) + .await + .unwrap(); + let retained = ctx + .meta + .read_managed_boot(key) + .await + .expect("permanent exact source retirement survives detach"); + assert_eq!(retained.phase, ManagedBootPhase::Interrupted); + assert_eq!( + ctx.meta + .managed_boot_open(key.creation.session_id) + .await + .unwrap(), + Some(false) + ); + let request = ManagedCaptureRequest { + key: key.clone(), + capture_id: SnapshotId::new(), + }; + assert!(ctx + .meta + .reserve_managed_capture(&request) + .await + .unwrap() + .is_none()); + let mut authority = ManagedCaptureAuthority { + version: 1, + request: request.clone(), + host_incarnation: row.prepared.unwrap().host_incarnation, + disk_manifest: request.disk_manifest(), + disk_sha256: "d".repeat(64), + disk_size_bytes: 4096, + image: row.request.cold.image.clone(), + kernel_sha256: row.request.cold.kernel_sha256.clone(), + aux_ro_drives: row.request.cold.aux_ro_drives.clone(), + consistency: ManagedDiskConsistency::CrashConsistentDiskCutV1, + source_state: ManagedCapturedSourceState::Destroyed, + record_digest: String::new(), + }; + authority.record_digest = authority.digest().unwrap(); + ctx.meta.record_managed_capture(&authority).await.unwrap(); + assert_eq!( + ctx.meta.reserve_managed_capture(&request).await.unwrap(), + Some(authority.clone()) + ); + let mut next = row.request.clone(); + next.key.context.target.binding_epoch = 2; + next.key.context.target.sandbox_id = engram_core::SandboxId::new(); + next.key.boot_attempt.push_str("-recovered"); + next.cold.boot_attempt = next.key.boot_attempt.clone(); + next.cold.sandbox_id = next.key.context.target.sandbox_id; + next.cold.disk.manifest = authority.disk_manifest; + next.cold.disk.sha256 = authority.disk_sha256.clone(); + next.cold.disk.source = engram_core::types::managed_cold::ManagedColdSource::Capture { + capture_id: request.capture_id, + session_id: key.creation.session_id, + sandbox_id: key.context.target.sandbox_id, + binding_epoch: 1, + record_digest: authority.record_digest.clone(), + }; + let mut spec: engram_core::types::sandbox::SandboxSpec = + serde_json::from_str(&next.sandbox_spec_json).unwrap(); + spec.rootfs_manifest = Some(authority.disk_manifest); + next.sandbox_spec_json = serde_json::to_string(&spec).unwrap(); + next.capture_authority = Some(authority.clone()); + ctx.meta.claim_managed_boot(next.clone()).await.unwrap(); + assert!( + ctx.meta.read_managed_boot(key).await.is_err(), + "retained old proof cannot become current target authority" + ); + assert_eq!( + ctx.meta.reserve_managed_capture(&request).await.unwrap(), + Some(authority) + ); + assert_eq!( + ctx.meta.read_managed_boot(&next.key).await.unwrap().phase, + ManagedBootPhase::Claimed + ); + let (_, unknown) = managed_boot_prepared_fixture(ctx).await; + ctx.meta + .advance_managed_boot( + &unknown.request.key, + unknown.revision, + ManagedBootAdvance::Interrupt, + ) + .await + .unwrap(); + ctx.meta + .transition_session( + unknown.request.key.creation.session_id, + SessionState::Idle, + BindingDisposition::Detach, + ) + .await + .unwrap(); + assert!( + ctx.meta + .read_managed_boot(&unknown.request.key) + .await + .is_err(), + "interruption alone is not permanent source proof" + ); +} +conformance!( + t_managed_bridge_detached_retired_source_can_capture, + managed_bridge_detached_retired_source_can_capture +); + +async fn managed_bridge_claim_reconcile_and_recovery(ctx: &Ctx) { + use engram_core::types::{ + managed_assets::*, managed_boot::*, managed_bridge::*, managed_cold::ManagedColdSource, + managed_create::digest, + }; + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + let mut ws = managed_creation_ws(SessionId::new()); + let mut i = bridge_intent(&ws); + ws.managed_boot_selection = Some(i.selection.clone()); + ctx.meta + .reserve_and_persist_create(ws.clone(), &[], 0) + .await + .unwrap(); + let host = HostId::new(); + ctx.meta + .upsert_host(host_record(host, "bridge-host", ctx.clock.now_utc())) + .await + .unwrap(); + let mut request = managed_boot_request(&ws, host); + assert!( + ctx.meta.claim_managed_boot(request.clone()).await.is_err(), + "selected creation cannot bypass durable intent" + ); + i = ctx.meta.begin_managed_boot_intent(i).await.unwrap(); + i = ctx + .meta + .advance_managed_boot_intent( + &i.intent_id, + i.revision, + ManagedIntentAdvance::SelectHost(host), + ) + .await + .unwrap(); + let assets = ManagedColdAssetsReceipt { + request: i.assets_request().unwrap(), + host_incarnation: "asset-owner".into(), + backend: ManagedColdBackend::Firecracker, + kernel_sha256: "c".repeat(64), + disk_sha256: "d".repeat(64), + disk_size_bytes: 4096, + }; + i = ctx + .meta + .advance_managed_boot_intent( + &i.intent_id, + i.revision, + ManagedIntentAdvance::Assets(Box::new(assets.clone())), + ) + .await + .unwrap(); + request.key.boot_attempt = i.intent_id.clone(); + request.key.context.target.sandbox_id = i.sandbox_id; + request.cold.boot_attempt = i.intent_id.clone(); + request.cold.sandbox_id = i.sandbox_id; + request.cold.disk.manifest = i.disk_manifest; + request.cold.disk.sha256 = assets.disk_sha256; + request.cold.disk.source = ManagedColdSource::FreshImage { + base_manifest: i.selection.base_manifest, + }; + let mut spec: engram_core::types::sandbox::SandboxSpec = + serde_json::from_str(&request.sandbox_spec_json).unwrap(); + spec.rootfs_manifest = Some(i.disk_manifest); + spec.memory.max_mib = i.selection.memory_mib; + request.sandbox_spec_json = serde_json::to_string(&spec).unwrap(); + let mut wrong = request.clone(); + wrong.cold.disk.sha256 = "e".repeat(64); + assert!(ctx.meta.claim_managed_boot(wrong).await.is_err()); + let row = ctx.meta.claim_managed_boot(request.clone()).await.unwrap(); + let alternative = HostId::new(); + ctx.meta + .upsert_host(host_record( + alternative, + "alternative-host", + ctx.clock.now_utc(), + )) + .await + .unwrap(); + assert!( + ctx.meta + .advance_managed_boot_intent( + &i.intent_id, + i.revision, + ManagedIntentAdvance::SelectHost(alternative) + ) + .await + .is_err(), + "claim fences relocation before completion" + ); + // Commit succeeded but the caller lost its reply: exact retry returns the same owner. + assert_eq!( + serde_json::to_value(ctx.meta.claim_managed_boot(request.clone()).await.unwrap()).unwrap(), + serde_json::to_value(&row).unwrap() + ); + let prepared = ManagedBootPreparedReceipt { + request: request.clone(), + host_incarnation: "asset-owner".into(), + network_identity: digest(b"network"), + guest_ip: "192.0.2.2".parse().unwrap(), + receipt_id: digest(b"prepared"), + }; + let mut wrong = prepared.clone(); + wrong.host_incarnation = "restarted-host".into(); + assert!(ctx + .meta + .advance_managed_boot( + &request.key, + row.revision, + ManagedBootAdvance::Prepared(Box::new(wrong)) + ) + .await + .is_err()); + let row = ctx + .meta + .advance_managed_boot( + &request.key, + row.revision, + ManagedBootAdvance::Prepared(Box::new(prepared)), + ) + .await + .unwrap(); + i = ctx + .meta + .advance_managed_boot_intent(&i.intent_id, i.revision, ManagedIntentAdvance::Complete) + .await + .unwrap(); + assert!(i.completed); + assert!(ctx + .meta + .list_managed_boot_intents(None, 10) + .await + .unwrap() + .is_empty()); + let capture_id = SnapshotId::new(); + let mut recovery = ManagedBootIntent { + query: i.query.clone(), + intent_id: uuid::Uuid::new_v4().to_string(), + revision: 1, + selection: i.selection.clone(), + sandbox_id: engram_core::SandboxId::new(), + binding_epoch: 2, + disk_manifest: engram_core::types::ManifestRef { + manifest_id: capture_id.as_uuid(), + version: 1, + }, + recovery: Some(ManagedRecoveryIntent { + recovery_id: "recovery-one".into(), + source: request.key.clone(), + capture_id, + }), + host_id: None, + assets: None, + completed: false, + }; + let mut substituted = recovery.clone(); + substituted + .selection + .launch + .descriptor + .args + .push("changed-argument".into()); + assert!(ctx + .meta + .begin_managed_boot_intent(substituted) + .await + .is_err()); + recovery = ctx.meta.begin_managed_boot_intent(recovery).await.unwrap(); + assert_eq!( + ctx.meta + .read_managed_creation_state(&i.query) + .await + .unwrap() + .state, + ManagedCreationPhase::Recovering + ); + let mut retry = recovery.clone(); + retry.intent_id = uuid::Uuid::new_v4().to_string(); + retry.sandbox_id = engram_core::SandboxId::new(); + assert_eq!( + ctx.meta.begin_managed_boot_intent(retry).await.unwrap(), + recovery + ); + let mut competing = recovery.clone(); + competing.intent_id = uuid::Uuid::new_v4().to_string(); + competing.recovery.as_mut().unwrap().recovery_id = "other-recovery".into(); + assert!(matches!( + ctx.meta.begin_managed_boot_intent(competing).await, + Err(engram_core::MetaError::Conflict(_)) + )); + assert!(ctx + .meta + .list_managed_disk_manifests() + .await + .unwrap() + .contains(&recovery.disk_manifest)); + assert_eq!( + serde_json::to_value(ctx.meta.read_managed_boot(&request.key).await.unwrap()).unwrap(), + serde_json::to_value(&row).unwrap(), + "intent does not itself interrupt a guest" + ); + ctx.meta + .advance_managed_boot(&request.key, row.revision, ManagedBootAdvance::Interrupt) + .await + .unwrap(); + ctx.meta + .reserve_managed_capture(&ManagedCaptureRequest { + key: request.key.clone(), + capture_id, + }) + .await + .unwrap(); + ctx.meta + .transition_session( + ws.session_id, + SessionState::Failed, + BindingDisposition::Detach, + ) + .await + .unwrap(); + assert!(ctx + .meta + .advance_managed_boot_intent( + &recovery.intent_id, + recovery.revision, + ManagedIntentAdvance::SelectHost(host) + ) + .await + .is_err()); + assert_eq!( + ctx.meta + .read_managed_creation_state(&i.query) + .await + .unwrap() + .state, + ManagedCreationPhase::Terminal + ); + let history = managed_history(ctx, ws.session_id).await; + ctx.meta + .set_host_status(host, HostStatus::Dead) + .await + .unwrap(); + assert_eq!( + ctx.meta.delete_host(host).await.unwrap(), + engram_core::types::host::DeleteHostOutcome::Deleted + ); + assert!(ctx.meta.get_host(host).await.unwrap().is_none()); + assert!(matches!( + ctx.meta.claim_managed_boot(request.clone()).await, + Err(MetaError::NotFound) + )); + assert_eq!(managed_history(ctx, ws.session_id).await, history); + assert_eq!( + ctx.meta.managed_sandbox_open(i.sandbox_id).await.unwrap(), + Some(false) + ); + let mut held = managed_creation_ws(SessionId::new()); + held.spec.image = ws.spec.image.clone(); + ctx.meta + .reserve_and_persist_create(held.clone(), &[], 0) + .await + .unwrap(); + assert!(ctx + .meta + .claim_managed_boot(managed_boot_request(&held, host)) + .await + .is_err()); + assert!(ctx + .meta + .get_managed_boot(&held.managed_creation.unwrap().identity) + .await + .unwrap() + .is_none()); +} +conformance!( + t_managed_bridge_claim_reconcile_and_recovery, + managed_bridge_claim_reconcile_and_recovery +); + +async fn managed_history(ctx: &Ctx, sid: SessionId) -> serde_json::Value { + if let Some(sim) = &ctx.sim { + sim.with_db(|db| serde_json::json!({ + "boot": db.managed_boots.get(&sid), + "captures": db.managed_captures.values().filter(|(r, _)| r.key.creation.session_id == sid).collect::>(), + "intents": db.managed_boot_intents.values().filter(|i| i.query.identity.session_id == sid).collect::>(), + "selection": db.managed_boot_selections.get(&sid), + })) + } else { + sqlx::query_scalar("SELECT jsonb_build_object('boot',(SELECT record FROM session_managed_boots WHERE session_id=$1),'captures',(SELECT jsonb_agg(jsonb_build_array(request,authority) ORDER BY capture_id) FROM session_managed_captures WHERE session_id=$1),'intents',(SELECT jsonb_agg(record ORDER BY intent_id) FROM session_managed_boot_intents WHERE session_id=$1),'selection',(SELECT selection FROM session_managed_boot_selections WHERE session_id=$1))") + .bind(sid.as_uuid()).fetch_one(ctx._pg.as_ref().unwrap().store.pool()).await.unwrap() + } +} + +async fn wait_for_placement_lock(db: &engram_testkit::pg::TestDb, advisory: bool) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let waiting: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock' AND CASE WHEN $1 THEN wait_event='advisory' ELSE query LIKE '%hosts%' END)") + .bind(advisory).fetch_one(db.store.pool()).await.unwrap(); + if waiting { return; } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }).await.expect("placement did not reach its lock barrier"); +} + +async fn managed_placement_and_policy_child(ctx: &Ctx, claim: bool) { + use engram_core::types::managed_bridge::*; + engram_testkit::operator::publish_empty(ctx.meta.as_ref()).await; + let host = HostId::new(); + ctx.meta + .upsert_host(host_record( + host, + &format!("placement-{claim}"), + ctx.clock.now_utc(), + )) + .await + .unwrap(); + let mut heartbeat = heartbeat_fixture(); + heartbeat.utilization.allocatable_mib = 1024; + ctx.meta + .touch_host_heartbeat(host, heartbeat) + .await + .unwrap(); + let mut parent = managed_creation_ws(SessionId::new()); + let intent = bridge_intent(&parent); + if !claim { + parent.managed_boot_selection = Some(intent.selection.clone()); + } + ctx.meta + .reserve_and_persist_create(parent.clone(), &[], 0) + .await + .unwrap(); + if !claim { + ctx.meta + .begin_managed_boot_intent(intent.clone()) + .await + .unwrap(); + } + let request = managed_boot_request(&parent, host); + managed_operator_gate_fixture(ctx, &request.key, true).await; + let mut child = managed_creation_ws(SessionId::new()); + child.managed_creation = None; + child.mcp_policy.as_mut().unwrap().parent_session_id = Some(parent.session_id); + let child_id = child.session_id; + let mut barrier = None; + if let Some(db) = &ctx._pg { + // The child holds its host row before it can lock the policy parent. + let trigger = format!("CREATE OR REPLACE FUNCTION wait_policy_child() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.id='{}'::uuid THEN PERFORM pg_advisory_xact_lock(80402); END IF; RETURN NEW; END; $$", child_id); + sqlx::query(&trigger) + .execute(db.store.pool()) + .await + .unwrap(); + sqlx::query("CREATE TRIGGER wait_policy_child BEFORE INSERT ON sessions FOR EACH ROW EXECUTE FUNCTION wait_policy_child()") + .execute(db.store.pool()).await.unwrap(); + let mut tx = db.store.pool().begin().await.unwrap(); + sqlx::query("SELECT pg_advisory_xact_lock(80402)") + .execute(&mut *tx) + .await + .unwrap(); + barrier = Some(tx); + } + let child_meta = ctx.meta.clone(); + let child_task = tokio::spawn(async move { + child_meta + .reserve_and_persist_create(child, &[host], 0) + .await + }); + if let Some(db) = &ctx._pg { + wait_for_placement_lock(db, true).await; + } + let retry = request.clone(); + let parent_meta = ctx.meta.clone(); + let parent_task = tokio::spawn(async move { + if claim { + parent_meta.claim_managed_boot(request).await.map(|_| ()) + } else { + parent_meta + .advance_managed_boot_intent( + &intent.intent_id, + intent.revision, + ManagedIntentAdvance::SelectHost(host), + ) + .await + .map(|_| ()) + } + }); + if let Some(db) = &ctx._pg { + wait_for_placement_lock(db, false).await; + } + if let Some(tx) = barrier { + tx.commit().await.unwrap(); + } + let (child_result, parent_result) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(child_task, parent_task) + }) + .await + .expect("placement operations did not complete"); + assert!( + child_result.as_ref().unwrap().is_ok(), + "policy child creation failed: {child_result:?}" + ); + assert!( + parent_result.as_ref().unwrap().is_ok(), + "managed placement failed: {parent_result:?}" + ); + assert_eq!( + ctx.meta.get_session(child_id).await.unwrap().host_id, + Some(host) + ); + if claim { + assert_eq!( + ctx.meta.per_host_reserved().await.unwrap()[&host].mem_mib, + 1024 + ); + assert!( + ctx.meta.claim_managed_boot(retry).await.is_ok(), + "an exact retry needs no extra capacity" + ); + } + if let Some(db) = &ctx._pg { + sqlx::query("DROP TRIGGER wait_policy_child ON sessions") + .execute(db.store.pool()) + .await + .unwrap(); + } +} +async fn managed_placement_and_policy_child_do_not_deadlock(ctx: &Ctx) { + managed_placement_and_policy_child(ctx, false).await; +} +async fn managed_claim_and_policy_child_do_not_deadlock(ctx: &Ctx) { + managed_placement_and_policy_child(ctx, true).await; +} + +conformance!( + t_managed_placement_and_policy_child_do_not_deadlock, + managed_placement_and_policy_child_do_not_deadlock +); + +conformance!( + t_managed_claim_and_policy_child_do_not_deadlock, + managed_claim_and_policy_child_do_not_deadlock +); + +async fn teleport_and_ordinary_policy_child(ctx: &Ctx, commit: bool) { + use engram_core::traits::CreateDisposition; + use engram_core::types::teleport::*; + + let source = HostId::new(); + let dest = HostId::new(); + for host in [source, dest] { + ctx.meta + .upsert_host(host_record(host, "policy-teleport", ctx.clock.now_utc())) + .await + .unwrap(); + let mut heartbeat = heartbeat_fixture(); + heartbeat.utilization.allocatable_mib = 4096; + ctx.meta + .touch_host_heartbeat(host, heartbeat) + .await + .unwrap(); + } + let mut parent = managed_creation_ws(SessionId::new()); + parent.managed_creation = None; + assert_eq!( + ctx.meta + .reserve_and_persist_create(parent.clone(), &[source], 0) + .await + .unwrap(), + CreateDisposition::Placed(source) + ); + assert_eq!( + ctx.meta.managed_boot_open(parent.session_id).await.unwrap(), + None + ); + let policy = ctx + .meta + .read_mcp_write_policy(parent.session_id) + .await + .unwrap(); + assert_eq!( + policy.write_eligibility, + engram_core::types::mcp_policy::WriteEligibility::ReadOnly + ); + ctx.meta + .transition_session_created(parent.session_id, engram_core::SandboxId::new()) + .await + .unwrap(); + ctx.meta + .transition_session( + parent.session_id, + SessionState::Active, + BindingDisposition::Retain, + ) + .await + .unwrap(); + let mut child = managed_creation_ws(SessionId::new()); + child.managed_creation = None; + child.mcp_policy.as_mut().unwrap().parent_session_id = Some(parent.session_id); + let child_id = child.session_id; + let request = TeleportAdmitRequest { + id: engram_core::TeleportId::new(), + session_id: parent.session_id, + reason: TeleportReason::Ui, + epoch: 0, + candidates: vec![source, dest], + pinned_dest: Some(dest), + mem_budget_mib: parent.mem_budget_mib, + cpu_budget_vcpus: i64::from(parent.cpu_budget_vcpus), + max_open_per_dest: 1, + kind: TeleportKind::Live, + }; + if commit { + admit_teleport(ctx, request.clone()).await; + for (from, to, patch) in [ + ( + TeleportPhase::Admitted, + TeleportPhase::Captured, + TeleportPatch::default(), + ), + ( + TeleportPhase::Captured, + TeleportPhase::Restored, + TeleportPatch { + dest_sandbox_id: Some(engram_core::SandboxId::new()), + ..Default::default() + }, + ), + ] { + assert!(ctx + .meta + .teleport_advance(request.id, from, to, patch, 0) + .await + .unwrap()); + } + } + let mut barrier = None; + if let Some(db) = &ctx._pg { + // Pause the real create method after it locks H, before policy ancestry P. + let trigger = format!("CREATE FUNCTION wait_teleport_policy_child() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.id='{}'::uuid THEN PERFORM pg_advisory_xact_lock(80403); END IF; RETURN NEW; END; $$", child_id); + sqlx::query(&trigger) + .execute(db.store.pool()) + .await + .unwrap(); + sqlx::query("CREATE TRIGGER wait_teleport_policy_child BEFORE INSERT ON sessions FOR EACH ROW EXECUTE FUNCTION wait_teleport_policy_child()") + .execute(db.store.pool()).await.unwrap(); + let mut tx = db.store.pool().begin().await.unwrap(); + sqlx::query("SELECT pg_advisory_xact_lock(80403)") + .execute(&mut *tx) + .await + .unwrap(); + barrier = Some(tx); + } + let child_meta = ctx.meta.clone(); + let child_task = tokio::spawn(async move { + child_meta + .reserve_and_persist_create(child, &[dest], 0) + .await + }); + if let Some(db) = &ctx._pg { + wait_for_placement_lock(db, true).await; + } + let parent_meta = ctx.meta.clone(); + let mut parent_task = tokio::spawn(async move { + if commit { + assert!(parent_meta.teleport_commit(request.id, 0).await?.is_some()); + Ok(None) + } else { + parent_meta.teleport_admit(request).await.map(Some) + } + }); + let completed_parent = if commit { + // FK checks can finish while a placement writer still holds the host. + Some( + tokio::time::timeout(Duration::from_secs(5), &mut parent_task) + .await + .expect("teleport commit blocked behind child placement"), + ) + } else { + if let Some(db) = &ctx._pg { + wait_for_placement_lock(db, false).await; + } + None + }; + if let Some(tx) = barrier { + tx.commit().await.unwrap(); + } + let child_result = tokio::time::timeout(Duration::from_secs(5), child_task) + .await + .expect("policy child did not complete"); + let parent_result = match completed_parent { + Some(result) => result, + None => tokio::time::timeout(Duration::from_secs(5), parent_task) + .await + .expect("teleport admission did not complete"), + }; + assert_eq!( + child_result.unwrap().unwrap(), + CreateDisposition::Placed(dest) + ); + let parent_result = parent_result + .unwrap() + .expect("ordinary policy-parent teleport failed"); + if let Some(parent_result) = parent_result { + let TeleportAdmitOutcome::Admitted(row) = parent_result else { + panic!("teleport refused: {parent_result:?}") + }; + assert_eq!(row.source_host_id, source); + assert_eq!(row.dest_host_id, dest); + } else { + assert_eq!( + ctx.meta + .get_session(parent.session_id) + .await + .unwrap() + .host_id, + Some(dest) + ); + } + assert_eq!( + ctx.meta.get_session(child_id).await.unwrap().host_id, + Some(dest) + ); + assert_eq!( + ctx.meta.per_host_reserved().await.unwrap()[&dest].mem_mib, + 1024 + ); + if let Some(db) = &ctx._pg { + sqlx::query("DROP TRIGGER wait_teleport_policy_child ON sessions") + .execute(db.store.pool()) + .await + .unwrap(); + sqlx::query("DROP FUNCTION wait_teleport_policy_child()") + .execute(db.store.pool()) + .await + .unwrap(); + } +} + +async fn teleport_and_ordinary_policy_child_do_not_deadlock(ctx: &Ctx) { + teleport_and_ordinary_policy_child(ctx, false).await; +} +async fn teleport_commit_and_ordinary_policy_child_do_not_deadlock(ctx: &Ctx) { + teleport_and_ordinary_policy_child(ctx, true).await; +} + +async fn teleport_destination_refusals_preserve_session_precedence(ctx: &Ctx) { + use engram_core::types::teleport::*; + let mut request = teleport_admission_fixture(ctx).await; + let dest = request.candidates[1]; + request.pinned_dest = Some(HostId::new()); + assert!(matches!( + ctx.meta.teleport_admit(request.clone()).await.unwrap(), + TeleportAdmitOutcome::NoFit + )); + let mut missing_session = request.clone(); + missing_session.session_id = SessionId::new(); + assert!(matches!( + ctx.meta.teleport_admit(missing_session).await.unwrap(), + TeleportAdmitOutcome::Fenced + )); + let mut stale = request.clone(); + stale.epoch += 1; + assert!(matches!( + ctx.meta.teleport_admit(stale).await.unwrap(), + TeleportAdmitOutcome::Fenced + )); + assert!(ctx + .meta + .request_host_retirement( + dest, + engram_core::types::host::CordonOwner::Operator, + "retire", + ctx.clock.now_utc() + ) + .await + .unwrap()); + ctx.clock.advance(Duration::from_secs(1)); + ctx.meta + .touch_host_heartbeat(dest, heartbeat_fixture()) + .await + .unwrap(); + assert_eq!( + ctx.meta + .grant_host_retirement(dest, ctx.clock.now_utc()) + .await + .unwrap(), + engram_core::types::host::RetirementGrant::Granted + ); + request.pinned_dest = Some(dest); + assert!(matches!( + ctx.meta.teleport_admit(request.clone()).await.unwrap(), + TeleportAdmitOutcome::NoFit + )); + ctx.meta + .transition_session( + request.session_id, + SessionState::Evicting, + BindingDisposition::Retain, + ) + .await + .unwrap(); + assert!(matches!( + ctx.meta.teleport_admit(request.clone()).await.unwrap(), + TeleportAdmitOutcome::SessionNotActive(SessionState::Evicting) + )); + request.epoch += 1; + assert!(matches!( + ctx.meta.teleport_admit(request).await.unwrap(), + TeleportAdmitOutcome::Fenced + )); + let mut open = teleport_admission_fixture(ctx).await; + admit_teleport(ctx, open.clone()).await; + open.pinned_dest = Some(HostId::new()); + assert!(matches!( + ctx.meta.teleport_admit(open).await, + Err(MetaError::Conflict(_)) + )); +} + +conformance!( + t_teleport_and_ordinary_policy_child_do_not_deadlock, + teleport_and_ordinary_policy_child_do_not_deadlock +); +conformance!( + t_teleport_destination_refusals_preserve_session_precedence, + teleport_destination_refusals_preserve_session_precedence +); + +conformance!( + t_teleport_commit_and_ordinary_policy_child_do_not_deadlock, + teleport_commit_and_ordinary_policy_child_do_not_deadlock +); diff --git a/crates/engram-testkit/Cargo.toml b/crates/engram-testkit/Cargo.toml index 418054be7..156416a18 100644 --- a/crates/engram-testkit/Cargo.toml +++ b/crates/engram-testkit/Cargo.toml @@ -16,6 +16,7 @@ async-trait.workspace = true bytes.workspace = true futures.workspace = true sha2.workspace = true +serde_json.workspace = true sqlx.workspace = true uuid.workspace = true workspace-hack = { version = "0.1", path = "../../workspace-hack" } diff --git a/crates/engram-testkit/src/lib.rs b/crates/engram-testkit/src/lib.rs index 5399388cb..ffb3332fa 100644 --- a/crates/engram-testkit/src/lib.rs +++ b/crates/engram-testkit/src/lib.rs @@ -8,3 +8,5 @@ pub mod blob_conformance; pub mod net; pub mod pg; pub mod storage; + +pub mod operator; diff --git a/crates/engram-testkit/src/operator.rs b/crates/engram-testkit/src/operator.rs new file mode 100644 index 000000000..56d27308a --- /dev/null +++ b/crates/engram-testkit/src/operator.rs @@ -0,0 +1,30 @@ +//! Reusable operator publication fixture. It uses the real store transitions. +use engram_core::{ + traits::MetadataStore, + types::{managed_create::digest, managed_operator::*}, +}; +pub fn candidate() -> OperatorCandidate { + let d = digest(b"configured operator fixture"); + OperatorCandidate { version:1,zones:[("zone-a".into(),OperatorZone { + revision:"1".into(),worker_digest:d.clone(),authorities:vec!["tools.example".into()],aliases:Vec::new(), + configuration:serde_json::json!({"id":"zone-a","revision":"1","mode":"public","authorities":["tools.example"],"aliases":[],"cidrs":[],"denyCidrs":[]}), + })].into(),deployment_identity:d.clone(),network_identity:d.clone(),trust_identity:d.clone(),ca_content_digests:[("fixture-root".into(),d)].into() } +} +pub async fn publish_empty(meta: &dyn MetadataStore) { + if meta + .read_operator_publication() + .await + .unwrap() + .published + .is_some() + { + return; + } + let staged = meta + .stage_operator_publication(None, candidate()) + .await + .unwrap(); + meta.publish_operator_generation(&staged.pending.unwrap()) + .await + .unwrap(); +} diff --git a/crates/engram-tui/src/net.rs b/crates/engram-tui/src/net.rs index ac629cdf3..f686e59a7 100644 --- a/crates/engram-tui/src/net.rs +++ b/crates/engram-tui/src/net.rs @@ -317,6 +317,7 @@ impl Net { tokio::spawn(async move { let res = c .complete_tool_call(CompleteToolCallRequest { + expected_mcp_plan_review: None, session_id: session_id.clone(), tool_call_id: tool_call_id.clone(), result_json, diff --git a/deploy/helm/engram/MCP-CALLBACK-INGRESS.md b/deploy/helm/engram/MCP-CALLBACK-INGRESS.md new file mode 100644 index 000000000..f7f09c3fd --- /dev/null +++ b/deploy/helm/engram/MCP-CALLBACK-INGRESS.md @@ -0,0 +1,110 @@ +# MCP callback ingress protection + +Keep external MCP disabled until the deployment checks below pass. An OAuth +redirect sends a GET to `/integrations/mcp/callback` with transient `code`, +`state`, and `iss` query values. The browser removes the query before it loads +the application. That does not protect the initial request at a proxy. + +The completion request is an authenticated POST to +`/rpc/engram.app.v1.IntegrationService/CompleteMcpAuthorization`. Same-origin +deployments send it through web nginx. Split-host deployments send it directly +to the API host. Both paths must exclude request bodies, queries, and referrers +from logs, traces, error reports, and request capture. + +## Checked-in controls + +The web nginx server logs method, original path without query, status, duration, +and response size as JSON. The server-wide format applies after SPA internal +redirects and to `/rpc/` and `/api/`. Health requests do not create access records. +There is no body or header field in this format. nginx logs in the location +where request processing ends, so a callback-location override alone is not +sufficient. See the [nginx access-log documentation](https://nginx.org/en/docs/http/ngx_http_log_module.html). + +HTTP error diagnostics are sent to `/dev/null`. nginx error messages can include +the complete request and referrer; raising the severity threshold does not +redact them. This removes HTTP error detail for all web routes. Status and +timing remain in access logs, and main-process startup diagnostics remain. +Do not enable debug logging or add another request-bearing log destination. +See [nginx error-log contexts](https://nginx.org/en/docs/ngx_core_module.html#error_log). + +The shipped public ingress profiles explicitly disable unsafe access logging: + +| Profile | Configuration | +| --- | --- | +| GCP quickstart and manual GCP example | `engram-web` BackendConfig has `spec.logging.enable: false`. | +| GCP split-host IAP | Both `engram-web-iap` and `engram-api-no-iap` BackendConfigs have `spec.logging.enable: false`. The rendered web and webhook Services reference these names. | +| AWS ALB | `alb.ingress.kubernetes.io/load-balancer-attributes` includes `access_logs.s3.enabled=false` and retains the idle timeout. | + +GKE supports the logging field from version `1.16.8-gke.10`. Omitting it leaves +version-dependent behavior. See [GKE HTTP access logging](https://docs.cloud.google.com/kubernetes-engine/docs/how-to/ingress-configuration#http_access_logging). +The AWS controller updates only specified attributes; an explicit false is +required to disable previously enabled logs. See [AWS Load Balancer Controller +attributes](https://kubernetes-sigs.github.io/aws-load-balancer-controller/latest/guide/ingress/annotations/#custom-attributes). +These controls disable the backend or ALB access log as a whole. They do not +provide field redaction or control independent security, CDN, or application logs. + +## Local canary and CI + +Prerequisites are Python 3.9 or later, Ruby with YAML, Helm, and a working Docker +daemon. Pull the official nginx tag specified in `docker/web.Dockerfile`, then +run from the repository root: + +```sh +python3 -m unittest discover -s deploy/helm/engram/ci/tests -p 'test_callback_profiles.py' +python3 deploy/helm/engram/ci/callback-ingress-canary.py --stage var/mcp-callback-canary +``` + +Use a new stage directory for each run. It must be a real directory under this +checkout's `var/mcp-*`. The script does not install or pull a runtime. Missing +prerequisites, no served requests, invalid configuration, unexpected responses, +and missing access records fail the check. The inspected image must be at most +100 MiB. Its exact ID is used for all runs and its digest is saved in `image.json`. + +The script renders the chart, runs the official entrypoint and envsubst filter, +and saves `nginx -T`, container logs, and cleanup records. It starts three serial +containers: same-origin web, split-host web, and a deliberately unsafe negative +control. Each has no network or published ports, one CPU, 128 MiB memory, +32 processes, a read-only root, and bounded writable tmpfs mounts. Each run has +a 90-second deadline and removes its own container. Only synthetic values are used. + +The requests cover callback serving, SPA internal fallback, POST proxying, +duplicate-Host rejection, a missing-index redirect cycle, and a closed upstream. +The safe cases require useful access records without the canary query, +referrer, or body. The negative control must expose canaries in both access and +error logs. Its upstream only supplies HTTP fixture responses; it does not +implement authentication or OAuth. The split-host case checks the web server +and runtime API origin, not the external API ingress or its application logs. + +The existing `deploy-config` CI lane runs the profile tests and canary. Chart, +Terraform, canary, workflow, detector, and web-image changes select that lane. +It remains a dependency of `CI Gate`. No extra required status is needed. + +## Deployment qualification + +The local canary proves behavior of the rendered nginx configuration and image. +Rendered cloud objects prove configuration intent. Neither proves that a cloud +controller applied the settings, nor that a deployment's entire log path is safe. + +Before enabling external MCP in a deployment: + +1. Identify the first public ingress for the callback host and the API host. + Include any CDN, WAF, identity proxy, load balancer, ingress controller, + service mesh, request recorder, trace exporter, and log collector. +2. Check the effective cloud backend or ALB logging attributes, Service bindings, + nginx configuration, and downstream capture policies. Disable capture of the + query, referrer, and POST body at each source before those fields reach a + collector. A collector filter alone does not protect upstream copies. +3. Send neutral, unique canary values through the real GET and POST paths. + Include success, rejected requests, internal fallback, and upstream errors + in an isolated qualification environment. Use metadata or counters to prove + requests arrived. After the documented ingestion delay, inspect every + configured log, trace, and error sink for those values. Keep the observed + request counts, effective configuration, sink coverage, and result. +4. Establish the negative control only in an isolated validation environment. + Never enable unsafe logging on a real credential flow. Requalify after an + ingress, runtime, collector, or logging-policy change. + +An unverified first ingress or collector keeps the feature disabled. An +administrator's MCP zone selection cannot override operator logging or network +policy. These checks are separate from real Firecracker acceptance, VZ parity, +enforcing CNI checks, OAuth/provider checks, and the complete application suites. diff --git a/deploy/helm/engram/README.md b/deploy/helm/engram/README.md index be7f53d95..f277ab900 100644 --- a/deploy/helm/engram/README.md +++ b/deploy/helm/engram/README.md @@ -5,6 +5,10 @@ frontend (default on) to any K8s cluster. The chart is cloud-agnostic — cloud-specific bindings (Workload Identity, KMS, LB annotations) live in the `values-*.yaml.example` overlays. +Before enabling external MCP OAuth, qualify the deployment's complete logging +path. See [callback ingress protection](MCP-CALLBACK-INGRESS.md) for the chart's +controls, local canary, and required checks at the public ingress. + ## What this chart deploys **Coordinator (always):** diff --git a/deploy/helm/engram/ci/callback-ingress-canary.py b/deploy/helm/engram/ci/callback-ingress-canary.py new file mode 100644 index 000000000..0b8ec2b2e --- /dev/null +++ b/deploy/helm/engram/ci/callback-ingress-canary.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Exercise the rendered nginx configuration with synthetic callback values. + +Requires Helm, Docker, and the image used by docker/web.Dockerfile already pulled. +No published ports or container network are needed. This does not test a cloud +load balancer, an authenticated RPC, or an external log collector. +""" + +import argparse +import json +from pathlib import Path +import re +import signal +import subprocess +import time +import uuid + + +ROOT = Path(__file__).resolve().parents[4] +CHART = ROOT / 'deploy/helm/engram' +CALLBACK = '/integrations/mcp/callback' +RPC = '/rpc/engram.app.v1.IntegrationService/CompleteMcpAuthorization' +TOKENS = ['canary-code-8b', 'canary-state-8b', 'canary-issuer-8b', + 'canary-referrer-8b', 'canary-body-8b'] +QUERY = '?code=' + TOKENS[0] + '&state=' + TOKENS[1] + '&iss=' + TOKENS[2] +SHELL = 'callback-canary-shell' + + +def run(*args, timeout=15, check=True): + return subprocess.run(args, text=True, capture_output=True, check=check, timeout=timeout) + + +def require(condition, message): + if not condition: + raise AssertionError(message) + + +def render(split): + args = ['helm', 'template', 'callback-canary', str(CHART), '--namespace', 'canary', + '-f', str(CHART / 'ci/render-values.yaml'), '--show-only', + 'templates/web-configmap.yaml', '--set', 'orchestrator.service.port=18081'] + if split: + args += ['--set', 'web.apiBaseUrl=https://api.canary.example'] + rendered = run(*args, timeout=30).stdout + block = rendered.split(' default.conf.template: |\n', 1)[1] + require(all(not line or line.startswith(' ') for line in block.splitlines()), + 'Unexpected ConfigMap block; cannot extract the nginx configuration') + return rendered, '\n'.join(line[4:] for line in block.splitlines()) + '\n' + + +def exercise(stage, image, split=False, unsafe=False): + """Run one bounded container. The fixture upstream is not an RPC implementation.""" + stage.mkdir() + config_dir = stage / 'config' + assets = stage / 'assets' + config_dir.mkdir() + assets.mkdir() + index = assets / 'index.html' + index.write_text(SHELL) + rendered, config = render(split) + (stage / 'rendered.yaml').write_text(rendered) + if unsafe: + # Deliberate negative control: reinstate request-bearing access/error logs + # at the server level, without changing request routing or test traffic. + config = config.replace('server_name _;', 'server_name _;\n' + ' access_log /dev/stdout combined;\n' + ' error_log /dev/stderr info;', 1) + require('access_log /dev/stdout combined;' in config, 'Negative control was not installed') + (config_dir / 'default.conf.template').write_text(config) + # Only the upstream is a fixture. The chart's server and log directives are + # used without path, port, filesystem, or logging rewrites. + (config_dir / 'upstream.conf.template').write_text('''server { + listen 18081; + access_log off; + error_log /dev/null; + location / { return 200 "callback-canary-upstream"; } + location = /rpc/closed { return 444; } +} +''') + name = 'mcp-callback-canary-' + uuid.uuid4().hex[:12] + manifest = {'name': name, 'image': image, 'split': split, 'negative_control': unsafe, + 'container_id': None, 'removed': False} + manifest_path = stage / 'manifest.json' + manifest_path.write_text(json.dumps(manifest, indent=2) + '\n') + container = None + started = time.monotonic() + count = 0 + + def request(path, status, body='', malformed=False): + nonlocal count + method = 'POST' if body else 'GET' + if body: + result = run('docker', 'exec', container, 'wget', '-S', '-O-', '-T', '2', + '--header', f'Referer: https://referrer.canary.example/{TOKENS[3]}', + '--header', 'Content-Type: application/json', '--post-data', body, + 'http://127.0.0.1:8080' + path, timeout=5, check=False) + require(f'HTTP/1.1 {status} ' in result.stderr, + f'Unexpected POST status: {result.stderr}') + count += 1 + return result.stdout + data = (f'{method} {path} HTTP/1.1\r\nHost: canary.example\r\n' + f'Referer: https://referrer.canary.example/{TOKENS[3]}\r\n' + 'Connection: close\r\n') + if malformed: + data += 'Host: duplicate.canary.example\r\n' + data += '\r\n' + body + result = run('docker', 'exec', container, 'sh', '-c', + 'printf "%s" "$1" | nc -w 2 127.0.0.1 8080', 'canary', data, timeout=5) + require(result.stdout.startswith(f'HTTP/1.1 {status} '), + f'Unexpected response for {method} {path.split("?")[0]}: {result.stdout[:80]}') + count += 1 + return result.stdout + + try: + args = ['docker', 'run', '--detach', '--pull=never', '--name', name, + '--label', 'engram.test=callback-ingress', '--network', 'none', + '--cpus', '1', '--memory', '128m', '--pids-limit', '32', + '--read-only', '--user', '101:101', '--cap-drop', 'ALL', + '--security-opt', 'no-new-privileges', + '--tmpfs', '/etc/nginx/conf.d:uid=101,gid=101,size=1m', + '--tmpfs', '/var/cache/nginx:uid=101,gid=101,size=8m', + '--tmpfs', '/var/run:uid=101,gid=101,size=1m', + '--tmpfs', '/tmp:uid=101,gid=101,size=1m', + '--add-host', 'callback-canary-orchestrator.canary.svc.cluster.local:127.0.0.1', + '--mount', f'type=bind,source={config_dir},target=/etc/nginx/templates,readonly', + '--mount', f'type=bind,source={assets},target=/usr/share/nginx/html,readonly', + '--env', 'NGINX_ENVSUBST_FILTER=^ENGRAM_', image] + container = run(*args, timeout=20).stdout.strip() + manifest['container_id'] = container + manifest_path.write_text(json.dumps(manifest, indent=2) + '\n') + for _ in range(30): + ready = run('docker', 'exec', container, 'wget', '-qO-', + 'http://127.0.0.1:8080/healthz', timeout=3, check=False) + if ready.returncode == 0 and ready.stdout == 'ok\n': + break + time.sleep(0.1) + else: + raise AssertionError('nginx did not become ready') + effective = run('docker', 'exec', container, 'nginx', '-T') + (stage / 'effective.conf').write_text(effective.stdout) + (stage / 'config-check.log').write_text(effective.stderr) + require(SHELL in request(CALLBACK + QUERY, 200), 'Callback did not serve the SPA') + require(SHELL in request('/missing-asset' + QUERY, 200), 'SPA fallback did not run') + request(RPC + QUERY, 200, json.dumps({'code': TOKENS[4]})) + request('/rpc/closed' + QUERY, 502, json.dumps({'code': TOKENS[4]})) + request(CALLBACK + QUERY, 400, malformed=True) + index.unlink() + request(CALLBACK + QUERY, 500) + index.write_text(SHELL) + if split: + require('https://api.canary.example' in request('/engram-config.js', 200), + 'Split-host runtime configuration was not served') + require(count >= 6, 'No positive request count') + run('docker', 'stop', '--time', '3', container, timeout=8) + logs = run('docker', 'logs', container) + (stage / 'stdout.log').write_text(logs.stdout) + (stage / 'stderr.log').write_text(logs.stderr) + if unsafe: + require(all(token in logs.stdout for token in TOKENS[:4]), + 'Access-log negative control did not expose the canary query and referrer') + require(TOKENS[0] in logs.stderr and TOKENS[3] in logs.stderr, + 'Error-log negative control did not expose the canary query and referrer') + else: + for token in TOKENS: + require(token not in logs.stdout + logs.stderr, f'Canary value leaked: {token}') + records = [json.loads(line) for line in logs.stdout.splitlines() if line.startswith('{')] + require(len(records) == count, f'Expected {count} access records, got {len(records)}') + expected = [('GET', CALLBACK, 200), ('GET', '/missing-asset', 200), + ('POST', RPC, 200), ('POST', '/rpc/closed', 502), + ('GET', CALLBACK, 400), ('GET', CALLBACK, 500)] + if split: + expected.append(('GET', '/engram-config.js', 200)) + require([(r['method'], r['path'], r['status']) for r in records] == expected, + 'Request metadata did not survive fallback or error handling') + require(all(r['duration'] >= 0 and r['bytes'] > 0 for r in records), + 'Timing and size metadata is missing') + print(f'{stage.name}: {count} requests, ' + f'{"negative control detected leaks" if unsafe else "no canary in nginx logs"}') + finally: + if container: + # Capture startup failures too. Remove only the ID this invocation created. + logs = run('docker', 'logs', container, check=False) + (stage / 'stdout.log').write_text(logs.stdout) + (stage / 'stderr.log').write_text(logs.stderr) + run('docker', 'rm', '--force', container) + require(run('docker', 'container', 'inspect', container, check=False).returncode != 0, + 'Owned container still exists after cleanup') + manifest['removed'] = True + manifest['requests'] = count + manifest['elapsed_seconds'] = round(time.monotonic() - started, 3) + manifest_path.write_text(json.dumps(manifest, indent=2) + '\n') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--stage', required=True, type=Path, help='New output directory under var/mcp-*') + args = parser.parse_args() + stage = args.stage.absolute() + require(stage == stage.resolve(), 'Stage must not use symlinks') + require(stage.is_relative_to(ROOT / 'var') and stage.relative_to(ROOT / 'var').parts[0].startswith('mcp-'), + 'Stage must be under this checkout var/mcp-*') + stage.mkdir(parents=True, exist_ok=False) + image, = re.findall(r'^FROM (nginx:\S+)$', (ROOT / 'docker/web.Dockerfile').read_text(), re.M) + inspected = run('docker', 'image', 'inspect', image) + (stage / 'image.json').write_text(inspected.stdout) + metadata, = json.loads(inspected.stdout) + require(metadata['Size'] <= 100 * 1024 * 1024, 'Canary image exceeds 100 MiB') + # Use the exact locally inspected image ID in every run, even if the tag moves. + def deadline(_signal, _frame): + raise TimeoutError('Canary exceeded its 90 second bound') + signal.signal(signal.SIGALRM, deadline) + for name, split, unsafe in [('same-origin', False, False), ('split-host', True, False), + ('negative-control', False, True)]: + signal.alarm(90) + try: + exercise(stage / name, metadata['Id'], split, unsafe) + finally: + signal.alarm(0) + + +if __name__ == '__main__': + main() diff --git a/deploy/helm/engram/ci/tests/test_callback_profiles.py b/deploy/helm/engram/ci/tests/test_callback_profiles.py new file mode 100644 index 000000000..b34eb87b4 --- /dev/null +++ b/deploy/helm/engram/ci/tests/test_callback_profiles.py @@ -0,0 +1,74 @@ +"""Check the rendered public ingress profiles, without a cloud account.""" + +import json +from pathlib import Path +import re +import subprocess +import unittest + + +ROOT = Path(__file__).resolve().parents[5] +CHART = ROOT / 'deploy/helm/engram' + + +def yaml_documents(source): + result = subprocess.run( + ['ruby', '-ryaml', '-rjson', '-e', + 'puts JSON.generate(YAML.load_stream(STDIN.read))'], + input=source, text=True, capture_output=True, check=True, timeout=10, + ) + return json.loads(result.stdout) + + +def render(overlay): + source = re.sub(r'REPLACE_[A-Z_]+', 'stub', (CHART / overlay).read_text()) + result = subprocess.run( + ['helm', 'template', 'callback-canary', str(CHART), '-f', + str(CHART / 'ci/render-values.yaml'), '-f', '-'], + input=source, text=True, capture_output=True, check=True, timeout=30, + ) + return yaml_documents(result.stdout) + + +class CallbackProfiles(unittest.TestCase): + def test_aws_disables_access_logging_in_rendered_ingress(self): + ingresses = [d for d in render('values-aws.yaml.example') + if d and d['kind'] == 'Ingress'] + self.assertGreater(len(ingresses), 0) + for ingress in ingresses: + attributes = dict(item.split('=', 1) for item in ingress['metadata'] + ['annotations']['alb.ingress.kubernetes.io/load-balancer-attributes'].split(',')) + self.assertEqual(attributes.get('access_logs.s3.enabled'), 'false') + self.assertEqual(attributes['idle_timeout.timeout_seconds'], '3600') + + def test_gcp_quickstart_disables_access_logging_on_referenced_backend(self): + source = (ROOT / 'deploy/terraform/gcp/quickstart/eso.tf').read_text() + block = source.split('resource "kubectl_manifest" "web_backendconfig" {', 1)[1] + manifest = block.split('<<-YAML\n', 1)[1].split('\n YAML', 1)[0] + backend, = yaml_documents(manifest) + services = [d for d in render('values-gcp.yaml.example') + if d and d['kind'] == 'Service' and + 'cloud.google.com/backend-config' in d['metadata'].get('annotations', {})] + self.assertGreater(len(services), 0) + for service in services: + reference = json.loads(service['metadata']['annotations']['cloud.google.com/backend-config']) + self.assertEqual(reference['default'], backend['metadata']['name']) + self.assertIs(backend['spec'].get('logging', {}).get('enable'), False) + + def test_gcp_split_host_disables_logging_for_web_and_api(self): + backends = {d['metadata']['name']: d for d in yaml_documents( + (CHART / 'examples/backendconfig-api-no-iap.yaml').read_text())} + services = [d for d in render('values-iap.yaml.example') + if d and d['kind'] == 'Service' and + 'cloud.google.com/backend-config' in d['metadata'].get('annotations', {})] + referenced = set() + for service in services: + reference = json.loads(service['metadata']['annotations']['cloud.google.com/backend-config']) + for name in reference.values(): + referenced.add(name) + self.assertIs(backends[name]['spec'].get('logging', {}).get('enable'), False) + self.assertEqual(referenced, {'engram-web-iap', 'engram-api-no-iap'}) + + +if __name__ == '__main__': + unittest.main() diff --git a/deploy/helm/engram/examples/backendconfig-api-no-iap.yaml b/deploy/helm/engram/examples/backendconfig-api-no-iap.yaml index 51065a517..c8c911d52 100644 --- a/deploy/helm/engram/examples/backendconfig-api-no-iap.yaml +++ b/deploy/helm/engram/examples/backendconfig-api-no-iap.yaml @@ -30,6 +30,9 @@ kind: BackendConfig metadata: name: engram-web-iap spec: + # Protect the initial callback query and referrer at the public backend. + logging: + enable: false iap: enabled: true --- @@ -38,6 +41,10 @@ kind: BackendConfig metadata: name: engram-api-no-iap spec: + # The split API host receives transient OAuth values in POST bodies. + # Body/trace capture must also be disabled in external middleware. + logging: + enable: false iap: enabled: false timeoutSec: 3600 diff --git a/deploy/helm/engram/templates/mcp-worker.yaml b/deploy/helm/engram/templates/mcp-worker.yaml new file mode 100644 index 000000000..2948a34a6 --- /dev/null +++ b/deploy/helm/engram/templates/mcp-worker.yaml @@ -0,0 +1,176 @@ +{{- if .Values.mcpWorker.enabled }} +{{- range $zone := .Values.mcpWorker.zones }} +{{- if not $zone.isolationConfirmed }}{{ fail "MCP worker namespace isolation must be verified" }}{{- end }} +{{- $name := printf "%s-mcp-%s" (include "engram.fullname" $) $zone.policy.id | trunc 63 | trimSuffix "-" }} +{{- $namespace := required "MCP worker requires a dedicated namespace" $zone.namespace }} +{{- if eq $namespace $.Release.Namespace }}{{ fail "MCP worker namespace must differ from the application namespace" }}{{- end }} +{{- $tls := required "MCP worker TLS secret is required" $zone.tlsSecret }} +{{- $rules := $zone.egressRules }} +{{- if eq $zone.policy.mode "public" }} +{{- $deny4 := list "0.0.0.0/8" "10.0.0.0/8" "100.64.0.0/10" "127.0.0.0/8" "169.254.0.0/16" "172.16.0.0/12" "192.0.0.0/24" "192.0.2.0/24" "192.88.99.0/24" "192.168.0.0/16" "198.18.0.0/15" "198.51.100.0/24" "203.0.113.0/24" "224.0.0.0/4" "240.0.0.0/4" "168.63.129.16/32" }} +{{- $deny6 := list "2001::/23" "2001:db8::/32" "2002::/16" "3fff::/20" }} +{{- range $deny := $zone.policy.denyCidrs }} +{{- if not (contains ":" $deny) }}{{- $deny4 = append $deny4 $deny }} +{{- else if regexMatch "^[23]" $deny }}{{- $deny6 = append $deny6 $deny }}{{- end }} +{{- end }} +{{- $rules = list (dict "cidr" "0.0.0.0/0" "except" ($deny4 | uniq)) (dict "cidr" "2000::/3" "except" ($deny6 | uniq)) }} +{{- end }} +{{- if not $rules }}{{ fail "MCP worker egress rules are required" }}{{- end }} +{{- $callers := required "MCP worker caller certificate pins are required" $zone.callers }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ $namespace }} +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }} + namespace: {{ $namespace }} +data: + config.json: {{ dict "zone" $zone.policy "callers" $callers "egressRules" $rules | toJson | quote }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $name }} + namespace: {{ $namespace }} +spec: + replicas: {{ default 1 $zone.replicas }} + selector: + matchLabels: + app.kubernetes.io/instance: {{ $.Release.Name }} + app.kubernetes.io/component: mcp-worker + engrams.io/mcp-zone: {{ $zone.policy.id }} + template: + metadata: + labels: + app.kubernetes.io/instance: {{ $.Release.Name }} + app.kubernetes.io/component: mcp-worker + engrams.io/mcp-zone: {{ $zone.policy.id }} + annotations: + checksum/zone-policy: {{ dict "zone" $zone.policy "callers" $callers "egressRules" $rules | toJson | sha256sum }} + spec: + serviceAccountName: {{ $name }} + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: worker + image: "{{ $.Values.mcpWorker.image.repository }}:{{ default $.Chart.AppVersion $.Values.mcpWorker.image.tag }}" + imagePullPolicy: {{ $.Values.mcpWorker.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + env: + - name: MCP_WORKER_ENABLED + value: {{ default false $zone.dispatchEnabled | quote }} + ports: + - name: rpc + containerPort: 8443 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "2" + memory: 1Gi + volumeMounts: + - name: config + mountPath: /etc/mcp-worker + readOnly: true + - name: tls + mountPath: /etc/mcp-worker/tls + readOnly: true + volumes: + - name: config + configMap: + name: {{ $name }} + - name: tls + secret: + secretName: {{ $tls }} + defaultMode: 0440 +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ $namespace }} +spec: + type: ClusterIP + selector: + app.kubernetes.io/instance: {{ $.Release.Name }} + app.kubernetes.io/component: mcp-worker + engrams.io/mcp-zone: {{ $zone.policy.id }} + ports: + - name: rpc + port: 8443 + targetPort: rpc +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $name }} + namespace: {{ $namespace }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: {{ $.Release.Name }} + app.kubernetes.io/component: mcp-worker + engrams.io/mcp-zone: {{ $zone.policy.id }} + policyTypes: [Ingress, Egress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ $.Release.Namespace }} + podSelector: + matchLabels: + app.kubernetes.io/instance: {{ $.Release.Name }} + matchExpressions: + - key: app.kubernetes.io/component + operator: In + values: [coordinator, orchestrator] + ports: + - protocol: TCP + port: 8443 + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ required "DNS namespace is required" $zone.dns.namespace }} + podSelector: + matchLabels: + {{- required "DNS pod selector is required" $zone.dns.podLabels | toYaml | nindent 14 }} + ports: + - protocol: UDP + port: 53 + - protocol: TCP + port: 53 + - to: + {{- range $rule := $rules }} + - ipBlock: + cidr: {{ $rule.cidr }} + {{- with $rule.except }} + except: + {{- toYaml . | nindent 14 }} + {{- end }} + {{- end }} + ports: + {{- range $port := required "Approved HTTPS ports are required" $zone.ports }} + - protocol: TCP + port: {{ $port }} + {{- end }} +{{- end }} +{{- end }} diff --git a/deploy/helm/engram/templates/web-configmap.yaml b/deploy/helm/engram/templates/web-configmap.yaml index 3cdc92b63..59b1e1a95 100644 --- a/deploy/helm/engram/templates/web-configmap.yaml +++ b/deploy/helm/engram/templates/web-configmap.yaml @@ -16,6 +16,22 @@ metadata: {{- include "engram.web.labels" . | nindent 4 }} data: default.conf.template: | + # The initial OAuth callback GET contains transient credentials. Keep + # only the original path, including after the SPA's internal redirect. + # Never include request, args, headers, referrer, or body in this format. + map $request_uri $engram_log_path { + ~^(?[^?]*) $engram_request_path; + default ""; + } + log_format engram_safe escape=json + '{"method":"$request_method","path":"$engram_log_path",' + '"status":$status,"duration":$request_time,"bytes":$body_bytes_sent}'; + + # nginx request error messages can contain the full request and referrer. + # They have no field-level redaction. Suppress HTTP error diagnostics; + # status/timing remain in access logs and main-process diagnostics remain. + error_log /dev/null; + # Sets `Connection: upgrade` only when the client requested an # Upgrade; otherwise leaves it alone so keep-alive works for # plain HTTP. Standard nginx WS-proxy idiom. Must live at http{} @@ -44,6 +60,7 @@ data: server_name _; root /usr/share/nginx/html; index index.html; + access_log /dev/stdout engram_safe; # Health probe target. Cheap and auth-free; the readiness # probe hits this before the pod joins the Service. diff --git a/deploy/helm/engram/values-aws.yaml.example b/deploy/helm/engram/values-aws.yaml.example index 3dafa6e7d..b60b88871 100644 --- a/deploy/helm/engram/values-aws.yaml.example +++ b/deploy/helm/engram/values-aws.yaml.example @@ -142,7 +142,9 @@ web: # IDE, /events); 3600 matches the nginx proxy_read_timeout and # the GCP deployment's socket-lifetime fix — the 60 s default # would cut every quiet session leg. - alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=3600 + # Explicit false also disables access logs on an existing ALB. Omitting + # the attribute does not undo an earlier logging configuration. + alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=3600,access_logs.s3.enabled=false alb.ingress.kubernetes.io/healthcheck-path: /healthz hosts: - host: engrams.REPLACE_DOMAIN diff --git a/deploy/helm/engram/values-gcp.yaml.example b/deploy/helm/engram/values-gcp.yaml.example index 78568291a..4d79df8c5 100644 --- a/deploy/helm/engram/values-gcp.yaml.example +++ b/deploy/helm/engram/values-gcp.yaml.example @@ -142,7 +142,9 @@ web: # apiVersion: cloud.google.com/v1 # kind: BackendConfig # metadata: { name: engram-web } - # spec: { timeoutSec: 3600 } + # spec: { timeoutSec: 3600, logging: { enable: false } } + # + # Explicitly disable query-bearing access logs for OAuth callbacks. # # Allow ~10 minutes after applying for the load balancer to # program before judging WebSockets. (values-iap.yaml.example diff --git a/deploy/helm/engram/values-iap.yaml.example b/deploy/helm/engram/values-iap.yaml.example index 93223f64c..98e3e884f 100644 --- a/deploy/helm/engram/values-iap.yaml.example +++ b/deploy/helm/engram/values-iap.yaml.example @@ -76,6 +76,7 @@ # kind: BackendConfig # metadata: { name: engram-web-iap } # spec: +# logging: { enable: false } # iap: { enabled: true } # # 300s is fine here — no sockets or streams ride this host. # --- @@ -83,6 +84,7 @@ # kind: BackendConfig # metadata: { name: engram-api-no-iap } # spec: +# logging: { enable: false } # iap: { enabled: false } # # A backend timeout caps a SOCKET'S LIFETIME, not idleness: # # the default 30s (or the web host's 300s) would cut every diff --git a/deploy/helm/engram/values-mcp-worker.yaml.example b/deploy/helm/engram/values-mcp-worker.yaml.example new file mode 100644 index 000000000..c47982ba2 --- /dev/null +++ b/deploy/helm/engram/values-mcp-worker.yaml.example @@ -0,0 +1,39 @@ +# This example is disabled. Create the namespace and TLS Secret separately. +# The Secret contains tls.key, tls.crt, and the trusted caller CA in ca.crt. +# Set real SHA-256 client certificate fingerprints. Give the coordinator token +# access and the orchestrator tool access with different client certificates. +mcpWorker: + enabled: false + zones: + - namespace: mcp-tools + isolationConfirmed: false + dispatchEnabled: false + replicas: 1 + tlsSecret: mcp-worker-rpc-tls + callers: + - fingerprint: "REPLACE_WITH_ORCHESTRATOR_CERTIFICATE_SHA256" + role: orchestrator + - fingerprint: "REPLACE_WITH_COORDINATOR_CERTIFICATE_SHA256" + role: coordinator + policy: + id: private-tools + revision: "1" + mode: internal + authorities: [tools.example] + aliases: [] + cidrs: [10.20.0.0/16] + # Include every platform administration network, including API servers. + denyCidrs: [10.20.0.0/24] + # Optional private provider CA PEM. This is not the RPC caller CA. + # ca: | + # -----BEGIN CERTIFICATE----- + # ... + limits: {execute: 32, discover: 2, oauth: 4, schema: 4, toolMs: 60000} + egressRules: + - cidr: 10.20.0.0/16 + except: [10.20.0.0/24] + ports: [443] + dns: + namespace: kube-system + podLabels: + k8s-app: kube-dns diff --git a/deploy/helm/engram/values.yaml b/deploy/helm/engram/values.yaml index 2c3edae8c..216f99697 100644 --- a/deploy/helm/engram/values.yaml +++ b/deploy/helm/engram/values.yaml @@ -688,3 +688,15 @@ orchestrator: nodeSelector: {} tolerations: [] affinity: {} + +# Isolated external MCP workers. Only operators can edit these zone policies. +# Provision a separate namespace with enforced NetworkPolicy before enabling. +# Namespace policies are additive: no other policy may grant broader egress. +mcpWorker: + enabled: false + image: + repository: ghcr.io/cortexapps/engrams/mcp-worker + tag: "" + pullPolicy: IfNotPresent + zones: [] + # See values-mcp-worker.yaml.example for a complete, disabled zone. diff --git a/deploy/mcp-control.example.json b/deploy/mcp-control.example.json new file mode 100644 index 000000000..f38e568de --- /dev/null +++ b/deploy/mcp-control.example.json @@ -0,0 +1,65 @@ +{ + "registry": { + "enabled": false, + "deploymentId": "deployment-a", + "zones": { + "public": { + "policyDigest": "0000000000000000000000000000000000000000000000000000000000000000", + "policyRevision": "1", + "endpoints": [ + "https://tools.example/mcp", + "https://tools.example/.well-known/oauth-protected-resource", + "https://auth.example/.well-known/oauth-authorization-server", + "https://auth.example/authorize", + "https://auth.example/token" + ] + } + }, + "registrations": { + "registered-public": { + "id": "registered-public", + "clientId": "registered-client-id", + "confidential": false, + "redirectUri": "https://app.example/integrations/mcp/callback", + "issuer": "https://auth.example/", + "resource": { + "zoneId": "public", + "endpoint": "https://tools.example/mcp" + }, + "resourceMetadata": { + "zoneId": "public", + "endpoint": "https://tools.example/.well-known/oauth-protected-resource" + }, + "authorizationMetadata": { + "zoneId": "public", + "endpoint": "https://auth.example/.well-known/oauth-authorization-server" + }, + "authorization": { + "zoneId": "public", + "endpoint": "https://auth.example/authorize" + }, + "token": { + "zoneId": "public", + "endpoint": "https://auth.example/token" + }, + "scopes": ["tools.read"], + "issuerParameterRequired": true, + "accountField": null + } + } + }, + "listenAddr": "0.0.0.0:50062", + "serverCertificateFile": "/run/mcp-control/server.crt", + "serverPrivateKeyFile": "/run/mcp-control/server.key", + "orchestratorCaFile": "/run/mcp-control/orchestrator-ca.crt", + "orchestratorFingerprints": [ + "00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00:00" + ], + "workers": { + "public": { + "url": "https://mcp-public.example:8443", + "serverCaFile": "/run/mcp-worker/server-ca.crt", + "clientIdentityFile": "/run/mcp-worker/coordinator.pem" + } + } +} diff --git a/deploy/migrations/0129_mcp_secret_reservations.sql b/deploy/migrations/0129_mcp_secret_reservations.sql new file mode 100644 index 000000000..d56cdcc36 --- /dev/null +++ b/deploy/migrations/0129_mcp_secret_reservations.sql @@ -0,0 +1,10 @@ +-- Keep purpose after value deletion. A generic name can never become an MCP ref. +CREATE TABLE org_secret_purposes ( + name text PRIMARY KEY, + connection_id text, + role text CHECK (role IN ('api_key', 'client_secret')), + generation bigint NOT NULL DEFAULT 0 CHECK (generation >= 0), + CHECK ((connection_id IS NULL) = (role IS NULL)), + CHECK (connection_id IS NULL OR length(connection_id) > 0) +); +INSERT INTO org_secret_purposes (name) SELECT name FROM org_secrets; diff --git a/deploy/migrations/0130_mcp_oauth_authority.sql b/deploy/migrations/0130_mcp_oauth_authority.sql new file mode 100644 index 000000000..91d44647e --- /dev/null +++ b/deploy/migrations/0130_mcp_oauth_authority.sql @@ -0,0 +1,14 @@ +-- MCP encrypted redirect state stays with the existing flow authority. +ALTER TABLE oauth_flows ADD COLUMN mcp_state jsonb; +ALTER TABLE oauth_flows ADD CONSTRAINT mcp_state_subject CHECK (mcp_state IS NULL OR subject_kind='mcp'); +ALTER TABLE oauth_credentials ADD COLUMN mcp_refresh_claim uuid; +CREATE TABLE mcp_oauth_admission ( + deployment_id text NOT NULL, + slot smallint NOT NULL CHECK (slot BETWEEN 0 AND 3), + id uuid NOT NULL UNIQUE, + connection_id text NOT NULL, + expires_at timestamptz NOT NULL, + PRIMARY KEY (deployment_id, slot), + UNIQUE (deployment_id, connection_id) +); +CREATE INDEX mcp_oauth_admission_expiry ON mcp_oauth_admission (expires_at); diff --git a/deploy/migrations/0131_mcp_session_write_policy.sql b/deploy/migrations/0131_mcp_session_write_policy.sql new file mode 100644 index 000000000..89cf812bf --- /dev/null +++ b/deploy/migrations/0131_mcp_session_write_policy.sql @@ -0,0 +1,9 @@ +-- Nullable policy preserves unknown legacy authority. Parent identity is immutable. +ALTER TABLE sessions ADD COLUMN mcp_write_policy JSONB + CHECK (mcp_write_policy IS NULL OR jsonb_typeof(mcp_write_policy) = 'object'); +ALTER TABLE sessions ADD COLUMN mcp_policy_parent_id UUID REFERENCES sessions(id); +ALTER TABLE sessions ADD CONSTRAINT mcp_policy_no_self_parent CHECK (mcp_policy_parent_id <> id); +CREATE INDEX sessions_mcp_policy_parent_idx ON sessions(mcp_policy_parent_id) + WHERE mcp_policy_parent_id IS NOT NULL; +CREATE INDEX session_events_tool_call_idx ON session_events(session_id, (payload->>'tool_call_id'), idx) + WHERE kind IN ('tool_call_requested', 'tool_result_submitted', 'tool_call_completed'); diff --git a/deploy/migrations/0132_managed_creation_authority.sql b/deploy/migrations/0132_managed_creation_authority.sql new file mode 100644 index 000000000..3a9ccdfc0 --- /dev/null +++ b/deploy/migrations/0132_managed_creation_authority.sql @@ -0,0 +1,27 @@ +-- A held creation is never eligible for ordinary queue placement. +CREATE TABLE session_managed_creations ( + session_id UUID PRIMARY KEY REFERENCES sessions(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + task_id TEXT NOT NULL CHECK (octet_length(task_id) BETWEEN 1 AND 256), + input_id TEXT NOT NULL UNIQUE CHECK (octet_length(input_id) BETWEEN 1 AND 256), + actor_id TEXT NOT NULL CHECK (octet_length(actor_id) BETWEEN 1 AND 256), + parent_session_id UUID REFERENCES sessions(id), + eligibility TEXT NOT NULL CHECK (eligibility IN ('unknown', 'read_only', 'write_eligible')), + input_digest TEXT NOT NULL CHECK (input_digest ~ '^sha256:[0-9a-f]{64}$'), + native_version INTEGER NOT NULL CHECK (native_version = 1), + native_bytes BYTEA NOT NULL CHECK (octet_length(native_bytes) BETWEEN 1 AND 1048576), + native_digest TEXT NOT NULL CHECK (native_digest ~ '^sha256:[0-9a-f]{64}$'), + state TEXT NOT NULL DEFAULT 'held' CHECK (state = 'held'), + CHECK (parent_session_id <> session_id) +); +CREATE INDEX session_managed_creations_parent_idx ON session_managed_creations(parent_session_id) + WHERE parent_session_id IS NOT NULL; +CREATE INDEX session_managed_creations_task_idx ON session_managed_creations(task_id); + +-- Application retries never update authority. Prevent an accidental SQL upsert too. +CREATE FUNCTION reject_managed_creation_update() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'managed creation authority is immutable'; +END; +$$; +CREATE TRIGGER session_managed_creations_immutable BEFORE UPDATE ON session_managed_creations + FOR EACH ROW EXECUTE FUNCTION reject_managed_creation_update(); diff --git a/deploy/migrations/0133_managed_boot_authority.sql b/deploy/migrations/0133_managed_boot_authority.sql new file mode 100644 index 000000000..bb04dbf46 --- /dev/null +++ b/deploy/migrations/0133_managed_boot_authority.sql @@ -0,0 +1,26 @@ +-- Separate from the immutable creation source. A claim precedes all host I/O. +CREATE TABLE session_managed_boots ( + session_id UUID PRIMARY KEY REFERENCES session_managed_creations(session_id) ON DELETE CASCADE, + host_id UUID NOT NULL REFERENCES hosts(id), + sandbox_id UUID NOT NULL UNIQUE, + binding_epoch BIGINT NOT NULL CHECK (binding_epoch > 0 AND binding_epoch <= 9007199254740991), + boot_attempt TEXT NOT NULL UNIQUE CHECK (octet_length(boot_attempt) BETWEEN 1 AND 128), + revision BIGINT NOT NULL CHECK (revision > 0), + phase TEXT NOT NULL CHECK (phase IN ('claimed','prepared','finalizing','finalized','start_issued','started','interrupted')), + record JSONB NOT NULL CHECK (jsonb_typeof(record) = 'object'), + updated_at TIMESTAMPTZ NOT NULL, + CHECK (record->>'phase' = phase), + CHECK ((record->>'revision')::BIGINT = revision) +); +CREATE INDEX session_managed_boots_drive_idx ON session_managed_boots(updated_at, session_id) + WHERE phase IN ('claimed','finalizing','finalized','start_issued'); +CREATE INDEX session_managed_boots_host_idx ON session_managed_boots(host_id); +-- A cold recovery permanently closes delivery targets from the old physical run. +CREATE TABLE session_managed_interrupted_targets ( + session_id UUID NOT NULL REFERENCES session_managed_creations(session_id) ON DELETE CASCADE, + binding_epoch BIGINT NOT NULL CHECK (binding_epoch > 0), + boot_attempt TEXT NOT NULL, + interrupted_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY(session_id, binding_epoch), + UNIQUE(session_id, boot_attempt) +); diff --git a/deploy/migrations/0134_managed_capture_authority.sql b/deploy/migrations/0134_managed_capture_authority.sql new file mode 100644 index 000000000..12a4fc9cd --- /dev/null +++ b/deploy/migrations/0134_managed_capture_authority.sql @@ -0,0 +1,13 @@ +CREATE TABLE session_managed_captures ( + capture_id UUID PRIMARY KEY, + session_id UUID NOT NULL REFERENCES session_managed_creations(session_id) ON DELETE CASCADE, + binding_epoch BIGINT NOT NULL CHECK (binding_epoch > 0), + source_sandbox_id UUID NOT NULL, + request JSONB NOT NULL CHECK (jsonb_typeof(request)='object'), + authority JSONB CHECK (jsonb_typeof(authority)='object'), + record_digest TEXT UNIQUE CHECK (record_digest ~ '^sha256:[0-9a-f]{64}$'), + created_at TIMESTAMPTZ NOT NULL, + UNIQUE(session_id,binding_epoch), + CHECK ((authority IS NULL) = (record_digest IS NULL)) +); +CREATE INDEX session_managed_captures_source_idx ON session_managed_captures(source_sandbox_id); diff --git a/deploy/migrations/0135_managed_interrupted_event_floor.sql b/deploy/migrations/0135_managed_interrupted_event_floor.sql new file mode 100644 index 000000000..52118267a --- /dev/null +++ b/deploy/migrations/0135_managed_interrupted_event_floor.sql @@ -0,0 +1 @@ +ALTER TABLE session_managed_interrupted_targets ADD COLUMN event_floor BIGINT NOT NULL DEFAULT 0 CHECK (event_floor >= 0); diff --git a/deploy/migrations/0136_managed_operator_generation_gate.sql b/deploy/migrations/0136_managed_operator_generation_gate.sql new file mode 100644 index 000000000..182c3e140 --- /dev/null +++ b/deploy/migrations/0136_managed_operator_generation_gate.sql @@ -0,0 +1,12 @@ +-- Publication owns this row. It starts closed and has no published authority. +-- Readers take FOR SHARE before session/ancestor locks. A publisher takes +-- FOR UPDATE before staging a generation or changing participant readiness. +CREATE TABLE managed_operator_generation_gate ( + singleton BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (singleton), + published_generation BIGINT CHECK (published_generation > 0 AND published_generation <= 9007199254740991), + published_identity TEXT CHECK (published_identity ~ '^sha256:[0-9a-f]{64}$'), + admission_open BOOLEAN NOT NULL DEFAULT FALSE, + CHECK ((published_generation IS NULL) = (published_identity IS NULL)), + CHECK (NOT admission_open OR published_generation IS NOT NULL) +); +INSERT INTO managed_operator_generation_gate(singleton) VALUES(TRUE); diff --git a/deploy/migrations/0137_managed_operator_publication.sql b/deploy/migrations/0137_managed_operator_publication.sql new file mode 100644 index 000000000..11c3eabf8 --- /dev/null +++ b/deploy/migrations/0137_managed_operator_publication.sql @@ -0,0 +1,17 @@ +-- The gate is locked before these records and all session rows. +CREATE TABLE managed_operator_publications ( + generation BIGINT PRIMARY KEY CHECK (generation > 0 AND generation <= 9007199254740991), + record JSONB NOT NULL, + published BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMPTZ NOT NULL +); +CREATE UNIQUE INDEX managed_operator_one_pending ON managed_operator_publications ((TRUE)) WHERE NOT published; +CREATE TABLE managed_operator_readiness ( + generation BIGINT NOT NULL REFERENCES managed_operator_publications(generation), + session_id UUID NOT NULL REFERENCES sessions(id), + participant JSONB NOT NULL, + proof JSONB, + PRIMARY KEY (generation, session_id) +); +CREATE INDEX session_managed_policy_participants ON session_integration_policy(session_id) + WHERE policy_json::jsonb->'managed_deny' IS NOT NULL AND policy_json::jsonb->'managed_deny' <> 'null'::jsonb; diff --git a/deploy/migrations/0138_managed_boot_intents.sql b/deploy/migrations/0138_managed_boot_intents.sql new file mode 100644 index 000000000..a8eb7cfe9 --- /dev/null +++ b/deploy/migrations/0138_managed_boot_intents.sql @@ -0,0 +1,20 @@ +-- Immutable approved inputs are committed with the held reservation. +CREATE TABLE session_managed_boot_selections ( + session_id UUID PRIMARY KEY REFERENCES session_managed_creations(session_id) ON DELETE RESTRICT, + selection JSONB NOT NULL CHECK (jsonb_typeof(selection) = 'object') +); +-- One initial intent and one exact recovery operation per physical source epoch. +CREATE TABLE session_managed_boot_intents ( + intent_id TEXT PRIMARY KEY CHECK (length(intent_id) BETWEEN 1 AND 128), + session_id UUID NOT NULL REFERENCES session_managed_creations(session_id) ON DELETE RESTRICT, + operation_key TEXT NOT NULL, + binding_epoch BIGINT NOT NULL CHECK (binding_epoch BETWEEN 1 AND 9007199254740991), + revision BIGINT NOT NULL CHECK (revision > 0), + completed BOOLEAN NOT NULL DEFAULT FALSE, + record JSONB NOT NULL CHECK (jsonb_typeof(record) = 'object'), + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + UNIQUE(session_id, operation_key), + UNIQUE(session_id, binding_epoch) +); +CREATE UNIQUE INDEX session_managed_boot_intents_pending ON session_managed_boot_intents(session_id) WHERE NOT completed; diff --git a/deploy/migrations/0139_managed_boot_host_history.sql b/deploy/migrations/0139_managed_boot_host_history.sql new file mode 100644 index 000000000..d2eaaead1 --- /dev/null +++ b/deploy/migrations/0139_managed_boot_host_history.sql @@ -0,0 +1,2 @@ +-- Boot records retain physical host identity after host deregistration. +ALTER TABLE session_managed_boots DROP CONSTRAINT session_managed_boots_host_id_fkey; diff --git a/deploy/terraform/gcp/quickstart/eso.tf b/deploy/terraform/gcp/quickstart/eso.tf index 86358b2ac..38072b2b9 100644 --- a/deploy/terraform/gcp/quickstart/eso.tf +++ b/deploy/terraform/gcp/quickstart/eso.tf @@ -265,5 +265,8 @@ resource "kubectl_manifest" "web_backendconfig" { namespace: ${kubernetes_namespace_v1.app.metadata[0].name} spec: timeoutSec: 3600 + # Callback queries and referrers must not enter load-balancer logs. + logging: + enable: false YAML } diff --git a/docker/host-agent.Dockerfile b/docker/host-agent.Dockerfile index 04994ceec..5248eadc7 100644 --- a/docker/host-agent.Dockerfile +++ b/docker/host-agent.Dockerfile @@ -51,12 +51,17 @@ FROM debian:trixie-slim # bump that regresses it fails the image build loudly. # - iproute2: ip, for the tap device + guest netns # - iptables: egress NAT / firewall rules for guest networking +# - procps: sysctl, to set private managed network namespace controls +# - nftables: nft, to install and inspect the managed network fences # - thin-provisioning-tools: thin_dump + thin_delta, which read the # session-disk pool's metadata (device-mapper itself is an ioctl) # (ca-certificates: TLS to the blob backend — GCS/S3.) RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates e2fsprogs iproute2 iptables thin-provisioning-tools \ + ca-certificates e2fsprogs iproute2 iptables procps nftables thin-provisioning-tools \ && rm -rf /var/lib/apt/lists/* +RUN set -eu; \ + for tool in sysctl ip nft; do command -v "$tool"; done; \ + sysctl --version; ip -Version; nft --version # ADR 0080/0036: fail the build if the base's mke2fs predates the # SOURCE_DATE_EPOCH support that deterministic packing relies on. RUN set -eux; \ diff --git a/docker/mcp-worker.Dockerfile b/docker/mcp-worker.Dockerfile new file mode 100644 index 000000000..f4d865a52 --- /dev/null +++ b/docker/mcp-worker.Dockerfile @@ -0,0 +1,19 @@ +# The official Node 24 LTS release is pinned in both stages. +FROM node:24.22.2-bookworm-slim AS build +WORKDIR /app +COPY mcp-worker/package.json mcp-worker/package-lock.json ./ +RUN npm ci --ignore-scripts +COPY mcp-worker/tsconfig.json mcp-worker/tsconfig.build.json ./ +COPY mcp-worker/src ./src +RUN npm run build && npm prune --omit=dev --ignore-scripts + +FROM node:24.22.2-bookworm-slim +ENV NODE_ENV=production +WORKDIR /app +COPY --from=build --chown=node:node /app/package.json ./ +COPY --from=build --chown=node:node /app/node_modules ./node_modules +COPY --from=build --chown=node:node /app/dist ./dist +USER node +EXPOSE 8443 +# This is inherited by disposable schema children. No core or heap dump is kept. +ENTRYPOINT ["/bin/sh", "-c", "ulimit -c 0; exec node --max-old-space-size=256 dist/main.js"] diff --git a/docker/orchestrator.Dockerfile b/docker/orchestrator.Dockerfile index 8d29b11cf..6574399b9 100644 --- a/docker/orchestrator.Dockerfile +++ b/docker/orchestrator.Dockerfile @@ -5,8 +5,8 @@ # HTTP/WS surface. Entry is `bun src/index.ts` (package.json `start`), listening # on ORCHESTRATOR_PORT (default 8787). # -# Mirrors docker/web.Dockerfile's two-stage shape: a deps layer keyed on the -# lockfile (stays warm across source edits), then a slim runtime that ships only +# Uses dependency layers keyed on their lockfiles, which stay warm across +# source edits, then a slim runtime that ships only # the production node_modules + the source. Built from the repo root: # # docker buildx build -f docker/orchestrator.Dockerfile -t engram/orchestrator:dev . @@ -33,6 +33,12 @@ COPY orchestrator/packages/spec-document ./packages/spec-document RUN --mount=type=cache,target=/root/.bun/install/cache \ bun install --frozen-lockfile --production +# Worker source imports need the worker's locked production dependencies. +FROM node:24.22.2-bookworm-slim AS worker-deps +WORKDIR /app/mcp-worker +COPY mcp-worker/package.json mcp-worker/package-lock.json ./ +RUN npm ci --omit=dev --ignore-scripts + # ── runtime ────────────────────────────────────────────────────────────── # Keep in lockstep with the deps stage and `.bun-version` (see above). FROM oven/bun:1.4.4-alpine @@ -42,6 +48,10 @@ WORKDIR /app/orchestrator # step), so the runtime needs the production node_modules + the full source. COPY --from=deps /app/orchestrator/node_modules ./node_modules COPY orchestrator/ ./ +# Runtime imports worker modules and their dependencies through their repository path. +COPY --from=worker-deps /app/mcp-worker/node_modules /app/mcp-worker/node_modules +COPY --from=worker-deps /app/mcp-worker/package.json /app/mcp-worker/package-lock.json /app/mcp-worker/ +COPY mcp-worker/src/ /app/mcp-worker/src/ # The Google credential denylist is ONE checked-in table shared with the Rust # egress proxy, which is the enforcement point and owns the file (ADR 0109). diff --git a/docs/adr/0096-vz-parity-pass-2.md b/docs/adr/0096-vz-parity-pass-2.md index 920e89fd9..b5c9fd94a 100644 --- a/docs/adr/0096-vz-parity-pass-2.md +++ b/docs/adr/0096-vz-parity-pass-2.md @@ -229,6 +229,28 @@ fallback (machine.vzs deleted) keeps the un-synced disk write and loses tmpfs, parked snapshots stay parked, and pre-D7 manifests parse (`warm: None`) by unit test. +### D7 amendment (2026-11-03): saved configuration revision + +Status: **Accepted.** + +Each new VZ configuration includes one virtio entropy device. The warm +manifest records `configuration_revision = 1` for this device layout. +A missing revision decodes as 0. The existing warm gate requires exact +revision equality before the platform restore call. Legacy and unknown +future revisions use the existing cold-boot path; the saved disk remains +available, but the old memory state is not reused. New snapshots after +that cold boot use the current configuration. + +Future incompatible default device changes must increase this revision. +The revision is an explicit device-layout contract, not a fingerprint of +the full configuration. Existing identity, cmdline, bundle and MAC checks +still apply. Guest images do not need a rebuild for this host-only change. + +Implemented in `vz: supply boot entropy and gate saved configurations`. +Unit tests cover the native device, legacy revision decoding, exact revision +checks and manifest round-trip. The original live VZ lifecycle test passes +its shell, durable snapshot and restore checks with unchanged guest images. + ## Out of scope (permanent divergences, not regressions) UFFD lazy restore, base-shm density, dirty-page diff snapshots, the NBD chunked diff --git a/docs/adr/0127-isolated-external-mcp.md b/docs/adr/0127-isolated-external-mcp.md new file mode 100644 index 000000000..70c884df7 --- /dev/null +++ b/docs/adr/0127-isolated-external-mcp.md @@ -0,0 +1,802 @@ +# ADR 0127 — Isolated external MCP tools + +Status: Proposed (2026-11-01) + +## Context + +ADR 0058 deferred native MCP configuration in the guest. That design lets a +harness connect to an upstream server and does not provide a platform write gate. +External tools must instead use the authenticated ENGRAM_TOOLS rail from ADR 0089. Claude uses its existing bridge; Codex uses its existing dynamic tools. +Both use parked calls and the durable completion outbox. MCP metadata is +untrusted input, never a new process-global tool registry. + +ADR 0106 is the credential authority. MCP OAuth uses subject kind `Mcp` and the +named integration connection ID as subject ID. Organization API keys use the +existing KEK-sealed organization secret store. There is no parallel secret store. +Personal credentials, guest stdio servers, package installation, resources, +prompts, sampling, elicitation, MCP Apps, and asynchronous Tasks are out of scope. + +## Decision + +### Service and trust boundary + +Add a trusted standalone Node 24 ESM service, `mcp-worker`, with the official +`@modelcontextprotocol/client` pinned to 2.4.0, Ajv pinned to 8.20.0, and a +maintained IP parser. Its mTLS interface has fixed discover, execute, cancel, +OAuth metadata, and token-exchange operations. It has no generic proxy or guest +listener. Each operator zone has a separate deployment. It receives no database +credentials, KEK, cloud identity, host mounts, or service-account token. Disable +core dumps and credential disk writes. Credentials exist only in operation memory +and must never appear in logs. + +The orchestrator issues expiring, single-use intents bound to worker boot ID, +connection revision, exact destination, operation, and call digest. Discovery and +OAuth have separate budgets. The worker cannot authorize a tool call or refresh a +credential on its own. The coordinator is the only credential and refresh +authority. The worker receives secrets only after admission, near dispatch. + +The coordinator exposes credential control only on a separate, default-off mTLS +listener. The internal service name is `engram.mcp.v1.CredentialControlService`. +Its four status operations return method-specific response wrappers with one +`CredentialStatus` field. Clients reject a missing status. The unreleased service +has no alias for the earlier name. Exact client certificate pins identify the trusted orchestrator role. +The existing cleartext application gRPC listener never serves this service. The +orchestrator may hold a short-lived credential lease in memory for one admitted +worker operation. It must not queue, cache, log, persist, or return the value to a +guest or browser. Resource calls use the orchestrator worker role; OAuth metadata +and token calls use the coordinator worker role. A lease checks the actual sealed +identity, config revision, and authorization generation, including when publication +in the orchestrator database failed. Worker boot and operator policy must match +the frozen operation; a new identity lookup cannot replace a frozen boot identity. +Worker boot binds an operation, not durable credential authority. After a worker +restart, fresh operations can use the current boot with the same sealed authority. + +The credential delivery deadline is at most ten seconds. The separately frozen +operation deadline is at most five minutes and remains the worker deadline. +Admission must cover that full operation deadline. Credential control echoes both +deadlines; the orchestrator validates them and dispatches before delivery expires. +Status returns NotFound only for absent authority metadata. Transport and service +failures remain unavailable and cannot justify recovery or replacement. + +### Reviewed catalogs and session binding + +Reuse named integration connections and profile integration grants. Discovery +creates a draft. Only administrators can publish an immutable reviewed catalog, +using expected-revision compare-and-swap. Preserve upstream descriptions, +schemas, and fingerprints. Administrators assign explicit read/write classes; +server annotations confer no permission. + +An immutable task-session binding includes connection/config revision, credential +identity, authorization generation, catalog revision, grants, constraints, and +hashes. Tool names resolve only through that binding. The guest cannot provide +endpoints or credential subjects. Changes to endpoint, account, authentication, +policy, or reviewed tools invalidate the binding. A normal refresh with the same +account and scope preserves it. Rebind must be explicit and cannot widen an +existing session silently. Child grants intersect parent effective grants. Plan +mode permits only reads and comes from trusted state. + +Managed upstream authorities and aliases enter the guest deny set before any +allow or default-allow rule and before credential injection. Reject conflicts +with direct integrations. MCP secret references cannot use generic guest inject +or mint paths. Keep old denies when invalidating; install new denies before a +rebind becomes visible. This guarantee covers organization-managed credentials; +unknown alternate origins and user-supplied credentials are outside it. Upstream +servers must authenticate writes. + +### Network containment + +Managed policy has a durable monotone host union. The coordinator saves the +desired union in the session integration policy before host I/O. The host saves +the desired state before proxy application, drains existing session connections, +then saves an applied marker before ACK. Read and compare-and-apply bind the exact +session, sandbox, physical epoch, host and proxy incarnations, binding hash, +operator identity, revision and digest. A lost ACK cannot remove retained hosts. + +The trusted read returns retained coordinator state separately from the exact +host receipt. A required wrapper carries the checked physical read target and +an optional canonical retained policy; a missing wrapper is unsupported, not an +empty union. Consumers retain both policies and choose a revision above both. +Coordinator persistence is not proof of host application. Reads check placement +before host I/O and after reading the retained policy, without a transaction +across that I/O. + +Managed host mode requires mTLS for the whole host mutation listener, exact +coordinator certificate pins, and host-specific server identity verification. +There is no simultaneous plaintext mutation listener or downgrade fallback. + +A managed host owns the durable policy directory with one lifetime exclusive OS +lock. The lock file keeps its inode and is never removed during handoff. The +lock releases on process exit or crash. A successor quarantines every managed +session reported by the surviving proxy before it reads durable records or +replays ordinary policies. Missing or corrupt records fail startup and keep +admission closed. Valid replay checks the physical target and live network +capability before it produces a new receipt for the successor incarnation. +Managed control requests use separate local proxy streams; a session drain does +not hold a global network lock. Directory ownership requires a persistent host +volume and one managed host generation per directory in deployment. + +Generic start checks all durable managed session associations for the actual +sandbox before any bind or policy write. A duplicate ordinary binding cannot +hide a managed record, including one retained after unbind. Supplied fence and +policy identities must agree with the requested sandbox; the explicit unfenced +sentinel remains available to ordinary legacy callers. + +Known managed sessions reject ordinary restore, base restore, resume, and agent +start. Explicit disk-only cold recovery uses the durable source authority +described below. The management marker is trusted host state, and a corrupt +marker does not make the session ordinary. A fresh target host has no local +marker: coordinator classification and pre-restore fencing are still required +before cross-host migration can be enabled. + +Fresh create and restore use a durable coordinator pre-start gate at the real +physical binding epoch. The orchestrator pulls preparation state, compiles and +commits a final candidate, then finalizes the exact manifest and host receipt. +The coordinator never holds a transaction waiting for the orchestrator. Initial +prompts remain durable and blocked until finalization. The native-only manifest +is an immutable creation input with authenticated session association. Restore +must quarantine guest execution and network before it can run saved state. + +The complete deny superset includes every operator-declared selectable authority +and alias in every zone, including unused zones. A durable operator generation +stages additions, closes new managed boot admission, and requires all affected +sessions to ACK the union or have a proved network fence before publication. +The worker configuration digest and deployment/network identity are separate +inputs. Missing capabilities and mixed generations stay closed. Network policy, +boot finalization, and operator publication are separate implementation slices; +the feature stays disabled until all producers and their composition are verified. + +Accept canonical HTTPS URLs with exact approved authority and port (443 by +default). Reject userinfo, query, fragment, IP literals, and redirects. A public +zone denies private and reserved ranges. An internal zone allows only exact +operator-approved hosts and private CIDRs, with operator-approved CAs. An +administrator cannot widen CIDRs, CAs, or routes. Every zone denies loopback, +link-local, metadata, multicast, unspecified, and platform administration +addresses. Apply this policy to OAuth as well as tools. + +Resolve every A and AAAA record on each new dial. Reject mixed permitted and +forbidden answers. Parse addresses with an established library and pin the checked +IP while preserving TLS SNI and HTTP authority. Local loopback and private test +CA allowances exist only in fixture code, never in production policy. + +### Protocol and validation + +Support only Streamable HTTP with JSON or request-scoped SSE responses, using +modern 2026-07-28 and legacy 2025-11-25 / 2025-06-18. Modern requests have no +initialize handshake or transport session. Legacy sessions are scoped to platform +session and credential identity. Never fall back to the deprecated separate SSE +transport. Untrusted responses cannot trigger resource, link, or icon fetches. + +Preserve text, structuredContent, and isError. Reject binary and resource content. +A delivery failure is distinct from the upstream outcome. Support JSON Schema +Draft 7 and 2020-12, local references only, bounded complexity, and no unsupported +dialects, remote references, or executable formats. Disable coercion, default +insertion, and property deletion. Compile and validate in a disposable child with +a hard one-second deadline and 128 MiB heap. + +### OAuth + +Use pre-registered public or confidential clients, no dynamic client registration. +Require PKCE S256, encrypted durable single-use state with a ten-minute expiry, +exact redirect URI, issuer validation, and resource/audience checks. Independently +approve metadata, authorization, and token destinations. Require explicit admin +consent to share the organization-session account. + +Use single-flight refresh and rotating-token compare-and-swap. Do not fall back +to stale credentials or expand scope silently. Send access tokens only to the +resource; send client secrets and refresh tokens only to the token endpoint. + +MCP redirect state is sealed in the existing OAuth flow store and consumed +atomically before token exchange. Coordinator OAuth requests use durable admission +with four concurrent slots per deployment and one per connection. Refresh claims +are exclusive and version-fenced; a token deadline ends before its claim. An +uncertain rotating exchange leaves the credential unavailable until reconnect. +No expired-claim retry may reuse the previous refresh token. Ordinary refresh +preserves authority identity; reconnect and API-key rotation create a new identity +bound to the next expected authorization generation. Before switching credential +kinds, references, or registrations, the orchestrator serializes the connection +transition and revokes the old authority with its exact identity and version. It +verifies the durable revocation receipt before acquisition and publication of the +replacement. Failed publication cannot restore the old authority. + +The existing connection row holds only credential metadata and a durable transition +token. A transition reserves its final config revision before the coordinator seals +a replacement. Publication uses that same revision and compares the transition +token and authorization generation. Recovery changes the token and leaves old +authority revoked. Late control replies and callbacks cannot publish through an +old token. A lost fresh-reference response can leave a sealed orphan; recovery +cannot publish an unknown reference or restore the previous credential. + +A local transition token alone cannot fence a delayed coordinator callback. +Before recovery or replacement, the orchestrator cancels each recorded attempt +by exact flow UUID, MCP connection and registration. Cancellation clears sealed +state and marks the existing flow row cancelled. Callback consumption admits token +I/O; cancellation before consumption prevents that exchange. Credential publication +locks the same flow row, requires live consumed state, applies credential-version +CAS and marks success in one transaction. Cancellation after consumption therefore +fences publication even if external I/O cannot be stopped. If publication wins, +recovery queries and revokes that exact credential before clearing its local fence. +A lost begin response can leave an unknown pending flow until expiry and cleanup; +the stale orchestrator call cannot return its authorization URL or record a usable +callback after recovery. Unknown flow IDs are never inferred or broadly cancelled. + +### Trusted session write policy + +The coordinator owns a durable conservative MCP write-policy latch. This is server +permission, not proof of the guest mode. A trusted plan input sets read-only and +advances the revision in the existing prompt/outbox transaction. A later queued +default or absent directive cannot relax it. This can keep external writes blocked +after a default prompt starts locally. The session state explains the restriction; +the feature stays disabled until the UI presents the required review action. + +Only actual new creation over the dedicated pinned mTLS session-policy service can +initialize known policy, inside the existing creation write-set and after validation +of supported harness semantics. Legacy and generic creation remain unknown. The +initial plan restriction is part of creation, not its best-effort initial prompt. +An immutable parent link makes reads and dispatch intersect all current ancestors. +Missing ancestry fails closed. Durable policy survives restart and disk restore; +binding-epoch changes invalidate old request references. + +A human owner or administrator can relax a known restrictive policy only by accepting +the exact pending plan exit, bound to the restrictive revision, binding epoch, and +first durable requested-call payload. Changed duplicates, rewound requests, already +submitted/completed calls, and old plan requests cannot authorize relaxation. Fresh +orchestrator identity and ownership checks precede a pinned mTLS acceptance. The +coordinator commits the policy decision, submitted event and existing result outbox +in one transaction. Public CompleteToolCall carries additive expected review context; +without it, ordinary delivery is unchanged and cannot relax MCP policy. Automation, +guest completion, run-start, mode-shaped strings and disk stamps never relax policy. +The session-policy service shares the dedicated listener and client pins, with a +separate bounded create message budget; credential control keeps its 32 KiB limit. + +Request acceptance freezes eligibility and exact leaf/ancestor policy references. +An earlier read-only or unknown request cannot gain writes after relaxation. A stale +eligible request cannot rebase. After the orchestrator commits its irreversible +journal barrier, it must synchronously validate those references, the complete +stored ancestry, current binding and execution identity at the coordinator. The +coordinator locks all involved session rows in stable order. Its successful locked +check is the policy-admission ordering point. A restriction before that point blocks +the operation; one after it applies to already-admitted active I/O, canceled best +effort. This is not an atomic transaction across the two databases. Live orchestrator +permissions and journal admission are separate checks. A lost reply or crash after +the journal barrier never permits automatic write replay. Plan approval does not +replace the separate exact external-write approval or explicit automation grant. + +### Authorization and irreversible dispatch + +Before execution, validate the active session/run, immutable binding, principal, +ownership, live permissions, current revisions, immutable grants, tool input, and +admin constraints. Writes also need one-use approval or an explicit automation +grant. Owner/admin interactive approval expires in ten minutes and binds the exact +session, run, call, connection, catalog, generation, and canonical argument hash. +It cannot expand grants or be replayed. Automation-plan approval cannot approve +MCP writes. Unrestricted automation constraints `{}` must be selected explicitly. + +A successful model turn can suspend on deferred calls. The harness records their +exact call IDs atomically with completion; this does not grant access. The fence +still requires the same current run and unresolved request. Ordinary or failed +completion, interruption, a newer run, and invalid associations remain stale. +Ordinary coordinator/host traffic retains wire version 38 and its exact policy +payload shape. The ordinary conversion rejects every managed deny value; it +never drops authority to reach an older peer. Dedicated managed RPCs require +the exact managed protocol 43 marker and pinned mTLS before payload decoding or +effects. A protocol marker does not grant authorization. Registration and +placement continue to advertise the ordinary contract. + +The guest harness emits new cancellation and deferred events only for the MCP +contract. Ordinary tools retain predecessor-supported event behavior. The +explicit ordinary wire type is the single ordinary contract, not a temporary +second transport stack. Predecessor fixtures advance with the supported release +window; deny rejection and managed authentication remain required. + +Keep the authoritative call journal separate from the UI pending projection. +Use unique `(session_id, tool_call_id)` and immutable request digests. Enforce +`waiting_approval -> authorized -> dispatching -> succeeded/failed/canceled/outcome_unknown` +with explicit legal transitions. Entering `dispatching` is the irreversible write +barrier: never auto-replay a write after crash, lease loss, or timeout. External +effects stay outside retried DBOS steps. Only reviewed safe reads may have bounded +retries. Persist a verified result before the completion outbox obligation. + +An unknown write blocks later writes for the session/connection, even with new +call IDs, until an owner/admin resolves it. Established success with unusable +output stays `succeeded` with result unavailable; otherwise record outcome unknown. +Guest completion projection is not evidence of upstream success. Stop, interrupt, +revoke, and deadline cancel active I/O best effort; cancellation cannot undo an +upstream effect. Resume uses live policy without widening immutable grants and +retains durable late results. + +### Limits and audit + +| Limit | Value | +| ---------------------------------------------- | ----------------------------------- | +| Connections / tools per session | 16 / 128 | +| Manifest / arguments / normalized result | 1 MiB / 256 KiB / 1 MiB | +| Decompressed response | 2 MiB | +| Connection / tool deadline | 5 s / 60 s (operator ceiling 300 s) | +| Concurrency: session / connection / deployment | 4 / 8 / 32 | +| Queue size / expiry | 128 / 30 s | +| Pending approvals: session / deployment | 16 / 128 | +| Schema validation deadline / heap | 1 s / 128 MiB | + +Enforce connection and deployment admission across replicas. Audit metadata only: +identities, revisions, decision, actor, call, timing, sizes, and outcome. Never audit +raw arguments, results, headers, or secrets. Session payloads use existing retention +and access rules. Redact known secret echoes, without claiming protection against +a malicious credential recipient. + +Use additive indexed foreign-key migrations, unique IDs, and state constraints. +Never alter applied migration checksums. Version shared protocols. Coordinator +persistence changes require simulator and real-Postgres conformance tests. + +## Rollout, rollback, and acceptance + +All global, zone, connection, and tool flags default false. Each kill switch blocks +new dispatch and cancels active work while preserving journals. Rollback disables +dispatch and drains work; it never deletes live history or migrations. + +Stage rollout: public reads with API keys, private endpoints, OAuth, interactive +writes, then explicit automation writes. Before enabling any stage, run its hostile +fixture cases, current-head CI, and operator acceptance. Both Claude and Codex must +pass the shared protocol path, approval, denial, cancellation, eviction/resume, +secret containment, and unknown-outcome gates. Require real Firecracker production +acceptance. VZ proves local parity separately and cannot satisfy that gate. The +physical CI workflow requires dedicated self-hosted Linux X64 Firecracker and +macOS ARM64 VZ runners with operator configuration. Runner access and +configuration remain unverified; source wiring does not prove runner readiness. + +The feature stays disabled until its release gates pass. The historical QuickJS +deep-recursion failure remains in `docs/testing/external-mcp-baseline.md` beside +its later source-specific results. Do not weaken or skip the regression check. +This record stays Proposed until the final acceptance review records evidence and +the implementation commit chain. + +### Recorded qualification and remaining gates + +On 2026-11-10, the manual Firecracker matrix passed all 24 unique cases at +`c5dc8e7b19513f33a849da5a06b8e550956697bb`: 12 Claude and 12 Codex, +with no failed or duplicate cases. Retained evidence passed its integrity checks, +and owned test cleanup and cloud retirement completed. The earlier VZ 24/24 result +belongs to `b75b0d7f`. Each result qualifies its recorded source only. + +The later upstream integration changes shared disk, lazy-resume, fill, and host +paths. The merged implementation still requires source checks, current-head +`CI Gate`, and source-matched physical qualification. Before this integration, +PR conflicts prevented a current-head workflow result. The source simplification review is complete; it +is separate from final acceptance review. Managed activation, upgrade, recovery, +and rollback remain unqualified. Keep the feature disabled and this record +Proposed until those gates have explicit evidence. + +### Upstream disk and upgrade integration + +The host uses the session-disk abstraction from ADR 0125. Managed boot inputs and +captures remain immutable chunk manifests. Garbage collection retains those +manifests independently of ordinary snapshot rows. + +ADR 0126 adds the reattach generation and upgrade drain. Managed sessions still +reject ordinary checkpoint, eviction, restore, and migration paths. The upgrade +drain therefore stops before changing the deployment when it encounters a managed +session it cannot checkpoint or evict. Do not enable managed sessions until their +upgrade and recovery paths preserve the same authority and network fences. The +current reattach generation does not establish managed-session adoption support. + +The ordinary mixed-release transition retains the existing ordinary transport +with managed creation disabled. Activating a pinned TLS-only host listener is a +separate cutover: an unchanged plaintext predecessor cannot use it. Managed +rollback to a predecessor that does not understand its owner and policy records +is not established by ordinary wire compatibility. Keep those release gates +closed. Proxy replacement can preserve a VM while ending its active proxy +streams; it does not provide uninterrupted networking. + +Changes to host storage, materialization, and the shared wire contract require new +source-matched builds and physical acceptance. Earlier Firecracker and VZ evidence +remains evidence for its recorded source; it does not qualify the merged +implementation. + +## Sources + +- [Streamable HTTP 2026-07-28](https://modelcontextprotocol.io/specification/2026-07-28/basic/transports/streamable-http) +- [Official TypeScript SDK 2.4.0](https://github.com/modelcontextprotocol/typescript-sdk/releases/tag/v2.4.0) +- [MCP authorization](https://modelcontextprotocol.io/specification/2026-07-28/basic/authorization) + +### Managed network producer limits + +Managed sessions inspect otherwise allowed external TLS through the existing +HTTP interceptor. Literal mode does not add credentials. HTTP authority must +match the normalized TLS SNI and port. The proxy rejects opaque protocols, +HTTP CONNECT, upgrades, and chunked HTTP/1 requests that this adapter cannot +bound. Clients must trust the guest proxy CA; certificate-pinned clients and +custom non-HTTP TLS can fail. Ordinary sessions keep their current behavior. +Managed sessions cannot use connector-owned tunnels because the current +connector contract does not prove the external authority. The gateway and TLS +proxy share session connection cancellation and drain. + +A proxy acknowledgement alone does not prove complete guest enforcement. The +backend must supply a strict network proof tied to the exact host, sandbox, +physical epoch, guest IP, and network identity before a final binding ACK. +The default backend proof is absent. Ordinary Firecracker rules redirect +TCP 443 and DNS; they do not establish all-port enforcement. The managed cold +producer described below installs pre-execution controls for all TCP ports and +closes unobserved UDP paths. Ordinary restore cannot supply that proof. Bootstrap +receipts have no final binding hash and cannot authorize tool exposure. Local socket tests do not +establish Firecracker, VZ, or harness compatibility acceptance. + +### Private network transport + +The selected VZ producer uses one bounded gVisor stack per VM. The helper owns +no host IP listener and cannot dial an external destination. The host passes +three owned descriptors: the Ethernet datagram channel, a reliable parent +control stream, and a registered private proxy stream. A private flow carries +its original IPv4 address and port and an SCM_RIGHTS stream. The Rust proxy +uses the existing session lease, TLS interceptor, and DNS policy. The private +stream is the allocation capability; no bearer credential is stored or logged. +Registration starts closed. An exact placement epoch must be bound before +packet admission. Replacement, policy change, or loss of either reliable +channel closes admission and joins owned work. A transport-ready response is +not a network proof or a final binding acknowledgement. + +The helper pins gvisor.dev/gvisor at +`v0.0.0-20260413194555-9680d69bf798` and Go 1.26.2. Ethernet MTU is 1500. +The host selects the exact source MAC, guest IPv4 address, and gateway. +Only IPv4 TCP and DNS at that gateway can reach the private proxy. Established +stack and DNS parsers own protocol decoding. There is no NAT fallback. +Managed Firecracker uses an owned per-interface early fence, all-TCP redirection, +controlled DNS, and drops for other UDP, as described below. Managed recovery +cold-boots from eligible disk-only artifacts. Saved session RAM and device state +are not eligible. Source-matched physical qualification, packaging, and release +activation remain separate gates. This decision remains Proposed. + +A policy drain retains its session lease until the helper returns cleanup EOF +on the reliable private stream. The helper first closes packet admission and +joins its stack, packet, flow, DNS, and control work. The proxy drains pending +descriptor carriers while it waits, with a five-second limit. A failed barrier +poisons the stable session gate; policy replay or a fresh allocation cannot +reopen it in that proxy incarnation. Recovery needs a fresh proxy incarnation +and full policy replay with quarantine checks. Other sessions remain usable. +This barrier cannot revoke bytes that were delivered before it completed. + +### Immutable managed creation authority + +The session create transaction stores a held reservation with the exact native +ENGRAM_TOOLS bytes, version and SHA-256 digest. It binds the reserved session ID +to the task, create input, actor and immutable parent policy. The request digest +covers the exact serialized trusted creation request and its native version and +task context. A retry must match all original input; it cannot replace a manifest +with the current profile. Empty native input is an explicit `[]`, never an absent +value. Native classification follows the descriptor contract, not a name prefix. + +Held reservations use the existing queued session row and all existing creation +satellites. Ordinary queue listing and placement exclude them. No host request +occurs during reservation. The pinned mTLS policy service alone can read this +source. Public creation and managed admission stay closed until the operator +admission and physical boot gates pass. The boot transition claims +the same held row and binds the actual physical epoch; a reservation is not a +network receipt or permission to start a harness. Legacy sessions have no source +and cannot be backfilled from guest state or a current profile. + +### Closed cold preparation + +A managed cold prepare request names one exact private disk manifest version, +its materialized bytes, its trusted source record, and all selected boot files. +Fresh images first get a private writable disk lineage. Capture eligibility and +disk consistency come from the trusted durable producer, not from guest status +or the absence of a memory manifest. Legacy artifacts cannot supply this input. +The backend pins private copies and verifies their bytes before construction. + +The VZ producer owns a fresh stopped machine, its exact file-handle network +attachment, and the helper's reliable channels in one opaque preparation object. +It is outside the ordinary sandbox map and has no start or resume operation. +Its receipt means closed preparation only; it is not an enforcement ACK. +The separate durable start producer binds actual placement, final manifest, +policy and start intent, checks current liveness, and enforces the managed +lifecycle gates. Drop must terminate and reap the helper without running the VM. + +VZ uses a host-selected static IPv4 address and gateway in the same /24, +a fixed unicast MAC on each private point-to-point attachment, and MTU 1500. +Kernel IP configuration also names that gateway as DNS. Managed init replaces +any existing resolver file with the selected gateway. +The changed init shim uses materializer layout version 2 and distinct memo keys. +Earlier memo records, manifests, and disks remain readable; stored data is not +rewritten. +Guest configuration supplies functionality; it is not host enforcement proof. +The backend does not widen the helper's DNS acceptance to old resolver addresses. + +The registration API returns an opaque allocation-and-stream owner. The VZ +backend accepts that owner, not independently supplied allocation metadata and +a raw stream. The helper inherits only the three selected descriptors, an empty +environment, and no arguments. Native attachment identity, reliable control +liveness, and child-process state are checked before a closed receipt. A helper +cleanup timeout stays failed after forced reaping. This local cleanup result +cannot replace the proxy's lease barrier or clear its stable session poison. + + +### Durable managed boot and disk capture authority + +The immutable creation row remains unchanged. A companion boot row claims the +actual host, sandbox, physical epoch and boot attempt before host I/O. Each +transition takes the session or policy-ancestor locks and compares its exact +revision. The pinned policy service reads this row and finalizes its manifest; +it never waits for the orchestrator while holding a transaction. Finalization +preserves every original native descriptor and its order. Only the full sealed +AgentSpec is durable; the plaintext environment is a transport-only input. + +The prepared receipt names the actual network allocation and host incarnation. +It is closed preparation only. Finalization requires an authenticated current +policy receipt with the final binding hash, the exact native source and the +combined manifest digest. The coordinator commits one start intent before its +first host call. After a lost reply, the scanner reads the same host intent and +state. Missing state does not authorize another start. Scanner pages use stable +session IDs so failed rows cannot starve later claims. +The scanner also reads prepared, finalizing and started owners so a retained +retirement proof cannot be hidden by a failed finalization or start reply. An +older exact retirement receipt may be recorded when the stored desired deny +policy is already a newer superset. This keeps the newer next-boot requirement; +it does not acknowledge that policy as applied to the retired owner. + +The singleton `managed_operator_generation_gate` starts closed with no published +generation or identity. Boot claim, finalization, start intent and started CAS +acquire its `FOR SHARE` lock before session or ancestor locks and require an +open exact generation and identity. The authenticated operator publisher must +acquire `FOR UPDATE` first, close admission, stage the generation and invalidate +participant readiness in a short transaction. It may open the gate only after +all required participants have current actual receipts. No host I/O occurs +under either lock. The publisher must also use this lock for held-create +admission and placement invalidation. There is no public or configuration +setter. The producer uses an optional separate operator mTLS listener with a +separate client CA and exact leaf pins that cannot overlap orchestrator pins. +Stage accepts only expected generation and configured candidate identity. It +resolves every candidate byte from operator-owned configuration. There is no +manual publish endpoint. The pinned orchestrator service has only a read-only +published projection. Rollout and registry flags remain separate admission gates. + +Migration 0137 stores immutable generation records and per-session requirements +and readiness. The participant query unions managed creations, registered boots, +and retained managed policies; it does not filter by active executions or terminal +status. The scanner commits the historical deny union before host I/O, including +unused zone authorities and aliases. A route or trust change advances the deny +revision once even when hosts are unchanged. Exact immutable Retired evidence can +prove a source nonrunning while its durable next-boot requirement advances; it +never counts as a new applied ACK or capture. A destroyed Captured source is a +separate readiness case. An old source can still be addressed after SQL detaches +its sandbox only when its recorded session, epoch, and retained host match. +Publication accepts only never-allocated held creation or exact permanent +Retired/Captured source evidence. Live applied-policy receipts do not supply a +durably invalidated host/proxy lifetime, so they cannot authorize publication. +Policy-only historical rows without immutable managed source lineage remain +enumerated and retain their complete deny union, but block publication with no +legacy host mutation. They need a separately reviewed durable lifecycle or +destruction migration; terminal labels, row deletion and synthetic boot/capture +records are not supported workarounds. This session-lineage restriction is +separate from required legacy external MCP protocol support. Every MCP-bound root +and child must use immutable managed creation. Current held, prepared, started +and captured managed sources retain their existing progress paths. +All participant snapshots and proofs are checked again under the exclusive gate +and stable session locks before publication. Host I/O holds no database locks. +The deployment configuration and pinned read contract are described in +[operator MCP publication](../operator-mcp-publication.md). + +A cold capture first interrupts the old physical target. It preserves history +and pending prompts, retires pending old tool-result deliveries, and makes old +call targets permanently unavailable. A capture intent fixes its source key, +capture ID, and independent disk manifest (capture UUID, version one) before +host I/O. The intent and claimed disk are GC roots; a missing pending manifest +fails GC collection closed. The authenticated producer returns the actual +retirement and disk receipt. Its digest binds the exact image, kernel, ordered +bundles, disk manifest, byte hash and size, and `CrashConsistentDiskCutV1` result. +That result does not assert clean guest caches or application transactions. + +Recovery requires the exact stored capture authority and an authoritatively +destroyed source. A capture-shaped request is insufficient. Destination prepare +also receives the exact record, so it verifies the selected disk against the +same retired source. Same-source capture retries return the same durable result, +including after replacement. No path infers a disk from a file name or restores +memory, device state, or a pending effect. Generic coordinator lifecycle and +execution gates remain closed until the appropriate durable start proof exists; +generic resume, snapshot, teleport, and harness restart cannot replace this path. + +### Firecracker closed cold producer + +Managed Firecracker uses one anonymous network namespace per preparation. The +namespace has only its loopback device and one TAP. It has no veth, external +route, shared address slot, or host-root firewall rule. A dedicated native +thread creates it; retained namespace descriptors select every later command +and the Firecracker child. Namespace device/inode, TAP ifindex, MAC, IPv4 +addresses, MTU, routes and installed rules are read back. Carrier state and +TAP queue count can change when Firecracker opens the same attachment; they +are not attachment identity. Administrative state remains checked. + +Before the TAP is brought up, owned netdev ingress/egress and inet +input/output/forward chains drop all traffic. Separate strict chains remain +installed behind that fence. They check the selected source tuple and narrow +ARP exchange, redirect all IPv4 TCP to a namespace-local listener, allow DNS +only at the selected gateway, and drop other protocols. Port inspection is +after defragmentation. No host-root NAT or established-flow rule participates. +The namespace owns its conntrack table; destroying all namespace, socket and +process owners discards it without a host-wide flush or address-slot reuse. + +The bounded Rust adapter transfers Unix streams only to the registered private +policy ingress. TCP requires the kernel original destination and port, with +no default-port fallback. Gateway TCP/UDP DNS uses the bounded private DNS +operation. The adapter makes no destination dial. Proxy loss, an invalid +handoff, owner loss or runtime shutdown is terminal. Owned flow tasks stop +before private cleanup EOF. Native child cleanup runs on a reserved thread; +Drop initiates it, while explicit close reports its result. Timeout or failed +cleanup cannot become a successful receipt. + +The cold constructor verifies private copies of the selected disk, kernel +and auxiliary files, configures fresh devices, and checks the Firecracker +state is Not started. It does not load memory or saved device state and has +no start, resume or ordinary-map conversion. The closed receipt binds the +exact request, registration, network identity, rules digest and configuration +digest. It does not certify capture consistency or final policy enforcement. +The durable start producer consumes the owner, binds placement and final +policy, and performs the sole release/start operation. Its guest resolver +must use the selected gateway. Root installed-rule, packet and VM acceptance +remain required on Linux; local socket fixtures and cross-compilation are not +that evidence. This decision remains Proposed. + +## CLOSED namespace CI qualification + +The required Firecracker root lane builds the current library test executable and +runs `managed::network::linux::tests::actual_closed_namespace_readback_detects_route_tampering` +with `--ignored --exact`. The gate requires one selected test and one passing test. +It requires root with CAP_SYS_ADMIN and CAP_NET_ADMIN, a TAP device, `ip`, +`sysctl`, nftables JSON support, and kernel netdev egress support (Linux 5.16 or +later). A missing capability or unsupported kernel fails the test. The lane checks +namespace enforcement independently of session-disk storage. + +This is source wiring until the assigned CI lane runs and records a pass. It checks +installed CLOSED state and detects route tampering. Packet forwarding, original +destination, live DNS, Firecracker VM, VZ parity, full-stack and CNI acceptance +remain separate gates. This check does not authorize activation. + +### Managed host ownership and the CLOSED policy handoff + +The host keeps each managed machine in one private owner from preparation to +retirement. Durable session and sandbox markers precede backend I/O. Generic +restore, execution, snapshot, pause, resume and attach commands cannot adopt +these owners. A lost owner stays closed. A durable start intent precedes the +single `SpawnHarness` write into a fresh guest. The host persists only digests +of the final agent environment, not plaintext credentials. + +Managed execution and session file reads and writes use the same exact Started +owner check: current host incarnation, start receipt, durable session binding, +boot identity, epoch and fresh backend readiness. The lifecycle lock covers +connection setup, with a bounded readiness and connection check. Body transfer +runs outside that lock, without a new body deadline. A fresh close signal belongs +to each prepared owner and cancels pending transfer work before native close. +File transfers use one connection from that owner and the existing agentd +protocol. An open transfer cannot reconnect to a replacement owner. Download +responses own their source streams, and socket reads observe receiver closure. +Cancellation, source failure or owner closure ends the transfer I/O; +a missing owner never permits an ordinary backend fallback. + +The initial private allocation holds a deny-only bootstrap policy lease. There +is one narrow exception to ordinary policy replacement: a NEVER-BOUND private +allocation with no flow work can transfer that lease to its final policy. The +private ingress checks the complete allocation, old policy and current session +Arc. Under the existing per-session apply mutex it closes admission, drops its +own lease, drains every other lease, checks the cancellation generation, then +publishes the new policy and acquires the exact current lease before ACK. An +unrelated close, poison, cancellation or unknown outcome leaves admission +closed. No transfer is permitted after Bind. Ordinary policy replacement keeps +its existing drain and terminal-owner semantics. The private control protocol +is version 5; mixed binaries cannot be adopted. + +The guest remains stopped through this transfer. VZ requires the actual helper +Bind ACK for the exact epoch and the retained native descriptor, MAC and MTU. +Firecracker requires that same private ACK before it removes only its two owned +CLOSED nftables tables. It then reads back the exact namespace, process, device, +route and remaining rule shape. Init replaces any existing resolver with the +selected gateway. The VZ helper comes from an explicit operator-owned path and +SHA-256; the private copy is checked before execution. + +A newer operator generation can also retire an exact never-started preparation: +expected policy and final binding hash must both be absent, the bootstrap deny +union must be monotone, and the same owner lock serializes this path against +start. The backend verifies its actual stopped state before teardown. A missing +owner or start-uncertain state cannot use this branch. The original boot key +stays immutable. Same-generation final policy handoff is unchanged. + +A live managed policy change retires the old machine and network. It preserves +an exact synced local disk cut and returns a retired-source receipt rather than +an applied-policy ACK. `ManagedRecoveryRequired` has an exact bounded gRPC +status marker, but is not retirement proof; callers must read the exact Retired +receipt. Later requests preserve that proof. Source retention latches before +native start and survives unknown replies, Drop and close until explicit artifact +cleanup. Both backends release a recorded capture source or an authorized known +live source only after VM and network cleanup succeeds. VZ confirms the stopped +attachment before file release; a stop or cleanup error retains the source. +Explicit capture reserves its deterministic manifest +before upload. Capture pauses and retires the source, syncs the disk, stores its +hash and size, and uploads that exact cut. Retries use the same source key, +capture ID and immutable manifest. A completed manifest write with a lost reply +is accepted only if its chunk list is identical. A path without a complete +record cannot become authority. `CrashConsistentDiskCutV1` does not promise that +guest page-cache or application writes were flushed. Recovery uses a fresh +kernel, device context and empty swap; it never restores old RAM. Physical FC +and VZ acceptance remains required before activation. + +### Managed cold asset inspection + +The coordinator selects an immutable approved disk manifest and ordered bundle +hashes before asking the authenticated host for cold asset inspection. The +version-one JSON request binds host, session and durable preparation intent. +The reply reports the actual backend and host incarnation plus verified logical +disk byte hash/size and kernel hash. Inspection creates no VM or policy lease. +Preparation checks the same bytes again. An unsupported host remains closed. +This additive gRPC method does not change the bincode wire layout. + +The host implementation reads the immutable manifest and content-addressed +chunks from its configured blob store. The enabled-image materializer publishes +all chunks before the manifest; the coordinator checks capture recoverability +before it selects a base. The managed path uses its bounded verified manifest reader instead of the +ordinary lazy-restore path. A missing chunk fails inspection. The reader validates +chunk lengths and hashes and hashes every logical disk byte, including sparse +zero ranges and a short final chunk. A manifest digest is not a disk digest. + +The host uses the same configured kernel path and bundle directory that managed +preparation uses. It checks all ordered bundle contents and reports the kernel +content hash. Preparation copies and checks the disk, kernel and bundles again +before it constructs a VM. Inspection creates no sandbox, network, helper or +policy owner. A receipt binds the exact request and current host incarnation. +The image digest in that request is an association with coordinator approval; +it is not independent host proof of catalog lineage. + +Only the pinned mutual-TLS host service exposes this operation. One bounded +inspection runs at a time without a lifecycle or inventory lock. Disk chunks +and files are read as streams; hashing yields between 64 KiB blocks. The +manifest is bounded to 8 MiB and decoded off the async executor. Cancellation +stops the read future; a pending bounded manifest decode retains the slot until +it exits. An inspection has a five-minute read deadline. File reads reject +symlinks and special files and check identity, length and modification metadata +before returning a hash. Local file and TLS tests do not prove physical VM +boot or network isolation; those acceptance gates remain separate. + +### Identity-only managed preparation and recovery bridge + +The internal policy service exposes `ReadManagedCreationState`, `PrepareManagedBoot`, +and `RecoverManagedBoot`. Each method has its own response envelope. Requests bind +only the immutable creation identity and native manifest version/digest; recovery +also binds an exact previously read source key and a bounded recovery ID. The +coordinator selects and pins approved physical inputs. Callers cannot provide a +host, disk path, kernel hash, or replacement epoch. + +The sanitized state distinguishes absent, held, preparing, boot, recovering, and +terminal. It includes the exact current boot and operation identity, but excludes +sealed agent ciphertext. Absence does not fence an in-flight reserve. Recovery +retries retain their original operation identity even when a newer boot exists. +The JSON contracts are versioned by their existing nested authorities; additive +protobuf methods preserve protocol and WIRE versions. These contracts alone do +not enable creation or prove physical host execution. + +The coordinator now commits approved image digest, base manifest, ordered bundle +hashes, and resource selection with the held reservation. Migration 0138 stores +these immutable inputs and one pending boot intent per session. Initial and +recovery operation keys are unique; revision CAS preserves the chosen sandbox, +epoch, capture ID, and private manifest reference across retries. The shared +operator gate is locked before the session. Blob and host I/O occur outside +transactions. Selection, private fork, capture, and bundle references are GC roots. + +The scanner forks the complete approved disk manifest under the pinned reference, +checks an authenticated cold-assets receipt, and claims the physical target through +the existing capacity transaction. Prepared receipts must match the inspected host +incarnation. Explicit recovery captures the exact old source before claiming a new +epoch. A detached source remains captureable only with its exact permanent Retired +receipt and unchanged host/epoch; it cannot acknowledge or replace a newer target. +Old run interruption and prompt delivery continue through the existing atomic event +and Started gates. Test fixtures exercise coordinator retry ordering and paired +metadata semantics; physical backend qualification remains a separate requirement. + +Pre-claim placement checks capacity before it saves a candidate. If that host loses +capacity or eligibility, a revision CAS can select another host and discard the old +asset receipt. The intent, sandbox, epoch, capture ID, and private manifest stay +unchanged. The store serializes this transition with claim under the session lock; +any claimed target prevents relocation, including after a lost claim reply. + +The immutable selection also stores the original nonsecret harness descriptor and +workdir with the ordered bundle hashes. Managed reservation seals the complete +original session environment in the existing session-secrets row. Finalization and +explicit recovery use those saved launch inputs, without resolving a newer image, +harness catalog generation, or creation secret. Live write authorization, operator +generation, retained denies, and exact applied binding still gate finalization and +start. Frozen launch metadata grants no permission. Migration 0138 is unchanged; +selection records without the required launch authority remain closed. diff --git a/docs/operator-mcp-publication.md b/docs/operator-mcp-publication.md new file mode 100644 index 000000000..94a549b06 --- /dev/null +++ b/docs/operator-mcp-publication.md @@ -0,0 +1,96 @@ +# Publish an external MCP operator configuration + +The coordinator can run a separate operator mTLS listener. Its client CA and +exact SHA-256 leaf pins identify the operator. Operator pins must not overlap +with orchestrator pins. Shared operator-controlled PKI roots are allowed; the +verified leaf pin and listener role determine authorization. The application and orchestrator listeners do not serve +the operator mutation service. The service accepts no caller identity header. + +The optional `publication` object in the coordinator MCP configuration contains: + +```json +{ + "listenAddr": "127.0.0.1:50063", + "serverCertificateFile": "/run/mcp-operator/server.crt", + "serverPrivateKeyFile": "/run/mcp-operator/server.key", + "operatorCaFile": "/run/mcp-operator/client-ca.crt", + "operatorFingerprints": ["REPLACE_WITH_UPPERCASE_COLON_SEPARATED_SHA256_LEAF_PIN"], + "candidateFile": "/run/mcp-operator/candidate.json", + "caFiles": { "worker-root": "/run/mcp-worker/server-ca.crt" } +} +``` + +This is a file fragment, not an enabled deployment. Omitting `publication` starts +no operator listener or scanner. Configuring it does not enable the MCP registry. +The listener can reconcile a staged generation while the registry is disabled. +All input files have a 128 KiB limit. Keep private keys in the dedicated identity +files. Candidate contents and published projections must contain only policy. + +The candidate uses the `OperatorCandidate` JSON shape in `engram-core`: + +- `version` is 1. `zones` maps every selectable zone ID to its exact revision, + `worker_digest`, sorted unique `authorities`, sorted unique `aliases`, and + `configuration`. +- `configuration` is the worker zone policy: `id`, `revision`, `mode`, + `authorities`, `aliases`, `cidrs`, `denyCidrs`, optional `ca`, and optional + `limits`. Unknown fields are rejected. Authorities are canonical DNS hosts, + with an optional nondefault port. Aliases are canonical DNS hosts. +- `deployment_identity`, `network_identity`, and `trust_identity` are distinct + identity fields. A worker digest is not evidence of network enforcement. +- `ca_content_digests` maps named CA inputs to SHA-256 digests of their actual + bytes. `caFiles` must have the same names and contents. A path hash cannot + replace a content hash. Inline zone CA contents must match one of these hashes. + +Identity digests use `sha256:` followed by 64 lowercase hexadecimal digits. The +candidate identity is the SHA-256 digest of the exact compact serialization of +`OperatorCandidate`: struct fields use their declared order, maps use sorted +keys, and nested configuration objects use sorted keys. Preserve all array order +in `configuration`. Use `OperatorCandidate::identity()` to compute the staging +identity. It includes policy changes that do not change the hostname set, such +as CIDRs, limits, CA contents, and deployment identities. The worker digest is a +separate value that its authenticated handshake must validate. + +`engram.mcp.v1.OperatorPublicationService.Stage` accepts protocol 1, an optional +`expected_generation`, and `configured_candidate_identity`. It resolves the +candidate from the operator-owned file. It does not accept policy bytes. Use an +absent expected generation for the first publication. An exact retry returns +the existing pending or completed generation. A different pending candidate or +stale generation fails. `Read` returns the durable published and pending state. +There is no Publish RPC. + +Staging closes managed creation and boot/start admission in the same transaction +that saves the candidate. The scanner enumerates all managed creations, boot +attempts, and retained managed policies, including idle, queued, and terminal +rows. It stores the union of historical denies and every candidate authority and +alias before host I/O. A policy-only generation change still advances the deny +revision once. A restart resumes from this durable requirement. + +Readiness requires an immutable managed creation with authoritative never-allocated +state, or exact permanent host evidence for its source and placement. A live +Applied receipt is not publication authority: a host or proxy restart can expire +it without changing session rows. A permanent Retired receipt proves that the old +owner is nonrunning; it is not a new applied policy ACK or a capture. A Captured receipt proves the +exact source was destroyed, but publication does not create a recovery artifact. +A held creation with no allocation or boot attempt can use its durable next-boot +requirement. Missing owners, errors, timeouts, and terminal session labels do not +prove readiness. Policy or placement changes invalidate a saved observation. +Policy-only historical rows without immutable managed creation/boot lineage remain +in the scan. Their complete deny union is retained, but they block publication, +even if a fresh live receipt is available. The scanner makes no host call for +that unsupported lineage. Such rows require a separately reviewed durable +lifecycle or destruction migration before publication can proceed. A terminal SQL +status or row deletion is not a supported workaround. Do not invent a boot or +capture record to adopt an old owner. This restriction concerns session lineage; +it does not remove support for legacy external MCP protocol transports. All +MCP-bound root and child creation must use the immutable managed creation path. + +The final transaction locks and checks all participants again before it publishes +the generation and opens admission. No host call holds a database transaction. + +The pinned orchestrator `SessionPolicyService.ReadPublishedOperator` returns a method-specific +`ReadPublishedOperatorResponse` with the same protocol-1 JSON payload only when admission is open and no candidate is +pending. It returns the published record with its complete configuration. Its +consumer must compare the worker digest to a validated worker handshake and +validate the separate deployment, network, and trust projection before selection. +Configuration hashes do not certify an enforcing CNI or physical guest isolation. +Those properties require deployment and backend acceptance tests. diff --git a/docs/testing/external-mcp-baseline.md b/docs/testing/external-mcp-baseline.md new file mode 100644 index 000000000..086563221 --- /dev/null +++ b/docs/testing/external-mcp-baseline.md @@ -0,0 +1,78 @@ +# External MCP validation baseline + +This records the primary development setup before external MCP changes on +2026-11-01. These are inherited preflight results, not new measurements of the +worker change. They are retained so later checks do not hide an existing failure. +The feature must stay disabled until the release gates pass. + +| Surface | Recorded result | Evidence boundary | +| ------------------------------- | ---------------------------------- | -------------------------------------------------------- | +| Orchestrator without database | 2,427 passed, 219 skipped | Source tests without live Postgres | +| Orchestrator with full Postgres | 2,619 passed, 18 skipped, 1 failed | Full local database suite | +| Web | 1,069 passed; build passed | Local source tests and production build | +| Native backends | Builds passed; VZ demo passed | Local native build and VZ demonstration | +| Firecracker production | Not established | No real production run; fork runners are not provisioned | + +The full Postgres suite failed in `sandbox.test.ts`: the deep-recursion case +expected a stack error (`StackError`, `InternalError`, or `RangeError`), but +observed `SandboxError`. The +focused file then passed all 15 tests. A 100-repeat recursion probe observed +`StackError`. This is an unrelated known baseline issue. Keep its assertion and +suite intact; do not use the focused pass to describe the full suite as green. + +## Verification on 2026-11-08 + +At source `c8154095`, the unchanged sandbox file passed 15 tests in one run +with Bun 1.4.4. One complete orchestrator run then passed 3,248 tests, skipped +18, and failed none across 300 files. Its command was +`bun --no-env-file --no-install test`, with no filter or concurrency override. +The deep-recursion assertion passed in that full run. The tracked orchestrator +files did not change during the run. + +The full run used two new isolated databases on the owned loopback test +cluster: one application database and one separate DBOS test database. The +existing helper verified both identities before migration or test writes. +Only those new databases were migrated or used for this check. Both were +removed after the test process and its children stopped. + +This is no reproduction of the earlier failure, not a repair claim. The +original failure remains in the baseline above, and its assertion and suite +are unchanged. The local Node version was 24.13.0; CI pins 24.22.2. This local +result does not establish current-head remote CI, provider behavior, or physical +Firecracker acceptance. + +The new `mcp-worker` package uses Node 24. Its deterministic local fixture has +real TLS and the pinned official MCP client. It covers modern and legacy +Streamable HTTP, JSON and SSE, catalog pages and schema drift, read/write +counters, held calls, transport and notification cancellation, hostile result +bodies, and OAuth metadata, PKCE, code replay, and refresh rotation. Its private +CA and loopback transport adapter are test-only. Local fixture success does not +prove isolation, live provider behavior, or production Firecracker acceptance. + +Run the package gates from the repository root: + +```sh +npm ci --prefix mcp-worker --ignore-scripts +npm run format:check --prefix mcp-worker +npm run typecheck --prefix mcp-worker +npm test --prefix mcp-worker +npm run build --prefix mcp-worker +``` + +The current worker CI job runs these commands, is selected by the path detector, +and participates in the required `CI Gate`. Workflow wiring does not establish +a passing current-head result or enable production dispatch. + +## Unreleased migration numbering + +Upstream uses migrations 0126–0128 for session-disk storage. The unreleased MCP +migrations use 0129–0138, in their original order with unchanged SQL checksums. +The migration-version test rejects duplicate numbers after later merges. + +An earlier development build applied the MCP migrations as 0126–0135. A database +with those versions is not compatible with the merged build. Keep it unchanged; +do not edit its migration ledger or bypass checksum validation. Use a fresh +isolated PostgreSQL cluster for acceptance. A new database on the old cluster is +insufficient because testkit cleanup can remove older test databases and templates. +This development reset is not a data-preserving upgrade path for a deployed MCP +installation. Such an installation requires a separate migration plan. diff --git a/mcp-worker/.gitignore b/mcp-worker/.gitignore new file mode 100644 index 000000000..849ddff3b --- /dev/null +++ b/mcp-worker/.gitignore @@ -0,0 +1 @@ +dist/ diff --git a/mcp-worker/README.md b/mcp-worker/README.md new file mode 100644 index 000000000..bf4dce64e --- /dev/null +++ b/mcp-worker/README.md @@ -0,0 +1,166 @@ +# Restricted MCP worker + +This package runs remote HTTPS MCP operations in a separate Node 24 service. +The official SDK is pinned to 2.4.0. The image pins the maintained Node 24.22.2 +release. The application and each zone keep dispatch disabled by default. + +```sh +npm ci --ignore-scripts +npm run format:check +npm run typecheck +npm test +npm run build +``` + +Tests use Node 24, OpenSSL, and Helm. Test files run serially because schema +checks have hard process budgets. Fixtures use local TLS listeners only. The +production entrypoint has no loopback bypass or alternate transport setting. + +## Internal contract + +`src/index.ts` exports the pure DTOs in `contracts.ts`, runtime request validation, +and canonical JSON/hash helpers. These modules do not import the application +store, SDK, network service, or credentials. Hashes use UTF-8 canonical JSON: +object keys sorted with JavaScript string order, array order retained, scalar +encoding from `JSON.stringify`, omitted undefined object properties, and SHA-256 +hex output. Non-JSON values are rejected. Callers in other languages must use +matching canonicalization; token-grant field names are fixed ASCII strings. + +The service accepts only mTLS on port 8443. A caller must present a certificate +signed by the configured caller CA and match an exact SHA-256 certificate pin. +The server certificate also needs the service DNS name and server-auth usage. +The client must check that name and its operator-provided CA. No public ingress, +bearer fallback, generic proxy, or guest listener is provided. + +- `GET /identity` returns protocol version, boot ID, zone ID, policy revision, + and effective policy digest. It requires an approved client certificate. +- `POST /rpc` accepts a versioned `WorkerRequest`. Orchestrator certificates can + discover, execute, validate schemas/destinations, and cancel. Coordinator + certificates can request metadata, exchange tokens, validate destinations, + and cancel. Each operation is fixed; no caller-selected method or header set + is forwarded to a provider. +- Execution scope contains the real session, run, tool-call, and immutable + binding hash. Administration scope contains the trusted request and actor + identities. Execute requires execution scope. Schema validation accepts either + scope, has no credential or destination field, and requires the orchestrator + caller role. Discovery, destination validation, and OAuth require administration scope. Cancel must + match the original scope, caller, connection revision, and credential identity. + +Before acquiring or sending a provider credential, the trusted client must +verify the worker identity against the approved zone and frozen policy digest. +The resource client in `orchestrator/src/mcp/worker-client.ts` retains that exact +TLS socket for one RPC. Create one client per operation with its cancellation +signal, and dispose it after use. An unused identity check can be replaced only +before credential acquisition. A closed socket cannot reconnect for dispatch. +The server permits two requests per socket and at most 35 seconds of idle time. +The client verifies the server CA, hostname, certificate pin, protocol, zone, +policy, and boot identity. It does not use a proxy, follow redirects, or retry. + +The session binding uses `sha256:` digests. The worker scope uses bare hexadecimal +for its binding hash. Use the strict conversion in `worker-encoding.ts`; do not +change the frozen binding or journal digest. Worker policy digests and coordinator +credential receipts stay in their original bare hexadecimal form. + +It must authorize current platform state and acquire durable global admission. +Then it creates a new intent bound to that worker boot, policy, connection +revision, credential identity, operation, exact destination, scope, expiry, +and admission lease. The immutable journal `requestDigest` is separate from +`operationDigest`, which hashes the complete transient operation, including its +credential. A routine token refresh can change the latter without changing the +binding or journal identity. Do not persist the transient operation or token. + +The worker consumes each intent before I/O and rejects reuse, expiry, wrong +boot/policy, changed payload, or exhausted local admission. Responses echo the +boot/policy identity, full credential-free intent (including lease expiry and +scope), intent ID, both digests, and lease ID for verification. +A worker restart changes the boot ID. It does not authorize a write retry. +The worker cannot verify database state: only the trusted caller can issue +admission and authorization. It has no database or KEK access. + +Discovery returns upstream DTOs, not a published catalog or a permission grant. +Schema and name fields are retained exactly. Descriptions are untrusted text. +Each input, output, and constraint document is compiled in a separate Ajv +instance inside a disposable process, with its own root and dialect. Independent +constraint documents are never wrapped in a new `allOf` root or URI registry. + +## Runtime bounds + +Every dial validates the exact canonical HTTPS authority and every A/AAAA answer, +then pins one checked address while retaining TLS SNI and HTTP authority. Public +zones reject private and reserved addresses. Internal zones permit only exact +operator hosts within approved private CIDRs. Permanent metadata, loopback, +link-local, multicast, and unspecified ranges cannot be enabled by a zone. +Operator administration denies apply in both modes. OAuth endpoints need their +own approved zone; a private MCP zone does not approve a public OAuth authority. +No redirect, automatic OAuth refresh, or alternate separate-SSE transport runs. + +Each operation owns its SDK client and credential. Legacy sessions never cross +operations, platform sessions, or credential identities. The worker accepts the +three approved protocol versions and JSON/SSE replies. It rejects unsupported +capabilities and content and does not fetch icons, links, resources, or prompts. +It uses low-level SDK requests, blocks synchronous SDK schema compilation, and +permits at most one tool POST per intent, including on protocol or auth errors. + +Limits include 256 KiB arguments, 1 MiB catalog and normalized output, 2 MiB +**decompressed** HTTP bodies, 128 tools, 128 pages, 5 seconds per connection, +60 seconds per tool by default, and a 300-second ceiling. Outer deadlines and +abort signals cover all I/O. Local active caps are 4 per session, 8 per +connection, and 32 executions per worker. Discovery has two slots, OAuth four, +and schema/destination checks four. Schema compilation also has a +shared four-child cap, including children started by execute/discover. These +local caps supplement durable admission across replicas; they do not replace it. + +Schema children have a 128 MiB heap from launch and an empty environment. +Trusted module startup has a separate five-second bound. The parent starts a +hard one-second compile/validation timer **before** sending any schema or data. +No schema enters the child during startup. Deadline, cancellation, process +failure, excess complexity, unsupported dialects, remote references, and unknown +formats fail closed. There is no schema retry or executable custom format. +The Node permission model is not used or claimed as a network sandbox. + +Execution returns an upstream outcome separately from output delivery. A valid +completed response can establish success even when unsupported content, +normalization, size, or output-schema checks prevent delivery. Malformed JSON, +a lost reply, or an HTTP binary reply after a dispatched write leaves the +outcome unknown. Neither a guest error nor cancellation proves that a provider +write failed. Cancellation aborts active I/O; it cannot undo external effects. +Client disconnect and shutdown also abort active operations. The trusted caller +must persist verified results and handle late results and unknown outcomes. + +Known literal credential echoes are redacted from text and structured output. +A discovered name or schema that contains a known secret is rejected, not +rewritten. Error responses contain fixed codes only; raw SDK errors and causes +never leave the boundary. Token-exchange replies go only to coordinator callers. +No provider payload is logged or stored on disk. This does not promise protection +against encoded echoes or a malicious server that already received a credential. + +## Deployment + +`docker/mcp-worker.Dockerfile` builds only this package. The Helm chart has one +Deployment, ClusterIP Service, ConfigMap, ServiceAccount, and NetworkPolicy per +zone. See `deploy/helm/engram/values-mcp-worker.yaml.example`. Provision separate +worker namespaces and RPC TLS secrets before enabling them. Workers get no +service-account token, workload-identity annotation, host mount, DB credential, +KEK, or application environment. They run nonroot with all capabilities dropped, +RuntimeDefault seccomp, a read-only filesystem, and core dumps disabled. Only +the operator config and scoped RPC certificate secret are mounted read-only. + +Public policies use fixed public CIDRs with reserved, metadata, and operator +administration exclusions. Internal policies use operator-supplied CIDRs and +exceptions. Startup validates each rule and refuses missing or invalid exclusions. +DNS is limited to the selected DNS namespace and pods; ingress requires both +the application namespace and trusted component selector. Each zone has its +own identity, service, and policy. Zone policy changes restart workers and change the +policy digest. There is no automatic authorization expansion. + +Kubernetes policies are additive. Operators must verify that no other matching +policy widens these routes, that the CNI enforces them, and that service NAT does +not bypass the intended exclusions. Rendered YAML and local TLS tests do not +prove effective cluster isolation or Firecracker guest isolation. Those release +gates remain required before dispatch can be enabled. + +The coordinator credential API uses `engram.mcp.v1.CredentialControlService` on +the pinned mTLS listener. Its status responses wrap `CredentialStatus`; a missing +status is an error. The credential API remains limited to 32 KiB. The separate +`engram.mcp.v1.SessionPolicyService` uses a 4 MiB create budget on that same +listener. Neither service is on the application listener or in its reflection. diff --git a/mcp-worker/package-lock.json b/mcp-worker/package-lock.json new file mode 100644 index 000000000..cb6bde17a --- /dev/null +++ b/mcp-worker/package-lock.json @@ -0,0 +1,312 @@ +{ + "name": "@engrams/mcp-worker", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@engrams/mcp-worker", + "version": "0.0.0", + "dependencies": { + "@modelcontextprotocol/client": "2.4.0", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "ipaddr.js": "2.2.0" + }, + "devDependencies": { + "@types/node": "24.10.0", + "prettier": "3.6.2", + "typescript": "5.9.3", + "yaml": "2.9.1" + }, + "engines": { + "node": ">=24 <25" + } + }, + "node_modules/@modelcontextprotocol/client": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/client/-/client-2.4.0.tgz", + "integrity": "sha512-LgrD2OphHuG12vfKFKz6SOnq7bKHQmhGJIAYdD+MZBn2npHEyry6uD8zvktodtl5CQM85MO0anUyD9JwmUmxww==", + "license": "Apache-2.0", + "dependencies": { + "@modelcontextprotocol/core": "2.4.0", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.8", + "jose": "^6.1.3", + "pkce-challenge": "^5.0.0", + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@modelcontextprotocol/core": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/core/-/core-2.4.0.tgz", + "integrity": "sha512-z8QBKqdfXLPzkLQuG4xp8hrVEMm1PZ1IgxMN2sMbJiFiAKOWXnAzogXn2W4aqIBlP1ea6U5yiqVquvUNM1Cj6Q==", + "license": "Apache-2.0", + "dependencies": { + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@types/node": { + "version": "24.10.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.0.tgz", + "integrity": "sha512-qzQZRBqkFsYyaSWXuEHc2WR9c0a0CXwiE5FWUvn7ZM+vdy1uZLfCunD38UzhuB7YN/J11ndbDBcTmOdxJo9Q7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ipaddr.js": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.2.0.tgz", + "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.15", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.15.tgz", + "integrity": "sha512-qdF6ewcZARtef63fJ9AKKZPgxw+t7USH6HLXYcD4CrdmSvDp79rpJSPFEKc90QBeGS63u1KQtN2MZQHC0yDIzA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/prettier": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", + "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/yaml": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz", + "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/zod": { + "version": "4.6.7", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.7.tgz", + "integrity": "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/mcp-worker/package.json b/mcp-worker/package.json new file mode 100644 index 000000000..e1cc8f3f4 --- /dev/null +++ b/mcp-worker/package.json @@ -0,0 +1,28 @@ +{ + "name": "@engrams/mcp-worker", + "version": "0.0.0", + "private": true, + "type": "module", + "engines": { + "node": ">=24 <25" + }, + "scripts": { + "test": "node --test --test-concurrency=1 --test-timeout=30000 test/*.test.ts", + "typecheck": "tsc --noEmit", + "build": "tsc -p tsconfig.build.json", + "format": "prettier --write .", + "format:check": "prettier --check ." + }, + "dependencies": { + "@modelcontextprotocol/client": "2.4.0", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "ipaddr.js": "2.2.0" + }, + "devDependencies": { + "@types/node": "24.10.0", + "prettier": "3.6.2", + "typescript": "5.9.3", + "yaml": "2.9.1" + } +} diff --git a/mcp-worker/src/contracts.ts b/mcp-worker/src/contracts.ts new file mode 100644 index 000000000..38bb6d322 --- /dev/null +++ b/mcp-worker/src/contracts.ts @@ -0,0 +1,140 @@ +// This wire contract has no application-store or runtime dependencies. +export const WORKER_PROTOCOL = 1; +export const PROTOCOL_VERSIONS = [ + "2026-07-28", + "2025-11-25", + "2025-06-18", +] as const; +export type JsonObject = Record; +export interface ZoneConfig { + id: string; + revision: string; + mode: "public" | "internal"; + authorities: string[]; + aliases: string[]; + cidrs: string[]; + denyCidrs: string[]; + ca?: string | undefined; + limits?: { + execute?: number; + discover?: number; + oauth?: number; + schema?: number; + toolMs?: number; + }; +} +export interface WorkerIdentity { + protocol: 1; + bootId: string; + zoneId: string; + policyDigest: string; + policyRevision: string; +} +export interface ProviderCredential { + header: "authorization" | "x-api-key" | "api-key"; + value: string; +} +export interface UpstreamTool { + name: string; + description?: string; + inputSchema: JsonObject; + outputSchema?: JsonObject; + annotations?: JsonObject; +} +export interface ToolOutput { + content: { type: "text"; text: string }[]; + structuredContent?: unknown; + isError?: boolean; +} +export interface ExecutionResult { + outcome: "succeeded" | "failed" | "outcome_unknown"; + delivery: "available" | "unavailable"; + output?: ToolOutput; + reason?: + | "unsupported_content" + | "result_limit" + | "output_schema" + | "upstream_failure"; +} +export type WorkerOperation = + | { kind: "discover"; endpoint: string; credential?: ProviderCredential } + | { + kind: "execute"; + endpoint: string; + tool: string; + arguments: JsonObject; + inputSchema: JsonObject; + outputSchema?: JsonObject; + argumentConstraintDocuments: JsonObject[]; + classification: "read" | "write"; + credential?: ProviderCredential; + } + | { kind: "validateSchemas"; documents: JsonObject[]; value?: unknown } + | { kind: "validateDestination"; endpoint: string } + | { kind: "metadata"; endpoint: string } + | { + kind: "token"; + endpoint: string; + grant: + | { + grant_type: "authorization_code"; + code: string; + redirect_uri: string; + code_verifier: string; + client_id: string; + resource: string; + client_secret?: string; + } + | { + grant_type: "refresh_token"; + refresh_token: string; + client_id: string; + resource: string; + client_secret?: string; + }; + } + | { kind: "cancel"; targetIntentId: string }; +// A trusted caller creates this only after global durable admission. The worker +// checks its bindings but has no database access to acquire or renew that lease. +export type WorkerScope = + | { + kind: "execution"; + sessionId: string; + runId: string; + toolCallId: string; + bindingHash: string; + } + | { kind: "administration"; requestId: string; actorId: string }; +export interface WorkerIntent { + id: string; + bootId: string; + zoneId: string; + policyDigest: string; + connectionId: string; + connectionRevision: number; + credentialIdentity: string; + scope: WorkerScope; + operation: WorkerOperation["kind"]; + destination: string; + requestDigest: string; + operationDigest: string; + expiresAt: number; + lease: { id: string; deploymentId: string; expiresAt: number }; +} +export interface WorkerRequest { + protocol: 1; + intent: WorkerIntent; + operation: WorkerOperation; +} +export type WorkerResponse = + | { + ok: true; + identity: WorkerIdentity; + intent: WorkerIntent; + intentId: string; + requestDigest: string; + operationDigest: string; + leaseId: string; + result: unknown; + } + | { ok: false; error: string }; diff --git a/mcp-worker/src/deployment.ts b/mcp-worker/src/deployment.ts new file mode 100644 index 000000000..0b9384047 --- /dev/null +++ b/mcp-worker/src/deployment.ts @@ -0,0 +1,69 @@ +import ipaddr from "ipaddr.js"; +import { PERMANENT_DENY_CIDRS } from "./network.ts"; +export interface EgressRule { + cidr: string; + except: string[]; +} +export const RESERVED_CIDRS = [ + "0.0.0.0/8", + "100.64.0.0/10", + "127.0.0.0/8", + "169.254.0.0/16", + "192.0.0.0/24", + "192.0.2.0/24", + "192.88.99.0/24", + "198.18.0.0/15", + "198.51.100.0/24", + "203.0.113.0/24", + "224.0.0.0/4", + "240.0.0.0/4", + "2001::/23", + "2001:db8::/32", + "2002::/16", + ...PERMANENT_DENY_CIDRS, +]; +function contains(outer: string, inner: string) { + const [a, ap] = ipaddr.parseCIDR(outer), + [b, bp] = ipaddr.parseCIDR(inner); + return a.kind() === b.kind() && ap <= bp && b.match(a, ap); +} +export function validateEgressRules( + rules: EgressRule[], + denies: string[], + allowedCidrs?: string[], + mode: "public" | "internal" = "internal", +): void { + if (!rules.length || rules.length > 128) throw Error("egress_rules_required"); + const blocked = [ + ...RESERVED_CIDRS, + ...denies, + ...(mode === "public" + ? ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7"] + : []), + ]; + for (const rule of rules) { + if ( + mode === "internal" && + allowedCidrs && + !allowedCidrs.some((cidr) => contains(cidr, rule.cidr)) + ) + throw Error("egress_exceeds_zone"); + const [address] = ipaddr.parseCIDR(rule.cidr); + if ( + address.kind() === "ipv6" && + !["2000::/3", "fc00::/7"].some((base) => contains(base, rule.cidr)) + ) + throw Error("ipv6_route_denied"); + for (const exception of rule.except) + if (!contains(rule.cidr, exception)) + throw Error("invalid_egress_exception"); + for (const deny of blocked) { + if (contains(deny, rule.cidr)) throw Error("egress_route_denied"); + if ( + contains(rule.cidr, deny) && + !rule.except.some((exception) => contains(exception, deny)) + ) + throw Error("egress_exception_required"); + } + } +} diff --git a/mcp-worker/src/encoding.ts b/mcp-worker/src/encoding.ts new file mode 100644 index 000000000..b89de981c --- /dev/null +++ b/mcp-worker/src/encoding.ts @@ -0,0 +1,29 @@ +import { createHash } from "node:crypto"; +export function object(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +export function canonical(value: unknown): string { + if (value === null || typeof value === "string" || typeof value === "boolean") + return JSON.stringify(value); + if (typeof value === "number" && Number.isFinite(value)) + return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (object(value)) + return `{${Object.keys(value) + .sort() + .filter((k) => value[k] !== undefined) + .map((k) => `${JSON.stringify(k)}:${canonical(value[k])}`) + .join(",")}}`; + throw Error("invalid_json"); +} +export function digest(value: unknown) { + return createHash("sha256").update(canonical(value)).digest("hex"); +} +export function bytes(value: unknown) { + return Buffer.byteLength(JSON.stringify(value)); +} +export function redact(text: string, secrets: string[]): string { + for (const secret of secrets) + if (secret) text = text.split(secret).join("[redacted]"); + return text; +} diff --git a/mcp-worker/src/engine.ts b/mcp-worker/src/engine.ts new file mode 100644 index 000000000..d9768239d --- /dev/null +++ b/mcp-worker/src/engine.ts @@ -0,0 +1,363 @@ +import { + Client, + StreamableHTTPClientTransport, +} from "@modelcontextprotocol/client"; +import type { Tool, FetchLike } from "@modelcontextprotocol/client"; +import { PROTOCOL_VERSIONS } from "./contracts.ts"; +import type { + WorkerOperation, + ExecutionResult, + ToolOutput, + UpstreamTool, +} from "./contracts.ts"; +import { bytes, object, redact } from "./encoding.ts"; +import { validateSchemas } from "./schema.ts"; +import { abortable, RESPONSE_BYTES } from "./network.ts"; + +export async function runOperation( + operation: WorkerOperation, + fetch: typeof globalThis.fetch, + signal: AbortSignal, +): Promise { + signal.throwIfAborted(); + if (operation.kind === "validateSchemas") + return validateSchemas(operation.documents, operation.value, signal); + if (operation.kind === "validateDestination") + throw Error("destination_requires_service"); + if (operation.kind === "cancel") throw Error("cancel_requires_service"); + if (operation.kind === "metadata" || operation.kind === "token") { + const init: RequestInit = { + method: operation.kind === "metadata" ? "GET" : "POST", + redirect: "error", + signal, + headers: { accept: "application/json" }, + }; + if (operation.kind === "token") { + init.body = new URLSearchParams(operation.grant).toString(); + init.headers = { + "content-type": "application/x-www-form-urlencoded", + accept: "application/json", + }; + } + const response = await fetch(operation.endpoint, init); + if (!response.ok) throw Error("upstream_rejected"); + if (!response.headers.get("content-type")?.includes("application/json")) + throw Error("unsupported_response"); + const result: unknown = JSON.parse(await boundedText(response)); + if (!object(result)) throw Error("invalid_response"); + return result; + } + if (operation.kind === "execute") { + if (bytes(operation.arguments) > 256 * 1024) throw Error("arguments_limit"); + if ( + !(await validateSchemas( + [operation.inputSchema, ...operation.argumentConstraintDocuments], + operation.arguments, + signal, + )) + ) + throw Error("arguments_invalid"); + if (operation.outputSchema) + await validateSchemas([operation.outputSchema], undefined, signal); + } + const secrets = + "credential" in operation && operation.credential + ? [ + operation.credential.value, + operation.credential.value.replace(/^Bearer /, ""), + ] + : []; + let verifiedOutcome: ExecutionResult["outcome"] | undefined; + let callSent = false; + let requests = 0; + let cleanupSignal: AbortSignal | undefined; + let cleanupSession: string | undefined; + const scopedFetch: FetchLike = async (input, init) => { + signal.throwIfAborted(); + const target = input instanceof Request ? input.url : String(input); + if (target !== operation.endpoint) throw Error("destination_changed"); + if (++requests > 140) throw Error("request_limit"); + const method = + init?.method ?? (input instanceof Request ? input.method : "GET"); + // Do not open background subscription streams or recover legacy sessions. + if (method === "GET") return new Response(null, { status: 405 }); + const headers = new Headers(init?.headers); + if (cleanupSignal) { + cleanupSignal.throwIfAborted(); + if ( + method !== "DELETE" || + !cleanupSession || + headers.get("mcp-session-id") !== cleanupSession + ) + throw Error("session_cleanup_denied"); + } else { + if (method !== "POST") throw Error("method_denied"); + if (typeof init?.body !== "string") throw Error("request_body_invalid"); + const message: unknown = JSON.parse(init.body); + if ( + !object(message) || + typeof message.method !== "string" || + ![ + "server/discover", + "initialize", + "notifications/initialized", + "tools/list", + "tools/call", + "notifications/cancelled", + ].includes(message.method) + ) + throw Error("rpc_denied"); + if (message.method === "tools/call") { + if (operation.kind !== "execute" || callSent) + throw Error("repeat_call_denied"); + callSent = true; + } + } + if (operation.credential) + headers.set(operation.credential.header, operation.credential.value); + // The SDK does not receive the credential through an auth provider or cache. + return fetch(target, { + ...init, + headers, + redirect: "error", + signal: AbortSignal.any([ + cleanupSignal ?? signal, + ...(init?.signal ? [init.signal] : []), + ]), + }); + }; + const client = new Client( + { name: "engrams-mcp-worker", version: "1" }, + { + capabilities: {}, + supportedProtocolVersions: [...PROTOCOL_VERSIONS], + versionNegotiation: { + mode: "auto", + probe: { maxRetries: 0, timeoutMs: 5000 }, + }, + inputRequired: { autoFulfill: false }, + // Low-level requests do not compile schemas. Fail closed if an SDK change + // ever invokes its synchronous output validator instead of our child. + jsonSchemaValidator: { + getValidator() { + throw Error("in_process_schema_forbidden"); + }, + }, + }, + ); + client.onerror = () => {}; + const transport = new StreamableHTTPClientTransport( + new URL(operation.endpoint), + { + fetch: scopedFetch, + requestInit: { redirect: "error" }, + reconnectionOptions: { + maxRetries: 0, + initialReconnectionDelay: 0, + maxReconnectionDelay: 0, + reconnectionDelayGrowFactor: 1, + }, + }, + ); + const options = { + signal, + timeout: 300000, + maxTotalTimeout: 300000, + resetTimeoutOnProgress: false, + }; + try { + await abortable( + client.connect(transport, { signal, timeout: 5000 }), + signal, + ); + const version = client.getNegotiatedProtocolVersion(); + if (!PROTOCOL_VERSIONS.some((v) => v === version)) + throw Error("protocol_denied"); + const capabilities = client.getServerCapabilities(); + if ( + !capabilities?.tools || + Object.keys(capabilities).some((k) => !["tools"].includes(k)) + ) + throw Error("capability_denied"); + if (operation.kind === "discover") { + const tools: UpstreamTool[] = []; + let cursor: string | undefined; + const cursors = new Set(); + do { + const page = await client.request( + { method: "tools/list", params: cursor ? { cursor } : {} }, + options, + ); + for (const tool of page.tools) + tools.push(discoveredTool(tool, secrets)); + if (tools.length > 128 || bytes(tools) > 1024 * 1024) + throw Error("catalog_limit"); + cursor = page.nextCursor; + if (cursor) { + if (cursors.has(cursor) || cursors.size >= 128) + throw Error("pagination_limit"); + cursors.add(cursor); + } + } while (cursor); + if (new Set(tools.map((t) => t.name)).size !== tools.length) + throw Error("duplicate_tools"); + await validateSchemas( + tools.flatMap((t) => + t.outputSchema ? [t.inputSchema, t.outputSchema] : [t.inputSchema], + ), + undefined, + signal, + ); + return tools; + } + // Avoid callTool(): it can derive headers, compile schemas, and resend calls. + const result = await client.request( + { + method: "tools/call", + params: { name: operation.tool, arguments: operation.arguments }, + }, + options, + ); + const outcome = result.isError ? "failed" : "succeeded"; + verifiedOutcome = outcome; + if (result.content.some((item) => item.type !== "text")) + return { + outcome, + delivery: "unavailable", + reason: "unsupported_content", + } satisfies ExecutionResult; + const output: ToolOutput = { + content: result.content.flatMap((item) => + item.type === "text" + ? [{ type: "text" as const, text: redact(item.text, secrets) }] + : [], + ), + ...(result.structuredContent !== undefined + ? { structuredContent: redactJson(result.structuredContent, secrets) } + : {}), + ...(result.isError !== undefined ? { isError: result.isError } : {}), + }; + if (bytes(output) > 1024 * 1024) + return { + outcome, + delivery: "unavailable", + reason: "result_limit", + } satisfies ExecutionResult; + if (operation.outputSchema && !result.isError) { + try { + if ( + result.structuredContent === undefined || + !(await validateSchemas( + [operation.outputSchema], + result.structuredContent, + signal, + )) + ) + return { + outcome, + delivery: "unavailable", + reason: "output_schema", + } satisfies ExecutionResult; + } catch { + return { + outcome, + delivery: "unavailable", + reason: "output_schema", + } satisfies ExecutionResult; + } + } + return { outcome, delivery: "available", output } satisfies ExecutionResult; + } catch { + if (operation.kind === "execute") + return { + outcome: verifiedOutcome ?? (callSent ? "outcome_unknown" : "failed"), + delivery: "unavailable", + reason: "upstream_failure", + } satisfies ExecutionResult; + throw Error("discovery_failed"); + } finally { + cleanupSession = transport.sessionId; + if (cleanupSession && !signal.aborted) { + // Legacy sessions are operation-owned. Cleanup has its own short bound + // inside the approved intent lifetime and never changes a known result. + cleanupSignal = AbortSignal.any([signal, AbortSignal.timeout(1000)]); + await abortable(transport.terminateSession(), cleanupSignal).catch( + () => {}, + ); + } + await client.close().catch(() => {}); + } +} +export function discoveredTool(tool: Tool, secrets: string[]): UpstreamTool { + if ( + containsSecret( + { + name: tool.name, + inputSchema: tool.inputSchema, + outputSchema: tool.outputSchema, + }, + secrets, + ) + ) + throw Error("secret_in_definition"); + if (tool.execution?.taskSupport && tool.execution.taskSupport !== "forbidden") + throw Error("task_tool_denied"); + return { + name: tool.name, + ...(tool.description !== undefined + ? { description: redact(tool.description, secrets) } + : {}), + inputSchema: tool.inputSchema, + ...(tool.outputSchema ? { outputSchema: tool.outputSchema } : {}), + ...(tool.annotations + ? { annotations: redactJson(tool.annotations, secrets) } + : {}), + }; +} +function redactJson(value: T, secrets: string[]): T; +function redactJson(value: unknown, secrets: string[]): unknown { + if (typeof value === "string") return redact(value, secrets); + if (Array.isArray(value)) + return value.map((item) => redactJson(item, secrets)); + if (object(value)) + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [ + redact(key, secrets), + redactJson(item, secrets), + ]), + ); + return value; +} +async function boundedText(response: Response) { + const reader = response.body?.getReader(); + if (!reader) throw Error("empty_response"); + const chunks: Uint8Array[] = []; + let length = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + length += value.length; + if (length > RESPONSE_BYTES) throw Error("response_limit"); + chunks.push(value); + } + return Buffer.concat(chunks).toString("utf8"); + } finally { + await reader.cancel().catch(() => {}); + } +} + +function containsSecret(value: unknown, secrets: string[]): boolean { + if (typeof value === "string") + return secrets.some( + (secret) => secret.length > 0 && value.includes(secret), + ); + if (Array.isArray(value)) + return value.some((item) => containsSecret(item, secrets)); + if (object(value)) + return Object.entries(value).some( + ([key, item]) => + containsSecret(key, secrets) || containsSecret(item, secrets), + ); + return false; +} diff --git a/mcp-worker/src/index.ts b/mcp-worker/src/index.ts new file mode 100644 index 000000000..fb942909a --- /dev/null +++ b/mcp-worker/src/index.ts @@ -0,0 +1,4 @@ +export type * from "./contracts.ts"; +export { WORKER_PROTOCOL, PROTOCOL_VERSIONS } from "./contracts.ts"; +export { canonical, digest } from "./encoding.ts"; +export { parseWorkerRequest } from "./validation.ts"; diff --git a/mcp-worker/src/main.ts b/mcp-worker/src/main.ts new file mode 100644 index 000000000..52182943b --- /dev/null +++ b/mcp-worker/src/main.ts @@ -0,0 +1,98 @@ +import { readFile } from "node:fs/promises"; +import type { WorkerOptions } from "./service.ts"; +import { createWorker } from "./service.ts"; +import { ZonePolicy } from "./network.ts"; +import { validateEgressRules } from "./deployment.ts"; +import { object } from "./encoding.ts"; +import type { ZoneConfig } from "./contracts.ts"; + +async function main() { + const raw: unknown = JSON.parse( + await readFile("/etc/mcp-worker/config.json", "utf8"), + ); + if ( + !object(raw) || + !isZone(raw.zone) || + !Array.isArray(raw.callers) || + !Array.isArray(raw.egressRules) + ) + throw Error("invalid_config"); + const callers = raw.callers.map((c): WorkerOptions["callers"][number] => { + if ( + !object(c) || + typeof c.fingerprint !== "string" || + !/^([A-F0-9]{2}:){31}[A-F0-9]{2}$/.test(c.fingerprint) || + (c.role !== "orchestrator" && c.role !== "coordinator") + ) + throw Error("invalid_caller"); + return { fingerprint: c.fingerprint, role: c.role }; + }); + const egress = raw.egressRules.map((rule) => { + if (!object(rule) || typeof rule.cidr !== "string" || !strings(rule.except)) + throw Error("invalid_egress"); + return { cidr: rule.cidr, except: rule.except }; + }); + validateEgressRules( + egress, + raw.zone.denyCidrs, + raw.zone.cidrs, + raw.zone.mode, + ); + const tls = { + key: await readFile("/etc/mcp-worker/tls/tls.key", "utf8"), + cert: await readFile("/etc/mcp-worker/tls/tls.crt", "utf8"), + ca: await readFile("/etc/mcp-worker/tls/ca.crt", "utf8"), + }; + const worker = createWorker({ + zone: new ZonePolicy(raw.zone), + tls, + callers, + enabled: process.env.MCP_WORKER_ENABLED === "true", + }); + await worker.listen(); + for (const signal of ["SIGTERM", "SIGINT"] as const) + process.once(signal, () => { + void worker.close().then(() => process.exit(0)); + }); +} +function strings(v: unknown): v is string[] { + return Array.isArray(v) && v.every((s) => typeof s === "string"); +} +function isZone(v: unknown): v is ZoneConfig { + if ( + !object(v) || + typeof v.id !== "string" || + typeof v.revision !== "string" || + !["public", "internal"].includes(String(v.mode)) || + !strings(v.authorities) || + !strings(v.aliases) || + !strings(v.cidrs) || + !strings(v.denyCidrs) || + (v.ca !== undefined && typeof v.ca !== "string") + ) + return false; + if ( + v.limits !== undefined && + (!object(v.limits) || + Object.entries(v.limits).some( + ([key, value]) => + !["execute", "discover", "oauth", "schema", "toolMs"].includes(key) || + !Number.isSafeInteger(value) || + Number(value) < 1, + )) + ) + return false; + return true; +} +process.on("uncaughtException", () => { + process.stderr.write("worker_failed\n"); + process.exit(1); +}); +process.on("unhandledRejection", () => { + process.stderr.write("worker_failed\n"); + process.exit(1); +}); +void main().catch(() => { + process.stderr.write("worker_start_failed\n"); + process.exit(1); +}); diff --git a/mcp-worker/src/network.ts b/mcp-worker/src/network.ts new file mode 100644 index 000000000..5ebb60f45 --- /dev/null +++ b/mcp-worker/src/network.ts @@ -0,0 +1,352 @@ +import { resolve4, resolve6 } from "node:dns/promises"; +import { request as httpsRequest } from "node:https"; +import type { RequestOptions } from "node:https"; +import type { ClientRequest } from "node:http"; +import type { Transform } from "node:stream"; +import { createGunzip, createInflate, createBrotliDecompress } from "node:zlib"; +import ipaddr from "ipaddr.js"; +import { digest } from "./encoding.ts"; +import type { ZoneConfig } from "./contracts.ts"; + +export const PERMANENT_DENY_CIDRS = [ + "168.63.129.16/32", + "fd00:ec2::254/128", + "fd20:ce::254/128", + "3fff::/20", +]; +export const RESPONSE_BYTES = 2 * 1024 * 1024; +const allowedHeaders = new Set([ + "accept", + "content-type", + "authorization", + "x-api-key", + "api-key", + "mcp-session-id", + "mcp-protocol-version", + "mcp-method", + "mcp-name", + "last-event-id", +]); +export type NetworkDependencies = { + resolve: (hostname: string) => Promise; + request: (url: URL, options: RequestOptions) => ClientRequest; +}; +const defaults: NetworkDependencies = { + async resolve(hostname) { + const results = await Promise.all( + [resolve4(hostname), resolve6(hostname)].map(async (p) => { + try { + return await p; + } catch (e) { + if ( + e instanceof Error && + "code" in e && + ["ENODATA", "ENOTFOUND"].includes(String(e.code)) + ) + return []; + throw Error("dns_failed"); + } + }), + ); + return results.flat(); + }, + request: (url, options) => httpsRequest(url, options), +}; +export class ZonePolicy { + readonly config: ZoneConfig; + readonly digest: string; + constructor(config: ZoneConfig) { + this.config = structuredClone(config); + if ( + !config.id || + !config.revision || + !["public", "internal"].includes(config.mode) || + !config.authorities.length + ) + throw Error("invalid_zone"); + for (const authority of config.authorities) { + const u = new URL(`https://${authority}/`); + if ( + u.host !== authority || + !validHost(u.hostname) || + u.username || + u.password + ) + throw Error("invalid_authority"); + } + for (const alias of config.aliases) + if (!validHost(alias)) throw Error("invalid_alias"); + for (const cidr of [...config.cidrs, ...config.denyCidrs]) + ipaddr.parseCIDR(cidr); + if (config.mode === "internal") + for (const cidr of config.cidrs) { + const [address, prefix] = ipaddr.parseCIDR(cidr); + if ( + !["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7"].some( + (base) => { + const [network, bits] = ipaddr.parseCIDR(base); + return ( + address.kind() === network.kind() && + prefix >= bits && + address.match(network, bits) + ); + }, + ) + ) + throw Error("internal_cidr_denied"); + } + this.digest = digest({ + version: 1, + policy: config, + permanentDenyCidrs: PERMANENT_DENY_CIDRS, + caIdentity: config.ca ?? "system-roots", + limits: { + execute: 32, + discover: 2, + oauth: 4, + schema: 4, + toolMs: 60000, + ...config.limits, + }, + }); + for (const list of [ + this.config.authorities, + this.config.aliases, + this.config.cidrs, + this.config.denyCidrs, + ]) + Object.freeze(list); + if (this.config.limits) Object.freeze(this.config.limits); + Object.freeze(this.config); + } + endpoint(input: string): URL { + const u = new URL(input); + if ( + input !== u.href || + u.protocol !== "https:" || + u.username || + u.password || + u.search || + u.hash || + !validHost(u.hostname) || + !this.config.authorities.includes(u.host) + ) + throw Error("destination_denied"); + return u; + } + checkAddresses(addresses: string[]) { + if (!addresses.length || addresses.length > 64) throw Error("dns_denied"); + for (const raw of addresses) { + const address = ipaddr.parse(raw); + const range = address.range(); + // IPv4-mapped, translation, transition and reserved IPv6 ranges fail closed. + if (!["unicast", "private", "uniqueLocal"].includes(range)) + throw Error("address_denied"); + if ( + address.kind() === "ipv6" && + range === "unicast" && + !matches(address, "2000::/3") + ) + throw Error("address_denied"); + if ( + [...PERMANENT_DENY_CIDRS, ...this.config.denyCidrs].some((cidr) => + matches(address, cidr), + ) + ) + throw Error("address_denied"); + if (this.config.mode === "public" && range !== "unicast") + throw Error("address_denied"); + if ( + this.config.mode === "internal" && + !this.config.cidrs.some((cidr) => matches(address, cidr)) + ) + throw Error("address_denied"); + } + } +} +function validHost(host: string) { + return ( + !ipaddr.isValid(host.replace(/^\[|\]$/g, "")) && + host.length <= 253 && + host.includes(".") && + host + .split(".") + .every((s) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(s)) + ); +} +function matches(address: ipaddr.IPv4 | ipaddr.IPv6, cidr: string) { + const [network, prefix] = ipaddr.parseCIDR(cidr); + return address.kind() === network.kind() && address.match(network, prefix); +} +// Dependencies are a code-level transport seam, never an operator configuration +// bypass. Production uses only the defaults below. +export function restrictedFetch( + zone: ZonePolicy, + dependencies: NetworkDependencies = defaults, +): ( + input: Parameters[0], + init?: Parameters[1], +) => ReturnType { + return async (input, init) => { + const raw = input instanceof Request ? input.url : String(input); + const url = zone.endpoint(raw); + const source = input instanceof Request ? input : undefined; + const signal = init?.signal ?? source?.signal; + signal?.throwIfAborted(); + const headers = new Headers(init?.headers ?? source?.headers); + for (const key of headers.keys()) + if (!allowedHeaders.has(key)) throw Error("header_denied"); + headers.set("accept-encoding", "gzip, deflate, br"); + const method = init?.method ?? source?.method ?? "GET"; + if (!["GET", "POST", "DELETE"].includes(method)) + throw Error("method_denied"); + const body = init?.body ?? (source?.body ? await source.text() : undefined); + if (body !== undefined && body !== null && typeof body !== "string") + throw Error("body_denied"); + if (typeof body === "string" && Buffer.byteLength(body) > 1024 * 1024) + throw Error("request_limit"); + const connectionDeadline = performance.now() + 5000; + const connectionSignal = AbortSignal.any([ + ...(signal ? [signal] : []), + AbortSignal.timeout(5000), + ]); + const addresses = await abortable( + dependencies.resolve(url.hostname), + connectionSignal, + ); + zone.checkAddresses(addresses); + connectionSignal.throwIfAborted(); + const address = addresses[0]!; + return new Promise((resolve, reject) => { + const req = dependencies.request(url, { + method, + headers: { ...Object.fromEntries(headers), host: url.host }, + servername: url.hostname, + family: ipaddr.parse(address).kind() === "ipv4" ? 4 : 6, + rejectUnauthorized: true, + ...(zone.config.ca ? { ca: zone.config.ca } : {}), + agent: false, + // This callback returns the already checked address; it never repeats DNS. + lookup: (_host, _options, callback) => + callback( + null, + address, + ipaddr.parse(address).kind() === "ipv4" ? 4 : 6, + ), + ...(signal ? { signal } : {}), + maxHeaderSize: 16384, + }); + const timer = setTimeout( + () => req.destroy(Error("connect_deadline")), + Math.max(1, connectionDeadline - performance.now()), + ); + req.once("socket", (socket) => + socket.once("secureConnect", () => clearTimeout(timer)), + ); + req.once("error", () => { + clearTimeout(timer); + reject(Error("transport_failed")); + }); + req.once("response", (res) => { + clearTimeout(timer); + let decoder: Transform | undefined; + try { + const status = res.statusCode; + if ( + status === undefined || + !Number.isInteger(status) || + status < 200 || + status > 599 + ) + throw Error("response_invalid"); + if (status >= 300 && status < 400) { + res.destroy(); + reject(Error("redirect_denied")); + return; + } + const encoding = res.headers["content-encoding"]; + if ( + encoding && + !["identity", "gzip", "deflate", "br"].includes(encoding) + ) { + res.destroy(); + reject(Error("encoding_denied")); + return; + } + decoder = + encoding === "gzip" + ? createGunzip() + : encoding === "deflate" + ? createInflate() + : encoding === "br" + ? createBrotliDecompress() + : undefined; + const stream = decoder ? res.pipe(decoder) : res; + let received = 0; + const responseHeaders = new Headers(); + for (const [name, value] of Object.entries(res.headers)) + if ( + value !== undefined && + !["content-encoding", "content-length", "set-cookie"].includes( + name, + ) + ) + responseHeaders.set( + name, + Array.isArray(value) ? value.join(",") : value, + ); + const bodyStream = new ReadableStream({ + start(controller) { + stream.on("data", (chunk: Buffer) => { + received += chunk.length; + if (received > RESPONSE_BYTES) { + stream.destroy(Error("response_limit")); + res.destroy(); + return; + } + controller.enqueue(chunk); + }); + stream.once("end", () => controller.close()); + stream.once("error", () => + controller.error(Error("response_failed")), + ); + res.once("error", () => { + if (decoder) decoder.destroy(Error("response_failed")); + }); + }, + cancel() { + stream.destroy(); + res.destroy(); + req.destroy(); + }, + }); + resolve( + new Response([204, 205, 304].includes(status) ? null : bodyStream, { + status, + headers: responseHeaders, + }), + ); + } catch { + decoder?.destroy(); + res.destroy(); + req.destroy(); + reject(Error("response_invalid")); + } + }); + req.end(body); + }); + }; +} +export async function abortable( + promise: Promise, + signal: AbortSignal, +): Promise { + signal.throwIfAborted(); + return new Promise((resolve, reject) => { + const abort = () => reject(Error("operation_aborted")); + signal.addEventListener("abort", abort, { once: true }); + promise + .then(resolve, reject) + .finally(() => signal.removeEventListener("abort", abort)); + }); +} diff --git a/mcp-worker/src/schema-child.ts b/mcp-worker/src/schema-child.ts new file mode 100644 index 000000000..010056cff --- /dev/null +++ b/mcp-worker/src/schema-child.ts @@ -0,0 +1,101 @@ +import { getHeapStatistics } from "node:v8"; +import { fullFormats } from "ajv-formats/dist/formats.js"; +import { Ajv } from "ajv"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { object } from "./encoding.ts"; +// No network loader, custom keyword, executable format, coercion or mutation. +if (getHeapStatistics().heap_size_limit > 128 * 1024 * 1024) process.exit(1); +process.stdout.write("ready\n"); +let raw = ""; +process.stdin.setEncoding("utf8"); +for await (const part of process.stdin) { + raw += String(part); + if (raw.length > 1024 * 1024) process.exit(1); +} +try { + const request: unknown = JSON.parse(raw); + if (!object(request) || !Array.isArray(request.documents)) throw Error(); + let valid = true; + for (const document of request.documents) { + let nodes = 0; + function guard(node: unknown, depth = 0) { + if (++nodes > 10000 || depth > 64) throw Error(); + if (typeof node === "boolean") return; + if (!object(node)) throw Error(); + if ( + node.$schema !== undefined && + ![ + "http://json-schema.org/draft-07/schema#", + "http://json-schema.org/draft-07/schema", + "https://json-schema.org/draft/2020-12/schema", + ].includes(String(node.$schema)) + ) + throw Error(); + for (const key of ["$ref", "$dynamicRef", "$recursiveRef"]) + if ( + node[key] !== undefined && + (typeof node[key] !== "string" || !node[key].startsWith("#")) + ) + throw Error(); + for (const key of [ + "properties", + "patternProperties", + "$defs", + "definitions", + "dependentSchemas", + ]) + if (object(node[key])) + for (const schema of Object.values(node[key])) + guard(schema, depth + 1); + for (const key of ["allOf", "anyOf", "oneOf", "prefixItems"]) + if (Array.isArray(node[key])) + for (const schema of node[key]) guard(schema, depth + 1); + for (const key of [ + "items", + "additionalProperties", + "additionalItems", + "contains", + "propertyNames", + "not", + "if", + "then", + "else", + "unevaluatedProperties", + "unevaluatedItems", + ]) + if (node[key] !== undefined) { + if (Array.isArray(node[key])) + for (const schema of node[key]) guard(schema, depth + 1); + else guard(node[key], depth + 1); + } + if (object(node.dependencies)) + for (const schema of Object.values(node.dependencies)) + if (!Array.isArray(schema)) guard(schema, depth + 1); + } + guard(document); + const schema = object(document) ? document : {}; + const Constructor = + schema.$schema === "https://json-schema.org/draft/2020-12/schema" + ? Ajv2020 + : Ajv; + const ajv = new Constructor({ + strictSchema: true, + strictTypes: false, + strictTuples: false, + strictRequired: false, + allErrors: false, + validateFormats: true, + coerceTypes: false, + useDefaults: false, + removeAdditional: false, + inlineRefs: false, + }); + for (const [name, format] of Object.entries(fullFormats)) + ajv.addFormat(name, format); + const check = ajv.compile(document); + if ("value" in request && !check(request.value)) valid = false; + } + process.stdout.write(String(valid)); +} catch { + process.exitCode = 1; +} diff --git a/mcp-worker/src/schema.ts b/mcp-worker/src/schema.ts new file mode 100644 index 000000000..9923f4632 --- /dev/null +++ b/mcp-worker/src/schema.ts @@ -0,0 +1,63 @@ +import childProcess from "node:child_process"; +let activeChildren = 0; +export async function validateSchemas( + documents: unknown[], + value?: unknown, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + const payload = JSON.stringify({ + documents, + ...(value === undefined ? {} : { value }), + }); + if (documents.length > 384 || Buffer.byteLength(payload) > 1024 * 1024) + throw Error("schema_limit"); + if (activeChildren >= 4) throw Error("schema_capacity"); + const file = new URL( + import.meta.url.endsWith(".ts") ? "./schema-child.ts" : "./schema-child.js", + import.meta.url, + ); + activeChildren++; + return new Promise((resolve, reject) => { + const child = childProcess.spawn( + process.execPath, + ["--max-old-space-size=125", "--max-semi-space-size=1", file.pathname], + { env: {}, stdio: ["pipe", "pipe", "ignore"], cwd: "/" }, + ); + let output = ""; + let failure: string | undefined; + const kill = (reason: string) => { + failure = reason; + child.kill("SIGKILL"); + }; + let ready = false; + let timer = setTimeout(() => kill("schema_bootstrap_deadline"), 5000); + const abort = () => kill("schema_aborted"); + signal?.addEventListener("abort", abort, { once: true }); + child.stdout.on("data", (data: Buffer) => { + output += data.toString(); + if (!ready && output.startsWith("ready\n")) { + ready = true; + output = output.slice(6); + clearTimeout(timer); + timer = setTimeout(() => kill("schema_deadline"), 1000); + // Start the hard schema budget before untrusted bytes enter the child. + child.stdin.end(payload); + } + if (output.length > 1024) kill("schema_output_limit"); + }); + child.stdin.on("error", () => {}); + child.once("error", () => { + failure = "schema_process"; + }); + child.once("close", (code) => { + activeChildren--; + clearTimeout(timer); + signal?.removeEventListener("abort", abort); + if (failure) reject(Error(failure)); + else if (code !== 0 || !["true", "false"].includes(output)) + reject(Error("schema_invalid")); + else resolve(output === "true"); + }); + }); +} diff --git a/mcp-worker/src/service.ts b/mcp-worker/src/service.ts new file mode 100644 index 000000000..651c6d5c2 --- /dev/null +++ b/mcp-worker/src/service.ts @@ -0,0 +1,283 @@ +import { parseWorkerRequest } from "./validation.ts"; +import { createServer } from "node:https"; +import type { ServerOptions } from "node:https"; +import type { IncomingMessage } from "node:http"; +import { TLSSocket } from "node:tls"; +import { randomUUID } from "node:crypto"; +import { resolve4, resolve6 } from "node:dns/promises"; +import { ZonePolicy, restrictedFetch, abortable } from "./network.ts"; +import type { NetworkDependencies } from "./network.ts"; +import { digest, object } from "./encoding.ts"; +import { runOperation } from "./engine.ts"; +import type { + WorkerIdentity, + WorkerRequest, + WorkerOperation, + WorkerResponse, + WorkerIntent, +} from "./contracts.ts"; +export interface WorkerOptions { + zone: ZonePolicy; + tls: Pick; + callers: { fingerprint: string; role: "orchestrator" | "coordinator" }[]; + enabled?: boolean; + network?: NetworkDependencies; +} +export function createWorker(options: WorkerOptions) { + if (Number(process.versions.node.split(".")[0]) !== 24) + throw Error("node_24_required"); + const identity: WorkerIdentity = { + protocol: 1, + bootId: randomUUID(), + zoneId: options.zone.config.id, + policyDigest: options.zone.digest, + policyRevision: options.zone.config.revision, + }; + const used = new Map(); + const leases = new Map(); + const active = new Map< + string, + { + controller: AbortController; + intent: WorkerIntent; + principal: string; + bucket: string; + } + >(); + const fetch = restrictedFetch(options.zone, options.network); + const server = createServer( + { + ...options.tls, + requestCert: true, + rejectUnauthorized: true, + minVersion: "TLSv1.3", + maxHeaderSize: 8192, + }, + (req, res) => { + void handle(req).then( + ({ status, body }) => { + res.writeHead(status, { + "content-type": "application/json", + "cache-control": "no-store", + }); + res.end(JSON.stringify(body)); + }, + () => { + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: "invalid_request" })); + }, + ); + }, + ); + server.maxConnections = 128; + server.requestTimeout = 5000; + server.headersTimeout = 5000; + server.timeout = 305000; + // A resource client verifies identity, then sends one RPC on that same TLS socket. + // This keeps credentials on the verified worker boot, including during a roll. + server.maxRequestsPerSocket = 2; + server.keepAliveTimeout = 35000; + server.keepAliveTimeoutBuffer = 0; + async function handle( + req: IncomingMessage, + ): Promise<{ status: number; body: unknown }> { + const deny = (status: number, error: string) => ({ + status, + body: { ok: false, error }, + }); + if (!(req.socket instanceof TLSSocket) || !req.socket.authorized) + return deny(403, "caller_denied"); + const principal = req.socket.getPeerCertificate().fingerprint256; + const caller = options.callers.find((c) => c.fingerprint === principal); + if (!caller) return deny(403, "caller_denied"); + if (req.method === "GET" && req.url === "/identity") + return { status: 200, body: identity }; + if (req.method !== "POST" || req.url !== "/rpc") + return deny(404, "route_denied"); + if (!options.enabled) return deny(503, "worker_disabled"); + const request = parseWorkerRequest(await readBody(req)); + const { intent, operation } = request; + const now = Date.now(); + for (const [id, expiry] of used) if (expiry <= now) used.delete(id); + for (const [id, expiry] of leases) if (expiry <= now) leases.delete(id); + if ( + intent.bootId !== identity.bootId || + intent.zoneId !== identity.zoneId || + intent.policyDigest !== identity.policyDigest || + intent.operation !== operation.kind || + intent.destination !== + ("endpoint" in operation ? operation.endpoint : "") || + intent.operationDigest !== digest(operation) || + intent.expiresAt <= now || + intent.expiresAt > now + 300000 || + intent.lease.expiresAt < intent.expiresAt || + used.has(intent.id) || + leases.has(intent.lease.id) || + used.size >= 4096 || + leases.size >= 4096 || + intent.lease.expiresAt > now + 300000 + ) + return deny(403, "intent_denied"); + const allowed = + caller.role === "coordinator" + ? ["metadata", "token", "validateDestination", "cancel"] + : [ + "discover", + "execute", + "cancel", + "validateSchemas", + "validateDestination", + ]; + if (!allowed.includes(operation.kind)) return deny(403, "operation_denied"); + if ("endpoint" in operation) options.zone.endpoint(operation.endpoint); + // Consume before admission or I/O. Failed admission needs a new intent. + used.set(intent.id, intent.expiresAt); + leases.set(intent.lease.id, intent.lease.expiresAt); + const bucket = + operation.kind === "execute" + ? "execute" + : operation.kind === "discover" + ? "discover" + : operation.kind === "metadata" || operation.kind === "token" + ? "oauth" + : operation.kind === "cancel" + ? "cancel" + : "schema"; + const limits = options.zone.config.limits; + const caps = { + execute: Math.min(limits?.execute ?? 32, 32), + discover: Math.min(limits?.discover ?? 2, 8), + oauth: Math.min(limits?.oauth ?? 4, 8), + schema: Math.min(limits?.schema ?? 4, 8), + cancel: 32, + }; + const running = [...active.values()]; + if ( + running.filter((a) => a.bucket === bucket).length >= caps[bucket] || + (bucket === "execute" && + (running.filter( + (a) => + a.bucket === "execute" && + a.intent.scope.kind === "execution" && + intent.scope.kind === "execution" && + a.intent.scope.sessionId === intent.scope.sessionId, + ).length >= 4 || + running.filter( + (a) => + a.bucket === "execute" && + a.intent.connectionId === intent.connectionId, + ).length >= 8)) + ) + return deny(429, "capacity_exceeded"); + const controller = new AbortController(); + const disconnected = () => controller.abort(); + req.socket.once("close", disconnected); + const maxMs = + bucket === "execute" + ? Math.min(limits?.toolMs ?? 60000, 300000) + : bucket === "cancel" + ? 5000 + : 10000; + const timer = setTimeout( + () => controller.abort(), + Math.min(intent.expiresAt - now, maxMs), + ); + active.set(intent.id, { controller, intent, principal, bucket }); + try { + let result: unknown; + if (operation.kind === "cancel") { + const target = active.get(operation.targetIntentId); + if ( + target && + (target.principal !== principal || + digest(target.intent.scope) !== digest(intent.scope) || + target.intent.connectionId !== intent.connectionId || + target.intent.connectionRevision !== intent.connectionRevision || + target.intent.credentialIdentity !== intent.credentialIdentity) + ) + return deny(403, "cancel_denied"); + target?.controller.abort(); + result = { cancelRequested: !!target }; + } else if (operation.kind === "validateDestination") { + const u = options.zone.endpoint(operation.endpoint); + const addresses = options.network + ? await abortable( + options.network.resolve(u.hostname), + controller.signal, + ) + : ( + await abortable( + Promise.all( + [resolve4(u.hostname), resolve6(u.hostname)].map( + async (p) => { + try { + return await p; + } catch (e) { + if ( + e instanceof Error && + "code" in e && + ["ENODATA", "ENOTFOUND"].includes(String(e.code)) + ) + return []; + throw Error("dns_failed"); + } + }, + ), + ), + controller.signal, + ) + ).flat(); + options.zone.checkAddresses(addresses); + result = { approved: true }; + } else result = await runOperation(operation, fetch, controller.signal); + const body: WorkerResponse = { + ok: true, + identity, + intent, + intentId: intent.id, + requestDigest: intent.requestDigest, + operationDigest: intent.operationDigest, + leaseId: intent.lease.id, + result, + }; + return { status: 200, body }; + } catch { + return deny(422, "operation_failed"); + } finally { + clearTimeout(timer); + req.socket.removeListener("close", disconnected); + controller.abort(); + active.delete(intent.id); + } + } + return { + identity, + async listen(port = 8443, host = "0.0.0.0") { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, host, resolve); + }); + const a = server.address(); + if (!a || typeof a === "string") throw Error("listen_failed"); + return a.port; + }, + async close() { + for (const item of active.values()) item.controller.abort(); + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((e) => (e ? reject(e) : resolve())), + ); + }, + }; +} +async function readBody(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + let length = 0; + for await (const chunk of req) { + const part = Buffer.from(chunk); + length += part.length; + if (length > 1536 * 1024) throw Error("request_limit"); + chunks.push(part); + } + return JSON.parse(Buffer.concat(chunks).toString("utf8")); +} diff --git a/mcp-worker/src/validation.ts b/mcp-worker/src/validation.ts new file mode 100644 index 000000000..d33539b66 --- /dev/null +++ b/mcp-worker/src/validation.ts @@ -0,0 +1,158 @@ +import { object } from "./encoding.ts"; +import type { + WorkerIntent, + WorkerOperation, + WorkerRequest, + WorkerScope, +} from "./contracts.ts"; +function text(value: unknown): value is string { + return typeof value === "string" && value.length > 0 && value.length <= 8192; +} +function schema(value: unknown): value is Record { + return object(value); +} +function schemas(value: unknown): value is Record[] { + return Array.isArray(value) && value.length <= 384 && value.every(schema); +} +function operationValid(value: unknown): value is WorkerOperation { + if (!object(value) || !text(value.kind)) return false; + if ( + "credential" in value && + (!object(value.credential) || + typeof value.credential.header !== "string" || + !["authorization", "api-key", "x-api-key"].includes( + String(value.credential.header), + ) || + !text(value.credential.value) || + /[\r\n]/.test(value.credential.value)) + ) + return false; + if ( + [ + "discover", + "execute", + "metadata", + "token", + "validateDestination", + ].includes(value.kind) && + !text(value.endpoint) + ) + return false; + switch (value.kind) { + case "discover": + case "metadata": + case "validateDestination": + return true; + case "cancel": + return text(value.targetIntentId); + case "validateSchemas": + return ( + schemas(value.documents) && + Object.keys(value).every((key) => + ["kind", "documents", "value"].includes(key), + ) + ); + case "execute": + return ( + text(value.tool) && + object(value.arguments) && + schema(value.inputSchema) && + (value.outputSchema === undefined || schema(value.outputSchema)) && + schemas(value.argumentConstraintDocuments) && + (value.classification === "read" || value.classification === "write") + ); + case "token": { + const g = value.grant; + if ( + !object(g) || + !text(g.client_id) || + !text(g.resource) || + (g.client_secret !== undefined && !text(g.client_secret)) + ) + return false; + const base = ["grant_type", "client_id", "resource", "client_secret"]; + if (g.grant_type === "authorization_code") + return ( + text(g.code) && + text(g.redirect_uri) && + text(g.code_verifier) && + Object.keys(g).every((k) => + [...base, "code", "redirect_uri", "code_verifier"].includes(k), + ) + ); + if (g.grant_type === "refresh_token") + return ( + text(g.refresh_token) && + Object.keys(g).every((k) => [...base, "refresh_token"].includes(k)) + ); + return false; + } + default: + return false; + } +} +function intentValid(i: unknown): i is WorkerIntent { + if (!object(i)) return false; + for (const key of [ + "id", + "bootId", + "zoneId", + "policyDigest", + "connectionId", + "credentialIdentity", + "operation", + "requestDigest", + "operationDigest", + ]) + if (!text(i[key])) return false; + return ( + scopeValid(i.scope) && + typeof i.destination === "string" && + Number.isSafeInteger(i.connectionRevision) && + Number(i.connectionRevision) >= 1 && + Number.isSafeInteger(i.expiresAt) && + object(i.lease) && + text(i.lease.id) && + text(i.lease.deploymentId) && + Number.isSafeInteger(i.lease.expiresAt) + ); +} +export function parseWorkerRequest(value: unknown): WorkerRequest { + if ( + !object(value) || + value.protocol !== 1 || + !intentValid(value.intent) || + !operationValid(value.operation) + ) + throw Error("invalid_request"); + if (value.intent.operation !== value.operation.kind) + throw Error("operation_mismatch"); + if ( + value.operation.kind === "execute" && + value.intent.scope.kind !== "execution" + ) + throw Error("execution_scope_required"); + if ( + !["execute", "cancel", "validateSchemas"].includes(value.operation.kind) && + value.intent.scope.kind !== "administration" + ) + throw Error("administration_scope_required"); + return { protocol: 1, intent: value.intent, operation: value.operation }; +} + +function scopeValid(value: unknown): value is WorkerScope { + if (!object(value)) return false; + if (value.kind === "execution") + return ( + text(value.sessionId) && + text(value.runId) && + text(value.toolCallId) && + typeof value.bindingHash === "string" && + /^[a-f0-9]{64}$/.test(value.bindingHash) + ); + return ( + value.kind === "administration" && + text(value.requestId) && + text(value.actorId) + ); +} diff --git a/mcp-worker/test/contracts.test.ts b/mcp-worker/test/contracts.test.ts new file mode 100644 index 000000000..bcb7eb503 --- /dev/null +++ b/mcp-worker/test/contracts.test.ts @@ -0,0 +1,104 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { digest, canonical, parseWorkerRequest } from "../src/index.ts"; +import type { WorkerRequest } from "../src/contracts.ts"; +const request: WorkerRequest = { + protocol: 1, + intent: { + id: "intent", + bootId: "boot", + zoneId: "zone", + policyDigest: "a".repeat(64), + connectionId: "connection", + connectionRevision: 1, + credentialIdentity: "credential", + scope: { kind: "administration", requestId: "request", actorId: "actor" }, + operation: "validateSchemas", + destination: "", + requestDigest: "b".repeat(64), + operationDigest: "c".repeat(64), + expiresAt: 100, + lease: { id: "lease", deploymentId: "deployment", expiresAt: 100 }, + }, + operation: { kind: "validateSchemas", documents: [{}] }, +}; +test("wire contract has distinct administration and execution scopes", () => { + assert.deepEqual(parseWorkerRequest(request), request); + assert.throws(() => + parseWorkerRequest({ + ...request, + operation: { + kind: "execute", + endpoint: "https://tools.example/mcp", + tool: "write", + arguments: {}, + inputSchema: {}, + argumentConstraintDocuments: [], + classification: "write", + }, + }), + ); + const execution = { + ...request, + intent: { + ...request.intent, + scope: { + kind: "execution" as const, + sessionId: "session", + runId: "run", + toolCallId: "call", + bindingHash: "d".repeat(64), + }, + }, + }; + assert.deepEqual(parseWorkerRequest(execution), execution); + for (const field of ["sessionId", "runId", "toolCallId", "bindingHash"]) { + assert.throws(() => + parseWorkerRequest({ + ...execution, + intent: { + ...execution.intent, + scope: { ...execution.intent.scope, [field]: "" }, + }, + }), + ); + } + for (const extra of [ + { credential: { header: "authorization", value: "Bearer private" } }, + { endpoint: "https://tools.example/mcp" }, + ]) { + assert.throws(() => + parseWorkerRequest({ + ...execution, + operation: { ...execution.operation, ...extra }, + }), + ); + } + for (const kind of ["discover", "validateDestination", "metadata"]) { + assert.throws(() => + parseWorkerRequest({ + ...execution, + intent: { ...execution.intent, operation: kind }, + operation: { kind, endpoint: "https://tools.example/mcp" }, + }), + ); + } + assert.throws(() => + parseWorkerRequest({ + ...request, + intent: { ...request.intent, operation: "unknown" }, + }), + ); +}); +test("canonical JSON sorts only object keys and operation digest changes with credential bytes", () => { + assert.equal( + canonical({ z: [2, 1], a: { c: true, b: "text" }, omitted: undefined }), + '{"a":{"b":"text","c":true},"z":[2,1]}', + ); + assert.equal(digest({ a: 1, b: 2 }), digest({ b: 2, a: 1 })); + assert.notEqual( + digest({ credential: { value: "old" } }), + digest({ credential: { value: "new" } }), + ); + assert.throws(() => canonical({ value: Infinity })); +}); diff --git a/mcp-worker/test/deployment.test.ts b/mcp-worker/test/deployment.test.ts new file mode 100644 index 000000000..c783dc268 --- /dev/null +++ b/mcp-worker/test/deployment.test.ts @@ -0,0 +1,205 @@ +import assert from "node:assert/strict"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseAllDocuments } from "yaml"; +import { test } from "node:test"; +import { mkdtempSync, writeFileSync, rmSync, readFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { validateEgressRules, RESERVED_CIDRS } from "../src/deployment.ts"; +test("operator rules cannot open metadata, loopback or platform administration", () => { + assert.throws(() => + validateEgressRules([{ cidr: "0.0.0.0/0", except: [] }], []), + ); + assert.throws(() => + validateEgressRules([{ cidr: "fd00::/8", except: [] }], []), + ); + assert.throws(() => + validateEgressRules([{ cidr: "10.0.0.0/8", except: [] }], ["10.1.0.0/16"]), + ); + validateEgressRules( + [{ cidr: "10.0.0.0/8", except: ["10.1.0.0/16"] }], + ["10.1.0.0/16"], + ); + assert.throws(() => + validateEgressRules([{ cidr: "169.254.169.254/32", except: [] }], []), + ); +}); +test("worker chart defaults off and image pins maintained Node 24", () => { + const chart = new URL("../../deploy/helm/engram", import.meta.url).pathname; + const rendered = execFileSync( + "helm", + ["template", "test", chart, "-f", `${chart}/ci/render-values.yaml`], + { encoding: "utf8", env: { ...process.env, HELM_PLUGINS: "/nonexistent" } }, + ); + assert.equal(rendered.includes("component: mcp-worker"), false); + const dockerfile = readFileSync( + new URL("../../docker/mcp-worker.Dockerfile", import.meta.url), + "utf8", + ); + assert.ok(dockerfile.includes("node:24.22.2-bookworm-slim")); + assert.ok(dockerfile.includes("ulimit -c 0")); +}); + +test("orchestrator image includes locked worker production dependencies beside worker source", () => { + const dockerfile = readFileSync( + new URL("../../docker/orchestrator.Dockerfile", import.meta.url), + "utf8", + ); + const stages = dockerfile.split(/^FROM /m); + const deps = stages.find((stage) => + stage.startsWith("node:24.22.2-bookworm-slim AS worker-deps\n"), + ); + assert.ok(deps, "worker dependencies need a separate locked install stage"); + assert.match(deps, /^WORKDIR \/app\/mcp-worker$/m); + assert.match( + deps, + /^COPY mcp-worker\/package.json mcp-worker\/package-lock.json \.\/$/m, + ); + assert.match(deps, /^RUN npm ci --omit=dev --ignore-scripts$/m); + const runtime = stages.at(-1)!; + assert.match( + runtime, + /^COPY --from=worker-deps \/app\/mcp-worker\/node_modules \/app\/mcp-worker\/node_modules$/m, + ); + assert.match( + runtime, + /^COPY --from=worker-deps \/app\/mcp-worker\/package.json \/app\/mcp-worker\/package-lock.json \/app\/mcp-worker\/$/m, + ); + assert.match(runtime, /^COPY mcp-worker\/src\/ \/app\/mcp-worker\/src\/$/m); + assert.match(runtime, /^USER bun$/m); + assert.match(runtime, /^CMD \["bun", "run", "start"\]$/m); +}); + +test("both zone deployments have fixed identity, mounts, namespace and network limits", (t) => { + const chart = new URL("../../deploy/helm/engram", import.meta.url).pathname; + const dir = mkdtempSync(join(tmpdir(), "mcp-render-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const zone = { + namespace: "worker-tools", + isolationConfirmed: true, + dispatchEnabled: false, + tlsSecret: "rpc-tls", + callers: [{ fingerprint: "fixture", role: "orchestrator" }], + policy: { + id: "private", + revision: "1", + mode: "internal", + authorities: ["tools.example"], + aliases: [], + cidrs: ["10.20.0.0/16"], + denyCidrs: ["10.20.0.0/24"], + }, + egressRules: [{ cidr: "10.20.0.0/16", except: ["10.20.0.0/24"] }], + ports: [443], + dns: { namespace: "kube-system", podLabels: { "k8s-app": "kube-dns" } }, + }; + const values = { + mcpWorker: { + enabled: true, + zones: [ + zone, + { + ...zone, + namespace: "worker-public", + policy: { + ...zone.policy, + id: "public", + mode: "public", + cidrs: [], + denyCidrs: ["8.8.4.0/24"], + }, + }, + ], + }, + }; + const file = join(dir, "values.json"); + writeFileSync(file, JSON.stringify(values)); + const rendered = execFileSync( + "helm", + [ + "template", + "test", + chart, + "-f", + `${chart}/ci/render-values.yaml`, + "-f", + file, + ], + { encoding: "utf8", env: { ...process.env, HELM_PLUGINS: "/nonexistent" } }, + ); + const docs = parseAllDocuments(rendered) + .map((d) => d.toJS()) + .filter((d) => d?.metadata?.namespace?.startsWith("worker-")); + assert.equal(docs.length, 10); + for (const d of docs.filter((d) => d.kind === "Deployment")) { + const spec = d.spec.template.spec; + assert.equal(spec.automountServiceAccountToken, false); + assert.equal(spec.enableServiceLinks, false); + assert.equal(spec.securityContext.runAsNonRoot, true); + assert.equal(spec.securityContext.seccompProfile.type, "RuntimeDefault"); + assert.equal(spec.containers.length, 1); + const c = spec.containers[0]; + assert.equal(c.securityContext.readOnlyRootFilesystem, true); + assert.equal(c.securityContext.allowPrivilegeEscalation, false); + assert.deepEqual(c.securityContext.capabilities.drop, ["ALL"]); + assert.deepEqual(c.env, [{ name: "MCP_WORKER_ENABLED", value: "false" }]); + assert.equal(spec.volumes.length, 2); + assert.ok(spec.volumes.every((v: { hostPath?: unknown }) => !v.hostPath)); + assert.equal(spec.hostNetwork, undefined); + } + for (const d of docs.filter((d) => d.kind === "NetworkPolicy")) { + assert.equal(d.spec.ingress[0].from.length, 1); + assert.ok(d.spec.ingress[0].from[0].namespaceSelector); + assert.ok(d.spec.ingress[0].from[0].podSelector); + assert.equal(d.spec.egress[0].to.length, 1); + assert.ok(d.spec.egress[0].to[0].namespaceSelector); + assert.ok(d.spec.egress[0].to[0].podSelector); + assert.deepEqual(d.spec.egress[1].ports, [{ protocol: "TCP", port: 443 }]); + } + for (const d of docs.filter((d) => d.kind === "ConfigMap")) { + const config = JSON.parse(d.data["config.json"]); + validateEgressRules( + config.egressRules, + config.zone.denyCidrs, + config.zone.cidrs, + config.zone.mode, + ); + } + assert.ok( + docs + .filter((d) => d.kind === "Service") + .every((d) => d.spec.type === "ClusterIP"), + ); + assert.ok( + docs + .filter((d) => d.kind === "ServiceAccount") + .every( + (d) => + d.automountServiceAccountToken === false && !d.metadata.annotations, + ), + ); +}); + +test("rendered routes cannot exceed private CIDRs or admit private ranges in public zones", () => { + assert.throws(() => + validateEgressRules( + [{ cidr: "10.0.0.0/8", except: [] }], + [], + ["10.20.0.0/16"], + "internal", + ), + ); + assert.throws(() => + validateEgressRules( + [ + { + cidr: "0.0.0.0/0", + except: RESERVED_CIDRS.filter((c) => !c.includes(":")), + }, + ], + [], + [], + "public", + ), + ); +}); diff --git a/mcp-worker/test/engine.test.ts b/mcp-worker/test/engine.test.ts new file mode 100644 index 000000000..6aa9ff478 --- /dev/null +++ b/mcp-worker/test/engine.test.ts @@ -0,0 +1,416 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { runOperation, discoveredTool } from "../src/engine.ts"; +import { startMcpFixture } from "./fixtures/server.ts"; +import type { WorkerOperation } from "../src/contracts.ts"; +const execute = ( + endpoint: string, +): Extract => ({ + kind: "execute", + endpoint, + tool: "write_counter", + arguments: { delta: 2 }, + inputSchema: { + type: "object", + properties: { delta: { type: "integer" } }, + required: ["delta"], + }, + argumentConstraintDocuments: [], + classification: "write", +}); +for (const version of ["2026-07-28", "2025-11-25", "2025-06-18"] as const) + for (const reply of ["json", "sse"] as const) + test(`worker uses SDK with bounded discovery and exact results ${version} ${reply}`, async (t) => { + const fixture = await startMcpFixture({ version, reply }); + t.after(() => fixture.close()); + const tools = await runOperation( + { kind: "discover", endpoint: fixture.url }, + fixture.fetch, + AbortSignal.timeout(20000), + ); + assert.equal(Array.isArray(tools) && tools.length, 2); + const result = await runOperation( + execute(fixture.url), + fixture.fetch, + AbortSignal.timeout(20000), + ); + assert.deepEqual(result, { + outcome: "succeeded", + delivery: "available", + output: { + content: [{ type: "text", text: "count=2" }], + structuredContent: { count: 2 }, + isError: false, + }, + }); + assert.equal(fixture.writes, 1); + if (version !== "2026-07-28") + assert.notEqual( + fixture.requests.find((r) => r.method === "tools/list")?.sessionId, + fixture.requests.find((r) => r.method === "tools/call")?.sessionId, + ); + }); +for (const version of ["2025-11-25", "2025-06-18", "2026-07-28"] as const) + test(`worker releases only its authenticated legacy session ${version}`, async (t) => { + const fixture = await startMcpFixture({ + version, + requireAuth: true, + apiKey: "cleanup-token", + }); + t.after(() => fixture.close()); + const deletions: { + target: string; + session: string | null; + credential: string | null; + }[] = []; + const deletionStatuses: number[] = []; + const fetch: typeof globalThis.fetch = async (input, init) => { + if (init?.method === "DELETE") { + const headers = new Headers(init.headers); + deletions.push({ + target: String(input), + session: headers.get("mcp-session-id"), + credential: headers.get("authorization"), + }); + } + const response = await fixture.fetch(input, init); + if (init?.method === "DELETE") deletionStatuses.push(response.status); + return response; + }; + const result = await runOperation( + { + ...execute(fixture.url), + credential: { header: "authorization", value: "Bearer cleanup-token" }, + }, + fetch, + AbortSignal.timeout(20000), + ); + assert.equal((result as { outcome: string }).outcome, "succeeded"); + assert.equal(fixture.writes, 1); + assert.deepEqual(deletionStatuses, version === "2026-07-28" ? [] : [200]); + if (version === "2026-07-28") assert.deepEqual(deletions, []); + else + assert.deepEqual(deletions, [ + { + target: fixture.url, + session: fixture.requests.find( + (request) => request.method === "tools/call", + )?.sessionId, + credential: "Bearer cleanup-token", + }, + ]); + }); + +for (const cleanup of ["refused", "failed", "stalled", "expired"] as const) + test(`legacy cleanup ${cleanup} preserves the result without replay`, async (t) => { + const fixture = await startMcpFixture({ version: "2025-11-25" }); + t.after(() => fixture.close()); + const controller = new AbortController(); + let deletions = 0; + let cleanupAborted = false; + const fetch: typeof globalThis.fetch = async (input, init) => { + if (init?.method !== "DELETE") return fixture.fetch(input, init); + deletions++; + if (cleanup === "refused") return new Response(null, { status: 405 }); + if (cleanup === "failed") throw Error("cleanup failed"); + assert.ok(init.signal); + const signal = init.signal; + const blocked = new Promise((_, reject) => { + signal.addEventListener( + "abort", + () => { + cleanupAborted = true; + reject(signal.reason); + }, + { once: true }, + ); + }); + if (cleanup === "expired") controller.abort(); + return blocked; + }; + const result = await runOperation( + execute(fixture.url), + fetch, + controller.signal, + ); + assert.deepEqual(result, { + outcome: "succeeded", + delivery: "available", + output: { + content: [{ type: "text", text: "count=2" }], + structuredContent: { count: 2 }, + isError: false, + }, + }); + assert.equal(fixture.writes, 1); + assert.equal(deletions, 1); + assert.equal( + cleanupAborted, + cleanup === "stalled" || cleanup === "expired", + ); + }); + +for (const response of [ + "image", + "audio", + "resource", + "resource_link", + "malformed", + "binary", + "error", +] as const) + test(`worker retains upstream outcome independently of ${response} delivery`, async (t) => { + const fixture = await startMcpFixture({ response }); + t.after(() => fixture.close()); + const result = await runOperation( + execute(fixture.url), + fixture.fetch, + AbortSignal.timeout(20000), + ); + assert.equal(fixture.writes, 1); + assert.ok( + typeof result === "object" && result !== null && "outcome" in result, + ); + assert.equal( + result.outcome, + ["malformed", "binary"].includes(response) + ? "outcome_unknown" + : response === "error" + ? "failed" + : "succeeded", + ); + assert.equal( + "delivery" in result && result.delivery, + response === "error" ? "available" : "unavailable", + ); + }); +test("worker rejects input before a tools call and cancels held I/O", async (t) => { + const fixture = await startMcpFixture({ holdCalls: true }); + t.after(() => fixture.close()); + await assert.rejects( + runOperation( + { + ...execute(fixture.url), + argumentConstraintDocuments: [ + { properties: { delta: { maximum: 1 } } }, + ], + }, + fixture.fetch, + AbortSignal.timeout(20000), + ), + ); + assert.equal(fixture.writes, 0); + const controller = new AbortController(); + const pending = runOperation( + execute(fixture.url), + fixture.fetch, + controller.signal, + ); + await fixture.waitForCall(); + controller.abort(); + const result = await pending; + assert.equal( + typeof result === "object" && + result !== null && + "outcome" in result && + result.outcome, + "outcome_unknown", + ); + await fixture.waitForCancellation(); + assert.equal(fixture.writes, 1); +}); + +for (const attack of [ + "header_mismatch", + "input_required", + "version_correction", + "unauthorized", +] as const) + test(`write is sent once after ${attack}`, async (t) => { + const fixture = await startMcpFixture(); + t.after(() => fixture.close()); + let calls = 0; + const fetch: typeof globalThis.fetch = async (input, init) => { + const response = await fixture.fetch(input, init); + const message = + typeof init?.body === "string" ? JSON.parse(init.body) : {}; + if (message.method !== "tools/call") return response; + calls++; + await response.body?.cancel(); + const body = + attack === "input_required" + ? { + jsonrpc: "2.0", + id: message.id, + result: { + resultType: "input_required", + inputRequests: {}, + requestState: "state", + }, + } + : { + jsonrpc: "2.0", + id: message.id, + error: { + code: attack === "header_mismatch" ? -32021 : -32022, + message: "credential-secret", + }, + }; + return new Response(JSON.stringify(body), { + status: attack === "unauthorized" ? 401 : 200, + headers: { "content-type": "application/json" }, + }); + }; + const result = await runOperation( + execute(fixture.url), + fetch, + AbortSignal.timeout(20000), + ); + assert.equal(fixture.writes, 1); + assert.equal(calls, 1); + assert.equal(JSON.stringify(result).includes("credential-secret"), false); + assert.ok( + typeof result === "object" && + result !== null && + "outcome" in result && + result.outcome === "outcome_unknown", + ); + }); +test("discovery redacts descriptions and rejects secret-bearing names and schemas", async (t) => { + const fixture = await startMcpFixture(); + t.after(() => fixture.close()); + let mode = "description"; + const fetch: typeof globalThis.fetch = async (input, init) => { + const response = await fixture.fetch(input, init); + const request = typeof init?.body === "string" ? JSON.parse(init.body) : {}; + if (request.method !== "tools/list") return response; + const body = await response.json(); + if (mode === "description") + body.result.tools[0].description = "echo provider-secret"; + if (mode === "schema") + body.result.tools[0].inputSchema.const = "provider-secret"; + if (mode === "name") body.result.tools[0].name = "provider-secret"; + return new Response(JSON.stringify(body), { + headers: { "content-type": "application/json" }, + }); + }; + const operation: WorkerOperation = { + kind: "discover", + endpoint: fixture.url, + credential: { header: "authorization", value: "Bearer provider-secret" }, + }; + const tools = await runOperation( + operation, + fetch, + AbortSignal.timeout(20000), + ); + assert.ok(Array.isArray(tools)); + assert.equal(tools[0].description, "echo [redacted]"); + for (mode of ["schema", "name"]) + await assert.rejects( + runOperation(operation, fetch, AbortSignal.timeout(20000)), + /discovery_failed/, + ); +}); +test("output schemas and normalized limits do not erase established success", async (t) => { + const fixture = await startMcpFixture(); + t.after(() => fixture.close()); + const result = await runOperation( + { + ...execute(fixture.url), + outputSchema: { type: "object", properties: { count: { const: 999 } } }, + }, + fixture.fetch, + AbortSignal.timeout(20000), + ); + assert.deepEqual(result, { + outcome: "succeeded", + delivery: "unavailable", + reason: "output_schema", + }); + const fetch: typeof globalThis.fetch = async (input, init) => { + const response = await fixture.fetch(input, init); + const request = typeof init?.body === "string" ? JSON.parse(init.body) : {}; + if (request.method !== "tools/call") return response; + const body = await response.json(); + body.result.content = [{ type: "text", text: "x".repeat(1024 * 1024) }]; + return new Response(JSON.stringify(body), { + headers: { "content-type": "application/json" }, + }); + }; + assert.deepEqual( + await runOperation(execute(fixture.url), fetch, AbortSignal.timeout(20000)), + { outcome: "succeeded", delivery: "unavailable", reason: "result_limit" }, + ); + assert.equal(fixture.writes, 2); +}); +test("secret echoes in structured output and text are redacted", async (t) => { + const fixture = await startMcpFixture(); + t.after(() => fixture.close()); + const fetch: typeof globalThis.fetch = async (input, init) => { + const response = await fixture.fetch(input, init); + const request = typeof init?.body === "string" ? JSON.parse(init.body) : {}; + if (request.method !== "tools/call") return response; + const body = await response.json(); + body.result.content = [{ type: "text", text: "echo provider-secret" }]; + body.result.structuredContent = { value: "provider-secret" }; + return new Response(JSON.stringify(body), { + headers: { "content-type": "application/json" }, + }); + }; + const result = await runOperation( + { + ...execute(fixture.url), + credential: { header: "x-api-key", value: "provider-secret" }, + }, + fetch, + AbortSignal.timeout(20000), + ); + assert.equal(JSON.stringify(result).includes("provider-secret"), false); + assert.ok(JSON.stringify(result).includes("[redacted]")); +}); +test("schema secret detection examines strings rather than serialized escapes", () => { + const secret = 'key"with\\escape'; + assert.throws( + () => + discoveredTool( + { + name: "read", + inputSchema: { + type: "object", + properties: { value: { const: secret } }, + }, + }, + [secret], + ), + /secret_in_definition/, + ); +}); +test("unusable deeply nested structured output retains established success", async (t) => { + const fixture = await startMcpFixture(); + t.after(() => fixture.close()); + const fetch: typeof globalThis.fetch = async (input, init) => { + const response = await fixture.fetch(input, init); + const request = typeof init?.body === "string" ? JSON.parse(init.body) : {}; + if (request.method !== "tools/call") return response; + await response.body?.cancel(); + const nested = '{"child":'.repeat(10000) + "0" + "}".repeat(10000); + return new Response( + `{"jsonrpc":"2.0","id":${JSON.stringify(request.id)},"result":{"resultType":"complete","content":[],"isError":false,"structuredContent":${nested}}}`, + { headers: { "content-type": "application/json" } }, + ); + }; + const result = await runOperation( + execute(fixture.url), + fetch, + AbortSignal.timeout(20000), + ); + assert.ok( + typeof result === "object" && + result !== null && + "outcome" in result && + result.outcome === "succeeded", + ); + assert.ok("delivery" in result && result.delivery === "unavailable"); + assert.equal(fixture.writes, 1); +}); diff --git a/mcp-worker/test/fixture.test.ts b/mcp-worker/test/fixture.test.ts new file mode 100644 index 000000000..cff1cd667 --- /dev/null +++ b/mcp-worker/test/fixture.test.ts @@ -0,0 +1,631 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { test } from "node:test"; +import { + Client, + StreamableHTTPClientTransport, +} from "@modelcontextprotocol/client"; +import { startMcpFixture } from "./fixtures/server.ts"; + +for (const version of ["2026-07-28", "2025-11-25", "2025-06-18"] as const) { + for (const reply of ["json", "sse"] as const) { + test(`SDK discovers pages and calls tools over TLS: ${version} ${reply}`, async (t) => { + const fixture = await startMcpFixture({ version, reply }); + t.after(() => fixture.close()); + const client = new Client( + { name: "fixture-client", version: "1" }, + { + versionNegotiation: { mode: "auto" }, + }, + ); + t.after(() => client.close()); + const transport = new StreamableHTTPClientTransport( + new URL(fixture.url), + { fetch: fixture.fetch }, + ); + await client.connect(transport); + const first = await client.request({ method: "tools/list", params: {} }); + assert.equal(first.tools[0]?.name, "read_counter"); + assert.equal( + first.tools[0]?.inputSchema.$schema, + "http://json-schema.org/draft-07/schema#", + ); + assert.equal(first.nextCursor, "page-2"); + const second = await client.listTools({ cursor: first.nextCursor }); + assert.equal(second.tools[0]?.name, "write_counter"); + assert.equal( + second.tools[0]?.inputSchema.$schema, + "https://json-schema.org/draft/2020-12/schema", + ); + assert.equal(second.nextCursor, undefined); + const read = await client.callTool({ + name: "read_counter", + arguments: {}, + }); + assert.deepEqual(read.structuredContent, { count: 0 }); + const written = await client.callTool({ + name: "write_counter", + arguments: { delta: 2 }, + }); + assert.deepEqual(written.structuredContent, { count: 2 }); + assert.deepEqual(written.content, [{ type: "text", text: "count=2" }]); + assert.equal(written.isError, false); + assert.equal(fixture.writes, 1); + assert.equal(fixture.reads, 1); + if (version === "2026-07-28") { + assert.equal( + fixture.requests.some((r) => r.method === "initialize"), + false, + ); + assert.equal(transport.sessionId, undefined); + } else { + assert.ok(fixture.requests.some((r) => r.method === "initialize")); + assert.equal(transport.sessionId, "fixture-session-1"); + assert.equal( + fixture.requests.find((r) => r.method === "tools/call")?.sessionId, + transport.sessionId, + ); + } + }); + } +} + +test("default trust rejects the private fixture CA; explicit test trust accepts it", async (t) => { + const fixture = await startMcpFixture(); + t.after(() => fixture.close()); + await assert.rejects( + fetch(`${fixture.origin}/.well-known/oauth-protected-resource`), + ); + const response = await fixture.fetch( + `${fixture.origin}/.well-known/oauth-protected-resource`, + ); + assert.equal(response.status, 200); +}); + +test("catalog revision changes description and schema on the wire", async (t) => { + const fixture = await startMcpFixture(); + t.after(() => fixture.close()); + const before = await rpc(fixture, "tools/list"); + fixture.setCatalogRevision(2); + const after = await rpc(fixture, "tools/list"); + assert.notDeepEqual(before.body, after.body); + assert.match(JSON.stringify(after.body), /revision 2/); + assert.match(JSON.stringify(after.body), /required_tag/); +}); + +test("legacy transport sessions reject a different credential identity", async (t) => { + const fixture = await startMcpFixture({ version: "2025-11-25" }); + t.after(() => fixture.close()); + const initialized = await rpc( + fixture, + "initialize", + { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "test", version: "1" }, + }, + { authorization: "Bearer account-one" }, + ); + const sessionId = initialized.response.headers.get("mcp-session-id"); + assert.equal(sessionId, "fixture-session-1"); + const response = await rpc( + fixture, + "tools/list", + {}, + { + authorization: "Bearer account-two", + "mcp-session-id": sessionId!, + }, + ); + assert.equal(response.response.status, 404); +}); + +test("modern metadata header mismatches fail before a write", async (t) => { + const fixture = await startMcpFixture(); + t.after(() => fixture.close()); + const response = await rpc( + fixture, + "tools/call", + { name: "write_counter", arguments: { delta: 1 } }, + { + "mcp-method": "tools/list", + }, + ); + assert.equal(response.response.status, 400); + assert.equal(fixture.writes, 0); +}); + +test("held calls release explicitly and transport abort records cancellation", async (t) => { + const fixture = await startMcpFixture({ holdCalls: true, reply: "sse" }); + t.after(() => fixture.close()); + const controller = new AbortController(); + const pending = fixture.fetch(fixture.url, { + method: "POST", + headers: modernHeaders("tools/call", "read_counter"), + body: JSON.stringify( + message("tools/call", { name: "read_counter", arguments: {} }), + ), + signal: controller.signal, + }); + const response = await pending; + const body = response.text(); + await fixture.waitForCall(); + controller.abort(); + await assert.rejects(body); + await fixture.waitForCancellation(); + assert.equal(fixture.cancellations, 1); + assert.equal(fixture.writes, 0); + const released = rpc(fixture, "tools/call", { + name: "read_counter", + arguments: {}, + }); + await fixture.waitForCall(); + fixture.releaseCalls(); + assert.equal((await released).response.status, 200); +}); + +for (const mode of [ + "malformed", + "oversize", + "image", + "audio", + "resource", + "resource_link", + "error", +] as const) { + test(`fixture serves ${mode} output without hiding an upstream write`, async (t) => { + const fixture = await startMcpFixture({ response: mode }); + t.after(() => fixture.close()); + const response = await fixture.fetch(fixture.url, { + method: "POST", + headers: modernHeaders("tools/call", "write_counter"), + body: JSON.stringify( + message("tools/call", { + name: "write_counter", + arguments: { delta: 1 }, + }), + ), + }); + const body = await response.text(); + assert.equal(response.status, 200); + assert.equal(fixture.writes, 1); + if (mode === "malformed") assert.throws(() => JSON.parse(body)); + else if (mode === "oversize") + assert.ok(Buffer.byteLength(body) > 2 * 1024 * 1024); + else if (mode === "error") + assert.equal(JSON.parse(body).result.isError, true); + else assert.equal(JSON.parse(body).result.content[0].type, mode); + }); +} + +test("OAuth metadata, PKCE code exchange, rotation and replay use real TLS", async (t) => { + const fixture = await startMcpFixture({ requireAuth: true }); + t.after(() => fixture.close()); + const unauthorized = await rpc(fixture, "tools/list"); + assert.equal(unauthorized.response.status, 401); + assert.match( + unauthorized.response.headers.get("www-authenticate")!, + /resource_metadata=/, + ); + const resource = await ( + await fixture.fetch( + `${fixture.origin}/.well-known/oauth-protected-resource`, + ) + ).json(); + assert.equal(resource.resource, fixture.url); + assert.deepEqual(resource.authorization_servers, [fixture.origin]); + const metadata = await ( + await fixture.fetch( + `${fixture.origin}/.well-known/oauth-authorization-server`, + ) + ).json(); + assert.equal(metadata.issuer, fixture.origin); + assert.equal(metadata.authorization_response_iss_parameter_supported, true); + assert.deepEqual(metadata.code_challenge_methods_supported, ["S256"]); + const verifier = "v".repeat(43); + const authorizationUrl = new URL(metadata.authorization_endpoint); + authorizationUrl.search = new URLSearchParams({ + response_type: "code", + client_id: "fixture-public", + redirect_uri: `${fixture.origin}/callback`, + state: "state-one", + code_challenge: createHash("sha256").update(verifier).digest("base64url"), + code_challenge_method: "S256", + resource: fixture.url, + scope: "tools:read tools:write", + }).toString(); + const authorization = await fixture.fetch(authorizationUrl, { + redirect: "manual", + }); + assert.equal(authorization.status, 302); + const callback = new URL(authorization.headers.get("location")!); + assert.equal(callback.searchParams.get("iss"), metadata.issuer); + assert.equal(callback.searchParams.get("state"), "state-one"); + const code = callback.searchParams.get("code")!; + const grant = { + grant_type: "authorization_code", + code, + code_verifier: verifier, + client_id: "fixture-public", + redirect_uri: `${fixture.origin}/callback`, + resource: fixture.url, + }; + assert.equal( + (await token(fixture, { ...grant, code_verifier: "wrong" })).status, + 400, + ); + assert.equal( + (await token(fixture, { ...grant, resource: "https://other.example/mcp" })) + .status, + 400, + ); + const exchanged = await token(fixture, grant); + assert.equal(exchanged.status, 200); + const tokens = await exchanged.json(); + assert.equal(tokens.access_token, "fixture-access-1"); + assert.equal(tokens.expires_in, 3600); + assert.equal((await token(fixture, grant)).status, 400); + assert.equal((await fixture.fetch(authorizationUrl)).status, 400); + const authorized = await rpc( + fixture, + "tools/list", + {}, + { authorization: `Bearer ${tokens.access_token}` }, + ); + assert.equal(authorized.response.status, 200); + const refresh = { + grant_type: "refresh_token", + refresh_token: tokens.refresh_token, + client_id: "fixture-public", + resource: fixture.url, + }; + const rotated = await (await token(fixture, refresh)).json(); + assert.equal(rotated.refresh_token, "fixture-refresh-2"); + assert.equal((await token(fixture, refresh)).status, 400); + assert.equal( + ( + await rpc( + fixture, + "tools/list", + {}, + { authorization: `Bearer ${tokens.access_token}` }, + ) + ).response.status, + 401, + ); +}); + +test("OAuth fixture can return a mismatched callback issuer", async (t) => { + const fixture = await startMcpFixture({ + callbackIssuer: "https://other.example", + }); + t.after(() => fixture.close()); + const url = new URL(`${fixture.origin}/authorize`); + url.search = new URLSearchParams({ + response_type: "code", + client_id: "fixture-public", + redirect_uri: `${fixture.origin}/callback`, + state: "issuer-case", + resource: fixture.url, + code_challenge: "c".repeat(43), + code_challenge_method: "S256", + }).toString(); + const response = await fixture.fetch(url); + assert.equal( + new URL(response.headers.get("location")!).searchParams.get("iss"), + "https://other.example", + ); +}); + +test("OAuth rejects missing PKCE, bad redirect, unknown client, and scope expansion", async (t) => { + const fixture = await startMcpFixture(); + t.after(() => fixture.close()); + const url = new URL(`${fixture.origin}/authorize`); + const params = { + response_type: "code", + client_id: "fixture-public", + redirect_uri: `${fixture.origin}/callback`, + state: "state", + resource: fixture.url, + code_challenge: "c".repeat(43), + code_challenge_method: "S256", + scope: "tools:read", + }; + for (const change of [ + { code_challenge_method: "plain" }, + { redirect_uri: "https://other.example/callback" }, + { client_id: "unknown" }, + { scope: "admin" }, + { resource: "https://other.example" }, + ]) { + url.search = new URLSearchParams({ ...params, ...change }).toString(); + assert.equal((await fixture.fetch(url)).status, 400); + } +}); + +function message(method: string, params: Record = {}) { + return { + jsonrpc: "2.0", + id: 1, + method, + params: { + ...params, + _meta: { + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientInfo": { + name: "fixture-client", + version: "1", + }, + "io.modelcontextprotocol/clientCapabilities": {}, + }, + }, + }; +} +function modernHeaders(method: string, name?: string) { + return { + "content-type": "application/json", + accept: "application/json, text/event-stream", + "mcp-protocol-version": "2026-07-28", + "mcp-method": method, + ...(name ? { "mcp-name": name } : {}), + }; +} +async function rpc( + fixture: Awaited>, + method: string, + params: Record = {}, + headers: Record = {}, +) { + const response = await fixture.fetch(fixture.url, { + method: "POST", + headers: { + ...modernHeaders( + method, + typeof params.name === "string" ? params.name : undefined, + ), + ...headers, + }, + body: JSON.stringify(message(method, params)), + }); + const text = await response.text(); + const data = text.startsWith("data: ") ? text.slice(6).trim() : text; + return { response, body: data ? JSON.parse(data) : undefined }; +} +function token( + fixture: Awaited>, + params: Record, +) { + return fixture.fetch(`${fixture.origin}/token`, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams(params).toString(), + }); +} + +test("binary HTTP output retains the committed write counter", async (t) => { + const fixture = await startMcpFixture({ response: "binary" }); + t.after(() => fixture.close()); + const response = await fixture.fetch(fixture.url, { + method: "POST", + headers: modernHeaders("tools/call", "write_counter"), + body: JSON.stringify( + message("tools/call", { name: "write_counter", arguments: { delta: 1 } }), + ), + }); + assert.equal( + response.headers.get("content-type"), + "application/octet-stream", + ); + assert.deepEqual( + new Uint8Array(await response.arrayBuffer()), + new Uint8Array([0, 255, 128, 1]), + ); + assert.equal(fixture.writes, 1); +}); + +test("legacy cancellation stops only the matching held call", async (t) => { + const fixture = await startMcpFixture({ + version: "2025-11-25", + holdCalls: true, + reply: "sse", + }); + t.after(() => fixture.close()); + const initialized = await rpc(fixture, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "fixture-client", version: "1" }, + }); + const headers = { + ...modernHeaders("tools/call", "read_counter"), + "mcp-session-id": initialized.response.headers.get("mcp-session-id")!, + }; + const first = await fixture.fetch(fixture.url, { + method: "POST", + headers, + body: JSON.stringify({ + ...message("tools/call", { name: "read_counter", arguments: {} }), + id: 10, + }), + }); + const firstBody = first.text().catch(() => "canceled"); + const second = await fixture.fetch(fixture.url, { + method: "POST", + headers, + body: JSON.stringify({ + ...message("tools/call", { name: "read_counter", arguments: {} }), + id: 20, + }), + }); + const secondBody = second.text().catch(() => "canceled"); + const canceled = await fixture.fetch(fixture.url, { + method: "POST", + headers, + body: JSON.stringify({ + jsonrpc: "2.0", + method: "notifications/cancelled", + params: { requestId: 10 }, + }), + }); + assert.equal(canceled.status, 202); + await fixture.waitForCancellation(); + fixture.releaseCalls(); + assert.equal(await firstBody, "canceled"); + assert.match(await secondBody, /"id":20/); + assert.equal(fixture.cancellations, 1); +}); + +test("legacy cancellation cannot cross a transport session with the same request ID", async (t) => { + const fixture = await startMcpFixture({ + version: "2025-11-25", + holdCalls: true, + reply: "sse", + }); + t.after(() => fixture.close()); + const firstSession = await rpc(fixture, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "fixture-client", version: "1" }, + }); + const secondSession = await rpc(fixture, "initialize", { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "fixture-client", version: "1" }, + }); + const firstHeaders = { + ...modernHeaders("tools/call", "read_counter"), + "mcp-session-id": firstSession.response.headers.get("mcp-session-id")!, + }; + const secondHeaders = { + ...modernHeaders("tools/call", "read_counter"), + "mcp-session-id": secondSession.response.headers.get("mcp-session-id")!, + }; + const body = JSON.stringify({ + ...message("tools/call", { name: "read_counter", arguments: {} }), + id: 10, + }); + const first = await fixture.fetch(fixture.url, { + method: "POST", + headers: firstHeaders, + body, + }); + const firstBody = first.text().catch(() => "canceled"); + const second = await fixture.fetch(fixture.url, { + method: "POST", + headers: secondHeaders, + body, + }); + const secondBody = second.text().catch(() => "canceled"); + await fixture.fetch(fixture.url, { + method: "POST", + headers: firstHeaders, + body: JSON.stringify({ + jsonrpc: "2.0", + method: "notifications/cancelled", + params: { requestId: 10 }, + }), + }); + await fixture.waitForCancellation(); + fixture.releaseCalls(); + assert.equal(await firstBody, "canceled"); + assert.match(await secondBody, /"id":10/); + assert.equal(fixture.cancellations, 1); +}); + +for (const version of ["2025-11-25", "2025-06-18"] as const) { + test(`legacy session IDs remain distinct after termination: ${version}`, async (t) => { + const fixture = await startMcpFixture({ version }); + t.after(() => fixture.close()); + const initialize = (authorization: string) => + rpc( + fixture, + "initialize", + { + protocolVersion: version, + capabilities: {}, + clientInfo: { name: "test", version: "1" }, + }, + { authorization }, + ); + const first = await initialize("Bearer account-one"); + const second = await initialize("Bearer account-two"); + const firstId = first.response.headers.get("mcp-session-id")!; + const secondId = second.response.headers.get("mcp-session-id")!; + const deleted = await fixture.fetch(fixture.url, { + method: "DELETE", + headers: { + authorization: "Bearer account-one", + "mcp-session-id": firstId, + }, + }); + assert.equal(deleted.status, 200); + const third = await initialize("Bearer account-three"); + const thirdId = third.response.headers.get("mcp-session-id"); + assert.notEqual(thirdId, firstId); + assert.notEqual(thirdId, secondId); + assert.equal(thirdId, "fixture-session-3"); + const stillActive = await rpc( + fixture, + "tools/list", + {}, + { + authorization: "Bearer account-two", + "mcp-session-id": secondId, + }, + ); + assert.equal(stillActive.response.status, 200); + assert.equal(stillActive.body.result.tools[0].name, "read_counter"); + }); + + test(`legacy session termination requires its credential identity: ${version}`, async (t) => { + const fixture = await startMcpFixture({ version }); + t.after(() => fixture.close()); + const initialized = await rpc( + fixture, + "initialize", + { + protocolVersion: version, + capabilities: {}, + clientInfo: { name: "test", version: "1" }, + }, + { authorization: "Bearer account-one" }, + ); + const sessionId = initialized.response.headers.get("mcp-session-id")!; + const denied = await fixture.fetch(fixture.url, { + method: "DELETE", + headers: { + authorization: "Bearer account-two", + "mcp-session-id": sessionId, + }, + }); + assert.equal(denied.status, 404); + const stillActive = await rpc( + fixture, + "tools/list", + {}, + { + authorization: "Bearer account-one", + "mcp-session-id": sessionId, + }, + ); + assert.equal(stillActive.response.status, 200); + assert.equal(stillActive.body.result.tools[0].name, "read_counter"); + const deleted = await fixture.fetch(fixture.url, { + method: "DELETE", + headers: { + authorization: "Bearer account-one", + "mcp-session-id": sessionId, + }, + }); + assert.equal(deleted.status, 200); + const terminated = await rpc( + fixture, + "tools/list", + {}, + { + authorization: "Bearer account-one", + "mcp-session-id": sessionId, + }, + ); + assert.equal(terminated.response.status, 404); + }); +} diff --git a/mcp-worker/test/fixtures/effect-worker.ts b/mcp-worker/test/fixtures/effect-worker.ts new file mode 100644 index 000000000..a3167c1bc --- /dev/null +++ b/mcp-worker/test/fixtures/effect-worker.ts @@ -0,0 +1,144 @@ +// Actual worker and provider HTTPS servers in Node. DNS/dial routing is a test boundary. +import { createInterface } from "node:readline"; +import { createServer, request } from "node:https"; +import { createWorker } from "../../src/service.ts"; +import { ZonePolicy } from "../../src/network.ts"; +const lines = createInterface({ input: process.stdin }); +const input = lines[Symbol.asyncIterator](); +const { tls, response = "normal" } = JSON.parse((await input.next()).value!); +const held: (() => void)[] = []; +let lastCall: string | undefined; +let calls = 0, + credentialRequests = 0; +const provider = createServer(tls, (req, res) => { + let raw = ""; + req.on("data", (chunk: Buffer) => { + raw += chunk; + }); + req.on("end", () => { + if (req.headers.authorization) credentialRequests++; + const message = JSON.parse(raw); + let result: unknown; + if (message.method === "server/discover") + result = { + supportedVersions: ["2026-07-28"], + capabilities: { tools: {} }, + }; + else if (message.method === "tools/list") + result = { + ttlMs: 0, + cacheScope: "private", + tools: [ + { + name: "Lookup.Record", + description: "Find a fixture record", + inputSchema: { type: "object" }, + }, + ], + }; + else if (message.method === "tools/call") { + calls++; + lastCall = raw; + if (response === "hold") { + held.push(() => { + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + id: message.id, + result: { + resultType: "complete", + content: [{ type: "text", text: "completed" }], + }, + }), + ); + }); + return; + } + if (response === "malformed") { + res.writeHead(200, { "content-type": "application/json" }); + res.end("{broken"); + return; + } + const secret = String(req.headers.authorization ?? ""); + const text = + response === "oversize" + ? "x".repeat(1100000) + : response === "normalized-limit" + ? "x".repeat(550000) + : `${secret} | ${secret.replace(/^Bearer /, "")}`; + result = { + content: + response === "image" + ? [{ type: "image", data: "AA==", mimeType: "image/png" }] + : [{ type: "text", text }], + structuredContent: { echo: text }, + isError: response === "error", + }; + } else { + res.writeHead(202); + res.end(); + return; + } + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + jsonrpc: "2.0", + id: message.id, + result: { ...Object(result), resultType: "complete" }, + }), + ); + }); +}); +await new Promise((resolve) => provider.listen(0, "127.0.0.1", resolve)); +const address = provider.address(); +if (!address || typeof address === "string") throw Error("fixture address"); +const zone = new ZonePolicy({ + id: "public", + revision: "1", + mode: "public", + authorities: ["tools.example.test"], + aliases: [], + cidrs: [], + denyCidrs: [], +}); +const make = () => + createWorker({ + tls, + zone, + enabled: true, + callers: [{ fingerprint: tls.fingerprint, role: "orchestrator" }], + network: { + resolve: async () => ["93.184.216.34"], + request: (_url, options) => + request(new URL(`https://127.0.0.1:${address.port}/mcp`), { + ...options, + ca: tls.ca, + servername: "localhost", + lookup: (_host, _options, callback) => callback(null, "127.0.0.1", 4), + }), + }, + }); +let worker = make(); +const port = await worker.listen(0, "127.0.0.1"); +const send = (value: unknown) => + process.stdout.write(`${JSON.stringify(value)}\n`); +send({ port, identity: worker.identity, node: process.versions.node }); +for await (const command of { [Symbol.asyncIterator]: () => input }) { + if (command === "stats") send({ calls, credentialRequests, lastCall }); + else if (command === "release") { + for (const release of held.splice(0)) release(); + send({ released: true }); + } else if (command === "restart") { + await worker.close(); + worker = make(); + await worker.listen(port, "127.0.0.1"); + send({ identity: worker.identity }); + } else if (command === "close") { + await worker.close(); + provider.closeAllConnections(); + await new Promise((resolve) => provider.close(() => resolve())); + lines.close(); + break; + } +} diff --git a/mcp-worker/test/fixtures/http.ts b/mcp-worker/test/fixtures/http.ts new file mode 100644 index 000000000..4f3155e45 --- /dev/null +++ b/mcp-worker/test/fixtures/http.ts @@ -0,0 +1,52 @@ +import { request } from "node:https"; +import { TEST_CA } from "./tls.ts"; + +// Test-only HTTPS adapter: explicit private CA, streaming bodies, and abort support. +// It never changes process TLS settings and never follows redirects. +export const fixtureFetch: typeof fetch = async (input, init) => { + const source = input instanceof Request ? input : new Request(input, init); + const headers = new Headers(init?.headers ?? source.headers); + const signal = init?.signal ?? source.signal; + const body = init?.body ?? (source.body ? await source.text() : undefined); + if (body !== undefined && body !== null && typeof body !== "string") { + throw new Error("Fixture fetch accepts string request bodies only"); + } + return new Promise((resolve, reject) => { + const req = request(source.url, { + method: init?.method ?? source.method, + headers: Object.fromEntries(headers), + ca: TEST_CA, + signal, + agent: false, + }); + req.on("error", reject); + req.on("response", (res) => { + const responseHeaders = new Headers(); + for (const [key, value] of Object.entries(res.headers)) { + if (value !== undefined) + responseHeaders.set( + key, + Array.isArray(value) ? value.join(", ") : value, + ); + } + const stream = new ReadableStream({ + start(controller) { + res.on("data", (chunk: Buffer) => controller.enqueue(chunk)); + res.on("end", () => controller.close()); + res.on("error", (error) => controller.error(error)); + }, + cancel() { + res.destroy(); + }, + }); + const status = res.statusCode ?? 500; + resolve( + new Response([204, 205, 304].includes(status) ? null : stream, { + status, + headers: responseHeaders, + }), + ); + }); + req.end(body); + }); +}; diff --git a/mcp-worker/test/fixtures/oauth.ts b/mcp-worker/test/fixtures/oauth.ts new file mode 100644 index 000000000..5e0f57fcf --- /dev/null +++ b/mcp-worker/test/fixtures/oauth.ts @@ -0,0 +1,265 @@ +import { createHash } from "node:crypto"; +import type { IncomingMessage, ServerResponse } from "node:http"; + +interface CodeGrant { + challenge: string; + client: string; + redirect: string; + resource: string; + scope: string; +} +interface RefreshGrant { + client: string; + scope: string; +} + +// A deterministic authorization server for hostile-boundary tests, not a provider driver. +export function createOAuthFixture( + origin: () => string, + callbackIssuer?: string, + advertised?: { + issuer: string; + authorization: string; + token: string; + redirectUri: string; + }, +) { + const codes = new Map(); + const states = new Set(); + const refreshes = new Map(); + let codeSequence = 0; + let tokenSequence = 0; + let accessToken: string | undefined; + let accessExpiresAt = 0; + let expiresInSeconds = 3600; + const resource = () => `${origin()}/mcp`; + const issuer = () => advertised?.issuer ?? origin(); + const redirectUri = () => advertised?.redirectUri ?? `${origin()}/callback`; + let account = "account-one"; + let scopeOverride: string | undefined; + let authorizations = 0; + let exchanges = 0; + let rotations = 0; + let rejections = 0; + const respond = (res: ServerResponse, value: unknown, status = 200) => { + res.writeHead(status, { + "content-type": "application/json", + "cache-control": "no-store", + }); + res.end(JSON.stringify(value)); + }; + const fail = (res: ServerResponse, error = "invalid_request") => { + ++rejections; + respond(res, { error }, 400); + }; + return { + observe() { + return { + account, + scope: scopeOverride ?? null, + expiresInSeconds, + authorizations, + exchanges, + rotations, + rejections, + }; + }, + configure(value: { + account?: string; + scope?: string; + issuer?: "configured" | "mismatch"; + expiresInSeconds?: number; + }) { + if ( + value.expiresInSeconds !== undefined && + (!Number.isInteger(value.expiresInSeconds) || + value.expiresInSeconds < 1 || + value.expiresInSeconds > 3600) + ) + throw new Error("Invalid token lifetime"); + if (value.expiresInSeconds !== undefined) + expiresInSeconds = value.expiresInSeconds; + if (value.account !== undefined) account = value.account; + if (value.scope !== undefined) scopeOverride = value.scope; + if (value.issuer !== undefined) + callbackIssuer = + value.issuer === "mismatch" ? "https://other.example" : undefined; + }, + accepts(authorization: string | undefined) { + return ( + accessToken !== undefined && + authorization === `Bearer ${accessToken}` && + Date.now() < accessExpiresAt + ); + }, + handle( + req: IncomingMessage, + res: ServerResponse, + url: URL, + body: string, + ): boolean { + if ( + url.pathname === "/.well-known/oauth-protected-resource" || + url.pathname === "/.well-known/oauth-protected-resource/mcp" + ) { + respond(res, { + resource: resource(), + authorization_servers: [issuer()], + scopes_supported: ["tools:read", "tools:write"], + bearer_methods_supported: ["header"], + }); + return true; + } + if (url.pathname === "/.well-known/oauth-authorization-server") { + respond(res, { + issuer: issuer(), + authorization_endpoint: `${advertised?.authorization ?? origin()}/authorize`, + token_endpoint: `${advertised?.token ?? origin()}/token`, + response_types_supported: ["code"], + grant_types_supported: ["authorization_code", "refresh_token"], + token_endpoint_auth_methods_supported: ["none", "client_secret_post"], + code_challenge_methods_supported: ["S256"], + authorization_response_iss_parameter_supported: true, + scopes_supported: ["tools:read", "tools:write"], + }); + return true; + } + if (url.pathname === "/authorize") { + const p = url.searchParams; + const client = p.get("client_id"); + const state = p.get("state"); + const challenge = p.get("code_challenge"); + const scope = p.get("scope") ?? "tools:read"; + if ( + req.method !== "GET" || + p.get("response_type") !== "code" || + !["fixture-public", "fixture-confidential"].includes(client ?? "") || + p.get("redirect_uri") !== redirectUri() || + p.get("resource") !== resource() || + p.get("code_challenge_method") !== "S256" || + !challenge || + !/^[A-Za-z0-9_-]{43}$/.test(challenge) || + !state || + states.has(state) || + !validScope(scope) + ) { + fail(res); + return true; + } + if (codes.size >= 128 || states.size >= 1024) { + fail(res); + return true; + } + ++authorizations; + const code = `fixture-code-${++codeSequence}`; + states.add(state); + codes.set(code, { + client: client!, + challenge, + redirect: redirectUri(), + resource: resource(), + scope, + }); + const callback = new URL(redirectUri()); + callback.search = new URLSearchParams({ + code, + state, + iss: callbackIssuer ?? issuer(), + }).toString(); + res.writeHead(302, { + location: callback.href, + "cache-control": "no-store", + }); + res.end(); + return true; + } + if (url.pathname !== "/token") return false; + if (req.method !== "POST") { + fail(res); + return true; + } + const p = new URLSearchParams(body); + const client = p.get("client_id"); + if ( + !["fixture-public", "fixture-confidential"].includes(client ?? "") || + (client === "fixture-confidential" && + p.get("client_secret") !== "fixture-client-secret") || + p.get("resource") !== resource() + ) { + fail(res); + return true; + } + let scope: string; + if (p.get("grant_type") === "authorization_code") { + const code = p.get("code") ?? ""; + const grant = codes.get(code); + const challenge = createHash("sha256") + .update(p.get("code_verifier") ?? "") + .digest("base64url"); + if ( + !grant || + grant.client !== client || + grant.redirect !== p.get("redirect_uri") || + grant.resource !== p.get("resource") || + grant.challenge !== challenge + ) { + fail(res, "invalid_grant"); + return true; + } + codes.delete(code); + ++exchanges; + scope = grant.scope; + } else if (p.get("grant_type") === "refresh_token") { + const refresh = p.get("refresh_token") ?? ""; + const grant = refreshes.get(refresh); + if ( + !grant || + grant.client !== client || + (p.has("scope") && !isScopeSubset(p.get("scope")!, grant.scope)) + ) { + fail(res, "invalid_grant"); + return true; + } + refreshes.delete(refresh); + ++rotations; + scope = p.get("scope") ?? grant.scope; + } else { + fail(res, "unsupported_grant_type"); + return true; + } + if (refreshes.size >= 128) { + fail(res); + return true; + } + scope = scopeOverride ?? scope; + ++tokenSequence; + accessToken = `fixture-access-${tokenSequence}`; + accessExpiresAt = Date.now() + expiresInSeconds * 1000; + const refreshToken = `fixture-refresh-${tokenSequence}`; + refreshes.set(refreshToken, { client: client!, scope }); + respond(res, { + access_token: accessToken, + refresh_token: refreshToken, + token_type: "Bearer", + expires_in: expiresInSeconds, + scope, + ...(advertised ? { fixture_account: account } : {}), + }); + return true; + }, + }; +} +function validScope(scope: string) { + return ( + scope.length > 0 && + scope + .split(" ") + .every((part) => ["tools:read", "tools:write"].includes(part)) + ); +} +function isScopeSubset(scope: string, previous: string) { + return ( + validScope(scope) && + scope.split(" ").every((part) => previous.split(" ").includes(part)) + ); +} diff --git a/mcp-worker/test/fixtures/resource-client-worker.ts b/mcp-worker/test/fixtures/resource-client-worker.ts new file mode 100644 index 000000000..4b90d7329 --- /dev/null +++ b/mcp-worker/test/fixtures/resource-client-worker.ts @@ -0,0 +1,80 @@ +// Actual Node worker fixture. Admission receipts are test inputs, not database proof. +import { createInterface } from "node:readline"; +import { createServer } from "node:https"; +import { createWorker } from "../../src/service.ts"; +import { ZonePolicy } from "../../src/network.ts"; + +const lines = createInterface({ input: process.stdin }); +const input = lines[Symbol.asyncIterator](); +const first = await input.next(); +const tls = JSON.parse(first.value!); +let providerDials = 0; +const makeWorker = () => + createWorker({ + tls, + zone: new ZonePolicy({ + id: "public", + revision: "1", + mode: "public", + authorities: ["tools.example"], + aliases: [], + cidrs: [], + denyCidrs: [], + }), + callers: [{ fingerprint: tls.fingerprint, role: "orchestrator" }], + enabled: true, + network: { + resolve: async () => { + providerDials++; + throw new Error("fixture has no provider"); + }, + request: () => { + providerDials++; + throw new Error("fixture has no provider"); + }, + }, + }); +let worker = makeWorker(); +const port = await worker.listen(0, "127.0.0.1"); +const send = (value: unknown) => + process.stdout.write(`${JSON.stringify(value)}\n`); +send({ port, identity: worker.identity, node: process.versions.node }); +let replacement: ReturnType | undefined; +let replacementBytes = 0; +let replacementRequests = 0; +for await (const line of { [Symbol.asyncIterator]: () => input }) { + if (line === "restart") { + await worker.close(); + worker = makeWorker(); + await worker.listen(port, "127.0.0.1"); + send({ identity: worker.identity }); + } else if (line === "replace") { + await worker.close(); + replacement = createServer( + { ...tls, requestCert: true, rejectUnauthorized: true }, + (req, res) => { + replacementRequests++; + req.on("data", (chunk: Buffer) => { + replacementBytes += chunk.length; + }); + req.on("end", () => { + res.writeHead(500); + res.end(); + }); + }, + ); + await new Promise((resolve) => + replacement!.listen(port, "127.0.0.1", resolve), + ); + send({ replaced: true }); + } else if (line === "stats") { + send({ providerDials, replacementBytes, replacementRequests }); + } else if (line === "close") { + if (replacement) { + replacement.closeAllConnections(); + await new Promise((resolve) => replacement!.close(() => resolve())); + } else await worker.close(); + lines.close(); + break; + } +} diff --git a/mcp-worker/test/fixtures/server.ts b/mcp-worker/test/fixtures/server.ts new file mode 100644 index 000000000..b141adafc --- /dev/null +++ b/mcp-worker/test/fixtures/server.ts @@ -0,0 +1,558 @@ +import { createServer } from "node:https"; +import type { IncomingMessage, ServerResponse } from "node:http"; +import type { Tool } from "@modelcontextprotocol/client"; +import { fixtureFetch } from "./http.ts"; +import { createOAuthFixture } from "./oauth.ts"; +import { TEST_CA, TEST_CERT, TEST_KEY } from "./tls.ts"; + +export type FixtureVersion = "2026-07-28" | "2025-11-25" | "2025-06-18"; +export type FixtureResponse = + | "normal" + | "malformed" + | "oversize" + | "binary" + | "image" + | "audio" + | "resource" + | "resource_link" + | "error"; +export interface FixtureOptions { + version?: FixtureVersion; + reply?: "json" | "sse"; + response?: FixtureResponse; + holdCalls?: boolean; + requireAuth?: boolean; + callbackIssuer?: string; + apiKey?: string; + initialCount?: number; + commitWrite?: (callIdentity: string, delta: number) => number; + network?: { + host: string; + port: number; + key: string; + cert: string; + ca: string; + origin: string; + oauth: { + issuer: string; + authorization: string; + token: string; + redirectUri: string; + }; + allowedAuthorities: string[]; + }; +} +export interface ObservedRequest { + method: string; + sessionId: string | undefined; + protocolVersion: string | undefined; +} +interface RpcMessage { + jsonrpc: "2.0"; + id?: number | string; + method: string; + params?: Record; +} + +export async function startMcpFixture(options: FixtureOptions = {}) { + const version = options.version ?? "2026-07-28"; + const modern = version === "2026-07-28"; + let origin = options.network?.origin ?? ""; + let revision = 1; + let count = options.initialCount ?? 0; + let reads = 0; + let writes = 0; + let cancellations = 0; + let sessionSequence = 0; + const requests: ObservedRequest[] = []; + const sessions = new Map(); + const pending = new Map< + ServerResponse, + { + id: number | string | undefined; + sessionId: string | undefined; + complete: () => void; + } + >(); + const calls = eventBarrier(); + const canceled = eventBarrier(); + const oauth = createOAuthFixture( + () => origin, + options.callbackIssuer, + options.network?.oauth, + ); + const server = createServer( + { + key: options.network?.key ?? TEST_KEY, + cert: options.network?.cert ?? TEST_CERT, + }, + (req, res) => { + void handle(req, res).catch(() => { + if (!res.headersSent) res.writeHead(500); + res.end("Fixture request failed"); + }); + }, + ); + server.requestTimeout = 5000; + server.headersTimeout = 5000; + server.keepAliveTimeout = 1000; + server.maxConnections = 64; + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen( + options.network?.port ?? 0, + options.network?.host ?? "127.0.0.1", + resolve, + ); + }); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Missing fixture listener"); + if (!options.network) origin = `https://127.0.0.1:${address.port}`; + + async function handle(req: IncomingMessage, res: ServerResponse) { + if ( + options.network && + (!req.headers.host || + !options.network.allowedAuthorities.includes(req.headers.host)) + ) { + res.writeHead(421).end(); + return; + } + if (!req.url?.startsWith("/") || req.url.startsWith("//")) { + res.writeHead(400).end(); + return; + } + const url = new URL( + req.url, + `https://${req.headers.host ?? new URL(origin).host}`, + ); + if (options.network) { + const expected = + url.pathname === "/authorize" + ? options.network.oauth.authorization + : url.pathname === "/token" + ? options.network.oauth.token + : url.pathname === "/.well-known/oauth-authorization-server" + ? options.network.oauth.issuer + : origin; + if (url.origin !== expected) { + res.writeHead(421).end(); + return; + } + } + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of req) { + bytes += Buffer.byteLength(chunk); + if (bytes > 65536) { + res.writeHead(413).end(); + return; + } + chunks.push(Buffer.from(chunk)); + } + const raw = Buffer.concat(chunks).toString("utf8"); + if (oauth.handle(req, res, url, raw)) return; + if (url.pathname !== "/mcp") { + res.writeHead(404).end(); + return; + } + if ( + options.requireAuth && + (!options.apiKey || + req.headers.authorization !== `Bearer ${options.apiKey}`) && + !oauth.accepts(req.headers.authorization) + ) { + res.writeHead(401, { + "content-type": "application/json", + "www-authenticate": `Bearer resource_metadata="${origin}/.well-known/oauth-protected-resource"`, + }); + res.end(JSON.stringify({ error: "unauthorized" })); + return; + } + if (req.method === "GET") { + res.writeHead(405).end(); + return; + } + const sessionId = header(req, "mcp-session-id"); + if (req.method === "DELETE" && !modern) { + if ( + !sessionId || + !sessions.has(sessionId) || + sessions.get(sessionId) !== req.headers.authorization + ) { + res.writeHead(404).end(); + return; + } + sessions.delete(sessionId); + res.writeHead(200).end(); + return; + } + if (req.method !== "POST") { + res.writeHead(405).end(); + return; + } + let body: unknown; + try { + body = JSON.parse(raw); + } catch { + res.writeHead(400).end(); + return; + } + if (!isRpcMessage(body)) { + res.writeHead(400).end(); + return; + } + const message = body; + if (requests.length === 128) requests.shift(); + requests.push({ + method: message.method, + sessionId, + protocolVersion: header(req, "mcp-protocol-version"), + }); + const rpcError = (code: number, text: string, status = 200) => + send( + res, + { + jsonrpc: "2.0", + id: message.id ?? null, + error: { code, message: text }, + }, + status, + ); + if (!modern && message.method === "server/discover") { + rpcError(-32601, "Method not found"); + return; + } + if (!modern && message.method === "initialize") { + if (sessions.size >= 128) { + rpcError(-32000, "Session limit", 429); + return; + } + const id = `fixture-session-${++sessionSequence}`; + sessions.set(id, req.headers.authorization); + res.setHeader("mcp-session-id", id); + sendResult(res, message, { + protocolVersion: version, + capabilities: { tools: { listChanged: true } }, + serverInfo: { name: "mcp-fixture", version: "1" }, + }); + return; + } + if ( + !modern && + (!sessionId || + !sessions.has(sessionId) || + sessions.get(sessionId) !== req.headers.authorization) + ) { + rpcError(-32001, "Unknown session", 404); + return; + } + if (message.method === "notifications/initialized") { + res.writeHead(202).end(); + return; + } + if (message.method === "notifications/cancelled") { + for (const [response, call] of pending) { + if ( + call.id === message.params?.requestId && + call.sessionId === sessionId + ) + response.destroy(); + } + res.writeHead(202).end(); + return; + } + if (modern && !validModernHeaders(req, message)) { + rpcError(-32021, "Request metadata mismatch", 400); + return; + } + switch (message.method) { + case "server/discover": + sendResult(res, message, { + supportedVersions: [version], + capabilities: { tools: {} }, + _meta: { + "io.modelcontextprotocol/serverInfo": { + name: "mcp-fixture", + version: "1", + }, + }, + }); + return; + case "ping": + sendResult(res, message, {}); + return; + case "tools/list": { + const page = message.params?.cursor; + if (page !== undefined && page !== "page-2") { + rpcError(-32602, "Unknown cursor"); + return; + } + sendResult(res, message, { + tools: [ + catalogTool( + page === "page-2" ? "write_counter" : "read_counter", + revision, + ), + ], + ...(page === "page-2" ? {} : { nextCursor: "page-2" }), + }); + return; + } + case "tools/call": { + const args = message.params?.arguments; + if (!isObject(args)) { + rpcError(-32602, "Arguments must be an object"); + return; + } + if (pending.size >= 64) { + rpcError(-32000, "Held call limit", 429); + return; + } + if (message.params?.name === "read_counter") ++reads; + else if ( + message.params?.name === "write_counter" && + Number.isSafeInteger(args.delta) + ) { + const delta = Number(args.delta); + count = options.commitWrite + ? options.commitWrite( + JSON.stringify([sessionId ?? null, message.id ?? null]), + delta, + ) + : count + delta; + ++writes; + } else { + rpcError(-32602, "Unknown tool or invalid delta"); + return; + } + const complete = () => toolReply(res, message, count); + if (options.holdCalls) { + if (options.reply === "sse") { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.flushHeaders(); + } + pending.set(res, { id: message.id, sessionId, complete }); + res.on("close", () => { + if (pending.delete(res) && !res.writableEnded) { + ++cancellations; + canceled.signal(); + } + }); + } + calls.signal(); + if (!options.holdCalls) complete(); + return; + } + default: + rpcError(-32601, "Method not found"); + } + } + function send(res: ServerResponse, body: unknown, status = 200) { + const json = JSON.stringify(body); + if (options.reply === "sse" && status === 200) { + if (!res.headersSent) + res.writeHead(status, { "content-type": "text/event-stream" }); + res.end(`data: ${json}\n\n`); + } else { + res.writeHead(status, { "content-type": "application/json" }); + res.end(json); + } + } + function sendResult( + res: ServerResponse, + message: RpcMessage, + result: unknown, + ) { + send(res, { + jsonrpc: "2.0", + id: message.id, + result: + modern && isObject(result) + ? { + ...result, + resultType: "complete", + ...(message.method === "tools/list" + ? { ttlMs: 0, cacheScope: "private" } + : {}), + } + : result, + }); + } + function toolReply(res: ServerResponse, message: RpcMessage, value: number) { + if (options.response === "binary") { + res.writeHead(200, { "content-type": "application/octet-stream" }); + res.end(Buffer.from([0, 255, 128, 1])); + return; + } + if (options.response === "malformed") { + res.writeHead(200, { "content-type": "application/json" }); + res.end("{broken"); + return; + } + let content: unknown[] = [{ type: "text", text: `count=${value}` }]; + switch (options.response) { + case "oversize": + content = [{ type: "text", text: "x".repeat(2 * 1024 * 1024 + 1) }]; + break; + case "image": + content = [{ type: "image", data: "AA==", mimeType: "image/png" }]; + break; + case "audio": + content = [{ type: "audio", data: "AA==", mimeType: "audio/wav" }]; + break; + case "resource": + content = [ + { + type: "resource", + resource: { + uri: "https://resource.example/data", + text: "untrusted", + }, + }, + ]; + break; + case "resource_link": + content = [ + { + type: "resource_link", + uri: "https://resource.example/data", + name: "untrusted", + }, + ]; + break; + } + sendResult(res, message, { + content, + structuredContent: { count: value }, + isError: options.response === "error", + }); + } + return { + get url() { + return `${origin}/mcp`; + }, + get origin() { + return origin; + }, + ca: options.network?.ca ?? TEST_CA, + fetch: fixtureFetch, + requests, + oauth, + get heldCalls() { + return pending.size; + }, + setCase( + value: Pick< + FixtureOptions, + "reply" | "response" | "holdCalls" | "requireAuth" + >, + ) { + Object.assign(options, value); + }, + cancelCalls() { + for (const [response] of pending) response.destroy(); + }, + get reads() { + return reads; + }, + get writes() { + return writes; + }, + get cancellations() { + return cancellations; + }, + setCatalogRevision(value: number) { + revision = value; + }, + waitForCall: calls.wait, + waitForCancellation: canceled.wait, + releaseCalls() { + const held = [...pending.values()]; + pending.clear(); + for (const call of held) call.complete(); + }, + async close() { + for (const [res] of pending) res.destroy(); + pending.clear(); + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + }, + }; +} +function header(req: IncomingMessage, name: string): string | undefined { + const value = req.headers[name]; + return typeof value === "string" ? value : undefined; +} +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function isRpcMessage(value: unknown): value is RpcMessage { + return ( + isObject(value) && + value.jsonrpc === "2.0" && + typeof value.method === "string" && + (value.id === undefined || + typeof value.id === "number" || + typeof value.id === "string") && + (value.params === undefined || isObject(value.params)) + ); +} +function validModernHeaders(req: IncomingMessage, message: RpcMessage) { + const meta = message.params?._meta; + return ( + isObject(meta) && + meta["io.modelcontextprotocol/protocolVersion"] === "2026-07-28" && + header(req, "mcp-protocol-version") === "2026-07-28" && + header(req, "mcp-method") === message.method && + (message.method !== "tools/call" || + header(req, "mcp-name") === message.params?.name) + ); +} +function catalogTool(name: string, revision: number): Tool { + return { + name, + description: `Counter tool revision ${revision}`, + inputSchema: { + type: "object", + $schema: + name === "read_counter" + ? "http://json-schema.org/draft-07/schema#" + : "https://json-schema.org/draft/2020-12/schema", + properties: { + ...(name === "write_counter" ? { delta: { type: "integer" } } : {}), + ...(revision > 1 ? { required_tag: { type: "string" } } : {}), + }, + required: [ + ...(name === "write_counter" ? ["delta"] : []), + ...(revision > 1 ? ["required_tag"] : []), + ], + additionalProperties: false, + }, + annotations: { + readOnlyHint: name === "read_counter", + destructiveHint: name === "write_counter", + }, + }; +} +function eventBarrier() { + let available = 0; + const waiters: (() => void)[] = []; + return { + signal() { + const waiter = waiters.shift(); + if (waiter) waiter(); + else ++available; + }, + wait() { + if (available > 0) { + --available; + return Promise.resolve(); + } + return new Promise((resolve) => waiters.push(resolve)); + }, + }; +} diff --git a/mcp-worker/test/fixtures/tls.ts b/mcp-worker/test/fixtures/tls.ts new file mode 100644 index 000000000..0035d94eb --- /dev/null +++ b/mcp-worker/test/fixtures/tls.ts @@ -0,0 +1,69 @@ +// Public test credentials. Never import this module from production code. +export const TEST_CA = `-----BEGIN CERTIFICATE----- +MIIC/DCCAeSgAwIBAgIUfAw9gD43biKgT5RrLP/+JNGoikMwDQYJKoZIhvcNAQEL +BQAwHjEcMBoGA1UEAwwTTUNQIGZpeHR1cmUgdGVzdCBDQTAeFw0yNjExMDIwMTQ2 +MTBaFw0zNjEwMzAwMTQ2MTBaMB4xHDAaBgNVBAMME01DUCBmaXh0dXJlIHRlc3Qg +Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCoUOEVWF/JV91busqi +/EYyOp2m+Y5jvVHioE5HWTUFFR7IwF3Cvgb2OrXvThSDoeoO/eijbybYzudMzfH5 +UB3C+Kc0V9SQl06qIEGXa1uWP5peOsIrGkmSnX4O0t7LJ9NBoLDXqPb1tCw4HPdP +Fr6H4VaJTQaNYNBHNBkm47TRYUE79RsyZ5ODErbiN/ZN+nT/Ql0S5Y3KJdyJykn3 +JFAA9J/Jepwlmut7qZzWGokwmNm2vMoXtarMsLj/Ka0R+Po/KhFgvBUnuelFcCS4 +DQD1YpO61ALTi50aTznoTitB93/nTK6Vn0cFb2cJcWPwF5H7UnKGejTZLLKIh3/k +D6yfAgMBAAGjMjAwMB0GA1UdDgQWBBQv02S/NFnklRHjU1G2w9LFmjG03DAPBgNV +HRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCVnj6//yvrGjVxVIqjtuVl +Mft3Av5oHkTbmjhlHIS0HtAq3Z+w35IkGWVAMdcVsxzsM/jN5NNHWBJEjdmppCUz +T2tYm0usD9jnqxXm6UgnrlD+wO9hauEOZoyEJT+2rbUqmrlh6PW7D0LELboHd8/j +jFK1mzZ0s8giWHCukUNeFVPwVP7kMoeydBo3AJG2eerH30wjQshOjyCjMiqZ5vGk +Z13z8ZIQzs30Dct6UO8YvU+h33S/+CDC585IwBJgf0wGv6+7WoFKUvaTg/UXPezZ +lOdOQC+aZ8oCX6K9Z3vf/1tiOXeBydROxp25CVsGAqK8SnbpeZNvqk6NahnvlYGK +-----END CERTIFICATE----- +`; +export const TEST_CERT = `-----BEGIN CERTIFICATE----- +MIIDCzCCAfOgAwIBAgIUdlie7U+wrUMgQWBB/ag6VDSSzcEwDQYJKoZIhvcNAQEL +BQAwHjEcMBoGA1UEAwwTTUNQIGZpeHR1cmUgdGVzdCBDQTAeFw0yNjExMDIwMTQ2 +MTBaFw0zNjEwMzAwMTQ2MTBaMBQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJ +KoZIhvcNAQEBBQADggEPADCCAQoCggEBAKneoKMRRkFVDy5TL82Y9yW8z8gT54J6 +7hPNp/Ez3EPMPqQsB9Z9NMYms4pLZ4HTO71ZdPtQ5doi2MRGHvbkLZRWZZnWHHK3 +D6DkP+5kExqiCIuU3bSK90bv1L6cIFK4bBV0MyYRfLULtmQ/5mHWdCGuifmlKASO +NlruI5/tZRHM0pdLTGgy/FylIc3IQsLBQcIbvQMmGLnTxRUwv4ulRo4SruUm02I9 +Vlm/WnKXxdOZNa51tbldouMJ3L1iy1chXK45bBrMhNxAKaAvEzo96MDubKou9PN/ +pmIr9ahyIhwa3oxt80VAwMRdXikdGq9c83DrDpLBbBF7OJBF158+hcECAwEAAaNL +MEkwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMAkGA1UdEwQCMAAwCwYDVR0P +BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4IBAQBG +kU7MiyRbKFT4Zy88epm23cGyS5c96k9Bip2ZgSrKaiqfgWrMZ5wjIKq2gP7zgKG4 +LY56AGPPc5BYEk+1NQwnhcTsN2omYHXPLEnoJAqwZpJ341jY6SpfexbFWAqVBOhq +GvOfxapMblv26UYOYzkiLE9Fmr1SpAkU9ajAFIjdzwE2A//CVgV8AHv4cUTD+ILL +h73/m9eEvJmxfBz2Ee5PxaVtHVPgXZIZHR3ir5v2yEJVIGfJvh5Tui9/5RG4UxVV +icT+7k45Ut1PFLTRIuezQ18X0hzs+TV5Gug0YMsF9U23OVAWfekp81RIWC9aGak5 +ymz4n6rOKfDjvlvT+dcW +-----END CERTIFICATE----- +`; +export const TEST_KEY = `-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCp3qCjEUZBVQ8u +Uy/NmPclvM/IE+eCeu4TzafxM9xDzD6kLAfWfTTGJrOKS2eB0zu9WXT7UOXaItjE +Rh725C2UVmWZ1hxytw+g5D/uZBMaogiLlN20ivdG79S+nCBSuGwVdDMmEXy1C7Zk +P+Zh1nQhron5pSgEjjZa7iOf7WURzNKXS0xoMvxcpSHNyELCwUHCG70DJhi508UV +ML+LpUaOEq7lJtNiPVZZv1pyl8XTmTWudbW5XaLjCdy9YstXIVyuOWwazITcQCmg +LxM6PejA7myqLvTzf6ZiK/WociIcGt6MbfNFQMDEXV4pHRqvXPNw6w6SwWwReziQ +RdefPoXBAgMBAAECggEAG/L8XTg3ZgJ3x/h58gsMBxKE4Xec7NQLHgkFa+NLh4Aw +glSYSYs6RqwReFylwC+gMaxpWfxahUScg9bBu3+ZnEGG4APhlGJYWPGVL0UZ+5TT +cRviSpVe0DYfu+j2TIX5iqIe1FGmQieWL/JvloNxgiU0oHWzGuTHPrjV9bdanM3g +4w2HdJS0nje5jNKCVcXeejZ2BPx2udF1P0qZZxVVkcbHOFuN9/6QXB8lXo6hlmrQ +xZXKAlwdzECpn1o11pgknZU7T3GGHN2WsYEniQv4hHjQEg+RNqrx0UK6QDAIon3y +pE7CEQpRDMsmL0GH0u+v6KdMar1AIWQWOCWHUNUyUQKBgQDXr+0ZKaEyw15umpuh +ulg3raCoB6XxS3ohw7CYz0FDbjjFZeXr5yrW0bzyBJJYlAWZFwW+wtIA151s64HE +wuuAGHPs2n0pO1PyQQlRLInZTK7mKkLuyIaWCBdq3QX9l3FNMqkVtSN9Co40Xkop ++315FIfFA8SR3P1FM/eOSehNlQKBgQDJnnMCT7r/TYi0qKrhXU21G9eMRUzVKVqu +VxVY9K/fiK79Ak9Cg8XISAn4PIt3JbBz3birZEYRYan6BCqkagv0JIe5r4p1wh6t +MkfNQSl4ZyKen6YNeXsbIB2HI/hKWVhGcQaYezB4DGxmoYuGc0LoxcUBAULqVXB/ +qi65PX0UfQKBgQCymnaumH9opslCs3peoQ0+BeP8YGD0f7jBzuckkrMLYHFYXLnY +79LUkldw3YJGIUTFcYALaA44LlOpBxetVG9o4lbUclSkWQF5oItTkZsQuuQ1qhdE +3f48rLSfhCjaxpZHA8VqOVYKzDvXd6DYRO2w3uRSq30SADomHRNI20CnRQKBgHsN +sQYFIY12hew1UW4Q4z+ZSZo8JaOoBMeJMOdFGB5nLt4Ozc8727fcd/FBCqHip+pf +tbktjzJGmFaWumv0qW0N6qSjBe/PVafBm18j2Sq6T7/ekpuAvGkW1c7IXl8KXr1H +eI4SCcgLVRHioOSjbn/hTNbLLUsWcH4KoU+/vpqFAoGBAIbz/H/q+lcKXBaZYzxJ +TdYJZrH1cdOIecqlWYqZyO75VfJXuikcZAh8fRg2hMF4MpvLEt9H6akKStji/gSD +ICTOEFZsc8szg0mdSTCI7IVBYrgsImdKYtsFVRhwf+vhypAne5yvau7Y+zw1xWct +DrSTp+bphObufZ4IzPYHPThU +-----END PRIVATE KEY----- +`; diff --git a/mcp-worker/test/fixtures/worker-tls.ts b/mcp-worker/test/fixtures/worker-tls.ts new file mode 100644 index 000000000..8435e6fca --- /dev/null +++ b/mcp-worker/test/fixtures/worker-tls.ts @@ -0,0 +1,77 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; +import { X509Certificate } from "node:crypto"; +import { request } from "node:https"; +export async function workerTls(hostname = "tools.example") { + const dir = await mkdtemp(join(tmpdir(), "mcp-worker-tls-")); + try { + execFileSync( + "openssl", + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + join(dir, "key.pem"), + "-out", + join(dir, "cert.pem"), + "-days", + "1", + "-subj", + `/CN=${hostname}`, + "-addext", + `subjectAltName=DNS:${hostname}${hostname === "localhost" ? ",IP:127.0.0.1" : ""}`, + "-addext", + "extendedKeyUsage=serverAuth,clientAuth", + ], + { stdio: "ignore" }, + ); + const key = await readFile(join(dir, "key.pem"), "utf8"), + cert = await readFile(join(dir, "cert.pem"), "utf8"); + return { + key, + cert, + ca: cert, + fingerprint: new X509Certificate(cert).fingerprint256, + }; + } finally { + await rm(dir, { recursive: true, force: true }); + } +} +export async function localRpc( + port: number, + tls: Awaited>, + path: string, + body?: unknown, + withCert = true, +) { + return new Promise<{ status: number; body: unknown }>((resolve, reject) => { + const req = request( + { + hostname: "127.0.0.1", + port, + servername: "tools.example", + path, + method: body === undefined ? "GET" : "POST", + ca: tls.ca, + ...(withCert ? { key: tls.key, cert: tls.cert } : {}), + agent: false, + }, + (res) => { + let raw = ""; + res.on("data", (chunk) => { + raw += String(chunk); + }); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, body: JSON.parse(raw) }), + ); + }, + ); + req.on("error", reject); + req.end(body === undefined ? undefined : JSON.stringify(body)); + }); +} diff --git a/mcp-worker/test/network.test.ts b/mcp-worker/test/network.test.ts new file mode 100644 index 000000000..95c3523fa --- /dev/null +++ b/mcp-worker/test/network.test.ts @@ -0,0 +1,248 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { ZonePolicy, restrictedFetch } from "../src/network.ts"; + +const config = { + id: "public", + revision: "1", + mode: "public" as const, + authorities: ["tools.example"], + aliases: [], + cidrs: [], + denyCidrs: ["203.0.113.0/24"], + ca: undefined, +}; +test("destination and all DNS answers are checked before any dial", async () => { + let dials = 0; + for (const answers of [ + ["127.0.0.1"], + ["169.254.169.254"], + ["::1"], + ["::ffff:127.0.0.1"], + ["8.8.8.8", "10.0.0.1"], + ["8.8.8.8", "fd00::1"], + ["203.0.113.1"], + ]) { + const fetch = restrictedFetch(new ZonePolicy(config), { + resolve: async () => answers, + request: () => { + dials++; + throw Error("dial"); + }, + }); + await assert.rejects(fetch("https://tools.example/mcp")); + } + const fetch = restrictedFetch(new ZonePolicy(config), { + resolve: async () => ["8.8.8.8"], + request: () => { + dials++; + throw Error("dial"); + }, + }); + for (const url of [ + "http://tools.example/mcp", + "https://evil.example/mcp", + "https://user@tools.example/mcp", + "https://tools.example/mcp?q=1", + "https://tools.example/mcp#x", + "https://127.0.0.1/mcp", + "https://tools.example:8443/mcp", + "https://tools.example./mcp", + ]) + await assert.rejects(fetch(url)); + assert.equal(dials, 0); +}); +test("internal approval never admits permanent denies and policy digest covers authority", () => { + const zone = new ZonePolicy({ + ...config, + mode: "internal", + cidrs: ["10.0.0.0/8"], + }); + zone.checkAddresses(["10.1.2.3"]); + assert.throws(() => zone.checkAddresses(["127.0.0.1"])); + assert.throws(() => zone.checkAddresses(["169.254.169.254"])); + assert.notEqual( + zone.digest, + new ZonePolicy({ ...config, authorities: ["other.example"] }).digest, + ); +}); + +test("real TLS preserves authority and pins one checked DNS answer on each dial", async (t) => { + const { createServer, request } = await import("node:https"); + const { workerTls } = await import("./fixtures/worker-tls.ts"); + const tls = await workerTls(); + let host = ""; + let sni = ""; + let hits = 0; + const server = createServer(tls, (req, res) => { + hits++; + host = req.headers.host ?? ""; + sni = "servername" in req.socket ? String(req.socket.servername) : ""; + res.end("ok"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => server.close(() => resolve()))); + const address = server.address(); + assert.ok(address && typeof address !== "string"); + let resolutions = 0; + let pinned = ""; + const zone = new ZonePolicy({ ...config, ca: tls.ca }); + const fetch = restrictedFetch(zone, { + resolve: async () => { + resolutions++; + return resolutions === 1 ? ["8.8.8.8"] : ["127.0.0.1"]; + }, + request: (url, options) => { + assert.equal(options.servername, "tools.example"); + return request(url, { + ...options, + port: address.port, + lookup: (hostname, lookupOptions, callback) => { + assert.ok(options.lookup); + options.lookup(hostname, lookupOptions, (error, checked, family) => { + assert.equal(error, null); + pinned = String(checked); + callback(null, "127.0.0.1", family); + }); + }, + }); + }, + }); + assert.equal(await (await fetch("https://tools.example/mcp")).text(), "ok"); + assert.equal(pinned, "8.8.8.8"); + assert.equal(host, "tools.example"); + assert.equal(sni, "tools.example"); + await assert.rejects(fetch("https://tools.example/mcp")); + assert.equal(hits, 1); + assert.equal(resolutions, 2); + for (const [ca, authority] of [ + [undefined, "tools.example"], + [tls.ca, "other.example"], + ] as const) { + const bad = restrictedFetch( + new ZonePolicy({ ...config, ca, authorities: [authority] }), + { + resolve: async () => ["8.8.8.8"], + request: (url, options) => + request(url, { + ...options, + hostname: "127.0.0.1", + port: address.port, + }), + }, + ); + await assert.rejects(bad(`https://${authority}/mcp`)); + } + assert.equal(hits, 1); +}); +test("real HTTPS refuses redirects and bounds decompressed bodies", async (t) => { + const { createServer, request } = await import("node:https"); + const { gzipSync } = await import("node:zlib"); + const { workerTls } = await import("./fixtures/worker-tls.ts"); + const tls = await workerTls(); + let hits = 0; + const server = createServer(tls, (req, res) => { + hits++; + if (req.url === "/redirect") { + res.writeHead(302, { location: "https://tools.example/next" }).end(); + return; + } + res + .writeHead(200, { "content-encoding": "gzip" }) + .end(gzipSync("x".repeat(2 * 1024 * 1024 + 1))); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => server.close(() => resolve()))); + const address = server.address(); + assert.ok(address && typeof address !== "string"); + const fetch = restrictedFetch(new ZonePolicy({ ...config, ca: tls.ca }), { + resolve: async () => ["8.8.8.8"], + request: (url, options) => + request(url, { ...options, hostname: "127.0.0.1", port: address.port }), + }); + await assert.rejects(fetch("https://tools.example/redirect")); + assert.equal(hits, 1); + await assert.rejects((await fetch("https://tools.example/large")).text()); + assert.equal(hits, 2); +}); +test("permanent metadata and non-global IPv6 denies cannot be widened by an internal zone", () => { + const zone = new ZonePolicy({ + ...config, + mode: "internal", + cidrs: ["fc00::/7"], + }); + for (const ip of [ + "168.63.129.16", + "fd00:ec2::254", + "fd20:ce::254", + "fec0::1", + "3fff::1", + "64:ff9b:1::1", + ]) + assert.throws(() => zone.checkAddresses([ip]), ip); +}); +test("internal CIDRs cannot grant public routes and defaults enter policy identity", () => { + assert.throws( + () => + new ZonePolicy({ ...config, mode: "internal", cidrs: ["8.8.8.0/24"] }), + ); + assert.throws( + () => + new ZonePolicy({ ...config, mode: "internal", cidrs: ["10.0.0.0/7"] }), + ); + const a = new ZonePolicy(config); + const b = new ZonePolicy({ ...config, limits: { execute: 31 } }); + assert.notEqual(a.digest, b.digest); +}); + +for (const fault of ["status", "header"] as const) { + test( + `invalid upstream ${fault} rejects only its operation`, + { timeout: 3000 }, + async (t) => { + const { createServer, request } = await import("node:https"); + const { workerTls } = await import("./fixtures/worker-tls.ts"); + const tls = await workerTls(); + let requests = 0; + const server = createServer(tls, (_req, res) => { + requests++; + res + .writeHead(requests === 1 && fault === "status" ? 600 : 200) + .end("ok"); + }); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const address = server.address(); + assert.ok(address && typeof address !== "string"); + let conversions = 0; + const fetch = restrictedFetch(new ZonePolicy({ ...config, ca: tls.ca }), { + resolve: async () => ["8.8.8.8"], + request: (url, options) => { + const req = request(url, { + ...options, + hostname: "127.0.0.1", + port: address.port, + }); + req.prependOnceListener("response", (res) => { + if (++conversions === 1 && fault === "header") + res.headers["invalid\nname"] = "untrusted"; + }); + return req; + }, + }); + await assert.rejects(fetch("https://tools.example/mcp"), { + message: "response_invalid", + }); + assert.equal( + await (await fetch("https://tools.example/mcp")).text(), + "ok", + ); + assert.equal(requests, 2); + }, + ); +} diff --git a/mcp-worker/test/schema.test.ts b/mcp-worker/test/schema.test.ts new file mode 100644 index 000000000..1addbaadc --- /dev/null +++ b/mcp-worker/test/schema.test.ts @@ -0,0 +1,196 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { validateSchemas } from "../src/schema.ts"; +test("independent dialects, roots and local references retain their meaning", async () => { + const a = { + $id: "https://schema.example/a", + $schema: "http://json-schema.org/draft-07/schema#", + definitions: { value: { type: "integer", minimum: 1 } }, + properties: { value: { $ref: "#/definitions/value" } }, + }; + const b = { + $id: "https://schema.example/a", + $schema: "https://json-schema.org/draft/2020-12/schema", + $defs: { value: { type: "integer", maximum: 3 } }, + properties: { value: { $ref: "#/$defs/value" } }, + }; + assert.equal(await validateSchemas([a, b], { value: 2 }), true); + assert.equal(await validateSchemas([a, b], { value: 0 }), false); + assert.equal(await validateSchemas([a, b], { value: 4 }), false); + assert.equal(await validateSchemas([{ type: "number" }], "2"), false); +}); +test("unsupported dialects, remote references and executable formats fail closed", async () => { + for (const schema of [ + { $schema: "https://example.com/schema" }, + { $ref: "https://example.com/schema" }, + { format: "executable" }, + { type: "string", pattern: "[" }, + ]) + await assert.rejects(validateSchemas([schema])); +}); +test("hostile regular expression is killed in disposable process", async () => { + const begin = performance.now(); + await assert.rejects( + validateSchemas( + [{ type: "string", pattern: "^(a+)+$" }], + "a".repeat(100) + "!", + ), + { message: "schema_deadline" }, + ); + assert.ok(performance.now() - begin < 6500); + assert.equal(await validateSchemas([{ type: "integer" }], 1), true); +}); +test("standard formats validate in the child and ordinary data keys remain data", async () => { + assert.equal( + await validateSchemas( + [{ type: "string", format: "date-time" }], + "2026-11-02T12:00:00Z", + ), + true, + ); + assert.equal( + await validateSchemas([{ type: "string", format: "date-time" }], "invalid"), + false, + ); + assert.equal( + await validateSchemas( + [ + { + properties: { + $ref: { const: "https://example.com/a" }, + $schema: { const: "data" }, + }, + }, + ], + { $ref: "https://example.com/a", $schema: "data" }, + ), + true, + ); +}); +test("schema processes have a shared cap and cancellation kills the child", async () => { + const controllers = Array.from({ length: 4 }, () => new AbortController()); + const pending = controllers.map((c) => + validateSchemas( + [{ type: "string", pattern: "^(a+)+$" }], + "a".repeat(100) + "!", + c.signal, + ), + ); + await assert.rejects(validateSchemas([{}]), /capacity/); + for (const controller of controllers) controller.abort(); + for (const result of await Promise.allSettled(pending)) + assert.equal(result.status, "rejected"); +}); +test("trusted bootstrap delay cannot extend the schema timer and can be canceled", async (t) => { + const { default: childProcess } = await import("node:child_process"); + const { pathToFileURL } = await import("node:url"); + const original = childProcess.spawn; + let firstWrite = 0; + const mock = t.mock.method( + childProcess, + "spawn", + ( + command: string, + args: readonly string[], + options: import("node:child_process").SpawnOptionsWithoutStdio, + ) => { + const path = args.at(-1)!; + const code = `await new Promise(r=>setTimeout(r,600)); await import(${JSON.stringify(pathToFileURL(path).href)});`; + const child = original( + command, + [...args.slice(0, -1), "--input-type=module", "--eval", code], + options, + ); + child.stdin?.once("finish", () => { + firstWrite = performance.now(); + }); + return child; + }, + ); + await assert.rejects( + validateSchemas( + [{ type: "string", pattern: "^(a+)+$" }], + "a".repeat(100) + "!", + ), + /schema_deadline/, + ); + assert.ok(firstWrite > 0); + assert.ok(performance.now() - firstWrite < 2500); + const controller = new AbortController(); + const pending = validateSchemas([{}], {}, controller.signal); + controller.abort(); + await assert.rejects(pending, /schema_aborted/); + mock.mock.restore(); +}); +test("the actual disposable V8 heap ceiling is at most 128 MiB", async (t) => { + const { default: childProcess } = await import("node:child_process"); + const { pathToFileURL } = await import("node:url"); + const original = childProcess.spawn; + t.mock.method( + childProcess, + "spawn", + ( + command: string, + args: readonly string[], + options: import("node:child_process").SpawnOptionsWithoutStdio, + ) => { + const code = `import{getHeapStatistics}from'node:v8';if(getHeapStatistics().heap_size_limit>128*1024*1024)process.exit(9);await import(${JSON.stringify(pathToFileURL(args.at(-1)!).href)});`; + return original( + command, + [...args.slice(0, -1), "--input-type=module", "--eval", code], + options, + ); + }, + ); + assert.equal(await validateSchemas([{ type: "integer" }], 1), true); +}); + +test("split UTF-8 preserves schema constants and argument values", async () => { + const { spawn } = await import("node:child_process"); + const results: string[] = []; + for (const [value, splitValue] of [ + ["é", false], + ["��", false], + ["é", true], + ] as const) { + const child = spawn( + process.execPath, + [ + "--max-old-space-size=125", + "--max-semi-space-size=1", + new URL("../src/schema-child.ts", import.meta.url).pathname, + ], + { + env: {}, + stdio: ["pipe", "pipe", "ignore"], + signal: AbortSignal.timeout(5000), + }, + ); + const result = new Promise((resolve, reject) => { + let output = ""; + let sent = false; + child.on("error", reject); + child.stdout.on("data", (chunk: Buffer) => { + output += chunk.toString(); + if (!sent && output.startsWith("ready\n")) { + sent = true; + const payload = Buffer.from( + JSON.stringify({ documents: [{ const: "é" }], value }), + ); + const split = + (splitValue + ? payload.lastIndexOf(Buffer.from("é")) + : payload.indexOf(Buffer.from("é"))) + 1; + child.stdin.write(payload.subarray(0, split)); + setTimeout(() => child.stdin.end(payload.subarray(split)), 50); + } + }); + child.on("close", (code) => { + if (code !== 0) reject(Error("child_failed")); + else resolve(output); + }); + }); + results.push(await result); + } + assert.deepEqual(results, ["ready\ntrue", "ready\nfalse", "ready\ntrue"]); +}); diff --git a/mcp-worker/test/service.test.ts b/mcp-worker/test/service.test.ts new file mode 100644 index 000000000..2c7a0eae8 --- /dev/null +++ b/mcp-worker/test/service.test.ts @@ -0,0 +1,314 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { randomUUID } from "node:crypto"; +import { createWorker } from "../src/service.ts"; +import { digest } from "../src/encoding.ts"; +import { ZonePolicy } from "../src/network.ts"; +import { workerTls, localRpc } from "./fixtures/worker-tls.ts"; +import type { + WorkerIdentity, + WorkerOperation, + WorkerRequest, +} from "../src/contracts.ts"; +const zone = () => + new ZonePolicy({ + id: "public", + revision: "1", + mode: "public", + authorities: ["tools.example"], + aliases: [], + cidrs: [], + denyCidrs: [], + }); +function envelope( + identity: WorkerIdentity, + operation: WorkerOperation, +): WorkerRequest { + return { + protocol: 1, + intent: { + id: randomUUID(), + ...identity, + connectionId: "connection", + connectionRevision: 1, + credentialIdentity: "credential", + scope: { + kind: "administration", + requestId: "admin-request", + actorId: "actor", + }, + operation: operation.kind, + destination: "endpoint" in operation ? operation.endpoint : "", + requestDigest: "a".repeat(64), + operationDigest: digest(operation), + expiresAt: Date.now() + 10000, + lease: { + id: randomUUID(), + deploymentId: "deployment", + expiresAt: Date.now() + 10000, + }, + }, + operation, + }; +} +test("mTLS RPC binds boot, policy, payload and lease; each intent is single use", async (t) => { + const tls = await workerTls(); + const worker = createWorker({ + zone: zone(), + tls, + callers: [{ fingerprint: tls.fingerprint, role: "orchestrator" }], + enabled: true, + }); + const port = await worker.listen(0, "127.0.0.1"); + t.after(() => worker.close()); + await assert.rejects(localRpc(port, tls, "/identity", undefined, false)); + const hello = await localRpc(port, tls, "/identity"); + assert.equal(hello.status, 200); + assert.deepEqual(hello.body, worker.identity); + const op: WorkerOperation = { + kind: "validateSchemas", + documents: [{ type: "integer" }], + value: 1, + }; + for (const field of ["bootId", "policyDigest", "operationDigest"] as const) { + const request = envelope(worker.identity, op); + request.intent[field] = "wrong"; + assert.equal((await localRpc(port, tls, "/rpc", request)).status, 403); + } + const expired = envelope(worker.identity, op); + expired.intent.lease.expiresAt = 0; + assert.equal((await localRpc(port, tls, "/rpc", expired)).status, 403); + const request = envelope(worker.identity, op); + const result = await localRpc(port, tls, "/rpc", request); + assert.equal(result.status, 200); + assert.ok( + typeof result.body === "object" && + result.body !== null && + "intent" in result.body, + ); + assert.deepEqual(result.body.intent, request.intent); + assert.ok( + typeof result.body === "object" && + result.body !== null && + "result" in result.body && + result.body.result === true, + ); + assert.equal((await localRpc(port, tls, "/rpc", request)).status, 403); + assert.equal((await localRpc(port, tls, "/proxy", request)).status, 404); + assert.equal( + ( + await localRpc( + port, + tls, + "/rpc", + envelope(worker.identity, { + kind: "token", + endpoint: "https://tools.example/token", + grant: { + grant_type: "refresh_token", + refresh_token: "secret", + client_id: "client", + resource: "https://tools.example/mcp", + }, + }), + ) + ).status, + 403, + ); +}); +test("disabled worker and unknown client identity cannot dispatch", async (t) => { + const tls = await workerTls(); + const disabled = createWorker({ + zone: zone(), + tls, + callers: [{ fingerprint: tls.fingerprint, role: "orchestrator" }], + }); + const port = await disabled.listen(0, "127.0.0.1"); + t.after(() => disabled.close()); + assert.equal( + ( + await localRpc( + port, + tls, + "/rpc", + envelope(disabled.identity, { kind: "validateSchemas", documents: [] }), + ) + ).status, + 503, + ); + const unknown = createWorker({ + zone: zone(), + tls, + callers: [{ fingerprint: "00", role: "orchestrator" }], + enabled: true, + }); + const other = await unknown.listen(0, "127.0.0.1"); + t.after(() => unknown.close()); + assert.equal((await localRpc(other, tls, "/identity")).status, 403); +}); +test("execution schema validation keeps the orchestrator role and exact bound scope", async (t) => { + const tls = await workerTls(); + for (const role of ["orchestrator", "coordinator"] as const) { + let dials = 0; + const worker = createWorker({ + zone: zone(), + tls, + callers: [{ fingerprint: tls.fingerprint, role }], + enabled: true, + network: { + resolve: async () => { + dials++; + throw Error("unexpected provider lookup"); + }, + request: () => { + dials++; + throw Error("unexpected provider request"); + }, + }, + }); + const port = await worker.listen(0, "127.0.0.1"); + t.after(() => worker.close()); + const request = envelope(worker.identity, { + kind: "validateSchemas", + documents: [{ type: "integer" }], + value: 2, + }); + request.intent.scope = { + kind: "execution", + sessionId: "session", + runId: "run", + toolCallId: "call", + bindingHash: "b".repeat(64), + }; + const result = await localRpc(port, tls, "/rpc", request); + assert.equal(result.status, role === "orchestrator" ? 200 : 403); + if (role === "orchestrator") { + assert.ok( + typeof result.body === "object" && + result.body !== null && + "intent" in result.body, + ); + assert.deepEqual(result.body.intent, request.intent); + assert.equal((await localRpc(port, tls, "/rpc", request)).status, 403); + const credential = { header: "authorization", value: "Bearer private" }; + const secretRequest = envelope(worker.identity, { + kind: "validateSchemas", + documents: [{}], + }); + const invalid = { + ...secretRequest, + operation: { ...secretRequest.operation, credential }, + }; + assert.equal((await localRpc(port, tls, "/rpc", invalid)).status, 400); + } + assert.equal(dials, 0); + } +}); +test("OAuth admission is separate, destination validation does not dial, and scoped cancel aborts I/O", async (t) => { + const { createServer, request } = await import("node:https"); + const tls = await workerTls(); + let hits = 0; + let arrived: () => void = () => {}; + const arrival = new Promise((resolve) => { + arrived = resolve; + }); + const provider = createServer(tls, () => { + hits++; + arrived(); + }); + await new Promise((resolve) => + provider.listen(0, "127.0.0.1", resolve), + ); + t.after(() => { + provider.closeAllConnections(); + return new Promise((resolve) => provider.close(() => resolve())); + }); + const address = provider.address(); + assert.ok(address && typeof address !== "string"); + const policy = new ZonePolicy({ + id: "public", + revision: "1", + mode: "public", + authorities: ["tools.example"], + aliases: [], + cidrs: [], + denyCidrs: [], + ca: tls.ca, + limits: { oauth: 1 }, + }); + const worker = createWorker({ + zone: policy, + tls, + callers: [{ fingerprint: tls.fingerprint, role: "coordinator" }], + enabled: true, + network: { + resolve: async () => ["8.8.8.8"], + request: (url, options) => + request(url, { ...options, hostname: "127.0.0.1", port: address.port }), + }, + }); + const port = await worker.listen(0, "127.0.0.1"); + t.after(() => worker.close()); + const first = envelope(worker.identity, { + kind: "metadata", + endpoint: "https://tools.example/metadata", + }); + const pending = localRpc(port, tls, "/rpc", first); + void pending.catch(() => {}); + await arrival; + assert.equal( + ( + await localRpc( + port, + tls, + "/rpc", + envelope(worker.identity, { + kind: "metadata", + endpoint: "https://tools.example/metadata", + }), + ) + ).status, + 429, + ); + assert.equal( + ( + await localRpc( + port, + tls, + "/rpc", + envelope(worker.identity, { + kind: "validateDestination", + endpoint: "https://tools.example/metadata", + }), + ) + ).status, + 200, + ); + assert.equal(hits, 1); + const wrong = envelope(worker.identity, { + kind: "cancel", + targetIntentId: first.intent.id, + }); + wrong.intent.scope = { + kind: "administration", + requestId: "different", + actorId: "actor", + }; + assert.equal((await localRpc(port, tls, "/rpc", wrong)).status, 403); + assert.equal( + ( + await localRpc( + port, + tls, + "/rpc", + envelope(worker.identity, { + kind: "cancel", + targetIntentId: first.intent.id, + }), + ) + ).status, + 200, + ); + assert.equal((await pending).status, 422); + assert.equal(hits, 1); +}); diff --git a/mcp-worker/tsconfig.build.json b/mcp-worker/tsconfig.build.json new file mode 100644 index 000000000..98a8d519f --- /dev/null +++ b/mcp-worker/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "rewriteRelativeImportExtensions": true, + "outDir": "dist", + "rootDir": "src", + "declaration": true + }, + "include": ["src/**/*.ts"] +} diff --git a/mcp-worker/tsconfig.json b/mcp-worker/tsconfig.json new file mode 100644 index 000000000..f268372b1 --- /dev/null +++ b/mcp-worker/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2024", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "verbatimModuleSyntax": true, + "erasableSyntaxOnly": true, + "allowImportingTsExtensions": true, + "noEmit": true, + "types": ["node"] + }, + "include": ["src/**/*.ts", "test/**/*.ts"] +} diff --git a/orchestrator/README.md b/orchestrator/README.md new file mode 100644 index 000000000..53913a89f --- /dev/null +++ b/orchestrator/README.md @@ -0,0 +1,51 @@ +# Orchestrator + +The orchestrator uses Bun. Run `bun run typecheck` and the focused `bun test` targets for a change. Database tests require an isolated test database. + +## External MCP startup + +External MCP is disabled by default. `MCP_ENABLED=true` permits new managed operations. The only accepted flag values are `true` and `false`. Enabling it also requires `MCP_RUNTIME_CONFIG`, an operator-owned JSON file path. A worker zone must separately have `enabled: true`; its default is false. + +The configuration has this structure: + +```json +{ + "version": 1, + "infrastructure": { + "deploymentIdentity": "sha256:<64 lowercase hexadecimal characters>", + "networkIdentity": "sha256:<64 lowercase hexadecimal characters>", + "trustIdentity": "sha256:<64 lowercase hexadecimal characters>" + }, + "caFiles": { "control": "/etc/engrams/mcp/control-ca.pem" }, + "control": { + "url": "https://control.example.test/", + "caName": "control", + "certFile": "/etc/engrams/mcp/client.pem", + "keyFile": "/etc/engrams/mcp/client-key.pem", + "serverFingerprint": "" + }, + "workers": [ + { + "zoneId": "public", + "deploymentId": "public-workers", + "enabled": false, + "tls": { + "url": "https://worker.example.test/", + "caName": "control", + "certFile": "/etc/engrams/mcp/client.pem", + "keyFile": "/etc/engrams/mcp/client-key.pem", + "serverFingerprint": "" + } + } + ], + "registrations": [] +} +``` + +Replace the example identities, paths, addresses and fingerprints with operator values. The loader rejects unknown fields and duplicate zones. It reads certificate and key files into memory and hashes the actual CA file contents. The live coordinator publication must match the configured infrastructure and CA identities. A configured address alone does not authorize a destination. Each worker deployment ID is a stable admission scope shared by all orchestrator processes that use that worker deployment. + +Optional OAuth registrations have `id`, `clientId`, `confidential`, `redirectUri`, `issuer`, `resource` (`zoneId` and `endpoint`), and `scopes`. Resource credentials and OAuth client secrets are stored through the credential control service. They do not belong in this file. + +Keep the configuration path set when disabling new dispatch to retain authenticated lifecycle reads and terminal reconciliation. Existing journal expiry, cancellation bookkeeping, and committed completion recovery stay installed even without a configuration file. Expiry scans persisted scopes, including removed worker zones. Disabling dispatch does not replay external effects. A held or uncertain managed creation retains its durable identity; the orchestrator does not reconstruct lost secret-bearing creation bytes. + +The application installs the shared runtime before starting workflows. Recovery and managed-session scanners stop before the listener manager and workflow engine during shutdown. A new managed binding becomes dispatchable only after the coordinator reports the exact finalized boot as started and the current host policy acknowledgement matches it. diff --git a/orchestrator/drizzle/0101_mcp_authority.sql b/orchestrator/drizzle/0101_mcp_authority.sql new file mode 100644 index 000000000..0fbe12193 --- /dev/null +++ b/orchestrator/drizzle/0101_mcp_authority.sql @@ -0,0 +1,138 @@ +CREATE TABLE "mcp_admission" ( + "execution_id" text PRIMARY KEY NOT NULL, + "deployment_id" text NOT NULL, + "connection_id" text NOT NULL, + "session_id" text NOT NULL, + "state" text NOT NULL, + "expires_at" timestamp with time zone NOT NULL, + "created_at" timestamp with time zone NOT NULL, + CONSTRAINT "mcp_admission_state_check" CHECK ("mcp_admission"."state" IN ('queued','admitted')) +); +--> statement-breakpoint +CREATE TABLE "mcp_admission_scope" ( + "id" text PRIMARY KEY NOT NULL +); +--> statement-breakpoint +CREATE TABLE "mcp_approval" ( + "execution_id" text PRIMARY KEY NOT NULL, + "deployment_id" text NOT NULL, + "session_id" text NOT NULL, + "request_digest" text NOT NULL, + "expires_at" timestamp with time zone NOT NULL, + "decision" text NOT NULL, + "actor_id" text, + "decided_at" timestamp with time zone, + "consumed_at" timestamp with time zone, + CONSTRAINT "mcp_approval_decision_check" CHECK ("mcp_approval"."decision" IN ('pending','approved','denied','expired')), + CONSTRAINT "mcp_approval_consumed_check" CHECK ("mcp_approval"."consumed_at" IS NULL OR ("mcp_approval"."decision" = 'approved' AND "mcp_approval"."actor_id" IS NOT NULL)) +); +--> statement-breakpoint +CREATE TABLE "mcp_catalog" ( + "id" text PRIMARY KEY NOT NULL, + "connection_id" text NOT NULL, + "kind" text NOT NULL, + "revision" integer NOT NULL, + "config_revision" integer NOT NULL, + "auth_generation" integer NOT NULL, + "credential_identity" text NOT NULL, + "tools" jsonb NOT NULL, + "fingerprint" text NOT NULL, + "actor_id" text NOT NULL, + "created_at" timestamp with time zone NOT NULL, + CONSTRAINT "mcp_catalog_kind_check" CHECK ("mcp_catalog"."kind" IN ('draft','published') AND (("mcp_catalog"."kind" = 'draft' AND "mcp_catalog"."revision" = 0) OR ("mcp_catalog"."kind" = 'published' AND "mcp_catalog"."revision" > 0))) +); +--> statement-breakpoint +CREATE TABLE "mcp_execution" ( + "id" text PRIMARY KEY NOT NULL, + "task_id" text NOT NULL, + "session_id" text NOT NULL, + "run_id" text NOT NULL, + "tool_call_id" text NOT NULL, + "connection_id" text NOT NULL, + "catalog_id" text NOT NULL, + "principal_id" text NOT NULL, + "binding_hash" text NOT NULL, + "request_digest" text NOT NULL, + "argument_hash" text NOT NULL, + "upstream_name" text NOT NULL, + "config_revision" integer NOT NULL, + "auth_generation" integer NOT NULL, + "classification" text NOT NULL, + "state" text NOT NULL, + "version" integer DEFAULT 1 NOT NULL, + "boot_id" text, + "intent_id" text, + "intent_expires_at" timestamp with time zone, + "deadline" timestamp with time zone NOT NULL, + "result" jsonb, + "result_unavailable" boolean DEFAULT false NOT NULL, + "created_at" timestamp with time zone NOT NULL, + "updated_at" timestamp with time zone NOT NULL, + CONSTRAINT "mcp_execution_state_check" CHECK ("mcp_execution"."state" IN ('waiting_approval','authorized','dispatching','succeeded','failed','canceled','outcome_unknown')), + CONSTRAINT "mcp_execution_classification_check" CHECK ("mcp_execution"."classification" IN ('read','write')), + CONSTRAINT "mcp_execution_dispatch_check" CHECK ("mcp_execution"."state" NOT IN ('dispatching','outcome_unknown','succeeded') OR ("mcp_execution"."boot_id" IS NOT NULL AND "mcp_execution"."intent_id" IS NOT NULL AND "mcp_execution"."intent_expires_at" IS NOT NULL)) +); +--> statement-breakpoint +CREATE TABLE "mcp_resolution" ( + "execution_id" text PRIMARY KEY NOT NULL, + "actor_id" text NOT NULL, + "outcome" text NOT NULL, + "created_at" timestamp with time zone NOT NULL, + CONSTRAINT "mcp_resolution_outcome_check" CHECK ("mcp_resolution"."outcome" IN ('succeeded','failed','acknowledged')) +); +--> statement-breakpoint +ALTER TABLE "integration_connection" ADD COLUMN "config_revision" integer DEFAULT 1 NOT NULL;--> statement-breakpoint +ALTER TABLE "integration_connection" ADD COLUMN "auth_generation" integer DEFAULT 1 NOT NULL;--> statement-breakpoint +ALTER TABLE "integration_connection" ADD COLUMN "credential_identity" text DEFAULT '' NOT NULL;--> statement-breakpoint +ALTER TABLE "integration_connection" ADD COLUMN "catalog_revision" integer DEFAULT 0 NOT NULL;--> statement-breakpoint +ALTER TABLE "integration_connection" ADD COLUMN "revoked" boolean DEFAULT false NOT NULL;--> statement-breakpoint +ALTER TABLE "mcp_admission" ADD CONSTRAINT "mcp_admission_execution_id_mcp_execution_id_fk" FOREIGN KEY ("execution_id") REFERENCES "public"."mcp_execution"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_admission" ADD CONSTRAINT "mcp_admission_deployment_id_mcp_admission_scope_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "public"."mcp_admission_scope"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_admission" ADD CONSTRAINT "mcp_admission_connection_id_integration_connection_id_fk" FOREIGN KEY ("connection_id") REFERENCES "public"."integration_connection"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_approval" ADD CONSTRAINT "mcp_approval_execution_id_mcp_execution_id_fk" FOREIGN KEY ("execution_id") REFERENCES "public"."mcp_execution"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_catalog" ADD CONSTRAINT "mcp_catalog_connection_id_integration_connection_id_fk" FOREIGN KEY ("connection_id") REFERENCES "public"."integration_connection"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_execution" ADD CONSTRAINT "mcp_execution_connection_id_integration_connection_id_fk" FOREIGN KEY ("connection_id") REFERENCES "public"."integration_connection"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_execution" ADD CONSTRAINT "mcp_execution_catalog_id_mcp_catalog_id_fk" FOREIGN KEY ("catalog_id") REFERENCES "public"."mcp_catalog"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_execution" ADD CONSTRAINT "mcp_execution_task_id_session_id_task_session_task_id_session_id_fk" FOREIGN KEY ("task_id","session_id") REFERENCES "public"."task_session"("task_id","session_id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_resolution" ADD CONSTRAINT "mcp_resolution_execution_id_mcp_execution_id_fk" FOREIGN KEY ("execution_id") REFERENCES "public"."mcp_execution"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "mcp_admission_deployment_idx" ON "mcp_admission" USING btree ("deployment_id","state","expires_at");--> statement-breakpoint +CREATE INDEX "mcp_admission_connection_idx" ON "mcp_admission" USING btree ("connection_id","state");--> statement-breakpoint +CREATE INDEX "mcp_admission_session_idx" ON "mcp_admission" USING btree ("session_id","state");--> statement-breakpoint +CREATE INDEX "mcp_approval_pending_idx" ON "mcp_approval" USING btree ("deployment_id","decision","expires_at");--> statement-breakpoint +CREATE INDEX "mcp_approval_session_idx" ON "mcp_approval" USING btree ("session_id","decision");--> statement-breakpoint +CREATE INDEX "mcp_catalog_connection_idx" ON "mcp_catalog" USING btree ("connection_id","created_at");--> statement-breakpoint +CREATE UNIQUE INDEX "mcp_catalog_published_revision_idx" ON "mcp_catalog" USING btree ("connection_id","revision") WHERE kind = 'published';--> statement-breakpoint +CREATE INDEX "mcp_execution_task_session_idx" ON "mcp_execution" USING btree ("task_id","session_id");--> statement-breakpoint +CREATE UNIQUE INDEX "mcp_execution_call_idx" ON "mcp_execution" USING btree ("session_id","tool_call_id");--> statement-breakpoint +CREATE INDEX "mcp_execution_connection_state_idx" ON "mcp_execution" USING btree ("connection_id","session_id","state");--> statement-breakpoint +CREATE INDEX "mcp_execution_catalog_idx" ON "mcp_execution" USING btree ("catalog_id");--> statement-breakpoint +CREATE INDEX "mcp_execution_pending_idx" ON "mcp_execution" USING btree ("state","deadline");--> statement-breakpoint +CREATE UNIQUE INDEX "mcp_execution_intent_idx" ON "mcp_execution" USING btree ("intent_id");--> statement-breakpoint +CREATE FUNCTION mcp_catalog_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'MCP catalogs are immutable'; +END; +$$; +--> statement-breakpoint +CREATE TRIGGER mcp_catalog_immutable_update BEFORE UPDATE ON mcp_catalog FOR EACH ROW EXECUTE FUNCTION mcp_catalog_immutable(); +--> statement-breakpoint +CREATE FUNCTION mcp_execution_transition() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF (to_jsonb(NEW) - ARRAY['state','version','boot_id','intent_id','intent_expires_at','result','result_unavailable','updated_at']) IS DISTINCT FROM + (to_jsonb(OLD) - ARRAY['state','version','boot_id','intent_id','intent_expires_at','result','result_unavailable','updated_at']) THEN + RAISE EXCEPTION 'MCP request identity is immutable'; + END IF; + IF NOT ((OLD.state = 'waiting_approval' AND NEW.state IN ('authorized','canceled')) OR + (OLD.state = 'authorized' AND NEW.state IN ('dispatching','canceled')) OR + (OLD.state = 'dispatching' AND NEW.state IN ('succeeded','failed','outcome_unknown')) OR + (OLD.state = 'outcome_unknown' AND NEW.state IN ('succeeded','failed'))) OR NEW.version <> OLD.version + 1 THEN + RAISE EXCEPTION 'Invalid MCP execution transition'; + END IF; + IF OLD.intent_id IS NOT NULL AND (NEW.intent_id, NEW.boot_id, NEW.intent_expires_at) IS DISTINCT FROM (OLD.intent_id, OLD.boot_id, OLD.intent_expires_at) THEN + RAISE EXCEPTION 'MCP dispatch intent is immutable'; + END IF; + RETURN NEW; +END; +$$; +--> statement-breakpoint +CREATE TRIGGER mcp_execution_transition_update BEFORE UPDATE ON mcp_execution FOR EACH ROW EXECUTE FUNCTION mcp_execution_transition(); diff --git a/orchestrator/drizzle/0102_mcp_authorization_kind.sql b/orchestrator/drizzle/0102_mcp_authorization_kind.sql new file mode 100644 index 000000000..41d4d8be9 --- /dev/null +++ b/orchestrator/drizzle/0102_mcp_authorization_kind.sql @@ -0,0 +1,3 @@ +ALTER TABLE "mcp_execution" ADD COLUMN "authorization_kind" text NOT NULL;--> statement-breakpoint +ALTER TABLE "mcp_execution" ADD COLUMN "automation_grant_hash" text;--> statement-breakpoint +ALTER TABLE "mcp_execution" ADD CONSTRAINT "mcp_execution_authorization_check" CHECK (("mcp_execution"."authorization_kind" = 'read' AND "mcp_execution"."classification" = 'read' AND "mcp_execution"."automation_grant_hash" IS NULL) OR ("mcp_execution"."authorization_kind" = 'approval' AND "mcp_execution"."automation_grant_hash" IS NULL) OR ("mcp_execution"."authorization_kind" = 'automation' AND "mcp_execution"."classification" = 'write' AND "mcp_execution"."automation_grant_hash" IS NOT NULL)); \ No newline at end of file diff --git a/orchestrator/drizzle/0103_mcp_session_retention.sql b/orchestrator/drizzle/0103_mcp_session_retention.sql new file mode 100644 index 000000000..1807c1bf0 --- /dev/null +++ b/orchestrator/drizzle/0103_mcp_session_retention.sql @@ -0,0 +1,12 @@ +ALTER TABLE "mcp_admission" DROP CONSTRAINT "mcp_admission_execution_id_mcp_execution_id_fk"; +--> statement-breakpoint +ALTER TABLE "mcp_approval" DROP CONSTRAINT "mcp_approval_execution_id_mcp_execution_id_fk"; +--> statement-breakpoint +ALTER TABLE "mcp_execution" DROP CONSTRAINT "mcp_execution_task_id_session_id_task_session_task_id_session_id_fk"; +--> statement-breakpoint +ALTER TABLE "mcp_resolution" DROP CONSTRAINT "mcp_resolution_execution_id_mcp_execution_id_fk"; +--> statement-breakpoint +ALTER TABLE "mcp_admission" ADD CONSTRAINT "mcp_admission_execution_id_mcp_execution_id_fk" FOREIGN KEY ("execution_id") REFERENCES "public"."mcp_execution"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_approval" ADD CONSTRAINT "mcp_approval_execution_id_mcp_execution_id_fk" FOREIGN KEY ("execution_id") REFERENCES "public"."mcp_execution"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_execution" ADD CONSTRAINT "mcp_execution_task_id_session_id_task_session_task_id_session_id_fk" FOREIGN KEY ("task_id","session_id") REFERENCES "public"."task_session"("task_id","session_id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_resolution" ADD CONSTRAINT "mcp_resolution_execution_id_mcp_execution_id_fk" FOREIGN KEY ("execution_id") REFERENCES "public"."mcp_execution"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/orchestrator/drizzle/0104_mcp_credential_metadata.sql b/orchestrator/drizzle/0104_mcp_credential_metadata.sql new file mode 100644 index 000000000..0e00b3cc6 --- /dev/null +++ b/orchestrator/drizzle/0104_mcp_credential_metadata.sql @@ -0,0 +1,3 @@ +-- Coordinator receipts and serialized transition metadata only; no credential values. +ALTER TABLE integration_connection ADD COLUMN mcp_credential jsonb NOT NULL DEFAULT '{}'::jsonb; +ALTER TABLE integration_connection ADD CONSTRAINT integration_connection_mcp_metadata_object CHECK (jsonb_typeof(mcp_credential) = 'object'); diff --git a/orchestrator/drizzle/0105_mcp_binding_envelope.sql b/orchestrator/drizzle/0105_mcp_binding_envelope.sql new file mode 100644 index 000000000..abcad55a8 --- /dev/null +++ b/orchestrator/drizzle/0105_mcp_binding_envelope.sql @@ -0,0 +1 @@ +ALTER TABLE "task_session" ADD COLUMN "mcp_binding_envelope" jsonb; diff --git a/orchestrator/drizzle/0106_mcp_request_review.sql b/orchestrator/drizzle/0106_mcp_request_review.sql new file mode 100644 index 000000000..ea4737f84 --- /dev/null +++ b/orchestrator/drizzle/0106_mcp_request_review.sql @@ -0,0 +1,2 @@ +ALTER TABLE "mcp_execution" ADD COLUMN "original_arguments" text;--> statement-breakpoint +ALTER TABLE "mcp_execution" ADD COLUMN "request_context" jsonb; \ No newline at end of file diff --git a/orchestrator/drizzle/0107_mcp_admission_generation.sql b/orchestrator/drizzle/0107_mcp_admission_generation.sql new file mode 100644 index 000000000..1b29fa51d --- /dev/null +++ b/orchestrator/drizzle/0107_mcp_admission_generation.sql @@ -0,0 +1,2 @@ +ALTER TABLE "mcp_admission" ADD COLUMN "reservation_id" text DEFAULT gen_random_uuid()::text NOT NULL;--> statement-breakpoint +CREATE UNIQUE INDEX "mcp_admission_reservation_idx" ON "mcp_admission" USING btree ("reservation_id"); \ No newline at end of file diff --git a/orchestrator/drizzle/0108_mcp_execution_authority.sql b/orchestrator/drizzle/0108_mcp_execution_authority.sql new file mode 100644 index 000000000..da8c99ee1 --- /dev/null +++ b/orchestrator/drizzle/0108_mcp_execution_authority.sql @@ -0,0 +1,7 @@ +ALTER TABLE "mcp_admission" DROP CONSTRAINT "mcp_admission_state_check";--> statement-breakpoint +ALTER TABLE "mcp_admission" ADD COLUMN "phase" text DEFAULT 'execute' NOT NULL;--> statement-breakpoint +ALTER TABLE "mcp_admission" ADD COLUMN "schema_queue_expires_at" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "mcp_admission" ADD COLUMN "execute_queue_expires_at" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "mcp_admission" ADD COLUMN "schema_digest" text;--> statement-breakpoint +ALTER TABLE "profile" ADD COLUMN "mcp_grant_publication" jsonb;--> statement-breakpoint +ALTER TABLE "mcp_admission" ADD CONSTRAINT "mcp_admission_state_check" CHECK ("mcp_admission"."state" IN ('queued','admitted','released')); \ No newline at end of file diff --git a/orchestrator/drizzle/0109_mcp_fixed_execute_deadline.sql b/orchestrator/drizzle/0109_mcp_fixed_execute_deadline.sql new file mode 100644 index 000000000..b261d5aab --- /dev/null +++ b/orchestrator/drizzle/0109_mcp_fixed_execute_deadline.sql @@ -0,0 +1,2 @@ +ALTER TABLE "mcp_admission" ADD COLUMN "execute_expires_at" timestamp with time zone;--> statement-breakpoint +ALTER TABLE "mcp_admission" ADD CONSTRAINT "mcp_admission_phase_check" CHECK ("mcp_admission"."phase" IN ('schema','execute')); \ No newline at end of file diff --git a/orchestrator/drizzle/0110_mcp_preflight_rejection.sql b/orchestrator/drizzle/0110_mcp_preflight_rejection.sql new file mode 100644 index 000000000..9b51016f3 --- /dev/null +++ b/orchestrator/drizzle/0110_mcp_preflight_rejection.sql @@ -0,0 +1,18 @@ +CREATE OR REPLACE FUNCTION mcp_execution_transition() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF (to_jsonb(NEW) - ARRAY['state','version','boot_id','intent_id','intent_expires_at','result','result_unavailable','updated_at']) IS DISTINCT FROM + (to_jsonb(OLD) - ARRAY['state','version','boot_id','intent_id','intent_expires_at','result','result_unavailable','updated_at']) THEN + RAISE EXCEPTION 'MCP request identity is immutable'; + END IF; + IF NOT ((OLD.state = 'waiting_approval' AND NEW.state IN ('authorized','canceled','failed')) OR + (OLD.state = 'authorized' AND NEW.state IN ('dispatching','canceled','failed')) OR + (OLD.state = 'dispatching' AND NEW.state IN ('succeeded','failed','outcome_unknown')) OR + (OLD.state = 'outcome_unknown' AND NEW.state IN ('succeeded','failed'))) OR NEW.version <> OLD.version + 1 THEN + RAISE EXCEPTION 'Invalid MCP execution transition'; + END IF; + IF OLD.intent_id IS NOT NULL AND (NEW.intent_id, NEW.boot_id, NEW.intent_expires_at) IS DISTINCT FROM (OLD.intent_id, OLD.boot_id, OLD.intent_expires_at) THEN + RAISE EXCEPTION 'MCP dispatch intent is immutable'; + END IF; + RETURN NEW; +END; +$$; diff --git a/orchestrator/drizzle/0111_mcp_completion_receipt.sql b/orchestrator/drizzle/0111_mcp_completion_receipt.sql new file mode 100644 index 000000000..7527b3175 --- /dev/null +++ b/orchestrator/drizzle/0111_mcp_completion_receipt.sql @@ -0,0 +1,28 @@ +ALTER TABLE "mcp_execution" ADD COLUMN "completion_enqueued_version" integer DEFAULT 0 NOT NULL;--> statement-breakpoint +CREATE INDEX "mcp_execution_recovery_idx" ON "mcp_execution" USING btree ("id") WHERE "mcp_execution"."completion_enqueued_version" < "mcp_execution"."version";--> statement-breakpoint +ALTER TABLE "mcp_execution" ADD CONSTRAINT "mcp_execution_completion_version_check" CHECK ("mcp_execution"."completion_enqueued_version" >= 0 AND "mcp_execution"."completion_enqueued_version" <= "mcp_execution"."version"); +--> statement-breakpoint +CREATE OR REPLACE FUNCTION mcp_execution_transition() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF (to_jsonb(NEW) - 'completion_enqueued_version') = (to_jsonb(OLD) - 'completion_enqueued_version') + AND OLD.state IN ('succeeded','failed','canceled','outcome_unknown') + AND NEW.completion_enqueued_version >= OLD.completion_enqueued_version + AND NEW.completion_enqueued_version <= OLD.version THEN + RETURN NEW; + END IF; + IF (to_jsonb(NEW) - ARRAY['state','version','boot_id','intent_id','intent_expires_at','result','result_unavailable','updated_at']) IS DISTINCT FROM + (to_jsonb(OLD) - ARRAY['state','version','boot_id','intent_id','intent_expires_at','result','result_unavailable','updated_at']) THEN + RAISE EXCEPTION 'MCP request identity is immutable'; + END IF; + IF NOT ((OLD.state = 'waiting_approval' AND NEW.state IN ('authorized','canceled','failed')) OR + (OLD.state = 'authorized' AND NEW.state IN ('dispatching','canceled','failed')) OR + (OLD.state = 'dispatching' AND NEW.state IN ('succeeded','failed','outcome_unknown')) OR + (OLD.state = 'outcome_unknown' AND NEW.state IN ('succeeded','failed'))) OR NEW.version <> OLD.version + 1 THEN + RAISE EXCEPTION 'Invalid MCP execution transition'; + END IF; + IF OLD.intent_id IS NOT NULL AND (NEW.intent_id, NEW.boot_id, NEW.intent_expires_at) IS DISTINCT FROM (OLD.intent_id, OLD.boot_id, OLD.intent_expires_at) THEN + RAISE EXCEPTION 'MCP dispatch intent is immutable'; + END IF; + RETURN NEW; +END; +$$; diff --git a/orchestrator/drizzle/0112_mcp_lifecycle_receipts.sql b/orchestrator/drizzle/0112_mcp_lifecycle_receipts.sql new file mode 100644 index 000000000..b78a4cd14 --- /dev/null +++ b/orchestrator/drizzle/0112_mcp_lifecycle_receipts.sql @@ -0,0 +1,14 @@ +CREATE TABLE "mcp_cancellation" ( + "execution_id" text PRIMARY KEY NOT NULL, + "actor_id" text, + "cause" text NOT NULL, + "expected_journal_revision" integer NOT NULL, + "journal_revision" integer NOT NULL, + "state" text NOT NULL, + "created_at" timestamp with time zone NOT NULL, + CONSTRAINT "mcp_cancellation_revision_check" CHECK ("mcp_cancellation"."expected_journal_revision" > 0 AND "mcp_cancellation"."journal_revision" = "mcp_cancellation"."expected_journal_revision" + 1), + CONSTRAINT "mcp_cancellation_state_check" CHECK ("mcp_cancellation"."state" IN ('canceled','outcome_unknown')) +); +--> statement-breakpoint +ALTER TABLE "mcp_resolution" ADD COLUMN "expected_context" jsonb;--> statement-breakpoint +ALTER TABLE "mcp_cancellation" ADD CONSTRAINT "mcp_cancellation_execution_id_mcp_execution_id_fk" FOREIGN KEY ("execution_id") REFERENCES "public"."mcp_execution"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/orchestrator/drizzle/0113_mcp_lifecycle_active.sql b/orchestrator/drizzle/0113_mcp_lifecycle_active.sql new file mode 100644 index 000000000..d0cfe9d38 --- /dev/null +++ b/orchestrator/drizzle/0113_mcp_lifecycle_active.sql @@ -0,0 +1 @@ +CREATE INDEX "mcp_execution_lifecycle_active_idx" ON "mcp_execution" USING btree ("id") WHERE "mcp_execution"."state" IN ('waiting_approval','authorized','dispatching'); \ No newline at end of file diff --git a/orchestrator/drizzle/0114_mcp_discovery_admission.sql b/orchestrator/drizzle/0114_mcp_discovery_admission.sql new file mode 100644 index 000000000..553598336 --- /dev/null +++ b/orchestrator/drizzle/0114_mcp_discovery_admission.sql @@ -0,0 +1,42 @@ +CREATE TABLE "mcp_discovery_admission" ( + "request_id" text PRIMARY KEY NOT NULL, + "intent_id" text NOT NULL, + "deployment_id" text NOT NULL, + "connection_id" text NOT NULL, + "actor_id" text NOT NULL, + "config_revision" integer NOT NULL, + "auth_generation" integer NOT NULL, + "credential_identity" text NOT NULL, + "identity" jsonb NOT NULL, + "destination" text NOT NULL, + "request_digest" text NOT NULL, + "status_digest" text NOT NULL, + "state" text NOT NULL, + "expires_at" timestamp with time zone NOT NULL, + "created_at" timestamp with time zone NOT NULL, + CONSTRAINT "mcp_discovery_admission_state_check" CHECK ("mcp_discovery_admission"."state" IN ('reserved','dispatched','finished')), + CONSTRAINT "mcp_discovery_admission_expiry_check" CHECK ("mcp_discovery_admission"."expires_at" = "mcp_discovery_admission"."created_at" + interval '30 seconds') +); +--> statement-breakpoint +ALTER TABLE "mcp_discovery_admission" ADD CONSTRAINT "mcp_discovery_admission_deployment_id_mcp_admission_scope_id_fk" FOREIGN KEY ("deployment_id") REFERENCES "public"."mcp_admission_scope"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "mcp_discovery_admission" ADD CONSTRAINT "mcp_discovery_admission_connection_id_integration_connection_id_fk" FOREIGN KEY ("connection_id") REFERENCES "public"."integration_connection"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "mcp_discovery_admission_intent_idx" ON "mcp_discovery_admission" USING btree ("intent_id");--> statement-breakpoint +CREATE INDEX "mcp_discovery_admission_deployment_idx" ON "mcp_discovery_admission" USING btree ("deployment_id","state","expires_at");--> statement-breakpoint +CREATE INDEX "mcp_discovery_admission_connection_idx" ON "mcp_discovery_admission" USING btree ("connection_id","state","expires_at");--> statement-breakpoint +CREATE FUNCTION mcp_discovery_admission_transition() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF (to_jsonb(NEW) - 'state') IS DISTINCT FROM (to_jsonb(OLD) - 'state') THEN + RAISE EXCEPTION 'MCP discovery identity is immutable'; + END IF; + IF NEW.state <> OLD.state AND NOT ( + (OLD.state = 'reserved' AND NEW.state IN ('dispatched', 'finished')) OR + (OLD.state = 'dispatched' AND NEW.state = 'finished') + ) THEN + RAISE EXCEPTION 'invalid MCP discovery transition'; + END IF; + RETURN NEW; +END; +$$; +--> statement-breakpoint +CREATE TRIGGER mcp_discovery_admission_transition BEFORE UPDATE ON mcp_discovery_admission + FOR EACH ROW EXECUTE FUNCTION mcp_discovery_admission_transition(); diff --git a/orchestrator/drizzle/0115_mcp_runtime_composition.sql b/orchestrator/drizzle/0115_mcp_runtime_composition.sql new file mode 100644 index 000000000..319bb474d --- /dev/null +++ b/orchestrator/drizzle/0115_mcp_runtime_composition.sql @@ -0,0 +1,24 @@ +ALTER TABLE "mcp_discovery_admission" ADD COLUMN "operation" jsonb DEFAULT '{"kind":"discover"}'::jsonb NOT NULL;--> statement-breakpoint +ALTER TABLE "task_session" ADD COLUMN "mcp_managed_lifecycle" jsonb;--> statement-breakpoint +ALTER TABLE "mcp_discovery_admission" ADD CONSTRAINT "mcp_administrative_operation_check" CHECK (coalesce( + "mcp_discovery_admission"."operation" = '{"kind":"discover"}'::jsonb OR ( + "mcp_discovery_admission"."operation"->>'kind' = 'validateSchemas' AND + "mcp_discovery_admission"."operation"->>'documentsDigest' ~ '^sha256:[0-9a-f]{64}$' AND + "mcp_discovery_admission"."operation" - 'kind' - 'documentsDigest' = '{}'::jsonb AND "mcp_discovery_admission"."destination" = '' + ), false));--> statement-breakpoint +CREATE FUNCTION mcp_managed_lifecycle_identity() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF OLD.mcp_managed_lifecycle IS NOT NULL AND ( + NEW.mcp_managed_lifecycle IS NULL OR + NEW.mcp_managed_lifecycle->'identity' IS DISTINCT FROM OLD.mcp_managed_lifecycle->'identity' OR + NEW.mcp_managed_lifecycle->'nativeVersion' IS DISTINCT FROM OLD.mcp_managed_lifecycle->'nativeVersion' OR + NEW.mcp_managed_lifecycle->'nativeDigest' IS DISTINCT FROM OLD.mcp_managed_lifecycle->'nativeDigest' + ) THEN + RAISE EXCEPTION 'MCP creation identity is immutable'; + END IF; + RETURN NEW; +END; +$$; +--> statement-breakpoint +CREATE TRIGGER mcp_managed_lifecycle_identity BEFORE UPDATE ON task_session + FOR EACH ROW EXECUTE FUNCTION mcp_managed_lifecycle_identity(); diff --git a/orchestrator/drizzle/meta/0115_snapshot.json b/orchestrator/drizzle/meta/0115_snapshot.json new file mode 100644 index 000000000..1d713a02e --- /dev/null +++ b/orchestrator/drizzle/meta/0115_snapshot.json @@ -0,0 +1,8867 @@ +{ + "id": "be95abdb-f0a6-4282-ba6d-1d6574010f13", + "prevId": "785b7a9a-c956-4d05-81af-fdf4b2849339", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.apikey": { + "name": "apikey", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_request": { + "name": "last_request", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + { + "expression": "reference_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "apikey_reference_id_user_id_fk": { + "name": "apikey_reference_id_user_id_fk", + "tableFrom": "apikey", + "tableTo": "user", + "columnsFrom": [ + "reference_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "file_name": { + "name": "file_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'private'" + }, + "current_version": { + "name": "current_version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "artifact_owner_idx": { + "name": "artifact_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "artifact_visibility_idx": { + "name": "artifact_visibility_idx", + "columns": [ + { + "expression": "visibility", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "artifact_owner_user_id_user_id_fk": { + "name": "artifact_owner_user_id_user_id_fk", + "tableFrom": "artifact", + "tableTo": "user", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact_version": { + "name": "artifact_version", + "schema": "", + "columns": { + "artifact_id": { + "name": "artifact_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "task_id": { + "name": "task_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "coord_artifact_id": { + "name": "coord_artifact_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "artifact_version_session_idx": { + "name": "artifact_version_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "artifact_version_artifact_id_artifact_id_fk": { + "name": "artifact_version_artifact_id_artifact_id_fk", + "tableFrom": "artifact_version", + "tableTo": "artifact", + "columnsFrom": [ + "artifact_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "artifact_version_artifact_id_version_pk": { + "name": "artifact_version_artifact_id_version_pk", + "columns": [ + "artifact_id", + "version" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation": { + "name": "automation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user'" + }, + "builtin_key": { + "name": "builtin_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "current_version": { + "name": "current_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "inputs": { + "name": "inputs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "block_overrides": { + "name": "block_overrides", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "end_sessions_on_finish": { + "name": "end_sessions_on_finish", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_by_user_id": { + "name": "created_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "next_fire_at": { + "name": "next_fire_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_fired_at": { + "name": "last_fired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "draft_session_id": { + "name": "draft_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "automation_due_idx": { + "name": "automation_due_idx", + "columns": [ + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "next_fire_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "automation_draft_session_idx": { + "name": "automation_draft_session_idx", + "columns": [ + { + "expression": "draft_session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "automation_builtin_key_unique": { + "name": "automation_builtin_key_unique", + "columns": [ + { + "expression": "builtin_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "builtin_key is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation_concurrency_claim": { + "name": "automation_concurrency_claim", + "schema": "", + "columns": { + "automation_id": { + "name": "automation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "concurrency_key": { + "name": "concurrency_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automation_concurrency_claim_automation_id_automation_id_fk": { + "name": "automation_concurrency_claim_automation_id_automation_id_fk", + "tableFrom": "automation_concurrency_claim", + "tableTo": "automation", + "columnsFrom": [ + "automation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "automation_concurrency_claim_automation_id_concurrency_key_pk": { + "name": "automation_concurrency_claim_automation_id_concurrency_key_pk", + "columns": [ + "automation_id", + "concurrency_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation_drop": { + "name": "automation_drop", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "automation_id": { + "name": "automation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entrypoint_id": { + "name": "entrypoint_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'main'" + }, + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "dropped_at": { + "name": "dropped_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "automation_drop_automation_idx": { + "name": "automation_drop_automation_idx", + "columns": [ + { + "expression": "automation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "automation_drop_automation_id_automation_id_fk": { + "name": "automation_drop_automation_id_automation_id_fk", + "tableFrom": "automation_drop", + "tableTo": "automation", + "columnsFrom": [ + "automation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation_instance": { + "name": "automation_instance", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "automation_id": { + "name": "automation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "inputs": { + "name": "inputs", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "opened_by": { + "name": "opened_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "closed_at": { + "name": "closed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "close_reason": { + "name": "close_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "automation_instance_open_key_unique": { + "name": "automation_instance_open_key_unique", + "columns": [ + { + "expression": "automation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "status = 'open'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "automation_instance_automation_idx": { + "name": "automation_instance_automation_idx", + "columns": [ + { + "expression": "automation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "opened_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "automation_instance_automation_id_automation_id_fk": { + "name": "automation_instance_automation_id_automation_id_fk", + "tableFrom": "automation_instance", + "tableTo": "automation", + "columnsFrom": [ + "automation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation_instance_handle": { + "name": "automation_instance_handle", + "schema": "", + "columns": { + "automation_id": { + "name": "automation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "handle": { + "name": "handle", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "written_by": { + "name": "written_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "automation_instance_handle_instance_idx": { + "name": "automation_instance_handle_instance_idx", + "columns": [ + { + "expression": "instance_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "automation_instance_handle_automation_id_automation_id_fk": { + "name": "automation_instance_handle_automation_id_automation_id_fk", + "tableFrom": "automation_instance_handle", + "tableTo": "automation", + "columnsFrom": [ + "automation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "automation_instance_handle_instance_id_automation_instance_id_fk": { + "name": "automation_instance_handle_instance_id_automation_instance_id_fk", + "tableFrom": "automation_instance_handle", + "tableTo": "automation_instance", + "columnsFrom": [ + "instance_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "automation_instance_handle_automation_id_handle_pk": { + "name": "automation_instance_handle_automation_id_handle_pk", + "columns": [ + "automation_id", + "handle" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation_run": { + "name": "automation_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "automation_id": { + "name": "automation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "trigger": { + "name": "trigger", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "delivery_key": { + "name": "delivery_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "concurrency_key": { + "name": "concurrency_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context": { + "name": "context", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "rendered_prompt": { + "name": "rendered_prompt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rendered_title": { + "name": "rendered_title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "task_id": { + "name": "task_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "entrypoint_id": { + "name": "entrypoint_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'main'" + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scheduled_for": { + "name": "scheduled_for", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_owner": { + "name": "lease_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "dry_run": { + "name": "dry_run", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "details_pruned_at": { + "name": "details_pruned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "automation_run_retention_idx": { + "name": "automation_run_retention_idx", + "columns": [ + { + "expression": "ended_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "ended_at is not null and details_pruned_at is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "automation_run_occurrence_unique": { + "name": "automation_run_occurrence_unique", + "columns": [ + { + "expression": "automation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entrypoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "instance_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_for", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "scheduled_for is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "automation_run_delivery_unique": { + "name": "automation_run_delivery_unique", + "columns": [ + { + "expression": "automation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "entrypoint_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delivery_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "delivery_key is not null and scheduled_for is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "automation_run_automation_created_idx": { + "name": "automation_run_automation_created_idx", + "columns": [ + { + "expression": "automation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "automation_run_lease_idx": { + "name": "automation_run_lease_idx", + "columns": [ + { + "expression": "lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "automation_run_concurrency_idx": { + "name": "automation_run_concurrency_idx", + "columns": [ + { + "expression": "automation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "concurrency_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "automation_run_automation_id_automation_id_fk": { + "name": "automation_run_automation_id_automation_id_fk", + "tableFrom": "automation_run", + "tableTo": "automation", + "columnsFrom": [ + "automation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "automation_run_task_id_task_id_fk": { + "name": "automation_run_task_id_task_id_fk", + "tableFrom": "automation_run", + "tableTo": "task", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation_session": { + "name": "automation_session", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'primary'" + }, + "keep": { + "name": "keep", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "automation_session_run_idx": { + "name": "automation_session_run_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "automation_session_run_id_automation_run_id_fk": { + "name": "automation_session_run_id_automation_run_id_fk", + "tableFrom": "automation_session", + "tableTo": "automation_run", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation_session_relay": { + "name": "automation_session_relay", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "automation_id": { + "name": "automation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target": { + "name": "target", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automation_session_relay_session_id_automation_session_session_id_fk": { + "name": "automation_session_relay_session_id_automation_session_session_id_fk", + "tableFrom": "automation_session_relay", + "tableTo": "automation_session", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "session_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "automation_session_relay_automation_id_automation_id_fk": { + "name": "automation_session_relay_automation_id_automation_id_fk", + "tableFrom": "automation_session_relay", + "tableTo": "automation", + "columnsFrom": [ + "automation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation_state": { + "name": "automation_state", + "schema": "", + "columns": { + "automation_id": { + "name": "automation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "writer": { + "name": "writer", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automation_state_automation_id_automation_id_fk": { + "name": "automation_state_automation_id_automation_id_fk", + "tableFrom": "automation_state", + "tableTo": "automation", + "columnsFrom": [ + "automation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "automation_state_automation_id_key_pk": { + "name": "automation_state_automation_id_key_pk", + "columns": [ + "automation_id", + "key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation_step_run": { + "name": "automation_step_run", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "block_id": { + "name": "block_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "inputs": { + "name": "inputs", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "outputs": { + "name": "outputs", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "automation_step_run_run_id_automation_run_id_fk": { + "name": "automation_step_run_run_id_automation_run_id_fk", + "tableFrom": "automation_step_run", + "tableTo": "automation_run", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "automation_step_run_run_id_block_id_attempt_pk": { + "name": "automation_step_run_run_id_block_id_attempt_pk", + "columns": [ + "run_id", + "block_id", + "attempt" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.automation_version": { + "name": "automation_version", + "schema": "", + "columns": { + "automation_id": { + "name": "automation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "trigger": { + "name": "trigger", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "blocks": { + "name": "blocks", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "entrypoints": { + "name": "entrypoints", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "agent": { + "name": "agent", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prompt_template": { + "name": "prompt_template", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "inputs_schema": { + "name": "inputs_schema", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "settings": { + "name": "settings", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_by_user_id": { + "name": "created_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "automation_version_automation_id_automation_id_fk": { + "name": "automation_version_automation_id_automation_id_fk", + "tableFrom": "automation_version", + "tableTo": "automation", + "columnsFrom": [ + "automation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "automation_version_automation_id_version_pk": { + "name": "automation_version_automation_id_version_pk", + "columns": [ + "automation_id", + "version" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connector": { + "name": "connector", + "schema": "", + "columns": { + "provider": { + "name": "provider", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connector_logo": { + "name": "connector_logo", + "schema": "", + "columns": { + "provider": { + "name": "provider", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "media_type": { + "name": "media_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.consumer_cursors": { + "name": "consumer_cursors", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "consumer": { + "name": "consumer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_idx": { + "name": "last_idx", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "consumer_cursors_session_id_consumer_pk": { + "name": "consumer_cursors_session_id_consumer_pk", + "columns": [ + "session_id", + "consumer" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.coordination_operation": { + "name": "coordination_operation", + "schema": "", + "columns": { + "caller_session_id": { + "name": "caller_session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_hash": { + "name": "request_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'reserved'" + }, + "reserved_task_id": { + "name": "reserved_task_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reserved_session_id": { + "name": "reserved_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "prompt_id": { + "name": "prompt_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "coordination_operation_reserved_session_idx": { + "name": "coordination_operation_reserved_session_idx", + "columns": [ + { + "expression": "reserved_session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "coordination_operation_caller_session_id_operation_idempotency_key_pk": { + "name": "coordination_operation_caller_session_id_operation_idempotency_key_pk", + "columns": [ + "caller_session_id", + "operation", + "idempotency_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dbos_sweep_lease": { + "name": "dbos_sweep_lease", + "schema": "", + "columns": { + "name": { + "name": "name", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dbos_sweep_ledger": { + "name": "dbos_sweep_ledger", + "schema": "", + "columns": { + "workflow_uuid": { + "name": "workflow_uuid", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "workflow_name": { + "name": "workflow_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sweep_count": { + "name": "sweep_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "first_swept_at": { + "name": "first_swept_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_swept_at": { + "name": "last_swept_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "suppressed": { + "name": "suppressed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "alerted_at": { + "name": "alerted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "terminal_alerted_at": { + "name": "terminal_alerted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dbos_version_heartbeats": { + "name": "dbos_version_heartbeats", + "schema": "", + "columns": { + "application_version": { + "name": "application_version", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pod_name": { + "name": "pod_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_seen": { + "name": "last_seen", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dbos_version_heartbeats_application_version_pod_name_pk": { + "name": "dbos_version_heartbeats_application_version_pod_name_pk", + "columns": [ + "application_version", + "pod_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.device_code": { + "name": "device_code", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "device_code": { + "name": "device_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_polled_at": { + "name": "last_polled_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "polling_interval": { + "name": "polling_interval", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "device_code_device_code_idx": { + "name": "device_code_device_code_idx", + "columns": [ + { + "expression": "device_code", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "device_code_user_code_idx": { + "name": "device_code_user_code_idx", + "columns": [ + { + "expression": "user_code", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_connection": { + "name": "integration_connection", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "alias": { + "name": "alias", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mcp_credential": { + "name": "mcp_credential", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "config_revision": { + "name": "config_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "auth_generation": { + "name": "auth_generation", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "credential_identity": { + "name": "credential_identity", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "catalog_revision": { + "name": "catalog_revision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "revoked": { + "name": "revoked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "tested_at": { + "name": "tested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integration_connection_provider_idx": { + "name": "integration_connection_provider_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_connection_provider_default_unique": { + "name": "integration_connection_provider_default_unique", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "is_default", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "integration_connection_alias_unique": { + "name": "integration_connection_alias_unique", + "nullsNotDistinct": false, + "columns": [ + "alias" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_event": { + "name": "integration_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "scope_value": { + "name": "scope_value", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "integration_event_delivery_unique": { + "name": "integration_event_delivery_unique", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "delivery_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "integration_event_conn_key_idx": { + "name": "integration_event_conn_key_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "received_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "integration_event_connection_id_integration_connection_id_fk": { + "name": "integration_event_connection_id_integration_connection_id_fk", + "tableFrom": "integration_event", + "tableTo": "integration_connection", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration_oidc_key": { + "name": "integration_oidc_key", + "schema": "", + "columns": { + "kid": { + "name": "kid", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "public_jwk": { + "name": "public_jwk", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "nonce": { + "name": "nonce", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "publish_until": { + "name": "publish_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "integration_oidc_key_active_unique": { + "name": "integration_oidc_key_active_unique", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "state = 'active'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_admission": { + "name": "mcp_admission", + "schema": "", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "reservation_id": { + "name": "reservation_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "gen_random_uuid()::text" + }, + "deployment_id": { + "name": "deployment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'execute'" + }, + "schema_queue_expires_at": { + "name": "schema_queue_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "execute_queue_expires_at": { + "name": "execute_queue_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "execute_expires_at": { + "name": "execute_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "schema_digest": { + "name": "schema_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_admission_reservation_idx": { + "name": "mcp_admission_reservation_idx", + "columns": [ + { + "expression": "reservation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_admission_deployment_idx": { + "name": "mcp_admission_deployment_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_admission_connection_idx": { + "name": "mcp_admission_connection_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_admission_session_idx": { + "name": "mcp_admission_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_admission_execution_id_mcp_execution_id_fk": { + "name": "mcp_admission_execution_id_mcp_execution_id_fk", + "tableFrom": "mcp_admission", + "tableTo": "mcp_execution", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "mcp_admission_deployment_id_mcp_admission_scope_id_fk": { + "name": "mcp_admission_deployment_id_mcp_admission_scope_id_fk", + "tableFrom": "mcp_admission", + "tableTo": "mcp_admission_scope", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "mcp_admission_connection_id_integration_connection_id_fk": { + "name": "mcp_admission_connection_id_integration_connection_id_fk", + "tableFrom": "mcp_admission", + "tableTo": "integration_connection", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_admission_phase_check": { + "name": "mcp_admission_phase_check", + "value": "\"mcp_admission\".\"phase\" IN ('schema','execute')" + }, + "mcp_admission_state_check": { + "name": "mcp_admission_state_check", + "value": "\"mcp_admission\".\"state\" IN ('queued','admitted','released')" + } + }, + "isRLSEnabled": false + }, + "public.mcp_admission_scope": { + "name": "mcp_admission_scope", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_approval": { + "name": "mcp_approval", + "schema": "", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_digest": { + "name": "request_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "decision": { + "name": "decision", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "decided_at": { + "name": "decided_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "consumed_at": { + "name": "consumed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "mcp_approval_pending_idx": { + "name": "mcp_approval_pending_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "decision", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_approval_session_idx": { + "name": "mcp_approval_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "decision", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_approval_execution_id_mcp_execution_id_fk": { + "name": "mcp_approval_execution_id_mcp_execution_id_fk", + "tableFrom": "mcp_approval", + "tableTo": "mcp_execution", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_approval_decision_check": { + "name": "mcp_approval_decision_check", + "value": "\"mcp_approval\".\"decision\" IN ('pending','approved','denied','expired')" + }, + "mcp_approval_consumed_check": { + "name": "mcp_approval_consumed_check", + "value": "\"mcp_approval\".\"consumed_at\" IS NULL OR (\"mcp_approval\".\"decision\" = 'approved' AND \"mcp_approval\".\"actor_id\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.mcp_cancellation": { + "name": "mcp_cancellation", + "schema": "", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cause": { + "name": "cause", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expected_journal_revision": { + "name": "expected_journal_revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "journal_revision": { + "name": "journal_revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_cancellation_execution_id_mcp_execution_id_fk": { + "name": "mcp_cancellation_execution_id_mcp_execution_id_fk", + "tableFrom": "mcp_cancellation", + "tableTo": "mcp_execution", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_cancellation_revision_check": { + "name": "mcp_cancellation_revision_check", + "value": "\"mcp_cancellation\".\"expected_journal_revision\" > 0 AND \"mcp_cancellation\".\"journal_revision\" = \"mcp_cancellation\".\"expected_journal_revision\" + 1" + }, + "mcp_cancellation_state_check": { + "name": "mcp_cancellation_state_check", + "value": "\"mcp_cancellation\".\"state\" IN ('canceled','outcome_unknown')" + } + }, + "isRLSEnabled": false + }, + "public.mcp_catalog": { + "name": "mcp_catalog", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision": { + "name": "revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "config_revision": { + "name": "config_revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "auth_generation": { + "name": "auth_generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "credential_identity": { + "name": "credential_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tools": { + "name": "tools", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_catalog_connection_idx": { + "name": "mcp_catalog_connection_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_catalog_published_revision_idx": { + "name": "mcp_catalog_published_revision_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "revision", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "kind = 'published'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_catalog_connection_id_integration_connection_id_fk": { + "name": "mcp_catalog_connection_id_integration_connection_id_fk", + "tableFrom": "mcp_catalog", + "tableTo": "integration_connection", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_catalog_kind_check": { + "name": "mcp_catalog_kind_check", + "value": "\"mcp_catalog\".\"kind\" IN ('draft','published') AND ((\"mcp_catalog\".\"kind\" = 'draft' AND \"mcp_catalog\".\"revision\" = 0) OR (\"mcp_catalog\".\"kind\" = 'published' AND \"mcp_catalog\".\"revision\" > 0))" + } + }, + "isRLSEnabled": false + }, + "public.mcp_discovery_admission": { + "name": "mcp_discovery_admission", + "schema": "", + "columns": { + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "intent_id": { + "name": "intent_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config_revision": { + "name": "config_revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "auth_generation": { + "name": "auth_generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "credential_identity": { + "name": "credential_identity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity": { + "name": "identity", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "destination": { + "name": "destination", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{\"kind\":\"discover\"}'::jsonb" + }, + "request_digest": { + "name": "request_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status_digest": { + "name": "status_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_discovery_admission_intent_idx": { + "name": "mcp_discovery_admission_intent_idx", + "columns": [ + { + "expression": "intent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_discovery_admission_deployment_idx": { + "name": "mcp_discovery_admission_deployment_idx", + "columns": [ + { + "expression": "deployment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_discovery_admission_connection_idx": { + "name": "mcp_discovery_admission_connection_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_discovery_admission_deployment_id_mcp_admission_scope_id_fk": { + "name": "mcp_discovery_admission_deployment_id_mcp_admission_scope_id_fk", + "tableFrom": "mcp_discovery_admission", + "tableTo": "mcp_admission_scope", + "columnsFrom": [ + "deployment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "mcp_discovery_admission_connection_id_integration_connection_id_fk": { + "name": "mcp_discovery_admission_connection_id_integration_connection_id_fk", + "tableFrom": "mcp_discovery_admission", + "tableTo": "integration_connection", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_administrative_operation_check": { + "name": "mcp_administrative_operation_check", + "value": "coalesce(\n \"mcp_discovery_admission\".\"operation\" = '{\"kind\":\"discover\"}'::jsonb OR (\n \"mcp_discovery_admission\".\"operation\"->>'kind' = 'validateSchemas' AND\n \"mcp_discovery_admission\".\"operation\"->>'documentsDigest' ~ '^sha256:[0-9a-f]{64}$' AND\n \"mcp_discovery_admission\".\"operation\" - 'kind' - 'documentsDigest' = '{}'::jsonb AND \"mcp_discovery_admission\".\"destination\" = ''\n ), false)" + }, + "mcp_discovery_admission_state_check": { + "name": "mcp_discovery_admission_state_check", + "value": "\"mcp_discovery_admission\".\"state\" IN ('reserved','dispatched','finished')" + }, + "mcp_discovery_admission_expiry_check": { + "name": "mcp_discovery_admission_expiry_check", + "value": "\"mcp_discovery_admission\".\"expires_at\" = \"mcp_discovery_admission\".\"created_at\" + interval '30 seconds'" + } + }, + "isRLSEnabled": false + }, + "public.mcp_execution": { + "name": "mcp_execution", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "task_id": { + "name": "task_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connection_id": { + "name": "connection_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "catalog_id": { + "name": "catalog_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "binding_hash": { + "name": "binding_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_digest": { + "name": "request_digest", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "argument_hash": { + "name": "argument_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "original_arguments": { + "name": "original_arguments", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_context": { + "name": "request_context", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "upstream_name": { + "name": "upstream_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config_revision": { + "name": "config_revision", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "auth_generation": { + "name": "auth_generation", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "classification": { + "name": "classification", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_kind": { + "name": "authorization_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "automation_grant_hash": { + "name": "automation_grant_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "boot_id": { + "name": "boot_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "intent_id": { + "name": "intent_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "intent_expires_at": { + "name": "intent_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deadline": { + "name": "deadline", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "result_unavailable": { + "name": "result_unavailable", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "completion_enqueued_version": { + "name": "completion_enqueued_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_execution_task_session_idx": { + "name": "mcp_execution_task_session_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_execution_call_idx": { + "name": "mcp_execution_call_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_execution_connection_state_idx": { + "name": "mcp_execution_connection_state_idx", + "columns": [ + { + "expression": "connection_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_execution_catalog_idx": { + "name": "mcp_execution_catalog_idx", + "columns": [ + { + "expression": "catalog_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_execution_pending_idx": { + "name": "mcp_execution_pending_idx", + "columns": [ + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "deadline", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_execution_lifecycle_active_idx": { + "name": "mcp_execution_lifecycle_active_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"mcp_execution\".\"state\" IN ('waiting_approval','authorized','dispatching')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_execution_recovery_idx": { + "name": "mcp_execution_recovery_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"mcp_execution\".\"completion_enqueued_version\" < \"mcp_execution\".\"version\"", + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_execution_intent_idx": { + "name": "mcp_execution_intent_idx", + "columns": [ + { + "expression": "intent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "mcp_execution_connection_id_integration_connection_id_fk": { + "name": "mcp_execution_connection_id_integration_connection_id_fk", + "tableFrom": "mcp_execution", + "tableTo": "integration_connection", + "columnsFrom": [ + "connection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "mcp_execution_catalog_id_mcp_catalog_id_fk": { + "name": "mcp_execution_catalog_id_mcp_catalog_id_fk", + "tableFrom": "mcp_execution", + "tableTo": "mcp_catalog", + "columnsFrom": [ + "catalog_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "mcp_execution_task_id_session_id_task_session_task_id_session_id_fk": { + "name": "mcp_execution_task_id_session_id_task_session_task_id_session_id_fk", + "tableFrom": "mcp_execution", + "tableTo": "task_session", + "columnsFrom": [ + "task_id", + "session_id" + ], + "columnsTo": [ + "task_id", + "session_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_execution_completion_version_check": { + "name": "mcp_execution_completion_version_check", + "value": "\"mcp_execution\".\"completion_enqueued_version\" >= 0 AND \"mcp_execution\".\"completion_enqueued_version\" <= \"mcp_execution\".\"version\"" + }, + "mcp_execution_state_check": { + "name": "mcp_execution_state_check", + "value": "\"mcp_execution\".\"state\" IN ('waiting_approval','authorized','dispatching','succeeded','failed','canceled','outcome_unknown')" + }, + "mcp_execution_classification_check": { + "name": "mcp_execution_classification_check", + "value": "\"mcp_execution\".\"classification\" IN ('read','write')" + }, + "mcp_execution_authorization_check": { + "name": "mcp_execution_authorization_check", + "value": "(\"mcp_execution\".\"authorization_kind\" = 'read' AND \"mcp_execution\".\"classification\" = 'read' AND \"mcp_execution\".\"automation_grant_hash\" IS NULL) OR (\"mcp_execution\".\"authorization_kind\" = 'approval' AND \"mcp_execution\".\"automation_grant_hash\" IS NULL) OR (\"mcp_execution\".\"authorization_kind\" = 'automation' AND \"mcp_execution\".\"classification\" = 'write' AND \"mcp_execution\".\"automation_grant_hash\" IS NOT NULL)" + }, + "mcp_execution_dispatch_check": { + "name": "mcp_execution_dispatch_check", + "value": "\"mcp_execution\".\"state\" NOT IN ('dispatching','outcome_unknown','succeeded') OR (\"mcp_execution\".\"boot_id\" IS NOT NULL AND \"mcp_execution\".\"intent_id\" IS NOT NULL AND \"mcp_execution\".\"intent_expires_at\" IS NOT NULL)" + } + }, + "isRLSEnabled": false + }, + "public.mcp_resolution": { + "name": "mcp_resolution", + "schema": "", + "columns": { + "execution_id": { + "name": "execution_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expected_context": { + "name": "expected_context", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "mcp_resolution_execution_id_mcp_execution_id_fk": { + "name": "mcp_resolution_execution_id_mcp_execution_id_fk", + "tableFrom": "mcp_resolution", + "tableTo": "mcp_execution", + "columnsFrom": [ + "execution_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "mcp_resolution_outcome_check": { + "name": "mcp_resolution_outcome_check", + "value": "\"mcp_resolution\".\"outcome\" IN ('succeeded','failed','acknowledged')" + } + }, + "isRLSEnabled": false + }, + "public.org_setting": { + "name": "org_setting", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_by_user_id": { + "name": "updated_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.papercuts": { + "name": "papercuts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tags": { + "name": "tags", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "task_id": { + "name": "task_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "papercuts_session_tool_call_unique": { + "name": "papercuts_session_tool_call_unique", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "papercuts_created_at_idx": { + "name": "papercuts_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "papercuts_archived_at_idx": { + "name": "papercuts_archived_at_idx", + "columns": [ + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pending_tool_calls": { + "name": "pending_tool_calls", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_name": { + "name": "tool_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "handling": { + "name": "handling", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "submitted_at": { + "name": "submitted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "pending_tool_calls_session_tool_call_unique": { + "name": "pending_tool_calls_session_tool_call_unique", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pending_tool_calls_session_idx": { + "name": "pending_tool_calls_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pending_tool_calls_awaiting_idx": { + "name": "pending_tool_calls_awaiting_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"pending_tool_calls\".\"handling\" = 'session' AND \"pending_tool_calls\".\"submitted_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.pr_ref": { + "name": "pr_ref", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pr_number": { + "name": "pr_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "authoring_task_id": { + "name": "authoring_task_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "head_branch": { + "name": "head_branch", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_branch": { + "name": "base_branch", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "observed_at": { + "name": "observed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "pr_ref_repo_pr_number_unique": { + "name": "pr_ref_repo_pr_number_unique", + "columns": [ + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "pr_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pr_ref_authoring_task_idx": { + "name": "pr_ref_authoring_task_idx", + "columns": [ + { + "expression": "authoring_task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "pr_ref_session_idx": { + "name": "pr_ref_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "pr_ref_authoring_task_id_task_id_fk": { + "name": "pr_ref_authoring_task_id_task_id_fk", + "tableFrom": "pr_ref", + "tableTo": "task", + "columnsFrom": [ + "authoring_task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.profile": { + "name": "profile", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'Bot'" + }, + "image_id": { + "name": "image_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "harness": { + "name": "harness", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model_router": { + "name": "model_router", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "effort": { + "name": "effort", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "programmatic_only": { + "name": "programmatic_only", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "include_user_tokens": { + "name": "include_user_tokens", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "env_vars": { + "name": "env_vars", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "skills": { + "name": "skills", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "mcp_grant_publication": { + "name": "mcp_grant_publication", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "integration_grants": { + "name": "integration_grants", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "network": { + "name": "network", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{\"default\":\"deny\",\"allowHosts\":[],\"allowHostPatterns\":[]}'::jsonb" + }, + "secrets": { + "name": "secrets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "repos": { + "name": "repos", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "apps": { + "name": "apps", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.review": { + "name": "review", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "target_id": { + "name": "target_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "task_id": { + "name": "task_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "base_sha": { + "name": "base_sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "github_review_id": { + "name": "github_review_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "finder_session_id": { + "name": "finder_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "verifier_session_id": { + "name": "verifier_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "automation_run_id": { + "name": "automation_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary_md": { + "name": "summary_md", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "head_branch": { + "name": "head_branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "base_branch": { + "name": "base_branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "additions": { + "name": "additions", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deletions": { + "name": "deletions", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "changed_files": { + "name": "changed_files", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "review_target_idx": { + "name": "review_target_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "review_task_idx": { + "name": "review_task_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "review_one_active_per_target_idx": { + "name": "review_one_active_per_target_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"review\".\"status\" IN ('queued', 'finding', 'verifying')", + "concurrently": false, + "method": "btree", + "with": {} + }, + "review_finder_session_idx": { + "name": "review_finder_session_idx", + "columns": [ + { + "expression": "finder_session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "review_verifier_session_idx": { + "name": "review_verifier_session_idx", + "columns": [ + { + "expression": "verifier_session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "review_target_id_review_target_id_fk": { + "name": "review_target_id_review_target_id_fk", + "tableFrom": "review", + "tableTo": "review_target", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "review_task_id_task_id_fk": { + "name": "review_task_id_task_id_fk", + "tableFrom": "review", + "tableTo": "task", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.review_event": { + "name": "review_event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "review_id": { + "name": "review_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "review_event_review_idx": { + "name": "review_event_review_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "review_event_review_id_review_id_fk": { + "name": "review_event_review_id_review_id_fk", + "tableFrom": "review_event", + "tableTo": "review", + "columnsFrom": [ + "review_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.review_finding": { + "name": "review_finding", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "review_id": { + "name": "review_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "start_line": { + "name": "start_line", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "end_line": { + "name": "end_line", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "side": { + "name": "side", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "confidence": { + "name": "confidence", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "body_md": { + "name": "body_md", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "suggested_fix": { + "name": "suggested_fix", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence": { + "name": "evidence", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'candidate'" + }, + "verdict_reason": { + "name": "verdict_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "github_thread_id": { + "name": "github_thread_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolution": { + "name": "resolution", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "review_finding_session_tool_call_unique": { + "name": "review_finding_session_tool_call_unique", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "review_finding_review_idx": { + "name": "review_finding_review_idx", + "columns": [ + { + "expression": "review_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "review_finding_review_id_review_id_fk": { + "name": "review_finding_review_id_review_id_fk", + "tableFrom": "review_finding", + "tableTo": "review", + "columnsFrom": [ + "review_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.review_target": { + "name": "review_target", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'github'" + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "repo": { + "name": "repo", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "number": { + "name": "number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_updated_at": { + "name": "provider_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "hydration_failed_at": { + "name": "hydration_failed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "review_target_provider_id_unique": { + "name": "review_target_provider_id_unique", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "review_target_coordinate_idx": { + "name": "review_target_coordinate_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "repo", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.review_verdict": { + "name": "review_verdict", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "finding_id": { + "name": "finding_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "verdict": { + "name": "verdict", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "confidence": { + "name": "confidence", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reasoning": { + "name": "reasoning", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "review_verdict_session_tool_call_unique": { + "name": "review_verdict_session_tool_call_unique", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "tool_call_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "review_verdict_finding_unique": { + "name": "review_verdict_finding_unique", + "columns": [ + { + "expression": "finding_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "review_verdict_finding_id_review_finding_id_fk": { + "name": "review_verdict_finding_id_review_finding_id_fk", + "tableFrom": "review_verdict", + "tableTo": "review_finding", + "columnsFrom": [ + "finding_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.router_model": { + "name": "router_model", + "schema": "", + "columns": { + "router_id": { + "name": "router_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_slug": { + "name": "canonical_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author": { + "name": "author", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "context_length": { + "name": "context_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "prompt_price": { + "name": "prompt_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "completion_price": { + "name": "completion_price", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "input_modalities": { + "name": "input_modalities", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "output_modalities": { + "name": "output_modalities", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "supported_parameters": { + "name": "supported_parameters", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "hugging_face_id": { + "name": "hugging_face_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "upstream": { + "name": "upstream", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "available": { + "name": "available", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "router_model_available_idx": { + "name": "router_model_available_idx", + "columns": [ + { + "expression": "router_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "available", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "router_model_router_id_model_id_pk": { + "name": "router_model_router_id_model_id_pk", + "columns": [ + "router_id", + "model_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.router_model_policy": { + "name": "router_model_policy", + "schema": "", + "columns": { + "router_id": { + "name": "router_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "user_enabled": { + "name": "user_enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "router_model_policy_router_id_model_id_router_model_router_id_model_id_fk": { + "name": "router_model_policy_router_id_model_id_router_model_router_id_model_id_fk", + "tableFrom": "router_model_policy", + "tableTo": "router_model", + "columnsFrom": [ + "router_id", + "model_id" + ], + "columnsTo": [ + "router_id", + "model_id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "router_model_policy_router_id_model_id_pk": { + "name": "router_model_policy_router_id_model_id_pk", + "columns": [ + "router_id", + "model_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "router_model_policy_user_implies_enabled": { + "name": "router_model_policy_user_implies_enabled", + "value": "NOT \"router_model_policy\".\"user_enabled\" OR \"router_model_policy\".\"enabled\"" + } + }, + "isRLSEnabled": false + }, + "public.router_sync_state": { + "name": "router_sync_state", + "schema": "", + "columns": { + "router_id": { + "name": "router_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "last_successful_sync_at": { + "name": "last_successful_sync_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_attempt_at": { + "name": "last_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_app": { + "name": "session_app", + "schema": "", + "columns": { + "host_label": { + "name": "host_label", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "port": { + "name": "port", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "visibility": { + "name": "visibility", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'org'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "session_app_session_name_idx": { + "name": "session_app_session_name_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "session_app_session_port_idx": { + "name": "session_app_session_port_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "port", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "session_app_session_idx": { + "name": "session_app_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "session_app_owner_idx": { + "name": "session_app_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_app_owner_user_id_user_id_fk": { + "name": "session_app_owner_user_id_user_id_fk", + "tableFrom": "session_app", + "tableTo": "user", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session_listeners": { + "name": "session_listeners", + "schema": "", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "terminal_at": { + "name": "terminal_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "dormant_at": { + "name": "dormant_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "woken_at": { + "name": "woken_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec": { + "name": "spec", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner_user_id": { + "name": "owner_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "template_id": { + "name": "template_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "current_doc_seq": { + "name": "current_doc_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "current_semantic_doc_seq": { + "name": "current_semantic_doc_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "published_checkpoint_id": { + "name": "published_checkpoint_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "published_by": { + "name": "published_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "published_at": { + "name": "published_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "spec_org_updated_idx": { + "name": "spec_org_updated_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "spec_owner_idx": { + "name": "spec_owner_idx", + "columns": [ + { + "expression": "owner_user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "spec_session_idx": { + "name": "spec_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_owner_user_id_user_id_fk": { + "name": "spec_owner_user_id_user_id_fk", + "tableFrom": "spec", + "tableTo": "user", + "columnsFrom": [ + "owner_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "spec_template_id_spec_template_id_fk": { + "name": "spec_template_id_spec_template_id_fk", + "tableFrom": "spec", + "tableTo": "spec_template", + "columnsFrom": [ + "template_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "spec_published_by_user_id_fk": { + "name": "spec_published_by_user_id_fk", + "tableFrom": "spec", + "tableTo": "user", + "columnsFrom": [ + "published_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_chat_message": { + "name": "spec_chat_message", + "schema": "", + "columns": { + "prompt_id": { + "name": "prompt_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "author_user_id": { + "name": "author_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_name": { + "name": "author_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "spec_chat_message_spec_created_idx": { + "name": "spec_chat_message_spec_created_idx", + "columns": [ + { + "expression": "spec_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_chat_message_spec_id_spec_id_fk": { + "name": "spec_chat_message_spec_id_spec_id_fk", + "tableFrom": "spec_chat_message", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "spec_chat_message_author_user_id_user_id_fk": { + "name": "spec_chat_message_author_user_id_user_id_fk", + "tableFrom": "spec_chat_message", + "tableTo": "user", + "columnsFrom": [ + "author_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_checkpoint": { + "name": "spec_checkpoint", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "state_vector": { + "name": "state_vector", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "rendered_markdown": { + "name": "rendered_markdown", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "doc_seq": { + "name": "doc_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author_user_id": { + "name": "author_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "spec_checkpoint_spec_created_idx": { + "name": "spec_checkpoint_spec_created_idx", + "columns": [ + { + "expression": "spec_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_checkpoint_spec_id_spec_id_fk": { + "name": "spec_checkpoint_spec_id_spec_id_fk", + "tableFrom": "spec_checkpoint", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "spec_checkpoint_author_user_id_user_id_fk": { + "name": "spec_checkpoint_author_user_id_user_id_fk", + "tableFrom": "spec_checkpoint", + "tableTo": "user", + "columnsFrom": [ + "author_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_drafting_seed": { + "name": "spec_drafting_seed", + "schema": "", + "columns": { + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "prompt_id": { + "name": "prompt_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "spec_drafting_seed_due_idx": { + "name": "spec_drafting_seed_due_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"spec_drafting_seed\".\"state\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_drafting_seed_spec_id_spec_id_fk": { + "name": "spec_drafting_seed_spec_id_spec_id_fk", + "tableFrom": "spec_drafting_seed", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "spec_drafting_seed_prompt_id_unique": { + "name": "spec_drafting_seed_prompt_id_unique", + "nullsNotDistinct": false, + "columns": [ + "prompt_id" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_gap_check_finding": { + "name": "spec_gap_check_finding", + "schema": "", + "columns": { + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "finding_id": { + "name": "finding_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "layer_key": { + "name": "layer_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "section_id": { + "name": "section_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "section_title": { + "name": "section_title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "requirement_id": { + "name": "requirement_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detail": { + "name": "detail", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "proposed_diff": { + "name": "proposed_diff", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "disposition": { + "name": "disposition", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "open_question_id": { + "name": "open_question_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "disposed_by": { + "name": "disposed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disposed_at": { + "name": "disposed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "spec_gap_check_finding_run_id_spec_gap_check_run_id_fk": { + "name": "spec_gap_check_finding_run_id_spec_gap_check_run_id_fk", + "tableFrom": "spec_gap_check_finding", + "tableTo": "spec_gap_check_run", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "spec_gap_check_finding_disposed_by_user_id_fk": { + "name": "spec_gap_check_finding_disposed_by_user_id_fk", + "tableFrom": "spec_gap_check_finding", + "tableTo": "user", + "columnsFrom": [ + "disposed_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "spec_gap_check_finding_run_id_finding_id_pk": { + "name": "spec_gap_check_finding_run_id_finding_id_pk", + "columns": [ + "run_id", + "finding_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_gap_check_run": { + "name": "spec_gap_check_run", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "request_fingerprint": { + "name": "request_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "semantic_doc_seq": { + "name": "semantic_doc_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "stopped_at_layer_key": { + "name": "stopped_at_layer_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "suppressed_count": { + "name": "suppressed_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "matrix": { + "name": "matrix", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "started_by": { + "name": "started_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "spec_gap_check_run_fingerprint_idx": { + "name": "spec_gap_check_run_fingerprint_idx", + "columns": [ + { + "expression": "spec_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_fingerprint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "spec_gap_check_run_spec_created_idx": { + "name": "spec_gap_check_run_spec_created_idx", + "columns": [ + { + "expression": "spec_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_gap_check_run_spec_id_spec_id_fk": { + "name": "spec_gap_check_run_spec_id_spec_id_fk", + "tableFrom": "spec_gap_check_run", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "spec_gap_check_run_started_by_user_id_fk": { + "name": "spec_gap_check_run_started_by_user_id_fk", + "tableFrom": "spec_gap_check_run", + "tableTo": "user", + "columnsFrom": [ + "started_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_open_question": { + "name": "spec_open_question", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "section_id": { + "name": "section_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "text": { + "name": "text", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "opened_by": { + "name": "opened_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "request_fingerprint": { + "name": "request_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resolution_note": { + "name": "resolution_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "spec_open_question_spec_section_idx": { + "name": "spec_open_question_spec_section_idx", + "columns": [ + { + "expression": "spec_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "section_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_open_question_spec_id_spec_id_fk": { + "name": "spec_open_question_spec_id_spec_id_fk", + "tableFrom": "spec_open_question", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "spec_open_question_opened_by_user_id_fk": { + "name": "spec_open_question_opened_by_user_id_fk", + "tableFrom": "spec_open_question", + "tableTo": "user", + "columnsFrom": [ + "opened_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + }, + "spec_open_question_resolved_by_user_id_fk": { + "name": "spec_open_question_resolved_by_user_id_fk", + "tableFrom": "spec_open_question", + "tableTo": "user", + "columnsFrom": [ + "resolved_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_participant": { + "name": "spec_participant", + "schema": "", + "columns": { + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connection_epoch": { + "name": "connection_epoch", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "connected_at": { + "name": "connected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "disconnected_at": { + "name": "disconnected_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "spec_participant_live_idx": { + "name": "spec_participant_live_idx", + "columns": [ + { + "expression": "spec_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_participant_spec_id_spec_id_fk": { + "name": "spec_participant_spec_id_spec_id_fk", + "tableFrom": "spec_participant", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "spec_participant_user_id_user_id_fk": { + "name": "spec_participant_user_id_user_id_fk", + "tableFrom": "spec_participant", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "spec_participant_spec_id_client_id_pk": { + "name": "spec_participant_spec_id_client_id_pk", + "columns": [ + "spec_id", + "client_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_projection": { + "name": "spec_projection", + "schema": "", + "columns": { + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "rev": { + "name": "rev", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "doc_seq": { + "name": "doc_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "semantic_doc_seq": { + "name": "semantic_doc_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rendered": { + "name": "rendered", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "document_state": { + "name": "document_state", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "digest": { + "name": "digest", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "digest_sha256": { + "name": "digest_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "staging_path": { + "name": "staging_path", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "requested_source": { + "name": "requested_source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "discard_notice": { + "name": "discard_notice", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "pushed_at": { + "name": "pushed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "spec_projection_session_state_idx": { + "name": "spec_projection_session_state_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "rev", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_projection_spec_id_spec_id_fk": { + "name": "spec_projection_spec_id_spec_id_fk", + "tableFrom": "spec_projection", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "spec_projection_spec_id_rev_pk": { + "name": "spec_projection_spec_id_rev_pk", + "columns": [ + "spec_id", + "rev" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_publish": { + "name": "spec_publish", + "schema": "", + "columns": { + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "checkpoint_id": { + "name": "checkpoint_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "artifact_id": { + "name": "artifact_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "artifact_version": { + "name": "artifact_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "acknowledged_question_count": { + "name": "acknowledged_question_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "acknowledged_question_ids": { + "name": "acknowledged_question_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "gap_check_run_id": { + "name": "gap_check_run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pinned_at": { + "name": "pinned_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "spec_publish_due_idx": { + "name": "spec_publish_due_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"spec_publish\".\"state\" <> 'complete'", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_publish_spec_id_spec_id_fk": { + "name": "spec_publish_spec_id_spec_id_fk", + "tableFrom": "spec_publish", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "spec_publish_requested_by_user_id_fk": { + "name": "spec_publish_requested_by_user_id_fk", + "tableFrom": "spec_publish", + "tableTo": "user", + "columnsFrom": [ + "requested_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_section_state": { + "name": "spec_section_state", + "schema": "", + "columns": { + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "section_id": { + "name": "section_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "na_reason": { + "name": "na_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "settled_by": { + "name": "settled_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "spec_section_state_spec_id_spec_id_fk": { + "name": "spec_section_state_spec_id_spec_id_fk", + "tableFrom": "spec_section_state", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "spec_section_state_settled_by_user_id_fk": { + "name": "spec_section_state_settled_by_user_id_fk", + "tableFrom": "spec_section_state", + "tableTo": "user", + "columnsFrom": [ + "settled_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "spec_section_state_spec_id_section_id_pk": { + "name": "spec_section_state_spec_id_section_id_pk", + "columns": [ + "spec_id", + "section_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_snapshot": { + "name": "spec_snapshot", + "schema": "", + "columns": { + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "state": { + "name": "state", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "state_vector": { + "name": "state_vector", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "covered_seq": { + "name": "covered_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "covered_semantic_doc_seq": { + "name": "covered_semantic_doc_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "spec_snapshot_spec_id_spec_id_fk": { + "name": "spec_snapshot_spec_id_spec_id_fk", + "tableFrom": "spec_snapshot", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_template": { + "name": "spec_template", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "layers": { + "name": "layers", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "sections": { + "name": "sections", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "spec_template_org_idx": { + "name": "spec_template_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_ticket_draft": { + "name": "spec_ticket_draft", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "parent_id": { + "name": "parent_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "section_id": { + "name": "section_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "depends_on": { + "name": "depends_on", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "sync_state": { + "name": "sync_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'draft'" + }, + "linear_id": { + "name": "linear_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sync_error": { + "name": "sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "spec_ticket_draft_tree_idx": { + "name": "spec_ticket_draft_tree_idx", + "columns": [ + { + "expression": "spec_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "parent_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "ordinal", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_ticket_draft_spec_id_spec_id_fk": { + "name": "spec_ticket_draft_spec_id_spec_id_fk", + "tableFrom": "spec_ticket_draft", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "spec_ticket_draft_parent_id_fk": { + "name": "spec_ticket_draft_parent_id_fk", + "tableFrom": "spec_ticket_draft", + "tableTo": "spec_ticket_draft", + "columnsFrom": [ + "parent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_ticket_sync_config": { + "name": "spec_ticket_sync_config", + "schema": "", + "columns": { + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "team_id": { + "name": "team_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "team_name": { + "name": "team_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_name": { + "name": "project_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "label_ids": { + "name": "label_ids", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "label_names": { + "name": "label_names", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "spec_ticket_sync_config_spec_id_spec_id_fk": { + "name": "spec_ticket_sync_config_spec_id_spec_id_fk", + "tableFrom": "spec_ticket_sync_config", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_ticket_sync_operation": { + "name": "spec_ticket_sync_operation", + "schema": "", + "columns": { + "caller_spec_id": { + "name": "caller_spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "idempotency_key": { + "name": "idempotency_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_hash": { + "name": "request_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'reserved'" + }, + "reserved_ticket_id": { + "name": "reserved_ticket_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "reserved_external_id": { + "name": "reserved_external_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "spec_ticket_sync_operation_ticket_idx": { + "name": "spec_ticket_sync_operation_ticket_idx", + "columns": [ + { + "expression": "reserved_ticket_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_ticket_sync_operation_caller_spec_id_spec_id_fk": { + "name": "spec_ticket_sync_operation_caller_spec_id_spec_id_fk", + "tableFrom": "spec_ticket_sync_operation", + "tableTo": "spec", + "columnsFrom": [ + "caller_spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "spec_ticket_sync_operation_caller_spec_id_operation_idempotency_key_pk": { + "name": "spec_ticket_sync_operation_caller_spec_id_operation_idempotency_key_pk", + "columns": [ + "caller_spec_id", + "operation", + "idempotency_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_transcript_action": { + "name": "spec_transcript_action", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "section_id": { + "name": "section_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_fingerprint": { + "name": "request_fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "chip": { + "name": "chip", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "result": { + "name": "result", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "actor_user_id": { + "name": "actor_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "delivered_at": { + "name": "delivered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "spec_transcript_action_pending_idx": { + "name": "spec_transcript_action_pending_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"spec_transcript_action\".\"delivered_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "spec_transcript_action_spec_id_spec_id_fk": { + "name": "spec_transcript_action_spec_id_spec_id_fk", + "tableFrom": "spec_transcript_action", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "spec_transcript_action_actor_user_id_user_id_fk": { + "name": "spec_transcript_action_actor_user_id_user_id_fk", + "tableFrom": "spec_transcript_action", + "tableTo": "user", + "columnsFrom": [ + "actor_user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.spec_update_log": { + "name": "spec_update_log", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "spec_id": { + "name": "spec_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "update": { + "name": "update", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "semantic_doc_seq": { + "name": "semantic_doc_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "changed_section_ids": { + "name": "changed_section_ids", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'::text[]" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "spec_update_log_spec_id_spec_id_fk": { + "name": "spec_update_log_spec_id_spec_id_fk", + "tableFrom": "spec_update_log", + "tableTo": "spec", + "columnsFrom": [ + "spec_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "spec_update_log_spec_id_seq_pk": { + "name": "spec_update_log_spec_id_seq_pk", + "columns": [ + "spec_id", + "seq" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.task": { + "name": "task", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "suggested_title": { + "name": "suggested_title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "custom_title": { + "name": "custom_title", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "created_by_user_id": { + "name": "created_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "workflow_run_id": { + "name": "workflow_run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "harness": { + "name": "harness", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model_router": { + "name": "model_router", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "effort": { + "name": "effort", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "parent_task_id": { + "name": "parent_task_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "root_task_id": { + "name": "root_task_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "local_task_name": { + "name": "local_task_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "canonical_task_name": { + "name": "canonical_task_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "spawning_session_id": { + "name": "spawning_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "launch_policy": { + "name": "launch_policy", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "task_slack_channel_created_idx": { + "name": "task_slack_channel_created_idx", + "columns": [ + { + "expression": "(source->>'team')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "(source->>'channel')", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "where": "source->>'provider' = 'slack'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "task_parent_idx": { + "name": "task_parent_idx", + "columns": [ + { + "expression": "parent_task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "task_root_idx": { + "name": "task_root_idx", + "columns": [ + { + "expression": "root_task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "task_root_canonical_name_unique": { + "name": "task_root_canonical_name_unique", + "columns": [ + { + "expression": "root_task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "canonical_task_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "canonical_task_name IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.task_session": { + "name": "task_session", + "schema": "", + "columns": { + "task_id": { + "name": "task_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "profile_id": { + "name": "profile_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "capabilities": { + "name": "capabilities", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "integration_grants": { + "name": "integration_grants", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "integration_connections": { + "name": "integration_connections", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "integration_principal_id": { + "name": "integration_principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mcp_binding_envelope": { + "name": "mcp_binding_envelope", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mcp_managed_lifecycle": { + "name": "mcp_managed_lifecycle", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "integration_snapshot_hash": { + "name": "integration_snapshot_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "task_session_session_idx": { + "name": "task_session_session_idx", + "columns": [ + { + "expression": "session_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "task_session_task_id_task_id_fk": { + "name": "task_session_task_id_task_id_fk", + "tableFrom": "task_session", + "tableTo": "task", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "task_session_profile_id_profile_id_fk": { + "name": "task_session_profile_id_profile_id_fk", + "tableFrom": "task_session", + "tableTo": "profile", + "columnsFrom": [ + "profile_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "task_session_task_id_session_id_pk": { + "name": "task_session_task_id_session_id_pk", + "columns": [ + "task_id", + "session_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "banned": { + "name": "banned", + "type": "boolean", + "primaryKey": false, + "notNull": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "timestamp", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user_session_secrets": { + "name": "user_session_secrets", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "env_var_name": { + "name": "env_var_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "wrapped_dek": { + "name": "wrapped_dek", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "nonce": { + "name": "nonce", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_id": { + "name": "key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "user_session_secrets_user_id_user_id_fk": { + "name": "user_session_secrets_user_id_user_id_fk", + "tableFrom": "user_session_secrets", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "user_session_secrets_user_id_env_var_name_pk": { + "name": "user_session_secrets_user_id_env_var_name_pk", + "columns": [ + "user_id", + "env_var_name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_registration": { + "name": "webhook_registration", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "verification": { + "name": "verification", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "provider_hint": { + "name": "provider_hint", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "disabled_reason": { + "name": "disabled_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_by_user_id": { + "name": "created_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "webhook_registration_provider_idx": { + "name": "webhook_registration_provider_idx", + "columns": [ + { + "expression": "provider_hint", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_sample": { + "name": "webhook_sample", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "registration_id": { + "name": "registration_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "webhook_sample_registration_event_idx": { + "name": "webhook_sample_registration_event_idx", + "columns": [ + { + "expression": "registration_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "received_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "webhook_sample_registration_id_webhook_registration_id_fk": { + "name": "webhook_sample_registration_id_webhook_registration_id_fk", + "tableFrom": "webhook_sample", + "tableTo": "webhook_registration", + "columnsFrom": [ + "registration_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/orchestrator/drizzle/meta/_journal.json b/orchestrator/drizzle/meta/_journal.json index 7288b7157..a01636a0e 100644 --- a/orchestrator/drizzle/meta/_journal.json +++ b/orchestrator/drizzle/meta/_journal.json @@ -708,6 +708,111 @@ "when": 1793241391817, "tag": "0100_profile_programmatic_only", "breakpoints": true + }, + { + "idx": 101, + "version": "7", + "when": 1793585018303, + "tag": "0101_mcp_authority", + "breakpoints": true + }, + { + "idx": 102, + "version": "7", + "when": 1793585413259, + "tag": "0102_mcp_authorization_kind", + "breakpoints": true + }, + { + "idx": 103, + "version": "7", + "when": 1793585592275, + "tag": "0103_mcp_session_retention", + "breakpoints": true + }, + { + "idx": 104, + "version": "7", + "when": 1793585592276, + "tag": "0104_mcp_credential_metadata", + "breakpoints": true + }, + { + "idx": 105, + "version": "7", + "when": 1793602718404, + "tag": "0105_mcp_binding_envelope", + "breakpoints": true + }, + { + "idx": 106, + "version": "7", + "when": 1793604760959, + "tag": "0106_mcp_request_review", + "breakpoints": true + }, + { + "idx": 107, + "version": "7", + "when": 1793605822727, + "tag": "0107_mcp_admission_generation", + "breakpoints": true + }, + { + "idx": 108, + "version": "7", + "when": 1793609323674, + "tag": "0108_mcp_execution_authority", + "breakpoints": true + }, + { + "idx": 109, + "version": "7", + "when": 1793609726214, + "tag": "0109_mcp_fixed_execute_deadline", + "breakpoints": true + }, + { + "idx": 110, + "version": "7", + "when": 1793611948671, + "tag": "0110_mcp_preflight_rejection", + "breakpoints": true + }, + { + "idx": 111, + "version": "7", + "when": 1793613661405, + "tag": "0111_mcp_completion_receipt", + "breakpoints": true + }, + { + "idx": 112, + "version": "7", + "when": 1793616759320, + "tag": "0112_mcp_lifecycle_receipts", + "breakpoints": true + }, + { + "idx": 113, + "version": "7", + "when": 1793619176833, + "tag": "0113_mcp_lifecycle_active", + "breakpoints": true + }, + { + "idx": 114, + "version": "7", + "when": 1793653206394, + "tag": "0114_mcp_discovery_admission", + "breakpoints": true + }, + { + "idx": 115, + "version": "7", + "when": 1793654535744, + "tag": "0115_mcp_runtime_composition", + "breakpoints": true } ] } \ No newline at end of file diff --git a/orchestrator/scripts/mcp-stack/automation-ownership.test.ts b/orchestrator/scripts/mcp-stack/automation-ownership.test.ts new file mode 100644 index 000000000..637986e27 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/automation-ownership.test.ts @@ -0,0 +1,433 @@ +import { expect, test } from "bun:test"; +import { + chmod, + link, + mkdtemp, + readdir, + readFile, + realpath, + rename, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { create } from "@bufbuild/protobuf"; +import { + AutomationSchema, + AutomationRunSchema, +} from "../../src/gen/engram/app/v1/automation_pb.ts"; +import { TaskSchema } from "../../src/gen/engram/app/v1/task_pb.ts"; +import { SessionSchema } from "../../src/gen/engram/app/v1/session_pb.ts"; +import { registerEngineBlocks } from "../../src/automations/engine/blocks/index.ts"; +import { validateDefinition } from "../../src/automations/engine/definition.ts"; +import { definitionOf } from "../../src/db/automations.ts"; +import { canonicalJson } from "./guest.ts"; +import { sha256 } from "./context.ts"; +import { + recordOwnedAutomation, + readOwnedAutomationRuns, + ownedAutomationCleanupSessions, + type AutomationCleanupClient, + type ManagedAutomationSession, +} from "./automation-ownership.ts"; + +const hostId = "11111111-1111-4111-8111-111111111111"; +const sessionId = "22222222-2222-4222-8222-222222222222"; +const foreignId = "33333333-3333-4333-8333-333333333333"; +const taskId = "fixture-root-task"; +const runId = "autorun:fixture-automation:manual:fixture-event"; +const definition = { + engine: 1, + trigger: { kind: "manual" }, + blocks: [ + { + id: "launch", + type: "create_session", + config: { + profileId: "fixture-profile", + promptTemplate: "", + titleTemplate: "Fixture automation task", + includeEventContext: false, + keepOnFinish: true, + }, + }, + { + id: "lifetime", + type: "wait_session", + config: { session: { blockId: "launch" }, until: "ended" }, + }, + ], + inputsSchema: [], + settings: { endSessionsOnFinish: false, runDeadlineSeconds: 120 }, +}; +registerEngineBlocks(); +const normalized = validateDefinition(definition); +const projectedDefinition = definitionOf({ + automationId: "fixture-automation", + version: 1, + trigger: normalized.trigger, + blocks: normalized.blocks, + entrypoints: normalized.entrypoints ?? [], + agent: normalized.agent ?? null, + role: normalized.role ?? null, + promptTemplate: normalized.promptTemplate ?? null, + inputsSchema: normalized.inputsSchema, + settings: normalized.settings, + createdByUserId: "fixture-admin", + createdAt: new Date("2026-01-01T00:00:00Z"), +}); + +const owned = { + automationId: "fixture-automation", + definitionActorId: "fixture-admin", + definitionSha256: sha256(canonicalJson(definition)), + profileId: "fixture-profile", +}; +const launchOutput = { + session_id: sessionId, + task_id: taskId, + initial_prompt: false, + prompt: "", +}; +async function fixture() { + const root = await mkdtemp( + join(await realpath(tmpdir()), "mcp-automation-owner-"), + ); + const ctx = { + runRoot: root, + runId: "fixture-run", + sourceCommit: "a".repeat(40), + hostId, + }; + await recordOwnedAutomation(ctx, owned); + await recordOwnedAutomation(ctx, { + ...owned, + runId: runId, + }); + const automation = create(AutomationSchema, { + id: owned.automationId, + kind: "user", + currentVersion: 1, + createdByUserId: owned.definitionActorId, + version: { + automationId: owned.automationId, + number: 1, + createdByUserId: owned.definitionActorId, + definitionJson: JSON.stringify(projectedDefinition), + }, + }); + const run = create(AutomationRunSchema, { + brief: { + id: runId, + automationId: owned.automationId, + version: 1, + triggerSource: "manual", + dryRun: false, + status: "running", + }, + steps: [ + { + blockId: "launch", + status: "succeeded", + sessionId, + outputsJson: JSON.stringify(launchOutput), + }, + ], + sessionIds: [sessionId], + }); + const task = create(TaskSchema, { + id: taskId, + type: "automation", + rootTaskId: taskId, + sourceJson: JSON.stringify({ + provider: "automation", + automationId: owned.automationId, + runId: runId, + }), + sessions: [{ sessionId, role: "primary" }], + }); + const client: AutomationCleanupClient = { + automations: { + getAutomation: async (request) => { + expect(request.lookup).toEqual({ + case: "id", + value: owned.automationId, + }); + return { $typeName: "engram.app.v1.GetAutomationResponse", automation }; + }, + }, + automationRuns: { + getRun: async (request) => { + expect(request.runId).toBe(runId); + return { $typeName: "engram.app.v1.GetRunResponse", run }; + }, + }, + tasks: { + getTask: async (request) => { + expect(request.taskId).toBe(taskId); + return { $typeName: "engram.app.v1.GetTaskResponse", task }; + }, + }, + }; + const managed = new Map(); + const proof = () => + ownedAutomationCleanupSessions( + ctx, + client, + managed, + new Set([owned.definitionActorId]), + new Set([sessionId]), + ); + return { root, ctx, automation, run, task, client, managed, proof }; +} +type Fixture = Awaited>; +async function withFixture(action: (value: Fixture) => Promise) { + const value = await fixture(); + try { + await action(value); + } finally { + await rm(value.root, { recursive: true }); + } +} + +test("production definition validation and stored projection preserve the fixture definition hash", () => { + expect(sha256(canonicalJson(projectedDefinition))).toBe( + owned.definitionSha256, + ); +}); + +test("automation ownership retains exact session and task after launch output", async () => { + await withFixture(async (f) => { + await recordOwnedAutomation(f.ctx, { ...owned, runId, sessionId, taskId }); + expect(await readOwnedAutomationRuns(f.ctx)).toEqual([ + { + ...owned, + runId, + sessionId, + taskId, + version: 1, + fixtureRunId: f.ctx.runId, + sourceCommit: f.ctx.sourceCommit, + hostId, + }, + ]); + }); +}); + +for (const boot of [false, true]) { + test(`automation cleanup proves exact session ownership ${boot ? "with a boot record" : "before host assignment"}`, async () => { + await withFixture(async (f) => { + if (boot) + f.managed.set(sessionId, { + sessionId, + actorId: `automation:${owned.automationId}`, + hostId, + taskId, + }); + expect([...(await f.proof())]).toEqual([sessionId]); + expect(await readOwnedAutomationRuns(f.ctx)).toEqual([ + { + ...owned, + runId: runId, + version: 1, + fixtureRunId: f.ctx.runId, + sourceCommit: f.ctx.sourceCommit, + hostId, + }, + ]); + }); + }); +} +for (const status of ["completed", "failed", "deadline"]) { + test(`terminal ${status} automation is provable only for cleanup`, async () => { + await withFixture(async (f) => { + f.run.brief!.status = status; + expect([...(await f.proof())]).toEqual([sessionId]); + }); + }); +} +const conflicts: [string, (f: Fixture) => void][] = [ + [ + "foreign automation", + (f) => { + f.automation.id = "foreign-automation"; + }, + ], + [ + "foreign creator", + (f) => { + f.automation.createdByUserId = "foreign-creator"; + }, + ], + [ + "foreign version creator", + (f) => { + f.automation.version!.createdByUserId = "foreign-creator"; + }, + ], + [ + "changed definition", + (f) => { + f.automation.version!.definitionJson = "{}"; + }, + ], + [ + "wrong run", + (f) => { + f.run.brief!.id = "other-run"; + }, + ], + [ + "run from another automation", + (f) => { + f.run.brief!.automationId = "foreign-automation"; + }, + ], + [ + "wrong run version", + (f) => { + f.run.brief!.version = 2; + }, + ], + [ + "dry run", + (f) => { + f.run.brief!.dryRun = true; + }, + ], + [ + "wrong run session", + (f) => { + f.run.sessionIds = [foreignId]; + }, + ], + [ + "wrong launch session", + (f) => { + f.run.steps[0]!.sessionId = foreignId; + }, + ], + [ + "wrong launch output session", + (f) => { + f.run.steps[0]!.outputsJson = JSON.stringify({ + ...launchOutput, + session_id: foreignId, + }); + }, + ], + [ + "wrong root task", + (f) => { + f.task.rootTaskId = "foreign-task"; + }, + ], + [ + "wrong task id", + (f) => { + f.task.id = "foreign-task"; + }, + ], + [ + "wrong task run", + (f) => { + f.task.sourceJson = JSON.stringify({ + provider: "automation", + automationId: owned.automationId, + runId: "foreign-run", + }); + }, + ], + [ + "wrong task automation", + (f) => { + f.task.sourceJson = JSON.stringify({ + provider: "automation", + automationId: "foreign-automation", + runId: runId, + }); + }, + ], + [ + "wrong task session", + (f) => { + f.task.sessions[0]!.sessionId = foreignId; + }, + ], + [ + "foreign public host", + (f) => { + f.task.sessions[0]!.session = create(SessionSchema, { + id: sessionId, + hostId: foreignId, + status: "active", + }); + }, + ], +]; +for (const [name, conflict] of conflicts) { + test(`automation cleanup rejects ${name}`, async () => { + await withFixture(async (f) => { + conflict(f); + if (name === "wrong launch output session") + expect([...(await f.proof())]).toEqual([]); + else await expect(f.proof()).rejects.toThrow(); + }); + }); +} +for (const [name, value] of [ + ["actorId", "automation:foreign"], + ["taskId", "foreign-task"], + ["hostId", foreignId], + ["sessionId", foreignId], +] as const) { + test(`automation cleanup rejects existing boot ${name} conflict`, async () => { + await withFixture(async (f) => { + f.managed.set(sessionId, { + sessionId, + actorId: `automation:${owned.automationId}`, + hostId, + taskId, + [name]: value, + }); + await expect(f.proof()).rejects.toThrow( + "cleanup_automation_managed_identity", + ); + }); + }); +} +for (const mode of [ + "substituted", + "wrong-name", + "public", + "symlink", + "hardlink", +] as const) { + test(`automation ownership rejects a ${mode} receipt`, async () => { + await withFixture(async (f) => { + const name = (await readdir(f.root)).find((name) => + name.startsWith("automation-run-owner-"), + ); + expect(name).toBeDefined(); + const path = join(f.root, name!); + if (mode === "substituted") { + const receipt = JSON.parse(await readFile(path, "utf8")); + receipt.fixtureRunId = "foreign-run"; + await writeFile(path, JSON.stringify(receipt), { mode: 0o600 }); + } else if (mode === "wrong-name") { + await rename( + path, + join(f.root, `automation-run-owner-${"0".repeat(64)}.json`), + ); + } else if (mode === "public") await chmod(path, 0o644); + else if (mode === "hardlink") await link(path, path + ".other"); + else { + await rename(path, path + ".other"); + await symlink(path + ".other", path); + } + await expect(readOwnedAutomationRuns(f.ctx)).rejects.toThrow(); + }); + }); +} diff --git a/orchestrator/scripts/mcp-stack/automation-ownership.ts b/orchestrator/scripts/mcp-stack/automation-ownership.ts new file mode 100644 index 000000000..b4e19bbc5 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/automation-ownership.ts @@ -0,0 +1,298 @@ +import { readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { z } from "zod"; +import { + privateRoot, + put, + readOwned, +} from "../../../tests/fixtures/mcp-stack/platform/owned.ts"; +import { requireThat, sha256, type Context } from "./context.ts"; +import { canonicalJson } from "./guest.ts"; +import type { PublicClient } from "./transport.ts"; + +type OwnershipContext = Pick< + Context, + "runRoot" | "runId" | "sourceCommit" | "hostId" +>; +const id = z.string().regex(/^[A-Za-z0-9_-]{1,96}$/); +const reference = z.strictObject({ + automationId: id, + definitionActorId: id, + definitionSha256: z.string().regex(/^[a-f0-9]{64}$/), + profileId: id, + runId: z + .string() + .regex(/^[A-Za-z0-9_:-]{1,256}$/) + .optional(), + sessionId: z.uuid().optional(), + taskId: z.string().min(1).max(256).optional(), +}); +const receiptSchema = reference.extend({ + version: z.literal(1), + fixtureRunId: id, + sourceCommit: z.string().regex(/^[a-f0-9]{40}$/), + hostId: z.uuid(), +}); +type Receipt = z.infer; +export type OwnedAutomationRun = Receipt & { runId: string }; +export type ManagedAutomationSession = { + sessionId: string; + actorId: string; + hostId: string; + taskId: string; +}; +export type AutomationCleanupClient = { + automations: Pick; + automationRuns: Pick; + tasks: Pick; +}; +function receiptPath( + ctx: OwnershipContext, + receipt: z.infer, +) { + return join( + ctx.runRoot, + receipt.sessionId && receipt.runId + ? `automation-session-owner-${sha256(receipt.runId)}.json` + : receipt.runId + ? `automation-run-owner-${sha256(receipt.runId)}.json` + : `automation-owner-${sha256(receipt.automationId)}.json`, + ); +} +async function readReceipt( + ctx: OwnershipContext, + path: string, +): Promise { + const receipt = receiptSchema.parse( + JSON.parse((await readOwned(path, 4096)).toString()), + ); + requireThat( + Boolean(receipt.sessionId) === Boolean(receipt.taskId) && + (!receipt.sessionId || Boolean(receipt.runId)), + "automation_ownership_session_reference", + ); + requireThat( + receipt.fixtureRunId === ctx.runId && + receipt.sourceCommit === ctx.sourceCommit && + receipt.hostId === ctx.hostId && + receiptPath(ctx, receipt) === path, + "automation_ownership_receipt_identity", + ); + return receipt; +} +async function matchCreation(ctx: OwnershipContext, receipt: Receipt) { + const creation = { ...receipt }; + delete creation.runId; + delete creation.sessionId; + delete creation.taskId; + const actual = await readReceipt(ctx, receiptPath(ctx, creation)); + requireThat( + canonicalJson(actual) === canonicalJson(creation), + "automation_ownership_creation", + ); +} +async function matchRun(ctx: OwnershipContext, receipt: Receipt) { + const run = { ...receipt }; + delete run.sessionId; + delete run.taskId; + const actual = await readReceipt(ctx, receiptPath(ctx, run)); + requireThat( + canonicalJson(actual) === canonicalJson(run), + "automation_ownership_run_reference", + ); +} +export async function recordOwnedAutomation( + ctx: OwnershipContext, + owned: z.infer, +): Promise { + const receipt = receiptSchema.parse({ + ...reference.parse(owned), + version: 1, + fixtureRunId: ctx.runId, + sourceCommit: ctx.sourceCommit, + hostId: ctx.hostId, + }); + requireThat( + Boolean(receipt.sessionId) === Boolean(receipt.taskId) && + (!receipt.sessionId || Boolean(receipt.runId)), + "automation_ownership_session_reference", + ); + if (receipt.runId) await matchCreation(ctx, receipt); + if (receipt.sessionId) await matchRun(ctx, receipt); + await put(receiptPath(ctx, receipt), JSON.stringify(receipt)); +} +export async function readOwnedAutomationRuns( + ctx: OwnershipContext, +): Promise { + await privateRoot(ctx.runRoot); + const allNames = new Set(await readdir(ctx.runRoot)); + const names = [...allNames].filter((name) => + /^automation-run-owner-[a-f0-9]{64}\.json$/.test(name), + ); + requireThat(names.length <= 128, "automation_ownership_receipt_bound"); + const runs: OwnedAutomationRun[] = []; + for (const name of names.sort()) { + let receipt = await readReceipt(ctx, join(ctx.runRoot, name)); + requireThat(receipt.runId, "automation_ownership_run"); + await matchCreation(ctx, receipt); + const sessionName = `automation-session-owner-${sha256(receipt.runId)}.json`; + if (allNames.has(sessionName)) { + receipt = await readReceipt(ctx, join(ctx.runRoot, sessionName)); + requireThat( + receipt.runId && receipt.sessionId && receipt.taskId, + "automation_ownership_session_reference", + ); + await matchCreation(ctx, receipt); + await matchRun(ctx, receipt); + } + requireThat(receipt.runId, "automation_ownership_run"); + runs.push({ ...receipt, runId: receipt.runId }); + } + return runs; +} + +/** Cleanup ownership only. A terminal run does not give runtime authority. */ +export async function ownedAutomationCleanupSessions( + ctx: OwnershipContext, + client: AutomationCleanupClient, + managed: ReadonlyMap, + actors: ReadonlySet, + candidates: ReadonlySet, +): Promise> { + const sessions = new Set(); + for (const receipt of await readOwnedAutomationRuns(ctx)) { + if (receipt.sessionId && !candidates.has(receipt.sessionId)) continue; + requireThat( + actors.has(receipt.definitionActorId), + "cleanup_automation_creator", + ); + const { automation } = await client.automations.getAutomation({ + lookup: { case: "id", value: receipt.automationId }, + }); + requireThat( + automation?.id === receipt.automationId && + automation.kind === "user" && + automation.createdByUserId === receipt.definitionActorId && + automation.version && + automation.version.automationId === receipt.automationId && + automation.version.createdByUserId === receipt.definitionActorId && + automation.version.number === automation.currentVersion && + automation.version.definitionJson.length <= 65536, + "cleanup_automation_definition", + ); + const definition = JSON.parse(automation.version.definitionJson); + requireThat( + sha256(canonicalJson(definition)) === receipt.definitionSha256, + "cleanup_automation_definition_hash", + ); + const blocks = z + .object({ + blocks: z + .array( + z.object({ + id: z.string(), + type: z.string(), + config: z.record(z.string(), z.unknown()), + }), + ) + .max(128), + }) + .parse(definition).blocks; + const launch = blocks.filter( + (block) => block.id === "launch" && block.type === "create_session", + ); + requireThat( + launch.length === 1 && launch[0]!.config.profileId === receipt.profileId, + "cleanup_automation_profile", + ); + const { run } = await client.automationRuns.getRun({ + runId: receipt.runId, + }); + requireThat( + run?.brief?.id === receipt.runId && + run.brief.automationId === receipt.automationId && + run.brief.version === automation.version.number && + run.brief.triggerSource === "manual" && + !run.brief.dryRun && + run.steps.length <= 128 && + run.sessionIds.length <= 128, + "cleanup_automation_run", + ); + const steps = run.steps.filter( + (step) => step.blockId === "launch" && step.status === "succeeded", + ); + if (steps.length === 0 && run.sessionIds.length === 0) continue; + requireThat( + steps.length === 1 && steps[0]!.outputsJson.length <= 65536, + "cleanup_automation_launch", + ); + const output = z + .object({ + session_id: z.uuid(), + task_id: z.string().min(1).max(256), + initial_prompt: z.literal(false), + prompt: z.literal(""), + }) + .parse(JSON.parse(steps[0]!.outputsJson)); + if (!candidates.has(output.session_id)) continue; + requireThat( + !receipt.sessionId || + (receipt.sessionId === output.session_id && + receipt.taskId === output.task_id), + "cleanup_automation_recorded_session", + ); + requireThat( + run.sessionIds.length === 1 && + run.sessionIds[0] === output.session_id && + steps[0]!.sessionId === output.session_id, + "cleanup_automation_session", + ); + const { task } = await client.tasks.getTask({ taskId: output.task_id }); + requireThat( + task?.id === output.task_id && + task.type === "automation" && + (task.rootTaskId ?? task.id) === output.task_id && + task.sessions.length <= 128 && + task.sourceJson.length <= 65536, + "cleanup_automation_task", + ); + const source = z + .object({ + provider: z.literal("automation"), + automationId: z.string(), + runId: z.string(), + }) + .parse(JSON.parse(task.sourceJson)); + requireThat( + source.automationId === receipt.automationId && + source.runId === receipt.runId, + "cleanup_automation_task_source", + ); + const primary = task.sessions.filter( + (ref) => ref.sessionId === output.session_id && ref.role === "primary", + ); + requireThat( + primary.length === 1 && + (!primary[0]!.session || + (primary[0]!.session.id === output.session_id && + (!primary[0]!.session.hostId || + primary[0]!.session.hostId === ctx.hostId))), + "cleanup_automation_task_session", + ); + const boot = managed.get(output.session_id); + requireThat( + !boot || + (boot.sessionId === output.session_id && + boot.hostId === ctx.hostId && + boot.actorId === `automation:${receipt.automationId}` && + boot.taskId === output.task_id), + "cleanup_automation_managed_identity", + ); + requireThat( + !sessions.has(output.session_id), + "cleanup_automation_duplicate_session", + ); + sessions.add(output.session_id); + } + return sessions; +} diff --git a/orchestrator/scripts/mcp-stack/automation.ts b/orchestrator/scripts/mcp-stack/automation.ts new file mode 100644 index 000000000..53ee3c562 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/automation.ts @@ -0,0 +1,140 @@ +import { z } from "zod"; +import { waitFor } from "./bootstrap.ts"; +import { requireThat, sha256, type Context } from "./context.ts"; +import { canonicalJson } from "./guest.ts"; +import type { Principals } from "./scenario-runtime.ts"; +import { recordOwnedAutomation } from "./automation-ownership.ts"; + +export type CreatedScenarioTask = { + taskId: string; + sessionId: string; + automation?: { + id: string; + runId: string; + runtimeActorId: string; + definitionActorId: string; + triggerActorId: string; + definitionSha256: string; + }; +}; +export async function createAutomationTask( + ctx: Context, + principals: Principals, + profileId: string, + suffix: string, +): Promise { + const definition = { + engine: 1, + trigger: { kind: "manual" }, + blocks: [ + { + id: "launch", + type: "create_session", + config: { + profileId, + promptTemplate: "", + titleTemplate: "Fixture automation task", + includeEventContext: false, + keepOnFinish: true, + }, + }, + { + id: "lifetime", + type: "wait_session", + config: { session: { blockId: "launch" }, until: "ended" }, + }, + ], + inputsSchema: [], + settings: { endSessionsOnFinish: false, runDeadlineSeconds: 120 }, + }; + const created = await principals.admin.automations.createAutomation({ + name: `Fixture ${suffix}`, + enabled: true, + definitionJson: JSON.stringify(definition), + inputsJson: "{}", + }); + requireThat(created.automation?.id, "public_automation_created"); + const automationId = created.automation.id; + const ownership = { + automationId, + definitionActorId: principals.actorIds.admin, + definitionSha256: sha256(canonicalJson(definition)), + profileId, + }; + await recordOwnedAutomation(ctx, ownership); + const launched = await principals.automation.automations.runNow({ + automationId, + }); + requireThat(launched.runId, "public_automation_run"); + await recordOwnedAutomation(ctx, { ...ownership, runId: launched.runId }); + const observed = await waitFor( + ctx, + "automation_actual_launch", + () => + principals.automation.automationRuns.getRun({ runId: launched.runId }), + (value) => { + requireThat( + value.run?.brief && + !value.run.brief.dryRun && + value.run.brief.automationId === automationId && + value.run.brief.triggerSource === "manual", + "actual_manual_automation_run", + ); + requireThat( + ["pending", "running"].includes(value.run.brief.status) && + !value.run.brief.endedAt, + "automation_launch_failed", + ); + return ( + value.run.brief.status === "running" && + value.run.steps.some( + (step) => step.blockId === "launch" && step.status === "succeeded", + ) + ); + }, + ); + const steps = observed.run!.steps.filter( + (step) => step.blockId === "launch" && step.status === "succeeded", + ); + requireThat(steps.length === 1, "single_automation_launch"); + const outputs = z + .object({ + session_id: z.uuid(), + task_id: z.string().min(1), + initial_prompt: z.literal(false), + prompt: z.literal(""), + }) + .safeParse(JSON.parse(steps[0]!.outputsJson)); + requireThat( + outputs.success && + observed.run!.sessionIds.length === 1 && + observed.run!.sessionIds[0] === outputs.data.session_id, + "automation_public_session_identity", + ); + await recordOwnedAutomation(ctx, { + ...ownership, + runId: launched.runId, + sessionId: outputs.data.session_id, + taskId: outputs.data.task_id, + }); + const titled = await principals.admin.tasks.updateTask({ + taskId: outputs.data.task_id, + title: "Fixture automation task", + }); + requireThat( + titled.task?.titleIsCustom === true, + "automation_custom_title_before_prompt", + ); + return { + taskId: outputs.data.task_id, + sessionId: outputs.data.session_id, + automation: { + id: automationId, + runId: launched.runId, + runtimeActorId: `automation:${automationId}`, + definitionActorId: principals.actorIds.admin, + triggerActorId: principals.actorIds.automation, + definitionSha256: ownership.definitionSha256, + }, + }; +} diff --git a/orchestrator/scripts/mcp-stack/bootstrap.ts b/orchestrator/scripts/mcp-stack/bootstrap.ts new file mode 100644 index 000000000..1e917028b --- /dev/null +++ b/orchestrator/scripts/mcp-stack/bootstrap.ts @@ -0,0 +1,677 @@ +import { writeFile } from "node:fs/promises"; +import { Code, ConnectError } from "@connectrpc/connect"; +import { z } from "zod"; +import { + privateFile, + requireThat, + RunnerFailure, + sha256, + type Context, + type FixtureKind, +} from "./context.ts"; +import { + jsonRequest, + operatorClient, + publicClient, + type PublicClient, +} from "./transport.ts"; +import { canonicalJson } from "./guest.ts"; +import type { Harness } from "./manifest.ts"; +import { encodeCallPlan, type CallPlan } from "./call-plan.ts"; + +export async function waitFor( + ctx: Pick, + seam: string, + read: () => Promise, + ready: (value: T) => boolean, +): Promise { + const deadline = performance.now() + ctx.deadlines.stepMs; + do { + const value = await read(); + if (ready(value)) return value; + await new Promise((resolve) => setTimeout(resolve, 100)); + } while (performance.now() < deadline); + throw new RunnerFailure("ASSERTION_FAILED", seam); +} +export async function bootstrapHumans(ctx: Context) { + const clients: Partial> = + {}; + const identities: string[] = []; + for (const role of ["admin", "owner", "other"] as const) { + const config = ctx.identities[role]; + const password = (await privateFile(ctx.runRoot, config.passwordFile)) + .toString("utf8") + .trim(); + requireThat( + password.length >= 16 && password.length <= 128, + "neutral_password_bounds", + ); + const signup = await jsonRequest( + ctx, + ctx.endpoints.app, + "POST", + "/api/auth/sign-up/email", + { email: config.email, password, name: `Fixture ${role}` }, + { origin: ctx.endpoints.app.origin }, + ); + requireThat(signup.status === 200, "password_signup"); + const login = await jsonRequest( + ctx, + ctx.endpoints.app, + "POST", + "/api/auth/sign-in/email", + { email: config.email, password }, + { origin: ctx.endpoints.app.origin }, + ); + requireThat(login.status === 200, "password_login"); + const cookies = login.headers["set-cookie"]; + requireThat(cookies?.length && cookies.length <= 8, "human_session_cookie"); + const cookie = cookies.map((v) => v.split(";", 1)[0]).join("; "); + requireThat(cookie.length <= 8192, "cookie_size"); + await writeFile(config.cookieFile, cookie, { mode: 0o600, flag: "wx" }); + const client = await publicClient(ctx, { kind: "cookie", value: cookie }); + const identity = await client.identity.whoAmI({}); + requireThat( + identity.email === config.email && + !identity.serviceAccount && + identity.role === (role === "admin" ? "admin" : "user") && + identity.userId, + "human_principal", + ); + identities.push(identity.userId); + clients[role] = client; + } + requireThat(new Set(identities).size === 3, "separate_human_principals"); + requireThat(clients.admin && clients.owner && clients.other, "human_clients"); + const key = await clients.admin.identity.createApiKey({ + name: "Fixture unattended", + role: "user", + }); + requireThat( + key.key.startsWith("engk_") && key.meta?.role === "user", + "public_api_key_creation", + ); + await writeFile(ctx.identities.unattendedKeyFile, key.key, { + mode: 0o600, + flag: "wx", + }); + const unattended = await publicClient(ctx, { + kind: "api-key", + value: key.key, + }); + const identity = await unattended.identity.whoAmI({}); + requireThat( + identity.serviceAccount && + identity.role === "user" && + !identities.includes(identity.userId), + "unattended_principal", + ); + const automationKey = await clients.admin.identity.createApiKey({ + name: "Fixture automation trigger", + role: "admin", + }); + requireThat( + automationKey.key.startsWith("engk_") && + automationKey.meta?.role === "admin", + "automation_api_key_creation", + ); + await writeFile(ctx.identities.automationKeyFile, automationKey.key, { + mode: 0o600, + flag: "wx", + }); + const automation = await publicClient(ctx, { + kind: "api-key", + value: automationKey.key, + }); + const automationIdentity = await automation.identity.whoAmI({}); + requireThat( + automationIdentity.serviceAccount && + automationIdentity.role === "admin" && + automationIdentity.userId && + !identities.includes(automationIdentity.userId) && + automationIdentity.userId !== identity.userId, + "separate_automation_trigger_principal", + ); + const sentinel = ( + await privateFile(ctx.runRoot, ctx.identities.routerSentinelFile) + ) + .toString("utf8") + .trim(); + requireThat( + /^fixture-neutral-[A-Za-z0-9_-]{16,96}$/.test(sentinel), + "neutral_router_sentinel", + ); + await clients.admin.secrets.putSecret({ + name: "openrouter.api_key", + value: sentinel, + }); + for (const client of [clients.owner, clients.other, unattended, automation]) { + const models = await client.models.listRouterModels({ + routerId: "openrouter", + }); + const model = models.models.find((v) => v.id === "z-ai/glm-5.2"); + requireThat( + model?.available && model.enabled && model.userEnabled, + "migrated_router_model", + ); + } + return { + admin: clients.admin, + owner: clients.owner, + other: clients.other, + unattended, + automation, + actorIds: { + admin: identities[0]!, + owner: identities[1]!, + other: identities[2]!, + unattended: identity.userId, + automation: automationIdentity.userId, + }, + }; +} + +function parseOperatorPublication(bytes: Uint8Array) { + try { + const generation = z.object({ + identity: z.string().regex(/^sha256:[a-f0-9]{64}$/), + generation: z.number().int().positive().safe(), + candidate: z.object({ + deployment_identity: z.string(), + network_identity: z.string(), + trust_identity: z.string(), + }), + }); + return z + .strictObject({ + admission_open: z.boolean(), + pending: generation.nullable(), + published: generation.nullable(), + }) + .parse(JSON.parse(new TextDecoder().decode(bytes))); + } catch { + throw new RunnerFailure("ASSERTION_FAILED", "operator_publication_json"); + } +} +export async function readOperatorPublication(ctx: Context) { + const response = await (await operatorClient(ctx)).read({ protocol: 1 }); + requireThat(response.protocol === 1, "operator_protocol"); + return parseOperatorPublication(response.publicationJson); +} +export async function publishOperator(ctx: Context) { + const client = await operatorClient(ctx); + const prior = await readOperatorPublication(ctx); + const candidateIdentity = ctx.publication.candidateIdentity; + const expectedGeneration = ctx.publication.expectedGeneration; + requireThat( + prior.pending === null && + (prior.published?.generation ?? null) === expectedGeneration, + "operator_expected_generation", + ); + await client.stage({ + protocol: 1, + configuredCandidateIdentity: candidateIdentity, + ...(expectedGeneration === null + ? {} + : { expectedGeneration: BigInt(expectedGeneration) }), + }); + return waitFor( + ctx, + "publication_open", + async () => { + const response = await client.read({ protocol: 1 }); + requireThat(response.protocol === 1, "operator_protocol"); + return parseOperatorPublication(response.publicationJson); + }, + (value) => + value.pending === null && + value.admission_open === true && + value.published?.identity === candidateIdentity && + value.published.generation === (expectedGeneration ?? 0) + 1 && + value.published.candidate?.deployment_identity === + ctx.publication.deploymentIdentity && + value.published.candidate.network_identity === + ctx.publication.networkIdentity && + value.published.candidate.trust_identity === + ctx.publication.trustIdentity, + ); +} + +export async function createCatalog( + ctx: Context, + admin: PublicClient, + suffix: string, + protocol: FixtureKind = "modern", + authKind: "api_key" | "oauth" = "api_key", +) { + let registrationId: string | undefined; + if (authKind === "oauth") { + const setup = await admin.integration.getMcpSetup({}); + const matches = setup.registrations.filter( + (registration) => + registration.resource?.endpoint === + `${ctx.protocolFixtures[protocol].resourceOrigin}/mcp` && + registration.issuer === ctx.protocolFixtures[protocol].issuerOrigin && + registration.redirectUri === ctx.protocolFixtures[protocol].redirectUri, + ); + requireThat( + setup.enabled && matches.length === 1, + "exact_oauth_registration_before_create", + ); + registrationId = matches[0]!.id; + } + requireThat(/^[a-z0-9-]{1,40}$/.test(suffix), "connection_suffix"); + const created = await admin.integration.createConnection({ + alias: `fixture-${suffix}`, + provider: "mcp", + displayName: "Fixture counter", + mcp: { + endpoint: `${ctx.protocolFixtures[protocol].resourceOrigin}/mcp`, + zone: ctx.publication.zone, + authKind, + ...(registrationId + ? { + oauthClientRegistrationId: registrationId, + shareWithOrgSessions: true, + } + : {}), + }, + }); + requireThat( + created.connection?.id && + created.connection.mcp?.endpoint === + `${ctx.protocolFixtures[protocol].resourceOrigin}/mcp`, + "created_connection", + ); + let authorizedConnection; + if (authKind === "oauth") { + authorizedConnection = await authorizeOAuth( + ctx, + admin, + created.connection.id, + protocol, + ); + } else { + const resourceKey = ( + await privateFile( + ctx.runRoot, + ctx.protocolFixtures[protocol].resourceKeyFile, + ) + ) + .toString("utf8") + .trim(); + requireThat( + /^[A-Za-z0-9_-]{16,128}$/.test(resourceKey), + "neutral_resource_key", + ); + const credential = await admin.integration.putMcpCredential({ + connectionId: created.connection.id, + expectedRevision: created.connection.configRevision, + expectedGeneration: created.connection.authGeneration, + kind: "api_key", + header: "authorization", + value: `Bearer ${resourceKey}`, + }); + requireThat(credential.connection, "credential_connection"); + authorizedConnection = credential.connection; + } + const discovered = await admin.integration.discoverMcp({ + connectionId: created.connection.id, + expectedRevision: authorizedConnection.configRevision, + }); + requireThat( + discovered.catalog?.tools.length === 2 && + new Set(discovered.catalog.tools.map((t) => t.upstreamName)).size === 2 && + discovered.catalog.tools.every((t) => + ["read_counter", "write_counter"].includes(t.upstreamName), + ), + "reviewed_counter_catalog", + ); + const reviewed = discovered.catalog.tools.map((tool) => ({ + ...tool, + classification: tool.upstreamName === "read_counter" ? "read" : "write", + enabled: true, + })); + const current = await admin.integration.getMcpCatalogs({ + connectionId: created.connection.id, + }); + requireThat( + current.draft?.id === discovered.catalog.id, + "exact_reviewed_draft", + ); + const published = await admin.integration.publishMcp({ + connectionId: created.connection.id, + draftId: discovered.catalog.id, + expectedRevision: current.catalogRevision, + tools: reviewed, + }); + requireThat( + published.catalog?.kind === "published" && + published.catalog.tools.length === 2, + "published_catalog", + ); + const enabled = await admin.integration.setConnectionEnabled({ + id: created.connection.id, + expectedRevision: authorizedConnection.configRevision, + enabled: true, + }); + requireThat( + enabled.connection?.id === created.connection.id && + enabled.connection.configRevision === + authorizedConnection.configRevision && + enabled.connection.enabled, + "enabled_connection", + ); + return { connection: enabled.connection, catalog: published.catalog }; +} + +/** Stable compiled name from actual public connection identity; verify against public binding before prompt. */ +export function modelToolName( + connectionId: string, + alias: string, + upstreamName: string, +) { + requireThat( + /^[a-z][a-z0-9-]{1,62}$/.test(alias) && + connectionId.length > 0 && + ["read_counter", "write_counter"].includes(upstreamName), + "actual_connection_alias", + ); + return `mcp_${alias.replaceAll("-", "_").slice(0, 24)}_${sha256(JSON.stringify([connectionId, upstreamName])).slice(0, 20)}`; +} +export async function bindingTools(client: PublicClient, sessionId: string) { + let cursor = ""; + const seen = new Set(); + const tools = []; + for (let page = 0; page < 32; page++) { + const state = await client.integration.getSessionMcpState({ + sessionId, + omitExecutions: true, + omitBindings: false, + bindingPageSize: 16, + afterBindingToolName: cursor, + }); + requireThat(state.bindingReceipt?.current, "current_binding"); + tools.push(...state.bindingTools); + requireThat( + tools.length <= 128 && + new Set(tools.map((tool) => tool.modelToolName)).size === tools.length, + "binding_tool_set_bound", + ); + if (!state.hasMoreBindingTools) return tools; + requireThat( + state.nextBindingToolName && !seen.has(state.nextBindingToolName), + "binding_cursor", + ); + seen.add(state.nextBindingToolName); + cursor = state.nextBindingToolName; + } + throw new RunnerFailure("ASSERTION_FAILED", "binding_page_limit"); +} +export async function bindingTool( + client: PublicClient, + sessionId: string, + connectionId: string, + upstreamName: string, +) { + const matches = (await bindingTools(client, sessionId)).filter( + (tool) => + tool.connectionId === connectionId && + tool.reviewedTool?.upstreamName === upstreamName, + ); + requireThat(matches.length === 1, "exact_public_binding"); + return matches[0]!; +} +export async function createHumanTask( + ctx: Context, + owner: PublicClient, + profileId: string, + harness: Harness, +) { + const response = await owner.tasks.createTask({ + type: "chat", + profileId, + harness, + deferInitialPrompt: true, + title: "Fixture acceptance", + }); + requireThat( + response.task?.id && response.task.sessions.length === 1, + "fresh_task_session", + ); + const task = await owner.tasks.updateTask({ + taskId: response.task.id, + title: "Fixture acceptance", + }); + requireThat(task.task?.titleIsCustom === true, "custom_title_before_prompt"); + const sessionId = response.task.sessions[0]!.sessionId; + await waitFor( + ctx, + "session_started", + () => owner.sessions.getSession({ sessionId }), + (value) => + value.session?.status === "active" && value.session.hostId === ctx.hostId, + ); + return { taskId: response.task.id, sessionId }; +} +export async function expectRejected( + action: () => Promise, + codes: readonly Code[], +) { + try { + await action(); + } catch (error) { + requireThat( + error instanceof ConnectError && codes.includes(error.code), + "expected_public_rejection", + ); + return error.code; + } + throw new RunnerFailure("ASSERTION_FAILED", "unexpected_public_success"); +} + +export async function authorizeOAuth( + ctx: Context, + admin: PublicClient, + connectionId: string, + protocol: FixtureKind = "modern", +) { + const setup = await admin.integration.getMcpSetup({ connectionId }); + requireThat( + setup.enabled && setup.current?.connection?.mcp, + "oauth_public_setup", + ); + const connection = setup.current.connection; + const registration = setup.registrations.find( + (r) => r.id === connection.mcp!.oauthClientRegistrationId, + ); + requireThat( + registration && + registration.redirectUri === ctx.protocolFixtures[protocol].redirectUri && + registration.issuer === ctx.protocolFixtures[protocol].issuerOrigin && + registration.resource?.endpoint === + `${ctx.protocolFixtures[protocol].resourceOrigin}/mcp`, + "oauth_exact_registration", + ); + const begun = await admin.integration.authorizeMcp({ + connectionId, + expectedRevision: connection.configRevision, + expectedGeneration: connection.authGeneration, + shareWithOrgSessions: true, + }); + const authorization = new URL(begun.authorizationUrl); + requireThat( + authorization.origin === + ctx.protocolFixtures[protocol].authorizationOrigin && + authorization.pathname === "/authorize" && + authorization.searchParams.get("resource") === + `${ctx.protocolFixtures[protocol].resourceOrigin}/mcp` && + authorization.searchParams.get("redirect_uri") === + registration.redirectUri && + authorization.searchParams.get("client_id") === registration.clientId && + authorization.searchParams.get("code_challenge_method") === "S256" && + /^[A-Za-z0-9_-]{43}$/.test( + authorization.searchParams.get("code_challenge") ?? "", + ), + "oauth_exact_authorization", + ); + const state = authorization.searchParams.get("state"); + requireThat(state && state.length <= 256, "oauth_state"); + const response = await jsonRequest( + ctx, + { ...ctx.protocolFixtures[protocol], origin: authorization.origin }, + "GET", + authorization.pathname + authorization.search, + ); + requireThat( + response.status === 302 && response.headers.location, + "oauth_fixture_redirect", + ); + const callback = new URL(response.headers.location); + requireThat( + callback.origin + callback.pathname === registration.redirectUri && + !callback.hash && + callback.searchParams.get("state") === state && + callback.searchParams.get("iss") === registration.issuer, + "oauth_exact_callback", + ); + const code = callback.searchParams.get("code"); + requireThat(code && code.length <= 512, "oauth_code"); + const completed = await admin.integration.completeMcpAuthorization({ + connectionId, + expectedRevision: begun.configRevision, + expectedGeneration: begun.authGeneration, + flowId: begun.flowId, + state, + code, + issuer: registration.issuer, + redirectUri: registration.redirectUri, + }); + requireThat( + completed.connection?.id === connectionId && + completed.credential?.kind === "oauth", + "oauth_completed", + ); + return completed.connection; +} + +export async function createFixtureProfile( + ctx: Context, + admin: PublicClient, + input: { + harness: Harness; + suffix: string; + connectionId: string; + connectionAlias: string; + upstreamName: "read_counter" | "write_counter"; + mode: "read" | "write" | "denied" | "refire" | "held-call" | "forged"; + arguments: Record; + callId: string; + expectedNativeResult?: Record; + callPlan?: CallPlan; + completionBarrier?: boolean; + prematureBarrier?: boolean; + grant: boolean; + automationWrite: boolean; + constraints: object[]; + defaultAllow: boolean; + }, +) { + requireThat(/^[a-z0-9-]{1,40}$/.test(input.suffix), "profile_suffix"); + const alias = modelToolName( + input.connectionId, + input.connectionAlias, + input.upstreamName, + ); + const proof = `${ctx.guest.stateDirectory}/${input.suffix}-{launch}.json`; + const argumentsJson = JSON.stringify(input.arguments); + requireThat( + Buffer.byteLength(argumentsJson) <= 4096 && + /^[A-Za-z0-9_-]{1,64}$/.test(input.callId), + "fixture_call_bounds", + ); + const expectedJson = + input.expectedNativeResult === undefined + ? undefined + : JSON.stringify(input.expectedNativeResult); + requireThat( + expectedJson === undefined || + Buffer.byteLength(expectedJson) <= 1024 * 1024, + "expected_native_result_bounds", + ); + requireThat( + !input.callPlan?.steps.some( + (step) => step.completionProjection === "premature", + ) || input.prematureBarrier === true, + "premature_barrier_required", + ); + const response = await admin.profiles.createProfile({ + name: `Fixture ${input.suffix}`, + imageId: ctx.imageId, + harness: input.harness, + modelRouter: "openrouter", + model: "z-ai/glm-5.2", + expectedMcpGrantRevision: 0n, + envVars: { + ENGRAM_FIXTURE_MODE: input.mode, + ...(input.callPlan + ? { ENGRAM_FIXTURE_CALL_PLAN_JSON: encodeCallPlan(input.callPlan) } + : { + ENGRAM_FIXTURE_TOOL: alias, + ENGRAM_FIXTURE_ARGUMENTS_JSON: argumentsJson, + ENGRAM_FIXTURE_CALL_ID: input.callId, + ...(expectedJson === undefined + ? {} + : { ENGRAM_FIXTURE_EXPECTED_RESULT_JSON: expectedJson }), + }), + ENGRAM_FIXTURE_STATE_DIR: ctx.guest.stateDirectory, + ...(input.completionBarrier + ? { + ENGRAM_FIXTURE_COMPLETION_BARRIER: `${ctx.guest.stateDirectory}/completion-release`, + } + : {}), + ...(input.prematureBarrier + ? { + ENGRAM_FIXTURE_PREMATURE_BARRIER: `${ctx.guest.stateDirectory}/premature-release`, + } + : {}), + ENGRAM_FIXTURE_PROOF: proof, + [input.harness === "claude" ? "ENGRAM_CLAUDE_BIN" : "ENGRAM_CODEX_BIN"]: + ctx.guest.cliPath, + }, + network: { default: input.defaultAllow ? "allow" : "deny" }, + integrationGrants: input.grant + ? [ + { + connectionId: input.connectionId, + operation: "invoke", + credentialScope: "org", + mcp: { + upstreamName: input.upstreamName, + automationWrite: input.automationWrite, + unrestricted: input.constraints.length === 0, + argumentConstraintDocumentsJson: input.constraints.map((v) => + JSON.stringify(v), + ), + }, + }, + ] + : [], + }); + requireThat( + response.profile?.id && + response.profile.harness === input.harness && + response.profile.modelRouter === "openrouter" && + response.profile.model === "z-ai/glm-5.2" && + response.profile.mcpGrantRevision === 1n, + "published_fixture_profile", + ); + return { + profileId: response.profile.id, + alias, + proof, + planSha256: input.callPlan + ? `sha256:${sha256(canonicalJson(input.callPlan))}` + : undefined, + }; +} diff --git a/orchestrator/scripts/mcp-stack/call-plan.ts b/orchestrator/scripts/mcp-stack/call-plan.ts new file mode 100644 index 000000000..2d785861c --- /dev/null +++ b/orchestrator/scripts/mcp-stack/call-plan.ts @@ -0,0 +1,83 @@ +import { z } from "zod"; +import { requireThat } from "./context.ts"; +import { canonicalJson } from "./guest.ts"; + +const id = z.string().regex(/^[A-Za-z0-9_-]{1,64}$/); +export const CallPlanSchema = z.strictObject({ + steps: z + .array( + z.strictObject({ + marker: id, + tool: z.string().regex(/^[A-Za-z0-9_]{1,128}$/), + arguments: z.record(z.string(), z.unknown()), + callId: id, + resultFormat: z.enum(["mcp", "json"]), + expectedResult: z.record(z.string(), z.unknown()).optional(), + expectExposure: z.enum(["present", "absent"]).optional(), + completionProjection: z.literal("premature").optional(), + }), + ) + .min(1) + .max(16), +}); +export type CallPlan = z.infer; +function depth(value: unknown, current = 0): number { + if (value === null || typeof value !== "object") return current; + return Math.max( + current + 1, + ...Object.values(value).map((child) => depth(child, current + 1)), + ); +} +export function encodeCallPlan(input: CallPlan) { + const plan = CallPlanSchema.parse(input); + requireThat( + new Set(plan.steps.map((step) => step.marker)).size === plan.steps.length, + "call_plan_unique_identity", + ); + for (const callId of new Set(plan.steps.map((step) => step.callId))) { + const pair = plan.steps.filter((step) => step.callId === callId); + if (pair.length === 1) continue; + const [present, absent] = pair; + requireThat( + pair.length === 2 && + present && + absent && + present.expectExposure !== "absent" && + absent.expectExposure === "absent" && + present.resultFormat === "mcp" && + absent.resultFormat === "mcp" && + present.tool.startsWith("mcp_") && + present.tool === absent.tool && + canonicalJson(present.arguments) === canonicalJson(absent.arguments) && + present.completionProjection === undefined && + absent.completionProjection === undefined && + absent.expectedResult === undefined, + "call_plan_duplicate_rebind_pair", + ); + } + for (const step of plan.steps) { + requireThat( + step.expectExposure !== "absent" || + (step.resultFormat === "mcp" && + step.tool.startsWith("mcp_") && + step.expectedResult === undefined && + step.completionProjection === undefined), + "call_plan_absent_contract", + ); + requireThat( + step.completionProjection === undefined || + (step.resultFormat === "mcp" && + step.expectExposure !== "absent" && + step.expectedResult === undefined), + "call_plan_premature_contract", + ); + requireThat( + Buffer.byteLength(JSON.stringify(step.arguments)) <= 4096 && + depth(step.arguments) <= 16, + "call_plan_argument_bounds", + ); + } + const encoded = JSON.stringify(plan); + requireThat(Buffer.byteLength(encoded) <= 65536, "call_plan_size"); + return encoded; +} diff --git a/orchestrator/scripts/mcp-stack/case-approval.ts b/orchestrator/scripts/mcp-stack/case-approval.ts new file mode 100644 index 000000000..4e638a84a --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-approval.ts @@ -0,0 +1,146 @@ +import { requireThat, sha256, type Context } from "./context.ts"; +import { canonicalJson, expectedCounterNative } from "./guest.ts"; +import type { Harness } from "./manifest.ts"; +import { DatabaseObserver, FixtureObserver } from "./observer.ts"; +import { + caseBodyFailure, + completeCaseCleanup, + approveOnce, + closeScenario, + delivered, + execution, + sendScenarioPrompt, + startScenario, + verifiedCounterResult, + type Principals, +} from "./scenario-runtime.ts"; + +export async function interactiveApproval( + ctx: Context, + principals: Principals, + database: DatabaseObserver, + harness: Harness, +) { + const observations = []; + for (const actor of ["owner", "admin"] as const) { + const fixture = new FixtureObserver(ctx); + await fixture.configure({ + reply: "json", + response: "normal", + holdCalls: false, + revision: 1, + }); + const before = await fixture.observe(); + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-approval-${actor}`, + protocol: "modern", + upstreamName: "write_counter", + mode: "write", + caller: "owner", + automationWrite: false, + arguments: { delta: 1 }, + constraints: [ + { + type: "object", + properties: { delta: { const: 1 } }, + required: ["delta"], + additionalProperties: false, + }, + ], + callId: "fixture-write", + expectedNativeResult: expectedCounterNative( + harness, + before.effects.count + 1, + false, + ), + }); + try { + await sendScenarioPrompt(scenario); + const approval = await approveOnce(ctx, principals, scenario, actor); + const call = await execution(ctx, scenario); + requireThat( + call?.executionId === approval.executionId, + "approved_execution_identity", + ); + const result = await verifiedCounterResult( + scenario, + call.executionId, + before.effects.count + 1, + false, + ); + const completion = await delivered(ctx, database, scenario); + const after = await fixture.observe(); + requireThat( + after.effects.executions === before.effects.executions + 1 && + after.effects.count === before.effects.count + 1, + "one_approved_write_effect", + ); + observations.push({ + protocol: ctx.protocolFixtures.modern.protocol, + actor, + approvalContextHash: approval.contextHash, + rejections: approval.rejections, + resultHash: result.resultHash, + proofHashes: completion.proofHashes, + acknowledged: completion.outbox.acknowledged, + beforeHash: sha256(canonicalJson(before)), + afterHash: sha256(canonicalJson(after)), + }); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, scenario); + }); + } + } + requireThat(observations.length === 2, "owner_and_admin_approvals"); + return observations; +} + +export const interactiveApprovalDriver: import("./driver.ts").CaseDriver = + async ({ ctx, principals, database, harness }) => { + const { completeAssertions } = await import("./driver.ts"); + const rows = await interactiveApproval(ctx, principals, database, harness); + const views = rows.map((row) => ({ + source: "public-rpc" as const, + identitySha256: row.approvalContextHash, + state: "authorized" as const, + contentSha256: sha256( + canonicalJson({ actor: row.actor, resultHash: row.resultHash }), + ), + })); + const effects = rows.map((row) => ({ + source: "fixture-observer" as const, + identitySha256: row.approvalContextHash, + state: "succeeded" as const, + contentSha256: sha256( + canonicalJson({ before: row.beforeHash, after: row.afterHash }), + ), + count: 1, + })); + const denied = (field: "otherDenied" | "changedDenied" | "replayDenied") => + rows.map((row) => ({ + source: "public-rpc" as const, + identitySha256: row.approvalContextHash, + state: "denied" as const, + contentSha256: sha256( + canonicalJson({ field, code: row.rejections[field] }), + ), + })); + return completeAssertions("interactive-approval", [ + { assertion: "owner-one-use", observations: [views[0]!] }, + { assertion: "admin-one-use", observations: [views[1]!] }, + { assertion: "other-member-denied", observations: denied("otherDenied") }, + { + assertion: "changed-context-denied", + observations: denied("changedDenied"), + }, + { + assertion: "replayed-decision-denied", + observations: denied("replayDenied"), + }, + { assertion: "one-effect", observations: effects }, + ]); + }; diff --git a/orchestrator/scripts/mcp-stack/case-controls.test.ts b/orchestrator/scripts/mcp-stack/case-controls.test.ts new file mode 100644 index 000000000..155298cc2 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-controls.test.ts @@ -0,0 +1,92 @@ +import { expect, test } from "bun:test"; +import { + assertHistoryPreserved, + cancellationRequest, + requireKillVariants, + assertKillPublication, +} from "./case-controls.ts"; + +test("verified history must retain exact state, result and availability", () => { + const history = { + state: "succeeded", + resultUnavailable: false, + resultJson: '{"outcome":"succeeded"}', + }; + expect(() => assertHistoryPreserved(history, history)).not.toThrow(); + for (const after of [ + { ...history, state: "canceled" }, + { ...history, resultUnavailable: true }, + { ...history, resultJson: "{}" }, + ]) + expect(() => assertHistoryPreserved(history, after)).toThrow(); +}); +test("cancellation uses current exact journal revision", () => { + expect( + cancellationRequest({ + executionId: "execution", + journalRevision: 9, + state: "dispatching", + }), + ).toEqual({ executionId: "execution", expectedJournalRevision: 9 }); + expect(() => + cancellationRequest({ + executionId: "execution", + journalRevision: 0, + state: "succeeded", + }), + ).toThrow(); +}); +test("only real orchestrator switches satisfy mandatory kill prerequisites", () => { + const complete = { + configurations: { + orchestrator: { normal: {}, "global-disabled": {}, "zone-disabled": {} }, + coordinator: { normal: {} }, + }, + operatorCandidates: { normal: {} }, + }; + expect(() => requireKillVariants(complete)).not.toThrow(); + const wrongRole = { + configurations: { + orchestrator: { normal: {} }, + coordinator: { normal: {}, "global-disabled": {}, "zone-disabled": {} }, + }, + operatorCandidates: { + normal: {}, + "global-disabled": {}, + "zone-disabled": {}, + }, + }; + expect(() => requireKillVariants(wrongRole)).toThrow("NEEDS_CONTEXT"); + expect(() => + requireKillVariants({ + configurations: { orchestrator: { normal: {} } }, + }), + ).toThrow("NEEDS_CONTEXT"); +}); +test("kill cannot republish or change the actual operator generation", () => { + const before = { + admission_open: true, + pending: null, + published: { + generation: 1, + identity: `sha256:${"a".repeat(64)}`, + candidate: { + deployment_identity: "deployment", + network_identity: "network", + trust_identity: "trust", + }, + }, + }; + expect(() => + assertKillPublication(before, structuredClone(before)), + ).not.toThrow(); + expect(() => + assertKillPublication(before, { + ...before, + published: { ...before.published, generation: 2 }, + }), + ).toThrow(); + expect(() => + assertKillPublication(before, { ...before, admission_open: false }), + ).toThrow(); +}); diff --git a/orchestrator/scripts/mcp-stack/case-controls.ts b/orchestrator/scripts/mcp-stack/case-controls.ts new file mode 100644 index 000000000..faa083ff6 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-controls.ts @@ -0,0 +1,359 @@ +import { readOperatorPublication, waitFor } from "./bootstrap.ts"; +import { canonicalJson } from "./guest.ts"; +import { requireThat, RunnerFailure } from "./context.ts"; +import { + completeAssertions, + type CaseDriver, + type DriverAssertion, +} from "./driver.ts"; +import { + caseBodyFailure, + completeCaseCleanup, + approveOnce, + closeScenario, + execution, + pollProof, + sendScenarioPrompt, + startScenario, + verifiedCounterResult, +} from "./scenario-runtime.ts"; +import { Supervisor } from "./supervisor.ts"; +import { deltaOne, deniedCall, policyObservation } from "./case-grants.ts"; + +export function requireKillVariants(supervisor: { + configurations: { orchestrator: object }; +}) { + for (const variant of ["normal", "global-disabled", "zone-disabled"]) + if (!(variant in supervisor.configurations.orchestrator)) + throw new RunnerFailure( + "NEEDS_CONTEXT", + `supervisor_orchestrator_${variant}_configuration`, + ); +} +export function assertKillPublication( + before: Awaited>, + after: Awaited>, +) { + requireThat( + before.admission_open && + before.pending === null && + before.published && + canonicalJson(before) === canonicalJson(after), + "kill_preserves_actual_operator_publication", + ); +} +export function cancellationRequest(call: { + executionId: string; + journalRevision: number; + state: string; +}) { + requireThat( + call.executionId.length > 0 && + Number.isSafeInteger(call.journalRevision) && + call.journalRevision > 0 && + call.state === "dispatching", + "current_dispatching_cancellation_revision", + ); + return { + executionId: call.executionId, + expectedJournalRevision: call.journalRevision, + }; +} +export function assertHistoryPreserved( + before: { state: string; resultUnavailable: boolean; resultJson?: string }, + after: typeof before, +) { + requireThat( + before.state === "succeeded" && + !before.resultUnavailable && + before.resultJson && + after.state === before.state && + after.resultUnavailable === before.resultUnavailable && + after.resultJson === before.resultJson, + "verified_history_preserved", + ); +} + +export const killCancelDriver: CaseDriver = async (input) => { + const { ctx, principals, database, harness } = input; + requireKillVariants(ctx.supervisor); + const supervisor = new Supervisor(ctx); + const assertions: DriverAssertion[] = []; + const historyObservations = []; + const deniedObservations = []; + for (const control of [ + "global", + "zone", + "connection", + "tool", + "cancel", + ] as const) { + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-control-${control}`, + protocol: "modern", + upstreamName: "write_counter", + mode: "write", + caller: "owner", + automationWrite: false, + arguments: { delta: 1 }, + constraints: [deltaOne], + callId: "control-history", + callPlan: (alias) => ({ + steps: [ + { + marker: "history", + tool: alias, + arguments: { delta: 1 }, + callId: "control-history", + resultFormat: "mcp", + }, + { + marker: "held", + tool: alias, + arguments: { delta: 1 }, + callId: "control-held", + resultFormat: "mcp", + }, + { + marker: "blocked", + tool: alias, + arguments: { delta: 1 }, + callId: "control-blocked", + resultFormat: "mcp", + }, + ], + }), + }); + let variantChanged = false; + let publicationBefore: + Awaited> | undefined; + try { + await scenario.fixture.configure({ + reply: "json", + response: "normal", + holdCalls: false, + revision: 1, + }); + const before = await scenario.fixture.observe(); + await sendScenarioPrompt(scenario, false, "history"); + await approveOnce(ctx, principals, scenario, "owner"); + const first = await execution(ctx, scenario); + requireThat( + first?.state === "succeeded", + "control_verified_history_exists", + ); + await verifiedCounterResult( + scenario, + first.executionId, + before.effects.count + 1, + false, + ); + await pollProof(ctx, scenario, { + callId: "control-history", + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + const history = await principals.admin.integration.getMcpExecutionResult({ + sessionId: scenario.sessionId, + executionId: first.executionId, + }); + // Commit a real write, then keep its response pending while the control changes. + await scenario.fixture.configure({ holdCalls: true }); + await sendScenarioPrompt(scenario, false, "held"); + await approveOnce( + ctx, + principals, + { ...scenario, input: { ...scenario.input, callId: "control-held" } }, + "owner", + ); + const held = await waitFor( + ctx, + "control_held_effect", + () => scenario.fixture.observe(), + (value) => + value.held === 1 && + value.effects.executions === before.effects.executions + 2, + ); + requireThat( + held.effects.count === before.effects.count + 2, + "held_write_committed_before_control", + ); + const inflight = await execution(ctx, scenario, false, "control-held"); + requireThat( + inflight?.state === "dispatching", + "actual_inflight_control_target", + ); + let mutation: unknown; + if (control === "global" || control === "zone") { + publicationBefore = await readOperatorPublication(ctx); + requireThat( + publicationBefore.published?.identity === + ctx.publication.candidateIdentity, + "kill_normal_publication", + ); + variantChanged = true; + const variant = + control === "global" ? "global-disabled" : "zone-disabled"; + const status = await supervisor.restart("orchestrator", variant); + const publication = await readOperatorPublication(ctx); + assertKillPublication(publicationBefore, publication); + mutation = { configuration: variant, roles: status.roles, publication }; + } else if (control === "connection") { + const setup = await principals.admin.integration.getMcpSetup({ + connectionId: scenario.connection.id, + }); + requireThat( + setup.current?.connection, + "connection_kill_current_revision", + ); + const killed = await principals.admin.integration.setConnectionEnabled({ + id: scenario.connection.id, + enabled: false, + expectedRevision: setup.current.connection.configRevision, + }); + requireThat( + killed.connection?.enabled === false, + "connection_kill_applied", + ); + mutation = { + revision: killed.connection.configRevision, + enabled: killed.connection.enabled, + }; + } else if (control === "tool") { + const setup = await principals.admin.integration.getMcpSetup({ + connectionId: scenario.connection.id, + }); + requireThat(setup.current?.connection, "tool_kill_current_connection"); + const discovery = await principals.admin.integration.discoverMcp({ + connectionId: scenario.connection.id, + expectedRevision: setup.current.connection.configRevision, + }); + requireThat(discovery.catalog, "tool_kill_reviewed_draft"); + const catalogs = await principals.admin.integration.getMcpCatalogs({ + connectionId: scenario.connection.id, + }); + const published = await principals.admin.integration.publishMcp({ + connectionId: scenario.connection.id, + draftId: discovery.catalog.id, + expectedRevision: catalogs.catalogRevision, + tools: discovery.catalog.tools.map((tool) => ({ + ...tool, + classification: + tool.upstreamName === "read_counter" ? "read" : "write", + enabled: tool.upstreamName !== "write_counter", + })), + }); + requireThat( + published.catalog?.tools.find( + (tool) => tool.upstreamName === "write_counter", + )?.enabled === false, + "tool_kill_applied", + ); + mutation = { + revision: published.catalog.revision, + fingerprint: published.catalog.fingerprint, + }; + } else { + const canceled = await principals.owner.integration.cancelMcpCall( + cancellationRequest(inflight), + ); + requireThat( + canceled.receipt?.executionId === inflight.executionId && + canceled.receipt.expectedJournalRevision === + inflight.journalRevision && + canceled.receipt.actorId === principals.actorIds.owner, + "actual_cancel_receipt", + ); + mutation = { + state: canceled.state, + revision: canceled.receipt.journalRevision, + cause: canceled.receipt.cause, + }; + } + const terminal = await execution(ctx, scenario, true, "control-held"); + requireThat( + terminal && ["canceled", "outcome_unknown"].includes(terminal.state), + "control_interrupts_held_call", + ); + const aborted = await waitFor( + ctx, + "control_upstream_abort", + () => scenario.fixture.observe(), + (value) => value.held === 0 && value.cancellations > held.cancellations, + ); + requireThat( + aborted.effects.executions === held.effects.executions && + aborted.effects.count === held.effects.count, + "control_does_not_undo_committed_effect", + ); + const afterHistory = + await principals.admin.integration.getMcpExecutionResult({ + sessionId: scenario.sessionId, + executionId: first.executionId, + }); + assertHistoryPreserved(history, afterHistory); + historyObservations.push( + policyObservation("public-rpc", first.executionId, "succeeded", { + state: afterHistory.state, + result: afterHistory.resultJson, + }), + ); + const denied = await deniedCall( + input, + scenario, + "blocked", + "control-blocked", + ); + deniedObservations.push(...denied); + assertions.push({ + assertion: control === "cancel" ? "cancel" : `${control}-kill`, + observations: [ + policyObservation( + "public-rpc", + inflight.executionId, + terminal.state === "canceled" ? "canceled" : "outcome_unknown", + mutation, + ), + policyObservation( + "fixture-observer", + scenario.sessionId, + "canceled", + { held, aborted }, + aborted.cancellations - held.cancellations, + ), + ], + }); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + try { + await scenario.fixture.release(); + await scenario.fixture.configure({ holdCalls: false }); + } finally { + try { + if (variantChanged) { + await supervisor.restart("orchestrator", "normal"); + requireThat(publicationBefore, "kill_publication_baseline"); + assertKillPublication( + publicationBefore, + await readOperatorPublication(ctx), + ); + } + } finally { + await closeScenario(principals, scenario); + } + } + }); + } + } + assertions.push( + { + assertion: "journal-history-preserved", + observations: historyObservations, + }, + { assertion: "no-new-dispatch", observations: deniedObservations }, + ); + return completeAssertions("kill-cancel", assertions); +}; diff --git a/orchestrator/scripts/mcp-stack/case-fidelity.test.ts b/orchestrator/scripts/mcp-stack/case-fidelity.test.ts new file mode 100644 index 000000000..1644c6f51 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-fidelity.test.ts @@ -0,0 +1,161 @@ +import { expect, test } from "bun:test"; + +// Isolate substitutes from the other driver tests; exercise the real call, +// journal, proof selection, outbox ACK and native completion helpers. +const exercise = String.raw` +import { mock } from "bun:test"; +import { strict as assert } from "node:assert"; +const runtime = await import("./scenario-runtime.ts"); +const bootstrap = await import("./bootstrap.ts"); +const guest = await import("./guest.ts"); +const observer = await import("./observer.ts"); +const { sha256 } = await import("./context.ts"); +const fault = process.env.FIDELITY_FAULT; +const ctx = { deadlines: { stepMs: 1 }, protocolFixtures: { + modern: { protocol: "modern" }, legacy: { protocol: "legacy" } +} }; +let boots = 0, cleanups = 0, reads = 0, configuredError = false; +const calls = [], acknowledgements = [], events = [], turns = [], completedTurns = [], proofs = [], setups = []; +let active; +const snapshot = () => ({ effects: { count: 7, executions: 0 }, reads }); +mock.module("./observer.ts", () => ({ ...observer, FixtureObserver: class { + async configure(input) { + if (input.response === "error") { + assert.equal(acknowledgements.at(-1), "fixture-read-ok"); + assert.equal(events.at(-1).tool_call_id, "fixture-read-ok"); + assert.equal(completedTurns.at(-1), "fixture-read-ok"); + } + configuredError = input.response === "error"; + } + async observe() { return snapshot(); } +} })); +mock.module("./bootstrap.ts", () => ({ ...bootstrap, + bindingTools: async () => [{ modelToolName: active.alias }], +})); +mock.module("./guest.ts", () => ({ ...guest, + proofPaths: async () => ["proof"], + readGuestProofRecords: async () => proofs, +})); +mock.module("./scenario-runtime.ts", () => ({ ...runtime, + async startScenario(ctx, principals, database, input) { + boots++; + const alias = "mcp_fixture_read"; + const plan = input.callPlan?.(alias, { id: "connection", alias: "fixture" }); + assert.equal(plan?.steps.length, 2, "two actual calls per boot"); + assert.notEqual(plan.steps[0].marker, plan.steps[1].marker); + assert.notEqual(plan.steps[0].callId, plan.steps[1].callId); + setups.push([input.harness, input.protocol, input.suffix]); + proofs.length = 0; calls.length = 0; events.length = 0; turns.length = 0; + active = { ctx, input, alias, connection: { id: "connection" }, + planSha256: "sha256:" + sha256(guest.canonicalJson(plan)), + sessionId: "session-" + boots, client: {}, + catalog: { tools: [{ upstreamName: "read_counter", inputSchemaJson: "{}" }] }, + reviewClient: { integration: { + async getSessionMcpState() { return { executions: calls, hasMoreExecutions: false }; }, + async getMcpExecutionResult({ executionId }) { + const row = calls.find(call => call.executionId === executionId); + return { state: row.state, resultUnavailable: false, resultJson: JSON.stringify({ + outcome: row.state, delivery: "available", output: { + content: [{ type: "text", text: "count=7" }], + structuredContent: { count: 7 }, isError: row.state === "failed", + } + }) }; + } + }, sessions: { async listSessionEvents({ kinds }) { + if (kinds[0] === "run_completed") { + completedTurns.push(calls.at(-1).toolCallId); + return { events: turns.map((event, idx) => ({ kind: "run_completed", idx: BigInt(idx * 2 + 1), payloadJson: JSON.stringify(event) })) }; + } + return { events: events.map((event, idx) => ({ kind: "tool_call_completed", idx: BigInt(idx * 2), payloadJson: JSON.stringify(event) })) }; + } } }, plan }; + return active; + }, + async sendScenarioPrompt(scenario, planMode, marker) { + const step = active.plan.steps.find(step => step.marker === marker); + assert(step, "dispatch selects a real plan marker"); + assert.equal(step.resultFormat, "mcp"); + assert.equal(step.callId, scenario.input.callId); + assert.equal(step.tool, scenario.alias); + assert.deepEqual(step.arguments, {}); + assert.deepEqual(step.expectedResult, guest.expectedCounterNative(scenario.input.harness, 7, configuredError)); + if (fault === "first-call" && !configuredError) throw new Error("first-call-failed"); + if (fault === "second-call" && configuredError) throw new Error("second-call-failed"); + reads++; + calls.push({ toolCallId: step.callId, connectionId: "connection", executionId: "execution-" + reads, state: configuredError ? "failed" : "succeeded" }); + const hash = "sha256:" + sha256(guest.canonicalJson(step.expectedResult)); + const proof = { callId: step.callId, tool: step.tool, planSha256: active.planSha256, + resultReceived: true, selectedToolName: step.tool, + selectedSchemaSha256: "sha256:" + sha256("{}"), + exposedMcpToolNamesSha256: "sha256:" + sha256(guest.canonicalJson([step.tool])), + delivery: "available", resultMatchesExpected: true, resultSha256: hash, expectedResultSha256: hash }; + if (!(configuredError && fault === "old-proof")) + proofs.push({ proof: configuredError && fault === "wrong-plan" ? { ...proof, planSha256: "sha256:" + "0".repeat(64) } : proof, sha256: sha256(guest.canonicalJson(proof)) }); + if (!(configuredError && fault === "old-completion")) + events.push({ tool_call_id: step.callId, tool_name: scenario.input.harness === "claude" ? "mcp__engrams__" + step.tool : step.tool, run_id: step.callId, ok: !configuredError }); + if (!(configuredError && fault === "old-turn")) + turns.push({ run_id: step.callId, ok: true }); + }, + async closeScenario() { cleanups++; } +})); +const database = { + async outbox(sessionId, callId) { + acknowledgements.push(callId); + return { acknowledged: !(configuredError && fault === "old-ack"), relayed: true, attempts: 1 }; + }, + async deliveryTarget() { return { targetGone: false }; } +}; +const { catalogFidelity } = await import("./case-fidelity.ts"); +for (const harness of ["claude", "codex"]) { + try { + const rows = await catalogFidelity(ctx, {}, database, harness); + assert.equal(fault, "none", "fault must fail the driver"); + assert.equal(rows.length, 8); + assert.deepEqual(rows.map(row => [row.protocol, row.reply, row.isError]), + ["modern", "legacy"].flatMap(protocol => ["json", "sse"].flatMap(reply => [false, true].map(isError => [protocol, reply, isError])))); + assert(rows.every(row => row.proofHashes.length === 1 && row.acknowledged)); + } catch (error) { + if (fault === "none") throw error; + assert(error instanceof runtime.CaseBodyFailure); + const seam = { "first-call": "first-call-failed", "second-call": "second-call-failed", "old-proof": "native_completed_proof_fixture-read-error", "wrong-plan": "native_completed_proof_fixture-read-error", "old-completion": "actual_native_completion_event", "old-ack": "actual_guest_ack", "old-turn": "native_turn_completed" }[fault]; + assert(error.failure.message.includes(seam), String(error.failure)); + assert.equal(cleanups, 1); + console.log("fault verified"); + process.exit(0); + } +} +assert.equal(boots, 8); +assert.equal(cleanups, 8); +assert.equal(reads, 16); +assert.equal(new Set(setups.map(([h, p, s]) => h + p + s)).size, 8); +assert.deepEqual(acknowledgements, Array.from({ length: 8 }, () => ["fixture-read-ok", "fixture-read-error"]).flat()); +console.log("eight boots and sixteen real planned calls verified"); +`; + +for (const fault of [ + "none", + "first-call", + "second-call", + "old-proof", + "wrong-plan", + "old-completion", + "old-ack", + "old-turn", +]) { + test(`fidelity pairs retain call identity, delivery and cleanup: ${fault}`, async () => { + const child = Bun.spawn([process.execPath, "--eval", exercise], { + cwd: import.meta.dir, + env: { ...process.env, FIDELITY_FAULT: fault }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exit] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + expect({ exit, stderr }).toEqual({ exit: 0, stderr: "" }); + expect(stdout).toContain( + fault === "none" ? "sixteen real" : "fault verified", + ); + }); +} diff --git a/orchestrator/scripts/mcp-stack/case-fidelity.ts b/orchestrator/scripts/mcp-stack/case-fidelity.ts new file mode 100644 index 000000000..9fcce1e58 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-fidelity.ts @@ -0,0 +1,267 @@ +import { + requireThat, + sha256, + type Context, + type FixtureKind, +} from "./context.ts"; +import { canonicalJson, expectedCounterNative } from "./guest.ts"; +import type { Harness } from "./manifest.ts"; +import { z } from "zod"; +import { bindingTools, waitFor } from "./bootstrap.ts"; +import { DatabaseObserver, FixtureObserver } from "./observer.ts"; +import { + caseBodyFailure, + completeCaseCleanup, + closeScenario, + delivered, + execution, + pollProof, + sendScenarioPrompt, + startScenario, + verifiedCounterResult, + type Principals, + type StartedScenario, +} from "./scenario-runtime.ts"; + +/** A tool ACK can precede the end of its native turn. */ +async function waitForNativeTurn( + ctx: Pick, + scenario: StartedScenario, + nativeCompletions: Awaited>["completion"], +) { + requireThat( + nativeCompletions.length > 0, + "native_turn_requires_tool_completion", + ); + return waitFor( + ctx, + "native_turn_completed", + async () => { + const page = await scenario.reviewClient.sessions.listSessionEvents({ + sessionId: scenario.sessionId, + kinds: ["run_completed"], + limit: 129n, + }); + requireThat(page.events.length < 129, "native_turn_event_bound"); + return page.events.some((event) => { + const idx = event.idx; + requireThat( + event.kind === "run_completed" && idx !== undefined, + "native_turn_event_identity", + ); + const payload = z + .object({ run_id: z.string(), ok: z.boolean() }) + .parse(JSON.parse(event.payloadJson)); + return ( + payload.ok && + nativeCompletions.every( + (native) => + payload.run_id === native.run_id && idx > BigInt(native.idx), + ) + ); + }); + }, + (finished) => finished, + ); +} + +/** Actual journal, fixture and guest observations; none is a supplied success receipt. */ +export async function catalogFidelity( + ctx: Context, + principals: Principals, + database: DatabaseObserver, + harness: Harness, +) { + const observations = []; + for (const protocol of [ + "modern", + "legacy", + ] as const satisfies readonly FixtureKind[]) { + const fixture = new FixtureObserver(ctx, protocol); + for (const reply of ["json", "sse"] as const) { + await fixture.configure({ + reply, + response: "normal", + holdCalls: false, + revision: 1, + }); + const initial = await fixture.observe(); + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-${protocol}-${reply}`, + protocol, + upstreamName: "read_counter", + mode: "read", + caller: "owner", + automationWrite: false, + arguments: {}, + constraints: [], + callId: "fixture-read-ok", + callPlan: (alias) => ({ + steps: [false, true].map((isError) => ({ + marker: isError ? "error" : "ok", + tool: alias, + arguments: {}, + callId: isError ? "fixture-read-error" : "fixture-read-ok", + resultFormat: "mcp", + expectedResult: expectedCounterNative( + harness, + initial.effects.count, + isError, + ), + })), + }), + }); + try { + requireThat(scenario.planSha256, "fidelity_call_plan_identity"); + for (const isError of [false, true]) { + const marker = isError ? "error" : "ok"; + // The normal call is fully verified and acknowledged before this switch. + if (isError) + await fixture.configure({ + reply, + response: "error", + holdCalls: false, + revision: 1, + }); + const before = await fixture.observe(); + const expectedNative = expectedCounterNative( + harness, + before.effects.count, + isError, + ); + const callScenario: StartedScenario = { + ...scenario, + input: { ...scenario.input, callId: `fixture-read-${marker}` }, + }; + await sendScenarioPrompt(callScenario, false, marker); + const call = await execution(ctx, callScenario); + requireThat(call, "fidelity_execution"); + const result = await verifiedCounterResult( + callScenario, + call.executionId, + before.effects.count, + isError, + ); + const selected = await pollProof(ctx, callScenario, { + callId: callScenario.input.callId, + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + const completion = await delivered(ctx, database, callScenario); + const reviewed = scenario.catalog.tools.find( + (tool) => tool.upstreamName === "read_counter", + ); + requireThat( + reviewed && selected.length > 0, + "fidelity_selected_tool", + ); + const bound = await bindingTools(scenario.client, scenario.sessionId); + requireThat( + bound.length === 1 && bound[0]?.modelToolName === scenario.alias, + "reviewed_only_binding_set", + ); + const expectedHash = `sha256:${sha256(canonicalJson(expectedNative))}`; + const schemaHash = `sha256:${sha256(canonicalJson(JSON.parse(reviewed.inputSchemaJson)))}`; + requireThat( + selected.every( + ({ proof }) => + proof.selectedToolName === scenario.alias && + proof.exposedMcpToolNamesSha256 === + `sha256:${sha256(canonicalJson([scenario.alias]))}` && + proof.selectedSchemaSha256 === schemaHash && + proof.delivery === "available" && + proof.resultMatchesExpected === true && + proof.resultSha256 === expectedHash && + proof.expectedResultSha256 === expectedHash, + ), + "exact_native_delivery_fidelity", + ); + requireThat( + selected.every((record) => + completion.proofHashes.includes(record.sha256), + ), + "fidelity_selected_proof_delivered", + ); + const after = await fixture.observe(); + requireThat( + after.effects.executions === before.effects.executions && + after.effects.count === before.effects.count && + after.reads === before.reads + 1, + "one_read_no_effect", + ); + await waitForNativeTurn(ctx, callScenario, completion.completion); + observations.push({ + protocol: ctx.protocolFixtures[protocol].protocol, + reply, + isError, + schemaHash, + expectedHash, + resultHash: result.resultHash, + proofHashes: selected.map((record) => record.sha256), + acknowledged: completion.outbox.acknowledged, + fixtureBeforeHash: sha256(canonicalJson(before)), + fixtureAfterHash: sha256(canonicalJson(after)), + }); + } + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, scenario); + }); + } + } + } + requireThat(observations.length === 8, "all_fidelity_variants"); + return observations; +} + +/** All five assertions derive from the eight completed real transport/protocol variants above. */ +export const catalogFidelityDriver: import("./driver.ts").CaseDriver = async ({ + ctx, + principals, + database, + harness, +}) => { + const { completeAssertions } = await import("./driver.ts"); + const rows = await catalogFidelity(ctx, principals, database, harness); + const publicViews = rows.map((row) => ({ + source: "public-rpc" as const, + identitySha256: sha256( + canonicalJson({ + protocol: row.protocol, + reply: row.reply, + isError: row.isError, + }), + ), + state: "current" as const, + contentSha256: sha256( + canonicalJson({ schemaHash: row.schemaHash, resultHash: row.resultHash }), + ), + })); + const guestViews = rows.flatMap((row) => + row.proofHashes.map((hash) => ({ + source: "guest-proof" as const, + identitySha256: sha256( + canonicalJson({ + protocol: row.protocol, + reply: row.reply, + isError: row.isError, + }), + ), + state: row.isError ? ("failed" as const) : ("succeeded" as const), + contentSha256: hash, + })), + ); + return completeAssertions("catalog-fidelity", [ + { assertion: "reviewed-tool-only-exposure", observations: guestViews }, + { + assertion: "actual-alias-schema", + observations: [...publicViews, ...guestViews], + }, + { assertion: "http-json-sse", observations: publicViews }, + { assertion: "modern-legacy", observations: publicViews }, + { assertion: "exact-text-structured-is-error", observations: guestViews }, + ]); +}; diff --git a/orchestrator/scripts/mcp-stack/case-grants.test.ts b/orchestrator/scripts/mcp-stack/case-grants.test.ts new file mode 100644 index 000000000..85fbf3f65 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-grants.test.ts @@ -0,0 +1,439 @@ +import { describe, expect, test } from "bun:test"; +import { + assertPolicyDenial, + assertPlanWriteDenial, + assertReadOnlyPolicy, + assertNoDispatch, + grantUpdateRequest, + fixedPolicyPlan, + assertPlanApprovalCompletion, + assertAbsentToolDenial, + assertNoProjectedAuthority, + assertDeltaOneConstraints, +} from "./case-grants.ts"; +import { create } from "@bufbuild/protobuf"; +import { sha256 } from "./context.ts"; +import { canonicalJson } from "./guest.ts"; +import { + GetSessionMcpStateResponseSchema, + McpExecutionReviewSchema, + McpApprovalSchema, +} from "../../src/gen/engram/app/v1/integration_pb.ts"; +import { + ProfileSchema, + UpdateProfileRequestSchema, +} from "../../src/gen/engram/app/v1/profile_pb.ts"; + +describe("policy case evidence", () => { + test("requires a received local refusal, not upstream error or missing event", () => { + expect(() => + assertPolicyDenial({ + resultReceived: true, + delivery: "available", + outcome: null, + isError: true, + }), + ).not.toThrow(); + for (const proof of [ + {}, + { resultReceived: true }, + { + resultReceived: true, + delivery: "available", + outcome: "failed", + isError: true, + }, + { + resultReceived: true, + delivery: "unavailable", + outcome: null, + isError: true, + }, + ]) + expect(() => assertPolicyDenial(proof)).toThrow(); + }); + test("requires unchanged reads, writes and count", () => { + const before = { reads: 3, effects: { count: 4, executions: 2 } }; + expect(() => assertNoDispatch(before, before)).not.toThrow(); + for (const after of [ + { ...before, reads: 4 }, + { ...before, effects: { count: 5, executions: 2 } }, + { ...before, effects: { count: 4, executions: 3 } }, + ]) + expect(() => assertNoDispatch(before, after)).toThrow(); + }); + test("grant update carries exact current revision and preserves profile fields", () => { + const profile = create(ProfileSchema, { + id: "profile", + name: "Fixture", + imageId: "image", + mcpGrantRevision: 7n, + envVars: { FIXTURE: "neutral" }, + integrationGrants: [ + { + connectionId: "connection", + operation: "invoke", + credentialScope: "org", + mcp: { upstreamName: "read_counter", unrestricted: true }, + }, + ], + }); + const request = create( + UpdateProfileRequestSchema, + grantUpdateRequest(profile, []), + ); + expect(request.expectedMcpGrantRevision).toBe(7n); + expect(request.envVars).toEqual(profile.envVars); + expect(request.integrationGrants).toEqual([]); + expect(request.imageId).toBe("image"); + }); + test("fixed child invocation has only the real native input fields", () => { + const plan = fixedPolicyPlan("mcp_fixture", "read"); + const spawn = plan.steps.find((step) => step.tool === "spawn_session")!; + expect(spawn.arguments).toEqual({ + task_name: "fixture-child", + message: "[[fixture:child]]", + idempotency_key: "fixture-child", + }); + expect(plan.steps.find((step) => step.marker === "child")?.tool).toBe( + "mcp_fixture", + ); + expect(spawn.resultFormat).toBe("json"); + }); +}); + +const nativePlanApprovals = { + claude: { + content: [ + { + type: "text", + text: "Your plan was approved. Implement it now, following the plan you presented.", + }, + ], + }, + codex: { + success: true, + contentItems: [ + { + type: "inputText", + text: "Plan approved. A fresh build turn starts next.", + }, + ], + }, +}; +for (const harness of ["claude", "codex"] as const) { + test(`${harness} plan completion requires one received native approval result`, () => { + const resultHash = (value: unknown) => + `sha256:${sha256(canonicalJson(value))}`; + const proof = { + resultReceived: true, + resultSha256: resultHash(nativePlanApprovals[harness]), + }; + expect(() => + assertPlanApprovalCompletion(harness, [{ proof }]), + ).not.toThrow(); + const rawDecision = '{"decision":"approve"}'; + const rawResult = + harness === "claude" + ? { content: [{ type: "text", text: rawDecision }] } + : { + success: true, + contentItems: [{ type: "inputText", text: rawDecision }], + }; + for (const completed of [ + [], + [{ proof }, { proof }], + [{ proof: { resultSha256: proof.resultSha256 } }], + [{ proof: { ...proof, resultReceived: false } }], + [{ proof: { ...proof, resultSha256: resultHash(rawResult) } }], + [ + { + proof: { + ...proof, + resultSha256: resultHash( + nativePlanApprovals[harness === "claude" ? "codex" : "claude"], + ), + }, + }, + ], + ]) { + expect(() => assertPlanApprovalCompletion(harness, completed)).toThrow( + "actual_generic_plan_completion", + ); + } + }); +} + +test("absent-tool proof requires the actual native unknown-tool refusal", () => { + const proof = { + expectExposure: "absent", + requestAttempted: true, + protocolDenialCode: "unknown_tool", + nativeDenialCode: -32602, + exposedMcpToolNamesSha256: "sha256:" + "1".repeat(64), + }; + expect(() => assertAbsentToolDenial("claude", proof)).not.toThrow(); + for (const changed of [ + { ...proof, requestAttempted: false }, + { ...proof, nativeDenialCode: -32603 }, + { ...proof, expectExposure: "present" }, + { ...proof, resultReceived: true }, + { ...proof, selectedSchemaSha256: "sha256:" + "2".repeat(64) }, + ]) + expect(() => assertAbsentToolDenial("claude", changed)).toThrow(); +}); + +test("guest success projection cannot become journal authority or result", () => { + const pending = { state: "waiting_approval", approvalDecision: "pending" }; + expect(() => assertNoProjectedAuthority(pending, undefined)).not.toThrow(); + expect(() => + assertNoProjectedAuthority({ ...pending, state: "canceled" }, undefined), + ).toThrow(); + for (const state of ["authorized", "dispatching", "succeeded"]) + expect(() => + assertNoProjectedAuthority({ ...pending, state }, undefined), + ).toThrow(); + expect(() => + assertNoProjectedAuthority( + { ...pending, approvalDecision: "approved" }, + undefined, + ), + ).toThrow(); + expect(() => + assertNoProjectedAuthority(pending, '{"outcome":"succeeded"}'), + ).toThrow(); +}); + +test("a real pending approval has no projected authority", () => { + expect(() => + assertNoProjectedAuthority( + { state: "waiting_approval", approvalDecision: "pending" }, + undefined, + ), + ).not.toThrow(); +}); + +test("an unknown outcome cannot prove that no dispatch occurred", () => { + expect(() => + assertNoProjectedAuthority( + { state: "outcome_unknown", approvalDecision: "" }, + undefined, + ), + ).toThrow(); +}); + +test("retired guest projection can be denied without creating authority", () => { + expect(() => + assertNoProjectedAuthority( + { + state: "canceled", + approvalDecision: "denied", + executionId: "call", + cancellationReceipt: { executionId: "call", state: "canceled" }, + }, + undefined, + ), + ).not.toThrow(); + expect(() => + assertNoProjectedAuthority( + { + state: "canceled", + approvalDecision: "denied", + executionId: "call", + cancellationReceipt: { executionId: "other", state: "canceled" }, + }, + undefined, + ), + ).toThrow(); +}); + +test("intersection retains independent repeated schema roots without widening", () => { + const schema = JSON.stringify({ + type: "object", + properties: { delta: { const: 1 } }, + required: ["delta"], + additionalProperties: false, + }); + expect(() => assertDeltaOneConstraints([schema, schema])).not.toThrow(); + expect(() => assertDeltaOneConstraints([])).toThrow(); + expect(() => + assertDeltaOneConstraints([schema, JSON.stringify({ type: "object" })]), + ).toThrow(); +}); + +test("generic plan approval requires an observed exact read-only MCP policy", () => { + expect(() => + assertReadOnlyPolicy({ writeEligibility: "read_only" }), + ).not.toThrow(); + for (const policy of [ + undefined, + { writeEligibility: "" }, + { writeEligibility: "unknown" }, + { writeEligibility: "write_eligible" }, + ]) { + expect(() => assertReadOnlyPolicy(policy)).toThrow(); + } +}); + +for (const harness of ["claude", "codex"] as const) { + test(`${harness} plan write denial requires the exact native receipt and trusted read-only grant`, () => { + const metadata = { + source: "runtime", + denial: "write_not_allowed", + delivery: "available", + isError: true, + }; + const native = { + claude: { + content: [{ type: "text", text: "External tool request denied." }], + isError: true, + _meta: { "engrams/toolResult": metadata }, + }, + codex: { + success: false, + contentItems: [ + { type: "inputText", text: "External tool request denied." }, + { + type: "inputText", + text: '{"delivery":"available","denial":"write_not_allowed","isError":true,"source":"runtime"}', + }, + ], + }, + }; + const digest = (value: unknown) => `sha256:${sha256(canonicalJson(value))}`; + const resultSha256 = digest(native[harness]); + const proof = { + resultReceived: true, + delivery: "available", + outcome: null, + isError: true, + denialPlanMode: false, + resultSha256, + expectedResultSha256: resultSha256, + resultMatchesExpected: true, + }; + const legacyNative = JSON.parse( + JSON.stringify(native[harness]).replaceAll( + "write_not_allowed", + "plan_mode", + ), + ); + const expected = { + sessionId: "session", + connectionId: "connection", + alias: "mcp_fixture", + }; + const state = create(GetSessionMcpStateResponseSchema, { + bindingReceipt: { current: true }, + availability: "available", + writePolicy: { + policy: { + current: true, + initialized: true, + writeEligibility: "read_only", + ref: { sessionId: "session" }, + }, + }, + bindingTools: [ + { + connectionId: "connection", + modelToolName: "mcp_fixture", + reviewedTool: { + upstreamName: "write_counter", + classification: "write", + enabled: true, + }, + argumentConstraintDocumentsJson: [ + '{"type":"object","properties":{"delta":{"const":1}},"required":["delta"],"additionalProperties":false}', + ], + }, + ], + }); + expect(() => + assertPlanWriteDenial(harness, [{ proof }], state, expected), + ).not.toThrow(); + for (const completed of [ + [], + [{ proof }, { proof }], + [{ proof: { ...proof, resultReceived: false } }], + [{ proof: { ...proof, delivery: "unavailable" } }], + [{ proof: { ...proof, outcome: "failed" } }], + [{ proof: { ...proof, isError: false } }], + [{ proof: { ...proof, resultSha256: undefined } }], + [{ proof: { ...proof, resultMatchesExpected: false } }], + [{ proof: { ...proof, expectedResultSha256: undefined } }], + [ + { + proof: { + ...proof, + resultSha256: digest( + native[harness === "claude" ? "codex" : "claude"], + ), + }, + }, + ], + [ + { + proof: { + ...proof, + denialPlanMode: true, + resultSha256: digest(legacyNative), + }, + }, + ], + ]) + expect(() => + assertPlanWriteDenial(harness, completed, state, expected), + ).toThrow(); + for (const change of [ + (value: typeof state) => { + value.writePolicy = undefined; + }, + (value: typeof state) => { + value.writePolicy!.policy!.writeEligibility = "write_eligible"; + }, + (value: typeof state) => { + value.writePolicy!.policy!.current = false; + }, + (value: typeof state) => { + value.writePolicy!.policy!.initialized = false; + }, + (value: typeof state) => { + value.writePolicy!.policy!.ref!.sessionId = "other"; + }, + (value: typeof state) => { + value.bindingReceipt!.current = false; + }, + (value: typeof state) => { + value.bindingTools = []; + }, + (value: typeof state) => { + value.hasMoreBindingTools = true; + }, + (value: typeof state) => { + value.bindingTools[0]!.reviewedTool!.classification = "read"; + }, + (value: typeof state) => { + value.bindingTools[0]!.unrestricted = true; + }, + (value: typeof state) => { + value.bindingTools[0]!.argumentConstraintDocumentsJson = []; + }, + (value: typeof state) => { + value.hasMoreExecutions = true; + }, + (value: typeof state) => { + value.executions.push(create(McpExecutionReviewSchema)); + }, + (value: typeof state) => { + value.approvals.push(create(McpApprovalSchema)); + }, + ]) { + const changed = structuredClone(state); + change(changed); + expect(() => + assertPlanWriteDenial(harness, [{ proof }], changed, expected), + ).toThrow(); + } + }); +} diff --git a/orchestrator/scripts/mcp-stack/case-grants.ts b/orchestrator/scripts/mcp-stack/case-grants.ts new file mode 100644 index 000000000..cfd38d88e --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-grants.ts @@ -0,0 +1,1199 @@ +import { Code, ConnectError } from "@connectrpc/connect"; +import { create } from "@bufbuild/protobuf"; +import { z } from "zod"; +import { ProfileIntegrationGrantSchema } from "../../src/gen/engram/app/v1/profile_pb.ts"; +import type { GetSessionMcpStateResponse } from "../../src/gen/engram/app/v1/integration_pb.ts"; +import { + bindingTools, + expectRejected, + modelToolName, + waitFor, +} from "./bootstrap.ts"; +import { grantUpdateRequest } from "./rebind.ts"; +export { grantUpdateRequest } from "./rebind.ts"; +import type { CallPlan } from "./call-plan.ts"; +import { requireThat, RunnerFailure, sha256 } from "./context.ts"; +import { + completeAssertions, + type CaseDriver, + type DriverAssertion, + type DriverObservation, +} from "./driver.ts"; +import { canonicalJson, releaseGuestBarrier } from "./guest.ts"; +import { + caseBodyFailure, + completeCaseCleanup, + approveOnce, + closeScenario, + closeScenarioSession, + execution, + pollProof, + pollGuestObservation, + sendScenarioPrompt, + startScenario, + scenarioBindingTools, + verifiedCounterResult, + type StartedScenario, +} from "./scenario-runtime.ts"; + +export function assertReadOnlyPolicy( + policy: { writeEligibility: string } | undefined, +) { + requireThat( + policy?.writeEligibility === "read_only", + "generic_plan_preserves_observed_read_only_policy", + ); +} +export function assertPolicyDenial(proof: { + resultReceived?: boolean; + delivery?: string; + outcome?: string | null; + isError?: boolean | null; +}) { + requireThat( + proof.resultReceived === true && + proof.delivery === "available" && + proof.outcome === null && + proof.isError === true, + "actual_runtime_policy_refusal", + ); +} +export function assertNoDispatch( + before: { reads: number; effects: { count: number; executions: number } }, + after: typeof before, +) { + requireThat( + after.reads === before.reads && + after.effects.count === before.effects.count && + after.effects.executions === before.effects.executions, + "no_upstream_dispatch_or_effect", + ); +} +export function assertAbsentToolDenial( + harness: "claude" | "codex", + proof: { + expectExposure?: string; + requestAttempted?: boolean; + protocolDenialCode?: string; + nativeDenialCode?: number; + exposedMcpToolNamesSha256?: string; + resultReceived?: boolean; + selectedSchemaSha256?: string; + }, +) { + requireThat( + proof.expectExposure === "absent" && + proof.requestAttempted === true && + proof.protocolDenialCode === "unknown_tool" && + (harness !== "claude" || proof.nativeDenialCode === -32602) && + !!proof.exposedMcpToolNamesSha256 && + proof.resultReceived === undefined && + proof.selectedSchemaSha256 === undefined, + "actual_absent_tool_native_refusal", + ); +} +export function assertNoProjectedAuthority( + call: { + state: string; + approvalDecision: string; + executionId?: string; + cancellationReceipt?: { executionId: string; state: string }; + }, + resultJson: string | undefined, +) { + requireThat( + (call.state === "waiting_approval" && + call.approvalDecision === "pending") || + (call.state === "canceled" && + call.approvalDecision === "denied" && + !!call.executionId && + call.cancellationReceipt?.executionId === call.executionId && + call.cancellationReceipt.state === "canceled"), + `guest_projection_has_no_journal_authority_${projectionState(call)}`, + ); + if (resultJson !== undefined) { + const result = z + .object({ outcome: z.string() }) + .parse(JSON.parse(resultJson)); + requireThat( + result.outcome !== "succeeded", + "guest_projection_has_no_success_result", + ); + } +} +function projectionState( + call: { state: string; approvalDecision: string } | undefined, +) { + const state = [ + "waiting_approval", + "authorized", + "dispatching", + "succeeded", + "denied", + "failed", + "canceled", + "outcome_unknown", + ].includes(call?.state ?? "") + ? call!.state + : "other"; + const decision = ["", "pending", "approved", "denied"].includes( + call?.approvalDecision ?? "", + ) + ? call?.approvalDecision || "empty" + : "other"; + return `${state}_${decision}`; +} +async function absentCall( + input: Parameters[0], + scenario: StartedScenario, + marker: string, + callId: string, + tool: string, +) { + const before = await scenario.fixture.observe(); + const tools = await scenarioBindingTools(scenario); + requireThat( + !tools.some((value) => value.modelToolName === tool), + "absent_alias_public_binding", + ); + await sendScenarioPrompt(scenario, false, marker); + const proofs = await pollProof(input.ctx, scenario, { + callId, + tool, + planSha256: scenario.planSha256, + }); + requireThat(proofs.length === 1, "single_absent_call_proof"); + assertAbsentToolDenial(input.harness, proofs[0]!.proof); + requireThat( + proofs[0]!.proof.exposedMcpToolNamesSha256 === + `sha256:${sha256(canonicalJson(tools.map((value) => value.modelToolName).sort()))}`, + "absent_call_actual_exposure_matches_binding", + ); + const after = await scenario.fixture.observe(); + assertNoDispatch(before, after); + return [ + policyObservation("guest-proof", scenario.sessionId, "denied", { + hash: proofs[0]!.sha256, + }), + policyObservation( + "fixture-observer", + scenario.sessionId, + "denied", + { before, after }, + 0, + ), + ]; +} + +export function fixedPolicyPlan( + alias: string, + kind: "read" | "write", +): CallPlan { + return { + steps: [ + { + marker: "allowed", + tool: alias, + arguments: kind === "read" ? {} : { delta: 1 }, + callId: "policy-allowed", + resultFormat: "mcp", + }, + { + marker: "restricted", + tool: alias, + arguments: { delta: 2 }, + callId: "policy-restricted", + resultFormat: "mcp", + }, + { + marker: "after-plan", + tool: alias, + arguments: kind === "read" ? {} : { delta: 1 }, + callId: "policy-after-plan", + resultFormat: "mcp", + }, + { + marker: "plan", + tool: "exit_plan_mode", + arguments: { plan: "Run the neutral counter call." }, + callId: "policy-plan", + resultFormat: "json", + }, + { + marker: "spawn", + tool: "spawn_session", + arguments: { + task_name: "fixture-child", + message: "[[fixture:child]]", + idempotency_key: "fixture-child", + }, + callId: "policy-spawn", + resultFormat: "json", + }, + { + marker: "child", + tool: alias, + arguments: kind === "read" ? {} : { delta: 1 }, + callId: "policy-child", + resultFormat: "mcp", + }, + ], + }; +} +export function expectedPlanApproval(harness: "claude" | "codex") { + return harness === "claude" + ? { + content: [ + { + type: "text", + text: "Your plan was approved. Implement it now, following the plan you presented.", + }, + ], + } + : { + success: true, + contentItems: [ + { + type: "inputText", + text: "Plan approved. A fresh build turn starts next.", + }, + ], + }; +} +export function assertPlanApprovalCompletion( + harness: "claude" | "codex", + completed: { + proof: { resultReceived?: boolean; resultSha256?: string }; + }[], +) { + requireThat( + completed.length === 1 && + completed[0]!.proof.resultReceived && + completed[0]!.proof.resultSha256 === + `sha256:${sha256(canonicalJson(expectedPlanApproval(harness)))}`, + "actual_generic_plan_completion", + ); +} +export function policyObservation( + source: DriverObservation["source"], + identity: string, + state: DriverObservation["state"], + metadata: unknown, + count?: number, +): DriverObservation { + return { + source, + identitySha256: sha256(identity), + state, + contentSha256: sha256(canonicalJson(metadata)), + ...(count === undefined ? {} : { count }), + }; +} +export const deltaOne = { + type: "object", + properties: { delta: { const: 1 } }, + required: ["delta"], + additionalProperties: false, +}; +export function assertDeltaOneConstraints(documents: string[]) { + requireThat( + documents.length > 0 && + documents.every( + (document) => + canonicalJson(JSON.parse(document)) === canonicalJson(deltaOne), + ), + "exact_delta_one_constraint_roots", + ); +} +export async function deniedCall( + input: Parameters[0], + scenario: StartedScenario, + marker: string, + callId: string, +) { + const before = await scenario.fixture.observe(); + await sendScenarioPrompt(scenario, false, marker); + const proofs = await pollProof(input.ctx, scenario, { + callId, + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + requireThat(proofs.length === 1, "single_denied_guest_completion"); + assertPolicyDenial(proofs[0]!.proof); + const after = await scenario.fixture.observe(); + assertNoDispatch(before, after); + return [ + policyObservation("guest-proof", scenario.sessionId, "denied", { + hash: proofs[0]!.sha256, + callId, + }), + policyObservation( + "fixture-observer", + scenario.sessionId, + "denied", + { before, after }, + 0, + ), + ]; +} + +export const unattendedGrantDriver: CaseDriver = async (input) => { + const { ctx, principals, database, harness } = input; + const actor = await principals.automation.identity.whoAmI({}); + const human = await principals.admin.identity.whoAmI({}); + requireThat( + actor.serviceAccount && + !human.serviceAccount && + actor.userId !== human.userId, + "distinct_unattended_and_review_actors", + ); + const actors = [ + policyObservation("public-rpc", actor.userId, "created", { + actor: sha256(actor.userId), + reviewer: sha256(human.userId), + serviceAccount: actor.serviceAccount, + }), + ]; + const assertions = [{ assertion: "api-key-principal", observations: actors }]; + for (const automationWrite of [false, true]) { + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-unattended-${automationWrite ? "allow" : "deny"}`, + protocol: "modern", + upstreamName: "write_counter", + mode: "write", + caller: "automation", + automationWrite, + arguments: { delta: 1 }, + constraints: [deltaOne], + callId: "policy-allowed", + callPlan: (alias) => fixedPolicyPlan(alias, "write"), + }); + try { + requireThat( + scenario.automation && + scenario.automation.runtimeActorId === + `automation:${scenario.automation.id}` && + scenario.automation.triggerActorId === actor.userId && + scenario.automation.definitionActorId === human.userId && + scenario.coordinator.source_key.creation.policy.actorId === + scenario.automation.runtimeActorId, + "authoritative_automation_runtime_and_trigger_actors", + ); + const forbiddenTrigger = await expectRejected( + () => + principals.unattended.automations.runNow({ + automationId: scenario.automation!.id, + }), + [Code.PermissionDenied], + ); + actors.push( + policyObservation( + "public-rpc", + scenario.automation.runtimeActorId, + "created", + { automation: scenario.automation, forbiddenTrigger }, + ), + ); + const tools = await bindingTools(principals.admin, scenario.sessionId); + requireThat( + tools.length === 1 && + tools[0]!.automationWrite === automationWrite && + !tools[0]!.unrestricted, + "exact_admin_automation_grant", + ); + assertDeltaOneConstraints(tools[0]!.argumentConstraintDocumentsJson); + if (!automationWrite) { + assertions.push({ + assertion: "default-deny", + observations: await deniedCall( + input, + scenario, + "allowed", + "policy-allowed", + ), + }); + // The generic approval path returns a native decision; it is not MCP authority. + await sendScenarioPrompt(scenario, true, "plan"); + const completed = await pollProof(ctx, scenario, { + callId: "policy-plan", + tool: "exit_plan_mode", + planSha256: scenario.planSha256, + }); + assertPlanApprovalCompletion(harness, completed); + const afterPlan = await principals.admin.sessions.getMcpWritePolicy({ + sessionId: scenario.sessionId, + }); + assertReadOnlyPolicy(afterPlan.policy); + assertions.push({ + assertion: "plan-approval-insufficient", + observations: await deniedCall( + input, + scenario, + "after-plan", + "policy-after-plan", + ), + }); + } else { + const before = await scenario.fixture.observe(); + await sendScenarioPrompt(scenario, false, "allowed"); + const call = await execution(ctx, scenario); + requireThat( + call?.state === "succeeded" && !call.approvalDecision, + "unattended_write_without_human_approval", + ); + const result = await verifiedCounterResult( + scenario, + call.executionId, + before.effects.count + 1, + false, + ); + const proofs = await pollProof(ctx, scenario, { + callId: "policy-allowed", + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + requireThat( + proofs.length === 1 && + proofs[0]!.proof.outcome === "succeeded" && + proofs[0]!.proof.delivery === "available", + "unattended_actual_result", + ); + const after = await scenario.fixture.observe(); + requireThat( + after.effects.executions === before.effects.executions + 1 && + after.effects.count === before.effects.count + 1, + "unattended_one_effect", + ); + assertions.push({ + assertion: "explicit-admin-grant", + observations: [ + policyObservation("public-rpc", scenario.sessionId, "succeeded", { + resultHash: result.resultHash, + grant: tools[0], + }), + policyObservation( + "fixture-observer", + scenario.sessionId, + "succeeded", + { before, after }, + 1, + ), + ], + }); + assertions.push({ + assertion: "argument-constraint", + observations: await deniedCall( + input, + scenario, + "restricted", + "policy-restricted", + ), + }); + } + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, scenario); + }); + } + } + return completeAssertions("unattended-grant", assertions); +}; + +export const grantDenialsDriver: CaseDriver = async (input) => { + const { ctx, principals, database, harness } = input; + const assertions: DriverAssertion[] = []; + const noEffects: DriverObservation[] = []; + const missing = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-grant-missing`, + protocol: "modern", + upstreamName: "read_counter", + mode: "read", + caller: "owner", + automationWrite: false, + arguments: {}, + constraints: [], + callId: "grant-missing", + grant: false, + callPlan: (alias) => ({ + steps: [ + { + marker: "missing", + tool: alias, + arguments: {}, + callId: "grant-missing", + resultFormat: "mcp", + expectExposure: "absent", + }, + ], + }), + }); + try { + const current = await principals.admin.profiles.getProfile({ + id: missing.profileId, + }); + requireThat( + current.profile && current.profile.integrationGrants.length === 0, + "missing_profile_grant", + ); + const forgedGrant = create(ProfileIntegrationGrantSchema, { + connectionId: missing.connection.id, + operation: "invoke", + credentialScope: "org", + mcp: { + upstreamName: "write_counter", + unrestricted: true, + automationWrite: true, + }, + }); + const refusal = await expectRejected( + () => + principals.other.profiles.updateProfile( + grantUpdateRequest(current.profile!, [forgedGrant]), + ), + [Code.PermissionDenied], + ); + const unchanged = await principals.admin.profiles.getProfile({ + id: missing.profileId, + }); + requireThat( + unchanged.profile?.mcpGrantRevision === + current.profile.mcpGrantRevision && + unchanged.profile.integrationGrants.length === 0, + "forged_grant_not_persisted", + ); + assertions.push({ + assertion: "forged-grant", + observations: [ + policyObservation("public-rpc", missing.profileId, "denied", { + refusal, + revision: unchanged.profile.mcpGrantRevision.toString(), + }), + ], + }); + const observations = await absentCall( + input, + missing, + "missing", + "grant-missing", + missing.alias, + ); + assertions.push({ assertion: "missing-grant", observations }); + noEffects.push( + ...observations.filter((value) => value.source === "fixture-observer"), + ); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, missing); + }); + } + const revoked = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-grant-revoked`, + protocol: "modern", + upstreamName: "read_counter", + mode: "read", + caller: "owner", + automationWrite: false, + arguments: {}, + constraints: [], + callId: "policy-allowed", + callPlan: (alias) => fixedPolicyPlan(alias, "read"), + }); + try { + const current = await principals.admin.profiles.getProfile({ + id: revoked.profileId, + }); + requireThat(current.profile, "current_profile_before_revoke"); + const updated = await principals.admin.profiles.updateProfile( + grantUpdateRequest(current.profile, []), + ); + requireThat( + updated.profile?.mcpGrantRevision === + current.profile.mcpGrantRevision + 1n && + updated.profile.integrationGrants.length === 0, + "admin_revoked_exact_grant", + ); + const observations = await deniedCall( + input, + revoked, + "allowed", + "policy-allowed", + ); + assertions.push({ assertion: "revoked-grant", observations }); + noEffects.push( + ...observations.filter((value) => value.source === "fixture-observer"), + ); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, revoked); + }); + } + const forged = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-guest-projection`, + protocol: "modern", + upstreamName: "write_counter", + mode: "read", + caller: "owner", + automationWrite: false, + arguments: { delta: 1 }, + constraints: [deltaOne], + callId: "grant-projection", + prematureBarrier: true, + callPlan: (alias) => ({ + steps: [ + { + marker: "projection", + tool: alias, + arguments: { delta: 1 }, + callId: "grant-projection", + resultFormat: "mcp", + completionProjection: "premature", + }, + ], + }), + }); + try { + const before = await forged.fixture.observe(); + await sendScenarioPrompt(forged, false, "projection"); + const held = await pollGuestObservation(ctx, forged, { + callId: "grant-projection", + tool: forged.alias, + planSha256: forged.planSha256, + observation: "prematureBarrierHeld", + }); + requireThat(held.length === 1, "single_premature_barrier"); + let lastPending: Awaited>; + const pending = await waitFor( + ctx, + "real_pending_call_before_guest_projection", + async () => (lastPending = await execution(ctx, forged, false)), + (call) => + call?.state === "waiting_approval" && + call.approvalDecision === "pending" && + !!call.runId, + ).catch((error: unknown) => { + if ( + !(error instanceof RunnerFailure) || + error.seam !== "real_pending_call_before_guest_projection" + ) + throw error; + requireThat( + false, + `real_pending_call_before_guest_projection_${projectionState(lastPending)}`, + ); + }); + requireThat( + pending?.state === "waiting_approval" && + pending.approvalDecision === "pending" && + pending.runId, + "real_pending_call_before_guest_projection", + ); + assertNoDispatch(before, await forged.fixture.observe()); + await releaseGuestBarrier( + ctx, + forged.client, + forged.sessionId, + "premature", + ); + const emitted = await pollGuestObservation(ctx, forged, { + callId: "grant-projection", + tool: forged.alias, + planSha256: forged.planSha256, + observation: "prematureCompletionEmitted", + }); + requireThat( + emitted.length === 1 && emitted[0]!.proof.eventSha256, + "actual_guest_completion_emission", + ); + const completedEvent = z.object({ + run_id: z.string(), + tool_call_id: z.string(), + tool_name: z.string(), + ok: z.boolean(), + }); + const runEvent = z.object({ run_id: z.string() }); + const transported = await waitFor( + ctx, + "actual_transported_guest_completion", + async () => { + const page = await forged.reviewClient.sessions.listSessionEvents({ + sessionId: forged.sessionId, + limit: 16n, + kinds: ["tool_call_completed", "run_completed", "run_interrupted"], + }); + requireThat(page.events.length < 16, "fresh_projection_event_bound"); + const completions = page.events + .filter((event) => event.kind === "tool_call_completed") + .map((event) => ({ + idx: event.idx, + value: completedEvent.parse(JSON.parse(event.payloadJson)), + })) + .filter( + (event) => + event.value.tool_call_id === pending.toolCallId && + event.value.run_id === pending.runId, + ); + requireThat(completions.length <= 1, "single_transported_projection"); + const completion = completions[0]; + const finished = page.events.find( + (event) => + event.kind !== "tool_call_completed" && + runEvent.parse(JSON.parse(event.payloadJson)).run_id === + pending.runId, + ); + if (!completion || !finished) return undefined; + requireThat( + completion.idx !== undefined && + finished.idx !== undefined && + finished.idx > completion.idx && + completion.value.ok && + completion.value.tool_name === + (harness === "claude" + ? `mcp__engrams__${forged.alias}` + : forged.alias), + "native_projection_matches_real_call", + ); + return { + completion: completion.value, + completionIdx: completion.idx.toString(), + finishedIdx: finished.idx.toString(), + }; + }, + (value) => !!value, + ); + const afterCall = await execution(ctx, forged, false); + requireThat( + afterCall && afterCall.executionId === pending.executionId, + "same_pending_journal_identity", + ); + const result = await forged.reviewClient.integration.getMcpExecutionResult({ + sessionId: forged.sessionId, + executionId: pending.executionId, + }); + assertNoProjectedAuthority(afterCall, result.resultJson); + requireThat( + result.state === afterCall.state, + "projection_result_state_matches_journal", + ); + const after = await forged.fixture.observe(); + assertNoDispatch(before, after); + const effects = policyObservation( + "fixture-observer", + forged.sessionId, + "denied", + { before, after }, + 0, + ); + noEffects.push(effects); + assertions.push({ + assertion: "forged-guest-completion", + observations: [ + policyObservation("guest-proof", forged.sessionId, "denied", { + emission: emitted[0]!.sha256, + }), + policyObservation("public-rpc", pending.executionId, "denied", { + transported, + state: afterCall.state, + approvalDecision: afterCall.approvalDecision, + resultHash: result.resultJson ? sha256(result.resultJson) : null, + }), + effects, + ], + }); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, forged); + }); + } + assertions.push({ assertion: "no-effect", observations: noEffects }); + return completeAssertions("grant-denials", assertions); +}; + +export function expectedWriteAuthorityDenial(harness: "claude" | "codex") { + const text = "External tool request denied."; + const metadata = { + source: "runtime", + denial: "write_not_allowed", + delivery: "available", + isError: true, + }; + return harness === "claude" + ? { + content: [{ type: "text", text }], + isError: true, + _meta: { "engrams/toolResult": metadata }, + } + : { + success: false, + contentItems: [ + { type: "inputText", text }, + { type: "inputText", text: canonicalJson(metadata) }, + ], + }; +} +export function assertPlanWriteDenial( + harness: "claude" | "codex", + completed: { + proof: Parameters[0] & { + denialPlanMode?: boolean; + resultSha256?: string; + expectedResultSha256?: string; + resultMatchesExpected?: boolean; + }; + }[], + state: GetSessionMcpStateResponse, + expected: { sessionId: string; connectionId: string; alias: string }, +) { + requireThat(completed.length === 1, "one_plan_call_proof"); + const proof = completed[0]!.proof; + assertPolicyDenial(proof); + const expectedHash = `sha256:${sha256(canonicalJson(expectedWriteAuthorityDenial(harness)))}`; + requireThat( + proof.resultMatchesExpected === true && + proof.resultSha256 === expectedHash && + proof.expectedResultSha256 === expectedHash, + "trusted_plan_write_denial", + ); + const policy = state.writePolicy?.policy; + assertReadOnlyPolicy(policy); + requireThat( + policy?.current === true && + policy.initialized === true && + policy.ref?.sessionId === expected.sessionId, + "trusted_current_plan_policy", + ); + const tool = state.bindingTools[0]; + requireThat( + state.bindingReceipt?.current === true && + state.availability === "available" && + !state.hasMoreBindingTools && + state.bindingTools.length === 1 && + tool?.connectionId === expected.connectionId && + tool.modelToolName === expected.alias && + tool.reviewedTool?.upstreamName === "write_counter" && + tool.reviewedTool.classification === "write" && + tool.reviewedTool.enabled === true && + !tool.unrestricted, + "exact_plan_write_grant", + ); + assertDeltaOneConstraints(tool!.argumentConstraintDocumentsJson); + requireThat( + !state.hasMoreExecutions && + state.executions.length === 0 && + state.approvals.length === 0, + "plan_write_denied_before_journal_authority", + ); +} + +export const planChildIntersectionDriver: CaseDriver = async (input) => { + const { ctx, principals, database, harness } = input; + const assertions: DriverAssertion[] = []; + for (const upstreamName of ["read_counter", "write_counter"] as const) { + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-plan-${upstreamName === "read_counter" ? "read" : "write"}`, + protocol: "modern", + upstreamName, + mode: upstreamName === "read_counter" ? "read" : "write", + caller: "owner", + automationWrite: false, + arguments: upstreamName === "read_counter" ? {} : { delta: 1 }, + constraints: upstreamName === "read_counter" ? [] : [deltaOne], + callId: "policy-allowed", + callPlan: (alias) => { + const plan = fixedPolicyPlan( + alias, + upstreamName === "read_counter" ? "read" : "write", + ); + if (upstreamName === "write_counter") + plan.steps[0]!.expectedResult = expectedWriteAuthorityDenial(harness); + return plan; + }, + }); + try { + const before = await scenario.fixture.observe(); + await sendScenarioPrompt(scenario, true, "allowed"); + const proofs = await pollProof(ctx, scenario, { + callId: "policy-allowed", + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + requireThat(proofs.length === 1, "one_plan_call_proof"); + const after = await scenario.fixture.observe(); + if (upstreamName === "read_counter") { + const call = await execution(ctx, scenario); + requireThat( + call?.state === "succeeded" && + after.reads === before.reads + 1 && + after.effects.executions === before.effects.executions, + "plan_read_succeeds", + ); + await verifiedCounterResult( + scenario, + call.executionId, + before.effects.count, + false, + ); + } else { + const state = + await scenario.reviewClient.integration.getSessionMcpState({ + sessionId: scenario.sessionId, + omitBindings: false, + omitExecutions: false, + bindingPageSize: 16, + pageSize: 16, + }); + assertPlanWriteDenial(harness, proofs, state, { + sessionId: scenario.sessionId, + connectionId: scenario.connection.id, + alias: scenario.alias, + }); + assertNoDispatch(before, after); + } + assertions.push({ + assertion: + upstreamName === "read_counter" + ? "plan-read-only" + : "plan-write-denied", + observations: [ + policyObservation( + "guest-proof", + scenario.sessionId, + upstreamName === "read_counter" ? "succeeded" : "denied", + { proof: proofs[0]!.sha256 }, + ), + policyObservation( + "fixture-observer", + scenario.sessionId, + upstreamName === "read_counter" ? "succeeded" : "denied", + { before, after }, + 0, + ), + ], + }); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, scenario); + }); + } + } + let outsideAlias = ""; + const parent = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-child`, + protocol: "modern", + upstreamName: "write_counter", + mode: "write", + caller: "owner", + automationWrite: false, + arguments: { delta: 1 }, + constraints: [deltaOne], + callId: "policy-spawn", + callPlan: (alias, connection) => { + outsideAlias = modelToolName( + connection.id, + connection.alias, + "read_counter", + ); + return { + steps: [ + ...fixedPolicyPlan(alias, "write").steps, + { + marker: "outside", + tool: outsideAlias, + arguments: {}, + callId: "policy-outside", + resultFormat: "mcp", + expectExposure: "absent", + }, + ], + }; + }, + }); + let child: StartedScenario | undefined; + try { + const parentTools = await bindingTools(principals.owner, parent.sessionId); + await sendScenarioPrompt(parent, false, "spawn"); + const spawned = await waitFor( + ctx, + "real_native_child_task", + async () => { + const value = await principals.owner.tasks.getTask({ + taskId: parent.taskId, + }); + const matches = + value.task?.descendants.filter( + (task) => + task.localTaskName === "fixture-child" && + task.parentTaskId === parent.taskId && + task.spawningSessionId === parent.sessionId, + ) ?? []; + requireThat(matches.length <= 1, "single_real_native_child"); + return matches[0]; + }, + (task) => !!task && task.sessions.length === 1, + ); + requireThat( + spawned && spawned.sessions.length === 1, + "native_child_identity", + ); + child = { + ...parent, + taskId: spawned.id, + sessionId: spawned.sessions[0]!.sessionId, + input: { ...parent.input, callId: "policy-child" }, + }; + database.own(child.taskId, child.sessionId); + await waitFor( + ctx, + "child_public_active", + async () => { + try { + return await principals.owner.sessions.getSession({ + sessionId: child!.sessionId, + }); + } catch (error) { + if (error instanceof ConnectError && error.code === Code.NotFound) + return undefined; + throw error; + } + }, + (value) => + value?.session?.status === "active" && + value.session.hostId === ctx.hostId, + ); + const started = await waitFor( + ctx, + "child_actual_started", + () => database.started(child!.sessionId, ctx.hostId), + (value) => value.phase === "started", + ); + requireThat( + started.source_key.creation.policy.parentSessionId === parent.sessionId, + "child_frozen_parent_source_identity", + ); + await waitFor( + ctx, + "child_managed_started", + () => database.managedLifecycle(child!.sessionId, started), + (value) => value.phase === "started", + ); + const childTools = await bindingTools(principals.owner, child.sessionId); + requireThat( + parentTools.length === 1 && childTools.length === 1, + "single_parent_and_child_tool", + ); + const { + argumentConstraintDocumentsJson: parentConstraints, + ...parentIdentity + } = parentTools[0]!; + const { + argumentConstraintDocumentsJson: childConstraints, + ...childIdentity + } = childTools[0]!; + assertDeltaOneConstraints(parentConstraints); + assertDeltaOneConstraints(childConstraints); + // Intersections preserve each schema root, including repeated equivalent restrictions. + requireThat( + canonicalJson(childIdentity) === canonicalJson(parentIdentity) && + !childIdentity.unrestricted && + childConstraints.length >= parentConstraints.length, + "native_child_parent_derived_grant_intersection", + ); + const before = await parent.fixture.observe(); + await approveOnce(ctx, principals, child, "owner"); + const completed = await execution(ctx, child); + requireThat( + completed?.state === "succeeded", + "allowed_child_write_succeeds", + ); + await verifiedCounterResult( + child, + completed.executionId, + before.effects.count + 1, + false, + ); + await pollProof(ctx, child, { + callId: "policy-child", + tool: child.alias, + planSha256: child.planSha256, + }); + const spawnProof = await pollProof(ctx, parent, { + callId: "policy-spawn", + tool: "spawn_session", + planSha256: parent.planSha256, + }); + requireThat( + spawnProof.length === 1 && spawnProof[0]!.proof.resultSha256, + "actual_native_spawn_result", + ); + const after = await parent.fixture.observe(); + requireThat( + after.effects.executions === before.effects.executions + 1 && + after.effects.count === before.effects.count + 1, + "allowed_child_exactly_one_effect", + ); + const absent = await absentCall( + input, + child, + "outside", + "policy-outside", + outsideAlias, + ); + assertions.push({ + assertion: "child-grant-intersection", + observations: [ + policyObservation("public-rpc", child.sessionId, "succeeded", { + parentTools, + childTools, + parentSessionHash: sha256(parent.sessionId), + spawnProof: spawnProof[0]!.sha256, + }), + ...absent, + ], + }); + const denied = await deniedCall( + input, + child, + "restricted", + "policy-restricted", + ); + assertions.push({ + assertion: "child-argument-intersection", + observations: [ + policyObservation( + "fixture-observer", + child.sessionId, + "succeeded", + { before, after }, + 1, + ), + ...denied, + ], + }); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + if (child) { + await closeScenarioSession(principals.admin.sessions, child); + await principals.admin.tasks.deleteTask({ taskId: child.taskId }); + } + await closeScenario(principals, parent); + }); + } + return completeAssertions("plan-child-intersection", assertions); +}; diff --git a/orchestrator/scripts/mcp-stack/case-oauth.test.ts b/orchestrator/scripts/mcp-stack/case-oauth.test.ts new file mode 100644 index 000000000..a2a40948b --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-oauth.test.ts @@ -0,0 +1,229 @@ +import { expect, test } from "bun:test"; +import * as oauth from "./case-oauth.ts"; +import { + assertStableAuthority, + assertRejectedRefreshAuthority, + assertOAuthRefusal, + assertOAuthRefreshFailure, + assertInvalidatedAuthority, + oauthCallPlan, +} from "./case-oauth.ts"; + +test("a reviewed catalog is enabled with its current connection revision", async () => { + expect(typeof oauth.enableReviewedOAuthConnection).toBe("function"); + let enabled = false; + const integration = { + async setConnectionEnabled(request: { + id: string; + expectedRevision: number; + enabled: boolean; + }) { + expect(request).toEqual({ + id: "fixture", + expectedRevision: 7, + enabled: true, + }); + enabled = true; + return { connection: { id: "fixture", configRevision: 7, enabled } }; + }, + }; + await oauth.enableReviewedOAuthConnection(integration, "fixture", 7); + expect(enabled).toBe(true); + await expect( + oauth.enableReviewedOAuthConnection( + { + async setConnectionEnabled() { + return { + connection: { id: "fixture", configRevision: 7, enabled: false }, + }; + }, + }, + "fixture", + 7, + ), + ).rejects.toThrow(); +}); + +test("ordinary token rotation cannot silently replace bound authority", () => { + const before = { + bindingHash: "sha256:binding", + current: true, + authGeneration: 3, + credentialIdentity: "sha256:identity", + }; + expect(() => assertStableAuthority(before, before)).not.toThrow(); + for (const after of [ + { ...before, bindingHash: "sha256:new" }, + { ...before, current: false }, + { ...before, authGeneration: 4 }, + { ...before, credentialIdentity: "sha256:other" }, + ]) + expect(() => assertStableAuthority(before, after)).toThrow(); +}); +test("authority changes must stale the original frozen binding", () => { + expect(() => + assertInvalidatedAuthority("old", { + bindingHash: "old", + current: false, + staleReasons: ["credential_changed"], + }), + ).not.toThrow(); + for (const after of [ + { bindingHash: "new", current: true, staleReasons: [] }, + { bindingHash: "old", current: false, staleReasons: [] }, + ]) + expect(() => assertInvalidatedAuthority("old", after)).toThrow(); +}); +test("OAuth retries use bounded distinct immutable native call IDs", () => { + const plan = oauthCallPlan("mcp_fixture"); + expect(plan.steps.length).toBeLessThanOrEqual(16); + expect(new Set(plan.steps.map((s) => s.callId)).size).toBe(plan.steps.length); + expect( + plan.steps.every( + (s) => s.tool === "mcp_fixture" && s.resultFormat === "mcp", + ), + ).toBe(true); +}); + +test("changed OAuth authority requires a received available runtime refusal", () => { + const refusal = { + resultReceived: true, + delivery: "available", + outcome: null, + isError: true, + }; + expect(() => assertOAuthRefusal(refusal)).not.toThrow(); + for (const invalid of [ + {}, + { ...refusal, resultReceived: false }, + { ...refusal, delivery: "unavailable" }, + { ...refusal, delivery: undefined }, + { ...refusal, isError: undefined }, + { ...refusal, isError: false }, + { ...refusal, outcome: undefined }, + { ...refusal, outcome: "succeeded" }, + { ...refusal, outcome: "failed" }, + { ...refusal, outcome: "outcome_unknown" }, + { ...refusal, outcome: "canceled" }, + { + resultReceived: true, + delivery: "unavailable", + outcome: "failed", + isError: null, + }, + ]) + expect(() => assertOAuthRefusal(invalid)).toThrow(); +}); + +test("refresh-trigger failure requires the exact persisted unavailable read failure", () => { + const proof = { + resultReceived: true, + delivery: "unavailable", + outcome: "failed", + isError: null, + resultUnavailable: false, + }; + const result = { + state: "failed", + resultUnavailable: false, + resultJson: JSON.stringify({ + outcome: "failed", + delivery: "unavailable", + reason: "upstream_failure", + }), + }; + expect(() => assertOAuthRefreshFailure(proof, result)).not.toThrow(); + for (const invalid of [ + { ...proof, resultReceived: false }, + { ...proof, outcome: null }, + { ...proof, outcome: "canceled" }, + { ...proof, outcome: "outcome_unknown" }, + { ...proof, delivery: "available" }, + { ...proof, isError: true }, + { ...proof, resultUnavailable: true }, + ]) + expect(() => assertOAuthRefreshFailure(invalid, result)).toThrow(); + for (const invalid of [ + { ...result, state: "canceled" }, + { ...result, resultUnavailable: true }, + { ...result, resultJson: undefined }, + { + ...result, + resultJson: JSON.stringify({ + outcome: "failed", + delivery: "unavailable", + reason: "execution_interrupted", + }), + }, + { + ...result, + resultJson: JSON.stringify({ + outcome: "failed", + delivery: "unavailable", + reason: "upstream_failure", + output: { isError: true }, + }), + }, + ]) + expect(() => assertOAuthRefreshFailure(proof, invalid)).toThrow(); +}); + +test("each authority change has a distinct immutable stale refusal call before recovery", () => { + const plan = oauthCallPlan("mcp_fixture"); + for (const change of ["account", "scope", "reconnect"]) { + const trigger = plan.steps.find((step) => step.marker === change); + const refusal = plan.steps.find( + (step) => + step.marker === + (change === "reconnect" + ? `${change}-stale` + : `${change}-reauthorized-stale`), + ); + expect(refusal).toBeDefined(); + expect(refusal?.callId).not.toBe(trigger?.callId); + expect(plan.steps.indexOf(refusal!)).toBeGreaterThan( + plan.steps.indexOf(trigger!), + ); + expect(plan.steps.indexOf(refusal!)).toBeLessThan( + plan.steps.findIndex((step) => step.marker === `${change}-rebound`), + ); + } +}); + +test("a rejected refresh leaves accepted identity unchanged but credential broken", () => { + const before = { + bindingHash: "old", + current: true, + authGeneration: 3, + credentialIdentity: "identity", + }; + const after = { ...before, credentialStatus: "broken" }; + expect(() => assertRejectedRefreshAuthority(before, after)).not.toThrow(); + expect(() => + assertRejectedRefreshAuthority(before, { ...after, current: false }), + ).not.toThrow(); + for (const invalid of [ + { ...after, credentialStatus: "active" }, + { ...after, bindingHash: "new" }, + { ...after, authGeneration: 4 }, + { ...after, credentialIdentity: "other" }, + ]) + expect(() => assertRejectedRefreshAuthority(before, invalid)).toThrow(); +}); + +test("rejected refresh is retried before reauthorization and stale refusal", () => { + const plan = oauthCallPlan("mcp_fixture"); + for (const change of ["account", "scope"]) { + const names = [ + `${change}-stale`, + `${change}-reauthorized-stale`, + `${change}-rebound`, + ]; + const positions = names.map((marker) => + plan.steps.findIndex((step) => step.marker === marker), + ); + expect(positions.every((position) => position >= 0)).toBe(true); + expect(positions[0]!).toBeLessThan(positions[1]!); + expect(positions[1]!).toBeLessThan(positions[2]!); + } +}); diff --git a/orchestrator/scripts/mcp-stack/case-oauth.ts b/orchestrator/scripts/mcp-stack/case-oauth.ts new file mode 100644 index 000000000..59df3a921 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-oauth.ts @@ -0,0 +1,642 @@ +import { z } from "zod"; +import { authorizeOAuth, waitFor } from "./bootstrap.ts"; +import type { CallPlan } from "./call-plan.ts"; +import { requireThat, RunnerFailure } from "./context.ts"; +import { + completeAssertions, + type CaseDriver, + type DriverAssertion, +} from "./driver.ts"; +import { FixtureObserver } from "./observer.ts"; +import { recoverCurrentBinding } from "./rebind.ts"; +import { + caseBodyFailure, + completeCaseCleanup, + closeScenario, + execution, + pollProof, + sendScenarioPrompt, + startScenario, + verifiedCounterResult, +} from "./scenario-runtime.ts"; +import { + assertNoDispatch, + assertPolicyDenial, + policyObservation, +} from "./case-grants.ts"; + +export async function enableReviewedOAuthConnection( + integration: { + setConnectionEnabled(request: { + id: string; + expectedRevision: number; + enabled: boolean; + }): Promise<{ + connection?: { id: string; configRevision: number; enabled: boolean }; + }>; + }, + id: string, + expectedRevision: number, +) { + const enabled = await integration.setConnectionEnabled({ + id, + expectedRevision, + enabled: true, + }); + requireThat( + enabled.connection?.id === id && + enabled.connection.configRevision === expectedRevision && + enabled.connection.enabled, + "oauth_reviewed_connection_enabled", + ); +} + +export function assertOAuthRefusal( + proof: Parameters[0], +) { + assertPolicyDenial(proof); +} + +export function assertOAuthRefreshFailure( + proof: Parameters[0] & { + resultUnavailable?: boolean | null; + }, + result: { state: string; resultUnavailable: boolean; resultJson?: string }, +) { + requireThat( + proof.resultReceived === true && + proof.outcome === "failed" && + proof.delivery === "unavailable" && + proof.isError === null && + proof.resultUnavailable === false && + result.state === "failed" && + !result.resultUnavailable && + result.resultJson !== undefined, + "oauth_refresh_trigger_unavailable_read_failure", + ); + z.strictObject({ + outcome: z.literal("failed"), + delivery: z.literal("unavailable"), + reason: z.literal("upstream_failure"), + }).parse(JSON.parse(result.resultJson)); +} + +type Authority = { + bindingHash: string; + current: boolean; + authGeneration: number; + credentialIdentity: string; +}; +export function assertStableAuthority(before: Authority, after: Authority) { + requireThat( + before.current && + after.current && + after.bindingHash === before.bindingHash && + after.authGeneration === before.authGeneration && + after.credentialIdentity === before.credentialIdentity, + "same_oauth_authority_binding_stable", + ); +} +export function assertRejectedRefreshAuthority( + before: Authority, + after: Authority & { credentialStatus: string }, +) { + requireThat( + after.bindingHash === before.bindingHash && + after.authGeneration === before.authGeneration && + after.credentialIdentity === before.credentialIdentity && + after.credentialStatus === "broken", + "oauth_rejected_refresh_authority", + ); +} +export function assertInvalidatedAuthority( + oldHash: string, + after: { bindingHash: string; current: boolean; staleReasons: string[] }, +) { + requireThat( + after.bindingHash === oldHash && + !after.current && + after.staleReasons.length > 0, + "changed_oauth_authority_requires_explicit_rebind", + ); +} +export function oauthCallPlan(alias: string): CallPlan { + return { + steps: [ + "initial", + "rotation", + "account", + "account-stale", + "account-reauthorized-stale", + "account-rebound", + "scope", + "scope-stale", + "scope-reauthorized-stale", + "scope-rebound", + "reconnect", + "reconnect-stale", + "reconnect-rebound", + "credential-probe", + ].map((marker) => ({ + marker, + tool: alias, + arguments: {}, + callId: `oauth-${marker}`, + resultFormat: "mcp", + })), + }; +} + +export const oauthAuthorityDriver: CaseDriver = async ({ + ctx, + principals, + database, + harness, +}) => { + const publicSetup = await principals.admin.integration.getMcpSetup({}); + const registrations = publicSetup.registrations.filter( + (r) => + r.issuer === ctx.protocolFixtures.modern.issuerOrigin && + r.resource?.endpoint === + `${ctx.protocolFixtures.modern.resourceOrigin}/mcp` && + r.redirectUri === ctx.protocolFixtures.modern.redirectUri, + ); + if ( + registrations.length !== 1 || + registrations[0]!.scopes.length !== 1 || + registrations[0]!.scopes[0] !== "tools:read" + ) + throw new RunnerFailure( + "NEEDS_CONTEXT", + "oauth_fixture_registration_exact_read_scope", + ); + const fixture = new FixtureObserver(ctx); + const fixtureBefore = await fixture.observe(); + await fixture.configure({ + reply: "json", + response: "normal", + holdCalls: false, + revision: 1, + }); + await fixture.oauth({ + account: "account-one", + scope: "tools:read", + issuer: "configured", + expiresInSeconds: 2, + }); + try { + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-oauth`, + protocol: "modern", + upstreamName: "read_counter", + authKind: "oauth", + mode: "read", + caller: "owner", + automationWrite: false, + arguments: {}, + constraints: [], + callId: "oauth-initial", + callPlan: oauthCallPlan, + }); + const assertions: DriverAssertion[] = []; + try { + const authority = async () => { + const state = await principals.admin.integration.getSessionMcpState({ + sessionId: scenario.sessionId, + omitExecutions: true, + bindingPageSize: 16, + }); + const setup = await principals.admin.integration.getMcpSetup({ + connectionId: scenario.connection.id, + }); + requireThat( + state.bindingReceipt && + setup.current?.connection && + setup.current.credential, + "oauth_actual_binding_and_connection", + ); + return { + bindingHash: state.bindingReceipt.bindingHash, + current: state.bindingReceipt.current, + staleReasons: state.bindingReceipt.staleReasons, + authGeneration: setup.current.connection.authGeneration, + credentialIdentity: setup.current.connection.credentialIdentity, + credentialStatus: setup.current.credential.status, + }; + }; + const successfulRead = async (marker: string) => { + const before = await fixture.observe(); + await sendScenarioPrompt(scenario, false, marker); + const call = await execution(ctx, scenario, true, `oauth-${marker}`); + requireThat(call?.state === "succeeded", "oauth_actual_read_succeeded"); + const result = await verifiedCounterResult( + scenario, + call.executionId, + before.effects.count, + false, + ); + const proofs = await pollProof(ctx, scenario, { + callId: `oauth-${marker}`, + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + requireThat( + proofs.length === 1 && + proofs[0]!.proof.outcome === "succeeded" && + proofs[0]!.proof.delivery === "available", + "oauth_native_read_delivered", + ); + const after = await fixture.observe(); + requireThat( + after.reads === before.reads + 1 && + after.effects.executions === before.effects.executions && + after.effects.count === before.effects.count, + "oauth_exact_read_no_write", + ); + return { + before, + after, + resultHash: result.resultHash, + proofHash: proofs[0]!.sha256, + }; + }; + const initial = await successfulRead("initial"); + requireThat( + initial.after.oauth.authorizations === + fixtureBefore.oauth.authorizations + 1 && + initial.after.oauth.exchanges === fixtureBefore.oauth.exchanges + 1 && + initial.after.oauth.expiresInSeconds === 2, + "real_oauth_pkce_exchange_and_short_expiry", + ); + const setup = await principals.admin.integration.getMcpSetup({ + connectionId: scenario.connection.id, + }); + const connection = setup.current?.connection; + const registration = setup.registrations.find( + (r) => r.id === connection?.mcp?.oauthClientRegistrationId, + ); + requireThat( + registration?.issuer === ctx.protocolFixtures.modern.issuerOrigin && + registration.resource?.endpoint === + `${ctx.protocolFixtures.modern.resourceOrigin}/mcp` && + registration.redirectUri === ctx.protocolFixtures.modern.redirectUri, + "oauth_exact_resource_issuer_and_callback", + ); + assertions.push( + { + assertion: "real-pkce", + observations: [ + policyObservation( + "fixture-observer", + scenario.connection.id, + "authorized", + initial.after.oauth, + ), + ], + }, + { + assertion: "exact-resource-issuer", + observations: [ + policyObservation( + "public-rpc", + scenario.connection.id, + "authorized", + { + issuer: registration.issuer, + resource: registration.resource.endpoint, + callback: registration.redirectUri, + }, + ), + ], + }, + ); + const beforeRotation = await authority(); + // Wait for the actual issued two-second token to expire. No clock or stored timestamp changes. + await new Promise((resolve) => setTimeout(resolve, 2100)); + const rotation = await successfulRead("rotation"); + requireThat( + rotation.after.oauth.rotations > rotation.before.oauth.rotations, + "coordinator_expiry_refresh_rotation", + ); + const afterRotation = await authority(); + assertStableAuthority(beforeRotation, afterRotation); + assertions.push( + { + assertion: "expiry-rotation", + observations: [ + policyObservation( + "fixture-observer", + scenario.connection.id, + "succeeded", + { + before: rotation.before.oauth, + after: rotation.after.oauth, + }, + ), + ], + }, + { + assertion: "same-identity-binding-stable", + observations: [ + policyObservation("public-rpc", scenario.sessionId, "current", { + beforeRotation, + afterRotation, + }), + ], + }, + ); + for (const change of ["account", "scope", "reconnect"] as const) { + const old = await authority(); + requireThat(old.current, "oauth_current_before_change"); + if (change === "reconnect") { + const current = await principals.admin.integration.getMcpSetup({ + connectionId: scenario.connection.id, + }); + requireThat( + current.current?.connection, + "oauth_disconnect_current_revision", + ); + await principals.admin.integration.disconnectMcp({ + connectionId: scenario.connection.id, + expectedRevision: current.current.connection.configRevision, + expectedGeneration: current.current.connection.authGeneration, + }); + // Short-expiry checks are complete. Keep the probe credential valid through rebind. + await fixture.oauth({ expiresInSeconds: 3600 }); + await authorizeOAuth(ctx, principals.admin, scenario.connection.id); + } else { + await fixture.oauth( + change === "account" + ? { account: "account-two" } + : { scope: "tools:read tools:write" }, + ); + await new Promise((resolve) => setTimeout(resolve, 2100)); + } + const before = await fixture.observe(); + await sendScenarioPrompt(scenario, false, change); + const proofs = await pollProof(ctx, scenario, { + callId: `oauth-${change}`, + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + requireThat(proofs.length === 1, "single_changed_oauth_completion"); + if (change === "reconnect") { + assertOAuthRefusal(proofs[0]!.proof); + } else { + const trigger = await execution( + ctx, + scenario, + true, + `oauth-${change}`, + ); + requireThat( + trigger?.state === "failed", + "oauth_refresh_trigger_journal_failed", + ); + const result = + await scenario.reviewClient.integration.getMcpExecutionResult({ + sessionId: scenario.sessionId, + executionId: trigger.executionId, + }); + assertOAuthRefreshFailure(proofs[0]!.proof, result); + } + const after = await fixture.observe(); + assertNoDispatch(before, after); + let repeatedHash: string | undefined; + if (change !== "reconnect") { + requireThat( + after.oauth.rotations > before.oauth.rotations, + "changed_authority_actual_refresh_attempt", + ); + const rejected = await waitFor( + ctx, + "oauth_refresh_credential_broken", + authority, + (value) => value.credentialStatus === "broken", + ); + assertRejectedRefreshAuthority(old, rejected); + // A broken credential fails closed without another rotating exchange. + await sendScenarioPrompt(scenario, false, `${change}-stale`); + const repeated = await pollProof(ctx, scenario, { + callId: `oauth-${change}-stale`, + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + requireThat(repeated.length === 1, "single_broken_oauth_failure"); + const retry = await execution( + ctx, + scenario, + true, + `oauth-${change}-stale`, + ); + requireThat(retry?.state === "failed", "oauth_broken_retry_failed"); + const retryResult = + await scenario.reviewClient.integration.getMcpExecutionResult({ + sessionId: scenario.sessionId, + executionId: retry.executionId, + }); + assertOAuthRefreshFailure(repeated[0]!.proof, retryResult); + repeatedHash = repeated[0]!.sha256; + const afterRetry = await fixture.observe(); + assertNoDispatch(after, afterRetry); + requireThat( + afterRetry.oauth.rotations === after.oauth.rotations, + "broken_oauth_does_not_refresh_again", + ); + assertRejectedRefreshAuthority(old, await authority()); + if (change === "scope") await fixture.oauth({ scope: "tools:read" }); + await authorizeOAuth(ctx, principals.admin, scenario.connection.id); + } + const stale = await waitFor( + ctx, + "oauth_binding_invalidated", + authority, + (value) => !value.current, + ); + assertInvalidatedAuthority(old.bindingHash, stale); + const refusalMarker = + change === "reconnect" + ? `${change}-stale` + : `${change}-reauthorized-stale`; + await sendScenarioPrompt(scenario, false, refusalMarker); + const refusals = await pollProof(ctx, scenario, { + callId: `oauth-${refusalMarker}`, + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + requireThat(refusals.length === 1, "single_stale_oauth_refusal"); + assertOAuthRefusal(refusals[0]!.proof); + const afterRefusal = await fixture.observe(); + assertNoDispatch(after, afterRefusal); + requireThat( + afterRefusal.oauth.rotations === after.oauth.rotations, + "stale_oauth_refusal_does_not_refresh_again", + ); + assertInvalidatedAuthority(old.bindingHash, await authority()); + // The new credential requires a new reviewed catalog and an explicit human rebind. + const current = await principals.admin.integration.getMcpSetup({ + connectionId: scenario.connection.id, + }); + requireThat( + current.current?.connection, + "oauth_current_for_catalog_review", + ); + const draft = await principals.admin.integration.discoverMcp({ + connectionId: scenario.connection.id, + expectedRevision: current.current.connection.configRevision, + }); + requireThat(draft.catalog, "oauth_reconnect_reviewed_draft"); + const catalogs = await principals.admin.integration.getMcpCatalogs({ + connectionId: scenario.connection.id, + }); + await principals.admin.integration.publishMcp({ + connectionId: scenario.connection.id, + draftId: draft.catalog.id, + expectedRevision: catalogs.catalogRevision, + tools: draft.catalog.tools.map((tool) => ({ + ...tool, + classification: + tool.upstreamName === "read_counter" ? "read" : "write", + enabled: true, + })), + }); + await enableReviewedOAuthConnection( + principals.admin.integration, + scenario.connection.id, + current.current.connection.configRevision, + ); + const recovery = await recoverCurrentBinding( + ctx, + scenario, + database, + change, + ); + requireThat( + recovery.old.binding_hash === old.bindingHash, + "oauth_recovery_old_binding", + ); + const fresh = await waitFor( + ctx, + "oauth_rebound_current", + authority, + (value) => + value.current && value.bindingHash === recovery.finalBindingHash, + ); + const read = await successfulRead(`${change}-rebound`); + assertions.push({ + assertion: + change === "account" + ? "account-change-invalidates" + : change === "scope" + ? "scope-expansion-invalidates" + : "reconnect", + observations: [ + policyObservation("public-rpc", scenario.sessionId, "current", { + old, + stale, + fresh, + recovery: { + oldBoot: recovery.old.boot_attempt, + newBoot: recovery.current.boot_attempt, + candidateHash: recovery.candidate.bindingHash, + }, + resultHash: read.resultHash, + }), + policyObservation( + "guest-proof", + scenario.sessionId, + change === "reconnect" ? "denied" : "failed", + { + hash: proofs[0]!.sha256, + delivery: change === "reconnect" ? "available" : "unavailable", + }, + ), + ...(repeatedHash + ? [ + policyObservation( + "guest-proof", + scenario.sessionId, + "failed", + { + hash: repeatedHash, + }, + ), + ] + : []), + policyObservation("guest-proof", scenario.sessionId, "denied", { + hash: refusals[0]!.sha256, + }), + policyObservation( + "fixture-observer", + scenario.connection.id, + "denied", + { before, after, afterRefusal }, + 0, + ), + ], + }); + } + await fixture.configure({ holdCalls: true }); + await sendScenarioPrompt(scenario, false, "credential-probe"); + await waitFor( + ctx, + "oauth_probe_live_model", + () => fixture.observe(), + (value) => value.held === 1, + ); + // The common helper observes both real processes and inline managed manifest, without printing secrets. + const { probeOAuthCredentialAbsence } = await import("./guest.ts"); + const probe = await probeOAuthCredentialAbsence( + ctx, + scenario.client, + scenario.sessionId, + ); + assertions.push({ + assertion: "no-credential-leak", + observations: [ + policyObservation( + "guest-proof", + scenario.sessionId, + "current", + probe, + ), + ], + }); + await fixture.release(); + const probeCall = await execution( + ctx, + scenario, + true, + "oauth-credential-probe", + ); + requireThat( + probeCall?.state === "succeeded", + "oauth_probe_call_finished", + ); + await pollProof(ctx, scenario, { + callId: "oauth-credential-probe", + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + return completeAssertions("oauth-authority", assertions); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, scenario); + }); + } + } finally { + await completeCaseCleanup(async () => { + await fixture.release(); + await fixture.configure({ holdCalls: false }); + await fixture.oauth({ + account: "account-one", + scope: "tools:read", + issuer: "configured", + expiresInSeconds: 3600, + }); + }); + } +}; diff --git a/orchestrator/scripts/mcp-stack/case-physical.ts b/orchestrator/scripts/mcp-stack/case-physical.ts new file mode 100644 index 000000000..93b23367a --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-physical.ts @@ -0,0 +1,396 @@ +import { waitFor } from "./bootstrap.ts"; +import { requireThat, sha256 } from "./context.ts"; +import { + completeAssertions, + type CaseDriver, + type DriverObservation, +} from "./driver.ts"; +import { + canonicalJson, + expectedCounterNative, + probeOAuthCredentialAbsence, +} from "./guest.ts"; +import { + guestColdObservation, + probeManagedAuthority, + probeWorkerControl, +} from "./guest-physical.ts"; +import { + HostObserver, + verifyColdChain, + verifyPhysicalBoot, +} from "./host-observer.ts"; +import { FixtureObserver } from "./observer.ts"; +import { recoverCurrentBinding, revokeProfileGrants } from "./rebind.ts"; +import { + caseBodyFailure, + completeCaseCleanup, + closeScenario, + delivered, + execution, + pollProof, + sendScenarioPrompt, + startScenario, + verifiedCounterResult, +} from "./scenario-runtime.ts"; + +function observe( + identity: string, + source: DriverObservation["source"], + state: DriverObservation["state"], + value: unknown, +): DriverObservation { + return { + source, + state, + identitySha256: sha256(identity), + contentSha256: sha256(canonicalJson(value)), + }; +} + +export function selectColdDenial( + positive: Awaited>, + records: Awaited>, +) { + requireThat(positive.length === 1, "cold_positive_proof_count"); + const original = positive[0]!; + requireThat( + original.proof.resultReceived === true && + original.proof.delivery === "available" && + original.proof.outcome === "succeeded" && + original.proof.isError === false && + original.proof.planSha256 !== undefined, + "cold_positive_proof_success", + ); + // Raw JSONL hashes identify retained records regardless of launch filename order. + requireThat( + records.length === positive.length + 1 && + records.filter((record) => record.sha256 === original.sha256).length === + 1, + "cold_positive_proof_retained", + ); + const denied = records.filter((record) => record.sha256 !== original.sha256); + requireThat( + denied.length === 1 && + denied.every( + ({ proof }) => + proof.callId === original.proof.callId && + proof.tool === original.proof.tool && + proof.planSha256 === original.proof.planSha256 && + proof.expectExposure === "absent" && + proof.protocolDenialCode === "unknown_tool" && + proof.requestAttempted === true && + proof.exposedMcpToolNamesSha256 === + `sha256:${sha256(canonicalJson([]))}`, + ), + "cold_old_execution_actual_denial", + ); + return denied; +} + +export const credentialNetworkIsolationDriver: CaseDriver = async ({ + ctx, + principals, + database, + harness, +}) => { + const fixture = new FixtureObserver(ctx); + await fixture.configure({ + reply: "json", + response: "normal", + holdCalls: true, + revision: 1, + }); + const before = await fixture.observe(); + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-physical-network`, + protocol: "modern", + upstreamName: "read_counter", + mode: "read", + caller: "owner", + automationWrite: false, + arguments: {}, + constraints: [], + callId: "physical-read", + defaultAllow: true, + authKind: "oauth", + expectedNativeResult: expectedCounterNative( + harness, + before.effects.count, + false, + ), + }); + try { + await sendScenarioPrompt(scenario); + await waitFor( + ctx, + "physical_actual_call_held", + () => fixture.observe(), + (v) => v.held === before.held + 1, + ); + const credentials = await probeOAuthCredentialAbsence( + ctx, + scenario.client, + scenario.sessionId, + ); + const host = new HostObserver(ctx); + const baseline = await host.logBaseline(); + const probe = await probeManagedAuthority( + ctx, + scenario.client, + scenario.sessionId, + ); + const rejected = await waitFor( + ctx, + "actual_managed_proxy_rejection", + () => host.proxyDenials(baseline, scenario.sessionId, probe.sni), + (v) => v.length > 0, + ); + const tls = await probeWorkerControl(ctx); + await fixture.release(); + const row = await execution(ctx, scenario); + requireThat(row?.state === "succeeded", "physical_guest_return_call"); + const result = await verifiedCounterResult( + scenario, + row.executionId, + before.effects.count, + false, + ); + const completion = await delivered(ctx, database, scenario); + const after = await fixture.observe(); + requireThat( + after.reads === before.reads + 1 && + after.effects.executions === before.effects.executions && + after.effects.count === before.effects.count, + "physical_network_no_extra_provider_call", + ); + const identity = scenario.sessionId; + const denial = observe(identity, "host-read-only", "denied", rejected); + return completeAssertions("credential-network-isolation", [ + { + assertion: "no-credential-in-guest", + observations: [ + observe(identity, "guest-proof", "current", credentials), + ], + }, + { + assertion: "default-allow-managed-authority-denied", + observations: [denial], + }, + { + assertion: "worker-control-denied", + observations: [observe(identity, "fixture-observer", "denied", tls)], + }, + { assertion: "positive-network-denial", observations: [denial] }, + { + assertion: "guest-return", + observations: [ + observe(identity, "guest-proof", "succeeded", completion), + observe(identity, "public-rpc", "succeeded", result), + observe(identity, "fixture-observer", "current", after), + ], + }, + ]); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await fixture.release(); + await fixture.configure({ holdCalls: false }); + await closeScenario(principals, scenario); + }); + } +}; + +export const diskOnlyColdRebindDriver: CaseDriver = async ({ + ctx, + principals, + database, + harness, +}) => { + const fixture = new FixtureObserver(ctx); + await fixture.configure({ + reply: "json", + response: "normal", + holdCalls: false, + revision: 1, + }); + const before = await fixture.observe(); + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-physical-cold`, + protocol: "modern", + upstreamName: "read_counter", + mode: "read", + caller: "owner", + automationWrite: false, + arguments: {}, + constraints: [], + callId: "cold-old-read", + callPlan: (alias) => ({ + steps: [ + { + marker: "before", + tool: alias, + arguments: {}, + callId: "cold-old-read", + resultFormat: "mcp", + expectedResult: expectedCounterNative( + harness, + before.effects.count, + false, + ), + }, + { + marker: "after", + tool: alias, + arguments: {}, + callId: "cold-old-read", + resultFormat: "mcp", + expectExposure: "absent", + }, + ], + }), + }); + try { + await sendScenarioPrompt(scenario, false, "before"); + const row = await execution(ctx, scenario); + requireThat(row?.state === "succeeded", "cold_old_actual_execution"); + await delivered(ctx, database, scenario); + const positive = await pollProof(ctx, scenario, { + callId: scenario.input.callId, + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + const host = new HostObserver(ctx); + const oldHost = await host.boot( + scenario.coordinator.source_key.context.target.sandbox_id, + ); + const oldAssets = await database.physicalBoot( + scenario.sessionId, + scenario.coordinator.boot_attempt, + ); + const oldPhysical = verifyPhysicalBoot( + ctx, + oldHost.record, + scenario.coordinator, + oldAssets, + ); + const oldGuest = await guestColdObservation( + ctx, + scenario.client, + scenario.sessionId, + true, + ); + requireThat( + oldGuest.manifestSha256 === oldPhysical.combinedSha256, + "old_actual_live_frozen_manifest", + ); + await revokeProfileGrants(principals.admin, scenario.profileId); + const recovery = await recoverCurrentBinding(ctx, scenario, database); + requireThat( + recovery.candidate.bindingTools.length === 0, + "physical_new_empty_binding", + ); + const captured = await host.boot( + recovery.old.source_key.context.target.sandbox_id, + ); + const currentHost = await host.boot( + recovery.current.source_key.context.target.sandbox_id, + ); + const chain = verifyColdChain(captured.record, currentHost.record); + const disk = await host.captureFile(captured.record); + const currentAssets = await database.physicalBoot( + scenario.sessionId, + recovery.current.boot_attempt, + ); + const currentPhysical = verifyPhysicalBoot( + ctx, + currentHost.record, + recovery.current, + currentAssets, + ); + const currentGuest = await guestColdObservation( + ctx, + scenario.client, + scenario.sessionId, + false, + ); + requireThat( + currentGuest.bootId !== oldGuest.bootId && + currentGuest.manifestSha256 === currentPhysical.combinedSha256 && + oldPhysical.nativeSha256 === currentPhysical.nativeSha256 && + oldPhysical.combinedSha256 !== currentPhysical.combinedSha256 && + oldPhysical.combinedCount === oldPhysical.nativeCount + 1 && + currentPhysical.combinedCount === currentPhysical.nativeCount, + "actual_cold_guest_and_frozen_bytes", + ); + await sendScenarioPrompt(scenario, false, "after"); + const records = await pollProof(ctx, scenario, { + callId: scenario.input.callId, + tool: scenario.alias, + planSha256: scenario.planSha256, + afterCount: positive.length, + }); + const denied = selectColdDenial(positive, records); + const after = await fixture.observe(); + requireThat( + after.reads === before.reads + 1 && + after.effects.executions === before.effects.executions && + after.effects.count === before.effects.count, + "cold_no_old_provider_reexecution", + ); + const currentRow = await execution(ctx, scenario); + requireThat( + currentRow?.executionId === row.executionId && + currentRow.state === "succeeded", + "cold_no_replacement_execution", + ); + const identity = scenario.sessionId; + const cold = observe(identity, "host-read-only", "retired", { + chain, + disk, + old: captured.sha256, + current: currentHost.sha256, + }); + const guest = observe(identity, "guest-proof", "started", { + old: oldGuest, + current: currentGuest, + }); + return completeAssertions("disk-only-cold-rebind", [ + { assertion: "disk-only-restore", observations: [cold, guest] }, + { + assertion: "new-native-and-mcp-frozen-bytes", + observations: [ + observe(identity, "coordinator-read-only", "started", { + old: oldPhysical, + current: currentPhysical, + }), + guest, + ], + }, + { + assertion: "old-target-retired-before-spawn", + observations: [ + cold, + observe(identity, "coordinator-read-only", "started", recovery), + ], + }, + { assertion: "no-old-ram", observations: [guest] }, + { + assertion: "no-old-execution", + observations: [ + observe(identity, "guest-proof", "denied", { positive, denied }), + observe(identity, "fixture-observer", "current", after), + ], + }, + ]); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, scenario); + }); + } +}; diff --git a/orchestrator/scripts/mcp-stack/case-recovery.ts b/orchestrator/scripts/mcp-stack/case-recovery.ts new file mode 100644 index 000000000..835a9613d --- /dev/null +++ b/orchestrator/scripts/mcp-stack/case-recovery.ts @@ -0,0 +1,864 @@ +import { requireThat, sha256 } from "./context.ts"; +import { waitFor } from "./bootstrap.ts"; +import { + canonicalJson, + proofPaths, + readGuestProofRecords, + expectedCounterNative, + releaseGuestBarrier, +} from "./guest.ts"; +import { + completeAssertions, + type CaseDriver, + type DriverObservation, +} from "./driver.ts"; +import { FixtureObserver } from "./observer.ts"; +import { Supervisor } from "./supervisor.ts"; +import { + caseBodyFailure, + completeCaseCleanup, + delivered, + pollGuestObservation, + pollProof, + toolCompletionEvents, + verifiedCounterResult, + closeScenario, + execution, + sendScenarioPrompt, + startScenario, + type StartedScenario, +} from "./scenario-runtime.ts"; +import { recoverCurrentBinding, revokeProfileGrants } from "./rebind.ts"; +import { McpCompletionSettlement } from "../../src/gen/engram/app/v1/integration_pb.ts"; +import type { Context } from "./context.ts"; + +async function completions( + ctx: Context, + scenario: StartedScenario, + callId: string, +) { + const paths = await proofPaths( + ctx, + scenario.client, + scenario.sessionId, + scenario.input.suffix, + ); + requireThat(paths.length <= 16, "recovery_proof_launch_bound"); + const records = ( + await Promise.all( + paths.map((path) => + readGuestProofRecords(ctx, scenario.client, scenario.sessionId, path), + ), + ) + ).flat(); + return records.filter( + (record) => + record.proof.callId === callId && + record.proof.tool === scenario.alias && + record.proof.planSha256 === scenario.planSha256 && + record.proof.resultReceived, + ); +} + +export function assertNewIdWriteDenial( + denied: Awaited>, +) { + requireThat( + denied.length > 0 && + denied.every( + ({ proof }) => + proof.resultReceived === true && + proof.delivery === "available" && + proof.outcome === null && + proof.isError === true, + ), + "new_id_actual_runtime_policy_refusal", + ); +} + +export function assertDuplicateWriteUnknown( + repeated: Awaited>, +) { + requireThat( + repeated.length > 0 && + repeated.every( + ({ proof }) => + proof.outcome === "outcome_unknown" && + proof.delivery === "unavailable", + ), + "duplicate_id_stays_unknown", + ); +} + +type SettledUnknownSnapshot = { + execution: { + executionId: string; + state: string; + runId: string; + toolCallId: string; + connectionId: string; + }; + result: { state: string; resultUnavailable: boolean; resultJson?: string }; + outbox: { attempts: number; relayed: boolean; acknowledged: boolean }; +}; + +export async function recoverSettledUnknown(input: { + before: SettledUnknownSnapshot; + restart: () => Promise; + readAfter: () => Promise; +}) { + const validate = (snapshot: SettledUnknownSnapshot) => { + requireThat( + snapshot.execution.state === "outcome_unknown" && + snapshot.result.state === "outcome_unknown" && + !snapshot.result.resultUnavailable && + snapshot.outbox.attempts >= 1 && + snapshot.outbox.relayed && + snapshot.outbox.acknowledged, + "settled_unknown_snapshot", + ); + if (snapshot.result.resultJson) { + const receipt = JSON.parse(snapshot.result.resultJson); + requireThat( + receipt.outcome === "outcome_unknown" && + receipt.delivery === "unavailable" && + receipt.output === undefined, + "unknown_has_no_invented_output", + ); + } + }; + validate(input.before); + const restart = await input.restart(); + const snapshot = await input.readAfter(); + validate(snapshot); + requireThat( + snapshot.execution.executionId === input.before.execution.executionId && + snapshot.execution.runId === input.before.execution.runId && + snapshot.execution.toolCallId === input.before.execution.toolCallId && + snapshot.execution.connectionId === input.before.execution.connectionId && + snapshot.result.resultJson === input.before.result.resultJson && + snapshot.outbox.attempts === input.before.outbox.attempts && + snapshot.outbox.relayed === input.before.outbox.relayed && + snapshot.outbox.acknowledged === input.before.outbox.acknowledged, + "settled_unknown_recovery_changed", + ); + return { restart, snapshot }; +} + +export const heldWriteRestartDriver: CaseDriver = async ({ + ctx, + principals, + database, + harness, +}) => { + const fixture = new FixtureObserver(ctx); + await fixture.configure({ + reply: "json", + response: "normal", + holdCalls: true, + revision: 1, + }); + const before = await fixture.observe(); + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-held-restart`, + protocol: "modern", + upstreamName: "write_counter", + mode: "write", + caller: "automation", + automationWrite: true, + arguments: { delta: 1 }, + constraints: [], + callId: "held-write-1", + callPlan: (alias) => ({ + steps: [ + { + marker: "first", + tool: alias, + arguments: { delta: 1 }, + callId: "held-write-1", + resultFormat: "mcp", + }, + { + marker: "second", + tool: alias, + arguments: { delta: 1 }, + callId: "held-write-2", + resultFormat: "mcp", + }, + ], + }), + }); + const identitySha256 = sha256( + canonicalJson({ + task: scenario.taskId, + session: scenario.sessionId, + call: scenario.input.callId, + }), + ); + const observations = ( + source: DriverObservation["source"], + state: DriverObservation["state"], + value: unknown, + ): DriverObservation[] => [ + { + source, + state, + identitySha256, + contentSha256: sha256(canonicalJson(value)), + ...(source === "public-rpc" + ? { actorSha256: sha256(scenario.reviewActorId) } + : {}), + }, + ]; + try { + await sendScenarioPrompt(scenario, false, "first"); + const held = await waitFor( + ctx, + "durable_committed_held_write", + () => fixture.observe(), + (value) => + value.held === 1 && + value.effects.executions === before.effects.executions + 1 && + value.effects.count === before.effects.count + 1, + ); + const dispatch = await execution(ctx, scenario, false); + requireThat(dispatch?.state === "dispatching", "held_write_dispatching"); + const supervisor = new Supervisor(ctx); + const worker = await supervisor.restart("worker"); + const orchestrator = await supervisor.restart("orchestrator"); + const terminal = await execution(ctx, scenario); + requireThat( + terminal?.state === "outcome_unknown" && + terminal.executionId === dispatch.executionId, + "durable_unknown_after_restart", + ); + const result = + await scenario.reviewClient.integration.getMcpExecutionResult({ + sessionId: scenario.sessionId, + executionId: terminal.executionId, + }); + requireThat( + result.state === "outcome_unknown" && !result.resultUnavailable, + "unknown_is_not_success_with_missing_result", + ); + if (result.resultJson) { + const receipt = JSON.parse(result.resultJson); + requireThat( + receipt.outcome === "outcome_unknown" && + receipt.delivery === "unavailable" && + receipt.output === undefined, + "unknown_has_no_invented_output", + ); + } + await fixture.release(); + await fixture.configure({ holdCalls: false }); + const first = await waitFor( + ctx, + "unknown_guest_receipt", + () => completions(ctx, scenario, "held-write-1"), + (records) => + records.some( + (record) => + record.proof.outcome === "outcome_unknown" && + record.proof.delivery === "unavailable", + ), + ); + assertDuplicateWriteUnknown(first); + const outbox = await waitFor( + ctx, + "unknown_guest_ack", + () => database.outbox(scenario.sessionId, scenario.input.callId), + (row) => row.attempts >= 1 && row.relayed && row.acknowledged, + ); + const recovered = await recoverSettledUnknown({ + before: { execution: terminal, result, outbox }, + restart: () => supervisor.restart("orchestrator"), + readAfter: async () => { + const current = await execution(ctx, scenario); + requireThat(current, "settled_unknown_execution_missing"); + return { + execution: current, + result: await scenario.reviewClient.integration.getMcpExecutionResult( + { + sessionId: scenario.sessionId, + executionId: current.executionId, + }, + ), + outbox: await database.outbox( + scenario.sessionId, + scenario.input.callId, + ), + }; + }, + }); + await sendScenarioPrompt(scenario, false, "second"); + const denied = await waitFor( + ctx, + "new_id_restriction_guest_receipt", + () => completions(ctx, scenario, "held-write-2"), + (records) => records.length > 0, + ); + assertNewIdWriteDenial(denied); + const state = await scenario.reviewClient.integration.getSessionMcpState({ + sessionId: scenario.sessionId, + omitBindings: true, + pageSize: 16, + }); + requireThat( + !state.hasMoreExecutions && + state.unknownWriteBlocks.includes(scenario.connection.id) && + state.executions.filter((row) => row.toolCallId === "held-write-1") + .length === 1 && + !state.executions.some( + (row) => + row.toolCallId === "held-write-2" && + ["dispatching", "succeeded", "outcome_unknown"].includes(row.state), + ), + "unknown_write_replay_restricted", + ); + const after = await fixture.observe(); + requireThat( + after.effects.executions === before.effects.executions + 1 && + after.effects.count === before.effects.count + 1 && + after.held === 0, + "one_effect_after_both_retries", + ); + return completeAssertions("held-write-restart", [ + { + assertion: "committed-held-barrier", + observations: observations("fixture-observer", "dispatching", held), + }, + { + assertion: "worker-restart", + observations: observations("supervisor", "running", worker), + }, + { + assertion: "orchestrator-restart", + observations: observations("supervisor", "running", orchestrator), + }, + { + assertion: "outcome-unknown", + observations: observations("public-rpc", "outcome_unknown", { + executionId: terminal.executionId, + state: terminal.state, + }), + }, + { + assertion: "one-effect", + observations: observations("fixture-observer", "outcome_unknown", { + before, + after, + }), + }, + { + assertion: "settled-unknown-recovery", + observations: [ + ...observations("guest-proof", "outcome_unknown", first), + ...observations("supervisor", "running", recovered.restart), + ...observations( + "public-rpc", + "outcome_unknown", + recovered.snapshot.result, + ), + ...observations( + "coordinator-read-only", + "relayed", + recovered.snapshot.outbox, + ), + ], + }, + { + assertion: "new-id-restricted", + observations: [ + ...observations("guest-proof", "denied", denied), + ...observations("public-rpc", "denied", { + blocks: state.unknownWriteBlocks, + executions: state.executions.map((row) => ({ + id: row.executionId, + call: row.toolCallId, + state: row.state, + })), + }), + ], + }, + ]); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await fixture.release(); + await fixture.configure({ holdCalls: false }); + await closeScenario(principals, scenario); + }); + } +}; + +export function assertAcknowledgedNativeCompletion( + completion: { tool_call_id: string; run_id: string; ok: boolean }[], + callId: string, + executionRunId: string, +) { + requireThat( + executionRunId.length > 0 && + completion.length === 1 && + completion[0]!.tool_call_id === callId && + completion[0]!.run_id.length > 0 && + completion[0]!.ok, + "ack_exact_native_run_call", + ); +} + +export const outboxGuestAckDriver: CaseDriver = async ({ + ctx, + principals, + database, + harness, +}) => { + const fixture = new FixtureObserver(ctx); + await fixture.configure({ + reply: "json", + response: "normal", + holdCalls: false, + revision: 1, + }); + const observed: Record = {}; + for (const targetGone of [false, true]) { + const before = await fixture.observe(); + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-outbox-${targetGone ? "gone" : "ack"}`, + protocol: "modern", + upstreamName: "write_counter", + mode: "write", + caller: "automation", + automationWrite: true, + arguments: { delta: 1 }, + constraints: [], + callId: "outbox-write-1", + completionBarrier: true, + expectedNativeResult: expectedCounterNative( + harness, + before.effects.count + 1, + false, + ), + }); + const identitySha256 = sha256( + canonicalJson({ + session: scenario.sessionId, + call: scenario.input.callId, + }), + ); + const observation = ( + source: DriverObservation["source"], + state: DriverObservation["state"], + value: unknown, + ): DriverObservation => ({ + source, + state, + identitySha256, + contentSha256: sha256(canonicalJson(value)), + ...(source === "public-rpc" + ? { actorSha256: sha256(scenario.reviewActorId) } + : {}), + }); + try { + await sendScenarioPrompt(scenario); + const row = await execution(ctx, scenario); + requireThat(row?.state === "succeeded", "outbox_verified_success"); + const result = await verifiedCounterResult( + scenario, + row.executionId, + before.effects.count + 1, + false, + ); + const proofs = await pollProof(ctx, scenario, { + callId: scenario.input.callId, + tool: scenario.alias, + }); + requireThat( + proofs.length === 1 && + proofs[0]!.proof.resultReceived === true && + proofs[0]!.proof.resultMatchesExpected === true && + proofs[0]!.proof.delivery === "available", + "outbox_actual_native_result", + ); + const held = await pollGuestObservation(ctx, scenario, { + callId: scenario.input.callId, + tool: scenario.alias, + observation: "completionBarrierHeld", + }); + const unacked = await database.outbox( + scenario.sessionId, + scenario.input.callId, + ); + requireThat( + unacked.relayed && + unacked.attempts >= 1 && + !unacked.acknowledged && + (await toolCompletionEvents(scenario)).length === 0, + "native_completion_held_before_ack", + ); + if (!targetGone) { + const supervisor = new Supervisor(ctx); + const orchestrator = await supervisor.restart("orchestrator"); + const coordinator = await supervisor.restart("coordinator"); + const recovered = await database.outbox( + scenario.sessionId, + scenario.input.callId, + ); + requireThat( + recovered.relayed && + !recovered.acknowledged && + (await toolCompletionEvents(scenario)).length === 0, + "unacked_delivery_survives_restart", + ); + await releaseGuestBarrier( + ctx, + scenario.client, + scenario.sessionId, + "completion", + ); + const released = await pollGuestObservation(ctx, scenario, { + callId: scenario.input.callId, + tool: scenario.alias, + observation: "completionBarrierReleased", + }); + const ack = await delivered(ctx, database, scenario); + assertAcknowledgedNativeCompletion( + ack.completion, + scenario.input.callId, + row.runId, + ); + observed["verified-persisted-result"] = [ + observation("public-rpc", "succeeded", result), + ]; + observed["coordinator-outbox"] = [ + observation("coordinator-read-only", "relayed", { + unacked, + recovered, + }), + ]; + observed.restart = [ + observation("supervisor", "running", { orchestrator, coordinator }), + ]; + observed["guest-proof"] = [ + observation("guest-proof", "succeeded", { proofs, held, released }), + ]; + observed["guest-ack"] = [ + observation("public-rpc", "acknowledged", { + executionRunId: row.runId, + completion: ack.completion, + }), + observation("coordinator-read-only", "acknowledged", { + outbox: ack.outbox, + target: ack.target, + }), + ]; + } else { + const recovery = await recoverCurrentBinding(ctx, scenario, database); + const target = await waitFor( + ctx, + "old_delivery_target_retired", + () => + database.deliveryTarget(scenario.sessionId, scenario.input.callId), + (value) => value.targetGone, + ); + const settled = await waitFor( + ctx, + "target_gone_outbox_settled", + () => database.outbox(scenario.sessionId, scenario.input.callId), + (value) => value.acknowledged, + ); + const afterResult = await verifiedCounterResult( + scenario, + row.executionId, + before.effects.count + 1, + false, + ); + requireThat( + afterResult.resultHash === result.resultHash && + afterResult.settlement === + McpCompletionSettlement.SETTLED_OR_TARGET_GONE && + (await toolCompletionEvents(scenario)).length === 0, + "target_gone_is_not_guest_ack", + ); + observed["target-gone-not-delivery"] = [ + observation("coordinator-read-only", "target_gone", { + target, + settled, + old: recovery.old.source_key, + current: recovery.current.source_key, + }), + observation("public-rpc", "target_gone", { + settlement: afterResult.settlement, + resultHash: afterResult.resultHash, + completions: 0, + }), + ]; + } + const after = await fixture.observe(); + requireThat( + after.effects.executions === before.effects.executions + 1 && + after.effects.count === before.effects.count + 1 && + after.held === 0, + "outbox_no_provider_replay", + ); + (observed["no-provider-replay"] ??= []).push( + observation("fixture-observer", "succeeded", { before, after }), + ); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, scenario); + }); + } + } + return completeAssertions( + "outbox-guest-ack", + Object.entries(observed).map(([assertion, observations]) => ({ + assertion, + observations, + })), + ); +}; + +export const revokeResumeDriver: CaseDriver = async ({ + ctx, + principals, + database, + harness, +}) => { + const fixture = new FixtureObserver(ctx); + await fixture.configure({ + reply: "json", + response: "normal", + holdCalls: false, + revision: 1, + }); + const before = await fixture.observe(); + const scenario = await startScenario(ctx, principals, database, { + harness, + suffix: `${harness}-revoke-resume`, + protocol: "modern", + upstreamName: "read_counter", + mode: "read", + caller: "owner", + automationWrite: false, + arguments: {}, + constraints: [], + callId: "resume-read-1", + callPlan: (alias) => ({ + steps: [ + { + marker: "initial", + tool: alias, + arguments: {}, + callId: "resume-read-1", + resultFormat: "mcp", + expectedResult: expectedCounterNative( + harness, + before.effects.count, + false, + ), + }, + { + marker: "old", + tool: alias, + arguments: {}, + callId: "resume-read-1", + resultFormat: "mcp", + expectExposure: "absent", + }, + { + marker: "new", + tool: alias, + arguments: {}, + callId: "resume-read-2", + resultFormat: "mcp", + expectExposure: "absent", + }, + ], + }), + }); + const identitySha256 = sha256( + canonicalJson({ + session: scenario.sessionId, + connection: scenario.connection.id, + }), + ); + const observation = ( + source: DriverObservation["source"], + state: DriverObservation["state"], + value: unknown, + ): DriverObservation => ({ + source, + state, + identitySha256, + contentSha256: sha256(canonicalJson(value)), + ...(source === "public-rpc" + ? { actorSha256: sha256(scenario.reviewActorId) } + : {}), + }); + try { + await sendScenarioPrompt(scenario, false, "initial"); + const initial = await execution(ctx, scenario); + requireThat( + initial?.state === "succeeded", + "revoke_initial_read_succeeded", + ); + await verifiedCounterResult( + scenario, + initial.executionId, + before.effects.count, + false, + ); + await delivered(ctx, database, scenario); + const prior = await pollProof(ctx, scenario, { + callId: scenario.input.callId, + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + requireThat( + prior.length === 1 && prior[0]!.proof.resultMatchesExpected === true, + "revoke_initial_actual_proof", + ); + const profile = await revokeProfileGrants( + principals.admin, + scenario.profileId, + ); + const disconnected = await principals.admin.integration.disconnectMcp({ + connectionId: scenario.connection.id, + expectedRevision: scenario.connection.configRevision, + expectedGeneration: scenario.connection.authGeneration, + }); + requireThat( + disconnected.connection?.id === scenario.connection.id && + disconnected.connection.authGeneration > + scenario.connection.authGeneration, + "revoke_actual_credential_generation", + ); + const recovery = await recoverCurrentBinding(ctx, scenario, database); + requireThat( + recovery.candidate.bindingTools.length === 0, + "revoke_current_empty_binding", + ); + await sendScenarioPrompt(scenario, false, "old"); + const retried = await pollProof(ctx, scenario, { + callId: scenario.input.callId, + tool: scenario.alias, + planSha256: scenario.planSha256, + expectExposure: "absent", + }); + const oldDenied = retried.filter( + (record) => record.proof.expectExposure === "absent", + ); + requireThat( + oldDenied.length === 1 && + oldDenied[0]!.proof.requestAttempted === true && + oldDenied[0]!.proof.protocolDenialCode === "unknown_tool", + "old_call_actual_protocol_denial", + ); + await sendScenarioPrompt(scenario, false, "new"); + const newDenied = await pollProof(ctx, scenario, { + callId: "resume-read-2", + tool: scenario.alias, + planSha256: scenario.planSha256, + }); + requireThat( + newDenied.length === 1 && + newDenied[0]!.proof.expectExposure === "absent" && + newDenied[0]!.proof.requestAttempted === true && + newDenied[0]!.proof.protocolDenialCode === "unknown_tool", + "old_alias_new_id_actual_protocol_denial", + ); + const emptyExposure = `sha256:${sha256(canonicalJson([]))}`; + requireThat( + [...oldDenied, ...newDenied].every( + (record) => record.proof.exposedMcpToolNamesSha256 === emptyExposure, + ), + "resumed_actual_empty_mcp_exposure", + ); + const after = await fixture.observe(); + const state = await scenario.reviewClient.integration.getSessionMcpState({ + sessionId: scenario.sessionId, + omitBindings: true, + pageSize: 16, + }); + requireThat( + !state.hasMoreExecutions && + state.executions.filter( + (row) => row.toolCallId === scenario.input.callId, + ).length === 1 && + state.executions.some( + (row) => + row.executionId === initial.executionId && + row.state === "succeeded", + ) && + !state.executions.some( + (row) => + row.toolCallId === "resume-read-2" && + [ + "authorized", + "dispatching", + "succeeded", + "outcome_unknown", + ].includes(row.state), + ) && + after.reads === before.reads + 1 && + after.effects.executions === before.effects.executions && + after.effects.count === before.effects.count, + "revoke_no_old_call_reexecution", + ); + return completeAssertions("revoke-resume", [ + { + assertion: "revocation", + observations: [ + observation("public-rpc", "denied", { + profileId: profile.id, + revision: String(profile.mcpGrantRevision), + authGeneration: disconnected.connection.authGeneration, + }), + ], + }, + { + assertion: "actual-resume", + observations: [ + observation("coordinator-read-only", "started", { + api: "managed_cold_recovery", + old: recovery.old.source_key, + current: recovery.current.source_key, + ordinaryResumeDenied: recovery.ordinaryResumeDenied, + }), + ], + }, + { + assertion: "current-binding", + observations: [ + observation("public-rpc", "current", { + hash: recovery.finalBindingHash, + tools: recovery.candidate.bindingTools, + }), + ], + }, + { + assertion: "old-alias-denied", + observations: [observation("guest-proof", "denied", newDenied)], + }, + { + assertion: "old-call-denied", + observations: [ + observation("guest-proof", "denied", oldDenied), + observation("fixture-observer", "denied", { before, after }), + ], + }, + ]); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + await closeScenario(principals, scenario); + }); + } +}; diff --git a/orchestrator/scripts/mcp-stack/cleanup-observer.test.ts b/orchestrator/scripts/mcp-stack/cleanup-observer.test.ts new file mode 100644 index 000000000..705e6670a --- /dev/null +++ b/orchestrator/scripts/mcp-stack/cleanup-observer.test.ts @@ -0,0 +1,210 @@ +import { afterAll, beforeAll, expect, spyOn, test } from "bun:test"; +import { mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import pg from "pg"; +import { DatabaseObserver } from "./observer.ts"; +import type { Context } from "./context.ts"; + +const sessionId = "11111111-1111-4111-8111-111111111111"; +const hostId = "22222222-2222-4222-8222-222222222222"; +const sandboxId = "33333333-3333-4333-8333-333333333333"; +const otherId = "44444444-4444-4444-8444-444444444444"; +let root: string; +beforeAll(async () => { + root = await mkdtemp(join(await realpath(tmpdir()), "mcp-cleanup-observer-")); + await writeFile( + join(root, "observer.dsn"), + "postgres://viewer@database.fixture.example/mcp_unit", + { mode: 0o600 }, + ); +}); +afterAll(async () => { + if (root) await rm(root, { recursive: true }); +}); + +async function withObserver( + run: (fixture: { + database: DatabaseObserver; + target: Record; + settled: Record; + queries: { sql: string; parameters: unknown[] }[]; + failQuery: () => void; + }) => Promise, +) { + // Connection setup reads only these fields; the pool query is the external boundary. + const ctx = { + runId: "unit", + runRoot: root, + deadlines: { requestMs: 1000 }, + observer: { + coordinatorDsnFile: join(root, "observer.dsn"), + orchestratorDsnFile: join(root, "observer.dsn"), + }, + } as Context; + const database = await DatabaseObserver.connect(ctx); + const target: Record = { + host_id: hostId, + sandbox_id: sandboxId, + binding_epoch: "7", + managed_task_id: "fixture-task", + managed_host_id: hostId, + managed_sandbox_id: sandboxId, + managed_binding_epoch: "7", + }; + const settled: Record = { + ...target, + host_id: null, + sandbox_id: null, + status: "completed", + pending_operations: false, + pending_boots: false, + pending_tombstone: false, + }; + const queries: { sql: string; parameters: unknown[] }[] = []; + let fail = false; + const query = spyOn(pg.Pool.prototype, "query").mockImplementation( + async (...args: unknown[]) => { + const sql = String(args[0]); + const parameters = args[1] as unknown[]; + queries.push({ sql, parameters }); + if (fail) throw new Error("observer unavailable"); + return { + rows: [sql.includes("pending_operations") ? settled : target], + rowCount: 1, + command: "SELECT", + oid: 0, + fields: [], + }; + }, + ); + database.own("fixture-task", sessionId); + try { + await run({ + database, + target, + settled, + queries, + failQuery: () => { + fail = true; + }, + }); + } finally { + query.mockRestore(); + await database.close(); + } +} + +test("cleanup observer retains exact ownership before reading settled obligations", async () => { + await withObserver(async ({ database, queries }) => { + await expect(database.cleanupSettled(sessionId)).rejects.toThrow( + "cleanup_target_not_retained", + ); + expect(queries).toHaveLength(0); + await database.retainCleanupTarget(sessionId, hostId); + expect(await database.cleanupSettled(sessionId)).toBe(true); + expect(queries[0]!.parameters).toEqual([sessionId]); + expect(queries[1]!.parameters).toEqual([sessionId, hostId, sandboxId]); + expect(queries[1]!.sql).toContain("state IN ('queued','running')"); + expect(queries[1]!.sql).toContain( + "session_managed_boot_intents WHERE session_id=$1 AND NOT completed", + ); + expect(queries[1]!.sql).toContain( + "sandbox_tombstones WHERE host_id=$2 AND sandbox_id=$3", + ); + await expect(database.retainCleanupTarget(otherId, hostId)).rejects.toThrow( + "observer_unowned_session", + ); + await expect(database.cleanupSettled(otherId)).rejects.toThrow( + "observer_unowned_session", + ); + expect(queries).toHaveLength(2); + }); +}); + +for (const pending of [ + "pending_operations", + "pending_boots", + "pending_tombstone", +]) + test(`destroy completion cannot settle cleanup with ${pending}`, async () => { + await withObserver(async ({ database, settled }) => { + await database.retainCleanupTarget(sessionId, hostId); + settled[pending] = true; + expect(await database.cleanupSettled(sessionId)).toBe(false); + settled[pending] = false; + expect(await database.cleanupSettled(sessionId)).toBe(true); + }); + }); + +for (const [field, value] of [ + ["host_id", otherId], + ["sandbox_id", otherId], + ["binding_epoch", "8"], + ["managed_task_id", "foreign-task"], + ["managed_host_id", otherId], + ["managed_sandbox_id", otherId], + ["managed_binding_epoch", "8"], + ["pending_tombstone", undefined], +] as const) + test(`cleanup fails closed when the retained ${field} identity or observation changes`, async () => { + await withObserver(async ({ database, settled }) => { + await database.retainCleanupTarget(sessionId, hostId); + settled[field] = value; + await expect(database.cleanupSettled(sessionId)).rejects.toThrow(); + }); + }); + +test("terminal status with a retained sandbox is not settled cleanup", async () => { + await withObserver(async ({ database, settled }) => { + await database.retainCleanupTarget(sessionId, hostId); + settled.sandbox_id = sandboxId; + expect(await database.cleanupSettled(sessionId)).toBe(false); + settled.sandbox_id = null; + settled.status = "active"; + expect(await database.cleanupSettled(sessionId)).toBe(false); + }); +}); + +test("ordinary and already-terminal managed targets retain their exact identity", async () => { + for (const managed of [false, true]) + await withObserver(async ({ database, target, settled }) => { + if (managed) { + target.host_id = null; + target.sandbox_id = null; + } else + for (const row of [target, settled]) + for (const field of [ + "managed_task_id", + "managed_host_id", + "managed_sandbox_id", + "managed_binding_epoch", + ]) + row[field] = null; + await database.retainCleanupTarget(sessionId, hostId); + expect(await database.cleanupSettled(sessionId)).toBe(true); + }); +}); + +test("unknown target, wrong owner, changed target and query failure do not prove cleanup", async () => { + await withObserver(async ({ database, target, failQuery }) => { + target.managed_task_id = "foreign-task"; + await expect( + database.retainCleanupTarget(sessionId, hostId), + ).rejects.toThrow("cleanup_exact_owned_target"); + target.managed_task_id = "fixture-task"; + await expect( + database.retainCleanupTarget(sessionId, otherId), + ).rejects.toThrow("cleanup_exact_owned_target"); + await database.retainCleanupTarget(sessionId, hostId); + target.sandbox_id = otherId; + target.managed_sandbox_id = otherId; + await expect( + database.retainCleanupTarget(sessionId, hostId), + ).rejects.toThrow("cleanup_target_changed"); + failQuery(); + await expect(database.cleanupSettled(sessionId)).rejects.toThrow( + "observer unavailable", + ); + }); +}); diff --git a/orchestrator/scripts/mcp-stack/cli.ts b/orchestrator/scripts/mcp-stack/cli.ts new file mode 100644 index 000000000..afdc0ae53 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/cli.ts @@ -0,0 +1,37 @@ +import { isAbsolute, resolve } from "node:path"; +import { RunnerFailure } from "./context.ts"; +import { FULL_MATRIX, selectedCases, type RunSelection } from "./manifest.ts"; + +export function parseArgs(args: readonly string[]) { + if (![2, 4].includes(args.length)) + throw new RunnerFailure("INVALID_CONTEXT", "arguments"); + const values = new Map(); + for (let index = 0; index < args.length; index += 2) { + const flag = args[index]!; + const value = args[index + 1]!; + if ( + !["--context", "--matrix", "--stage"].includes(flag) || + values.has(flag) || + value.startsWith("--") + ) + throw new RunnerFailure("INVALID_CONTEXT", "arguments"); + values.set(flag, value); + } + const context = values.get("--context"); + const matrix = values.get("--matrix"); + const stage = values.get("--stage"); + if ( + (stage !== undefined && stage !== "infrastructure") || + (stage !== undefined && matrix !== undefined) + ) + throw new RunnerFailure("INVALID_CONTEXT", "arguments"); + const selection: RunSelection = + stage === "infrastructure" + ? { stage } + : matrix === undefined + ? FULL_MATRIX + : { stage: "matrix", matrix }; + if (!context || !isAbsolute(context) || resolve(context) !== context) + throw new RunnerFailure("INVALID_CONTEXT", "arguments"); + return { context, selection, selected: selectedCases(selection) }; +} diff --git a/orchestrator/scripts/mcp-stack/context.ts b/orchestrator/scripts/mcp-stack/context.ts new file mode 100644 index 000000000..6f1d668e4 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/context.ts @@ -0,0 +1,350 @@ +import { createHash } from "node:crypto"; +import { lstat, open, realpath } from "node:fs/promises"; +import { constants } from "node:fs"; +import { dirname, isAbsolute, join, resolve, sep } from "node:path"; +import { z } from "zod"; +import { RunnerFailure } from "./failure.ts"; + +export { RunnerFailure } from "./failure.ts"; +export function requireThat( + condition: unknown, + seam: string, +): asserts condition { + if (!condition) throw new RunnerFailure("ASSERTION_FAILED", seam); +} +const hash = z.string().regex(/^[a-f0-9]{64}$/); +const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/); +const id = z.string().regex(/^[a-zA-Z0-9_-]{1,96}$/); +const path = z.string().refine((v) => isAbsolute(v) && resolve(v) === v); +const origin = z.string().refine((v) => { + try { + const u = new URL(v); + return ( + u.origin === v && + u.protocol === "https:" && + u.hostname.endsWith(".fixture.example") && + !u.username && + !u.password + ); + } catch { + return false; + } +}); +const file = z.strictObject({ path, sha256: hash }); +const endpoint = z.strictObject({ + origin, + ca: file, + serverFingerprint: z.string().regex(/^(?:[A-F0-9]{2}:){31}[A-F0-9]{2}$/), +}); +const fixtureEndpoint = endpoint.extend({ + configuration: file, + runRoot: path, + tokenFile: path, + resourceKeyFile: path, + resourceOrigin: origin, + issuerOrigin: origin, + authorizationOrigin: origin, + tokenOrigin: origin, + redirectUri: z.string().url(), +}); +const variants = z.strictObject({ + normal: file, + "global-disabled": file.optional(), + "zone-disabled": file.optional(), +}); +export type ConfigurationId = "normal" | "global-disabled" | "zone-disabled"; +export type FixtureKind = "modern" | "legacy"; +const human = z.strictObject({ + email: z.string().regex(/^[a-z0-9-]+@fixture\.example$/), + passwordFile: path, + cookieFile: path, +}); +export const ContextSchema = z.strictObject({ + version: z.literal(1), + runId: id, + runRoot: path, + backend: z.enum(["firecracker", "vz"]), + sourceCommit: z.string().regex(/^[a-f0-9]{40}$/), + hostId: z.uuid(), + imageId: id, + imageDigest: digest, + artifacts: z.strictObject({ + coordinator: file, + orchestrator: file, + worker: file, + hostAgent: file, + fixture: file, + kernel: file, + guestImage: file, + claudeHarness: file, + codexHarness: file, + modelCli: file, + claudeBundle: file, + codexBundle: file, + helper: file, + }), + endpoints: z.strictObject({ + app: endpoint, + coordinator: endpoint, + workerControl: endpoint, + operator: endpoint.extend({ certificate: file, privateKeyFile: path }), + }), + protocolFixtures: z.strictObject({ + modern: fixtureEndpoint.extend({ protocol: z.literal("2026-07-28") }), + legacy: fixtureEndpoint.extend({ + protocol: z.enum(["2025-11-25", "2025-06-18"]), + }), + }), + publication: z.strictObject({ + candidateIdentity: digest, + expectedGeneration: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER) + .nullable(), + deploymentIdentity: digest, + networkIdentity: digest, + trustIdentity: digest, + zone: id, + }), + identities: z.strictObject({ + admin: human, + owner: human, + other: human, + unattendedKeyFile: path, + automationKeyFile: path, + routerSentinelFile: path, + }), + supervisor: z.strictObject({ + socket: path, + tokenFile: path, + configurations: z.strictObject({ + orchestrator: variants, + worker: z.strictObject({ normal: file }), + coordinator: z.strictObject({ normal: file }), + "host-agent": z.strictObject({ normal: file }), + }), + operatorCandidates: z.strictObject({ + normal: z.strictObject({ file, identity: digest }), + }), + }), + observer: z.strictObject({ + orchestratorDsnFile: path, + coordinatorDsnFile: path, + hostManagedRoot: path, + hostLogFile: path, + }), + guest: z.strictObject({ + stateDirectory: z.literal("/tmp/mcp-stack-fixture"), + cliPath: z.string().regex(/^\/[^\s]*\/engram-mcp-model-fixture$/), + }), + deadlines: z.strictObject({ + requestMs: z.number().int().min(100).max(30000), + stepMs: z.number().int().min(100).max(120000), + totalMs: z.number().int().min(1000).max(7200000), + }), + evidenceFile: path, +}); +export type Context = z.infer; +export type Endpoint = z.infer; +export type Artifact = z.infer; +export const sha256 = (value: string | Uint8Array) => + createHash("sha256").update(value).digest("hex"); + +export async function realOwnedPath( + root: string, + target: string, + absent = false, +): Promise { + if ( + !isAbsolute(root) || + resolve(root) !== root || + !isAbsolute(target) || + resolve(target) !== target || + (target !== root && !target.startsWith(root + sep)) + ) + throw new RunnerFailure("INVALID_CONTEXT", "owned_path"); + let cursor: string = sep; + for (const part of target.split(sep).filter(Boolean)) { + cursor = join(cursor, part); + try { + const stat = await lstat(cursor); + if (stat.isSymbolicLink() || (cursor !== target && !stat.isDirectory())) + throw new Error(); + if (cursor.startsWith(root) && stat.uid !== process.getuid?.()) + throw new Error(); + } catch (error) { + if ( + absent && + cursor === target && + error instanceof Error && + "code" in error && + error.code === "ENOENT" + ) + return; + throw new RunnerFailure("INVALID_CONTEXT", "owned_path"); + } + } +} +export async function privateFile( + root: string, + target: string, + limit = 65536, +): Promise { + await realOwnedPath(root, target); + const handle = await open(target, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stat = await handle.stat(); + if ( + !stat.isFile() || + stat.size > limit || + stat.size === 0 || + (stat.mode & 0o077) !== 0 + ) + throw new RunnerFailure("INVALID_CONTEXT", "private_file"); + return await handle.readFile(); + } finally { + await handle.close(); + } +} +export async function verifyArtifact( + root: string, + artifact: Artifact, +): Promise { + await realOwnedPath(root, artifact.path); + const handle = await open( + artifact.path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size === 0) throw new Error(); + const digest = createHash("sha256"); + for await (const chunk of handle.createReadStream({ autoClose: false })) + digest.update(chunk); + if (digest.digest("hex") !== artifact.sha256) throw new Error(); + } catch { + throw new RunnerFailure("INVALID_CONTEXT", "artifact_digest"); + } finally { + await handle.close(); + } +} +export async function loadContext(filename: string): Promise { + // The private context is a direct child of the run root; the root never comes from an environment default. + const root = dirname(filename); + const raw = await privateFile(root, filename); + let ctx: Context; + try { + ctx = ContextSchema.parse(JSON.parse(raw.toString("utf8"))); + } catch { + throw new RunnerFailure("INVALID_CONTEXT", "context_schema"); + } + if ( + ctx.runRoot !== root || + (await realpath(root)) !== root || + ((await lstat(root)).mode & 0o777) !== 0o700 + ) + throw new RunnerFailure("INVALID_CONTEXT", "private_root"); + const humanValues = Object.values({ + admin: ctx.identities.admin, + owner: ctx.identities.owner, + other: ctx.identities.other, + }); + if ( + new Set(humanValues.map((v) => v.email)).size !== 3 || + new Set(humanValues.map((v) => v.cookieFile)).size !== 3 + ) + throw new RunnerFailure("INVALID_CONTEXT", "distinct_identities"); + const outputs = [ + ...humanValues.map((v) => v.cookieFile), + ctx.identities.unattendedKeyFile, + ctx.identities.automationKeyFile, + ctx.evidenceFile, + ]; + if (new Set(outputs).size !== outputs.length) + throw new RunnerFailure("INVALID_CONTEXT", "distinct_outputs"); + for (const output of outputs) { + await realOwnedPath(root, output, true); + try { + await lstat(output); + } catch (e) { + if (e instanceof Error && "code" in e && e.code === "ENOENT") continue; + throw e; + } + throw new RunnerFailure("INVALID_CONTEXT", "output_exists"); + } + for (const fixture of Object.values(ctx.protocolFixtures)) { + await realOwnedPath(root, fixture.runRoot); + await verifyArtifact(fixture.runRoot, fixture.configuration); + await privateFile(fixture.runRoot, fixture.tokenFile); + await privateFile(fixture.runRoot, fixture.resourceKeyFile); + if (((await lstat(fixture.runRoot)).mode & 0o777) !== 0o700) + throw new RunnerFailure("INVALID_CONTEXT", "fixture_private_root"); + if ( + new URL(fixture.redirectUri).origin !== ctx.endpoints.app.origin || + new URL(fixture.redirectUri).pathname !== "/integrations/mcp/callback" || + new URL(fixture.redirectUri).search || + new URL(fixture.redirectUri).hash + ) + throw new RunnerFailure("INVALID_CONTEXT", "fixture_redirect"); + } + if ( + ctx.protocolFixtures.modern.runRoot === + ctx.protocolFixtures.legacy.runRoot || + ctx.protocolFixtures.modern.origin === ctx.protocolFixtures.legacy.origin || + ctx.protocolFixtures.modern.resourceOrigin === + ctx.protocolFixtures.legacy.resourceOrigin + ) + throw new RunnerFailure("INVALID_CONTEXT", "distinct_protocol_fixtures"); + if ( + ctx.deadlines.requestMs > ctx.deadlines.stepMs || + ctx.deadlines.stepMs > ctx.deadlines.totalMs + ) + throw new RunnerFailure("INVALID_CONTEXT", "deadlines"); + for (const artifact of Object.values(ctx.artifacts)) + await verifyArtifact(root, artifact); + for (const ep of [ + ...Object.values(ctx.endpoints), + ...Object.values(ctx.protocolFixtures), + ]) + await verifyArtifact(root, ep.ca); + await verifyArtifact(root, ctx.endpoints.operator.certificate); + for (const secret of [ + ...humanValues.map((v) => v.passwordFile), + ...Object.values(ctx.protocolFixtures).flatMap((fixture) => [ + fixture.resourceKeyFile, + fixture.tokenFile, + ]), + ctx.identities.routerSentinelFile, + ctx.endpoints.operator.privateKeyFile, + ctx.supervisor.tokenFile, + ctx.observer.orchestratorDsnFile, + ctx.observer.coordinatorDsnFile, + ]) + await privateFile(root, secret); + for (const configs of Object.values(ctx.supervisor.configurations)) + for (const artifact of Object.values(configs)) + await verifyArtifact(root, artifact); + for (const candidate of Object.values(ctx.supervisor.operatorCandidates)) + await verifyArtifact(root, candidate.file); + await realOwnedPath(root, ctx.observer.hostManagedRoot); + const hostRoot = await lstat(ctx.observer.hostManagedRoot); + if (!hostRoot.isDirectory() || (hostRoot.mode & 0o777) !== 0o700) + throw new RunnerFailure("INVALID_CONTEXT", "host_managed_private_root"); + await realOwnedPath(root, ctx.observer.hostLogFile); + const hostLog = await lstat(ctx.observer.hostLogFile); + if ( + !hostLog.isFile() || + (hostLog.mode & 0o777) !== 0o600 || + hostLog.nlink !== 1 + ) + throw new RunnerFailure("INVALID_CONTEXT", "host_private_log"); + await realOwnedPath(root, ctx.supervisor.socket); + if ( + !(await lstat(ctx.supervisor.socket)).isSocket() || + ((await lstat(ctx.supervisor.socket)).mode & 0o077) !== 0 + ) + throw new RunnerFailure("INVALID_CONTEXT", "supervisor_socket"); + return ctx; +} diff --git a/orchestrator/scripts/mcp-stack/contracts.test.ts b/orchestrator/scripts/mcp-stack/contracts.test.ts new file mode 100644 index 000000000..2e546a45e --- /dev/null +++ b/orchestrator/scripts/mcp-stack/contracts.test.ts @@ -0,0 +1,2392 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { + chmod, + mkdtemp, + mkdir, + writeFile, + symlink, + rm, + readFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; +import { createServer } from "node:https"; +import { X509Certificate } from "node:crypto"; +import { + ContextSchema, + RunnerFailure, + privateFile, + realOwnedPath, + sha256, +} from "./context.ts"; +import { + HARNESS, + REQUIRED_CASES, + selectMatrix, + verifyResults, + type CaseResult, +} from "./manifest.ts"; +import { credentialHeaders, publicClient } from "./transport.ts"; +import { createCatalog, expectRejected, modelToolName } from "./bootstrap.ts"; +import { connectNodeAdapter } from "@connectrpc/connect-node"; +import { Code, ConnectError } from "@connectrpc/connect"; +import { ExecOutputSchema } from "../../src/gen/engram/app/v1/session_pb.ts"; +import { StatusSchema } from "./supervisor.ts"; +import { Evidence, verifyEvidence } from "./evidence.ts"; +import { requirePrerequisites } from "./preflight.ts"; +import { safeFailure, validateDrivers } from "./runner.ts"; +import { caseBodyFailure, completeCaseCleanup } from "./scenario-runtime.ts"; +import { drivers } from "./run.ts"; +import { + isProofFilename, + probeOAuthCredentialAbsence, + readGuestFile, +} from "./guest.ts"; +import { + guestColdObservation, + probeManagedAuthority, +} from "./guest-physical.ts"; +import { parseArgs } from "./cli.ts"; +import { create, fromBinary, toBinary } from "@bufbuild/protobuf"; +import { + IntegrationService, + IntegrationConnectionSchema, + McpCatalogSchema, + PutMcpCredentialRequestSchema, + PublishMcpRequestSchema, +} from "../../src/gen/engram/app/v1/integration_pb.ts"; +import { OperatorPublicationServiceStageRequestSchema } from "../../src/gen/engram/mcp/v1/operator_pb.ts"; +import { FixtureObserver } from "./observer.ts"; + +let root: string; +beforeAll(async () => { + // Resolve the repository from the source file; never use the primary checkout or OS temp directory. + const actual = join(import.meta.dir, "../../..", "var/mcp-stack-runner"); + await realOwnedPath(actual, actual); + root = await mkdtemp(join(actual, "unit-")); +}); +afterAll(async () => { + if (root) await rm(root, { recursive: true }); +}); +const h = "a".repeat(64); +const artifact = () => ({ path: "/owned/a", sha256: h }); +const ep = (name: string) => ({ + origin: `https://${name}.fixture.example`, + ca: artifact(), + serverFingerprint: Array(32).fill("AA").join(":"), +}); +function validContext() { + return { + version: 1, + runId: "unit", + runRoot: "/owned", + backend: "vz", + sourceCommit: "a".repeat(40), + hostId: "11111111-1111-4111-8111-111111111111", + imageId: "fixture-image", + imageDigest: `sha256:${h}`, + artifacts: Object.fromEntries( + [ + "coordinator", + "orchestrator", + "worker", + "hostAgent", + "fixture", + "kernel", + "guestImage", + "claudeHarness", + "codexHarness", + "modelCli", + "claudeBundle", + "codexBundle", + "helper", + ].map((key) => [key, artifact()]), + ), + endpoints: { + app: ep("app"), + coordinator: ep("coordinator"), + workerControl: ep("worker-control"), + operator: { + ...ep("operator"), + certificate: artifact(), + privateKeyFile: "/owned/key", + }, + }, + protocolFixtures: { + modern: { + ...ep("observer"), + configuration: artifact(), + runRoot: "/owned/modern", + resourceKeyFile: "/owned/modern/resource.key", + tokenFile: "/owned/token", + resourceOrigin: ep("resource").origin, + issuerOrigin: ep("issuer").origin, + authorizationOrigin: ep("authorization").origin, + tokenOrigin: ep("token").origin, + redirectUri: ep("app").origin + "/integrations/mcp/callback", + protocol: "2026-07-28", + }, + legacy: { + ...ep("observer-legacy"), + configuration: artifact(), + runRoot: "/owned/legacy", + resourceKeyFile: "/owned/legacy/resource.key", + tokenFile: "/owned/legacy/token", + resourceOrigin: ep("resource-legacy").origin, + issuerOrigin: ep("issuer-legacy").origin, + authorizationOrigin: ep("authorization-legacy").origin, + tokenOrigin: ep("token-legacy").origin, + redirectUri: ep("app").origin + "/integrations/mcp/callback", + protocol: "2025-11-25", + }, + }, + publication: { + candidateIdentity: `sha256:${h}`, + expectedGeneration: null, + deploymentIdentity: `sha256:${h}`, + networkIdentity: `sha256:${h}`, + trustIdentity: `sha256:${h}`, + zone: "fixture", + }, + identities: { + ...Object.fromEntries( + ["admin", "owner", "other"].map((role) => [ + role, + { + email: `${role}@fixture.example`, + passwordFile: `/owned/${role}.password`, + cookieFile: `/owned/${role}.cookie`, + }, + ]), + ), + unattendedKeyFile: "/owned/unattended.key", + automationKeyFile: "/owned/automation.key", + routerSentinelFile: "/owned/router.key", + }, + supervisor: { + socket: "/owned/control.sock", + tokenFile: "/owned/supervisor.token", + configurations: Object.fromEntries( + ["orchestrator", "worker", "coordinator", "host-agent"].map((role) => [ + role, + { normal: artifact() }, + ]), + ), + operatorCandidates: { + normal: { file: artifact(), identity: `sha256:${h}` }, + }, + }, + observer: { + coordinatorDsnFile: "/owned/c.dsn", + hostManagedRoot: "/owned/host/managed-boots", + hostLogFile: "/owned/host.log", + orchestratorDsnFile: "/owned/o.dsn", + }, + guest: { + stateDirectory: "/tmp/mcp-stack-fixture", + cliPath: "/opt/fixture/engram-mcp-model-fixture", + }, + deadlines: { requestMs: 1000, stepMs: 3000, totalMs: 10000 }, + evidenceFile: "/owned/evidence.jsonl", + }; +} +test("manifest has exactly twelve cases for each real harness", () => { + expect(selectMatrix("all")).toHaveLength(24); + for (const harness of HARNESS) + expect(REQUIRED_CASES.filter((c) => c.harness === harness)).toHaveLength( + 12, + ); + expect(() => selectMatrix("claude")).toThrow("matrix_selection"); + expect(() => selectMatrix("claude")).toThrow(RunnerFailure); + expect(() => verifyResults([])).toThrow(RunnerFailure); +}); +test("preflight rejects incomplete fixed controls before publication or signup", () => { + const ctx = ContextSchema.parse(validContext()); + expect(() => requirePrerequisites(ctx)).toThrow( + "guest_tls_request_deadline_6000ms", + ); + ctx.deadlines = { requestMs: 6000, stepMs: 6000, totalMs: 10000 }; + expect(() => requirePrerequisites(ctx)).toThrow( + "supervisor_orchestrator_global-disabled_configuration", + ); + for (const variant of ["global-disabled", "zone-disabled"] as const) { + ctx.supervisor.configurations.orchestrator[variant] = artifact(); + } + expect(() => requirePrerequisites(ctx)).not.toThrow(); + ctx.supervisor.operatorCandidates.normal.identity = `sha256:${"b".repeat(64)}`; + expect(() => requirePrerequisites(ctx)).toThrow( + "normal_candidate_publication_identity", + ); +}); +test("runner refuses missing drivers and removes raw error payloads", () => { + expect(() => validateDrivers(drivers)).not.toThrow(); + expect(() => validateDrivers({})).toThrow("complete_driver_registry"); + const failure = safeFailure(new Error("private-token-value")); + expect(failure.seam).toBe("unexpected_runner_error"); + expect(JSON.stringify(failure)).not.toContain("private-token-value"); +}); +test("acceptance proof discovery requires a fresh launch nonce, never PID-only fallback", () => { + expect( + isProofFilename("claude-cold", `claude-cold-12-${"a".repeat(32)}.json`), + ).toBe(true); + for (const name of [ + "claude-cold-12.json", + `claude-cold-12-${"A".repeat(32)}.json`, + `other-12-${"a".repeat(32)}.json`, + `claude-cold-12-${"a".repeat(31)}.json`, + `../claude-cold-12-${"a".repeat(32)}.json`, + ]) + expect(isProofFilename("claude-cold", name)).toBe(false); +}); +test("CLI requires an explicit private context and complete matrix", () => { + expect( + parseArgs(["--context", "/owned/context.json", "--matrix", "all"]).selected, + ).toHaveLength(24); + for (const args of [ + [], + ["--context", "relative.json", "--matrix", "all"], + ["--context", "/owned/context.json", "--matrix", "claude"], + ["--context", "/owned/context.json", "--context", "/owned/other.json"], + ["--context", "/owned/context.json", "--key", "secret"], + ]) + expect(() => parseArgs(args)).toThrow(); +}); +test("CLI rejects a present empty stage while retaining the full default", () => { + expect(() => + parseArgs(["--context", "/owned/context.json", "--stage", ""]), + ).toThrow("arguments"); + expect(parseArgs(["--context", "/owned/context.json"]).selection).toEqual({ + stage: "matrix", + matrix: "all", + }); + expect( + parseArgs(["--context", "/owned/context.json", "--matrix", "all"]) + .selection, + ).toEqual({ + stage: "matrix", + matrix: "all", + }); +}); +test("zero and missing case results fail", () => { + expect(() => verifyResults([])).toThrow(); + expect(() => + verifyResults( + REQUIRED_CASES.map((c) => ({ + key: c.key, + status: "NEEDS_CONTEXT", + assertions: [], + })), + ), + ).toThrow("matrix_incomplete"); +}); +test("every subassertion needs evidence", () => { + const results: CaseResult[] = REQUIRED_CASES.map((c) => ({ + key: c.key, + status: "PASS", + assertions: c.assertions.map((assertion) => ({ + assertion, + observationHashes: [h], + })), + })); + expect(() => verifyResults(results)).not.toThrow(); + results[0]!.assertions.pop(); + expect(() => verifyResults(results)).toThrow("missing_subassertion_evidence"); +}); +test("duplicate case cannot replace missing case", () => { + const results: CaseResult[] = REQUIRED_CASES.map((c) => ({ + key: c.key, + status: "PASS", + assertions: [], + })); + results[1] = results[0]!; + expect(() => verifyResults(results)).toThrow("missing_or_duplicate_case"); +}); +test("strict versioned context rejects unknown and credential-bearing fields", () => { + expect(ContextSchema.safeParse(validContext()).success).toBe(true); + expect( + ContextSchema.safeParse({ ...validContext(), version: 2 }).success, + ).toBe(false); + expect( + ContextSchema.safeParse({ ...validContext(), apiKey: "sensitive" }).success, + ).toBe(false); +}); +test("only physical backends and fixture HTTPS origins are valid", () => { + for (const backend of ["process", "fake", ""]) + expect( + ContextSchema.safeParse({ ...validContext(), backend }).success, + ).toBe(false); + for (const origin of [ + "http://app.fixture.example", + "https://localhost:5173", + "https://provider.example", + "https://app.fixture.example/", + "https://secret@app.fixture.example", + ]) { + const ctx = validContext(); + ctx.endpoints.app.origin = origin; + expect(ContextSchema.safeParse(ctx).success).toBe(false); + } +}); +test("artifact hashes and pin formats are strict", () => { + const ctx = validContext(); + ctx.endpoints.app.serverFingerprint = h; + expect(ContextSchema.safeParse(ctx).success).toBe(false); + ctx.endpoints.app = ep("app"); + ctx.imageDigest = h; + expect(ContextSchema.safeParse(ctx).success).toBe(false); +}); +test("owned file rejects traversal and symlink", async () => { + await writeFile(join(root, "secret"), "neutral", { mode: 0o600 }); + await symlink(join(root, "secret"), join(root, "link")); + await expect(privateFile(root, join(root, "link"))).rejects.toThrow( + "owned_path", + ); + await expect(realOwnedPath(root, root + "/../elsewhere")).rejects.toThrow( + "owned_path", + ); + expect((await privateFile(root, join(root, "secret"))).toString()).toBe( + "neutral", + ); +}); +test("owned file rejects loose permissions and oversize", async () => { + await writeFile(join(root, "loose"), "neutral", { mode: 0o644 }); + await chmod(join(root, "loose"), 0o644); + await expect(privateFile(root, join(root, "loose"))).rejects.toThrow( + "private_file", + ); + await expect(privateFile(root, join(root, "secret"), 2)).rejects.toThrow( + "private_file", + ); +}); +test("identity headers cannot mix cookie and unattended principals", () => { + expect( + credentialHeaders({ kind: "cookie", value: "session=neutral" }), + ).toEqual({ + cookie: "session=neutral", + }); + expect(credentialHeaders({ kind: "api-key", value: "engk_neutral" })).toEqual( + { + "x-api-key": "engk_neutral", + }, + ); + expect(() => + credentialHeaders({ kind: "cookie", value: "session=x\r\nx-api-key:bad" }), + ).toThrow(); + expect(() => + credentialHeaders({ kind: "api-key", value: "other" }), + ).toThrow(); +}); +test("compiled alias uses actual connection identity and exact upstream", () => { + expect(modelToolName("connection-1", "fixture-one", "read_counter")).toBe( + "mcp_fixture_one_" + sha256('["connection-1","read_counter"]').slice(0, 20), + ); + expect(modelToolName("connection-2", "fixture-one", "read_counter")).not.toBe( + modelToolName("connection-1", "fixture-one", "read_counter"), + ); +}); +test("generated wire request preserves optimistic revisions and exact operator identity", () => { + const stage = create(OperatorPublicationServiceStageRequestSchema, { + protocol: 1, + expectedGeneration: 0n, + configuredCandidateIdentity: `sha256:${h}`, + }); + expect( + fromBinary( + OperatorPublicationServiceStageRequestSchema, + toBinary(OperatorPublicationServiceStageRequestSchema, stage), + ), + ).toEqual(stage); + const request = create(PutMcpCredentialRequestSchema, { + connectionId: "connection-1", + expectedRevision: 7, + expectedGeneration: 9, + kind: "api_key", + header: "Authorization", + value: "Bearer neutral", + }); + expect( + fromBinary( + PutMcpCredentialRequestSchema, + toBinary(PutMcpCredentialRequestSchema, request), + ), + ).toEqual(request); + const publish = create(PublishMcpRequestSchema, { + connectionId: "connection-1", + draftId: "draft-1", + expectedRevision: 4, + }); + expect( + fromBinary( + PublishMcpRequestSchema, + toBinary(PublishMcpRequestSchema, publish), + ).expectedRevision, + ).toBe(4); +}); +test("supervisor rejects arbitrary commands and missing network enforcement", () => { + expect( + StatusSchema.safeParse({ version: 1, command: "arbitrary" }).success, + ).toBe(false); + expect( + StatusSchema.safeParse({ + version: 1, + network: { fixtureOnlyEgress: false }, + }).success, + ).toBe(false); +}); +test("evidence refuses supplied success receipts and raw result fields", async () => { + const ctx = ContextSchema.parse(validContext()); + ctx.evidenceFile = join(root, "evidence.jsonl"); + const evidence = await Evidence.create(ctx); + try { + await expect( + evidence.result({ + key: REQUIRED_CASES[0]!.key, + status: "PASS", + assertions: REQUIRED_CASES[0]!.assertions.map((assertion) => ({ + assertion, + observationHashes: [h], + })), + }), + ).rejects.toThrow("case_evidence_provenance"); + await expect( + evidence.observe({ + source: "fixture-observer", + caseKey: REQUIRED_CASES[0]!.key, + identitySha256: h, + state: "succeeded", + contentSha256: h, + ...{ rawResult: "secret" }, + }), + ).rejects.toThrow(); + await expect(evidence.finish()).rejects.toThrow( + "missing_or_duplicate_case", + ); + } finally { + await evidence.close(); + } +}); + +test("native counter expectations preserve exact Codex strings and Claude metadata", async () => { + const { expectedCounterNative, canonicalJson } = await import("./guest.ts"); + expect(expectedCounterNative("claude", 1, false)).toEqual({ + content: [{ type: "text", text: "count=1" }], + structuredContent: { count: 1 }, + isError: false, + _meta: { + "engrams/toolResult": { + outcome: "succeeded", + delivery: "available", + resultUnavailable: false, + isError: false, + }, + }, + }); + expect(expectedCounterNative("codex", 1, false)).toEqual({ + success: true, + contentItems: [ + { type: "inputText", text: "count=1" }, + { type: "inputText", text: '{"structuredContent":{"count":1}}' }, + { + type: "inputText", + text: '{"delivery":"available","isError":false,"outcome":"succeeded","resultUnavailable":false}', + }, + ], + }); + expect(canonicalJson({ a: '{"z":1,"a":2}' })).not.toBe( + canonicalJson({ a: '{"a":2,"z":1}' }), + ); + expect(canonicalJson({ a: false })).not.toBe(canonicalJson({})); + expect(() => canonicalJson({ a: undefined })).toThrow(); +}); + +test("protocol fixtures and restart variants stay explicit", () => { + const ctx = validContext(); + expect(ContextSchema.safeParse(ctx).success).toBe(true); + ctx.protocolFixtures.legacy.protocol = "2026-07-28"; + expect(ContextSchema.safeParse(ctx).success).toBe(false); + expect( + ContextSchema.safeParse({ + ...validContext(), + supervisor: { + ...validContext().supervisor, + configurations: { orchestrator: { arbitrary: artifact() } }, + }, + }).success, + ).toBe(false); +}); + +test("fixed plans reject duplicate IDs, deep or large arguments and unknown fields", async () => { + const { encodeCallPlan } = await import("./call-plan.ts"); + const step = { + marker: "read", + tool: "mcp_counter", + arguments: {}, + callId: "read-1", + resultFormat: "mcp" as const, + }; + expect(JSON.parse(encodeCallPlan({ steps: [step] })).steps).toHaveLength(1); + expect(() => encodeCallPlan({ steps: [step, step] })).toThrow(); + expect(() => + encodeCallPlan({ + steps: [{ ...step, arguments: { text: "x".repeat(4096) } }], + }), + ).toThrow(); + let deep: Record = {}; + for (let i = 0; i < 16; i++) deep = { child: deep }; + expect(() => + encodeCallPlan({ steps: [{ ...step, arguments: deep }] }), + ).toThrow(); +}); +test("guest JSONL preserves each flushed completion and does not invent partial output", async () => { + const { parseGuestProofRecords } = await import("./guest.ts"); + const first = { callId: "call-1", tool: "mcp_counter", resultReceived: true }; + const second = { + callId: "call-2", + tool: "mcp_counter", + resultReceived: true, + }; + const records = parseGuestProofRecords( + Buffer.from(JSON.stringify(first) + "\n" + JSON.stringify(second) + "\n"), + ); + expect(records.map((value) => value.proof.callId)).toEqual([ + "call-1", + "call-2", + ]); + expect(records[0]?.sha256).not.toBe(records[1]?.sha256); + expect(parseGuestProofRecords(Buffer.from(JSON.stringify(first)))).toEqual( + [], + ); + expect(() => parseGuestProofRecords(Buffer.from("bad\n"))).toThrow(); +}); + +test("automation trigger requires its own private output and cannot reuse the user key field", () => { + const context = validContext(); + expect(ContextSchema.parse(context).identities.automationKeyFile).toBe( + "/owned/automation.key", + ); + const { automationKeyFile: omitted, ...identities } = context.identities; + expect(omitted).toBeDefined(); + expect(() => ContextSchema.parse({ ...context, identities })).toThrow(); +}); + +test("absent and premature plans cannot request fabricated expected results", async () => { + const { encodeCallPlan } = await import("./call-plan.ts"); + const step = { + marker: "missing", + tool: "mcp_counter", + arguments: {}, + callId: "missing-1", + resultFormat: "mcp" as const, + }; + expect( + JSON.parse( + encodeCallPlan({ steps: [{ ...step, expectExposure: "absent" }] }), + ).steps[0].expectExposure, + ).toBe("absent"); + for (const invalid of [ + { ...step, expectExposure: "absent" as const, expectedResult: {} }, + { ...step, expectExposure: "absent" as const, tool: "spawn_session" }, + { + ...step, + expectExposure: "absent" as const, + resultFormat: "json" as const, + }, + { + ...step, + expectExposure: "absent" as const, + completionProjection: "premature" as const, + }, + { ...step, completionProjection: "premature" as const, expectedResult: {} }, + { + ...step, + completionProjection: "premature" as const, + resultFormat: "json" as const, + }, + ]) + expect(() => encodeCallPlan({ steps: [invalid] })).toThrow(); +}); + +test("native barrier and absent denial records cannot masquerade as result receipts", async () => { + const { NativeProofSchema } = await import("./guest.ts"); + const identity = { + callId: "call-1", + tool: "mcp_counter", + planSha256: `sha256:${h}`, + }; + const held = { ...identity, observation: "completionBarrierHeld" }; + expect(NativeProofSchema.safeParse(held).success).toBe(true); + expect( + NativeProofSchema.safeParse({ ...held, resultReceived: true }).success, + ).toBe(false); + expect( + NativeProofSchema.safeParse({ ...held, eventSha256: `sha256:${h}` }) + .success, + ).toBe(false); + expect( + NativeProofSchema.safeParse({ + ...identity, + observation: "prematureCompletionEmitted", + eventSha256: `sha256:${h}`, + }).success, + ).toBe(true); + expect( + NativeProofSchema.safeParse({ + ...identity, + observation: "prematureCompletionEmitted", + }).success, + ).toBe(false); + const denied = { + ...identity, + expectExposure: "absent", + selectedToolName: identity.tool, + exposedMcpToolNamesSha256: `sha256:${h}`, + requestAttempted: true, + protocolDenialCode: "unknown_tool", + nativeDenialCode: -32602, + }; + expect(NativeProofSchema.safeParse(denied).success).toBe(true); + expect( + NativeProofSchema.safeParse({ ...denied, resultReceived: true }).success, + ).toBe(false); + expect( + NativeProofSchema.safeParse({ + ...denied, + selectedSchemaSha256: `sha256:${h}`, + }).success, + ).toBe(false); + expect( + NativeProofSchema.safeParse({ ...denied, nativeDenialCode: -32603 }) + .success, + ).toBe(false); +}); + +test("grant replacement preserves public profile state and exact publication CAS", async () => { + const { ProfileSchema, UpdateProfileRequestSchema } = + await import("../../src/gen/engram/app/v1/profile_pb.ts"); + const { grantUpdateRequest } = await import("./rebind.ts"); + const profile = create(ProfileSchema, { + id: "profile-1", + name: "Fixture", + imageId: "image-1", + envVars: { FIXTURE: "value" }, + network: { default: "allow" }, + mcpGrantRevision: 7n, + programmaticOnly: true, + integrationGrants: [{ connectionId: "connection-1", operation: "invoke" }], + }); + const request = create( + UpdateProfileRequestSchema, + grantUpdateRequest(profile, []), + ); + const decoded = fromBinary( + UpdateProfileRequestSchema, + toBinary(UpdateProfileRequestSchema, request), + ); + expect(decoded.expectedMcpGrantRevision).toBe(7n); + expect(decoded.integrationGrants).toEqual([]); + expect(decoded.envVars).toEqual({ FIXTURE: "value" }); + expect(decoded.network?.default).toBe("allow"); + expect(decoded.programmaticOnly).toBe(true); +}); + +test("only an exact ordered present-to-absent pair can reuse a native call ID", async () => { + const { encodeCallPlan } = await import("./call-plan.ts"); + const present = { + marker: "before", + tool: "mcp_counter", + arguments: { delta: 1 }, + callId: "same-call", + resultFormat: "mcp" as const, + }; + const absent = { + ...present, + marker: "after", + expectExposure: "absent" as const, + }; + expect( + JSON.parse(encodeCallPlan({ steps: [present, absent] })).steps, + ).toHaveLength(2); + for (const steps of [ + [absent, present], + [present, { ...absent, arguments: { delta: 2 } }], + [present, { ...absent, tool: "mcp_other" }], + [present, { ...absent, expectExposure: "present" as const }], + [present, absent, { ...absent, marker: "third" }], + ]) + expect(() => encodeCallPlan({ steps })).toThrow(); +}); + +test("fixture readiness waits for transport startup and rejects invalid responses without retry", async () => { + const keyPath = join(root, "observer.key"); + const certPath = join(root, "observer.pem"); + const certificate = Bun.spawn( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + keyPath, + "-out", + certPath, + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + ], + { stdout: "ignore", stderr: "ignore" }, + ); + expect(await certificate.exited).toBe(0); + await chmod(certPath, 0o600); + const cert = await readFile(certPath); + const token = "a".repeat(48); + const tokenPath = join(root, "observer.token"); + await writeFile(tokenPath, token, { mode: 0o600 }); + const ctx = ContextSchema.parse(validContext()); + ctx.runRoot = root; + ctx.deadlines = { requestMs: 100, stepMs: 250, totalMs: 10000 }; + const endpoint = ctx.protocolFixtures.legacy; + let status = 200; + let resource = endpoint.resourceOrigin + "/mcp"; + let issuer = endpoint.issuerOrigin; + let connections = 0; + let transportFailures = 1; + const requests: { + method?: string; + route?: string; + authorization?: string; + }[] = []; + const server = createServer( + { key: await readFile(keyPath), cert }, + (req, res) => { + requests.push({ + method: req.method, + route: req.url, + authorization: req.headers.authorization, + }); + res.writeHead(status, { "content-type": "application/json" }); + res.end(JSON.stringify({ version: 1, ready: true, resource, issuer })); + }, + ); + server.on("connection", (socket) => { + connections++; + if (transportFailures-- > 0) socket.destroy(); + }); + await new Promise((resolve) => server.listen(0, "localhost", resolve)); + try { + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("server_address"); + endpoint.origin = `https://localhost:${address.port}`; + endpoint.ca = { path: certPath, sha256: sha256(cert) }; + endpoint.serverFingerprint = new X509Certificate(cert).fingerprint256; + endpoint.tokenFile = tokenPath; + const observer = new FixtureObserver(ctx, "legacy"); + await observer.ready(); + expect(connections).toBe(2); + expect(requests).toEqual([ + { method: "GET", route: "/v1/ready", authorization: `Bearer ${token}` }, + ]); + + for (const failure of ["status", "resource", "issuer", "token"] as const) { + const before = connections; + status = failure === "status" ? 401 : 200; + resource = + endpoint.resourceOrigin + (failure === "resource" ? "/wrong" : "/mcp"); + issuer = endpoint.issuerOrigin + (failure === "issuer" ? "/wrong" : ""); + await writeFile(tokenPath, failure === "token" ? "invalid" : token); + await expect(observer.ready()).rejects.toThrow( + failure === "status" + ? "fixture_observer_status" + : failure === "token" + ? "observer_token" + : "fixture_identity", + ); + expect(connections - before).toBe(failure === "token" ? 0 : 1); + } + await writeFile(tokenPath, token); + const beforeWrite = connections; + transportFailures = 1; + await expect(observer.configure({ revision: 2 })).rejects.toThrow( + "https_request", + ); + expect(connections - beforeWrite).toBe(1); + + for (const failure of ["transport", "certificate"] as const) { + status = 200; + resource = endpoint.resourceOrigin + "/mcp"; + issuer = endpoint.issuerOrigin; + transportFailures = failure === "transport" ? 1000 : 0; + endpoint.serverFingerprint = + failure === "certificate" + ? Array(32).fill("00").join(":") + : new X509Certificate(cert).fingerprint256; + const before = connections; + const begun = performance.now(); + await expect(observer.ready()).rejects.toThrow( + "TRANSPORT_FAILED:https_request", + ); + expect(connections - before).toBeGreaterThan(1); + expect(performance.now() - begun).toBeLessThan(1000); + } + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test("guest proof reads use authenticated downloads with bounded verified bytes", async () => { + const keyPath = join(root, "download.key"); + const certPath = join(root, "download.pem"); + const certificate = Bun.spawn( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + keyPath, + "-out", + certPath, + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + ], + { stdout: "ignore", stderr: "ignore" }, + ); + expect(await certificate.exited).toBe(0); + await chmod(certPath, 0o600); + const cert = await readFile(certPath); + const bytes = Buffer.from('{"callId":"proof"}\n'); + const requests: { + url: string; + cookie?: string; + apiKey?: string; + origin?: string; + }[] = []; + const uploads: { + bytes: Buffer; + size?: string; + hash?: string; + contentType?: string; + }[] = []; + let receipt: unknown; + let status = 200; + let rpcCode: string | string[] | undefined; + const secret = "private-file-response-token"; + let body = bytes; + let size = String(body.length); + let hash = sha256(body); + let stall = false; + let chunked = false; + const server = createServer( + { key: await readFile(keyPath), cert }, + (req, res) => { + requests.push({ + url: req.url ?? "", + cookie: req.headers.cookie, + apiKey: req.headers["x-api-key"]?.toString(), + origin: req.headers.origin, + }); + if (stall) return; + if (req.method === "POST") { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + const uploaded = Buffer.concat(chunks); + uploads.push({ + bytes: uploaded, + size: req.headers["x-upload-size"]?.toString(), + hash: req.headers["x-upload-sha256"]?.toString(), + contentType: req.headers["content-type"], + }); + res.writeHead(status, { + "content-type": "application/json", + ...(rpcCode === undefined ? {} : { "x-engram-rpc-code": rpcCode }), + ...(status === 302 ? { location: "/redirect-target" } : {}), + }); + res.end( + JSON.stringify( + receipt ?? { + path: new URL(req.url!, "https://localhost").searchParams.get( + "path", + ), + size_bytes: uploaded.length, + sha256: sha256(uploaded), + }, + ), + ); + }); + return; + } + res.writeHead(status, { + "content-type": "application/octet-stream", + ...(chunked ? {} : { "content-length": size }), + "x-content-sha256": hash, + "x-private": secret, + ...(rpcCode === undefined ? {} : { "x-engram-rpc-code": rpcCode }), + ...(status === 302 ? { location: "/redirect-target" } : {}), + }); + res.end(body); + }, + ); + await new Promise((resolve) => server.listen(0, "localhost", resolve)); + try { + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("server_address"); + const ctx = ContextSchema.parse(validContext()); + ctx.runRoot = root; + ctx.endpoints.app = { + origin: `https://localhost:${address.port}`, + ca: { path: certPath, sha256: sha256(cert) }, + serverFingerprint: new X509Certificate(cert).fingerprint256, + }; + ctx.deadlines.requestMs = 2000; + const path = ctx.guest.stateDirectory + "/proof.json"; + for (const credential of [ + { kind: "cookie" as const, value: "fixture_session=owner" }, + { kind: "api-key" as const, value: "engk_owner" }, + ]) { + const client = await publicClient(ctx, credential); + expect(await readGuestFile(ctx, client, "owned-session", path)).toEqual( + bytes, + ); + expect(requests.at(-1)).toEqual({ + url: "/api/v1/sessions/owned-session/files?path=%2Ftmp%2Fmcp-stack-fixture%2Fproof.json", + cookie: credential.kind === "cookie" ? credential.value : undefined, + apiKey: credential.kind === "api-key" ? credential.value : undefined, + origin: ctx.endpoints.app.origin, + }); + } + const fixture = Buffer.alloc(4096, 65); + for (const credential of [ + { kind: "cookie" as const, value: "fixture_session=owner" }, + { kind: "api-key" as const, value: "engk_owner" }, + ]) { + const uploader = await publicClient(ctx, credential); + expect( + await uploader.uploadGuestFile("owned-session", path, fixture), + ).toEqual({ + sizeBytes: 4096, + sha256: sha256(fixture), + }); + expect(uploads.at(-1)).toEqual({ + bytes: fixture, + size: "4096", + hash: sha256(fixture), + contentType: "application/octet-stream", + }); + expect(requests.at(-1)).toMatchObject({ + cookie: credential.kind === "cookie" ? credential.value : undefined, + apiKey: credential.kind === "api-key" ? credential.value : undefined, + origin: ctx.endpoints.app.origin, + }); + } + const client = await publicClient(ctx, { + kind: "cookie", + value: "fixture_session=owner", + }); + for (receipt of [ + {}, + { path, size_bytes: 4095, sha256: sha256(fixture) }, + { path, size_bytes: 4096, sha256: "0".repeat(64) }, + { path: path + "wrong", size_bytes: 4096, sha256: sha256(fixture) }, + { path, size_bytes: 4096, sha256: sha256(fixture), extra: secret }, + secret.repeat(65536), + ]) { + await expect( + client.uploadGuestFile("owned-session", path, fixture), + ).rejects.toThrow("guest_file_upload_receipt"); + } + receipt = undefined; + const beforeOversize = requests.length; + await expect( + client.uploadGuestFile( + "owned-session", + path, + Buffer.alloc(1024 * 1024 + 1), + ), + ).rejects.toThrow("guest_file_upload_size"); + expect(requests.length).toBe(beforeOversize); + const { anonymousGuestFileStatus } = await import("./transport.ts"); + status = 401; + expect(await anonymousGuestFileStatus(ctx, "owned-session", path)).toBe( + 401, + ); + expect(requests.at(-1)).toMatchObject({ + cookie: undefined, + apiKey: undefined, + origin: ctx.endpoints.app.origin, + }); + status = 404; + const nonOwner = await publicClient(ctx, { + kind: "cookie", + value: "fixture_session=other", + }); + expect(await nonOwner.nonOwnerGuestFileStatus("owned-session", path)).toBe( + 404, + ); + expect(requests.at(-1)?.cookie).toBe("fixture_session=other"); + status = 200; + await expect( + anonymousGuestFileStatus(ctx, "owned-session", path), + ).rejects.toThrow("guest_file_status_http_200"); + status = 400; + rpcCode = "3"; + for (const action of [ + () => client.uploadGuestFile("owned-session", path, fixture), + () => anonymousGuestFileStatus(ctx, "owned-session", path), + () => nonOwner.nonOwnerGuestFileStatus("owned-session", path), + ]) { + const before = requests.length; + await expect(action()).rejects.toMatchObject({ + code: "TRANSPORT_FAILED", + seam: "guest_file_status_http_400_rpc_code_3", + }); + expect(requests.length).toBe(before + 1); + } + status = 302; + rpcCode = undefined; + const beforeRedirect = requests.length; + await expect( + client.uploadGuestFile("owned-session", path, fixture), + ).rejects.toThrow("guest_file_status_http_302"); + await expect( + anonymousGuestFileStatus(ctx, "owned-session", path), + ).rejects.toThrow("guest_file_status_http_302"); + expect(requests.length).toBe(beforeRedirect + 2); + status = 200; + + for (const invalid of [ + "../proof", + path + "/child", + ctx.guest.stateDirectory + "/..", + path + "?x=1", + path + ";id", + path + "\n", + ]) { + const before = requests.length; + await expect( + readGuestFile(ctx, client, "owned-session", invalid), + ).rejects.toThrow("guest_proof_path"); + expect(requests.length).toBe(before); + } + for (const denied of [401, 403, 404, 302]) { + status = denied; + const before = requests.length; + await expect( + readGuestFile(ctx, client, "owned-session", path), + ).rejects.toMatchObject({ + code: "TRANSPORT_FAILED", + seam: `guest_file_status_http_${denied}`, + }); + expect(requests.length).toBe(before + 1); + } + body = Buffer.from(JSON.stringify({ error: secret })); + size = String(body.length); + status = 400; + const fileErrors: Array<[string | string[] | undefined, string]> = [ + ["1", "guest_file_status_http_400_rpc_code_1"], + ["9", "guest_file_status_http_400_rpc_code_9"], + ["16", "guest_file_status_http_400_rpc_code_16"], + [undefined, "guest_file_status_http_400"], + ["", "guest_file_status_http_400"], + ["0", "guest_file_status_http_400"], + ["17", "guest_file_status_http_400"], + ["09", "guest_file_status_http_400"], + ["+9", "guest_file_status_http_400"], + ["9junk", "guest_file_status_http_400"], + ["9 9", "guest_file_status_http_400"], + ["9, 9", "guest_file_status_http_400"], + [["9", "9"], "guest_file_status_http_400"], + [secret, "guest_file_status_http_400"], + ]; + for (const [header, seam] of fileErrors) { + rpcCode = header; + for (const action of [ + () => readGuestFile(ctx, client, "owned-session", path), + () => client.uploadGuestFile("owned-session", path, fixture), + () => anonymousGuestFileStatus(ctx, "owned-session", path), + () => nonOwner.nonOwnerGuestFileStatus("owned-session", path), + ]) { + let failure: unknown; + try { + await action(); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(RunnerFailure); + if (!(failure instanceof RunnerFailure)) + throw new Error("expected_runner_failure"); + expect(failure.code).toBe("TRANSPORT_FAILED"); + expect(failure.seam).toBe(seam); + expect(failure.message).toBe(`TRANSPORT_FAILED:${seam}`); + expect(Object.keys(failure).sort()).toEqual(["code", "seam"]); + expect(JSON.stringify(safeFailure(failure))).not.toContain(secret); + expect(failure.stack).not.toContain(secret); + } + } + status = 600; + rpcCode = "9"; + await expect( + readGuestFile(ctx, client, "owned-session", path), + ).rejects.toMatchObject({ + code: "TRANSPORT_FAILED", + seam: "guest_file_status", + }); + status = 200; + body = bytes; + size = String(bytes.length); + // Success uses file metadata and bytes, regardless of an error diagnostic header. + expect(await readGuestFile(ctx, client, "owned-session", path)).toEqual( + bytes, + ); + rpcCode = undefined; + const pin = ctx.endpoints.app.serverFingerprint; + ctx.endpoints.app.serverFingerprint = Array(32).fill("00").join(":"); + const beforeBadPin = requests.length; + await expect( + readGuestFile(ctx, client, "owned-session", path), + ).rejects.toThrow("guest_file_download"); + await expect( + client.uploadGuestFile("owned-session", path, fixture), + ).rejects.toThrow("guest_file_upload"); + await expect( + anonymousGuestFileStatus(ctx, "owned-session", path), + ).rejects.toThrow("guest_file_denial"); + expect(requests.length).toBe(beforeBadPin); + ctx.endpoints.app.serverFingerprint = pin; + size = String(bytes.length + 1); + await expect( + readGuestFile(ctx, client, "owned-session", path), + ).rejects.toThrow("guest_file_download"); + size = String(bytes.length); + hash = "invalid"; + await expect( + readGuestFile(ctx, client, "owned-session", path), + ).rejects.toThrow("guest_file_metadata"); + hash = "0".repeat(64); + await expect( + readGuestFile(ctx, client, "owned-session", path), + ).rejects.toThrow("guest_file_integrity"); + hash = sha256(bytes); + size = String(1024 * 1024 + 1); + await expect( + readGuestFile(ctx, client, "owned-session", path), + ).rejects.toThrow("guest_file_metadata"); + chunked = true; + await expect( + readGuestFile(ctx, client, "owned-session", path), + ).rejects.toThrow("guest_file_metadata"); + chunked = false; + body = Buffer.alloc(1024 * 1024, 65); + hash = sha256(body); + size = String(body.length); + expect(await readGuestFile(ctx, client, "owned-session", path)).toEqual( + body, + ); + body = Buffer.alloc(0); + hash = sha256(body); + size = "0"; + expect(await readGuestFile(ctx, client, "owned-session", path)).toEqual( + body, + ); + ctx.deadlines.requestMs = 50; + stall = true; + await expect( + client.uploadGuestFile("owned-session", path, fixture), + ).rejects.toThrow("guest_file_upload"); + await expect( + anonymousGuestFileStatus(ctx, "owned-session", path), + ).rejects.toThrow("guest_file_denial"); + await expect( + readGuestFile(ctx, client, "owned-session", path), + ).rejects.toThrow("guest_file_download"); + expect( + requests.every((request) => request.url.startsWith("/api/v1/sessions/")), + ).toBe(true); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test("public RPC diagnostics retain descriptor methods and keep error data private", async () => { + const keyPath = join(root, "rpc-method.key"); + const certPath = join(root, "rpc-method.pem"); + const certificate = Bun.spawn( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + keyPath, + "-out", + certPath, + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + ], + { stdout: "ignore", stderr: "ignore" }, + ); + expect(await certificate.exited).toBe(0); + await chmod(certPath, 0o600); + const cert = await readFile(certPath); + const secret = "private-body-message-token"; + const errorBody = { code: "failed_precondition", message: secret }; + let streamFailure = true; + const envelope = (flags: number, bytes: Uint8Array) => { + const header = Buffer.alloc(5); + header[0] = flags; + header.writeUInt32BE(bytes.length, 1); + return Buffer.concat([header, bytes]); + }; + const first = create(ExecOutputSchema, { + event: { case: "started", value: { execId: "exec-fixed" } }, + }); + const server = createServer( + { key: await readFile(keyPath), cert }, + (req, res) => { + if (req.url === "/rpc/engram.app.v1.IntegrationService/DiscoverMcp") { + res.writeHead(400, { + "content-type": "application/json", + "x-private": secret, + }); + res.end(JSON.stringify(errorBody)); + } else if (req.url === "/rpc/engram.app.v1.SessionService/Exec") { + res.writeHead(200, { "content-type": "application/connect+proto" }); + res.write(envelope(0, toBinary(ExecOutputSchema, first))); + res.end( + envelope( + 2, + Buffer.from( + JSON.stringify( + streamFailure + ? { error: errorBody, metadata: { "x-private": [secret] } } + : {}, + ), + ), + ), + ); + } else { + res.writeHead(404); + res.end(); + } + }, + ); + await new Promise((resolve) => server.listen(0, "localhost", resolve)); + try { + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("server_address"); + const ctx = ContextSchema.parse(validContext()); + ctx.runRoot = root; + ctx.endpoints.app = { + origin: `https://localhost:${address.port}`, + ca: { path: certPath, sha256: sha256(cert) }, + serverFingerprint: new X509Certificate(cert).fingerprint256, + }; + const client = await publicClient(ctx, { + kind: "cookie", + value: "fixture_session=private-cookie", + }); + let original: unknown; + try { + await client.integration.discoverMcp({ connectionId: "private-input" }); + } catch (error) { + original = error; + } + expect(original).toBeInstanceOf(ConnectError); + if (!(original instanceof ConnectError)) + throw new Error("expected_connect_error"); + expect(original.code).toBe(Code.FailedPrecondition); + expect(original.rawMessage).toBe(secret); + expect(original.metadata.get("x-private")).toBe(secret); + expect( + await expectRejected( + () => Promise.reject(original), + [Code.FailedPrecondition], + ), + ).toBe(Code.FailedPrecondition); + const failure = safeFailure(original); + expect(failure.seam).toBe( + "public_rpc_IntegrationService.DiscoverMcp_code_9", + ); + expect(safeFailure(original).seam).toBe(failure.seam); + expect(original.rawMessage).toBe(secret); + expect(Object.keys(original).sort()).toEqual( + Object.keys(new ConnectError(secret, Code.FailedPrecondition)).sort(), + ); + expect(JSON.stringify(failure)).not.toContain("private"); + expect( + safeFailure(new ConnectError(secret, Code.FailedPrecondition)).seam, + ).toBe("public_rpc_code_9"); + expect(safeFailure(new Error(secret)).seam).toBe("unexpected_runner_error"); + const stream = client.sessions + .exec({ sessionId: "private-session", argv: ["private-arg"] }) + [Symbol.asyncIterator](); + expect((await stream.next()).value).toEqual(first); + let deferred: unknown; + try { + await stream.next(); + } catch (error) { + deferred = error; + } + expect(deferred).toBeInstanceOf(ConnectError); + if (!(deferred instanceof ConnectError)) + throw new Error("expected_stream_error"); + expect(deferred.rawMessage).toBe(secret); + expect(deferred.metadata.get("x-private")).toBe(secret); + expect(safeFailure(deferred).seam).toBe( + "public_rpc_SessionService.Exec_code_9", + ); + expect(JSON.stringify(safeFailure(deferred))).not.toContain("private"); + streamFailure = false; + const complete = []; + for await (const message of client.sessions.exec({ sessionId: "session" })) + complete.push(message); + expect(complete).toEqual([first]); + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test("catalog bootstrap enables the published connection with the authorized revision and checks the response", async () => { + const keyPath = join(root, "catalog.key"); + const certPath = join(root, "catalog.pem"); + const certificate = Bun.spawn( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + keyPath, + "-out", + certPath, + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + ], + { stdout: "ignore", stderr: "ignore" }, + ); + expect(await certificate.exited).toBe(0); + await chmod(certPath, 0o600); + const cert = await readFile(certPath); + const ctx = ContextSchema.parse(validContext()); + ctx.runRoot = root; + ctx.protocolFixtures.modern.resourceKeyFile = join( + root, + "catalog-resource.key", + ); + await writeFile( + ctx.protocolFixtures.modern.resourceKeyFile, + "fixture-neutral-resource-key", + { + mode: 0o600, + }, + ); + const created = create(IntegrationConnectionSchema, { + id: "connection-1", + alias: "fixture-catalog", + provider: "mcp", + enabled: false, + configRevision: 3, + authGeneration: 5, + catalogRevision: 0, + mcp: { + endpoint: ctx.protocolFixtures.modern.resourceOrigin + "/mcp", + zone: "fixture", + authKind: "api_key", + }, + }); + const authorized = create(IntegrationConnectionSchema, { + ...created, + configRevision: 7, + authGeneration: 6, + credentialIdentity: "credential-1", + }); + const draft = create(McpCatalogSchema, { + id: "draft-1", + connectionId: "connection-1", + kind: "draft", + revision: 11, + configRevision: 7, + authGeneration: 6, + tools: ["read_counter", "write_counter"].map((upstreamName) => ({ + upstreamName, + inputSchemaJson: '{"type":"object"}', + })), + }); + const published = create(McpCatalogSchema, { + ...draft, + id: "published-1", + kind: "published", + revision: 12, + }); + const enabled = create(IntegrationConnectionSchema, { + ...authorized, + enabled: true, + catalogRevision: 12, + testedAt: "2026-01-01T00:00:00Z", + }); + let mode = "success"; + const calls: string[] = []; + const handler = connectNodeAdapter({ + requestPathPrefix: "/rpc", + routes(router) { + router.service(IntegrationService, { + createConnection(req, rpc) { + calls.push("create"); + expect(rpc.requestHeader.get("cookie")).toBe("fixture_session=admin"); + expect(rpc.requestHeader.has("x-api-key")).toBe(false); + expect(req.alias).toBe("fixture-catalog"); + return { connection: created }; + }, + putMcpCredential(req) { + calls.push("authorize"); + expect(req.connectionId).toBe("connection-1"); + expect(req.expectedRevision).toBe(3); + expect(req.expectedGeneration).toBe(5); + return { connection: authorized }; + }, + discoverMcp(req) { + calls.push("discover"); + expect(req.connectionId).toBe("connection-1"); + expect(req.expectedRevision).toBe(7); + return { catalog: draft }; + }, + getMcpCatalogs(req) { + calls.push("review"); + expect(req.connectionId).toBe("connection-1"); + return { draft, configRevision: 7, catalogRevision: 11 }; + }, + publishMcp(req) { + calls.push("publish"); + expect(req.connectionId).toBe("connection-1"); + expect(req.draftId).toBe("draft-1"); + expect(req.expectedRevision).toBe(11); + expect( + req.tools.map((tool) => [ + tool.upstreamName, + tool.classification, + tool.enabled, + ]), + ).toEqual([ + ["read_counter", "read", true], + ["write_counter", "write", true], + ]); + return { catalog: published }; + }, + setConnectionEnabled(req, rpc) { + calls.push("enable"); + expect(calls).toEqual([ + "create", + "authorize", + "discover", + "review", + "publish", + "enable", + ]); + expect(rpc.requestHeader.get("cookie")).toBe("fixture_session=admin"); + expect(rpc.requestHeader.has("x-api-key")).toBe(false); + expect(req.id).toBe("connection-1"); + expect(req.expectedRevision).toBe(7); + expect(req.enabled).toBe(true); + if (mode === "rpc-failure") + throw new ConnectError("enable refused", Code.FailedPrecondition); + if (mode === "missing") return {}; + return { + connection: create(IntegrationConnectionSchema, { + ...enabled, + ...(mode === "wrong-id" ? { id: "connection-2" } : {}), + ...(mode === "wrong-revision" ? { configRevision: 8 } : {}), + ...(mode === "disabled" ? { enabled: false } : {}), + }), + }; + }, + }); + }, + }); + const server = createServer({ key: await readFile(keyPath), cert }, handler); + await new Promise((resolve) => server.listen(0, "localhost", resolve)); + try { + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("server_address"); + ctx.endpoints.app = { + origin: `https://localhost:${address.port}`, + ca: { path: certPath, sha256: sha256(cert) }, + serverFingerprint: new X509Certificate(cert).fingerprint256, + }; + const client = await publicClient(ctx, { + kind: "cookie", + value: "fixture_session=admin", + }); + const result = await createCatalog(ctx, client, "catalog"); + expect(calls).toEqual([ + "create", + "authorize", + "discover", + "review", + "publish", + "enable", + ]); + expect(result.connection).toEqual(enabled); + expect(result.catalog).toEqual(published); + for (mode of [ + "rpc-failure", + "wrong-id", + "wrong-revision", + "disabled", + "missing", + ]) { + calls.length = 0; + await expect(createCatalog(ctx, client, "catalog")).rejects.toThrow( + mode === "rpc-failure" ? "enable refused" : "enabled_connection", + ); + expect(calls).toEqual([ + "create", + "authorize", + "discover", + "review", + "publish", + "enable", + ]); + } + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } +}); + +test("matrix selectors reject empty, unknown and duplicate cases and retain canonical order", () => { + const first = REQUIRED_CASES[0]!, + last = REQUIRED_CASES.at(-1)!; + expect(selectMatrix(`${last.key},${first.key}`)).toEqual([first, last]); + for (const value of [ + "", + "claude", + "all,all", + `${first.key},`, + `,${first.key}`, + `${first.key},${first.key}`, + ` ${first.key}`, + "claude/unknown", + ]) + expect(() => selectMatrix(value)).toThrow("matrix_selection"); + expect(selectMatrix(REQUIRED_CASES.map((c) => c.key).join(","))).toEqual( + REQUIRED_CASES, + ); + expect(parseArgs(["--context", "/owned/context.json"]).selection).toEqual({ + stage: "matrix", + matrix: "all", + }); + expect( + parseArgs(["--context", "/owned/context.json", "--stage", "infrastructure"]) + .selection, + ).toEqual({ stage: "infrastructure" }); + expect( + parseArgs(["--context", "/owned/context.json", "--matrix", first.key]) + .selected, + ).toEqual([first]); + for (const args of [ + ["--context", "/owned/context.json", "--stage", "other"], + [ + "--context", + "/owned/context.json", + "--stage", + "infrastructure", + "--matrix", + "all", + ], + ]) + expect(() => parseArgs(args)).toThrow(); +}); + +const infrastructureRows = () => + (["ordinary", "managed"] as const).map((mode) => ({ + mode, + execSha256: h, + uploadSha256: h, + downloadSha256: h, + sizeBytes: 32, + unauthenticatedStatus: 401 as const, + nonOwnerStatus: 404 as const, + })); +for (const selection of [ + { stage: "infrastructure" }, + { stage: "matrix", matrix: REQUIRED_CASES[0]!.key }, + { stage: "matrix", matrix: "all" }, +] as const) + for (const mode of ["writer", "replay"] as const) + test(`evidence ${mode} requires prerequisite provenance for ${selection.stage === "matrix" ? selection.matrix : selection.stage}`, async () => { + const ctx = ContextSchema.parse(validContext()); + ctx.evidenceFile = join( + root, + `prerequisites-${mode}-${selection.stage === "matrix" ? selection.matrix.replaceAll("/", "-") : selection.stage}.jsonl`, + ); + const evidence = await Evidence.create(ctx, selection); + try { + const cases = + selection.stage === "matrix" ? selectMatrix(selection.matrix) : []; + const full = selection.stage === "matrix" && selection.matrix === "all"; + await recordCases(evidence, cases); + await evidence.infrastructure(infrastructureRows()); + if (mode === "writer") { + await expect(evidence.finish()).rejects.toThrow( + "prerequisites_evidence_required", + ); + } else { + const source = await readFile(ctx.evidenceFile, "utf8"); + const terminal = { + version: 1, + kind: full ? "complete" : "stage-complete", + ...selection, + selected: cases.length, + passed: cases.length, + stageCompleted: !full, + matrixCompleted: full, + physicalAcceptance: full, + }; + expect(() => + verifyEvidence( + ctx, + selection, + source + JSON.stringify(terminal) + "\n", + ), + ).toThrow("prerequisites_evidence_required"); + } + await recordPrerequisites(evidence); + await expect(recordPrerequisites(evidence)).rejects.toThrow( + "prerequisites_evidence_duplicate", + ); + await evidence.finish(); + const source = await readFile(ctx.evidenceFile, "utf8"); + expect(() => verifyEvidence(ctx, selection, source)).not.toThrow(); + const prerequisite = source + .split("\n") + .find((line) => line.includes('"kind":"prerequisites"'))!; + expect(() => + verifyEvidence( + ctx, + selection, + source.replace( + prerequisite + "\n", + prerequisite + "\n" + prerequisite + "\n", + ), + ), + ).toThrow("prerequisites_evidence_duplicate"); + } finally { + await evidence.close(); + } + }); +async function recordPrerequisites(evidence: Evidence) { + await evidence.prerequisites({ + supervisorSha256: h, + publicationSha256: h, + publicStateSha256: h, + }); +} +async function recordCases(evidence: Evidence, cases = REQUIRED_CASES) { + for (const selected of cases) { + const digest = await evidence.observe({ + source: "guest-proof", + caseKey: selected.key, + identitySha256: h, + state: "succeeded", + contentSha256: h, + }); + await evidence.result({ + key: selected.key, + status: "PASS", + assertions: selected.assertions.map((assertion) => ({ + assertion, + observationHashes: [digest], + })), + }); + } +} +test("full evidence needs successful ordered infrastructure before acceptance", async () => { + const ctx = ContextSchema.parse(validContext()); + ctx.evidenceFile = join(root, "full-infrastructure.jsonl"); + const evidence = await Evidence.create(ctx); + try { + await recordCases(evidence); + await expect(evidence.finish()).rejects.toThrow( + "infrastructure_evidence_required", + ); + for (const rows of [ + [], + infrastructureRows().reverse(), + infrastructureRows().slice(0, 1), + infrastructureRows().map((v) => ({ ...v, unauthenticatedStatus: 200 })), + infrastructureRows().map((v) => ({ ...v, execSha256: "A".repeat(64) })), + infrastructureRows().map((v) => ({ ...v, physicalAcceptance: true })), + ]) + await expect(evidence.infrastructure(rows)).rejects.toThrow(); + await evidence.infrastructure(infrastructureRows()); + await expect(evidence.infrastructure(infrastructureRows())).rejects.toThrow( + "infrastructure_evidence_duplicate", + ); + await recordPrerequisites(evidence); + await evidence.finish(); + } finally { + await evidence.close(); + } + const records = (await readFile(ctx.evidenceFile, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(records.at(-1)).toMatchObject({ + kind: "complete", + physicalAcceptance: true, + selected: 24, + passed: 24, + }); + expect(records.find((v) => v.kind === "infrastructure").rows).toEqual( + infrastructureRows(), + ); +}); +for (const stage of [ + "infrastructure", + "filtered", + "explicit-all-keys", +] as const) { + test(`${stage} evidence cannot claim full acceptance`, async () => { + const ctx = ContextSchema.parse(validContext()); + ctx.evidenceFile = join(root, `${stage}.jsonl`); + const matrix = + stage === "explicit-all-keys" + ? REQUIRED_CASES.map((v) => v.key).join(",") + : REQUIRED_CASES[0]!.key; + const selection = + stage === "infrastructure" + ? { stage: "infrastructure" as const } + : { stage: "matrix" as const, matrix }; + const cases = stage === "infrastructure" ? [] : selectMatrix(matrix); + const evidence = await Evidence.create(ctx, selection); + try { + if (stage === "infrastructure") { + await expect(evidence.finish()).rejects.toThrow( + "infrastructure_evidence_required", + ); + await evidence.infrastructure(infrastructureRows()); + } else { + await expect(evidence.finish()).rejects.toThrow( + "selected_cases_incomplete", + ); + await recordCases(evidence, cases); + await evidence.infrastructure(infrastructureRows()); + } + await recordPrerequisites(evidence); + await evidence.finish(); + } finally { + await evidence.close(); + } + const records = (await readFile(ctx.evidenceFile, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(records[0].selected).toBe(cases.length); + expect(records.at(-1)).toMatchObject({ + kind: "stage-complete", + stageCompleted: true, + matrixCompleted: false, + physicalAcceptance: false, + selected: cases.length, + passed: cases.length, + }); + expect(records.some((v) => v.kind === "complete")).toBe(false); + expect(() => verifyResults([])).toThrow("missing_or_duplicate_case"); + }); +} + +test("observer accepts the production automation task id and retains ownership guards", async () => { + const { DatabaseObserver } = await import("./observer.ts"); + const { automationRunId } = await import("../../src/automations/ids.ts"); + const ctx = ContextSchema.parse(validContext()); + ctx.runRoot = root; + ctx.observer.coordinatorDsnFile = join(root, "observer-own-c.dsn"); + ctx.observer.orchestratorDsnFile = join(root, "observer-own-o.dsn"); + for (const path of [ + ctx.observer.coordinatorDsnFile, + ctx.observer.orchestratorDsnFile, + ]) + await writeFile( + path, + "postgres://fixture:neutral@database.fixture.example/mcp_unit", + { + mode: 0o600, + }, + ); + // The pools are lazy. Registering ownership does not send a database query. + const database = await DatabaseObserver.connect(ctx); + const taskId = `automation:${automationRunId( + "11111111-1111-4111-8111-111111111111", + "manual:22222222-2222-4222-8222-222222222222", + )}`; + const sessionId = "33333333-3333-4333-8333-333333333333"; + try { + expect(() => database.own(taskId, sessionId)).not.toThrow(); + expect(() => database.own(taskId, sessionId)).toThrow( + "observer_owned_session", + ); + expect(() => database.own("different-task", sessionId)).toThrow( + "observer_owned_session", + ); + expect(() => database.own(taskId, "not-a-session")).toThrow( + "observer_owned_session", + ); + for (const invalid of [ + "", + "task space", + "task\nname", + "task/name", + "task\u0000name", + "a".repeat(129), + ]) + expect(() => + database.own(invalid, "44444444-4444-4444-8444-444444444444"), + ).toThrow("observer_owned_session"); + } finally { + await database.close(); + } +}); + +test("selected dispatcher collects cleaned case failures and stops on unsafe failures", async () => { + const { runSelectedCases } = await import("./runner.ts"); + const { DatabaseObserver } = await import("./observer.ts"); + const { SCENARIOS } = await import("./manifest.ts"); + const ctx = ContextSchema.parse(validContext()); + ctx.runRoot = root; + ctx.observer.coordinatorDsnFile = join(root, "selection-c.dsn"); + ctx.observer.orchestratorDsnFile = join(root, "selection-o.dsn"); + for (const path of [ + ctx.observer.coordinatorDsnFile, + ctx.observer.orchestratorDsnFile, + ]) + await writeFile( + path, + "postgres://fixture:neutral@database.fixture.example/mcp_unit", + { + mode: 0o600, + }, + ); + ctx.endpoints.app.ca.path = join(root, "selection-ca.pem"); + await writeFile(ctx.endpoints.app.ca.path, "unused-neutral-ca", { + mode: 0o600, + }); + // Pools and public clients are lazy. No query or request is made by these test drivers. + const database = await DatabaseObserver.connect(ctx); + const client = await publicClient(ctx, { + kind: "cookie", + value: "neutral=fixture", + }); + const principals = { + admin: client, + owner: client, + other: client, + unattended: client, + automation: client, + actorIds: { + admin: "1", + owner: "2", + other: "3", + unattended: "4", + automation: "5", + }, + }; + const input = { ctx, database, principals }; + const calls: string[] = []; + let mode = "pass"; + const events: string[] = []; + const registry: Record = Object.fromEntries( + SCENARIOS.map((s) => [ + s.id, + async ({ harness }: { harness: string }) => { + calls.push(`${harness}/${s.id}`); + if (harness === "claude" && mode === "setup") + throw new RunnerFailure("ASSERTION_FAILED", "setup_failed"); + try { + if ( + (harness === "claude" && ["body", "body-cleanup"].includes(mode)) || + mode === "both-body" + ) + throw new RunnerFailure( + "ASSERTION_FAILED", + `${harness}_case_assertion`, + ); + if (harness === "claude" && mode === "nested-cleanup") + await completeCaseCleanup(async () => { + throw new RunnerFailure( + "ASSERTION_FAILED", + "nested_cleanup_failed", + ); + }); + if (harness === "claude" && mode === "transport") + throw new RunnerFailure("TRANSPORT_FAILED", "https_request"); + if (harness === "claude" && mode === "unexpected") + throw new Error("private-error"); + if (harness === "claude" && mode === "unclassified") + throw new RunnerFailure( + "ASSERTION_FAILED", + "oauth_account_recovery_prepare_phase_unavailable_unclassified", + ); + if (harness === "claude" && mode === "unsafe-seam") + throw new RunnerFailure("ASSERTION_FAILED", "private error text"); + if (mode === "incomplete") return []; + return s.assertions.map((assertion) => ({ + assertion, + observations: [ + { + source: "guest-proof", + identitySha256: h, + state: "succeeded", + contentSha256: h, + }, + ], + })); + } catch (error) { + throw caseBodyFailure(error); + } finally { + await completeCaseCleanup(async () => { + events.push(`cleanup:${harness}`); + if ( + harness === "claude" && + ["cleanup", "body-cleanup"].includes(mode) + ) + throw new RunnerFailure("ASSERTION_FAILED", "cleanup_failed"); + }); + } + }, + ]), + ); + validateDrivers(registry); + const selection = { + stage: "matrix" as const, + matrix: "codex/catalog-fidelity,claude/catalog-fidelity", + }; + ctx.evidenceFile = join(root, "selected-dispatch.jsonl"); + const evidence = await Evidence.create(ctx, selection); + try { + await runSelectedCases(registry, selection, input, evidence); + expect(calls).toEqual([ + "claude/catalog-fidelity", + "codex/catalog-fidelity", + ]); + await evidence.infrastructure(infrastructureRows()); + await recordPrerequisites(evidence); + await evidence.finish(); + } finally { + await evidence.close(); + } + ctx.evidenceFile = join(root, "selected-failure.jsonl"); + const failed = await Evidence.create(ctx, selection); + mode = "incomplete"; + calls.length = 0; + events.length = 0; + try { + await expect( + runSelectedCases(registry, selection, input, failed, async () => { + events.push("ready"); + }), + ).rejects.toThrow("driver_subassertions_complete"); + expect(calls).toEqual([ + "claude/catalog-fidelity", + "codex/catalog-fidelity", + ]); + expect(events).toEqual(["cleanup:claude", "ready", "cleanup:codex"]); + await expect(failed.finish()).rejects.toThrow("selected_cases_incomplete"); + } finally { + await failed.close(); + } + const records = (await readFile(ctx.evidenceFile, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(records.at(-1)).toMatchObject({ + kind: "case", + key: "codex/catalog-fidelity", + status: "FAIL", + seam: "driver_subassertions_complete", + }); + expect(records.filter((record) => record.kind === "case")).toHaveLength(2); + expect( + records.some( + (record) => + record.kind === "complete" || record.kind === "stage-complete", + ), + ).toBe(false); + try { + for (const variant of [ + { mode: "body", seam: "claude_case_assertion", calls: 2, ready: true }, + { + mode: "both-body", + seam: "claude_case_assertion", + calls: 2, + ready: true, + }, + { mode: "cleanup", seam: "cleanup_failed", calls: 1, ready: false }, + { mode: "body-cleanup", seam: "cleanup_failed", calls: 1, ready: false }, + { + mode: "nested-cleanup", + seam: "nested_cleanup_failed", + calls: 1, + ready: false, + }, + { mode: "setup", seam: "setup_failed", calls: 1, ready: false }, + { mode: "transport", seam: "https_request", calls: 1, ready: false }, + { + mode: "unclassified", + seam: "oauth_account_recovery_prepare_phase_unavailable_unclassified", + calls: 1, + ready: false, + }, + { + mode: "unsafe-seam", + seam: "unexpected_runner_error", + calls: 1, + ready: false, + }, + { + mode: "unexpected", + seam: "unexpected_runner_error", + calls: 1, + ready: false, + }, + { + mode: "body", + seam: "infrastructure_unhealthy", + calls: 1, + ready: true, + unhealthy: true, + }, + { + mode: "body", + seam: "runner_total_deadline", + calls: 1, + ready: false, + expired: true, + }, + { + mode: "body", + seam: "runner_total_deadline", + calls: 1, + ready: true, + expiresDuringHealth: true, + }, + ]) { + mode = variant.mode; + calls.length = 0; + events.length = 0; + ctx.evidenceFile = join( + root, + `selected-${variant.mode}-${variant.seam}.jsonl`, + ); + // Deadline cases use distinct files without changing the run identity. + if (variant.expiresDuringHealth) ctx.evidenceFile += ".during-health"; + const output = await Evidence.create(ctx, selection); + let ready = false; + try { + const deadline = variant.expired + ? performance.now() - 1 + : performance.now() + + (variant.expiresDuringHealth ? 500 : ctx.deadlines.totalMs); + await expect( + runSelectedCases( + registry, + selection, + input, + output, + async () => { + ready = true; + events.push("ready"); + if (variant.unhealthy) + throw new RunnerFailure( + "ASSERTION_FAILED", + "infrastructure_unhealthy", + ); + if (variant.expiresDuringHealth) + await new Promise((resolve) => setTimeout(resolve, 510)); + }, + deadline, + ), + ).rejects.toThrow(variant.seam); + expect(calls).toHaveLength(variant.calls); + expect(ready).toBe(variant.ready); + if (variant.calls === 2) + expect(events).toEqual(["cleanup:claude", "ready", "cleanup:codex"]); + await expect(output.finish()).rejects.toThrow(); + } finally { + await output.close(); + } + const rows = (await readFile(ctx.evidenceFile, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(rows.filter((row) => row.kind === "case")).toHaveLength( + variant.calls, + ); + expect(rows.find((row) => row.kind === "case")).toMatchObject({ + key: "claude/catalog-fidelity", + status: "FAIL", + }); + if (variant.calls === 2) + expect(rows.at(-1)).toMatchObject({ + key: "codex/catalog-fidelity", + status: variant.mode === "both-body" ? "FAIL" : "PASS", + }); + expect( + rows.some( + (row) => row.kind === "complete" || row.kind === "stage-complete", + ), + ).toBe(false); + } + } finally { + await database.close(); + } +}); + +// Run the generated guest shell against a private proc tree, with no live guest. +test("guest process probes identify the staged harness and keep their guards", async () => { + const dir = await mkdtemp(join(root, "proc-probe-")); + const proc = join(dir, "proc"); + const shm = join(dir, "shm"); + const state = join(dir, "state"); + await mkdir(join(proc, "sys/kernel/random"), { recursive: true }); + await mkdir(shm); + await mkdir(state); + await writeFile(join(proc, "mounts"), `tmpfs ${shm} tmpfs rw 0 0\n`); + await writeFile( + join(proc, "sys/kernel/random/boot_id"), + "11111111-1111-4111-8111-111111111111\n", + ); + const ctx = ContextSchema.parse(validContext()); + const session = "probe-session"; + await writeFile( + join(state, "cold-disk-sentinel"), + sha256(`${ctx.runId}:${session}:cold-sentinel`), + ); + async function process(pid: number, argv: string[], env: string[]) { + const path = join(proc, String(pid)); + await mkdir(path, { recursive: true }); + await writeFile(join(path, "cmdline"), argv.join("\0") + "\0"); + await writeFile(join(path, "environ"), env.join("\0") + "\0"); + } + const harness = "/opt/fixture/harness"; + await process( + 1, + [harness, "--connect", "local", "--session-id", session], + ["ENGRAM_TOOLS=manifest"], + ); + await process(2, [ctx.guest.cliPath], []); + await process(3, [harness, "mcp-bridge"], ["ENGRAM_TOOLS=manifest"]); + await process(4, [harness, "hook-bridge"], ["ENGRAM_TOOLS=manifest"]); + async function generated(cold: boolean) { + let argv: string[] = []; + const client = { + sessions: { + async *exec(request: { argv?: string[] }) { + argv = request.argv ?? []; + yield create(ExecOutputSchema, { + event: { case: "started", value: {} }, + }); + yield create(ExecOutputSchema, { + event: { case: "exit", value: { exitStatus: 1 } }, + }); + }, + }, + }; + try { + if (cold) await guestColdObservation(ctx, client, session, false); + else await probeOAuthCredentialAbsence(ctx, client, session); + } catch { + /* The stub only captures the command. */ + } + expect(argv.length).toBeGreaterThan(3); + const script = argv[2]! + .replaceAll("/proc/", proc + "/") + .replaceAll("/dev/shm", shm) + .replaceAll("/tmp/mcp-stack-fixture", state); + return () => + spawnSync(argv[0]!, [argv[1]!, script, ...argv.slice(3)], { + encoding: "utf8", + }); + } + const oauth = await generated(false); + const cold = await generated(true); + expect(oauth().status).toBe(0); + expect(cold().status).toBe(0); + await process( + 1, + [harness, "--vsock-host", "123", "--session-id", session], + ["ENGRAM_TOOLS=manifest"], + ); + expect(cold().status).toBe(0); + await process(5, [harness, "--connect", "local"], ["ENGRAM_TOOLS=manifest"]); + expect(cold().status).toBe(12); + await rm(join(proc, "5"), { recursive: true }); + await process( + 3, + [harness, "mcp-bridge"], + ["ENGRAM_TOOLS=manifest", "TOKEN=fixture-access-123"], + ); + expect(oauth().status).toBe(15); + await process(3, [harness, "mcp-bridge"], []); + expect(oauth().status).toBe(16); + await process(3, [harness, "mcp-bridge"], ["ENGRAM_TOOLS=manifest"]); + await rm(join(proc, "2"), { recursive: true }); + expect(oauth().status).toBe(16); +}); + +test("managed proxy probe uses the fixture IP and SNI without guest DNS", async () => { + const dir = await mkdtemp(join(root, "managed-probe-")); + const bin = join(dir, "bin"); + await mkdir(bin); + const ctx = ContextSchema.parse(validContext()); + const fixture = ctx.protocolFixtures.modern; + fixture.runRoot = dir; + fixture.resourceOrigin = "https://resource.fixture.example:8443"; + const config = JSON.stringify({ + listen: { host: "10.82.7.30", port: 8443 }, + advertised: { resource: fixture.resourceOrigin }, + }); + const configPath = join(dir, "config.json"); + await writeFile(configPath, config, { mode: 0o600 }); + fixture.configuration = { path: configPath, sha256: sha256(config) }; + const managed = new URL(fixture.resourceOrigin).hostname; + const calls: string[][] = []; + let tlsStatus = 1; + async function executable(name: string, body: string) { + const path = join(bin, name); + await writeFile(path, `#!/bin/sh\n${body}\n`, { mode: 0o700 }); + await chmod(path, 0o700); + } + await executable("timeout", 'shift; exec "$@"'); + await executable("getent", "exit 99"); + const client = { + sessions: { + async *exec(request: { argv?: string[] }) { + const argv = request.argv ?? []; + calls.push(argv); + await executable("openssl", `exit ${tlsStatus}`); + const i = argv.indexOf("-c"); + const result = spawnSync( + "/bin/sh", + ["-c", argv[i + 1]!, ...argv.slice(i + 2)], + { env: { PATH: `${bin}:/usr/bin:/bin` }, encoding: "utf8" }, + ); + yield create(ExecOutputSchema, { + event: { case: "started", value: {} }, + }); + yield create(ExecOutputSchema, { + event: { case: "stdout", value: Buffer.from(result.stdout) }, + }); + yield create(ExecOutputSchema, { + event: { case: "exit", value: { exitStatus: result.status ?? 255 } }, + }); + }, + }, + }; + const result = await probeManagedAuthority(ctx, client, "session"); + expect(result.sni).toBe(managed); + expect(calls).toHaveLength(1); + expect(calls[0]).toContain("10.82.7.30:443"); + expect(calls[0]).not.toContain("10.82.7.30:8443"); + expect(calls[0]).toContain(managed); + expect(calls[0]!.join(" ")).not.toContain("getent"); + fixture.configuration.sha256 = "0".repeat(64); + await expect(probeManagedAuthority(ctx, client, "session")).rejects.toThrow( + "managed_fixture_config_digest", + ); + fixture.configuration.sha256 = sha256(config); + const wrongOrigin = JSON.stringify({ + listen: { host: "10.82.7.30", port: 8443 }, + advertised: { resource: "https://other.fixture.example:8443" }, + }); + await writeFile(configPath, wrongOrigin); + fixture.configuration.sha256 = sha256(wrongOrigin); + await expect(probeManagedAuthority(ctx, client, "session")).rejects.toThrow( + "managed_fixture_config_binding", + ); + await writeFile(configPath, config); + fixture.configuration.sha256 = sha256(config); + for (const status of [0, 124, 126, 127, 137]) { + tlsStatus = status; + await expect( + probeManagedAuthority(ctx, client, "session"), + ).rejects.toThrow(); + } +}); + +test("FC storage checkpoint must pass before infrastructure evidence permits cases", async () => { + const { completeInfrastructure } = await import("./runner.ts"); + for (const failure of [false, true]) { + const order: string[] = []; + const checkpoint = async () => { + order.push("physical"); + if (failure) throw Error("physical storage failure"); + }; + const evidence = { + infrastructure: async () => { + order.push("evidence"); + }, + }; + const task = completeInfrastructure( + "firecracker", + infrastructureRows(), + evidence, + { infrastructureReady: checkpoint }, + ); + if (failure) { + await expect(task).rejects.toThrow("physical storage failure"); + expect(order).toEqual(["physical"]); + } else { + await task; + expect(order).toEqual(["physical", "evidence"]); + } + } + let called = false; + await completeInfrastructure( + "vz", + infrastructureRows(), + { infrastructure: async () => {} }, + { + infrastructureReady: async () => { + called = true; + }, + }, + ); + expect(called).toBe(false); +}); diff --git a/orchestrator/scripts/mcp-stack/driver.ts b/orchestrator/scripts/mcp-stack/driver.ts new file mode 100644 index 000000000..b951285ee --- /dev/null +++ b/orchestrator/scripts/mcp-stack/driver.ts @@ -0,0 +1,38 @@ +import type { Context } from "./context.ts"; +import { requireThat } from "./context.ts"; +import type { Observation } from "./evidence.ts"; +import { SCENARIOS, type Harness, type ScenarioId } from "./manifest.ts"; +import type { DatabaseObserver } from "./observer.ts"; +import type { Principals } from "./scenario-runtime.ts"; + +export type DriverObservation = Omit; +export type DriverAssertion = { + assertion: string; + observations: DriverObservation[]; +}; +export type DriverInput = { + ctx: Context; + principals: Principals; + database: DatabaseObserver; + harness: Harness; +}; +export type CaseDriver = (input: DriverInput) => Promise; + +/** Validate completeness only. The driver must make each assertion against actual observations. */ +export function completeAssertions( + id: ScenarioId, + assertions: DriverAssertion[], +) { + const required = SCENARIOS.find((value) => value.id === id)!; + requireThat( + assertions.length === required.assertions.length && + new Set(assertions.map((value) => value.assertion)).size === + assertions.length && + required.assertions.every((name) => + assertions.some((value) => value.assertion === name), + ) && + assertions.every((value) => value.observations.length > 0), + "driver_subassertions_complete", + ); + return assertions; +} diff --git a/orchestrator/scripts/mcp-stack/evidence.ts b/orchestrator/scripts/mcp-stack/evidence.ts new file mode 100644 index 000000000..d8b685d52 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/evidence.ts @@ -0,0 +1,355 @@ +import { open, type FileHandle } from "node:fs/promises"; +import { isDeepStrictEqual } from "node:util"; +import { z } from "zod"; +import { requireThat, sha256, type Context } from "./context.ts"; +import { + FULL_MATRIX, + REQUIRED_CASES, + selectedCases, + verifyResults, + type CaseResult, + type RunSelection, +} from "./manifest.ts"; + +const observation = z.strictObject({ + source: z.enum([ + "public-rpc", + "fixture-observer", + "coordinator-read-only", + "orchestrator-read-only", + "host-read-only", + "guest-proof", + "supervisor", + ]), + caseKey: z.enum(REQUIRED_CASES.map((v) => v.key)), + // Object identities are hashed before reaching this sink. Never store arguments, results or headers. + identitySha256: z.string().regex(/^[a-f0-9]{64}$/), + actorSha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + state: z.enum([ + "created", + "started", + "current", + "denied", + "waiting_approval", + "authorized", + "dispatching", + "succeeded", + "failed", + "canceled", + "outcome_unknown", + "relayed", + "acknowledged", + "target_gone", + "retired", + "stopped", + "running", + ]), + contentSha256: z.string().regex(/^[a-f0-9]{64}$/), + count: z.number().int().nonnegative().safe().optional(), + bytes: z + .number() + .int() + .nonnegative() + .max(4 * 1024 * 1024) + .optional(), +}); +const infrastructureRow = z.strictObject({ + mode: z.enum(["ordinary", "managed"]), + execSha256: z.string().regex(/^[a-f0-9]{64}$/), + uploadSha256: z.string().regex(/^[a-f0-9]{64}$/), + downloadSha256: z.string().regex(/^[a-f0-9]{64}$/), + sizeBytes: z + .number() + .int() + .positive() + .safe() + .max(4 * 1024 * 1024), + unauthenticatedStatus: z.literal(401), + nonOwnerStatus: z.literal(404), +}); +export type InfrastructureRow = z.infer; +const infrastructure = z.tuple([ + infrastructureRow.extend({ mode: z.literal("ordinary") }), + infrastructureRow.extend({ mode: z.literal("managed") }), +]); +export type Observation = z.infer; +const hash = z.string().regex(/^[a-f0-9]{64}$/); +const prerequisites = z.strictObject({ + supervisorSha256: hash, + publicationSha256: hash, + publicStateSha256: hash, +}); +const caseResult = z.strictObject({ + key: z.string(), + status: z.enum(["PASS", "FAIL", "NEEDS_CONTEXT"]), + assertions: z.array( + z.strictObject({ + assertion: z.string(), + observationHashes: z.array(hash), + }), + ), + seam: z + .string() + .regex(/^[a-zA-Z0-9_./:-]{1,200}$/) + .optional(), +}); +const runSelection = z.discriminatedUnion("stage", [ + z.strictObject({ stage: z.literal("infrastructure") }), + z.strictObject({ stage: z.literal("matrix"), matrix: z.string() }), +]); + +/** Writer and reader use the same assertion, provenance and completion checks. */ +class EvidenceState { + private sequence = 0; + private readonly observed = new Map>(); + private readonly results: CaseResult[] = []; + private infrastructureVerified = false; + private prerequisitesRecorded = false; + private completed = false; + private readonly selected; + readonly selection: RunSelection; + constructor(selection: RunSelection) { + this.selection = runSelection.parse(selection); + this.selected = selectedCases(this.selection); + } + private active() { + requireThat(!this.completed, "evidence_already_complete"); + } + header(ctx: Context) { + return { + version: 1, + kind: "run", + runId: ctx.runId, + backend: ctx.backend, + sourceCommit: ctx.sourceCommit, + selected: this.selected.length, + ...this.selection, + physicalAcceptance: false, + artifactSha256: Object.fromEntries( + Object.entries(ctx.artifacts).map(([key, value]) => [ + key, + value.sha256, + ]), + ), + publicationIdentity: ctx.publication.candidateIdentity, + }; + } + observe(input: unknown) { + this.active(); + const value = observation.parse(input); + requireThat( + this.selected.some((c) => c.key === value.caseKey), + "evidence_case_identity", + ); + const digest = sha256(JSON.stringify(value)); + const hashes = this.observed.get(value.caseKey) ?? new Set(); + hashes.add(digest); + this.observed.set(value.caseKey, hashes); + return { + version: 1, + kind: "observation", + sequence: ++this.sequence, + ...value, + sha256: digest, + }; + } + prerequisites(input: unknown) { + this.active(); + const value = prerequisites.parse(input); + requireThat( + !this.prerequisitesRecorded, + "prerequisites_evidence_duplicate", + ); + this.prerequisitesRecorded = true; + return { version: 1, kind: "prerequisites", ...value }; + } + infrastructure(input: unknown) { + this.active(); + const rows = infrastructure.parse(input); + requireThat( + !this.infrastructureVerified, + "infrastructure_evidence_duplicate", + ); + requireThat( + rows.every((row) => row.uploadSha256 === row.downloadSha256), + "infrastructure_file_digest", + ); + this.infrastructureVerified = true; + return { version: 1, kind: "infrastructure", rows }; + } + result(input: unknown) { + this.active(); + const result = caseResult.parse(input); + const expected = this.selected.find((c) => c.key === result.key); + requireThat( + expected && !this.results.some((r) => r.key === result.key), + "evidence_case_identity", + ); + requireThat( + result.assertions.every( + (a) => + expected.assertions.some((name) => name === a.assertion) && + a.observationHashes.length > 0 && + a.observationHashes.every((h) => + this.observed.get(result.key)?.has(h), + ), + ), + "case_evidence_provenance", + ); + if (result.status === "PASS") + requireThat( + result.assertions.length === expected.assertions.length && + new Set(result.assertions.map((a) => a.assertion)).size === + expected.assertions.length, + "case_subassertion_count", + ); + this.results.push(result); + return { version: 1, kind: "case", ...result }; + } + finish() { + this.active(); + const full = + this.selection.stage === "matrix" && this.selection.matrix === "all"; + if (full) verifyResults(this.results); + else + requireThat( + this.results.length === this.selected.length && + this.selected.every((c) => + this.results.some((r) => r.key === c.key && r.status === "PASS"), + ), + "selected_cases_incomplete", + ); + requireThat( + this.infrastructureVerified, + "infrastructure_evidence_required", + ); + requireThat(this.prerequisitesRecorded, "prerequisites_evidence_required"); + this.completed = true; + return { + version: 1, + kind: full ? "complete" : "stage-complete", + ...this.selection, + selected: this.selected.length, + passed: this.results.length, + stageCompleted: !full, + matrixCompleted: full, + physicalAcceptance: full, + }; + } +} + +export const EVIDENCE_MAX_BYTES = 16 * 1024 * 1024; + +/** Replay the bounded stream through the writer's validators before adapter success. */ +export function verifyEvidence( + ctx: Context, + selection: RunSelection, + source: string, +) { + requireThat( + Buffer.byteLength(source) <= EVIDENCE_MAX_BYTES && source.endsWith("\n"), + "evidence_stream_bound", + ); + const lines = source.split("\n"); + lines.pop(); + requireThat(lines.length >= 2, "evidence_stream_records"); + const state = new EvidenceState(selection); + requireThat( + isDeepStrictEqual(JSON.parse(lines[0]!), state.header(ctx)), + "evidence_run_header", + ); + let completed = false; + for (let index = 1; index < lines.length; index++) { + const record = z + .looseObject({ version: z.literal(1), kind: z.string() }) + .parse(JSON.parse(lines[index]!)); + const { version, kind, ...body } = record; + switch (kind) { + case "prerequisites": + requireThat( + isDeepStrictEqual(record, state.prerequisites(body)), + "evidence_prerequisites_record", + ); + break; + case "infrastructure": { + const value = z.strictObject({ rows: infrastructure }).parse(body); + requireThat( + isDeepStrictEqual(record, state.infrastructure(value.rows)), + "evidence_infrastructure_record", + ); + break; + } + case "observation": { + const { sequence, sha256: digest, ...value } = body; + const expected = state.observe(value); + requireThat( + sequence === expected.sequence && digest === expected.sha256, + "evidence_observation_digest", + ); + break; + } + case "case": + state.result(body); + break; + case "complete": + case "stage-complete": + requireThat(index === lines.length - 1, "evidence_terminal_position"); + requireThat( + isDeepStrictEqual(record, state.finish()), + "evidence_completion_record", + ); + completed = true; + break; + default: + requireThat(false, "evidence_record_kind"); + } + } + requireThat(completed, "evidence_completion_missing"); +} + +export class Evidence { + private constructor( + private readonly handle: FileHandle, + private readonly state: EvidenceState, + ) {} + static async create(ctx: Context, selection: RunSelection = FULL_MATRIX) { + const state = new EvidenceState(selection); + const handle = await open(ctx.evidenceFile, "wx", 0o600); + const evidence = new Evidence(handle, state); + try { + await evidence.line(state.header(ctx)); + } catch (error) { + await handle.close(); + throw error; + } + return evidence; + } + private async line(value: unknown) { + await this.handle.writeFile(JSON.stringify(value) + "\n"); + } + async observe(input: Observation) { + const value = this.state.observe(input); + await this.line(value); + return value.sha256; + } + async prerequisites(input: z.infer) { + await this.line(this.state.prerequisites(input)); + } + async infrastructure(input: unknown) { + await this.line(this.state.infrastructure(input)); + } + async result(result: CaseResult) { + await this.line(this.state.result(result)); + } + async finish() { + await this.line(this.state.finish()); + await this.handle.sync(); + } + async close() { + await this.handle.sync(); + await this.handle.close(); + } +} diff --git a/orchestrator/scripts/mcp-stack/failure.ts b/orchestrator/scripts/mcp-stack/failure.ts new file mode 100644 index 000000000..02aff064c --- /dev/null +++ b/orchestrator/scripts/mcp-stack/failure.ts @@ -0,0 +1,12 @@ +export class RunnerFailure extends Error { + constructor( + readonly code: + | "INVALID_CONTEXT" + | "ASSERTION_FAILED" + | "NEEDS_CONTEXT" + | "TRANSPORT_FAILED", + readonly seam: string, + ) { + super(`${code}:${seam}`); + } +} diff --git a/orchestrator/scripts/mcp-stack/guest-physical.ts b/orchestrator/scripts/mcp-stack/guest-physical.ts new file mode 100644 index 000000000..be5abbc7d --- /dev/null +++ b/orchestrator/scripts/mcp-stack/guest-physical.ts @@ -0,0 +1,182 @@ +import { connect } from "node:tls"; +import { z } from "zod"; +import { + requireThat, + privateFile, + sha256, + verifyArtifact, + RunnerFailure, + type Context, +} from "./context.ts"; +import { guestExec, guestHarnessPath, type GuestExecClient } from "./guest.ts"; +import { tlsOptions } from "./transport.ts"; + +/** Verify the real worker rejects a TLS client without a certificate. */ +export async function probeWorkerControl(ctx: Context) { + const endpoint = ctx.endpoints.workerControl; + await verifyArtifact(ctx.runRoot, endpoint.ca); + const options = await tlsOptions(ctx, endpoint); + const url = new URL(endpoint.origin); + return await new Promise<{ + classification: "mtls_authentication_denial"; + alert: 116; + leafSha256: string; + }>((resolve, reject) => { + const socket = connect({ + ...options, + host: url.hostname, + port: Number(url.port || "443"), + servername: url.hostname, + minVersion: "TLSv1.3", + maxVersion: "TLSv1.3", + }); + let leafSha256 = ""; + let settled = false; + const finish = (ok: boolean) => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + if (ok) + resolve({ + classification: "mtls_authentication_denial", + alert: 116, + leafSha256, + }); + else + reject( + new RunnerFailure( + "ASSERTION_FAILED", + "worker_tls_authentication_denial", + ), + ); + }; + const timer = setTimeout(() => finish(false), ctx.deadlines.requestMs); + socket.on("secureConnect", () => { + const leaf = socket.getPeerCertificate().raw; + if (socket.authorized && socket.getProtocol() === "TLSv1.3" && leaf) + leafSha256 = sha256(leaf); + }); + socket.on("error", (error: NodeJS.ErrnoException) => { + finish( + !!leafSha256 && + (error.code === "ERR_SSL_TLSV13_ALERT_CERTIFICATE_REQUIRED" || + error.code === "ERR_SSL_TLSV1_ALERT_CERTIFICATE_REQUIRED"), + ); + }); + socket.on("data", () => finish(false)); + socket.on("close", () => finish(false)); + }); +} + +export async function probeManagedAuthority( + ctx: Context, + client: GuestExecClient, + sessionId: string, +) { + const fixture = ctx.protocolFixtures.modern; + const url = new URL(fixture.resourceOrigin); + const bytes = await privateFile(fixture.runRoot, fixture.configuration.path); + requireThat( + sha256(bytes) === fixture.configuration.sha256, + "managed_fixture_config_digest", + ); + const config = z + .object({ + listen: z.object({ + host: z.ipv4(), + port: z.number().int().min(1).max(65535), + }), + advertised: z.object({ resource: z.string() }), + }) + .parse(JSON.parse(bytes.toString("utf8"))); + requireThat( + config.advertised.resource === fixture.resourceOrigin && + config.listen.port === Number(url.port || "443"), + "managed_fixture_config_binding", + ); + const response = await guestExec(ctx, client, sessionId, [ + "/usr/bin/env", + "-i", + "PATH=/usr/bin:/bin", + "/bin/sh", + "-c", + `command -v openssl >/dev/null && command -v timeout >/dev/null || exit 127 +timeout 5s openssl s_client -connect "$1" -servername "$2" -tls1_3 < /dev/null >/dev/null 2>&1 +`, + "fixture-managed-destination", + // Port 443 follows the enforced proxy path; denial happens before upstream dial. + `${config.listen.host}:443`, + url.hostname, + ]); + requireThat( + response.status !== 0 && ![124, 126, 127, 137].includes(response.status), + "managed_destination_probe_denied", + ); + return { sni: url.hostname, sha256: sha256(String(response.status)) }; +} + +/** Fixed sentinels test loss of the old tmpfs state, not removal of every old RAM byte. */ +export async function guestColdObservation( + ctx: Context, + client: GuestExecClient, + sessionId: string, + prepare: boolean, +) { + const sentinel = sha256(`${ctx.runId}:${sessionId}:cold-sentinel`); + const script = `set -eu +cd /tmp/mcp-stack-fixture +awk '$2 == "/dev/shm" && $3 == "tmpfs" {found=1} END {exit !found}' /proc/mounts +if [ "$1" = prepare ]; then + [ ! -e /dev/shm/mcp-stack-sentinel ] && [ ! -L /dev/shm/mcp-stack-sentinel ] + [ ! -e cold-disk-sentinel ] && [ ! -L cold-disk-sentinel ] + umask 077 + set -C + printf '%s' "$2" > /dev/shm/mcp-stack-sentinel + printf '%s' "$2" > cold-disk-sentinel + sync -f cold-disk-sentinel +else + [ ! -e /dev/shm/mcp-stack-sentinel ] && [ ! -L /dev/shm/mcp-stack-sentinel ] +fi +[ ! -L cold-disk-sentinel ] && [ "$(cat cold-disk-sentinel)" = "$2" ] +boot=$(cat /proc/sys/kernel/random/boot_id) +count=0 +manifest_hash='' +for path in /proc/[0-9]*/cmdline; do + [ -r "$path" ] || continue + first=$(tr '\\000' '\\n' < "$path" | head -n 1) + [ "$first" = "$3" ] || continue + second=$(tr '\\000' '\\n' < "$path" | sed -n '2p') + case "$second" in --connect|--vsock-host) ;; *) continue;; esac + count=$((count+1)) + [ "$count" -le 1 ] || exit 12 + env_path=$(dirname "$path")/environ + values=$(head -c 1048577 "$env_path" | tr '\\000' '\\n') + [ "$(printf '%s' "$values" | wc -c)" -le 1048576 ] || exit 13 + manifest=$(printf '%s\\n' "$values" | sed -n 's/^ENGRAM_TOOLS=//p') + [ -n "$manifest" ] || exit 14 + manifest_hash=$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1) +done +[ "$count" -eq 1 ] +printf '{"bootId":"%s","manifestSha256":"%s","tmpfsVerified":true,"diskRetained":true}\\n' "$boot" "$manifest_hash" +`; + const response = await guestExec(ctx, client, sessionId, [ + "/bin/sh", + "-c", + script, + "fixture-cold-observe", + prepare ? "prepare" : "verify", + sentinel, + guestHarnessPath(ctx), + ]); + requireThat(response.status === 0, "guest_cold_actual_observation"); + const observed = z + .strictObject({ + bootId: z.uuid(), + manifestSha256: z.string().regex(/^[a-f0-9]{64}$/), + tmpfsVerified: z.literal(true), + diskRetained: z.literal(true), + }) + .parse(JSON.parse(response.stdout.toString("utf8"))); + return { ...observed, sha256: sha256(response.stdout) }; +} diff --git a/orchestrator/scripts/mcp-stack/guest-ready.test.ts b/orchestrator/scripts/mcp-stack/guest-ready.test.ts new file mode 100644 index 000000000..63a784bac --- /dev/null +++ b/orchestrator/scripts/mcp-stack/guest-ready.test.ts @@ -0,0 +1,138 @@ +import { expect, spyOn, test } from "bun:test"; +import { create } from "@bufbuild/protobuf"; +import { ExecOutputSchema } from "../../src/gen/engram/app/v1/session_pb.ts"; +import type { Context } from "./context.ts"; +import { guestReady } from "./guest.ts"; +import { Supervisor } from "./supervisor.ts"; + +for (const backend of ["firecracker", "vz"] as const) + test(`guest readiness waits for the native read proof only on FC (${backend})`, async () => { + const order: string[] = []; + const commands: string[][] = []; + const prepare = spyOn( + Supervisor.prototype, + "prepareDiskRead", + ).mockImplementation(async () => { + order.push("prepare"); + }); + const verify = spyOn( + Supervisor.prototype, + "verifyDiskRead", + ).mockImplementation(async () => { + order.push("verify"); + }); + // Only backend is read while the native control methods are replaced. + const ctx = { backend, deadlines: { requestMs: 1000 } } as Context; + const client = { + sessions: { + async *exec(request: { argv?: string[] }) { + const argv = request.argv ?? []; + commands.push(argv); + const reading = argv.includes("fixture-disk-read"); + order.push( + reading + ? "read" + : argv.includes("fixture-disk-read-create") + ? "create" + : argv.includes("fixture-disk-read-remove") + ? "remove" + : "ready", + ); + yield create(ExecOutputSchema, { + event: { case: "started", value: {} }, + }); + if (reading) + yield create(ExecOutputSchema, { + event: { case: "stdout", value: Buffer.alloc(4096) }, + }); + yield create(ExecOutputSchema, { + event: { case: "exit", value: { exitStatus: 0 } }, + }); + }, + }, + }; + try { + await guestReady(ctx, client, "session"); + order.push("caller"); + expect(order).toEqual( + backend === "firecracker" + ? ["ready", "create", "prepare", "read", "verify", "remove", "caller"] + : ["ready", "caller"], + ); + if (backend === "firecracker") { + const createCommand = commands[1]!; + const readCommand = commands[2]!; + expect(createCommand[2]).toContain("conv=excl,fsync"); + expect(createCommand[2]).toContain('"$(stat -c %d /)"'); + expect(createCommand[2]).toContain('"$(stat -c %b "$1")" -ge 8'); + expect(readCommand[2]).toContain( + "iflag=direct,nofollow bs=4096 count=1", + ); + expect(createCommand.at(-1)).toMatch( + /^\/tmp\/mcp-stack-fixture\/disk-read-[a-f0-9]{32}\.bin$/, + ); + expect(readCommand.at(-1)).toBe(createCommand.at(-1)); + } + } finally { + prepare.mockRestore(); + verify.mockRestore(); + } + }); + +for (const failure of ["create", "prepare", "read", "verify"] as const) + test(`guest readiness does not return before a successful read ACK (${failure})`, async () => { + const order: string[] = []; + const prepare = spyOn( + Supervisor.prototype, + "prepareDiskRead", + ).mockImplementation(async () => { + order.push("prepare"); + if (failure === "prepare") throw Error("prepare failed"); + }); + const verify = spyOn( + Supervisor.prototype, + "verifyDiskRead", + ).mockImplementation(async () => { + order.push("verify"); + if (failure === "verify") throw Error("verify failed"); + }); + // The stub supplies only the fields consumed before native control I/O. + const ctx = { + backend: "firecracker", + deadlines: { requestMs: 1000 }, + } as Context; + const client = { + sessions: { + async *exec(request: { argv?: string[] }) { + const argv = request.argv ?? []; + const stage = argv.includes("fixture-disk-read-create") + ? "create" + : argv.includes("fixture-disk-read") + ? "read" + : "ready"; + order.push(stage); + yield create(ExecOutputSchema, { + event: { case: "started", value: {} }, + }); + if (stage === "read") + yield create(ExecOutputSchema, { + event: { case: "stdout", value: Buffer.alloc(4096) }, + }); + yield create(ExecOutputSchema, { + event: { + case: "exit", + value: { exitStatus: stage === failure ? 1 : 0 }, + }, + }); + }, + }, + }; + try { + await expect(guestReady(ctx, client, "session")).rejects.toThrow(); + const steps = ["ready", "create", "prepare", "read", "verify"]; + expect(order).toEqual(steps.slice(0, steps.indexOf(failure) + 1)); + } finally { + prepare.mockRestore(); + verify.mockRestore(); + } + }); diff --git a/orchestrator/scripts/mcp-stack/guest.ts b/orchestrator/scripts/mcp-stack/guest.ts new file mode 100644 index 000000000..4e2128121 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/guest.ts @@ -0,0 +1,484 @@ +import { z } from "zod"; +import { randomBytes } from "node:crypto"; +import { posix } from "node:path"; +import { requireThat, RunnerFailure, sha256, type Context } from "./context.ts"; +import type { PublicClient } from "./transport.ts"; +import { Supervisor } from "./supervisor.ts"; + +export function canonicalJson(value: unknown): string { + if ( + value === null || + typeof value === "boolean" || + typeof value === "string" || + (typeof value === "number" && Number.isFinite(value)) + ) + return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value) + return `{${Object.keys(value) + .sort() + .map( + (key) => + `${JSON.stringify(key)}:${canonicalJson(Reflect.get(value, key))}`, + ) + .join(",")}}`; + throw new RunnerFailure("ASSERTION_FAILED", "canonical_json_value"); +} + +export async function readGuestFile( + ctx: Context, + client: PublicClient, + sessionId: string, + path: string, +) { + const name = path.slice(ctx.guest.stateDirectory.length + 1); + requireThat( + path.startsWith(ctx.guest.stateDirectory + "/") && + name.length > 0 && + !/[^A-Za-z0-9_.-]/.test(name) && + name !== "." && + name !== "..", + "guest_proof_path", + ); + return await client.downloadGuestProofFile(sessionId, path); +} + +export const NativeProofSchema = z + .object({ + callId: z.string().regex(/^[A-Za-z0-9_-]{1,64}$/), + tool: z.string().regex(/^[A-Za-z0-9_]{1,128}$/), + observation: z + .enum([ + "completionBarrierHeld", + "completionBarrierReleased", + "prematureBarrierHeld", + "prematureBarrierReleased", + "prematureCompletionEmitted", + ]) + .optional(), + eventSha256: z + .string() + .regex(/^sha256:[a-f0-9]{64}$/) + .optional(), + expectExposure: z.enum(["present", "absent"]).optional(), + requestAttempted: z.literal(true).optional(), + protocolDenialCode: z.literal("unknown_tool").optional(), + nativeDenialCode: z.literal(-32602).optional(), + resultReceived: z.boolean().optional(), + forgedRejected: z.boolean().optional(), + delivery: z.enum(["available", "unavailable"]).optional(), + outcome: z + .enum(["succeeded", "failed", "canceled", "outcome_unknown"]) + .nullable() + .optional(), + resultUnavailable: z.boolean().nullable().optional(), + isError: z.boolean().nullable().optional(), + denialPlanMode: z.boolean().optional(), + selectedToolName: z + .string() + .regex(/^[A-Za-z0-9_]{1,128}$/) + .optional(), + selectedSchemaSha256: z + .string() + .regex(/^sha256:[a-f0-9]{64}$/) + .optional(), + exposedMcpToolNamesSha256: z + .string() + .regex(/^sha256:[a-f0-9]{64}$/) + .optional(), + planSha256: z + .string() + .regex(/^sha256:[a-f0-9]{64}$/) + .optional(), + resultMatchesExpected: z.boolean().optional(), + resultSha256: z + .string() + .regex(/^sha256:[a-f0-9]{64}$/) + .optional(), + expectedResultSha256: z + .string() + .regex(/^sha256:[a-f0-9]{64}$/) + .optional(), + }) + .superRefine((value, ctx) => { + const resultFields = [ + value.resultReceived, + value.forgedRejected, + value.delivery, + value.outcome, + value.resultUnavailable, + value.isError, + value.denialPlanMode, + value.resultMatchesExpected, + value.resultSha256, + value.expectedResultSha256, + ]; + if (value.observation !== undefined) { + if ( + resultFields.some((field) => field !== undefined) || + value.expectExposure !== undefined || + value.requestAttempted !== undefined || + value.protocolDenialCode !== undefined || + value.nativeDenialCode !== undefined || + value.selectedSchemaSha256 !== undefined || + value.selectedToolName !== undefined || + value.exposedMcpToolNamesSha256 !== undefined || + (value.observation === "prematureCompletionEmitted") !== + (value.eventSha256 !== undefined) + ) + ctx.addIssue({ code: "custom", message: "observation_is_not_result" }); + } else if (value.eventSha256 !== undefined) { + ctx.addIssue({ code: "custom", message: "event_requires_observation" }); + } + if ( + value.expectExposure === "absent" && + (value.observation !== undefined || + !value.tool.startsWith("mcp_") || + value.selectedToolName !== value.tool || + value.requestAttempted !== true || + value.protocolDenialCode !== "unknown_tool" || + !value.planSha256 || + !value.exposedMcpToolNamesSha256 || + value.selectedSchemaSha256 !== undefined || + resultFields.some((field) => field !== undefined)) + ) + ctx.addIssue({ + code: "custom", + message: "absent_is_protocol_denial_only", + }); + if ( + value.expectExposure !== "absent" && + [ + value.requestAttempted, + value.protocolDenialCode, + value.nativeDenialCode, + ].some((field) => field !== undefined) + ) + ctx.addIssue({ code: "custom", message: "denial_requires_absence" }); + }); + +export function parseGuestProofRecords(bytes: Buffer) { + requireThat(bytes.length <= 1024 * 1024, "guest_proof_size"); + const text = bytes.toString("utf8"); + // Each flushed record is one line. A writer may currently be appending the final line. + const lines = text.split("\n"); + lines.pop(); + requireThat(lines.length <= 128, "guest_proof_record_bound"); + return lines.map((line) => { + let raw: unknown; + try { + raw = JSON.parse(line); + } catch { + throw new RunnerFailure("ASSERTION_FAILED", "guest_proof_json"); + } + const proof = NativeProofSchema.safeParse(raw); + requireThat(proof.success, "guest_proof_shape"); + return { proof: proof.data, sha256: sha256(line + "\n") }; + }); +} +export async function readGuestProofRecords( + ctx: Context, + client: PublicClient, + sessionId: string, + path: string, +) { + return parseGuestProofRecords( + await readGuestFile(ctx, client, sessionId, path), + ); +} +export async function readGuestProof( + ctx: Context, + client: PublicClient, + sessionId: string, + path: string, +) { + return ( + (await readGuestProofRecords(ctx, client, sessionId, path)).at(-1) ?? null + ); +} + +export type GuestExecClient = { + sessions: Pick; +}; + +export async function guestExec( + ctx: Context, + client: GuestExecClient, + sessionId: string, + argv: string[], +) { + let started = false; + let exited = false; + let status: number | undefined; + let bytes = 0; + const output: Uint8Array[] = []; + for await (const response of client.sessions.exec( + { + sessionId, + argv, + timeoutSecs: BigInt( + Math.max(1, Math.floor(ctx.deadlines.requestMs / 1000)), + ), + }, + { timeoutMs: ctx.deadlines.requestMs }, + )) { + requireThat(!exited, "guest_exec_after_exit"); + switch (response.event.case) { + case "started": + requireThat(!started, "guest_exec_single_start"); + started = true; + break; + case "stdout": + case "stderr": + requireThat(started, "guest_exec_start_first"); + bytes += response.event.value.length; + requireThat(bytes <= 65536, "guest_exec_output_bound"); + if (response.event.case === "stdout") output.push(response.event.value); + break; + case "exit": + requireThat(started, "guest_exec_start_first"); + exited = true; + status = response.event.value.exitStatus; + break; + default: + throw new RunnerFailure("ASSERTION_FAILED", "guest_exec_frame"); + } + } + requireThat( + started && exited && status !== undefined, + "guest_exec_completion", + ); + return { status, stdout: Buffer.concat(output) }; +} + +export async function guestReady( + ctx: Context, + client: GuestExecClient, + sessionId: string, +) { + const response = await guestExec(ctx, client, sessionId, [ + "/bin/sh", + "-c", + 'test -d /tmp/mcp-stack-fixture && test ! -L /tmp/mcp-stack-fixture && test "$(stat -c %a /tmp/mcp-stack-fixture)" = 700', + ]); + requireThat(response.status === 0, "guest_fixture_directory_ready"); + if (ctx.backend !== "firecracker") return; + const path = `/tmp/mcp-stack-fixture/disk-read-${randomBytes(16).toString("hex")}.bin`; + const created = await guestExec(ctx, client, sessionId, [ + "/bin/sh", + "-c", + `set -eu +root=/tmp/mcp-stack-fixture +test -d "$root" && test ! -L "$root" +test "$(stat -c %a "$root")" = 700 +test "$(stat -c %u "$root")" = "$(id -u)" +test "$(stat -c %d "$root")" = "$(stat -c %d /)" +umask 077 +dd if=/dev/zero of="$1" bs=4096 count=1 conv=excl,fsync status=none +test -f "$1" && test ! -L "$1" +test "$(stat -c %h "$1")" = 1 +test "$(stat -c %u "$1")" = "$(id -u)" +test "$(stat -c %a "$1")" = 600 +test "$(stat -c %s "$1")" = 4096 +test "$(stat -c %b "$1")" -ge 8 +`, + "fixture-disk-read-create", + path, + ]); + requireThat( + created.status === 0 && created.stdout.length === 0, + "guest_disk_read_file", + ); + const supervisor = new Supervisor(ctx); + await supervisor.prepareDiskRead(); + const read = await guestExec(ctx, client, sessionId, [ + "/bin/sh", + "-c", + `set -eu +test -f "$1" && test ! -L "$1" +test "$(stat -c %h "$1")" = 1 +test "$(stat -c %u "$1")" = "$(id -u)" +test "$(stat -c %a "$1")" = 600 +test "$(stat -c %s "$1")" = 4096 +test "$(stat -c %d "$1")" = "$(stat -c %d /)" +dd if="$1" iflag=direct,nofollow bs=4096 count=1 status=none +`, + "fixture-disk-read", + path, + ]); + requireThat( + read.status === 0 && read.stdout.equals(Buffer.alloc(4096)), + "guest_disk_read_complete", + ); + await supervisor.verifyDiskRead(); + const removed = await guestExec(ctx, client, sessionId, [ + "/bin/sh", + "-c", + `set -eu +test -f "$1" && test ! -L "$1" +test "$(stat -c %h "$1")" = 1 +test "$(stat -c %u "$1")" = "$(id -u)" +rm -- "$1" +`, + "fixture-disk-read-remove", + path, + ]); + requireThat( + removed.status === 0 && removed.stdout.length === 0, + "guest_disk_read_cleanup", + ); +} + +export function guestHarnessPath(ctx: Context) { + return posix.join(posix.dirname(ctx.guest.cliPath), "harness"); +} + +/** Observe live harness/model environments, including inline ENGRAM_TOOLS. This is not a disk scan. */ +export async function probeOAuthCredentialAbsence( + ctx: Context, + client: GuestExecClient, + sessionId: string, +) { + const script = `set -eu +harness_count=0 +model_count=0 +manifest_count=0 +scanned=0 +for path in /proc/[0-9]*/cmdline; do + [ -r "$path" ] || continue + first=$(tr '\\000' '\\n' < "$path" | head -n 1) + kind=other + case "$first" in + "$2") kind=harness;; + "$1") kind=model;; + esac + [ "$kind" != other ] || continue + scanned=$((scanned+1)) + [ "$scanned" -le 256 ] || exit 12 + env_path=\${path%cmdline}environ + [ -r "$env_path" ] || exit 13 + values=$(head -c 1048577 "$env_path" | tr '\\000' '\\n') + bytes=$(printf '%s' "$values" | wc -c) + [ "$bytes" -le 1048576 ] || exit 14 + if printf '%s' "$values" | grep -Eq 'fixture-(access|refresh)-[0-9]+'; then exit 15; fi + if [ "$kind" = harness ]; then + harness_count=$((harness_count+1)) + if printf '%s' "$values" | grep -q '^ENGRAM_TOOLS=.'; then manifest_count=$((manifest_count+1)); fi + else + model_count=$((model_count+1)) + fi +done +[ "$harness_count" -ge 1 ] && [ "$model_count" -ge 1 ] && [ "$manifest_count" -eq "$harness_count" ] || exit 16 +printf '{"harnesses":%s,"models":%s,"manifests":%s,"credentialFound":false}\\n' "$harness_count" "$model_count" "$manifest_count" +`; + const result = await guestExec(ctx, client, sessionId, [ + "/bin/sh", + "-c", + script, + "fixture-env-probe", + ctx.guest.cliPath, + guestHarnessPath(ctx), + ]); + requireThat(result.status === 0, "live_oauth_environment_clean"); + const parsed = z + .strictObject({ + harnesses: z.number().int().min(1).max(256), + models: z.number().int().min(1).max(256), + manifests: z.number().int().min(1).max(256), + credentialFound: z.literal(false), + }) + .safeParse(JSON.parse(result.stdout.toString("utf8"))); + requireThat( + parsed.success && parsed.data.manifests === parsed.data.harnesses, + "live_oauth_environment_observation", + ); + return { ...parsed.data, sha256: sha256(result.stdout) }; +} + +export function isProofFilename(suffix: string, name: string) { + requireThat(/^[a-z0-9-]{1,40}$/.test(suffix), "guest_proof_suffix"); + return new RegExp(`^${suffix}-[0-9]+-[a-f0-9]{32}\\.json$`).test(name); +} +export async function proofPaths( + ctx: Context, + client: PublicClient, + sessionId: string, + suffix: string, +) { + requireThat(/^[a-z0-9-]{1,40}$/.test(suffix), "guest_proof_suffix"); + const result = await guestExec(ctx, client, sessionId, [ + "/bin/ls", + "-1", + ctx.guest.stateDirectory, + ]); + requireThat(result.status === 0, "guest_proof_listing"); + return result.stdout + .toString("utf8") + .trim() + .split("\n") + .filter((name) => isProofFilename(suffix, name)) + .map((name) => `${ctx.guest.stateDirectory}/${name}`); +} + +/** Literal native response contract from engram-harness-proto/tool_result.rs. */ +export function expectedCounterNative( + harness: "claude" | "codex", + count: number, + isError: boolean, +): Record { + const output = { + content: [{ type: "text", text: `count=${count}` }], + structuredContent: { count }, + isError, + }; + const metadata = { + outcome: isError ? "failed" : "succeeded", + delivery: "available", + resultUnavailable: false, + isError, + }; + return harness === "claude" + ? { ...output, _meta: { "engrams/toolResult": metadata } } + : { + success: !isError, + contentItems: [ + { type: "inputText", text: `count=${count}` }, + { + type: "inputText", + text: canonicalJson({ structuredContent: { count } }), + }, + { type: "inputText", text: canonicalJson(metadata) }, + ], + }; +} + +/** Publish only the fixed exclusive release file; the CLI independently checks its owner/type/mode. */ +export async function releaseGuestBarrier( + ctx: Context, + client: PublicClient, + sessionId: string, + kind: "completion" | "premature", +) { + requireThat( + kind === "completion" || kind === "premature", + "guest_barrier_kind", + ); + const response = await guestExec(ctx, client, sessionId, [ + "/bin/sh", + "-c", + `set -eu +root=/tmp/mcp-stack-fixture +path="$root/${kind}-release" +test -d "$root" && test ! -L "$root" +test "$(stat -c %a "$root")" = 700 +test "$(stat -c %u "$root")" = "$(id -u)" +umask 077 +set -C +: > "$path" +`, + ]); + requireThat( + response.status === 0 && response.stdout.length === 0, + "guest_barrier_exclusive_release", + ); +} diff --git a/orchestrator/scripts/mcp-stack/host-observer.ts b/orchestrator/scripts/mcp-stack/host-observer.ts new file mode 100644 index 000000000..592eabb69 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/host-observer.ts @@ -0,0 +1,562 @@ +import { open, lstat } from "node:fs/promises"; +import { constants, type Stats } from "node:fs"; +import { createHash } from "node:crypto"; +import { join } from "node:path"; +import { z } from "zod"; +import { realOwnedPath, requireThat, sha256, type Context } from "./context.ts"; +import { sourceKeySchema, type DatabaseObserver } from "./observer.ts"; +import { canonicalJson } from "./guest.ts"; + +const hex = z.string().regex(/^[a-f0-9]{64}$/); +const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/); +const bytes = z + .number() + .int() + .positive() + .max(16 * 1024 ** 3); +const manifest = z.strictObject({ + manifest_id: z.uuid(), + version: z.number().int().positive().safe(), +}); +const object = z.record(z.string(), z.unknown()); +const auxDrives = z + .array( + z.strictObject({ + drive_id: z.string(), + guest_mount: z.string(), + fs_type: z.literal("squashfs"), + sha256: hex, + }), + ) + .min(3) + .max(12) + .refine((items) => + items.every( + (v, i) => + v.drive_id === `dyn_${i}` && v.guest_mount === `/opt/engram/dyn/${i}`, + ), + ); +const captureRequest = z.strictObject({ + key: sourceKeySchema, + capture_id: z.uuid(), +}); +const captureAuthority = z.strictObject({ + version: z.literal(1), + request: captureRequest, + host_incarnation: z.string().min(1).max(128), + disk_manifest: manifest, + disk_sha256: hex, + disk_size_bytes: bytes, + image: z.string().min(1).max(256), + kernel_sha256: hex, + aux_ro_drives: auxDrives, + consistency: z.literal("crash_consistent_disk_cut_v1"), + source_state: z.literal("destroyed"), + record_digest: digest, +}); +const cold = z.strictObject({ + version: z.literal(1), + session_id: z.uuid(), + sandbox_id: z.uuid(), + boot_attempt: z.string().min(1).max(128), + image: z.string().min(1).max(256), + disk: z.strictObject({ + manifest, + sha256: hex, + size_bytes: bytes, + source: z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("fresh_image"), + base_manifest: manifest, + }), + z.strictObject({ + kind: z.literal("capture"), + capture_id: z.uuid(), + session_id: z.uuid(), + sandbox_id: z.uuid(), + binding_epoch: z.number().int().positive().safe(), + record_digest: digest, + }), + ]), + }), + kernel_sha256: hex, + aux_ro_drives: auxDrives, +}); +const request = z.strictObject({ + key: sourceKeySchema, + cold, + bootstrap_policy: object, + capture_authority: captureAuthority.nullable(), + sandbox_spec_json: z.string().max(1024 * 1024), +}); +const recordSchema = z.strictObject({ + version: z.literal(1), + request, + incarnation: z.string().min(1).max(128), + state: object, + allocation: object.nullable(), + prepared: object.nullable(), + applied: object.nullable(), + apply_digest: digest.nullable(), + finalization: digest.nullable(), + capture: z + .strictObject({ + request: captureRequest, + manifest, + artifact: z.tuple([hex, bytes]).nullable(), + authority: captureAuthority.nullable(), + }) + .nullable(), + retired: z.tuple([hex, bytes]).nullable(), +}); +export type HostRecord = z.infer; +const startSchema = z.strictObject({ + receipt: z.strictObject({ + key: sourceKeySchema, + prepared_receipt_id: digest, + finalization_digest: digest, + host_incarnation: z.string().min(1).max(128), + network_identity: digest, + receipt_id: digest, + }), + start_id: digest, +}); +export function verifyPhysicalBoot( + ctx: Pick & { + artifacts: { kernel: { sha256: string } }; + }, + record: HostRecord, + current: Awaited>, + observed: Awaited>, +) { + const assets = z + .strictObject({ + request: z.strictObject({ + version: z.literal(1), + host_id: z.uuid(), + session_id: z.uuid(), + intent_id: z.string(), + image: z.string(), + image_digest: digest, + disk_manifest: manifest, + aux_ro_drives: auxDrives, + }), + host_incarnation: z.string(), + backend: z.enum(["firecracker", "vz"]), + kernel_sha256: hex, + disk_sha256: hex, + disk_size_bytes: bytes, + }) + .parse(boundedHostJson(observed.assets, 65536)); + const start = startSchema.parse(boundedHostJson(observed.start, 65536)); + const hostState = z + .strictObject({ state: z.literal("started"), intent: startSchema }) + .parse(record.state); + const prepared = z + .object({ + request, + host_incarnation: z.string(), + network_identity: digest, + receipt_id: digest, + }) + .parse(record.prepared); + requireThat( + current.phase === "started" && + canonicalJson(record.request.key) === canonicalJson(current.source_key) && + canonicalJson(start) === canonicalJson(hostState.intent) && + canonicalJson(start.receipt.key) === canonicalJson(current.source_key) && + start.receipt.host_incarnation === record.incarnation && + start.receipt.finalization_digest === record.finalization && + canonicalJson(prepared.request) === canonicalJson(record.request) && + prepared.host_incarnation === record.incarnation && + prepared.receipt_id === start.receipt.prepared_receipt_id && + prepared.network_identity === start.receipt.network_identity && + assets.host_incarnation === record.incarnation && + assets.backend === ctx.backend && + assets.request.host_id === ctx.hostId && + assets.request.session_id === current.source_key.creation.session_id && + assets.request.intent_id === current.boot_attempt && + assets.request.image_digest === ctx.imageDigest && + assets.request.image === record.request.cold.image && + canonicalJson(assets.request.disk_manifest) === + canonicalJson(record.request.cold.disk.manifest) && + assets.kernel_sha256 === record.request.cold.kernel_sha256 && + assets.kernel_sha256 === ctx.artifacts.kernel.sha256 && + assets.disk_sha256 === record.request.cold.disk.sha256 && + assets.disk_size_bytes === record.request.cold.disk.size_bytes && + canonicalJson(assets.request.aux_ro_drives) === + canonicalJson(record.request.cold.aux_ro_drives), + "physical_actual_backend_chain", + ); + const combined = Buffer.from( + z + .array(z.number().int().min(0).max(255)) + .min(1) + .max(1048576) + .parse(boundedHostJson(observed.combined_manifest, 8388608)), + ); + requireThat( + observed.native_digest === current.native_digest && + observed.native_digest === `sha256:${sha256(observed.native_bytes)}` && + observed.combined_manifest_digest === current.combined_manifest_digest && + observed.combined_manifest_digest === `sha256:${sha256(combined)}`, + "physical_manifest_integrity", + ); + const entry = z.object({ + name: z.string().min(1), + description: z.string(), + inputSchema: object, + }); + const native = z + .array(entry.passthrough()) + .max(128) + .parse(boundedHostJson(observed.native_bytes.toString("utf8"), 1048576)); + const all = z + .array(entry.passthrough()) + .max(128) + .parse(boundedHostJson(combined.toString("utf8"), 1048576)); + requireThat( + native.length <= all.length && + new Set(all.map((v) => v.name)).size === all.length && + canonicalJson(all.slice(0, native.length)) === canonicalJson(native), + "physical_native_manifest_prefix", + ); + return { + backend: assets.backend, + nativeSha256: sha256(observed.native_bytes), + combinedSha256: sha256(combined), + nativeCount: native.length, + combinedCount: all.length, + assetsSha256: sha256(observed.assets), + startSha256: sha256(observed.start), + }; +} +export function verifyColdChain(old: HostRecord, next: HostRecord) { + const authority = old.capture?.authority; + requireThat( + authority && + old.state.state === "captured" && + canonicalJson(old.state.authority) === canonicalJson(authority), + "physical_old_capture_destroyed", + ); + const source = next.request.cold.disk.source; + requireThat( + source.kind === "capture" && + authority.image === old.request.cold.image && + authority.kernel_sha256 === old.request.cold.kernel_sha256 && + canonicalJson(authority.aux_ro_drives) === + canonicalJson(old.request.cold.aux_ro_drives) && + canonicalJson(next.request.capture_authority) === + canonicalJson(authority) && + source.capture_id === authority.request.capture_id && + source.session_id === old.request.key.creation.session_id && + source.sandbox_id === old.request.key.context.target.sandbox_id && + source.binding_epoch === old.request.key.context.target.binding_epoch && + source.record_digest === authority.record_digest && + canonicalJson(next.request.key.creation) === + canonicalJson(old.request.key.creation) && + next.request.key.native_digest === old.request.key.native_digest && + next.request.key.context.target.binding_epoch === + source.binding_epoch + 1 && + next.request.key.context.target.sandbox_id !== source.sandbox_id && + next.request.key.boot_attempt !== old.request.key.boot_attempt && + next.request.cold.disk.sha256 === authority.disk_sha256 && + next.request.cold.disk.size_bytes === authority.disk_size_bytes && + canonicalJson(next.request.cold.disk.manifest) === + canonicalJson(authority.disk_manifest) && + next.request.cold.image === authority.image && + next.request.cold.kernel_sha256 === authority.kernel_sha256 && + canonicalJson(next.request.cold.aux_ro_drives) === + canonicalJson(authority.aux_ro_drives), + "physical_exact_capture_to_cold_chain", + ); + return { + captureSha256: authority.disk_sha256, + authoritySha256: sha256(canonicalJson(authority)), + }; +} +export function boundedHostJson(text: string, limit: number) { + requireThat(Buffer.byteLength(text) <= limit, "host_json_bytes"); + const value: unknown = JSON.parse(text); + const stack = [{ value, depth: 0 }]; + while (stack.length) { + const current = stack.pop()!; + requireThat(current.depth <= 64, "host_json_depth"); + if (current.value !== null && typeof current.value === "object") + for (const child of Object.values(current.value)) + stack.push({ value: child, depth: current.depth + 1 }); + } + return value; +} +export function parseHostRecord(encoded: Buffer, sandboxId: string) { + requireThat(z.uuid().safeParse(sandboxId).success, "host_sandbox_identity"); + const envelope = z + .strictObject({ + schema: z.literal(1), + content_hash: hex, + id: z.uuid(), + body: z.string(), + }) + .parse( + boundedHostJson( + new TextDecoder("utf-8", { fatal: true }).decode(encoded), + 16 * 1024 ** 2, + ), + ); + requireThat( + envelope.id === sandboxId && + envelope.content_hash === sha256(envelope.body), + "host_envelope_integrity", + ); + const raw = boundedHostJson(envelope.body, 8 * 1024 ** 2); + const record = recordSchema.parse(raw); + requireThat( + record.request.key.context.target.sandbox_id === sandboxId && + record.request.cold.sandbox_id === sandboxId && + record.request.cold.session_id === + record.request.key.creation.session_id && + record.request.cold.boot_attempt === record.request.key.boot_attempt, + "host_record_target", + ); + const spec = z + .object({ env: object }) + .parse(boundedHostJson(record.request.sandbox_spec_json, 1024 ** 2)); + requireThat( + Object.keys(spec.env).length === 0, + "host_spec_has_no_environment", + ); + // The production digest uses Rust struct field order. Verify the raw serialized object order. + const rawRecord = object.parse(raw); + if (record.capture?.authority) { + const authority = object.parse(object.parse(rawRecord.capture).authority); + requireThat( + authority.record_digest === + `sha256:${sha256(JSON.stringify({ ...authority, record_digest: "" }))}`, + "host_capture_record_digest", + ); + requireThat( + canonicalJson(record.capture.request.key) === + canonicalJson(record.request.key) && + record.capture.authority.host_incarnation === record.incarnation && + canonicalJson(record.capture.request) === + canonicalJson(record.capture.authority.request) && + canonicalJson(record.capture.manifest) === + canonicalJson(record.capture.authority.disk_manifest) && + record.capture.manifest.manifest_id === + record.capture.request.capture_id && + record.capture.manifest.version === 1 && + canonicalJson(record.capture.artifact) === + canonicalJson([ + record.capture.authority.disk_sha256, + record.capture.authority.disk_size_bytes, + ]), + "host_capture_exact_authority", + ); + } + return { record, sha256: sha256(encoded) }; +} +function privateStat(stat: Stats) { + requireThat( + stat.isFile() && + stat.uid === process.getuid?.() && + (stat.mode & 0o777) === 0o600 && + stat.nlink === 1, + "host_private_file", + ); +} +function unchanged(before: Stats, after: Stats) { + requireThat( + before.ino === after.ino && + before.dev === after.dev && + before.size === after.size && + before.mtimeMs === after.mtimeMs && + before.ctimeMs === after.ctimeMs, + "host_file_changed_during_read", + ); +} +export class HostObserver { + constructor(private readonly ctx: Context) {} + async boot(sandboxId: string) { + requireThat(z.uuid().safeParse(sandboxId).success, "host_sandbox_identity"); + const path = join(this.ctx.observer.hostManagedRoot, `${sandboxId}.boot`); + await realOwnedPath(this.ctx.runRoot, path); + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = await file.stat(); + privateStat(before); + requireThat( + before.size > 0 && before.size <= 16 * 1024 ** 2, + "host_record_bytes", + ); + const data = Buffer.alloc(before.size); + let offset = 0; + const deadline = performance.now() + this.ctx.deadlines.stepMs; + while (offset < data.length) { + requireThat(performance.now() < deadline, "host_record_read_deadline"); + const read = await file.read( + data, + offset, + data.length - offset, + offset, + ); + requireThat(read.bytesRead > 0, "host_record_complete"); + offset += read.bytesRead; + } + unchanged(before, await file.stat()); + unchanged(before, await lstat(path)); + return parseHostRecord(data, sandboxId); + } finally { + await file.close(); + } + } + async captureFile(record: HostRecord) { + requireThat( + record.capture?.authority && record.capture.artifact, + "host_capture_available", + ); + const authority = record.capture.authority; + const path = join( + this.ctx.observer.hostManagedRoot, + `${authority.request.capture_id}.disk`, + ); + await realOwnedPath(this.ctx.runRoot, path); + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = await file.stat(); + privateStat(before); + requireThat( + before.size === authority.disk_size_bytes && + before.size <= 16 * 1024 ** 3, + "host_capture_size", + ); + const hash = createHash("sha256"); + const buffer = Buffer.alloc(65536); + const deadline = performance.now() + this.ctx.deadlines.stepMs; + let offset = 0; + while (offset < before.size) { + requireThat(performance.now() < deadline, "host_capture_read_deadline"); + const read = await file.read( + buffer, + 0, + Math.min(buffer.length, before.size - offset), + offset, + ); + requireThat(read.bytesRead > 0, "host_capture_complete"); + hash.update(buffer.subarray(0, read.bytesRead)); + offset += read.bytesRead; + } + unchanged(before, await file.stat()); + unchanged(before, await lstat(path)); + const actual = hash.digest("hex"); + requireThat(actual === authority.disk_sha256, "host_capture_actual_hash"); + return { sha256: actual, bytes: offset }; + } finally { + await file.close(); + } + } + async logBaseline() { + await realOwnedPath(this.ctx.runRoot, this.ctx.observer.hostLogFile); + const file = await open( + this.ctx.observer.hostLogFile, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const stat = await file.stat(); + privateStat(stat); + const last = Buffer.alloc(1); + if (stat.size) await file.read(last, 0, 1, stat.size - 1); + return { + inode: stat.ino, + device: stat.dev, + offset: stat.size, + partial: stat.size > 0 && last[0] !== 10, + }; + } finally { + await file.close(); + } + } + async proxyDenials( + baseline: Awaited>, + sessionId: string, + sni: string, + ) { + requireThat( + z.uuid().safeParse(sessionId).success && + /^[a-z0-9.-]+\.fixture\.example$/.test(sni), + "proxy_observer_identity", + ); + await realOwnedPath(this.ctx.runRoot, this.ctx.observer.hostLogFile); + const file = await open( + this.ctx.observer.hostLogFile, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const before = await file.stat(); + privateStat(before); + requireThat( + before.ino === baseline.inode && + before.dev === baseline.device && + before.size >= baseline.offset && + before.size - baseline.offset <= 8 * 1024 ** 2, + "host_log_window", + ); + const data = Buffer.alloc(before.size - baseline.offset); + let offset = 0; + const deadline = performance.now() + this.ctx.deadlines.stepMs; + while (offset < data.length) { + requireThat(performance.now() < deadline, "host_log_read_deadline"); + const read = await file.read( + data, + offset, + data.length - offset, + baseline.offset + offset, + ); + requireThat(read.bytesRead > 0, "host_log_complete"); + offset += read.bytesRead; + } + const after = await file.stat(); + const current = await lstat(this.ctx.observer.hostLogFile); + requireThat( + after.ino === before.ino && + current.ino === before.ino && + current.dev === before.dev && + after.size >= before.size && + current.size >= before.size, + "host_log_rotation_or_truncation", + ); + return parseProxyDenials(data, baseline.partial, sessionId, sni); + } finally { + await file.close(); + } + } +} +export function parseProxyDenials( + data: Buffer, + partial: boolean, + sessionId: string, + sni: string, +) { + requireThat(data.length <= 8 * 1024 ** 2, "host_log_window"); + const lines = data.toString("utf8").split("\n"); + requireThat( + lines.every((line) => Buffer.byteLength(line) <= 65536), + "host_log_line_bound", + ); + lines.pop(); + if (partial) lines.shift(); + let malformedDenial = false; + const records = lines.flatMap((line) => { + const match = line.match( + /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z) INFO engram_egress_proxy::proxy: egress rejected — destination not in any allow_hosts session_id=([0-9a-f-]{36}) sni=([a-z0-9.-]+) original_dst=(?:None|Some\(\((?:\d{1,3}\.){3}\d{1,3}, \d{1,5}\)\))$/, + ); + if ( + !match && + line.includes("egress rejected — destination not in any allow_hosts") + ) + malformedDenial = true; + return match && match[2] === sessionId && match[3] === sni + ? [{ sha256: sha256(line + "\n"), bytes: Buffer.byteLength(line) }] + : []; + }); + return !malformedDenial && records.length === 1 ? records : []; +} diff --git a/orchestrator/scripts/mcp-stack/infrastructure.test.ts b/orchestrator/scripts/mcp-stack/infrastructure.test.ts new file mode 100644 index 000000000..0d9ba48a0 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/infrastructure.test.ts @@ -0,0 +1,1030 @@ +import { afterAll, beforeAll, expect, spyOn, test } from "bun:test"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createServer } from "node:https"; +import { X509Certificate } from "node:crypto"; +import pg from "pg"; +import { connectNodeAdapter } from "@connectrpc/connect-node"; +import { Code, ConnectError } from "@connectrpc/connect"; +import { + ContextSchema, + realOwnedPath, + sha256, + RunnerFailure, +} from "./context.ts"; +import { bindingTools, modelToolName } from "./bootstrap.ts"; +import { DatabaseObserver } from "./observer.ts"; +import { publicClient } from "./transport.ts"; +import { IntegrationService } from "../../src/gen/engram/app/v1/integration_pb.ts"; +import { ProfileService } from "../../src/gen/engram/app/v1/profile_pb.ts"; +import { TaskService } from "../../src/gen/engram/app/v1/task_pb.ts"; +import { SessionService } from "../../src/gen/engram/app/v1/session_pb.ts"; +import { + AutomationService, + AutomationRunService, +} from "../../src/gen/engram/app/v1/automation_pb.ts"; +import { createAutomationTask } from "./automation.ts"; +import { readOwnedAutomationRuns } from "./automation-ownership.ts"; + +let root: string; +let cert: Buffer; +let key: Buffer; +beforeAll(async () => { + const actual = join(import.meta.dir, "../../..", "var/mcp-stack-runner"); + await realOwnedPath(actual, actual); + root = await mkdtemp(join(actual, "infrastructure-unit-")); + const keyPath = join(root, "https.key"), + certPath = join(root, "https.pem"); + const process = Bun.spawn( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + keyPath, + "-out", + certPath, + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost", + ], + { stdout: "ignore", stderr: "ignore" }, + ); + expect(await process.exited).toBe(0); + await chmod(certPath, 0o600); + cert = await readFile(certPath); + key = await readFile(keyPath); +}); +afterAll(async () => { + if (root) await rm(root, { recursive: true }); +}); +const h = "a".repeat(64); +const artifact = () => ({ path: "/owned/a", sha256: h }); +const ep = (name: string) => ({ + origin: `https://${name}.fixture.example`, + ca: artifact(), + serverFingerprint: Array(32).fill("AA").join(":"), +}); +function validContext() { + return { + version: 1, + runId: "unit", + runRoot: "/owned", + backend: "vz", + sourceCommit: "a".repeat(40), + hostId: "11111111-1111-4111-8111-111111111111", + imageId: "fixture-image", + imageDigest: `sha256:${h}`, + artifacts: Object.fromEntries( + [ + "coordinator", + "orchestrator", + "worker", + "hostAgent", + "fixture", + "kernel", + "guestImage", + "claudeHarness", + "codexHarness", + "modelCli", + "claudeBundle", + "codexBundle", + "helper", + ].map((key) => [key, artifact()]), + ), + endpoints: { + app: ep("app"), + coordinator: ep("coordinator"), + workerControl: ep("worker-control"), + operator: { + ...ep("operator"), + certificate: artifact(), + privateKeyFile: "/owned/key", + }, + }, + protocolFixtures: { + modern: { + ...ep("observer"), + configuration: artifact(), + runRoot: "/owned/modern", + resourceKeyFile: "/owned/modern/resource.key", + tokenFile: "/owned/token", + resourceOrigin: ep("resource").origin, + issuerOrigin: ep("issuer").origin, + authorizationOrigin: ep("authorization").origin, + tokenOrigin: ep("token").origin, + redirectUri: ep("app").origin + "/integrations/mcp/callback", + protocol: "2026-07-28", + }, + legacy: { + ...ep("observer-legacy"), + configuration: artifact(), + runRoot: "/owned/legacy", + resourceKeyFile: "/owned/legacy/resource.key", + tokenFile: "/owned/legacy/token", + resourceOrigin: ep("resource-legacy").origin, + issuerOrigin: ep("issuer-legacy").origin, + authorizationOrigin: ep("authorization-legacy").origin, + tokenOrigin: ep("token-legacy").origin, + redirectUri: ep("app").origin + "/integrations/mcp/callback", + protocol: "2025-11-25", + }, + }, + publication: { + candidateIdentity: `sha256:${h}`, + expectedGeneration: null, + deploymentIdentity: `sha256:${h}`, + networkIdentity: `sha256:${h}`, + trustIdentity: `sha256:${h}`, + zone: "fixture", + }, + identities: { + ...Object.fromEntries( + ["admin", "owner", "other"].map((role) => [ + role, + { + email: `${role}@fixture.example`, + passwordFile: `/owned/${role}.password`, + cookieFile: `/owned/${role}.cookie`, + }, + ]), + ), + unattendedKeyFile: "/owned/unattended.key", + automationKeyFile: "/owned/automation.key", + routerSentinelFile: "/owned/router.key", + }, + supervisor: { + socket: "/owned/control.sock", + tokenFile: "/owned/supervisor.token", + configurations: Object.fromEntries( + ["orchestrator", "worker", "coordinator", "host-agent"].map((role) => [ + role, + { normal: artifact() }, + ]), + ), + operatorCandidates: { + normal: { file: artifact(), identity: `sha256:${h}` }, + }, + }, + observer: { + coordinatorDsnFile: "/owned/c.dsn", + hostManagedRoot: "/owned/host/managed-boots", + hostLogFile: "/owned/host.log", + orchestratorDsnFile: "/owned/o.dsn", + }, + guest: { + stateDirectory: "/tmp/mcp-stack-fixture", + cliPath: "/opt/fixture/engram-mcp-model-fixture", + }, + deadlines: { requestMs: 1000, stepMs: 3000, totalMs: 10000 }, + evidenceFile: "/owned/evidence.jsonl", + }; +} + +for (const failure of [ + "none", + "upload", + "download", + "update", + "binding", + "counter", + "cleanup", + "destroy-failed", + "destroy-cancelled", + "managed-upload", + "managed-download", + "upload-cleanup", + "publication", + "owner", + "profile", + "scenario-ordinary", + "scenario-managed", + "scenario-owner", + "scenario-receipt", + "scenario-tools", + "scenario-executions", + "scenario-policy", + "scenario-pagination", + "scenario-malformed", +] as const) + test(`infrastructure uses public owner tasks, managed authority and exact cleanup: ${failure}`, async () => { + const scenarioTest = failure.startsWith("scenario-"); + const ordinaryScenario = scenarioTest && failure !== "scenario-managed"; + const managedIndex = scenarioTest ? 0 : 1; + const ctx = ContextSchema.parse(validContext()); + ctx.runRoot = root; + ctx.protocolFixtures.modern.resourceKeyFile = join(root, "resource.key"); + ctx.protocolFixtures.modern.tokenFile = join(root, "observer.token"); + ctx.observer.coordinatorDsnFile = join(root, "coordinator.dsn"); + ctx.observer.orchestratorDsnFile = join(root, "orchestrator.dsn"); + await writeFile( + ctx.protocolFixtures.modern.resourceKeyFile, + "fixture-neutral-resource-key", + { mode: 0o600 }, + ); + await writeFile(ctx.protocolFixtures.modern.tokenFile, "t".repeat(32), { + mode: 0o600, + }); + for (const path of [ + ctx.observer.coordinatorDsnFile, + ctx.observer.orchestratorDsnFile, + ]) + await writeFile(path, "postgres://unit@unused.fixture.example/mcp_unit", { + mode: 0o600, + }); + // Pools remain disconnected. Read-only observations use controlled values. + const database = await DatabaseObserver.connect(ctx); + const events: string[] = []; + const deletions = new Set(); + const destroyReads = new Map(); + const destroyQuery = spyOn(pg.Pool.prototype, "query").mockImplementation( + async (...args: unknown[]) => { + expect(args[0]).toContain("FROM session_ops"); + const sessionId = (args[1] as string[])[0]!; + expect(deletions.has(sessionId)).toBe(true); + const reads = (destroyReads.get(sessionId) ?? 0) + 1; + destroyReads.set(sessionId, reads); + const state = + failure === "destroy-failed" + ? "failed" + : failure === "destroy-cancelled" + ? "cancelled" + : reads === 1 + ? "missing" + : reads === 2 + ? "queued" + : reads === 3 + ? "running" + : "done"; + events.push(`destroy-${sessionId}-${state}`); + return { + rows: state === "missing" ? [] : [{ id: "1", state }], + rowCount: state === "missing" ? 0 : 1, + command: "SELECT", + oid: 0, + fields: [], + }; + }, + ); + const sessionIds = [ + "11111111-1111-4111-8111-111111111112", + "11111111-1111-4111-8111-111111111113", + ]; + if (failure === "none") { + await expect( + database.destroyOperation("11111111-1111-4111-8111-111111111199"), + ).rejects.toThrow("observer_unowned_session"); + expect(destroyQuery).not.toHaveBeenCalled(); + } + const source_key = { + creation: { + session_id: sessionIds[managedIndex]!, + task_id: scenarioTest ? "task-1" : "task-2", + policy: { + actorId: failure === "owner" ? "other" : "owner", + inputId: "input", + parentSessionId: null, + eligibility: "interactive", + }, + input_digest: `sha256:${h}`, + }, + context: { + target: { + session_id: sessionIds[managedIndex]!, + sandbox_id: sessionIds[managedIndex]!, + host_id: ctx.hostId, + binding_epoch: 1, + }, + operator_generation: 1, + operator_identity: + failure === "publication" + ? `sha256:${"b".repeat(64)}` + : ctx.publication.candidateIdentity, + binding_hash: null, + }, + boot_attempt: "boot", + native_digest: `sha256:${h}`, + }; + const started = spyOn(database, "started").mockImplementation( + async (sessionId, hostId) => { + if (ordinaryScenario) + throw new RunnerFailure( + "ASSERTION_FAILED", + "coordinator_managed_boot_row", + ); + expect(sessionId).toBe(sessionIds[managedIndex]!); + expect(hostId).toBe(ctx.hostId); + events.push("managed-started"); + return { + phase: "started", + host_id: hostId, + binding_epoch: "1", + boot_attempt: "boot", + native_digest: `sha256:${h}`, + source_key, + combined_manifest_digest: `sha256:${h}`, + binding_hash: `sha256:${h}`, + }; + }, + ); + const lifecycle = spyOn(database, "managedLifecycle").mockImplementation( + async (sessionId) => { + expect(sessionId).toBe(sessionIds[managedIndex]!); + events.push("managed-lifecycle"); + return { + phase: "started", + nativeDigest: `sha256:${h}`, + bindingHash: `sha256:${h}`, + }; + }, + ); + const connection = { + id: "connection", + alias: "fixture-infrastructure-managed", + provider: "mcp", + enabled: true, + configRevision: 7, + authGeneration: 6, + mcp: { + endpoint: ctx.protocolFixtures.modern.resourceOrigin + "/mcp", + zone: "fixture", + authKind: "api_key", + }, + }; + const tools = ["read_counter", "write_counter"].map((upstreamName) => ({ + upstreamName, + inputSchemaJson: '{"type":"object"}', + })); + const alias = modelToolName( + connection.id, + connection.alias, + "read_counter", + ); + let profiles = 0, + tasks = 0, + observations = 0; + const deletedTasks: string[] = [], + deletedProfiles: string[] = []; + const bytes = Buffer.alloc(4096, 65); + const handler = connectNodeAdapter({ + requestPathPrefix: "/rpc", + routes(router) { + router.service(ProfileService, { + createProfile(req, rpc) { + expect(rpc.requestHeader.get("cookie")).toBe( + "fixture_session=admin", + ); + profiles++; + events.push(`profile-${profiles}`); + expect(req.imageId).toBe(ctx.imageId); + expect(req.harness).toBe("claude"); + expect(req.modelRouter).toBe("openrouter"); + expect(req.model).toBe("z-ai/glm-5.2"); + if (profiles === 1 && failure !== "scenario-managed") + expect(req.integrationGrants).toEqual([]); + else { + expect(req.integrationGrants).toHaveLength(1); + expect(req.integrationGrants[0]).toMatchObject({ + connectionId: connection.id, + mcp: { + upstreamName: "read_counter", + automationWrite: false, + unrestricted: true, + }, + }); + expect(req.network?.default).toBe("deny"); + } + return { + profile: { + id: `profile-${profiles}`, + harness: req.harness, + modelRouter: req.modelRouter, + model: req.model, + mcpGrantRevision: failure === "profile" ? 0n : 1n, + }, + }; + }, + deleteProfile(req) { + deletedProfiles.push(req.id); + events.push(`delete-${req.id}`); + return {}; + }, + }); + router.service(TaskService, { + createTask(req, rpc) { + expect(rpc.requestHeader.get("cookie")).toBe( + "fixture_session=owner", + ); + expect(req.deferInitialPrompt).toBe(true); + expect(req.prompt).toBeUndefined(); + expect(req.type).toBe("chat"); + tasks++; + events.push(`task-${tasks}`); + return { + task: { + id: `task-${tasks}`, + sessions: [ + { sessionId: sessionIds[tasks - 1]!, isPrimary: true }, + ], + }, + }; + }, + updateTask(req) { + if (failure === "update") + throw new ConnectError("private", Code.FailedPrecondition); + return { task: { id: req.taskId, titleIsCustom: true } }; + }, + getTask(req) { + return { + task: { + id: req.taskId, + createdByUserId: + failure === "scenario-owner" ? "other" : "owner", + sessions: [{ sessionId: sessionIds[0]! }], + }, + }; + }, + deleteTask(req) { + deletedTasks.push(req.taskId); + events.push(`delete-${req.taskId}`); + if (failure === "cleanup" || failure === "upload-cleanup") + throw new ConnectError("private", Code.Unavailable); + return {}; + }, + }); + router.service(SessionService, { + deleteSession(req) { + deletions.add(req.sessionId); + events.push(`delete-session-${req.sessionId}`); + return {}; + }, + getSession(req) { + return { + session: { + id: req.sessionId, + status: deletions.has(req.sessionId) ? "completed" : "active", + hostId: ctx.hostId, + }, + }; + }, + async *exec(req, rpc) { + expect(rpc.requestHeader.get("cookie")).toBe( + "fixture_session=owner", + ); + events.push(`exec-${tasks}`); + expect(req.argv[0]).toBe("/bin/sh"); + const script = req.argv[2]!; + yield { + event: { case: "started" as const, value: { execId: "fixed" } }, + }; + if (script.includes("infrastructure-exec-ok")) { + expect(script).toContain("umask 077"); + expect(script).toContain("set -C"); + expect(script).toContain("stat -c %u"); + expect(script).toContain("stat -c %h"); + expect(script).toContain("stat -c %a"); + expect(script).toContain("sha256sum"); + expect(req.argv[4]).toBe(ctx.guest.stateDirectory); + yield { + event: { + case: "stdout" as const, + value: Buffer.from("infrastructure-exec-ok\n"), + }, + }; + } + yield { + event: { case: "exit" as const, value: { exitStatus: 0 } }, + }; + }, + }); + router.service(IntegrationService, { + createConnection() { + events.push("catalog"); + return { connection }; + }, + putMcpCredential() { + return { connection }; + }, + discoverMcp() { + return { catalog: { id: "draft", kind: "draft", tools } }; + }, + getMcpCatalogs() { + return { draft: { id: "draft" }, catalogRevision: 1 }; + }, + publishMcp() { + return { catalog: { id: "published", kind: "published", tools } }; + }, + setConnectionEnabled() { + return { connection }; + }, + getSessionMcpState(req) { + if (ordinaryScenario) { + expect(req.sessionId).toBe(sessionIds[0]!); + expect(req.omitBindings).toBe(false); + return { + availability: + failure === "scenario-malformed" ? "" : "unavailable", + ...(failure === "scenario-receipt" + ? { bindingReceipt: { current: false } } + : {}), + ...(failure === "scenario-policy" ? { writePolicy: {} } : {}), + ...(failure === "scenario-tools" + ? { bindingTools: [{ modelToolName: alias }] } + : {}), + ...(failure === "scenario-executions" + ? { executions: [{ executionId: "unexpected" }] } + : {}), + ...(failure === "scenario-pagination" + ? { hasMoreBindingTools: true } + : {}), + }; + } + expect(req.sessionId).toBe(sessionIds[managedIndex]!); + events.push("binding"); + return { + bindingReceipt: { current: true }, + bindingTools: + failure === "binding" + ? [] + : [ + { + connectionId: connection.id, + modelToolName: alias, + reviewedTool: tools[0], + }, + ], + }; + }, + }); + }, + }); + const server = createServer({ key, cert }, (req, res) => { + if (req.url === "/v1/observe") { + observations++; + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + version: 1, + healthy: true, + effects: { version: 1, executions: 0, count: 0, events: [] }, + reads: failure === "counter" && observations > 1 ? 1 : 0, + held: 0, + cancellations: 0, + oauth: { + account: "account-one", + scope: null, + authorizations: 0, + exchanges: 0, + rotations: 0, + rejections: 0, + expiresInSeconds: 600, + }, + }), + ); + return; + } + if (req.url?.startsWith("/api/v1/sessions/")) { + const owner = req.headers.cookie === "fixture_session=owner"; + if (!owner) { + res.writeHead(req.headers.cookie ? 404 : 401); + res.end("private"); + return; + } + expect(req.headers.origin).toBe(ctx.endpoints.app.origin); + if (req.method === "POST") { + events.push(`upload-${tasks}`); + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + expect(Buffer.concat(chunks)).toEqual(bytes); + if ( + failure === "upload" || + failure === "upload-cleanup" || + (failure === "managed-upload" && tasks === 2) + ) { + res.writeHead(400, { "x-engram-rpc-code": "3" }); + res.end("private"); + return; + } + res.writeHead(200, { "content-type": "application/json" }); + res.end( + JSON.stringify({ + path: new URL( + req.url!, + ctx.endpoints.app.origin, + ).searchParams.get("path"), + size_bytes: 4096, + sha256: sha256(bytes), + }), + ); + }); + return; + } + events.push(`download-${tasks}`); + res.writeHead( + failure === "download" || + (failure === "managed-download" && tasks === 2) + ? 404 + : 200, + { "content-length": "4096", "x-content-sha256": sha256(bytes) }, + ); + res.end(bytes); + return; + } + void handler(req, res); + }); + await new Promise((resolve) => + server.listen(0, "localhost", resolve), + ); + try { + const address = server.address(); + if (!address || typeof address === "string") throw new Error("address"); + const endpoint = { + origin: `https://localhost:${address.port}`, + ca: { path: join(root, "https.pem"), sha256: sha256(cert) }, + serverFingerprint: new X509Certificate(cert).fingerprint256, + }; + ctx.endpoints.app = endpoint; + Object.assign(ctx.protocolFixtures.modern, endpoint); + const admin = await publicClient(ctx, { + kind: "cookie", + value: "fixture_session=admin", + }); + const owner = await publicClient(ctx, { + kind: "cookie", + value: "fixture_session=owner", + }); + const other = await publicClient(ctx, { + kind: "cookie", + value: "fixture_session=other", + }); + const principals = { + admin, + owner, + other, + unattended: other, + automation: other, + actorIds: { + admin: "admin", + owner: "owner", + other: "other", + unattended: "unattended", + automation: "automation", + }, + }; + const { runInfrastructure } = await import("./infrastructure.ts"); + if (scenarioTest) { + const { startScenario, scenarioBindingTools } = + await import("./scenario-runtime.ts"); + const pending = startScenario(ctx, principals, database, { + harness: "claude", + suffix: "claude-grant-missing", + protocol: "modern", + upstreamName: "read_counter", + mode: "read", + caller: "owner", + automationWrite: false, + arguments: {}, + constraints: [], + callId: "grant-missing", + ...(ordinaryScenario ? { grant: false } : {}), + }); + if (failure === "scenario-ordinary" || failure === "scenario-managed") { + const scenario = await pending; + expect(scenario.sessionId).toBe(sessionIds[0]!); + const observed = await scenarioBindingTools(scenario); + if (ordinaryScenario) { + expect(observed).toEqual([]); + expect(() => scenario.coordinator).toThrow( + "managed_scenario_required", + ); + await expect( + bindingTools(owner, scenario.sessionId), + ).rejects.toThrow("current_binding"); + } else { + expect(observed.map((tool) => tool.modelToolName)).toEqual([alias]); + expect(scenario.coordinator.phase).toBe("started"); + } + } else { + await expect(pending).rejects.toThrow( + failure === "scenario-owner" + ? "ordinary_owner_task_session" + : "ordinary_empty_mcp_authority", + ); + } + expect(started).toHaveBeenCalledTimes(ordinaryScenario ? 0 : 1); + expect(lifecycle).toHaveBeenCalledTimes(ordinaryScenario ? 0 : 1); + return; + } + if (failure === "none") { + const rows = await runInfrastructure(ctx, principals, database); + expect(rows).toEqual( + (["ordinary", "managed"] as const).map((mode) => ({ + mode, + execSha256: sha256("infrastructure-exec-ok\n"), + uploadSha256: sha256(bytes), + downloadSha256: sha256(bytes), + sizeBytes: 4096, + unauthenticatedStatus: 401, + nonOwnerStatus: 404, + })), + ); + for (let i = 0; i < 2; i++) { + expect(deletions.has(sessionIds[i]!)).toBe(true); + expect( + events.indexOf(`destroy-${sessionIds[i]}-done`), + ).toBeGreaterThan(-1); + expect(events.indexOf(`destroy-${sessionIds[i]}-done`)).toBeLessThan( + events.indexOf(`delete-task-${i + 1}`), + ); + } + expect(events.indexOf("delete-task-1")).toBeLessThan( + events.indexOf("catalog"), + ); + expect(events.indexOf("managed-lifecycle")).toBeLessThan( + events.indexOf("binding"), + ); + expect(events.indexOf("binding")).toBeLessThan( + events.indexOf("upload-2"), + ); + expect(started).toHaveBeenCalledTimes(1); + expect(lifecycle).toHaveBeenCalledTimes(1); + } else { + const seams: Partial> = { + upload: "guest_file_status_http_400_rpc_code_3", + download: "guest_file_status_http_404", + update: "private", + binding: "exact_public_binding", + counter: "infrastructure_no_tool_execution", + cleanup: "private", + "destroy-failed": "infrastructure_destroy_failed", + "destroy-cancelled": "infrastructure_destroy_failed", + "managed-upload": "guest_file_status_http_400_rpc_code_3", + "managed-download": "guest_file_status_http_404", + "upload-cleanup": "guest_file_status_http_400_rpc_code_3", + publication: "infrastructure_started_owner_publication", + owner: "infrastructure_started_owner_publication", + profile: "published_fixture_profile", + }; + const seam = seams[failure]; + expect(seam).toBeDefined(); + await expect( + runInfrastructure(ctx, principals, database), + ).rejects.toThrow(seam); + if ( + [ + "upload", + "download", + "update", + "cleanup", + "upload-cleanup", + ].includes(failure) + ) + expect(tasks).toBe(1); + } + expect(deletedTasks).toEqual( + failure.startsWith("destroy-") + ? [] + : Array.from({ length: tasks }, (_, i) => `task-${i + 1}`), + ); + expect(deletedProfiles).toEqual( + Array.from({ length: profiles }, (_, i) => `profile-${i + 1}`), + ); + } finally { + destroyQuery.mockRestore(); + started.mockRestore(); + lifecycle.mockRestore(); + await database.close(); + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }); + +for (const state of [ + "running", + "pending-visible-launch", + "completed", + "filtered", + "failed", + "superseded", + "halted", + "deadline", + "ended", +] as const) + test(`automation fixture keeps live authority through the case: ${state}`, async () => { + const ctx = ContextSchema.parse(validContext()); + ctx.runRoot = root; + const automationId = `fixture-automation-${state}`; + const runId = `autorun:${automationId}:manual:fixture`; + const sessionId = "11111111-1111-4111-8111-111111111112"; + const taskId = "fixture-task"; + const events: string[] = []; + let definitionJson: string | undefined; + let observations = 0; + const terminal = !["running", "pending-visible-launch"].includes(state); + const handler = connectNodeAdapter({ + requestPathPrefix: "/rpc", + routes(router) { + router.service(AutomationService, { + createAutomation(req, rpc) { + expect(rpc.requestHeader.get("cookie")).toBe( + "fixture_session=admin", + ); + expect(req.name).toBe(`Fixture ${state}`); + expect(req.enabled).toBe(true); + expect(req.inputsJson).toBe("{}"); + events.push("create"); + definitionJson = req.definitionJson; + return { automation: { id: automationId } }; + }, + runNow(req, rpc) { + expect(rpc.requestHeader.get("x-api-key")).toBe( + "engk_fixture_automation", + ); + expect(req.automationId).toBe(automationId); + events.push("run"); + return { runId }; + }, + }); + router.service(AutomationRunService, { + async getRun(req, rpc) { + expect(rpc.requestHeader.get("x-api-key")).toBe( + "engk_fixture_automation", + ); + expect(req.runId).toBe(runId); + expect( + (await readOwnedAutomationRuns(ctx)).find( + (receipt) => receipt.runId === runId, + ), + ).toMatchObject({ + automationId, + runId, + definitionActorId: "definition-actor", + profileId: "fixture-profile", + fixtureRunId: ctx.runId, + sourceCommit: ctx.sourceCommit, + hostId: ctx.hostId, + }); + events.push("observe"); + observations++; + return { + run: { + brief: { + id: runId, + automationId, + status: + state === "pending-visible-launch" + ? observations === 1 + ? "pending" + : "running" + : state === "ended" + ? "running" + : state, + triggerSource: "manual", + dryRun: false, + ...(state === "ended" + ? { endedAt: "2026-01-01T00:00:00Z" } + : {}), + }, + sessionIds: [sessionId], + steps: [ + { + blockId: "launch", + status: "succeeded", + outputsJson: JSON.stringify({ + session_id: sessionId, + task_id: taskId, + initial_prompt: false, + prompt: "", + }), + }, + ], + }, + }; + }, + }); + router.service(TaskService, { + async updateTask(req, rpc) { + expect(rpc.requestHeader.get("cookie")).toBe( + "fixture_session=admin", + ); + expect(req.taskId).toBe(taskId); + expect(req.title).toBe("Fixture automation task"); + expect( + (await readOwnedAutomationRuns(ctx)).find( + (receipt) => receipt.runId === runId, + ), + ).toMatchObject({ automationId, runId, sessionId, taskId }); + events.push("title"); + return { task: { id: taskId, titleIsCustom: true } }; + }, + }); + router.service(SessionService, { + sendPrompt() { + events.push("prompt"); + return {}; + }, + }); + }, + }); + const server = createServer({ cert, key }, handler); + await new Promise((resolve) => + server.listen(0, "localhost", resolve), + ); + try { + const address = server.address(); + if (!address || typeof address === "string") throw new Error("address"); + ctx.endpoints.app = { + origin: `https://localhost:${address.port}`, + ca: { path: join(root, "https.pem"), sha256: sha256(cert) }, + serverFingerprint: new X509Certificate(cert).fingerprint256, + }; + const admin = await publicClient(ctx, { + kind: "cookie", + value: "fixture_session=admin", + }); + const automation = await publicClient(ctx, { + kind: "api-key", + value: "engk_fixture_automation", + }); + const principals = { + admin, + automation, + owner: admin, + other: admin, + unattended: admin, + actorIds: { + admin: "definition-actor", + automation: "trigger-actor", + owner: "owner", + other: "other", + unattended: "unattended", + }, + }; + const pending = createAutomationTask( + ctx, + principals, + "fixture-profile", + state, + ); + if (terminal) { + await expect(pending).rejects.toThrow("automation_launch_failed"); + expect(events).toEqual(["create", "run", "observe"]); + } else { + const result = await pending; + expect(result).toMatchObject({ + taskId, + sessionId, + automation: { + id: automationId, + runId, + runtimeActorId: `automation:${automationId}`, + definitionActorId: "definition-actor", + triggerActorId: "trigger-actor", + }, + }); + expect(events).toEqual( + state === "pending-visible-launch" + ? ["create", "run", "observe", "observe", "title"] + : ["create", "run", "observe", "title"], + ); + expect(JSON.parse(definitionJson!)).toEqual({ + engine: 1, + trigger: { kind: "manual" }, + blocks: [ + { + id: "launch", + type: "create_session", + config: { + profileId: "fixture-profile", + promptTemplate: "", + titleTemplate: "Fixture automation task", + includeEventContext: false, + keepOnFinish: true, + }, + }, + { + id: "lifetime", + type: "wait_session", + config: { session: { blockId: "launch" }, until: "ended" }, + }, + ], + inputsSchema: [], + settings: { endSessionsOnFinish: false, runDeadlineSeconds: 120 }, + }); + } + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }); diff --git a/orchestrator/scripts/mcp-stack/infrastructure.ts b/orchestrator/scripts/mcp-stack/infrastructure.ts new file mode 100644 index 000000000..b6407718a --- /dev/null +++ b/orchestrator/scripts/mcp-stack/infrastructure.ts @@ -0,0 +1,277 @@ +import { + bindingTool, + createCatalog, + createFixtureProfile, + createHumanTask, + waitFor, +} from "./bootstrap.ts"; +import { requireThat, sha256, type Context } from "./context.ts"; +import type { InfrastructureRow } from "./evidence.ts"; +import { + canonicalJson, + guestExec, + guestReady, + readGuestFile, +} from "./guest.ts"; +import { DatabaseObserver, FixtureObserver } from "./observer.ts"; +import type { Principals } from "./scenario-runtime.ts"; +import { anonymousGuestFileStatus } from "./transport.ts"; + +export type InfrastructureEvidence = InfrastructureRow[]; + +/** Exercise the public owner file routes before any model or MCP tool runs. */ +export async function runInfrastructure( + ctx: Context, + principals: Principals, + database: DatabaseObserver, +): Promise { + const fixture = new FixtureObserver(ctx, "modern"); + const before = await fixture.observe(); + const rows: InfrastructureEvidence = []; + for (const mode of ["ordinary", "managed"] as const) { + let profileId: string | undefined; + let taskId: string | undefined; + let sessionId: string | undefined; + let failed = false; + // Retain ownership as soon as Create returns, including later helper failures. + const admin = { + ...principals.admin, + profiles: { + ...principals.admin.profiles, + createProfile: async ( + ...args: Parameters + ) => { + const response = await principals.admin.profiles.createProfile( + ...args, + ); + profileId = response.profile?.id; + return response; + }, + }, + }; + const owner = { + ...principals.owner, + tasks: { + ...principals.owner.tasks, + createTask: async ( + ...args: Parameters + ) => { + const response = await principals.owner.tasks.createTask(...args); + taskId = response.task?.id; + if (response.task?.sessions.length === 1) { + sessionId = response.task.sessions[0]!.sessionId; + if (taskId && sessionId) database.own(taskId, sessionId); + } + return response; + }, + }, + }; + try { + let managed: Awaited> | undefined; + let alias: string | undefined; + if (mode === "ordinary") { + const response = await admin.profiles.createProfile({ + name: "Fixture infrastructure ordinary", + imageId: ctx.imageId, + harness: "claude", + modelRouter: "openrouter", + model: "z-ai/glm-5.2", + integrationGrants: [], + }); + requireThat(response.profile?.id, "infrastructure_ordinary_profile"); + } else { + managed = await createCatalog( + ctx, + admin, + "infrastructure-managed", + "modern", + "api_key", + ); + const profile = await createFixtureProfile(ctx, admin, { + harness: "claude", + suffix: "infrastructure-managed", + connectionId: managed.connection.id, + connectionAlias: managed.connection.alias, + upstreamName: "read_counter", + mode: "read", + grant: true, + automationWrite: false, + arguments: {}, + callId: "infrastructure-read", + constraints: [], + defaultAllow: false, + }); + alias = profile.alias; + } + requireThat(profileId, "infrastructure_profile"); + const task = await createHumanTask(ctx, owner, profileId, "claude"); + if (managed) { + const coordinator = await waitFor( + ctx, + "infrastructure_coordinator_started", + () => database.started(task.sessionId, ctx.hostId), + (state) => state.phase === "started", + ); + requireThat( + coordinator.source_key.context.operator_identity === + ctx.publication.candidateIdentity && + coordinator.source_key.context.target.session_id === + task.sessionId && + coordinator.source_key.context.target.host_id === ctx.hostId && + coordinator.source_key.creation.task_id === task.taskId && + coordinator.source_key.creation.session_id === task.sessionId && + coordinator.source_key.creation.policy.actorId === + principals.actorIds.owner, + "infrastructure_started_owner_publication", + ); + await waitFor( + ctx, + "infrastructure_orchestrator_started", + () => database.managedLifecycle(task.sessionId, coordinator), + (state) => state.phase === "started", + ); + const bound = await bindingTool( + owner, + task.sessionId, + managed.connection.id, + "read_counter", + ); + const reviewed = managed.catalog.tools.find( + (tool) => tool.upstreamName === "read_counter", + ); + requireThat( + bound.modelToolName === alias && + bound.reviewedTool && + reviewed && + canonicalJson(JSON.parse(bound.reviewedTool.inputSchemaJson)) === + canonicalJson(JSON.parse(reviewed.inputSchemaJson)), + "infrastructure_delivered_read_binding", + ); + } + await guestReady(ctx, owner, task.sessionId); + const bytes = Buffer.alloc(4096, 65); + const uploadedPath = `${ctx.guest.stateDirectory}/infrastructure-${mode}-upload.bin`; + const createdPath = `${ctx.guest.stateDirectory}/infrastructure-${mode}-exec.bin`; + const upload = await owner.uploadGuestFile( + task.sessionId, + uploadedPath, + bytes, + ); + const exec = await guestExec(ctx, owner, task.sessionId, [ + "/bin/sh", + "-c", + `set -eu +root="$1" +upload="$2" +created="$3" +expected="$4" +uid=$(id -u) +test -d "$root" && test ! -L "$root" +test "$(stat -c %a "$root")" = 700 +test "$(stat -c %u "$root")" = "$uid" +check_file() { + test -f "$1" && test ! -L "$1" + test "$(stat -c %h "$1")" = 1 + test "$(stat -c %u "$1")" = "$uid" + test "$(stat -c %a "$1")" = 600 + test "$(stat -c %s "$1")" = 4096 + test "$(sha256sum "$1" | cut -d ' ' -f 1)" = "$expected" +} +check_file "$upload" +umask 077 +set -C +head -c 4096 /dev/zero | tr '\\000' A > "$created" +check_file "$created" +printf 'infrastructure-exec-ok\\n' +`, + "fixture-infrastructure", + ctx.guest.stateDirectory, + uploadedPath, + createdPath, + sha256(bytes), + ]); + requireThat( + exec.status === 0 && + exec.stdout.equals(Buffer.from("infrastructure-exec-ok\n")), + "infrastructure_exec_integrity", + ); + const download = await readGuestFile( + ctx, + owner, + task.sessionId, + createdPath, + ); + requireThat( + download.equals(bytes) && + upload.sizeBytes === bytes.length && + upload.sha256 === sha256(download), + "infrastructure_transfer_integrity", + ); + const unauthenticatedStatus = await anonymousGuestFileStatus( + ctx, + task.sessionId, + createdPath, + ); + const nonOwnerStatus = await principals.other.nonOwnerGuestFileStatus( + task.sessionId, + createdPath, + ); + const after = await fixture.observe(); + requireThat( + after.reads === before.reads && + after.held === before.held && + after.cancellations === before.cancellations && + canonicalJson(after.effects) === canonicalJson(before.effects), + "infrastructure_no_tool_execution", + ); + rows.push({ + mode, + execSha256: sha256(exec.stdout), + uploadSha256: upload.sha256, + downloadSha256: sha256(download), + sizeBytes: download.length, + unauthenticatedStatus, + nonOwnerStatus, + }); + } catch (error) { + failed = true; + throw error; + } finally { + let cleanupError: unknown; + try { + if (sessionId) { + const cleanupSessionId = sessionId; + await principals.admin.sessions.deleteSession({ + sessionId: cleanupSessionId, + }); + // The terminal session status can precede physical host teardown. + await waitFor( + ctx, + "infrastructure_destroy_done", + () => database.destroyOperation(cleanupSessionId), + (operation) => { + requireThat( + operation?.state !== "failed" && + operation?.state !== "cancelled", + "infrastructure_destroy_failed", + ); + return operation?.state === "done"; + }, + ); + } + if (taskId) await principals.admin.tasks.deleteTask({ taskId }); + } catch (error) { + cleanupError = error; + } + try { + if (profileId) + await principals.admin.profiles.deleteProfile({ id: profileId }); + } catch (error) { + cleanupError ??= error; + } + // Preserve the first failed boundary; the platform still audits all owned resources. + if (!failed && cleanupError) throw cleanupError; + } + } + return rows; +} diff --git a/orchestrator/scripts/mcp-stack/manifest.ts b/orchestrator/scripts/mcp-stack/manifest.ts new file mode 100644 index 000000000..2be5c15de --- /dev/null +++ b/orchestrator/scripts/mcp-stack/manifest.ts @@ -0,0 +1,205 @@ +import { RunnerFailure } from "./failure.ts"; + +export const HARNESS = ["claude", "codex"] as const; +export type Harness = (typeof HARNESS)[number]; +export const SCENARIOS = [ + { + id: "catalog-fidelity", + assertions: [ + "reviewed-tool-only-exposure", + "actual-alias-schema", + "http-json-sse", + "modern-legacy", + "exact-text-structured-is-error", + ], + }, + { + id: "grant-denials", + assertions: [ + "missing-grant", + "revoked-grant", + "forged-grant", + "forged-guest-completion", + "no-effect", + ], + }, + { + id: "interactive-approval", + assertions: [ + "owner-one-use", + "admin-one-use", + "other-member-denied", + "changed-context-denied", + "replayed-decision-denied", + "one-effect", + ], + }, + { + id: "unattended-grant", + assertions: [ + "api-key-principal", + "default-deny", + "explicit-admin-grant", + "argument-constraint", + "plan-approval-insufficient", + ], + }, + { + id: "plan-child-intersection", + assertions: [ + "plan-read-only", + "plan-write-denied", + "child-grant-intersection", + "child-argument-intersection", + ], + }, + { + id: "credential-network-isolation", + assertions: [ + "no-credential-in-guest", + "default-allow-managed-authority-denied", + "worker-control-denied", + "positive-network-denial", + "guest-return", + ], + }, + { + id: "disk-only-cold-rebind", + assertions: [ + "disk-only-restore", + "new-native-and-mcp-frozen-bytes", + "old-target-retired-before-spawn", + "no-old-ram", + "no-old-execution", + ], + }, + { + id: "revoke-resume", + assertions: [ + "revocation", + "actual-resume", + "current-binding", + "old-alias-denied", + "old-call-denied", + ], + }, + { + id: "held-write-restart", + assertions: [ + "committed-held-barrier", + "worker-restart", + "orchestrator-restart", + "outcome-unknown", + "one-effect", + "settled-unknown-recovery", + "new-id-restricted", + ], + }, + { + id: "oauth-authority", + assertions: [ + "real-pkce", + "exact-resource-issuer", + "expiry-rotation", + "same-identity-binding-stable", + "account-change-invalidates", + "scope-expansion-invalidates", + "reconnect", + "no-credential-leak", + ], + }, + { + id: "kill-cancel", + assertions: [ + "global-kill", + "zone-kill", + "connection-kill", + "tool-kill", + "cancel", + "journal-history-preserved", + "no-new-dispatch", + ], + }, + { + id: "outbox-guest-ack", + assertions: [ + "verified-persisted-result", + "coordinator-outbox", + "restart", + "guest-proof", + "guest-ack", + "no-provider-replay", + "target-gone-not-delivery", + ], + }, +] as const; +export type ScenarioId = (typeof SCENARIOS)[number]["id"]; +export const REQUIRED_CASES = HARNESS.flatMap((harness) => + SCENARIOS.map((scenario) => ({ + ...scenario, + harness, + key: `${harness}/${scenario.id}`, + })), +); +export type RunSelection = + | { stage: "infrastructure" } + | { stage: "matrix"; matrix: string }; +export const FULL_MATRIX = { stage: "matrix", matrix: "all" } as const; +export function selectMatrix(value: string) { + if ( + REQUIRED_CASES.length !== 24 || + new Set(REQUIRED_CASES.map((v) => v.key)).size !== 24 + ) + throw new RunnerFailure("ASSERTION_FAILED", "required_case_manifest"); + if (value === "all") return REQUIRED_CASES; + const keys = value.split(","); + if ( + !value || + new Set(keys).size !== keys.length || + keys.some((key) => !REQUIRED_CASES.some((c) => c.key === key)) + ) + throw new RunnerFailure("INVALID_CONTEXT", "matrix_selection"); + return REQUIRED_CASES.filter((c) => keys.includes(c.key)); +} +export function selectedCases(selection: RunSelection) { + return selection.stage === "infrastructure" + ? [] + : selectMatrix(selection.matrix); +} +export type AssertionEvidence = { + assertion: string; + observationHashes: string[]; +}; +export type CaseResult = { + key: string; + status: "PASS" | "FAIL" | "NEEDS_CONTEXT"; + assertions: AssertionEvidence[]; + seam?: string; +}; +export function verifyResults(results: readonly CaseResult[]): void { + if ( + results.length !== REQUIRED_CASES.length || + new Set(results.map((v) => v.key)).size !== REQUIRED_CASES.length + ) + throw new RunnerFailure("ASSERTION_FAILED", "missing_or_duplicate_case"); + for (const required of REQUIRED_CASES) { + const result = results.find((v) => v.key === required.key); + if (!result || result.status !== "PASS") + throw new RunnerFailure("ASSERTION_FAILED", "matrix_incomplete"); + const actual = result.assertions.map((v) => v.assertion); + if ( + actual.length !== required.assertions.length || + new Set(actual).size !== actual.length || + required.assertions.some((v) => !actual.includes(v)) || + result.assertions.some( + (v) => + v.observationHashes.length === 0 || + v.observationHashes.some((h) => !/^[a-f0-9]{64}$/.test(h)), + ) + ) + throw new RunnerFailure( + "ASSERTION_FAILED", + "missing_subassertion_evidence", + ); + } +} diff --git a/orchestrator/scripts/mcp-stack/observer.ts b/orchestrator/scripts/mcp-stack/observer.ts new file mode 100644 index 000000000..3396411ea --- /dev/null +++ b/orchestrator/scripts/mcp-stack/observer.ts @@ -0,0 +1,645 @@ +import pg from "pg"; +import { z } from "zod"; +import { + privateFile, + requireThat, + RunnerFailure, + type FixtureKind, + type Context, +} from "./context.ts"; +import { jsonRequest } from "./transport.ts"; +import { canonicalJson } from "./guest.ts"; + +const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/); +const cleanupIdentitySchema = z.strictObject({ + host_id: z.uuid().nullable(), + sandbox_id: z.uuid().nullable(), + binding_epoch: z.string().regex(/^[1-9][0-9]*$/), + managed_task_id: z.string().min(1).nullable(), + managed_host_id: z.uuid().nullable(), + managed_sandbox_id: z.uuid().nullable(), + managed_binding_epoch: z + .string() + .regex(/^[1-9][0-9]*$/) + .nullable(), +}); +export const sourceKeySchema = z.object({ + creation: z.object({ + session_id: z.uuid(), + task_id: z.string(), + policy: z.object({ + actorId: z.string(), + inputId: z.string(), + parentSessionId: z.string().nullable().optional(), + eligibility: z.string(), + }), + input_digest: digest, + }), + context: z.object({ + target: z.object({ + session_id: z.uuid(), + sandbox_id: z.uuid(), + host_id: z.uuid(), + binding_epoch: z.number().int().positive().safe(), + }), + operator_generation: z.number().int().positive().safe(), + operator_identity: digest, + binding_hash: z.null(), + }), + boot_attempt: z.string(), + native_digest: digest, +}); +const FixtureObservationSchema = z.strictObject({ + version: z.literal(1), + healthy: z.literal(true), + effects: z.strictObject({ + version: z.literal(1), + executions: count, + count: z.number().int().safe(), + events: z + .array( + z.strictObject({ + sequence: count, + callHash: z.string().regex(/^[a-f0-9]{64}$/), + }), + ) + .max(64), + }), + reads: count, + held: count, + cancellations: count, + oauth: z.strictObject({ + account: z.enum(["account-one", "account-two"]), + scope: z + .enum(["tools:read", "tools:write", "tools:read tools:write"]) + .nullable(), + authorizations: count, + exchanges: count, + rotations: count, + rejections: count, + expiresInSeconds: z.number().int().min(1).max(3600), + }), +}); +export class FixtureObserver { + constructor( + private readonly ctx: Context, + private readonly protocol: FixtureKind = "modern", + ) {} + private async request( + method: "GET" | "POST", + route: string, + body?: unknown, + requestMs = this.ctx.deadlines.requestMs, + ) { + const token = ( + await privateFile( + this.ctx.runRoot, + this.ctx.protocolFixtures[this.protocol].tokenFile, + ) + ) + .toString("utf8") + .trim(); + requireThat(/^[A-Za-z0-9_-]{32,128}$/.test(token), "observer_token"); + const response = await jsonRequest( + { ...this.ctx, deadlines: { ...this.ctx.deadlines, requestMs } }, + this.ctx.protocolFixtures[this.protocol], + method, + route, + body, + { authorization: `Bearer ${token}` }, + ); + requireThat(response.status === 200, "fixture_observer_status"); + return response.body; + } + async ready() { + const deadline = performance.now() + this.ctx.deadlines.stepMs; + let body: unknown; + // Container start can complete before the HTTPS listener accepts requests. + for (;;) { + try { + body = await this.request( + "GET", + "/v1/ready", + undefined, + Math.max( + 1, + Math.min( + this.ctx.deadlines.requestMs, + deadline - performance.now(), + ), + ), + ); + break; + } catch (error) { + if ( + !(error instanceof RunnerFailure) || + error.code !== "TRANSPORT_FAILED" || + error.seam !== "https_request" + ) + throw error; + const remaining = deadline - performance.now(); + if (remaining <= 0) throw error; + await new Promise((resolve) => + setTimeout(resolve, Math.min(100, remaining)), + ); + if (performance.now() >= deadline) throw error; + } + } + const response = z + .strictObject({ + version: z.literal(1), + ready: z.literal(true), + resource: z.string(), + issuer: z.string(), + }) + .safeParse(body); + requireThat( + response.success && + response.data.resource === + `${this.ctx.protocolFixtures[this.protocol].resourceOrigin}/mcp` && + response.data.issuer === + this.ctx.protocolFixtures[this.protocol].issuerOrigin, + "fixture_identity", + ); + } + async observe() { + const response = FixtureObservationSchema.safeParse( + await this.request("GET", "/v1/observe"), + ); + requireThat(response.success, "fixture_durable_observation"); + return response.data; + } + async configure(input: { + reply?: "json" | "sse"; + response?: "normal" | "error"; + holdCalls?: boolean; + revision?: number; + }) { + const checked = z + .strictObject({ + reply: z.enum(["json", "sse"]).optional(), + response: z.enum(["normal", "error"]).optional(), + holdCalls: z.boolean().optional(), + revision: z.number().int().min(1).max(100).optional(), + }) + .parse(input); + await this.request("POST", "/v1/case", checked); + } + async release() { + await this.request("POST", "/v1/release", {}); + } + async cancel() { + await this.request("POST", "/v1/cancel", {}); + } + async oauth(input: { + account?: "account-one" | "account-two"; + scope?: "tools:read" | "tools:write" | "tools:read tools:write"; + issuer?: "configured" | "mismatch"; + expiresInSeconds?: number; + }) { + const checked = z + .strictObject({ + account: z.enum(["account-one", "account-two"]).optional(), + scope: z + .enum(["tools:read", "tools:write", "tools:read tools:write"]) + .optional(), + issuer: z.enum(["configured", "mismatch"]).optional(), + expiresInSeconds: z.number().int().min(1).max(3600).optional(), + }) + .parse(input); + await this.request("POST", "/v1/oauth", checked); + } +} + +/** No caller SQL. Every observation is scoped to a task/session obtained from public CreateTask. */ +export class DatabaseObserver { + private readonly owned = new Map(); + private readonly cleanupTargets = new Map< + string, + { + hostId: string; + sandboxId: string; + bindingEpoch: string; + managed: boolean; + } + >(); + private constructor( + private readonly coordinator: pg.Pool, + private readonly orchestrator: pg.Pool, + private readonly databaseName: string, + ) {} + static async connect(ctx: Context) { + const pool = async (path: string) => { + const dsn = (await privateFile(ctx.runRoot, path)) + .toString("utf8") + .trim(); + let url: URL; + try { + url = new URL(dsn); + } catch { + throw new RunnerFailure("INVALID_CONTEXT", "observer_dsn"); + } + requireThat( + ["postgres:", "postgresql:"].includes(url.protocol) && + url.hostname.endsWith(".fixture.example") && + url.pathname === `/mcp_${ctx.runId.replaceAll("-", "_")}` && + url.search === "", + "observer_database_scope", + ); + return new pg.Pool({ + connectionString: dsn, + max: 1, + connectionTimeoutMillis: ctx.deadlines.requestMs, + idleTimeoutMillis: 1000, + statement_timeout: ctx.deadlines.requestMs, + query_timeout: ctx.deadlines.requestMs, + options: "-c default_transaction_read_only=on", + }); + }; + return new DatabaseObserver( + await pool(ctx.observer.coordinatorDsnFile), + await pool(ctx.observer.orchestratorDsnFile), + `mcp_${ctx.runId.replaceAll("-", "_")}`, + ); + } + /** Fixed schema-only prerequisites, before public signup or publication. No authority rows are read. */ + async prerequisites() { + const expected = { + coordinator: [ + "sessions.id:uuid", + "sessions.status:text", + "sessions.host_id:uuid", + "sessions.sandbox_id:uuid", + "sessions.binding_epoch:bigint", + "sandbox_tombstones.host_id:uuid", + "sandbox_tombstones.sandbox_id:uuid", + "session_ops.id:bigint", + "session_ops.session_id:uuid", + "session_ops.kind:text", + "session_ops.state:text", + "session_managed_creations.session_id:uuid", + "session_managed_creations.task_id:text", + "session_managed_creations.native_version:integer", + "session_managed_creations.native_bytes:bytea", + "session_managed_creations.native_digest:text", + "session_managed_boots.session_id:uuid", + "session_managed_boots.host_id:uuid", + "session_managed_boots.sandbox_id:uuid", + "session_managed_boots.binding_epoch:bigint", + "session_managed_boots.boot_attempt:text", + "session_managed_boots.phase:text", + "session_managed_boots.record:jsonb", + "session_managed_boot_intents.intent_id:text", + "session_managed_boot_intents.session_id:uuid", + "session_managed_boot_intents.binding_epoch:bigint", + "session_managed_boot_intents.completed:boolean", + "session_managed_boot_intents.record:jsonb", + "session_outbox.session_id:uuid", + "session_outbox.prompt_id:text", + "session_outbox.kind:text", + "session_outbox.attempts:integer", + "session_outbox.delivered_at:timestamp with time zone", + "session_outbox.acked_at:timestamp with time zone", + "session_events.session_id:uuid", + "session_events.idx:bigint", + "session_events.kind:text", + "session_events.payload:jsonb", + "session_managed_interrupted_targets.session_id:uuid", + "session_managed_interrupted_targets.event_floor:bigint", + ], + orchestrator: [ + "task_session.task_id:text", + "task_session.session_id:text", + "task_session.mcp_managed_lifecycle:jsonb", + ], + }; + for (const role of ["coordinator", "orchestrator"] as const) { + const pool = this[role]; + const tables = [...new Set(expected[role].map((v) => v.split(".")[0]!))]; + const result = await pool.query( + "SELECT current_database() AS database,table_name,column_name,data_type FROM information_schema.columns WHERE table_schema='public' AND table_name=ANY($1::text[])", + [tables], + ); + const rows = z + .array( + z.strictObject({ + database: z.string(), + table_name: z.string(), + column_name: z.string(), + data_type: z.string(), + }), + ) + .max(256) + .parse(result.rows); + requireThat( + rows.every((v) => v.database === this.databaseName), + "observer_actual_database", + ); + const actual = new Set( + rows.map((v) => `${v.table_name}.${v.column_name}:${v.data_type}`), + ); + for (const column of expected[role]) + if (!actual.has(column)) + throw new RunnerFailure( + "NEEDS_CONTEXT", + `${role}_schema_${column.split(":")[0]}`, + ); + } + } + own(taskId: string, sessionId: string) { + requireThat( + /^[A-Za-z0-9_:-]{1,128}$/.test(taskId) && + z.uuid().safeParse(sessionId).success && + !this.owned.has(sessionId), + "observer_owned_session", + ); + this.owned.set(sessionId, taskId); + } + private task(sessionId: string) { + const task = this.owned.get(sessionId); + requireThat(task, "observer_unowned_session"); + return task; + } + async retainCleanupTarget(sessionId: string, hostId: string) { + const taskId = this.task(sessionId); + requireThat(z.uuid().safeParse(hostId).success, "cleanup_expected_host"); + const result = await this.coordinator.query( + "SELECT s.host_id::text,s.sandbox_id::text,s.binding_epoch::text,c.task_id AS managed_task_id,b.host_id::text AS managed_host_id,b.sandbox_id::text AS managed_sandbox_id,b.binding_epoch::text AS managed_binding_epoch FROM sessions s LEFT JOIN session_managed_creations c ON c.session_id=s.id LEFT JOIN session_managed_boots b ON b.session_id=s.id WHERE s.id=$1", + [sessionId], + ); + requireThat(result.rowCount === 1, "cleanup_target_row"); + const row = cleanupIdentitySchema.parse(result.rows[0]); + const managed = row.managed_task_id !== null; + const sandboxId = row.sandbox_id ?? row.managed_sandbox_id; + requireThat( + sandboxId && + (row.host_id ?? row.managed_host_id) === hostId && + (managed + ? row.managed_task_id === taskId && + row.managed_host_id === hostId && + row.managed_sandbox_id === sandboxId && + row.managed_binding_epoch === row.binding_epoch + : row.managed_host_id === null && + row.managed_sandbox_id === null && + row.managed_binding_epoch === null), + "cleanup_exact_owned_target", + ); + const target = { + hostId, + sandboxId, + bindingEpoch: row.binding_epoch, + managed, + }; + const prior = this.cleanupTargets.get(sessionId); + requireThat( + !prior || canonicalJson(prior) === canonicalJson(target), + "cleanup_target_changed", + ); + this.cleanupTargets.set(sessionId, target); + } + /** Completed cleanup obligations are metadata confirmation, not physical owner proof. */ + async cleanupSettled(sessionId: string) { + const taskId = this.task(sessionId); + const target = this.cleanupTargets.get(sessionId); + requireThat(target, "cleanup_target_not_retained"); + const result = await this.coordinator.query( + "SELECT s.host_id::text,s.sandbox_id::text,s.binding_epoch::text,c.task_id AS managed_task_id,b.host_id::text AS managed_host_id,b.sandbox_id::text AS managed_sandbox_id,b.binding_epoch::text AS managed_binding_epoch,s.status,EXISTS(SELECT 1 FROM session_ops WHERE session_id=$1 AND state IN ('queued','running')) AS pending_operations,EXISTS(SELECT 1 FROM session_managed_boot_intents WHERE session_id=$1 AND NOT completed) AS pending_boots,EXISTS(SELECT 1 FROM sandbox_tombstones WHERE host_id=$2 AND sandbox_id=$3) AS pending_tombstone FROM sessions s LEFT JOIN session_managed_creations c ON c.session_id=s.id LEFT JOIN session_managed_boots b ON b.session_id=s.id WHERE s.id=$1", + [sessionId, target.hostId, target.sandboxId], + ); + requireThat(result.rowCount === 1, "cleanup_settled_row"); + const row = cleanupIdentitySchema + .extend({ + status: z.string(), + pending_operations: z.boolean(), + pending_boots: z.boolean(), + pending_tombstone: z.boolean(), + }) + .parse(result.rows[0]); + requireThat( + row.binding_epoch === target.bindingEpoch && + (row.host_id === null || row.host_id === target.hostId) && + (row.sandbox_id === null || row.sandbox_id === target.sandboxId) && + (target.managed + ? row.managed_task_id === taskId && + row.managed_host_id === target.hostId && + row.managed_sandbox_id === target.sandboxId && + row.managed_binding_epoch === target.bindingEpoch + : row.managed_task_id === null && + row.managed_host_id === null && + row.managed_sandbox_id === null && + row.managed_binding_epoch === null), + "cleanup_target_changed", + ); + return ( + ["completed", "failed", "dead"].includes(row.status) && + row.sandbox_id === null && + !row.pending_operations && + !row.pending_boots && + !row.pending_tombstone + ); + } + async destroyOperation(sessionId: string) { + this.task(sessionId); + const result = await this.coordinator.query( + "SELECT id::text,state FROM session_ops WHERE session_id=$1 AND kind='destroy' ORDER BY id DESC LIMIT 1", + [sessionId], + ); + if (result.rowCount === 0) return null; + requireThat(result.rowCount === 1, "coordinator_destroy_row"); + const parsed = z + .strictObject({ + id: z.string().regex(/^[1-9][0-9]*$/), + state: z.enum(["queued", "running", "done", "failed", "cancelled"]), + }) + .safeParse(result.rows[0]); + requireThat(parsed.success, "coordinator_destroy_shape"); + return parsed.data; + } + async started(sessionId: string, hostId: string) { + const taskId = this.task(sessionId); + const result = await this.coordinator.query( + "SELECT b.phase,b.host_id::text,b.binding_epoch::text,b.boot_attempt,c.native_digest,b.record->'request'->'key' AS source_key,b.record->'finalization'->>'combined_manifest_digest' AS combined_manifest_digest,b.record->'finalization'->'policy'->'context'->>'binding_hash' AS binding_hash FROM session_managed_boots b JOIN session_managed_creations c USING(session_id) WHERE b.session_id=$1 AND c.task_id=$2 AND b.host_id=$3", + [sessionId, taskId, hostId], + ); + requireThat(result.rowCount === 1, "coordinator_managed_boot_row"); + const row = z + .strictObject({ + phase: z.string(), + host_id: z.string(), + binding_epoch: z.string().regex(/^[0-9]+$/), + boot_attempt: z.string(), + native_digest: z.string().regex(/^sha256:[a-f0-9]{64}$/), + source_key: sourceKeySchema, + combined_manifest_digest: digest.nullable(), + binding_hash: digest.nullable(), + }) + .safeParse(result.rows[0]); + requireThat(row.success, "coordinator_managed_boot_shape"); + return row.data; + } + /** Select only nonsecret assets, start receipt and frozen manifests for the exact current boot. */ + async physicalBoot(sessionId: string, bootAttempt: string) { + const taskId = this.task(sessionId); + requireThat( + /^[A-Za-z0-9-]{1,128}$/.test(bootAttempt), + "physical_boot_attempt", + ); + const result = await this.coordinator.query( + "SELECT i.completed,c.native_version,c.native_bytes,c.native_digest,CASE WHEN octet_length((i.record->'assets')::text)<=65536 THEN (i.record->'assets')::text END AS assets,CASE WHEN octet_length((b.record->'start')::text)<=65536 THEN (b.record->'start')::text END AS start,CASE WHEN octet_length((b.record->'finalization'->'combined_manifest')::text)<=8388608 THEN (b.record->'finalization'->'combined_manifest')::text END AS combined_manifest,b.record->'finalization'->>'combined_manifest_digest' AS combined_manifest_digest FROM session_managed_boot_intents i JOIN session_managed_boots b ON b.session_id=i.session_id AND b.boot_attempt=i.intent_id JOIN session_managed_creations c ON c.session_id=i.session_id WHERE i.session_id=$1 AND c.task_id=$2 AND i.intent_id=$3 AND b.phase='started' AND i.binding_epoch=b.binding_epoch", + [sessionId, taskId, bootAttempt], + ); + requireThat(result.rowCount === 1, "physical_current_intent_row"); + return z + .strictObject({ + completed: z.literal(true), + native_version: z.literal(1), + native_bytes: z + .instanceof(Buffer) + .refine((v) => v.length > 0 && v.length <= 1048576), + native_digest: digest, + assets: z.string().min(1).max(65536), + start: z.string().min(1).max(65536), + combined_manifest: z.string().min(1).max(8388608), + combined_manifest_digest: digest, + }) + .parse(result.rows[0]); + } + async managedLifecycle( + sessionId: string, + expected: Awaited>, + ) { + const taskId = this.task(sessionId); + const result = await this.orchestrator.query( + "SELECT mcp_managed_lifecycle FROM task_session WHERE task_id=$1 AND session_id=$2", + [taskId, sessionId], + ); + requireThat(result.rowCount === 1, "orchestrator_lifecycle_row"); + const phase = z + .object({ + phase: z.enum([ + "reserving", + "held", + "preparing", + "finalizing", + "started", + "recovering", + "terminal", + ]), + }) + .safeParse(result.rows[0]?.mcp_managed_lifecycle); + requireThat(phase.success, "orchestrator_lifecycle_phase"); + if (phase.data.phase !== "started") return { phase: phase.data.phase }; + const parsed = z + .object({ + version: z.literal(1), + identity: z.object({ + sessionId: z.uuid(), + taskId: z.string(), + inputId: z.string(), + actorId: z.string(), + inputDigest: digest, + writeEligibility: z.string(), + parentSessionId: z.string().optional(), + }), + nativeVersion: z.literal(1), + nativeDigest: digest, + phase: z.enum([ + "reserving", + "held", + "preparing", + "finalizing", + "started", + "recovering", + "terminal", + ]), + bindingHash: digest, + sourceKey: sourceKeySchema, + }) + .safeParse(result.rows[0]?.mcp_managed_lifecycle); + requireThat(parsed.success, "orchestrator_lifecycle_shape"); + const value = parsed.data; + requireThat( + value.identity.sessionId === sessionId && + value.identity.taskId === taskId && + value.nativeDigest === expected.native_digest && + value.bindingHash === expected.binding_hash && + canonicalJson(value.sourceKey) === canonicalJson(expected.source_key) && + value.identity.inputId === + expected.source_key.creation.policy.inputId && + value.identity.actorId === + expected.source_key.creation.policy.actorId && + value.identity.inputDigest === + expected.source_key.creation.input_digest && + value.identity.writeEligibility === + expected.source_key.creation.policy.eligibility && + (value.identity.parentSessionId ?? null) === + (expected.source_key.creation.policy.parentSessionId ?? null), + "orchestrator_coordinator_source_identity", + ); + return { + phase: value.phase, + nativeDigest: value.nativeDigest, + bindingHash: value.bindingHash, + }; + } + async listener(sessionId: string) { + const taskId = this.task(sessionId); + const result = await this.orchestrator.query( + "SELECT l.owner IS NOT NULL AS leased,l.terminal_at IS NOT NULL AS terminal FROM session_listeners l JOIN task_session ts ON ts.session_id=l.session_id WHERE l.session_id=$1 AND ts.task_id=$2", + [sessionId, taskId], + ); + requireThat(result.rowCount === 1, "orchestrator_listener_row"); + const row = z + .strictObject({ leased: z.boolean(), terminal: z.boolean() }) + .safeParse(result.rows[0]); + requireThat(row.success, "orchestrator_listener_shape"); + return row.data; + } + async outbox(sessionId: string, callId: string) { + this.task(sessionId); + requireThat(/^[A-Za-z0-9_-]{1,64}$/.test(callId), "outbox_call_id"); + const result = await this.coordinator.query( + "SELECT attempts,delivered_at IS NOT NULL AS relayed,acked_at IS NOT NULL AS acknowledged FROM session_outbox WHERE session_id=$1 AND prompt_id=$2 AND kind='tool_result'", + [sessionId, `tool_result:${sessionId}:${callId}`], + ); + requireThat(result.rowCount === 1, "coordinator_outbox_row"); + const row = z + .strictObject({ + attempts: count, + relayed: z.boolean(), + acknowledged: z.boolean(), + }) + .safeParse(result.rows[0]); + requireThat(row.success, "coordinator_outbox_shape"); + return row.data; + } + async deliveryTarget(sessionId: string, callId: string) { + const taskId = this.task(sessionId); + requireThat( + /^[A-Za-z0-9_-]{1,64}$/.test(callId), + "delivery_target_call_id", + ); + const result = await this.coordinator.query( + "SELECT (SELECT min(e.idx)::text FROM session_events e WHERE e.session_id=c.session_id AND e.kind='tool_call_requested' AND e.payload->>'tool_call_id'=$3) AS requested_index,COALESCE((SELECT max(event_floor) FROM session_managed_interrupted_targets WHERE session_id=c.session_id),0)::text AS interruption_floor FROM session_managed_creations c WHERE c.session_id=$1 AND c.task_id=$2", + [sessionId, taskId, callId], + ); + requireThat(result.rowCount === 1, "delivery_target_creation"); + const parsed = z + .strictObject({ + requested_index: z.string().regex(/^[0-9]+$/), + interruption_floor: z.string().regex(/^[0-9]+$/), + }) + .safeParse(result.rows[0]); + requireThat(parsed.success, "delivery_target_actual_request"); + return { + ...parsed.data, + targetGone: + BigInt(parsed.data.requested_index) < + BigInt(parsed.data.interruption_floor), + }; + } + async close() { + await Promise.all([this.coordinator.end(), this.orchestrator.end()]); + } +} diff --git a/orchestrator/scripts/mcp-stack/physical.test.ts b/orchestrator/scripts/mcp-stack/physical.test.ts new file mode 100644 index 000000000..68f755c59 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/physical.test.ts @@ -0,0 +1,383 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createServer } from "node:tls"; +import { + TEST_CA, + TEST_CERT, + TEST_KEY, +} from "../../../mcp-worker/test/fixtures/tls.ts"; +import { X509Certificate } from "node:crypto"; +import { sha256 } from "./context.ts"; +import { + boundedHostJson, + parseHostRecord, + parseProxyDenials, + verifyColdChain, + verifyPhysicalBoot, +} from "./host-observer.ts"; +import { probeWorkerControl } from "./guest-physical.ts"; +import type { Context } from "./context.ts"; + +const id = (n: number) => + `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; +const h = "a".repeat(64), + d = `sha256:${h}`; +const aux = Array.from({ length: 3 }, (_, i) => ({ + drive_id: `dyn_${i}`, + guest_mount: `/opt/engram/dyn/${i}`, + fs_type: "squashfs", + sha256: h, +})); +const manifest = { manifest_id: id(4), version: 1 }; +const key = () => ({ + creation: { + session_id: id(1), + task_id: "task", + policy: { actorId: "actor", inputId: "input", eligibility: "interactive" }, + input_digest: d, + }, + context: { + target: { + session_id: id(1), + sandbox_id: id(2), + host_id: id(3), + binding_epoch: 1, + }, + operator_generation: 1, + operator_identity: d, + binding_hash: null, + }, + boot_attempt: "boot-first", + native_digest: d, +}); +function rawRecord() { + const request = { + key: key(), + cold: { + version: 1, + session_id: id(1), + sandbox_id: id(2), + boot_attempt: "boot-first", + image: "fixture-image", + disk: { + manifest, + sha256: h, + size_bytes: 4, + source: { + kind: "fresh_image", + base_manifest: { manifest_id: id(5), version: 1 }, + }, + }, + kernel_sha256: h, + aux_ro_drives: aux, + }, + bootstrap_policy: {}, + capture_authority: null, + sandbox_spec_json: '{"env":{}}', + }; + return { + version: 1, + request, + incarnation: "host-first", + state: { + state: "started", + intent: { + receipt: { + key: key(), + prepared_receipt_id: d, + finalization_digest: d, + host_incarnation: "host-first", + network_identity: d, + receipt_id: d, + }, + start_id: d, + }, + }, + allocation: null, + prepared: { + request, + host_incarnation: "host-first", + network_identity: d, + receipt_id: d, + guest_ip: "192.0.2.2", + }, + applied: null, + apply_digest: null, + finalization: d, + capture: null, + retired: null, + }; +} +function encoded(value: unknown) { + const body = JSON.stringify(value); + return Buffer.from( + JSON.stringify({ schema: 1, content_hash: sha256(body), id: id(2), body }), + ); +} +test("host envelope accepts production slot IDs in failed boot records", () => { + const raw = { + ...rawRecord(), + state: { state: "failed", key: key() }, + }; + expect( + parseHostRecord(encoded(raw), id(2)).record.request.cold.aux_ro_drives.map( + (drive) => drive.drive_id, + ), + ).toEqual(["dyn_0", "dyn_1", "dyn_2"]); +}); +test("host envelope rejects hyphenated slot IDs", () => { + const raw = rawRecord(); + raw.request.cold.aux_ro_drives = aux.map((drive, i) => ({ + ...drive, + drive_id: `dyn-${i}`, + })); + expect(() => parseHostRecord(encoded(raw), id(2))).toThrow(); +}); +test("host envelope rejects corruption, target substitution, environment and unknown fields", () => { + expect(parseHostRecord(encoded(rawRecord()), id(2)).record.version).toBe(1); + const corrupt = JSON.parse(encoded(rawRecord()).toString()); + corrupt.content_hash = "b".repeat(64); + expect(() => + parseHostRecord(Buffer.from(JSON.stringify(corrupt)), id(2)), + ).toThrow(); + expect(() => parseHostRecord(encoded(rawRecord()), id(9))).toThrow(); + const changed = rawRecord(); + changed.request.sandbox_spec_json = '{"env":{"TOKEN":"test"}}'; + expect(() => parseHostRecord(encoded(changed), id(2))).toThrow(); + expect(() => + parseHostRecord(encoded({ ...rawRecord(), pass: true }), id(2)), + ).toThrow(); + expect(() => boundedHostJson(JSON.stringify([[[1]]]), 2)).toThrow(); + expect(() => + boundedHostJson("[".repeat(65) + "0" + "]".repeat(65), 1024), + ).toThrow(); +}); +function captured() { + const request = { key: key(), capture_id: id(6) }; + const authority = { + version: 1, + request, + host_incarnation: "host-first", + disk_manifest: { manifest_id: id(6), version: 1 }, + disk_sha256: h, + disk_size_bytes: 4, + image: "fixture-image", + kernel_sha256: h, + aux_ro_drives: aux, + consistency: "crash_consistent_disk_cut_v1", + source_state: "destroyed", + record_digest: "", + }; + authority.record_digest = `sha256:${sha256(JSON.stringify(authority))}`; + return { + ...rawRecord(), + state: { state: "captured", authority }, + capture: { + request, + manifest: authority.disk_manifest, + artifact: [h, 4], + authority, + }, + }; +} +test("capture digest and cold destination bind exact source, artifact and epoch", () => { + const raw = captured(); + const old = parseHostRecord(encoded(raw), id(2)).record; + const next = structuredClone(old); + next.request.key.context.target.sandbox_id = id(7); + next.request.key.context.target.binding_epoch = 2; + next.request.key.boot_attempt = "boot-second"; + next.request.cold.sandbox_id = id(7); + next.request.cold.boot_attempt = "boot-second"; + next.request.capture_authority = old.capture!.authority; + next.request.cold.disk = { + manifest: { manifest_id: id(6), version: 1 }, + sha256: h, + size_bytes: 4, + source: { + kind: "capture", + capture_id: id(6), + session_id: id(1), + sandbox_id: id(2), + binding_epoch: 1, + record_digest: old.capture!.authority!.record_digest, + }, + }; + expect(verifyColdChain(old, next).captureSha256).toBe(h); + if (next.request.cold.disk.source.kind !== "capture") + throw new Error("expected capture"); + next.request.cold.disk.source.record_digest = `sha256:${"b".repeat(64)}`; + expect(() => verifyColdChain(old, next)).toThrow(); + raw.capture.authority.disk_size_bytes = 5; + expect(() => parseHostRecord(encoded(raw), id(2))).toThrow(); +}); +test("actual backend chain rejects backend substitution and changed native extension fields", () => { + const native = Buffer.from( + JSON.stringify([ + { + name: "native", + description: "test", + inputSchema: {}, + extension: "original", + }, + ]), + ); + const raw = rawRecord(); + raw.request.key.native_digest = `sha256:${sha256(native)}`; + raw.state.intent.receipt.key = raw.request.key; + const host = parseHostRecord(encoded(raw), id(2)).record; + const assets = { + request: { + version: 1, + host_id: id(3), + session_id: id(1), + intent_id: "boot-first", + image: "fixture-image", + image_digest: d, + disk_manifest: manifest, + aux_ro_drives: aux, + }, + host_incarnation: "host-first", + backend: "vz", + kernel_sha256: h, + disk_sha256: h, + disk_size_bytes: 4, + }; + const current = { + phase: "started", + host_id: id(3), + binding_epoch: "1", + boot_attempt: "boot-first", + native_digest: raw.request.key.native_digest, + source_key: host.request.key, + combined_manifest_digest: `sha256:${sha256(native)}`, + binding_hash: d, + }; + const observed = { + completed: true as const, + native_version: 1 as const, + native_bytes: native, + native_digest: current.native_digest, + assets: JSON.stringify(assets), + start: JSON.stringify(raw.state.intent), + combined_manifest: JSON.stringify([...native]), + combined_manifest_digest: current.combined_manifest_digest, + }; + const ctx = { + backend: "vz" as const, + hostId: id(3), + imageDigest: d, + artifacts: { kernel: { sha256: h } }, + }; + expect(verifyPhysicalBoot(ctx, host, current, observed).backend).toBe("vz"); + expect(() => + verifyPhysicalBoot( + { ...ctx, backend: "firecracker" }, + host, + current, + observed, + ), + ).toThrow(); + const wrong = Buffer.from(native.toString().replace("original", "changed")); + const wrongHash = `sha256:${sha256(wrong)}`; + expect(() => + verifyPhysicalBoot( + ctx, + host, + { ...current, combined_manifest_digest: wrongHash }, + { + ...observed, + combined_manifest: JSON.stringify([...wrong]), + combined_manifest_digest: wrongHash, + }, + ), + ).toThrow(); +}); +test("proxy denial requires one complete exact daemon event", () => { + const line = `2026-11-07T00:00:00.000000Z INFO engram_egress_proxy::proxy: egress rejected — destination not in any allow_hosts session_id=${id(1)} sni=resource.fixture.example original_dst=Some((10.88.2.30, 443))`; + const parse = ( + text: string, + partial = false, + session = id(1), + sni = "resource.fixture.example", + ) => parseProxyDenials(Buffer.from(text), partial, session, sni); + expect(parse(line + "\n")).toHaveLength(1); + expect(parse(line)).toHaveLength(0); + expect(parse(line + "\n", true)).toHaveLength(0); + expect(parse(line + "\n", false, id(9))).toHaveLength(0); + expect( + parse(line + "\n", false, id(1), "other.fixture.example"), + ).toHaveLength(0); + expect(parse(line.replace(" INFO ", " WARN ") + "\n")).toHaveLength(0); + expect( + parse(line.replace("engram_egress_proxy::proxy", "other::proxy") + "\n"), + ).toHaveLength(0); + expect(parse("prefix " + line + "\n")).toHaveLength(0); + expect(parse(line + " session_id=" + id(1) + "\n")).toHaveLength(0); + expect(parse(line + "\n" + line + "\n")).toHaveLength(0); + expect(parse(line + "\n" + "prefix " + line + "\n")).toHaveLength(0); + expect( + parse(line.replace("sni=resource", "sni=bad\nsni=resource") + "\n"), + ).toHaveLength(0); + expect(() => parse("x".repeat(65537))).toThrow(); +}); +test("worker TLS requires a verified server and certificate-required alert", async () => { + const root = await realpath( + await mkdtemp(join(tmpdir(), "worker-tls-test-")), + ); + const caPath = join(root, "ca.pem"); + await writeFile(caPath, TEST_CA, { mode: 0o600 }); + const server = createServer({ + key: TEST_KEY, + cert: TEST_CERT, + ca: TEST_CA, + requestCert: true, + rejectUnauthorized: true, + minVersion: "TLSv1.3", + maxVersion: "TLSv1.3", + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const port = (server.address() as { port: number }).port; + const endpoint = { + origin: `https://localhost:${port}`, + ca: { path: caPath, sha256: sha256(Buffer.from(TEST_CA)) }, + serverFingerprint: new X509Certificate(TEST_CERT).fingerprint256, + }; + const ctx = { + runRoot: root, + deadlines: { requestMs: 6000 }, + endpoints: { workerControl: endpoint }, + } as Context; + try { + expect((await probeWorkerControl(ctx)).classification).toBe( + "mtls_authentication_denial", + ); + endpoint.serverFingerprint = Array(32).fill("AA").join(":"); + await expect(probeWorkerControl(ctx)).rejects.toThrow(); + endpoint.serverFingerprint = new X509Certificate(TEST_CERT).fingerprint256; + await new Promise((resolve) => server.close(() => resolve())); + const permissive = createServer( + { + key: TEST_KEY, + cert: TEST_CERT, + minVersion: "TLSv1.3", + maxVersion: "TLSv1.3", + }, + (socket) => socket.destroy(), + ); + await new Promise((resolve) => + permissive.listen(port, "127.0.0.1", resolve), + ); + try { + await expect(probeWorkerControl(ctx)).rejects.toThrow(); + } finally { + await new Promise((resolve) => permissive.close(() => resolve())); + } + } finally { + if (server.listening) + await new Promise((resolve) => server.close(() => resolve())); + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/orchestrator/scripts/mcp-stack/preflight.ts b/orchestrator/scripts/mcp-stack/preflight.ts new file mode 100644 index 000000000..8f0c92ea4 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/preflight.ts @@ -0,0 +1,66 @@ +import { requireThat, RunnerFailure, type Context } from "./context.ts"; +import { FixtureObserver, type DatabaseObserver } from "./observer.ts"; +import { Supervisor } from "./supervisor.ts"; +import type { PublicClient } from "./transport.ts"; +import { waitFor } from "./bootstrap.ts"; +import { requireKillVariants } from "./case-controls.ts"; + +/** Validate all fixed variants before the first mutation, not midway through kill cases. */ +export function requirePrerequisites(ctx: Context) { + if (ctx.deadlines.requestMs < 6000) + throw new RunnerFailure( + "NEEDS_CONTEXT", + "guest_tls_request_deadline_6000ms", + ); + requireKillVariants(ctx.supervisor); + requireThat( + ctx.supervisor.operatorCandidates.normal.identity === + ctx.publication.candidateIdentity, + "normal_candidate_publication_identity", + ); +} +export async function preflight(ctx: Context, database: DatabaseObserver) { + requirePrerequisites(ctx); + const status = await new Supervisor(ctx).status(); + requireThat( + status.roles.every( + (v) => v.state === "running" && v.configurationId === "normal", + ), + "initial_six_roles_running_normal", + ); + await database.prerequisites(); + for (const protocol of ["modern", "legacy"] as const) { + const fixture = new FixtureObserver(ctx, protocol); + await fixture.ready(); + const observed = await fixture.observe(); + requireThat(observed.held === 0, "fixture_has_no_existing_held_call"); + } + return status; +} +export async function publicPrerequisites(ctx: Context, admin: PublicClient) { + const images = await admin.images.listEnabledImages({}); + requireThat(images.images.length <= 128, "public_image_catalog_bound"); + const matches = images.images.filter((v) => v.id === ctx.imageId); + requireThat( + matches.length === 1 && matches[0]!.manifestDigest === ctx.imageDigest, + "actual_public_enabled_image", + ); + const response = await waitFor( + ctx, + "actual_enrolled_host_ready", + () => admin.fleet.getHost({ hostId: ctx.hostId }), + (value) => value.host?.status === "ready", + ); + requireThat( + response.host?.id === ctx.hostId && + response.host.capacityTotalMib > 0n && + response.host.readyImageDigests.includes(ctx.imageDigest), + "actual_host_image_prefetched", + ); + return { + imageId: ctx.imageId, + imageDigest: matches[0]!.manifestDigest, + hostId: response.host.id, + status: response.host.status, + }; +} diff --git a/orchestrator/scripts/mcp-stack/proof-selection.test.ts b/orchestrator/scripts/mcp-stack/proof-selection.test.ts new file mode 100644 index 000000000..8db49787b --- /dev/null +++ b/orchestrator/scripts/mcp-stack/proof-selection.test.ts @@ -0,0 +1,55 @@ +import { expect, test } from "bun:test"; +import { parseGuestProofRecords } from "./guest.ts"; +import { selectProofRecords } from "./scenario-runtime.ts"; + +const plan = `sha256:${"a".repeat(64)}`; +const tool = "mcp_fixture_read"; +const initial = { + callId: "resume-read-1", + tool, + planSha256: plan, + resultReceived: true, +}; +const denied = { + callId: "resume-read-1", + tool, + planSha256: plan, + expectExposure: "absent", + selectedToolName: tool, + requestAttempted: true, + protocolDenialCode: "unknown_tool", + exposedMcpToolNamesSha256: `sha256:${"b".repeat(64)}`, +} as const; +const selection = { + callId: "resume-read-1", + tool, + planSha256: plan, + expectExposure: "absent" as const, +}; +function records(rows: object[]) { + return parseGuestProofRecords( + Buffer.from(rows.map((row) => JSON.stringify(row) + "\n").join("")), + ); +} + +test("selects the revoked proof whether the old proof file survives or is empty", () => { + for (const old of [[], [initial]]) { + const found = selectProofRecords( + [...records(old), ...records([denied])], + selection, + ); + expect(found).toHaveLength(1); + expect(found[0]!.proof).toEqual(denied); + } +}); + +test("rejects positive or mismatched proofs for the revoked call", () => { + for (const row of [ + initial, + { ...denied, callId: "other" }, + { ...denied, tool: "mcp_other", selectedToolName: "mcp_other" }, + { ...denied, planSha256: `sha256:${"c".repeat(64)}` }, + ]) { + expect(selectProofRecords(records([row]), selection)).toHaveLength(0); + } +}); diff --git a/orchestrator/scripts/mcp-stack/rebind.test.ts b/orchestrator/scripts/mcp-stack/rebind.test.ts new file mode 100644 index 000000000..1fc0a118d --- /dev/null +++ b/orchestrator/scripts/mcp-stack/rebind.test.ts @@ -0,0 +1,289 @@ +import { expect, test } from "bun:test"; +import { create } from "@bufbuild/protobuf"; +import { Code, ConnectError } from "@connectrpc/connect"; +import { + McpRebindDrainReason, + McpRebindDrainSchema, +} from "../../src/gen/engram/app/v1/integration_pb.ts"; +import { commitReviewedRebind } from "./rebind.ts"; +import * as rebind from "./rebind.ts"; +import { RunnerFailure } from "./context.ts"; + +const oldHash = `sha256:${"1".repeat(64)}`; +const previewHash = `sha256:${"2".repeat(64)}`; +const finalHash = `sha256:${"3".repeat(64)}`; +const request = { + sessionId: "00000000-0000-4000-8000-000000000001", + expectedBindingHash: oldHash, + expectedCandidateHash: previewHash, +}; +const ctx = { deadlines: { stepMs: 1000, requestMs: 1000, totalMs: 1000 } }; +function pending(reason = McpRebindDrainReason.DISPATCH_IN_PROGRESS) { + return new ConnectError("pending", Code.FailedPrecondition, undefined, [ + { + desc: McpRebindDrainSchema, + value: create(McpRebindDrainSchema, { + reason, + retryAfter: "2026-11-07T00:00:00Z", + }), + }, + ]); +} + +test("replays the exact reviewed request and returns the finalized hash", async () => { + const calls: (typeof request)[] = []; + const result = await commitReviewedRebind(ctx, request, async (arg) => { + calls.push({ ...arg }); + if (calls.length === 1) throw pending(); + return { bindingHash: finalHash }; + }); + expect(calls).toEqual([request, request]); + expect(result.bindingHash).toBe(finalHash); + expect(result.bindingHash).not.toBe(previewHash); +}); + +test("does not retry unexpected errors or other drain reasons", async () => { + for (const error of [ + new ConnectError("denied", Code.PermissionDenied), + pending(McpRebindDrainReason.UNKNOWN_INTENT), + new Error("unavailable"), + ]) { + let calls = 0; + await expect( + commitReviewedRebind(ctx, request, async () => { + calls++; + throw error; + }), + ).rejects.toThrow(); + expect(calls).toBe(1); + } +}); + +test("reports only fixed precondition categories without retrying unexpected details", async () => { + const cases = [ + { + error: new ConnectError("private-data", Code.FailedPrecondition), + seam: "rebind_precondition_details_0_reason_none_retry_na_category_other", + }, + { + error: new ConnectError( + "MCP operation could not be completed", + Code.FailedPrecondition, + ), + seam: "rebind_precondition_details_0_reason_none_retry_na_category_operation", + }, + { + error: pending(McpRebindDrainReason.UNKNOWN_INTENT), + seam: "rebind_precondition_details_1_reason_unknown_intent_retry_valid_category_other", + }, + { + error: new ConnectError( + "private-data", + Code.FailedPrecondition, + undefined, + [ + { + desc: McpRebindDrainSchema, + value: create(McpRebindDrainSchema, { + reason: McpRebindDrainReason.DISPATCH_IN_PROGRESS, + retryAfter: "private-data", + }), + }, + ], + ), + seam: "rebind_precondition_details_1_reason_dispatch_retry_invalid_category_other", + }, + { + error: new ConnectError( + "private-data", + Code.FailedPrecondition, + undefined, + [ + { desc: McpRebindDrainSchema, value: create(McpRebindDrainSchema) }, + { desc: McpRebindDrainSchema, value: create(McpRebindDrainSchema) }, + ], + ), + seam: "rebind_precondition_details_many_reason_none_retry_na_category_other", + }, + ]; + for (const [message, category] of [ + ["external MCP service is not configured", "service_unconfigured"], + ["managed boot authority changed or unavailable", "boot_authority"], + ["session policy changed or unavailable", "policy_changed"], + ["host policy unavailable", "host_policy"], + ["retained policy unavailable", "retained_policy"], + ["managed guest identity unavailable", "guest_identity"], + ["managed egress source unavailable", "egress_source"], + ["managed policy does not support tunnels", "tunnels"], + ["desired policy changed", "desired_policy"], + ["managed guest identity changed", "guest_identity_changed"], + ["host acknowledgement mismatch", "host_ack"], + ["physical placement changed", "placement"], + ["managed creation authority changed", "creation_authority"], + ] as const) { + cases.push({ + error: new ConnectError(message, Code.FailedPrecondition), + seam: `rebind_precondition_details_0_reason_none_retry_na_category_${category}`, + }); + } + for (const { error, seam } of cases) { + let calls = 0; + try { + await commitReviewedRebind(ctx, request, async () => { + calls++; + throw error; + }); + throw new Error("unexpected success"); + } catch (caught) { + expect(caught).toHaveProperty("seam", seam); + expect(String(caught)).not.toContain("private-data"); + expect(calls).toBe(1); + } + } +}); + +test("rejects malformed drain detail and final binding hashes", async () => { + const invalidDate = new ConnectError( + "pending", + Code.FailedPrecondition, + undefined, + [ + { + desc: McpRebindDrainSchema, + value: create(McpRebindDrainSchema, { + reason: McpRebindDrainReason.DISPATCH_IN_PROGRESS, + retryAfter: "invalid", + }), + }, + ], + ); + let calls = 0; + await expect( + commitReviewedRebind(ctx, request, async () => { + calls++; + throw invalidDate; + }), + ).rejects.toThrow(); + expect(calls).toBe(1); + for (const bindingHash of [oldHash, "bad"]) { + await expect( + commitReviewedRebind(ctx, request, async () => ({ bindingHash })), + ).rejects.toThrow(); + } +}); + +test("OAuth recovery marks its branch, stage and observed phase without copying arbitrary failures", async () => { + for (const change of ["account", "scope", "reconnect"] as const) { + for (const stage of ["prepare", "commit", "started"] as const) { + for (const error of [ + new Error("private-data"), + new ConnectError("private-data", Code.FailedPrecondition), + new RunnerFailure("ASSERTION_FAILED", "private-data"), + ]) { + let calls = 0; + await expect( + rebind.withRebindDiagnostics( + change, + stage, + async () => ({ phase: "recovering" }), + async () => { + calls++; + throw error; + }, + ), + ).rejects.toHaveProperty( + "seam", + `oauth_${change}_recovery_${stage}_phase_recovering_unclassified`, + ); + expect(calls).toBe(1); + } + } + } + for (const readPhase of [ + async () => ({ phase: "private-data" }), + async () => { + throw new Error("private-data"); + }, + ]) { + await expect( + rebind.withRebindDiagnostics("account", "commit", readPhase, async () => { + throw new Error("private-data"); + }), + ).rejects.toHaveProperty( + "seam", + "oauth_account_recovery_commit_phase_unavailable_unclassified", + ); + } +}); + +test("OAuth diagnostics retain fixed preconditions and never authorize another commit", async () => { + for (const error of [ + new ConnectError("private-data", Code.FailedPrecondition), + pending(McpRebindDrainReason.UNKNOWN_INTENT), + pending(McpRebindDrainReason.LEGACY_INTENT), + pending(McpRebindDrainReason.UNSPECIFIED), + // Unknown wire enum values must also fail closed. + pending(99 as McpRebindDrainReason), + ]) { + let calls = 0; + try { + await rebind.withRebindDiagnostics( + "scope", + "commit", + async () => ({ phase: "recovering" }), + () => + commitReviewedRebind(ctx, request, async () => { + calls++; + throw error; + }), + ); + throw new Error("unexpected success"); + } catch (caught) { + expect(caught).toBeInstanceOf(RunnerFailure); + expect(caught).toHaveProperty( + "seam", + expect.stringMatching( + /^oauth_scope_recovery_commit_phase_recovering_rebind_precondition_details_(0|1)_reason_(none|unknown_intent|legacy_intent|unspecified|other)_retry_(na|valid)_category_other$/, + ), + ); + expect(String(caught)).not.toContain("private-data"); + expect(String(caught).length).toBeLessThanOrEqual(217); + expect(calls).toBe(1); + } + } + const calls: (typeof request)[] = []; + const result = await rebind.withRebindDiagnostics( + "reconnect", + "commit", + async () => { + throw new Error("must not read on success"); + }, + () => + commitReviewedRebind(ctx, request, async (arg) => { + calls.push({ ...arg }); + if (calls.length === 1) throw pending(); + return { bindingHash: finalHash }; + }), + ); + expect(calls).toEqual([request, request]); + expect(result.bindingHash).toBe(finalHash); +}); + +test("recovery diagnostics retain the fixed identity refusal seam", async () => { + await expect( + rebind.withRebindDiagnostics( + "reconnect", + "started", + async () => ({ phase: "started" }), + async () => { + throw new RunnerFailure( + "ASSERTION_FAILED", + "managed_cold_recovery_identity", + ); + }, + ), + ).rejects.toHaveProperty( + "seam", + "oauth_reconnect_recovery_started_phase_started_managed_cold_recovery_identity", + ); +}); diff --git a/orchestrator/scripts/mcp-stack/rebind.ts b/orchestrator/scripts/mcp-stack/rebind.ts new file mode 100644 index 000000000..071261408 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/rebind.ts @@ -0,0 +1,316 @@ +import { Code, ConnectError } from "@connectrpc/connect"; +import type { MessageInitShape } from "@bufbuild/protobuf"; +import { + type Profile, + type ProfileIntegrationGrant, + type UpdateProfileRequestSchema, +} from "../../src/gen/engram/app/v1/profile_pb.ts"; +import { + McpRebindDrainReason, + McpRebindDrainSchema, +} from "../../src/gen/engram/app/v1/integration_pb.ts"; +import { bindingTools, expectRejected, waitFor } from "./bootstrap.ts"; +import { requireThat, RunnerFailure, type Context } from "./context.ts"; +import { canonicalJson, guestReady } from "./guest.ts"; +import type { DatabaseObserver } from "./observer.ts"; +import type { StartedScenario } from "./scenario-runtime.ts"; +import type { PublicClient } from "./transport.ts"; + +/** UpdateProfile replaces the profile body; retain every unrelated public field. */ +export function grantUpdateRequest( + profile: Profile, + integrationGrants: ProfileIntegrationGrant[], +): MessageInitShape { + return { + id: profile.id, + name: profile.name, + description: profile.description, + icon: profile.icon, + imageId: profile.imageId, + includeUserTokens: profile.includeUserTokens, + envVars: profile.envVars, + skills: profile.skills, + network: profile.network, + secrets: profile.secrets, + harness: profile.harness, + model: profile.model, + effort: profile.effort, + integrationGrants, + repos: profile.repos, + modelRouter: profile.modelRouter, + apps: profile.apps, + programmaticOnly: profile.programmaticOnly, + expectedMcpGrantRevision: profile.mcpGrantRevision, + }; +} +export async function revokeProfileGrants( + admin: PublicClient, + profileId: string, +) { + const current = await admin.profiles.getProfile({ id: profileId }); + requireThat(current.profile?.id === profileId, "revoke_actual_profile"); + const updated = await admin.profiles.updateProfile( + grantUpdateRequest(current.profile, []), + ); + requireThat( + updated.profile?.id === profileId && + updated.profile.integrationGrants.length === 0 && + updated.profile.mcpGrantRevision === + current.profile.mcpGrantRevision + 1n, + "empty_grants_publication", + ); + return updated.profile; +} + +type RebindRequest = { + sessionId: string; + expectedBindingHash: string; + expectedCandidateHash: string; +}; + +type OAuthRebindChange = "account" | "scope" | "reconnect"; +type RecoveryStage = "prepare" | "commit" | "started"; +class RebindPreconditionFailure extends RunnerFailure {} +const recoverySeams = new Set([ + "reviewed_rebind_candidate", + "managed_rebind_commit_started", + "rebind_final_binding_hash", + "managed_cold_recovery_deadline", + "managed_cold_recovery_started", + "managed_cold_recovery_identity", + "managed_cold_recovery_orchestrator_started", + "managed_cold_recovery_reviewed_binding", + "managed_cold_recovery_active", +]); + +/** Only local fixed markers and validated phases enter failure evidence. */ +export async function withRebindDiagnostics( + change: OAuthRebindChange | undefined, + stage: RecoveryStage, + readPhase: () => Promise<{ phase: string }>, + run: () => Promise, +): Promise { + if (!change) return run(); + try { + return await run(); + } catch (error) { + let phase = "unavailable"; + try { + const observed = await readPhase(); + if ( + [ + "reserving", + "held", + "preparing", + "finalizing", + "started", + "recovering", + "terminal", + ].includes(observed.phase) + ) + phase = observed.phase; + } catch { + // An unavailable observation cannot replace the original failed boundary. + } + const reason = + error instanceof RebindPreconditionFailure || + (error instanceof RunnerFailure && recoverySeams.has(error.seam)) + ? error.seam + : "unclassified"; + throw new RunnerFailure( + "ASSERTION_FAILED", + `oauth_${change}_recovery_${stage}_phase_${phase}_${reason}`, + ); + } +} + +/** Replay the same reviewed request only while the server reports a typed drain. */ +export async function commitReviewedRebind( + ctx: Pick, + request: RebindRequest, + commit: (request: RebindRequest) => Promise<{ bindingHash: string }>, +) { + const result = await waitFor( + ctx, + "managed_rebind_commit_started", + async () => { + try { + return await commit(request); + } catch (error) { + if ( + !(error instanceof ConnectError) || + error.code !== Code.FailedPrecondition + ) + throw error; + const details = error.findDetails(McpRebindDrainSchema); + const validRetry = + details.length === 1 && + Number.isFinite(Date.parse(details[0]!.retryAfter)); + if ( + details.length !== 1 || + details[0]!.reason !== McpRebindDrainReason.DISPATCH_IN_PROGRESS || + !validRetry + ) { + const count = + details.length === 0 ? "0" : details.length === 1 ? "1" : "many"; + const reason = + details.length !== 1 + ? "none" + : details[0]!.reason === McpRebindDrainReason.DISPATCH_IN_PROGRESS + ? "dispatch" + : details[0]!.reason === McpRebindDrainReason.UNKNOWN_INTENT + ? "unknown_intent" + : details[0]!.reason === McpRebindDrainReason.LEGACY_INTENT + ? "legacy_intent" + : details[0]!.reason === McpRebindDrainReason.UNSPECIFIED + ? "unspecified" + : "other"; + const retry = + details.length === 1 ? (validRetry ? "valid" : "invalid") : "na"; + const category = + new Map([ + ["MCP calls are still draining", "draining"], + [ + "An administrator must publish MCP grants before rebind", + "publication", + ], + ["MCP operation could not be completed", "operation"], + [ + "external MCP service is not configured", + "service_unconfigured", + ], + [ + "managed boot authority changed or unavailable", + "boot_authority", + ], + ["session policy changed or unavailable", "policy_changed"], + ["host policy unavailable", "host_policy"], + ["retained policy unavailable", "retained_policy"], + ["managed guest identity unavailable", "guest_identity"], + ["managed egress source unavailable", "egress_source"], + ["managed policy does not support tunnels", "tunnels"], + ["desired policy changed", "desired_policy"], + ["managed guest identity changed", "guest_identity_changed"], + ["host acknowledgement mismatch", "host_ack"], + ["physical placement changed", "placement"], + ["managed creation authority changed", "creation_authority"], + ]).get(error.rawMessage) ?? "other"; + throw new RebindPreconditionFailure( + "ASSERTION_FAILED", + `rebind_precondition_details_${count}_reason_${reason}_retry_${retry}_category_${category}`, + ); + } + return null; + } + }, + (value) => value !== null, + ); + requireThat( + result && + /^sha256:[a-f0-9]{64}$/.test(result.bindingHash) && + result.bindingHash !== request.expectedBindingHash, + "rebind_final_binding_hash", + ); + return result; +} + +/** The supported managed resume is reviewed cold recovery; ordinary Resume stays fenced. */ +export async function recoverCurrentBinding( + ctx: Context, + scenario: StartedScenario, + database: DatabaseObserver, + change?: OAuthRebindChange, +) { + const old = await database.started(scenario.sessionId, ctx.hostId); + requireThat( + old.phase === "started" && old.binding_hash, + "rebind_started_source", + ); + const oldHash = old.binding_hash; + const ordinaryResumeDenied = await expectRejected( + () => + scenario.reviewClient.sessions.resume({ sessionId: scenario.sessionId }), + [Code.Unavailable], + ); + const observe = (stage: RecoveryStage, run: () => Promise) => + withRebindDiagnostics( + change, + stage, + () => database.managedLifecycle(scenario.sessionId, old), + run, + ); + const candidate = await observe("prepare", async () => { + const prepared = await scenario.reviewClient.integration.prepareMcpRebind({ + sessionId: scenario.sessionId, + expectedBindingHash: oldHash, + }); + requireThat( + /^sha256:[a-f0-9]{64}$/.test(prepared.bindingHash) && + prepared.bindingHash !== oldHash && + prepared.bindingEpoch === String(Number(old.binding_epoch) + 1) && + prepared.drain === undefined, + "reviewed_rebind_candidate", + ); + return prepared; + }); + const deadline = performance.now() + ctx.deadlines.stepMs; + const committed = await observe("commit", () => + commitReviewedRebind( + ctx, + { + sessionId: scenario.sessionId, + expectedBindingHash: oldHash, + expectedCandidateHash: candidate.bindingHash, + }, + (request) => scenario.reviewClient.integration.rebindMcp(request), + ), + ); + return observe("started", async () => { + const remainingMs = deadline - performance.now(); + requireThat(remainingMs > 0, "managed_cold_recovery_deadline"); + const current = await waitFor( + { deadlines: { ...ctx.deadlines, stepMs: remainingMs } }, + "managed_cold_recovery_started", + () => database.started(scenario.sessionId, ctx.hostId), + (value) => + value.phase === "started" && + value.binding_hash === committed.bindingHash, + ); + requireThat( + current.binding_epoch === candidate.bindingEpoch && + current.source_key.context.target.sandbox_id !== + old.source_key.context.target.sandbox_id && + current.boot_attempt !== old.boot_attempt && + current.native_digest === old.native_digest && + canonicalJson(current.source_key.creation) === + canonicalJson(old.source_key.creation), + "managed_cold_recovery_identity", + ); + await waitFor( + ctx, + "managed_cold_recovery_orchestrator_started", + () => database.managedLifecycle(scenario.sessionId, current), + (value) => value.phase === "started", + ); + const bound = await bindingTools(scenario.reviewClient, scenario.sessionId); + requireThat( + canonicalJson(bound) === canonicalJson(candidate.bindingTools), + "managed_cold_recovery_reviewed_binding", + ); + const session = await scenario.reviewClient.sessions.getSession({ + sessionId: scenario.sessionId, + }); + requireThat( + session.session?.status === "active", + "managed_cold_recovery_active", + ); + await guestReady(ctx, scenario.client, scenario.sessionId); + return { + old, + current, + candidate, + finalBindingHash: committed.bindingHash, + ordinaryResumeDenied, + }; + }); +} diff --git a/orchestrator/scripts/mcp-stack/review-regressions.test.ts b/orchestrator/scripts/mcp-stack/review-regressions.test.ts new file mode 100644 index 000000000..5647be879 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/review-regressions.test.ts @@ -0,0 +1,350 @@ +import { expect, test } from "bun:test"; +import { create } from "@bufbuild/protobuf"; +import { ListSessionEventsResponseSchema } from "../../src/gen/engram/app/v1/session_pb.ts"; +import { toolCompletionEvents } from "./scenario-runtime.ts"; +import { selectColdDenial } from "./case-physical.ts"; +import { + assertDuplicateWriteUnknown, + recoverSettledUnknown, + assertNewIdWriteDenial, + assertAcknowledgedNativeCompletion, +} from "./case-recovery.ts"; +import { canonicalJson, parseGuestProofRecords } from "./guest.ts"; +import { sha256 } from "./context.ts"; + +const alias = "mcp_fixture_read"; +const callId = "cold-old-read"; +const planSha256 = `sha256:${"a".repeat(64)}`; +const emptyExposure = `sha256:${sha256(canonicalJson([]))}`; + +test("acknowledged completion allows a delivery run but requires one successful matching call", () => { + const completion = { + tool_call_id: "call-1", + run_id: "delivery-run", + ok: true, + }; + expect(() => + assertAcknowledgedNativeCompletion([completion], "call-1", "original-run"), + ).not.toThrow(); + for (const events of [ + [], + [completion, completion], + [{ ...completion, tool_call_id: "other" }], + [{ ...completion, run_id: "" }], + [{ ...completion, ok: false }], + ]) + expect(() => + assertAcknowledgedNativeCompletion(events, "call-1", "original-run"), + ).toThrow(); + expect(() => + assertAcknowledgedNativeCompletion([completion], "call-1", ""), + ).toThrow(); +}); +const records = (proof: object) => + parseGuestProofRecords(Buffer.from(JSON.stringify(proof) + "\n")); +const positive = () => + records({ + callId, + tool: alias, + planSha256, + resultReceived: true, + delivery: "available", + outcome: "succeeded", + isError: false, + }); +const negative = () => + records({ + callId, + tool: alias, + planSha256, + expectExposure: "absent", + selectedToolName: alias, + requestAttempted: true, + protocolDenialCode: "unknown_tool", + exposedMcpToolNamesSha256: emptyExposure, + }); + +for (const harness of ["claude", "codex"] as const) { + const nativeName = harness === "claude" ? `mcp__engrams__${alias}` : alias; + const payload = (id: string, name = nativeName) => + JSON.stringify({ + tool_call_id: id, + tool_name: name, + run_id: "actual-run", + ok: true, + }); + test(`${harness} completion uses exact native query and preserves call/run identity`, async () => { + const selected = await toolCompletionEvents({ + alias, + sessionId: "session", + input: { harness, callId }, + reviewClient: { + sessions: { + listSessionEvents: async (request) => { + expect(request).toEqual({ + sessionId: "session", + kinds: ["tool_call_completed"], + toolNames: [nativeName], + limit: 129n, + }); + return create(ListSessionEventsResponseSchema, { + events: [ + { + idx: 7n, + kind: "tool_call_completed", + payloadJson: payload(callId), + }, + { + idx: 8n, + kind: "tool_call_completed", + payloadJson: payload("other-call"), + }, + ], + }); + }, + }, + }, + }); + expect(selected).toEqual([ + { + tool_call_id: callId, + tool_name: nativeName, + run_id: "actual-run", + ok: true, + idx: "7", + sha256: sha256(payload(callId)), + }, + ]); + }); + test(`${harness} completion rejects a non-native name even if returned by server`, async () => { + await expect( + toolCompletionEvents({ + alias, + sessionId: "session", + input: { harness, callId }, + reviewClient: { + sessions: { + listSessionEvents: async () => + create(ListSessionEventsResponseSchema, { + events: [ + { + idx: 7n, + kind: "tool_call_completed", + payloadJson: payload( + callId, + harness === "claude" ? alias : `mcp__engrams__${alias}`, + ), + }, + ], + }), + }, + }, + }), + ).rejects.toThrow("completion_event_shape"); + }); + test(`${harness} target-gone absence detects an actual native completion`, async () => { + const scenario = { + alias, + sessionId: "session", + input: { harness, callId }, + reviewClient: { + sessions: { + listSessionEvents: async ( + request: Parameters< + Parameters< + typeof toolCompletionEvents + >[0]["reviewClient"]["sessions"]["listSessionEvents"] + >[0], + ) => + create(ListSessionEventsResponseSchema, { + events: request.toolNames?.includes(nativeName) + ? [ + { + idx: 7n, + kind: "tool_call_completed", + payloadJson: payload(callId), + }, + ] + : [], + }), + }, + }, + }; + expect((await toolCompletionEvents(scenario)).length === 0).toBe(false); + expect((await toolCompletionEvents(scenario, "other-call")).length).toBe(0); + }); +} + +for (const order of ["old-first", "new-first"] as const) { + test(`cold refusal selects new record with retained positive JSONL in ${order} filename order`, () => { + const baseline = positive(); + const refused = negative(); + const all = + order === "old-first" + ? [...baseline, ...refused] + : [...refused, ...baseline]; + expect(selectColdDenial(baseline, all)).toEqual(refused); + expect(baseline).toEqual(positive()); + }); +} +test("cold refusal rejects lost, replaced, duplicate or extra baseline/proof records", () => { + const baseline = positive(), + refused = negative(); + const replaced = records({ ...baseline[0]!.proof, isError: true }); + for (const all of [ + refused, + [...replaced, ...refused], + [...baseline, ...refused, ...refused], + baseline, + ]) { + expect(() => selectColdDenial(baseline, all)).toThrow(); + } + expect(() => selectColdDenial([], refused)).toThrow(); +}); +test("cold refusal requires exact same call, tool, plan and empty actual exposure", () => { + const baseline = positive(), + refused = negative(); + for (const change of [ + { callId: "other-call" }, + { tool: "mcp_other", selectedToolName: "mcp_other" }, + { planSha256: `sha256:${"b".repeat(64)}` }, + { exposedMcpToolNamesSha256: `sha256:${"b".repeat(64)}` }, + ]) { + expect(() => + selectColdDenial(baseline, [ + ...baseline, + ...records({ ...refused[0]!.proof, ...change }), + ]), + ).toThrow(); + } +}); +test("new-ID unknown-write restriction requires actual available runtime refusal", () => { + expect(() => + assertNewIdWriteDenial( + records({ + callId: "held-write-2", + tool: alias, + planSha256, + resultReceived: true, + delivery: "available", + outcome: null, + isError: true, + }), + ), + ).not.toThrow(); +}); +test("new-ID unknown-write restriction rejects unavailable or missing runtime refusal", () => { + for (const change of [ + { delivery: "unavailable", outcome: "outcome_unknown" }, + { delivery: "unavailable", outcome: "failed" }, + { outcome: "failed" }, + { outcome: "succeeded", isError: false }, + { isError: false }, + { resultReceived: false }, + ]) + expect(() => + assertNewIdWriteDenial( + records({ + callId: "held-write-2", + tool: alias, + planSha256, + resultReceived: true, + delivery: "available", + outcome: null, + isError: true, + ...change, + }), + ), + ).toThrow(); + expect(() => assertNewIdWriteDenial([])).toThrow(); +}); + +test("duplicate unknown-write proof rejects invented success in either file order", () => { + const unknown = records({ + callId: "held-write-1", + tool: alias, + planSha256, + resultReceived: true, + delivery: "unavailable", + outcome: "outcome_unknown", + }); + const success = records({ + ...unknown[0]!.proof, + delivery: "available", + outcome: "succeeded", + isError: false, + }); + for (const unordered of [ + [...success, ...unknown], + [...unknown, ...success], + ]) + expect(() => assertDuplicateWriteUnknown(unordered)).toThrow(); + expect(() => + assertDuplicateWriteUnknown([...unknown, ...unknown]), + ).not.toThrow(); + expect(() => assertDuplicateWriteUnknown([])).toThrow(); +}); + +test("settled unknown recovery does not need a second guest receipt", async () => { + const before = { + execution: { + executionId: "execution-1", + state: "outcome_unknown", + runId: "run-1", + toolCallId: "call-1", + connectionId: "connection-1", + }, + result: { + state: "outcome_unknown", + resultUnavailable: false, + resultJson: JSON.stringify({ + outcome: "outcome_unknown", + delivery: "unavailable", + }), + }, + outbox: { attempts: 1, relayed: true, acknowledged: true }, + }; + let restarts = 0; + let reads = 0; + const after = await recoverSettledUnknown({ + before, + restart: async () => { + restarts++; + return { role: "orchestrator" }; + }, + readAfter: async () => { + reads++; + return structuredClone(before); + }, + }); + expect(restarts).toBe(1); + expect(reads).toBe(1); + expect(after.snapshot).toEqual(before); + for (const change of [ + { execution: { ...before.execution, executionId: "other" } }, + { execution: { ...before.execution, runId: "other" } }, + { execution: { ...before.execution, toolCallId: "other" } }, + { result: { ...before.result, state: "succeeded" } }, + { result: { ...before.result, resultUnavailable: true } }, + { outbox: { ...before.outbox, attempts: 2 } }, + { outbox: { ...before.outbox, acknowledged: false } }, + { + result: { + ...before.result, + resultJson: JSON.stringify({ + outcome: "outcome_unknown", + delivery: "unavailable", + output: {}, + }), + }, + }, + ]) { + await expect( + recoverSettledUnknown({ + before, + restart: async () => ({}), + readAfter: async () => ({ ...structuredClone(before), ...change }), + }), + ).rejects.toThrow(); + } +}); diff --git a/orchestrator/scripts/mcp-stack/run.ts b/orchestrator/scripts/mcp-stack/run.ts new file mode 100644 index 000000000..30203ca75 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/run.ts @@ -0,0 +1,35 @@ +import { interactiveApprovalDriver } from "./case-approval.ts"; +import { killCancelDriver } from "./case-controls.ts"; +import { catalogFidelityDriver } from "./case-fidelity.ts"; +import { + grantDenialsDriver, + unattendedGrantDriver, + planChildIntersectionDriver, +} from "./case-grants.ts"; +import { oauthAuthorityDriver } from "./case-oauth.ts"; +import { + credentialNetworkIsolationDriver, + diskOnlyColdRebindDriver, +} from "./case-physical.ts"; +import { + heldWriteRestartDriver, + outboxGuestAckDriver, + revokeResumeDriver, +} from "./case-recovery.ts"; +import { main, type DriverRegistry } from "./runner.ts"; + +export const drivers: DriverRegistry = { + "catalog-fidelity": catalogFidelityDriver, + "grant-denials": grantDenialsDriver, + "interactive-approval": interactiveApprovalDriver, + "unattended-grant": unattendedGrantDriver, + "plan-child-intersection": planChildIntersectionDriver, + "credential-network-isolation": credentialNetworkIsolationDriver, + "disk-only-cold-rebind": diskOnlyColdRebindDriver, + "revoke-resume": revokeResumeDriver, + "held-write-restart": heldWriteRestartDriver, + "oauth-authority": oauthAuthorityDriver, + "kill-cancel": killCancelDriver, + "outbox-guest-ack": outboxGuestAckDriver, +}; +if (import.meta.main) await main(drivers, process.argv.slice(2)); diff --git a/orchestrator/scripts/mcp-stack/runner.ts b/orchestrator/scripts/mcp-stack/runner.ts new file mode 100644 index 000000000..05b2c80b2 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/runner.ts @@ -0,0 +1,245 @@ +import { ConnectError } from "@connectrpc/connect"; +import { bootstrapHumans, publishOperator } from "./bootstrap.ts"; +import { parseArgs } from "./cli.ts"; +import { + loadContext, + requireThat, + RunnerFailure, + sha256, + type Context, +} from "./context.ts"; +import { + completeAssertions, + type CaseDriver, + type DriverInput, +} from "./driver.ts"; +import { Supervisor } from "./supervisor.ts"; +import { CaseBodyFailure, CaseCleanupFailure } from "./scenario-runtime.ts"; +import { Evidence } from "./evidence.ts"; +import { canonicalJson } from "./guest.ts"; +import { runInfrastructure } from "./infrastructure.ts"; +import { + FULL_MATRIX, + selectedCases, + SCENARIOS, + type RunSelection, + type ScenarioId, +} from "./manifest.ts"; +import { DatabaseObserver } from "./observer.ts"; +import { preflight, publicPrerequisites } from "./preflight.ts"; +import { publicRpcMethod } from "./transport.ts"; + +export type DriverRegistry = Record; +export function validateDrivers( + drivers: Record, +): asserts drivers is DriverRegistry { + const names = Object.keys(drivers); + requireThat( + names.length === SCENARIOS.length && + SCENARIOS.every((s) => typeof drivers[s.id] === "function") && + names.every((name) => SCENARIOS.some((s) => s.id === name)), + "complete_driver_registry", + ); +} +export function safeFailure(error: unknown): RunnerFailure { + if (error instanceof CaseBodyFailure || error instanceof CaseCleanupFailure) + return safeFailure(error.failure); + if ( + error instanceof RunnerFailure && + /^[A-Za-z0-9_./:-]{1,200}$/.test(error.seam) + ) + return error; + if (error instanceof ConnectError) { + const method = publicRpcMethod(error); + return new RunnerFailure( + "TRANSPORT_FAILED", + `public_rpc_${method ? method + "_" : ""}code_${error.code}`, + ); + } + return new RunnerFailure("ASSERTION_FAILED", "unexpected_runner_error"); +} + +/** Dispatch the validated selection through the same complete driver registry. */ +export async function runSelectedCases( + drivers: DriverRegistry, + selection: RunSelection, + input: Omit, + evidence: Evidence, + assertReady = async () => { + await preflight(input.ctx, input.database); + await publicPrerequisites(input.ctx, input.principals.admin); + }, + deadline = performance.now() + input.ctx.deadlines.totalMs, +) { + validateDrivers(drivers); + const cases = selectedCases(selection); + let firstFailure: RunnerFailure | undefined; + for (const [index, selected] of cases.entries()) { + let driverReturned = false; + try { + const driverAssertions = await drivers[selected.id]({ + ...input, + harness: selected.harness, + }); + driverReturned = true; + const assertions = completeAssertions(selected.id, driverAssertions); + driverReturned = false; + const recorded = []; + for (const assertion of assertions) { + const observationHashes = []; + for (const observation of assertion.observations) + observationHashes.push( + await evidence.observe({ ...observation, caseKey: selected.key }), + ); + recorded.push({ assertion: assertion.assertion, observationHashes }); + } + await evidence.result({ + key: selected.key, + status: "PASS", + assertions: recorded, + }); + } catch (error) { + const caseError = + error instanceof CaseBodyFailure ? error.failure : error; + const failure = safeFailure(error); + await evidence.result({ + key: selected.key, + status: failure.code === "NEEDS_CONTEXT" ? "NEEDS_CONTEXT" : "FAIL", + assertions: [], + seam: failure.seam, + }); + if ( + !(error instanceof CaseBodyFailure || driverReturned) || + !(caseError instanceof RunnerFailure) || + failure !== caseError || + failure.code !== "ASSERTION_FAILED" || + // Unclassified diagnostics can hide transport or unexpected errors. + failure.seam.endsWith("_unclassified") + ) + throw failure; + firstFailure ??= failure; + if (index + 1 < cases.length) { + requireThat(performance.now() < deadline, "runner_total_deadline"); + await assertReady(); + requireThat(performance.now() < deadline, "runner_total_deadline"); + } + } + } + if (firstFailure) throw firstFailure; +} + +/** Physical release is checked before infrastructure can permit MCP cases. */ +export async function completeInfrastructure( + backend: Context["backend"], + rows: Awaited>, + evidence: Pick, + supervisor: { infrastructureReady(): Promise }, +) { + if (backend === "firecracker") await supervisor.infrastructureReady(); + await evidence.infrastructure(rows); +} + +/** Failed cases leave an incomplete artifact, even when later cases can run. */ +export async function runMatrix( + ctx: Context, + drivers: DriverRegistry, + selection: RunSelection = FULL_MATRIX, + deadline = performance.now() + ctx.deadlines.totalMs, +) { + selectedCases(selection); + validateDrivers(drivers); + const evidence = await Evidence.create(ctx, selection); + let database: DatabaseObserver | undefined; + try { + database = await DatabaseObserver.connect(ctx); + const supervisor = await preflight(ctx, database); + const publication = await publishOperator(ctx); + const principals = await bootstrapHumans(ctx); + const ready = await publicPrerequisites(ctx, principals.admin); + await evidence.prerequisites({ + supervisorSha256: sha256(canonicalJson(supervisor)), + publicationSha256: sha256(canonicalJson(publication)), + publicStateSha256: sha256(canonicalJson(ready)), + }); + await completeInfrastructure( + ctx.backend, + await runInfrastructure(ctx, principals, database), + evidence, + new Supervisor(ctx), + ); + await runSelectedCases( + drivers, + selection, + { ctx, principals, database }, + evidence, + undefined, + deadline, + ); + await evidence.finish(); + } finally { + const results = await Promise.allSettled([ + evidence.close(), + ...(database ? [database.close()] : []), + ]); + requireThat( + results.every((result) => result.status === "fulfilled"), + "runner_observer_evidence_close", + ); + } +} + +/** Only the executable invokes this watchdog. The adapter owns service teardown on every exit. */ +export async function main(drivers: DriverRegistry, args: readonly string[]) { + const begun = performance.now(); + const expired = () => { + process.stderr.write('{"status":"FAIL","seam":"runner_total_deadline"}\n'); + process.exit(124); + }; + let watchdog = setTimeout(expired, 7200000); + try { + const selected = parseArgs(args); + validateDrivers(drivers); + const ctx = await loadContext(selected.context); + clearTimeout(watchdog); + const remaining = ctx.deadlines.totalMs - (performance.now() - begun); + requireThat(remaining > 0, "runner_context_deadline"); + watchdog = setTimeout(expired, remaining); + await runMatrix( + ctx, + drivers, + selected.selection, + begun + ctx.deadlines.totalMs, + ); + process.stdout.write( + JSON.stringify({ + status: "PASS", + selected: selected.selected.length, + passed: selected.selected.length, + ...selected.selection, + stageCompleted: !( + selected.selection.stage === "matrix" && + selected.selection.matrix === "all" + ), + matrixCompleted: + selected.selection.stage === "matrix" && + selected.selection.matrix === "all", + physicalAcceptance: + selected.selection.stage === "matrix" && + selected.selection.matrix === "all", + evidenceFile: ctx.evidenceFile, + }) + "\n", + ); + } catch (error) { + const failure = safeFailure(error); + process.stderr.write( + JSON.stringify({ + status: failure.code === "NEEDS_CONTEXT" ? "NEEDS_CONTEXT" : "FAIL", + code: failure.code, + seam: failure.seam, + }) + "\n", + ); + process.exitCode = 1; + } finally { + clearTimeout(watchdog); + } +} diff --git a/orchestrator/scripts/mcp-stack/scenario-cleanup.test.ts b/orchestrator/scripts/mcp-stack/scenario-cleanup.test.ts new file mode 100644 index 000000000..043fad948 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/scenario-cleanup.test.ts @@ -0,0 +1,316 @@ +import { expect, test } from "bun:test"; +import { + Code, + ConnectError, + createClient, + createRouterTransport, +} from "@connectrpc/connect"; +import { closeScenario, closeScenarioSession } from "./scenario-runtime.ts"; +import { + AutomationService, + AutomationRunService, +} from "../../src/gen/engram/app/v1/automation_pb.ts"; +import { SessionService } from "../../src/gen/engram/app/v1/session_pb.ts"; +import { TaskService } from "../../src/gen/engram/app/v1/task_pb.ts"; +import { ProfileService } from "../../src/gen/engram/app/v1/profile_pb.ts"; + +function cleanupFixture( + input: { + automation?: boolean; + runStatus?: string; + statuses?: string[]; + wrongSession?: boolean; + failure?: "stop" | "delete" | "get" | "retain" | "destroy" | "settled"; + destroyStates?: Array< + "queued" | "running" | "done" | "failed" | "cancelled" + >; + settled?: boolean[]; + } = {}, +) { + const events: string[] = []; + const sessionId = "11111111-1111-4111-8111-111111111111"; + const ctx = { + hostId: "22222222-2222-4222-8222-222222222222", + deadlines: { requestMs: 1000, stepMs: 0, totalMs: 10000 }, + }; + const statuses = [...(input.statuses ?? ["completed"])]; + const destroyStates = [...(input.destroyStates ?? ["done"])]; + const settled = [...(input.settled ?? [true])]; + const transport = createRouterTransport((router) => { + router.service(AutomationRunService, { + getRun(request) { + expect(request.runId).toBe("fixture-run"); + events.push("getRun"); + return { run: { brief: { status: input.runStatus ?? "running" } } }; + }, + stopRun(request) { + expect(request.runId).toBe("fixture-run"); + events.push("stopRun"); + if (input.failure === "stop") + throw new ConnectError("stop failed", Code.Unavailable); + return {}; + }, + }); + router.service(SessionService, { + deleteSession(request) { + expect(request.sessionId).toBe(sessionId); + events.push("deleteSession"); + if (input.failure === "delete") + throw new ConnectError("delete failed", Code.Unavailable); + return {}; + }, + getSession(request) { + expect(request.sessionId).toBe(sessionId); + events.push("getSession"); + if (input.failure === "get") + throw new ConnectError("get failed", Code.Unavailable); + return { + session: { + id: input.wrongSession ? "other-session" : sessionId, + status: statuses.length > 1 ? statuses.shift() : statuses[0], + }, + }; + }, + }); + router.service(AutomationService, { + archiveAutomation(request) { + expect(request.id).toBe("fixture-automation"); + events.push("archiveAutomation"); + return {}; + }, + }); + router.service(TaskService, { + deleteTask(request) { + expect(request.taskId).toBe("fixture-task"); + events.push("deleteTask"); + return {}; + }, + }); + router.service(ProfileService, { + deleteProfile(request) { + expect(request.id).toBe("fixture-profile"); + events.push("deleteProfile"); + return {}; + }, + }); + }); + const principals = { + automation: { + automationRuns: createClient(AutomationRunService, transport), + }, + admin: { + sessions: createClient(SessionService, transport), + automations: createClient(AutomationService, transport), + tasks: createClient(TaskService, transport), + profiles: createClient(ProfileService, transport), + }, + }; + const scenario = { + ctx, + database: { + async retainCleanupTarget(id: string, hostId: string) { + expect(id).toBe(sessionId); + expect(hostId).toBe(ctx.hostId); + events.push("retainTarget"); + if (input.failure === "retain") throw new Error("retain failed"); + }, + async destroyOperation(id: string) { + expect(id).toBe(sessionId); + const state = + destroyStates.length > 1 ? destroyStates.shift()! : destroyStates[0]!; + events.push(`destroy:${state}`); + if (input.failure === "destroy") throw new Error("destroy failed"); + return { id: "1", state }; + }, + async cleanupSettled(id: string) { + expect(id).toBe(sessionId); + const value = settled.length > 1 ? settled.shift()! : settled[0]!; + events.push(`settled:${value}`); + if (input.failure === "settled") throw new Error("settled failed"); + return value; + }, + }, + taskId: "fixture-task", + sessionId, + profileId: "fixture-profile", + automation: + input.automation === false + ? undefined + : { + id: "fixture-automation", + runId: "fixture-run", + runtimeActorId: "fixture-runtime", + definitionActorId: "fixture-definition", + triggerActorId: "fixture-trigger", + definitionSha256: "a".repeat(64), + }, + }; + return { + events, + ctx, + close: () => closeScenario(principals, scenario), + closeSession: () => + closeScenarioSession(principals.admin.sessions, scenario), + }; +} + +test("automation cleanup waits for the exact primary session before removing ownership", async () => { + const fixture = cleanupFixture({ statuses: ["active", "completed"] }); + fixture.ctx.deadlines.stepMs = 500; + await fixture.close(); + expect(fixture.events).toEqual([ + "getRun", + "stopRun", + "retainTarget", + "deleteSession", + "getSession", + "getSession", + "destroy:done", + "settled:true", + "archiveAutomation", + "deleteTask", + "deleteProfile", + ]); +}); + +test("terminal automation runs still require primary session cleanup", async () => { + for (const status of ["completed", "failed", "dead"]) { + const fixture = cleanupFixture({ + runStatus: "completed", + statuses: [status], + }); + await fixture.close(); + expect(fixture.events).toEqual([ + "getRun", + "retainTarget", + "deleteSession", + "getSession", + "destroy:done", + "settled:true", + "archiveAutomation", + "deleteTask", + "deleteProfile", + ]); + } +}); + +test("human cleanup deletes and confirms the exact primary session", async () => { + const fixture = cleanupFixture({ automation: false }); + await fixture.close(); + expect(fixture.events).toEqual([ + "retainTarget", + "deleteSession", + "getSession", + "destroy:done", + "settled:true", + "deleteTask", + "deleteProfile", + ]); +}); + +test("cleanup errors retain the task and definition", async () => { + for (const failure of [ + "stop", + "delete", + "get", + "retain", + "destroy", + "settled", + ] as const) { + const fixture = cleanupFixture({ failure }); + await expect(fixture.close()).rejects.toThrow(`${failure} failed`); + expect(fixture.events).not.toContain("archiveAutomation"); + expect(fixture.events).not.toContain("deleteTask"); + expect(fixture.events).not.toContain("deleteProfile"); + if (failure === "stop") + expect(fixture.events).not.toContain("deleteSession"); + } +}); + +test("wrong session identity cannot authorize ownership deletion", async () => { + const fixture = cleanupFixture({ wrongSession: true }); + await expect(fixture.close()).rejects.toThrow(); + expect(fixture.events).toEqual([ + "getRun", + "stopRun", + "retainTarget", + "deleteSession", + "getSession", + ]); +}); + +test("nonterminal session timeout retains ownership", async () => { + const fixture = cleanupFixture({ statuses: ["active"] }); + await expect(fixture.close()).rejects.toThrow(); + expect(fixture.events).toEqual([ + "getRun", + "stopRun", + "retainTarget", + "deleteSession", + "getSession", + ]); +}); + +for (const state of ["queued", "running", "failed", "cancelled"] as const) + test(`terminal session with destroy ${state} retains ownership`, async () => { + const fixture = cleanupFixture({ destroyStates: [state] }); + await expect(fixture.close()).rejects.toThrow(); + expect(fixture.events).toContain(`destroy:${state}`); + expect(fixture.events).not.toContain("archiveAutomation"); + expect(fixture.events).not.toContain("deleteTask"); + expect(fixture.events).not.toContain("deleteProfile"); + }); + +test("completed destroy with pending cleanup obligations retains ownership", async () => { + const fixture = cleanupFixture({ settled: [false] }); + await expect(fixture.close()).rejects.toThrow(); + expect(fixture.events).toContain("settled:false"); + expect(fixture.events).not.toContain("deleteTask"); + expect(fixture.events).not.toContain("deleteProfile"); +}); + +test("child session uses the same retirement barrier before task ownership can be removed", async () => { + const blocked = cleanupFixture({ automation: false, settled: [false] }); + await expect(blocked.closeSession()).rejects.toThrow( + "scenario_cleanup_obligations", + ); + expect(blocked.events).toEqual([ + "retainTarget", + "deleteSession", + "getSession", + "destroy:done", + "settled:false", + ]); + const settled = cleanupFixture({ automation: false }); + await settled.closeSession(); + expect(settled.events).toEqual([ + "retainTarget", + "deleteSession", + "getSession", + "destroy:done", + "settled:true", + ]); +}); + +test("cleanup waits for destroy and obligations before the next case can start", async () => { + const fixture = cleanupFixture({ + destroyStates: ["running", "done"], + settled: [false, true], + }); + fixture.ctx.deadlines.stepMs = 1000; + await fixture.close(); + fixture.events.push("nextCase"); + expect(fixture.events.slice(-8)).toEqual([ + "destroy:done", + "settled:false", + "destroy:done", + "settled:true", + "archiveAutomation", + "deleteTask", + "deleteProfile", + "nextCase", + ]); + expect(fixture.events.indexOf("retainTarget")).toBeLessThan( + fixture.events.indexOf("deleteSession"), + ); +}); diff --git a/orchestrator/scripts/mcp-stack/scenario-runtime.ts b/orchestrator/scripts/mcp-stack/scenario-runtime.ts new file mode 100644 index 000000000..4a8fd9dff --- /dev/null +++ b/orchestrator/scripts/mcp-stack/scenario-runtime.ts @@ -0,0 +1,732 @@ +import { Code } from "@connectrpc/connect"; +import { z } from "zod"; +import { + bindingTool, + bindingTools, + modelToolName, + createCatalog, + createFixtureProfile, + createHumanTask, + expectRejected, + waitFor, + type bootstrapHumans, +} from "./bootstrap.ts"; +import { + requireThat, + sha256, + type Context, + type FixtureKind, +} from "./context.ts"; +import { + canonicalJson, + guestReady, + proofPaths, + readGuestProofRecords, +} from "./guest.ts"; +import type { Harness } from "./manifest.ts"; +import { DatabaseObserver, FixtureObserver } from "./observer.ts"; +import { + createAutomationTask, + type CreatedScenarioTask, +} from "./automation.ts"; +import type { CallPlan } from "./call-plan.ts"; +import type { PublicClient } from "./transport.ts"; + +/** A body error can escape its finally block only after successful cleanup. */ +export class CaseBodyFailure extends Error { + constructor(readonly failure: unknown) { + super("case_body_failed"); + } +} +export class CaseCleanupFailure extends Error { + constructor(readonly failure: unknown) { + super("case_cleanup_failed"); + } +} +export function caseBodyFailure(error: unknown) { + return error instanceof CaseBodyFailure || error instanceof CaseCleanupFailure + ? error + : new CaseBodyFailure(error); +} +export async function completeCaseCleanup(cleanup: () => Promise) { + try { + await cleanup(); + } catch (error) { + throw new CaseCleanupFailure(error); + } +} + +export type Principals = Awaited>; +export type StartedScenario = Awaited>; +export async function startScenario( + ctx: Context, + principals: Principals, + database: DatabaseObserver, + input: { + harness: Harness; + suffix: string; + protocol: FixtureKind; + upstreamName: "read_counter" | "write_counter"; + mode: "read" | "write" | "denied" | "refire" | "held-call" | "forged"; + caller: "owner" | "unattended" | "automation"; + automationWrite: boolean; + arguments: Record; + constraints: object[]; + callId: string; + defaultAllow?: boolean; + grant?: boolean; + authKind?: "api_key" | "oauth"; + callPlan?: ( + alias: string, + connection: { id: string; alias: string }, + ) => CallPlan; + completionBarrier?: boolean; + prematureBarrier?: boolean; + expectedNativeResult?: Record; + }, +) { + const fixture = new FixtureObserver(ctx, input.protocol); + const before = await fixture.observe(); + const { connection, catalog } = await createCatalog( + ctx, + principals.admin, + input.suffix, + input.protocol, + input.authKind, + ); + const profile = await createFixtureProfile(ctx, principals.admin, { + ...input, + connectionId: connection.id, + connectionAlias: connection.alias, + grant: input.grant ?? true, + callPlan: input.callPlan?.( + modelToolName(connection.id, connection.alias, input.upstreamName), + { id: connection.id, alias: connection.alias }, + ), + defaultAllow: input.defaultAllow ?? false, + }); + const client = principals[input.caller]; + const reviewClient = input.caller !== "owner" ? principals.admin : client; + const task: CreatedScenarioTask = + input.caller === "automation" + ? await createAutomationTask( + ctx, + principals, + profile.profileId, + input.suffix, + ) + : await createHumanTask(ctx, client, profile.profileId, input.harness); + await waitFor( + ctx, + "scenario_public_active", + () => client.sessions.getSession({ sessionId: task.sessionId }), + (value) => + value.session?.status === "active" && value.session.hostId === ctx.hostId, + ); + database.own(task.taskId, task.sessionId); + let coordinator: Awaited> | undefined; + if (input.grant === false) { + requireThat(input.caller === "owner", "ordinary_missing_grant_owner"); + const observed = await client.tasks.getTask({ taskId: task.taskId }); + requireThat( + observed.task?.id === task.taskId && + observed.task.createdByUserId === principals.actorIds.owner && + observed.task.sessions.length === 1 && + observed.task.sessions[0]?.sessionId === task.sessionId, + "ordinary_owner_task_session", + ); + await scenarioBindingTools({ + input, + reviewClient, + sessionId: task.sessionId, + }); + } else { + coordinator = await waitFor( + ctx, + "coordinator_started", + () => database.started(task.sessionId, ctx.hostId), + (state) => state.phase === "started", + ); + requireThat( + coordinator.source_key.context.operator_identity === + ctx.publication.candidateIdentity && + coordinator.source_key.context.target.session_id === task.sessionId && + coordinator.source_key.context.target.host_id === ctx.hostId, + "started_target_publication", + ); + if (task.automation) + requireThat( + coordinator.source_key.creation.policy.actorId === + task.automation.runtimeActorId, + "actual_automation_runtime_actor", + ); + const managedCoordinator = coordinator; + await waitFor( + ctx, + "orchestrator_started", + () => database.managedLifecycle(task.sessionId, managedCoordinator), + (state) => state.phase === "started", + ); + const bound = await bindingTool( + reviewClient, + task.sessionId, + connection.id, + input.upstreamName, + ); + const reviewed = catalog.tools.find( + (tool) => tool.upstreamName === input.upstreamName, + ); + requireThat( + bound.modelToolName === profile.alias && + bound.reviewedTool && + reviewed && + canonicalJson(JSON.parse(bound.reviewedTool.inputSchemaJson)) === + canonicalJson(JSON.parse(reviewed.inputSchemaJson)), + "exact_delivered_binding_schema", + ); + } + await guestReady(ctx, client, task.sessionId); + return { + ctx, + database, + ...task, + ...profile, + client, + reviewClient, + callerActorId: principals.actorIds[input.caller], + reviewActorId: + principals.actorIds[input.caller === "owner" ? "owner" : "admin"], + fixture, + connection, + catalog, + before, + get coordinator() { + requireThat(coordinator, "managed_scenario_required"); + return coordinator; + }, + input, + }; +} +/** Only the explicit no-grant fixture may observe ordinary empty authority. */ +export async function scenarioBindingTools( + scenario: Pick, +) { + if (scenario.input.grant !== false) + return bindingTools(scenario.reviewClient, scenario.sessionId); + const state = await scenario.reviewClient.integration.getSessionMcpState({ + sessionId: scenario.sessionId, + omitBindings: false, + omitExecutions: false, + bindingPageSize: 16, + pageSize: 16, + }); + requireThat( + state.bindingReceipt === undefined && + state.writePolicy === undefined && + state.availability === "unavailable" && + state.bindingTools.length === 0 && + state.executions.length === 0 && + state.approvals.length === 0 && + state.unknownWriteBlocks.length === 0 && + state.resolutionReceipts.length === 0 && + state.allowedActions.length === 0 && + state.hasMoreBindingTools === false && + state.hasMoreExecutions === false && + state.nextBindingToolName === "" && + state.nextExecutionId === "", + "ordinary_empty_mcp_authority", + ); + return state.bindingTools; +} + +export async function sendScenarioPrompt( + scenario: StartedScenario, + plan = false, + marker?: string, + promptAttempt = 1, +) { + requireThat( + Number.isInteger(promptAttempt) && + promptAttempt >= 1 && + promptAttempt <= 16, + "prompt_attempt_bound", + ); + requireThat( + marker === undefined || /^[A-Za-z0-9_-]{1,64}$/.test(marker), + "call_plan_marker", + ); + await scenario.client.sessions.sendPrompt({ + sessionId: scenario.sessionId, + text: marker ? `[[fixture:${marker}]]` : "Run the neutral fixture call.", + promptId: `fixture-${scenario.input.suffix}${marker ? "-" + marker : ""}-${promptAttempt}`, + ...(plan ? { harnessMode: "plan" } : {}), + }); +} +export async function execution( + ctx: Context, + scenario: StartedScenario, + terminal = true, + callId = scenario.input.callId, +) { + return waitFor( + ctx, + "public_execution", + async () => { + const state = await scenario.reviewClient.integration.getSessionMcpState({ + sessionId: scenario.sessionId, + omitBindings: true, + pageSize: 16, + }); + requireThat(!state.hasMoreExecutions, "fresh_session_execution_bound"); + const matches = state.executions.filter( + (row) => + row.toolCallId === callId && + row.connectionId === scenario.connection.id, + ); + requireThat(matches.length <= 1, "single_call_identity"); + return matches[0]; + }, + (value) => + !!value && + (!terminal || + ["succeeded", "failed", "canceled", "outcome_unknown"].includes( + value.state, + )), + ); +} +export async function approveOnce( + ctx: Context, + principals: Principals, + scenario: StartedScenario, + actor: "owner" | "admin", +) { + const pending = await waitFor( + ctx, + "interactive_approval", + async () => { + const response = await scenario.reviewClient.integration.listMcpApprovals( + { + sessionId: scenario.sessionId, + }, + ); + requireThat( + !response.nextExecutionId && response.approvals.length <= 1, + "single_approval", + ); + return response.approvals[0]; + }, + (value) => !!value, + ); + requireThat( + pending?.expectedReviewContext && + pending.toolCallId === scenario.input.callId && + pending.sessionId === scenario.sessionId && + pending.connectionId === scenario.connection.id, + "exact_approval_context", + ); + const reviewContext = pending.expectedReviewContext; + const request = { + executionId: pending.executionId, + approve: true, + expectedReviewContext: reviewContext, + }; + const otherDenied = await expectRejected( + () => principals.other.integration.decideMcpApproval(request), + [Code.PermissionDenied, Code.NotFound], + ); + const changedDenied = await expectRejected( + () => + principals[actor].integration.decideMcpApproval({ + ...request, + expectedReviewContext: { + ...reviewContext, + argumentHash: `sha256:${"0".repeat(64)}`, + }, + }), + [Code.Aborted, Code.FailedPrecondition], + ); + const approved = + await principals[actor].integration.decideMcpApproval(request); + requireThat( + approved.receipt?.executionId === pending.executionId && + approved.receipt.decision === "approved", + "owner_admin_approval_receipt", + ); + const reversedDenied = await expectRejected( + () => + principals[actor].integration.decideMcpApproval({ + ...request, + approve: false, + }), + [Code.Aborted, Code.FailedPrecondition], + ); + const replayDenied = await expectRejected( + () => principals[actor].integration.decideMcpApproval(request), + [Code.Aborted, Code.FailedPrecondition], + ); + return { + executionId: pending.executionId, + contextHash: sha256(canonicalJson(pending.expectedReviewContext)), + rejections: { otherDenied, changedDenied, reversedDenied, replayDenied }, + }; +} +export async function verifiedCounterResult( + scenario: StartedScenario, + executionId: string, + count: number, + isError: boolean, +) { + const response = + await scenario.reviewClient.integration.getMcpExecutionResult({ + sessionId: scenario.sessionId, + executionId, + }); + requireThat( + response.resultJson && !response.resultUnavailable, + "persisted_verified_result_available", + ); + const parsed = z + .strictObject({ + outcome: z.enum(["succeeded", "failed"]), + delivery: z.literal("available"), + output: z.strictObject({ + content: z.array( + z.strictObject({ type: z.literal("text"), text: z.string() }), + ), + structuredContent: z.strictObject({ count: z.number().int() }), + isError: z.boolean(), + }), + }) + .safeParse(JSON.parse(response.resultJson)); + requireThat( + parsed.success && + parsed.data.outcome === (isError ? "failed" : "succeeded") && + response.state === parsed.data.outcome && + canonicalJson(parsed.data.output) === + canonicalJson({ + content: [{ type: "text", text: `count=${count}` }], + structuredContent: { count }, + isError, + }), + "exact_counter_journal_result", + ); + return { + state: response.state, + resultHash: sha256(canonicalJson(parsed.data)), + output: parsed.data.output, + settlement: response.settlement, + }; +} +export async function toolCompletionEvents( + scenario: { + input: Pick; + alias: string; + sessionId: string; + reviewClient: { + sessions: Pick; + }; + }, + callId = scenario.input.callId, +) { + const nativeName = + scenario.input.harness === "claude" + ? `mcp__engrams__${scenario.alias}` + : scenario.alias; + const page = await scenario.reviewClient.sessions.listSessionEvents({ + sessionId: scenario.sessionId, + kinds: ["tool_call_completed"], + toolNames: [nativeName], + limit: 129n, + }); + requireThat(page.events.length < 129, "completion_event_bound"); + return page.events.flatMap((event) => { + requireThat( + event.kind === "tool_call_completed" && event.idx !== undefined, + "completion_event_identity", + ); + const parsed = z + .object({ + tool_call_id: z.string(), + tool_name: z.string(), + run_id: z.string(), + ok: z.boolean(), + }) + .safeParse(JSON.parse(event.payloadJson)); + requireThat( + parsed.success && parsed.data.tool_name === nativeName, + "completion_event_shape", + ); + return parsed.data.tool_call_id === callId + ? [ + { + ...parsed.data, + idx: String(event.idx), + sha256: sha256(event.payloadJson), + }, + ] + : []; + }); +} + +export async function delivered( + ctx: Context, + database: DatabaseObserver, + scenario: StartedScenario, +) { + const paths = await waitFor( + ctx, + "guest_proof_created", + () => + proofPaths( + ctx, + scenario.client, + scenario.sessionId, + scenario.input.suffix, + ), + (paths) => paths.length > 0, + ); + requireThat(paths.length <= 4, "bounded_cli_launches"); + const observed = await waitFor( + ctx, + "guest_result_received", + async () => { + const currentPaths = await proofPaths( + ctx, + scenario.client, + scenario.sessionId, + scenario.input.suffix, + ); + requireThat(currentPaths.length <= 4, "bounded_cli_launches"); + const proofs = await Promise.all( + currentPaths.map((path) => + readGuestProofRecords(ctx, scenario.client, scenario.sessionId, path), + ), + ); + return proofs + .flat() + .filter( + (value) => + value.proof.resultReceived && + value.proof.callId === scenario.input.callId && + value.proof.tool === scenario.alias, + ); + }, + (values) => values.length >= 1, + ); + const outbox = await waitFor( + ctx, + "actual_guest_ack", + () => database.outbox(scenario.sessionId, scenario.input.callId), + (value) => value.acknowledged, + ); + requireThat( + outbox.relayed && outbox.attempts >= 1, + "outbox_relay_before_ack", + ); + const completion = await waitFor( + ctx, + "actual_native_completion_event", + () => toolCompletionEvents(scenario), + (events) => events.length > 0, + ); + const target = await database.deliveryTarget( + scenario.sessionId, + scenario.input.callId, + ); + requireThat(!target.targetGone, "guest_ack_is_not_target_gone_settlement"); + return { + proofHashes: observed.map((value) => value.sha256), + outbox, + completion, + target, + }; +} + +/** A native receipt is separate from outbox ACK and upstream outcome. */ +type ProofSelection = { + callId: string; + tool: string; + planSha256?: string; + afterCount?: number; + expectExposure?: "absent"; +}; +type ProofObservation = NonNullable< + Awaited< + ReturnType + >[number]["proof"]["observation"] +>; +export async function pollProof( + ctx: Context, + scenario: StartedScenario, + input: ProofSelection, +) { + return pollSelectedProof(ctx, scenario, input); +} +export async function pollGuestObservation( + ctx: Context, + scenario: StartedScenario, + input: ProofSelection & { + observation: ProofObservation; + }, +) { + return pollSelectedProof(ctx, scenario, input); +} +async function pollSelectedProof( + ctx: Context, + scenario: StartedScenario, + input: ProofSelection & { observation?: ProofObservation }, +) { + requireThat( + /^[A-Za-z0-9_-]{1,64}$/.test(input.callId) && + /^[A-Za-z0-9_]{1,128}$/.test(input.tool), + "proof_selection_identity", + ); + const afterCount = input.afterCount ?? 0; + requireThat( + Number.isInteger(afterCount) && afterCount >= 0 && afterCount <= 128, + "proof_selection_count", + ); + return waitFor( + ctx, + `native_completed_proof_${input.callId}`, + async () => { + const paths = await proofPaths( + ctx, + scenario.client, + scenario.sessionId, + scenario.input.suffix, + ); + requireThat(paths.length <= 16, "proof_launch_bound"); + const records = ( + await Promise.all( + paths.map((path) => + readGuestProofRecords( + ctx, + scenario.client, + scenario.sessionId, + path, + ), + ), + ) + ).flat(); + return selectProofRecords(records, input); + }, + (records) => records.length > afterCount, + ); +} +export function selectProofRecords( + records: Awaited>, + input: ProofSelection & { observation?: ProofObservation }, +) { + return records.filter( + (record) => + record.proof.callId === input.callId && + record.proof.tool === input.tool && + (input.planSha256 === undefined || + record.proof.planSha256 === input.planSha256) && + (input.expectExposure === undefined || + record.proof.expectExposure === input.expectExposure) && + (input.observation !== undefined + ? record.proof.observation === input.observation + : record.proof.observation === undefined && + (record.proof.resultReceived === true || + record.proof.forgedRejected === true || + (record.proof.expectExposure === "absent" && + record.proof.requestAttempted === true && + record.proof.protocolDenialCode === "unknown_tool"))), + ); +} + +type ScenarioCleanup = Pick & { + ctx: Pick; + database: Pick< + DatabaseObserver, + "retainCleanupTarget" | "destroyOperation" | "cleanupSettled" + >; +}; + +export async function closeScenarioSession( + sessions: Pick, + scenario: ScenarioCleanup, +) { + await scenario.database.retainCleanupTarget( + scenario.sessionId, + scenario.ctx.hostId, + ); + await sessions.deleteSession({ sessionId: scenario.sessionId }); + await waitFor( + scenario.ctx, + "scenario_cleanup_session_terminal", + () => sessions.getSession({ sessionId: scenario.sessionId }), + (value) => { + requireThat( + value.session?.id === scenario.sessionId, + "scenario_cleanup_session_identity", + ); + return ["completed", "failed", "dead"].includes(value.session.status); + }, + ); + await waitFor( + scenario.ctx, + "scenario_cleanup_obligations", + async () => { + const destroy = await scenario.database.destroyOperation( + scenario.sessionId, + ); + requireThat( + destroy?.state !== "failed" && destroy?.state !== "cancelled", + "scenario_cleanup_destroy_failed", + ); + return ( + destroy?.state === "done" && + (await scenario.database.cleanupSettled(scenario.sessionId)) + ); + }, + (settled) => settled, + ); +} + +export async function closeScenario( + principals: { + automation: Pick; + admin: Pick< + PublicClient, + "sessions" | "automations" | "tasks" | "profiles" + >; + }, + scenario: Pick< + StartedScenario, + "automation" | "sessionId" | "taskId" | "profileId" + > & + ScenarioCleanup, +) { + if (scenario.automation) { + const run = await principals.automation.automationRuns.getRun({ + runId: scenario.automation.runId, + }); + if ( + run.run?.brief && + ![ + "completed", + "failed", + "filtered", + "superseded", + "halted", + "deadline", + ].includes(run.run.brief.status) + ) + await principals.automation.automationRuns.stopRun({ + runId: scenario.automation.runId, + reason: "Fixture cleanup", + }); + } + await closeScenarioSession(principals.admin.sessions, scenario); + if (scenario.automation) { + await principals.admin.automations.archiveAutomation({ + id: scenario.automation.id, + }); + } + await principals.admin.tasks.deleteTask({ taskId: scenario.taskId }); + await principals.admin.profiles.deleteProfile({ id: scenario.profileId }); + // Archived profiles retain grants. Keep the connection for run-owned teardown. +} diff --git a/orchestrator/scripts/mcp-stack/supervisor.ts b/orchestrator/scripts/mcp-stack/supervisor.ts new file mode 100644 index 000000000..e0b03b241 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/supervisor.ts @@ -0,0 +1,234 @@ +import { request } from "node:http"; +import { z } from "zod"; +import { + privateFile, + requireThat, + RunnerFailure, + type Context, + type ConfigurationId, +} from "./context.ts"; + +export const ROLES = [ + "orchestrator", + "worker", + "coordinator", + "host-agent", + "fixture-modern", + "fixture-legacy", +] as const; +export type Role = (typeof ROLES)[number]; +export type MutableRole = Exclude; +const hash = z.string().regex(/^[a-f0-9]{64}$/); +const roleState = z.strictObject({ + role: z.enum(ROLES), + state: z.enum(["running", "stopped"]), + bootId: z.string().regex(/^[A-Za-z0-9_-]{1,96}$/), + artifactSha256: hash, + configurationId: z.enum(["normal", "global-disabled", "zone-disabled"]), + configurationSha256: hash, +}); +export const StatusSchema = z.strictObject({ + version: z.literal(1), + runId: z.string(), + backend: z.enum(["firecracker", "vz"]), + sourceCommit: z.string(), + hostId: z.string(), + roles: z.array(roleState).length(6), + network: z.strictObject({ + generation: z.string().regex(/^[A-Za-z0-9_-]{1,96}$/), + enforcementSha256: hash, + fixtureOnlyEgress: z.literal(true), + externalDnsBlocked: z.literal(true), + roles: z.array(z.enum(["orchestrator", "coordinator", "worker"])).length(3), + }), +}); +export type SupervisorStatus = z.infer; + +/** The adapter implements these fixed operations. No commands, paths or arbitrary URLs cross this socket. */ +export class Supervisor { + constructor(private readonly ctx: Context) {} + private async call( + operation: + | "status" + | "restart" + | "stop" + | "infrastructure-ready" + | "prepare-disk-read" + | "verify-disk-read", + role?: MutableRole, + configurationId?: ConfigurationId, + ): Promise { + if ((operation === "restart" || operation === "stop") && !role) + throw new RunnerFailure("INVALID_CONTEXT", "restart_domain"); + const token = ( + await privateFile(this.ctx.runRoot, this.ctx.supervisor.tokenFile) + ) + .toString("utf8") + .trim(); + requireThat(/^[A-Za-z0-9_-]{32,128}$/.test(token), "supervisor_token"); + const body = JSON.stringify({ + version: 1, + runId: this.ctx.runId, + operation, + ...(role ? { role } : {}), + ...(configurationId ? { configurationId } : {}), + }); + const value = await new Promise((resolve, reject) => { + const fail = () => + reject(new RunnerFailure("TRANSPORT_FAILED", "supervisor")); + const req = request( + { + socketPath: this.ctx.supervisor.socket, + path: "/v1/control", + method: "POST", + agent: false, + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + }, + }, + (res) => { + const chunks: Buffer[] = []; + let bytes = 0; + res.on("data", (chunk: Buffer) => { + bytes += chunk.length; + if (bytes > 65536) req.destroy(); + else chunks.push(chunk); + }); + res.on("error", fail); + res.on("end", () => { + clearTimeout(timer); + try { + if (res.statusCode !== 200) throw new Error(); + resolve(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + } catch { + fail(); + } + }); + }, + ); + const timer = setTimeout( + () => req.destroy(), + this.ctx.deadlines.requestMs, + ); + req.on("error", () => { + clearTimeout(timer); + fail(); + }); + req.end(body); + }); + const parsed = StatusSchema.safeParse(value); + requireThat(parsed.success, "supervisor_response"); + const state = parsed.data; + requireThat( + state.runId === this.ctx.runId && + state.backend === this.ctx.backend && + state.sourceCommit === this.ctx.sourceCommit && + state.hostId === this.ctx.hostId, + "supervisor_identity", + ); + requireThat( + new Set(state.roles.map((v) => v.role)).size === 6 && + new Set(state.network.roles).size === 3, + "supervisor_role_set", + ); + const artifacts = { + orchestrator: this.ctx.artifacts.orchestrator, + worker: this.ctx.artifacts.worker, + coordinator: this.ctx.artifacts.coordinator, + "host-agent": this.ctx.artifacts.hostAgent, + "fixture-modern": this.ctx.artifacts.fixture, + "fixture-legacy": this.ctx.artifacts.fixture, + }; + for (const entry of state.roles) + requireThat( + entry.artifactSha256 === artifacts[entry.role].sha256, + "supervisor_artifact", + ); + for (const entry of state.roles) { + if (entry.role === "fixture-modern" || entry.role === "fixture-legacy") { + requireThat( + entry.configurationId === "normal" && + entry.configurationSha256 === + this.ctx.protocolFixtures[ + entry.role === "fixture-modern" ? "modern" : "legacy" + ].configuration.sha256, + "fixture_configuration", + ); + } else { + requireThat( + entry.role === "orchestrator" || entry.configurationId === "normal", + "only_orchestrator_has_disable_variants", + ); + const configuration = + entry.role === "orchestrator" + ? this.ctx.supervisor.configurations.orchestrator[ + entry.configurationId + ] + : this.ctx.supervisor.configurations[entry.role].normal; + requireThat( + configuration && configuration.sha256 === entry.configurationSha256, + "supervisor_configuration", + ); + } + } + return state; + } + async prepareDiskRead() { + await this.call("prepare-disk-read"); + } + async verifyDiskRead() { + await this.call("verify-disk-read"); + } + async infrastructureReady() { + return this.call("infrastructure-ready"); + } + async status() { + return this.call("status"); + } + async restart( + role: MutableRole, + configurationId: ConfigurationId = "normal", + ) { + requireThat( + role === "orchestrator" + ? this.ctx.supervisor.configurations.orchestrator[configurationId] + : configurationId === "normal", + "declared_configuration", + ); + const before = await this.status(); + requireThat( + before.roles.every((r) => r.state === "running"), + "roles_running_before_restart", + ); + const after = await this.call("restart", role, configurationId); + requireThat( + after.roles.find((r) => r.role === role)?.configurationId === + configurationId, + "selected_configuration", + ); + requireThat( + after.roles.every((r) => r.state === "running"), + "roles_running_after_restart", + ); + for (const current of after.roles) { + const prior = before.roles.find((r) => r.role === current.role)!; + requireThat( + current.role === role + ? current.bootId !== prior.bootId + : current.bootId === prior.bootId, + "restart_scope", + ); + } + requireThat( + after.network.generation === before.network.generation && + after.network.enforcementSha256 === before.network.enforcementSha256, + "restart_network_preserved", + ); + return after; + } + async stop(role: MutableRole) { + return this.call("stop", role); + } +} diff --git a/orchestrator/scripts/mcp-stack/transport.ts b/orchestrator/scripts/mcp-stack/transport.ts new file mode 100644 index 000000000..a3fadc517 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/transport.ts @@ -0,0 +1,456 @@ +import { request as httpsRequest } from "node:https"; +import { checkServerIdentity, type ConnectionOptions } from "node:tls"; +import { + ConnectError, + createClient, + type Interceptor, +} from "@connectrpc/connect"; +import { + createConnectTransport, + createGrpcTransport, +} from "@connectrpc/connect-node"; +import { + AutomationService, + AutomationRunService, +} from "../../src/gen/engram/app/v1/automation_pb.ts"; +import { ApiKeyService } from "../../src/gen/engram/app/v1/api_key_pb.ts"; +import { IntegrationService } from "../../src/gen/engram/app/v1/integration_pb.ts"; +import { ProfileService } from "../../src/gen/engram/app/v1/profile_pb.ts"; +import { TaskService } from "../../src/gen/engram/app/v1/task_pb.ts"; +import { SessionService } from "../../src/gen/engram/app/v1/session_pb.ts"; +import { OrgSecretService } from "../../src/gen/engram/app/v1/org_secret_pb.ts"; +import { ModelRouterService } from "../../src/gen/engram/app/v1/model_router_pb.ts"; +import { FleetService } from "../../src/gen/engram/app/v1/fleet_pb.ts"; +import { ImageService } from "../../src/gen/engram/app/v1/image_pb.ts"; +import { OperatorPublicationService } from "../../src/gen/engram/mcp/v1/operator_pb.ts"; +import { + privateFile, + requireThat, + sha256, + RunnerFailure, + type Context, + type Endpoint, +} from "./context.ts"; + +const publicRpcMethods = new WeakMap(); +export function publicRpcMethod(error: ConnectError): string | undefined { + return publicRpcMethods.get(error); +} + +export type Credential = + | { kind: "cookie"; value: string } + | { kind: "api-key"; value: string }; +export function credentialHeaders( + credential: Credential, +): Record { + if (!credential.value || /[\r\n]/.test(credential.value)) + throw new RunnerFailure("INVALID_CONTEXT", "credential_shape"); + if ( + credential.kind === "api-key" && + !/^engk_[A-Za-z0-9_-]+$/.test(credential.value) + ) + throw new RunnerFailure("INVALID_CONTEXT", "api_key_shape"); + return credential.kind === "cookie" + ? { cookie: credential.value } + : { "x-api-key": credential.value }; +} +export async function tlsOptions( + ctx: Context, + endpoint: Endpoint, +): Promise { + return { + ca: await privateFile(ctx.runRoot, endpoint.ca.path), + rejectUnauthorized: true, + checkServerIdentity(host, cert) { + const error = checkServerIdentity(host, cert); + return ( + error ?? + (cert.fingerprint256 === endpoint.serverFingerprint + ? undefined + : new Error("fixture_certificate_pin")) + ); + }, + }; +} +export async function publicClient(ctx: Context, credential: Credential) { + const headers = credentialHeaders(credential); + const authenticate: Interceptor = (next) => async (req) => { + for (const [key, value] of Object.entries(headers)) + req.header.set(key, value); + req.header.set("origin", ctx.endpoints.app.origin); + const method = `${req.service.name}.${req.method.name}`; + const remember = (error: unknown) => { + if ( + error instanceof ConnectError && + /^[A-Za-z0-9_]{1,80}\.[A-Za-z0-9_]{1,80}$/.test(method) + ) + publicRpcMethods.set(error, method); + }; + try { + const response = await next(req); + if (!response.stream) return response; + const message = response.message; + return { + ...response, + message: (async function* () { + try { + yield* message; + } catch (error) { + remember(error); + throw error; + } + })(), + }; + } catch (error) { + remember(error); + throw error; + } + }; + const transport = createConnectTransport({ + baseUrl: ctx.endpoints.app.origin + "/rpc", + httpVersion: "1.1", + nodeOptions: { + ...(await tlsOptions(ctx, ctx.endpoints.app)), + agent: false, + }, + interceptors: [authenticate], + defaultTimeoutMs: ctx.deadlines.requestMs, + readMaxBytes: 4 * 1024 * 1024, + writeMaxBytes: 1024 * 1024, + acceptCompression: [], + useBinaryFormat: true, + }); + return { + uploadGuestFile: (sessionId: string, path: string, bytes: Uint8Array) => + uploadGuestFile(ctx, headers, sessionId, path, bytes), + nonOwnerGuestFileStatus: (sessionId: string, path: string) => + fileDenialStatus(ctx, headers, sessionId, path, 404), + downloadGuestProofFile: (sessionId: string, path: string) => + downloadGuestProofFile(ctx, headers, sessionId, path), + identity: createClient(ApiKeyService, transport), + automations: createClient(AutomationService, transport), + automationRuns: createClient(AutomationRunService, transport), + integration: createClient(IntegrationService, transport), + profiles: createClient(ProfileService, transport), + tasks: createClient(TaskService, transport), + sessions: createClient(SessionService, transport), + secrets: createClient(OrgSecretService, transport), + models: createClient(ModelRouterService, transport), + fleet: createClient(FleetService, transport), + images: createClient(ImageService, transport), + }; +} +export type PublicClient = Awaited>; +export async function operatorClient(ctx: Context) { + const ep = ctx.endpoints.operator; + return createClient( + OperatorPublicationService, + createGrpcTransport({ + baseUrl: ep.origin, + nodeOptions: { + ...(await tlsOptions(ctx, ep)), + cert: await privateFile(ctx.runRoot, ep.certificate.path), + key: await privateFile(ctx.runRoot, ep.privateKeyFile), + }, + defaultTimeoutMs: ctx.deadlines.requestMs, + readMaxBytes: 1024 * 1024, + writeMaxBytes: 65536, + acceptCompression: [], + idleConnectionTimeoutMs: 1000, + }), + ); +} + +/** Read the owner-gated file route with the same credentials and TLS pin as RPC. */ +async function downloadGuestProofFile( + ctx: Context, + headers: Record, + sessionId: string, + path: string, +): Promise { + const tls = await tlsOptions(ctx, ctx.endpoints.app); + const route = `/api/v1/sessions/${encodeURIComponent(sessionId)}/files?path=${encodeURIComponent(path)}`; + return await new Promise((resolve, reject) => { + const fail = (detail = "guest_file_download") => { + clearTimeout(timer); + reject(new RunnerFailure("TRANSPORT_FAILED", detail)); + req.destroy(); + }; + const req = httpsRequest( + new URL(route, ctx.endpoints.app.origin), + { + ...tls, + method: "GET", + agent: false, + headers: { ...headers, origin: ctx.endpoints.app.origin }, + }, + (res) => { + if (res.statusCode !== 200) { + return fail( + fileStatusSeam(res.statusCode, res.headers["x-engram-rpc-code"]), + ); + } + const size = res.headers["content-length"]; + const hash = res.headers["x-content-sha256"]; + if ( + typeof size !== "string" || + !/^[0-9]+$/.test(size) || + BigInt(size) > 1024n * 1024n || + typeof hash !== "string" || + !/^[a-f0-9]{64}$/.test(hash) + ) + return fail("guest_file_metadata"); + const chunks: Buffer[] = []; + let bytes = 0; + res.on("data", (chunk: Buffer) => { + bytes += chunk.length; + if (bytes > 1024 * 1024) fail("guest_proof_size"); + else chunks.push(chunk); + }); + res.on("error", () => fail()); + res.on("aborted", () => fail()); + res.on("end", () => { + clearTimeout(timer); + const value = Buffer.concat(chunks); + try { + requireThat( + BigInt(value.length) === BigInt(size) && sha256(value) === hash, + "guest_file_integrity", + ); + resolve(value); + } catch (error) { + reject(error); + } + }); + }, + ); + const timer = setTimeout(() => fail(), ctx.deadlines.requestMs); + req.on("error", () => fail()); + req.end(); + }); +} + +function fileStatusSeam( + status: number | undefined, + rpcCode: string | string[] | undefined, +) { + if (!status || !Number.isInteger(status) || status < 100 || status > 599) + return "guest_file_status"; + return ( + `guest_file_status_http_${status}` + + (typeof rpcCode === "string" && /^(?:[1-9]|1[0-6])$/.test(rpcCode) + ? `_rpc_code_${Number(rpcCode)}` + : "") + ); +} + +/** Upload fixed bounded bytes and require the server's completed-transfer receipt. */ +async function uploadGuestFile( + ctx: Context, + headers: Record, + sessionId: string, + path: string, + bytes: Uint8Array, +): Promise<{ sizeBytes: number; sha256: string }> { + requireThat(bytes.length <= 1024 * 1024, "guest_file_upload_size"); + const payload = Buffer.from(bytes); + const hash = sha256(payload); + const tls = await tlsOptions(ctx, ctx.endpoints.app); + const route = `/api/v1/sessions/${encodeURIComponent(sessionId)}/files?path=${encodeURIComponent(path)}`; + return await new Promise((resolve, reject) => { + let settled = false; + const fail = (detail = "guest_file_upload") => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(new RunnerFailure("TRANSPORT_FAILED", detail)); + req.destroy(); + }; + const req = httpsRequest( + new URL(route, ctx.endpoints.app.origin), + { + ...tls, + method: "POST", + agent: false, + headers: { + ...headers, + origin: ctx.endpoints.app.origin, + "content-type": "application/octet-stream", + "content-length": String(payload.length), + "x-upload-size": String(payload.length), + "x-upload-sha256": hash, + }, + }, + (res) => { + if (res.statusCode !== 200) + return fail( + fileStatusSeam(res.statusCode, res.headers["x-engram-rpc-code"]), + ); + const chunks: Buffer[] = []; + let size = 0; + res.on("data", (chunk: Buffer) => { + size += chunk.length; + if (size > 65536) fail("guest_file_upload_receipt"); + else chunks.push(chunk); + }); + res.on("error", () => fail()); + res.on("aborted", () => fail()); + res.on("end", () => { + if (settled) return; + let value: unknown; + try { + value = JSON.parse(Buffer.concat(chunks).toString("utf8")); + } catch { + return fail("guest_file_upload_receipt"); + } + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + Object.keys(value).sort().join(",") !== "path,sha256,size_bytes" || + !("path" in value) || + value.path !== path || + !("size_bytes" in value) || + !Number.isSafeInteger(value.size_bytes) || + value.size_bytes !== payload.length || + !("sha256" in value) || + value.sha256 !== hash + ) + return fail("guest_file_upload_receipt"); + settled = true; + clearTimeout(timer); + resolve({ sizeBytes: payload.length, sha256: hash }); + }); + }, + ); + const timer = setTimeout(() => fail(), ctx.deadlines.requestMs); + req.on("error", () => fail()); + req.end(payload); + }); +} + +/** The anonymous probe has no credential variant and only accepts a 401 response. */ +export function anonymousGuestFileStatus( + ctx: Context, + sessionId: string, + path: string, +) { + return fileDenialStatus(ctx, {}, sessionId, path, 401); +} + +/** Check guard status without reading or retaining any response body. */ +async function fileDenialStatus( + ctx: Context, + headers: Record, + sessionId: string, + path: string, + expected: S, +): Promise { + const tls = await tlsOptions(ctx, ctx.endpoints.app); + const route = `/api/v1/sessions/${encodeURIComponent(sessionId)}/files?path=${encodeURIComponent(path)}`; + return await new Promise((resolve, reject) => { + let settled = false; + const fail = (detail = "guest_file_denial") => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(new RunnerFailure("TRANSPORT_FAILED", detail)); + req.destroy(); + }; + const req = httpsRequest( + new URL(route, ctx.endpoints.app.origin), + { + ...tls, + method: "GET", + agent: false, + headers: { ...headers, origin: ctx.endpoints.app.origin }, + }, + (res) => { + if (res.statusCode !== expected) + return fail( + fileStatusSeam(res.statusCode, res.headers["x-engram-rpc-code"]), + ); + settled = true; + clearTimeout(timer); + res.destroy(); + resolve(expected); + }, + ); + const timer = setTimeout(() => fail(), ctx.deadlines.requestMs); + req.on("error", () => fail()); + req.end(); + }); +} + +/** One bounded HTTPS request. Never follows a redirect or logs response bodies. */ +export async function jsonRequest( + ctx: Context, + ep: Endpoint, + method: "GET" | "POST", + route: string, + body?: unknown, + headers: Record = {}, +) { + if (!route.startsWith("/") || route.startsWith("//") || /[\r\n#]/.test(route)) + throw new RunnerFailure("INVALID_CONTEXT", "request_route"); + const payload = + body === undefined ? undefined : Buffer.from(JSON.stringify(body)); + if (payload && payload.length > 65536) + throw new RunnerFailure("INVALID_CONTEXT", "request_size"); + const tls = await tlsOptions(ctx, ep); + return await new Promise<{ + status: number; + headers: import("node:http").IncomingHttpHeaders; + body: unknown; + }>((resolve, reject) => { + const fail = () => + reject(new RunnerFailure("TRANSPORT_FAILED", "https_request")); + const req = httpsRequest( + new URL(route, ep.origin), + { + ...tls, + method, + agent: false, + headers: { + ...headers, + ...(payload + ? { + "content-type": "application/json", + "content-length": String(payload.length), + } + : {}), + }, + }, + (res) => { + const chunks: Buffer[] = []; + let bytes = 0; + res.on("data", (chunk: Buffer) => { + bytes += chunk.length; + if (bytes > 1024 * 1024) req.destroy(); + else chunks.push(chunk); + }); + res.on("error", fail); + res.on("end", () => { + clearTimeout(timer); + try { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: bytes + ? JSON.parse(Buffer.concat(chunks).toString("utf8")) + : null, + }); + } catch { + fail(); + } + }); + }, + ); + const timer = setTimeout(() => req.destroy(), ctx.deadlines.requestMs); + req.on("error", () => { + clearTimeout(timer); + fail(); + }); + req.end(payload); + }); +} diff --git a/orchestrator/scripts/mcp-stack/tsconfig.json b/orchestrator/scripts/mcp-stack/tsconfig.json new file mode 100644 index 000000000..5b15acaa8 --- /dev/null +++ b/orchestrator/scripts/mcp-stack/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { "noEmit": true }, + "include": ["./*.ts"] +} diff --git a/orchestrator/src/__tests__/connection-credential-broker.test.ts b/orchestrator/src/__tests__/connection-credential-broker.test.ts index 249ff08e6..8c9c51117 100644 --- a/orchestrator/src/__tests__/connection-credential-broker.test.ts +++ b/orchestrator/src/__tests__/connection-credential-broker.test.ts @@ -229,3 +229,46 @@ describe("named-connection credential broker", () => { expect(exchanges).toBe(0); }); }); + +test("mixed session audit hash includes the MCP binding envelope", async () => { + const { bindingAuthority } = await import("./mcp-binding-fixture.ts"); + const { compileSessionBindings } = await import("../mcp/session-bindings.ts"); + const { bindingSnapshotColumns } = await import("../mcp/binding-store.ts"); + const authority = bindingAuthority(); + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: authority.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + }); + const source = authority.connections[0]!; + const columns = bindingSnapshotColumns(snapshot, [ + { + id: source.id, + alias: source.alias, + displayName: "Docs", + provider: "mcp", + config: source.config, + }, + connection, + ]); + const session = { + userId: "user-1", + principalId: "user-1", + profileId: authority.profileId, + integrationGrants: sessionGrants, + ...columns, + }; + let auditHash = ""; + const app = makeConnectionCredentialBrokerRoute({ + db: {} as never, + bearer: "broker-secret", + sessions: { get: async () => session }, + sessionStatus: async () => "active", + exchange: async (_config, identity) => { + auditHash = identity.profileSnapshotId; + return { accessToken: "token", expiresAt: new Date(Date.now() + 60000) }; + }, + }); + expect((await app.request(path, post(body))).status).toBe(200); + expect(auditHash).toBe(integrationSnapshotHash(session)); +}); diff --git a/orchestrator/src/__tests__/helpers/mcp-dbos-child.ts b/orchestrator/src/__tests__/helpers/mcp-dbos-child.ts new file mode 100644 index 000000000..05add1b4c --- /dev/null +++ b/orchestrator/src/__tests__/helpers/mcp-dbos-child.ts @@ -0,0 +1,275 @@ +import { assertSweepPoliciesExhaustive } from "../../sweep/policy.ts"; +import { DBOS } from "@dbos-inc/dbos-sdk"; +import { createInterface } from "node:readline"; +import { Client } from "pg"; +import { fromJson } from "@bufbuild/protobuf"; +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { and, eq } from "drizzle-orm"; +import { getDb } from "../../db/client.ts"; +import { mcpExecution } from "../../db/schema.ts"; +import { CredentialControlService } from "../../gen/engram/mcp/v1/control_pb.ts"; +import { SessionPolicyService } from "../../gen/engram/mcp/v1/session_policy_pb.ts"; +import { McpWritePolicyRefSchema } from "../../gen/engram/app/v1/session_pb.ts"; +import { makeStoredSessionBindingResolver } from "../../mcp/binding-store.ts"; +import { unwrapCredentialControlClient } from "../../mcp/control-client.ts"; +import { makeMcpEffectDriver } from "../../mcp/effect-driver.ts"; +import { makeMcpJournal } from "../../mcp/journal.ts"; +import { makeMcpRecovery, installMcpRecovery } from "../../mcp/recovery.ts"; +import { makeMcpRecoveryScanner } from "../../mcp/recovery-scanner.ts"; +import { startMcpRequestWorkflow } from "../../workflows/mcp-exec.ts"; +import { makeToolConsumer } from "../../listeners/tool-consumer.ts"; +import { createToolRegistry } from "../../tools/registry.ts"; +import { makePendingToolCallStore } from "../../tools/pending-tool-calls.ts"; +import { verifyMcpTestDatabases } from "./mcp-test-isolation.ts"; +import type { effectFixture } from "../mcp-effect-fixture.ts"; + +type Fixture = Awaited>; +interface Input { + remote: Fixture["remote"]; + request: Fixture["request"]; + deploymentId: string; + checkpoint?: string; + workflowId: string; + resume?: boolean; + clockOffset?: number; + eventIdx?: string; + requestReadFailures?: number; +} +const lines = createInterface({ input: process.stdin }); +const input: Input = JSON.parse((await lines[Symbol.asyncIterator]().next()).value!); +const { system } = await verifyMcpTestDatabases(); +const now = () => new Date(Date.now() + (input.clockOffset ?? 0)); +let releaseCheckpoint: (() => void) | undefined; +let postLeaseFences = 0; +let responseRead = false; +async function checkpoint(name: string) { + if (name !== input.checkpoint) return; + process.stdout.write(`CHECKPOINT ${name}\n`); + await new Promise((resolve) => { + releaseCheckpoint = resolve; + }); // Parent kills or releases the observed process. +} +// Test-only driver instrumentation observes real SQL before/after execution and commit. +// It never changes SQL, transaction results, authority, or the production effect path. +const transitions = new WeakMap(); +Client.prototype.query = new Proxy(Client.prototype.query, { + apply(target, client, args) { + const query = typeof args[0] === "string" ? args[0] : args[0]?.text; + const values = args[1] ?? args[0]?.values ?? []; + return (async () => { + if (responseRead && query?.startsWith("select") && query.includes('from "mcp_execution"')) { + responseRead = false; + await checkpoint("response_before_persist"); + } + if (query?.startsWith('update "mcp_admission"') && values.includes("execute")) + await checkpoint("before_admission"); + if (query?.startsWith('update "mcp_approval" set "consumed_at"')) + await checkpoint("approval_before_consume"); + if (query?.startsWith('insert into "mcp_execution"')) { + await checkpoint("before_insert"); + transitions.set(client, "after_insert"); + } + if (query?.startsWith('update "mcp_execution" set "state"')) { + if (values[0] === "dispatching") { + await checkpoint("intent_issued"); + transitions.set(client, "barrier_committed"); + } + if (["succeeded", "failed"].includes(values[0])) { + await checkpoint("response_verified"); + transitions.set(client, "result_committed"); + } + } + const result = await Reflect.apply(target, client, args); + if (query?.startsWith('update "mcp_approval" set "consumed_at"')) + await checkpoint("approval_consumed_uncommitted"); + if (query?.toLowerCase() === "commit") { + const boundary = transitions.get(client); + transitions.delete(client); + if (boundary) await checkpoint(boundary); + } + if (query?.startsWith('update "mcp_execution" set "completion_enqueued_version"')) + await checkpoint("receipt_committed"); + return result; + })(); + }, +}); +const db = getDb(), + journal = makeMcpJournal(db); +const ref = fromJson(McpWritePolicyRefSchema, input.remote.policyRef); +let leases = 0; +async function fence(req: { + toolCallId: string; + originalArguments: string; + modelToolName: string; +}) { + const [call] = await db + .select() + .from(mcpExecution) + .where( + and( + eq(mcpExecution.sessionId, input.request.sessionId), + eq(mcpExecution.toolCallId, input.request.toolCallId), + ), + ); + if (call) { + const lease = await journal.getAdmission(call.id); + if (call.state === "authorized" && lease?.phase === "execute" && lease.state === "admitted") + await checkpoint("admission_committed"); + if (call.state === "dispatching" && leases > 0) await checkpoint("intent_leased"); + } + if (leases > 0 && ++postLeaseFences === 2) responseRead = true; + return { + protocol: 1, + status: + req.toolCallId === input.request.toolCallId && + req.originalArguments === input.request.originalArguments && + req.modelToolName === input.request.modelToolName + ? "current" + : "stale", + }; +} +const policy = createClient( + SessionPolicyService, + createRouterTransport((router) => + router.service(SessionPolicyService, { + readWritePolicy: async () => ({ + state: { + policy: { + current: true, + initialized: input.remote.eligibility !== "unknown", + writeEligibility: input.remote.eligibility, + ref, + ancestorPolicyRefs: [], + }, + }, + }), + validateExecutionIdentity: fence, + validateDispatchFence: async (req) => + input.remote.eligibility === "write_eligible" + ? fence(req) + : { protocol: 1, status: "stale" }, + }), + ), +); +const control = unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => + router.service(CredentialControlService, { + lease: async (req) => { + leases++; + await checkpoint("before_lease"); + return { + protocol: 1, + intentId: req.intentId, + expiresAt: req.expiresAt, + operationExpiresAt: req.operationExpiresAt, + header: "authorization", + value: "Bearer known-literal-token", + }; + }, + }), + ), + ), +); +const resolver = makeStoredSessionBindingResolver( + { + control: policy, + readOperatorPolicies: async () => input.remote.operatorPolicies, + enforceManagedDeny: async (expected) => expected, + }, + db, +); +const driver = makeMcpEffectDriver({ + db, + deploymentId: input.deploymentId, + control, + policy, + resolver, + workerTls: () => input.remote.workerTls, + enabled: () => true, + now, +}); +// The coordinator completion boundary is a parent-owned deterministic fixture. It stores a +// fixed-key outbox record before returning, and survives driver process death. +const completions: (() => void)[] = []; +lines.on("line", (line) => { + if (line === "ACK") completions.shift()?.(); + if (line === "CONTINUE") releaseCheckpoint?.(); +}); +const runtime = makeMcpRecovery({ + db, + driver, + deploymentId: input.deploymentId, + now, + readRequestEvent: async (sessionId, index) => { + if (input.requestReadFailures && input.requestReadFailures-- > 0) { + await checkpoint("request_read_failed"); + throw Error("private-control-error"); + } + if (sessionId !== input.request.sessionId || index !== BigInt(input.eventIdx ?? "1")) + return undefined; + return { + idx: index, + kind: "tool_call_requested", + payloadJson: JSON.stringify({ + run_id: input.request.runId, + tool_call_id: input.request.toolCallId, + name: input.request.modelToolName, + args_json: input.request.originalArguments, + }), + }; + }, + completer: { + async completeToolCall(request) { + await checkpoint("before_enqueue"); + const ack = new Promise((resolve) => completions.push(resolve)); + process.stdout.write(`COMPLETE ${JSON.stringify(request)}\n`); + await ack; + await checkpoint("after_enqueue_ack"); + }, + }, +}); +installMcpRecovery(runtime); +assertSweepPoliciesExhaustive(); +DBOS.setConfig({ + name: "mcp-restart-test", + systemDatabaseUrl: system, + executorID: input.workflowId, + applicationVersion: "mcp-restart-v1", + runAdminServer: false, +}); +await DBOS.launch(); +const scanner = makeMcpRecoveryScanner({ + deploymentId: input.deploymentId, + journal, + runtime: () => runtime, + now, +}); +if (!input.resume) { + const consumer = makeToolConsumer({ + registry: createToolRegistry(), + pendingCalls: makePendingToolCallStore(db), + now, + startWorkflow: async () => { + throw Error("static workflow used"); + }, + startMcpRequest: startMcpRequestWorkflow, + }); + await consumer.handle( + { + idx: BigInt(input.eventIdx ?? "1"), + kind: "tool_call_requested", + payloadJson: JSON.stringify({ + run_id: input.request.runId, + tool_call_id: input.request.toolCallId, + name: input.request.modelToolName, + args_json: input.request.originalArguments, + }), + }, + { sessionId: input.request.sessionId }, + ); +} +await scanner.start(); +process.stdout.write("READY\n"); +setInterval(() => {}, 1000); diff --git a/orchestrator/src/__tests__/helpers/mcp-test-isolation.ts b/orchestrator/src/__tests__/helpers/mcp-test-isolation.ts new file mode 100644 index 000000000..0055a324d --- /dev/null +++ b/orchestrator/src/__tests__/helpers/mcp-test-isolation.ts @@ -0,0 +1,30 @@ +import { Pool } from "pg"; + +/** Verify both actual database identities before any DBOS configuration or fixture write. */ +export async function verifyMcpTestDatabases() { + const app = process.env.ORCHESTRATOR_DATABASE_URL; + const system = process.env.MCP_TEST_DBOS_DATABASE_URL; + const expected = process.env.MCP_TEST_DATABASE_NAME; + if (!app || !system || !expected) throw Error("explicit isolated MCP test databases required"); + const a = new URL(app), + b = new URL(system); + if ( + !/^engram_mcp_\d+_\d+_\d+$/.test(expected) || + a.pathname !== `/${expected}` || + !/^\/engram_mcp_dbos_test_[a-z0-9_]+$/.test(b.pathname) || + a.pathname === b.pathname || + [a, b].some((u) => u.hostname !== "127.0.0.1" || u.port !== "5435") + ) + throw Error("MCP test database isolation mismatch"); + for (const url of [a, b]) { + const pool = new Pool({ connectionString: url.href, connectionTimeoutMillis: 2000 }); + try { + const result = await pool.query("select current_database() as name"); + if (result.rows[0]?.name !== url.pathname.slice(1)) + throw Error("MCP test database identity mismatch"); + } finally { + await pool.end(); + } + } + return { app, system }; +} diff --git a/orchestrator/src/__tests__/http-log.test.ts b/orchestrator/src/__tests__/http-log.test.ts new file mode 100644 index 000000000..fe00a1915 --- /dev/null +++ b/orchestrator/src/__tests__/http-log.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from "bun:test"; +import { Hono } from "hono"; +import { makeHttpLog } from "../http-log.ts"; +test("HTTP logs retain method path status and time without callback query or request body", async () => { + const records: unknown[] = []; + let now = 100; + const app = new Hono(); + app.use( + makeHttpLog( + (record) => records.push(record), + () => now, + ), + ); + app.get("/oauth/mcp", (c) => { + now = 125; + return c.text("login required", 401); + }); + app.post("/credential", (c) => c.text("ok")); + await app.request( + "https://app.example.test/oauth/mcp?code=secret-code&state=secret-state&iss=https://issuer.example", + ); + await app.request("https://app.example.test/credential", { + method: "POST", + headers: { authorization: "Bearer private" }, + body: "entered-secret", + }); + expect(records[0]).toEqual({ method: "GET", path: "/oauth/mcp", status: 401, durationMs: 25 }); + expect(records[1]).toEqual({ method: "POST", path: "/credential", status: 200, durationMs: 0 }); + const text = JSON.stringify(records); + for (const secret of [ + "secret-code", + "secret-state", + "issuer.example", + "entered-secret", + "private", + ]) + expect(text).not.toContain(secret); +}); diff --git a/orchestrator/src/__tests__/mcp-actions-rpc.test.ts b/orchestrator/src/__tests__/mcp-actions-rpc.test.ts new file mode 100644 index 000000000..5c9f09ad6 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-actions-rpc.test.ts @@ -0,0 +1,108 @@ +import { expect, test } from "bun:test"; +import { Code, ConnectError, createClient } from "@connectrpc/connect"; +import { createConnectTransport } from "@connectrpc/connect-node"; +import type { AddressInfo } from "node:net"; +import { Hono } from "hono"; +import { buildServer } from "../server.ts"; +import { IntegrationService } from "../gen/engram/app/v1/integration_pb.ts"; +import { mcpHandlers } from "../mcp/contracts.ts"; +import { reviewContext } from "../mcp/journal-review.ts"; +import { effectFixture } from "./mcp-effect-fixture.ts"; + +test("typed cancellation and unknown resolution require the live owner and exact context", async () => { + const f = await effectFixture({ classification: "write", response: "hold" }); + let actorId = "other"; + const server = buildServer(new Hono(), (router) => + router.service( + IntegrationService, + mcpHandlers({ + services: { + enabled: true, + journal: f.journal, + repository: f.repository, + runtime: { + cancel: async (id) => f.driver.cancel(id), + prepareRebind: async () => { + throw Error("unavailable"); + }, + rebind: async () => { + throw Error("unavailable"); + }, + }, + }, + connections: f.connections, + getSession: async () => ({ + user: { id: actorId, role: "user", email: "user@example.test" }, + }), + now: () => new Date(), + connectionToProto: (row) => ({ id: row.id }), + }), + ), + ); + try { + const call = await f.driver.acceptToolRequest(f.request); + await f.driver.driveOnce(call.id); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); + const pending = f.driver.driveOnce(call.id); + const deadline = Date.now() + 5000; + while (!(await f.command("stats")).calls && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 10)); + const dispatched = (await f.journal.get(call.id))!; + expect(dispatched.state).toBe("dispatching"); + const url = await new Promise((resolve) => + server.listen(0, "127.0.0.1", () => + resolve(`http://127.0.0.1:${(server.address() as AddressInfo).port}/rpc`), + ), + ); + const client = createClient( + IntegrationService, + createConnectTransport({ baseUrl: url, httpVersion: "1.1" }), + ); + try { + await client.cancelMcpCall({ + executionId: call.id, + expectedJournalRevision: dispatched.version, + }); + throw Error("expected denial"); + } catch (error) { + expect(error).toBeInstanceOf(ConnectError); + expect((error as ConnectError).code).toBe(Code.PermissionDenied); + } + expect((await f.journal.get(call.id))!.state).toBe("dispatching"); + actorId = f.actorId; + const canceled = await client.cancelMcpCall({ + executionId: call.id, + expectedJournalRevision: dispatched.version, + }); + expect(canceled.receipt?.state).toBe("outcome_unknown"); + expect((await pending).state).toBe("outcome_unknown"); + const unknown = (await f.journal.get(call.id))!; + const expected = { + executionId: call.id, + journalRevision: unknown.version, + requestDigest: unknown.requestDigest, + bindingHash: unknown.bindingHash, + intentId: unknown.intentId!, + }; + await expect( + client.resolveMcpOutcome({ + executionId: call.id, + outcome: "acknowledged", + expectedUnknownContext: { ...expected, intentId: "wrong" }, + }), + ).rejects.toThrow(); + const receipt = await client.resolveMcpOutcome({ + executionId: call.id, + outcome: "acknowledged", + expectedUnknownContext: expected, + }); + expect(receipt.receipt?.expectedContext?.intentId).toBe(expected.intentId); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await new Promise((resolve) => + server.listening ? server.close(() => resolve()) : resolve(), + ); + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-active-watch.test.ts b/orchestrator/src/__tests__/mcp-active-watch.test.ts new file mode 100644 index 000000000..e16ddd4be --- /dev/null +++ b/orchestrator/src/__tests__/mcp-active-watch.test.ts @@ -0,0 +1,47 @@ +import { expect, test } from "bun:test"; +import { makeMcpActiveWatch } from "../mcp/active-watch.ts"; + +test("registers before its first authority read and aborts if it fails", async () => { + let registered = false; + const watch = makeMcpActiveWatch({ + check: async () => { + expect(registered).toBe(true); + throw Error("unavailable"); + }, + intervalMs: 10, + }); + const active = watch.register("execution"); + registered = true; + await active.ready; + expect(active.signal.aborted).toBe(true); + active.close(); +}); + +test("periodic authority checks and local cancellation abort active work", async () => { + let calls = 0; + const watch = makeMcpActiveWatch({ + check: async () => { + if (++calls > 1) throw Error("changed"); + }, + intervalMs: 10, + }); + const active = watch.register("execution"); + await active.ready; + await new Promise((resolve) => + active.signal.addEventListener("abort", () => resolve(), { once: true }), + ); + expect(calls).toBeGreaterThan(1); + active.close(); + const other = watch.register("other"); + watch.cancel("other"); + expect(other.signal.aborted).toBe(true); + other.close(); +}); + +test("a stalled authority read aborts within the check budget", async () => { + const watch = makeMcpActiveWatch({ check: () => new Promise(() => {}), intervalMs: 10 }); + const active = watch.register("execution"); + await Promise.race([active.ready, new Promise((resolve) => setTimeout(resolve, 50))]); + expect(active.signal.aborted).toBe(true); + active.close(); +}); diff --git a/orchestrator/src/__tests__/mcp-arguments.test.ts b/orchestrator/src/__tests__/mcp-arguments.test.ts new file mode 100644 index 000000000..0fb9e6c74 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-arguments.test.ts @@ -0,0 +1,134 @@ +import { expect, spyOn, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { assertOriginalArguments } from "../mcp/journal-review.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { MCP_LIMITS } from "../mcp/limits.ts"; + +const accepted = [ + "0", + "-0", + "0.0e999999999999999999999", + "1e3", + "1000.00", + "1.2300e2", + "0.1", + "1e-7", + "9007199254740991", + "-9007199254740991", + "5e-324", + "1e+0003", + "1e-0007", +]; +const rejected = [ + "9007199254740992", + "9007199254740993", + "-9007199254740992", + "-9007199254740993", + "9.007199254740993e15", + "0.10000000000000001", + "1.0000000000000001", + "9007199254740991.1", + "1e400", + "-1e400", + "1e-400", + "-1e-400", + "4e-324", + "1e20", +]; +for (const token of accepted) + test(`original arguments preserve decimal value ${token}`, () => { + const raw = `{"x":${token}}`; + expect(() => assertOriginalArguments(raw, canonicalDigest(JSON.parse(raw)))).not.toThrow(); + }); +for (const token of rejected) + test(`original arguments reject changed decimal value ${token}`, () => { + const raw = `{"x":${token}}`; + const parsed = JSON.parse(raw); + const hash = Number.isFinite(parsed.x) ? canonicalDigest(parsed) : canonicalDigest({ x: null }); + expect(() => assertOriginalArguments(raw, hash)).toThrow("invalid argument JSON"); + }); + +test("numeric checks cover nested arrays and objects, with numeric strings unchanged", () => { + const safe = '{"items":[{"x":0.1},[5e-324]],"id":"9007199254740993"}'; + expect(() => assertOriginalArguments(safe, canonicalDigest(JSON.parse(safe)))).not.toThrow(); + const unsafe = '{"items":[{"x":9007199254740993}]}'; + expect(() => assertOriginalArguments(unsafe, canonicalDigest(JSON.parse(unsafe)))).toThrow(); +}); + +test("bounded numeric tokens handle zero, exponent leading zeros and underflow", () => { + for (const raw of [ + `{"x":0.${"0".repeat(MCP_LIMITS.argumentBytes - 40)}e999999999999999999999}`, + `{"x":1e${"0".repeat(MCP_LIMITS.argumentBytes - 20)}3}`, + `{"x":1${"0".repeat(MCP_LIMITS.argumentBytes - 30)}e-${MCP_LIMITS.argumentBytes - 29}}`, + ]) { + expect(Buffer.byteLength(raw)).toBeLessThanOrEqual(MCP_LIMITS.argumentBytes); + expect(() => assertOriginalArguments(raw, canonicalDigest(JSON.parse(raw)))).not.toThrow(); + } + for (const raw of [ + `{"x":0.${"0".repeat(MCP_LIMITS.argumentBytes - 20)}1}`, + `{"x":1e${"9".repeat(MCP_LIMITS.argumentBytes - 20)}}`, + ]) { + expect(Buffer.byteLength(raw)).toBeLessThanOrEqual(MCP_LIMITS.argumentBytes); + expect(() => assertOriginalArguments(raw, canonicalDigest(JSON.parse(raw)))).toThrow(); + } +}); + +test("runtime supplies the original numeric token to the reviver", () => { + let source: string | undefined; + JSON.parse("9007199254740993", (_key, value, context?: { source?: string }) => { + source = context?.source; + return value; + }); + expect(source).toBe("9007199254740993"); +}); + +test("missing numeric source fails closed", () => { + const parse = JSON.parse; + const hash = canonicalDigest({ x: 1 }); + const patched = spyOn(JSON, "parse").mockImplementation((raw, reviver) => + parse( + raw, + reviver && + function (key, value) { + return reviver.call(this, key, value); + }, + ), + ); + try { + expect(() => assertOriginalArguments('{"x":1}', hash)).toThrow("invalid argument JSON"); + } finally { + patched.mockRestore(); + } +}); + +test("near-budget internal and trailing zero runs finish in a bounded child", () => { + const parser = new URL("../mcp/arguments.ts", import.meta.url).href; + const child = spawnSync( + process.execPath, + [ + "--no-env-file", + "--no-install", + "--eval", + ` + import { parseArgumentJson } from ${JSON.stringify(parser)}; + const limit = ${MCP_LIMITS.argumentBytes}; + const accepted = '{"x":0.1' + "0".repeat(limit - 9) + '}'; + const rejected = '{"x":0.1' + "0".repeat(limit - 10) + '1}'; + if (Buffer.byteLength(accepted) !== limit || Buffer.byteLength(rejected) !== limit) + throw Error("fixture byte count changed"); + if (JSON.stringify(parseArgumentJson(accepted)) !== '{"x":0.1}') + throw Error("trailing-zero value changed"); + let denied = false; + try { parseArgumentJson(rejected); } + catch (error) { denied = error instanceof Error && error.message === "numeric argument loses precision"; } + if (!denied) throw Error("internal-zero precision loss accepted"); + console.log("checked two full-limit numeric tokens"); + `, + ], + { encoding: "utf8", timeout: 10000, killSignal: "SIGKILL" }, + ); + expect(child.error).toBeUndefined(); + expect(child.status).toBe(0); + expect(child.stderr).toBe(""); + expect(child.stdout.trim()).toBe("checked two full-limit numeric tokens"); +}, 15000); diff --git a/orchestrator/src/__tests__/mcp-authority-notification.test.ts b/orchestrator/src/__tests__/mcp-authority-notification.test.ts new file mode 100644 index 000000000..022c0f44a --- /dev/null +++ b/orchestrator/src/__tests__/mcp-authority-notification.test.ts @@ -0,0 +1,80 @@ +import { expect, test } from "bun:test"; +import { effectFixture } from "./mcp-effect-fixture.ts"; +import { makeCredentialMetadataStore } from "../mcp/credential-store.ts"; +import { observeMcpAuthorityChanges } from "../mcp/authority-notification.ts"; + +test("a committed authority mutation requests reconciliation and notification failure does not undo it", async () => { + const { eq } = await import("drizzle-orm"); + const { integrationConnection } = await import("../db/schema.ts"); + const f = await effectFixture({ classification: "read" }); + let calls = 0; + const stop = observeMcpAuthorityChanges(() => { + calls++; + throw Error("unavailable"); + }); + try { + const credentials = makeCredentialMetadataStore(f.db); + const [current] = await f.db + .select() + .from(integrationConnection) + .where(eq(integrationConnection.id, f.connection.id)); + await credentials.claim( + f.connection.id, + current!.configRevision, + current!.authGeneration, + f.actorId, + current!.config, + new Date(), + ); + expect((await credentials.get(f.connection.id)).revoked).toBe(true); + expect(calls).toBe(1); + await expect( + credentials.claim( + f.connection.id, + current!.configRevision, + current!.authGeneration, + f.actorId, + current!.config, + new Date(), + ), + ).rejects.toThrow(); + expect(calls).toBe(1); + } finally { + stop(); + await f.cleanup(); + } +}, 20000); + +test("a credential transition requests durable reconciliation of active work", async () => { + const { eq } = await import("drizzle-orm"); + const { integrationConnection } = await import("../db/schema.ts"); + const { makeMcpLifecycleReconciler } = await import("../mcp/lifecycle-reconciler.ts"); + const f = await effectFixture({ classification: "read" }); + const scanner = makeMcpLifecycleReconciler({ db: f.db, driver: f.driver }); + let scan: Promise | undefined; + const stop = observeMcpAuthorityChanges(() => { + scan = scanner.reconcileAuthorityChange(); + }); + try { + const call = await f.driver.acceptToolRequest(f.request); + const [current] = await f.db + .select() + .from(integrationConnection) + .where(eq(integrationConnection.id, f.connection.id)); + await makeCredentialMetadataStore(f.db).claim( + f.connection.id, + current!.configRevision, + current!.authGeneration, + f.actorId, + current!.config, + new Date(), + ); + expect(scan).toBeDefined(); + await scan; + expect((await f.journal.get(call.id))!.state).toBe("canceled"); + expect((await f.command("stats")).calls).toBe(0); + } finally { + stop(); + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-automation-denial.test.ts b/orchestrator/src/__tests__/mcp-automation-denial.test.ts new file mode 100644 index 000000000..6b48de890 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-automation-denial.test.ts @@ -0,0 +1,182 @@ +import { expect, spyOn, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { + automation, + automationRun, + automationSession, + mcpAdmission, + mcpApproval, + mcpExecution, + task, + taskSession, +} from "../db/schema.ts"; +import { makeMcpRecovery } from "../mcp/recovery.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +import { effectFixture } from "./mcp-effect-fixture.ts"; + +await verifyMcpTestDatabases(); + +async function linkAutomation(f: Awaited>) { + const automationId = crypto.randomUUID(), + runId = crypto.randomUUID(); + await f.db + .insert(automation) + .values({ id: automationId, name: "Fixture automation", createdByUserId: f.actorId }); + await f.db.insert(automationRun).values({ + id: runId, + automationId, + taskId: f.taskId, + sessionId: f.sessionId, + status: "running", + trigger: { source: "manual", receivedAt: new Date().toISOString() }, + }); + await f.db.insert(automationSession).values({ sessionId: f.sessionId, runId, blockId: "start" }); + await f.db + .update(taskSession) + .set({ integrationPrincipalId: `automation:${automationId}` }) + .where(eq(taskSession.sessionId, f.sessionId)); + await f.db.update(task).set({ createdByUserId: null }).where(eq(task.id, f.taskId)); + return automationId; +} + +function recovery( + f: Awaited>, + completeToolCall: (input: { resultJson: string }) => Promise = async () => {}, +) { + return makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + completer: { completeToolCall }, + async readRequestEvent(_sessionId, idx) { + return { + idx, + kind: "tool_call_requested", + payloadJson: JSON.stringify({ + tool_call_id: f.request.toolCallId, + run_id: f.request.runId, + name: f.request.modelToolName, + args_json: f.request.originalArguments, + }), + }; + }, + }); +} + +function requestMarker(f: Awaited>) { + return { + sessionId: f.sessionId, + runId: f.request.runId, + toolCallId: f.request.toolCallId, + requestEventIdx: "1", + deadlineMs: Date.now() + 930000, + }; +} + +test("write-eligible automation without an explicit write grant completes a policy denial without effects", async () => { + const f = await effectFixture({ classification: "write", automationWrite: false }); + let automationId: string | undefined; + try { + automationId = await linkAutomation(f); + const delivered: string[] = []; + let deliveryUnavailable = true; + const runtime = recovery(f, async ({ resultJson }) => { + if (deliveryUnavailable) throw Error("private-transport-error"); + delivered.push(resultJson); + }); + const ref = requestMarker(f); + expect(f.snapshot.immutableBindings[0]!.tools[0]!.grant.automationWrite).toBe(false); + f.failAuthorityReads(1); + await expect(runtime.decideRequest(ref)).rejects.toThrow("MCP request acceptance unavailable"); + const decision = await runtime.decideRequest(ref); + expect(decision).toEqual({ kind: "denied", code: "write_not_allowed" }); + if (decision.kind !== "denied") throw Error("expected policy denial"); + await expect(runtime.deliverDenial(ref, decision.code)).rejects.toThrow( + "MCP denial completion unavailable", + ); + deliveryUnavailable = false; + await runtime.deliverDenial(ref, decision.code); + expect(delivered).toHaveLength(1); + expect(JSON.parse(delivered[0]!)).toMatchObject({ + source: "runtime", + denial: "write_not_allowed", + delivery: "available", + output: { isError: true }, + }); + expect( + await f.db.select().from(mcpExecution).where(eq(mcpExecution.sessionId, f.sessionId)), + ).toHaveLength(0); + for (const table of [mcpAdmission, mcpApproval]) + expect( + await f.db + .select() + .from(table) + .innerJoin(mcpExecution, eq(table.executionId, mcpExecution.id)) + .where(eq(mcpExecution.sessionId, f.sessionId)), + ).toHaveLength(0); + expect(f.events).not.toContain("lease"); + expect(f.leases).toBe(0); + expect(await f.command("stats")).toMatchObject({ calls: 0, credentialRequests: 0 }); + } finally { + if (automationId) await f.db.delete(automation).where(eq(automation.id, automationId)); + await f.cleanup(); + } +}, 20000); + +test("explicit constrained automation grant preserves unavailable errors, original acceptance, and argument limits", async () => { + const f = await effectFixture({ + classification: "write", + constraints: [{ properties: { x: { maximum: 1 } } }], + }); + let automationId: string | undefined; + try { + automationId = await linkAutomation(f); + const runtime = recovery(f), + ref = requestMarker(f); + const transaction = spyOn(f.db, "transaction").mockRejectedValueOnce( + Error("private-database-error"), + ); + try { + await expect(runtime.decideRequest(ref)).rejects.toThrow( + "MCP request acceptance unavailable", + ); + } finally { + transaction.mockRestore(); + } + const decision = await runtime.decideRequest(ref); + expect(decision.kind).toBe("accepted"); + if (decision.kind !== "accepted") throw Error("expected automation acceptance"); + const call = (await f.journal.get(decision.execution.executionId))!; + expect(call.authorizationKind).toBe("automation"); + expect((await f.driver.driveOnce(call.id)).state).toBe("succeeded"); + expect( + await f.db.select().from(mcpApproval).where(eq(mcpApproval.executionId, call.id)), + ).toHaveLength(0); + f.request.toolCallId = "restricted-call"; + f.request.originalArguments = '{"x":2}'; + f.requests.set(f.request.toolCallId, { ...f.request }); + const restricted = await runtime.decideRequest(requestMarker(f)); + if (restricted.kind !== "accepted") throw Error("expected schema validation path"); + const failed = await f.driver.driveOnce(restricted.execution.executionId); + expect(failed.state).toBe("failed"); + expect(failed.intentId).toBeNull(); + expect(failed.result).toMatchObject({ source: "local_validation", output: { isError: true } }); + expect(await f.journal.getAdmission(failed.id)).toBeNull(); + await f.db.update(automation).set({ enabled: false }).where(eq(automation.id, automationId)); + expect( + await runtime.decideRequest({ + ...ref, + deadlineMs: 0, + requestEventIdx: "2", + runId: "changed-run", + }), + ).toEqual(decision); + expect((await f.journal.get(call.id))!.originalArguments).toBe('{ "x": 1 }'); + expect(f.leases).toBe(1); + expect((await f.command("stats")).calls).toBe(1); + } finally { + if (automationId) await f.db.delete(automation).where(eq(automation.id, automationId)); + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-binding-fixture.ts b/orchestrator/src/__tests__/mcp-binding-fixture.ts new file mode 100644 index 000000000..4bfe713f7 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-binding-fixture.ts @@ -0,0 +1,73 @@ +import { reviewCatalog } from "../mcp/catalog.ts"; +import type { ProfileIntegrationGrant } from "../db/schema.ts"; +export const operatorPublication = { + generation: 1, + identity: `sha256:${"b".repeat(64)}`, + deploymentIdentity: `sha256:${"c".repeat(64)}`, + networkIdentity: `sha256:${"d".repeat(64)}`, + trustIdentity: `sha256:${"e".repeat(64)}`, + caContentDigests: { control: `sha256:${"f".repeat(64)}` }, +}; +export const schema = { + $schema: "https://json-schema.org/draft/2020-12/schema", + $defs: { name: { type: "string" } }, + type: "object", + properties: { name: { $ref: "#/$defs/name" } }, +}; +export const bindingGrant: ProfileIntegrationGrant = { + connectionId: "connection-a", + operation: "invoke", + credentialScope: "org", + resourceConstraints: [], + mcp: { + upstreamName: "Document.Search", + automationWrite: false, + unrestricted: true, + argumentConstraintDocuments: [], + }, +}; +export function bindingAuthority() { + return { + profileId: "profile-a", + profileGrants: [structuredClone(bindingGrant)], + connections: [ + { + id: "connection-a", + catalogId: "catalog-a-1", + alias: "docs", + enabled: true, + config: { + endpoint: "https://tools.example.test/mcp", + zone: "public", + auth: { kind: "api_key" as const, secretRef: "mcp.key" }, + }, + state: { + configRevision: 1, + authGeneration: 1, + catalogRevision: 1, + credentialIdentity: "identity-a", + revoked: false, + }, + tools: reviewCatalog([ + { + upstreamName: "Document.Search", + description: "Search", + inputSchema: schema, + classification: "read", + enabled: true, + }, + ]), + }, + ], + operatorPolicies: [ + { + publication: structuredClone(operatorPublication), + zoneId: "public", + policyRevision: "1", + policyDigest: `sha256:${"a".repeat(64)}`, + authorities: ["tools.example.test"], + aliases: ["alias.example.test"], + }, + ], + }; +} diff --git a/orchestrator/src/__tests__/mcp-binding-store.test.ts b/orchestrator/src/__tests__/mcp-binding-store.test.ts new file mode 100644 index 000000000..fcdf2563d --- /dev/null +++ b/orchestrator/src/__tests__/mcp-binding-store.test.ts @@ -0,0 +1,325 @@ +import { expect, test } from "bun:test"; +import { eq, inArray } from "drizzle-orm"; +import { checkDb, getDb } from "../db/client.ts"; +import { task, taskSession } from "../db/schema.ts"; +import { compileSessionBindings } from "../mcp/session-bindings.ts"; +import { bindingSnapshotColumns, readStoredBindingSession } from "../mcp/binding-store.ts"; +import { bindingAuthority } from "./mcp-binding-fixture.ts"; +const reachable = process.env.ORCHESTRATOR_DATABASE_URL ? await checkDb() : false; +test.skipIf(!reachable)( + "binding envelope and existing connection snapshots round-trip atomically", + async () => { + const db = getDb(), + taskId = crypto.randomUUID(), + sessionId = crypto.randomUUID(); + const authority = bindingAuthority(); + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: authority.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + }); + const columns = bindingSnapshotColumns( + snapshot, + authority.connections.map((c) => ({ + id: c.id, + alias: c.alias, + provider: "mcp", + displayName: "Docs", + config: c.config, + })), + ); + try { + await db.insert(task).values({ id: taskId, type: "chat" }); + await expect( + db.transaction(async (tx) => { + await tx.insert(taskSession).values({ taskId, sessionId, ...columns }); + throw new Error("reservation rollback"); + }), + ).rejects.toThrow("reservation rollback"); + expect(await readStoredBindingSession(sessionId, db)).toBeNull(); + await db.insert(taskSession).values({ taskId, sessionId, ...columns }); + expect((await readStoredBindingSession(sessionId, db))?.snapshot).toEqual(snapshot); + expect(columns.mcpBindingEnvelope).not.toHaveProperty("immutableBindings"); + await db.update(task).set({ status: "cancelled" }).where(eq(task.id, taskId)); + expect((await readStoredBindingSession(sessionId, db))?.active).toBe(false); + await db + .update(taskSession) + .set({ integrationConnections: [] }) + .where(eq(taskSession.sessionId, sessionId)); + await expect(readStoredBindingSession(sessionId, db)).rejects.toThrow(); + } finally { + await db.delete(task).where(eq(task.id, taskId)); + } + }, +); +test.skipIf(!reachable)( + "resolver reads current stored profile and immutable published catalog authority", + async () => { + const { profile, integrationConnection, mcpCatalog } = await import("../db/schema.ts"); + const { readLiveBindingAuthority } = await import("../mcp/binding-store.ts"); + const { canonicalDigest } = await import("../mcp/hash.ts"); + const { checkBindingCurrent } = await import("../mcp/session-bindings.ts"); + const { integrationSnapshotHash } = await import("../integrations/grants.ts"); + const db = getDb(), + pid = crypto.randomUUID(), + cid = crypto.randomUUID(), + catalogId = crypto.randomUUID(); + const fixture = bindingAuthority(), + source = fixture.connections[0]!; + const grants = fixture.profileGrants.map((grant) => ({ ...grant, connectionId: cid })); + try { + await db.insert(profile).values({ + id: pid, + name: "Binding test", + imageId: "image", + harness: "claude", + integrationGrants: grants, + }); + await db.insert(integrationConnection).values({ + id: cid, + alias: `docs-${cid}`, + provider: "mcp", + displayName: "Docs", + config: source.config, + enabled: true, + ...source.state, + }); + await db.insert(mcpCatalog).values({ + id: catalogId, + connectionId: cid, + kind: "published", + revision: 1, + configRevision: 1, + authGeneration: 1, + credentialIdentity: source.state.credentialIdentity, + tools: source.tools, + fingerprint: canonicalDigest(source.tools), + actorId: "test-operator", + createdAt: new Date(), + }); + const authority = await readLiveBindingAuthority(pid, fixture.operatorPolicies, db); + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: pid, integrationGrants: grants }, + liveAuthority: authority, + bindingEpoch: 0, + }); + expect(snapshot.immutableBindings[0]!.catalogId).toBe(catalogId); + const columns = bindingSnapshotColumns(snapshot, [ + { + id: cid, + alias: `docs-${cid}`, + provider: "mcp", + displayName: "Docs", + config: source.config, + }, + ]); + const fullHash = integrationSnapshotHash({ + profileId: pid, + integrationGrants: grants, + ...columns, + }); + expect(fullHash).not.toBe( + integrationSnapshotHash({ + profileId: pid, + integrationGrants: grants, + integrationConnections: columns.integrationConnections, + }), + ); + expect(fullHash).toBe( + integrationSnapshotHash( + JSON.parse(JSON.stringify({ profileId: pid, integrationGrants: grants, ...columns })), + ), + ); + // Credential refresh metadata changes alone do not alter the authority tuple. + await db + .update(integrationConnection) + .set({ testedAt: new Date(), updatedAt: new Date() }) + .where(eq(integrationConnection.id, cid)); + expect( + checkBindingCurrent({ + snapshot, + liveAuthority: await readLiveBindingAuthority(pid, fixture.operatorPolicies, db), + }).current, + ).toBe(true); + await db.update(profile).set({ integrationGrants: [] }).where(eq(profile.id, pid)); + expect( + checkBindingCurrent({ + snapshot, + liveAuthority: await readLiveBindingAuthority(pid, fixture.operatorPolicies, db), + }).current, + ).toBe(false); + await db.update(profile).set({ integrationGrants: grants }).where(eq(profile.id, pid)); + await db + .update(integrationConnection) + .set({ authGeneration: 2 }) + .where(eq(integrationConnection.id, cid)); + expect( + checkBindingCurrent({ + snapshot, + liveAuthority: await readLiveBindingAuthority(pid, fixture.operatorPolicies, db), + }).current, + ).toBe(false); + } finally { + await db.delete(mcpCatalog).where(eq(mcpCatalog.id, catalogId)); + await db.delete(integrationConnection).where(eq(integrationConnection.id, cid)); + await db.delete(profile).where(eq(profile.id, pid)); + } + }, +); + +test.skipIf(!reachable)( + "live authority reads all current catalogs in two queries and retains unavailable connections", + async () => { + const { drizzle } = await import("drizzle-orm/node-postgres"); + const { getPool } = await import("../db/client.ts"); + const schema = await import("../db/schema.ts"); + const { profile, integrationConnection, mcpCatalog } = schema; + const { readLiveBindingAuthority } = await import("../mcp/binding-store.ts"); + const { canonicalDigest } = await import("../mcp/hash.ts"); + const { reviewCatalog } = await import("../mcp/catalog.ts"); + const { checkBindingCurrent } = await import("../mcp/session-bindings.ts"); + const queries: string[] = []; + const db = drizzle(getPool(), { + schema, + logger: { logQuery: (query) => queries.push(query) }, + }); + const pid = crypto.randomUUID(); + const ids = { + current: crypto.randomUUID(), + stale: crypto.randomUUID(), + draft: crypto.randomUUID(), + credentials: crypto.randomUUID(), + revoked: crypto.randomUUID(), + missing: crypto.randomUUID(), + other: crypto.randomUUID(), + }; + const fixture = bindingAuthority(), + source = fixture.connections[0]!; + const grants = [ids.current, ids.revoked].map((connectionId) => ({ + ...fixture.profileGrants[0]!, + connectionId, + })); + const updatedTools = reviewCatalog([ + { ...source.tools[0]!, description: "Search current documents" }, + ]); + const catalogs = [ + { connectionId: ids.current, revision: 1, kind: "published" as const }, + { connectionId: ids.current, revision: 2, kind: "published" as const }, + { connectionId: ids.stale, revision: 1, kind: "published" as const }, + { connectionId: ids.draft, revision: 0, kind: "draft" as const }, + { connectionId: ids.credentials, revision: 1, kind: "published" as const }, + { connectionId: ids.revoked, revision: 1, kind: "published" as const }, + { connectionId: ids.current, revision: 3, kind: "published" as const }, + ].map((value) => { + const tools = + value.revision === 3 + ? [{ ...updatedTools[0]!, fingerprint: "invalid" }] + : value.revision === 2 + ? updatedTools + : source.tools; + return { + ...value, + id: crypto.randomUUID(), + configRevision: 1, + authGeneration: value.connectionId === ids.credentials ? 2 : 1, + credentialIdentity: source.state.credentialIdentity, + tools, + fingerprint: canonicalDigest(tools), + actorId: "test-operator", + createdAt: new Date(), + }; + }); + async function read() { + queries.length = 0; + const authority = await readLiveBindingAuthority(pid, fixture.operatorPolicies, db); + expect(queries).toHaveLength(2); + return authority; + } + try { + await db.insert(profile).values({ + id: pid, + name: "Catalog join test", + imageId: "image", + harness: "claude", + integrationGrants: grants, + }); + await db.insert(integrationConnection).values( + Object.entries(ids).map(([name, id]) => ({ + id, + alias: `docs-${id}`, + provider: name === "other" ? "github" : "mcp", + displayName: "Docs", + config: name === "other" ? {} : source.config, + enabled: true, + ...source.state, + catalogRevision: name === "stale" ? 2 : name === "draft" ? 0 : 1, + })), + ); + await db.insert(mcpCatalog).values(catalogs); + const authority = await read(); + const connections = new Map( + authority.connections.map((connection) => [connection.id, connection]), + ); + expect(connections.has(ids.other)).toBe(false); + expect(connections.get(ids.current)).toMatchObject({ + catalogId: catalogs[0]!.id, + enabled: true, + tools: source.tools, + }); + for (const id of [ids.stale, ids.draft, ids.missing]) { + expect(connections.get(id)).toMatchObject({ catalogId: "", enabled: false, tools: [] }); + } + expect(connections.get(ids.credentials)).toMatchObject({ + catalogId: catalogs[4]!.id, + enabled: false, + tools: [], + }); + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: pid, integrationGrants: grants }, + liveAuthority: authority, + bindingEpoch: 0, + }); + expect(checkBindingCurrent({ snapshot, liveAuthority: authority }).current).toBe(true); + await db + .update(integrationConnection) + .set({ catalogRevision: 2 }) + .where(eq(integrationConnection.id, ids.current)); + await db + .update(integrationConnection) + .set({ revoked: true }) + .where(eq(integrationConnection.id, ids.revoked)); + const current = await read(); + expect(current.connections.find((connection) => connection.id === ids.current)).toMatchObject( + { + catalogId: catalogs[1]!.id, + enabled: true, + tools: updatedTools, + state: { catalogRevision: 2 }, + }, + ); + expect(current.connections.find((connection) => connection.id === ids.revoked)).toMatchObject( + { + catalogId: catalogs[5]!.id, + state: { revoked: true }, + }, + ); + expect(checkBindingCurrent({ snapshot, liveAuthority: current })).toMatchObject({ + current: false, + staleReasons: expect.arrayContaining(["catalogRevision", "connection_unavailable"]), + }); + await db + .update(integrationConnection) + .set({ catalogRevision: 3 }) + .where(eq(integrationConnection.id, ids.current)); + await expect(readLiveBindingAuthority(pid, fixture.operatorPolicies, db)).rejects.toThrow( + "MCP catalog fingerprint mismatch", + ); + } finally { + await db.delete(mcpCatalog).where(inArray(mcpCatalog.connectionId, Object.values(ids))); + await db + .delete(integrationConnection) + .where(inArray(integrationConnection.id, Object.values(ids))); + await db.delete(profile).where(eq(profile.id, pid)); + } + }, +); diff --git a/orchestrator/src/__tests__/mcp-cancellation-composition.test.ts b/orchestrator/src/__tests__/mcp-cancellation-composition.test.ts new file mode 100644 index 000000000..a3b992074 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-cancellation-composition.test.ts @@ -0,0 +1,240 @@ +import { expect, test } from "bun:test"; +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { eq } from "drizzle-orm"; +import { reviewFixture } from "./mcp-review-fixture.ts"; +import { + makeMcpRecovery, + type McpRequestDecision, + type McpRequestMarker, +} from "../mcp/recovery.ts"; +import { SessionPolicyService } from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import { mcpExecution, taskSession } from "../db/schema.ts"; +import { effectFixture } from "./mcp-effect-fixture.ts"; +import { makeToolConsumer } from "../listeners/tool-consumer.ts"; +import { makePendingToolCallStore } from "../tools/pending-tool-calls.ts"; +import { createToolRegistry } from "../tools/registry.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +await verifyMcpTestDatabases(); + +test("authenticated cancellation checks exact current accepted identity before durable mutation", async () => { + const f = await reviewFixture(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + let status = "current", + checks = 0, + completions = 0; + const policy = createClient( + SessionPolicyService, + createRouterTransport((router) => + router.service(SessionPolicyService, { + validateExecutionIdentity: async (req) => { + checks++; + expect(req.originalArguments).toBe(f.input.originalArguments); + expect(req.modelToolName).toBe(f.input.resolvedTool.modelToolName); + expect(req.sessionId).toBe(f.sessionId); + expect(req.runId).toBe(f.input.runId); + return { protocol: 1, status }; + }, + }), + ), + ); + const runtime = makeMcpRecovery({ + db: f.db, + deploymentId: f.deploymentId, + now: () => f.now, + cancellationPolicy: policy, + completer: { + completeToolCall: async () => { + completions++; + }, + }, + }); + const input = { + sessionId: f.sessionId, + runId: f.input.runId, + toolCallId: f.input.toolCallId, + modelToolName: f.input.resolvedTool.modelToolName, + }; + await runtime.cancelRequest({ ...input, runId: "old-run" }); + await runtime.cancelRequest({ ...input, modelToolName: "mcp_other" }); + expect(checks).toBe(0); + expect((await f.journal.get(call.id))!.state).toBe("waiting_approval"); + status = "stale"; + await runtime.cancelRequest(input); + expect((await f.journal.get(call.id))!.state).toBe("waiting_approval"); + status = "unknown"; + await expect(runtime.cancelRequest(input)).rejects.toThrow("unavailable"); + status = "current"; + const envelope = f.snapshot; + await f.db + .update(taskSession) + .set({ mcpBindingEnvelope: { ...envelope, bindingHash: `sha256:${"0".repeat(64)}` } }) + .where(eq(taskSession.sessionId, f.sessionId)); + await runtime.cancelRequest(input); + expect((await f.journal.get(call.id))!.state).toBe("waiting_approval"); + const { immutableBindings: _bindings, ...original } = envelope; + await f.db + .update(taskSession) + .set({ mcpBindingEnvelope: original }) + .where(eq(taskSession.sessionId, f.sessionId)); + await runtime.cancelRequest(input); + const cancelled = (await f.journal.get(call.id))!; + expect(cancelled.state).toBe("canceled"); + expect(cancelled.result).toBeNull(); + expect(cancelled.completionEnqueuedVersion).toBe(0); + const version = cancelled.version; + await runtime.cancelRequest(input); + expect((await f.journal.get(call.id))!.version).toBe(version); + expect(completions).toBe(0); + } finally { + await f.cleanup(); + } +}); + +for (const cancelBeforeDecision of [true, false]) + test(`cancellation ${cancelBeforeDecision ? "before acceptance" : "after denial"} without a journal row permits the next tool request`, async () => { + const f = await effectFixture(); + try { + const request = (toolCallId: string, idx: bigint) => ({ + idx, + kind: "tool_call_requested", + payloadJson: JSON.stringify({ + run_id: f.request.runId, + tool_call_id: toolCallId, + name: f.request.modelToolName, + args_json: f.request.originalArguments, + }), + }); + const first = request(f.request.toolCallId, 1n); + const next = request("next-call", 3n); + const events = [first, next]; + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + readRequestEvent: async (_sessionId, idx) => events.find((event) => event.idx === idx), + }); + const started: McpRequestMarker[] = []; + const consumer = makeToolConsumer({ + registry: createToolRegistry(), + pendingCalls: makePendingToolCallStore(f.db), + now: () => new Date(), + startWorkflow: async () => { + throw Error("unexpected native tool dispatch"); + }, + startMcpRequest: async (ref) => { + started.push(ref); + }, + cancelMcpRequest: runtime.cancelRequest, + }); + await consumer.handle(first, { sessionId: f.sessionId }); + // The persisted native cancellation makes the exact request fence stale. + f.setFence(false); + const decide = () => runtime.decideRequest(started[0]!); + if (!cancelBeforeDecision) + expect(await decide()).toEqual({ kind: "denied", code: "authority_changed" }); + await consumer.handle( + { + idx: 2n, + kind: "tool_call_cancel_requested", + payloadJson: JSON.stringify({ + run_id: f.request.runId, + tool_call_id: f.request.toolCallId, + name: f.request.modelToolName, + }), + }, + { sessionId: f.sessionId }, + ); + if (cancelBeforeDecision) + expect(await decide()).toEqual({ kind: "denied", code: "authority_changed" }); + expect( + await f.db.select().from(mcpExecution).where(eq(mcpExecution.sessionId, f.sessionId)), + ).toEqual([]); + await consumer.handle(next, { sessionId: f.sessionId }); + expect(started.map((ref) => ref.toolCallId)).toEqual([f.request.toolCallId, "next-call"]); + f.requests.set("next-call", { ...f.request, toolCallId: "next-call" }); + f.setFence(true); + expect((await runtime.decideRequest(started[1]!)).kind).toBe("accepted"); + expect((await f.command("stats")).calls).toBe(0); + expect(f.leases).toBe(0); + } finally { + await f.cleanup(); + } + }, 20_000); + +test("cancellation during acceptance leaves the inserted call for stale-authority reconciliation", async () => { + const f = await effectFixture(); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let inserted!: () => void; + const reached = new Promise((resolve) => { + inserted = resolve; + }); + let accepting: Promise | undefined; + try { + let fences = 0; + f.beforeFence(async () => { + if (++fences === 2) { + inserted(); + await held; + } + }); + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + cancellationPolicy: createClient( + SessionPolicyService, + createRouterTransport((router) => + router.service(SessionPolicyService, { + validateExecutionIdentity: async () => ({ protocol: 1, status: "stale" }), + }), + ), + ), + readRequestEvent: async (_sessionId, idx) => ({ + idx, + kind: "tool_call_requested", + payloadJson: JSON.stringify({ + run_id: f.request.runId, + tool_call_id: f.request.toolCallId, + name: f.request.modelToolName, + args_json: f.request.originalArguments, + }), + }), + }); + accepting = runtime.decideRequest({ + ...f.request, + requestEventIdx: "1", + deadlineMs: Date.now() + 60_000, + }); + await Promise.race([ + reached, + accepting.then(() => { + throw Error("acceptance ended before the inserted-call fence"); + }), + ]); + f.setFence(false); + await runtime.cancelRequest(f.request); + release(); + const decision = await accepting; + expect(decision.kind).toBe("accepted"); + if (decision.kind !== "accepted") throw Error("inserted request lost"); + await f.driver.reconcile(decision.execution.executionId); + const call = (await f.journal.get(decision.execution.executionId))!; + expect(call.state).toBe("canceled"); + expect(call.intentId).toBeNull(); + expect((await f.command("stats")).calls).toBe(0); + expect(f.leases).toBe(0); + } finally { + release(); + try { + await accepting; + } finally { + await f.cleanup(); + } + } +}, 20_000); diff --git a/orchestrator/src/__tests__/mcp-catalog-publication.test.ts b/orchestrator/src/__tests__/mcp-catalog-publication.test.ts new file mode 100644 index 000000000..d3f44728a --- /dev/null +++ b/orchestrator/src/__tests__/mcp-catalog-publication.test.ts @@ -0,0 +1,128 @@ +import { expect, test } from "bun:test"; +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { eq, sql } from "drizzle-orm"; +import { user, integrationConnection, mcpCatalog } from "../db/schema.ts"; +import { IntegrationService } from "../gen/engram/app/v1/integration_pb.ts"; +import { mcpHandlers } from "../mcp/contracts.ts"; +import { reviewFixture } from "./mcp-review-fixture.ts"; + +for (const [change, patch] of [ + ["demoted", { role: "user" }], + ["banned", { banned: true }], + ["service account", { email: "apikey+publisher@service.local" }], +] as const) { + test(`catalog publication rejects a ${change} publisher after schema validation`, async () => { + const f = await reviewFixture(); + const entered = Promise.withResolvers(), + release = Promise.withResolvers(); + let pending: Promise | undefined; + try { + const client = createClient( + IntegrationService, + createRouterTransport((router) => + router.service( + IntegrationService, + mcpHandlers({ + services: { + enabled: true, + repository: f.repository, + journal: f.journal, + async validateSchemas() { + entered.resolve(); + await release.promise; + }, + }, + connections: f.connections, + getSession: async () => ({ user: { id: f.publisherId, role: "admin" } }), + now: () => f.now, + connectionToProto: (connection) => ({ id: connection.id }), + }), + ), + ), + ); + const before = await f.connections.get(f.connection.id); + const history = await f.db + .select() + .from(mcpCatalog) + .where(eq(mcpCatalog.connectionId, f.connection.id)); + pending = client.publishMcp({ + connectionId: f.connection.id, + draftId: f.draft.id, + expectedRevision: 1, + tools: f.catalog.tools.map((tool) => ({ + upstreamName: tool.upstreamName, + description: tool.description, + inputSchemaJson: JSON.stringify(tool.inputSchema), + classification: "read", + enabled: true, + })), + }); + await entered.promise; + await f.db.update(user).set(patch).where(eq(user.id, f.publisherId)); + release.resolve(); + await expect(pending).rejects.toThrow(); + expect(await f.connections.get(f.connection.id)).toEqual(before); + expect( + await f.db.select().from(mcpCatalog).where(eq(mcpCatalog.connectionId, f.connection.id)), + ).toEqual(history); + } finally { + release.resolve(); + await pending?.catch(() => {}); + await f.cleanup(); + } + }); +} + +test("catalog publication waits for a publisher role write before locking its connection", async () => { + const f = await reviewFixture(); + const locked = Promise.withResolvers(), + release = Promise.withResolvers(); + let mutation: Promise | undefined, pending: Promise | undefined; + try { + mutation = f.db.transaction(async (tx) => { + await tx.update(user).set({ role: "user" }).where(eq(user.id, f.publisherId)); + const result = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + locked.resolve(result.rows[0]!.pid); + await release.promise; + }); + const blocker = await locked.promise; + pending = f.repository + .publish(f.connection.id, f.draft.id, 1, f.catalog.tools, f.publisherId, f.now) + .then( + () => "published", + () => "denied", + ); + let blocked = false; + const deadline = Date.now() + 2000; + while (Date.now() < deadline) { + const result = await f.db.execute<{ waiting: boolean }>(sql` + select exists ( + select 1 from pg_stat_activity where ${blocker} = any(pg_blocking_pids(pid)) + ) as waiting`); + if (result.rows[0]?.waiting) { + blocked = true; + break; + } + await Bun.sleep(5); + } + expect(blocked).toBe(true); + // Actor-before-connection lock order leaves the connection available to other work. + await f.db.transaction(async (tx) => { + await tx.execute(sql`set local lock_timeout = '500ms'`); + await tx + .select() + .from(integrationConnection) + .where(eq(integrationConnection.id, f.connection.id)) + .for("update"); + }); + release.resolve(); + await mutation; + expect(await pending).toBe("denied"); + expect((await f.connections.get(f.connection.id))!.mcp!.catalogRevision).toBe(1); + } finally { + release.resolve(); + await mutation; + await pending; + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-catalog-read.test.ts b/orchestrator/src/__tests__/mcp-catalog-read.test.ts new file mode 100644 index 000000000..7cddf6008 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-catalog-read.test.ts @@ -0,0 +1,25 @@ +import { expect, test } from "bun:test"; +import { reviewFixture } from "./mcp-review-fixture.ts"; + +test("reads only the latest persisted draft and selected published revision", async () => { + const f = await reviewFixture(); + try { + const current = (await f.connections.get(f.connection.id))!; + const newer = await f.repository.saveDraft( + f.connection.id, + current.mcp!, + [{ upstreamName: "Read.New", description: "New", inputSchema: { type: "object" } }], + f.actorId, + new Date(f.now.getTime() + 1000), + ); + const draft = await f.repository.getLatestDraft(f.connection.id); + const published = await f.repository.getCatalog(f.connection.id, current.mcp!.catalogRevision); + expect(draft?.id).toBe(newer.id); + expect(draft?.tools[0]?.upstreamName).toBe("Read.New"); + expect(published?.id).toBe(f.catalog.id); + expect(published?.revision).toBe(1); + expect(await f.repository.getCatalog(f.connection.id, 2)).toBeNull(); + } finally { + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-child-bindings.test.ts b/orchestrator/src/__tests__/mcp-child-bindings.test.ts new file mode 100644 index 000000000..bb1f8575a --- /dev/null +++ b/orchestrator/src/__tests__/mcp-child-bindings.test.ts @@ -0,0 +1,86 @@ +import { expect, test } from "bun:test"; +import { + compileChildSessionBindings, + assertChildMcpCreationBlocked, +} from "../mcp/session-child.ts"; +import { compileSessionBindings } from "../mcp/session-bindings.ts"; +import { bindingAuthority, bindingGrant } from "./mcp-binding-fixture.ts"; +import { checkDb, getDb } from "../db/client.ts"; +import { task, taskSession } from "../db/schema.ts"; +import { eq } from "drizzle-orm"; + +test("child compiler reads actual spawning session and defaults to no MCP request", async () => { + const parentAuthority = bindingAuthority(); + parentAuthority.profileGrants[0]!.mcp = { + ...bindingGrant.mcp!, + unrestricted: false, + argumentConstraintDocuments: [ + { + $schema: "http://json-schema.org/draft-07/schema#", + definitions: { p: { const: "parent" } }, + properties: { project: { $ref: "#/definitions/p" } }, + }, + ], + }; + const parent = compileSessionBindings({ + launchPolicy: { + profileId: parentAuthority.profileId, + integrationGrants: parentAuthority.profileGrants, + }, + liveAuthority: parentAuthority, + bindingEpoch: 0, + }); + const child = bindingAuthority(); + child.profileId = "child-profile"; + let selected = ""; + const args = { + spawningSessionId: "actual-parent-session", + launchPolicy: { profileId: child.profileId, integrationGrants: child.profileGrants }, + liveAuthority: child, + bindingEpoch: 0, + readParentCurrent: async (id: string) => { + selected = id; + return parent; + }, + }; + expect((await compileChildSessionBindings(args)).immutableBindings).toEqual([]); + const explicit = await compileChildSessionBindings({ ...args, requestedGrants: [bindingGrant] }); + expect(selected).toBe("actual-parent-session"); + expect(explicit.immutableBindings[0]!.tools[0]!.grant.argumentConstraintDocuments).toEqual( + parentAuthority.profileGrants[0]!.mcp!.argumentConstraintDocuments, + ); + await expect( + compileChildSessionBindings({ + ...args, + readParentCurrent: async () => { + throw new Error("parent revoked"); + }, + }), + ).rejects.toThrow("parent revoked"); +}); +const reachable = process.env.ORCHESTRATOR_DATABASE_URL ? await checkDb() : false; +test.skipIf(!reachable)( + "existing child path checks actual session authority even when launch policy is empty", + async () => { + const db = getDb(), + tid = crypto.randomUUID(), + sid = crypto.randomUUID(); + try { + await db.insert(task).values({ id: tid, type: "chat" }); + await db + .insert(taskSession) + .values({ taskId: tid, sessionId: sid, integrationGrants: [bindingGrant] }); + await expect(assertChildMcpCreationBlocked(tid, sid, [], db)).rejects.toThrow( + "activation is not configured", + ); + await db + .update(taskSession) + .set({ integrationGrants: [] }) + .where(eq(taskSession.sessionId, sid)); + await assertChildMcpCreationBlocked(tid, sid, [], db); + await expect(assertChildMcpCreationBlocked(tid, "other-session", [], db)).rejects.toThrow(); + } finally { + await db.delete(task).where(eq(task.id, tid)); + } + }, +); diff --git a/orchestrator/src/__tests__/mcp-cold-fixture.ts b/orchestrator/src/__tests__/mcp-cold-fixture.ts new file mode 100644 index 000000000..eaa7d2779 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-cold-fixture.ts @@ -0,0 +1,388 @@ +import { create, type MessageInitShape } from "@bufbuild/protobuf"; +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { createHash } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { makeProfileStore } from "../db/profiles.ts"; +import { task, taskSession, user } from "../db/schema.ts"; +import { + SessionPolicyService, + ManagedPolicyPhase, + ManagedAppliedStateSchema, + ManagedPolicyTargetSchema, + ManagedDenyPolicySchema, + type ManagedAppliedState, +} from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import { bindingSnapshotColumns, readLiveBindingAuthority } from "../mcp/binding-store.ts"; +import { canonicalDigest, canonicalJson } from "../mcp/hash.ts"; +import { bytesDigest } from "../mcp/managed-create-input.ts"; +import { makeManagedLifecycleStore } from "../mcp/managed-lifecycle-store.ts"; +import type { ManagedLifecycle } from "../mcp/managed-lifecycle.ts"; +import { makeManagedPolicyAuthority, managedPolicy } from "../mcp/managed-policy.ts"; +import { makeManagedSessionRuntime } from "../mcp/managed-runtime.ts"; +import { makeManagedRebind } from "../mcp/managed-rebind.ts"; +import { managedQuery, type ManagedCreationState } from "../mcp/managed-state.ts"; +import { + checkBindingCurrent, + compileSessionBindings, + type McpOperatorPolicy, + type McpSessionBindingSnapshot, +} from "../mcp/session-bindings.ts"; +import { reviewFixture } from "./mcp-review-fixture.ts"; + +/** Real O stores and generated client handlers; placement/capture/host are explicit producer fixtures. */ +export async function coldFixture( + existing?: Awaited>, + suppliedPolicies?: McpOperatorPolicy[], + initialSnapshot?: McpSessionBindingSnapshot, +) { + const f = existing ?? (await reviewFixture()); + const profiles = makeProfileStore(f.db); + let currentProfile = (await profiles.get(f.profileId))!; + if (!currentProfile.mcpGrantPublication) { + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + await profiles.update(f.profileId, currentProfile, { actorId: f.actorId, expectedRevision: 0 }); + await f.db.update(user).set({ role: "user" }).where(eq(user.id, f.actorId)); + currentProfile = (await profiles.get(f.profileId))!; + } + let policies = structuredClone(suppliedPolicies ?? f.authority.operatorPolicies); + const authority = await readLiveBindingAuthority(f.profileId, policies, f.db); + const snapshot = + initialSnapshot ?? + compileSessionBindings({ + launchPolicy: { profileId: f.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 1, + }); + const [row] = await f.db.select().from(taskSession).where(eq(taskSession.sessionId, f.sessionId)); + const [owner] = await f.db.select().from(task).where(eq(task.id, f.taskId)); + const identity = { + ...(owner?.spawningSessionId ? { parentSessionId: owner.spawningSessionId } : {}), + sessionId: f.sessionId, + taskId: f.taskId, + inputId: f.sessionId, + actorId: f.actorId, + writeEligibility: "write_eligible", + inputDigest: `sha256:${"a".repeat(64)}`, + }; + const initial: ManagedLifecycle = { + version: 1, + nativeVersion: 1, + nativeDigest: bytesDigest(new TextEncoder().encode("[]")), + bindingHash: snapshot.bindingHash, + identity, + phase: "started", + }; + const publication = () => ({ + generation: policies[0]!.publication.generation, + identity: policies[0]!.publication.identity, + policies, + }); + function boot( + epoch: number, + deny: McpSessionBindingSnapshot["managedDenyPolicy"], + ): NonNullable { + const request = { + key: { + creation: managedQuery(initial).identity, + native_digest: initial.nativeDigest, + boot_attempt: crypto.randomUUID(), + context: { + target: { + session_id: f.sessionId, + sandbox_id: crypto.randomUUID(), + host_id: crypto.randomUUID(), + binding_epoch: epoch, + }, + operator_generation: publication().generation, + operator_identity: publication().identity, + binding_hash: null, + }, + }, + bootstrap_policy: deny, + }; + return { + request, + revision: 2, + phase: "prepared", + sealed_agent: null, + finalization: null, + retired: null, + prepared: { + request, + host_incarnation: "host-incarnation", + network_identity: `sha256:${"8".repeat(64)}`, + guest_ip: "192.0.2.7", + receipt_id: `sha256:${"e".repeat(64)}`, + }, + }; + } + const state: ManagedCreationState = { + query: managedQuery(initial), + state: "boot", + boot: boot(1, snapshot.managedDenyPolicy), + operation: null, + terminal_reason: null, + }; + state.boot!.phase = "started"; + initial.sourceKey = state.boot!.request.key; + await f.db + .update(taskSession) + .set({ + ...bindingSnapshotColumns(snapshot, row!.integrationConnections ?? []), + mcpManagedLifecycle: initial, + }) + .where(eq(taskSession.sessionId, f.sessionId)); + let retained = snapshot.managedDenyPolicy, + applied: ManagedAppliedState | undefined; + let native = "[]", + loseRecover = false, + loseFinalize = false; + let recoverCalls = 0, + applies = 0, + finalizes = 0, + hostReads = 0; + const recoveryIds = new Set(); + let beforeApply: () => Promise = async () => {}; + let beforeRecover: () => Promise = async () => {}; + let validateExtra: () => Promise = async () => {}; + let parentReader: (id: string) => Promise = async () => { + throw Error("unexpected parent"); + }; + const target = () => { + const t = state.boot!.request.key.context.target; + return create(ManagedPolicyTargetSchema, { + sessionId: t.session_id, + sandboxId: t.sandbox_id, + hostId: t.host_id, + bindingEpoch: BigInt(t.binding_epoch), + }); + }; + const receipt = ( + request: MessageInitShape["context"], + p: MessageInitShape, + ) => { + const policy = managedPolicy(create(ManagedDenyPolicySchema, p)); + const json = { + context: { + target: state.boot!.request.key.context.target, + operator_generation: publication().generation, + operator_identity: publication().identity, + binding_hash: request!.bindingHash ?? null, + }, + policy, + host_incarnation: "host-incarnation", + proxy_incarnation: "proxy-incarnation", + state_id: "", + network_proof: { + target: state.boot!.request.key.context.target, + guest_ip: "192.0.2.7", + network_identity: state.boot!.prepared!.network_identity, + }, + }; + return create(ManagedAppliedStateSchema, { + context: request, + policy: p, + hostIncarnation: json.host_incarnation, + proxyIncarnation: json.proxy_incarnation, + stateId: `sha256:${createHash("sha256").update(JSON.stringify(json)).digest("hex")}`, + networkProof: { + target: target(), + guestIp: "192.0.2.7", + networkIdentity: state.boot!.prepared!.network_identity, + }, + }); + }; + applied = receipt( + { + target: target(), + operatorGeneration: BigInt(publication().generation), + operatorIdentity: publication().identity, + phase: ManagedPolicyPhase.FINAL, + bindingHash: snapshot.bindingHash, + }, + { ...retained, revision: BigInt(retained.revision) }, + ); + const store = makeManagedLifecycleStore(f.db); + const control = createClient( + SessionPolicyService, + createRouterTransport((router) => + router.service(SessionPolicyService, { + async readNativeManifest() { + return { + protocol: 1, + identity, + nativeVersion: 1, + nativeDigest: initial.nativeDigest, + nativeBytes: new TextEncoder().encode(native), + }; + }, + async readManagedPolicy() { + hostReads++; + return { + protocol: 1, + retainedDesired: { + target: target(), + policy: { ...retained, revision: BigInt(retained.revision) }, + }, + applied, + }; + }, + async applyManagedPolicy(request) { + applies++; + await beforeApply(); + applied = receipt(request.context, request.policy!); + retained = managedPolicy(applied.policy!); + return { protocol: 1, applied }; + }, + async finalizeManagedBoot(request) { + finalizes++; + const input: { + native_version: 1; + combined_manifest: number[]; + combined_manifest_digest: string; + } = JSON.parse(new TextDecoder().decode(request.inputJson)); + state.boot!.finalization = { ...input, prepared: state.boot!.prepared! }; + state.boot!.phase = "finalizing"; + if (loseFinalize) { + loseFinalize = false; + throw Error("lost finalize reply"); + } + return { protocol: 1, bootJson: new TextEncoder().encode(JSON.stringify(state.boot)) }; + }, + }), + ), + ); + const encode = () => ({ + protocol: 1, + stateJson: new TextEncoder().encode(JSON.stringify(state)), + }); + const bridge = { + readManagedCreationState: async () => encode(), + prepareManagedBoot: async () => { + throw Error("unexpected initial prepare"); + }, + async recoverManagedBoot(input: { recoveryId: string; expectedSourceKeyJson: Uint8Array }) { + recoverCalls++; + await beforeRecover(); + if (!recoveryIds.has(input.recoveryId)) { + if ( + canonicalJson(JSON.parse(new TextDecoder().decode(input.expectedSourceKeyJson))) !== + canonicalJson(state.boot!.request.key) + ) + throw Error("source changed"); + recoveryIds.add(input.recoveryId); + const source = state.boot!.request.key; + state.boot = boot(source.context.target.binding_epoch + 1, retained); + applied = undefined; + state.operation = { + intent_id: crypto.randomUUID(), + revision: 1, + recovery_id: input.recoveryId, + source_key: source, + capture_id: crypto.randomUUID(), + completed: true, + }; + } + if (loseRecover) { + loseRecover = false; + throw Error("lost recovery reply"); + } + return encode(); + }, + }; + const managed = makeManagedPolicyAuthority({ + control, + readPublished: async () => publication(), + readPlacement: async () => { + const prepared = state.boot?.prepared; + if (!prepared) throw Error("MCP managed placement unavailable"); + return { + target: target(), + networkIdentity: prepared.network_identity, + guestIp: prepared.guest_ip, + hostIncarnation: prepared.host_incarnation, + }; + }, + }); + const makeRuntime = () => + makeManagedSessionRuntime({ + store, + control, + bridge, + managed, + readPublished: async () => publication(), + enabled: () => true, + authorize: async () => {}, + enroll: async () => {}, + async validate(candidate) { + if ( + !checkBindingCurrent({ + snapshot: candidate, + liveAuthority: await readLiveBindingAuthority(f.profileId, policies, f.db), + }).current + ) + throw Error("authority changed"); + await validateExtra(); + }, + }); + const lifecycle = makeRuntime(); + let clockOffset = 0; + const service = makeManagedRebind({ + db: f.db, + store, + lifecycle, + managed, + readOperatorPolicies: async () => policies, + readParentCurrent: (id) => parentReader(id), + now: () => new Date(Date.now() + clockOffset), + }); + return { + ...f, + snapshot, + store, + lifecycle, + service, + profiles, + state, + restartRunOnce: () => makeRuntime().runOnce(f.sessionId), + stats: () => ({ recoverCalls, recoveryIds: recoveryIds.size, applies, finalizes, hostReads }), + advanceColdNow: (offset: number) => { + clockOffset += offset; + }, + loseRecover: () => { + loseRecover = true; + }, + loseFinalize: () => { + loseFinalize = true; + }, + start: () => { + state.boot!.phase = "started"; + }, + setNative: (value: string) => { + native = value; + }, + beforeApply: (fn: () => Promise) => { + beforeApply = fn; + }, + beforeRecover: (fn: () => Promise) => { + beforeRecover = fn; + }, + validateExtra: (fn: () => Promise) => { + validateExtra = fn; + }, + parentReader: (fn: typeof parentReader) => { + parentReader = fn; + }, + setPolicies: (value: McpOperatorPolicy[]) => { + policies = value; + }, + addRetained(host: string) { + const body = { + version: 1 as const, + revision: retained.revision + 1, + hosts: [...new Set([...retained.hosts, host])].sort(), + }; + retained = { ...body, digest: canonicalDigest(body) }; + }, + }; +} diff --git a/orchestrator/src/__tests__/mcp-cold-rebind.test.ts b/orchestrator/src/__tests__/mcp-cold-rebind.test.ts new file mode 100644 index 000000000..64007d351 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-cold-rebind.test.ts @@ -0,0 +1,349 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { integrationConnection, profile, task, taskSession } from "../db/schema.ts"; +import type { ManagedLifecycle } from "../mcp/managed-lifecycle.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { McpRebindDraining } from "../mcp/rebind.ts"; +import { coldFixture } from "./mcp-cold-fixture.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +await verifyMcpTestDatabases(); + +const prepare = (f: Awaited>) => + f.service.prepare(f.sessionId, f.snapshot.bindingHash, f.actorId); +const commit = (f: Awaited>, hash: string) => + f.service.commit(f.sessionId, f.snapshot.bindingHash, hash, f.actorId); + +test("cold rebind persists intent before capture and opens only the exact finalized new boot", async () => { + const f = await coldFixture(); + try { + const candidate = await prepare(f); + f.beforeRecover(async () => { + f.beforeRecover(async () => {}); + const row = await f.store.read(f.sessionId); + expect(row.session.mcpManagedLifecycle?.phase).toBe("recovering"); + expect(row.session.mcpManagedLifecycle?.recovery?.candidateHash).toBe(candidate.bindingHash); + }); + f.beforeApply(async () => { + const row = await f.store.read(f.sessionId); + expect(row.session.mcpManagedLifecycle?.phase).toBe("finalizing"); + expect(row.session.mcpBindingEnvelope?.bindingEpoch).toBe(2); + expect(row.session.mcpManagedLifecycle?.pending?.combinedManifestDigest).toBeTruthy(); + }); + await expect(commit(f, candidate.bindingHash)).rejects.toBeInstanceOf(McpRebindDraining); + expect((await f.store.read(f.sessionId)).session.mcpManagedLifecycle?.phase).toBe("recovering"); + expect(f.stats().recoverCalls).toBe(0); + await f.restartRunOnce(); + const pending = (await f.store.read(f.sessionId)).session.mcpManagedLifecycle!; + expect(pending.phase).toBe("finalizing"); + expect(f.stats()).toMatchObject({ recoveryIds: 1, applies: 1, finalizes: 1 }); + f.start(); + await f.restartRunOnce(); + expect(await commit(f, candidate.bindingHash)).toEqual({ bindingHash: pending.bindingHash }); + expect((await f.store.read(f.sessionId)).session.mcpManagedLifecycle?.phase).toBe("started"); + expect(f.stats()).toMatchObject({ recoveryIds: 1, applies: 1, finalizes: 1 }); + } finally { + await f.cleanup(); + } +}); + +test("commit leaves recovery to the scanner and accepts only its exact completed intent", async () => { + const f = await coldFixture(); + const read = f.store.read; + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + let enter!: () => void; + const entered = new Promise((resolve) => { + enter = resolve; + }); + let scanner: Promise | undefined; + let armed = true; + let captures = 0; + try { + const candidate = await prepare(f); + f.beforeRecover(async () => { + if (++captures > 1) throw Error("concurrent recovery driver"); + enter(); + await blocked; + }); + f.store.read = async (id) => { + const row = await read(id); + if (armed && row.session.mcpManagedLifecycle?.phase === "recovering") { + armed = false; + scanner = f.lifecycle.runOnce(id); + await entered; + } + return row; + }; + await expect(commit(f, candidate.bindingHash)).rejects.toBeInstanceOf(McpRebindDraining); + expect((await read(f.sessionId)).session.mcpManagedLifecycle?.phase).toBe("recovering"); + expect(f.stats()).toMatchObject({ recoveryIds: 0, recoverCalls: 1 }); + release(); + await scanner; + f.beforeRecover(async () => {}); + f.start(); + await f.restartRunOnce(); + const finished = (await read(f.sessionId)).session.mcpManagedLifecycle!; + expect(finished.phase).toBe("started"); + expect(await commit(f, candidate.bindingHash)).toEqual({ bindingHash: finished.bindingHash }); + await expect(commit(f, "wrong")).rejects.toThrow(); + await expect( + f.service.commit( + f.sessionId, + f.snapshot.bindingHash, + candidate.bindingHash, + crypto.randomUUID(), + ), + ).rejects.toThrow(); + expect(f.stats()).toMatchObject({ recoveryIds: 1, applies: 1, finalizes: 1 }); + } finally { + release(); + await scanner; + f.store.read = read; + await f.cleanup(); + } +}); + +const changedRecoveries: Array<{ + name: string; + replace(value: ManagedLifecycle): ManagedLifecycle; +}> = [ + { + name: "missing intent", + replace: ({ recovery: _recovery, ...value }) => ({ ...value, phase: "started" }), + }, + { name: "terminal", replace: (value) => ({ ...value, phase: "terminal" }) }, + { name: "held", replace: (value) => ({ ...value, phase: "held" }) }, + { + name: "replaced pending intent", + replace: (value) => ({ ...value, recovery: { ...value.recovery!, id: crypto.randomUUID() } }), + }, + { + name: "replaced started intent", + replace: (value) => ({ + ...value, + phase: "started", + recovery: { ...value.recovery!, id: crypto.randomUUID() }, + }), + }, + { + name: "changed actor", + replace: (value) => ({ + ...value, + recovery: { ...value.recovery!, actorId: crypto.randomUUID() }, + }), + }, + { + name: "changed source binding", + replace: (value) => ({ + ...value, + recovery: { ...value.recovery!, expectedBindingHash: `sha256:${"f".repeat(64)}` }, + }), + }, + { + name: "changed reviewed candidate", + replace: (value) => { + const { bindingHash: _hash, ...body } = value.recovery!.candidate; + const changed = { ...body, profileId: crypto.randomUUID() }; + const candidate = { ...changed, bindingHash: canonicalDigest(changed) }; + return { + ...value, + recovery: { ...value.recovery!, candidate, candidateHash: candidate.bindingHash }, + }; + }, + }, +]; +for (const change of changedRecoveries) { + test(`commit rejects a ${change.name} recovery after its authority check`, async () => { + const f = await coldFixture(); + const read = f.store.read; + try { + const candidate = await prepare(f); + await expect(commit(f, candidate.bindingHash)).rejects.toBeInstanceOf(McpRebindDraining); + f.store.read = async (id) => { + const row = await read(id); + await f.db + .update(taskSession) + .set({ mcpManagedLifecycle: change.replace(row.session.mcpManagedLifecycle!) }) + .where(eq(taskSession.sessionId, id)); + return read(id); + }; + await expect(commit(f, candidate.bindingHash)).rejects.toThrow( + "MCP rebind reservation changed", + ); + } finally { + f.store.read = read; + await f.cleanup(); + } + }); +} + +test("lost recovery and finalization replies retain one recovery and one final manifest", async () => { + const f = await coldFixture(); + try { + const candidate = await prepare(f); + await expect(commit(f, candidate.bindingHash)).rejects.toBeInstanceOf(McpRebindDraining); + f.loseRecover(); + await expect(f.restartRunOnce()).rejects.toThrow("lost recovery"); + expect((await f.store.read(f.sessionId)).session.mcpManagedLifecycle?.phase).toBe("recovering"); + f.loseFinalize(); + await expect(f.restartRunOnce()).rejects.toThrow("internal"); + f.start(); + await f.restartRunOnce(); + await commit(f, candidate.bindingHash); + expect(f.stats()).toMatchObject({ recoveryIds: 1, applies: 1, finalizes: 1 }); + } finally { + await f.cleanup(); + } +}); + +test("stale review, owner, connection and publication fail before capture", async () => { + const f = await coldFixture(); + try { + const candidate = await prepare(f); + await expect(commit(f, "wrong")).rejects.toThrow("stale"); + await f.db.update(task).set({ createdByUserId: null }).where(eq(task.id, f.taskId)); + await expect(commit(f, candidate.bindingHash)).rejects.toThrow(); + await f.db.update(task).set({ createdByUserId: f.actorId }).where(eq(task.id, f.taskId)); + await f.db + .update(integrationConnection) + .set({ enabled: false }) + .where(eq(integrationConnection.id, f.connection.id)); + await expect(commit(f, candidate.bindingHash)).rejects.toThrow(); + await f.db + .update(integrationConnection) + .set({ enabled: true }) + .where(eq(integrationConnection.id, f.connection.id)); + await f.db + .update(profile) + .set({ mcpGrantPublication: null }) + .where(eq(profile.id, f.profileId)); + await expect(commit(f, candidate.bindingHash)).rejects.toThrow(); + expect(f.stats().recoverCalls).toBe(0); + } finally { + await f.cleanup(); + } +}); + +test("cold rebind rejects changed native bytes and source placement before capture", async () => { + const f = await coldFixture(); + try { + const candidate = await prepare(f); + f.setNative("[{}"); + await expect(commit(f, candidate.bindingHash)).rejects.toThrow(); + f.setNative("[]"); + f.state.boot!.request.key.context.target.sandbox_id = crypto.randomUUID(); + await expect(commit(f, candidate.bindingHash)).rejects.toThrow("source changed"); + expect(f.stats().recoverCalls).toBe(0); + } finally { + await f.cleanup(); + } +}); + +test("cold rebind preserves newer retained denies and stops on authority change during apply", async () => { + const f = await coldFixture(); + try { + f.addRetained("retained.example.test"); + const candidate = await prepare(f); + expect(candidate.managedDenyPolicy.hosts).toContain("retained.example.test"); + f.beforeApply(async () => { + await f.db + .update(profile) + .set({ mcpGrantPublication: null }) + .where(eq(profile.id, f.profileId)); + }); + await expect(commit(f, candidate.bindingHash)).rejects.toBeInstanceOf(McpRebindDraining); + await expect(f.restartRunOnce()).rejects.toThrow("authority changed"); + expect(f.stats().finalizes).toBe(0); + const row = await f.store.read(f.sessionId); + expect(row.session.mcpManagedLifecycle?.phase).toBe("finalizing"); + expect(row.session.mcpManagedLifecycle?.pending?.snapshot.managedDenyPolicy.hosts).toContain( + "retained.example.test", + ); + } finally { + await f.cleanup(); + } +}); + +test("cold host calls do not hold session or owner database locks", async () => { + const f = await coldFixture(); + try { + const candidate = await prepare(f); + f.beforeRecover(async () => { + await f.db.transaction(async (tx) => { + await tx.execute("SET LOCAL lock_timeout = '500ms'"); + await tx.update(task).set({ title: "Unrelated local update" }).where(eq(task.id, f.taskId)); + await tx + .update(taskSession) + .set({ role: "primary" }) + .where(eq(taskSession.sessionId, f.sessionId)); + }); + }); + await expect(commit(f, candidate.bindingHash)).rejects.toBeInstanceOf(McpRebindDraining); + await f.restartRunOnce(); + expect(f.stats().recoveryIds).toBe(1); + } finally { + await f.cleanup(); + } +}); + +test("retired source is deny evidence only and the new boot retains later desired hosts", async () => { + const f = await coldFixture(); + try { + const boot = f.state.boot!; + boot.phase = "interrupted"; + boot.retired = { + prepared: boot.prepared!, + retained_policy: f.snapshot.managedDenyPolicy, + receipt_id: `sha256:${"f".repeat(64)}`, + }; + const candidate = await prepare(f); + expect(f.stats().hostReads).toBe(0); + f.addRetained("later.example.test"); + await expect(commit(f, candidate.bindingHash)).rejects.toBeInstanceOf(McpRebindDraining); + await f.restartRunOnce(); + expect(f.stats().applies).toBe(1); + const value = (await f.store.read(f.sessionId)).session.mcpManagedLifecycle!; + expect(value.pending!.snapshot.managedDenyPolicy.hosts).toContain("later.example.test"); + expect(value.phase).toBe("finalizing"); + f.start(); + await f.restartRunOnce(); + expect((await f.store.read(f.sessionId)).session.mcpManagedLifecycle?.phase).toBe("started"); + expect(await commit(f, candidate.bindingHash)).toEqual({ bindingHash: value.bindingHash }); + } finally { + await f.cleanup(); + } +}); + +test("sanitized state accepts the bounded JSON expansion of a maximum-size final manifest", async () => { + const { decodeManagedState } = await import("../mcp/managed-state.ts"); + const { bytesDigest } = await import("../mcp/managed-create-input.ts"); + const f = await coldFixture(); + try { + const prefix = '[{"name":"native","description":"', + suffix = '","inputSchema":{},"execution":"sync"}]'; + const bytes = new TextEncoder().encode( + prefix + "x".repeat(1024 * 1024 - prefix.length - suffix.length) + suffix, + ); + expect(bytes.length).toBe(1024 * 1024); + f.state.boot!.finalization = { + prepared: f.state.boot!.prepared!, + native_version: 1, + combined_manifest: [...bytes], + combined_manifest_digest: bytesDigest(bytes), + }; + const stateJson = new TextEncoder().encode(JSON.stringify(f.state)); + expect(stateJson.length).toBeGreaterThan(4 * 1024 * 1024); + const value = (await f.store.read(f.sessionId)).session.mcpManagedLifecycle!; + expect( + decodeManagedState({ protocol: 1, stateJson }, value).boot!.finalization! + .combined_manifest_digest, + ).toBe(bytesDigest(bytes)); + expect(() => + decodeManagedState({ protocol: 1, stateJson: new Uint8Array(8 * 1024 * 1024 + 1) }, value), + ).toThrow("unavailable"); + } finally { + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-control.test.ts b/orchestrator/src/__tests__/mcp-control.test.ts new file mode 100644 index 000000000..27fc34090 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-control.test.ts @@ -0,0 +1,167 @@ +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { expect, test } from "bun:test"; +import { createSecureServer } from "node:http2"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { X509Certificate } from "node:crypto"; +import { connectNodeAdapter } from "@connectrpc/connect-node"; +import { CredentialControlService } from "../gen/engram/mcp/v1/control_pb.ts"; +import { + makeCredentialControlClient, + unwrapCredentialControlClient, + makeSessionPolicyControlClient, +} from "../mcp/control-client.ts"; + +import { SessionPolicyService } from "../gen/engram/mcp/v1/session_policy_pb.ts"; + +function certificates() { + const dir = mkdtempSync(join(tmpdir(), "mcp-control-test-")); + try { + execFileSync( + "openssl", + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + join(dir, "key"), + "-out", + join(dir, "cert"), + "-days", + "1", + "-subj", + "/CN=localhost", + "-addext", + "subjectAltName=DNS:localhost,IP:127.0.0.1", + "-addext", + "extendedKeyUsage=serverAuth,clientAuth", + ], + { stdio: "ignore" }, + ); + const cert = readFileSync(join(dir, "cert"), "utf8"); + return { + cert, + key: readFileSync(join(dir, "key"), "utf8"), + ca: cert, + serverFingerprint: new X509Certificate(cert).fingerprint256, + }; + } finally { + rmSync(dir, { recursive: true }); + } +} +test("control client requires pinned mTLS and preserves int64 metadata", async () => { + const tls = certificates(); + let calls = 0; + const server = createSecureServer( + { key: tls.key, cert: tls.cert, ca: tls.ca, requestCert: true, rejectUnauthorized: true }, + connectNodeAdapter({ + routes(router) { + router.service(SessionPolicyService, { + async readWritePolicy(req) { + return { + state: { + policy: { + ref: { + sessionId: req.sessionId, + policyRevision: 9007199254740993n, + bindingEpoch: 9007199254740995n, + }, + writeEligibility: "read_only", + }, + }, + }; + }, + }); + router.service(CredentialControlService, { + async getStatus(req) { + calls++; + expect(req.protocol).toBe(1); + return { status: { protocol: 1, version: 9007199254740993n, status: "connected" } }; + }, + }); + }, + }), + ); + const sessions = new Set(); + server.on("session", (s) => { + sessions.add(s); + s.on("close", () => sessions.delete(s)); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("address"); + const url = `https://127.0.0.1:${address.port}`; + try { + const good = makeCredentialControlClient({ url, ...tls }); + expect((await good.getStatus({ protocol: 1, connectionId: "c" })).version).toBe( + 9007199254740993n, + ); + const policy = makeSessionPolicyControlClient({ url, ...tls }); + expect( + (await policy.readWritePolicy({ protocol: 1, sessionId: "s" })).state?.policy?.ref + ?.policyRevision, + ).toBe(9007199254740993n); + const badPolicy = makeSessionPolicyControlClient({ + url, + ...tls, + serverFingerprint: "00:".repeat(31) + "00", + }); + await expect(badPolicy.readWritePolicy({ protocol: 1 })).rejects.toThrow("control unavailable"); + const bad = makeCredentialControlClient({ + url, + ...tls, + serverFingerprint: "00:".repeat(31) + "00", + }); + await expect(bad.getStatus({ protocol: 1 })).rejects.toThrow("credential control unavailable"); + const other = certificates(); + const untrusted = makeCredentialControlClient({ + url, + ...tls, + key: other.key, + cert: other.cert, + }); + await expect(untrusted.getStatus({ protocol: 1 })).rejects.toThrow( + "credential control unavailable", + ); + expect(calls).toBe(1); + expect(() => + makeCredentialControlClient({ url: url.replace("https:", "http:"), ...tls }), + ).toThrow(); + expect(() => makeCredentialControlClient({ url, ...tls, key: "" })).toThrow(); + } finally { + for (const s of sessions) s.destroy(); + await new Promise((r) => server.close(() => r())); + } +}); + +test("credential response wrappers fail closed when status is absent", async () => { + const client = unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => + router.service(CredentialControlService, { + async putSecret() { + return {}; + }, + async completeAuthorization() { + return {}; + }, + async getStatus() { + return {}; + }, + async revoke() { + return {}; + }, + }), + ), + ), + ); + await expect(client.putSecret({})).rejects.toThrow("status unavailable"); + await expect(client.completeAuthorization({})).rejects.toThrow("status unavailable"); + await expect(client.getStatus({})).rejects.toThrow("status unavailable"); + await expect(client.revoke({})).rejects.toThrow("status unavailable"); +}); diff --git a/orchestrator/src/__tests__/mcp-credential-admin.test.ts b/orchestrator/src/__tests__/mcp-credential-admin.test.ts new file mode 100644 index 000000000..a3094c90f --- /dev/null +++ b/orchestrator/src/__tests__/mcp-credential-admin.test.ts @@ -0,0 +1,606 @@ +import { unwrapCredentialControlClient } from "../mcp/control-client.ts"; +import { expect, test } from "bun:test"; +import { Code, ConnectError, createClient, createRouterTransport } from "@connectrpc/connect"; +import { create, toJson } from "@bufbuild/protobuf"; +import { checkDb, getDb } from "../db/client.ts"; +import { eq, sql } from "drizzle-orm"; +import { integrationConnection, mcpCatalog } from "../db/schema.ts"; +import { makeIntegrationConnectionStore } from "../db/integration-connections.ts"; +import { makeCredentialMetadataStore } from "../mcp/credential-store.ts"; +import { makeCredentialAdmin } from "../mcp/credential-admin.ts"; +import { + CredentialControlService, + CredentialKind, + CredentialStatusSchema, + type CredentialStatus, +} from "../gen/engram/mcp/v1/control_pb.ts"; +const reachable = process.env.ORCHESTRATOR_DATABASE_URL ? await checkDb() : false; +const config = { + endpoint: "https://tools.example.test/mcp", + zone: "public", + auth: { kind: "api_key" as const, secretRef: "" }, +}; +async function fixture() { + const connections = makeIntegrationConnectionStore(), + store = makeCredentialMetadataStore(); + const row = await connections.create({ + alias: `mcp-${crypto.randomUUID()}`, + displayName: "Tools", + provider: "mcp", + config, + }); + let current: CredentialStatus | undefined; + let failRevoke = false, + lostRevoke = false; + const calls: string[] = []; + const control = unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => + router.service(CredentialControlService, { + async putSecret(req) { + calls.push("put"); + current = create(CredentialStatusSchema, { + protocol: 1, + context: { + connectionId: req.context!.connectionId, + configRevision: req.context!.configRevision, + resource: req.context!.resource, + authorizationGeneration: req.context!.previousAuthorizationGeneration + 1n, + credentialIdentity: `identity-${calls.length}`, + policyDigest: "a".repeat(64), + policyRevision: "v1", + }, + reference: { + kind: req.kind, + name: req.expected?.name || "mcp.fresh", + connectionId: row.id, + generation: (req.expected?.generation ?? 0n) + 1n, + }, + version: (req.expected?.generation ?? 0n) + 1n, + status: "connected", + }); + return { status: current }; + }, + async getStatus() { + if (!current) throw new Error("absent"); + return { status: current }; + }, + async revoke() { + calls.push("revoke"); + if (failRevoke) throw new Error("refused"); + current = create(CredentialStatusSchema, { + ...current!, + context: { + ...current!.context!, + authorizationGeneration: current!.context!.authorizationGeneration + 1n, + credentialIdentity: "revoked-identity", + }, + reference: { + ...current!.reference!, + generation: current!.reference!.generation + 1n, + }, + version: current!.version + 1n, + status: "revoked", + }); + if (lostRevoke) throw new Error("lost reply"); + return { status: current }; + }, + }), + ), + ), + ); + let now = new Date(); + const admin = makeCredentialAdmin({ store, control, registrations: [], now: () => now }); + return { + connections, + store, + row, + admin, + calls, + advance(milliseconds: number) { + now = new Date(now.getTime() + milliseconds); + }, + setFailure(value = true) { + failRevoke = value; + }, + setLost() { + lostRevoke = true; + }, + async cleanup() { + await getDb().delete(mcpCatalog).where(eq(mcpCatalog.connectionId, row.id)); + await connections.delete(row.id); + }, + }; +} +test.skipIf(!reachable)( + "fresh ingestion publishes generated reference at sealed revision and never stores entered value", + async () => { + const f = await fixture(); + try { + await f.admin.putSecret( + f.row.id, + 1, + 1, + "admin", + CredentialKind.API_KEY, + "x-api-key", + "entered-secret", + ); + const row = await f.store.get(f.row.id); + expect(row.config).toEqual({ ...config, auth: { kind: "api_key", secretRef: "mcp.fresh" } }); + expect(row.configRevision).toBe(2); + expect(row.authGeneration).toBe(3); + expect(JSON.stringify(row)).not.toContain("entered-secret"); + await expect( + f.admin.putSecret(f.row.id, 1, 1, "admin", CredentialKind.API_KEY, "x-api-key", "next"), + ).rejects.toThrow("stale"); + expect(f.calls).toEqual(["put"]); + } finally { + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "failed revocation prevents replacement and leaves old binding disabled", + async () => { + const f = await fixture(); + try { + await f.admin.putSecret( + f.row.id, + 1, + 1, + "admin", + CredentialKind.API_KEY, + "x-api-key", + "first", + ); + f.setFailure(); + await expect( + f.admin.putSecret(f.row.id, 2, 3, "admin", CredentialKind.API_KEY, "x-api-key", "second"), + ).rejects.toThrow(); + expect(f.calls).toEqual(["put", "revoke"]); + const row = await f.store.get(f.row.id); + expect(row.revoked).toBe(true); + expect(row.mcpCredential.transition).toBeDefined(); + } finally { + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "lost revoke reply is recovered from receipt without replay before rotation", + async () => { + const f = await fixture(); + try { + await f.admin.putSecret( + f.row.id, + 1, + 1, + "admin", + CredentialKind.API_KEY, + "x-api-key", + "first", + ); + f.setLost(); + await f.admin.putSecret( + f.row.id, + 2, + 3, + "admin", + CredentialKind.API_KEY, + "x-api-key", + "second", + ); + expect(f.calls).toEqual(["put", "revoke", "put"]); + const row = await f.store.get(f.row.id); + expect(row.revoked).toBe(false); + expect(row.authGeneration).toBe(5); + } finally { + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "same-reference rotation changes actual identity and preserves published context", + async () => { + const f = await fixture(); + try { + await f.admin.putSecret( + f.row.id, + 1, + 1, + "admin", + CredentialKind.API_KEY, + "x-api-key", + "first", + ); + const first = await f.store.get(f.row.id); + await f.admin.putSecret( + f.row.id, + 2, + 3, + "admin", + CredentialKind.API_KEY, + "x-api-key", + "second", + ); + const next = await f.store.get(f.row.id); + expect(next.config).toEqual(first.config); + expect(next.credentialIdentity).not.toBe(first.credentialIdentity); + expect(next.authGeneration).toBe(5); + expect(next.configRevision).toBe(3); + expect(f.calls).toEqual(["put", "revoke", "put"]); + } finally { + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "public credential response contains status and references but no entered value", + async () => { + const f = await fixture(); + try { + const { credentialHandlers } = await import("../mcp/credential-rpc.ts"); + const { makeMcpRepository } = await import("../mcp/repository.ts"); + const { makeMcpJournal } = await import("../mcp/journal.ts"); + const { IntegrationService } = await import("../gen/engram/app/v1/integration_pb.ts"); + const client = createClient( + IntegrationService, + createRouterTransport((router) => + router.service( + IntegrationService, + credentialHandlers({ + services: { + enabled: true, + credentials: f.admin, + repository: makeMcpRepository(), + journal: makeMcpJournal(), + }, + connections: f.connections, + getSession: async () => ({ user: { id: "admin", role: "admin" } }), + connectionToProto: (row) => ({ id: row.id }), + }), + ), + ), + ); + const response = await client.putMcpCredential({ + connectionId: f.row.id, + expectedRevision: 1, + expectedGeneration: 1, + kind: "api_key", + header: "x-api-key", + value: "browser-entered-secret", + }); + expect(response.credential!.reference).toBe("mcp.fresh"); + expect(response.credential!.status).toBe("connected"); + expect(response.credential).not.toHaveProperty("value"); + expect(response.credential).not.toHaveProperty("header"); + const setup = await client.getMcpSetup({ connectionId: f.row.id }); + expect(setup.current!.credential!.reference).toBe("mcp.fresh"); + expect(JSON.stringify(await f.store.get(f.row.id))).not.toContain("browser-entered-secret"); + } finally { + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "named connection CRUD admits an unconfigured draft and refuses browser reference injection", + async () => { + const f = await fixture(); + let createdId: string | undefined; + try { + const { registerIntegration } = await import("../rpc/integration.ts"); + const { makeMcpRepository } = await import("../mcp/repository.ts"); + const { makeMcpJournal } = await import("../mcp/journal.ts"); + const { IntegrationService } = await import("../gen/engram/app/v1/integration_pb.ts"); + const client = createClient( + IntegrationService, + createRouterTransport((router) => + registerIntegration(router, { + getSession: async () => ({ user: { id: "admin", role: "admin" } }), + connections: f.connections, + mcp: { + enabled: true, + credentials: f.admin, + repository: makeMcpRepository(), + journal: makeMcpJournal(), + validateDestination: async () => {}, + }, + }), + ), + ); + const input = { + alias: `mcp-${crypto.randomUUID()}`, + displayName: "Tools", + provider: "mcp", + mcp: { endpoint: config.endpoint, zone: config.zone, authKind: "api_key" }, + }; + await expect( + client.createConnection({ ...input, mcp: { ...input.mcp, apiKeySecretRef: "mcp.stolen" } }), + ).rejects.toThrow("fresh credential"); + const created = await client.createConnection(input); + createdId = created.connection!.id; + expect(created.connection!.enabled).toBe(false); + expect(created.connection!.credentialIdentity).toBe(""); + expect(created.connection!.mcp!.apiKeySecretRef).toBe(""); + await expect( + client.updateConnection({ + id: createdId, + alias: input.alias, + displayName: "Tools", + expectedRevision: 1, + expectedGeneration: 1, + mcp: { ...input.mcp, apiKeySecretRef: "mcp.stolen" }, + }), + ).rejects.toThrow("references cannot"); + expect((await f.store.get(createdId)).configRevision).toBe(1); + await expect( + client.discoverMcp({ connectionId: createdId, expectedRevision: 1 }), + ).rejects.toThrow(); + } finally { + if (createdId) await f.connections.delete(createdId); + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "disabled dispatch preserves registered credential recovery and disconnect with exact admin context", + async () => { + const f = await fixture(); + try { + const { credentialHandlers } = await import("../mcp/credential-rpc.ts"); + const { makeMcpRepository } = await import("../mcp/repository.ts"); + const { makeMcpJournal } = await import("../mcp/journal.ts"); + const { IntegrationService, GetMcpSetupResponseSchema } = + await import("../gen/engram/app/v1/integration_pb.ts"); + await f.admin.putSecret( + f.row.id, + 1, + 1, + "admin", + CredentialKind.API_KEY, + "x-api-key", + "entered-secret", + ); + f.setFailure(); + await expect(f.admin.disconnect(f.row.id, 2, 3, "admin")).rejects.toThrow(); + f.advance(35001); + const pending = await f.store.get(f.row.id); + const services = { + enabled: false, + credentials: f.admin, + repository: makeMcpRepository(), + journal: makeMcpJournal(), + }; + let email = "admin@example.test"; + const client = createClient( + IntegrationService, + createRouterTransport((router) => + router.service( + IntegrationService, + credentialHandlers({ + services, + connections: f.connections, + getSession: async () => ({ user: { id: "admin", role: "admin", email } }), + connectionToProto: (row) => ({ id: row.id }), + }), + ), + ), + ); + const setup = await client.getMcpSetup({ connectionId: f.row.id }); + expect(setup.enabled).toBe(false); + expect(setup.recoveryAvailable).toBe(true); + expect(setup.transitionPending).toBe(true); + expect(JSON.stringify(toJson(GetMcpSetupResponseSchema, setup))).not.toContain( + "entered-secret", + ); + const request = { + connectionId: f.row.id, + expectedRevision: pending.configRevision, + expectedGeneration: pending.authGeneration, + }; + await expect( + client.putMcpCredential({ + ...request, + kind: "api_key", + header: "x-api-key", + value: "forbidden", + }), + ).rejects.toThrow("disabled"); + await expect(client.authorizeMcp(request)).rejects.toThrow("disabled"); + await expect(client.completeMcpAuthorization(request)).rejects.toThrow("disabled"); + email = "apikey+agent@service.local"; + await expect(client.recoverMcpCredential(request)).rejects.toThrow("human administrator"); + await expect(client.disconnectMcp(request)).rejects.toThrow("human administrator"); + email = "admin@example.test"; + await expect( + client.recoverMcpCredential({ + ...request, + expectedGeneration: request.expectedGeneration + 1, + }), + ).rejects.toThrow("state changed"); + await expect(client.recoverMcpCredential(request)).rejects.toThrow("state changed"); + expect((await f.store.get(f.row.id)).mcpCredential.transition).toBeDefined(); + f.setFailure(false); + f.advance(35001); + const recovered = await client.recoverMcpCredential(request); + expect(recovered.credential!.status).toBe("disconnected"); + const ready = await f.store.get(f.row.id); + expect(ready.mcpCredential.transition).toBeUndefined(); + await client.disconnectMcp({ + connectionId: f.row.id, + expectedRevision: ready.configRevision, + expectedGeneration: ready.authGeneration, + }); + expect(f.calls.filter((x) => x === "put")).toHaveLength(1); + } finally { + await f.cleanup(); + } + }, +); + +for (const discovered of [true, false]) { + test.skipIf(!reachable)( + discovered + ? "public removal preserves a discovered connection and its credential authority" + : "public removal deletes an unused connection after credential revocation", + async () => { + const f = await fixture(); + try { + const { registerIntegration } = await import("../rpc/integration.ts"); + const { makeMcpRepository } = await import("../mcp/repository.ts"); + const { makeMcpJournal } = await import("../mcp/journal.ts"); + const { IntegrationService } = await import("../gen/engram/app/v1/integration_pb.ts"); + const client = createClient( + IntegrationService, + createRouterTransport((router) => + registerIntegration(router, { + getSession: async () => ({ user: { id: "admin", role: "admin" } }), + connections: f.connections, + mcp: { + enabled: true, + credentials: f.admin, + repository: makeMcpRepository(), + journal: makeMcpJournal(), + discovery: { + discover: async () => [ + { + upstreamName: "Read.Record", + description: "Read a record", + inputSchema: { type: "object" }, + }, + ], + }, + }, + }), + ), + ); + await client.putMcpCredential({ + connectionId: f.row.id, + expectedRevision: 1, + expectedGeneration: 1, + kind: "api_key", + header: "x-api-key", + value: "test-secret", + }); + const before = await f.store.get(f.row.id); + const catalog = discovered + ? (await client.discoverMcp({ connectionId: f.row.id, expectedRevision: 2 })).catalog + : undefined; + let error: unknown; + let deleted: boolean | undefined; + try { + deleted = ( + await client.deleteConnection({ + id: f.row.id, + expectedRevision: before.configRevision, + expectedGeneration: before.authGeneration, + }) + ).deleted; + } catch (caught) { + error = caught; + } + if (discovered) { + // Refusal must precede both durable transition and external revocation. + expect(await f.store.get(f.row.id)).toEqual(before); + expect(f.calls).toEqual(["put"]); + expect((await f.admin.status(f.row.id)).status?.status).toBe("connected"); + expect(error).toBeInstanceOf(ConnectError); + expect((error as ConnectError).code).toBe(Code.FailedPrecondition); + expect((error as ConnectError).rawMessage).toContain("disable or disconnect"); + expect((await client.getMcpCatalogs({ connectionId: f.row.id })).draft?.id).toBe( + catalog!.id, + ); + } else { + expect(error).toBeUndefined(); + expect(deleted).toBe(true); + expect(await f.connections.get(f.row.id)).toBeNull(); + expect(f.calls).toEqual(["put", "revoke"]); + } + } finally { + await f.cleanup(); + } + }, + ); +} + +test.skipIf(!reachable)( + "removal checks history after a concurrent catalog writer commits", + async () => { + const f = await fixture(); + const db = getDb(); + const locked = Promise.withResolvers(); + const release = Promise.withResolvers(); + let writer: Promise | undefined; + let removal: Promise | undefined; + try { + await f.admin.putSecret( + f.row.id, + 1, + 1, + "admin", + CredentialKind.API_KEY, + "x-api-key", + "test-secret", + ); + const before = await f.store.get(f.row.id); + writer = db.transaction(async (tx) => { + await tx + .select() + .from(integrationConnection) + .where(eq(integrationConnection.id, f.row.id)) + .for("update"); + const pid = await tx.execute<{ pid: number }>(sql`SELECT pg_backend_pid() AS pid`); + locked.resolve(pid.rows[0]!.pid); + await release.promise; + // A catalog producer commits its history while it owns the connection lock. + await tx.insert(mcpCatalog).values({ + id: crypto.randomUUID(), + connectionId: f.row.id, + kind: "draft", + revision: 0, + configRevision: before.configRevision, + authGeneration: before.authGeneration, + credentialIdentity: before.credentialIdentity, + tools: [], + fingerprint: "fixture", + actorId: "admin", + createdAt: new Date(), + }); + }); + const blocker = await locked.promise; + removal = f.admin + .remove(f.row.id, before.configRevision, before.authGeneration, "admin") + .then( + () => undefined, + (error: unknown) => error, + ); + const deadline = Date.now() + 1000; + let waiting = false; + while (Date.now() < deadline) { + const result = await db.execute<{ waiting: boolean }>(sql` + SELECT EXISTS (SELECT 1 FROM pg_stat_activity + WHERE ${blocker} = ANY(pg_blocking_pids(pid))) AS waiting`); + if (result.rows[0]?.waiting) { + waiting = true; + break; + } + await Bun.sleep(5); + } + expect(waiting).toBe(true); + release.resolve(); + await writer; + expect(await removal).toBeInstanceOf(ConnectError); + expect(await f.store.get(f.row.id)).toEqual(before); + expect(f.calls).toEqual(["put"]); + } finally { + release.resolve(); + await writer; + await removal; + await f.cleanup(); + } + }, +); diff --git a/orchestrator/src/__tests__/mcp-credential-broker.test.ts b/orchestrator/src/__tests__/mcp-credential-broker.test.ts new file mode 100644 index 000000000..523e6e6c5 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-credential-broker.test.ts @@ -0,0 +1,402 @@ +import { unwrapCredentialControlClient } from "../mcp/control-client.ts"; +import { expect, test } from "bun:test"; +import { Code, ConnectError, createClient, createRouterTransport } from "@connectrpc/connect"; +import { create } from "@bufbuild/protobuf"; +import { + CredentialControlService, + CredentialKind, + CredentialStatusSchema, +} from "../gen/engram/mcp/v1/control_pb.ts"; +import { + makeCredentialBroker, + type AdmittedCredentialOperation, +} from "../mcp/credential-broker.ts"; +import type { WorkerRequest } from "../../../mcp-worker/src/contracts.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +const now = 100000; +const identity = { + protocol: 1 as const, + bootId: "boot", + zoneId: "public", + policyDigest: "a".repeat(64), + policyRevision: "v1", +}; +const status = create(CredentialStatusSchema, { + protocol: 1, + context: { + connectionId: "connection", + configRevision: 2n, + authorizationGeneration: 3n, + credentialIdentity: "identity", + resource: { zoneId: "public", endpoint: "https://tools.example.test/mcp" }, + policyDigest: identity.policyDigest, + policyRevision: "v1", + }, + reference: { + kind: CredentialKind.API_KEY, + name: "mcp-key", + connectionId: "connection", + generation: 1n, + }, + status: "connected", + version: 1n, +}); +function fixture(expire = false, finalFenceMs = 0, responseDigest?: string) { + let time = now, + deliveries = 0, + leases = 0; + const seen: unknown[] = []; + const admitted: AdmittedCredentialOperation[] = []; + const requests: WorkerRequest[] = []; + const control = unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => + router.service(CredentialControlService, { + async lease(req) { + leases++; + seen.push(req); + // Match McpAuthority::lease: exactly 64 ASCII hexadecimal characters. + if (req.requestDigest.length !== 64 || !/^[0-9a-fA-F]{64}$/.test(req.requestDigest)) + throw new ConnectError("invalid request digest", Code.PermissionDenied); + if (expire) time = now + 10000; + return { + protocol: 1, + header: "x-api-key", + value: "transient-value", + expiresAt: req.expiresAt, + operationExpiresAt: req.operationExpiresAt, + intentId: req.intentId, + }; + }, + }), + ), + ), + ); + const broker = makeCredentialBroker({ + control, + now: () => time, + async beforeDispatch() { + time += finalFenceMs; + }, + admission: { + async assertAdmitted(operation) { + admitted.push(structuredClone(operation)); + }, + }, + worker: { + async identity() { + return identity; + }, + async dispatch(request) { + deliveries++; + requests.push(structuredClone(request)); + return { + ok: true, + identity, + intent: request.intent, + intentId: request.intent.id, + requestDigest: responseDigest ?? request.intent.requestDigest, + operationDigest: request.intent.operationDigest, + leaseId: request.intent.lease.id, + result: { outcome: "succeeded" }, + }; + }, + }, + }); + const input: AdmittedCredentialOperation = { + status, + identity, + intent: { + id: "intent", + bootId: "boot", + zoneId: "public", + policyDigest: identity.policyDigest, + connectionId: "connection", + connectionRevision: 2, + credentialIdentity: "identity", + scope: { + kind: "execution" as const, + sessionId: "s", + runId: "r", + toolCallId: "call", + bindingHash: "b".repeat(64), + }, + operation: "execute" as const, + destination: "https://tools.example.test/mcp", + requestDigest: `sha256:${"b".repeat(64)}`, + expiresAt: now + 60000, + lease: { id: "admission", deploymentId: "deployment", expiresAt: now + 60000 }, + }, + operation: { + kind: "execute" as const, + endpoint: "https://tools.example.test/mcp", + tool: "Read", + arguments: {}, + inputSchema: { type: "object" }, + argumentConstraintDocuments: [], + classification: "read" as const, + }, + }; + return { broker, input, seen, admitted, requests, counters: () => ({ deliveries, leases }) }; +} +for (const [operation, rawDigest] of [ + ["discover", "1d8346198aa04c4188491657f648995a8657213ae8bde54920a7588ca74f778e"], + ["execute", "4307c51da41ceb7da86053004e56d21d4b7b6fe78811d8ce64a2e1468b936a77"], +] as const) + test(`${operation} leases raw hex while admission and worker retain the canonical intent`, async () => { + const f = fixture(); + f.input.intent.requestDigest = canonicalDigest({ kind: operation }); + if (operation === "discover") { + f.input.intent.operation = operation; + f.input.intent.scope = { kind: "administration", requestId: "discovery", actorId: "admin" }; + f.input.operation = { kind: operation, endpoint: "https://tools.example.test/mcp" }; + } + const original = structuredClone(f.input); + const response = await f.broker.dispatch(f.input); + expect(f.counters()).toEqual({ deliveries: 1, leases: 1 }); + expect(f.seen[0]).toMatchObject({ + protocol: 1, + context: original.status.context, + reference: original.status.reference, + worker: original.identity, + intentId: "intent", + requestDigest: rawDigest, + operation, + expiresAt: 110000n, + operationExpiresAt: 160000n, + admission: { id: "admission", deploymentId: "deployment", expiresAt: 160000n }, + scope: + operation === "discover" + ? { case: "administration", value: { requestId: "discovery", actorId: "admin" } } + : { + case: "execution", + value: { + sessionId: "s", + runId: "r", + toolCallId: "call", + bindingHash: "b".repeat(64), + }, + }, + }); + expect(f.admitted).toEqual([original, original]); + expect(response.ok).toBe(true); + if (!response.ok) throw Error("unexpected worker failure"); + expect(f.requests).toEqual([ + { + protocol: 1, + intent: { ...original.intent, operationDigest: response.operationDigest }, + operation: { + ...original.operation, + credential: { header: "x-api-key", value: "transient-value" }, + }, + }, + ]); + expect(response.requestDigest).toBe(`sha256:${rawDigest}`); + expect(response.intent).toEqual(f.requests[0]!.intent); + expect(f.input).toEqual(original); + }); + +test("malformed canonical request digests fail before credential leasing", async () => { + const raw = "b".repeat(64); + for (const requestDigest of [ + "", + raw, + `sha256:sha256:${raw}`, + `SHA256:${raw}`, + `sha256:${raw.toUpperCase()}`, + `sha256:${raw} `, + `sha256:${raw}\n`, + `sha256:${raw.slice(1)}`, + `sha256:${raw}0`, + `sha256:${"g".repeat(64)}`, + ]) { + const f = fixture(); + f.input.intent.requestDigest = requestDigest; + await expect(f.broker.dispatch(f.input)).rejects.toThrow("MCP request digest invalid"); + expect(f.counters()).toEqual({ deliveries: 0, leases: 0 }); + } +}); + +test("raw control digest cannot replace the tagged worker response binding", async () => { + const f = fixture(false, 0, "b".repeat(64)); + await expect(f.broker.dispatch(f.input)).rejects.toThrow("MCP worker response mismatch"); + expect(f.counters()).toEqual({ deliveries: 1, leases: 1 }); +}); + +test("broker preserves 60 second intent while credential delivery expires in 10 seconds", async () => { + const f = fixture(); + await f.broker.dispatch(f.input); + expect(f.counters()).toEqual({ deliveries: 1, leases: 1 }); + expect(f.seen[0]).toMatchObject({ expiresAt: 110000n, operationExpiresAt: 160000n }); +}); +test("expired handoff causes no resource call and no re-lease", async () => { + const f = fixture(true); + await expect(f.broker.dispatch(f.input)).rejects.toThrow("expired"); + expect(f.counters()).toEqual({ deliveries: 0, leases: 1 }); +}); +test("worker policy and boot mismatch prevents secret lease", async () => { + const f = fixture(); + f.input.identity = { ...identity, bootId: "wrong" }; + await expect(f.broker.dispatch(f.input)).rejects.toThrow(); + expect(f.counters()).toEqual({ deliveries: 0, leases: 0 }); +}); +test("admission rejection prevents worker and credential access", async () => { + const f = fixture(); + let touched = false; + const control = unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => + router.service(CredentialControlService, { + async lease() { + touched = true; + throw new Error("unexpected"); + }, + }), + ), + ), + ); + const broker = makeCredentialBroker({ + control, + now: () => now, + admission: { + async assertAdmitted() { + throw new Error("admission denied"); + }, + }, + worker: { + async identity() { + touched = true; + return identity; + }, + async dispatch() { + touched = true; + throw new Error("unexpected"); + }, + }, + }); + await expect(broker.dispatch(f.input)).rejects.toThrow("admission denied"); + expect(touched).toBe(false); +}); +test("changed destination, credential identity and insufficient operation admission fail before leasing", async () => { + for (const mutate of [ + (f: ReturnType) => { + f.input.operation.endpoint = "https://other.example.test/"; + }, + (f: ReturnType) => { + f.input.intent.credentialIdentity = "old"; + }, + (f: ReturnType) => { + f.input.intent.lease.expiresAt--; + }, + ]) { + const f = fixture(); + mutate(f); + await expect(f.broker.dispatch(f.input)).rejects.toThrow(); + expect(f.counters()).toEqual({ deliveries: 0, leases: 0 }); + } +}); +test("authority invalidated after lease is checked again before resource dispatch", async () => { + const f = fixture(); + let checks = 0, + sent = false; + const control = unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => + router.service(CredentialControlService, { + async lease(req) { + return { + protocol: 1, + header: "x-api-key", + value: "secret", + expiresAt: req.expiresAt, + operationExpiresAt: req.operationExpiresAt, + intentId: req.intentId, + }; + }, + }), + ), + ), + ); + const broker = makeCredentialBroker({ + control, + now: () => now, + admission: { + async assertAdmitted() { + if (++checks === 2) throw new Error("revoked"); + }, + }, + worker: { + async identity() { + return identity; + }, + async dispatch(request) { + sent = true; + return { + ok: true, + identity, + intent: request.intent, + intentId: request.intent.id, + requestDigest: request.intent.requestDigest, + operationDigest: request.intent.operationDigest, + leaseId: request.intent.lease.id, + result: {}, + }; + }, + }, + }); + await expect(broker.dispatch(f.input)).rejects.toThrow("revoked"); + expect(sent).toBe(false); +}); +test("worker transport errors cannot expose the transient credential", async () => { + const f = fixture(); + const control = unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => + router.service(CredentialControlService, { + async lease(req) { + return { + protocol: 1, + header: "x-api-key", + value: "secret-value", + expiresAt: req.expiresAt, + operationExpiresAt: req.operationExpiresAt, + intentId: req.intentId, + }; + }, + }), + ), + ), + ); + const broker = makeCredentialBroker({ + control, + now: () => now, + admission: { async assertAdmitted() {} }, + worker: { + async identity() { + return identity; + }, + async dispatch() { + throw new Error("request with secret-value failed"); + }, + }, + }); + await expect(broker.dispatch(f.input)).rejects.toThrow("MCP worker operation unavailable"); +}); + +for (const finalFenceMs of [9999, 10000, 11000]) + test(`final fence at ${finalFenceMs} ms preserves the delivery boundary and full operation deadline`, async () => { + const f = fixture(false, finalFenceMs); + if (finalFenceMs < 10000) { + await f.broker.dispatch(f.input); + expect(f.counters()).toEqual({ deliveries: 1, leases: 1 }); + } else { + await expect(f.broker.dispatch(f.input)).rejects.toThrow("credential delivery expired"); + expect(f.counters()).toEqual({ deliveries: 0, leases: 1 }); + } + expect(f.seen[0]).toMatchObject({ expiresAt: 110000n, operationExpiresAt: 160000n }); + expect(f.input.intent.expiresAt).toBe(160000); + }); diff --git a/orchestrator/src/__tests__/mcp-credential-oauth.test.ts b/orchestrator/src/__tests__/mcp-credential-oauth.test.ts new file mode 100644 index 000000000..f5e1a3536 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-credential-oauth.test.ts @@ -0,0 +1,447 @@ +import { unwrapCredentialControlClient } from "../mcp/control-client.ts"; +import { expect, test } from "bun:test"; +import { Code, ConnectError, createClient, createRouterTransport } from "@connectrpc/connect"; +import { create } from "@bufbuild/protobuf"; +import { checkDb } from "../db/client.ts"; +import { makeIntegrationConnectionStore } from "../db/integration-connections.ts"; +import { makeCredentialMetadataStore } from "../mcp/credential-store.ts"; +import { makeCredentialAdmin } from "../mcp/credential-admin.ts"; +import { + CredentialControlService, + CredentialKind, + CredentialStatusSchema, + type AuthorityContext, + type CredentialStatus, +} from "../gen/engram/mcp/v1/control_pb.ts"; +const reachable = process.env.ORCHESTRATOR_DATABASE_URL ? await checkDb() : false; +const config = { + endpoint: "https://tools.example.test/mcp", + zone: "internal", + auth: { + kind: "oauth" as const, + clientRegistrationId: "public-client", + shareWithOrgSessions: true as const, + }, +}; +async function fixture() { + const connections = makeIntegrationConnectionStore(), + store = makeCredentialMetadataStore(); + const row = await connections.create({ + alias: `mcp-${crypto.randomUUID()}`, + displayName: "Tools", + provider: "mcp", + config, + }); + let clock = Date.now(), + flowContext: AuthorityContext | undefined, + current: CredentialStatus | undefined, + lostCallback = false, + outage = false; + const calls: string[] = []; + const cancelled = new Set(); + let flowNumber = 0; + let pauseComplete: (() => Promise) | undefined; + let pauseBegin: (() => Promise) | undefined; + let pauseStatus: (() => Promise) | undefined; + const control = unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => + router.service(CredentialControlService, { + async getStatus() { + if (outage) throw new ConnectError("outage", Code.Unavailable); + if (!current) throw new ConnectError("absent", Code.NotFound); + return { status: current }; + }, + async cancelAuthorization(req) { + if (outage) throw new ConnectError("outage", Code.Unavailable); + expect(req.connectionId).toBe(row.id); + cancelled.add(req.flowId); + return { + protocol: 1, + flowId: req.flowId, + connectionId: req.connectionId, + registrationId: req.registrationId, + }; + }, + async beginAuthorization(req) { + calls.push("begin"); + expect(req.sharingConsent).toBe(true); + expect(req.clientSecret).toBeUndefined(); + expect(req.registrationId).toBe("public-client"); + expect(req.context!.resource!.zoneId).toBe("internal"); + callback.flowId = `flow-${++flowNumber}`; + return { + protocol: 1, + flowId: callback.flowId, + context: { + connectionId: row.id, + configRevision: req.context!.configRevision, + authorizationGeneration: req.context!.previousAuthorizationGeneration + 1n, + credentialIdentity: "ceremony", + resource: req.context!.resource, + policyDigest: "a".repeat(64), + policyRevision: "v1", + }, + authorizationUrl: "https://auth.example.test/authorize?state=secret-state", + expiresAt: BigInt(clock + 600000), + }; + }, + async completeAuthorization(req) { + if (cancelled.has(req.flowId)) throw new ConnectError("cancelled", Code.Aborted); + calls.push("complete"); + expect(req.state).toBe("secret-state"); + expect(req.code).toBe("secret-code"); + flowContext = req.context!; + current = create(CredentialStatusSchema, { + protocol: 1, + context: flowContext, + reference: { kind: CredentialKind.OAUTH, registrationId: "public-client" }, + version: (current?.version ?? 0n) + 1n, + status: "connected", + }); + if (lostCallback) throw new ConnectError("lost reply", Code.NotFound); + return { status: current }; + }, + async revoke(req) { + calls.push("revoke"); + expect(req.expected!.version).toBe(current!.version); + current = create(CredentialStatusSchema, { + ...current!, + version: current!.version + 1n, + status: "revoked", + }); + return { status: current }; + }, + }), + ), + ), + ); + const admin = makeCredentialAdmin({ + store, + control: { + ...control, + async getStatus(req, options) { + const wait = pauseStatus; + pauseStatus = undefined; + if (wait) await wait(); + return control.getStatus(req, options); + }, + async beginAuthorization(req, options) { + const response = await control.beginAuthorization(req, options); + const wait = pauseBegin; + pauseBegin = undefined; + if (wait) await wait(); + return response; + }, + async completeAuthorization(req, options) { + const wait = pauseComplete; + pauseComplete = undefined; + if (wait) await wait(); + return control.completeAuthorization(req, options); + }, + }, + now: () => new Date(clock), + registrations: [ + { + id: "public-client", + clientId: "client-id", + confidential: false, + redirectUri: "https://app.example.test/oauth/mcp", + issuer: "https://auth.example.test/", + resource: { zoneId: "internal", endpoint: config.endpoint }, + scopes: ["tools"], + }, + ], + }); + const callback = { + flowId: "flow", + state: "secret-state", + code: "secret-code", + issuer: "https://auth.example.test/", + redirectUri: "https://app.example.test/oauth/mcp", + }; + return { + connections, + store, + row, + admin, + callback, + calls, + pauseStatus() { + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + pauseStatus = async () => { + entered.resolve(); + await release.promise; + }; + return { entered: entered.promise, resume: release.resolve }; + }, + pauseBegin() { + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + pauseBegin = async () => { + entered.resolve(); + await release.promise; + }; + return { entered: entered.promise, resume: release.resolve }; + }, + pause() { + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + pauseComplete = async () => { + entered.resolve(); + await release.promise; + }; + return { entered: entered.promise, resume: release.resolve }; + }, + lose() { + lostCallback = true; + }, + advance() { + clock += 35000; + }, + outage(value: boolean) { + outage = value; + }, + current: () => current, + cancelled: (id: string) => cancelled.has(id), + async cleanup() { + await connections.delete(row.id); + }, + }; +} +test.skipIf(!reachable)( + "OAuth admin consent, exact callback actor and redirect, one-use publication and metadata-only persistence", + async () => { + const f = await fixture(); + try { + await expect(f.admin.begin(f.row.id, 1, 1, "admin", false)).rejects.toThrow("consent"); + expect(f.calls).toEqual([]); + await f.admin.begin(f.row.id, 1, 1, "admin", true); + let row = await f.store.get(f.row.id); + expect(JSON.stringify(row)).not.toContain("secret-state"); + await expect(f.admin.complete(row.id, 2, 2, "other", f.callback)).rejects.toThrow("stale"); + await expect( + f.admin.complete(row.id, 2, 2, "admin", { + ...f.callback, + redirectUri: "https://evil.example/", + }), + ).rejects.toThrow("callback"); + await f.admin.complete(row.id, 2, 2, "admin", f.callback); + row = await f.store.get(row.id); + expect(row.revoked).toBe(false); + expect(row.authGeneration).toBe(3); + expect(row.configRevision).toBe(2); + expect(JSON.stringify(row)).not.toContain("secret-code"); + expect(JSON.stringify(row)).not.toContain("secret-state"); + await expect(f.admin.complete(row.id, 2, 3, "admin", f.callback)).rejects.toThrow("stale"); + expect(f.calls).toEqual(["begin", "complete"]); + } finally { + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "switch during pending OAuth blocks the old callback before exchange", + async () => { + const f = await fixture(); + try { + await f.admin.begin(f.row.id, 1, 1, "admin", true); + await f.admin.configure(f.row.id, 2, 2, "admin", { + endpoint: config.endpoint, + zone: "internal", + auth: { kind: "api_key", secretRef: "" }, + }); + await expect(f.admin.complete(f.row.id, 2, 2, "admin", f.callback)).rejects.toThrow(); + expect(f.calls).toEqual(["begin"]); + expect(f.cancelled(f.callback.flowId)).toBe(true); + } finally { + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "lost callback requires recovery that revokes the published coordinator authority", + async () => { + const f = await fixture(); + try { + await f.admin.begin(f.row.id, 1, 1, "admin", true); + f.lose(); + await expect(f.admin.complete(f.row.id, 2, 2, "admin", f.callback)).rejects.toThrow(); + expect(f.current()!.status).toBe("connected"); + f.advance(); + await f.admin.recover(f.row.id, 2, 2, "admin"); + expect(f.current()!.status).toBe("revoked"); + expect((await f.store.get(f.row.id)).revoked).toBe(true); + expect(f.calls).toEqual(["begin", "complete", "revoke"]); + } finally { + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "recovery fences a lost reconnect callback over a previously revoked receipt", + async () => { + const f = await fixture(); + try { + await f.admin.begin(f.row.id, 1, 1, "admin", true); + await f.admin.complete(f.row.id, 2, 2, "admin", f.callback); + await f.admin.begin(f.row.id, 2, 3, "admin", true); + f.lose(); + await expect(f.admin.complete(f.row.id, 3, 4, "admin", f.callback)).rejects.toThrow(); + f.advance(); + await f.admin.recover(f.row.id, 3, 4, "admin"); + expect(f.current()!.status).toBe("revoked"); + } finally { + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "recovery keeps its durable fence on outage and clears only confirmed absence", + async () => { + const f = await fixture(); + try { + const now = new Date(); + await f.store.claim(f.row.id, 1, 1, "admin", config, now); + f.advance(); + f.advance(); + f.outage(true); + await expect(f.admin.recover(f.row.id, 2, 2, "admin")).rejects.toThrow(); + expect((await f.store.get(f.row.id)).mcpCredential.transition).toBeDefined(); + f.outage(false); + f.advance(); + await f.admin.recover(f.row.id, 2, 2, "admin"); + const row = await f.store.get(f.row.id); + expect(row.revoked).toBe(true); + expect(row.mcpCredential.transition).toBeUndefined(); + } finally { + await f.cleanup(); + } + }, +); + +for (const reconnect of [false, true]) { + test.skipIf(!reachable)( + `recovery cancels pre-dispatch callback before kind switch (reconnect=${reconnect})`, + async () => { + const f = await fixture(); + try { + await f.admin.begin(f.row.id, 1, 1, "admin", true); + if (reconnect) { + await f.admin.complete(f.row.id, 2, 2, "admin", f.callback); + await f.admin.begin(f.row.id, 2, 3, "admin", true); + } + const row = await f.store.get(f.row.id); + const pause = f.pause(); + const late = f.admin.complete(row.id, row.configRevision, row.authGeneration, "admin", { + ...f.callback, + }); + const outcome = late.then( + () => false, + () => true, + ); + await pause.entered; + const exchanges = f.calls.filter((c) => c === "complete").length; + f.advance(); + await f.admin.recover(row.id, row.configRevision, row.authGeneration, "admin"); + await f.admin.configure(row.id, row.configRevision, row.authGeneration, "admin", { + endpoint: config.endpoint, + zone: config.zone, + auth: { kind: "api_key", secretRef: "" }, + }); + pause.resume(); + expect(await outcome).toBe(true); + expect(f.calls.filter((c) => c === "complete")).toHaveLength(exchanges); + expect(f.current()?.status).not.toBe("connected"); + let next = await f.store.get(row.id); + await f.admin.configure(row.id, next.configRevision, next.authGeneration, "admin", config); + next = await f.store.get(row.id); + await f.admin.begin(row.id, next.configRevision, next.authGeneration, "admin", true); + next = await f.store.get(row.id); + await f.admin.complete( + row.id, + next.configRevision, + next.authGeneration, + "admin", + f.callback, + ); + expect((await f.store.get(row.id)).revoked).toBe(false); + } finally { + await f.cleanup(); + } + }, + ); +} + +test.skipIf(!reachable)( + "lost begin response cannot return a public authorization URL after recovery", + async () => { + const f = await fixture(); + try { + const pause = f.pauseBegin(); + const begin = f.admin.begin(f.row.id, 1, 1, "admin", true); + const result = begin.then( + (value) => ({ value }), + () => ({ value: undefined }), + ); + await pause.entered; + f.advance(); + await f.admin.recover(f.row.id, 2, 2, "admin"); + await f.admin.configure(f.row.id, 2, 2, "admin", { + endpoint: config.endpoint, + zone: config.zone, + auth: { kind: "api_key", secretRef: "" }, + }); + pause.resume(); + expect((await result).value).toBeUndefined(); + const row = await f.store.get(f.row.id); + expect(row.mcpCredential.flow).toBeUndefined(); + await expect( + f.admin.complete(row.id, row.configRevision, row.authGeneration, "admin", f.callback), + ).rejects.toThrow(); + expect(f.calls).toEqual(["begin"]); + } finally { + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "old recovery cannot revoke a newer same-registration authority", + async () => { + const f = await fixture(); + try { + await f.admin.begin(f.row.id, 1, 1, "admin", true); + const callbackPause = f.pause(); + const oldCallback = f.admin.complete(f.row.id, 2, 2, "admin", { ...f.callback }).then( + () => false, + () => true, + ); + await callbackPause.entered; + f.advance(); + const statusPause = f.pauseStatus(); + const oldRecovery = f.admin.recover(f.row.id, 2, 2, "admin").then( + () => false, + () => true, + ); + await statusPause.entered; + f.advance(); + await f.admin.recover(f.row.id, 2, 2, "admin"); + await f.admin.begin(f.row.id, 2, 2, "admin", true); + await f.admin.complete(f.row.id, 3, 3, "admin", f.callback); + const replacement = structuredClone(f.current()); + statusPause.resume(); + expect(await oldRecovery).toBe(true); + callbackPause.resume(); + expect(await oldCallback).toBe(true); + expect(f.current()).toEqual(replacement); + expect(f.current()!.status).toBe("connected"); + expect(f.calls.filter((c) => c === "revoke")).toHaveLength(0); + } finally { + await f.cleanup(); + } + }, +); diff --git a/orchestrator/src/__tests__/mcp-credential-store.test.ts b/orchestrator/src/__tests__/mcp-credential-store.test.ts new file mode 100644 index 000000000..de06047c5 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-credential-store.test.ts @@ -0,0 +1,117 @@ +import { expect, test } from "bun:test"; +import { makeIntegrationConnectionStore } from "../db/integration-connections.ts"; +import { makeCredentialMetadataStore } from "../mcp/credential-store.ts"; +import { getDb, checkDb } from "../db/client.ts"; +const reachable = process.env.ORCHESTRATOR_DATABASE_URL ? await checkDb() : false; +const config = { + endpoint: "https://tools.example.test/mcp", + zone: "public", + auth: { kind: "api_key", secretRef: "" }, +}; +test.skipIf(!reachable)( + "credential transitions serialize across replicas and publish exact target revision", + async () => { + const connections = makeIntegrationConnectionStore(); + const row = await connections.create({ + alias: `mcp-${crypto.randomUUID()}`, + displayName: "Tools", + provider: "mcp", + config, + }); + const a = makeCredentialMetadataStore(), + b = makeCredentialMetadataStore(); + try { + const now = new Date(); + const results = await Promise.allSettled( + [a, b].map((s) => s.claim(row.id, 1, 1, "admin", config, now)), + ); + expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1); + const won = results.find((r) => r.status === "fulfilled"); + if (!won || won.status !== "fulfilled") throw new Error("claim"); + const claim = won.value; + const stale = (await connections.get(row.id))!; + expect(stale.mcp!.revoked).toBe(true); + await expect( + b.claim(row.id, stale.mcp!.configRevision, stale.mcp!.authGeneration, "admin", config, now), + ).rejects.toThrow("stale"); + const status = { + protocol: 1, + context: { + connectionId: row.id, + configRevision: String(claim.revision), + authorizationGeneration: String(claim.generation + 1), + credentialIdentity: "identity-new", + resource: { zoneId: "public", endpoint: config.endpoint }, + policyDigest: "a".repeat(64), + policyRevision: "v1", + }, + reference: { + kind: "CREDENTIAL_KIND_API_KEY", + name: "mcp.fresh", + connectionId: row.id, + generation: "1", + }, + version: "1", + status: "connected", + }; + await expect( + a.publish( + claim, + { ...config, auth: { kind: "api_key", secretRef: "mcp.fresh" } }, + { ...status, reference: { ...status.reference, kind: "CREDENTIAL_KIND_CLIENT_SECRET" } }, + now, + ), + ).rejects.toThrow("receipt"); + await a.publish( + claim, + { ...config, auth: { kind: "api_key", secretRef: "mcp.fresh" } }, + status, + now, + ); + const live = (await connections.get(row.id))!; + expect(live.mcp!.configRevision).toBe(claim.revision); + expect(live.mcp!.authGeneration).toBe(claim.generation + 1); + expect(live.mcp!.credentialIdentity).toBe("identity-new"); + expect(live.mcp!.revoked).toBe(false); + await expect(b.publish(claim, config, status, now)).rejects.toThrow("stale"); + } finally { + await connections.delete(row.id); + } + }, +); +test.skipIf(!reachable)( + "recovery replaces durable token and blocks late publication from the old owner", + async () => { + const connections = makeIntegrationConnectionStore(), + a = makeCredentialMetadataStore(), + b = makeCredentialMetadataStore(); + const row = await connections.create({ + alias: `mcp-${crypto.randomUUID()}`, + provider: "mcp", + displayName: "Tools", + config, + }); + try { + const now = new Date(); + const old = await a.claim(row.id, 1, 1, "admin", config, now); + await expect( + b.recover(row.id, old.revision, old.generation, "admin", new Date(now.getTime() + 34999)), + ).rejects.toThrow(); + const recovered = await b.recover( + row.id, + old.revision, + old.generation, + "admin", + new Date(now.getTime() + 35000), + ); + await expect(a.publish(old, config, undefined, now)).rejects.toThrow("stale"); + await b.publish(recovered, config, undefined, now); + const next = await a.claim(row.id, old.revision, old.generation, "admin", config, now); + await expect(b.stage(recovered, {}, now)).rejects.toThrow("stale"); + await a.publish(next, config, undefined, now); + expect((await a.get(row.id)).revoked).toBe(true); + } finally { + await connections.delete(row.id); + } + }, +); diff --git a/orchestrator/src/__tests__/mcp-cross-zone-recovery.test.ts b/orchestrator/src/__tests__/mcp-cross-zone-recovery.test.ts new file mode 100644 index 000000000..e339030d2 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-cross-zone-recovery.test.ts @@ -0,0 +1,88 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/node-postgres"; +import { Pool } from "pg"; +import * as schema from "../db/schema.ts"; +import { makeMcpJournal } from "../mcp/journal.ts"; +import { makeMcpRecoveryScanner } from "../mcp/recovery-scanner.ts"; +import { schemaVerifiedFixture } from "./mcp-review-fixture.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; + +test("sixteenth approval is global to the session across racing deployment clients", async () => { + const { app } = await verifyMcpTestDatabases(); + const f = await schemaVerifiedFixture(); + const pool = new Pool({ connectionString: app, max: 2 }); + const other = makeMcpJournal(drizzle(pool, { schema })); + const deployments = [f.deploymentId, crypto.randomUUID()]; + try { + const calls = []; + for (let i = 0; i < 17; i++) { + const toolCallId = `approval-${i}`; + const call = await f.journal.create( + { + ...f.input, + toolCallId, + requestPolicyContext: { ...f.input.requestPolicyContext, toolCallId }, + }, + { kind: "approval" }, + f.now, + ); + calls.push(call); + if (i < 15) await f.journal.requestApproval(call.id, deployments[i % 2]!, f.now); + } + const outcomes = await Promise.allSettled([ + f.journal.requestApproval(calls[15]!.id, deployments[0]!, f.now), + other.requestApproval(calls[16]!.id, deployments[1]!, f.now), + ]); + expect(outcomes.filter((o) => o.status === "fulfilled")).toHaveLength(1); + expect(outcomes.filter((o) => o.status === "rejected")).toHaveLength(1); + expect(await f.journal.pending(f.sessionId, f.actorId, f.now)).toHaveLength(16); + } finally { + await f.cleanup(); + await f.db + .delete(schema.mcpAdmissionScope) + .where(eq(schema.mcpAdmissionScope.id, deployments[1]!)); + await pool.end(); + } +}); + +test("disabled scanner expires persisted removed deployment approvals and admissions", async () => { + await verifyMcpTestDatabases(); + const f = await schemaVerifiedFixture(); + const currentDeployment = crypto.randomUUID(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const base = makeMcpJournal(f.db); + const toolCallId = "active-schema"; + const active = await base.create( + { + ...f.input, + toolCallId, + requestPolicyContext: { ...f.input.requestPolicyContext, toolCallId }, + }, + { kind: "approval" }, + f.now, + ); + await base.admitSchema(active.id, f.deploymentId, f.now); + const scanner = makeMcpRecoveryScanner({ + journal: f.journal, + deploymentId: currentDeployment, + runtime: () => undefined, + now: () => new Date(f.now.getTime() + 600_001), + }); + await scanner.runOnce(); + const [approval] = await f.db + .select() + .from(schema.mcpApproval) + .where(eq(schema.mcpApproval.executionId, call.id)); + expect(approval!.decision).toBe("expired"); + expect((await f.journal.get(call.id))!.state).toBe("canceled"); + expect((await f.journal.getAdmission(active.id))!.state).toBe("released"); + } finally { + await f.cleanup(); + await f.db + .delete(schema.mcpAdmissionScope) + .where(eq(schema.mcpAdmissionScope.id, currentDeployment)); + } +}); diff --git a/orchestrator/src/__tests__/mcp-currentness-graph.test.ts b/orchestrator/src/__tests__/mcp-currentness-graph.test.ts new file mode 100644 index 000000000..048551c1f --- /dev/null +++ b/orchestrator/src/__tests__/mcp-currentness-graph.test.ts @@ -0,0 +1,123 @@ +import { expect, test } from "bun:test"; +import { eq, inArray } from "drizzle-orm"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +import { reviewFixture, seedReviewConnectionAuthority } from "./mcp-review-fixture.ts"; +import { task, taskSession, profile, mcpCatalog } from "../db/schema.ts"; +import { + bindingSnapshotColumns, + readLiveBindingAuthority, + readStoredBindingSession, +} from "../mcp/binding-store.ts"; +import { compileSessionBindings } from "../mcp/session-bindings.ts"; +import { validateMcpConfig } from "../mcp/config.ts"; +import { makeSessionBindingResolver } from "../mcp/session-resolver.ts"; + +test("currentness reads a deep multi-connection graph within the one-second fence", async () => { + await verifyMcpTestDatabases(); + const f = await reviewFixture(); + const connectionIds: string[] = [], + taskIds: string[] = []; + try { + const grants = [...f.authority.profileGrants]; + for (let i = 0; i < 15; i++) { + const connection = await f.connections.create({ + alias: `test-${crypto.randomUUID()}`, + provider: "mcp", + displayName: "Graph tools", + config: f.connection.config, + }); + connectionIds.push(connection.id); + await seedReviewConnectionAuthority(connection.id, f.db, f.now); + const row = (await f.connections.get(connection.id))!; + const tools = [ + { + upstreamName: "Write.Record", + description: "Graph tool", + inputSchema: { type: "object" }, + }, + ]; + const draft = await f.repository.saveDraft(connection.id, row.mcp!, tools, f.actorId, f.now); + await f.repository.publish( + connection.id, + draft.id, + 0, + tools.map((t) => ({ ...t, classification: "write", enabled: true })), + f.publisherId, + f.now, + ); + await f.repository.setEnabled(connection.id, 1, true, f.now); + grants.push({ ...grants[0]!, connectionId: connection.id }); + } + await f.db + .update(profile) + .set({ integrationGrants: grants }) + .where(eq(profile.id, f.profileId)); + const live = await readLiveBindingAuthority(f.profileId, f.authority.operatorPolicies, f.db); + const connections = live.connections.map((c) => ({ + id: c.id, + alias: c.alias, + displayName: c.alias, + provider: "mcp", + config: validateMcpConfig(c.config), + })); + let parent: + | { sessionId: string; snapshot: ReturnType } + | undefined; + for (let depth = 0; depth < 24; depth++) { + const sessionId = crypto.randomUUID(), + taskId = crypto.randomUUID(); + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: f.profileId, integrationGrants: grants }, + liveAuthority: live, + bindingEpoch: 1, + ...(parent ? { parentAuthority: parent } : {}), + }); + await f.db.insert(task).values({ + id: taskId, + type: "chat", + createdByUserId: f.actorId, + ...(parent ? { spawningSessionId: parent.sessionId } : {}), + }); + taskIds.push(taskId); + await f.db.insert(taskSession).values({ + taskId, + sessionId, + profileId: f.profileId, + integrationPrincipalId: f.actorId, + ...bindingSnapshotColumns(snapshot, connections), + }); + parent = { sessionId, snapshot }; + } + let externalReads = 0; + const resolver = makeSessionBindingResolver({ + readSession: (id) => readStoredBindingSession(id, f.db), + readAuthority: (id) => readLiveBindingAuthority(id, f.authority.operatorPolicies, f.db), + readBindingEpoch: async () => { + externalReads++; + return 1; + }, + enforceManagedDeny: async (expected) => { + externalReads++; + return expected; + }, + }); + const start = performance.now(); + expect((await resolver.readCurrent(parent!.sessionId)).bindingHash).toBe( + parent!.snapshot.bindingHash, + ); + const elapsed = performance.now() - start; + console.info( + `MCP graph: 24 sessions, 16 connections, ${externalReads} injected external reads, ${Math.round(elapsed)} ms`, + ); + expect(externalReads).toBe(25); + expect(elapsed).toBeLessThan(1000); + await f.db.update(profile).set({ integrationGrants: [] }).where(eq(profile.id, f.profileId)); + await expect(resolver.readCurrent(parent!.sessionId)).rejects.toThrow("authority changed"); + } finally { + for (const id of taskIds.reverse()) await f.db.delete(task).where(eq(task.id, id)); + if (connectionIds.length) + await f.db.delete(mcpCatalog).where(inArray(mcpCatalog.connectionId, connectionIds)); + for (const id of connectionIds) await f.connections.delete(id); + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-dbos-restart.test.ts b/orchestrator/src/__tests__/mcp-dbos-restart.test.ts new file mode 100644 index 000000000..77f73e6d6 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-dbos-restart.test.ts @@ -0,0 +1,661 @@ +import { unknownContext } from "../mcp/journal.ts"; +import { beforeAll, afterAll, expect, test } from "bun:test"; +import { spawn } from "node:child_process"; +import { createInterface } from "node:readline"; +import { once } from "node:events"; +import { Pool } from "pg"; +import { and, eq } from "drizzle-orm"; +import { effectFixture } from "./mcp-effect-fixture.ts"; +import { mcpExecution, mcpApproval } from "../db/schema.ts"; +import { reviewContext } from "../mcp/journal-review.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; + +const isolated = await verifyMcpTestDatabases(); +const system = new Pool({ connectionString: isolated.system }); +beforeAll(async () => { + await system.query("create schema if not exists mcp_test"); + await system.query( + `create table if not exists mcp_test.completions (session_id text, call_id text, result_json text not null, attempts integer not null default 1, primary key(session_id,call_id))`, + ); +}); +afterAll(() => system.end()); +async function until(description: string, check: () => Promise, timeout = 20000) { + const end = Date.now() + timeout; + while (Date.now() < end) { + if (await check()) return; + await Bun.sleep(20); + } + throw Error(`timeout: ${description}`); +} +type Fixture = Awaited>; +async function start( + f: Fixture, + executor: string, + checkpoint?: string, + resume = false, + clockOffset = 0, + eventIdx = "1", + requestReadFailures = 0, +) { + const child = spawn( + process.execPath, + [new URL("./helpers/mcp-dbos-child.ts", import.meta.url).pathname], + { + env: { + PATH: process.env.PATH, + ORCHESTRATOR_DATABASE_URL: isolated.app, + MCP_TEST_DBOS_DATABASE_URL: isolated.system, + MCP_TEST_DATABASE_NAME: process.env.MCP_TEST_DATABASE_NAME, + NODE_ENV: "test", + LOG_LEVEL: "silent", + }, + stdio: ["pipe", "pipe", "pipe"], + }, + ); + let reached = false, + ready = false, + stderr = "", + failure: unknown; + let completions = Promise.resolve(); + const lines = createInterface({ input: child.stdout }); + lines.on("line", (line) => { + if (line === `CHECKPOINT ${checkpoint}`) reached = true; + if (line === "READY") ready = true; + if (line.startsWith("COMPLETE ")) { + const data: { sessionId: string; toolCallId: string; resultJson: string } = JSON.parse( + line.slice(9), + ); + completions = completions + .then(async () => { + // Explicit coordinator fixture: fixed-key durable enqueue precedes ACK. No effect proof. + await system.query( + `insert into mcp_test.completions(session_id,call_id,result_json) values($1,$2,$3) + on conflict(session_id,call_id) do update set attempts=mcp_test.completions.attempts+1`, + [data.sessionId, data.toolCallId, data.resultJson], + ); + if (checkpoint === "enqueue_before_ack") { + reached = true; + return; + } + child.stdin.write("ACK\n"); + }) + .catch((error) => { + failure = error; + }); + } + }); + child.stderr.on("data", (value) => { + stderr += String(value); + }); + child.stdin.write( + JSON.stringify({ + remote: f.remote, + request: f.request, + deploymentId: f.deploymentId, + workflowId: executor, + checkpoint, + resume, + clockOffset, + eventIdx, + requestReadFailures, + }) + "\n", + ); + return { + async reached() { + await until(`child ${checkpoint}`, async () => { + if (failure) throw failure; + if (child.exitCode !== null) + throw Error( + `driver exited ${child.exitCode}: ${stderr.replace(/postgres(?:ql)?:\/\/[^\s]+/g, "[database]")}`, + ); + return reached; + }); + }, + async ready() { + await until("driver ready", async () => { + if (child.exitCode !== null) throw Error(`driver exited: ${stderr}`); + return ready; + }); + }, + continue() { + child.stdin.write("CONTINUE\n"); + }, + async stop() { + if (child.exitCode === null && child.signalCode === null) { + const done = once(child, "exit"); + child.kill("SIGKILL"); + await done; + } + await completions; + lines.close(); + }, + }; +} +async function callFor(f: Fixture) { + return ( + await f.db + .select() + .from(mcpExecution) + .where( + and( + eq(mcpExecution.sessionId, f.sessionId), + eq(mcpExecution.toolCallId, f.request.toolCallId), + ), + ) + )[0]; +} +async function approve(f: Fixture) { + const call = await f.driver.acceptToolRequest(f.request); + await f.driver.driveOnce(call.id); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); +} +const boundaries = [ + "before_insert", + "after_insert", + "before_admission", + "admission_committed", + "approval_before_consume", + "approval_consumed_uncommitted", + "intent_issued", + "barrier_committed", + "before_lease", + "intent_leased", + "response_verified", + "result_committed", + "before_enqueue", + "enqueue_before_ack", + "after_enqueue_ack", + "receipt_committed", +]; +for (const boundary of boundaries) + test(`actual DBOS driver process restart at ${boundary}`, async () => { + const insert = boundary === "before_insert" || boundary === "after_insert"; + const f = await effectFixture({ classification: "write" }); + const executor = `mcp-test-${crypto.randomUUID()}`; + let child: Awaited> | undefined; + try { + if (!insert) await approve(f); + child = await start(f, executor, boundary); + await child.reached(); + const before = await callFor(f); + const beforeCalls = (await f.command("stats")).calls; + const beforeLease = before ? await f.journal.getAdmission(before.id) : null; + await child.stop(); + const afterBarrier = [ + "barrier_committed", + "before_lease", + "intent_leased", + "response_verified", + ].includes(boundary); + child = await start( + f, + executor, + undefined, + boundary !== "before_insert", + afterBarrier ? 61000 : 0, + ); + await child.ready(); + if (insert) { + await until("recovered schema and approval", async () => { + const call = await callFor(f); + return ( + !!call && (await f.journal.getAdmission(call.id))?.schemaDigest === call.requestDigest + ); + }); + const call = (await callFor(f))!; + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); + } + await until("committed completion receipt", async () => { + const call = await callFor(f); + return !!call && call.completionEnqueuedVersion === call.version; + }); + const final = (await callFor(f))!; + expect(final.originalArguments).toBe(f.request.originalArguments); + if (before) { + expect(final.requestDigest).toBe(before.requestDigest); + expect(final.deadline).toEqual(before.deadline); + } + if (beforeLease?.phase === "execute") + expect(final.intentExpiresAt).toEqual(beforeLease.expiresAt); + expect(final.state).toBe(afterBarrier ? "outcome_unknown" : "succeeded"); + const finalCalls = (await f.command("stats")).calls; + expect(finalCalls).toBe(afterBarrier ? beforeCalls : 1); + expect(finalCalls).toBeLessThanOrEqual(1); + if (!insert) { + const approval = ( + await f.db.select().from(mcpApproval).where(eq(mcpApproval.executionId, final.id)) + )[0]!; + expect(approval.consumedAt).not.toBeNull(); + } + const completion = ( + await system.query( + "select * from mcp_test.completions where session_id=$1 and call_id=$2", + [f.sessionId, f.request.toolCallId], + ) + ).rows[0]; + expect(completion).toBeDefined(); + const delivered = JSON.parse(completion.result_json); + if (afterBarrier) + expect(delivered).toMatchObject({ + source: "runtime", + outcome: "outcome_unknown", + delivery: "unavailable", + }); + else + expect(delivered).toMatchObject({ + outcome: "succeeded", + delivery: "available", + output: { isError: false }, + }); + if (["enqueue_before_ack", "after_enqueue_ack"].includes(boundary)) + expect(completion.attempts).toBeGreaterThanOrEqual(2); + // Inspect the actual DBOS database, not a serialization mock. + const status = await system.query( + "select inputs,output,error from dbos.workflow_status where executor_id=$1", + [executor], + ); + expect(status.rowCount).toBeGreaterThan(0); + for (const row of status.rows) { + const input = JSON.parse(row.inputs); + expect(input.json).toHaveLength(1); + expect(Object.keys(input.json[0]).sort()).toEqual( + input.json[0].requestEventIdx + ? ["deadlineMs", "requestEventIdx", "runId", "sessionId", "toolCallId"] + : ["executionId", "runId", "sessionId", "toolCallId"], + ); + if (row.output) expect(JSON.parse(row.output).json).toBeNull(); + expect(row.error).toBeNull(); + } + const durable = JSON.stringify(status.rows); + expect(durable).not.toContain("known-literal-token"); + expect(durable).not.toContain("originalArguments"); + expect(durable).not.toContain("structuredContent"); + expect(durable).not.toContain("resultJson"); + const steps = await system.query( + "select o.* from dbos.operation_outputs o join dbos.workflow_status w using(workflow_uuid) where w.executor_id=$1", + [executor], + ); + expect( + steps.rows.every((row) => + ["DBOS.sleep", "mcp-request-decision", "McpExecWorkflow"].includes(row.function_name), + ), + ).toBe(true); + expect(JSON.stringify(steps.rows)).not.toContain("known-literal-token"); + expect(JSON.stringify(steps.rows)).not.toContain("structuredContent"); + } finally { + await child?.stop(); + await system.query("delete from mcp_test.completions where session_id=$1", [f.sessionId]); + await f.cleanup(); + } + }, 45000); + +for (const eligibility of ["plan", "unknown"]) + test(`actual consumer and DBOS allow reads with ${eligibility} write policy`, async () => { + const f = await effectFixture({ eligibility }); + let child: Awaited> | undefined; + try { + child = await start(f, `mcp-test-${crypto.randomUUID()}`); + await child.ready(); + await until("read completion", async () => { + const row = await callFor(f); + return row?.state === "succeeded" && row.completionEnqueuedVersion === row.version; + }); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await child?.stop(); + await system.query("delete from mcp_test.completions where session_id=$1", [f.sessionId]); + await f.cleanup(); + } + }, 30000); + +for (const eligibility of ["plan", "unknown"]) + test(`actual consumer completes a ${eligibility} write denial and permits the next read`, async () => { + const f = await effectFixture({ eligibility, classification: "write" }); + let child: Awaited> | undefined; + try { + child = await start(f, `mcp-test-${crypto.randomUUID()}`); + await child.ready(); + await until( + "negative completion", + async () => + !!( + await system.query( + "select 1 from mcp_test.completions where session_id=$1 and call_id=$2", + [f.sessionId, f.request.toolCallId], + ) + ).rowCount, + ); + expect(await callFor(f)).toBeUndefined(); + expect((await f.command("stats")).calls).toBe(0); + const denied = ( + await system.query( + "select result_json from mcp_test.completions where session_id=$1 and call_id=$2", + [f.sessionId, f.request.toolCallId], + ) + ).rows[0]; + expect(JSON.parse(denied.result_json)).toMatchObject({ + source: "runtime", + denial: "write_not_allowed", + output: { isError: true }, + }); + expect(JSON.parse(denied.result_json).outcome).toBeUndefined(); + await child.stop(); + f.request.toolCallId = "next-read"; + f.request.modelToolName = f.snapshot.immutableBindings[0]!.tools.find( + (x) => x.tool.classification === "read", + )!.name; + child = await start(f, `mcp-test-${crypto.randomUUID()}`, undefined, false, 0, "2"); + await child.ready(); + await until("subsequent read", async () => { + const row = await callFor(f); + return row?.state === "succeeded" && row.completionEnqueuedVersion === row.version; + }); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await child?.stop(); + await system.query("delete from mcp_test.completions where session_id=$1", [f.sessionId]); + await f.cleanup(); + } + }, 30000); + +test("late verified outcome after expiry gets a new completion receipt without another effect", async () => { + const f = await effectFixture({ classification: "write" }); + let child: Awaited> | undefined; + try { + await approve(f); + child = await start(f, `mcp-test-${crypto.randomUUID()}`, "response_before_persist"); + await child.reached(); + const dispatched = (await callFor(f))!; + expect(dispatched.state).toBe("dispatching"); + expect((await f.command("stats")).calls).toBe(1); + await f.journal.expire(f.deploymentId, new Date(dispatched.intentExpiresAt!.getTime() + 1)); + const { makeMcpRecovery } = await import("../mcp/recovery.ts"); + const deliveries: unknown[] = []; + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + completer: { + async completeToolCall(req) { + deliveries.push(JSON.parse(req.resultJson)); + await system.query( + `insert into mcp_test.completions(session_id,call_id,result_json) values($1,$2,$3) on conflict(session_id,call_id) do update set attempts=mcp_test.completions.attempts+1`, + [req.sessionId, req.toolCallId, req.resultJson], + ); + }, + }, + }); + const marker = { + executionId: dispatched.id, + sessionId: f.sessionId, + runId: f.request.runId, + toolCallId: f.request.toolCallId, + }; + await runtime.runOnce(marker); + const unknown = (await callFor(f))!; + expect(unknown.state).toBe("outcome_unknown"); + expect(unknown.result).toBeNull(); + expect(deliveries).toMatchObject([ + { source: "runtime", outcome: "outcome_unknown", delivery: "unavailable" }, + ]); + expect(unknown.completionEnqueuedVersion).toBe(unknown.version); + f.requests.set("new-write", { ...f.request, toolCallId: "new-write" }); + await expect(f.driver.acceptToolRequest(f.requests.get("new-write")!)).rejects.toThrow(); + await f.journal.resolveUnknown( + dispatched.id, + f.actorId, + "acknowledged", + new Date(), + unknownContext((await f.journal.get(dispatched.id))!), + ); + const resolved = (await callFor(f))!; + expect(resolved.state).toBe("outcome_unknown"); + expect(resolved.result).toBeNull(); + expect(resolved.version).toBe(unknown.version); + child.continue(); + await until("late completion", async () => { + const row = (await callFor(f))!; + return row.state === "succeeded" && row.completionEnqueuedVersion === row.version; + }); + const known = (await callFor(f))!; + expect(known.version).toBeGreaterThan(unknown.version); + expect(known.intentId).toBe(dispatched.intentId); + expect(known.result).toMatchObject({ outcome: "succeeded", delivery: "available" }); + const queued = ( + await system.query( + "select result_json,attempts from mcp_test.completions where session_id=$1 and call_id=$2", + [f.sessionId, f.request.toolCallId], + ) + ).rows[0]; + expect(queued.attempts).toBeGreaterThanOrEqual(2); + expect(JSON.parse(queued.result_json)).toMatchObject({ + source: "runtime", + outcome: "outcome_unknown", + }); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await child?.stop(); + await system.query("delete from mcp_test.completions where session_id=$1", [f.sessionId]); + await f.cleanup(); + } +}, 30000); + +for (const classification of ["read", "write"] as const) + test(`kill after provider send expires unknown and never repeats the ${classification}`, async () => { + const f = await effectFixture({ classification, response: "hold" }); + const executor = `mcp-test-${crypto.randomUUID()}`; + let child: Awaited> | undefined; + try { + if (classification === "write") await approve(f); + child = await start(f, executor); + await child.ready(); + await until("provider tools/call", async () => (await f.command("stats")).calls === 1); + const sent = (await callFor(f))!; + expect(sent.state).toBe("dispatching"); + await child.stop(); + await f.command("release"); + child = await start(f, executor, undefined, true, 61000); + await child.ready(); + await until("unknown completion", async () => { + const row = (await callFor(f))!; + return row.state === "outcome_unknown" && row.completionEnqueuedVersion === row.version; + }); + expect((await f.command("stats")).calls).toBe(1); + expect((await callFor(f))!.intentId).toBe(sent.intentId); + } finally { + await child?.stop(); + await f.command("release"); + await system.query("delete from mcp_test.completions where session_id=$1", [f.sessionId]); + await f.cleanup(); + } + }, 30000); + +for (const cause of ["policy", "invalid_json"] as const) + for (const boundary of ["before_enqueue", "enqueue_before_ack"]) + test(`durable ${cause} denial survives ${boundary} and changed request policy`, async () => { + const f = await effectFixture({ classification: "write", eligibility: "plan" }); + if (cause === "invalid_json") f.request.originalArguments = "{private-invalid-arguments"; + const executor = `mcp-test-${crypto.randomUUID()}`; + const originalRun = f.request.runId; + let child: Awaited> | undefined; + try { + child = await start(f, executor, boundary); + await child.reached(); + expect(await callFor(f)).toBeUndefined(); + await child.stop(); + // Both policy and the duplicate event change after the negative decision commits. + f.remote.eligibility = "write_eligible"; + f.setEligibility("write_eligible"); + f.request.originalArguments = '{"x":1}'; + f.request.runId = "changed-run"; + child = await start(f, executor, undefined, false, 0, "2"); + await child.ready(); + await until( + "denial redelivery", + async () => + ( + await system.query( + "select attempts from mcp_test.completions where session_id=$1 and call_id=$2", + [f.sessionId, f.request.toolCallId], + ) + ).rows[0]?.attempts >= (boundary === "enqueue_before_ack" ? 2 : 1), + ); + expect(await callFor(f)).toBeUndefined(); + expect((await f.command("stats")).calls).toBe(0); + const workflowId = `mcp-request:${f.sessionId}:${f.request.toolCallId}`; + const state = ( + await system.query("select inputs from dbos.workflow_status where workflow_uuid=$1", [ + workflowId, + ]) + ).rows[0]; + expect(JSON.parse(state.inputs).json).toEqual([ + { + sessionId: f.sessionId, + runId: originalRun, + toolCallId: f.request.toolCallId, + requestEventIdx: "1", + deadlineMs: expect.any(Number), + }, + ]); + const steps = ( + await system.query( + "select output,error from dbos.operation_outputs where workflow_uuid=$1 and function_name='mcp-request-decision'", + [workflowId], + ) + ).rows; + expect(steps).toHaveLength(1); + expect(JSON.parse(steps[0].output).json).toEqual({ + kind: "denied", + code: cause === "policy" ? "write_not_allowed" : "invalid_arguments", + }); + expect(steps[0].error).toBeNull(); + expect(JSON.stringify(steps)).not.toContain("private-invalid-arguments"); + expect(JSON.stringify(steps)).not.toContain("known-literal-token"); + } finally { + await child?.stop(); + await system.query("delete from mcp_test.completions where session_id=$1", [f.sessionId]); + await f.cleanup(); + } + }, 30000); + +test("transient request read retries with only a fixed error in DBOS", async () => { + const f = await effectFixture(); + const executor = `mcp-test-${crypto.randomUUID()}`; + let child: Awaited> | undefined; + try { + child = await start(f, executor, undefined, false, 0, "1", 1); + await child.ready(); + await until("retry completes", async () => { + const row = await callFor(f); + return row?.state === "succeeded" && row.completionEnqueuedVersion === row.version; + }); + const steps = ( + await system.query( + "select output,error from dbos.operation_outputs where workflow_uuid=$1 and function_name='mcp-request-decision' order by function_id", + [`mcp-request:${f.sessionId}:${f.request.toolCallId}`], + ) + ).rows; + expect(steps).toHaveLength(2); + expect(steps[0].error).toContain("MCP request acceptance unavailable"); + expect(steps[1].error).toBeNull(); + expect(JSON.stringify(steps)).not.toContain("private-control-error"); + expect(JSON.stringify(steps)).not.toContain("known-literal-token"); + expect(JSON.stringify(steps)).not.toContain("originalArguments"); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await child?.stop(); + await system.query("delete from mcp_test.completions where session_id=$1", [f.sessionId]); + await f.cleanup(); + } +}, 30000); + +test("request decision restart keeps its fixed deadline and expires without an effect", async () => { + const f = await effectFixture(); + const executor = `mcp-test-${crypto.randomUUID()}`; + const workflowId = `mcp-request:${f.sessionId}:${f.request.toolCallId}`; + let child: Awaited> | undefined; + try { + child = await start(f, executor, "request_read_failed", false, 0, "1", 1); + await child.reached(); + const first = ( + await system.query("select inputs from dbos.workflow_status where workflow_uuid=$1", [ + workflowId, + ]) + ).rows[0].inputs; + expect(JSON.parse(first).json[0].deadlineMs).toBeNumber(); + await child.stop(); + child = await start(f, executor, undefined, false, 931000); + await child.ready(); + await until( + "expired denial", + async () => + !!( + await system.query( + "select 1 from mcp_test.completions where session_id=$1 and call_id=$2", + [f.sessionId, f.request.toolCallId], + ) + ).rowCount, + ); + expect(await callFor(f)).toBeUndefined(); + expect((await f.command("stats")).calls).toBe(0); + const completion = ( + await system.query( + "select result_json from mcp_test.completions where session_id=$1 and call_id=$2", + [f.sessionId, f.request.toolCallId], + ) + ).rows[0]; + expect(JSON.parse(completion.result_json).denial).toBe("deadline_expired"); + const restored = ( + await system.query("select inputs from dbos.workflow_status where workflow_uuid=$1", [ + workflowId, + ]) + ).rows[0].inputs; + expect(restored).toBe(first); + } finally { + await child?.stop(); + await system.query("delete from mcp_test.completions where session_id=$1", [f.sessionId]); + await f.cleanup(); + } +}, 30000); + +test("changed duplicate after accepted completion retains the first workflow and result", async () => { + const f = await effectFixture(); + let child: Awaited> | undefined; + try { + child = await start(f, `mcp-test-${crypto.randomUUID()}`); + await child.ready(); + await until("first receipt", async () => { + const row = await callFor(f); + return row?.state === "succeeded" && row.completionEnqueuedVersion === row.version; + }); + const first = (await callFor(f))!; + const delivered = ( + await system.query( + "select result_json,attempts from mcp_test.completions where session_id=$1 and call_id=$2", + [f.sessionId, f.request.toolCallId], + ) + ).rows[0]; + await child.stop(); + f.request.runId = "changed-run"; + f.request.originalArguments = "{invalid-duplicate}"; + child = await start(f, `mcp-test-${crypto.randomUUID()}`, undefined, false, 0, "2"); + await child.ready(); + expect(await callFor(f)).toEqual(first); + expect( + ( + await system.query( + "select result_json,attempts from mcp_test.completions where session_id=$1 and call_id=$2", + [f.sessionId, f.request.toolCallId], + ) + ).rows[0], + ).toEqual(delivered); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await child?.stop(); + await system.query("delete from mcp_test.completions where session_id=$1", [f.sessionId]); + await f.cleanup(); + } +}, 30000); diff --git a/orchestrator/src/__tests__/mcp-discovered-catalog.test.ts b/orchestrator/src/__tests__/mcp-discovered-catalog.test.ts new file mode 100644 index 000000000..3066329d8 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-discovered-catalog.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from "bun:test"; +import { discoveredCatalog } from "../mcp/discovery.ts"; +test("discovery projects only bounded reviewed catalog input and rejects injection", () => { + const input = { + name: "lookup", + inputSchema: { type: "object" }, + annotations: { readOnlyHint: true }, + }; + expect(discoveredCatalog([input])).toEqual([ + { upstreamName: "lookup", description: "", inputSchema: { type: "object" } }, + ]); + for (const value of [ + [input, input], + [{ ...input, inputSchema: { $ref: "https://schema.example.test/" } }], + [{ ...input, name: "bad\nname" }], + [{ ...input, endpoint: "https://other.example.test/" }], + [{ ...input, description: "x".repeat(16385) }], + ]) + expect(() => discoveredCatalog(value)).toThrow(); +}); diff --git a/orchestrator/src/__tests__/mcp-discovery-admission.test.ts b/orchestrator/src/__tests__/mcp-discovery-admission.test.ts new file mode 100644 index 000000000..371c14bca --- /dev/null +++ b/orchestrator/src/__tests__/mcp-discovery-admission.test.ts @@ -0,0 +1,410 @@ +import { expect, test } from "bun:test"; +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { unwrapCredentialControlClient } from "../mcp/control-client.ts"; +import { create } from "@bufbuild/protobuf"; +import { eq } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/node-postgres"; +import { Pool } from "pg"; +import * as schema from "../db/schema.ts"; +import { getDb } from "../db/client.ts"; +import { + CredentialControlService, + CredentialKind, + CredentialStatusSchema, +} from "../gen/engram/mcp/v1/control_pb.ts"; +import { statusMetadata } from "../mcp/credential-store.ts"; +import { makeMcpDiscoveryAdmission } from "../mcp/discovery-admission.ts"; +import type { AdmittedCredentialOperation } from "../mcp/credential-broker.ts"; +import { makeCredentialBroker } from "../mcp/credential-broker.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; + +async function fixture() { + const { app } = await verifyMcpTestDatabases(); + const db = getDb(), + now = new Date("2026-01-01T00:00:00Z"); + const actorId = crypto.randomUUID(), + connectionId = crypto.randomUUID(), + deploymentId = crypto.randomUUID(); + const identity = { + protocol: 1 as const, + bootId: crypto.randomUUID(), + zoneId: "public", + policyDigest: canonicalDigest({ policy: 1 }), + policyRevision: "1", + }; + const status = create(CredentialStatusSchema, { + protocol: 1, + status: "connected", + version: 1n, + context: { + connectionId, + configRevision: 1n, + authorizationGeneration: 2n, + credentialIdentity: "account", + resource: { endpoint: "https://tools.example.test/mcp", zoneId: "public" }, + policyDigest: identity.policyDigest, + policyRevision: "1", + }, + reference: { kind: CredentialKind.API_KEY, connectionId, name: "mcp.org.key", generation: 1n }, + }); + await db.insert(schema.user).values({ + id: actorId, + name: "Discovery administrator", + email: `${actorId}@example.test`, + role: "admin", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.integrationConnection).values({ + id: connectionId, + alias: `discovery-${connectionId}`, + displayName: "Discovery tools", + provider: "mcp", + configRevision: 1, + authGeneration: 2, + credentialIdentity: "account", + config: { + endpoint: "https://tools.example.test/mcp", + zone: "public", + auth: { kind: "api_key", secretRef: "mcp.org.key" }, + }, + mcpCredential: { status: statusMetadata(status) }, + }); + const input = { + requestId: crypto.randomUUID(), + actorId, + connectionId, + deploymentId, + configRevision: 1, + authGeneration: 2, + credentialIdentity: "account", + identity, + destination: "https://tools.example.test/mcp", + }; + const admission = makeMcpDiscoveryAdmission(db); + const deployments = new Set([deploymentId]); + return { + app, + db, + now, + input, + identity, + status, + admission, + deployments, + operation(intent: Awaited>): AdmittedCredentialOperation { + return { + identity, + status, + intent, + operation: { kind: "discover", endpoint: input.destination }, + }; + }, + async cleanup() { + await db + .delete(schema.mcpDiscoveryAdmission) + .where(eq(schema.mcpDiscoveryAdmission.connectionId, connectionId)); + await db + .delete(schema.integrationConnection) + .where(eq(schema.integrationConnection.id, connectionId)); + await db.delete(schema.user).where(eq(schema.user.id, actorId)); + for (const id of deployments) + await db.delete(schema.mcpAdmissionScope).where(eq(schema.mcpAdmissionScope.id, id)); + }, + }; +} + +test("durable discovery has a one-use dispatch barrier, immutable identity and fixed expiry", async () => { + const f = await fixture(); + try { + const intent = await f.admission.reserve(f.input, f.now), + operation = f.operation(intent); + expect(intent.scope).toEqual({ + kind: "administration", + actorId: f.input.actorId, + requestId: f.input.requestId, + }); + expect(intent.expiresAt).toBe(f.now.getTime() + 30_000); + await expect(f.admission.assertAdmitted(operation, f.now)).rejects.toThrow(); + await f.admission.beginDispatch(f.input.requestId, intent.id, f.now); + await expect(f.admission.beginDispatch(f.input.requestId, intent.id, f.now)).rejects.toThrow(); + await f.admission.assertAdmitted(operation, f.now); + await f.admission.assertAdmitted(operation, f.now); + await expect( + f.db + .update(schema.mcpDiscoveryAdmission) + .set({ destination: "https://other.example.test/mcp" }) + .where(eq(schema.mcpDiscoveryAdmission.requestId, f.input.requestId)) + .execute(), + ).rejects.toThrow(); + await expect(f.admission.reserve(f.input, f.now)).rejects.toThrow("consumed"); + await expect( + f.admission.reserve({ ...f.input, destination: "https://other.example.test/mcp" }, f.now), + ).rejects.toThrow(); + await expect( + f.admission.assertAdmitted(operation, new Date(intent.expiresAt)), + ).rejects.toThrow(); + await f.admission.finish(f.input.requestId, intent.id, f.now); + await f.admission.finish(f.input.requestId, intent.id, f.now); + await expect(f.admission.assertAdmitted(operation, f.now)).rejects.toThrow(); + await expect(f.admission.reserve(f.input, new Date(intent.expiresAt + 1))).rejects.toThrow( + "consumed", + ); + await expect( + f.db + .update(schema.mcpDiscoveryAdmission) + .set({ state: "reserved" }) + .where(eq(schema.mcpDiscoveryAdmission.requestId, f.input.requestId)) + .execute(), + ).rejects.toThrow(); + const next = await f.admission.reserve({ ...f.input, requestId: crypto.randomUUID() }, f.now); + expect(next.id).not.toBe(intent.id); + } finally { + await f.cleanup(); + } +}); + +test("discovery checks live admin, config, authorization, account and exact worker operation on both broker fences", async () => { + const f = await fixture(); + try { + const intent = await f.admission.reserve(f.input, f.now), + op = f.operation(intent); + await f.admission.beginDispatch(f.input.requestId, intent.id, f.now); + for (const changed of [ + { ...op, identity: { ...op.identity, bootId: "other" } }, + { ...op, identity: { ...op.identity, policyRevision: "2" } }, + { ...op, intent: { ...intent, destination: "https://other.example.test/mcp" } }, + { ...op, intent: { ...intent, requestDigest: canonicalDigest({ changed: true }) } }, + { + ...op, + intent: { + ...intent, + scope: { + kind: "administration" as const, + requestId: f.input.requestId, + actorId: "other", + }, + }, + }, + { + ...op, + operation: { kind: "discover" as const, endpoint: "https://other.example.test/mcp" }, + }, + { + ...op, + status: create(CredentialStatusSchema, { + ...f.status, + context: { ...f.status.context!, authorizationGeneration: 3n }, + }), + }, + ]) + await expect(f.admission.assertAdmitted(changed, f.now)).rejects.toThrow(); + for (const patch of [ + { role: "user" }, + { banned: true }, + { email: "apikey+fixture@service.local" }, + ]) { + await f.db.update(schema.user).set(patch).where(eq(schema.user.id, f.input.actorId)); + await expect(f.admission.assertAdmitted(op, f.now)).rejects.toThrow(); + await f.db + .update(schema.user) + .set({ role: "admin", banned: false, email: `${f.input.actorId}@example.test` }) + .where(eq(schema.user.id, f.input.actorId)); + } + for (const patch of [ + { configRevision: 2 }, + { authGeneration: 3 }, + { credentialIdentity: "different-account" }, + { revoked: true }, + { + mcpCredential: { + transition: { id: "transition", actorId: f.input.actorId, startedAt: f.now.getTime() }, + }, + }, + ]) { + await f.db + .update(schema.integrationConnection) + .set(patch) + .where(eq(schema.integrationConnection.id, f.input.connectionId)); + await expect(f.admission.assertAdmitted(op, f.now)).rejects.toThrow(); + await f.db + .update(schema.integrationConnection) + .set({ + configRevision: 1, + authGeneration: 2, + credentialIdentity: "account", + revoked: false, + mcpCredential: { status: statusMetadata(f.status) }, + }) + .where(eq(schema.integrationConnection.id, f.input.connectionId)); + } + let leases = 0, + sends = 0; + const broker = makeCredentialBroker({ + admission: { assertAdmitted: (operation) => f.admission.assertAdmitted(operation, f.now) }, + now: () => f.now.getTime(), + worker: { + async identity() { + return f.identity; + }, + async dispatch() { + sends++; + throw Error("unexpected send"); + }, + }, + control: unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => + router.service(CredentialControlService, { + async lease() { + leases++; + await f.db + .update(schema.user) + .set({ role: "user" }) + .where(eq(schema.user.id, f.input.actorId)); + return { + protocol: 1, + intentId: intent.id, + expiresAt: BigInt(f.now.getTime() + 10_000), + operationExpiresAt: BigInt(intent.expiresAt), + header: "authorization", + value: "fixture-credential", + }; + }, + }), + ), + ), + ), + }); + await expect(broker.dispatch(op)).rejects.toThrow(); + expect(leases).toBe(1); + expect(sends).toBe(0); + await expect(broker.dispatch(op)).rejects.toThrow(); + expect(leases).toBe(1); + } finally { + await f.cleanup(); + } +}); + +test("discovery connection budget is global across deployments, clients and fixed death expiry", async () => { + const f = await fixture(); + const pool = new Pool({ connectionString: f.app, max: 10 }); + try { + const other = makeMcpDiscoveryAdmission(drizzle(pool, { schema })); + const inputs = Array.from({ length: 6 }, () => ({ + ...f.input, + requestId: crypto.randomUUID(), + deploymentId: crypto.randomUUID(), + })); + inputs.forEach((input) => f.deployments.add(input.deploymentId)); + const results = await Promise.allSettled( + inputs.map((input, i) => (i % 2 ? other : f.admission).reserve(input, f.now)), + ); + expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1); + expect(results.filter((r) => r.status === "rejected")).toHaveLength(5); + const winner = results.find((r) => r.status === "fulfilled")!; + if (winner.status !== "fulfilled") throw Error("missing winner"); + const later = new Date(f.now.getTime() + 30_000); + await expect( + other.beginDispatch( + (winner.value.scope as { requestId: string }).requestId, + winner.value.id, + later, + ), + ).rejects.toThrow(); + expect( + (await other.reserve({ ...f.input, requestId: crypto.randomUUID() }, later)).expiresAt, + ).toBe(later.getTime() + 30_000); + } finally { + await pool.end(); + await f.cleanup(); + } +}); + +test("discovery deployment budget admits four independent connections with no queue", async () => { + const fixtures = []; + try { + for (let i = 0; i < 5; i++) fixtures.push(await fixture()); + const first = fixtures[0]!; + const states = await Promise.allSettled( + fixtures.map((f) => + f.admission.reserve({ ...f.input, deploymentId: first.input.deploymentId }, f.now), + ), + ); + expect(states.filter((r) => r.status === "fulfilled")).toHaveLength(4); + expect(states.filter((r) => r.status === "rejected")).toHaveLength(1); + const winner = states.findIndex((r) => r.status === "fulfilled"), + rejected = states.findIndex((r) => r.status === "rejected"); + const intent = states[winner]!; + if (intent.status !== "fulfilled") throw Error("missing winner"); + await first.admission.finish(fixtures[winner]!.input.requestId, intent.value.id, first.now); + await fixtures[rejected]!.admission.reserve( + { ...fixtures[rejected]!.input, deploymentId: first.input.deploymentId }, + first.now, + ); + // Delete exact owned rows before their shared deployment scope. + for (const f of fixtures) + await f.db + .delete(schema.mcpDiscoveryAdmission) + .where(eq(schema.mcpDiscoveryAdmission.connectionId, f.input.connectionId)); + } finally { + for (const f of fixtures.slice().reverse()) await f.cleanup(); + } +}); + +test("administrative schema admission is credential-free, one-use, bounded and exact", async () => { + const f = await fixture(); + try { + const documents = [{ type: "object", properties: { value: { type: "integer" } } }]; + const input = { ...f.input, documents }; + const intents = []; + for (let i = 0; i < 4; i++) + intents.push( + await f.admission.reserveSchema({ ...input, requestId: crypto.randomUUID() }, f.now), + ); + await expect( + f.admission.reserveSchema({ ...input, requestId: crypto.randomUUID() }, f.now), + ).rejects.toThrow("capacity"); + // Schema capacity does not consume discovery capacity, including this connection. + const discovery = await f.admission.reserve(f.input, f.now); + const intent = intents[0]!; + expect(intent.operation).toBe("validateSchemas"); + expect(intent.destination).toBe(""); + if (intent.scope.kind !== "administration") throw Error("scope"); + const requestId = intent.scope.requestId; + const operation = { identity: input.identity, intent, documents }; + await expect(f.admission.assertSchemaAdmitted(operation, f.now)).rejects.toThrow(); + await f.admission.beginDispatch(requestId, intent.id, f.now); + await f.admission.assertSchemaAdmitted(operation, f.now); + await expect(f.admission.beginDispatch(requestId, intent.id, f.now)).rejects.toThrow(); + await expect( + f.admission.assertSchemaAdmitted({ ...operation, documents: [{ type: "string" }] }, f.now), + ).rejects.toThrow(); + await expect( + f.admission.assertSchemaAdmitted( + { ...operation, identity: { ...input.identity, bootId: "changed" } }, + f.now, + ), + ).rejects.toThrow(); + await expect( + f.admission.assertSchemaAdmitted(operation, new Date(intent.expiresAt)), + ).rejects.toThrow(); + await f.db.update(schema.user).set({ role: "user" }).where(eq(schema.user.id, input.actorId)); + await expect(f.admission.assertSchemaAdmitted(operation, f.now)).rejects.toThrow(); + await f.db.update(schema.user).set({ role: "admin" }).where(eq(schema.user.id, input.actorId)); + await f.admission.finish(requestId, intent.id, f.now); + await expect(f.admission.reserveSchema({ ...input, requestId }, f.now)).rejects.toThrow( + "consumed", + ); + await expect(f.admission.assertSchemaAdmitted(operation, f.now)).rejects.toThrow(); + await f.admission.reserveSchema( + { ...input, requestId: crypto.randomUUID() }, + new Date(intent.expiresAt), + ); + await f.admission.finish(f.input.requestId, discovery.id, f.now); + } finally { + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-domain.test.ts b/orchestrator/src/__tests__/mcp-domain.test.ts new file mode 100644 index 000000000..2391af6fb --- /dev/null +++ b/orchestrator/src/__tests__/mcp-domain.test.ts @@ -0,0 +1,291 @@ +import { describe, expect, test } from "bun:test"; +import { validateMcpConfig } from "../mcp/config.ts"; +import { reviewCatalog } from "../mcp/catalog.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { compileBindings, assertBindingCurrent } from "../mcp/bindings.ts"; +import { validateMcpGrant } from "../mcp/types.ts"; + +const config = { + endpoint: "https://tools.example.test/mcp", + zone: "public", + auth: { kind: "api_key" as const, secretRef: "mcp.service.api-key" }, +}; +const tool = { + upstreamName: "Search.Documents", + description: "Search documents", + inputSchema: { type: "object" }, + classification: "read" as const, + enabled: true, +}; +const state = { + configRevision: 1, + authGeneration: 1, + credentialIdentity: "account-a", + catalogRevision: 1, + revoked: false, +}; +const grant = { + connectionId: "c1", + operation: "invoke", + credentialScope: "org" as const, + resourceConstraints: [], + mcp: { + upstreamName: tool.upstreamName, + automationWrite: false, + argumentConstraintDocuments: [], + unrestricted: true, + }, +}; + +describe("MCP domain", () => { + test("accepts canonical endpoint and references only", () => { + expect(validateMcpConfig(config)).toEqual(config); + expect( + validateMcpConfig({ ...config, endpoint: "https://tools/mcp", zone: "internal" }).zone, + ).toBe("internal"); + expect(() => + validateMcpConfig({ ...config, endpoint: "https://bad..example.test/mcp" }), + ).toThrow(); + for (const endpoint of [ + "http://tools.example.test/mcp", + "https://127.0.0.1/mcp", + "https://[::1]/mcp", + "https://u:p@tools.example.test/mcp", + "https://tools.example.test/mcp?q=1", + "https://tools.example.test/mcp#x", + "https://tools.example.test:443/mcp", + ]) { + expect(() => validateMcpConfig({ ...config, endpoint })).toThrow(); + } + expect(() => validateMcpConfig({ ...config, token: "secret" })).toThrow(); + expect(() => + validateMcpConfig({ + ...config, + auth: { kind: "api_key" as const, secretRef: "unrelated.key" }, + }), + ).toThrow(); + }); + test("requires explicit tool scope and unrestricted selection", () => { + expect(() => validateMcpGrant(grant)).not.toThrow(); + expect(() => validateMcpGrant({ ...grant, credentialScope: "user" })).toThrow(); + expect(() => + validateMcpGrant({ ...grant, mcp: { ...grant.mcp, unrestricted: false } }), + ).toThrow(); + expect(() => validateMcpGrant({ ...grant, operation: tool.upstreamName })).toThrow(); + expect(() => + validateMcpGrant({ + ...grant, + mcp: { ...grant.mcp, unrestricted: false, argumentConstraintDocuments: [{}] }, + }), + ).toThrow(); + expect(() => + validateMcpGrant({ + ...grant, + mcp: { ...grant.mcp, argumentConstraintDocuments: [{ type: "object" }] }, + }), + ).toThrow(); + }); + test("catalog fingerprints bind upstream schema and explicit review", () => { + const reviewed = reviewCatalog([tool]); + expect(reviewed[0]!.fingerprint).toBe(canonicalDigest(tool)); + expect(() => reviewCatalog([{ ...tool, classification: undefined }])).toThrow(); + expect(() => reviewCatalog([tool, tool])).toThrow(); + expect(() => + reviewCatalog([{ ...tool, inputSchema: { $ref: "https://example.test/schema" } }]), + ).toThrow(); + }); + test("freezes names and authority and rejects live changes", () => { + const bindings = compileBindings([ + { + id: "c1", + alias: "docs", + config, + state, + enabled: true, + tools: reviewCatalog([tool]), + grants: [grant], + }, + ]); + expect(bindings[0]!.tools[0]!.name).toMatch(/^mcp_docs_/); + expect(() => assertBindingCurrent(bindings[0]!, { ...state, enabled: true })).not.toThrow(); + expect(() => + assertBindingCurrent(bindings[0]!, { ...state, enabled: true, authGeneration: 2 }), + ).toThrow(); + expect(() => assertBindingCurrent(bindings[0]!, { ...state, enabled: false })).toThrow(); + const before = bindings[0]!.hash; + config.auth.secretRef = "mcp.service.replacement"; + expect(bindings[0]!.config.auth).toEqual({ + kind: "api_key" as const, + secretRef: "mcp.service.api-key", + }); + expect(bindings[0]!.hash).toBe(before); + config.auth.secretRef = "mcp.service.api-key"; + }); + test("canonical hash rejects unsupported and non-finite JSON", () => { + expect(canonicalDigest({ b: 1, a: 2 })).toBe(canonicalDigest({ a: 2, b: 1 })); + expect(() => canonicalDigest({ n: NaN })).toThrow(); + expect(() => canonicalDigest({ n: undefined })).toThrow(); + }); +}); + +test("MCP child grants intersect constraints and automation permission", async () => { + const { intersectMcpGrants } = await import("../mcp/bindings.ts"); + const parent = { + ...grant, + mcp: { + ...grant.mcp, + unrestricted: false, + argumentConstraintDocuments: [ + { properties: { project: { const: "allowed" } }, required: ["project"] }, + ], + }, + }; + const child = { ...grant, mcp: { ...grant.mcp, automationWrite: true } }; + const [effective] = intersectMcpGrants([parent], [child]); + expect(effective!.mcp!.automationWrite).toBe(false); + expect(effective!.mcp!.argumentConstraintDocuments).toEqual( + parent.mcp.argumentConstraintDocuments, + ); + expect(() => intersectMcpGrants([], [child])).toThrow(); +}); + +test("MCP grants cannot become generic guest capabilities", async () => { + const { grantsToCapabilities } = await import("../integrations/grants.ts"); + const connection = { + id: "c1", + alias: "docs", + provider: "mcp", + displayName: "Docs", + config, + enabled: true, + isDefault: false, + testedAt: null, + createdAt: new Date(0), + updatedAt: new Date(0), + }; + expect(grantsToCapabilities([{ grant, connection }])).toEqual([]); +}); + +test("grant intersection removes only exact canonical constraint duplicates", async () => { + const { intersectMcpGrants } = await import("../mcp/bindings.ts"); + const first = { type: "object", properties: { count: { minimum: 1, maximum: 5 } } }; + const duplicate = { properties: { count: { maximum: 5, minimum: 1 } }, type: "object" }; + const distinct = { type: "object", properties: { count: { maximum: 3 } } }; + const constrained = (documents: Record[]) => ({ + ...grant, + mcp: { ...grant.mcp, unrestricted: false, argumentConstraintDocuments: documents }, + }); + const [effective] = intersectMcpGrants( + [constrained([first, first])], + [constrained([duplicate, distinct, distinct])], + ); + expect(effective!.mcp!.argumentConstraintDocuments).toEqual([first, distinct]); +}); + +test("schema keyword checks do not reinterpret ordinary property names or constants", async () => { + const { assertSchemaShape } = await import("../mcp/catalog.ts"); + expect(() => + assertSchemaShape({ + type: "object", + properties: { $ref: { type: "string" } }, + const: { $schema: "ordinary data" }, + }), + ).not.toThrow(); + expect(() => + assertSchemaShape({ properties: { child: { $ref: "https://example.test/schema" } } }), + ).toThrow("local"); +}); + +test("child intersections preserve independent local references with colliding definition names", async () => { + const { intersectMcpGrants } = await import("../mcp/bindings.ts"); + const parentSchema = { + $schema: "http://json-schema.org/draft-07/schema#", + $id: "https://schemas.example.test/constraint", + $defs: { item: { const: "allowed" } }, + type: "object", + properties: { project: { $ref: "#/$defs/item" } }, + required: ["project"], + }; + const childSchema = { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: "https://schemas.example.test/constraint", + $defs: { item: { type: "integer", minimum: 1, maximum: 5 } }, + type: "object", + properties: { count: { $ref: "#/$defs/item" } }, + required: ["count"], + }; + const parent = { + ...grant, + mcp: { ...grant.mcp, unrestricted: false, argumentConstraintDocuments: [parentSchema] }, + }; + const child = { + ...grant, + mcp: { ...grant.mcp, unrestricted: false, argumentConstraintDocuments: [childSchema] }, + }; + const [effective] = intersectMcpGrants([parent], [child]); + expect(effective!.mcp).toHaveProperty("argumentConstraintDocuments", [parentSchema, childSchema]); + expect(effective!.mcp!.unrestricted).toBe(false); + const documents = effective!.mcp!.argumentConstraintDocuments; + const resolve = (document: Record, pointer: string): unknown => + pointer + .slice(2) + .split("/") + .reduce((value, key) => { + if (!value || typeof value !== "object" || Array.isArray(value)) + throw new Error("missing reference target"); + return (value as Record)[key]; + }, document); + expect(resolve(documents[0]!, parentSchema.properties.project.$ref)).toEqual({ + const: "allowed", + }); + expect(resolve(documents[1]!, childSchema.properties.count.$ref)).toEqual({ + type: "integer", + minimum: 1, + maximum: 5, + }); + const [descendant] = intersectMcpGrants([effective!], [grant]); + expect(descendant!.mcp!.argumentConstraintDocuments).toEqual(documents); + const [binding] = compileBindings([ + { + id: "c1", + alias: "docs", + config, + state, + enabled: true, + tools: reviewCatalog([tool]), + grants: [effective!], + }, + ]); + expect(binding!.tools[0]!.grant.argumentConstraintDocuments).toEqual(documents); + parentSchema.$defs.item.const = "changed"; + childSchema.$defs.item.maximum = 100; + expect(resolve(documents[0]!, "#/$defs/item")).toEqual({ const: "allowed" }); + expect(resolve(documents[1]!, "#/$defs/item")).toEqual({ + type: "integer", + minimum: 1, + maximum: 5, + }); +}); +test("API-key draft has no reference and cannot produce a ready binding", () => { + const draft = { ...config, auth: { kind: "api_key" as const, secretRef: "" } }; + expect(validateMcpConfig(draft)).toEqual(draft); + expect( + validateMcpConfig({ + ...config, + auth: { kind: "api_key", secretRef: "mcp-coordinator_generated" }, + }).auth.kind, + ).toBe("api_key"); + expect(() => + compileBindings([ + { + id: "c1", + alias: "tools", + enabled: true, + state, + config: draft, + grants: [grant], + tools: reviewCatalog([tool]), + }, + ]), + ).toThrow("unavailable"); +}); diff --git a/orchestrator/src/__tests__/mcp-effect-driver.test.ts b/orchestrator/src/__tests__/mcp-effect-driver.test.ts new file mode 100644 index 000000000..7ccedc68f --- /dev/null +++ b/orchestrator/src/__tests__/mcp-effect-driver.test.ts @@ -0,0 +1,823 @@ +import { canonicalDigest } from "../mcp/hash.ts"; +import { mcpExecution } from "../db/schema.ts"; +import { expect, test } from "bun:test"; +import { effectFixture } from "./mcp-effect-fixture.ts"; +import { reviewContext } from "../mcp/journal-review.ts"; + +test("actual worker read in unknown policy preserves raw args, schema lease and redacted outcome once", async () => { + const f = await effectFixture({ eligibility: "unknown" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + expect(call.originalArguments).toBe(f.request.originalArguments); + const prepared = await f.driver.driveOnce(call.id); + expect(prepared.state).toBe("succeeded"); + expect(prepared.result).toMatchObject({ + outcome: "succeeded", + delivery: "available", + output: { isError: false, structuredContent: { echo: "[redacted] | [redacted]" } }, + }); + expect(JSON.stringify(prepared)).not.toContain("known-literal-token"); + expect((await f.command("stats")).calls).toBe(1); + expect(f.leases).toBe(1); + expect(f.events.at(-1)).toBe("read-fence"); + expect((await f.driver.acceptToolRequest(f.request)).id).toBe(call.id); + expect((await f.driver.driveOnce(call.id)).state).toBe("succeeded"); + expect((await f.command("stats")).calls).toBe(1); + await expect( + f.driver.acceptToolRequest({ ...f.request, originalArguments: '{"x":1}' }), + ).rejects.toThrow(); + } finally { + await f.cleanup(); + } +}, 20000); + +test("write schema releases capacity before approval and uses a distinct execute generation", async () => { + const f = await effectFixture({ classification: "write" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + expect((await f.driver.driveOnce(call.id)).state).toBe("waiting_approval"); + const schema = (await f.journal.getAdmission(call.id))!; + expect(schema.state).toBe("released"); + expect(schema.schemaDigest).toBe(call.requestDigest); + expect(f.leases).toBe(0); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); + let executeId = ""; + f.beforeLease(async () => { + executeId = (await f.journal.getAdmission(call.id))!.reservationId; + }); + expect((await f.driver.driveOnce(call.id)).state).toBe("succeeded"); + expect(executeId).not.toBe(schema.reservationId); + expect(f.events.at(-1)).toBe("write-fence"); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await f.cleanup(); + } +}, 20000); + +for (const response of ["image", "error", "malformed", "oversize", "normalized-limit"] as const) + test(`actual worker ${response} retains outcome and delivery without replay`, async () => { + const f = await effectFixture({ classification: "write", response }); + try { + const call = await f.driver.acceptToolRequest(f.request); + await f.driver.driveOnce(call.id); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); + const result = await f.driver.driveOnce(call.id); + expect(result.state).toBe( + response === "error" + ? "failed" + : response === "malformed" || response === "oversize" + ? "outcome_unknown" + : "succeeded", + ); + expect(result.resultUnavailable).toBe( + response === "image" || response === "normalized-limit", + ); + expect(JSON.stringify(result)).not.toContain("known-literal-token"); + if (response === "error") + expect(result.result).toMatchObject({ + output: { isError: true, structuredContent: { echo: "[redacted] | [redacted]" } }, + }); + await f.driver.driveOnce(call.id); + expect((await f.command("stats")).calls).toBe(1); + if (result.state === "outcome_unknown") { + const next = { ...f.request, toolCallId: "call-2" }; + f.requests.set(next.toolCallId, next); + await expect(f.driver.acceptToolRequest(next)).rejects.toThrow("write authority"); + } + } finally { + await f.cleanup(); + } + }, 20000); + +test("plan reads pass while plan and unknown writes deny before schema or secrets", async () => { + for (const classification of ["read", "write"] as const) { + const f = await effectFixture({ classification, eligibility: "plan" }); + try { + if (classification === "read") { + const call = await f.driver.acceptToolRequest(f.request); + expect((await f.driver.driveOnce(call.id)).state).toBe("succeeded"); + } else { + await expect(f.driver.acceptToolRequest(f.request)).rejects.toThrow(); + f.setEligibility("unknown"); + await expect(f.driver.acceptToolRequest(f.request)).rejects.toThrow(); + expect(f.leases).toBe(0); + expect((await f.command("stats")).credentialRequests).toBe(0); + } + } finally { + await f.cleanup(); + } + } +}, 20000); + +test("independent input constraints reject without approval or credentials", async () => { + const f = await effectFixture({ + classification: "write", + constraints: [ + { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: "https://schema.example/constraint", + properties: { x: { maximum: 0 } }, + }, + ], + }); + try { + const call = await f.driver.acceptToolRequest(f.request); + const denied = await f.driver.driveOnce(call.id); + expect(denied.state).toBe("failed"); + expect(denied.intentId).toBeNull(); + expect(denied.result).toMatchObject({ + source: "local_validation", + delivery: "available", + output: { isError: true }, + }); + expect(await f.journal.getAdmission(call.id)).toBeNull(); + expect(await f.journal.pending(f.sessionId, f.actorId, new Date())).toHaveLength(0); + expect(f.leases).toBe(0); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("output schema compiles separately and failed output validation preserves known success", async () => { + const f = await effectFixture({ + outputSchema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + required: ["missing"], + }, + }); + try { + const call = await f.driver.acceptToolRequest(f.request); + const result = await f.driver.driveOnce(call.id); + expect(result.state).toBe("succeeded"); + expect(result.resultUnavailable).toBe(true); + expect(result.result).toMatchObject({ reason: "output_schema" }); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await f.cleanup(); + } +}, 20000); + +test("changed raw bytes and oversized arguments never acquire credentials", async () => { + const f = await effectFixture(); + try { + await expect( + f.driver.acceptToolRequest({ ...f.request, originalArguments: '{"x":1}' }), + ).rejects.toThrow("authority"); + await expect( + f.driver.acceptToolRequest({ + ...f.request, + originalArguments: JSON.stringify({ x: "x".repeat(256 * 1024) }), + }), + ).rejects.toThrow("limit"); + expect(f.leases).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("duplicate concurrent drive crosses one barrier and calls upstream once", async () => { + const f = await effectFixture({ classification: "write" }); + try { + const [a, b] = await Promise.all([ + f.driver.acceptToolRequest(f.request), + f.driver.acceptToolRequest(f.request), + ]); + expect(a.id).toBe(b.id); + expect(a.deadline).toEqual(b.deadline); + await f.driver.driveOnce(a.id); + const approval = await f.journal.requestApproval(a.id, f.deploymentId, new Date()); + await f.journal.decide(a.id, f.actorId, true, reviewContext(a, approval), new Date()); + const results = await Promise.allSettled([f.driver.driveOnce(a.id), f.driver.driveOnce(a.id)]); + expect(results.some((r) => r.status === "fulfilled" && r.value.state === "succeeded")).toBe( + true, + ); + expect((await f.command("stats")).calls).toBe(1); + expect(f.leases).toBe(1); + } finally { + await f.cleanup(); + } +}, 20000); + +test("credential lease rotation preserves frozen binding and redacts the new token", async () => { + const f = await effectFixture(); + try { + const first = await f.driver.acceptToolRequest(f.request); + expect((await f.driver.driveOnce(first.id)).state).toBe("succeeded"); + f.refresh(); + const request = { ...f.request, toolCallId: "call-2" }; + f.requests.set(request.toolCallId, request); + const second = await f.driver.acceptToolRequest(request); + expect(second.bindingHash).toBe(first.bindingHash); + expect(second.requestContext?.resolvedTool.binding).toEqual( + first.requestContext?.resolvedTool.binding, + ); + const result = await f.driver.driveOnce(second.id); + expect(result.state).toBe("succeeded"); + expect(JSON.stringify(result)).not.toContain("refreshed-literal-token"); + expect((await f.command("stats")).calls).toBe(2); + } finally { + await f.cleanup(); + } +}, 20000); + +import { eq } from "drizzle-orm"; +import { integrationConnection, task, taskSession, user } from "../db/schema.ts"; +import { statusMetadata } from "../mcp/credential-store.ts"; +import { makeProfileStore } from "../db/profiles.ts"; + +for (const change of [ + "ban", + "owner", + "principal", + "publication", + "account", + "generation", + "endpoint", + "zone", +] as const) + test(`live ${change} change denies the frozen call before credential lease`, async () => { + const f = await effectFixture(); + try { + const call = await f.driver.acceptToolRequest(f.request); + if (change === "ban") + await f.db.update(user).set({ banned: true }).where(eq(user.id, f.actorId)); + else if (change === "owner") + await f.db.update(task).set({ createdByUserId: null }).where(eq(task.id, f.taskId)); + else if (change === "principal") + await f.db + .update(taskSession) + .set({ integrationPrincipalId: "other" }) + .where(eq(taskSession.sessionId, f.sessionId)); + else if (change === "publication") { + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + const profiles = makeProfileStore(f.db); + await profiles.update(f.profileId, (await profiles.get(f.profileId))!, { + actorId: f.actorId, + expectedRevision: 1, + }); + await f.db.update(user).set({ role: "user" }).where(eq(user.id, f.actorId)); + } else if (change === "account") + await f.db + .update(integrationConnection) + .set({ credentialIdentity: "different-account-scope" }) + .where(eq(integrationConnection.id, f.connection.id)); + else if (change === "generation") + await f.db + .update(integrationConnection) + .set({ authGeneration: call.authGeneration + 1 }) + .where(eq(integrationConnection.id, f.connection.id)); + else if (change === "endpoint") + await f.db + .update(integrationConnection) + .set({ config: { ...f.connection.config, endpoint: "https://changed.example.test/mcp" } }) + .where(eq(integrationConnection.id, f.connection.id)); + else f.operatorPolicies[0]!.policyDigest = `sha256:${"a".repeat(64)}`; + // A duplicate returns the original authority even after revocation, but cannot dispatch it. + expect((await f.driver.acceptToolRequest(f.request)).requestDigest).toBe(call.requestDigest); + await expect(f.driver.driveOnce(call.id)).rejects.toThrow(); + expect(f.leases).toBe(0); + expect((await f.command("stats")).credentialRequests).toBe(0); + } finally { + await f.cleanup(); + } + }, 20000); + +test("changed worker boot between barrier and broker identity releases zero credentials", async () => { + const f = await effectFixture(); + try { + const call = await f.driver.acceptToolRequest(f.request); + let changed = false; + f.beforeFence(async () => { + if (!changed && (await f.journal.get(call.id))?.state === "dispatching") { + changed = true; + await f.command("restart"); + } + }); + expect((await f.driver.driveOnce(call.id)).state).toBe("failed"); + expect(changed).toBe(true); + expect(f.leases).toBe(0); + expect((await f.command("stats")).credentialRequests).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("raw coordinator publication policy mismatch is denied before secret leasing", async () => { + const f = await effectFixture(); + try { + const call = await f.driver.acceptToolRequest(f.request); + f.status.context!.policyDigest = "a".repeat(64); + await f.db + .update(integrationConnection) + .set({ mcpCredential: { status: statusMetadata(f.status) } }) + .where(eq(integrationConnection.id, f.connection.id)); + expect((await f.driver.driveOnce(call.id)).state).toBe("failed"); + expect(f.leases).toBe(0); + expect((await f.command("stats")).credentialRequests).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("revocation during credential lease is caught by the second same-generation assertion", async () => { + const f = await effectFixture({ classification: "write" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + await f.driver.driveOnce(call.id); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); + f.afterLease(async () => { + await f.db.update(user).set({ banned: true }).where(eq(user.id, f.actorId)); + }); + expect((await f.driver.driveOnce(call.id)).state).toBe("outcome_unknown"); + expect(f.leases).toBe(1); + expect((await f.command("stats")).credentialRequests).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("final coordinator fence after a lease blocks changed authority", async () => { + const f = await effectFixture(); + try { + const call = await f.driver.acceptToolRequest(f.request); + let postLeaseFences = 0; + f.beforeFence(async () => { + if (f.leases && ++postLeaseFences === 2) f.setFence(false); + }); + expect((await f.driver.driveOnce(call.id)).state).toBe("failed"); + expect(postLeaseFences).toBe(2); + expect((await f.command("stats")).credentialRequests).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("global kill during final fence prevents the immediate send", async () => { + const f = await effectFixture(); + try { + const call = await f.driver.acceptToolRequest(f.request); + let postLeaseFences = 0; + f.beforeFence(async () => { + if (f.leases && ++postLeaseFences === 2) f.setEnabled(false); + }); + expect((await f.driver.driveOnce(call.id)).state).toBe("failed"); + expect((await f.command("stats")).credentialRequests).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("unpublished legacy human grants require explicit administrator publication", async () => { + const f = await effectFixture({ published: false }); + try { + await expect(f.driver.acceptToolRequest(f.request)).rejects.toThrow( + "administrator publication", + ); + expect(f.leases).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("trusted shorter deadline freezes once and duplicate delivery cannot extend it", async () => { + const f = await effectFixture(); + try { + const deadline = new Date(Date.now() + 5000); + const call = await f.driver.acceptToolRequest(f.request, deadline); + expect(call.deadline).toEqual(deadline); + expect( + (await f.driver.acceptToolRequest(f.request, new Date(Date.now() + 60000))).deadline, + ).toEqual(deadline); + expect((await f.driver.driveOnce(call.id)).state).toBe("succeeded"); + } finally { + await f.cleanup(); + } +}, 20000); + +test("an unresolved write blocks new writes but permits the separately reviewed read", async () => { + const f = await effectFixture({ classification: "write", response: "malformed" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + await f.driver.driveOnce(call.id); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); + expect((await f.driver.driveOnce(call.id)).state).toBe("outcome_unknown"); + const readName = f.snapshot.immutableBindings[0]!.tools.find( + (tool) => tool.tool.classification === "read", + )!.name; + const readRequest = { ...f.request, toolCallId: "read-after-unknown", modelToolName: readName }; + f.requests.set(readRequest.toolCallId, readRequest); + f.setEligibility("plan"); + const read = await f.driver.acceptToolRequest(readRequest); + // The fixture still returns malformed output. It proves admission and one read call. + await f.driver.driveOnce(read.id); + expect((await f.command("stats")).calls).toBe(2); + expect(f.leases).toBe(2); + } finally { + await f.cleanup(); + } +}, 20000); + +test("cancel after provider send leaves an unknown write that cannot replay", async () => { + const f = await effectFixture({ classification: "write", response: "hold" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + await f.driver.driveOnce(call.id); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); + const controller = new AbortController(); + const pending = f.driver.driveOnce(call.id, controller.signal); + const until = Date.now() + 5000; + while (!(await f.command("stats")).calls && Date.now() < until) + await new Promise((resolve) => setTimeout(resolve, 10)); + expect((await f.command("stats")).calls).toBe(1); + await f.journal.cancel(call.id, f.actorId, new Date(), (await f.journal.get(call.id))!.version); + controller.abort(); + expect((await pending).state).toBe("outcome_unknown"); + await f.driver.driveOnce(call.id); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await f.cleanup(); + } +}, 20000); + +import { automation, automationRun, automationSession } from "../db/schema.ts"; + +test("published automation grant uses exact live automation linkage without human approval", async () => { + const f = await effectFixture({ classification: "write" }); + const automationId = crypto.randomUUID(), + runId = crypto.randomUUID(); + try { + await f.db + .insert(automation) + .values({ id: automationId, name: "Fixture automation", createdByUserId: f.actorId }); + await f.db.insert(automationRun).values({ + id: runId, + automationId, + taskId: f.taskId, + sessionId: f.sessionId, + status: "running", + trigger: { source: "manual", receivedAt: new Date().toISOString() }, + }); + await f.db + .insert(automationSession) + .values({ sessionId: f.sessionId, runId, blockId: "start" }); + await f.db + .update(taskSession) + .set({ integrationPrincipalId: `automation:${automationId}` }) + .where(eq(taskSession.sessionId, f.sessionId)); + await f.db.update(task).set({ createdByUserId: null }).where(eq(task.id, f.taskId)); + const call = await f.driver.acceptToolRequest(f.request); + expect(call.authorizationKind).toBe("automation"); + expect((await f.driver.driveOnce(call.id)).state).toBe("succeeded"); + const request = { ...f.request, toolCallId: "automation-next" }; + f.requests.set(request.toolCallId, request); + const next = await f.driver.acceptToolRequest(request); + await f.db.update(automation).set({ enabled: false }).where(eq(automation.id, automationId)); + await expect(f.driver.driveOnce(next.id)).rejects.toThrow("automation"); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await f.db.delete(automation).where(eq(automation.id, automationId)); + await f.cleanup(); + } +}, 20000); + +test("live admin role is required when the immutable principal is not the current owner", async () => { + const f = await effectFixture(); + const adminId = crypto.randomUUID(); + try { + await f.db.insert(user).values({ + id: adminId, + name: "Fixture admin", + email: `${adminId}@example.test`, + role: "admin", + createdAt: new Date(), + updatedAt: new Date(), + }); + await f.db + .update(taskSession) + .set({ integrationPrincipalId: adminId }) + .where(eq(taskSession.sessionId, f.sessionId)); + const call = await f.driver.acceptToolRequest(f.request); + await f.db.update(user).set({ role: "user" }).where(eq(user.id, adminId)); + await expect(f.driver.driveOnce(call.id)).rejects.toThrow(); + expect(f.leases).toBe(0); + } finally { + await f.cleanup(); + await f.db.delete(user).where(eq(user.id, adminId)); + } +}, 20000); + +import { create } from "@bufbuild/protobuf"; +import { McpWritePolicyRefSchema } from "../gen/engram/app/v1/session_pb.ts"; +import { bindingSnapshotColumns, readLiveBindingAuthority } from "../mcp/binding-store.ts"; +import { compileSessionBindings } from "../mcp/session-bindings.ts"; + +test("live ancestor restriction denies child execution", async () => { + const f = await effectFixture(); + const parentId = crypto.randomUUID(), + parentTaskId = crypto.randomUUID(); + try { + const [leaf] = await f.db + .select() + .from(taskSession) + .where(eq(taskSession.sessionId, f.sessionId)); + await f.db.insert(task).values({ id: parentTaskId, type: "chat", createdByUserId: f.actorId }); + await f.db.insert(taskSession).values({ + taskId: parentTaskId, + sessionId: parentId, + profileId: f.profileId, + integrationPrincipalId: f.actorId, + ...bindingSnapshotColumns(f.snapshot, leaf!.integrationConnections!), + }); + const authority = await readLiveBindingAuthority(f.profileId, f.operatorPolicies, f.db); + const child = compileSessionBindings({ + launchPolicy: { profileId: f.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + requestedGrants: authority.profileGrants, + parentAuthority: { sessionId: parentId, snapshot: f.snapshot }, + }); + await f.db.update(task).set({ spawningSessionId: parentId }).where(eq(task.id, f.taskId)); + await f.db + .update(taskSession) + .set(bindingSnapshotColumns(child, leaf!.integrationConnections!)) + .where(eq(taskSession.sessionId, f.sessionId)); + const parentRef = create(McpWritePolicyRefSchema, { + sessionId: parentId, + bindingEpoch: 0n, + policyRevision: 1n, + }); + f.policies.set(parentId, { ref: parentRef, ancestorPolicyRefs: [] }); + f.policies.get(f.sessionId)!.ancestorPolicyRefs = [parentRef]; + await f.resolver.readCurrent(f.sessionId); + const call = await f.driver.acceptToolRequest(f.request); + await f.db.update(task).set({ status: "failed" }).where(eq(task.id, parentTaskId)); + await expect(f.driver.driveOnce(call.id)).rejects.toThrow(); + expect(f.leases).toBe(0); + } finally { + await f.db.delete(task).where(eq(task.id, f.taskId)); + await f.db.delete(task).where(eq(task.id, parentTaskId)); + await f.cleanup(); + } +}, 20000); + +test("credential assertion rejects an altered execution reservation after leasing", async () => { + const f = await effectFixture(); + try { + const call = await f.driver.acceptToolRequest(f.request); + const { mcpAdmission } = await import("../db/schema.ts"); + f.afterLease(async () => { + await f.db + .update(mcpAdmission) + .set({ reservationId: crypto.randomUUID() }) + .where(eq(mcpAdmission.executionId, call.id)); + }); + expect((await f.driver.driveOnce(call.id)).state).toBe("failed"); + expect(f.leases).toBe(1); + expect((await f.command("stats")).credentialRequests).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("credential delivery expiry during only the final fence prevents send and replay", async () => { + const f = await effectFixture({ classification: "write" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + await f.driver.driveOnce(call.id); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); + let postLeaseFences = 0; + f.beforeFence(async () => { + if (f.leases && ++postLeaseFences === 2) f.advanceNow(11000); + }); + const result = await f.driver.driveOnce(call.id); + expect(postLeaseFences).toBe(2); + // Only the ten-second credential handoff expires; the full tool deadline stays live. + expect(result.intentExpiresAt!.getTime() - (Date.now() + 11000)).toBeGreaterThan(45000); + expect(result.state).toBe("outcome_unknown"); + expect(f.leases).toBe(1); + expect((await f.command("stats")).credentialRequests).toBe(0); + expect((await f.command("stats")).calls).toBe(0); + expect((await f.driver.acceptToolRequest(f.request)).id).toBe(call.id); + expect((await f.driver.driveOnce(call.id)).state).toBe("outcome_unknown"); + expect(f.leases).toBe(1); + expect((await f.command("stats")).credentialRequests).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("durable cancellation aborts an active provider call without a notification", async () => { + const f = await effectFixture({ classification: "write", response: "hold" }); + const controller = new AbortController(); + try { + const call = await f.driver.acceptToolRequest(f.request); + await f.driver.driveOnce(call.id); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); + const pending = f.driver.driveOnce(call.id, controller.signal); + const until = Date.now() + 5000; + while (!(await f.command("stats")).calls && Date.now() < until) + await new Promise((resolve) => setTimeout(resolve, 10)); + expect((await f.command("stats")).calls).toBe(1); + const dispatched = (await f.journal.get(call.id))!; + await f.journal.cancel(call.id, f.actorId, new Date(), dispatched.version); + const result = await Promise.race([ + pending, + new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 2500)), + ]); + expect(result).not.toBe("timeout"); + if (result !== "timeout") expect(result.state).toBe("outcome_unknown"); + expect((await f.command("stats")).calls).toBe(1); + } finally { + controller.abort(); + await f.cleanup(); + } +}, 20000); + +test("canceling a dispatched read does not invent an upstream failure", async () => { + const f = await effectFixture({ classification: "read", response: "hold" }); + const controller = new AbortController(); + try { + const call = await f.driver.acceptToolRequest(f.request); + const pending = f.driver.driveOnce(call.id, controller.signal); + const until = Date.now() + 5000; + while (!(await f.command("stats")).calls && Date.now() < until) + await new Promise((resolve) => setTimeout(resolve, 10)); + expect((await f.command("stats")).calls).toBe(1); + const dispatched = (await f.journal.get(call.id))!; + await f.journal.cancel(call.id, f.actorId, new Date(), dispatched.version); + const result = await Promise.race([ + pending, + new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 2500)), + ]); + expect(result).not.toBe("timeout"); + if (result !== "timeout") expect(result.state).toBe("outcome_unknown"); + } finally { + controller.abort(); + await f.cleanup(); + } +}, 20000); + +test("lifecycle reconciliation preserves a current newer execution", async () => { + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + await f.driver.reconcile(call.id); + expect((await f.journal.get(call.id))!.state).toBe("authorized"); + expect((await f.driver.driveOnce(call.id)).state).toBe("succeeded"); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await f.cleanup(); + } +}, 20000); + +test("lifecycle reconciliation cancels an old run before it can dispatch", async () => { + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + f.setFence(false); + await f.driver.reconcile(call.id); + expect((await f.journal.get(call.id))!.state).toBe("canceled"); + expect((await f.driver.driveOnce(call.id)).state).toBe("canceled"); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("lifecycle reconciliation cancels after the principal loses access", async () => { + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + await f.db.update(user).set({ banned: true }).where(eq(user.id, f.actorId)); + await f.driver.reconcile(call.id); + expect((await f.journal.get(call.id))!.state).toBe("canceled"); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("unavailable coordinator fence leaves execution pending for retry", async () => { + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + f.beforeFence(async () => { + throw Error("temporary control outage"); + }); + await expect(f.driver.reconcile(call.id)).rejects.toThrow(); + expect((await f.journal.get(call.id))!.state).toBe("authorized"); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("an older resume reconciles current authority without canceling a newer run", async () => { + const { makeMcpLifecycleEvents } = await import("../mcp/lifecycle-events.ts"); + const { consumerCursor } = await import("../db/schema.ts"); + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + const lifecycle = makeMcpLifecycleEvents({ + db: f.db, + now: () => new Date(), + list: async (_id, after) => + after < 1n ? [{ idx: 1n, kind: "resumed", payloadJson: "{}" }] : [], + reconcile: async () => f.driver.reconcile(call.id), + }); + await lifecycle.runOnce(f.sessionId); + expect((await f.journal.get(call.id))!.state).toBe("authorized"); + expect((await f.driver.driveOnce(call.id)).state).toBe("succeeded"); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await f.db.delete(consumerCursor).where(eq(consumerCursor.sessionId, f.sessionId)); + await f.cleanup(); + } +}, 20000); + +test("a late old-run acceptance is canceled by independent reconciliation", async () => { + const { makeMcpLifecycleEvents } = await import("../mcp/lifecycle-events.ts"); + const { consumerCursor } = await import("../db/schema.ts"); + const f = await effectFixture({ classification: "read" }); + try { + const lifecycle = makeMcpLifecycleEvents({ + db: f.db, + now: () => new Date(), + list: async (_id, after) => + after < 1n ? [{ idx: 1n, kind: "run_interrupted", payloadJson: '{"run_id":"run-1"}' }] : [], + reconcile: async () => {}, + }); + await lifecycle.runOnce(f.sessionId); + const call = await f.driver.acceptToolRequest(f.request); + f.setFence(false); + await f.driver.reconcile(call.id); + expect((await f.journal.get(call.id))!.state).toBe("canceled"); + expect((await f.driver.driveOnce(call.id)).state).toBe("canceled"); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.db.delete(consumerCursor).where(eq(consumerCursor.sessionId, f.sessionId)); + await f.cleanup(); + } +}, 20000); + +test("unsafe raw write arguments reject before journal, lease or worker I/O", async () => { + const f = await effectFixture({ classification: "write" }); + try { + for (const token of ["9007199254740993", "-9007199254740993", "9.007199254740993e15"]) { + const request = { ...f.request, originalArguments: `{"x":${token}}` }; + f.requests.set(request.toolCallId, request); + await expect(f.driver.acceptToolRequest(request)).rejects.toMatchObject({ + code: "invalid_arguments", + }); + await expect( + f.journal.create( + { + ...f.input, + originalArguments: request.originalArguments, + argumentHash: canonicalDigest(JSON.parse(request.originalArguments)), + }, + { kind: "approval" }, + f.now, + ), + ).rejects.toThrow("invalid argument JSON"); + expect( + await f.db.select().from(mcpExecution).where(eq(mcpExecution.sessionId, f.sessionId)), + ).toHaveLength(0); + expect(f.leases).toBe(0); + expect(await f.command("stats")).toMatchObject({ + calls: 0, + credentialRequests: 0, + }); + } + } finally { + await f.cleanup(); + } +}, 20000); + +test("safe boundary write reaches the serialized provider operation unchanged", async () => { + const f = await effectFixture({ classification: "write" }); + try { + f.request.originalArguments = '{ "x": 9007199254740991 }'; + const call = await f.driver.acceptToolRequest(f.request); + expect(call.originalArguments).toBe(f.request.originalArguments); + await f.driver.driveOnce(call.id); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()); + expect((await f.driver.driveOnce(call.id)).state).toBe("succeeded"); + const stats = await f.command("stats"); + expect(stats.calls).toBe(1); + expect(JSON.parse(stats.lastCall).params.arguments.x).toBe(9007199254740991); + expect(stats.lastCall).toContain('"x":9007199254740991'); + } finally { + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-effect-fixture.test.ts b/orchestrator/src/__tests__/mcp-effect-fixture.test.ts new file mode 100644 index 000000000..e2faace60 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-effect-fixture.test.ts @@ -0,0 +1,52 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { effectFixture } from "./mcp-effect-fixture.ts"; + +test("failed worker version setup closes the child before the fixture rejects", async () => { + const dir = await mkdtemp(join(tmpdir(), "mcp-effect-setup-")); + const executable = join(dir, "worker"), + pidFile = join(dir, "pid"); + const previous = process.env.MCP_WORKER_TEST_NODE; + let pid: number | undefined; + try { + await writeFile( + executable, + `#!/usr/bin/env node +const fs = require("node:fs"); +fs.writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); +process.stdin.resume(); +console.log(JSON.stringify({ node: "unsupported", port: 0, identity: {} })); +setInterval(() => {}, 1000); +`, + { mode: 0o700 }, + ); + process.env.MCP_WORKER_TEST_NODE = executable; + await expect(effectFixture()).rejects.toThrow(); + pid = Number(await readFile(pidFile, "utf8")); + expect(() => process.kill(pid!, 0)).toThrow(); + } finally { + if (previous === undefined) delete process.env.MCP_WORKER_TEST_NODE; + else process.env.MCP_WORKER_TEST_NODE = previous; + if (pid !== undefined) { + try { + process.kill(pid, "SIGKILL"); + } catch { + /* Already reaped. */ + } + } + await rm(dir, { recursive: true, force: true }); + } +}, 10000); + +test("spawn failure settles without waiting for a missing child exit", async () => { + const previous = process.env.MCP_WORKER_TEST_NODE; + try { + process.env.MCP_WORKER_TEST_NODE = "/missing-mcp-fixture-node"; + await expect(effectFixture()).rejects.toThrow(); + } finally { + if (previous === undefined) delete process.env.MCP_WORKER_TEST_NODE; + else process.env.MCP_WORKER_TEST_NODE = previous; + } +}, 10000); diff --git a/orchestrator/src/__tests__/mcp-effect-fixture.ts b/orchestrator/src/__tests__/mcp-effect-fixture.ts new file mode 100644 index 000000000..e489a071a --- /dev/null +++ b/orchestrator/src/__tests__/mcp-effect-fixture.ts @@ -0,0 +1,418 @@ +import { operatorPublication } from "./mcp-binding-fixture.ts"; +import { expect } from "bun:test"; +import { spawn } from "node:child_process"; +import { createInterface } from "node:readline"; +import { create, toJson } from "@bufbuild/protobuf"; +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { eq } from "drizzle-orm"; +import { workerTls } from "../../../mcp-worker/test/fixtures/worker-tls.ts"; +import type { WorkerIdentity } from "../../../mcp-worker/src/contracts.ts"; +import { reviewFixture } from "./mcp-review-fixture.ts"; +import { makeProfileStore } from "../db/profiles.ts"; +import { user, taskSession, integrationConnection, profile } from "../db/schema.ts"; +import { compileSessionBindings } from "../mcp/session-bindings.ts"; +import { + bindingSnapshotColumns, + readLiveBindingAuthority, + makeStoredSessionBindingResolver, +} from "../mcp/binding-store.ts"; +import { + CredentialControlService, + CredentialKind, + CredentialStatusSchema, +} from "../gen/engram/mcp/v1/control_pb.ts"; +import { SessionPolicyService } from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import { + McpWritePolicyRefSchema, + type McpWritePolicyRef, +} from "../gen/engram/app/v1/session_pb.ts"; +import { statusMetadata } from "../mcp/credential-store.ts"; +import { unwrapCredentialControlClient } from "../mcp/control-client.ts"; +import { makeMcpEffectDriver } from "../mcp/effect-driver.ts"; + +export async function effectFixture( + options: { + classification?: "read" | "write"; + response?: string; + constraints?: Record[]; + outputSchema?: Record; + eligibility?: string; + published?: boolean; + bindingEpoch?: number; + automationWrite?: boolean; + } = {}, +) { + const tls = await workerTls("localhost"); + const child = spawn( + process.env.MCP_WORKER_TEST_NODE ?? "node", + [new URL("../../../mcp-worker/test/fixtures/effect-worker.ts", import.meta.url).pathname], + { + stdio: ["pipe", "pipe", "pipe"], + env: { PATH: process.env.PATH ?? "", TMPDIR: process.env.TMPDIR ?? "/tmp" }, + }, + ); + const output = createInterface({ input: child.stdout }), + lines = output[Symbol.asyncIterator](); + let stderr = ""; + child.stderr.on("data", (data) => { + stderr += String(data); + }); + // Register completion before setup so spawn errors and early exits cannot miss cleanup. + const closed = new Promise((resolve) => child.once("close", () => resolve())); + let childError: Error | undefined; + child.on("error", (error) => { + childError = error; + }); + child.stdin.on("error", (error) => { + childError = error; + }); + let setupFixture: Awaited> | undefined; + async function stopWorker(graceful: boolean) { + const timeout = setTimeout(() => child.kill("SIGKILL"), 5000); + try { + if (child.exitCode === null && child.signalCode === null) { + if (graceful) child.stdin.end("close\n"); + else child.kill("SIGKILL"); + } + await closed; + } finally { + clearTimeout(timeout); + output.close(); + child.stdin.destroy(); + child.stdout.destroy(); + child.stderr.destroy(); + } + } + async function cleanup(graceful: boolean) { + try { + await setupFixture?.cleanup(); + } finally { + await stopWorker(graceful); + } + if (graceful) expect(stderr).toBe(""); + } + async function next() { + let timeout: ReturnType | undefined; + try { + const line = await Promise.race([ + lines.next(), + closed.then(() => { + throw Error("fixture worker exited"); + }), + new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(Error("fixture worker response timeout")), 5000); + }), + ]); + if (childError || line.done) throw Error("fixture worker unavailable"); + return JSON.parse(line.value); + } finally { + clearTimeout(timeout); + } + } + try { + child.stdin.write(`${JSON.stringify({ tls, response: options.response })}\n`); + const ready: { port: number; identity: WorkerIdentity; node: string } = await next(); + expect(ready.node.startsWith("24.")).toBe(true); + const f = await reviewFixture(options.constraints, options.classification === "write" ? 2 : 1); + setupFixture = f; + const classification = options.classification ?? "read"; + const tools = f.authority.profileGrants.map((grant) => ({ + upstreamName: grant.mcp!.upstreamName, + description: "Fixture operation", + inputSchema: { type: "object", properties: { x: { type: "integer" } }, required: ["x"] }, + ...(options.outputSchema ? { outputSchema: options.outputSchema } : {}), + })); + const current = (await f.connections.get(f.connection.id))!; + const draft = await f.repository.saveDraft( + current.id, + current.mcp!, + tools, + f.actorId, + new Date(), + ); + await f.repository.publish( + current.id, + draft.id, + 1, + tools.map((tool, index) => ({ + ...tool, + classification: index === 0 ? classification : "read", + enabled: true, + })), + f.publisherId, + new Date(), + ); + await f.repository.setEnabled(current.id, 1, true, new Date()); + const profiles = makeProfileStore(f.db); + if (options.automationWrite !== undefined) { + for (const grant of f.authority.profileGrants) + grant.mcp.automationWrite = options.automationWrite; + await f.db + .update(profile) + .set({ integrationGrants: f.authority.profileGrants }) + .where(eq(profile.id, f.profileId)); + } + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + if (options.published !== false) + await profiles.update(f.profileId, (await profiles.get(f.profileId))!, { + actorId: f.actorId, + expectedRevision: 0, + }); + await f.db.update(user).set({ role: "user" }).where(eq(user.id, f.actorId)); + const operatorPolicies = [ + { + publication: structuredClone(operatorPublication), + zoneId: ready.identity.zoneId, + policyRevision: ready.identity.policyRevision, + policyDigest: `sha256:${ready.identity.policyDigest}`, + authorities: ["tools.example.test"], + aliases: [], + }, + ]; + const authority = await readLiveBindingAuthority(f.profileId, operatorPolicies, f.db); + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: f.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: options.bindingEpoch ?? 0, + }); + await f.db + .update(taskSession) + .set( + bindingSnapshotColumns(snapshot, [ + { + id: current.id, + alias: current.alias, + provider: "mcp", + displayName: current.displayName, + config: current.config, + }, + ]), + ) + .where(eq(taskSession.sessionId, f.sessionId)); + const binding = snapshot.immutableBindings[0]!; + const request = { + sessionId: f.sessionId, + runId: "run-1", + toolCallId: "call-1", + modelToolName: binding.tools.find((bound) => bound.tool.upstreamName === "Write.Record")! + .name, + originalArguments: '{ "x": 1 }', + }; + const requests = new Map([[request.toolCallId, request]]); + const events: string[] = []; + let eligibility = options.eligibility ?? "write_eligible", + fenceCurrent = true, + token = "known-literal-token", + leases = 0; + let enabled = true; + let authorityFailures = 0; + let clockOffsetMs = 0; + let beforeFence: (() => Promise) | undefined; + let beforeLease: (() => Promise) | undefined; + let afterLease: (() => Promise) | undefined; + const policies = new Map< + string, + { ref: McpWritePolicyRef; ancestorPolicyRefs: McpWritePolicyRef[] } + >([ + [ + f.sessionId, + { + ref: create(McpWritePolicyRefSchema, { + ...f.input.requestPolicyContext.acceptedPolicyRef, + bindingEpoch: BigInt(snapshot.bindingEpoch), + }), + ancestorPolicyRefs: f.input.requestPolicyContext.ancestorPolicyRefs, + }, + ], + ]); + const policy = createClient( + SessionPolicyService, + createRouterTransport((router) => + router.service(SessionPolicyService, { + async readWritePolicy(req) { + const refs = policies.get(req.sessionId)!; + return { + state: { + policy: { + current: true, + initialized: eligibility !== "unknown", + writeEligibility: eligibility, + ...refs, + }, + }, + }; + }, + async validateExecutionIdentity(req) { + events.push("read-fence"); + await beforeFence?.(); + const original = requests.get(req.toolCallId); + return { + protocol: 1, + status: + fenceCurrent && + original?.originalArguments === req.originalArguments && + original?.modelToolName === req.modelToolName + ? "current" + : "stale", + }; + }, + async validateDispatchFence(req) { + events.push("write-fence"); + await beforeFence?.(); + const original = requests.get(req.toolCallId); + return { + protocol: 1, + status: + fenceCurrent && + eligibility === "write_eligible" && + original?.originalArguments === req.originalArguments && + original?.modelToolName === req.modelToolName + ? "current" + : "stale", + }; + }, + }), + ), + ); + const status = create(CredentialStatusSchema, { + protocol: 1, + status: "connected", + version: 1n, + context: { + connectionId: current.id, + configRevision: BigInt(binding.configRevision), + authorizationGeneration: BigInt(binding.authGeneration), + credentialIdentity: binding.credentialIdentity, + resource: { endpoint: binding.config.endpoint, zoneId: binding.config.zone }, + policyDigest: ready.identity.policyDigest, + policyRevision: ready.identity.policyRevision, + }, + reference: { + kind: CredentialKind.API_KEY, + connectionId: current.id, + name: "mcp.test.key", + generation: 1n, + }, + }); + await f.db + .update(integrationConnection) + .set({ mcpCredential: { status: statusMetadata(status) } }) + .where(eq(integrationConnection.id, current.id)); + const control = unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => + router.service(CredentialControlService, { + async lease(req) { + events.push("lease"); + await beforeLease?.(); + leases++; + await afterLease?.(); + return { + protocol: 1, + intentId: req.intentId, + expiresAt: req.expiresAt, + operationExpiresAt: req.operationExpiresAt, + header: "authorization", + value: `Bearer ${token}`, + }; + }, + }), + ), + ), + ); + // Operator policy, host enforcement ACK, and coordinator history are fixture trusted boundaries. + const resolver = makeStoredSessionBindingResolver( + { + control: policy, + readOperatorPolicies: async () => { + if (authorityFailures > 0) { + authorityFailures--; + throw Error("private-authority-transport-error"); + } + return operatorPolicies; + }, + enforceManagedDeny: async (expected) => expected, + }, + f.db, + ); + const driver = makeMcpEffectDriver({ + db: f.db, + deploymentId: f.deploymentId, + control, + policy, + resolver, + workerTls: () => ({ + ...tls, + url: `https://127.0.0.1:${ready.port}`, + serverFingerprint: tls.fingerprint, + }), + enabled: () => enabled, + now: () => new Date(Date.now() + clockOffsetMs), + }); + return { + ...f, + driver, + control, + remote: { + workerTls: { + ...tls, + url: `https://127.0.0.1:${ready.port}`, + serverFingerprint: tls.fingerprint, + }, + operatorPolicies, + status: toJson(CredentialStatusSchema, status), + eligibility, + policyRef: toJson(McpWritePolicyRefSchema, f.input.requestPolicyContext.acceptedPolicyRef), + }, + resolver, + request, + requests, + snapshot, + status, + events, + policies, + operatorPolicies, + advanceNow(milliseconds: number) { + clockOffsetMs += milliseconds; + }, + failAuthorityReads(count: number) { + authorityFailures = count; + }, + setEnabled(value: boolean) { + enabled = value; + }, + beforeFence(fn: () => Promise) { + beforeFence = fn; + }, + get leases() { + return leases; + }, + setEligibility(value: string) { + eligibility = value; + }, + setFence(value: boolean) { + fenceCurrent = value; + }, + beforeLease(fn: () => Promise) { + beforeLease = fn; + }, + afterLease(fn: () => Promise) { + afterLease = fn; + }, + refresh() { + token = "refreshed-literal-token"; + }, + async command(command: string) { + child.stdin.write(`${command}\n`); + return next(); + }, + async cleanup() { + await cleanup(true); + }, + }; + } catch (error) { + await cleanup(false); + throw error; + } +} diff --git a/orchestrator/src/__tests__/mcp-effect-result.test.ts b/orchestrator/src/__tests__/mcp-effect-result.test.ts new file mode 100644 index 000000000..c7af285fd --- /dev/null +++ b/orchestrator/src/__tests__/mcp-effect-result.test.ts @@ -0,0 +1,56 @@ +import { expect, test } from "bun:test"; +import { verifiedExecutionResult } from "../mcp/effect-driver.ts"; + +test("verified result keeps upstream outcome and delivery separate", () => { + expect( + verifiedExecutionResult({ + outcome: "succeeded", + delivery: "unavailable", + reason: "output_schema", + }), + ).toEqual({ + state: "succeeded", + result: { outcome: "succeeded", delivery: "unavailable", reason: "output_schema" }, + resultUnavailable: true, + }); + const error = { + outcome: "failed", + delivery: "available", + output: { + content: [{ type: "text", text: "rejected" }], + structuredContent: { code: 1 }, + isError: true, + }, + }; + expect(verifiedExecutionResult(error)).toEqual({ + state: "failed", + result: error, + resultUnavailable: false, + }); + expect( + verifiedExecutionResult({ + outcome: "outcome_unknown", + delivery: "unavailable", + reason: "upstream_failure", + }).state, + ).toBe("outcome_unknown"); +}); + +test("malformed and contradictory outcomes never establish success", () => { + for (const value of [ + null, + {}, + { outcome: "succeeded" }, + { + outcome: "succeeded", + delivery: "available", + output: { content: [{ type: "image", data: "AA==" }] }, + }, + { outcome: "succeeded", delivery: "available", output: { content: [], isError: true } }, + { outcome: "outcome_unknown", delivery: "available", output: { content: [] } }, + { outcome: "failed", delivery: "unavailable", reason: "invented" }, + { outcome: "succeeded", delivery: "unavailable", reason: "output_schema", secret: "untrusted" }, + ]) { + expect(() => verifiedExecutionResult(value)).toThrow(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-global-admission.test.ts b/orchestrator/src/__tests__/mcp-global-admission.test.ts new file mode 100644 index 000000000..c4e10071b --- /dev/null +++ b/orchestrator/src/__tests__/mcp-global-admission.test.ts @@ -0,0 +1,167 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/node-postgres"; +import { Pool } from "pg"; +import * as schema from "../db/schema.ts"; +import { makeMcpAdmission } from "../mcp/admission.ts"; +import { reviewFixture } from "./mcp-review-fixture.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; + +for (const limit of ["session", "connection"] as const) { + test(`global ${limit} budget serializes independent clients across worker deployments`, async () => { + const { app } = await verifyMcpTestDatabases(); + const f = await reviewFixture(); + const pool = new Pool({ connectionString: app, max: 10 }); + const other = makeMcpAdmission(drizzle(pool, { schema })); + const local = makeMcpAdmission(f.db); + const maximum = limit === "session" ? 4 : 8; + const deployments = Array.from({ length: maximum + 1 }, () => crypto.randomUUID()); + try { + const calls = []; + for (let i = 0; i < maximum + 1; i++) { + const toolCallId = `global-${i}`; + const call = await f.journal.create( + { + ...f.input, + toolCallId, + requestPolicyContext: { ...f.input.requestPolicyContext, toolCallId }, + }, + { kind: "approval" }, + f.now, + ); + if (limit === "connection") { + const independentSessionCall = { + ...call, + id: crypto.randomUUID(), + sessionId: crypto.randomUUID(), + }; + await f.db.insert(schema.taskSession).values({ + taskId: f.taskId, + sessionId: independentSessionCall.sessionId, + profileId: f.profileId, + integrationPrincipalId: f.actorId, + }); + await f.db.insert(schema.mcpExecution).values(independentSessionCall); + calls.push(independentSessionCall); + continue; + } + calls.push(call); + } + const states = await Promise.all( + calls + .slice(0, maximum + 1) + .map((call, i) => + (i % 2 ? other : local).reserve(call.id, deployments[i]!, "schema", f.now), + ), + ); + expect(states.filter((state) => state === "admitted")).toHaveLength(maximum); + expect(states.filter((state) => state === "queued")).toHaveLength(1); + // A separate session and connection can still progress while the first is saturated. + const independent = await reviewFixture(); + try { + const call = await independent.journal.create( + independent.input, + { kind: "approval" }, + f.now, + ); + expect(await other.reserve(call.id, independent.deploymentId, "schema", f.now)).toBe( + "admitted", + ); + } finally { + await independent.cleanup(); + } + } finally { + await f.db.delete(schema.task).where(eq(schema.task.id, f.taskId)); + for (const id of deployments) + await f.db.delete(schema.mcpAdmissionScope).where(eq(schema.mcpAdmissionScope.id, id)); + await f.cleanup(); + await pool.end(); + } + }); +} + +// Exercise the production reserve and cancellation transactions, including the +// connection FOR KEY SHARE lock PostgreSQL takes for the first admission INSERT. +test("initial admission and cancellation keep connection before execution without an FK deadlock", async () => { + const { app } = await verifyMcpTestDatabases(); + const f = await reviewFixture(); + const pool = new Pool({ connectionString: app, max: 2 }); + const first = await pool.connect(), + second = await pool.connect(); + const { makeMcpJournal } = await import("../mcp/journal.ts"); + function signal() { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; + } + const executionLocked = signal(), + cancellationConnectionAttempt = signal(), + cancellationExecutionAttempt = signal(); + let admissionHasConnectionLock = false; + function intercept(client: typeof first, lane: "admission" | "cancel") { + return new Proxy(client, { + get(target, key, receiver) { + if (key !== "query") return Reflect.get(target, key, receiver); + return async (query: string | import("pg").QueryConfig, values?: unknown[]) => { + const text = typeof query === "string" ? query : query.text; + const locksConnection = + text.includes('from "integration_connection"') && text.includes(" for "); + const locksExecution = + text.includes('from "mcp_execution"') && text.endsWith(" for update"); + if (lane === "cancel" && locksConnection) cancellationConnectionAttempt.resolve(); + if (lane === "cancel" && locksExecution) cancellationExecutionAttempt.resolve(); + const result = await target.query(query, values); + if (lane === "admission" && locksConnection) admissionHasConnectionLock = true; + if (lane === "admission" && locksExecution) { + executionLocked.resolve(); + // With connection-first locking cancellation waits on the connection. + // Without it, let cancellation lock the connection and wait on this + // execution before the admission INSERT checks its connection FK. + await (admissionHasConnectionLock + ? cancellationConnectionAttempt.promise + : cancellationExecutionAttempt.promise); + } + return result; + }; + }, + }); + } + try { + await first.query("set statement_timeout = '3s'"); + await second.query("set statement_timeout = '3s'"); + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + const admission = makeMcpAdmission(drizzle(intercept(first, "admission"), { schema })); + const cancellation = makeMcpJournal(drizzle(intercept(second, "cancel"), { schema })); + const reserved = admission.reserve(call.id, f.deploymentId, "schema", f.now); + const canceled = executionLocked.promise.then(() => + cancellation.cancelInternal(call.id, "interrupt", f.now), + ); + const results = await Promise.allSettled([reserved, canceled]); + function errorCode(error: unknown): string { + if (error && typeof error === "object") { + if ("code" in error && typeof error.code === "string") return error.code; + if ("cause" in error) return errorCode(error.cause); + } + return "unknown"; + } + expect( + results.map((result) => + result.status === "fulfilled" ? "fulfilled" : errorCode(result.reason), + ), + ).toEqual(["fulfilled", "fulfilled"]); + expect(admissionHasConnectionLock).toBe(true); + expect(await f.journal.getAdmission(call.id)).toBeNull(); + const [stored] = await f.db + .select() + .from(schema.mcpExecution) + .where(eq(schema.mcpExecution.id, call.id)); + expect(stored!.state).toBe("canceled"); + } finally { + first.release(); + second.release(); + await pool.end(); + await f.cleanup(); + } +}, 8_000); diff --git a/orchestrator/src/__tests__/mcp-inherited-write-fence.test.ts b/orchestrator/src/__tests__/mcp-inherited-write-fence.test.ts new file mode 100644 index 000000000..28dc10d1c --- /dev/null +++ b/orchestrator/src/__tests__/mcp-inherited-write-fence.test.ts @@ -0,0 +1,580 @@ +import { expect, test } from "bun:test"; +import { create } from "@bufbuild/protobuf"; +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { eq, sql } from "drizzle-orm"; +import { reviewFixture } from "./mcp-review-fixture.ts"; +import { makeProfileStore } from "../db/profiles.ts"; +import { task, taskSession, user, mcpExecution } from "../db/schema.ts"; +import { compileSessionBindings, type McpSessionBindingSnapshot } from "../mcp/session-bindings.ts"; +import { + bindingSnapshotColumns, + readLiveBindingAuthority, + makeStoredSessionBindingResolver, +} from "../mcp/binding-store.ts"; +import { compileChildSessionBindings } from "../mcp/session-child.ts"; +import { resolveSnapshotTool } from "../mcp/session-resolver.ts"; +import { reviewContext } from "../mcp/journal-review.ts"; +import { unknownContext, type McpCallInput } from "../mcp/journal.ts"; +import { makeMcpEffectDriver } from "../mcp/effect-driver.ts"; +import { unwrapCredentialControlClient } from "../mcp/control-client.ts"; +import { SessionPolicyService } from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import { CredentialControlService } from "../gen/engram/mcp/v1/control_pb.ts"; +import { McpWritePolicyRefSchema } from "../gen/engram/app/v1/session_pb.ts"; + +// Explicit bound-child fixture. Native child activation and host enforcement remain closed. +// The journal, resolver, permission checks, approvals and admission use real Postgres. +async function family(withOtherConnection = false) { + const f = await reviewFixture([], 2); + let extra: Awaited> | undefined; + try { + extra = withOtherConnection ? await reviewFixture() : undefined; + await f.repository.publish( + f.connection.id, + f.draft.id, + 1, + f.catalog.tools.map((tool, index) => ({ + ...tool, + classification: index === 0 ? "write" : "read", + enabled: true, + })), + f.publisherId, + f.now, + ); + await f.repository.setEnabled(f.connection.id, 1, true, f.now); + const tasks: string[] = []; + const snapshots = new Map(); + let workerCalls = 0; + let now = f.now; + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + const profiles = makeProfileStore(f.db); + await profiles.update( + f.profileId, + { + ...(await profiles.get(f.profileId))!, + integrationGrants: [ + ...f.authority.profileGrants, + ...(extra?.authority.profileGrants ?? []), + ], + }, + { actorId: f.actorId, expectedRevision: 0 }, + ); + await f.db.update(user).set({ role: "user" }).where(eq(user.id, f.actorId)); + const authority = await readLiveBindingAuthority( + f.profileId, + f.authority.operatorPolicies, + f.db, + ); + const nativeConnections = [f.connection, ...(extra ? [extra.connection] : [])].map((row) => ({ + id: row.id, + alias: row.alias, + provider: "mcp", + displayName: row.displayName, + config: row.config, + })); + const root = compileSessionBindings({ + launchPolicy: { profileId: f.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + }); + await f.db + .update(taskSession) + .set(bindingSnapshotColumns(root, nativeConnections)) + .where(eq(taskSession.sessionId, f.sessionId)); + snapshots.set(f.sessionId, root); + const policyRef = (sessionId: string) => + create(McpWritePolicyRefSchema, { + sessionId, + bindingEpoch: BigInt(snapshots.get(sessionId)!.bindingEpoch), + policyRevision: 1n, + }); + const policy = createClient( + SessionPolicyService, + createRouterTransport((router) => + router.service(SessionPolicyService, { + readWritePolicy: async (req) => ({ + state: { + policy: { + current: true, + initialized: true, + writeEligibility: "write_eligible", + ref: policyRef(req.sessionId), + ancestorPolicyRefs: snapshots + .get(req.sessionId)! + .ancestorAuthorityRefs.map((ref) => policyRef(ref.sessionId)), + }, + }, + }), + validateDispatchFence: async () => ({ protocol: 1, status: "current" }), + validateExecutionIdentity: async () => ({ protocol: 1, status: "current" }), + }), + ), + ); + const resolver = makeStoredSessionBindingResolver( + { + control: policy, + readOperatorPolicies: async () => f.authority.operatorPolicies, + enforceManagedDeny: async (expected) => expected, + }, + f.db, + ); + const driver = makeMcpEffectDriver({ + db: f.db, + deploymentId: f.deploymentId, + policy, + resolver, + control: unwrapCredentialControlClient( + createClient( + CredentialControlService, + createRouterTransport((router) => router.service(CredentialControlService, {})), + ), + ), + enabled: () => true, + now: () => now, + workerTls: () => { + workerCalls++; + throw Error("fixture worker must not be invoked"); + }, + }); + function input( + sessionId: string, + taskId: string, + callId = crypto.randomUUID(), + connectionId = f.connection.id, + toolIndex = 0, + ): McpCallInput { + const snapshot = snapshots.get(sessionId)!; + const binding = snapshot.immutableBindings.find( + (binding) => binding.connectionId === connectionId, + )!; + const resolved = resolveSnapshotTool(snapshot, binding.tools[toolIndex]!.name)!; + return { + ...f.input, + sessionId, + taskId, + toolCallId: callId, + bindingHash: snapshot.bindingHash, + resolvedTool: resolved, + connectionId, + catalogId: binding.catalogId, + classification: resolved.reviewedTool.classification, + upstreamName: resolved.reviewedTool.upstreamName, + configRevision: binding.configRevision, + authGeneration: binding.authGeneration, + requestPolicyContext: { + ...f.input.requestPolicyContext, + sessionId, + toolCallId: callId, + acceptedPolicyRef: policyRef(sessionId), + ancestorPolicyRefs: snapshot.ancestorAuthorityRefs.map((ref) => policyRef(ref.sessionId)), + }, + deadline: new Date(now.getTime() + 600_000), + }; + } + const request = (call: McpCallInput) => ({ + sessionId: call.sessionId, + runId: call.runId, + toolCallId: call.toolCallId, + modelToolName: call.resolvedTool.modelToolName, + originalArguments: call.originalArguments, + }); + async function child(parentId: string | null = f.sessionId) { + const taskId = crypto.randomUUID(), + sessionId = crypto.randomUUID(); + const compilation = { + launchPolicy: { profileId: f.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + requestedGrants: authority.profileGrants, + }; + const snapshot = parentId + ? await compileChildSessionBindings({ + ...compilation, + spawningSessionId: parentId, + readParentCurrent: resolver.readCurrent, + }) + : compileSessionBindings(compilation); + await f.db.insert(task).values({ + id: taskId, + type: "chat", + createdByUserId: f.actorId, + spawningSessionId: parentId, + }); + tasks.push(taskId); + await f.db.insert(taskSession).values({ + taskId, + sessionId, + profileId: f.profileId, + integrationPrincipalId: f.actorId, + ...bindingSnapshotColumns(snapshot, nativeConnections), + }); + snapshots.set(sessionId, snapshot); + return input(sessionId, taskId); + } + async function ready(input: McpCallInput, accepted?: typeof mcpExecution.$inferSelect) { + const call = + accepted ?? + (await f.journal.create( + input, + { kind: input.classification === "read" ? "read" : "approval" }, + now, + )); + await f.journal.admitSchema(call.id, f.deploymentId, now); + const schema = (await f.journal.getAdmission(call.id))!; + await f.journal.finishSchema( + call.id, + schema.reservationId, + schema.expiresAt, + call.requestDigest, + true, + now, + ); + let authorized = call; + if (call.authorizationKind === "approval") { + const approval = await f.journal.requestApproval(call.id, f.deploymentId, now); + authorized = ( + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), now) + ).call; + } + await f.journal.admit(call.id, f.deploymentId, now); + const lease = (await f.journal.getAdmission(call.id))!; + const proof = { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: authorized.version, + reservationId: lease.reservationId, + reservationExpiresAt: lease.expiresAt, + }; + const intent = { + bootId: "fixture-boot", + intentId: crypto.randomUUID(), + expiresAt: lease.expiresAt, + }; + return { call: authorized, proof, intent }; + } + async function unknownParent() { + const readyCall = await ready(input(f.sessionId, f.taskId)); + await f.journal.dispatch(readyCall.call.id, readyCall.intent, now, readyCall.proof); + const call = await f.journal.cancelInternal(readyCall.call.id, "interrupt", now); + expect(call.state).toBe("outcome_unknown"); + return call; + } + return { + ...f, + extraConnectionId: extra?.connection.id, + driver, + request, + input, + child, + ready, + unknownParent, + snapshots, + get workerCalls() { + return workerCalls; + }, + advance(value: Date) { + now = value; + }, + async cleanup() { + for (const id of tasks.reverse()) await f.db.delete(task).where(eq(task.id, id)); + await f.cleanup(); + await extra?.cleanup(); + }, + }; + } catch (error) { + await f.cleanup(); + await extra?.cleanup(); + throw error; + } +} + +test("parent unknown blocks fresh and already-bound child acceptance with new call IDs", async () => { + const f = await family(); + try { + const existing = await f.child(); + await f.unknownParent(); + const fresh = await f.child(); + for (const input of [existing, fresh]) { + await expect(f.driver.acceptToolRequest(f.request(input))).rejects.toThrow("write authority"); + expect( + await f.db.select().from(mcpExecution).where(eq(mcpExecution.sessionId, input.sessionId)), + ).toHaveLength(0); + } + expect(f.workerCalls).toBe(0); + } finally { + await f.cleanup(); + } +}); + +test("accepted child cannot start worker or cross final dispatch after parent becomes unknown", async () => { + const f = await family(); + try { + const input = await f.child(); + const accepted = await f.driver.acceptToolRequest(f.request(input)); + const ready = await f.ready(input, accepted); + await f.unknownParent(); + expect((await f.driver.acceptToolRequest(f.request(input))).id).toBe(accepted.id); + await expect(f.driver.driveOnce(accepted.id)).rejects.toThrow("unknown write"); + await expect( + f.journal.dispatch(ready.call.id, ready.intent, f.now, ready.proof), + ).rejects.toThrow("unknown write"); + expect((await f.journal.get(accepted.id))!.state).toBe("authorized"); + expect(f.workerCalls).toBe(0); + } finally { + await f.cleanup(); + } +}); + +test("resolved parent remains blocking until intent expiry; unresolved expiry alone never releases", async () => { + const f = await family(); + try { + const parent = await f.unknownParent(); + const child = await f.child(); + f.advance(new Date(parent.intentExpiresAt!.getTime() + 1)); + await expect(f.driver.acceptToolRequest(f.request(child))).rejects.toThrow("write authority"); + f.advance(f.now); + await f.journal.resolveUnknown( + parent.id, + f.actorId, + "acknowledged", + f.now, + unknownContext(parent), + ); + await expect(f.driver.acceptToolRequest(f.request(child))).rejects.toThrow("write authority"); + f.advance(new Date(parent.intentExpiresAt!.getTime() + 1)); + const accepted = await f.driver.acceptToolRequest(f.request(child)); + const ready = await f.ready(child, accepted); + expect( + ( + await f.journal.dispatch( + ready.call.id, + ready.intent, + new Date(parent.intentExpiresAt!.getTime() + 1), + ready.proof, + ) + ).state, + ).toBe("dispatching"); + expect(f.workerCalls).toBe(0); + } finally { + await f.cleanup(); + } +}); + +test("credential-broker current assertion checks ancestor outcomes after child barrier", async () => { + const f = await family(); + try { + const child = await f.ready(await f.child()); + await f.journal.dispatch(child.call.id, child.intent, f.now, child.proof); + await f.unknownParent(); + await expect( + f.journal.assertDispatchCurrent(child.call.id, child.proof, f.now), + ).rejects.toThrow("unknown write"); + } finally { + await f.cleanup(); + } +}); + +test("independent session sharing connection and parent of an unknown child remain allowed", async () => { + const f = await family(); + try { + const child = await f.ready(await f.child()); + await f.journal.dispatch(child.call.id, child.intent, f.now, child.proof); + await f.journal.cancelInternal(child.call.id, "interrupt", f.now); + const sibling = await f.child(); + for (const input of [f.input(f.sessionId, f.taskId), sibling, await f.child(null)]) + expect((await f.driver.acceptToolRequest(f.request(input))).state).toBe("waiting_approval"); + await f.unknownParent(); + expect((await f.driver.acceptToolRequest(f.request(await f.child(null)))).state).toBe( + "waiting_approval", + ); + } finally { + await f.cleanup(); + } +}); + +for (const change of ["detached", "other-parent", "parent-epoch", "parent-terminal"] as const) + test(`trusted ancestry rejects ${change} before new acceptance or worker`, async () => { + const f = await family(); + try { + const input = await f.child(); + if (change === "detached" || change === "other-parent") { + const other = change === "other-parent" ? await f.child(null) : null; + await f.db + .update(task) + .set({ spawningSessionId: other?.sessionId ?? null }) + .where(eq(task.id, input.taskId)); + } else if (change === "parent-terminal") { + await f.db.update(task).set({ status: "failed" }).where(eq(task.id, f.taskId)); + } else { + const envelope = ( + await f.db.select().from(taskSession).where(eq(taskSession.sessionId, f.sessionId)) + )[0]!.mcpBindingEnvelope!; + await f.db + .update(taskSession) + .set({ mcpBindingEnvelope: { ...envelope, bindingEpoch: 1 } }) + .where(eq(taskSession.sessionId, f.sessionId)); + } + await expect(f.driver.acceptToolRequest(f.request(input))).rejects.toThrow(); + await expect(f.journal.create(input, { kind: "approval" }, f.now)).rejects.toThrow(); + expect(f.workerCalls).toBe(0); + } finally { + await f.cleanup(); + } + }); + +// Hold the parent execution row. Real cancellation takes the connection lock before it +// waits here. Child dispatch must then wait behind that cancellation on the connection. +test("parent unknown and child final dispatch serialize on the connection lock", async () => { + const f = await family(); + try { + const child = await f.ready(await f.child()); + const parent = await f.ready(f.input(f.sessionId, f.taskId)); + await f.journal.dispatch(parent.call.id, parent.intent, f.now, parent.proof); + let release!: () => void; + let locked!: (pid: number) => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const acquired = new Promise((resolve) => { + locked = resolve; + }); + const holder = f.db.transaction(async (tx) => { + await tx.select().from(mcpExecution).where(eq(mcpExecution.id, parent.call.id)).for("update"); + const pid = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + locked(pid.rows[0]!.pid); + await gate; + }); + async function waitingOn(pid: number) { + for (let i = 0; i < 100; i++) { + const rows = await f.db.execute<{ pid: number }>( + sql`select pid from pg_stat_activity where ${pid} = any(pg_blocking_pids(pid))`, + ); + if (rows.rows[0]) return rows.rows[0].pid; + await Bun.sleep(5); + } + throw Error("journal operation did not reach the held row"); + } + const pid = await acquired; + const cancel = f.journal.cancelInternal(parent.call.id, "interrupt", f.now); + let result: Promise | undefined; + try { + const cancelPid = await waitingOn(pid); + result = f.journal.dispatch(child.call.id, child.intent, f.now, child.proof).then( + () => "sent", + (error) => String(error), + ); + expect(await waitingOn(cancelPid)).toBeNumber(); + } finally { + release(); + } + await holder; + expect((await cancel).state).toBe("outcome_unknown"); + expect(await result).toContain("unknown write"); + expect((await f.journal.get(child.call.id))!.state).toBe("authorized"); + } finally { + await f.cleanup(); + } +}); + +test("parent unknown permits child reads and writes on another connection", async () => { + const f = await family(true); + try { + const child = await f.child(); + await f.unknownParent(); + const read = f.input(child.sessionId, child.taskId, crypto.randomUUID(), f.connection.id, 1); + const other = f.input(child.sessionId, child.taskId, crypto.randomUUID(), f.extraConnectionId!); + const acceptedRead = await f.driver.acceptToolRequest(f.request(read)); + expect(acceptedRead.state).toBe("authorized"); + const readyRead = await f.ready(read, acceptedRead); + await f.journal.dispatch(readyRead.call.id, readyRead.intent, f.now, readyRead.proof); + expect( + (await f.journal.assertDispatchCurrent(readyRead.call.id, readyRead.proof, f.now)).state, + ).toBe("dispatching"); + const accepted = await f.driver.acceptToolRequest(f.request(other)); + const ready = await f.ready(other, accepted); + expect((await f.journal.dispatch(ready.call.id, ready.intent, f.now, ready.proof)).state).toBe( + "dispatching", + ); + expect((await f.journal.assertDispatchCurrent(ready.call.id, ready.proof, f.now)).state).toBe( + "dispatching", + ); + } finally { + await f.cleanup(); + } +}); + +test("grandparent restriction survives a new child binding and changed arguments", async () => { + const f = await family(); + try { + const parent = await f.child(); + const grandchild = await f.child(parent.sessionId); + await f.unknownParent(); + await expect( + f.driver.acceptToolRequest({ ...f.request(grandchild), originalArguments: '{"x":2}' }), + ).rejects.toThrow("write authority"); + const original = f.snapshots.get(grandchild.sessionId)!; + const { bindingHash: _, ...body } = original; + const replacement = { ...body, bindingEpoch: 1 }; + const { canonicalDigest } = await import("../mcp/hash.ts"); + const snapshot = { ...replacement, bindingHash: canonicalDigest(replacement) }; + f.snapshots.set(grandchild.sessionId, snapshot); + const [stored] = await f.db + .select() + .from(taskSession) + .where(eq(taskSession.sessionId, grandchild.sessionId)); + await f.db + .update(taskSession) + .set(bindingSnapshotColumns(snapshot, stored!.integrationConnections!)) + .where(eq(taskSession.sessionId, grandchild.sessionId)); + const changed = f.input(grandchild.sessionId, grandchild.taskId); + await expect(f.driver.acceptToolRequest(f.request(changed))).rejects.toThrow("write authority"); + expect(f.workerCalls).toBe(0); + } finally { + await f.cleanup(); + } +}); + +test("actual acceptance still checks the current principal before any worker operation", async () => { + const f = await family(); + try { + const child = await f.child(); + await f.db.update(user).set({ banned: true }).where(eq(user.id, f.actorId)); + await expect(f.driver.acceptToolRequest(f.request(child))).rejects.toThrow( + "owner or administrator", + ); + expect(f.workerCalls).toBe(0); + } finally { + await f.cleanup(); + } +}); + +test("ancestor unknown preserves verified child result and exact duplicate identity", async () => { + const f = await family(); + try { + const input = await f.child(); + const child = await f.ready(input); + const dispatched = await f.journal.dispatch(child.call.id, child.intent, f.now, child.proof); + await f.unknownParent(); + const completed = await f.journal.finish( + dispatched.id, + dispatched.version, + "succeeded", + { output: "fixture verified result" }, + false, + f.now, + ); + expect((await f.journal.create(input, { kind: "approval" }, f.now)).id).toBe(completed.id); + expect((await f.driver.acceptToolRequest(f.request(input))).result).toEqual(completed.result); + expect((await f.driver.driveOnce(completed.id)).state).toBe("succeeded"); + await expect( + f.driver.acceptToolRequest({ ...f.request(input), originalArguments: '{"x":2}' }), + ).rejects.toThrow("identity changed"); + await expect( + f.driver.acceptToolRequest(f.request(f.input(input.sessionId, input.taskId))), + ).rejects.toThrow("write authority"); + expect(f.workerCalls).toBe(0); + } finally { + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-lifecycle-actions.test.ts b/orchestrator/src/__tests__/mcp-lifecycle-actions.test.ts new file mode 100644 index 000000000..a398446e6 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-lifecycle-actions.test.ts @@ -0,0 +1,206 @@ +import { expect, test } from "bun:test"; +import { schemaVerifiedFixture } from "./mcp-review-fixture.ts"; +import { reviewContext } from "../mcp/journal-review.ts"; +import { persistVerifiedOutcome } from "../mcp/effect-driver.ts"; + +async function dispatch(f: Awaited>) { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const approved = ( + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), f.now) + ).call; + await f.journal.admit(call.id, f.deploymentId, f.now); + const lease = (await f.journal.getAdmission(call.id))!; + return f.journal.dispatch( + call.id, + { bootId: "boot", intentId: crypto.randomUUID(), expiresAt: lease.expiresAt }, + f.now, + { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: approved.version, + reservationId: lease.reservationId, + reservationExpiresAt: lease.expiresAt, + }, + ); +} + +test("cancel requires the observed revision and returns a durable receipt without undo", async () => { + const f = await schemaVerifiedFixture(); + try { + const call = await dispatch(f); + await expect(f.journal.cancel(call.id, f.actorId, f.now, call.version - 1)).rejects.toThrow( + "stale", + ); + expect((await f.journal.get(call.id))!.state).toBe("dispatching"); + const canceled = await f.journal.cancel(call.id, f.actorId, f.now, call.version); + expect(canceled.state).toBe("outcome_unknown"); + const data = await f.journal.readSession(f.sessionId, f.actorId, { limit: 16 }); + expect(data.rows[0]!.cancellation).toMatchObject({ + actorId: f.actorId, + expectedJournalRevision: call.version, + journalRevision: call.version + 1, + state: "outcome_unknown", + }); + expect(await f.journal.hasUnknown(f.sessionId, f.connection.id)).toBe(true); + } finally { + await f.cleanup(); + } +}); + +test("resolution requires exact unknown context and retains later verified truth", async () => { + const f = await schemaVerifiedFixture(); + try { + const dispatched = await dispatch(f); + const call = await f.journal.cancel(dispatched.id, f.actorId, f.now, dispatched.version); + const expected = { + executionId: call.id, + journalRevision: call.version, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + intentId: call.intentId!, + }; + await expect( + f.journal.resolveUnknown(call.id, f.actorId, "acknowledged", f.now, { + ...expected, + intentId: "other", + }), + ).rejects.toThrow("stale"); + expect(await f.journal.hasUnknown(f.sessionId, f.connection.id)).toBe(true); + const receipt = await f.journal.resolveUnknown( + call.id, + f.actorId, + "acknowledged", + f.now, + expected, + ); + expect(receipt.expectedContext).toEqual(expected); + expect(await f.journal.hasUnknown(f.sessionId, f.connection.id)).toBe(false); + const result = await persistVerifiedOutcome( + f.journal, + dispatched, + { + state: "succeeded", + result: { outcome: "succeeded", delivery: "available", output: { content: [] } }, + resultUnavailable: false, + }, + () => f.now, + ); + expect(result.state).toBe("succeeded"); + expect(result.intentId).toBe(dispatched.intentId); + const data = await f.journal.readSession(f.sessionId, f.actorId, { limit: 16 }); + expect(data.rows[0]!.resolution).toEqual(receipt); + expect(data.rows[0]!.call.result).toEqual(result.result); + } finally { + await f.cleanup(); + } +}); + +test("verified result before resolution makes the unknown context stale", async () => { + const f = await schemaVerifiedFixture(); + try { + const dispatched = await dispatch(f); + const unknown = await f.journal.cancel(dispatched.id, f.actorId, f.now, dispatched.version); + const expected = { + executionId: unknown.id, + journalRevision: unknown.version, + requestDigest: unknown.requestDigest, + bindingHash: unknown.bindingHash, + intentId: unknown.intentId!, + }; + const result = await persistVerifiedOutcome( + f.journal, + dispatched, + { + state: "succeeded", + result: { outcome: "succeeded", delivery: "available", output: { content: [] } }, + resultUnavailable: false, + }, + () => f.now, + ); + expect(result.state).toBe("succeeded"); + await expect( + f.journal.resolveUnknown(unknown.id, f.actorId, "acknowledged", f.now, expected), + ).rejects.toThrow("stale"); + const data = await f.journal.readSession(f.sessionId, f.actorId, { limit: 16 }); + expect(data.rows[0]!.resolution).toBeNull(); + expect(data.rows[0]!.call.result).toEqual(result.result); + } finally { + await f.cleanup(); + } +}); + +test("concurrent resolution and verified result preserve the committed evidence", async () => { + const f = await schemaVerifiedFixture(); + try { + const dispatched = await dispatch(f); + const unknown = await f.journal.cancel(dispatched.id, f.actorId, f.now, dispatched.version); + const expected = { + executionId: unknown.id, + journalRevision: unknown.version, + requestDigest: unknown.requestDigest, + bindingHash: unknown.bindingHash, + intentId: unknown.intentId!, + }; + const [resolution, result] = await Promise.allSettled([ + f.journal.resolveUnknown(unknown.id, f.actorId, "acknowledged", f.now, expected), + persistVerifiedOutcome( + f.journal, + dispatched, + { + state: "succeeded", + result: { outcome: "succeeded", delivery: "available", output: { content: [] } }, + resultUnavailable: false, + }, + () => f.now, + ), + ]); + expect(result.status).toBe("fulfilled"); + const data = await f.journal.readSession(f.sessionId, f.actorId, { limit: 16 }); + expect(data.rows[0]!.call.state).toBe("succeeded"); + expect(data.rows[0]!.call.result).toMatchObject({ outcome: "succeeded" }); + if (resolution.status === "fulfilled") + expect(data.rows[0]!.resolution).toEqual(resolution.value); + else { + expect(String(resolution.reason)).toContain("stale"); + expect(data.rows[0]!.resolution).toBeNull(); + } + } finally { + await f.cleanup(); + } +}); + +test("an unverified read failure cannot overwrite a concurrent cancellation", async () => { + const f = await schemaVerifiedFixture(); + try { + const dispatched = await dispatch(f); + let first = true; + const journal = { + ...f.journal, + async get(id: string) { + const before = await f.journal.get(id); + if (first) { + first = false; + await f.journal.cancel(id, f.actorId, f.now, dispatched.version); + } + return before; + }, + }; + const result = await persistVerifiedOutcome( + journal, + dispatched, + { + state: "failed", + result: { outcome: "failed", delivery: "unavailable" }, + resultUnavailable: false, + }, + () => f.now, + false, + ); + expect(result.state).toBe("outcome_unknown"); + } finally { + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-lifecycle-cursor.test.ts b/orchestrator/src/__tests__/mcp-lifecycle-cursor.test.ts new file mode 100644 index 000000000..86d5c0d79 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-lifecycle-cursor.test.ts @@ -0,0 +1,70 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { consumerCursor } from "../db/schema.ts"; +import { makeMcpLifecycleEvents } from "../mcp/lifecycle-events.ts"; + +test("a failed event cannot be skipped and competing replicas advance only after cancellation", async () => { + const db = getDb(); + const sessionId = crypto.randomUUID(); + const events = [ + { idx: 1n, kind: "run_interrupted", payloadJson: "{}" }, + { idx: 2n, kind: "resumed", payloadJson: "{}" }, + ]; + let fail = true; + const canceled: bigint[] = []; + const make = () => + makeMcpLifecycleEvents({ + db, + now: () => new Date(), + list: async (_id, after) => events.filter((event) => event.idx > after), + reconcile: async (_id, event) => { + if (event.idx === 1n && fail) throw Error("unavailable"); + canceled.push(event.idx); + }, + }); + try { + await expect(make().runOnce(sessionId)).rejects.toThrow("unavailable"); + expect( + (await db.select().from(consumerCursor).where(eq(consumerCursor.sessionId, sessionId)))[0]! + .lastIdx, + ).toBe(-1n); + expect(canceled).toEqual([]); + fail = false; + await Promise.all([make().runOnce(sessionId), make().runOnce(sessionId)]); + await make().runOnce(sessionId); + expect( + (await db.select().from(consumerCursor).where(eq(consumerCursor.sessionId, sessionId)))[0]! + .lastIdx, + ).toBe(2n); + expect(canceled).toContain(1n); + expect(canceled).toContain(2n); + expect(canceled.indexOf(1n)).toBeLessThan(canceled.indexOf(2n)); + } finally { + await db.delete(consumerCursor).where(eq(consumerCursor.sessionId, sessionId)); + } +}); + +test("invalid event order cannot advance beyond the last committed event", async () => { + const db = getDb(); + const sessionId = crypto.randomUUID(); + const reader = makeMcpLifecycleEvents({ + db, + now: () => new Date(), + list: async () => [ + { idx: 3n, kind: "run_interrupted", payloadJson: "{}" }, + { idx: 2n, kind: "resumed", payloadJson: "{}" }, + ], + reconcile: async () => {}, + }); + try { + await expect(reader.runOnce(sessionId)).rejects.toThrow("order"); + const [cursor] = await db + .select() + .from(consumerCursor) + .where(eq(consumerCursor.sessionId, sessionId)); + expect(cursor!.lastIdx).toBe(-1n); + } finally { + await db.delete(consumerCursor).where(eq(consumerCursor.sessionId, sessionId)); + } +}); diff --git a/orchestrator/src/__tests__/mcp-lifecycle-interceptor.test.ts b/orchestrator/src/__tests__/mcp-lifecycle-interceptor.test.ts new file mode 100644 index 000000000..7ec3e1469 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-lifecycle-interceptor.test.ts @@ -0,0 +1,65 @@ +import { expect, test } from "bun:test"; +import { createContextValues, type UnaryRequest, type UnaryResponse } from "@connectrpc/connect"; +import { create } from "@bufbuild/protobuf"; +import { + InterruptRequestSchema, + InterruptResponseSchema, + SessionService, +} from "../gen/engram/app/v1/session_pb.ts"; +import { makeMcpLifecycleInterceptor } from "../mcp/lifecycle-interceptor.ts"; + +test("trusted interrupt waits for durable invalidation before forwarding", async () => { + const order: string[] = []; + const req: UnaryRequest = { + stream: false, + service: SessionService, + method: SessionService.method.interrupt, + requestMethod: "POST", + url: "http://example.test", + signal: new AbortController().signal, + header: new Headers(), + contextValues: createContextValues(), + message: create(InterruptRequestSchema, { sessionId: "s1" }), + }; + const res: UnaryResponse = { + stream: false, + service: SessionService, + method: SessionService.method.interrupt, + header: new Headers(), + trailer: new Headers(), + message: create(InterruptResponseSchema), + }; + const interceptor = makeMcpLifecycleInterceptor(async (id, cause) => { + order.push(`${id}:${cause}`); + }); + await interceptor(async () => { + order.push("forward"); + return res; + })(req); + expect(order).toEqual(["s1:interrupt", "forward"]); +}); + +test("invalidation failure stops the lifecycle request", async () => { + const req: UnaryRequest = { + stream: false, + service: SessionService, + method: SessionService.method.interrupt, + requestMethod: "POST", + url: "http://example.test", + signal: new AbortController().signal, + header: new Headers(), + contextValues: createContextValues(), + message: create(InterruptRequestSchema, { sessionId: "s1" }), + }; + let forwarded = false; + const interceptor = makeMcpLifecycleInterceptor(async () => { + throw Error("store unavailable"); + }); + await expect( + interceptor(async () => { + forwarded = true; + throw Error("unexpected forward"); + })(req), + ).rejects.toThrow("store unavailable"); + expect(forwarded).toBe(false); +}); diff --git a/orchestrator/src/__tests__/mcp-lifecycle-invalidation.test.ts b/orchestrator/src/__tests__/mcp-lifecycle-invalidation.test.ts new file mode 100644 index 000000000..97bf1171b --- /dev/null +++ b/orchestrator/src/__tests__/mcp-lifecycle-invalidation.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from "bun:test"; +import { effectFixture } from "./mcp-effect-fixture.ts"; +import { makeMcpLifecycleInvalidator } from "../mcp/lifecycle-invalidation.ts"; + +test("internal lifecycle cancellation commits before local abort", async () => { + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + const invalidator = makeMcpLifecycleInvalidator({ + db: f.db, + abort: async (id) => { + expect((await f.journal.get(id))!.state).toBe("canceled"); + }, + }); + await invalidator(f.sessionId, "interrupt"); + expect((await f.journal.cancellationReceipt(call.id))?.cause).toBe("interrupt"); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-lifecycle-reconciler.test.ts b/orchestrator/src/__tests__/mcp-lifecycle-reconciler.test.ts new file mode 100644 index 000000000..9a9e3c622 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-lifecycle-reconciler.test.ts @@ -0,0 +1,184 @@ +import { expect, test } from "bun:test"; +import { effectFixture } from "./mcp-effect-fixture.ts"; +import { makeMcpLifecycleReconciler } from "../mcp/lifecycle-reconciler.ts"; + +test("independent keyset scan cancels stale accepted work without provider replay", async () => { + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + f.setFence(false); + const scanner = makeMcpLifecycleReconciler({ db: f.db, driver: f.driver }); + await scanner.runOnce(); + expect((await f.journal.get(call.id))!.state).toBe("canceled"); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("recovery lifecycle pass runs event catch-up and independent currentness", async () => { + const { makeMcpRecovery } = await import("../mcp/recovery.ts"); + const { makeMcpRecoveryScanner } = await import("../mcp/recovery-scanner.ts"); + const { consumerCursor } = await import("../db/schema.ts"); + const { eq } = await import("drizzle-orm"); + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + listLifecycleEvents: async (_id, after) => + after < 1n ? [{ idx: 1n, kind: "resumed", payloadJson: "{}" }] : [], + }); + const scanner = makeMcpRecoveryScanner({ + deploymentId: f.deploymentId, + runtime: () => runtime, + }); + await scanner.runLifecycleOnce(); + expect((await f.journal.get(call.id))!.state).toBe("authorized"); + expect( + ( + await f.db.select().from(consumerCursor).where(eq(consumerCursor.sessionId, f.sessionId)) + )[0]!.lastIdx, + ).toBe(1n); + f.setFence(false); + await scanner.runLifecycleOnce(); + expect((await f.journal.get(call.id))!.state).toBe("canceled"); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.db.delete(consumerCursor).where(eq(consumerCursor.sessionId, f.sessionId)); + await f.cleanup(); + } +}, 20000); + +test("a stalled authority check times out and aborts local I/O", async () => { + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + const canceled: string[] = []; + const scanner = makeMcpLifecycleReconciler({ + db: f.db, + timeoutMs: 10, + driver: { + reconcile: () => new Promise(() => {}), + cancel: (id) => { + canceled.push(id); + }, + }, + }); + await expect(scanner.runOnce()).rejects.toThrow("unavailable"); + expect(canceled).toContain(call.id); + expect((await f.journal.get(call.id))!.state).toBe("authorized"); + } finally { + await f.cleanup(); + } +}, 20000); + +test("a timed-out stale check cannot cancel later when its response arrives", async () => { + const f = await effectFixture({ classification: "read" }); + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + try { + const call = await f.driver.acceptToolRequest(f.request); + f.setFence(false); + f.beforeFence(() => blocked); + const scanner = makeMcpLifecycleReconciler({ db: f.db, driver: f.driver, timeoutMs: 10 }); + await expect(scanner.runOnce()).rejects.toThrow("unavailable"); + release(); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect((await f.journal.get(call.id))!.state).toBe("authorized"); + } finally { + release(); + await f.cleanup(); + } +}, 20000); + +test("the recovery scheduler runs lifecycle catch-up without a workflow delivery", async () => { + const { makeMcpRecovery } = await import("../mcp/recovery.ts"); + const { makeMcpRecoveryScanner } = await import("../mcp/recovery-scanner.ts"); + const { consumerCursor } = await import("../db/schema.ts"); + const { eq } = await import("drizzle-orm"); + const f = await effectFixture({ classification: "read" }); + let scanner: ReturnType | undefined; + try { + const call = await f.driver.acceptToolRequest(f.request); + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + listLifecycleEvents: async (_id, after) => + after < 1n ? [{ idx: 1n, kind: "resumed", payloadJson: "{}" }] : [], + }); + scanner = makeMcpRecoveryScanner({ + deploymentId: f.deploymentId, + journal: f.journal, + runtime: () => runtime, + startWorkflow: async () => {}, + }); + await scanner.start(); + const deadline = Date.now() + 3000; + while (Date.now() < deadline) { + const [current] = await f.db + .select() + .from(consumerCursor) + .where(eq(consumerCursor.sessionId, f.sessionId)); + if (current?.lastIdx === 1n) break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + const [cursor] = await f.db + .select() + .from(consumerCursor) + .where(eq(consumerCursor.sessionId, f.sessionId)); + expect(cursor?.lastIdx).toBe(1n); + expect((await f.journal.get(call.id))!.state).toBe("authorized"); + } finally { + await scanner?.stop(); + await f.db.delete(consumerCursor).where(eq(consumerCursor.sessionId, f.sessionId)); + await f.cleanup(); + } +}, 20000); + +test("active keyset uses the partial index and advances past a full page", async () => { + const { eq, sql } = await import("drizzle-orm"); + const { mcpExecution } = await import("../db/schema.ts"); + const { MCP_LIMITS } = await import("../mcp/limits.ts"); + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + const [base] = await f.db.select().from(mcpExecution).where(eq(mcpExecution.id, call.id)); + const copies = Array.from({ length: MCP_LIMITS.queueSize }, (_, i) => ({ + ...base!, + id: crypto.randomUUID(), + toolCallId: `scan-${i}-${crypto.randomUUID()}`, + })); + await f.db.insert(mcpExecution).values(copies); + const seen = new Set(); + const scanner = makeMcpLifecycleReconciler({ + db: f.db, + driver: { + async reconcile(id) { + seen.add(id); + }, + cancel() {}, + }, + }); + await scanner.runOnce(); + expect(seen.size).toBe(MCP_LIMITS.queueSize); + await scanner.runOnce(); + expect(seen.size).toBe(MCP_LIMITS.queueSize + 1); + const plan = await f.db.transaction(async (tx) => { + await tx.execute(sql`SET LOCAL enable_seqscan = off`); + return tx.execute( + sql`EXPLAIN SELECT id FROM mcp_execution WHERE state IN ('waiting_approval','authorized','dispatching') AND id > '' ORDER BY id LIMIT 128`, + ); + }); + expect(JSON.stringify(plan.rows)).toContain("mcp_execution_lifecycle_active_idx"); + } finally { + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-lifecycle-rpc.test.ts b/orchestrator/src/__tests__/mcp-lifecycle-rpc.test.ts new file mode 100644 index 000000000..026c879fd --- /dev/null +++ b/orchestrator/src/__tests__/mcp-lifecycle-rpc.test.ts @@ -0,0 +1,71 @@ +import { expect, test } from "bun:test"; +import { Code, ConnectError, createClient, createRouterTransport } from "@connectrpc/connect"; +import { createConnectTransport } from "@connectrpc/connect-node"; +import type { AddressInfo } from "node:net"; +import { Hono } from "hono"; +import { buildServer } from "../server.ts"; +import { SessionService } from "../gen/engram/app/v1/session_pb.ts"; +import { registerPassthrough } from "../rpc/passthrough.ts"; +import { makeMcpLifecycleInterceptor } from "../mcp/lifecycle-interceptor.ts"; +import { makeMcpLifecycleInvalidator } from "../mcp/lifecycle-invalidation.ts"; +import { effectFixture } from "./mcp-effect-fixture.ts"; + +test("public interrupt authorizes before durable cancellation and upstream forwarding", async () => { + const f = await effectFixture({ classification: "read" }); + const call = await f.driver.acceptToolRequest(f.request); + let owner = "other-owner"; + let forwarded = 0; + const invalidate = makeMcpLifecycleInvalidator({ + db: f.db, + abort: async (id) => f.driver.cancel(id), + }); + const upstream = createRouterTransport( + (router) => + router.service(SessionService, { + async interrupt(req) { + expect((await f.journal.get(call.id))!.state).toBe("canceled"); + forwarded++; + return { sessionId: req.sessionId }; + }, + }), + { transport: { interceptors: [makeMcpLifecycleInterceptor(invalidate)] } }, + ); + const server = buildServer(new Hono(), (router) => + registerPassthrough( + router, + [{ service: SessionService, methods: ["Interrupt"] }], + upstream, + async () => ({ user: { id: f.actorId, role: "user", email: "user@example.test" } }), + async () => owner, + ), + ); + try { + const url = await new Promise((resolve) => + server.listen(0, "127.0.0.1", () => + resolve(`http://127.0.0.1:${(server.address() as AddressInfo).port}/rpc`), + ), + ); + const client = createClient( + SessionService, + createConnectTransport({ baseUrl: url, httpVersion: "1.1" }), + ); + try { + await client.interrupt({ sessionId: f.sessionId }); + throw Error("expected denial"); + } catch (error) { + expect(error).toBeInstanceOf(ConnectError); + expect((error as ConnectError).code).toBe(Code.NotFound); + } + expect((await f.journal.get(call.id))!.state).toBe("authorized"); + expect(forwarded).toBe(0); + owner = f.actorId; + await client.interrupt({ sessionId: f.sessionId }); + expect(forwarded).toBe(1); + expect((await f.journal.cancellationReceipt(call.id))?.cause).toBe("interrupt"); + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-managed-create-input.test.ts b/orchestrator/src/__tests__/mcp-managed-create-input.test.ts new file mode 100644 index 000000000..3e16aec02 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-managed-create-input.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from "bun:test"; +import { + managedCreateInputDigest, + managedCreationInput, + frozenNativeManifest, +} from "../mcp/managed-create-input.ts"; +test("managed creation digest matches the Rust exact-wire golden vector", () => { + expect( + managedCreateInputDigest({ + protocol: 1, + nativeVersion: 1, + taskId: "task-a", + nativeDigest: `sha256:${"a".repeat(64)}`, + creationRequest: new Uint8Array([1, 2, 3]), + }), + ).toBe("sha256:fa03d5903a5d81b1127d47cc331d0d833eb68f5fe42012397d86958bf5b995b5"); +}); +test("creation association retains native bytes and excludes credential bytes", () => { + const input = { + requestedSessionId: crypto.randomUUID(), + imageUri: "image", + mode: "agent", + harness: "claude", + harnessMode: "plan", + harnessEnv: { ENGRAM_TOOLS: "[ ]", NATIVE_TOKEN: "fixture-sensitive-value" }, + }; + const prepared = managedCreationInput(input, { + taskId: "task", + actorId: "actor", + inputId: "source", + }); + expect(prepared.identity.writeEligibility).toBe("read_only"); + expect(prepared.nativeDigest).toBe(frozenNativeManifest("[ ]").digest); + expect(JSON.stringify(prepared.identity)).not.toContain("fixture-sensitive-value"); + expect(prepared.request.inputDigest).toBe(managedCreateInputDigest(prepared.request)); + expect( + managedCreationInput( + { ...input, harnessEnv: { ...input.harnessEnv, ENGRAM_TOOLS: "[]" } }, + { taskId: "task", actorId: "actor", inputId: "source" }, + ).request.inputDigest, + ).not.toBe(prepared.request.inputDigest); + expect(() => + frozenNativeManifest( + JSON.stringify([ + { name: "mcp_injected", description: "", inputSchema: {}, execution: "sync" }, + ]), + ), + ).toThrow(); +}); diff --git a/orchestrator/src/__tests__/mcp-managed-lifecycle-store.test.ts b/orchestrator/src/__tests__/mcp-managed-lifecycle-store.test.ts new file mode 100644 index 000000000..988f67d42 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-managed-lifecycle-store.test.ts @@ -0,0 +1,56 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { reviewFixture } from "./mcp-review-fixture.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +import { makeManagedLifecycleStore } from "../mcp/managed-lifecycle-store.ts"; +import type { ManagedLifecycle } from "../mcp/managed-lifecycle.ts"; +import { taskSession } from "../db/schema.ts"; +await verifyMcpTestDatabases(); +test("managed lifecycle retains exact nonsecret identity through lost-response and concurrent CAS", async () => { + const f = await reviewFixture(); + try { + const store = makeManagedLifecycleStore(f.db); + const initial: ManagedLifecycle = { + version: 1, + identity: { + taskId: f.taskId, + sessionId: f.sessionId, + actorId: f.actorId, + inputId: "input", + inputDigest: `sha256:${"a".repeat(64)}`, + writeEligibility: "read_only", + }, + nativeVersion: 1, + nativeDigest: `sha256:${"b".repeat(64)}`, + phase: "reserving", + bindingHash: f.snapshot.bindingHash, + }; + await store.initialize(initial); + await store.initialize(initial); + expect((await store.read(f.sessionId)).session.mcpManagedLifecycle).toEqual(initial); + const next = { ...initial, phase: "held" as const }; + const outcomes = await Promise.allSettled([ + store.compareAndSet(initial, next), + store.compareAndSet(initial, { ...initial, phase: "preparing" }), + ]); + expect(outcomes.filter((o) => o.status === "fulfilled")).toHaveLength(1); + await expect(store.compareAndSet(initial, next)).rejects.toThrow("changed"); + await expect( + f.db + .update(taskSession) + .set({ mcpManagedLifecycle: null }) + .where(eq(taskSession.sessionId, f.sessionId)) + .execute(), + ).rejects.toThrow(); + await expect( + f.db + .update(taskSession) + .set({ mcpManagedLifecycle: { ...next, nativeDigest: `sha256:${"c".repeat(64)}` } }) + .where(eq(taskSession.sessionId, f.sessionId)) + .execute(), + ).rejects.toThrow(); + expect(await store.pending()).toContainEqual({ sessionId: f.sessionId }); + } finally { + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-managed-policy.test.ts b/orchestrator/src/__tests__/mcp-managed-policy.test.ts new file mode 100644 index 000000000..334ae66dd --- /dev/null +++ b/orchestrator/src/__tests__/mcp-managed-policy.test.ts @@ -0,0 +1,255 @@ +import { expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { create } from "@bufbuild/protobuf"; +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { + ManagedAppliedStateSchema, + ManagedPolicyTargetSchema, + ManagedPolicyPhase, + SessionPolicyService, + type ManagedAppliedState, +} from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { + makeManagedPolicyAuthority, + managedAppliedJson, + unionManagedPolicies, +} from "../mcp/managed-policy.ts"; +import { operatorPublication } from "./mcp-binding-fixture.ts"; +const physicalNetworkIdentity = `sha256:${"8".repeat(64)}`; +const policy = (revision: number, hosts: string[]) => { + const body = { version: 1 as const, revision, hosts }; + return { ...body, digest: canonicalDigest(body) }; +}; + +test("managed deny union keeps orphaned applied and retained desired authority", () => { + expect( + unionManagedPolicies([ + policy(2, ["old.example.test"]), + policy(7, ["orphan.example.test"]), + policy(9, ["retained.example.test"]), + policy(3, ["next.example.test"]), + ]), + ).toEqual( + policy(10, [ + "next.example.test", + "old.example.test", + "orphan.example.test", + "retained.example.test", + ]), + ); +}); +function fixture() { + const target = create(ManagedPolicyTargetSchema, { + sessionId: crypto.randomUUID(), + sandboxId: crypto.randomUUID(), + hostId: crypto.randomUUID(), + bindingEpoch: 2n, + }); + const deny = policy(3, ["tools.example.test"]), + bindingHash = `sha256:${"a".repeat(64)}`; + const targetJson = { + session_id: target.sessionId, + sandbox_id: target.sandboxId, + host_id: target.hostId, + binding_epoch: 2, + }; + const json = { + context: { + target: targetJson, + operator_generation: 1, + operator_identity: operatorPublication.identity, + binding_hash: bindingHash, + }, + policy: deny, + host_incarnation: "host-incarnation", + proxy_incarnation: "proxy-incarnation", + state_id: "", + network_proof: { + target: targetJson, + guest_ip: "192.0.2.5", + network_identity: physicalNetworkIdentity, + }, + }; + const applied = create(ManagedAppliedStateSchema, { + context: { + target, + operatorGeneration: 1n, + operatorIdentity: operatorPublication.identity, + phase: ManagedPolicyPhase.FINAL, + bindingHash, + }, + policy: { ...deny, revision: 3n }, + hostIncarnation: json.host_incarnation, + proxyIncarnation: json.proxy_incarnation, + stateId: `sha256:${createHash("sha256").update(JSON.stringify(json)).digest("hex")}`, + networkProof: { + target, + guestIp: "192.0.2.5", + networkIdentity: physicalNetworkIdentity, + }, + }); + const placement = { + target, + networkIdentity: physicalNetworkIdentity, + guestIp: "192.0.2.5", + hostIncarnation: "host-incarnation", + }; + let present = true; + const publication = { + generation: 1, + identity: operatorPublication.identity, + policies: [ + { + publication: operatorPublication, + zoneId: "public", + policyRevision: "1", + policyDigest: operatorPublication.identity, + authorities: ["tools.example.test"], + aliases: [], + }, + ], + }; + const control = createClient( + SessionPolicyService, + createRouterTransport((router) => + router.service(SessionPolicyService, { + applyManagedPolicy: async () => ({ protocol: 1, applied }), + readManagedPolicy: async () => ({ + protocol: 1, + applied, + ...(present ? { retainedDesired: { target, policy: { ...deny, revision: 3n } } } : {}), + }), + }), + ), + ); + const authority = makeManagedPolicyAuthority({ + control, + readPlacement: async () => placement, + readPublished: async () => publication, + }); + const expected = { + sessionId: target.sessionId, + bindingEpoch: 2, + bindingHash, + denyRevision: 3, + denyDigest: deny.digest, + }; + return { + target, + deny, + bindingHash, + applied, + placement, + publication, + authority, + expected, + omitRetained: () => { + present = false; + }, + }; +} +function resign(applied: ManagedAppliedState) { + const target = applied.context!.target!; + const targetJson = { + session_id: target.sessionId, + sandbox_id: target.sandboxId, + host_id: target.hostId, + binding_epoch: Number(target.bindingEpoch), + }; + applied.stateId = `sha256:${createHash("sha256") + .update( + JSON.stringify({ + context: { + target: targetJson, + operator_generation: Number(applied.context!.operatorGeneration), + operator_identity: applied.context!.operatorIdentity, + binding_hash: applied.context!.bindingHash, + }, + policy: policy(Number(applied.policy!.revision), applied.policy!.hosts), + host_incarnation: applied.hostIncarnation, + proxy_incarnation: applied.proxyIncarnation, + state_id: "", + network_proof: applied.networkProof + ? { + target: targetJson, + guest_ip: applied.networkProof.guestIp, + network_identity: applied.networkProof.networkIdentity, + } + : null, + }), + ) + .digest("hex")}`; +} + +for (const operation of ["apply", "enforce"] as const) { + const run = (f: ReturnType) => + operation === "apply" + ? f.authority.apply(f.target, f.bindingHash, f.deny, undefined, f.publication) + : f.authority.enforce(f.expected, f.deny); + test(`${operation} accepts current prepared physical identity distinct from publication`, async () => { + const f = fixture(); + expect(f.placement.networkIdentity).not.toBe(operatorPublication.networkIdentity); + expect(managedAppliedJson(f.applied).state_id).toBe(f.applied.stateId); + expect(await run(f)).toEqual(operation === "apply" ? f.applied : f.expected); + }); + for (const field of ["networkIdentity", "guestIp", "hostIncarnation"] as const) + test(`${operation} rejects a re-digested same-target receipt with transplanted ${field}`, async () => { + const f = fixture(); + if (field === "hostIncarnation") f.applied.hostIncarnation = "other-host-incarnation"; + else + f.applied.networkProof![field] = + field === "guestIp" ? "192.0.2.8" : operatorPublication.networkIdentity; + resign(f.applied); + expect(managedAppliedJson(f.applied).state_id).toBe(f.applied.stateId); + await expect(run(f)).rejects.toThrow( + operation === "apply" + ? "MCP managed apply changed" + : "MCP final managed acknowledgement unavailable", + ); + }); + for (const field of ["sandboxId", "hostId", "bindingEpoch"] as const) + test(`${operation} rejects current placement with stale ${field}`, async () => { + const f = fixture(); + f.placement.target = create(ManagedPolicyTargetSchema, { + ...f.target, + [field]: field === "bindingEpoch" ? 3n : "other-placement", + }); + await expect(run(f)).rejects.toThrow(); + }); + test(`${operation} rejects absent network proof`, async () => { + const f = fixture(); + f.applied.networkProof = undefined; + resign(f.applied); + await expect(run(f)).rejects.toThrow("MCP managed receipt unavailable"); + }); + test(`${operation} rejects receipt with invalid digest`, async () => { + const f = fixture(); + f.applied.networkProof!.networkIdentity = `sha256:${"9".repeat(64)}`; + await expect(run(f)).rejects.toThrow("digest"); + }); + for (const field of ["operatorGeneration", "operatorIdentity", "bindingHash"] as const) + test(`${operation} rejects re-digested receipt with changed ${field}`, async () => { + const f = fixture(); + if (field === "operatorGeneration") f.applied.context![field] = 2n; + else f.applied.context![field] = `sha256:${"9".repeat(64)}`; + resign(f.applied); + await expect(run(f)).rejects.toThrow( + operation === "apply" + ? "MCP managed apply changed" + : "MCP final managed acknowledgement unavailable", + ); + }); +} +test("fresh final host receipt requires explicit retained presence and binding", async () => { + const f = fixture(); + f.omitRetained(); + await expect(f.authority.enforce(f.expected, f.deny)).rejects.toThrow("presence"); + const current = fixture(); + await expect( + current.authority.enforce( + { ...current.expected, bindingHash: operatorPublication.identity }, + current.deny, + ), + ).rejects.toThrow("acknowledgement"); +}); diff --git a/orchestrator/src/__tests__/mcp-managed-runtime.test.ts b/orchestrator/src/__tests__/mcp-managed-runtime.test.ts new file mode 100644 index 000000000..b3dd3b3d0 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-managed-runtime.test.ts @@ -0,0 +1,445 @@ +import { expect, spyOn, test } from "bun:test"; +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { SessionPolicyService } from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import { makeManagedLifecycleStore } from "../mcp/managed-lifecycle-store.ts"; +import { makeManagedSessionRuntime, McpManagedCreationPending } from "../mcp/managed-runtime.ts"; +import { managedQuery, decodeManagedState } from "../mcp/managed-state.ts"; +import { makeManagedPolicyAuthority } from "../mcp/managed-policy.ts"; +import { reviewFixture } from "./mcp-review-fixture.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +import type { ManagedLifecycle } from "../mcp/managed-lifecycle.ts"; +import { create } from "@bufbuild/protobuf"; +import { createHash } from "node:crypto"; +import { + ManagedAppliedStateSchema, + ManagedPolicyTargetSchema, + type ManagedAppliedState, +} from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import { managedPolicy } from "../mcp/managed-policy.ts"; +import type { ManagedCreationState } from "../mcp/managed-state.ts"; +import { operatorPublication } from "./mcp-binding-fixture.ts"; +import * as controlClients from "../mcp/control-client.ts"; +import * as bindingStore from "../mcp/binding-store.ts"; +import { makeMcpRuntime } from "../mcp/runtime.ts"; +import { McpAuthorityStale } from "../mcp/authority-error.ts"; + +// Real O persistence and generated in-process control transport; no physical boot is claimed. +test("lost reserve response retains immutable identity and never regenerates or resends secret input", async () => { + await verifyMcpTestDatabases(); + const f = await reviewFixture(); + try { + let reserves = 0, + prepares = 0, + enrollments = 0, + exists = false; + const store = makeManagedLifecycleStore(f.db); + const control = createClient( + SessionPolicyService, + createRouterTransport((router) => + router.service(SessionPolicyService, { + async reserveManagedCreate() { + reserves++; + throw Error("lost response"); + }, + }), + ), + ); + const unavailable = async (): Promise => { + throw Error("unexpected host operation"); + }; + const managed = makeManagedPolicyAuthority({ + control, + readPlacement: unavailable, + readPublished: unavailable, + }); + const response = async (phase: string) => { + const value = (await store.read(f.sessionId)).session.mcpManagedLifecycle!; + return { + protocol: 1, + stateJson: new TextEncoder().encode( + JSON.stringify({ + query: managedQuery(value), + state: phase, + boot: null, + operation: null, + terminal_reason: null, + }), + ), + }; + }; + const runtime = makeManagedSessionRuntime({ + store, + control, + managed, + readPublished: unavailable, + authorize: async () => {}, + validate: async () => {}, + enroll: async () => { + enrollments++; + }, + enabled: () => true, + bridge: { + readManagedCreationState: () => response(exists ? "held" : "absent"), + prepareManagedBoot: async () => { + prepares++; + return response("preparing"); + }, + recoverManagedBoot: unavailable, + }, + }); + const input = { + imageUri: "image", + mode: "agent", + requestedSessionId: f.sessionId, + harnessEnv: { ENGRAM_TOOLS: "[]", PRIVATE_NATIVE_TOKEN: "neutral-secret-value" }, + mcpBindingEnvelope: (await store.read(f.sessionId)).session.mcpBindingEnvelope!, + integrationConnections: (await store.read(f.sessionId)).session.integrationConnections!, + }; + await expect( + runtime.create(input, { actorId: f.actorId, inputId: f.sessionId }), + ).rejects.toBeInstanceOf(McpManagedCreationPending); + const durable = (await store.read(f.sessionId)).session.mcpManagedLifecycle!; + expect(durable.phase).toBe("reserving"); + expect(JSON.stringify(durable)).not.toContain("neutral-secret-value"); + await runtime.runOnce(f.sessionId); + expect(prepares).toBe(0); + expect(enrollments).toBe(0); + exists = true; + await runtime.runOnce(f.sessionId); + expect(prepares).toBe(1); + expect(enrollments).toBe(1); + await runtime.create( + { ...input, harnessEnv: { ENGRAM_TOOLS: "[]", PRIVATE_NATIVE_TOKEN: "regenerated-value" } }, + { actorId: f.actorId, inputId: f.sessionId }, + ); + expect(reserves).toBe(1); + expect((await store.read(f.sessionId)).session.mcpManagedLifecycle).toEqual(durable); + } finally { + await f.cleanup(); + } +}); + +test("managed creation state requires exact query, explicit presence and sanitized boot", () => { + const value: ManagedLifecycle = { + version: 1, + identity: { + sessionId: "session", + taskId: "automation:run", + inputId: "input", + actorId: "automation:source", + writeEligibility: "read_only", + inputDigest: `sha256:${"a".repeat(64)}`, + }, + nativeVersion: 1, + nativeDigest: `sha256:${"b".repeat(64)}`, + bindingHash: `sha256:${"c".repeat(64)}`, + phase: "reserving", + }; + const body = { + query: managedQuery(value), + state: "absent", + boot: null, + operation: null, + terminal_reason: null, + }; + const encode = (input: unknown) => ({ + protocol: 1, + stateJson: new TextEncoder().encode(JSON.stringify(input)), + }); + expect(decodeManagedState(encode(body), value).state).toBe("absent"); + expect(() => + decodeManagedState( + encode({ ...body, query: { ...body.query, native_digest: `sha256:${"d".repeat(64)}` } }), + value, + ), + ).toThrow("identity"); + expect(() => decodeManagedState({ protocol: 1, stateJson: new Uint8Array() }, value)).toThrow( + "unavailable", + ); + expect(() => + decodeManagedState(encode({ ...body, boot: { sealed_agent: { ciphertext: [1] } } }), value), + ).toThrow(); +}); + +for (const missingProof of [false, true]) + test(`final boot requires real receipt shape and durable exact manifest before start (missing proof ${missingProof})`, async () => { + await verifyMcpTestDatabases(); + const f = await reviewFixture(); + try { + const store = makeManagedLifecycleStore(f.db); + let state: ManagedCreationState | undefined, applied: ManagedAppliedState | undefined; + let finalizeCalls = 0, + applyCalls = 0; + const physicalNetworkIdentity = `sha256:${"8".repeat(64)}`; + const publication = { + generation: 1, + identity: operatorPublication.identity, + policies: f.authority.operatorPolicies, + }; + const target = create(ManagedPolicyTargetSchema, { + sessionId: f.sessionId, + sandboxId: crypto.randomUUID(), + hostId: crypto.randomUUID(), + bindingEpoch: 1n, + }); + const targetJson = { + session_id: f.sessionId, + sandbox_id: target.sandboxId, + host_id: target.hostId, + binding_epoch: 1, + }; + const control = createClient( + SessionPolicyService, + createRouterTransport((router) => + router.service(SessionPolicyService, { + async reserveManagedCreate() { + const v = (await store.read(f.sessionId)).session.mcpManagedLifecycle!; + state = { + query: managedQuery(v), + state: "held", + boot: null, + operation: null, + terminal_reason: null, + }; + return { protocol: 1, identity: v.identity, state: "held" }; + }, + async readNativeManifest() { + const v = (await store.read(f.sessionId)).session.mcpManagedLifecycle!; + return { + protocol: 1, + identity: v.identity, + nativeVersion: 1, + nativeDigest: v.nativeDigest, + nativeBytes: new TextEncoder().encode("[]"), + }; + }, + async readManagedPolicy() { + return { + protocol: 1, + retainedDesired: { + target, + policy: applied?.policy ?? { + ...f.snapshot.managedDenyPolicy, + revision: BigInt(f.snapshot.managedDenyPolicy.revision), + }, + }, + applied, + }; + }, + async applyManagedPolicy(request) { + applyCalls++; + const v = (await store.read(f.sessionId)).session.mcpManagedLifecycle!; + expect(v.phase).toBe("finalizing"); + expect(v.pending?.snapshot.bindingHash).toBe(request.context!.bindingHash!); + const p = managedPolicy( + create( + (await import("../gen/engram/mcp/v1/session_policy_pb.ts")) + .ManagedDenyPolicySchema, + request.policy!, + ), + ); + const json = { + context: { + target: targetJson, + operator_generation: 1, + operator_identity: publication.identity, + binding_hash: request.context!.bindingHash, + }, + policy: p, + host_incarnation: "host-incarnation", + proxy_incarnation: "proxy-incarnation", + state_id: "", + network_proof: missingProof + ? null + : { + target: targetJson, + guest_ip: "192.0.2.6", + network_identity: physicalNetworkIdentity, + }, + }; + applied = create(ManagedAppliedStateSchema, { + context: request.context, + policy: request.policy, + hostIncarnation: json.host_incarnation, + proxyIncarnation: json.proxy_incarnation, + stateId: `sha256:${createHash("sha256").update(JSON.stringify(json)).digest("hex")}`, + ...(missingProof + ? {} + : { + networkProof: { + target, + guestIp: "192.0.2.6", + networkIdentity: physicalNetworkIdentity, + }, + }), + }); + return { protocol: 1, applied }; + }, + async finalizeManagedBoot(request) { + finalizeCalls++; + const input: { + native_version: 1; + combined_manifest: number[]; + combined_manifest_digest: string; + } = JSON.parse(new TextDecoder().decode(request.inputJson)); + expect(applied).toBeDefined(); + expect( + (await store.read(f.sessionId)).session.mcpManagedLifecycle!.pending! + .combinedManifestDigest, + ).toBe(input.combined_manifest_digest); + state!.boot!.phase = "finalizing"; + state!.boot!.finalization = { ...input, prepared: state!.boot!.prepared! }; + throw Error("lost finalization reply"); + }, + }), + ), + ); + const managed = makeManagedPolicyAuthority({ + control, + readPublished: async () => publication, + readPlacement: async () => { + const prepared = state?.boot?.prepared; + if (!prepared) throw Error("MCP managed placement unavailable"); + return { + target, + networkIdentity: prepared.network_identity, + guestIp: prepared.guest_ip, + hostIncarnation: prepared.host_incarnation, + }; + }, + }); + const encode = () => ({ + protocol: 1, + stateJson: new TextEncoder().encode(JSON.stringify(state)), + }); + const runtime = makeManagedSessionRuntime({ + store, + control, + managed, + enabled: () => true, + authorize: async () => {}, + validate: async () => {}, + enroll: async () => {}, + readPublished: async () => publication, + bridge: { + readManagedCreationState: async () => encode(), + recoverManagedBoot: async () => { + throw Error("unexpected recovery"); + }, + async prepareManagedBoot() { + const v = (await store.read(f.sessionId)).session.mcpManagedLifecycle!; + const request = { + key: { + creation: state!.query.identity, + native_digest: v.nativeDigest, + boot_attempt: "boot-attempt", + context: { + target: targetJson, + operator_generation: 1, + operator_identity: publication.identity, + binding_hash: null, + }, + }, + bootstrap_policy: f.snapshot.managedDenyPolicy, + }; + state!.state = "boot"; + state!.boot = { + request, + revision: 2, + phase: "prepared", + sealed_agent: null, + finalization: null, + retired: null, + prepared: { + request, + host_incarnation: "host-incarnation", + network_identity: physicalNetworkIdentity, + guest_ip: "192.0.2.6", + receipt_id: `sha256:${"a".repeat(64)}`, + }, + }; + return encode(); + }, + }, + }); + const row = (await store.read(f.sessionId)).session; + await runtime.create( + { + imageUri: "image", + mode: "agent", + requestedSessionId: f.sessionId, + harnessEnv: { ENGRAM_TOOLS: "[]" }, + mcpBindingEnvelope: row.mcpBindingEnvelope!, + integrationConnections: row.integrationConnections!, + }, + { actorId: f.actorId, inputId: f.sessionId }, + ); + await expect(runtime.runOnce(f.sessionId)).rejects.toThrow(); + expect(applyCalls).toBe(1); + expect(finalizeCalls).toBe(missingProof ? 0 : 1); + expect((await store.read(f.sessionId)).session.mcpManagedLifecycle!.phase).toBe("finalizing"); + if (!missingProof) { + state!.boot!.phase = "started"; + const tls = { + url: "https://control.example.test/", + ca: "ca", + cert: "cert", + key: "key", + serverFingerprint: Array(32).fill("AA").join(":"), + }; + const clientSpy = spyOn(controlClients, "makeSessionPolicyControlClient").mockReturnValue( + control, + ); + const storedSpy = spyOn(bindingStore, "readStoredBindingSession").mockRejectedValue( + Error("stored binding reached"), + ); + try { + const composed = makeMcpRuntime({ + db: f.db, + deploymentId: f.deploymentId, + enabled: () => true, + configuration: { + infrastructure: operatorPublication, + control: tls, + registrations: [], + workers: new Map(), + }, + bridge: () => ({ + readManagedCreationState: async () => encode(), + prepareManagedBoot: async () => encode(), + recoverManagedBoot: async () => encode(), + }), + }); + if (!composed.resolver) throw Error("configured resolver missing"); + const error = await composed.resolver.readCurrent(f.sessionId).catch((e: unknown) => e); + expect(error).toBeInstanceOf(Error); + expect(error).not.toBeInstanceOf(McpAuthorityStale); + expect(error).toHaveProperty("message", "MCP managed session start is pending"); + expect(storedSpy).not.toHaveBeenCalled(); + const originalKey = state!.boot!.request.key; + state!.boot!.request.key = { + ...originalKey, + boot_attempt: "other-attempt", + }; + await expect(composed.resolver.readCurrent(f.sessionId)).rejects.toBeInstanceOf( + McpAuthorityStale, + ); + state!.boot!.request.key = originalKey; + await runtime.runOnce(f.sessionId); + expect(finalizeCalls).toBe(1); + expect(applyCalls).toBe(1); + expect((await store.read(f.sessionId)).session.mcpManagedLifecycle!.phase).toBe( + "started", + ); + await expect(composed.resolver.readCurrent(f.sessionId)).rejects.toThrow( + "stored binding reached", + ); + } finally { + storedSpy.mockRestore(); + clientSpy.mockRestore(); + } + } + } finally { + await f.cleanup(); + } + }); diff --git a/orchestrator/src/__tests__/mcp-operator-runtime.test.ts b/orchestrator/src/__tests__/mcp-operator-runtime.test.ts new file mode 100644 index 000000000..d440cb54b --- /dev/null +++ b/orchestrator/src/__tests__/mcp-operator-runtime.test.ts @@ -0,0 +1,65 @@ +import { expect, test } from "bun:test"; +import { makeMcpOperatorRuntime } from "../mcp/operator-runtime.ts"; +import { operatorPublication } from "./mcp-binding-fixture.ts"; +import type { McpRuntimeConfiguration } from "../mcp/runtime-config.ts"; + +test("worker selection requires fresh publication, exact destination, and both switches", async () => { + const tls = { + url: "https://worker.example.test/", + ca: "ca", + cert: "cert", + key: "key", + serverFingerprint: "AA", + }; + const configuration: McpRuntimeConfiguration = { + infrastructure: operatorPublication, + control: tls, + registrations: [], + workers: new Map([["public", { enabled: true, deploymentId: "zone-workers", tls }]]), + }; + let enabled = true, + available = true, + reads = 0; + const policy = { + publication: operatorPublication, + zoneId: "public", + policyRevision: "1", + policyDigest: `sha256:${"f".repeat(64)}`, + authorities: ["tools.example.test"], + aliases: [], + }; + const runtime = makeMcpOperatorRuntime({ + configuration, + enabled: () => enabled, + read: async () => { + reads++; + if (!available) throw Error("pending"); + return { + generation: 1, + identity: operatorPublication.identity, + policies: [structuredClone(policy)], + }; + }, + }); + const config = { + endpoint: "https://tools.example.test/mcp", + zone: "public", + auth: { kind: "api_key" as const, secretRef: "" }, + }; + const selected = await runtime.select(config); + expect(selected.deploymentId).toBe("zone-workers"); + await expect( + runtime.select({ ...config, endpoint: "https://alias.example.test/mcp" }), + ).rejects.toThrow(); + policy.publication = { ...operatorPublication, generation: 2 }; + await expect(runtime.assertCurrent(config, selected.policy)).rejects.toThrow("changed"); + available = false; + await expect(runtime.select(config)).rejects.toThrow("pending"); + expect(reads).toBe(4); + enabled = false; + await expect(runtime.select(config)).rejects.toThrow("disabled"); + enabled = true; + configuration.workers.get("public")!.enabled = false; + await expect(runtime.select(config)).rejects.toThrow("disabled"); + expect(reads).toBe(4); +}); diff --git a/orchestrator/src/__tests__/mcp-operator.test.ts b/orchestrator/src/__tests__/mcp-operator.test.ts new file mode 100644 index 000000000..533841549 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-operator.test.ts @@ -0,0 +1,105 @@ +import { expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { ZonePolicy } from "../../../mcp-worker/src/network.ts"; +import { canonicalJson } from "../mcp/hash.ts"; +import { parsePublishedOperator, makeOperatorReader } from "../mcp/operator.ts"; + +const hash = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; +function fixture() { + const config = { + id: "public", + revision: "1", + mode: "public" as const, + authorities: ["tools.example.test"], + aliases: ["alias.example.test"], + cidrs: [], + denyCidrs: [], + }; + const zone = { + revision: "1", + worker_digest: `sha256:${new ZonePolicy(config).digest}`, + authorities: config.authorities, + aliases: config.aliases, + configuration: JSON.parse(canonicalJson(config)), + }; + const expected = { + deploymentIdentity: hash("deployment"), + networkIdentity: hash("network"), + trustIdentity: hash("trust"), + caContentDigests: { control: hash("ca-content") }, + }; + const candidate = { + version: 1, + zones: { public: zone }, + deployment_identity: expected.deploymentIdentity, + network_identity: expected.networkIdentity, + trust_identity: expected.trustIdentity, + ca_content_digests: expected.caContentDigests, + }; + const value = { + published: { + generation: 1, + expected_generation: null, + identity: hash(JSON.stringify(candidate)), + candidate, + }, + pending: null, + admission_open: true, + }; + return { value, expected, encode: () => new TextEncoder().encode(JSON.stringify(value)) }; +} +test("published operator reader binds exact complete coordinator and worker identities", () => { + const f = fixture(); + const actual = parsePublishedOperator(f.encode(), f.expected); + expect(actual.generation).toBe(1); + expect(actual.policies[0]).toMatchObject({ + zoneId: "public", + authorities: ["tools.example.test"], + publication: { ...f.expected, generation: 1, identity: f.value.published.identity }, + }); +}); +test("operator reader rejects pending, incompatible, altered and absent authority", () => { + for (const mutate of [ + (f: ReturnType) => { + f.value.admission_open = false; + }, + (f: ReturnType) => { + Object.assign(f.value, { pending: f.value.published }); + }, + (f: ReturnType) => { + f.value.published.candidate.network_identity = hash("changed"); + }, + (f: ReturnType) => { + f.expected.caContentDigests.control = hash("changed-ca"); + }, + (f: ReturnType) => { + f.value.published.candidate.zones.public.worker_digest = hash("fake-worker"); + f.value.published.identity = hash(JSON.stringify(f.value.published.candidate)); + }, + (f: ReturnType) => { + f.value.published.candidate.zones.public.aliases.push("unpublished.example.test"); + }, + ]) { + const f = fixture(); + mutate(f); + expect(() => parsePublishedOperator(f.encode(), f.expected)).toThrow(); + } + expect(() => parsePublishedOperator(new Uint8Array(), fixture().expected)).toThrow(); +}); +test("operator selection performs a new published read for every operation", async () => { + const f = fixture(); + let calls = 0; + const reader = makeOperatorReader( + { + readPublishedOperator: async () => { + calls++; + return { protocol: 1, publicationJson: f.encode() }; + }, + }, + f.expected, + ); + expect((await reader.read()).policies).toHaveLength(1); + f.value.admission_open = false; + await expect(reader.read()).rejects.toThrow(); + expect(calls).toBe(2); +}); diff --git a/orchestrator/src/__tests__/mcp-phased-admission.test.ts b/orchestrator/src/__tests__/mcp-phased-admission.test.ts new file mode 100644 index 000000000..accac34f2 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-phased-admission.test.ts @@ -0,0 +1,299 @@ +import { expect, test } from "bun:test"; +import { reviewFixture } from "./mcp-review-fixture.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; + +// Real isolated PostgreSQL; schema success below is an explicit trusted-worker fixture. +test("schema lease is released before approval and execute gets a distinct generation", async () => { + const f = await reviewFixture(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await expect(f.journal.requestApproval(call.id, f.deploymentId, f.now)).rejects.toThrow( + "schema", + ); + expect(await f.journal.admitSchema(call.id, f.deploymentId, f.now)).toBe("admitted"); + const schema = (await f.journal.getAdmission(call.id))!; + await f.journal.finishSchema( + call.id, + schema.reservationId, + schema.expiresAt, + call.requestDigest, + true, + f.now, + ); + expect((await f.journal.getAdmission(call.id))!.state).toBe("released"); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, f.now); + expect(approval.expiresAt.getTime()).toBe(f.now.getTime() + 600_000); + expect((await f.journal.getAdmission(call.id))!.schemaDigest).toBe(call.requestDigest); + await expect( + f.journal.finishSchema( + call.id, + schema.reservationId, + schema.expiresAt, + canonicalDigest({ wrong: true }), + true, + f.now, + ), + ).rejects.toThrow(); + } finally { + await f.cleanup(); + } +}); + +test("released failed schema preserves first queue expiry across replacement generations", async () => { + const f = await reviewFixture(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.admitSchema(call.id, f.deploymentId, f.now); + const first = (await f.journal.getAdmission(call.id))!; + await f.journal.finishSchema( + call.id, + first.reservationId, + first.expiresAt, + call.requestDigest, + false, + f.now, + ); + const later = new Date(f.now.getTime() + 20_000); + await f.journal.admitSchema(call.id, f.deploymentId, later); + const second = (await f.journal.getAdmission(call.id))!; + expect(second.reservationId).not.toBe(first.reservationId); + expect(second.schemaQueueExpiresAt).toEqual(first.schemaQueueExpiresAt); + await expect( + f.journal.finishSchema( + call.id, + first.reservationId, + first.expiresAt, + call.requestDigest, + true, + later, + ), + ).rejects.toThrow(); + await f.journal.finishSchema( + call.id, + second.reservationId, + second.expiresAt, + call.requestDigest, + false, + later, + ); + await expect( + f.journal.admitSchema(call.id, f.deploymentId, new Date(f.now.getTime() + 30_001)), + ).rejects.toThrow("queue"); + } finally { + await f.cleanup(); + } +}); + +test("execute replacement preserves first queue and tool expiries after approval wait", async () => { + const f = await reviewFixture(); + try { + const call = await f.journal.create( + { ...f.input, deadline: new Date(f.now.getTime() + 900_000) }, + { kind: "approval" }, + f.now, + ); + await f.journal.admitSchema(call.id, f.deploymentId, f.now); + const schema = (await f.journal.getAdmission(call.id))!; + await f.journal.finishSchema( + call.id, + schema.reservationId, + schema.expiresAt, + call.requestDigest, + true, + f.now, + ); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const { reviewContext } = await import("../mcp/journal-review.ts"); + const afterWait = new Date(f.now.getTime() + 300_000); + const approved = await f.journal.decide( + call.id, + f.actorId, + true, + reviewContext(call, approval), + afterWait, + ); + await f.journal.admit(call.id, f.deploymentId, afterWait); + const first = (await f.journal.getAdmission(call.id))!; + expect(first.reservationId).not.toBe(schema.reservationId); + expect(first.expiresAt.getTime()).toBe(afterWait.getTime() + 60_000); + expect(first.executeQueueExpiresAt!.getTime()).toBe(afterWait.getTime() + 30_000); + const proof = { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: approved.call.version - 1, + reservationId: first.reservationId, + reservationExpiresAt: first.expiresAt, + }; + await expect( + f.journal.dispatch( + call.id, + { bootId: "boot", intentId: crypto.randomUUID(), expiresAt: first.expiresAt }, + afterWait, + proof, + ), + ).rejects.toThrow("stale"); + await f.journal.admit(call.id, f.deploymentId, new Date(afterWait.getTime() + 20_000)); + const replacement = (await f.journal.getAdmission(call.id))!; + expect(replacement.reservationId).not.toBe(first.reservationId); + expect(replacement.expiresAt).toEqual(first.expiresAt); + expect(replacement.executeQueueExpiresAt).toEqual(first.executeQueueExpiresAt); + await expect( + f.journal.finishSchema( + call.id, + schema.reservationId, + schema.expiresAt, + call.requestDigest, + true, + afterWait, + ), + ).rejects.toThrow(); + expect((await f.journal.getAdmission(call.id))!.reservationId).toBe(replacement.reservationId); + } finally { + await f.cleanup(); + } +}); + +test("schema capacity is shared across database clients and released capacity promotes without renewing queue", async () => { + const fixtures = await Promise.all(Array.from({ length: 5 }, () => reviewFixture())); + const first = fixtures[0]!; + try { + const calls = await Promise.all( + fixtures.map((f) => f.journal.create(f.input, { kind: "approval" }, f.now)), + ); + const states = await Promise.all( + fixtures.map((f, i) => f.journal.admitSchema(calls[i]!.id, first.deploymentId, f.now)), + ); + expect(states.filter((state) => state === "admitted")).toHaveLength(4); + const queuedIndex = states.indexOf("queued"), + activeIndex = states.indexOf("admitted"); + const waiting = (await first.journal.getAdmission(calls[queuedIndex]!.id))!; + const active = (await first.journal.getAdmission(calls[activeIndex]!.id))!; + await first.journal.finishSchema( + calls[activeIndex]!.id, + active.reservationId, + active.expiresAt, + calls[activeIndex]!.requestDigest, + true, + first.now, + ); + expect( + await first.journal.admitSchema( + calls[queuedIndex]!.id, + first.deploymentId, + new Date(first.now.getTime() + 5_000), + ), + ).toBe("admitted"); + expect( + (await first.journal.getAdmission(calls[queuedIndex]!.id))!.schemaQueueExpiresAt, + ).toEqual(waiting.schemaQueueExpiresAt); + } finally { + const { task } = await import("../db/schema.ts"); + const { eq } = await import("drizzle-orm"); + for (const f of fixtures) await f.db.delete(task).where(eq(task.id, f.taskId)); + for (const f of fixtures) await f.cleanup(); + } +}); + +test("capacity reads exclude retained history but count expired active leases until release", async () => { + const f = await reviewFixture(); + const { drizzle } = await import("drizzle-orm/node-postgres"); + const { getPool } = await import("../db/client.ts"); + const schema = await import("../db/schema.ts"); + const { makeMcpAdmission } = await import("../mcp/admission.ts"); + const capacityReads: { query: string; params: unknown[] }[] = []; + const capacity = makeMcpAdmission( + drizzle(getPool(), { + schema, + logger: { + logQuery(query, params) { + if ( + query.startsWith("select") && + query.includes('from "mcp_admission"') && + query.includes('"mcp_admission"."deployment_id" =') + ) { + capacityReads.push({ query, params }); + } + }, + }, + }), + ); + try { + const calls = []; + for (let i = 0; i < 7; i++) { + const toolCallId = `capacity-${i}`; + const call = await f.journal.create( + { + ...f.input, + toolCallId, + requestPolicyContext: { ...f.input.requestPolicyContext, toolCallId }, + }, + { kind: "approval" }, + f.now, + ); + calls.push(call); + if (i === 6) continue; + await f.journal.admitSchema(call.id, f.deploymentId, f.now); + if (i < 2) { + const lease = (await f.journal.getAdmission(call.id))!; + await f.journal.finishSchema( + call.id, + lease.reservationId, + lease.expiresAt, + call.requestDigest, + i === 0, + f.now, + ); + } + } + const successfulHistory = (await f.journal.getAdmission(calls[0]!.id))!; + const failedHistory = (await f.journal.getAdmission(calls[1]!.id))!; + const later = new Date(f.now.getTime() + 15_000); + // The four active leases are expired, but still own capacity until explicit release. + expect(await capacity.reserve(calls[6]!.id, f.deploymentId, "schema", later)).toBe("queued"); + const waiting = (await f.journal.getAdmission(calls[6]!.id))!; + expect(capacityReads).toHaveLength(1); + const read = capacityReads[0]!; + // Replay the actual emitted SQL: four active rows plus the newly queued row, no history. + const population = await getPool().query<{ state: string; execution_id: string }>( + read.query, + read.params, + ); + expect(population.rows).toHaveLength(5); + expect(population.rows.filter((row) => row.state === "admitted")).toHaveLength(4); + expect(population.rows.filter((row) => row.state === "queued")).toHaveLength(1); + expect(population.rows.map((row) => row.execution_id)).not.toContain(calls[0]!.id); + expect(population.rows.map((row) => row.execution_id)).not.toContain(calls[1]!.id); + expect(await f.journal.getAdmission(calls[0]!.id)).toEqual(successfulHistory); + expect(await f.journal.getAdmission(calls[1]!.id)).toEqual(failedHistory); + const active = (await f.journal.getAdmission(calls[2]!.id))!; + await f.journal.finishSchema( + calls[2]!.id, + active.reservationId, + active.expiresAt, + calls[2]!.requestDigest, + false, + later, + ); + const promotionTime = new Date(f.now.getTime() + 20_000); + expect(await capacity.reserve(calls[6]!.id, f.deploymentId, "schema", promotionTime)).toBe( + "admitted", + ); + expect((await f.journal.getAdmission(calls[6]!.id))!.schemaQueueExpiresAt).toEqual( + waiting.schemaQueueExpiresAt, + ); + // The separately fetched released row still carries its original retry deadline. + expect(await capacity.reserve(calls[1]!.id, f.deploymentId, "schema", promotionTime)).toBe( + "queued", + ); + const replacement = (await f.journal.getAdmission(calls[1]!.id))!; + expect(replacement.schemaQueueExpiresAt).toEqual(failedHistory.schemaQueueExpiresAt); + expect(replacement.reservationId).not.toBe(failedHistory.reservationId); + expect((await f.journal.getAdmission(calls[0]!.id))!.schemaDigest).toBe( + calls[0]!.requestDigest, + ); + } finally { + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-preflight-rejection.test.ts b/orchestrator/src/__tests__/mcp-preflight-rejection.test.ts new file mode 100644 index 000000000..0f1edf7fe --- /dev/null +++ b/orchestrator/src/__tests__/mcp-preflight-rejection.test.ts @@ -0,0 +1,115 @@ +import { expect, test } from "bun:test"; +import { schemaVerifiedFixture } from "./mcp-review-fixture.ts"; +import { reviewContext } from "../mcp/journal-review.ts"; + +test("preflight rejection cannot release another generation or a committed dispatch", async () => { + const f = await schemaVerifiedFixture(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const approved = ( + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), f.now) + ).call; + await f.journal.admit(call.id, f.deploymentId, f.now); + const lease = (await f.journal.getAdmission(call.id))!; + const rejection = { + requestDigest: call.requestDigest, + version: approved.version, + reservation: { reservationId: lease.reservationId, expiresAt: lease.expiresAt }, + }; + await expect( + f.journal.rejectBeforeDispatch( + call.id, + { ...rejection, reservation: { ...rejection.reservation, reservationId: "stale" } }, + f.now, + ), + ).rejects.toThrow(); + expect((await f.journal.getAdmission(call.id))!.reservationId).toBe(lease.reservationId); + const proof = { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: approved.version, + reservationId: lease.reservationId, + reservationExpiresAt: lease.expiresAt, + }; + const results = await Promise.allSettled([ + f.journal.dispatch( + call.id, + { bootId: "boot", intentId: crypto.randomUUID(), expiresAt: lease.expiresAt }, + f.now, + proof, + ), + f.journal.rejectBeforeDispatch(call.id, rejection, f.now), + ]); + expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1); + const row = (await f.journal.get(call.id))!; + if (row.state === "dispatching") + expect((await f.journal.getAdmission(call.id))!.reservationId).toBe(lease.reservationId); + else { + expect(row.state).toBe("failed"); + expect(row.intentId).toBeNull(); + expect(await f.journal.getAdmission(call.id)).toBeNull(); + } + } finally { + await f.cleanup(); + } +}); + +import { persistVerifiedOutcome } from "../mcp/effect-driver.ts"; +test("a verified late success survives cancellation between result read and result commit", async () => { + const f = await schemaVerifiedFixture(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const approved = ( + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), f.now) + ).call; + await f.journal.admit(call.id, f.deploymentId, f.now); + const lease = (await f.journal.getAdmission(call.id))!; + const dispatched = await f.journal.dispatch( + call.id, + { bootId: "boot", intentId: crypto.randomUUID(), expiresAt: lease.expiresAt }, + f.now, + { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: approved.version, + reservationId: lease.reservationId, + reservationExpiresAt: lease.expiresAt, + }, + ); + let commits = 0; + const result = await persistVerifiedOutcome( + { + ...f.journal, + async finish(...args) { + if (++commits === 1) + await f.journal.cancel( + call.id, + f.actorId, + f.now, + (await f.journal.get(call.id))!.version, + ); + return f.journal.finish(...args); + }, + }, + dispatched, + { + state: "succeeded", + result: { outcome: "succeeded", delivery: "available", output: { content: [] } }, + resultUnavailable: false, + }, + () => f.now, + ); + expect(result.state).toBe("succeeded"); + expect(result.intentId).toBe(dispatched.intentId); + expect(commits).toBe(2); + expect(await f.journal.getAdmission(call.id)).toBeNull(); + } finally { + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-profile-publication.test.ts b/orchestrator/src/__tests__/mcp-profile-publication.test.ts new file mode 100644 index 000000000..950239ccc --- /dev/null +++ b/orchestrator/src/__tests__/mcp-profile-publication.test.ts @@ -0,0 +1,182 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { user, profile } from "../db/schema.ts"; +import { makeProfileStore } from "../db/profiles.ts"; +import { reviewFixture } from "./mcp-review-fixture.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; + +test("MCP publication requires current admin and exact explicit revision; unrelated edits preserve provenance", async () => { + const f = await reviewFixture(); + try { + const store = makeProfileStore(f.db); + const input = (await store.get(f.profileId))!; + await expect( + store.update(f.profileId, input, { actorId: f.actorId, expectedRevision: 0 }), + ).rejects.toThrow("administrator"); + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + const published = await store.update(f.profileId, input, { + actorId: f.actorId, + expectedRevision: 0, + }); + expect(published!.mcpGrantPublication).toEqual({ + revision: 1, + actorId: f.actorId, + grantsHash: canonicalDigest(input.integrationGrants), + }); + await expect( + store.update(f.profileId, input, { actorId: f.actorId, expectedRevision: 0 }), + ).rejects.toThrow("revision"); + const unchanged = await store.update(f.profileId, { ...input, name: "Changed label" }); + expect(unchanged!.mcpGrantPublication).toEqual(published!.mcpGrantPublication); + await f.db + .update(profile) + .set({ mcpGrantPublication: null }) + .where(eq(profile.id, f.profileId)); + expect( + (await store.update(f.profileId, { ...input, name: "Legacy label" }))!.mcpGrantPublication, + ).toBeNull(); + await expect(store.update(f.profileId, { ...input, integrationGrants: [] })).rejects.toThrow( + "publication", + ); + await f.db.update(user).set({ role: "user" }).where(eq(user.id, f.actorId)); + await expect( + store.update(f.profileId, input, { actorId: f.actorId, expectedRevision: 0 }), + ).rejects.toThrow("administrator"); + } finally { + await f.cleanup(); + } +}); + +test("publication waits for a current-role write and concurrent revision CAS has one winner", async () => { + const f = await reviewFixture(); + const { sql } = await import("drizzle-orm"); + try { + const store = makeProfileStore(f.db), + input = (await store.get(f.profileId))!; + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + let unlock!: () => void, locked!: (pid: number) => void; + const release = new Promise((resolve) => { + unlock = resolve; + }); + const ready = new Promise((resolve) => { + locked = resolve; + }); + const mutation = f.db.transaction(async (tx) => { + await tx.update(user).set({ role: "user" }).where(eq(user.id, f.actorId)); + const rows = await tx.execute<{ pid: number }>(sql`SELECT pg_backend_pid() AS pid`); + locked(Number(rows.rows[0]!.pid)); + await release; + }); + const blocker = await ready; + const attempt = store + .update(f.profileId, input, { actorId: f.actorId, expectedRevision: 0 }) + .then( + () => "allowed", + () => "denied", + ); + try { + const deadline = Date.now() + 2000; + let blocked = false; + while (Date.now() < deadline) { + const rows = await f.db.execute<{ waiting: boolean }>( + sql`SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE ${blocker} = ANY(pg_blocking_pids(pid))) AS waiting`, + ); + if (rows.rows[0]?.waiting) { + blocked = true; + break; + } + await Bun.sleep(5); + } + expect(blocked).toBe(true); + } finally { + unlock(); + } + await mutation; + expect(await attempt).toBe("denied"); + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + const results = await Promise.allSettled( + [1, 2].map(() => + store.update(f.profileId, input, { actorId: f.actorId, expectedRevision: 0 }), + ), + ); + expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1); + expect((await store.get(f.profileId))!.mcpGrantPublication?.revision).toBe(1); + } finally { + await f.cleanup(); + } +}); + +test("stored resolver permits current unknown write policy but rejects terminal identity", async () => { + const f = await reviewFixture(); + const { createClient, createRouterTransport } = await import("@connectrpc/connect"); + const { SessionPolicyService } = await import("../gen/engram/mcp/v1/session_policy_pb.ts"); + const { makeStoredSessionBindingResolver } = await import("../mcp/binding-store.ts"); + let current = true; + const control = createClient( + SessionPolicyService, + createRouterTransport((router) => + router.service(SessionPolicyService, { + readWritePolicy: async () => ({ + state: { + policy: { + ref: f.input.requestPolicyContext.acceptedPolicyRef, + current, + initialized: false, + writeEligibility: "unknown", + }, + }, + }), + }), + ), + ); + try { + const resolver = makeStoredSessionBindingResolver( + { + control, + readOperatorPolicies: async () => f.authority.operatorPolicies, + enforceManagedDeny: async (expected) => expected, + }, + f.db, + ); + const { readLiveBindingAuthority, bindingSnapshotColumns } = + await import("../mcp/binding-store.ts"); + const { compileSessionBindings } = await import("../mcp/session-bindings.ts"); + const { taskSession } = await import("../db/schema.ts"); + const live = await readLiveBindingAuthority(f.profileId, f.authority.operatorPolicies, f.db); + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: f.profileId, integrationGrants: live.profileGrants }, + liveAuthority: live, + bindingEpoch: 0, + }); + await f.db + .update(taskSession) + .set( + bindingSnapshotColumns(snapshot, [ + { + id: f.connection.id, + alias: f.connection.alias, + provider: "mcp", + displayName: f.connection.displayName, + config: f.connection.config, + }, + ]), + ) + .where(eq(taskSession.sessionId, f.sessionId)); + await resolver.readCurrent(f.sessionId); + expect( + await resolver.resolveBoundTool({ + sessionId: f.sessionId, + modelToolName: f.input.resolvedTool.modelToolName, + }), + ).toBeDefined(); + current = false; + expect( + await resolver.resolveBoundTool({ + sessionId: f.sessionId, + modelToolName: f.input.resolvedTool.modelToolName, + }), + ).toBeUndefined(); + } finally { + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-public-create.test.ts b/orchestrator/src/__tests__/mcp-public-create.test.ts new file mode 100644 index 000000000..bf4452529 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-public-create.test.ts @@ -0,0 +1,267 @@ +import { expect, spyOn, test } from "bun:test"; +import { createClient, createRouterTransport } from "@connectrpc/connect"; +import { fromBinary } from "@bufbuild/protobuf"; +import { createHash } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { ZonePolicy } from "../../../mcp-worker/src/network.ts"; +import { makeProfileStore } from "../db/profiles.ts"; +import { sessionListener, task, taskSession } from "../db/schema.ts"; +import { + CreatePolicySessionRequestSchema, + SessionPolicyService, +} from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import * as controlClients from "../mcp/control-client.ts"; +import * as capabilities from "../mcp/session-capabilities.ts"; +import { canonicalJson } from "../mcp/hash.ts"; +import { McpManagedCreationPending } from "../mcp/managed-runtime.ts"; +import { managedQuery } from "../mcp/managed-state.ts"; +import { makeMcpRuntime } from "../mcp/runtime.ts"; +import { + createSessionForExistingTask, + createTaskWithSession, + registerSessionListener, + type CreateTaskDeps, +} from "../rpc/task-create.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +import { reviewFixture } from "./mcp-review-fixture.ts"; + +await verifyMcpTestDatabases(); + +for (const existing of [false, true]) { + for (const lostReply of [false, true]) { + test(`public ${existing ? "existing-task" : "root"} managed create retains one listener (${lostReply ? "lost reserve reply" : "successful reserve"})`, async () => { + const f = await reviewFixture(); + const taskId = existing ? f.taskId : crypto.randomUUID(); + const sessionId = crypto.randomUUID(); + let reserves = 0; + let enabled = true; + const deleted: string[] = []; + const hash = (text: string) => `sha256:${createHash("sha256").update(text).digest("hex")}`; + const zone = { + id: "public", + revision: "1", + mode: "public" as const, + authorities: ["tools.example.test"], + aliases: [], + cidrs: [], + denyCidrs: [], + }; + const infrastructure = { + deploymentIdentity: hash("deployment"), + networkIdentity: hash("network"), + trustIdentity: hash("trust"), + caContentDigests: { control: hash("ca") }, + }; + const candidate = { + version: 1, + zones: { + public: { + revision: "1", + worker_digest: `sha256:${new ZonePolicy(zone).digest}`, + authorities: zone.authorities, + aliases: zone.aliases, + configuration: JSON.parse(canonicalJson(zone)), + }, + }, + deployment_identity: infrastructure.deploymentIdentity, + network_identity: infrastructure.networkIdentity, + trust_identity: infrastructure.trustIdentity, + ca_content_digests: infrastructure.caContentDigests, + }; + const control = createClient( + SessionPolicyService, + createRouterTransport((router) => { + router.service(SessionPolicyService, { + readPublishedOperator: async () => ({ + protocol: 1, + publicationJson: new TextEncoder().encode( + JSON.stringify({ + published: { + generation: 1, + expected_generation: null, + identity: hash(JSON.stringify(candidate)), + candidate, + }, + pending: null, + admission_open: true, + }), + ), + }), + reserveManagedCreate: async (request) => { + reserves++; + const input = fromBinary(CreatePolicySessionRequestSchema, request.creationRequest); + expect(input.create?.requestedSessionId).toBe(sessionId); + if (lostReply) throw Error("reserve reply lost"); + return { + protocol: 1, + state: "held", + identity: { + sessionId, + taskId: request.taskId, + actorId: input.actorId, + inputId: input.inputId, + parentSessionId: input.parentSessionId, + inputDigest: request.inputDigest, + writeEligibility: "write_eligible", + }, + }; + }, + readManagedCreationState: async () => { + const [row] = await f.db + .select() + .from(taskSession) + .where(eq(taskSession.sessionId, sessionId)); + if (!row?.mcpManagedLifecycle) throw Error("managed reservation missing"); + return { + protocol: 1, + stateJson: new TextEncoder().encode( + JSON.stringify({ + query: managedQuery(row.mcpManagedLifecycle), + state: "held", + boot: null, + operation: null, + terminal_reason: null, + }), + ), + }; + }, + }); + }), + ); + // Only the physical control transport is replaced; enrollment uses the real runtime and PG. + const controlSpy = spyOn(controlClients, "makeSessionPolicyControlClient").mockReturnValue( + control, + ); + const tls = { + url: "https://control.example.test/", + ca: "ca", + cert: "cert", + key: "key", + serverFingerprint: Array(32).fill("AA").join(":"), + }; + const runtime = makeMcpRuntime({ + db: f.db, + deploymentId: f.deploymentId, + enabled: () => enabled, + configuration: { + infrastructure, + control: tls, + registrations: [], + workers: new Map([["public", { enabled: true, deploymentId: f.deploymentId, tls }]]), + }, + bridge: (client) => client, + }); + if (!runtime.capabilities || !runtime.lifecycle) throw Error("configured runtime missing"); + const capabilitySpy = spyOn(capabilities, "mcpSessionCapabilities").mockReturnValue( + runtime.capabilities, + ); + try { + const deps: CreateTaskDeps = { + db: f.db, + profiles: makeProfileStore(f.db), + connections: f.connections, + images: { + listEnabledImages: async () => ({ images: [{ id: "image", imageUri: "image" }] }), + }, + connectors: { list: async () => [] }, + harnessCatalog: { + listHarnesses: async () => ({ + harnesses: [ + { + name: "claude", + descriptor: { auth: { userEnv: "TEST_TOKEN" }, models: [], effort: [] }, + }, + ], + }), + }, + secrets: { get: async () => "neutral-token", getAll: async () => ({}) }, + oauth: { listCredentials: async () => ({ credentials: [] }) }, + orgSecret: { listSecrets: async () => ({ secrets: [] }) }, + sessions: { + createSession: async () => { + throw Error("generic creation must not run"); + }, + deleteSession: async (req) => { + deleted.push(req.sessionId); + }, + }, + newTaskId: () => taskId, + newSessionId: () => sessionId, + }; + const create = () => + existing + ? createSessionForExistingTask(deps, { + taskId, + profileId: f.profileId, + ownerUserId: f.actorId, + role: "primary", + registerListener: true, + }) + : createTaskWithSession(deps, { + type: "chat", + profileId: f.profileId, + ownerUserId: f.actorId, + prompt: "Neutral task", + }); + if (lostReply) { + await expect(create()).rejects.toBeInstanceOf(McpManagedCreationPending); + expect( + await f.db + .select() + .from(sessionListener) + .where(eq(sessionListener.sessionId, sessionId)), + ).toHaveLength(0); + enabled = false; + await runtime.lifecycle.runOnce(sessionId); + await runtime.lifecycle.runOnce(sessionId); + } else { + expect((await create()).sessionId).toBe(sessionId); + // Re-enrollment must not reset a listener lease or terminal marker. + await f.db + .update(sessionListener) + .set({ owner: "listener-owner", terminalAt: f.now }) + .where(eq(sessionListener.sessionId, sessionId)); + const [listener] = await f.db + .select() + .from(sessionListener) + .where(eq(sessionListener.sessionId, sessionId)); + await registerSessionListener(f.db, sessionId); + expect( + ( + await f.db + .select() + .from(sessionListener) + .where(eq(sessionListener.sessionId, sessionId)) + )[0], + ).toEqual(listener); + } + expect(reserves).toBe(1); + expect(deleted).toEqual([]); + expect(await f.db.select().from(task).where(eq(task.id, taskId))).toHaveLength(1); + const rows = await f.db + .select() + .from(taskSession) + .where(eq(taskSession.sessionId, sessionId)); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + taskId, + integrationPrincipalId: f.actorId, + mcpManagedLifecycle: { + phase: lostReply ? "reserving" : "held", + identity: { sessionId, taskId }, + }, + }); + expect( + await f.db.select().from(sessionListener).where(eq(sessionListener.sessionId, sessionId)), + ).toHaveLength(1); + } finally { + capabilitySpy.mockRestore(); + controlSpy.mockRestore(); + await f.db.delete(sessionListener).where(eq(sessionListener.sessionId, sessionId)); + await f.db.delete(taskSession).where(eq(taskSession.sessionId, sessionId)); + if (!existing) await f.db.delete(task).where(eq(task.id, taskId)); + await f.cleanup(); + } + }); + } +} diff --git a/orchestrator/src/__tests__/mcp-rebind-deny.test.ts b/orchestrator/src/__tests__/mcp-rebind-deny.test.ts new file mode 100644 index 000000000..ecb433beb --- /dev/null +++ b/orchestrator/src/__tests__/mcp-rebind-deny.test.ts @@ -0,0 +1,25 @@ +import { expect, test } from "bun:test"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { unionManagedPolicies } from "../mcp/managed-policy.ts"; +function policy(revision: number, hosts: string[]) { + const body = { version: 1 as const, revision, hosts }; + return { ...body, digest: canonicalDigest(body) }; +} +test("cold deny union retains old, new and lost acknowledgement hosts at a newer revision", () => { + const result = unionManagedPolicies([ + policy(2, ["old.example.test"]), + policy(3, ["new.example.test", "parent.example.test"]), + policy(3, ["lost.example.test", "old.example.test"]), + ]); + expect(result.revision).toBe(4); + expect(result.hosts).toEqual([ + "lost.example.test", + "new.example.test", + "old.example.test", + "parent.example.test", + ]); + expect(result.digest).toBe(canonicalDigest({ version: 1, revision: 4, hosts: result.hosts })); +}); +test("cold deny union rejects corrupted retained policy", () => { + expect(() => unionManagedPolicies([{ ...policy(1, ["old.example.test"]), hosts: [] }])).toThrow(); +}); diff --git a/orchestrator/src/__tests__/mcp-rebind-dispatch-race.test.ts b/orchestrator/src/__tests__/mcp-rebind-dispatch-race.test.ts new file mode 100644 index 000000000..73eea3707 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-rebind-dispatch-race.test.ts @@ -0,0 +1,331 @@ +import { expect, test } from "bun:test"; +import { effectFixture } from "./mcp-effect-fixture.ts"; +import { McpRebindDraining } from "../mcp/rebind.ts"; +import { coldFixture } from "./mcp-cold-fixture.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +await verifyMcpTestDatabases(); + +test("a rebind cannot expose a new binding while the old dispatch is in its final fence", async () => { + const f = await effectFixture({ classification: "read", bindingEpoch: 1 }); + const cold = await coldFixture(f, f.operatorPolicies); + let unblock!: () => void; + const blocked = new Promise((resolve) => { + unblock = resolve; + }); + let notify!: () => void; + const entered = new Promise((resolve) => { + notify = resolve; + }); + try { + const call = await f.driver.acceptToolRequest(f.request); + let afterLeaseFences = 0; + f.beforeFence(async () => { + if (f.leases > 0 && ++afterLeaseFences === 2) { + notify(); + await blocked; + } + }); + const pending = f.driver.driveOnce(call.id); + await entered; + const service = cold.service; + const candidate = await service.prepare(f.sessionId, f.snapshot.bindingHash, f.actorId); + expect(candidate.drain?.reason).toBe("dispatch_in_progress"); + expect(candidate.drain?.retryAfter).toBeTruthy(); + const { mcpOperation } = await import("../mcp/contracts.ts"); + const { McpRebindDrainSchema, McpRebindDrainReason } = + await import("../gen/engram/app/v1/integration_pb.ts"); + const { ConnectError } = await import("@connectrpc/connect"); + try { + await mcpOperation(() => + service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId), + ); + throw Error("expected drain"); + } catch (error) { + expect(error).toBeInstanceOf(ConnectError); + expect( + (error as InstanceType).findDetails(McpRebindDrainSchema)[0]?.reason, + ).toBe(McpRebindDrainReason.DISPATCH_IN_PROGRESS); + } + const attempt = await service + .commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId) + .then( + () => "committed", + (error: Error) => error.message, + ); + unblock(); + await pending; + expect(attempt).toContain("dispatch in progress"); + expect((await f.command("stats")).calls).toBe(1); + await expect( + service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId), + ).rejects.toBeInstanceOf(McpRebindDraining); + await cold.restartRunOnce(); + cold.start(); + await cold.restartRunOnce(); + await service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId); + expect((await cold.store.read(f.sessionId)).session.mcpManagedLifecycle?.phase).toBe("started"); + } finally { + unblock(); + await f.cleanup(); + } +}, 20000); + +test("cancel and manual resolution do not waive the original in-flight intent", async () => { + const f = await effectFixture({ classification: "read", bindingEpoch: 1 }); + const cold = await coldFixture(f, f.operatorPolicies); + let unblock!: () => void; + const blocked = new Promise((resolve) => { + unblock = resolve; + }); + let notify!: () => void; + const entered = new Promise((resolve) => { + notify = resolve; + }); + try { + const call = await f.driver.acceptToolRequest(f.request); + let afterLeaseFences = 0; + f.beforeFence(async () => { + if (f.leases > 0 && ++afterLeaseFences === 2) { + notify(); + await blocked; + } + }); + const pending = f.driver.driveOnce(call.id); + await entered; + const dispatched = (await f.journal.get(call.id))!; + const unknown = await f.journal.cancel(call.id, f.actorId, new Date(), dispatched.version); + const context = { + executionId: call.id, + journalRevision: unknown.version, + requestDigest: unknown.requestDigest, + bindingHash: unknown.bindingHash, + intentId: unknown.intentId!, + }; + await f.journal.resolveUnknown(call.id, f.actorId, "acknowledged", new Date(), context); + const service = cold.service; + const candidate = await service.prepare(f.sessionId, f.snapshot.bindingHash, f.actorId); + expect(candidate.drain?.reason).toBe("unknown_intent"); + expect(candidate.drain?.retryAfter).toBeTruthy(); + const attempt = await service + .commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId) + .then( + () => "committed", + (error: Error) => error.message, + ); + unblock(); + await pending; + expect(attempt).toContain("dispatch in progress"); + expect((await f.command("stats")).calls).toBe(1); + await expect( + service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId), + ).rejects.toBeInstanceOf(McpRebindDraining); + await cold.restartRunOnce(); + cold.start(); + await cold.restartRunOnce(); + await service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId); + expect((await cold.store.read(f.sessionId)).session.mcpManagedLifecycle?.phase).toBe("started"); + } finally { + unblock(); + await f.cleanup(); + } +}, 20000); + +test("an expired original intent permits rebind and cannot send afterward", async () => { + const f = await effectFixture({ classification: "read", bindingEpoch: 1 }); + const cold = await coldFixture(f, f.operatorPolicies); + let unblock!: () => void; + const blocked = new Promise((resolve) => { + unblock = resolve; + }); + let notify!: () => void; + const entered = new Promise((resolve) => { + notify = resolve; + }); + try { + const call = await f.driver.acceptToolRequest(f.request); + let afterLeaseFences = 0; + f.beforeFence(async () => { + if (f.leases > 0 && ++afterLeaseFences === 2) { + notify(); + await blocked; + } + }); + const pending = f.driver.driveOnce(call.id); + await entered; + const dispatched = (await f.journal.get(call.id))!; + await f.journal.cancel(call.id, f.actorId, new Date(), dispatched.version); + const service = cold.service; + const candidate = await service.prepare(f.sessionId, f.snapshot.bindingHash, f.actorId); + cold.advanceColdNow(310_000); + f.advanceNow(310_000); + await expect( + service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId), + ).rejects.toBeInstanceOf(McpRebindDraining); + await cold.restartRunOnce(); + cold.start(); + await cold.restartRunOnce(); + await service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId); + unblock(); + const outcome = await pending; + expect(outcome.state).toBe("outcome_unknown"); + expect((await f.command("stats")).calls).toBe(0); + } finally { + unblock(); + await f.cleanup(); + } +}, 20000); + +test("a different-scope barrier commits before the waiting rebind reads its drain", async () => { + const { and, eq, sql } = await import("drizzle-orm"); + const { mcpExecution } = await import("../db/schema.ts"); + const { reviewContext } = await import("../mcp/journal-review.ts"); + const f = await effectFixture({ classification: "write", bindingEpoch: 1 }); + const cold = await coldFixture(f, f.operatorPolicies); + let release!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + let locked!: () => void; + const ready = new Promise((resolve) => { + locked = resolve; + }); + let blocker: Promise | undefined; + try { + const call = await f.driver.acceptToolRequest(f.request); + await f.driver.driveOnce(call.id); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, new Date()); + const approved = ( + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), new Date()) + ).call; + await f.journal.admit(call.id, f.deploymentId, new Date()); + const lease = (await f.journal.getAdmission(call.id))!; + const service = cold.service; + const candidate = await service.prepare(f.sessionId, f.snapshot.bindingHash, f.actorId); + blocker = f.db.transaction(async (tx) => { + await tx.select().from(mcpExecution).where(eq(mcpExecution.id, call.id)).for("update"); + locked(); + await held; + }); + await ready; + const barrier = f.journal.dispatch( + call.id, + { bootId: "boot", intentId: crypto.randomUUID(), expiresAt: lease.expiresAt }, + new Date(), + { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: approved.version, + reservationId: lease.reservationId, + reservationExpiresAt: lease.expiresAt, + }, + ); + async function waitForLocks(minimum: number) { + const deadline = Date.now() + 3000; + while (Date.now() < deadline) { + const result = await f.db.execute( + sql`select count(*)::integer as n from pg_stat_activity where datname = current_database() and wait_event_type = 'Lock' and state = 'active'`, + ); + if (Number(result.rows[0]?.n) >= minimum) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw Error("expected lock wait unavailable"); + } + await waitForLocks(1); + const commit = service + .commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId) + .then( + () => "committed", + (error: Error) => error.message, + ); + await waitForLocks(2); + release(); + await blocker; + expect((await barrier).state).toBe("dispatching"); + expect(await commit).not.toBe("committed"); + expect(cold.stats().applies).toBe(0); + } finally { + release(); + await blocker; + await f.cleanup(); + } +}, 20000); + +test("an accepted old call cannot cross the barrier after rebind", async () => { + const f = await effectFixture({ classification: "read", bindingEpoch: 1 }); + const cold = await coldFixture(f, f.operatorPolicies); + try { + const call = await f.driver.acceptToolRequest(f.request); + const service = cold.service; + const candidate = await service.prepare(f.sessionId, f.snapshot.bindingHash, f.actorId); + await expect( + service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId), + ).rejects.toBeInstanceOf(McpRebindDraining); + await cold.restartRunOnce(); + cold.start(); + await cold.restartRunOnce(); + await service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId); + await f.driver.driveOnce(call.id).catch(() => undefined); + expect((await f.command("stats")).calls).toBe(0); + expect((await f.journal.get(call.id))!.state).not.toBe("dispatching"); + } finally { + await f.cleanup(); + } +}, 20000); + +test("a held cold host call allows another session to acquire a real worker credential lease", async () => { + // The cold fixture has no process. Only the other session starts one isolated worker fixture. + const first = await coldFixture(); + const other = await effectFixture({ classification: "read" }); + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + let enter!: () => void; + const entered = new Promise((resolve) => { + enter = resolve; + }); + let pending: Promise | undefined; + try { + const candidate = await first.service.prepare( + first.sessionId, + first.snapshot.bindingHash, + first.actorId, + ); + first.beforeApply(async () => { + enter(); + await blocked; + }); + await expect( + first.service.commit( + first.sessionId, + first.snapshot.bindingHash, + candidate.bindingHash, + first.actorId, + ), + ).rejects.toBeInstanceOf(McpRebindDraining); + pending = first.lifecycle.runOnce(first.sessionId); + await entered; + const call = await other.driver.acceptToolRequest(other.request); + const result = await other.driver.driveOnce(call.id); + expect(other.leases).toBe(1); + expect((await other.command("stats")).calls).toBe(1); + expect(result.state).toBe("succeeded"); + release(); + await pending; + await expect( + first.service.commit( + first.sessionId, + first.snapshot.bindingHash, + candidate.bindingHash, + first.actorId, + ), + ).rejects.toBeInstanceOf(McpRebindDraining); + } finally { + release(); + await pending; + await other.cleanup(); + await first.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-rebind.test.ts b/orchestrator/src/__tests__/mcp-rebind.test.ts new file mode 100644 index 000000000..0d7b5d52f --- /dev/null +++ b/orchestrator/src/__tests__/mcp-rebind.test.ts @@ -0,0 +1,249 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { profile, task, taskSession, user } from "../db/schema.ts"; +import { bindingSnapshotColumns, readLiveBindingAuthority } from "../mcp/binding-store.ts"; +import { compileSessionBindings } from "../mcp/session-bindings.ts"; +import { McpRebindDraining } from "../mcp/rebind.ts"; +import { coldFixture } from "./mcp-cold-fixture.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +await verifyMcpTestDatabases(); +import { reviewFixture } from "./mcp-review-fixture.ts"; + +async function finish(f: Awaited>, hash: string) { + await expect( + f.service.commit(f.sessionId, f.snapshot.bindingHash, hash, f.actorId), + ).rejects.toBeInstanceOf(McpRebindDraining); + await f.restartRunOnce(); + f.start(); + await f.restartRunOnce(); + return f.service.commit(f.sessionId, f.snapshot.bindingHash, hash, f.actorId); +} + +test("concurrent cold commits reserve one recovery and expose only its finalized binding", async () => { + const f = await coldFixture(); + try { + const candidate = await f.service.prepare(f.sessionId, f.snapshot.bindingHash, f.actorId); + const results = await Promise.allSettled( + Array.from({ length: 2 }, () => + f.service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId), + ), + ); + expect(results.every((r) => r.status === "rejected")).toBe(true); + expect(f.stats().recoveryIds).toBe(0); + await f.restartRunOnce(); + expect(f.stats().recoveryIds).toBe(1); + f.start(); + await f.restartRunOnce(); + const result = await f.service.commit( + f.sessionId, + f.snapshot.bindingHash, + candidate.bindingHash, + f.actorId, + ); + expect(result.bindingHash).toBe( + (await f.store.read(f.sessionId)).session.mcpBindingEnvelope!.bindingHash, + ); + } finally { + await f.cleanup(); + } +}); + +test("child cold rebind uses the current parent intersection and rejects a changed parent", async () => { + const base = await reviewFixture([], 2); + const parent = await coldFixture(base); + const childTask = crypto.randomUUID(), + childId = crypto.randomUUID(); + try { + const live = await readLiveBindingAuthority( + base.profileId, + base.authority.operatorPolicies, + base.db, + ); + const first = live.profileGrants[0]!; + const grant = { + ...first, + mcp: { + ...first.mcp!, + unrestricted: false, + automationWrite: false, + argumentConstraintDocuments: [{ type: "object", properties: { x: { maximum: 5 } } }], + }, + }; + const connections = + (await parent.store.read(base.sessionId)).session.integrationConnections ?? []; + const narrow = compileSessionBindings({ + launchPolicy: { profileId: base.profileId, integrationGrants: live.profileGrants }, + liveAuthority: live, + bindingEpoch: 1, + requestedGrants: [grant], + }); + await base.db + .update(taskSession) + .set(bindingSnapshotColumns(narrow, connections)) + .where(eq(taskSession.sessionId, base.sessionId)); + const child = compileSessionBindings({ + launchPolicy: { profileId: base.profileId, integrationGrants: live.profileGrants }, + liveAuthority: live, + bindingEpoch: 1, + parentAuthority: { sessionId: base.sessionId, snapshot: narrow }, + }); + await base.db.insert(task).values({ + id: childTask, + type: "chat", + createdByUserId: base.actorId, + spawningSessionId: base.sessionId, + }); + await base.db.insert(taskSession).values({ + taskId: childTask, + sessionId: childId, + profileId: base.profileId, + integrationPrincipalId: base.actorId, + ...bindingSnapshotColumns(child, connections), + }); + const f = await coldFixture( + { ...base, sessionId: childId, taskId: childTask }, + undefined, + child, + ); + let current = true; + f.parentReader(async (id) => { + expect(id).toBe(base.sessionId); + if (!current) throw Error("parent stale"); + return narrow; + }); + const candidate = await f.service.prepare(childId, child.bindingHash, f.actorId); + const effective = candidate.immutableBindings.flatMap((b) => b.tools); + expect(effective).toHaveLength(1); + expect(effective[0]!.grant).toMatchObject({ + unrestricted: false, + automationWrite: false, + argumentConstraintDocuments: grant.mcp.argumentConstraintDocuments, + }); + expect(candidate.ancestorAuthorityRefs[0]!.bindingHash).toBe(narrow.bindingHash); + current = false; + await expect( + f.service.commit(childId, child.bindingHash, candidate.bindingHash, f.actorId), + ).rejects.toThrow("parent stale"); + expect(f.stats().recoverCalls).toBe(0); + current = true; + await finish(f, candidate.bindingHash); + } finally { + await base.db.delete(taskSession).where(eq(taskSession.sessionId, childId)); + await base.db.delete(task).where(eq(task.id, childTask)); + await parent.cleanup(); + } +}); + +test("new administrator-published leaf grants require a fresh reviewed cold candidate", async () => { + const base = await reviewFixture([], 2); + const f = await coldFixture(base); + try { + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + const full = (await f.profiles.get(f.profileId))!; + await f.profiles.update( + f.profileId, + { ...full, integrationGrants: [full.integrationGrants[0]!] }, + { actorId: f.actorId, expectedRevision: 1 }, + ); + const candidate = await f.service.prepare(f.sessionId, f.snapshot.bindingHash, f.actorId); + expect(candidate.immutableBindings.flatMap((b) => b.tools)).toHaveLength(1); + const narrow = (await f.profiles.get(f.profileId))!; + await f.profiles.update( + f.profileId, + { ...narrow, integrationGrants: full.integrationGrants }, + { actorId: f.actorId, expectedRevision: 2 }, + ); + await expect( + f.service.commit(f.sessionId, f.snapshot.bindingHash, candidate.bindingHash, f.actorId), + ).rejects.toThrow("stale"); + expect(f.stats().recoverCalls).toBe(0); + const reviewed = await f.service.prepare(f.sessionId, f.snapshot.bindingHash, f.actorId); + expect(reviewed.profilePublication?.revision).toBe(3); + expect(reviewed.immutableBindings.flatMap((b) => b.tools)).toHaveLength(2); + await finish(f, reviewed.bindingHash); + } finally { + await f.cleanup(); + } +}); + +test("typed rebind returns drain details and requires the exact reviewed candidate", async () => { + const { createClient, createRouterTransport, ConnectError } = await import("@connectrpc/connect"); + const { IntegrationService, McpRebindDrainSchema, McpRebindDrainReason } = + await import("../gen/engram/app/v1/integration_pb.ts"); + const { mcpHandlers } = await import("../mcp/contracts.ts"); + const f = await coldFixture(); + try { + const client = createClient( + IntegrationService, + createRouterTransport((router) => + router.service( + IntegrationService, + mcpHandlers({ + services: { + enabled: true, + journal: f.journal, + repository: f.repository, + runtime: { + cancel: async () => {}, + prepareRebind: f.service.prepare, + rebind: f.service.commit, + }, + }, + connections: f.connections, + getSession: async () => ({ + user: { id: f.actorId, role: "user", email: "user@example.test" }, + }), + now: () => new Date(), + connectionToProto: (row) => ({ id: row.id }), + }), + ), + ), + ); + const candidate = await client.prepareMcpRebind({ + sessionId: f.sessionId, + expectedBindingHash: f.snapshot.bindingHash, + }); + expect(candidate.bindingTools).toHaveLength(1); + await expect( + client.rebindMcp({ + sessionId: f.sessionId, + expectedBindingHash: f.snapshot.bindingHash, + expectedCandidateHash: "wrong", + }), + ).rejects.toThrow(); + try { + await client.rebindMcp({ + sessionId: f.sessionId, + expectedBindingHash: f.snapshot.bindingHash, + expectedCandidateHash: candidate.bindingHash, + }); + throw Error("expected drain"); + } catch (error) { + expect(error).toBeInstanceOf(ConnectError); + if (!(error instanceof ConnectError)) throw error; + expect(error.findDetails(McpRebindDrainSchema)[0]?.reason).toBe( + McpRebindDrainReason.DISPATCH_IN_PROGRESS, + ); + } + await f.restartRunOnce(); + f.start(); + await f.restartRunOnce(); + const result = await client.rebindMcp({ + sessionId: f.sessionId, + expectedBindingHash: f.snapshot.bindingHash, + expectedCandidateHash: candidate.bindingHash, + }); + expect(result.bindingHash).toBe( + (await f.store.read(f.sessionId)).session.mcpManagedLifecycle!.bindingHash, + ); + await f.db + .update(profile) + .set({ mcpGrantPublication: null }) + .where(eq(profile.id, f.profileId)); + await expect( + client.prepareMcpRebind({ sessionId: f.sessionId, expectedBindingHash: result.bindingHash }), + ).rejects.toThrow("administrator must publish"); + } finally { + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-recovery.test.ts b/orchestrator/src/__tests__/mcp-recovery.test.ts new file mode 100644 index 000000000..eb606a084 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-recovery.test.ts @@ -0,0 +1,709 @@ +import { expect, test } from "bun:test"; +import { Code, ConnectError } from "@connectrpc/connect"; +import { eq } from "drizzle-orm"; +import { effectFixture } from "./mcp-effect-fixture.ts"; +import { mcpExecution, mcpAdmissionScope, profile } from "../db/schema.ts"; +import { makeMcpRecovery } from "../mcp/recovery.ts"; + +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +await verifyMcpTestDatabases(); + +test("committed result is redelivered after enqueue failure without another effect", async () => { + const f = await effectFixture(); + try { + let fail = true; + const delivered: string[] = []; + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + completer: { + async completeToolCall(input) { + if (fail) throw Error("transport failure"); + delivered.push(input.resultJson); + }, + }, + }); + const marker = await runtime.accept(f.request); + await expect(runtime.runOnce(marker)).rejects.toThrow("MCP completion unavailable"); + const committed = (await f.journal.get(marker.executionId))!; + expect(committed.state).toBe("succeeded"); + expect(committed.completionEnqueuedVersion).toBe(0); + fail = false; + expect(await runtime.runOnce(marker)).toBe("done"); + expect(JSON.parse(delivered[0]!)).toEqual({ ...committed.result, resultUnavailable: false }); + expect((await f.journal.get(marker.executionId))!.completionEnqueuedVersion).toBe( + committed.version, + ); + expect(await runtime.runOnce(marker)).toBe("done"); + expect(delivered).toHaveLength(1); + expect((await f.command("stats")).calls).toBe(1); + expect((await runtime.pending()).map((x) => x.executionId)).not.toContain(marker.executionId); + await expect( + f.db + .update(mcpExecution) + .set({ completionEnqueuedVersion: 0 }) + .where(eq(mcpExecution.id, marker.executionId)) + .execute(), + ).rejects.toThrow(); + await expect( + f.db + .update(mcpExecution) + .set({ completionEnqueuedVersion: committed.version + 1 }) + .where(eq(mcpExecution.id, marker.executionId)) + .execute(), + ).rejects.toThrow(); + } finally { + await f.cleanup(); + } +}, 20000); + +test("recovery expires unadmitted rows and delivers a local cancellation", async () => { + const f = await effectFixture({ classification: "write" }); + try { + let now = new Date(); + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => now, + completer: { + async completeToolCall(input) { + expect(JSON.parse(input.resultJson)).toMatchObject({ + source: "runtime", + outcome: "canceled", + delivery: "unavailable", + }); + }, + }, + }); + const marker = await runtime.accept(f.request); + const call = (await f.journal.get(marker.executionId))!; + now = new Date(call.deadline.getTime() + 1); + const { makeMcpRecoveryScanner } = await import("../mcp/recovery-scanner.ts"); + await makeMcpRecoveryScanner({ + deploymentId: f.deploymentId, + journal: f.journal, + runtime: () => undefined, + now: () => now, + }).runOnce(); + expect((await f.journal.get(call.id))!.state).toBe("canceled"); + expect((await runtime.pending()).map((x) => x.executionId)).toContain(call.id); + expect(await runtime.runOnce(marker)).toBe("done"); + expect((await runtime.pending()).map((x) => x.executionId)).not.toContain(call.id); + } finally { + await f.cleanup(); + } +}, 20000); + +for (const response of [ + "error", + "image", + "normalized-limit", + "malformed", + "oversize", + "local_validation", +]) + test(`completion retains the committed ${response} distinction`, async () => { + const f = await effectFixture({ + response: response === "local_validation" ? "normal" : response, + }); + try { + if (response === "local_validation") f.request.originalArguments = '{"x":"wrong"}'; + let result: unknown; + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + completer: { + async completeToolCall(request) { + result = JSON.parse(request.resultJson); + }, + }, + }); + const marker = await runtime.accept(f.request); + expect(await runtime.runOnce(marker)).toBe("done"); + const call = (await f.journal.get(marker.executionId))!; + expect(result).toEqual({ ...call.result, resultUnavailable: call.resultUnavailable }); + if (response === "local_validation") { + expect(result).toMatchObject({ + source: "local_validation", + delivery: "available", + output: { isError: true }, + }); + expect(call.intentId).toBeNull(); + expect((await f.command("stats")).calls).toBe(0); + } else { + expect((await f.command("stats")).calls).toBe(1); + if (response === "error") + expect(result).toMatchObject({ outcome: "failed", output: { isError: true } }); + if (response === "image" || response === "normalized-limit") + expect(result).toMatchObject({ + outcome: "succeeded", + delivery: "unavailable", + resultUnavailable: true, + }); + } + await runtime.runOnce(marker); + expect(JSON.stringify(result)).not.toContain("known-literal-token"); + } finally { + await f.cleanup(); + } + }, 20000); + +test("bounded keyset scans reach calls behind a full page and stop after shutdown", async () => { + const f = await effectFixture(); + try { + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + completer: { + async completeToolCall() { + throw Error("not run"); + }, + }, + }); + const expected = new Set(); + for (let i = 0; i < 130; i++) { + const request = { ...f.request, toolCallId: `queued-${i}` }; + f.requests.set(request.toolCallId, request); + expected.add((await runtime.accept(request)).executionId); + } + const { makeMcpRecoveryScanner } = await import("../mcp/recovery-scanner.ts"); + const scheduled = new Set(); + let calls = 0; + const scanner = makeMcpRecoveryScanner({ + deploymentId: f.deploymentId, + journal: f.journal, + runtime: () => runtime, + startWorkflow: async (ref) => { + calls++; + expect(Object.keys(ref).sort()).toEqual([ + "executionId", + "runId", + "sessionId", + "toolCallId", + ]); + scheduled.add(ref.executionId); + }, + }); + await scanner.runOnce(); + expect(calls).toBe(128); + for (let i = 0; i < 4 && [...expected].some((id) => !scheduled.has(id)); i++) + await scanner.runOnce(); + expect([...expected].every((id) => scheduled.has(id))).toBe(true); + await scanner.start(); + await scanner.stop(); + const stopped = calls; + await Bun.sleep(1100); + expect(calls).toBe(stopped); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("already-completed guest projection cannot replace or remove journal truth", async () => { + const f = await effectFixture(); + try { + const call = await f.driver.acceptToolRequest(f.request); + const committed = await f.driver.driveOnce(call.id); + const { makePendingToolCallStore } = await import("../tools/pending-tool-calls.ts"); + const pending = makePendingToolCallStore(f.db); + await pending.recordRequested({ + sessionId: f.sessionId, + toolCallId: f.request.toolCallId, + toolName: f.request.modelToolName, + handling: "handled", + requestedAt: new Date(), + }); + await pending.markCompleted(f.sessionId, f.request.toolCallId, new Date()); + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + // The coordinator acknowledges and drops results for an already-completed call. + completer: { + async completeToolCall() { + return {}; + }, + }, + }); + await runtime.runOnce({ + executionId: call.id, + sessionId: f.sessionId, + runId: f.request.runId, + toolCallId: f.request.toolCallId, + }); + const retained = (await f.journal.get(call.id))!; + expect(retained.result).toEqual(committed.result); + expect(retained.state).toBe("succeeded"); + expect(retained.completionEnqueuedVersion).toBe(retained.version); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await f.cleanup(); + } +}, 20000); + +test("test topology rejects a missing or primary-looking DBOS URL before connection", async () => { + const saved = process.env.MCP_TEST_DBOS_DATABASE_URL; + try { + delete process.env.MCP_TEST_DBOS_DATABASE_URL; + await expect(verifyMcpTestDatabases()).rejects.toThrow( + "explicit isolated MCP test databases required", + ); + process.env.MCP_TEST_DBOS_DATABASE_URL = "postgresql://127.0.0.1:5435/engram_orchestrator"; + await expect(verifyMcpTestDatabases()).rejects.toThrow("MCP test database isolation mismatch"); + } finally { + process.env.MCP_TEST_DBOS_DATABASE_URL = saved; + } +}); + +test("permanent input and policy denials have explicit typed codes", async () => { + const f = await effectFixture({ classification: "write", eligibility: "plan" }); + try { + await expect(f.driver.acceptToolRequest(f.request)).rejects.toMatchObject({ + code: "write_not_allowed", + }); + await expect( + f.driver.acceptToolRequest({ ...f.request, originalArguments: "{broken" }), + ).rejects.toMatchObject({ code: "invalid_arguments" }); + } finally { + await f.cleanup(); + } +}, 20000); + +test("request decision uses the exact int64 event and preserves first arguments and identity", async () => { + const f = await effectFixture(); + try { + const index = 9007199254740993n; + let fail = true; + let reads = 0; + const original = ' {"x": 1} '; + f.request.originalArguments = original; + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + async readRequestEvent(sessionId, idx) { + expect(sessionId).toBe(f.sessionId); + expect(idx).toBe(index); + reads++; + if (fail) throw Error("private-control-error"); + return { + idx, + kind: "tool_call_requested", + payloadJson: JSON.stringify({ + tool_call_id: f.request.toolCallId, + run_id: f.request.runId, + name: f.request.modelToolName, + args_json: original, + }), + }; + }, + }); + const ref = { + sessionId: f.sessionId, + runId: f.request.runId, + toolCallId: f.request.toolCallId, + requestEventIdx: String(index), + deadlineMs: Date.now() + 930000, + }; + await expect(runtime.decideRequest(ref)).rejects.toThrow("MCP request acceptance unavailable"); + fail = false; + const first = await runtime.decideRequest(ref); + expect(first.kind).toBe("accepted"); + if (first.kind !== "accepted") throw Error("expected accepted references"); + const call = (await f.journal.get(first.execution.executionId))!; + expect(call.originalArguments).toBe(original); + const duplicate = await runtime.decideRequest({ + ...ref, + runId: "changed-run", + requestEventIdx: "12", + deadlineMs: 0, + }); + expect(duplicate).toEqual(first); + expect(reads).toBe(2); + expect((await f.journal.get(call.id))!.deadline).toEqual(call.deadline); + expect((await f.command("stats")).calls).toBe(0); + expect(JSON.stringify(first)).not.toContain("originalArguments"); + } finally { + await f.cleanup(); + } +}); + +test("request decision rejects mismatched event references with a fixed private error", async () => { + const f = await effectFixture(); + try { + let kind = "tool_call_requested", + idx = 1n, + run = f.request.runId, + call = f.request.toolCallId; + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + async readRequestEvent() { + return { + idx, + kind, + payloadJson: JSON.stringify({ + run_id: run, + tool_call_id: call, + name: f.request.modelToolName, + args_json: f.request.originalArguments, + }), + }; + }, + }); + const ref = { + sessionId: f.sessionId, + runId: f.request.runId, + toolCallId: f.request.toolCallId, + requestEventIdx: "1", + deadlineMs: Date.now() + 930000, + }; + for (const bad of ["0", "-1", "1.1", "9223372036854775808"]) + await expect(runtime.decideRequest({ ...ref, requestEventIdx: bad })).rejects.toThrow( + "MCP request acceptance unavailable", + ); + idx = 2n; + await expect(runtime.decideRequest(ref)).rejects.toThrow("MCP request acceptance unavailable"); + idx = 1n; + kind = "tool_call_completed"; + await expect(runtime.decideRequest(ref)).rejects.toThrow("MCP request acceptance unavailable"); + kind = "tool_call_requested"; + run = "changed-run"; + await expect(runtime.decideRequest(ref)).rejects.toThrow("MCP request acceptance unavailable"); + run = f.request.runId; + call = "changed-call"; + await expect(runtime.decideRequest(ref)).rejects.toThrow("MCP request acceptance unavailable"); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}); + +test("revoked profile grants produce a local denial before execution", async () => { + const f = await effectFixture(); + try { + let delivered: unknown; + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + completer: { + async completeToolCall(input) { + delivered = JSON.parse(input.resultJson); + }, + }, + async readRequestEvent(_sessionId, idx) { + return { + idx, + kind: "tool_call_requested", + payloadJson: JSON.stringify({ + run_id: f.request.runId, + tool_call_id: idx === 1n ? f.request.toolCallId : "revoked-call", + name: f.request.modelToolName, + args_json: f.request.originalArguments, + }), + }; + }, + }); + const ref = { + sessionId: f.sessionId, + runId: f.request.runId, + toolCallId: f.request.toolCallId, + requestEventIdx: "1", + deadlineMs: Date.now() + 930000, + }; + const winner = await runtime.decideRequest(ref); + expect(winner.kind).toBe("accepted"); + await f.db.update(profile).set({ integrationGrants: [] }).where(eq(profile.id, f.profileId)); + expect(await runtime.decideRequest(ref)).toEqual(winner); + const revoked = { ...ref, toolCallId: "revoked-call", requestEventIdx: "2" }; + const decision = await runtime.decideRequest(revoked); + expect(decision).toEqual({ kind: "denied", code: "authority_changed" }); + if (decision.kind !== "denied") throw Error("expected denial"); + await runtime.deliverDenial(revoked, decision.code); + expect(delivered).toEqual({ + source: "runtime", + denial: "authority_changed", + delivery: "available", + output: { + isError: true, + content: [{ type: "text", text: "External tool request denied." }], + }, + }); + const calls = await f.db + .select() + .from(mcpExecution) + .where(eq(mcpExecution.sessionId, f.sessionId)); + expect(calls.map((call) => call.toolCallId)).toEqual([f.request.toolCallId]); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); + +test("transient binding authority failure is retried while a verified absent tool is denied", async () => { + const f = await effectFixture(); + try { + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + async readRequestEvent(_sessionId, idx) { + return { + idx, + kind: "tool_call_requested", + payloadJson: JSON.stringify({ + run_id: f.request.runId, + tool_call_id: f.request.toolCallId, + name: f.request.modelToolName, + args_json: f.request.originalArguments, + }), + }; + }, + }); + const ref = { + sessionId: f.sessionId, + runId: f.request.runId, + toolCallId: f.request.toolCallId, + requestEventIdx: "1", + deadlineMs: Date.now() + 930000, + }; + f.failAuthorityReads(1); + await expect(runtime.decideRequest(ref)).rejects.toThrow("MCP request acceptance unavailable"); + expect((await runtime.decideRequest(ref)).kind).toBe("accepted"); + f.request.toolCallId = "absent-tool"; + f.request.modelToolName = "mcp_unpublished_tool"; + expect( + await runtime.decideRequest({ + ...ref, + toolCallId: f.request.toolCallId, + requestEventIdx: "2", + }), + ).toEqual({ kind: "denied", code: "tool_unavailable" }); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}); + +for (const disposition of ["missing", "terminal", "host_lost", "conflict", "transport"] as const) + test(`completion ${disposition} settles only authoritative target loss`, async () => { + const f = await effectFixture(); + try { + let fail = true; + const error = new ConnectError( + "private-delivery-error", + disposition === "missing" + ? Code.NotFound + : disposition === "transport" + ? Code.Unavailable + : Code.FailedPrecondition, + new Headers({ + "engram-error-slug": disposition === "terminal" ? "snapshot_invalidated" : disposition, + }), + ); + const runtime = makeMcpRecovery({ + db: f.db, + driver: f.driver, + deploymentId: f.deploymentId, + now: () => new Date(), + completer: { + async completeToolCall() { + if (fail) throw error; + }, + }, + }); + const marker = await runtime.accept(f.request); + const ref = { + sessionId: f.sessionId, + runId: f.request.runId, + toolCallId: f.request.toolCallId, + requestEventIdx: "1", + deadlineMs: Date.now() + 930000, + }; + if (disposition === "missing" || disposition === "terminal") { + expect(await runtime.runOnce(marker)).toBe("done"); + await runtime.deliverDenial(ref, "write_not_allowed"); + } else { + await expect(runtime.runOnce(marker)).rejects.toThrow("MCP completion unavailable"); + await expect(runtime.deliverDenial(ref, "write_not_allowed")).rejects.toThrow( + "MCP denial completion unavailable", + ); + expect((await f.journal.get(marker.executionId))!.completionEnqueuedVersion).toBe(0); + expect((await runtime.pending()).map((x) => x.executionId)).toContain(marker.executionId); + fail = false; + expect(await runtime.runOnce(marker)).toBe("done"); + } + const settled = (await f.journal.get(marker.executionId))!; + expect(settled.state).toBe("succeeded"); + expect(settled.completionEnqueuedVersion).toBe(settled.version); + expect((await runtime.pending()).map((x) => x.executionId)).not.toContain(marker.executionId); + expect((await f.command("stats")).calls).toBe(1); + } finally { + await f.cleanup(); + } + }, 20000); + +test("disabled dispatch construction drains persisted calls without any dispatch dependency", async () => { + const { schemaVerifiedFixture } = await import("./mcp-review-fixture.ts"); + const { makeMcpRecoveryScanner } = await import("../mcp/recovery-scanner.ts"); + const f = await schemaVerifiedFixture(); + const recoveryDeploymentId = crypto.randomUUID(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + const delivered: string[] = []; + const runtime = makeMcpRecovery({ + db: f.db, + deploymentId: recoveryDeploymentId, + now: () => f.now, + completer: { + async completeToolCall(input) { + delivered.push(input.resultJson); + }, + }, + listLifecycleEvents: async () => [], + }); + const scanner = makeMcpRecoveryScanner({ + journal: f.journal, + deploymentId: recoveryDeploymentId, + runtime: () => runtime, + now: () => f.now, + startWorkflow: async (ref) => { + if (ref.sessionId === f.sessionId) await runtime.runOnce(ref); + }, + }); + await scanner.runLifecycleOnce(); + await scanner.runOnce(); + const retained = (await f.journal.get(call.id))!; + expect(retained.state).toBe("canceled"); + expect(retained.intentId).toBeNull(); + expect(retained.completionEnqueuedVersion).toBe(retained.version); + expect(JSON.parse(delivered[0]!)).toMatchObject({ + source: "runtime", + outcome: "canceled", + delivery: "unavailable", + }); + await expect( + runtime.accept({ ...f.input, modelToolName: f.input.resolvedTool.modelToolName }), + ).rejects.toMatchObject({ code: "disabled" }); + expect( + await runtime.decideRequest({ + sessionId: f.sessionId, + runId: f.input.runId, + toolCallId: "new-call", + requestEventIdx: "1", + deadlineMs: f.now.getTime() + 1000, + }), + ).toEqual({ kind: "denied", code: "disabled" }); + await expect(Promise.resolve().then(() => runtime.abortExecution(call.id))).rejects.toThrow( + "MCP cancellation unavailable", + ); + expect(delivered).toHaveLength(1); + } finally { + await f.db.delete(mcpAdmissionScope).where(eq(mcpAdmissionScope.id, recoveryDeploymentId)); + await f.cleanup(); + } +}); + +test("disabled recovery scanner settles committed results across deployment identities without another effect", async () => { + const f = await effectFixture(); + const recoveryDeploymentId = crypto.randomUUID(); + try { + const call = await f.driver.acceptToolRequest(f.request); + const committed = await f.driver.driveOnce(call.id); + const leases = f.leases, + events = [...f.events]; + f.setEnabled(false); + let attempts = 0; + const deliveries: string[] = []; + const { makeMcpRecoveryScanner } = await import("../mcp/recovery-scanner.ts"); + const runtime = makeMcpRecovery({ + db: f.db, + deploymentId: recoveryDeploymentId, + now: () => new Date(), + completer: { + async completeToolCall(input) { + attempts++; + deliveries.push(input.resultJson); + if (attempts === 1) throw Error("private-transport-error"); + }, + }, + }); + const scanner = makeMcpRecoveryScanner({ + journal: f.journal, + deploymentId: recoveryDeploymentId, + runtime: () => runtime, + startWorkflow: async (ref) => { + if (ref.sessionId === f.sessionId) await runtime.runOnce(ref); + }, + }); + await expect(scanner.runOnce()).rejects.toThrow("MCP completion unavailable"); + expect((await f.journal.get(call.id))!.completionEnqueuedVersion).toBe(0); + await scanner.runOnce(); + expect((await f.journal.get(call.id))!.completionEnqueuedVersion).toBe(committed.version); + expect((await f.journal.get(call.id))!.result).toEqual(committed.result); + expect(f.leases).toBe(leases); + expect(f.events).toEqual(events); + expect((await f.command("stats")).calls).toBe(1); + expect(attempts).toBe(2); + expect(JSON.parse(deliveries[1]!)).toEqual({ + ...committed.result, + resultUnavailable: committed.resultUnavailable, + }); + } finally { + await f.db.delete(mcpAdmissionScope).where(eq(mcpAdmissionScope.id, recoveryDeploymentId)); + await f.cleanup(); + } +}, 20000); + +test("live dispatch disable drains an accepted call without driving or leasing it", async () => { + const f = await effectFixture(); + try { + let enabled = true, + drives = 0, + aborts = 0; + const runtime = makeMcpRecovery({ + db: f.db, + deploymentId: f.deploymentId, + now: () => new Date(), + dispatchEnabled: () => enabled, + driver: { + ...f.driver, + async driveOnce(...args) { + drives++; + return f.driver.driveOnce(...args); + }, + cancel(id) { + aborts++; + f.driver.cancel(id); + }, + }, + completer: { async completeToolCall() {} }, + }); + const ref = await runtime.accept(f.request), + leases = f.leases; + enabled = false; + expect(await runtime.runOnce(ref)).toBe("done"); + expect((await f.journal.get(ref.executionId))!.state).toBe("canceled"); + expect(drives).toBe(0); + expect(aborts).toBe(1); + expect(f.leases).toBe(leases); + expect((await f.command("stats")).calls).toBe(0); + } finally { + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-reserved-session-id.test.ts b/orchestrator/src/__tests__/mcp-reserved-session-id.test.ts new file mode 100644 index 000000000..bd3cd513d --- /dev/null +++ b/orchestrator/src/__tests__/mcp-reserved-session-id.test.ts @@ -0,0 +1,20 @@ +import { expect, test } from "bun:test"; +import { + createReservedTaskSession, + McpSessionCreationMismatchError, +} from "../mcp/session-create.ts"; +test("reserved creation retains the actual response ID for child compensation", async () => { + let error: unknown; + try { + await createReservedTaskSession( + { sessions: { createSession: async () => ({ sessionId: "actual-session" }) } }, + { imageUri: "image", mode: "agent", requestedSessionId: "reserved-session" }, + { actorId: "actor", inputId: "input" }, + ); + } catch (caught) { + error = caught; + } + expect(error).toBeInstanceOf(McpSessionCreationMismatchError); + if (!(error instanceof McpSessionCreationMismatchError)) throw Error("missing identity"); + expect(error.createdSessionId).toBe("actual-session"); +}); diff --git a/orchestrator/src/__tests__/mcp-review-fixture.ts b/orchestrator/src/__tests__/mcp-review-fixture.ts new file mode 100644 index 000000000..ac3b3f62c --- /dev/null +++ b/orchestrator/src/__tests__/mcp-review-fixture.ts @@ -0,0 +1,288 @@ +import { operatorPublication } from "./mcp-binding-fixture.ts"; +import { create } from "@bufbuild/protobuf"; +import { eq } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { + user, + task, + taskSession, + profile, + mcpCatalog, + mcpAdmissionScope, + integrationConnection, +} from "../db/schema.ts"; +import { makeIntegrationConnectionStore } from "../db/integration-connections.ts"; +import { makeMcpRepository } from "../mcp/repository.ts"; +import { makeMcpJournal } from "../mcp/journal.ts"; +import { compileSessionBindings } from "../mcp/session-bindings.ts"; +import { bindingSnapshotColumns } from "../mcp/binding-store.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { McpWritePolicyRefSchema } from "../gen/engram/app/v1/session_pb.ts"; + +/** Local authority fixture only; no coordinator credential has been issued. */ +export async function seedReviewConnectionAuthority(id: string, db = getDb(), now = new Date()) { + await db + .update(integrationConnection) + .set({ + credentialIdentity: "test-account", + authGeneration: 2, + revoked: false, + enabled: false, + testedAt: null, + updatedAt: now, + }) + .where(eq(integrationConnection.id, id)); +} + +/** The stored binding and policy are test fixtures. No host ACK or VM enforcement is claimed. */ +export async function reviewFixture(documents: Record[] = [], toolCount = 1) { + const profileId = crypto.randomUUID(); + const db = getDb(), + actorId = crypto.randomUUID(), + publisherId = crypto.randomUUID(), + taskId = crypto.randomUUID(), + sessionId = crypto.randomUUID(), + deploymentId: string = crypto.randomUUID(); + const now = new Date("2026-01-01T00:00:00Z"); + const connections = makeIntegrationConnectionStore(db), + repository = makeMcpRepository(db); + await db.insert(user).values({ + id: actorId, + name: "Review owner", + email: `${actorId}@example.test`, + role: "user", + createdAt: now, + updatedAt: now, + }); + await db.insert(user).values({ + id: publisherId, + name: "Catalog publisher", + email: `${publisherId}@example.test`, + role: "admin", + createdAt: now, + updatedAt: now, + }); + const config = { + endpoint: "https://tools.example.test/mcp", + zone: "public", + auth: { kind: "api_key" as const, secretRef: "mcp.test.key" }, + }; + const connection = await connections.create({ + alias: `test-${crypto.randomUUID()}`, + provider: "mcp", + displayName: "Test tools", + config, + }); + await seedReviewConnectionAuthority(connection.id, db, now); + const current = (await connections.get(connection.id))!; + const tool = { + upstreamName: "Write.Record", + description: "Write a record", + inputSchema: { type: "object" }, + }; + const tools = Array.from({ length: toolCount }, (_, i) => ({ + ...tool, + upstreamName: i === 0 ? tool.upstreamName : `Write.Record${i}`, + })); + const draft = await repository.saveDraft(connection.id, current.mcp!, tools, actorId, now); + const catalog = await repository.publish( + connection.id, + draft.id, + 0, + tools.map((tool) => ({ ...tool, classification: "write", enabled: true })), + publisherId, + now, + ); + await repository.setEnabled(connection.id, 1, true, now); + const grant = { + connectionId: connection.id, + operation: "invoke", + resourceConstraints: [], + credentialScope: "org" as const, + mcp: { + upstreamName: tool.upstreamName, + automationWrite: true, + unrestricted: documents.length === 0, + argumentConstraintDocuments: documents, + }, + }; + const grants = tools.map((tool) => ({ + ...grant, + mcp: { ...grant.mcp, upstreamName: tool.upstreamName }, + })); + const authority = { + profileId, + profileGrants: grants, + connections: [ + { + id: connection.id, + catalogId: catalog.id, + alias: connection.alias, + enabled: true, + config, + state: { ...current.mcp!, catalogRevision: 1 }, + tools: [catalog.tools[0]!], + }, + ], + operatorPolicies: [ + { + publication: structuredClone(operatorPublication), + zoneId: "public", + policyRevision: "1", + policyDigest: canonicalDigest({ policy: 1 }), + authorities: ["tools.example.test"], + aliases: [], + }, + ], + }; + // The repository published this catalog as reviewed, with explicit write classification. + const { reviewCatalog } = await import("../mcp/catalog.ts"); + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: authority.profileId, integrationGrants: grants }, + liveAuthority: { + ...authority, + connections: [{ ...authority.connections[0]!, tools: reviewCatalog(catalog.tools) }], + }, + bindingEpoch: 0, + }); + await db.insert(profile).values({ + id: profileId, + name: "Review profile", + imageId: "image", + harness: "claude", + integrationGrants: grants, + }); + await db.insert(task).values({ id: taskId, type: "chat", createdByUserId: actorId }); + await db.insert(taskSession).values({ + taskId, + sessionId, + profileId, + integrationPrincipalId: actorId, + ...bindingSnapshotColumns(snapshot, [ + { + id: connection.id, + alias: connection.alias, + provider: "mcp", + displayName: connection.displayName, + config: connection.config, + }, + ]), + }); + const binding = snapshot.immutableBindings[0]!, + bound = binding.tools[0]!; + const resolvedTool = { + bindingHash: snapshot.bindingHash, + profilePublication: snapshot.profilePublication, + binding, + modelToolName: bound.name, + reviewedTool: bound.tool, + effectiveGrant: bound.grant, + ancestorAuthorityRefs: snapshot.ancestorAuthorityRefs, + execution: "deferred" as const, + }; + const requestPolicyContext = { + sessionId, + runId: "run-1", + toolCallId: "call-1", + acceptedPolicyRef: create(McpWritePolicyRefSchema, { + sessionId, + policyRevision: 1n, + bindingEpoch: 0n, + }), + acceptedWriteEligibility: "write_eligible", + ancestorPolicyRefs: [], + }; + const input = { + taskId, + sessionId, + runId: "run-1", + toolCallId: "call-1", + principalId: actorId, + connectionId: connection.id, + catalogId: catalog.id, + bindingHash: snapshot.bindingHash, + argumentHash: canonicalDigest({ x: 1 }), + originalArguments: '{ "x": 1 }', + resolvedTool, + requestPolicyContext, + upstreamName: tool.upstreamName, + configRevision: 1, + authGeneration: current.mcp!.authGeneration, + classification: "write" as const, + deadline: new Date(now.getTime() + 600_000), + }; + return { + db, + authority: { + ...authority, + connections: [{ ...authority.connections[0]!, tools: reviewCatalog(catalog.tools) }], + }, + profileId, + actorId, + publisherId, + draft, + actor: actorId, + taskId, + sessionId, + deploymentId, + now, + connections, + repository, + connection, + catalog, + snapshot, + input, + journal: makeMcpJournal(db), + async cleanup() { + await db.delete(task).where(eq(task.id, taskId)); + await db.delete(mcpAdmissionScope).where(eq(mcpAdmissionScope.id, deploymentId)); + await db.delete(mcpCatalog).where(eq(mcpCatalog.connectionId, connection.id)); + await connections.delete(connection.id); + await db.delete(profile).where(eq(profile.id, profileId)); + await db.delete(user).where(eq(user.id, actorId)); + await db.delete(user).where(eq(user.id, publisherId)); + }, + }; +} + +/** Stage fixture for tests of review/journal behavior after a successful worker preflight. + * It obtains and releases real durable leases; the boolean worker result is injected. */ +let priorSchemaPreparation: Promise = Promise.resolve(); +export async function schemaVerifiedFixture( + documents: Record[] = [], + toolCount = 1, + deploymentId?: string, +) { + const f = await reviewFixture(documents, toolCount); + if (deploymentId) f.deploymentId = deploymentId; + const base = f.journal; + return { + ...f, + journal: { + ...base, + async create(...args: Parameters) { + const call = await base.create(...args); + const ready = priorSchemaPreparation.then(async () => { + if ( + ["waiting_approval", "authorized"].includes(call.state) && + !(await base.getAdmission(call.id))?.schemaDigest + ) { + await base.admitSchema(call.id, f.deploymentId, args[2]); + const lease = (await base.getAdmission(call.id))!; + await base.finishSchema( + call.id, + lease.reservationId, + lease.expiresAt, + call.requestDigest, + true, + args[2], + ); + } + }); + priorSchemaPreparation = ready.catch(() => {}); + await ready; + return call; + }, + }, + }; +} diff --git a/orchestrator/src/__tests__/mcp-review-rpc.test.ts b/orchestrator/src/__tests__/mcp-review-rpc.test.ts new file mode 100644 index 000000000..3f10e7450 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-review-rpc.test.ts @@ -0,0 +1,882 @@ +import { expect, test } from "bun:test"; +import { createClient, createRouterTransport, Code, ConnectError } from "@connectrpc/connect"; +import { createConnectTransport } from "@connectrpc/connect-node"; +import { create, toBinary, toJson } from "@bufbuild/protobuf"; +import type { AddressInfo } from "node:net"; +import { Hono } from "hono"; +import { eq } from "drizzle-orm"; +import { checkDb } from "../db/client.ts"; +import { task, user, pendingToolCall, mcpApproval } from "../db/schema.ts"; +import { schemaVerifiedFixture as reviewFixture } from "./mcp-review-fixture.ts"; +import { buildServer } from "../server.ts"; +import { + IntegrationService, + McpCompletionSettlement, + GetMcpExecutionResultResponseSchema, + GetSessionMcpStateResponseSchema, +} from "../gen/engram/app/v1/integration_pb.ts"; +import { SessionService } from "../gen/engram/app/v1/session_pb.ts"; +import { mcpHandlers, type McpServices } from "../mcp/contracts.ts"; +import { makePendingToolCallStore } from "../tools/pending-tool-calls.ts"; +import { makeExternalToolCompletionGuard } from "../rpc/tool-completion-guard.ts"; +import { registerPassthrough } from "../rpc/passthrough.ts"; +import { assertNoProjectedAuthority } from "../../scripts/mcp-stack/case-grants.ts"; + +const reachable = process.env.ORCHESTRATOR_DATABASE_URL ? await checkDb() : false; +async function wire(f: Awaited>, producers = true) { + const { taskSession, profile } = await import("../db/schema.ts"); + const { canonicalDigest } = await import("../mcp/hash.ts"); + const publication = { revision: 1, actorId: f.actorId, grantsHash: f.snapshot.profileGrantsHash }; + const { bindingHash: _oldHash, ...body } = f.snapshot; + f.snapshot.profilePublication = publication; + f.snapshot.bindingHash = canonicalDigest({ ...body, profilePublication: publication }); + f.input.bindingHash = f.snapshot.bindingHash; + f.input.resolvedTool.bindingHash = f.snapshot.bindingHash; + f.input.resolvedTool.profilePublication = publication; + const [sessionRow] = await f.db + .select() + .from(taskSession) + .where(eq(taskSession.sessionId, f.sessionId)); + await f.db + .update(taskSession) + .set({ + mcpBindingEnvelope: { + ...sessionRow!.mcpBindingEnvelope!, + profilePublication: publication, + bindingHash: f.snapshot.bindingHash, + }, + }) + .where(eq(taskSession.sessionId, f.sessionId)); + await f.db + .update(profile) + .set({ mcpGrantPublication: publication }) + .where(eq(profile.id, f.profileId)); + const service: McpServices = { enabled: true, repository: f.repository, journal: f.journal }; + // Injected policy and resolver are fixtures, not a live coordinator/host ACK. + if (producers) + service.reviewProducers = { + bindings: { readCurrent: async () => structuredClone(f.snapshot) }, + policy: { + readWritePolicy: async () => ({ + state: { + policy: { + ref: f.input.requestPolicyContext.acceptedPolicyRef, + writeEligibility: "write_eligible", + current: true, + initialized: true, + }, + planReview: { + policyRef: f.input.requestPolicyContext.acceptedPolicyRef, + toolCallId: "plan-a", + runId: "run-1", + eventIdx: 1n, + requestDigest: "plan-digest", + argsJson: '{"plan":"original"}', + }, + }, + }), + }, + }; + let forwarded = 0; + const upstream = createRouterTransport((router) => + router.service(SessionService, { + completeToolCall: async (req) => { + forwarded++; + return { sessionId: req.sessionId }; + }, + }), + ); + const server = buildServer(new Hono(), (router) => { + const getSession = async () => ({ + user: { id: f.actorId, role: "admin", email: `${f.actorId}@example.test` }, + }); + router.service( + IntegrationService, + mcpHandlers({ + services: service, + connections: f.connections, + getSession, + now: () => f.now, + connectionToProto: (row) => ({ id: row.id }), + }), + ); + registerPassthrough( + router, + [{ service: SessionService, methods: ["CompleteToolCall"] }], + upstream, + getSession, + async () => f.actorId, + { + "SessionService.CompleteToolCall": makeExternalToolCompletionGuard({ + pendingCalls: makePendingToolCallStore(f.db), + }), + }, + ); + }); + const url = await new Promise((resolve) => + server.listen(0, "127.0.0.1", () => + resolve(`http://127.0.0.1:${(server.address() as AddressInfo).port}/rpc`), + ), + ); + const transport = createConnectTransport({ baseUrl: url, httpVersion: "1.1" }); + return { + client: createClient(IntegrationService, transport), + session: createClient(SessionService, transport), + service, + forwarded: () => forwarded, + close: () => + new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ), + }; +} +async function code(promise: Promise, expected: Code) { + try { + await promise; + throw new Error("expected denial"); + } catch (error) { + expect(error).toBeInstanceOf(ConnectError); + expect((error as ConnectError).code).toBe(expected); + } +} +test.skipIf(!reachable)( + "registered review handlers return original arguments and bind decisions to all context", + async () => { + const f = await reviewFixture(), + s = await wire(f); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const state = await s.client.getSessionMcpState({ + sessionId: f.sessionId, + planToolCallId: "plan-a", + }); + expect(state.availability).toBe("available"); + expect(state.writePolicy?.planReview?.argsJson).toBe('{"plan":"original"}'); + expect(state.approvals[0]!.originalArguments).toBe(f.input.originalArguments); + expect(state.executions[0]!.allowedActions).toContain("approve"); + expect(JSON.stringify(toJson(GetSessionMcpStateResponseSchema, state))).not.toContain( + "mcp.test.key", + ); + const expected = state.approvals[0]!.expectedReviewContext!; + await code( + s.client.decideMcpApproval({ executionId: call.id, approve: true }), + Code.InvalidArgument, + ); + await code( + s.client.decideMcpApproval({ + executionId: call.id, + approve: true, + expectedReviewContext: { ...expected, argumentHash: "changed" }, + }), + Code.Aborted, + ); + const result = await s.client.decideMcpApproval({ + executionId: call.id, + approve: true, + expectedReviewContext: expected, + }); + expect(result.state).toBe("authorized"); + expect(result.receipt?.reviewedContext).toEqual(expected); + expect(result.receipt?.actorId).toBe(f.actorId); + expect((await f.journal.get(call.id))!.bootId).toBeNull(); + } finally { + await s.close(); + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "fresh authorization denies cached admin on empty reads and changed permission decisions", + async () => { + const f = await reviewFixture(), + s = await wire(f); + try { + await f.db.update(task).set({ createdByUserId: "other-owner" }).where(eq(task.id, f.taskId)); + await code(s.client.getSessionMcpState({ sessionId: f.sessionId }), Code.PermissionDenied); + await code(s.client.listMcpApprovals({ sessionId: f.sessionId }), Code.PermissionDenied); + await f.db.update(task).set({ createdByUserId: f.actorId }).where(eq(task.id, f.taskId)); + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const read = await s.client.listMcpApprovals({ sessionId: f.sessionId }); + await f.db.update(user).set({ banned: true }).where(eq(user.id, f.actorId)); + await code( + s.client.decideMcpApproval({ + executionId: call.id, + approve: true, + expectedReviewContext: read.approvals[0]!.expectedReviewContext, + }), + Code.PermissionDenied, + ); + expect((await f.journal.get(call.id))!.state).toBe("waiting_approval"); + } finally { + await s.close(); + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "generic completion and forged guest projection cannot change MCP authority", + async () => { + const f = await reviewFixture(), + s = await wire(f); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const beforeProjection = (await f.journal.get(call.id))!; + const projection = makePendingToolCallStore(f.db); + await projection.recordRequested({ + sessionId: f.sessionId, + toolCallId: f.input.toolCallId, + toolName: f.input.resolvedTool.modelToolName, + handling: "handled", + requestedAt: f.now, + }); + await code( + s.session.completeToolCall({ + sessionId: f.sessionId, + toolCallId: f.input.toolCallId, + resultJson: '{"decision":"approve","ok":true}', + }), + Code.PermissionDenied, + ); + await projection.markSubmitted(f.sessionId, f.input.toolCallId, f.now); + await projection.markCompleted(f.sessionId, f.input.toolCallId, f.now); + const afterProjection = (await f.journal.get(call.id))!; + expect(afterProjection.state).toBe("waiting_approval"); + expect(afterProjection.version).toBe(beforeProjection.version); + expect(afterProjection.intentId).toBeNull(); + expect(afterProjection.result).toBeNull(); + const [approval] = await f.db + .select() + .from(mcpApproval) + .where(eq(mcpApproval.executionId, call.id)); + expect(approval!.decision).toBe("pending"); + expect(approval!.consumedAt).toBeNull(); + expect(s.forwarded()).toBe(0); + const state = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + const execution = state.executions.find((value) => value.executionId === call.id)!; + expect(execution.state).toBe("waiting_approval"); + expect(execution.approvalDecision).toBe("pending"); + const result = await s.client.getMcpExecutionResult({ + sessionId: f.sessionId, + executionId: call.id, + }); + expect(result.resultJson).toBeUndefined(); + expect(() => assertNoProjectedAuthority(execution, result.resultJson)).not.toThrow(); + } finally { + await f.db.delete(pendingToolCall).where(eq(pendingToolCall.sessionId, f.sessionId)); + await s.close(); + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "state pages are bounded and missing producers never offer approval", + async () => { + const f = await reviewFixture(), + s = await wire(f, false); + try { + for (let i = 0; i < 18; i++) + await f.journal.create( + { + ...f.input, + toolCallId: `page-${i}`, + requestPolicyContext: { ...f.input.requestPolicyContext, toolCallId: `page-${i}` }, + }, + { kind: "approval" }, + f.now, + ); + const first = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + expect(first.executions).toHaveLength(16); + expect(first.availability).toBe("unavailable"); + expect(first.approvals).toHaveLength(0); + expect(first.executions.flatMap((c) => c.allowedActions)).not.toContain("approve"); + const next = await s.client.getSessionMcpState({ + sessionId: f.sessionId, + afterExecutionId: first.nextExecutionId, + }); + expect(next.executions).toHaveLength(2); + expect( + new Set([...first.executions, ...next.executions].map((c) => c.executionId)).size, + ).toBe(18); + expect( + toBinary(GetSessionMcpStateResponseSchema, create(GetSessionMcpStateResponseSchema, first)) + .byteLength, + ).toBeLessThanOrEqual(2 * 1024 * 1024); + } finally { + await s.close(); + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "byte-limited pages preserve whole arguments, independent schema roots and int64 policy identity", + async () => { + const documents = [ + { + $schema: "http://json-schema.org/draft-07/schema#", + $id: "urn:test:constraint", + definitions: { value: { type: "string" } }, + type: "object", + properties: { value: { $ref: "#/definitions/value" } }, + }, + { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: "urn:test:constraint", + $defs: { value: { maxLength: 250000 } }, + type: "object", + properties: { value: { $ref: "#/$defs/value" } }, + }, + ]; + const f = await reviewFixture(documents), + s = await wire(f); + try { + f.input.requestPolicyContext.acceptedPolicyRef.policyRevision = 9007199254740993n; + const { canonicalDigest } = await import("../mcp/hash.ts"); + const args = { value: '"'.repeat(120000) }, + originalArguments = JSON.stringify(args); + for (let i = 0; i < 10; i++) { + const call = await f.journal.create( + { + ...f.input, + originalArguments, + argumentHash: canonicalDigest(args), + toolCallId: `large-${i}`, + requestPolicyContext: { ...f.input.requestPolicyContext, toolCallId: `large-${i}` }, + }, + { kind: "approval" }, + f.now, + ); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + } + const first = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + expect(first.executions.length).toBeLessThan(10); + expect(first.executions.length).toBeGreaterThan(0); + const pages = [first]; + while (pages.at(-1)!.hasMoreExecutions) { + pages.push( + await s.client.getSessionMcpState({ + sessionId: f.sessionId, + afterExecutionId: pages.at(-1)!.nextExecutionId, + omitBindings: true, + }), + ); + if (pages.length > 10) throw new Error("state cursor did not advance"); + } + const approvals = pages.flatMap((page) => page.approvals); + expect(approvals).toHaveLength(10); + for (const item of approvals) { + expect(item.originalArguments).toBe(originalArguments); + expect(item.argumentConstraintDocumentsJson.map((value) => JSON.parse(value))).toEqual( + documents, + ); + expect(item.expectedReviewContext!.policyRevision).toBe("9007199254740993"); + } + for (const page of pages) { + expect(toBinary(GetSessionMcpStateResponseSchema, page).byteLength).toBeLessThanOrEqual( + 2 * 1024 * 1024, + ); + expect( + Buffer.byteLength(JSON.stringify(toJson(GetSessionMcpStateResponseSchema, page))), + ).toBeLessThanOrEqual(2 * 1024 * 1024); + } + } finally { + await s.close(); + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "no-call sessions expose complete immutable binding tools with independent paging", + async () => { + const documents = [ + { $schema: "http://json-schema.org/draft-07/schema#", $id: "urn:shared", type: "object" }, + { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: "urn:shared", + type: "object", + }, + ]; + const f = await reviewFixture(documents, 18), + s = await wire(f, false); + try { + const first = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + expect(first.executions).toHaveLength(0); + expect(first.bindingTools).toHaveLength(16); + expect(first.bindingReceipt?.current).toBe(false); + expect(first.hasMoreBindingTools).toBe(true); + const second = await s.client.getSessionMcpState({ + sessionId: f.sessionId, + afterBindingToolName: first.nextBindingToolName, + omitExecutions: true, + }); + expect(second.bindingTools).toHaveLength(2); + expect(second.hasMoreBindingTools).toBe(false); + const tools = [...first.bindingTools, ...second.bindingTools]; + expect(new Set(tools.map((tool) => tool.modelToolName)).size).toBe(18); + for (const tool of tools) { + expect(tool.argumentConstraintDocumentsJson.map((value) => JSON.parse(value))).toEqual( + documents, + ); + expect(tool.catalogId).toBe(f.catalog.id); + expect(tool.unrestricted).toBe(false); + } + expect(JSON.stringify(toJson(GetSessionMcpStateResponseSchema, first))).not.toContain( + "mcp.test.key", + ); + expect(toBinary(GetSessionMcpStateResponseSchema, first).byteLength).toBeLessThanOrEqual( + 2 * 1024 * 1024, + ); + } finally { + await s.close(); + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)("a fresh policy lineage change invalidates a saved review", async () => { + const f = await reviewFixture(), + s = await wire(f); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const state = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + const expected = state.approvals[0]!.expectedReviewContext!; + s.service.reviewProducers!.policy.readWritePolicy = async () => ({ + state: { + policy: { + ref: f.input.requestPolicyContext.acceptedPolicyRef, + writeEligibility: "write_eligible", + current: true, + initialized: true, + ancestorPolicyRefs: [ + { sessionId: "ancestor-changed", bindingEpoch: 0n, policyRevision: 2n }, + ], + }, + }, + }); + await code( + s.client.decideMcpApproval({ + executionId: call.id, + approve: true, + expectedReviewContext: expected, + }), + Code.Aborted, + ); + const current = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + expect(current.approvals).toHaveLength(0); + expect(current.executions[0]!.allowedActions).not.toContain("approve"); + expect((await f.journal.get(call.id))!.state).toBe("waiting_approval"); + } finally { + await s.close(); + await f.cleanup(); + } +}); + +test.skipIf(!reachable)( + "typed state reports legacy leaf publication as unavailable", + async () => { + const { taskSession } = await import("../db/schema.ts"); + const { canonicalDigest } = await import("../mcp/hash.ts"); + const f = await reviewFixture(); + const s = await wire(f); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const published = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + expect(published.availability).toBe("available"); + expect(published.executions[0]!.allowedActions).toContain("approve"); + const { bindingHash: _old, ...body } = f.snapshot; + const legacy = { ...body, profilePublication: null }; + const hash = canonicalDigest(legacy); + f.snapshot.profilePublication = null; + f.snapshot.bindingHash = hash; + const [stored] = await f.db + .select() + .from(taskSession) + .where(eq(taskSession.sessionId, f.sessionId)); + await f.db + .update(taskSession) + .set({ + mcpBindingEnvelope: { + ...stored!.mcpBindingEnvelope!, + profilePublication: null, + bindingHash: hash, + }, + }) + .where(eq(taskSession.sessionId, f.sessionId)); + const state = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + expect(state.availability).toBe("unavailable"); + expect(state.bindingReceipt?.staleReasons).toContain("publication_required"); + expect(state.executions[0]!.allowedActions).not.toContain("approve"); + expect(state.writePolicy?.allowedActions ?? []).not.toContain("approve_current_plan"); + } finally { + await s.close(); + await f.cleanup(); + } + }, + 20000, +); + +test.skipIf(!reachable)( + "registered result read enforces session association and current human ownership with dispatch disabled", + async () => { + const f = await reviewFixture(), + s = await wire(f); + const { mcpExecution } = await import("../db/schema.ts"); + const other = await reviewFixture(); + s.service.enabled = false; + try { + await f.db.update(task).set({ createdByUserId: f.actorId }).where(eq(task.id, other.taskId)); + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + const result = { + outcome: "failed", + delivery: "available", + output: { + content: [{ type: "text", text: "" }], + structuredContent: { x: 1 }, + isError: true, + }, + }; + await f.db + .update(mcpExecution) + .set({ state: "failed", result, version: 2 }) + .where(eq(mcpExecution.id, call.id)); + await f.db + .update(mcpExecution) + .set({ completionEnqueuedVersion: 2 }) + .where(eq(mcpExecution.id, call.id)); + const response = await s.client.getMcpExecutionResult({ + sessionId: f.sessionId, + executionId: call.id, + }); + expect(JSON.parse(response.resultJson!)).toEqual(result); + expect(response.state).toBe("failed"); + expect(response.settlement).toBe(McpCompletionSettlement.SETTLED_OR_TARGET_GONE); + await code( + s.client.getMcpExecutionResult({ sessionId: f.sessionId, executionId: "missing" }), + Code.FailedPrecondition, + ); + await code( + s.client.getMcpExecutionResult({ sessionId: "other", executionId: call.id }), + Code.PermissionDenied, + ); + await code( + s.client.getMcpExecutionResult({ sessionId: other.sessionId, executionId: call.id }), + Code.FailedPrecondition, + ); + await f.db.update(task).set({ createdByUserId: "other-owner" }).where(eq(task.id, f.taskId)); + await code( + s.client.getMcpExecutionResult({ sessionId: f.sessionId, executionId: call.id }), + Code.PermissionDenied, + ); + } finally { + await s.close(); + await other.cleanup(); + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "result reader bounds data and preserves absent and pending settlement", + async () => { + const f = await reviewFixture(), + s = await wire(f); + const { mcpExecution } = await import("../db/schema.ts"); + const { canonicalJson } = await import("../mcp/hash.ts"); + async function make(name: string, result: Record) { + const call = await f.journal.create( + { + ...f.input, + toolCallId: name, + requestPolicyContext: { ...f.input.requestPolicyContext, toolCallId: name }, + }, + { kind: "approval" }, + f.now, + ); + await f.db + .update(mcpExecution) + .set({ state: "failed", result, version: 2 }) + .where(eq(mcpExecution.id, call.id)); + return call; + } + try { + const active = await f.journal.create(f.input, { kind: "approval" }, f.now); + const absent = await s.client.getMcpExecutionResult({ + sessionId: f.sessionId, + executionId: active.id, + }); + expect(absent.resultJson).toBeUndefined(); + expect(absent.settlement).toBe(McpCompletionSettlement.NOT_APPLICABLE); + const envelope = (text: string) => ({ + outcome: "failed", + delivery: "available", + output: { content: [{ type: "text", text }], isError: true }, + }); + const overhead = Buffer.byteLength(canonicalJson(envelope(""))); + const max = await make("max", envelope("x".repeat(1024 * 1024 - overhead))); + const response = await s.client.getMcpExecutionResult({ + sessionId: f.sessionId, + executionId: max.id, + }); + expect(Buffer.byteLength(response.resultJson!)).toBe(1024 * 1024); + expect(response.settlement).toBe(McpCompletionSettlement.PENDING); + const oversized = await make("oversized", envelope("x".repeat(1024 * 1024 - overhead + 1))); + await code( + s.client.getMcpExecutionResult({ sessionId: f.sessionId, executionId: oversized.id }), + Code.FailedPrecondition, + ); + const escaped = await make( + "escaped", + envelope("\\".repeat(Math.floor((1024 * 1024 - overhead) / 2))), + ); + const escapedResult = await s.client.getMcpExecutionResult({ + sessionId: f.sessionId, + executionId: escaped.id, + }); + expect( + Buffer.byteLength( + JSON.stringify(toJson(GetMcpExecutionResultResponseSchema, escapedResult)), + ), + ).toBeLessThanOrEqual(2 * 1024 * 1024); + const unavailable = await f.journal.create( + { + ...f.input, + toolCallId: "unavailable", + requestPolicyContext: { ...f.input.requestPolicyContext, toolCallId: "unavailable" }, + }, + { kind: "approval" }, + f.now, + ); + await f.db + .update(mcpExecution) + .set({ state: "authorized", version: 2 }) + .where(eq(mcpExecution.id, unavailable.id)); + await f.db + .update(mcpExecution) + .set({ + state: "dispatching", + version: 3, + bootId: "boot", + intentId: "intent-unavailable", + intentExpiresAt: f.now, + }) + .where(eq(mcpExecution.id, unavailable.id)); + await f.db + .update(mcpExecution) + .set({ state: "succeeded", version: 4, resultUnavailable: true }) + .where(eq(mcpExecution.id, unavailable.id)); + const noResult = await s.client.getMcpExecutionResult({ + sessionId: f.sessionId, + executionId: unavailable.id, + }); + expect(noResult.resultUnavailable).toBe(true); + expect(noResult.resultJson).toBeUndefined(); + const malformed = await make("malformed", { + outcome: "failed", + delivery: "available", + output: { content: [{ type: "image" }], isError: true }, + }); + await code( + s.client.getMcpExecutionResult({ sessionId: f.sessionId, executionId: malformed.id }), + Code.FailedPrecondition, + ); + } finally { + await s.close(); + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "result read checks current admin, banned and service identities", + async () => { + const f = await reviewFixture(), + s = await wire(f); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.db.update(task).set({ createdByUserId: "other-owner" }).where(eq(task.id, f.taskId)); + await code( + s.client.getMcpExecutionResult({ sessionId: f.sessionId, executionId: call.id }), + Code.PermissionDenied, + ); + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + expect( + (await s.client.getMcpExecutionResult({ sessionId: f.sessionId, executionId: call.id })) + .state, + ).toBe("waiting_approval"); + await f.db.update(user).set({ banned: true }).where(eq(user.id, f.actorId)); + await code( + s.client.getMcpExecutionResult({ sessionId: f.sessionId, executionId: call.id }), + Code.PermissionDenied, + ); + await f.db + .update(user) + .set({ banned: false, email: `apikey+${f.actorId}@service.local` }) + .where(eq(user.id, f.actorId)); + await code( + s.client.getMcpExecutionResult({ sessionId: f.sessionId, executionId: call.id }), + Code.PermissionDenied, + ); + } finally { + await s.close(); + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "disabled dispatch keeps registered history, pending reviews and exact cancellation", + async () => { + const f = await reviewFixture(), + s = await wire(f); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const enabled = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + const expected = enabled.approvals[0]!.expectedReviewContext!; + let cancels = 0; + const runtime = { + cancel: async () => { + cancels++; + }, + prepareRebind: async () => { + throw Error("must not rebind"); + }, + rebind: async () => { + throw Error("must not rebind"); + }, + }; + s.service.runtime = runtime; + s.service.enabled = false; + const state = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + expect(state.availability).toBe("unavailable"); + expect(state.executions[0]!.review!.originalArguments).toBe(f.input.originalArguments); + expect(state.executions[0]!.allowedActions).toEqual(["cancel"]); + expect(state.bindingReceipt!.allowedActions).toEqual([]); + expect(state.writePolicy?.allowedActions).toEqual(["approve_current_plan"]); + expect((await s.client.listMcpApprovals({ sessionId: f.sessionId })).approvals).toHaveLength( + 1, + ); + await code( + s.client.decideMcpApproval({ + executionId: call.id, + approve: true, + expectedReviewContext: expected, + }), + Code.FailedPrecondition, + ); + await code( + s.client.prepareMcpRebind({ + sessionId: f.sessionId, + expectedBindingHash: f.snapshot.bindingHash, + }), + Code.FailedPrecondition, + ); + await code(s.client.discoverMcp({}), Code.FailedPrecondition); + await code(s.client.publishMcp({}), Code.FailedPrecondition); + await code(s.client.rebindMcp({}), Code.FailedPrecondition); + await code( + s.client.cancelMcpCall({ + executionId: call.id, + expectedJournalRevision: call.version + 100, + }), + Code.Aborted, + ); + const current = (await f.journal.get(call.id))!; + const canceled = await s.client.cancelMcpCall({ + executionId: call.id, + expectedJournalRevision: current.version, + }); + expect(canceled.state).toBe("canceled"); + expect(cancels).toBe(1); + await f.db.update(task).set({ createdByUserId: "other-owner" }).where(eq(task.id, f.taskId)); + await code(s.client.getSessionMcpState({ sessionId: f.sessionId }), Code.PermissionDenied); + } finally { + await s.close(); + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "disabled dispatch preserves exact unknown resolution and journal-only cancellation authority", + async () => { + const f = await reviewFixture(), + s = await wire(f); + try { + const { reviewContext } = await import("../mcp/journal-review.ts"); + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const approved = ( + await f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), f.now) + ).call; + await f.journal.admit(call.id, f.deploymentId, f.now); + const lease = (await f.journal.getAdmission(call.id))!; + const dispatched = await f.journal.dispatch( + call.id, + { bootId: "boot", intentId: crypto.randomUUID(), expiresAt: lease.expiresAt }, + f.now, + { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: approved.version, + reservationId: lease.reservationId, + reservationExpiresAt: lease.expiresAt, + }, + ); + s.service.enabled = false; + expect(s.service.runtime).toBeUndefined(); + const canceled = await s.client.cancelMcpCall({ + executionId: call.id, + expectedJournalRevision: dispatched.version, + }); + expect(canceled.state).toBe("outcome_unknown"); + expect(canceled.receipt!.state).toBe("outcome_unknown"); + const state = await s.client.getSessionMcpState({ sessionId: f.sessionId }); + expect(state.unknownWriteBlocks).toContain(f.connection.id); + const expected = state.executions[0]!.expectedUnknownContext!; + const request = { + executionId: call.id, + outcome: "acknowledged", + expectedUnknownContext: expected, + }; + await code( + s.client.resolveMcpOutcome({ + ...request, + expectedUnknownContext: { ...expected, intentId: "other" }, + }), + Code.Aborted, + ); + await f.db + .update(user) + .set({ email: "apikey+fixture@service.local" }) + .where(eq(user.id, f.actorId)); + await code(s.client.getSessionMcpState({ sessionId: f.sessionId }), Code.PermissionDenied); + await code(s.client.resolveMcpOutcome(request), Code.PermissionDenied); + await f.db + .update(user) + .set({ email: `${f.actorId}@example.test` }) + .where(eq(user.id, f.actorId)); + await f.db.update(task).set({ createdByUserId: "other-owner" }).where(eq(task.id, f.taskId)); + await code(s.client.resolveMcpOutcome(request), Code.PermissionDenied); + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + const resolved = await s.client.resolveMcpOutcome(request); + expect(resolved.receipt!.expectedContext).toEqual(expected); + expect((await f.journal.get(call.id))!.state).toBe("outcome_unknown"); + expect((await f.journal.get(call.id))!.result).toBeNull(); + expect( + (await s.client.getSessionMcpState({ sessionId: f.sessionId })).resolutionReceipts, + ).toHaveLength(1); + } finally { + await s.close(); + await f.cleanup(); + } + }, +); diff --git a/orchestrator/src/__tests__/mcp-review-store.test.ts b/orchestrator/src/__tests__/mcp-review-store.test.ts new file mode 100644 index 000000000..19065b534 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-review-store.test.ts @@ -0,0 +1,531 @@ +import { unknownContext } from "../mcp/journal.ts"; +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { checkDb, getDb } from "../db/client.ts"; +import { task, taskSession, user } from "../db/schema.ts"; +import { makeMcpJournal } from "../mcp/journal.ts"; + +const reachable = process.env.ORCHESTRATOR_DATABASE_URL ? await checkDb() : false; +test.skipIf(!reachable)( + "empty approval reads still require a current human owner or admin", + async () => { + const db = getDb(); + const actorId = crypto.randomUUID(), + taskId = crypto.randomUUID(), + sessionId = crypto.randomUUID(); + try { + await db.insert(user).values({ + id: actorId, + name: "Review reader", + email: `${actorId}@example.test`, + role: "user", + createdAt: new Date(), + updatedAt: new Date(), + }); + await db.insert(task).values({ id: taskId, type: "chat", createdByUserId: "another-owner" }); + await db.insert(taskSession).values({ taskId, sessionId }); + const journal = makeMcpJournal(db); + // A cached administrator role must not grant access, even when no calls exist. + await expect(journal.pending(sessionId, actorId, new Date())).rejects.toThrow( + "owner or administrator", + ); + } finally { + await db.delete(task).where(eq(task.id, taskId)); + await db.delete(user).where(eq(user.id, actorId)); + } + }, +); + +import { schemaVerifiedFixture as reviewFixture } from "./mcp-review-fixture.ts"; +test.skipIf(!reachable)( + "journal freezes original bytes and rejects argument hash mismatch", + async () => { + const f = await reviewFixture(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + expect(call.originalArguments).toBe(f.input.originalArguments); + await expect( + f.journal.create( + { ...f.input, toolCallId: "changed", originalArguments: '{"x":2}' }, + { kind: "approval" }, + f.now, + ), + ).rejects.toThrow("argument"); + } finally { + await f.cleanup(); + } + }, +); + +import { sql } from "drizzle-orm"; +import { mcpApproval, mcpAdmission, mcpExecution, profile } from "../db/schema.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { reviewContext, requestPolicyContext } from "../mcp/journal-review.ts"; + +async function pending(f: Awaited>) { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, f.now); + return { call, approval, expected: reviewContext(call, approval) }; +} +test.skipIf(!reachable)( + "same request races keep the first bytes and immutable complete context", + async () => { + const f = await reviewFixture(); + try { + const calls = await Promise.all( + Array.from({ length: 4 }, () => f.journal.create(f.input, { kind: "approval" }, f.now)), + ); + expect(new Set(calls.map((c) => c.id)).size).toBe(1); + const original = calls[0]!; + expect(requestPolicyContext(original.requestContext!).acceptedPolicyRef.policyRevision).toBe( + 1n, + ); + const changed = { + ...f.input, + originalArguments: '{"x":2}', + argumentHash: canonicalDigest({ x: 2 }), + }; + await expect(f.journal.create(changed, { kind: "approval" }, f.now)).rejects.toThrow( + "digest", + ); + await expect( + f.journal.create({ ...f.input, originalArguments: '{"x":1}' }, { kind: "approval" }, f.now), + ).rejects.toThrow("digest"); + await expect( + Promise.resolve( + f.db + .update(mcpExecution) + .set({ originalArguments: '{"x":2}', state: "canceled", version: 2 }) + .where(eq(mcpExecution.id, original.id)), + ), + ).rejects.toThrow(); + expect((await f.journal.get(original.id))!.originalArguments).toBe(f.input.originalArguments); + } finally { + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "every expected review field is exact and concurrent decisions have one winner", + async () => { + const f = await reviewFixture(); + try { + const { call, expected } = await pending(f); + for (const key of Object.keys(expected) as (keyof typeof expected)[]) { + const value = expected[key]; + await expect( + f.journal.decide( + call.id, + f.actorId, + true, + { ...expected, [key]: typeof value === "number" ? value + 1 : `${value}-changed` }, + f.now, + ), + ).rejects.toThrow("stale"); + } + const results = await Promise.allSettled( + [true, false].map((approve) => + f.journal.decide(call.id, f.actorId, approve, expected, f.now), + ), + ); + expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1); + const [approval] = await f.db + .select() + .from(mcpApproval) + .where(eq(mcpApproval.executionId, call.id)); + expect(approval!.actorId).toBe(f.actorId); + expect(approval!.consumedAt).toBeNull(); + } finally { + await f.cleanup(); + } + }, +); +async function waitForBlocked(db: ReturnType, blocker: number) { + const deadline = Date.now() + 1000; + while (Date.now() < deadline) { + const rows = await db.execute<{ waiting: boolean }>( + sql`SELECT EXISTS (SELECT 1 FROM pg_stat_activity WHERE ${blocker} = ANY(pg_blocking_pids(pid))) AS waiting`, + ); + if (rows.rows[0]?.waiting === true) return; + await Bun.sleep(5); + } + throw new Error("decision did not reach the held authorization row"); +} +test.skipIf(!reachable)( + "decisions serialize with current owner, admin role, ban and service identity changes", + async () => { + const f = await reviewFixture(); + try { + const { call, expected } = await pending(f); + const changes = ["owner", "role", "ban", "service"]; + for (const change of changes) { + await f.db + .update(task) + .set({ createdByUserId: change === "role" ? "other-owner" : f.actorId }) + .where(eq(task.id, f.taskId)); + await f.db + .update(user) + .set({ + role: change === "role" ? "admin" : "user", + banned: false, + email: `${f.actorId}@example.test`, + }) + .where(eq(user.id, f.actorId)); + let unlock!: () => void, locked!: (pid: number) => void; + const release = new Promise((resolve) => { + unlock = resolve; + }); + const ready = new Promise((resolve) => { + locked = resolve; + }); + const mutation = f.db.transaction(async (tx) => { + if (change === "owner") + await tx + .update(task) + .set({ createdByUserId: "other-owner" }) + .where(eq(task.id, f.taskId)); + else + await tx + .update(user) + .set( + change === "role" + ? { role: "user" } + : change === "ban" + ? { banned: true } + : { email: `apikey+${f.actorId}@service.local` }, + ) + .where(eq(user.id, f.actorId)); + const rows = await tx.execute<{ pid: number }>(sql`SELECT pg_backend_pid() AS pid`); + locked(Number(rows.rows[0]!.pid)); + await release; + }); + const blocker = await ready; + const decision = f.journal.decide(call.id, f.actorId, true, expected, f.now).then( + () => "allowed", + () => "denied", + ); + try { + await waitForBlocked(f.db, blocker); + } finally { + unlock(); + } + await mutation; + expect(await decision).toBe("denied"); + expect((await f.journal.get(call.id))!.state).toBe("waiting_approval"); + } + } finally { + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "local profile changes and replaced bindings invalidate review", + async () => { + const f = await reviewFixture(); + try { + const { call, expected } = await pending(f); + await f.db.update(profile).set({ integrationGrants: [] }).where(eq(profile.id, f.profileId)); + await expect(f.journal.decide(call.id, f.actorId, true, expected, f.now)).rejects.toThrow( + "stale", + ); + } finally { + await f.cleanup(); + } + }, +); +test.skipIf(!reachable)( + "failed barrier releases reservation without consuming approval", + async () => { + const f = await reviewFixture(); + try { + const { call, expected } = await pending(f); + const decision = await f.journal.decide(call.id, f.actorId, true, expected, f.now); + await f.journal.admit(call.id, f.deploymentId, f.now); + const authorization = { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: decision.call.version - 1, + reservationId: (await f.journal.getAdmission(call.id))!.reservationId, + reservationExpiresAt: (await f.journal.getAdmission(call.id))!.expiresAt, + }; + await expect( + f.journal.dispatch( + call.id, + { + bootId: "boot-a", + intentId: crypto.randomUUID(), + expiresAt: new Date(f.now.getTime() + 1000), + }, + f.now, + authorization, + ), + ).rejects.toThrow("stale"); + expect((await f.journal.getAdmission(call.id))?.state).toBe("released"); + const [approval] = await f.db + .select() + .from(mcpApproval) + .where(eq(mcpApproval.executionId, call.id)); + expect(approval!.consumedAt).toBeNull(); + expect((await f.journal.get(call.id))!.state).toBe("authorized"); + } finally { + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "unknown write blocks are global to the session across state pages", + async () => { + const f = await reviewFixture(); + try { + const { call, expected } = await pending(f); + const approved = await f.journal.decide(call.id, f.actorId, true, expected, f.now); + await f.journal.admit(call.id, f.deploymentId, f.now); + const dispatched = await f.journal.dispatch( + call.id, + { + bootId: "boot-a", + intentId: crypto.randomUUID(), + expiresAt: new Date(f.now.getTime() + 1000), + }, + f.now, + { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: approved.call.version, + reservationId: (await f.journal.getAdmission(call.id))!.reservationId, + reservationExpiresAt: (await f.journal.getAdmission(call.id))!.expiresAt, + }, + ); + await f.journal.finish(call.id, dispatched.version, "outcome_unknown", null, false, f.now); + const page = await f.journal.readSession(f.sessionId, f.actorId, { after: "zzzz" }); + expect(page.rows).toHaveLength(0); + expect(page.unknownWriteBlocks).toEqual([f.connection.id]); + await f.journal.resolveUnknown( + call.id, + f.actorId, + "acknowledged", + f.now, + unknownContext((await f.journal.get(call.id))!), + ); + expect((await f.journal.readSession(f.sessionId, f.actorId)).unknownWriteBlocks).toHaveLength( + 0, + ); + expect( + (await f.journal.readSession(f.sessionId, f.actorId)).rows[0]!.resolution?.outcome, + ).toBe("acknowledged"); + } finally { + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "legacy requests cannot gain review authority and frozen read-only requests stay blocked", + async () => { + const f = await reviewFixture(); + try { + const { call } = await pending(f); + const legacyId = crypto.randomUUID(); + await f.db.insert(mcpExecution).values({ + ...call, + id: legacyId, + toolCallId: "legacy", + originalArguments: null, + requestContext: null, + }); + await expect(f.journal.requestApproval(legacyId, f.deploymentId, f.now)).rejects.toThrow( + "schema", + ); + const current = await f.journal.pending(f.sessionId, f.actorId, f.now); + expect(current.map((row) => row.call.id)).not.toContain(legacyId); + await expect( + f.journal.decide( + legacyId, + f.actorId, + true, + reviewContext(call, current[0]!.approval), + f.now, + ), + ).rejects.toThrow("context unavailable"); + const readonly = await f.journal.create( + { + ...f.input, + toolCallId: "read-only", + requestPolicyContext: { + ...f.input.requestPolicyContext, + toolCallId: "read-only", + acceptedWriteEligibility: "read_only", + }, + }, + { kind: "approval" }, + f.now, + ); + const approval = await f.journal.requestApproval(readonly.id, f.deploymentId, f.now); + await expect( + f.journal.decide(readonly.id, f.actorId, true, reviewContext(readonly, approval), f.now), + ).rejects.toThrow("policy denies"); + expect((await f.journal.get(readonly.id))!.state).toBe("waiting_approval"); + } finally { + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "ancestor profile revocation invalidates a child approval without changing the child profile", + async () => { + const f = await reviewFixture(); + const childTaskId = crypto.randomUUID(), + childSessionId = crypto.randomUUID(), + childProfileId = crypto.randomUUID(); + try { + const { compileSessionBindings } = await import("../mcp/session-bindings.ts"); + const { bindingSnapshotColumns } = await import("../mcp/binding-store.ts"); + const live = { ...f.authority, profileId: childProfileId }; + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: childProfileId, integrationGrants: live.profileGrants }, + liveAuthority: live, + bindingEpoch: 0, + requestedGrants: live.profileGrants, + parentAuthority: { sessionId: f.sessionId, snapshot: f.snapshot }, + }); + await f.db.insert(profile).values({ + id: childProfileId, + name: "Child review profile", + imageId: "image", + harness: "claude", + integrationGrants: live.profileGrants, + }); + await f.db.insert(task).values({ + id: childTaskId, + type: "chat", + createdByUserId: f.actorId, + spawningSessionId: f.sessionId, + }); + await f.db.insert(taskSession).values({ + taskId: childTaskId, + sessionId: childSessionId, + ...bindingSnapshotColumns(snapshot, [ + { + id: f.connection.id, + alias: f.connection.alias, + provider: "mcp", + displayName: "Tools", + config: f.connection.config, + }, + ]), + }); + const binding = snapshot.immutableBindings[0]!, + bound = binding.tools[0]!; + const call = await f.journal.create( + { + ...f.input, + taskId: childTaskId, + sessionId: childSessionId, + bindingHash: snapshot.bindingHash, + resolvedTool: { + ...f.input.resolvedTool, + bindingHash: snapshot.bindingHash, + binding, + effectiveGrant: bound.grant, + ancestorAuthorityRefs: snapshot.ancestorAuthorityRefs, + }, + requestPolicyContext: { + ...f.input.requestPolicyContext, + sessionId: childSessionId, + acceptedPolicyRef: { + ...f.input.requestPolicyContext.acceptedPolicyRef, + sessionId: childSessionId, + }, + ancestorPolicyRefs: [f.input.requestPolicyContext.acceptedPolicyRef], + }, + }, + { kind: "approval" }, + f.now, + ); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, f.now); + await f.db.update(profile).set({ integrationGrants: [] }).where(eq(profile.id, f.profileId)); + await expect( + f.journal.decide(call.id, f.actorId, true, reviewContext(call, approval), f.now), + ).rejects.toThrow("stale"); + expect((await f.journal.get(call.id))!.state).toBe("waiting_approval"); + } finally { + await f.db.delete(task).where(eq(task.id, childTaskId)); + await f.db.delete(profile).where(eq(profile.id, childProfileId)); + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "competing first requests with different argument bytes cannot share a call", + async () => { + const f = await reviewFixture(); + try { + const inputs = [ + f.input, + { ...f.input, originalArguments: '{"x":2}', argumentHash: canonicalDigest({ x: 2 }) }, + ]; + const outcomes = await Promise.allSettled( + inputs.map((input) => f.journal.create(input, { kind: "approval" }, f.now)), + ); + expect(outcomes.filter((result) => result.status === "fulfilled")).toHaveLength(1); + expect(outcomes.filter((result) => result.status === "rejected")).toHaveLength(1); + const state = await f.journal.readSession(f.sessionId, f.actorId); + expect(state.rows).toHaveLength(1); + const stored = state.rows[0]!.call; + expect(canonicalDigest(JSON.parse(stored.originalArguments!))).toBe(stored.argumentHash); + } finally { + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "a stale reservation attempt cannot release a replacement at the same clock time", + async () => { + const f = await reviewFixture(); + try { + const { call, expected } = await pending(f); + const approved = await f.journal.decide(call.id, f.actorId, true, expected, f.now); + await f.journal.admit(call.id, f.deploymentId, f.now); + const old = (await f.journal.getAdmission(call.id))!; + const authorization = { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: approved.call.version, + reservationId: old.reservationId, + reservationExpiresAt: old.expiresAt, + }; + const intent = { + bootId: "boot-a", + intentId: crypto.randomUUID(), + expiresAt: new Date(f.now.getTime() + 1000), + }; + await expect( + f.journal.dispatch(call.id, intent, f.now, { + ...authorization, + expectedJournalRevision: 0, + }), + ).rejects.toThrow(); + await f.journal.admit(call.id, f.deploymentId, f.now); + const replacement = (await f.journal.getAdmission(call.id))!; + expect(replacement.createdAt).toEqual(old.createdAt); + expect(replacement.expiresAt).toEqual(old.expiresAt); + expect(replacement.reservationId).not.toBe(old.reservationId); + await expect(f.journal.dispatch(call.id, intent, f.now, authorization)).rejects.toThrow(); + expect((await f.journal.getAdmission(call.id))?.reservationId).toBe( + replacement.reservationId, + ); + } finally { + await f.cleanup(); + } + }, +); diff --git a/orchestrator/src/__tests__/mcp-rpc.test.ts b/orchestrator/src/__tests__/mcp-rpc.test.ts new file mode 100644 index 000000000..98e5ff3f3 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-rpc.test.ts @@ -0,0 +1,397 @@ +import { describe, expect, test } from "bun:test"; +import { createClient, Code, ConnectError } from "@connectrpc/connect"; +import { createConnectTransport } from "@connectrpc/connect-node"; +import { Hono } from "hono"; +import type { AddressInfo } from "node:net"; +import { buildServer } from "../server.ts"; +import { IntegrationService } from "../gen/engram/app/v1/integration_pb.ts"; +import { mcpHandlers, type McpServices } from "../mcp/contracts.ts"; +import type { + IntegrationConnectionRow, + IntegrationConnectionStore, +} from "../db/integration-connections.ts"; +import { reviewCatalog } from "../mcp/catalog.ts"; +import { MCP_LIMITS } from "../mcp/limits.ts"; + +const unavailable = async (): Promise => { + throw new Error("unexpected service invocation"); +}; +const row: IntegrationConnectionRow = { + id: "connection-1", + alias: "tools", + provider: "mcp", + displayName: "Tools", + isDefault: false, + enabled: false, + testedAt: null, + createdAt: new Date(0), + updatedAt: new Date(0), + config: { + endpoint: "https://tools.example.test/mcp", + zone: "public", + auth: { kind: "api_key", secretRef: "mcp.tools.api-key" }, + }, + mcp: { + configRevision: 1, + authGeneration: 1, + credentialIdentity: "account-a", + catalogRevision: 0, + revoked: false, + }, +}; +const connections: IntegrationConnectionStore = { + list: async () => [row], + get: async (id) => (id === row.id ? row : null), + getMany: async () => [row], + getDefault: async () => null, + create: unavailable, + update: unavailable, + delete: unavailable, + markTested: unavailable, + setEnabled: unavailable, + setConfig: unavailable, + ensureDefault: unavailable, +}; +function services(): McpServices { + return { + enabled: true, + repository: { + setEnabled: unavailable, + saveDraft: unavailable, + publish: unavailable, + getCatalog: unavailable, + getLatestDraft: unavailable, + }, + journal: { + get: unavailable, + readSession: unavailable, + readExecutionResult: unavailable, + getAdmission: unavailable, + hasUnknown: unavailable, + assertCurrent: unavailable, + create: unavailable, + requestApproval: unavailable, + pending: unavailable, + decide: unavailable, + admit: unavailable, + admitSchema: unavailable, + finishSchema: unavailable, + rejectBeforeDispatch: unavailable, + assertDispatchCurrent: unavailable, + dispatch: unavailable, + finish: unavailable, + cancel: unavailable, + cancelInternal: unavailable, + cancellationReceipt: unavailable, + resolveUnknown: unavailable, + expire: unavailable, + expireUnadmitted: unavailable, + expiryScopes: unavailable, + }, + }; +} +async function spawn(service?: McpServices, role = "admin", email?: string, connection = row) { + const server = buildServer(new Hono(), (router) => + router.service( + IntegrationService, + mcpHandlers({ + services: service, + connections: { + ...connections, + get: async (id) => (id === connection.id ? connection : null), + }, + getSession: async () => ({ user: { id: "owner", role, email } }), + now: () => new Date(0), + connectionToProto: (c) => ({ id: c.id }), + }), + ), + ); + const url = await new Promise((resolve) => + server.listen(0, "127.0.0.1", () => + resolve(`http://127.0.0.1:${(server.address() as AddressInfo).port}/rpc`), + ), + ); + return { + client: createClient( + IntegrationService, + createConnectTransport({ baseUrl: url, httpVersion: "1.1" }), + ), + close: () => + new Promise((resolve, reject) => server.close((e) => (e ? reject(e) : resolve()))), + }; +} +async function code(promise: Promise, expected: Code) { + try { + await promise; + throw new Error("request should fail"); + } catch (e) { + expect(e).toBeInstanceOf(ConnectError); + expect((e as ConnectError).code).toBe(expected); + } +} +describe("MCP native wire contract", () => { + test("all new methods fail closed by default", async () => { + const s = await spawn(); + try { + await code( + s.client.discoverMcp({ connectionId: row.id, expectedRevision: 1 }), + Code.FailedPrecondition, + ); + await code(s.client.putMcpCredential({}), Code.FailedPrecondition); + await code(s.client.completeMcpAuthorization({}), Code.FailedPrecondition); + await code(s.client.recoverMcpCredential({}), Code.FailedPrecondition); + await code(s.client.publishMcp({}), Code.FailedPrecondition); + await code(s.client.authorizeMcp({}), Code.FailedPrecondition); + await code(s.client.disconnectMcp({}), Code.FailedPrecondition); + await code(s.client.getMcpExecutionResult({}), Code.FailedPrecondition); + await code(s.client.listMcpApprovals({}), Code.FailedPrecondition); + await code(s.client.decideMcpApproval({}), Code.FailedPrecondition); + await code(s.client.cancelMcpCall({}), Code.FailedPrecondition); + await code(s.client.resolveMcpOutcome({}), Code.FailedPrecondition); + await code(s.client.rebindMcp({}), Code.FailedPrecondition); + } finally { + await s.close(); + } + }); + test("admin controls and missing transport fail before services run", async () => { + const member = await spawn(services(), "user"); + const admin = await spawn(services()); + try { + await code(member.client.discoverMcp({ connectionId: row.id }), Code.PermissionDenied); + await code(member.client.publishMcp({}), Code.PermissionDenied); + await code(member.client.authorizeMcp({}), Code.PermissionDenied); + await code(admin.client.discoverMcp({ connectionId: row.id }), Code.FailedPrecondition); + await code( + admin.client.rebindMcp({ sessionId: "s1", expectedBindingHash: "hash" }), + Code.FailedPrecondition, + ); + } finally { + await member.close(); + await admin.close(); + } + }); + test("setup reports only a registered enabled discovery hook", async () => { + const service = services(); + const admin = await spawn(service); + try { + expect((await admin.client.getMcpSetup({})).discoveryAvailable).toBe(false); + service.discovery = { discover: unavailable }; + expect((await admin.client.getMcpSetup({})).discoveryAvailable).toBe(true); + service.enabled = false; + const disabled = await admin.client.getMcpSetup({}); + expect(disabled.discoveryAvailable).toBe(false); + expect(disabled.recoveryAvailable).toBe(false); + await code(admin.client.recoverMcpCredential({}), Code.FailedPrecondition); + await code(admin.client.disconnectMcp({}), Code.FailedPrecondition); + } finally { + await admin.close(); + } + }); + test("catalog read requires an admin and returns the persisted draft and current publication", async () => { + const service = services(); + const base = { + connectionId: row.id, + configRevision: 1, + authGeneration: 1, + credentialIdentity: "account-a", + actorId: "owner", + createdAt: new Date(0), + fingerprint: "hash", + }; + service.repository.getLatestDraft = async () => ({ + ...base, + id: "draft", + kind: "draft", + revision: 0, + tools: [ + { + upstreamName: "read.item", + description: "", + inputSchema: { type: "object" }, + }, + ], + }); + service.repository.getCatalog = async (_id, revision) => ({ + ...base, + id: "published", + kind: "published", + revision, + tools: [ + { + upstreamName: "read.item", + description: "old", + inputSchema: { type: "object" }, + classification: "read", + enabled: true, + fingerprint: "tool", + }, + ], + }); + const current = { ...row, mcp: { ...row.mcp!, catalogRevision: 2 } }; + const admin = await spawn(service, "admin", undefined, current); + const member = await spawn(service, "user"); + const serviceAccount = await spawn(service, "admin", "apikey+agent@service.local"); + try { + await code( + serviceAccount.client.getMcpCatalogs({ connectionId: row.id }), + Code.PermissionDenied, + ); + await code(member.client.getMcpCatalogs({ connectionId: row.id }), Code.PermissionDenied); + const result = await admin.client.getMcpCatalogs({ connectionId: row.id }); + expect(result.draft?.tools[0]?.classification).toBe(""); + expect(result.draft?.tools[0]?.description).toBe(""); + expect(result.published?.revision).toBe(2); + expect(result.published?.tools[0]?.classification).toBe("read"); + expect(result.catalogRevision).toBe(2); + service.enabled = false; + expect( + (await admin.client.getMcpCatalogs({ connectionId: row.id })).published?.revision, + ).toBe(2); + service.repository.getLatestDraft = async () => ({ + ...base, + id: "oversized", + kind: "draft", + revision: 0, + tools: Array.from({ length: 129 }, (_, i) => ({ + upstreamName: `tool-${i}`, + description: "", + inputSchema: { type: "object" }, + })), + }); + await code(admin.client.getMcpCatalogs({ connectionId: row.id }), Code.FailedPrecondition); + } finally { + await member.close(); + await serviceAccount.close(); + await admin.close(); + } + }); + test("publication preserves schema and classification and maps stale CAS", async () => { + const service = services(); + let calls = 0; + service.validateSchemas = async (tools) => { + expect(tools[0]!.classification).toBe("write"); + }; + service.repository.publish = async (_connectionId, _draftId, expected, input) => { + calls++; + if (expected !== 0) throw new Error("stale revision"); + const tools = reviewCatalog(input); + return { + id: "catalog-1", + connectionId: row.id, + kind: "published", + revision: 1, + configRevision: 1, + authGeneration: 1, + credentialIdentity: "account-a", + tools, + fingerprint: "catalog-hash", + actorId: "owner", + createdAt: new Date(0), + }; + }; + const s = await spawn(service); + const request = { + connectionId: row.id, + draftId: "draft-1", + expectedRevision: 0, + tools: [ + { + upstreamName: "Create.Record", + description: "Create a record", + inputSchemaJson: + '{"type":"object","properties":{"count":{"type":"number","minimum":0.1,"maximum":1e3}}}', + outputSchemaJson: + '{"type":"object","properties":{"count":{"type":"number","minimum":0.1,"maximum":1000}}}', + classification: "write", + enabled: true, + }, + ], + }; + try { + const response = await s.client.publishMcp(request); + expect(response.catalog?.tools[0]?.classification).toBe("write"); + expect(response.catalog!.tools[0]!.inputSchemaJson).toBe(request.tools[0]!.outputSchemaJson); + expect(response.catalog!.tools[0]!.inputSchemaJson).toBe( + response.catalog!.tools[0]!.outputSchemaJson, + ); + await code(s.client.publishMcp({ ...request, expectedRevision: 1 }), Code.Aborted); + await code( + s.client.publishMcp({ + ...request, + tools: [{ ...request.tools[0]!, inputSchemaJson: "not json" }], + }), + Code.InvalidArgument, + ); + expect(calls).toBe(2); + } finally { + await s.close(); + } + }); + for (const field of ["inputSchemaJson", "outputSchemaJson"] as const) { + test.each([ + [ + "rounded decimal", + '{"type":"object","properties":{"count":{"minimum":0.10000000000000001}}}', + ], + ["unsafe integer", '{"type":"object","properties":{"count":{"maximum":9007199254740993}}}'], + ["raw byte limit", '{"type":"object"}'.padStart(MCP_LIMITS.argumentBytes + 1)], + ])( + `publication rejects ${field} with %s before validation or persistence`, + async (_name, raw) => { + const service = services(); + let validations = 0; + let publications = 0; + service.validateSchemas = async () => { + validations++; + }; + service.repository.publish = async () => { + publications++; + return unavailable(); + }; + const s = await spawn(service); + try { + await code( + s.client.publishMcp({ + connectionId: row.id, + draftId: "draft-1", + tools: [ + { + upstreamName: "Create.Record", + description: "Create a record", + inputSchemaJson: '{"type":"object"}', + [field]: raw, + classification: "write", + enabled: true, + }, + ], + }), + Code.InvalidArgument, + ); + expect(validations).toBe(0); + expect(publications).toBe(0); + } finally { + await s.close(); + } + }, + ); + } +}); + +test("credential lifecycle requires a human administrator before service access", async () => { + const member = await spawn(services(), "user"); + const serviceAccount = await spawn(services(), "admin", "apikey+agent@service.local"); + try { + for (const s of [member, serviceAccount]) { + await code(s.client.putMcpCredential({ value: "entered-value" }), Code.PermissionDenied); + await code(s.client.authorizeMcp({ shareWithOrgSessions: true }), Code.PermissionDenied); + await code( + s.client.completeMcpAuthorization({ code: "provider-code", state: "provider-state" }), + Code.PermissionDenied, + ); + await code(s.client.disconnectMcp({}), Code.PermissionDenied); + } + } finally { + await member.close(); + await serviceAccount.close(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-runtime-authority.test.ts b/orchestrator/src/__tests__/mcp-runtime-authority.test.ts new file mode 100644 index 000000000..fade3ff43 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-runtime-authority.test.ts @@ -0,0 +1,273 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { taskSession, user } from "../db/schema.ts"; +import { reviewFixture } from "./mcp-review-fixture.ts"; +import { reviewContext } from "../mcp/journal-review.ts"; + +async function ready() { + const f = await reviewFixture(); + await f.db + .update(taskSession) + .set({ integrationPrincipalId: f.actorId }) + .where(eq(taskSession.sessionId, f.sessionId)); + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.admitSchema(call.id, f.deploymentId, f.now); + const schema = (await f.journal.getAdmission(call.id))!; + await f.journal.finishSchema( + call.id, + schema.reservationId, + schema.expiresAt, + call.requestDigest, + true, + f.now, + ); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, f.now); + const approved = await f.journal.decide( + call.id, + f.actorId, + true, + reviewContext(call, approval), + f.now, + ); + await f.journal.admit(call.id, f.deploymentId, f.now); + const lease = (await f.journal.getAdmission(call.id))!; + const proof = { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: approved.call.version, + reservationId: lease.reservationId, + reservationExpiresAt: lease.expiresAt, + }; + const intent = { bootId: "boot-a", intentId: crypto.randomUUID(), expiresAt: lease.expiresAt }; + return { ...f, call, proof, intent }; +} +test("barrier checks the current immutable principal and ban in its transaction", async () => { + const f = await ready(); + try { + await f.db.update(user).set({ banned: true }).where(eq(user.id, f.actorId)); + await expect(f.journal.dispatch(f.call.id, f.intent, f.now, f.proof)).rejects.toThrow(); + expect((await f.journal.get(f.call.id))!.state).toBe("authorized"); + } finally { + await f.cleanup(); + } +}); +test("barrier denies a principal changed after review", async () => { + const f = await ready(); + try { + await f.db + .update(taskSession) + .set({ integrationPrincipalId: "other" }) + .where(eq(taskSession.sessionId, f.sessionId)); + await expect(f.journal.dispatch(f.call.id, f.intent, f.now, f.proof)).rejects.toThrow(); + } finally { + await f.cleanup(); + } +}); + +test("successful barrier and both live assertions retain exact lease generation", async () => { + const f = await ready(); + try { + expect((await f.journal.dispatch(f.call.id, f.intent, f.now, f.proof)).state).toBe( + "dispatching", + ); + expect((await f.journal.assertDispatchCurrent(f.call.id, f.proof, f.now)).state).toBe( + "dispatching", + ); + await expect( + f.journal.assertDispatchCurrent( + f.call.id, + { ...f.proof, reservationId: "other-generation" }, + f.now, + ), + ).rejects.toThrow("stale"); + await f.db.update(user).set({ banned: true }).where(eq(user.id, f.actorId)); + await expect(f.journal.assertDispatchCurrent(f.call.id, f.proof, f.now)).rejects.toThrow(); + } finally { + await f.cleanup(); + } +}); + +test("automation requires exact live linkage and published grant; creator role is not authority", async () => { + const { automation, automationRun, automationSession, task } = await import("../db/schema.ts"); + const { makeProfileStore } = await import("../db/profiles.ts"); + const { compileSessionBindings } = await import("../mcp/session-bindings.ts"); + const { bindingSnapshotColumns } = await import("../mcp/binding-store.ts"); + for (const publish of [false, true]) { + const f = await reviewFixture(); + const automationId = crypto.randomUUID(), + runId = crypto.randomUUID(), + principalId = `automation:${automationId}`; + try { + const profiles = makeProfileStore(f.db); + if (publish) { + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + await profiles.update(f.profileId, (await profiles.get(f.profileId))!, { + actorId: f.actorId, + expectedRevision: 0, + }); + await f.db.update(user).set({ role: "user" }).where(eq(user.id, f.actorId)); + } + const publication = (await profiles.get(f.profileId))!.mcpGrantPublication ?? null; + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: f.profileId, integrationGrants: f.authority.profileGrants }, + liveAuthority: { ...f.authority, profilePublication: publication }, + bindingEpoch: 0, + }); + await f.db + .update(taskSession) + .set({ + ...bindingSnapshotColumns(snapshot, [ + { + id: f.connection.id, + alias: f.connection.alias, + provider: "mcp", + displayName: f.connection.displayName, + config: f.connection.config, + }, + ]), + integrationPrincipalId: principalId, + }) + .where(eq(taskSession.sessionId, f.sessionId)); + await f.db.update(task).set({ createdByUserId: null }).where(eq(task.id, f.taskId)); + await f.db + .insert(automation) + .values({ id: automationId, name: "Test automation", createdByUserId: f.actorId }); + await f.db.insert(automationRun).values({ + id: runId, + automationId, + taskId: f.taskId, + sessionId: f.sessionId, + status: "running", + trigger: { source: "manual", receivedAt: f.now.toISOString() }, + }); + await f.db + .insert(automationSession) + .values({ sessionId: f.sessionId, runId, blockId: "start" }); + const input = { + ...f.input, + principalId, + bindingHash: snapshot.bindingHash, + resolvedTool: { + ...f.input.resolvedTool, + bindingHash: snapshot.bindingHash, + binding: snapshot.immutableBindings[0]!, + profilePublication: publication, + }, + }; + const call = await f.journal.create( + input, + { kind: "automation", grant: input.resolvedTool.effectiveGrant }, + f.now, + ); + await f.journal.admitSchema(call.id, f.deploymentId, f.now); + const schema = (await f.journal.getAdmission(call.id))!; + await f.journal.finishSchema( + call.id, + schema.reservationId, + schema.expiresAt, + call.requestDigest, + true, + f.now, + ); + await f.journal.admit(call.id, f.deploymentId, f.now); + const lease = (await f.journal.getAdmission(call.id))!; + const proof = { + executionId: call.id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId, + expectedJournalRevision: call.version, + reservationId: lease.reservationId, + reservationExpiresAt: lease.expiresAt, + }; + const intent = { bootId: "boot", intentId: crypto.randomUUID(), expiresAt: lease.expiresAt }; + if (!publish) + await expect(f.journal.dispatch(call.id, intent, f.now, proof)).rejects.toThrow( + "published", + ); + else { + expect((await f.journal.dispatch(call.id, intent, f.now, proof)).state).toBe("dispatching"); + await f.db + .update(automation) + .set({ enabled: false }) + .where(eq(automation.id, automationId)); + await expect(f.journal.assertDispatchCurrent(call.id, proof, f.now)).rejects.toThrow( + "automation", + ); + } + } finally { + await f.db.delete(automation).where(eq(automation.id, automationId)); + await f.cleanup(); + } + } +}); + +test("competing barriers commit once and losing cleanup preserves the committed lease", async () => { + const f = await ready(); + try { + const results = await Promise.allSettled([ + f.journal.dispatch(f.call.id, f.intent, f.now, f.proof), + f.journal.dispatch(f.call.id, f.intent, f.now, f.proof), + ]); + expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1); + expect((await f.journal.getAdmission(f.call.id))!.reservationId).toBe(f.proof.reservationId); + expect((await f.journal.assertDispatchCurrent(f.call.id, f.proof, f.now)).state).toBe( + "dispatching", + ); + } finally { + await f.cleanup(); + } +}); + +test("cancellation commits before a waiting dispatch", async () => { + const f = await ready(); + try { + const revision = (await f.journal.get(f.call.id))!.version; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const dispatch = (async () => { + await gate; + return f.journal.dispatch(f.call.id, f.intent, f.now, f.proof); + })(); + const canceled = await f.journal.cancel(f.call.id, f.actorId, f.now, revision); + expect(canceled.state).toBe("canceled"); + release(); + await expect(dispatch).rejects.toThrow(); + expect((await f.journal.get(f.call.id))!.state).toBe("canceled"); + expect(await f.journal.getAdmission(f.call.id)).toBeNull(); + await expect(f.journal.assertDispatchCurrent(f.call.id, f.proof, f.now)).rejects.toThrow(); + } finally { + await f.cleanup(); + } +}); + +test("committed dispatch rejects cancellation with the previously observed revision", async () => { + const f = await ready(); + try { + const revision = (await f.journal.get(f.call.id))!.version; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; + }); + const cancel = (async () => { + await gate; + return f.journal.cancel(f.call.id, f.actorId, f.now, revision); + })(); + expect((await f.journal.dispatch(f.call.id, f.intent, f.now, f.proof)).state).toBe( + "dispatching", + ); + release(); + await expect(cancel).rejects.toThrow("stale cancellation context"); + const call = (await f.journal.get(f.call.id))!; + expect(call.state).toBe("dispatching"); + expect(call.intentId).toBe(f.intent.intentId); + expect((await f.journal.getAdmission(f.call.id))!.reservationId).toBe(f.proof.reservationId); + await expect(f.journal.dispatch(f.call.id, f.intent, f.now, f.proof)).rejects.toThrow(); + } finally { + await f.cleanup(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-runtime-config.test.ts b/orchestrator/src/__tests__/mcp-runtime-config.test.ts new file mode 100644 index 000000000..e6a6d6b63 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-runtime-config.test.ts @@ -0,0 +1,46 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { loadMcpRuntimeConfiguration } from "../mcp/runtime-config.ts"; + +test("runtime configuration defaults zones off and binds actual CA bytes at unchanged paths", async () => { + const dir = await mkdtemp(join(tmpdir(), "mcp-config-")); + try { + await writeFile(join(dir, "ca"), "first-ca"); + await writeFile(join(dir, "cert"), "certificate"); + await writeFile(join(dir, "key"), "private-key", { mode: 0o600 }); + const tls = { + url: "https://control.example.test/", + caName: "control", + certFile: join(dir, "cert"), + keyFile: join(dir, "key"), + serverFingerprint: Array(32).fill("AA").join(":"), + }; + const config = { + version: 1, + infrastructure: { + deploymentIdentity: `sha256:${"a".repeat(64)}`, + networkIdentity: `sha256:${"b".repeat(64)}`, + trustIdentity: `sha256:${"c".repeat(64)}`, + }, + caFiles: { control: join(dir, "ca") }, + control: tls, + workers: [{ zoneId: "public", deploymentId: "public-workers", tls }], + }; + await writeFile(join(dir, "config"), JSON.stringify(config)); + const first = await loadMcpRuntimeConfiguration(join(dir, "config")); + expect(first.workers.get("public")!.enabled).toBe(false); + await writeFile(join(dir, "ca"), "second-ca"); + const next = await loadMcpRuntimeConfiguration(join(dir, "config")); + expect(next.infrastructure.caContentDigests.control).not.toBe( + first.infrastructure.caContentDigests.control, + ); + expect(next.control.ca).toBe("second-ca"); + config.workers.push(config.workers[0]!); + await writeFile(join(dir, "config"), JSON.stringify(config)); + await expect(loadMcpRuntimeConfiguration(join(dir, "config"))).rejects.toThrow("duplicated"); + } finally { + await rm(dir, { recursive: true }); + } +}); diff --git a/orchestrator/src/__tests__/mcp-schema-composition.test.ts b/orchestrator/src/__tests__/mcp-schema-composition.test.ts new file mode 100644 index 000000000..62a3d6b3b --- /dev/null +++ b/orchestrator/src/__tests__/mcp-schema-composition.test.ts @@ -0,0 +1,129 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { effectFixture } from "./mcp-effect-fixture.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +import { mcpDiscoveryAdmission, user } from "../db/schema.ts"; +import { makeMcpDiscoveryAdmission } from "../mcp/discovery-admission.ts"; +import { makeMcpOperatorRuntime } from "../mcp/operator-runtime.ts"; +import { makeMcpDiscovery } from "../mcp/discovery.ts"; +import { makeCredentialMetadataStore } from "../mcp/credential-store.ts"; +import { makeMcpSchemaValidator } from "../mcp/schema-validator.ts"; + +await verifyMcpTestDatabases(); +test("actual isolated worker validates admin schemas without credential acquisition", async () => { + const f = await effectFixture(); + try { + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + let enabled = true; + const publication = f.operatorPolicies[0]!.publication; + const operator = makeMcpOperatorRuntime({ + enabled: () => enabled, + configuration: { + infrastructure: publication, + control: f.remote.workerTls, + registrations: [], + workers: new Map([ + ["public", { enabled: true, deploymentId: f.deploymentId, tls: f.remote.workerTls }], + ]), + }, + read: async () => ({ + generation: publication.generation, + identity: publication.identity, + policies: f.operatorPolicies, + }), + }); + const admission = makeMcpDiscoveryAdmission(f.db); + const validate = makeMcpSchemaValidator({ admission, operator, now: () => new Date() }); + const connection = (await f.connections.get(f.connection.id))!; + const tools = f.snapshot.immutableBindings[0]!.tools.map((t) => t.tool); + await validate(tools, connection, f.actorId); + expect(f.leases).toBe(0); + expect(await f.command("stats")).toEqual({ calls: 0, credentialRequests: 0 }); + await expect( + validate( + [{ ...tools[0]!, inputSchema: { type: "not-a-schema-type" } }], + connection, + f.actorId, + ), + ).rejects.toThrow(); + const rows = await f.db + .select() + .from(mcpDiscoveryAdmission) + .where(eq(mcpDiscoveryAdmission.connectionId, f.connection.id)); + expect(rows).toHaveLength(2); + expect( + rows.every((row) => row.state === "finished" && row.operation.kind === "validateSchemas"), + ).toBe(true); + enabled = false; + await expect(validate(tools, connection, f.actorId)).rejects.toThrow("disabled"); + enabled = true; + await f.db.update(user).set({ role: "user" }).where(eq(user.id, f.actorId)); + await expect(validate(tools, connection, f.actorId)).rejects.toThrow("administrator"); + expect(f.leases).toBe(0); + } finally { + await f.db + .delete(mcpDiscoveryAdmission) + .where(eq(mcpDiscoveryAdmission.connectionId, f.connection.id)); + await f.cleanup(); + } +}, 30000); + +test("actual discovery uses one durable intent and fences role loss after credential delivery", async () => { + const f = await effectFixture(); + try { + await f.db.update(user).set({ role: "admin" }).where(eq(user.id, f.actorId)); + const publication = f.operatorPolicies[0]!.publication; + const operator = makeMcpOperatorRuntime({ + enabled: () => true, + configuration: { + infrastructure: publication, + control: f.remote.workerTls, + registrations: [], + workers: new Map([ + ["public", { enabled: true, deploymentId: f.deploymentId, tls: f.remote.workerTls }], + ]), + }, + read: async () => ({ + generation: publication.generation, + identity: publication.identity, + policies: f.operatorPolicies, + }), + }); + const discovery = makeMcpDiscovery({ + admission: makeMcpDiscoveryAdmission(f.db), + credentials: makeCredentialMetadataStore(f.db), + control: f.control, + operator, + now: () => new Date(), + }); + const connection = (await f.connections.get(f.connection.id))!; + expect(await discovery.discover(connection, f.actorId)).toEqual([ + { + upstreamName: "Lookup.Record", + description: "Find a fixture record", + inputSchema: { type: "object" }, + }, + ]); + expect(f.leases).toBe(1); + const before = await f.command("stats"); + f.afterLease(async () => { + await f.db.update(user).set({ role: "user" }).where(eq(user.id, f.actorId)); + }); + await expect(discovery.discover(connection, f.actorId)).rejects.toThrow("administrator"); + expect(f.leases).toBe(2); + expect(await f.command("stats")).toEqual(before); + const rows = await f.db + .select() + .from(mcpDiscoveryAdmission) + .where(eq(mcpDiscoveryAdmission.connectionId, f.connection.id)); + expect(rows).toHaveLength(2); + expect(rows.every((row) => row.state === "finished" && row.operation.kind === "discover")).toBe( + true, + ); + } finally { + await f.db + .delete(mcpDiscoveryAdmission) + .where(eq(mcpDiscoveryAdmission.connectionId, f.connection.id)); + await f.cleanup(); + } +}, 30000); diff --git a/orchestrator/src/__tests__/mcp-session-bindings.test.ts b/orchestrator/src/__tests__/mcp-session-bindings.test.ts new file mode 100644 index 000000000..47956b6c8 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-session-bindings.test.ts @@ -0,0 +1,256 @@ +import { expect, test } from "bun:test"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { reviewCatalog } from "../mcp/catalog.ts"; +import { + compileSessionBindings, + checkBindingCurrent, + compileMcpManifest, +} from "../mcp/session-bindings.ts"; + +import { schema, bindingGrant, bindingAuthority } from "./mcp-binding-fixture.ts"; +function compile(authority = bindingAuthority()) { + return compileSessionBindings({ + launchPolicy: { profileId: authority.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + }); +} +test("session bindings preserve reviewed schema and freeze every authority input", () => { + const authority = bindingAuthority(); + const snapshot = compile(authority); + const manifest = compileMcpManifest(snapshot); + expect(manifest[0]?.inputSchema).toEqual(schema); + expect(manifest[0]?.execution).toBe("sync"); + expect(manifest[0]?.resultFormat).toBe("mcp"); + expect(snapshot.managedDenyPolicy.hosts).toEqual(["alias.example.test", "tools.example.test"]); + const original = structuredClone(snapshot); + authority.connections[0]!.tools[0]!.inputSchema = { type: "null" }; + authority.operatorPolicies[0]!.aliases.push("new.example.test"); + expect(snapshot).toEqual(original); + expect(checkBindingCurrent({ snapshot, liveAuthority: authority }).current).toBe(false); +}); +test("all identity and authority changes invalidate; token refresh has no effect", () => { + const snapshot = compile(); + expect(checkBindingCurrent({ snapshot, liveAuthority: bindingAuthority() }).current).toBe(true); + const mutations = [ + (a: ReturnType) => { + a.connections[0]!.state.configRevision++; + }, + (a: ReturnType) => { + a.connections[0]!.state.authGeneration++; + }, + (a: ReturnType) => { + a.connections[0]!.state.catalogRevision++; + }, + (a: ReturnType) => { + a.connections[0]!.state.credentialIdentity = "identity-b"; + }, + (a: ReturnType) => { + a.connections[0]!.enabled = false; + }, + (a: ReturnType) => { + a.connections[0]!.state.revoked = true; + }, + (a: ReturnType) => { + a.connections[0]!.alias = "renamed"; + }, + (a: ReturnType) => { + a.connections[0]!.config.endpoint = "https://other.example.test/mcp"; + }, + (a: ReturnType) => { + a.operatorPolicies[0]!.policyRevision = "2"; + }, + (a: ReturnType) => { + a.operatorPolicies[0]!.policyDigest = `sha256:${"b".repeat(64)}`; + }, + (a: ReturnType) => { + a.operatorPolicies[0]!.aliases = []; + }, + (a: ReturnType) => { + a.profileGrants = []; + }, + ]; + for (const mutate of mutations) { + const authority = bindingAuthority(); + mutate(authority); + expect(checkBindingCurrent({ snapshot, liveAuthority: authority }).current).toBe(false); + } +}); +test("children intersect actual parent, profile and explicit request; empty never inherits", () => { + const authority = bindingAuthority(); + authority.profileGrants[0]!.mcp = { + ...bindingGrant.mcp!, + unrestricted: false, + argumentConstraintDocuments: [schema], + }; + const parent = compile(authority); + const childAuthority = bindingAuthority(); + childAuthority.profileGrants[0]!.mcp!.automationWrite = true; + const args = { + launchPolicy: { + profileId: childAuthority.profileId, + integrationGrants: childAuthority.profileGrants, + }, + liveAuthority: childAuthority, + bindingEpoch: 0, + parentAuthority: { sessionId: "parent", snapshot: parent }, + }; + const empty = compileSessionBindings({ ...args, requestedGrants: [] }); + expect(empty.immutableBindings).toEqual([]); + const child = compileSessionBindings({ ...args, requestedGrants: childAuthority.profileGrants }); + expect(child.immutableBindings[0]!.tools[0]!.grant.argumentConstraintDocuments).toEqual([schema]); + expect(child.immutableBindings[0]!.tools[0]!.grant.automationWrite).toBe(false); + expect(child.ancestorAuthorityRefs[0]?.bindingHash).toBe(parent.bindingHash); + expect(() => + compileSessionBindings({ + ...args, + requestedGrants: [ + { ...bindingGrant, mcp: { ...bindingGrant.mcp!, upstreamName: "Unknown" } }, + ], + }), + ).toThrow(); +}); +test("write descriptors stay deferred and catalog changes cannot mutate old manifest", () => { + const authority = bindingAuthority(); + authority.connections[0]!.tools = reviewCatalog([ + { + upstreamName: "Document.Search", + description: "Write", + inputSchema: schema, + classification: "write", + enabled: true, + }, + ]); + const snapshot = compile(authority); + const manifest = compileMcpManifest(snapshot); + expect(manifest[0]!.execution).toBe("deferred"); + expect(manifest[0]!.resultFormat).toBe("mcp"); + manifest[0]!.inputSchema = {}; + expect(compileMcpManifest(snapshot)[0]!.inputSchema).toEqual(schema); +}); +test("missing operator approval, altered review fingerprints and over-limit requests fail closed", () => { + const authority = bindingAuthority(); + authority.operatorPolicies = []; + expect(() => compile(authority)).toThrow(); + const changed = bindingAuthority(); + changed.connections[0]!.tools[0]!.description = "Changed after review"; + expect(() => compile(changed)).toThrow(); + const tooMany = bindingAuthority(); + tooMany.connections = Array.from({ length: 17 }, (_, i) => ({ + ...tooMany.connections[0]!, + id: `c${i}`, + alias: `docs${i}`, + })); + tooMany.profileGrants = tooMany.connections.map((c) => ({ ...bindingGrant, connectionId: c.id })); + expect(() => compile(tooMany)).toThrow(); +}); +test("changes to managed destinations outside selected grants invalidate the deny set", () => { + const authority = bindingAuthority(); + authority.connections.push({ + ...structuredClone(authority.connections[0]!), + id: "ungranted", + alias: "other", + }); + authority.operatorPolicies[0]!.authorities.push("second.example.test"); + const snapshot = compile(authority); + authority.connections[1]!.config.endpoint = "https://second.example.test/mcp"; + expect(checkBindingCurrent({ snapshot, liveAuthority: authority }).current).toBe(false); +}); +test("binding freezes the immutable catalog row identifier", () => { + const authority = bindingAuthority(); + const snapshot = compile(authority); + expect(snapshot.immutableBindings[0]!.catalogId).toBe(authority.connections[0]!.catalogId); + authority.connections[0]!.catalogId = "replacement-catalog"; + expect(checkBindingCurrent({ snapshot, liveAuthority: authority }).current).toBe(false); +}); +test("merged manifest checks native collisions and the total tool and byte limits", async () => { + const { appendMcpManifest } = await import("../tools/manifest.ts"); + const snapshot = compile(); + const native = { + name: "native", + description: "Native", + inputSchema: { type: "object" }, + execution: "sync", + nativeBindings: {}, + }; + const limit = Array.from({ length: 127 }, (_, i) => ({ ...native, name: `native${i}` })); + expect(appendMcpManifest(JSON.stringify(limit), snapshot)).toHaveLength(128); + expect(() => appendMcpManifest(JSON.stringify([...limit, native]), snapshot)).toThrow(); + expect(() => + appendMcpManifest( + JSON.stringify([{ ...native, name: snapshot.immutableBindings[0]!.tools[0]!.name }]), + snapshot, + ), + ).toThrow(); + expect(() => + appendMcpManifest( + JSON.stringify([{ ...native, description: "a".repeat(1024 * 1024) }]), + snapshot, + ), + ).toThrow(); +}); + +test("publication revision and publisher are frozen and cannot silently rebase a session", () => { + const source = bindingAuthority(); + const authority = { + ...source, + profilePublication: { + revision: 1, + actorId: "admin-a", + grantsHash: canonicalDigest(source.profileGrants), + }, + }; + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: authority.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + }); + expect(snapshot.profilePublication).toEqual(authority.profilePublication); + authority.profilePublication.revision = 2; + expect(checkBindingCurrent({ snapshot, liveAuthority: authority }).current).toBe(false); + expect(snapshot.profilePublication!.revision).toBe(1); + authority.profilePublication.grantsHash = canonicalDigest([]); + expect(() => + compileSessionBindings({ + launchPolicy: { profileId: authority.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + }), + ).toThrow("publication"); +}); + +test("complete operator publication denies unused zones and invalidates their changes", () => { + const authority = bindingAuthority(); + authority.operatorPolicies.push({ + ...authority.operatorPolicies[0]!, + zoneId: "internal", + authorities: ["unused.example.test", "unused.example.test:8443"], + aliases: ["unused-alias.example.test"], + }); + const snapshot = compile(authority); + expect(snapshot.managedDenyPolicy.hosts).toEqual([ + "alias.example.test", + "tools.example.test", + "unused-alias.example.test", + "unused.example.test", + ]); + authority.operatorPolicies[1]!.policyRevision = "2"; + expect(checkBindingCurrent({ snapshot, liveAuthority: authority }).current).toBe(false); +}); + +test("binding retains exact publication and distinct infrastructure trust identity", () => { + const authority = bindingAuthority(); + const publication = { + generation: 1, + identity: `sha256:${"b".repeat(64)}`, + deploymentIdentity: `sha256:${"c".repeat(64)}`, + networkIdentity: `sha256:${"d".repeat(64)}`, + trustIdentity: `sha256:${"e".repeat(64)}`, + caContentDigests: { control: `sha256:${"f".repeat(64)}` }, + }; + Object.assign(authority.operatorPolicies[0]!, { publication }); + const snapshot = compile(authority); + expect(snapshot.immutableBindings[0]!.operatorPolicy).toMatchObject({ publication }); + publication.caContentDigests.control = `sha256:${"0".repeat(64)}`; + expect(checkBindingCurrent({ snapshot, liveAuthority: authority }).current).toBe(false); +}); diff --git a/orchestrator/src/__tests__/mcp-session-compiler.test.ts b/orchestrator/src/__tests__/mcp-session-compiler.test.ts new file mode 100644 index 000000000..92fe82479 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-session-compiler.test.ts @@ -0,0 +1,306 @@ +import { expect, test } from "bun:test"; +import type { IntegrationPolicyJson } from "../connectors/registry.ts"; +import { makeMcpSessionCompiler } from "../mcp/session-compiler.ts"; +import { makeMcpOperatorRuntime } from "../mcp/operator-runtime.ts"; +import { compileSessionBindings } from "../mcp/session-bindings.ts"; +import { restoreBindingSnapshot } from "../mcp/binding-store.ts"; +import { buildLaunchPolicy } from "../rpc/task-create.ts"; +import { bindingAuthority, operatorPublication } from "./mcp-binding-fixture.ts"; + +const direct: IntegrationPolicyJson = { + network: { default: "allow", allow_hosts: [], allow_host_patterns: [] }, + injects: [], + observes: [], + secrets: [], + guest_services: [], + tunnels: [], +}; +function fixture(live = bindingAuthority()) { + let parent = compileSessionBindings({ + launchPolicy: { profileId: live.profileId, integrationGrants: live.profileGrants }, + liveAuthority: live, + bindingEpoch: 7, + requestedGrants: [ + { + ...live.profileGrants[0]!, + mcp: { + ...live.profileGrants[0]!.mcp!, + unrestricted: false, + automationWrite: false, + argumentConstraintDocuments: [ + { type: "object", properties: { name: { maxLength: 10 } } }, + ], + }, + }, + ], + }); + let currentParent = true, + enabled = true, + parentReads = 0; + const tls = { + url: "https://worker.example.test/", + ca: "ca", + cert: "cert", + key: "key", + serverFingerprint: "AA", + }; + const operator = makeMcpOperatorRuntime({ + configuration: { + control: tls, + infrastructure: operatorPublication, + workers: new Map([["public", { enabled: true, deploymentId: "workers", tls }]]), + registrations: [], + }, + enabled: () => enabled, + read: async () => ({ + generation: 1, + identity: operatorPublication.identity, + policies: live.operatorPolicies, + }), + }); + const compile = makeMcpSessionCompiler({ + operator, + readAuthority: async () => live, + readParentCurrent: async (id) => { + expect(id).toBe("parent"); + parentReads++; + if (!currentParent) throw Error("parent unavailable"); + return parent; + }, + reservedSecretReferences: async () => new Set(["reserved-reference"]), + }); + const input = { + imageUri: "image", + mode: "agent", + requestedSessionId: "child", + harnessEnv: { + ENGRAM_TOOLS: JSON.stringify([ + { + name: "native", + description: "Native", + inputSchema: {}, + execution: "sync", + nativeBindings: {}, + }, + ]), + }, + }; + const context = { + profileId: live.profileId, + profileGrants: live.profileGrants, + grants: live.profileGrants, + directPolicy: direct, + spawningSessionId: "parent", + }; + return { + live, + parent, + compile, + input, + context, + staleParent: () => { + currentParent = false; + }, + disable: () => { + enabled = false; + }, + reads: () => parentReads, + replaceParent: (snapshot: typeof parent) => { + parent = snapshot; + }, + }; +} + +test("child compilation uses the live source profile and the parent's frozen effective request", async () => { + const live = bindingAuthority(); + live.profileGrants[0]!.mcp = { + ...live.profileGrants[0]!.mcp!, + unrestricted: false, + argumentConstraintDocuments: [{ type: "object", properties: { name: { maxLength: 10 } } }], + }; + const f = fixture(live); + const root = await f.compile(f.input, { + profileId: f.live.profileId, + profileGrants: f.live.profileGrants, + grants: f.live.profileGrants, + directPolicy: direct, + }); + const policy = buildLaunchPolicy( + { + id: f.live.profileId, + name: "Fixture", + description: "", + icon: "", + imageId: "image", + harness: "claude", + model: null, + effort: null, + programmaticOnly: false, + includeUserTokens: false, + envVars: {}, + skills: [], + integrationGrants: f.live.profileGrants, + network: { default: "deny", allowHosts: [], allowHostPatterns: [] }, + secrets: [], + repos: [], + apps: [], + createdAt: new Date(0), + updatedAt: new Date(0), + deletedAt: null, + }, + root, + ); + expect(policy.integrationGrants[0]!.mcp!.argumentConstraintDocuments).toHaveLength(1); + const parent = restoreBindingSnapshot(root.mcpBindingEnvelope!, root.integrationConnections!); + f.replaceParent(parent); + const inherited = { + ...f.context, + profileGrants: policy.integrationGrants, + grants: policy.integrationGrants, + }; + const child = await f.compile(f.input, inherited); + const snapshot = restoreBindingSnapshot(child.mcpBindingEnvelope!, child.integrationConnections!); + expect(snapshot.ancestorAuthorityRefs[0]).toMatchObject({ + sessionId: "parent", + bindingHash: parent.bindingHash, + bindingEpoch: 0, + }); + expect(snapshot.immutableBindings[0]!.tools[0]!.grant).toMatchObject({ + unrestricted: false, + automationWrite: false, + argumentConstraintDocuments: f.live.profileGrants[0]!.mcp!.argumentConstraintDocuments, + }); + expect(f.reads()).toBe(1); + const empty = await f.compile(f.input, { ...inherited, grants: [] }); + expect( + restoreBindingSnapshot(empty.mcpBindingEnvelope!, empty.integrationConnections!) + .immutableBindings, + ).toHaveLength(0); + f.staleParent(); + await expect(f.compile(f.input, inherited)).rejects.toThrow("parent unavailable"); + const changedProfileGrants = structuredClone(policy.integrationGrants); + changedProfileGrants[0]!.mcp!.argumentConstraintDocuments = [ + { type: "object", properties: { name: { maxLength: 5 } } }, + ]; + await expect( + f.compile(f.input, { + profileId: f.live.profileId, + profileGrants: changedProfileGrants, + grants: policy.integrationGrants, + directPolicy: direct, + }), + ).rejects.toThrow("MCP profile changed during compilation"); +}); + +test("unchanged constrained grants stay bounded through deep child compilation", async () => { + const live = bindingAuthority(); + const constraint = { type: "object", properties: { name: { maxLength: 10 } } }; + live.profileGrants[0]!.mcp = { + ...live.profileGrants[0]!.mcp!, + unrestricted: false, + argumentConstraintDocuments: [constraint], + }; + let parent = compileSessionBindings({ + launchPolicy: { profileId: live.profileId, integrationGrants: live.profileGrants }, + liveAuthority: live, + bindingEpoch: 0, + requestedGrants: live.profileGrants, + }); + for (let depth = 0; depth < 12; depth++) { + const grants = [ + { ...live.profileGrants[0]!, mcp: parent.immutableBindings[0]!.tools[0]!.grant }, + ]; + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: live.profileId, integrationGrants: live.profileGrants }, + liveAuthority: live, + bindingEpoch: 0, + requestedGrants: grants, + parentAuthority: { sessionId: `parent-${depth}`, snapshot: parent }, + }); + expect(snapshot.immutableBindings[0]!.tools[0]!.grant.argumentConstraintDocuments).toEqual([ + constraint, + ]); + parent = snapshot; + } +}); + +test("actual child compiler freezes native tools and intersects the fresh parent authority", async () => { + const f = fixture(); + const child = await f.compile(f.input, f.context); + const snapshot = restoreBindingSnapshot(child.mcpBindingEnvelope!, child.integrationConnections!); + expect(snapshot.bindingEpoch).toBe(0); + expect(snapshot.ancestorAuthorityRefs[0]).toMatchObject({ + sessionId: "parent", + bindingHash: f.parent.bindingHash, + bindingEpoch: 7, + }); + expect(snapshot.immutableBindings[0]!.tools[0]!.grant).toMatchObject({ + unrestricted: false, + automationWrite: false, + argumentConstraintDocuments: + f.parent.immutableBindings[0]!.tools[0]!.grant.argumentConstraintDocuments, + }); + expect( + JSON.parse(child.harnessEnv!.ENGRAM_TOOLS!).map((tool: { name: string }) => tool.name), + ).toEqual(["native"]); + expect(f.reads()).toBe(1); + f.staleParent(); + await expect(f.compile(f.input, { ...f.context, grants: [] })).rejects.toThrow( + "parent unavailable", + ); +}); + +test("actual child compiler preserves managed ancestry without granting implicit tools", async () => { + const f = fixture(); + const child = await f.compile(f.input, { ...f.context, grants: [] }); + const snapshot = restoreBindingSnapshot(child.mcpBindingEnvelope!, child.integrationConnections!); + expect(snapshot.immutableBindings).toHaveLength(0); + expect(snapshot.ancestorAuthorityRefs[0]!.bindingHash).toBe(f.parent.bindingHash); + expect(snapshot.managedDenyPolicy.hosts).toContain("tools.example.test"); + expect(snapshot.managedDenyPolicy.hosts).toContain("alias.example.test"); +}); + +test("final child policy cannot bypass managed destinations or inject stored credential references", async () => { + const f = fixture(); + for (const policy of [ + { ...direct, network: { ...direct.network, allow_hosts: ["tools.example.test"] } }, + { ...direct, network: { ...direct.network, allow_host_patterns: ["*.example.test"] } }, + { + ...direct, + secrets: [ + { + secret_ref: "reserved-reference", + env_var: "TOKEN", + mode: "literal" as const, + allow_hosts: [], + allow_host_patterns: [], + }, + ], + }, + ]) + await expect(f.compile(f.input, { ...f.context, directPolicy: policy })).rejects.toThrow(); + f.disable(); + await expect(f.compile(f.input, f.context)).rejects.toThrow("disabled"); +}); + +test("child compiler rejects the combined tool limit before creating a held reservation", async () => { + const f = fixture(); + const native = (count: number) => + JSON.stringify( + Array.from({ length: count }, (_, i) => ({ + name: `native-${i}`, + description: "Native", + inputSchema: {}, + execution: "sync", + nativeBindings: {}, + })), + ); + const accepted = await f.compile( + { ...f.input, harnessEnv: { ENGRAM_TOOLS: native(127) } }, + f.context, + ); + expect(JSON.parse(accepted.harnessEnv!.ENGRAM_TOOLS!)).toHaveLength(127); + await expect( + f.compile({ ...f.input, harnessEnv: { ENGRAM_TOOLS: native(128) } }, f.context), + ).rejects.toThrow("manifest exceeds limits"); +}); diff --git a/orchestrator/src/__tests__/mcp-session-conflicts.test.ts b/orchestrator/src/__tests__/mcp-session-conflicts.test.ts new file mode 100644 index 000000000..df68b51a3 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-session-conflicts.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from "bun:test"; +import { assertMcpDirectConflicts } from "../mcp/session-conflicts.ts"; +import type { IntegrationPolicyJson } from "../connectors/registry.ts"; +test("final direct policy rejects managed hosts, aliases, patterns and authoritative secret references", () => { + const base: IntegrationPolicyJson = { + injects: [], + observes: [], + network: { default: "allow", allow_hosts: [], allow_host_patterns: [] }, + secrets: [], + guest_services: [], + tunnels: [], + }; + const hosts = ["tools.example.test", "alias.example.test"], + reserved = new Set(["reserved-reference"]); + expect(() => assertMcpDirectConflicts(base, hosts, reserved)).not.toThrow(); + for (const host of hosts) + expect(() => + assertMcpDirectConflicts( + { ...base, network: { ...base.network, allow_hosts: [host] } }, + hosts, + reserved, + ), + ).toThrow("conflicts"); + expect(() => + assertMcpDirectConflicts( + { ...base, network: { ...base.network, allow_host_patterns: ["*.example.test"] } }, + hosts, + reserved, + ), + ).toThrow("conflicts"); + expect(() => + assertMcpDirectConflicts( + { + ...base, + secrets: [ + { + secret_ref: "reserved-reference", + env_var: "TOKEN", + mode: "literal", + allow_hosts: [], + allow_host_patterns: [], + }, + ], + }, + hosts, + reserved, + ), + ).toThrow("credential"); +}); diff --git a/orchestrator/src/__tests__/mcp-session-create.test.ts b/orchestrator/src/__tests__/mcp-session-create.test.ts new file mode 100644 index 000000000..7a1d22b6f --- /dev/null +++ b/orchestrator/src/__tests__/mcp-session-create.test.ts @@ -0,0 +1,101 @@ +import { expect, test } from "bun:test"; +import { compileBoundSessionCandidate, createReservedTaskSession } from "../mcp/session-create.ts"; +import { managedCreationInput } from "../mcp/managed-create-input.ts"; +import { fromBinary } from "@bufbuild/protobuf"; +import { CreatePolicySessionRequestSchema } from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import { compileSessionBindings } from "../mcp/session-bindings.ts"; +import { bindingAuthority } from "./mcp-binding-fixture.ts"; + +const native = { + name: "native", + description: "Native tool", + inputSchema: { type: "object" }, + execution: "sync", +}; +function candidate() { + const authority = bindingAuthority(); + return compileBoundSessionCandidate( + { + imageUri: "image", + mode: "agent", + requestedSessionId: "reserved", + harnessEnv: { ENGRAM_TOOLS: JSON.stringify([native]) }, + }, + { + launchPolicy: { profileId: authority.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + }, + ); +} +test("candidate and held reserve carry only the frozen native manifest", () => { + const input = candidate(); + expect(JSON.parse(input.harnessEnv!.ENGRAM_TOOLS!)).toEqual([native]); + const request = managedCreationInput(input, { + actorId: "actor", + inputId: "input", + taskId: "task", + }); + const wire = fromBinary(CreatePolicySessionRequestSchema, request.request.creationRequest); + expect(JSON.parse(wire.create!.harnessEnv.ENGRAM_TOOLS!)).toEqual([native]); + expect(wire.create!.requestedSessionId).toBe("reserved"); +}); +test("reserved MCP creation requires the managed capability and never calls generic create", async () => { + let generic = 0; + const sessions = { + async createSession() { + generic++; + return { sessionId: "reserved" }; + }, + }; + await expect( + createReservedTaskSession({ sessions }, candidate(), { actorId: "actor", inputId: "input" }), + ).rejects.toThrow("not configured"); + expect(generic).toBe(0); + await createReservedTaskSession( + { sessions }, + { imageUri: "image", mode: "agent" }, + { actorId: "actor", inputId: "input" }, + ); + expect(generic).toBe(1); +}); +test("child reserve binds the actual parent while cleared harness mode cannot remove ancestry", () => { + const authority = bindingAuthority(); + const compile = { + launchPolicy: { profileId: authority.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + }; + const parent = compileSessionBindings(compile); + const input = compileBoundSessionCandidate( + { imageUri: "image", mode: "agent", requestedSessionId: "child" }, + { + ...compile, + parentAuthority: { sessionId: "actual-parent", snapshot: parent }, + requestedGrants: authority.profileGrants, + }, + ); + const request = managedCreationInput( + { ...input, harnessMode: undefined }, + { taskId: "task", actorId: "actor", inputId: "input", parentSessionId: "actual-parent" }, + ); + const wire = fromBinary(CreatePolicySessionRequestSchema, request.request.creationRequest); + expect(wire.parentSessionId).toBe("actual-parent"); + expect(input.mcpBindingEnvelope!.ancestorAuthorityRefs[0]!.sessionId).toBe("actual-parent"); + expect(request.identity.parentSessionId).toBe("actual-parent"); +}); +test("returned identity mismatch preserves the actual ID for native delegation compensation", async () => { + await expect( + createReservedTaskSession( + { + sessions: { + async createSession() { + return { sessionId: "actual-created" }; + }, + }, + }, + { imageUri: "image", mode: "agent", requestedSessionId: "reserved" }, + { actorId: "actor", inputId: "source" }, + ), + ).rejects.toMatchObject({ createdSessionId: "actual-created" }); +}); diff --git a/orchestrator/src/__tests__/mcp-session-policy-store.test.ts b/orchestrator/src/__tests__/mcp-session-policy-store.test.ts new file mode 100644 index 000000000..5df3ce4a1 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-session-policy-store.test.ts @@ -0,0 +1,47 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { checkDb, getDb } from "../db/client.ts"; +import { task, taskSession, user } from "../db/schema.ts"; +import { authorizePolicyActor } from "../mcp/session-policy.ts"; +const reachable = process.env.ORCHESTRATOR_DATABASE_URL ? await checkDb() : false; +test.skipIf(!reachable)( + "policy authority uses current human owner, role, and ban state", + async () => { + const db = getDb(), + actor = crypto.randomUUID(), + tid = crypto.randomUUID(), + sid = crypto.randomUUID(); + try { + await db + .insert(user) + .values({ + id: actor, + name: "Policy owner", + email: `${actor}@example.test`, + createdAt: new Date(), + updatedAt: new Date(), + role: "user", + }); + await db.insert(task).values({ id: tid, type: "chat", createdByUserId: actor }); + await db.insert(taskSession).values({ taskId: tid, sessionId: sid }); + expect((await authorizePolicyActor(sid, actor)).isAdmin).toBe(false); + await db.update(task).set({ createdByUserId: "another-owner" }).where(eq(task.id, tid)); + await expect(authorizePolicyActor(sid, actor)).rejects.toThrow("owner or administrator"); + await db.update(user).set({ role: "admin" }).where(eq(user.id, actor)); + expect((await authorizePolicyActor(sid, actor)).isAdmin).toBe(true); + await db.update(user).set({ role: "user" }).where(eq(user.id, actor)); + await expect(authorizePolicyActor(sid, actor)).rejects.toThrow(); + await db.update(task).set({ createdByUserId: actor }).where(eq(task.id, tid)); + await db.update(user).set({ banned: true }).where(eq(user.id, actor)); + await expect(authorizePolicyActor(sid, actor)).rejects.toThrow(); + await db + .update(user) + .set({ banned: false, email: `apikey+${actor}@service.local` }) + .where(eq(user.id, actor)); + await expect(authorizePolicyActor(sid, actor)).rejects.toThrow(); + } finally { + await db.delete(task).where(eq(task.id, tid)); + await db.delete(user).where(eq(user.id, actor)); + } + }, +); diff --git a/orchestrator/src/__tests__/mcp-session-policy.test.ts b/orchestrator/src/__tests__/mcp-session-policy.test.ts new file mode 100644 index 000000000..1bd3fd43b --- /dev/null +++ b/orchestrator/src/__tests__/mcp-session-policy.test.ts @@ -0,0 +1,194 @@ +import { expect, test } from "bun:test"; +import { create } from "@bufbuild/protobuf"; +import { createHandlerContext } from "@connectrpc/connect"; +import { + SessionService, + CompleteToolCallRequestSchema, + GetMcpWritePolicyRequestSchema, + McpPlanReviewSchema, +} from "../gen/engram/app/v1/session_pb.ts"; +import { makeSessionPolicyOverrides } from "../mcp/session-policy.ts"; + +const context = () => + createHandlerContext({ + service: SessionService, + method: SessionService.method.completeToolCall, + protocolName: "connect", + requestMethod: "POST", + url: "https://test.invalid/rpc", + requestHeader: new Headers(), + responseHeader: new Headers(), + responseTrailer: new Headers(), + }); +const review = () => + create(McpPlanReviewSchema, { + policyRef: { sessionId: "session-a", policyRevision: 3n, bindingEpoch: 2n }, + toolCallId: "plan-a", + runId: "run-a", + eventIdx: 7n, + requestDigest: "a".repeat(64), + argsJson: '{"plan":"reviewed"}', + }); +test("human plan acceptance uses exact expected context and never relaxes in a preflight", async () => { + let accepted = 0; + let checked = 0; + const overrides = makeSessionPolicyOverrides({ + getSession: async () => ({ user: { id: "owner-a" } }), + authorize: async (sid, actor) => { + checked++; + expect(sid).toBe("session-a"); + expect(actor).toBe("owner-a"); + return { id: actor, isAdmin: false }; + }, + control: { + async acceptPlanExit(req) { + accepted++; + expect(req.actorId).toBe("owner-a"); + expect(req.expected).toEqual(review()); + throw new Error("unavailable"); + }, + }, + }); + const req = create(CompleteToolCallRequestSchema, { + sessionId: "session-a", + toolCallId: "plan-a", + resultJson: '{"decision":"approve"}', + expectedMcpPlanReview: review(), + }); + await expect(overrides["SessionService.CompleteToolCall"]!(req, context())).rejects.toThrow( + "unavailable", + ); + expect(accepted).toBe(1); + expect(checked).toBe(1); +}); +test("generic approval does not call the policy authority", async () => { + const overrides = makeSessionPolicyOverrides(); + expect( + await overrides["SessionService.CompleteToolCall"]!( + create(CompleteToolCallRequestSchema, { + sessionId: "s", + toolCallId: "c", + resultJson: '{"decision":"approve"}', + }), + context(), + ), + ).toBeUndefined(); +}); +test("changed identity and rejected human authorization prevent policy mutation", async () => { + let accepted = 0; + const overrides = makeSessionPolicyOverrides({ + getSession: async () => ({ user: { id: "member-a" } }), + authorize: async () => { + throw new Error("owner or administrator required"); + }, + control: { + async acceptPlanExit() { + accepted++; + return {}; + }, + }, + }); + await expect( + overrides["SessionService.CompleteToolCall"]!( + create(CompleteToolCallRequestSchema, { + sessionId: "session-a", + toolCallId: "plan-a", + resultJson: '{"decision":"approve"}', + expectedMcpPlanReview: review(), + }), + context(), + ), + ).rejects.toThrow(); + expect(accepted).toBe(0); +}); +test("policy review is read from trusted control and exposes its action", async () => { + const overrides = makeSessionPolicyOverrides({ + getSession: async () => ({ user: { id: "owner-a" } }), + authorize: async () => ({ id: "owner-a", isAdmin: false }), + control: { + async readWritePolicy() { + return { + state: { + policy: { writeEligibility: "read_only" }, + planReview: review(), + reason: "External writes are blocked.", + }, + }; + }, + }, + }); + const result = await overrides["SessionService.GetMcpWritePolicy"]!( + create(GetMcpWritePolicyRequestSchema, { sessionId: "session-a", planToolCallId: "plan-a" }), + context(), + ); + expect(result?.response).toMatchObject({ + planReview: review(), + allowedActions: ["approve_current_plan"], + }); +}); + +test("rejection keeps ordinary result delivery and drops permissive review context", async () => { + const req = create(CompleteToolCallRequestSchema, { + sessionId: "session-a", + toolCallId: "plan-a", + resultJson: '{"decision":"reject"}', + expectedMcpPlanReview: review(), + }); + expect( + await makeSessionPolicyOverrides()["SessionService.CompleteToolCall"]!(req, context()), + ).toBeUndefined(); + expect(req.expectedMcpPlanReview).toBeUndefined(); +}); + +test("public approved review replaces generic forwarding only after the trusted commit", async () => { + const { createClient, createRouterTransport } = await import("@connectrpc/connect"); + const { registerPassthrough } = await import("../rpc/passthrough.ts"); + let generic = 0, + accepted = 0; + const getSession = async () => ({ user: { id: "owner-a", role: "user" } }); + const upstream = createRouterTransport((router) => + router.service(SessionService, { + async completeToolCall(req) { + generic++; + return { sessionId: req.sessionId, note: "generic" }; + }, + }), + ); + const transport = createRouterTransport((router) => + registerPassthrough( + router, + [{ service: SessionService, methods: ["CompleteToolCall"] }], + upstream, + getSession, + async () => "owner-a", + undefined, + makeSessionPolicyOverrides({ + getSession, + authorize: async () => ({ id: "owner-a", isAdmin: false }), + control: { + async acceptPlanExit() { + accepted++; + return {}; + }, + }, + }), + ), + ); + const client = createClient(SessionService, transport); + const result = await client.completeToolCall({ + sessionId: "session-a", + toolCallId: "plan-a", + resultJson: '{"decision":"approve"}', + expectedMcpPlanReview: review(), + }); + expect(result.note).toBe("tool result queued"); + expect(accepted).toBe(1); + expect(generic).toBe(0); + await client.completeToolCall({ + sessionId: "session-a", + toolCallId: "other", + resultJson: '{"decision":"approve"}', + }); + expect(generic).toBe(1); + expect(accepted).toBe(1); +}); diff --git a/orchestrator/src/__tests__/mcp-session-resolver.test.ts b/orchestrator/src/__tests__/mcp-session-resolver.test.ts new file mode 100644 index 000000000..50130500a --- /dev/null +++ b/orchestrator/src/__tests__/mcp-session-resolver.test.ts @@ -0,0 +1,215 @@ +import { expect, test } from "bun:test"; +import { compileSessionBindings } from "../mcp/session-bindings.ts"; +import { + makeSessionBindingResolver, + type BoundSessionAuthority, + type SessionBindingResolverDeps, +} from "../mcp/session-resolver.ts"; +import { bindingAuthority, bindingGrant } from "./mcp-binding-fixture.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { bindingSnapshotColumns, restoreBindingSnapshot } from "../mcp/binding-store.ts"; + +function fixture() { + const authority = bindingAuthority(); + const snapshot = compileSessionBindings({ + launchPolicy: { profileId: authority.profileId, integrationGrants: authority.profileGrants }, + liveAuthority: authority, + bindingEpoch: 0, + }); + const rows = new Map([ + ["session-a", { snapshot, parentSessionId: null, active: true }], + ]); + const deps: SessionBindingResolverDeps = { + readSession: async (id) => rows.get(id) ?? null, + readAuthority: async () => authority, + readBindingEpoch: async () => 0, + // This test adapter does not prove host enforcement. + enforceManagedDeny: async (expected) => expected, + }; + return { authority, snapshot, rows, deps }; +} +test("only exact names in this session resolve and results cannot mutate stored snapshots", async () => { + const f = fixture(); + const resolver = makeSessionBindingResolver(f.deps); + const name = f.snapshot.immutableBindings[0]!.tools[0]!.name; + const resolved = await resolver.resolveBoundTool({ sessionId: "session-a", modelToolName: name }); + expect(resolved?.bindingHash).toBe(f.snapshot.bindingHash); + resolved!.reviewedTool.inputSchema = {}; + expect( + (await resolver.resolveBoundTool({ sessionId: "session-a", modelToolName: name }))!.reviewedTool + .inputSchema, + ).not.toEqual({}); + for (const modelToolName of ["Document.Search", `${name}_spoof`, "mcp_docs_fake"]) + expect( + await resolver.resolveBoundTool({ sessionId: "session-a", modelToolName }), + ).toBeUndefined(); + expect( + await resolver.resolveBoundTool({ sessionId: "session-b", modelToolName: name }), + ).toBeUndefined(); +}); +test("absent ACK, wrong digest, restored epoch and changed ancestor grants deny", async () => { + const f = fixture(); + const childAuthority = bindingAuthority(); + childAuthority.profileId = "profile-child"; + const child = compileSessionBindings({ + launchPolicy: { + profileId: childAuthority.profileId, + integrationGrants: childAuthority.profileGrants, + }, + liveAuthority: childAuthority, + bindingEpoch: 0, + requestedGrants: [bindingGrant], + parentAuthority: { sessionId: "session-a", snapshot: f.snapshot }, + }); + f.rows.set("child", { snapshot: child, parentSessionId: "session-a", active: true }); + f.deps.readAuthority = async (id) => + id === childAuthority.profileId ? childAuthority : f.authority; + const input = { sessionId: "child", modelToolName: child.immutableBindings[0]!.tools[0]!.name }; + expect(await makeSessionBindingResolver(f.deps).resolveBoundTool(input)).toBeDefined(); + expect( + await makeSessionBindingResolver({ ...f.deps, enforceManagedDeny: undefined }).resolveBoundTool( + input, + ), + ).toBeUndefined(); + expect( + await makeSessionBindingResolver({ + ...f.deps, + enforceManagedDeny: async (expected) => ({ ...expected, denyDigest: "wrong" }), + }).resolveBoundTool(input), + ).toBeUndefined(); + expect( + await makeSessionBindingResolver({ + ...f.deps, + readBindingEpoch: async () => 1, + }).resolveBoundTool(input), + ).toBeUndefined(); + f.authority.profileGrants = []; + expect(await makeSessionBindingResolver(f.deps).resolveBoundTool(input)).toBeUndefined(); + f.authority.profileGrants = [bindingGrant]; + f.rows.get("child")!.parentSessionId = null; + expect(await makeSessionBindingResolver(f.deps).resolveBoundTool(input)).toBeUndefined(); +}); +test("two existing sessions resolve different catalogs without a global name registry", async () => { + const f = fixture(); + const other = bindingAuthority(); + other.profileId = "profile-b"; + other.connections[0]!.tools = []; + other.profileGrants = []; + const otherSnapshot = compileSessionBindings({ + launchPolicy: { profileId: other.profileId, integrationGrants: other.profileGrants }, + liveAuthority: other, + bindingEpoch: 0, + }); + f.rows.set("session-b", { snapshot: otherSnapshot, parentSessionId: null, active: true }); + f.deps.readAuthority = async (id) => (id === "profile-b" ? other : f.authority); + const resolver = makeSessionBindingResolver(f.deps), + name = f.snapshot.immutableBindings[0]!.tools[0]!.name; + expect( + await resolver.resolveBoundTool({ sessionId: "session-a", modelToolName: name }), + ).toBeDefined(); + expect( + await resolver.resolveBoundTool({ sessionId: "session-b", modelToolName: name }), + ).toBeUndefined(); +}); + +test("stored duplicate constraints retain their immutable binding until the actual profile changes", async () => { + const f = fixture(); + const constraint = { type: "object", properties: { name: { maxLength: 10 } } }; + f.authority.profileGrants[0]!.mcp = { + ...bindingGrant.mcp!, + unrestricted: false, + argumentConstraintDocuments: [constraint], + }; + const source = compileSessionBindings({ + launchPolicy: { + profileId: f.authority.profileId, + integrationGrants: f.authority.profileGrants, + }, + liveAuthority: f.authority, + bindingEpoch: 0, + }); + // Stored bindings can contain repeated whole schema roots. + const binding = source.immutableBindings[0]!; + binding.tools[0]!.grant.argumentConstraintDocuments = [constraint, structuredClone(constraint)]; + binding.policyHash = canonicalDigest([ + { ...f.authority.profileGrants[0]!, mcp: binding.tools[0]!.grant }, + ]); + const { hash: _hash, ...bindingBody } = binding; + binding.hash = canonicalDigest(bindingBody); + const { bindingHash: _bindingHash, ...body } = source; + source.bindingHash = canonicalDigest(body); + const columns = bindingSnapshotColumns(source, [ + { + id: binding.connectionId, + alias: binding.alias, + provider: "mcp", + displayName: "Tools", + config: binding.config, + }, + ]); + const stored = restoreBindingSnapshot(columns.mcpBindingEnvelope, columns.integrationConnections); + const original = structuredClone(stored); + f.rows.set("session-a", { snapshot: stored, parentSessionId: null, active: true }); + const resolver = makeSessionBindingResolver(f.deps); + const input = { sessionId: "session-a", modelToolName: binding.tools[0]!.name }; + const resolved = await resolver.resolveBoundTool(input); + expect(resolved?.bindingHash).toBe(original.bindingHash); + expect(resolved?.effectiveGrant.argumentConstraintDocuments).toEqual([constraint, constraint]); + expect(await resolver.readCurrent("session-a")).toEqual(original); + expect(stored).toEqual(original); + f.authority.profileGrants[0]!.mcp!.argumentConstraintDocuments = [ + { type: "object", properties: { name: { maxLength: 5 } } }, + ]; + expect(await resolver.resolveBoundTool(input)).toBeUndefined(); + expect(stored).toEqual(original); +}); + +test("every ACK identity field and parent binding replacement is fenced", async () => { + const f = fixture(), + name = f.snapshot.immutableBindings[0]!.tools[0]!.name; + const input = { sessionId: "session-a", modelToolName: name }; + for (const field of ["sessionId", "bindingHash", "denyDigest"] as const) { + const deps = { + ...f.deps, + enforceManagedDeny: async ( + expected: import("../mcp/session-resolver.ts").ManagedDenyAck, + ) => ({ ...expected, [field]: "changed" }), + }; + expect(await makeSessionBindingResolver(deps).resolveBoundTool(input)).toBeUndefined(); + } + for (const field of ["bindingEpoch", "denyRevision"] as const) { + const deps = { + ...f.deps, + enforceManagedDeny: async ( + expected: import("../mcp/session-resolver.ts").ManagedDenyAck, + ) => ({ ...expected, [field]: 1 }), + }; + expect(await makeSessionBindingResolver(deps).resolveBoundTool(input)).toBeUndefined(); + } + const child = compileSessionBindings({ + launchPolicy: { + profileId: f.authority.profileId, + integrationGrants: f.authority.profileGrants, + }, + liveAuthority: f.authority, + bindingEpoch: 0, + requestedGrants: f.authority.profileGrants, + parentAuthority: { sessionId: "session-a", snapshot: f.snapshot }, + }); + f.rows.set("child", { snapshot: child, parentSessionId: "session-a", active: true }); + f.rows.get("session-a")!.snapshot = compileSessionBindings({ + launchPolicy: { + profileId: f.authority.profileId, + integrationGrants: f.authority.profileGrants, + }, + liveAuthority: f.authority, + bindingEpoch: 0, + denyRevision: 1, + }); + expect( + await makeSessionBindingResolver(f.deps).resolveBoundTool({ + sessionId: "child", + modelToolName: name, + }), + ).toBeUndefined(); +}); diff --git a/orchestrator/src/__tests__/mcp-startup-recovery.test.ts b/orchestrator/src/__tests__/mcp-startup-recovery.test.ts new file mode 100644 index 000000000..49ca0dc33 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-startup-recovery.test.ts @@ -0,0 +1,94 @@ +import { expect, test } from "bun:test"; +import { eq } from "drizzle-orm"; +import { task, user } from "../db/schema.ts"; +import { coldFixture } from "./mcp-cold-fixture.ts"; +import { makeMcpRuntime } from "../mcp/runtime.ts"; +import { makeMcpRecoveryScanner } from "../mcp/recovery-scanner.ts"; +import { effectFixture } from "./mcp-effect-fixture.ts"; +import { operatorPublication } from "./mcp-binding-fixture.ts"; +import { verifyMcpTestDatabases } from "./helpers/mcp-test-isolation.ts"; +await verifyMcpTestDatabases(); + +test("actual startup composition retains committed completion through missing and disabled configuration", async () => { + const f = await effectFixture({ classification: "read" }); + try { + const call = await f.driver.acceptToolRequest(f.request); + expect((await f.driver.driveOnce(call.id)).state).toBe("succeeded"); + let delivered = 0, + fail = true, + bridgeConstructed = 0; + const common = { + db: f.db, + deploymentId: crypto.randomUUID(), + enabled: () => false, + bridge: () => { + bridgeConstructed++; + return { + readManagedCreationState: async () => { + throw Error("unexpected creation read"); + }, + prepareManagedBoot: async () => { + throw Error("unexpected prepare"); + }, + recoverManagedBoot: async () => { + throw Error("unexpected recovery"); + }, + }; + }, + completer: { + async completeToolCall(input: { resultJson: string }) { + if (fail) throw Error("reply unavailable"); + expect(JSON.parse(input.resultJson).outcome).toBe("succeeded"); + delivered++; + }, + }, + }; + let runtime = makeMcpRuntime(common); + expect(runtime.services.enabled).toBe(false); + expect(runtime.capabilities).toBeUndefined(); + expect(bridgeConstructed).toBe(0); + const scanner = makeMcpRecoveryScanner({ + deploymentId: common.deploymentId, + journal: runtime.services.journal, + runtime: () => runtime.recovery, + startWorkflow: async (marker) => { + if (marker.sessionId === f.sessionId) await runtime.recovery.runOnce(marker); + }, + }); + await expect(scanner.runOnce()).rejects.toThrow("completion unavailable"); + expect((await f.journal.get(call.id))!.completionEnqueuedVersion).toBe(0); + runtime = makeMcpRuntime({ + ...common, + configuration: { + infrastructure: operatorPublication, + control: f.remote.workerTls, + workers: new Map([ + ["public", { enabled: false, deploymentId: f.deploymentId, tls: f.remote.workerTls }], + ]), + registrations: [], + }, + }); + expect(runtime.services.enabled).toBe(false); + expect(runtime.capabilities).toBeDefined(); + expect(bridgeConstructed).toBe(1); + fail = false; + await scanner.runOnce(); + expect(delivered).toBe(1); + await expect(runtime.recovery.accept(f.request)).rejects.toMatchObject({ code: "disabled" }); + expect((await f.command("stats")).calls).toBe(1); + await scanner.stop(); + const cold = await coldFixture(f, f.operatorPolicies); + if (!runtime.validate) throw Error("configured runtime validation missing"); + await f.db.update(user).set({ banned: true }).where(eq(user.id, f.actorId)); + await expect(runtime.validate(cold.snapshot, f.sessionId)).rejects.toThrow( + "creation principal unavailable", + ); + await f.db.update(user).set({ banned: false }).where(eq(user.id, f.actorId)); + await f.db.update(task).set({ createdByUserId: null }).where(eq(task.id, f.taskId)); + await expect(runtime.validate(cold.snapshot, f.sessionId)).rejects.toThrow( + "creation principal unavailable", + ); + } finally { + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-startup.test.ts b/orchestrator/src/__tests__/mcp-startup.test.ts new file mode 100644 index 000000000..7d0f62a4f --- /dev/null +++ b/orchestrator/src/__tests__/mcp-startup.test.ts @@ -0,0 +1,185 @@ +import { expect, spyOn, test } from "bun:test"; +import { Code, ConnectError } from "@connectrpc/connect"; +import { mcpStartupSettings } from "../mcp/runtime-config.ts"; +import { makeManagedSessionScanner } from "../mcp/managed-scanner.ts"; +import { log } from "../log.ts"; + +test("MCP startup requires explicit global opt-in and retains configured recovery when disabled", () => { + expect(mcpStartupSettings({})).toEqual({ enabled: false, path: undefined }); + expect(mcpStartupSettings({ MCP_RUNTIME_CONFIG: "/operator/runtime.json" })).toEqual({ + enabled: false, + path: "/operator/runtime.json", + }); + expect( + mcpStartupSettings({ MCP_ENABLED: "true", MCP_RUNTIME_CONFIG: "/operator/runtime.json" }) + .enabled, + ).toBe(true); + expect(() => mcpStartupSettings({ MCP_ENABLED: "true" })).toThrow("configuration required"); + expect(() => mcpStartupSettings({ MCP_ENABLED: "1" })).toThrow("true or false"); +}); + +test("managed scanner advances past a failed row and wraps its durable keyset cursor", async () => { + const reads: (string | undefined)[] = [], + driven: string[] = []; + const scanner = makeManagedSessionScanner({ + store: { + async pending(after) { + reads.push(after); + return after === undefined ? [{ sessionId: "a" }, { sessionId: "b" }] : []; + }, + }, + runtime: { + async runOnce(id) { + driven.push(id); + if (id === "a") throw Error("unavailable"); + }, + }, + }); + await scanner.runOnce(); + await scanner.runOnce(); + await scanner.runOnce(); + expect(reads).toEqual([undefined, "b", undefined]); + expect(driven).toEqual(["a", "b", "a", "b"]); + await scanner.stop(); +}); + +test("managed scanner shutdown waits for its current reconciliation", async () => { + let release!: () => void, enter!: () => void; + const held = new Promise((resolve) => { + release = resolve; + }); + const entered = new Promise((resolve) => { + enter = resolve; + }); + const driven: string[] = []; + const scanner = makeManagedSessionScanner({ + store: { + async pending() { + return [{ sessionId: "pending" }, { sessionId: "later" }]; + }, + }, + runtime: { + async runOnce(id) { + driven.push(id); + enter(); + await held; + }, + }, + }); + const started = scanner.start(); + await entered; + let settled = false; + const stopped = scanner.stop().then(() => { + settled = true; + }); + await Promise.resolve(); + expect(settled).toBe(false); + release(); + await started; + await stopped; + expect(settled).toBe(true); + expect(driven).toEqual(["pending"]); +}); + +test.each([ + ["MCP prepared receipt unavailable", "prepared_receipt_unavailable"], + ["MCP boot publication changed", "boot_publication_changed"], + ["MCP finalization reservation changed", "finalization_reservation_changed"], +])("managed scanner reports the fixed reason for %s", async (message, reason) => { + const warning = spyOn(log, "warn").mockImplementation(() => {}); + const reads: (string | undefined)[] = [], + driven: string[] = []; + const scanner = makeManagedSessionScanner({ + store: { + async pending(after) { + reads.push(after); + return after === undefined ? [{ sessionId: "pending" }, { sessionId: "later" }] : []; + }, + }, + runtime: { + async runOnce(id) { + driven.push(id); + if (id === "pending") + throw new Error(message, { cause: { authorization: "Bearer test-secret" } }); + }, + }, + }); + try { + await scanner.runOnce(); + await scanner.runOnce(); + await scanner.runOnce(); + expect(reads).toEqual([undefined, "later", undefined]); + expect(driven).toEqual(["pending", "later", "pending", "later"]); + expect(warning.mock.calls).toEqual([ + [{ reason }, "MCP managed session reconciliation unavailable"], + [{ reason }, "MCP managed session reconciliation unavailable"], + ]); + } finally { + warning.mockRestore(); + } +}); + +test.each([ + new Error("Authorization: Bearer test-secret https://example.test/?token=test-secret"), + new Error("MCP prepared receipt unavailable: Bearer test-secret"), + new Error("wrapped", { cause: new Error("MCP prepared receipt unavailable") }), + { message: "MCP prepared receipt unavailable", authorization: "Bearer test-secret" }, + "MCP prepared receipt unavailable", + null, +])("managed scanner keeps unknown failures unclassified without error data: %#", async (error) => { + const warning = spyOn(log, "warn").mockImplementation(() => {}); + const scanner = makeManagedSessionScanner({ + store: { pending: async () => [{ sessionId: "pending" }] }, + runtime: { + async runOnce() { + throw error; + }, + }, + }); + try { + await scanner.runOnce(); + expect(warning.mock.calls).toEqual([ + [{ reason: "unclassified" }, "MCP managed session reconciliation unavailable"], + ]); + } finally { + warning.mockRestore(); + } +}); + +test.each([ + [Code.FailedPrecondition, "rpc_failed_precondition"], + [Code.Unavailable, "rpc_unavailable"], + [Code.Unauthenticated, "rpc_unauthenticated"], + [0, "unclassified"], + [17, "unclassified"], + [NaN, "unclassified"], + [1.5, "unclassified"], +])("managed scanner logs only a fixed RPC reason for code %s", async (code, reason) => { + const error = new ConnectError( + "Bearer test-secret https://example.test/?token=test-secret", + Code.Unknown, + { authorization: "Bearer test-secret" }, + ); + // The wire boundary can supply an invalid numeric code despite the declared enum type. + Object.defineProperty(error, "code", { value: code }); + const warning = spyOn(log, "warn").mockImplementation(() => {}); + const driven: string[] = []; + const scanner = makeManagedSessionScanner({ + store: { pending: async () => [{ sessionId: "pending" }, { sessionId: "later" }] }, + runtime: { + async runOnce(id) { + driven.push(id); + if (id === "pending") throw error; + }, + }, + }); + try { + await scanner.runOnce(); + expect(driven).toEqual(["pending", "later"]); + expect(warning.mock.calls).toEqual([ + [{ reason }, "MCP managed session reconciliation unavailable"], + ]); + } finally { + warning.mockRestore(); + } +}); diff --git a/orchestrator/src/__tests__/mcp-store.test.ts b/orchestrator/src/__tests__/mcp-store.test.ts new file mode 100644 index 000000000..aad25986d --- /dev/null +++ b/orchestrator/src/__tests__/mcp-store.test.ts @@ -0,0 +1,535 @@ +import { unknownContext } from "../mcp/journal.ts"; +import { describe, expect, test } from "bun:test"; +import { eq, sql } from "drizzle-orm"; +import { checkDb } from "../db/client.ts"; +import { mcpAdmission, mcpApproval, task, taskSession } from "../db/schema.ts"; +import { schemaVerifiedFixture as reviewFixture } from "./mcp-review-fixture.ts"; +import { bindingSnapshotColumns } from "../mcp/binding-store.ts"; +import { reviewContext } from "../mcp/journal-review.ts"; +import { type McpCallAuthorization, makeMcpJournal } from "../mcp/journal.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { makeCredentialMetadataStore } from "../mcp/credential-store.ts"; + +const reachable = process.env.ORCHESTRATOR_DATABASE_URL ? await checkDb() : false; +const tool = { + upstreamName: "Write.Record", + description: "Write a record", + inputSchema: { type: "object" }, +}; +const config = { + endpoint: "https://tools.example.test/mcp", + zone: "public", + auth: { kind: "api_key", secretRef: "mcp.service.api-key" }, +}; +// These adapters supply trusted producer fixtures only. They do not prove host or runtime enforcement. +async function fixture(deploymentId?: string) { + const f = await reviewFixture([], 1, deploymentId); + const base = f.journal; + const input = { ...f.input, deadline: new Date(f.now.getTime() + 60_000) }; + const actor = (id: string) => (id === "owner" ? f.actorId : id); + const journal = { + ...base, + async create(value: typeof input, authorization: McpCallAuthorization, now: Date) { + const { sessionId, runId, toolCallId } = value; + // Additional sessions in capacity tests use the same persisted fixture authority. + await f.db + .update(taskSession) + .set( + bindingSnapshotColumns(f.snapshot, [ + { + id: f.connection.id, + alias: f.connection.alias, + provider: "mcp", + displayName: f.connection.displayName, + config: f.connection.config, + }, + ]), + ) + .where(eq(taskSession.sessionId, sessionId)); + return base.create( + { + ...value, + requestPolicyContext: { + ...value.requestPolicyContext, + sessionId, + runId, + toolCallId, + acceptedPolicyRef: { ...value.requestPolicyContext.acceptedPolicyRef, sessionId }, + }, + }, + authorization, + now, + ); + }, + pending: (sid: string, id: string, _cachedAdmin: boolean, now: Date) => + base.pending(sid, actor(id), now), + async decide(id: string, who: string, _cachedAdmin: boolean, approve: boolean, now: Date) { + const call = (await base.get(id))!; + const [item] = await f.db.select().from(mcpApproval).where(eq(mcpApproval.executionId, id)); + return base.decide(id, actor(who), approve, reviewContext(call, item!), now); + }, + async dispatch( + id: string, + intent: { bootId: string; intentId: string; expiresAt: Date }, + now: Date, + ) { + const call = (await base.get(id))!; + return base.dispatch(id, intent, now, { + executionId: id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: call.version, + reservationId: (await base.getAdmission(id))!.reservationId, + reservationExpiresAt: (await base.getAdmission(id))!.expiresAt, + }); + }, + cancel: async (id: string, who: string, _cachedAdmin: boolean, now: Date) => + base.cancel(id, actor(who), now, (await base.get(id))!.version), + resolveUnknown: async ( + id: string, + who: string, + _cachedAdmin: boolean, + outcome: "succeeded" | "failed" | "acknowledged", + now: Date, + ) => base.resolveUnknown(id, actor(who), outcome, now, unknownContext((await base.get(id))!)), + }; + return { ...f, input, journal, row: f.connection, repo: f.repository, published: f.catalog }; +} + +describe("MCP durable authority", () => { + test.skipIf(!reachable)("publish CAS has one winner and refuses a stale discovery", async () => { + const f = await fixture(); + try { + const outcomes = await Promise.allSettled( + [1, 2].map(() => + f.repo.publish( + f.row.id, + f.draft.id, + 1, + [{ ...tool, classification: "write", enabled: true }], + f.publisherId, + f.now, + ), + ), + ); + expect(outcomes.filter((o) => o.status === "fulfilled")).toHaveLength(1); + const credentials = makeCredentialMetadataStore(f.db); + const claim = await credentials.claim( + f.row.id, + 1, + f.input.authGeneration, + "admin", + config, + f.now, + ); + await credentials.publish(claim, config, undefined, f.now); + await expect( + f.repo.publish( + f.row.id, + f.draft.id, + 2, + [{ ...tool, classification: "write", enabled: true }], + f.publisherId, + f.now, + ), + ).rejects.toThrow("stale"); + expect(await f.connections.markTested(f.row.id, f.now)).toBeNull(); + } finally { + await f.cleanup(); + } + }); + test.skipIf(!reachable)( + "call digest, approval and dispatch are single-use under concurrency", + async () => { + const f = await fixture(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + expect((await f.journal.create(f.input, { kind: "approval" }, f.now)).id).toBe(call.id); + await expect( + f.journal.create( + { ...f.input, argumentHash: canonicalDigest({ x: 2 }) }, + { kind: "approval" }, + f.now, + ), + ).rejects.toThrow("digest"); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + await expect(f.journal.decide(call.id, "intruder", false, true, f.now)).rejects.toThrow( + "owner", + ); + await f.journal.decide(call.id, "owner", false, true, f.now); + await f.journal.admit(call.id, f.deploymentId, f.now); + const results = await Promise.allSettled( + [1, 2].map((n) => + f.journal.dispatch( + call.id, + { + bootId: "boot-1", + intentId: `intent-${call.id}-${n}`, + expiresAt: new Date(f.now.getTime() + 10_000), + }, + f.now, + ), + ), + ); + expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1); + const current = (await f.journal.get(call.id))!; + expect(current.state).toBe("dispatching"); + expect((await f.journal.getAdmission(call.id))?.state).toBe("admitted"); + await f.journal.finish(call.id, current.version, "outcome_unknown", null, false, f.now); + await expect( + f.journal.create({ ...f.input, toolCallId: "call-2" }, { kind: "approval" }, f.now), + ).rejects.toThrow("unknown"); + // The original request remains readable and cannot become a second dispatch. + expect((await f.journal.create(f.input, { kind: "approval" }, f.now)).id).toBe(call.id); + await f.journal.resolveUnknown(call.id, "owner", false, "acknowledged", f.now); + expect(await f.journal.hasUnknown(f.input.sessionId, f.row.id)).toBe(false); + await expect( + Promise.resolve( + f.db.execute( + sql`UPDATE mcp_execution SET request_digest = 'changed' WHERE id = ${call.id}`, + ), + ), + ).rejects.toThrow(); + } finally { + await f.cleanup(); + } + }, + ); + test.skipIf(!reachable)( + "distributed admission bounds session concurrency and expires pending approvals", + async () => { + const f = await fixture(); + try { + const calls = await Promise.all( + Array.from({ length: 6 }, (_, n) => + f.journal.create( + { ...f.input, toolCallId: `call-${n}` }, + { kind: "automation", grant: f.input.resolvedTool.effectiveGrant }, + f.now, + ), + ), + ); + const pending = await f.journal.create( + { ...f.input, toolCallId: "approval" }, + { kind: "approval" }, + f.now, + ); + const statuses = await Promise.all( + calls.map((c) => f.journal.admit(c.id, f.deploymentId, f.now)), + ); + expect(statuses.filter((s) => s === "admitted")).toHaveLength(4); + expect(statuses.filter((s) => s === "queued")).toHaveLength(2); + await f.journal.requestApproval(pending.id, f.deploymentId, f.now); + await f.journal.expire(f.deploymentId, new Date(f.now.getTime() + 601_000)); + expect((await f.journal.get(pending.id))!.state).toBe("canceled"); + expect(await f.journal.pending(f.input.sessionId, "owner", false, f.now)).toHaveLength(0); + } finally { + await f.cleanup(); + } + }, + ); +}); + +describe("MCP authority isolation", () => { + test.skipIf(!reachable)( + "rejects cross-connection catalog and unapproved write transitions", + async () => { + const first = await fixture(); + const second = await fixture(); + try { + await expect( + first.journal.create(first.input, { kind: "read" }, first.now), + ).rejects.toThrow("write requires"); + await expect( + first.journal.create( + first.input, + { + kind: "automation", + grant: { + upstreamName: tool.upstreamName, + automationWrite: false, + argumentConstraintDocuments: [], + unrestricted: true, + }, + }, + first.now, + ), + ).rejects.toThrow("explicit automation"); + await expect( + first.journal.create( + { ...first.input, catalogId: second.published.id }, + { kind: "approval" }, + first.now, + ), + ).rejects.toThrow("identity"); + const call = await first.journal.create(first.input, { kind: "approval" }, first.now); + await expect( + Promise.resolve( + first.db.execute( + sql`UPDATE mcp_execution SET state = 'succeeded', version = version + 1 WHERE id = ${call.id}`, + ), + ), + ).rejects.toThrow(); + await expect( + Promise.resolve( + first.db.execute( + sql`UPDATE mcp_catalog SET tools = '[]'::jsonb WHERE id = ${first.published.id}`, + ), + ), + ).rejects.toThrow(); + await expect( + first.journal.pending(first.input.sessionId, "intruder", false, first.now), + ).rejects.toThrow("owner"); + } finally { + await first.cleanup(); + await second.cleanup(); + } + }, + ); + test.skipIf(!reachable)( + "pending approvals are owner-scoped and expired approvals cannot dispatch", + async () => { + const f = await fixture(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + await expect( + f.journal.pending(f.input.sessionId, "intruder", false, f.now), + ).rejects.toThrow("owner"); + expect(await f.journal.pending(f.input.sessionId, "owner", false, f.now)).toHaveLength(1); + await f.journal.decide(call.id, "owner", false, true, f.now); + await f.journal.admit(call.id, f.deploymentId, f.now); + const later = new Date(f.now.getTime() + 61_000); + await expect( + f.journal.dispatch( + call.id, + { + bootId: "boot", + intentId: crypto.randomUUID(), + expiresAt: new Date(later.getTime() + 1000), + }, + later, + ), + ).rejects.toThrow("cannot dispatch"); + await f.journal.expire(f.deploymentId, later); + expect((await f.journal.get(call.id))!.state).toBe("canceled"); + } finally { + await f.cleanup(); + } + }, + ); + test.skipIf(!reachable)( + "an expired dispatch becomes unknown and accepts one verified late result", + async () => { + const f = await fixture(); + try { + const call = await f.journal.create(f.input, { kind: "approval" }, f.now); + await f.journal.requestApproval(call.id, f.deploymentId, f.now); + await f.journal.decide(call.id, "owner", false, true, f.now); + await f.journal.admit(call.id, f.deploymentId, f.now); + await f.journal.dispatch( + call.id, + { + bootId: "boot", + intentId: crypto.randomUUID(), + expiresAt: new Date(f.now.getTime() + 1000), + }, + f.now, + ); + await f.journal.expire(f.deploymentId, new Date(f.now.getTime() + 61_000)); + const unknown = (await f.journal.get(call.id))!; + expect(unknown.state).toBe("outcome_unknown"); + expect(await f.journal.hasUnknown(f.input.sessionId, f.row.id)).toBe(true); + const late = await f.journal.finish( + call.id, + unknown.version, + "succeeded", + null, + true, + new Date(f.now.getTime() + 62_000), + ); + expect(late.resultUnavailable).toBe(true); + expect(await f.journal.hasUnknown(f.input.sessionId, f.row.id)).toBe(false); + await expect( + f.journal.finish(call.id, unknown.version, "failed", null, false, f.now), + ).rejects.toThrow("stale"); + } finally { + await f.cleanup(); + } + }, + ); +}); + +test.skipIf(!reachable)( + "approval has a ten-minute window separate from the tool deadline", + async () => { + const f = await fixture(); + try { + const call = await f.journal.create( + { ...f.input, deadline: new Date(f.now.getTime() + 690_000) }, + { kind: "approval" }, + f.now, + ); + const approval = await f.journal.requestApproval(call.id, f.deploymentId, f.now); + expect(approval.expiresAt.getTime() - f.now.getTime()).toBe(600_000); + const afterFiveMinutes = new Date(f.now.getTime() + 300_000); + await f.journal.decide(call.id, "owner", false, true, afterFiveMinutes); + await f.journal.admit(call.id, f.deploymentId, afterFiveMinutes); + const dispatch = await f.journal.dispatch( + call.id, + { + bootId: "boot", + intentId: crypto.randomUUID(), + expiresAt: new Date(afterFiveMinutes.getTime() + 1000), + }, + afterFiveMinutes, + ); + expect(dispatch.state).toBe("dispatching"); + await f.journal.expire(f.deploymentId, new Date(afterFiveMinutes.getTime() + 61_000)); + expect((await f.journal.get(call.id))!.state).toBe("outcome_unknown"); + } finally { + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)("replicas share connection and deployment admission ceilings", async () => { + const fixtures: Awaited>[] = []; + const auth = { + kind: "automation" as const, + grant: { + upstreamName: tool.upstreamName, + automationWrite: true, + argumentConstraintDocuments: [], + unrestricted: true, + }, + }; + try { + const sharedDeployment = crypto.randomUUID(); + for (let i = 0; i < 9; i++) fixtures.push(await fixture(sharedDeployment)); + const first = fixtures[0]!; + const allCalls = await Promise.all( + fixtures.flatMap((f) => + Array.from({ length: 4 }, (_, i) => + f.journal.create({ ...f.input, toolCallId: `limit-${i}` }, auth, f.now), + ), + ), + ); + const states = await Promise.all( + allCalls.map((c) => makeMcpJournal(first.db).admit(c.id, first.deploymentId, first.now)), + ); + expect(states.filter((state) => state === "admitted")).toHaveLength(32); + expect(states.filter((state) => state === "queued")).toHaveLength(4); + // Clear the deployment test through normal task retention. + for (const f of fixtures) await f.db.delete(task).where(eq(task.id, f.input.taskId)); + await first.db + .insert(task) + .values({ id: first.input.taskId, type: "chat", createdByUserId: first.actorId }); + const sessions = [first.input.sessionId, crypto.randomUUID(), crypto.randomUUID()]; + for (const sessionId of sessions) + await first.db.insert(taskSession).values({ taskId: first.input.taskId, sessionId }); + const connectionCalls = await Promise.all( + sessions.flatMap((sessionId) => + Array.from({ length: 4 }, (_, i) => + first.journal.create( + { ...first.input, sessionId, toolCallId: `connection-${i}` }, + auth, + first.now, + ), + ), + ), + ); + const connectionStates = await Promise.all( + connectionCalls.map((c) => + makeMcpJournal(first.db).admit(c.id, first.deploymentId, first.now), + ), + ); + expect(connectionStates.filter((state) => state === "admitted")).toHaveLength(8); + expect(connectionStates.filter((state) => state === "queued")).toHaveLength(4); + } finally { + // Session-owned call payloads and child rows use the existing task retention path. + for (const f of fixtures) await f.db.delete(task).where(eq(task.id, f.input.taskId)); + for (const f of fixtures) await f.cleanup(); + } +}); + +test.skipIf(!reachable)( + "recovers calls after a crash before approval or admission is recorded", + async () => { + const f = await fixture(); + try { + const waiting = await f.journal.create(f.input, { kind: "approval" }, f.now); + const ready = await f.journal.create( + { ...f.input, toolCallId: "before-admission" }, + { + kind: "automation", + grant: { + upstreamName: tool.upstreamName, + automationWrite: true, + argumentConstraintDocuments: [], + unrestricted: true, + }, + }, + f.now, + ); + const count = await f.journal.expireUnadmitted(new Date(f.now.getTime() + 61_000)); + expect(count).toBeGreaterThanOrEqual(2); + expect((await f.journal.get(waiting.id))!.state).toBe("canceled"); + expect((await f.journal.get(ready.id))!.state).toBe("canceled"); + } finally { + await f.cleanup(); + } + }, +); + +test.skipIf(!reachable)( + "admission retries preserve queue expiry and promotion starts a tool window", + async () => { + const f = await fixture(); + const authorization = { + kind: "automation" as const, + grant: { + upstreamName: tool.upstreamName, + automationWrite: true, + argumentConstraintDocuments: [], + unrestricted: true, + }, + }; + const at = (seconds: number) => new Date(f.now.getTime() + seconds * 1000); + try { + const calls = await Promise.all( + Array.from({ length: 6 }, (_, i) => + f.journal.create( + { ...f.input, toolCallId: `fixed-queue-${i}`, deadline: at(300) }, + authorization, + f.now, + ), + ), + ); + for (const call of calls.slice(0, 4)) + expect(await f.journal.admit(call.id, f.deploymentId, f.now)).toBe("admitted"); + for (const call of calls.slice(4)) { + expect(await f.journal.admit(call.id, f.deploymentId, f.now)).toBe("queued"); + expect(await f.journal.admit(call.id, f.deploymentId, at(20))).toBe("queued"); + const [lease] = await f.db + .select() + .from(mcpAdmission) + .where(eq(mcpAdmission.executionId, call.id)); + expect(lease!.expiresAt).toEqual(at(30)); + } + await f.journal.cancel(calls[0]!.id, "owner", false, at(25)); + expect(await f.journal.admit(calls[4]!.id, f.deploymentId, at(25))).toBe("admitted"); + const [promoted] = await f.db + .select() + .from(mcpAdmission) + .where(eq(mcpAdmission.executionId, calls[4]!.id)); + expect(promoted!.expiresAt).toEqual(at(85)); + await f.journal.expire(f.deploymentId, at(31)); + expect((await f.journal.get(calls[5]!.id))!.state).toBe("canceled"); + expect((await f.journal.get(calls[4]!.id))!.state).toBe("authorized"); + } finally { + await f.cleanup(); + } + }, +); diff --git a/orchestrator/src/__tests__/mcp-unknown-drain.test.ts b/orchestrator/src/__tests__/mcp-unknown-drain.test.ts new file mode 100644 index 000000000..9330883cd --- /dev/null +++ b/orchestrator/src/__tests__/mcp-unknown-drain.test.ts @@ -0,0 +1,64 @@ +import { expect, test } from "bun:test"; +import { reviewContext } from "../mcp/journal-review.ts"; +import { effectFixture } from "./mcp-effect-fixture.ts"; + +test("manual resolution cannot authorize a new write while the old intent can send", async () => { + const f = await effectFixture({ classification: "write" }); + let unblock!: () => void; + const blocked = new Promise((resolve) => { + unblock = resolve; + }); + let notify!: () => void; + const entered = new Promise((resolve) => { + notify = resolve; + }); + try { + const first = await f.driver.acceptToolRequest(f.request); + await f.driver.driveOnce(first.id); + const approval = await f.journal.requestApproval(first.id, f.deploymentId, new Date()); + await f.journal.decide(first.id, f.actorId, true, reviewContext(first, approval), new Date()); + let fences = 0; + f.beforeFence(async () => { + if (f.leases > 0 && ++fences === 2) { + notify(); + await blocked; + } + }); + const pending = f.driver.driveOnce(first.id); + await entered; + const earlierRequest = { ...f.request, toolCallId: "already-accepted" }; + f.requests.set(earlierRequest.toolCallId, earlierRequest); + const earlier = await f.driver.acceptToolRequest(earlierRequest); + const dispatched = (await f.journal.get(first.id))!; + const unknown = await f.journal.cancel(first.id, f.actorId, new Date(), dispatched.version); + const context = { + executionId: first.id, + journalRevision: unknown.version, + requestDigest: unknown.requestDigest, + bindingHash: unknown.bindingHash, + intentId: unknown.intentId!, + }; + await f.journal.resolveUnknown(first.id, f.actorId, "acknowledged", new Date(), context); + const earlierAttempt = await f.driver.driveOnce(earlier.id).then( + () => "accepted", + () => "blocked", + ); + const next = { ...f.request, toolCallId: "new-call" }; + f.requests.set(next.toolCallId, next); + const attempted = await f.driver.acceptToolRequest(next).then( + () => "accepted", + () => "blocked", + ); + f.advanceNow(310_000); + unblock(); + await pending; + expect(attempted).toBe("blocked"); + expect(earlierAttempt).toBe("blocked"); + expect((await f.command("stats")).calls).toBe(0); + const accepted = await f.driver.acceptToolRequest(next); + expect(accepted.toolCallId).toBe("new-call"); + } finally { + unblock(); + await f.cleanup(); + } +}, 20000); diff --git a/orchestrator/src/__tests__/mcp-worker-client.test.ts b/orchestrator/src/__tests__/mcp-worker-client.test.ts new file mode 100644 index 000000000..a7da0d8a2 --- /dev/null +++ b/orchestrator/src/__tests__/mcp-worker-client.test.ts @@ -0,0 +1,540 @@ +import { expect, test } from "bun:test"; +import { spawn } from "node:child_process"; +import { createInterface } from "node:readline"; +import { createServer } from "node:https"; +import type { IncomingMessage, ServerResponse } from "node:http"; +import { once } from "node:events"; +import { workerTls } from "../../../mcp-worker/test/fixtures/worker-tls.ts"; +import { digest } from "../../../mcp-worker/src/index.ts"; +import type { + WorkerIdentity, + WorkerOperation, + WorkerRequest, +} from "../../../mcp-worker/src/contracts.ts"; +import { makeResourceWorkerClient } from "../mcp/worker-client.ts"; +import { toWorkerBindingHash } from "../mcp/worker-encoding.ts"; + +function envelope(identity: WorkerIdentity, operation: WorkerOperation): WorkerRequest { + const expiresAt = Date.now() + 10000; + return { + protocol: 1, + intent: { + id: crypto.randomUUID(), + bootId: identity.bootId, + zoneId: identity.zoneId, + policyDigest: identity.policyDigest, + connectionId: "connection", + connectionRevision: 1, + credentialIdentity: "credential", + operation: operation.kind, + scope: { + kind: "execution", + sessionId: "session", + runId: "run", + toolCallId: "call", + bindingHash: toWorkerBindingHash(`sha256:${"b".repeat(64)}`), + }, + destination: "endpoint" in operation ? operation.endpoint : "", + requestDigest: `sha256:${"c".repeat(64)}`, + operationDigest: digest(operation), + expiresAt, + lease: { id: crypto.randomUUID(), deploymentId: "fixture-deployment", expiresAt }, + }, + operation, + }; +} +const schema: WorkerOperation = { + kind: "validateSchemas", + documents: [{ type: "integer" }], + value: 1, +}; +function options( + tls: Awaited>, + port: number, + identity: WorkerIdentity, +) { + return { + tls: { ...tls, url: `https://127.0.0.1:${port}`, serverFingerprint: tls.fingerprint }, + policy: { + zoneId: identity.zoneId, + policyRevision: identity.policyRevision, + policyDigest: `sha256:${identity.policyDigest}`, + }, + }; +} +async function actualWorker() { + const tls = await workerTls("localhost"); + const process = spawn( + globalThis.process.env.MCP_WORKER_TEST_NODE ?? "node", + [ + new URL("../../../mcp-worker/test/fixtures/resource-client-worker.ts", import.meta.url) + .pathname, + ], + { stdio: ["pipe", "pipe", "pipe"], env: { PATH: globalThis.process.env.PATH ?? "" } }, + ); + const output = createInterface({ input: process.stdout }); + const lines = output[Symbol.asyncIterator](); + let stderr = ""; + process.stderr.on("data", (data) => { + stderr += String(data); + }); + process.stdin.write(`${JSON.stringify(tls)}\n`); + async function next() { + const line = await lines.next(); + if (line.done) throw new Error(`worker fixture exited: ${stderr}`); + return JSON.parse(line.value); + } + const ready: { port: number; identity: WorkerIdentity; node: string } = await next(); + expect(ready.node.startsWith("24.")).toBe(true); + return { + ...ready, + tls, + async command(command: string) { + process.stdin.write(`${command}\n`); + return next(); + }, + async close() { + const exited = once(process, "exit"); + process.stdin.end("close\n"); + await exited; + output.close(); + expect(stderr).toBe(""); + }, + }; +} + +async function protocolFixture( + reply: (req: IncomingMessage, res: ServerResponse, body: WorkerRequest | undefined) => void, +) { + const tls = await workerTls("localhost"); + const identity: WorkerIdentity = { + protocol: 1, + bootId: "boot", + zoneId: "public", + policyRevision: "1", + policyDigest: "a".repeat(64), + }; + let rpcs = 0, + identities = 0, + bytes = 0; + const sockets = new Set(); + const server = createServer( + { ...tls, requestCert: true, rejectUnauthorized: true }, + (req, res) => { + sockets.add(req.socket); + if (req.url === "/identity") { + identities++; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify(identity)); + } else { + rpcs++; + let text = ""; + req.on("data", (chunk: Buffer) => { + bytes += chunk.length; + text += chunk; + }); + req.on("end", () => reply(req, res, text ? JSON.parse(text) : undefined)); + } + }, + ); + server.maxRequestsPerSocket = 2; + server.keepAliveTimeout = 35000; + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("test address"); + return { + identity, + client: (signal?: AbortSignal) => + makeResourceWorkerClient({ ...options(tls, address.port, identity), signal }), + counts: () => ({ rpcs, identities, bytes, sockets: sockets.size }), + async close() { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; +} +function success(identity: WorkerIdentity, request: WorkerRequest) { + return { + ok: true, + identity, + intent: request.intent, + intentId: request.intent.id, + requestDigest: request.intent.requestDigest, + operationDigest: request.intent.operationDigest, + leaseId: request.intent.lease.id, + result: true, + }; +} + +test("actual worker accepts execution-scoped disposable schemas on the pinned handshake channel", async () => { + const fixture = await actualWorker(); + const client = makeResourceWorkerClient(options(fixture.tls, fixture.port, fixture.identity)); + try { + expect(await client.identity()).toEqual(fixture.identity); + // The broker repeats the real identity check before acquiring a credential. + expect(await client.identity()).toEqual(fixture.identity); + const request = envelope(fixture.identity, schema); + const response = await client.dispatch(request); + expect(response.ok && response.result).toBe(true); + expect(response.ok && response.intent).toEqual(request.intent); + await expect(client.dispatch(request)).rejects.toThrow("MCP worker"); + expect((await fixture.command("stats")).providerDials).toBe(0); + } finally { + client.close(); + await fixture.close(); + } +}, 15000); + +test("a closed actual worker channel cannot send credentials to a replacement process", async () => { + const fixture = await actualWorker(); + const client = makeResourceWorkerClient(options(fixture.tls, fixture.port, fixture.identity)); + try { + await client.identity(); + await fixture.command("replace"); + const operation: WorkerOperation = { + kind: "execute", + endpoint: "https://tools.example/mcp", + tool: "write", + arguments: {}, + inputSchema: {}, + argumentConstraintDocuments: [], + classification: "write", + credential: { header: "authorization", value: "Bearer transient-provider-value" }, + }; + await expect(client.dispatch(envelope(fixture.identity, operation))).rejects.toThrow( + "MCP worker", + ); + expect(await fixture.command("stats")).toEqual({ + providerDials: 0, + replacementBytes: 0, + replacementRequests: 0, + }); + } finally { + client.close(); + await fixture.close(); + } +}, 15000); + +test("an actual worker restart changes boot and requires a new operation client", async () => { + const fixture = await actualWorker(); + const oldClient = makeResourceWorkerClient(options(fixture.tls, fixture.port, fixture.identity)); + const nextClient = makeResourceWorkerClient(options(fixture.tls, fixture.port, fixture.identity)); + try { + const oldIdentity = await oldClient.identity(); + const restarted: { identity: WorkerIdentity } = await fixture.command("restart"); + expect(restarted.identity.bootId).not.toBe(oldIdentity.bootId); + await expect(oldClient.dispatch(envelope(oldIdentity, schema))).rejects.toThrow("MCP worker"); + expect(await nextClient.identity()).toEqual(restarted.identity); + const response = await nextClient.dispatch(envelope(restarted.identity, schema)); + expect(response.ok && response.result).toBe(true); + expect((await fixture.command("stats")).providerDials).toBe(0); + } finally { + oldClient.close(); + nextClient.close(); + await fixture.close(); + } +}, 15000); + +test("actual schema children keep independent roots, reject coercion and do not dial providers", async () => { + const fixture = await actualWorker(); + try { + const documents = [ + { + $schema: "http://json-schema.org/draft-07/schema#", + $id: "https://schema.example/constraint", + definitions: { value: { type: "integer", maximum: 5 } }, + properties: { n: { $ref: "#/definitions/value" } }, + required: ["n"], + }, + { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: "https://schema.example/constraint", + $defs: { value: { type: "integer", minimum: 2 } }, + properties: { n: { $ref: "#/$defs/value" } }, + }, + ]; + for (const [value, expected] of [ + [{ n: 3 }, true], + [{ n: 1 }, false], + [{ n: 6 }, false], + [{ n: "3" }, false], + ] as const) { + const client = makeResourceWorkerClient(options(fixture.tls, fixture.port, fixture.identity)); + try { + const identity = await client.identity(); + const response = await client.dispatch( + envelope(identity, { kind: "validateSchemas", documents, value }), + ); + expect(response.ok && response.result).toBe(expected); + } finally { + client.close(); + } + } + expect((await fixture.command("stats")).providerDials).toBe(0); + } finally { + await fixture.close(); + } +}, 15000); + +test("separate operation clients have separate channels and never retry or reuse an RPC", async () => { + const fixture = await protocolFixture((_req, res, request) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify(success(fixture.identity, request!))); + }); + const clients = [fixture.client(), fixture.client()]; + try { + const identities = await Promise.all(clients.map((client) => client.identity())); + await Promise.all( + clients.map(async (client, index) => { + const request = envelope(identities[index]!, schema); + expect((await client.dispatch(request)).ok).toBe(true); + await expect(client.dispatch(request)).rejects.toThrow("MCP worker"); + await expect(client.identity()).rejects.toThrow("MCP worker"); + }), + ); + expect(fixture.counts().rpcs).toBe(2); + expect(fixture.counts().identities).toBe(2); + expect(fixture.counts().sockets).toBe(2); + } finally { + clients.forEach((client) => client.close()); + await fixture.close(); + } +}); + +test("response identity and full intent mismatches fail with sanitized errors and one send", async () => { + const changes = [ + (response: ReturnType) => ({ + ...response, + identity: { ...response.identity, bootId: "new-boot" }, + }), + (response: ReturnType) => ({ + ...response, + identity: { ...response.identity, policyRevision: "2" }, + }), + (response: ReturnType) => ({ + ...response, + intent: { ...response.intent, scope: { ...response.intent.scope, runId: "other" } }, + }), + (response: ReturnType) => ({ + ...response, + intent: { + ...response.intent, + lease: { ...response.intent.lease, expiresAt: response.intent.lease.expiresAt + 1 }, + }, + }), + (response: ReturnType) => ({ ...response, intentId: "other" }), + (response: ReturnType) => ({ ...response, operationDigest: "other" }), + (response: ReturnType) => ({ ...response, requestDigest: "other" }), + (response: ReturnType) => ({ ...response, leaseId: "other" }), + () => ({ ok: false, error: "Bearer private-value with upstream cause" }), + ]; + for (const change of changes) { + const fixture = await protocolFixture((_req, res, request) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify(change(success(fixture.identity, request!)))); + }); + const client = fixture.client(); + try { + await client.identity(); + await expect(client.dispatch(envelope(fixture.identity, schema))).rejects.toThrow( + /^MCP worker operation unavailable$/, + ); + expect(fixture.counts().rpcs).toBe(1); + } finally { + client.close(); + await fixture.close(); + } + } +}, 15000); + +test("redirect, oversized, truncated and invalid UTF-8 replies cannot trigger a retry", async () => { + for (const kind of ["redirect", "size", "truncated", "utf8"] as const) { + const fixture = await protocolFixture((_req, res) => { + if (kind === "redirect") { + res.writeHead(307, { location: "/identity" }); + res.end(); + } else if (kind === "size") { + res.writeHead(200, { "content-type": "application/json" }); + res.end("x".repeat(2 * 1024 * 1024 + 1)); + } else if (kind === "truncated") { + res.writeHead(200, { "content-type": "application/json", "content-length": 100 }); + res.end("{"); + } else { + res.writeHead(200, { "content-type": "application/json" }); + res.end(Buffer.from([0x22, 0xff, 0x22])); + } + }); + const client = fixture.client(); + try { + await client.identity(); + const request = envelope(fixture.identity, schema); + request.intent.expiresAt = Date.now() + 100; + request.intent.lease.expiresAt = request.intent.expiresAt; + await expect(client.dispatch(request)).rejects.toThrow(/^MCP worker operation unavailable$/); + expect(fixture.counts().rpcs).toBe(1); + expect(fixture.counts().identities).toBe(1); + } finally { + client.close(); + await fixture.close(); + } + } +}, 15000); + +test("abort ends an active RPC and concurrent calls cannot replace its channel", async () => { + let entered!: () => void; + const received = new Promise((resolve) => { + entered = resolve; + }); + const fixture = await protocolFixture(() => { + entered(); + }); + const controller = new AbortController(), + client = fixture.client(controller.signal); + try { + await client.identity(); + const request = envelope(fixture.identity, schema); + const pending = client.dispatch(request); + await received; + await expect(client.identity()).rejects.toThrow("MCP worker"); + await expect(client.dispatch(request)).rejects.toThrow("MCP worker"); + controller.abort(); + await expect(pending).rejects.toThrow(/^MCP worker operation unavailable$/); + expect(fixture.counts().rpcs).toBe(1); + } finally { + client.close(); + await fixture.close(); + } +}); + +test("changed boot, expired lease, invalid operation digest and oversized request send zero RPC bodies", async () => { + const fixture = await protocolFixture(() => { + throw new Error("unexpected RPC"); + }); + try { + for (const kind of ["boot", "lease", "digest", "size", "role"] as const) { + const client = fixture.client(); + try { + await client.identity(); + const request = envelope(fixture.identity, schema); + if (kind === "boot") request.intent.bootId = "other"; + else if (kind === "lease") request.intent.lease.expiresAt = Date.now() - 1; + else if (kind === "digest") request.intent.operationDigest = "wrong"; + else if (kind === "role") { + request.operation = { kind: "metadata", endpoint: "https://tools.example/metadata" }; + request.intent.operation = "metadata"; + request.intent.destination = request.operation.endpoint; + request.intent.scope = { kind: "administration", requestId: "request", actorId: "actor" }; + request.intent.operationDigest = digest(request.operation); + } else { + request.operation = { + kind: "validateSchemas", + documents: [{ description: "x".repeat(1536 * 1024) }], + }; + request.intent.operationDigest = digest(request.operation); + } + await expect(client.dispatch(request)).rejects.toThrow("MCP worker"); + } finally { + client.close(); + } + } + expect(fixture.counts().bytes).toBe(0); + expect(fixture.counts().rpcs).toBe(0); + } finally { + await fixture.close(); + } +}); + +test("TLS trust, caller identity, server pin and zone policy fail before any RPC", async () => { + const tls = await workerTls("localhost"), + other = await workerTls("localhost"); + const identity: WorkerIdentity = { + protocol: 1, + bootId: "boot", + zoneId: "public", + policyRevision: "1", + policyDigest: "a".repeat(64), + }; + let rpcs = 0; + const server = createServer( + { ...tls, requestCert: true, rejectUnauthorized: true }, + (req, res) => { + if (req.url !== "/identity") rpcs++; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify(identity)); + }, + ); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("test address"); + const good = options(tls, address.port, identity); + try { + for (const bad of [ + { ...good, tls: { ...good.tls, ca: other.ca } }, + { ...good, tls: { ...good.tls, cert: other.cert, key: other.key } }, + { ...good, tls: { ...good.tls, serverFingerprint: other.fingerprint } }, + { ...good, policy: { ...good.policy, zoneId: "other" } }, + { ...good, policy: { ...good.policy, policyRevision: "2" } }, + { ...good, policy: { ...good.policy, policyDigest: `sha256:${"b".repeat(64)}` } }, + ]) { + const client = makeResourceWorkerClient(bad); + try { + await expect(client.identity()).rejects.toThrow("MCP worker"); + } finally { + client.close(); + } + } + expect(() => makeResourceWorkerClient({ ...good, tls: { ...good.tls, key: "" } })).toThrow(); + expect(() => + makeResourceWorkerClient({ + ...good, + tls: { ...good.tls, url: good.tls.url.replace("https:", "http:") }, + }), + ).toThrow(); + for (const url of [ + "not a URL", + `${good.tls.url}/proxy`, + `${good.tls.url}?target=other`, + good.tls.url.replace("https://", "https://user:password@"), + ]) { + expect(() => makeResourceWorkerClient({ ...good, tls: { ...good.tls, url } })).toThrow( + /^MCP worker requires pinned mTLS and policy$/, + ); + } + expect(rpcs).toBe(0); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}, 15000); + +test("server certificate pin cannot replace hostname verification", async () => { + const tls = await workerTls(); // Valid, pinned certificate for tools.example, not 127.0.0.1. + const identity: WorkerIdentity = { + protocol: 1, + bootId: "boot", + zoneId: "public", + policyRevision: "1", + policyDigest: "a".repeat(64), + }; + let requests = 0; + const server = createServer( + { ...tls, requestCert: true, rejectUnauthorized: true }, + (_req, res) => { + requests++; + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify(identity)); + }, + ); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("test address"); + const client = makeResourceWorkerClient(options(tls, address.port, identity)); + try { + await expect(client.identity()).rejects.toThrow("MCP worker"); + expect(requests).toBe(0); + } finally { + client.close(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +}); diff --git a/orchestrator/src/__tests__/mcp-worker-encoding.test.ts b/orchestrator/src/__tests__/mcp-worker-encoding.test.ts new file mode 100644 index 000000000..822e8239d --- /dev/null +++ b/orchestrator/src/__tests__/mcp-worker-encoding.test.ts @@ -0,0 +1,34 @@ +import { expect, test } from "bun:test"; +import { toWorkerBindingHash, assertWorkerPolicy } from "../mcp/worker-encoding.ts"; + +test("worker digests use strict representation conversion without changing frozen identity", () => { + const raw = "a".repeat(64); + const frozen = { zoneId: "public", policyRevision: "1", policyDigest: `sha256:${raw}` }; + const identity = { protocol: 1 as const, bootId: "boot", ...frozen, policyDigest: raw }; + expect(toWorkerBindingHash(`sha256:${raw}`)).toBe(raw); + expect(() => assertWorkerPolicy(identity, frozen)).not.toThrow(); + expect(frozen.policyDigest).toBe(`sha256:${raw}`); + expect(identity.policyDigest).toBe(raw); + for (const value of [ + raw, + `sha256:sha256:${raw}`, + `SHA256:${raw}`, + `sha256:${raw.toUpperCase()}`, + `sha256:${raw} `, + `sha256:${raw.slice(1)}`, + ]) { + expect(() => toWorkerBindingHash(value)).toThrow(); + } + for (const policyDigest of [ + `sha256:${raw}`, + `sha256:sha256:${raw}`, + raw.toUpperCase(), + "b".repeat(64), + `${raw} `, + ]) { + expect(() => assertWorkerPolicy({ ...identity, policyDigest }, frozen)).toThrow(); + } + expect(() => assertWorkerPolicy({ ...identity, zoneId: "other" }, frozen)).toThrow(); + expect(() => assertWorkerPolicy({ ...identity, policyRevision: "2" }, frozen)).toThrow(); + expect(() => assertWorkerPolicy(identity, { ...frozen, policyDigest: raw })).toThrow(); +}); diff --git a/orchestrator/src/__tests__/profiles.test.ts b/orchestrator/src/__tests__/profiles.test.ts index 2cbff58cd..3101d8938 100644 --- a/orchestrator/src/__tests__/profiles.test.ts +++ b/orchestrator/src/__tests__/profiles.test.ts @@ -11,6 +11,7 @@ import { PR_REVIEW_CAPABILITY } from "../tools/review.ts"; import { capabilityGrant } from "../integrations/grants.ts"; import type { IntegrationConnectionStore } from "../db/integration-connections.ts"; import { invalidateRegistry } from "../connectors/registry.ts"; +import { MCP_LIMITS } from "../mcp/limits.ts"; const protoGrant = (capability: string) => create(ProfileIntegrationGrantSchema, capabilityGrant( @@ -766,3 +767,139 @@ describe("user-scoped grants (ADR 0115)", () => { } finally { await s.close(); } }); }); + +describe("MCP profile grant contract", () => { + test("numeric MCP constraints preserve decimal and equivalent exponent values", async () => { + const s = await spawn({ + getSession: makeGetSession("admin", "admin"), + store: makeFakeStore(), + images: fakeImages(["img-1"]), + }); + const schema = + '{"type":"object","properties":{"count":{"type":"number","minimum":0.1,"maximum":1e3}}}'; + const canonical = schema.replace("1e3", "1000"); + const grant = { + connectionId: "default-mcp", + operation: "invoke", + credentialScope: "org", + mcp: { + upstreamName: "Create.Record", + automationWrite: true, + argumentConstraintDocumentsJson: [schema], + unrestricted: false, + }, + }; + const input = { + name: "Tools", + imageId: "img-1", + harness: "claude", + integrationGrants: [grant], + }; + try { + const created = await s.client.createProfile(input); + expect(created.profile!.integrationGrants[0]!.mcp!.argumentConstraintDocumentsJson).toEqual([ + canonical, + ]); + const updated = await s.client.updateProfile({ + ...input, + id: created.profile!.id, + integrationGrants: [ + { ...grant, mcp: { ...grant.mcp, argumentConstraintDocumentsJson: [canonical] } }, + ], + }); + expect(updated.profile!.integrationGrants[0]!.mcp!.argumentConstraintDocumentsJson).toEqual([ + canonical, + ]); + } finally { + await s.close(); + } + }); + + test.each([ + ["rounded decimal", '{"type":"object","properties":{"count":{"minimum":0.10000000000000001}}}'], + ["unsafe integer", '{"type":"object","properties":{"count":{"maximum":9007199254740993}}}'], + ["raw byte limit", '{"type":"object"}'.padStart(MCP_LIMITS.argumentBytes + 1)], + ])( + "rejects MCP constraints with %s before create or update persistence", + async (_name, schema) => { + const store = makeFakeStore(); + const s = await spawn({ + getSession: makeGetSession("admin", "admin"), + store, + images: fakeImages(["img-1"]), + }); + const input = { name: "Tools", imageId: "img-1", harness: "claude" }; + const grant = { + connectionId: "default-mcp", + operation: "invoke", + credentialScope: "org", + mcp: { + upstreamName: "Create.Record", + automationWrite: true, + argumentConstraintDocumentsJson: [schema], + unrestricted: false, + }, + }; + try { + const created = await s.client.createProfile(input); + await expectErr( + s.client.createProfile({ ...input, integrationGrants: [grant] }), + Code.InvalidArgument, + ); + await expectErr( + s.client.updateProfile({ ...input, id: created.profile!.id, integrationGrants: [grant] }), + Code.InvalidArgument, + ); + expect(await store.list({ includeArchived: true })).toHaveLength(1); + expect((await store.get(created.profile!.id))!.integrationGrants).toEqual([]); + } finally { + await s.close(); + } + }, + ); + + test("create, update, and list preserve explicit organization scope and constraints", async () => { + const s = await spawn({ getSession: makeGetSession("admin", "admin"), store: makeFakeStore(), images: fakeImages(["img-1"]), mountCatalog: fakeCatalog([]) }); + const grant = { connectionId: "default-mcp", operation: "invoke", credentialScope: "org", resourceConstraints: [], mcp: { upstreamName: "Create.Record", automationWrite: true, argumentConstraintDocumentsJson: [ + JSON.stringify({ $schema: "http://json-schema.org/draft-07/schema#", $id: "https://schemas.example.test/constraint", $defs: { item: { const: "allowed" } }, type: "object", properties: { project: { $ref: "#/$defs/item" } }, required: ["project"] }), + JSON.stringify({ $schema: "https://json-schema.org/draft/2020-12/schema", $id: "https://schemas.example.test/constraint", $defs: { item: { type: "integer", minimum: 1 } }, type: "object", properties: { count: { $ref: "#/$defs/item" } }, required: ["count"] }), + ], unrestricted: false } }; + const input = { name: "Tools", imageId: "img-1", harness: "claude", integrationGrants: [grant] }; + try { + const created = await s.client.createProfile(input); + expect(created.profile!.integrationGrants[0]!.credentialScope).toBe("org"); + expect(created.profile!.integrationGrants[0]!.mcp?.argumentConstraintDocumentsJson).toEqual(grant.mcp.argumentConstraintDocumentsJson); + const updated = await s.client.updateProfile({ ...input, id: created.profile!.id, name: "Renamed tools" }); + expect(updated.profile!.integrationGrants[0]!.mcp?.automationWrite).toBe(true); + expect(updated.profile!.integrationGrants[0]!.mcp?.argumentConstraintDocumentsJson).toEqual(grant.mcp.argumentConstraintDocumentsJson); + const listed = await s.client.listProfiles({}); + expect(listed.profiles[0]!.integrationGrants[0]!.mcp?.upstreamName).toBe("Create.Record"); + expect(listed.profiles[0]!.integrationGrants[0]!.mcp?.argumentConstraintDocumentsJson).toEqual(grant.mcp.argumentConstraintDocumentsJson); + await expectErr(s.client.updateProfile({ ...input, id: created.profile!.id, integrationGrants: [{ ...grant, credentialScope: "user" }] }), Code.InvalidArgument); + await expectErr(s.client.updateProfile({ ...input, id: created.profile!.id, integrationGrants: [{ ...grant, mcp: { ...grant.mcp, argumentConstraintDocumentsJson: [] } }] }), Code.InvalidArgument); + await expectErr(s.client.updateProfile({ ...input, id: created.profile!.id, integrationGrants: [{ ...grant, mcp: { ...grant.mcp, argumentConstraintDocumentsJson: ["[]"] } }] }), Code.InvalidArgument); + const unrestricted = await s.client.updateProfile({ ...input, id: created.profile!.id, integrationGrants: [{ ...grant, mcp: { ...grant.mcp, argumentConstraintDocumentsJson: [], unrestricted: true } }] }); + expect(unrestricted.profile!.integrationGrants[0]!.mcp?.argumentConstraintDocumentsJson).toEqual([]); + expect(unrestricted.profile!.integrationGrants[0]!.mcp?.unrestricted).toBe(true); + } finally { await s.close(); } + }); +}); + +test("typed profile publication preserves optional revision presence and authenticates the publisher", async () => { + const store = makeFakeStore(); + const calls: unknown[] = []; + const originalCreate = store.create.bind(store); + store.create = async (input, publication) => { + calls.push(publication); + return originalCreate(input); + }; + const s = await spawn({ getSession: makeGetSession("admin-publisher", "admin"), store, images: fakeImages(["img-1"]) }); + const input = { name: "Profile", imageId: "img-1", harness: "claude" }; + try { + await s.client.createProfile(input); + await s.client.createProfile({ ...input, expectedMcpGrantRevision: 0n }); + expect(calls).toEqual([undefined, { actorId: "admin-publisher", expectedRevision: 0 }]); + await expectErr(s.client.createProfile({ ...input, expectedMcpGrantRevision: BigInt(Number.MAX_SAFE_INTEGER) + 1n }), Code.InvalidArgument); + expect(calls).toHaveLength(2); + } finally { await s.close(); } +}); diff --git a/orchestrator/src/__tests__/session-files.route.test.ts b/orchestrator/src/__tests__/session-files.route.test.ts index 5e8ee6c8c..8f4182940 100644 --- a/orchestrator/src/__tests__/session-files.route.test.ts +++ b/orchestrator/src/__tests__/session-files.route.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from "bun:test"; +import { Code, ConnectError } from "@connectrpc/connect"; import { makeSessionFilesRoute, @@ -42,6 +43,7 @@ describe("session file routes", () => { }, ); expect(response.status).toBe(200); + expect(response.headers.has("x-engram-rpc-code")).toBe(false); expect(await response.json()).toEqual({ path, size_bytes: bytes.length, @@ -115,8 +117,96 @@ describe("session file routes", () => { `/api/v1/sessions/${sessionId}/files?path=${encodeURIComponent(path)}`, ); expect(response.status).toBe(200); + expect(response.headers.has("x-engram-rpc-code")).toBe(false); expect(response.headers.get("content-length")).toBe(String(bytes.length)); expect(response.headers.get("x-content-sha256")).toBe(digest); expect(new Uint8Array(await response.arrayBuffer())).toEqual(bytes); }); + + test("keeps file RPC status and body while exposing only its numeric code in the diagnostic header", async () => { + const cases: Array<[Code, number]> = [ + [Code.Canceled, 400], + [Code.Unknown, 400], + [Code.InvalidArgument, 400], + [Code.DeadlineExceeded, 400], + [Code.NotFound, 404], + [Code.AlreadyExists, 409], + [Code.PermissionDenied, 403], + [Code.ResourceExhausted, 413], + [Code.FailedPrecondition, 409], + [Code.Aborted, 400], + [Code.OutOfRange, 400], + [Code.Unimplemented, 400], + [Code.Internal, 400], + [Code.Unavailable, 400], + [Code.DataLoss, 400], + [Code.Unauthenticated, 401], + ]; + for (const [code, status] of cases) { + const error = new ConnectError("file RPC failed", code, { + "x-private": "private-error-metadata", + }); + const client: SessionFileClient = { + async writeFile() { + throw error; + }, + async *readFile() { + throw error; + }, + }; + for (const method of ["GET", "POST"]) { + const response = await app(client).request( + `/api/v1/sessions/${sessionId}/files?path=${encodeURIComponent(path)}`, + { method, ...(method === "POST" ? { headers: { "x-upload-size": "0" } } : {}) }, + ); + expect(response.status).toBe(status); + expect(response.headers.get("x-engram-rpc-code")).toBe(String(code)); + expect(response.headers.has("x-private")).toBe(false); + expect(await response.json()).toEqual({ error: "file RPC failed" }); + } + } + }); + + test("omits the diagnostic header for invalid RPC error codes", async () => { + for (const code of [0, 17, -1, 1.5, NaN, Infinity, -Infinity]) { + const error = new ConnectError("file RPC failed", code); + const client: SessionFileClient = { + async writeFile() { + throw error; + }, + async *readFile() { + throw error; + }, + }; + for (const method of ["GET", "POST"]) { + const response = await app(client).request( + `/api/v1/sessions/${sessionId}/files?path=${encodeURIComponent(path)}`, + { method, ...(method === "POST" ? { headers: { "x-upload-size": "0" } } : {}) }, + ); + expect(response.status).toBe(400); + expect(response.headers.has("x-engram-rpc-code")).toBe(false); + expect(await response.json()).toEqual({ error: "file RPC failed" }); + } + } + }); + + test("does not give non-RPC file errors an RPC diagnostic code", async () => { + const client: SessionFileClient = { + async writeFile() { + throw new Error("file failed"); + }, + async *readFile() { + throw new Error("file failed"); + }, + }; + for (const method of ["GET", "POST"]) { + const response = await app(client).request( + `/api/v1/sessions/${sessionId}/files?path=${encodeURIComponent(path)}`, + { method, ...(method === "POST" ? { headers: { "x-upload-size": "0" } } : {}) }, + ); + expect(response.status).toBe(500); + expect(response.headers.has("x-engram-rpc-code")).toBe(false); + expect(await response.json()).toEqual({ error: "file failed" }); + } + }); }); diff --git a/orchestrator/src/__tests__/task-create.test.ts b/orchestrator/src/__tests__/task-create.test.ts index b5deeba5c..9c8fbfe66 100644 --- a/orchestrator/src/__tests__/task-create.test.ts +++ b/orchestrator/src/__tests__/task-create.test.ts @@ -26,11 +26,7 @@ import { type Db, } from "../rpc/task-create.ts"; import type { ProfileRow, ProfileStore } from "../db/profiles.ts"; -import type { - SessionAppStore, - SessionAppSpec, - SessionAppRow, -} from "../db/session-apps.ts"; +import type { SessionAppStore, SessionAppSpec, SessionAppRow } from "../db/session-apps.ts"; import type { ImagesClient } from "../rpc/profiles.ts"; import type { UserIdentity, UserIdentityStore } from "../db/users.ts"; import { createToolRegistry } from "../tools/registry.ts"; @@ -144,7 +140,7 @@ const routedModel = (over: Partial = {}): RouterModelRow => ({ const fakeModelRouters = (row = routedModel()): ModelRouterStore => ({ listModels: async () => [row], - getModel: async (_routerId, modelId) => modelId === row.modelId ? row : null, + getModel: async (_routerId, modelId) => (modelId === row.modelId ? row : null), replaceCatalog: async () => ({ markedUnavailable: 0 }), updatePolicy: async () => row, getSyncState: async () => null, @@ -283,9 +279,7 @@ describe("compileSessionCreateInput — user-scoped integrations (ADR 0115)", () ...deps(), connectors: { list: async () => [{ provider: "acme", config: acmeConnector }] }, oauthSubject: { kind: OauthSubjectKind.USER, id: "user-7" }, - ...(credentials - ? { listUserConnectorCredentials: async () => credentials } - : {}), + ...(credentials ? { listUserConnectorCredentials: async () => credentials } : {}), }); const policyOf = (inp: { integrationPolicyJson?: string }) => JSON.parse(inp.integrationPolicyJson ?? "{}") as { @@ -296,7 +290,7 @@ describe("compileSessionCreateInput — user-scoped integrations (ADR 0115)", () }; test("a human run with the credential connected stamps oauth_user", async () => { -const inp = await compileSessionCreateInput( + const inp = await compileSessionCreateInput( userScopedProfile(), userDeps([{ provider: "acme", status: "connected" }]), ); @@ -375,13 +369,11 @@ const inp = await compileSessionCreateInput( expect(inp.integrationConnections?.map((c) => c.provider)).toEqual(["github"]); // No acme inject in the policy, and no org fallback for it. const policy = policyOf(inp); - expect( - (policy.injects ?? []).some((inject) => inject.secret_ref === "acme.token"), - ).toBe(false); + expect((policy.injects ?? []).some((inject) => inject.secret_ref === "acme.token")).toBe(false); }); test("a programmatic run compiles the org credential and never gates", async () => { -const inp = await compileSessionCreateInput( + const inp = await compileSessionCreateInput( userScopedProfile(), { ...userDeps([]), @@ -397,7 +389,7 @@ const inp = await compileSessionCreateInput( }); test("an unscoped grant compiles the org credential for a human run", async () => { -const inp = await compileSessionCreateInput( + const inp = await compileSessionCreateInput( profile({ integrationGrants: [ { connectionId: "default-acme", operation: "issues:write", resourceConstraints: [] }, @@ -411,7 +403,7 @@ const inp = await compileSessionCreateInput( }); test("a capability override compiles org authority and never gates", async () => { -const inp = await compileSessionCreateInput( + const inp = await compileSessionCreateInput( userScopedProfile(), { ...userDeps([]), @@ -764,10 +756,12 @@ describe("compileSessionCreateInput", () => { connections, harnessCatalog: { listHarnesses: async () => ({ - harnesses: [{ - name: "claude", - descriptor: { models: [], effort: [], routerProtocols: [protocol] }, - }], + harnesses: [ + { + name: "claude", + descriptor: { models: [], effort: [], routerProtocols: [protocol] }, + }, + ], }), }, orgSecret: { listSecrets: async () => ({ secrets: [{ name: "openrouter.api_key" }] }) }, @@ -849,11 +843,21 @@ describe("compileSessionCreateInput", () => { ...deps(), harnessCatalog: providerHarnessCatalog(), orgSecret: { listSecrets: async () => ({ secrets: [{ name: "openrouter.api_key" }] }) }, - modelRouters: fakeModelRouters(routedModel({ supportedParameters: ["tools"], userEnabled: false })), + modelRouters: fakeModelRouters( + routedModel({ supportedParameters: ["tools"], userEnabled: false }), + ), }; - const routedProfile = profile({ modelRouter: "openrouter", model: "z-ai/glm-5.2", effort: "high" }); - await expect(compileSessionCreateInput(routedProfile, providerDeps)).rejects.toMatchObject({ code: Code.PermissionDenied }); - const input = await compileSessionCreateInput(routedProfile, providerDeps, { programmatic: true }); + const routedProfile = profile({ + modelRouter: "openrouter", + model: "z-ai/glm-5.2", + effort: "high", + }); + await expect(compileSessionCreateInput(routedProfile, providerDeps)).rejects.toMatchObject({ + code: Code.PermissionDenied, + }); + const input = await compileSessionCreateInput(routedProfile, providerDeps, { + programmatic: true, + }); expect(input.effort).toBeUndefined(); expect(input.harnessEnv?.MAX_THINKING_TOKENS).toBeUndefined(); }); @@ -899,6 +903,27 @@ describe("compileSessionCreateInput", () => { expect(inp.harnessEnv?.ENGRAM_USER_NAME).toBeUndefined(); }); + test("native tool manifest cannot be replaced by profile or late environment input", async () => { + const toolRegistry = createToolRegistry(); + toolRegistry.register({ + name: "native_tool", + description: "Native tool", + input: z.object({}), + output: z.object({}), + handling: "handled", + execution: "sync", + handler: async () => ({}), + }); + const result = await compileSessionCreateInput( + profile({ envVars: { ENGRAM_TOOLS: '[{"name":"mcp_injected"}]' } }), + { ...deps(), toolRegistry }, + { extraHarnessEnv: { ENGRAM_TOOLS: '[{"name":"late_injected"}]' } }, + ); + expect( + JSON.parse(result.harnessEnv!.ENGRAM_TOOLS!).map((tool: { name: string }) => tool.name), + ).toEqual(["native_tool"]); + }); + test("injects ENGRAM_TOOLS with capability-gated and ungated manifest tools", async () => { const toolRegistry = createToolRegistry(); toolRegistry.register({ @@ -972,16 +997,14 @@ describe("compileSessionCreateInput", () => { { taskType: "spec" }, ); - expect(JSON.parse(chat.harnessEnv!.ENGRAM_TOOLS!).map((tool: { name: string }) => tool.name)) - .toEqual(["always_available"]); - expect(chat.harnessEnv?.ENGRAM_APPEND_SYSTEM_PROMPT).not.toContain( - SPEC_MODE_SYSTEM_PROMPT, - ); - expect(JSON.parse(spec.harnessEnv!.ENGRAM_TOOLS!).map((tool: { name: string }) => tool.name)) - .toEqual(["always_available", "spec_read"]); - expect(spec.harnessEnv?.ENGRAM_APPEND_SYSTEM_PROMPT).toContain( - SPEC_MODE_SYSTEM_PROMPT, - ); + expect( + JSON.parse(chat.harnessEnv!.ENGRAM_TOOLS!).map((tool: { name: string }) => tool.name), + ).toEqual(["always_available"]); + expect(chat.harnessEnv?.ENGRAM_APPEND_SYSTEM_PROMPT).not.toContain(SPEC_MODE_SYSTEM_PROMPT); + expect( + JSON.parse(spec.harnessEnv!.ENGRAM_TOOLS!).map((tool: { name: string }) => tool.name), + ).toEqual(["always_available", "spec_read"]); + expect(spec.harnessEnv?.ENGRAM_APPEND_SYSTEM_PROMPT).toContain(SPEC_MODE_SYSTEM_PROMPT); }); test("carries the spec template into the spec prompt", async () => { @@ -1406,10 +1429,13 @@ function recordingDb( if (throwOnTx) throw new Error("db boom"); const tx = { insert: () => ({ - values: async (v: Record) => { - insertCount += 1; - if (insertCount === failInsertAt) throw new Error("insert boom"); - records.push(v); + values: (v: Record) => { + const result = Promise.resolve().then(() => { + insertCount += 1; + if (insertCount === failInsertAt) throw new Error("insert boom"); + records.push(v); + }); + return Object.assign(result, { onConflictDoNothing: () => result }); }, }), update: () => ({ @@ -1483,10 +1509,12 @@ describe("createTaskWithSession", () => { test(`rejects a human ${type} launch before any session or task write`, async () => { const sessions = fakeSessions(); const records: Record[] = []; - await expect(createTaskWithSession( - createDeps(sessions, recordingDb(records), { profileOver: { programmaticOnly: true } }), - { type, ownerUserId: "u", profileId: "p1" }, - )).rejects.toMatchObject({ code: Code.PermissionDenied }); + await expect( + createTaskWithSession( + createDeps(sessions, recordingDb(records), { profileOver: { programmaticOnly: true } }), + { type, ownerUserId: "u", profileId: "p1" }, + ), + ).rejects.toMatchObject({ code: Code.PermissionDenied }); expect(sessions.createReqs).toEqual([]); expect(records).toEqual([]); }); @@ -1790,10 +1818,11 @@ describe("createTaskWithSession", () => { test("reserves nothing and injects nothing when the profile declares no apps", async () => { const apps = fakeSessionApps(); const sessions = fakeSessions(); - await createTaskWithSession( - createDeps(sessions, recordingDb([]), { sessionApps: apps }), - { type: "chat", ownerUserId: "u", profileId: "p1" }, - ); + await createTaskWithSession(createDeps(sessions, recordingDb([]), { sessionApps: apps }), { + type: "chat", + ownerUserId: "u", + profileId: "p1", + }); expect(apps.calls).toHaveLength(0); const env = (sessions.createReqs[0] as { harnessEnv?: Record }).harnessEnv; expect(Object.keys(env ?? {}).some((k) => k.endsWith("_INGRESS_URL"))).toBe(false); @@ -1847,7 +1876,6 @@ describe("createTaskWithSession", () => { const env = (sessions.createReqs[0] as { harnessEnv?: Record }).harnessEnv; expect(env?.ENGRAM_USER_EMAIL).toBeUndefined(); }); - }); describe("createSessionForExistingTask", () => { diff --git a/orchestrator/src/__tests__/tasks.test.ts b/orchestrator/src/__tests__/tasks.test.ts index b4475bbf4..29259c7eb 100644 --- a/orchestrator/src/__tests__/tasks.test.ts +++ b/orchestrator/src/__tests__/tasks.test.ts @@ -540,7 +540,20 @@ function okDb(taskId = "fake-task", listFixture?: ListDbFixture): Db { }; return { transaction: async (fn: (tx: unknown) => Promise) => { - const tx = { insert: () => ({ values: async () => undefined }) }; + const tx = { + insert: (table: unknown) => ({ + values: () => { + const result = Promise.resolve(); + return Object.assign(result, { + onConflictDoNothing: ({ target }: { target: unknown }) => { + expect(table).toBe(sessionListenerTable); + expect(target).toBe(sessionListenerTable.sessionId); + return result; + }, + }); + }, + }), + }; return fn(tx); }, select: () => makeSelectChain(), @@ -754,6 +767,8 @@ function listSessionRef( integrationConnections: null, integrationPrincipalId: null, integrationSnapshotHash: null, + mcpBindingEnvelope: null, + mcpManagedLifecycle: null, createdAt: new Date(createdAt), }; } diff --git a/orchestrator/src/authz/policy-map.ts b/orchestrator/src/authz/policy-map.ts index 858345208..e3519338b 100644 --- a/orchestrator/src/authz/policy-map.ts +++ b/orchestrator/src/authz/policy-map.ts @@ -54,6 +54,11 @@ export const POLICY: Record = { subject: "Session", sessionIdField: "sessionId", }, + "SessionService.GetMcpWritePolicy": { + action: "read", + subject: "Session", + sessionIdField: "sessionId", + }, "SessionService.DeleteSession": { action: "delete", subject: "Session", diff --git a/orchestrator/src/authz/publisher.ts b/orchestrator/src/authz/publisher.ts new file mode 100644 index 000000000..10a4703bc --- /dev/null +++ b/orchestrator/src/authz/publisher.ts @@ -0,0 +1,13 @@ +import { eq } from "drizzle-orm"; +import type { getDb } from "../db/client.ts"; +import { user } from "../db/schema.ts"; +import { isServiceAccountEmail } from "../rpc/api-key.ts"; + +type PublisherTx = Parameters["transaction"]>[0]>[0]; + +/** Lock the actor before publication targets so demotion and publication serialize. */ +export async function lockMcpPublisher(tx: PublisherTx, actorId: string) { + const [actor] = await tx.select().from(user).where(eq(user.id, actorId)).for("share"); + if (!actor || actor.banned || actor.role !== "admin" || isServiceAccountEmail(actor.email)) + throw new Error("current human administrator required for MCP publication"); +} diff --git a/orchestrator/src/control-plane/transport.ts b/orchestrator/src/control-plane/transport.ts index 7b505cde2..5c28dcd67 100644 --- a/orchestrator/src/control-plane/transport.ts +++ b/orchestrator/src/control-plane/transport.ts @@ -21,6 +21,7 @@ import { config } from "../config.ts"; import { makeLeaseStore } from "../listeners/lease-store.ts"; import { makeListenerWakeInterceptor } from "../listeners/wake-interceptor.ts"; import { log } from "../log.ts"; +import { makeMcpLifecycleInterceptor } from "../mcp/lifecycle-interceptor.ts"; // Machine identity (ADR §5): one trusted caller, one static credential. export const bearerInterceptor: Interceptor = (next) => (req) => { @@ -67,6 +68,19 @@ export const controlPlaneTransport: Transport = makeTransport( config.controlPlaneGrpcUrl, config.controlPlaneBearer, [ + makeMcpLifecycleInterceptor(async (sessionId, cause) => { + const [{ getDb }, { makeMcpLifecycleInvalidator }, { mcpRecovery }] = await Promise.all([ + import("../db/client.ts"), + import("../mcp/lifecycle-invalidation.ts"), + import("../mcp/recovery.ts"), + ]); + await makeMcpLifecycleInvalidator({ + db: getDb(), + abort: async (id) => { + mcpRecovery()?.abortExecution(id); + }, + })(sessionId, cause); + }), // Every resuming session RPC wakes the session's listener row (ADR 0119 // amendment): this is the one seam all callers share. makeListenerWakeInterceptor( diff --git a/orchestrator/src/db/integration-connections.ts b/orchestrator/src/db/integration-connections.ts index c9a0c7c88..04dd00ea5 100644 --- a/orchestrator/src/db/integration-connections.ts +++ b/orchestrator/src/db/integration-connections.ts @@ -1,6 +1,8 @@ +import { notifyMcpAuthorityChanged } from "../mcp/authority-notification.ts"; /** Named integration connection store (ADR 0109). */ -import { and, asc, eq, inArray } from "drizzle-orm"; +import { validateMcpConfig } from "../mcp/config.ts"; +import { and, asc, eq, inArray, ne } from "drizzle-orm"; import { getDb } from "./client.ts"; import { integrationConnection as connectionTable } from "./schema.ts"; @@ -14,6 +16,7 @@ export interface GoogleCloudConnectionConfig { } export interface IntegrationConnectionRow { + mcp?: import("../mcp/types.ts").McpConnectionState; id: string; alias: string; provider: string; @@ -40,7 +43,10 @@ export interface IntegrationConnectionStore { getMany(ids: readonly string[]): Promise; getDefault(provider: string): Promise; create(input: IntegrationConnectionInput): Promise; - update(id: string, input: Omit): Promise; + update( + id: string, + input: Omit, + ): Promise; delete(id: string): Promise; markTested(id: string, testedAt: Date): Promise; setEnabled(id: string, enabled: boolean): Promise; @@ -53,6 +59,17 @@ export interface IntegrationConnectionStore { function toRow(r: typeof connectionTable.$inferSelect): IntegrationConnectionRow { return { + ...(r.provider === "mcp" + ? { + mcp: { + configRevision: r.configRevision, + authGeneration: r.authGeneration, + credentialIdentity: r.credentialIdentity, + catalogRevision: r.catalogRevision, + revoked: r.revoked, + }, + } + : {}), id: r.id, alias: r.alias, provider: r.provider, @@ -73,17 +90,16 @@ export function makeIntegrationConnectionStore( const rows = await db .select() .from(connectionTable) - .where(and( - eq(connectionTable.provider, provider), - eq(connectionTable.isDefault, true), - )) + .where(and(eq(connectionTable.provider, provider), eq(connectionTable.isDefault, true))) .limit(1); return rows[0] ? toRow(rows[0]) : null; } return { async list() { - return (await db.select().from(connectionTable).orderBy(asc(connectionTable.alias))).map(toRow); + return (await db.select().from(connectionTable).orderBy(asc(connectionTable.alias))).map( + toRow, + ); }, async get(id) { @@ -107,10 +123,12 @@ export function makeIntegrationConnectionStore( getDefault, async create(input) { + if (input.provider === "mcp") input = { ...input, config: validateMcpConfig(input.config) }; const rows = await db .insert(connectionTable) .values({ id: crypto.randomUUID(), ...input, enabled: false }) .returning(); + if (input.provider === "mcp") notifyMcpAuthorityChanged(); return toRow(rows[0]!); }, @@ -118,7 +136,7 @@ export function makeIntegrationConnectionStore( const rows = await db .update(connectionTable) .set({ ...input, enabled: false, testedAt: null, updatedAt: new Date() }) - .where(eq(connectionTable.id, id)) + .where(and(eq(connectionTable.id, id), ne(connectionTable.provider, "mcp"))) .returning(); return rows[0] ? toRow(rows[0]) : null; }, @@ -127,7 +145,8 @@ export function makeIntegrationConnectionStore( const rows = await db .delete(connectionTable) .where(eq(connectionTable.id, id)) - .returning({ id: connectionTable.id }); + .returning({ id: connectionTable.id, provider: connectionTable.provider }); + if (rows[0]?.provider === "mcp") notifyMcpAuthorityChanged(); return rows.length > 0; }, @@ -135,7 +154,7 @@ export function makeIntegrationConnectionStore( const rows = await db .update(connectionTable) .set({ testedAt, updatedAt: testedAt }) - .where(eq(connectionTable.id, id)) + .where(and(eq(connectionTable.id, id), ne(connectionTable.provider, "mcp"))) .returning(); return rows[0] ? toRow(rows[0]) : null; }, @@ -144,7 +163,7 @@ export function makeIntegrationConnectionStore( const rows = await db .update(connectionTable) .set({ enabled, updatedAt: new Date() }) - .where(eq(connectionTable.id, id)) + .where(and(eq(connectionTable.id, id), ne(connectionTable.provider, "mcp"))) .returning(); return rows[0] ? toRow(rows[0]) : null; }, @@ -153,24 +172,28 @@ export function makeIntegrationConnectionStore( const rows = await db .update(connectionTable) .set({ config, updatedAt: new Date() }) - .where(eq(connectionTable.id, id)) + .where(and(eq(connectionTable.id, id), ne(connectionTable.provider, "mcp"))) .returning(); return rows[0] ? toRow(rows[0]) : null; }, async ensureDefault(provider, displayName) { + if (provider === "mcp") throw new Error("MCP requires an explicit named connection"); const existing = await getDefault(provider); if (existing) return existing; - await db.insert(connectionTable).values({ - id: crypto.randomUUID(), - alias: `${provider}-default`, - provider, - displayName, - isDefault: true, - config: {}, - enabled: true, - testedAt: new Date(0), - }).onConflictDoNothing(); + await db + .insert(connectionTable) + .values({ + id: crypto.randomUUID(), + alias: `${provider}-default`, + provider, + displayName, + isDefault: true, + config: {}, + enabled: true, + testedAt: new Date(0), + }) + .onConflictDoNothing(); const created = await getDefault(provider); if (!created) { throw new Error(`failed to create default integration connection for "${provider}"`); diff --git a/orchestrator/src/db/profiles.ts b/orchestrator/src/db/profiles.ts index 3b409b89c..e5bf7c07f 100644 --- a/orchestrator/src/db/profiles.ts +++ b/orchestrator/src/db/profiles.ts @@ -6,8 +6,11 @@ * fake. Drizzle-backed by default. Soft delete only (deleted_at). */ -import { and, eq, inArray, isNull, ne } from "drizzle-orm"; +import { and, eq, inArray, isNull } from "drizzle-orm"; +import { notifyMcpAuthorityChanged } from "../mcp/authority-notification.ts"; +import { canonicalDigest } from "../mcp/hash.ts"; +import { lockMcpPublisher } from "../authz/publisher.ts"; import { getDb } from "./client.ts"; import { profile as profileTable, @@ -19,7 +22,10 @@ import { type ProfileIntegrationGrant, } from "./schema.ts"; +export type McpGrantPublication = { revision: number; actorId: string; grantsHash: string }; +export type PublishMcpGrants = { actorId: string; expectedRevision: number }; export interface ProfileRow { + mcpGrantPublication?: McpGrantPublication | null; id: string; name: string; description: string; @@ -82,9 +88,13 @@ export interface ProfileStore { getActive(id: string): Promise; /** Rows for the given ids (active or archived) — for snapshot enrichment. */ getByIds(ids: string[]): Promise; - create(input: ProfileInput): Promise; + create(input: ProfileInput, publication?: PublishMcpGrants): Promise; /** Returns the updated row, or null if the id is absent / archived. */ - update(id: string, input: ProfileInput): Promise; + update( + id: string, + input: ProfileInput, + publication?: PublishMcpGrants, + ): Promise; /** Idempotent soft delete (sets deleted_at). */ softDelete(id: string): Promise; } @@ -92,6 +102,7 @@ export interface ProfileStore { function toRow(r: typeof profileTable.$inferSelect): ProfileRow { return { id: r.id, + mcpGrantPublication: r.mcpGrantPublication, name: r.name, description: r.description, icon: r.icon, @@ -140,26 +151,110 @@ export function makeProfileStore(db: ReturnType = getDb()): Profil const rows = await db.select().from(profileTable).where(inArray(profileTable.id, ids)); return rows.map(toRow); }, - async create(input) { - const id = crypto.randomUUID(); - await db.insert(profileTable).values({ id, ...input }); - const row = await this.get(id); - return row!; + async create(input, publication) { + return db.transaction(async (tx) => { + if (publication) await lockMcpPublisher(tx, publication.actorId); + const provenance = publicationFor([], null, input.integrationGrants, publication); + const [row] = await tx + .insert(profileTable) + .values({ id: crypto.randomUUID(), ...input, mcpGrantPublication: provenance }) + .returning(); + return toRow(row!); + }); }, - async update(id, input) { - const existing = await this.getActive(id); - if (!existing) return null; - await db - .update(profileTable) - .set({ ...input, updatedAt: new Date() }) - .where(eq(profileTable.id, id)); - return this.get(id); + async update(id, input, publication) { + const result = await db.transaction(async (tx) => { + if (publication) await lockMcpPublisher(tx, publication.actorId); + const [existing] = await tx + .select() + .from(profileTable) + .where(and(eq(profileTable.id, id), isNull(profileTable.deletedAt))) + .for("update"); + if (!existing) return null; + const provenance = publicationFor( + existing.integrationGrants, + existing.mcpGrantPublication, + input.integrationGrants, + publication, + ); + // Select only writable fields; a caller cannot supply publication metadata through input. + const { + name, + description, + icon, + imageId, + harness, + modelRouter, + model, + effort, + programmaticOnly, + includeUserTokens, + envVars, + skills, + integrationGrants, + network, + secrets, + repos, + apps, + } = input; + const [row] = await tx + .update(profileTable) + .set({ + name, + description, + icon, + imageId, + harness, + modelRouter, + model, + effort, + programmaticOnly, + includeUserTokens, + envVars, + skills, + integrationGrants, + network, + secrets, + repos, + apps, + mcpGrantPublication: provenance, + updatedAt: new Date(), + }) + .where(eq(profileTable.id, id)) + .returning(); + return toRow(row!); + }); + if (result) notifyMcpAuthorityChanged(); + return result; }, async softDelete(id) { await db .update(profileTable) .set({ deletedAt: new Date() }) .where(and(eq(profileTable.id, id), isNull(profileTable.deletedAt))); + notifyMcpAuthorityChanged(); }, }; } + +function publicationFor( + previous: ProfileIntegrationGrant[], + provenance: McpGrantPublication | null, + next: ProfileIntegrationGrant[], + publication?: PublishMcpGrants, +): McpGrantPublication | null { + const hash = (grants: ProfileIntegrationGrant[]) => + canonicalDigest(grants.filter((g) => g.mcp !== undefined)); + if (!publication) { + if (hash(previous) !== hash(next)) throw new Error("explicit MCP grant publication required"); + return provenance; + } + const revision = provenance?.revision ?? 0; + if ( + !Number.isSafeInteger(publication.expectedRevision) || + publication.expectedRevision !== revision || + revision >= Number.MAX_SAFE_INTEGER + ) + throw new Error("MCP grant publication revision changed"); + return { revision: revision + 1, actorId: publication.actorId, grantsHash: hash(next) }; +} diff --git a/orchestrator/src/db/schema.ts b/orchestrator/src/db/schema.ts index 116a24488..bedbc2160 100644 --- a/orchestrator/src/db/schema.ts +++ b/orchestrator/src/db/schema.ts @@ -69,6 +69,7 @@ export interface TaskLaunchPolicy { integrationPolicyJson: string; integrationGrants: ProfileIntegrationGrant[]; integrationConnections: IntegrationConnectionSnapshot[]; + mcpBindingEnvelope?: import("../mcp/binding-store.ts").McpBindingEnvelope; network: ProfileNetwork; secrets: ProfileSecret[]; repos: ProfileRepo[]; @@ -173,6 +174,13 @@ export const taskSession = pgTable( // snapshot. This makes broker audit records attributable without consulting // mutable workflow state. integrationPrincipalId: text("integration_principal_id"), + // Candidate metadata only. The host must separately acknowledge deny enforcement. + mcpBindingEnvelope: + jsonb("mcp_binding_envelope").$type(), + mcpManagedLifecycle: + jsonb("mcp_managed_lifecycle").$type< + import("../mcp/managed-lifecycle.ts").ManagedLifecycle + >(), // ADR 0109: content-hash of the compiled authorization snapshot // (profileId + grants + connections). The broker emits it as the // `engrams_profile_snapshot` OIDC claim, so Google-side audit logs join @@ -1064,7 +1072,6 @@ export const consumerCursor = pgTable( (t) => [primaryKey({ columns: [t.sessionId, t.consumer] })], ); - // --------------------------------------------------------------------------- // DBOS orphan sweep (ADR 0104) // --------------------------------------------------------------------------- @@ -1155,6 +1162,7 @@ export interface ProfileIntegrationGrant { * while programmatic sessions always compile the org credential. Rides the * grant array through launch_policy and the task_session snapshot. */ credentialScope?: "org" | "user"; + mcp?: import("../mcp/types.ts").McpToolGrant; } /** A parsed forge remote — enough to join against the GitHub integration. */ @@ -1175,6 +1183,7 @@ export interface ProfileRepo { /** Immutable non-secret connection configuration stamped onto a session. */ export interface IntegrationConnectionSnapshot { + mcp?: import("../mcp/session-bindings.ts").McpSessionBinding; id: string; alias: string; provider: string; @@ -1188,60 +1197,62 @@ export const DEFAULT_PROFILE_NETWORK: ProfileNetwork = { allowHostPatterns: [], }; -export const profile = pgTable( - "profile", - { - id: text("id").primaryKey(), // uuid string (crypto.randomUUID()) - name: text("name").notNull(), - description: text("description").notNull().default(""), - icon: text("icon").notNull().default("Bot"), // lucide icon name - imageId: text("image_id").notNull(), // logical ref → enabled_images.id (§3) - // ADR 0062/0063: the default harness (a HarnessCatalogService catalog name) - // this profile's sessions run, with default model + effort (catalog option - // ids). `harness` is REQUIRED — a profile always names a concrete harness (the - // "inherit deployment default" semantics were superseded; existing rows were - // backfilled to `claude`). model/effort stay nullable → the harness - // descriptor's defaults. All overridable per session. - harness: text("harness").notNull(), - modelRouter: text("model_router"), - model: text("model"), - effort: text("effort"), - programmaticOnly: boolean("programmatic_only").notNull().default(false), - includeUserTokens: boolean("include_user_tokens").notNull().default(false), - envVars: jsonb("env_vars").notNull().default({}), // { KEY: VALUE } - // ADR 0055: dynamic skill bundle names this profile's sessions mount (e.g. - // ["skills", "browser"]). Resolved by the coordinator to reserved-slot - // mounts at session create. Empty = base session (no skills). - skills: jsonb("skills").$type().notNull().default([]), - // ADR 0109: structured integration authority. Unlike the retired flat - // capability array, this preserves which configured identity an operation - // uses and keeps its resource constraints attached. - integrationGrants: jsonb("integration_grants") - .$type() - .notNull() - .default([]), - // ADR 0057: egress network allow-list (deny by default) + secrets this - // profile's sessions get, lifted off the image manifest. Additive in B1; - // compiled into the per-session SessionPolicy + consumed at boot in B2. - network: jsonb("network").$type().notNull().default(DEFAULT_PROFILE_NETWORK), - secrets: jsonb("secrets").$type().notNull().default([]), - // Git repositories this profile's image contains — explicit, user-managed - // (by hand or via DiscoverProfileRepos). Fed to the routing picker's cards. - repos: jsonb("repos").$type().notNull().default([]), - // ADR 0118: the services every session from this profile hosts. The - // orchestrator reserves one hostname per app BEFORE it creates the session - // (inside the transaction below this one) and injects each app's address - // into the guest's env, so a sibling app is configured with an address the - // user's browser can reach too. Empty = no apps. - apps: jsonb("apps").$type().notNull().default([]), - createdAt: timestamp("created_at").notNull().defaultNow(), - updatedAt: timestamp("updated_at") - .notNull() - .defaultNow() - .$onUpdate(() => new Date()), - deletedAt: timestamp("deleted_at"), // null = active; soft delete only (§4) - }, -); +export const profile = pgTable("profile", { + id: text("id").primaryKey(), // uuid string (crypto.randomUUID()) + name: text("name").notNull(), + description: text("description").notNull().default(""), + icon: text("icon").notNull().default("Bot"), // lucide icon name + imageId: text("image_id").notNull(), // logical ref → enabled_images.id (§3) + // ADR 0062/0063: the default harness (a HarnessCatalogService catalog name) + // this profile's sessions run, with default model + effort (catalog option + // ids). `harness` is REQUIRED — a profile always names a concrete harness (the + // "inherit deployment default" semantics were superseded; existing rows were + // backfilled to `claude`). model/effort stay nullable → the harness + // descriptor's defaults. All overridable per session. + harness: text("harness").notNull(), + modelRouter: text("model_router"), + model: text("model"), + effort: text("effort"), + programmaticOnly: boolean("programmatic_only").notNull().default(false), + includeUserTokens: boolean("include_user_tokens").notNull().default(false), + envVars: jsonb("env_vars").notNull().default({}), // { KEY: VALUE } + // ADR 0055: dynamic skill bundle names this profile's sessions mount (e.g. + // ["skills", "browser"]). Resolved by the coordinator to reserved-slot + // mounts at session create. Empty = base session (no skills). + skills: jsonb("skills").$type().notNull().default([]), + // ADR 0109: structured integration authority. Unlike the retired flat + // capability array, this preserves which configured identity an operation + // uses and keeps its resource constraints attached. + mcpGrantPublication: jsonb("mcp_grant_publication").$type<{ + revision: number; + actorId: string; + grantsHash: string; + }>(), + integrationGrants: jsonb("integration_grants") + .$type() + .notNull() + .default([]), + // ADR 0057: egress network allow-list (deny by default) + secrets this + // profile's sessions get, lifted off the image manifest. Additive in B1; + // compiled into the per-session SessionPolicy + consumed at boot in B2. + network: jsonb("network").$type().notNull().default(DEFAULT_PROFILE_NETWORK), + secrets: jsonb("secrets").$type().notNull().default([]), + // Git repositories this profile's image contains — explicit, user-managed + // (by hand or via DiscoverProfileRepos). Fed to the routing picker's cards. + repos: jsonb("repos").$type().notNull().default([]), + // ADR 0118: the services every session from this profile hosts. The + // orchestrator reserves one hostname per app BEFORE it creates the session + // (inside the transaction below this one) and injects each app's address + // into the guest's env, so a sibling app is configured with an address the + // user's browser can reach too. Empty = no apps. + apps: jsonb("apps").$type().notNull().default([]), + createdAt: timestamp("created_at").notNull().defaultNow(), + updatedAt: timestamp("updated_at") + .notNull() + .defaultNow() + .$onUpdate(() => new Date()), + deletedAt: timestamp("deleted_at"), // null = active; soft delete only (§4) +}); // --------------------------------------------------------------------------- // Automations (ADR 0102 → ADR 0119): versioned block-graph definitions whose @@ -1301,7 +1312,10 @@ export const routerModel = pgTable( available: boolean("available").notNull().default(true), updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }, - (t) => [primaryKey({ columns: [t.routerId, t.modelId] }), index("router_model_available_idx").on(t.routerId, t.available)], + (t) => [ + primaryKey({ columns: [t.routerId, t.modelId] }), + index("router_model_available_idx").on(t.routerId, t.available), + ], ); export const routerModelPolicy = pgTable( @@ -1315,7 +1329,10 @@ export const routerModelPolicy = pgTable( }, (t) => [ primaryKey({ columns: [t.routerId, t.modelId] }), - foreignKey({ columns: [t.routerId, t.modelId], foreignColumns: [routerModel.routerId, routerModel.modelId] }).onDelete("cascade"), + foreignKey({ + columns: [t.routerId, t.modelId], + foreignColumns: [routerModel.routerId, routerModel.modelId], + }).onDelete("cascade"), check("router_model_policy_user_implies_enabled", sql`NOT ${t.userEnabled} OR ${t.enabled}`), ], ); @@ -1743,6 +1760,15 @@ export const integrationConnection = pgTable( { id: text("id").primaryKey(), alias: text("alias").notNull().unique(), + mcpCredential: jsonb("mcp_credential") + .$type() + .notNull() + .default({}), + configRevision: integer("config_revision").notNull().default(1), + authGeneration: integer("auth_generation").notNull().default(1), + credentialIdentity: text("credential_identity").notNull().default(""), + catalogRevision: integer("catalog_revision").notNull().default(0), + revoked: boolean("revoked").notNull().default(false), provider: text("provider").notNull(), displayName: text("display_name").notNull(), isDefault: boolean("is_default").notNull().default(false), @@ -2155,3 +2181,280 @@ export const accountRelations = relations(account, ({ one }) => ({ references: [user.id], }), })); + +/** Immutable discovery and review records. Published revisions never change. */ +export const mcpCatalog = pgTable( + "mcp_catalog", + { + id: text("id").primaryKey(), + connectionId: text("connection_id") + .notNull() + .references(() => integrationConnection.id), + kind: text("kind").$type<"draft" | "published">().notNull(), + revision: integer("revision").notNull(), + configRevision: integer("config_revision").notNull(), + authGeneration: integer("auth_generation").notNull(), + credentialIdentity: text("credential_identity").notNull(), + tools: jsonb("tools") + .$type< + import("../mcp/types.ts").McpDiscoveredTool[] | import("../mcp/types.ts").McpReviewedTool[] + >() + .notNull(), + fingerprint: text("fingerprint").notNull(), + actorId: text("actor_id").notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull(), + }, + (t) => [ + index("mcp_catalog_connection_idx").on(t.connectionId, t.createdAt), + uniqueIndex("mcp_catalog_published_revision_idx") + .on(t.connectionId, t.revision) + .where(sql`kind = 'published'`), + check( + "mcp_catalog_kind_check", + sql`${t.kind} IN ('draft','published') AND ((${t.kind} = 'draft' AND ${t.revision} = 0) OR (${t.kind} = 'published' AND ${t.revision} > 0))`, + ), + ], +); + +/** Authoritative upstream outcome. Guest completion is a separate projection. */ +export const mcpExecution = pgTable( + "mcp_execution", + { + id: text("id").primaryKey(), + taskId: text("task_id").notNull(), + sessionId: text("session_id").notNull(), + runId: text("run_id").notNull(), + toolCallId: text("tool_call_id").notNull(), + connectionId: text("connection_id") + .notNull() + .references(() => integrationConnection.id), + catalogId: text("catalog_id") + .notNull() + .references(() => mcpCatalog.id), + principalId: text("principal_id").notNull(), + bindingHash: text("binding_hash").notNull(), + requestDigest: text("request_digest").notNull(), + argumentHash: text("argument_hash").notNull(), + originalArguments: text("original_arguments"), + requestContext: + jsonb("request_context").$type(), + upstreamName: text("upstream_name").notNull(), + configRevision: integer("config_revision").notNull(), + authGeneration: integer("auth_generation").notNull(), + classification: text("classification").$type<"read" | "write">().notNull(), + authorizationKind: text("authorization_kind") + .$type<"read" | "approval" | "automation">() + .notNull(), + automationGrantHash: text("automation_grant_hash"), + state: text("state").$type().notNull(), + version: integer("version").notNull().default(1), + bootId: text("boot_id"), + intentId: text("intent_id"), + intentExpiresAt: timestamp("intent_expires_at", { withTimezone: true }), + deadline: timestamp("deadline", { withTimezone: true }).notNull(), + result: jsonb("result").$type>(), + resultUnavailable: boolean("result_unavailable").notNull().default(false), + // Delivery settlement: enqueue ACK or authoritative target-gone rejection, never guest/effect proof. + completionEnqueuedVersion: integer("completion_enqueued_version").notNull().default(0), + createdAt: timestamp("created_at", { withTimezone: true }).notNull(), + updatedAt: timestamp("updated_at", { withTimezone: true }).notNull(), + }, + (t) => [ + foreignKey({ + columns: [t.taskId, t.sessionId], + foreignColumns: [taskSession.taskId, taskSession.sessionId], + }).onDelete("cascade"), + index("mcp_execution_task_session_idx").on(t.taskId, t.sessionId), + uniqueIndex("mcp_execution_call_idx").on(t.sessionId, t.toolCallId), + index("mcp_execution_connection_state_idx").on(t.connectionId, t.sessionId, t.state), + index("mcp_execution_catalog_idx").on(t.catalogId), + index("mcp_execution_pending_idx").on(t.state, t.deadline), + index("mcp_execution_lifecycle_active_idx") + .on(t.id) + .where(sql`${t.state} IN ('waiting_approval','authorized','dispatching')`), + index("mcp_execution_recovery_idx") + .on(t.id) + .where(sql`${t.completionEnqueuedVersion} < ${t.version}`), + check( + "mcp_execution_completion_version_check", + sql`${t.completionEnqueuedVersion} >= 0 AND ${t.completionEnqueuedVersion} <= ${t.version}`, + ), + uniqueIndex("mcp_execution_intent_idx").on(t.intentId), + check( + "mcp_execution_state_check", + sql`${t.state} IN ('waiting_approval','authorized','dispatching','succeeded','failed','canceled','outcome_unknown')`, + ), + check("mcp_execution_classification_check", sql`${t.classification} IN ('read','write')`), + check( + "mcp_execution_authorization_check", + sql`(${t.authorizationKind} = 'read' AND ${t.classification} = 'read' AND ${t.automationGrantHash} IS NULL) OR (${t.authorizationKind} = 'approval' AND ${t.automationGrantHash} IS NULL) OR (${t.authorizationKind} = 'automation' AND ${t.classification} = 'write' AND ${t.automationGrantHash} IS NOT NULL)`, + ), + check( + "mcp_execution_dispatch_check", + sql`${t.state} NOT IN ('dispatching','outcome_unknown','succeeded') OR (${t.bootId} IS NOT NULL AND ${t.intentId} IS NOT NULL AND ${t.intentExpiresAt} IS NOT NULL)`, + ), + ], +); + +export const mcpApproval = pgTable( + "mcp_approval", + { + executionId: text("execution_id") + .primaryKey() + .references(() => mcpExecution.id, { onDelete: "cascade" }), + deploymentId: text("deployment_id").notNull(), + sessionId: text("session_id").notNull(), + requestDigest: text("request_digest").notNull(), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + decision: text("decision").$type<"pending" | "approved" | "denied" | "expired">().notNull(), + actorId: text("actor_id"), + decidedAt: timestamp("decided_at", { withTimezone: true }), + consumedAt: timestamp("consumed_at", { withTimezone: true }), + }, + (t) => [ + index("mcp_approval_pending_idx").on(t.deploymentId, t.decision, t.expiresAt), + index("mcp_approval_session_idx").on(t.sessionId, t.decision), + check( + "mcp_approval_decision_check", + sql`${t.decision} IN ('pending','approved','denied','expired')`, + ), + check( + "mcp_approval_consumed_check", + sql`${t.consumedAt} IS NULL OR (${t.decision} = 'approved' AND ${t.actorId} IS NOT NULL)`, + ), + ], +); + +/** A cancellation is a durable decision, not proof that an upstream effect was undone. */ +export const mcpCancellation = pgTable( + "mcp_cancellation", + { + executionId: text("execution_id") + .primaryKey() + .references(() => mcpExecution.id, { onDelete: "cascade" }), + actorId: text("actor_id"), + cause: text("cause").notNull(), + expectedJournalRevision: integer("expected_journal_revision").notNull(), + journalRevision: integer("journal_revision").notNull(), + state: text("state").$type<"canceled" | "outcome_unknown">().notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull(), + }, + (t) => [ + check( + "mcp_cancellation_revision_check", + sql`${t.expectedJournalRevision} > 0 AND ${t.journalRevision} = ${t.expectedJournalRevision} + 1`, + ), + check("mcp_cancellation_state_check", sql`${t.state} IN ('canceled','outcome_unknown')`), + ], +); + +export const mcpResolution = pgTable( + "mcp_resolution", + { + executionId: text("execution_id") + .primaryKey() + .references(() => mcpExecution.id, { onDelete: "cascade" }), + expectedContext: + jsonb("expected_context").$type(), + actorId: text("actor_id").notNull(), + outcome: text("outcome").$type<"succeeded" | "failed" | "acknowledged">().notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull(), + }, + (t) => [ + check( + "mcp_resolution_outcome_check", + sql`${t.outcome} IN ('succeeded','failed','acknowledged')`, + ), + ], +); + +/** Durable row locks serialize short admission decisions across replicas. */ +export const mcpAdmissionScope = pgTable("mcp_admission_scope", { id: text("id").primaryKey() }); +export const mcpAdmission = pgTable( + "mcp_admission", + { + executionId: text("execution_id") + .primaryKey() + .references(() => mcpExecution.id, { onDelete: "cascade" }), + reservationId: text("reservation_id") + .notNull() + .default(sql`gen_random_uuid()::text`), + deploymentId: text("deployment_id") + .notNull() + .references(() => mcpAdmissionScope.id), + connectionId: text("connection_id") + .notNull() + .references(() => integrationConnection.id), + sessionId: text("session_id").notNull(), + phase: text("phase").$type<"schema" | "execute">().notNull().default("execute"), + schemaQueueExpiresAt: timestamp("schema_queue_expires_at", { withTimezone: true }), + executeQueueExpiresAt: timestamp("execute_queue_expires_at", { withTimezone: true }), + executeExpiresAt: timestamp("execute_expires_at", { withTimezone: true }), + schemaDigest: text("schema_digest"), + state: text("state").$type<"queued" | "admitted" | "released">().notNull(), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull(), + }, + (t) => [ + uniqueIndex("mcp_admission_reservation_idx").on(t.reservationId), + index("mcp_admission_deployment_idx").on(t.deploymentId, t.state, t.expiresAt), + index("mcp_admission_connection_idx").on(t.connectionId, t.state), + index("mcp_admission_session_idx").on(t.sessionId, t.state), + check("mcp_admission_phase_check", sql`${t.phase} IN ('schema','execute')`), + check("mcp_admission_state_check", sql`${t.state} IN ('queued','admitted','released')`), + ], +); + +/** Single-use administrative discovery. Contains authority metadata only. */ +export const mcpDiscoveryAdmission = pgTable( + "mcp_discovery_admission", + { + requestId: text("request_id").primaryKey(), + intentId: text("intent_id").notNull(), + deploymentId: text("deployment_id") + .notNull() + .references(() => mcpAdmissionScope.id), + connectionId: text("connection_id") + .notNull() + .references(() => integrationConnection.id), + actorId: text("actor_id").notNull(), + configRevision: integer("config_revision").notNull(), + authGeneration: integer("auth_generation").notNull(), + credentialIdentity: text("credential_identity").notNull(), + identity: jsonb("identity") + .$type() + .notNull(), + destination: text("destination").notNull(), + operation: jsonb("operation") + .$type() + .notNull() + .default({ kind: "discover" }), + requestDigest: text("request_digest").notNull(), + statusDigest: text("status_digest").notNull(), + state: text("state").$type<"reserved" | "dispatched" | "finished">().notNull(), + expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), + createdAt: timestamp("created_at", { withTimezone: true }).notNull(), + }, + (t) => [ + uniqueIndex("mcp_discovery_admission_intent_idx").on(t.intentId), + index("mcp_discovery_admission_deployment_idx").on(t.deploymentId, t.state, t.expiresAt), + index("mcp_discovery_admission_connection_idx").on(t.connectionId, t.state, t.expiresAt), + check( + "mcp_administrative_operation_check", + sql`coalesce( + ${t.operation} = '{"kind":"discover"}'::jsonb OR ( + ${t.operation}->>'kind' = 'validateSchemas' AND + ${t.operation}->>'documentsDigest' ~ '^sha256:[0-9a-f]{64}$' AND + ${t.operation} - 'kind' - 'documentsDigest' = '{}'::jsonb AND ${t.destination} = '' + ), false)`, + ), + check( + "mcp_discovery_admission_state_check", + sql`${t.state} IN ('reserved','dispatched','finished')`, + ), + check( + "mcp_discovery_admission_expiry_check", + sql`${t.expiresAt} = ${t.createdAt} + interval '30 seconds'`, + ), + ], +); diff --git a/orchestrator/src/gen/engram/app/v1/integration_pb.ts b/orchestrator/src/gen/engram/app/v1/integration_pb.ts index aa87250b6..b39bb0b4c 100644 --- a/orchestrator/src/gen/engram/app/v1/integration_pb.ts +++ b/orchestrator/src/gen/engram/app/v1/integration_pb.ts @@ -2,15 +2,17 @@ // @generated from file engram/app/v1/integration.proto (package engram.app.v1, syntax proto3) /* eslint-disable */ -import type { GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv1"; -import { fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv1"; +import type { GenEnum, GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv1"; +import { enumDesc, fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv1"; +import type { GetMcpWritePolicyResponse } from "./session_pb"; +import { file_engram_app_v1_session } from "./session_pb"; import type { Message } from "@bufbuild/protobuf"; /** * Describes the file engram/app/v1/integration.proto. */ export const file_engram_app_v1_integration: GenFile = /*@__PURE__*/ - fileDesc("Ch9lbmdyYW0vYXBwL3YxL2ludGVncmF0aW9uLnByb3RvEg1lbmdyYW0uYXBwLnYxIpgBChtHb29nbGVDbG91ZENvbm5lY3Rpb25Db25maWcSIgoad29ya2xvYWRfaWRlbnRpdHlfcHJvdmlkZXIYASABKAkSHQoVc2VydmljZV9hY2NvdW50X2VtYWlsGAIgASgJEhEKCWVuZHBvaW50cxgDIAMoCRIjChtjbG91ZF9zcWxfcG9zdGdyZXNfaW5zdGFuY2UYBCABKAki/AEKFUludGVncmF0aW9uQ29ubmVjdGlvbhIKCgJpZBgBIAEoCRINCgVhbGlhcxgCIAEoCRIQCghwcm92aWRlchgDIAEoCRIUCgxkaXNwbGF5X25hbWUYBCABKAkSDwoHZW5hYmxlZBgFIAEoCBIRCgl0ZXN0ZWRfYXQYBiABKAkSEgoKY3JlYXRlZF9hdBgHIAEoCRISCgp1cGRhdGVkX2F0GAggASgJEkAKDGdvb2dsZV9jbG91ZBgJIAEoCzIqLmVuZ3JhbS5hcHAudjEuR29vZ2xlQ2xvdWRDb25uZWN0aW9uQ29uZmlnEhIKCmlzX2RlZmF1bHQYCiABKAgiGAoWTGlzdENvbm5lY3Rpb25zUmVxdWVzdCJUChdMaXN0Q29ubmVjdGlvbnNSZXNwb25zZRI5Cgtjb25uZWN0aW9ucxgBIAMoCzIkLmVuZ3JhbS5hcHAudjEuSW50ZWdyYXRpb25Db25uZWN0aW9uIpIBChdDcmVhdGVDb25uZWN0aW9uUmVxdWVzdBINCgVhbGlhcxgBIAEoCRIQCghwcm92aWRlchgCIAEoCRIUCgxkaXNwbGF5X25hbWUYAyABKAkSQAoMZ29vZ2xlX2Nsb3VkGAQgASgLMiouZW5ncmFtLmFwcC52MS5Hb29nbGVDbG91ZENvbm5lY3Rpb25Db25maWciVAoYQ3JlYXRlQ29ubmVjdGlvblJlc3BvbnNlEjgKCmNvbm5lY3Rpb24YASABKAsyJC5lbmdyYW0uYXBwLnYxLkludGVncmF0aW9uQ29ubmVjdGlvbiKMAQoXVXBkYXRlQ29ubmVjdGlvblJlcXVlc3QSCgoCaWQYASABKAkSDQoFYWxpYXMYAiABKAkSFAoMZGlzcGxheV9uYW1lGAMgASgJEkAKDGdvb2dsZV9jbG91ZBgEIAEoCzIqLmVuZ3JhbS5hcHAudjEuR29vZ2xlQ2xvdWRDb25uZWN0aW9uQ29uZmlnIlQKGFVwZGF0ZUNvbm5lY3Rpb25SZXNwb25zZRI4Cgpjb25uZWN0aW9uGAEgASgLMiQuZW5ncmFtLmFwcC52MS5JbnRlZ3JhdGlvbkNvbm5lY3Rpb24iJQoXRGVsZXRlQ29ubmVjdGlvblJlcXVlc3QSCgoCaWQYASABKAkiKwoYRGVsZXRlQ29ubmVjdGlvblJlc3BvbnNlEg8KB2RlbGV0ZWQYASABKAgiIwoVVGVzdENvbm5lY3Rpb25SZXF1ZXN0EgoKAmlkGAEgASgJIjUKFlRlc3RDb25uZWN0aW9uUmVzcG9uc2USCgoCb2sYASABKAgSDwoHbWVzc2FnZRgCIAEoCSI6ChtTZXRDb25uZWN0aW9uRW5hYmxlZFJlcXVlc3QSCgoCaWQYASABKAkSDwoHZW5hYmxlZBgCIAEoCCJYChxTZXRDb25uZWN0aW9uRW5hYmxlZFJlc3BvbnNlEjgKCmNvbm5lY3Rpb24YASABKAsyJC5lbmdyYW0uYXBwLnYxLkludGVncmF0aW9uQ29ubmVjdGlvbiIoChpHZXRHb29nbGVDbG91ZFNldHVwUmVxdWVzdBIKCgJpZBgBIAEoCSKiAQobR2V0R29vZ2xlQ2xvdWRTZXR1cFJlc3BvbnNlEg4KBmlzc3VlchgBIAEoCRIQCghhdWRpZW5jZRgCIAEoCRIZChFzdWJqZWN0X2F0dHJpYnV0ZRgDIAEoCRIcChRjb25uZWN0aW9uX2F0dHJpYnV0ZRgEIAEoCRIVCg1nY2xvdWRfc2NyaXB0GAUgASgJEhEKCXRlcnJhZm9ybRgGIAEoCSLmAQoJQ29ubmVjdG9yEhAKCHByb3ZpZGVyGAEgASgJEhMKC2NvbmZpZ19qc29uGAIgASgJEg8KB2J1aWx0aW4YAyABKAgSEgoKY3JlYXRlZF9hdBgEIAEoCRISCgp1cGRhdGVkX2F0GAUgASgJEg4KBnN0YXR1cxgGIAEoCRI4CghzZXR0aW5ncxgHIAMoCzImLmVuZ3JhbS5hcHAudjEuQ29ubmVjdG9yLlNldHRpbmdzRW50cnkaLwoNU2V0dGluZ3NFbnRyeRILCgNrZXkYASABKAkSDQoFdmFsdWUYAiABKAk6AjgBIhcKFUxpc3RDb25uZWN0b3JzUmVxdWVzdCJGChZMaXN0Q29ubmVjdG9yc1Jlc3BvbnNlEiwKCmNvbm5lY3RvcnMYASADKAsyGC5lbmdyYW0uYXBwLnYxLkNvbm5lY3RvciItChZVcHNlcnRDb25uZWN0b3JSZXF1ZXN0EhMKC2NvbmZpZ19qc29uGAEgASgJIkYKF1Vwc2VydENvbm5lY3RvclJlc3BvbnNlEisKCWNvbm5lY3RvchgBIAEoCzIYLmVuZ3JhbS5hcHAudjEuQ29ubmVjdG9yIioKFkRlbGV0ZUNvbm5lY3RvclJlcXVlc3QSEAoIcHJvdmlkZXIYASABKAkiKgoXRGVsZXRlQ29ubmVjdG9yUmVzcG9uc2USDwoHZGVsZXRlZBgBIAEoCCI6Cg1Db25uZWN0b3JJY29uEgwKBG1vbm8YASABKAkSDQoFY29sb3IYAiABKAkSDAoEbG9nbxgDIAEoCSJ/ChBDb25uZWN0b3JEaXNwbGF5EgwKBG5hbWUYASABKAkSEAoIY2F0ZWdvcnkYAiABKAkSDQoFYmx1cmIYAyABKAkSKgoEaWNvbhgEIAEoCzIcLmVuZ3JhbS5hcHAudjEuQ29ubmVjdG9ySWNvbhIQCghmZWF0dXJlZBgFIAEoCCJ2ChFDYXRhbG9nQ2FwYWJpbGl0eRIOCgZhY3Rpb24YASABKAkSDgoGYWNjZXNzGAIgASgJEg0KBWFzc2V0GAMgASgJEg0KBWxhYmVsGAQgASgJEgwKBGhvc3QYBSABKAkSFQoNZW5kcG9pbnRfcnVsZRgGIAEoCSK0AgoUUHJvdmlkZXJDYXRhbG9nRW50cnkSEAoIcHJvdmlkZXIYASABKAkSMAoHZGlzcGxheRgCIAEoCzIfLmVuZ3JhbS5hcHAudjEuQ29ubmVjdG9yRGlzcGxheRIZChFjcmVkZW50aWFsX3NvdXJjZRgDIAEoCRINCgVob3N0cxgEIAMoCRI2CgxjYXBhYmlsaXRpZXMYBSADKAsyIC5lbmdyYW0uYXBwLnYxLkNhdGFsb2dDYXBhYmlsaXR5Eh0KFWRlZmF1bHRfY29ubmVjdGlvbl9pZBgGIAEoCRIYChBjb25uZWN0aW9uX21vZGVsGAcgASgJEj0KD3VzZXJfY3JlZGVudGlhbBgIIAEoCzIkLmVuZ3JhbS5hcHAudjEuVXNlckNyZWRlbnRpYWxTdXBwb3J0IkkKFVVzZXJDcmVkZW50aWFsU3VwcG9ydBINCgVvYXV0aBgBIAEoCBINCgV0b2tlbhgCIAEoCBISCgp0b2tlbl9oaW50GAMgASgJIh4KHEdldEludGVncmF0aW9uQ2F0YWxvZ1JlcXVlc3QiVwodR2V0SW50ZWdyYXRpb25DYXRhbG9nUmVzcG9uc2USNgoJcHJvdmlkZXJzGAEgAygLMiMuZW5ncmFtLmFwcC52MS5Qcm92aWRlckNhdGFsb2dFbnRyeSKuAQoYU2V0TWludENyZWRlbnRpYWxSZXF1ZXN0EhAKCHByb3ZpZGVyGAEgASgJEgwKBGtpbmQYAiABKAkSQwoGdmFsdWVzGAMgAygLMjMuZW5ncmFtLmFwcC52MS5TZXRNaW50Q3JlZGVudGlhbFJlcXVlc3QuVmFsdWVzRW50cnkaLQoLVmFsdWVzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASIxChlTZXRNaW50Q3JlZGVudGlhbFJlc3BvbnNlEhQKDHNlY3JldF9uYW1lcxgBIAMoCSJQChpVcGxvYWRDb25uZWN0b3JMb2dvUmVxdWVzdBIQCghwcm92aWRlchgBIAEoCRIMCgRkYXRhGAIgASgMEhIKCm1lZGlhX3R5cGUYAyABKAkiLwobVXBsb2FkQ29ubmVjdG9yTG9nb1Jlc3BvbnNlEhAKCGxvZ29fdXJsGAEgASgJIq4CChRUZXN0Q29ubmVjdG9yUmVxdWVzdBIQCghwcm92aWRlchgBIAEoCRJKCgxkcmFmdF92YWx1ZXMYAiADKAsyNC5lbmdyYW0uYXBwLnYxLlRlc3RDb25uZWN0b3JSZXF1ZXN0LkRyYWZ0VmFsdWVzRW50cnkSTgoOZHJhZnRfc2V0dGluZ3MYAyADKAsyNi5lbmdyYW0uYXBwLnYxLlRlc3RDb25uZWN0b3JSZXF1ZXN0LkRyYWZ0U2V0dGluZ3NFbnRyeRoyChBEcmFmdFZhbHVlc0VudHJ5EgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCToCOAEaNAoSRHJhZnRTZXR0aW5nc0VudHJ5EgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCToCOAEiNAoVVGVzdENvbm5lY3RvclJlc3BvbnNlEgoKAm9rGAEgASgIEg8KB21lc3NhZ2UYAiABKAkirAEKG1NldENvbm5lY3RvclNldHRpbmdzUmVxdWVzdBIQCghwcm92aWRlchgBIAEoCRJKCghzZXR0aW5ncxgCIAMoCzI4LmVuZ3JhbS5hcHAudjEuU2V0Q29ubmVjdG9yU2V0dGluZ3NSZXF1ZXN0LlNldHRpbmdzRW50cnkaLwoNU2V0dGluZ3NFbnRyeRILCgNrZXkYASABKAkSDQoFdmFsdWUYAiABKAk6AjgBIksKHFNldENvbm5lY3RvclNldHRpbmdzUmVzcG9uc2USKwoJY29ubmVjdG9yGAEgASgLMhguZW5ncmFtLmFwcC52MS5Db25uZWN0b3IynQwKEkludGVncmF0aW9uU2VydmljZRJdCg5MaXN0Q29ubmVjdG9ycxIkLmVuZ3JhbS5hcHAudjEuTGlzdENvbm5lY3RvcnNSZXF1ZXN0GiUuZW5ncmFtLmFwcC52MS5MaXN0Q29ubmVjdG9yc1Jlc3BvbnNlEmAKD1Vwc2VydENvbm5lY3RvchIlLmVuZ3JhbS5hcHAudjEuVXBzZXJ0Q29ubmVjdG9yUmVxdWVzdBomLmVuZ3JhbS5hcHAudjEuVXBzZXJ0Q29ubmVjdG9yUmVzcG9uc2USYAoPRGVsZXRlQ29ubmVjdG9yEiUuZW5ncmFtLmFwcC52MS5EZWxldGVDb25uZWN0b3JSZXF1ZXN0GiYuZW5ncmFtLmFwcC52MS5EZWxldGVDb25uZWN0b3JSZXNwb25zZRJyChVHZXRJbnRlZ3JhdGlvbkNhdGFsb2cSKy5lbmdyYW0uYXBwLnYxLkdldEludGVncmF0aW9uQ2F0YWxvZ1JlcXVlc3QaLC5lbmdyYW0uYXBwLnYxLkdldEludGVncmF0aW9uQ2F0YWxvZ1Jlc3BvbnNlEmYKEVNldE1pbnRDcmVkZW50aWFsEicuZW5ncmFtLmFwcC52MS5TZXRNaW50Q3JlZGVudGlhbFJlcXVlc3QaKC5lbmdyYW0uYXBwLnYxLlNldE1pbnRDcmVkZW50aWFsUmVzcG9uc2USbAoTVXBsb2FkQ29ubmVjdG9yTG9nbxIpLmVuZ3JhbS5hcHAudjEuVXBsb2FkQ29ubmVjdG9yTG9nb1JlcXVlc3QaKi5lbmdyYW0uYXBwLnYxLlVwbG9hZENvbm5lY3RvckxvZ29SZXNwb25zZRJaCg1UZXN0Q29ubmVjdG9yEiMuZW5ncmFtLmFwcC52MS5UZXN0Q29ubmVjdG9yUmVxdWVzdBokLmVuZ3JhbS5hcHAudjEuVGVzdENvbm5lY3RvclJlc3BvbnNlEm8KFFNldENvbm5lY3RvclNldHRpbmdzEiouZW5ncmFtLmFwcC52MS5TZXRDb25uZWN0b3JTZXR0aW5nc1JlcXVlc3QaKy5lbmdyYW0uYXBwLnYxLlNldENvbm5lY3RvclNldHRpbmdzUmVzcG9uc2USYAoPTGlzdENvbm5lY3Rpb25zEiUuZW5ncmFtLmFwcC52MS5MaXN0Q29ubmVjdGlvbnNSZXF1ZXN0GiYuZW5ncmFtLmFwcC52MS5MaXN0Q29ubmVjdGlvbnNSZXNwb25zZRJjChBDcmVhdGVDb25uZWN0aW9uEiYuZW5ncmFtLmFwcC52MS5DcmVhdGVDb25uZWN0aW9uUmVxdWVzdBonLmVuZ3JhbS5hcHAudjEuQ3JlYXRlQ29ubmVjdGlvblJlc3BvbnNlEmMKEFVwZGF0ZUNvbm5lY3Rpb24SJi5lbmdyYW0uYXBwLnYxLlVwZGF0ZUNvbm5lY3Rpb25SZXF1ZXN0GicuZW5ncmFtLmFwcC52MS5VcGRhdGVDb25uZWN0aW9uUmVzcG9uc2USYwoQRGVsZXRlQ29ubmVjdGlvbhImLmVuZ3JhbS5hcHAudjEuRGVsZXRlQ29ubmVjdGlvblJlcXVlc3QaJy5lbmdyYW0uYXBwLnYxLkRlbGV0ZUNvbm5lY3Rpb25SZXNwb25zZRJdCg5UZXN0Q29ubmVjdGlvbhIkLmVuZ3JhbS5hcHAudjEuVGVzdENvbm5lY3Rpb25SZXF1ZXN0GiUuZW5ncmFtLmFwcC52MS5UZXN0Q29ubmVjdGlvblJlc3BvbnNlEm8KFFNldENvbm5lY3Rpb25FbmFibGVkEiouZW5ncmFtLmFwcC52MS5TZXRDb25uZWN0aW9uRW5hYmxlZFJlcXVlc3QaKy5lbmdyYW0uYXBwLnYxLlNldENvbm5lY3Rpb25FbmFibGVkUmVzcG9uc2USbAoTR2V0R29vZ2xlQ2xvdWRTZXR1cBIpLmVuZ3JhbS5hcHAudjEuR2V0R29vZ2xlQ2xvdWRTZXR1cFJlcXVlc3QaKi5lbmdyYW0uYXBwLnYxLkdldEdvb2dsZUNsb3VkU2V0dXBSZXNwb25zZWIGcHJvdG8z"); + fileDesc("Ch9lbmdyYW0vYXBwL3YxL2ludGVncmF0aW9uLnByb3RvEg1lbmdyYW0uYXBwLnYxIpgBChtHb29nbGVDbG91ZENvbm5lY3Rpb25Db25maWcSIgoad29ya2xvYWRfaWRlbnRpdHlfcHJvdmlkZXIYASABKAkSHQoVc2VydmljZV9hY2NvdW50X2VtYWlsGAIgASgJEhEKCWVuZHBvaW50cxgDIAMoCRIjChtjbG91ZF9zcWxfcG9zdGdyZXNfaW5zdGFuY2UYBCABKAkipwMKFUludGVncmF0aW9uQ29ubmVjdGlvbhIKCgJpZBgBIAEoCRINCgVhbGlhcxgCIAEoCRIQCghwcm92aWRlchgDIAEoCRIUCgxkaXNwbGF5X25hbWUYBCABKAkSDwoHZW5hYmxlZBgFIAEoCBIRCgl0ZXN0ZWRfYXQYBiABKAkSEgoKY3JlYXRlZF9hdBgHIAEoCRISCgp1cGRhdGVkX2F0GAggASgJEkAKDGdvb2dsZV9jbG91ZBgJIAEoCzIqLmVuZ3JhbS5hcHAudjEuR29vZ2xlQ2xvdWRDb25uZWN0aW9uQ29uZmlnEhIKCmlzX2RlZmF1bHQYCiABKAgSLwoDbWNwGAsgASgLMiIuZW5ncmFtLmFwcC52MS5NY3BDb25uZWN0aW9uQ29uZmlnEhcKD2NvbmZpZ19yZXZpc2lvbhgMIAEoDRIXCg9hdXRoX2dlbmVyYXRpb24YDSABKA0SGwoTY3JlZGVudGlhbF9pZGVudGl0eRgOIAEoCRIYChBjYXRhbG9nX3JldmlzaW9uGA8gASgNEg8KB3Jldm9rZWQYECABKAgiGAoWTGlzdENvbm5lY3Rpb25zUmVxdWVzdCJUChdMaXN0Q29ubmVjdGlvbnNSZXNwb25zZRI5Cgtjb25uZWN0aW9ucxgBIAMoCzIkLmVuZ3JhbS5hcHAudjEuSW50ZWdyYXRpb25Db25uZWN0aW9uIsMBChdDcmVhdGVDb25uZWN0aW9uUmVxdWVzdBINCgVhbGlhcxgBIAEoCRIQCghwcm92aWRlchgCIAEoCRIUCgxkaXNwbGF5X25hbWUYAyABKAkSQAoMZ29vZ2xlX2Nsb3VkGAQgASgLMiouZW5ncmFtLmFwcC52MS5Hb29nbGVDbG91ZENvbm5lY3Rpb25Db25maWcSLwoDbWNwGAUgASgLMiIuZW5ncmFtLmFwcC52MS5NY3BDb25uZWN0aW9uQ29uZmlnIlQKGENyZWF0ZUNvbm5lY3Rpb25SZXNwb25zZRI4Cgpjb25uZWN0aW9uGAEgASgLMiQuZW5ncmFtLmFwcC52MS5JbnRlZ3JhdGlvbkNvbm5lY3Rpb24i9QEKF1VwZGF0ZUNvbm5lY3Rpb25SZXF1ZXN0EhsKE2V4cGVjdGVkX2dlbmVyYXRpb24YByABKA0SGQoRZXhwZWN0ZWRfcmV2aXNpb24YBiABKA0SCgoCaWQYASABKAkSDQoFYWxpYXMYAiABKAkSFAoMZGlzcGxheV9uYW1lGAMgASgJEkAKDGdvb2dsZV9jbG91ZBgEIAEoCzIqLmVuZ3JhbS5hcHAudjEuR29vZ2xlQ2xvdWRDb25uZWN0aW9uQ29uZmlnEi8KA21jcBgFIAEoCzIiLmVuZ3JhbS5hcHAudjEuTWNwQ29ubmVjdGlvbkNvbmZpZyJUChhVcGRhdGVDb25uZWN0aW9uUmVzcG9uc2USOAoKY29ubmVjdGlvbhgBIAEoCzIkLmVuZ3JhbS5hcHAudjEuSW50ZWdyYXRpb25Db25uZWN0aW9uIl0KF0RlbGV0ZUNvbm5lY3Rpb25SZXF1ZXN0EhkKEWV4cGVjdGVkX3JldmlzaW9uGAIgASgNEhsKE2V4cGVjdGVkX2dlbmVyYXRpb24YAyABKA0SCgoCaWQYASABKAkiKwoYRGVsZXRlQ29ubmVjdGlvblJlc3BvbnNlEg8KB2RlbGV0ZWQYASABKAgiIwoVVGVzdENvbm5lY3Rpb25SZXF1ZXN0EgoKAmlkGAEgASgJIjUKFlRlc3RDb25uZWN0aW9uUmVzcG9uc2USCgoCb2sYASABKAgSDwoHbWVzc2FnZRgCIAEoCSJVChtTZXRDb25uZWN0aW9uRW5hYmxlZFJlcXVlc3QSCgoCaWQYASABKAkSDwoHZW5hYmxlZBgCIAEoCBIZChFleHBlY3RlZF9yZXZpc2lvbhgDIAEoDSJYChxTZXRDb25uZWN0aW9uRW5hYmxlZFJlc3BvbnNlEjgKCmNvbm5lY3Rpb24YASABKAsyJC5lbmdyYW0uYXBwLnYxLkludGVncmF0aW9uQ29ubmVjdGlvbiIoChpHZXRHb29nbGVDbG91ZFNldHVwUmVxdWVzdBIKCgJpZBgBIAEoCSKiAQobR2V0R29vZ2xlQ2xvdWRTZXR1cFJlc3BvbnNlEg4KBmlzc3VlchgBIAEoCRIQCghhdWRpZW5jZRgCIAEoCRIZChFzdWJqZWN0X2F0dHJpYnV0ZRgDIAEoCRIcChRjb25uZWN0aW9uX2F0dHJpYnV0ZRgEIAEoCRIVCg1nY2xvdWRfc2NyaXB0GAUgASgJEhEKCXRlcnJhZm9ybRgGIAEoCSLmAQoJQ29ubmVjdG9yEhAKCHByb3ZpZGVyGAEgASgJEhMKC2NvbmZpZ19qc29uGAIgASgJEg8KB2J1aWx0aW4YAyABKAgSEgoKY3JlYXRlZF9hdBgEIAEoCRISCgp1cGRhdGVkX2F0GAUgASgJEg4KBnN0YXR1cxgGIAEoCRI4CghzZXR0aW5ncxgHIAMoCzImLmVuZ3JhbS5hcHAudjEuQ29ubmVjdG9yLlNldHRpbmdzRW50cnkaLwoNU2V0dGluZ3NFbnRyeRILCgNrZXkYASABKAkSDQoFdmFsdWUYAiABKAk6AjgBIhcKFUxpc3RDb25uZWN0b3JzUmVxdWVzdCJGChZMaXN0Q29ubmVjdG9yc1Jlc3BvbnNlEiwKCmNvbm5lY3RvcnMYASADKAsyGC5lbmdyYW0uYXBwLnYxLkNvbm5lY3RvciItChZVcHNlcnRDb25uZWN0b3JSZXF1ZXN0EhMKC2NvbmZpZ19qc29uGAEgASgJIkYKF1Vwc2VydENvbm5lY3RvclJlc3BvbnNlEisKCWNvbm5lY3RvchgBIAEoCzIYLmVuZ3JhbS5hcHAudjEuQ29ubmVjdG9yIioKFkRlbGV0ZUNvbm5lY3RvclJlcXVlc3QSEAoIcHJvdmlkZXIYASABKAkiKgoXRGVsZXRlQ29ubmVjdG9yUmVzcG9uc2USDwoHZGVsZXRlZBgBIAEoCCI6Cg1Db25uZWN0b3JJY29uEgwKBG1vbm8YASABKAkSDQoFY29sb3IYAiABKAkSDAoEbG9nbxgDIAEoCSJ/ChBDb25uZWN0b3JEaXNwbGF5EgwKBG5hbWUYASABKAkSEAoIY2F0ZWdvcnkYAiABKAkSDQoFYmx1cmIYAyABKAkSKgoEaWNvbhgEIAEoCzIcLmVuZ3JhbS5hcHAudjEuQ29ubmVjdG9ySWNvbhIQCghmZWF0dXJlZBgFIAEoCCJ2ChFDYXRhbG9nQ2FwYWJpbGl0eRIOCgZhY3Rpb24YASABKAkSDgoGYWNjZXNzGAIgASgJEg0KBWFzc2V0GAMgASgJEg0KBWxhYmVsGAQgASgJEgwKBGhvc3QYBSABKAkSFQoNZW5kcG9pbnRfcnVsZRgGIAEoCSK0AgoUUHJvdmlkZXJDYXRhbG9nRW50cnkSEAoIcHJvdmlkZXIYASABKAkSMAoHZGlzcGxheRgCIAEoCzIfLmVuZ3JhbS5hcHAudjEuQ29ubmVjdG9yRGlzcGxheRIZChFjcmVkZW50aWFsX3NvdXJjZRgDIAEoCRINCgVob3N0cxgEIAMoCRI2CgxjYXBhYmlsaXRpZXMYBSADKAsyIC5lbmdyYW0uYXBwLnYxLkNhdGFsb2dDYXBhYmlsaXR5Eh0KFWRlZmF1bHRfY29ubmVjdGlvbl9pZBgGIAEoCRIYChBjb25uZWN0aW9uX21vZGVsGAcgASgJEj0KD3VzZXJfY3JlZGVudGlhbBgIIAEoCzIkLmVuZ3JhbS5hcHAudjEuVXNlckNyZWRlbnRpYWxTdXBwb3J0IkkKFVVzZXJDcmVkZW50aWFsU3VwcG9ydBINCgVvYXV0aBgBIAEoCBINCgV0b2tlbhgCIAEoCBISCgp0b2tlbl9oaW50GAMgASgJIh4KHEdldEludGVncmF0aW9uQ2F0YWxvZ1JlcXVlc3QiVwodR2V0SW50ZWdyYXRpb25DYXRhbG9nUmVzcG9uc2USNgoJcHJvdmlkZXJzGAEgAygLMiMuZW5ncmFtLmFwcC52MS5Qcm92aWRlckNhdGFsb2dFbnRyeSKuAQoYU2V0TWludENyZWRlbnRpYWxSZXF1ZXN0EhAKCHByb3ZpZGVyGAEgASgJEgwKBGtpbmQYAiABKAkSQwoGdmFsdWVzGAMgAygLMjMuZW5ncmFtLmFwcC52MS5TZXRNaW50Q3JlZGVudGlhbFJlcXVlc3QuVmFsdWVzRW50cnkaLQoLVmFsdWVzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASIxChlTZXRNaW50Q3JlZGVudGlhbFJlc3BvbnNlEhQKDHNlY3JldF9uYW1lcxgBIAMoCSJQChpVcGxvYWRDb25uZWN0b3JMb2dvUmVxdWVzdBIQCghwcm92aWRlchgBIAEoCRIMCgRkYXRhGAIgASgMEhIKCm1lZGlhX3R5cGUYAyABKAkiLwobVXBsb2FkQ29ubmVjdG9yTG9nb1Jlc3BvbnNlEhAKCGxvZ29fdXJsGAEgASgJIq4CChRUZXN0Q29ubmVjdG9yUmVxdWVzdBIQCghwcm92aWRlchgBIAEoCRJKCgxkcmFmdF92YWx1ZXMYAiADKAsyNC5lbmdyYW0uYXBwLnYxLlRlc3RDb25uZWN0b3JSZXF1ZXN0LkRyYWZ0VmFsdWVzRW50cnkSTgoOZHJhZnRfc2V0dGluZ3MYAyADKAsyNi5lbmdyYW0uYXBwLnYxLlRlc3RDb25uZWN0b3JSZXF1ZXN0LkRyYWZ0U2V0dGluZ3NFbnRyeRoyChBEcmFmdFZhbHVlc0VudHJ5EgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCToCOAEaNAoSRHJhZnRTZXR0aW5nc0VudHJ5EgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCToCOAEiNAoVVGVzdENvbm5lY3RvclJlc3BvbnNlEgoKAm9rGAEgASgIEg8KB21lc3NhZ2UYAiABKAkirAEKG1NldENvbm5lY3RvclNldHRpbmdzUmVxdWVzdBIQCghwcm92aWRlchgBIAEoCRJKCghzZXR0aW5ncxgCIAMoCzI4LmVuZ3JhbS5hcHAudjEuU2V0Q29ubmVjdG9yU2V0dGluZ3NSZXF1ZXN0LlNldHRpbmdzRW50cnkaLwoNU2V0dGluZ3NFbnRyeRILCgNrZXkYASABKAkSDQoFdmFsdWUYAiABKAk6AjgBIksKHFNldENvbm5lY3RvclNldHRpbmdzUmVzcG9uc2USKwoJY29ubmVjdG9yGAEgASgLMhguZW5ncmFtLmFwcC52MS5Db25uZWN0b3IiqwEKE01jcENvbm5lY3Rpb25Db25maWcSEAoIZW5kcG9pbnQYASABKAkSDAoEem9uZRgCIAEoCRIRCglhdXRoX2tpbmQYAyABKAkSGgoSYXBpX2tleV9zZWNyZXRfcmVmGAQgASgJEiQKHG9hdXRoX2NsaWVudF9yZWdpc3RyYXRpb25faWQYBSABKAkSHwoXc2hhcmVfd2l0aF9vcmdfc2Vzc2lvbnMYBiABKAgiqgEKB01jcFRvb2wSFQoNdXBzdHJlYW1fbmFtZRgBIAEoCRITCgtkZXNjcmlwdGlvbhgCIAEoCRIZChFpbnB1dF9zY2hlbWFfanNvbhgDIAEoCRIaChJvdXRwdXRfc2NoZW1hX2pzb24YBCABKAkSFgoOY2xhc3NpZmljYXRpb24YBSABKAkSDwoHZW5hYmxlZBgGIAEoCBITCgtmaW5nZXJwcmludBgHIAEoCSK9AQoKTWNwQ2F0YWxvZxIKCgJpZBgBIAEoCRIVCg1jb25uZWN0aW9uX2lkGAIgASgJEgwKBGtpbmQYAyABKAkSEAoIcmV2aXNpb24YBCABKA0SFwoPY29uZmlnX3JldmlzaW9uGAUgASgNEhcKD2F1dGhfZ2VuZXJhdGlvbhgGIAEoDRIlCgV0b29scxgHIAMoCzIWLmVuZ3JhbS5hcHAudjEuTWNwVG9vbBITCgtmaW5nZXJwcmludBgIIAEoCSIuChVHZXRNY3BDYXRhbG9nc1JlcXVlc3QSFQoNY29ubmVjdGlvbl9pZBgBIAEoCSKjAQoWR2V0TWNwQ2F0YWxvZ3NSZXNwb25zZRIXCg9jb25maWdfcmV2aXNpb24YASABKA0SGAoQY2F0YWxvZ19yZXZpc2lvbhgCIAEoDRIoCgVkcmFmdBgDIAEoCzIZLmVuZ3JhbS5hcHAudjEuTWNwQ2F0YWxvZxIsCglwdWJsaXNoZWQYBCABKAsyGS5lbmdyYW0uYXBwLnYxLk1jcENhdGFsb2ciRgoSRGlzY292ZXJNY3BSZXF1ZXN0EhUKDWNvbm5lY3Rpb25faWQYASABKAkSGQoRZXhwZWN0ZWRfcmV2aXNpb24YAiABKA0iQQoTRGlzY292ZXJNY3BSZXNwb25zZRIqCgdjYXRhbG9nGAEgASgLMhkuZW5ncmFtLmFwcC52MS5NY3BDYXRhbG9nIn4KEVB1Ymxpc2hNY3BSZXF1ZXN0EhUKDWNvbm5lY3Rpb25faWQYASABKAkSEAoIZHJhZnRfaWQYAiABKAkSGQoRZXhwZWN0ZWRfcmV2aXNpb24YAyABKA0SJQoFdG9vbHMYBCADKAsyFi5lbmdyYW0uYXBwLnYxLk1jcFRvb2wiQAoSUHVibGlzaE1jcFJlc3BvbnNlEioKB2NhdGFsb2cYASABKAsyGS5lbmdyYW0uYXBwLnYxLk1jcENhdGFsb2cihQEKE0F1dGhvcml6ZU1jcFJlcXVlc3QSGQoRZXhwZWN0ZWRfcmV2aXNpb24YBCABKA0SFQoNY29ubmVjdGlvbl9pZBgBIAEoCRIbChNleHBlY3RlZF9nZW5lcmF0aW9uGAIgASgNEh8KF3NoYXJlX3dpdGhfb3JnX3Nlc3Npb25zGAMgASgIIogBChRBdXRob3JpemVNY3BSZXNwb25zZRIXCg9jb25maWdfcmV2aXNpb24YBCABKA0SFwoPYXV0aF9nZW5lcmF0aW9uGAUgASgNEg8KB2Zsb3dfaWQYAiABKAkSEgoKZXhwaXJlc19hdBgDIAEoCRIZChFhdXRob3JpemF0aW9uX3VybBgBIAEoCSJlChREaXNjb25uZWN0TWNwUmVxdWVzdBIZChFleHBlY3RlZF9yZXZpc2lvbhgDIAEoDRIVCg1jb25uZWN0aW9uX2lkGAEgASgJEhsKE2V4cGVjdGVkX2dlbmVyYXRpb24YAiABKA0iUQoVRGlzY29ubmVjdE1jcFJlc3BvbnNlEjgKCmNvbm5lY3Rpb24YASABKAsyJC5lbmdyYW0uYXBwLnYxLkludGVncmF0aW9uQ29ubmVjdGlvbiKPAwoLTWNwQXBwcm92YWwSFAoMZXhlY3V0aW9uX2lkGAEgASgJEhIKCnNlc3Npb25faWQYAiABKAkSDgoGcnVuX2lkGAMgASgJEhQKDHRvb2xfY2FsbF9pZBgEIAEoCRIVCg1jb25uZWN0aW9uX2lkGAUgASgJEhUKDXVwc3RyZWFtX25hbWUYBiABKAkSFgoOcmVxdWVzdF9kaWdlc3QYByABKAkSFQoNYXJndW1lbnRfaGFzaBgIIAEoCRISCgpleHBpcmVzX2F0GAkgASgJEhoKEm9yaWdpbmFsX2FyZ3VtZW50cxgKIAEoCRItCg1yZXZpZXdlZF90b29sGAsgASgLMhYuZW5ncmFtLmFwcC52MS5NY3BUb29sEioKImFyZ3VtZW50X2NvbnN0cmFpbnRfZG9jdW1lbnRzX2pzb24YDCADKAkSSAoXZXhwZWN0ZWRfcmV2aWV3X2NvbnRleHQYDSABKAsyJy5lbmdyYW0uYXBwLnYxLk1jcEFwcHJvdmFsUmV2aWV3Q29udGV4dCJJChdMaXN0TWNwQXBwcm92YWxzUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEhoKEmFmdGVyX2V4ZWN1dGlvbl9pZBgCIAEoCSJkChhMaXN0TWNwQXBwcm92YWxzUmVzcG9uc2USLQoJYXBwcm92YWxzGAEgAygLMhouZW5ncmFtLmFwcC52MS5NY3BBcHByb3ZhbBIZChFuZXh0X2V4ZWN1dGlvbl9pZBgCIAEoCSKLAQoYRGVjaWRlTWNwQXBwcm92YWxSZXF1ZXN0EhQKDGV4ZWN1dGlvbl9pZBgBIAEoCRIPCgdhcHByb3ZlGAIgASgIEkgKF2V4cGVjdGVkX3Jldmlld19jb250ZXh0GAMgASgLMicuZW5ncmFtLmFwcC52MS5NY3BBcHByb3ZhbFJldmlld0NvbnRleHQiXgoZRGVjaWRlTWNwQXBwcm92YWxSZXNwb25zZRINCgVzdGF0ZRgBIAEoCRIyCgdyZWNlaXB0GAIgASgLMiEuZW5ncmFtLmFwcC52MS5NY3BBcHByb3ZhbFJlY2VpcHQiTwoUQ2FuY2VsTWNwQ2FsbFJlcXVlc3QSFAoMZXhlY3V0aW9uX2lkGAEgASgJEiEKGWV4cGVjdGVkX2pvdXJuYWxfcmV2aXNpb24YAiABKA0iXgoVQ2FuY2VsTWNwQ2FsbFJlc3BvbnNlEg0KBXN0YXRlGAEgASgJEjYKB3JlY2VpcHQYAiABKAsyJS5lbmdyYW0uYXBwLnYxLk1jcENhbmNlbGxhdGlvblJlY2VpcHQihQEKGFJlc29sdmVNY3BPdXRjb21lUmVxdWVzdBIUCgxleGVjdXRpb25faWQYASABKAkSDwoHb3V0Y29tZRgCIAEoCRJCChhleHBlY3RlZF91bmtub3duX2NvbnRleHQYAyABKAsyIC5lbmdyYW0uYXBwLnYxLk1jcFVua25vd25Db250ZXh0ImIKGVJlc29sdmVNY3BPdXRjb21lUmVzcG9uc2USDwoHb3V0Y29tZRgBIAEoCRI0CgdyZWNlaXB0GAIgASgLMiMuZW5ncmFtLmFwcC52MS5NY3BSZXNvbHV0aW9uUmVjZWlwdCJMChdQcmVwYXJlTWNwUmViaW5kUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEh0KFWV4cGVjdGVkX2JpbmRpbmdfaGFzaBgCIAEoCSKpAgoYUHJlcGFyZU1jcFJlYmluZFJlc3BvbnNlEhQKDGJpbmRpbmdfaGFzaBgBIAEoCRIVCg1iaW5kaW5nX2Vwb2NoGAIgASgJEjkKDWJpbmRpbmdfdG9vbHMYAyADKAsyIi5lbmdyYW0uYXBwLnYxLk1jcEJvdW5kVG9vbFN1bW1hcnkSEgoKZGVueV9ob3N0cxgEIAMoCRIVCg1kZW55X3JldmlzaW9uGAUgASgJEhMKC2RlbnlfZGlnZXN0GAYgASgJEjcKC2Nvbm5lY3Rpb25zGAcgAygLMiIuZW5ncmFtLmFwcC52MS5NY3BSZWJpbmRDb25uZWN0aW9uEiwKBWRyYWluGAggASgLMh0uZW5ncmFtLmFwcC52MS5NY3BSZWJpbmREcmFpbiJaCg5NY3BSZWJpbmREcmFpbhIzCgZyZWFzb24YASABKA4yIy5lbmdyYW0uYXBwLnYxLk1jcFJlYmluZERyYWluUmVhc29uEhMKC3JldHJ5X2FmdGVyGAIgASgJIsQBChNNY3BSZWJpbmRDb25uZWN0aW9uEhUKDWNvbm5lY3Rpb25faWQYASABKAkSDQoFYWxpYXMYAiABKAkSEAoIZW5kcG9pbnQYAyABKAkSDAoEem9uZRgEIAEoCRIbChNjcmVkZW50aWFsX2lkZW50aXR5GAUgASgJEhcKD2NvbmZpZ19yZXZpc2lvbhgGIAEoDRIXCg9hdXRoX2dlbmVyYXRpb24YByABKA0SGAoQY2F0YWxvZ19yZXZpc2lvbhgIIAEoDSJmChBSZWJpbmRNY3BSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkSHQoVZXhwZWN0ZWRfYmluZGluZ19oYXNoGAIgASgJEh8KF2V4cGVjdGVkX2NhbmRpZGF0ZV9oYXNoGAMgASgJIikKEVJlYmluZE1jcFJlc3BvbnNlEhQKDGJpbmRpbmdfaGFzaBgBIAEoCSKVAQoXUHV0TWNwQ3JlZGVudGlhbFJlcXVlc3QSFQoNY29ubmVjdGlvbl9pZBgBIAEoCRIZChFleHBlY3RlZF9yZXZpc2lvbhgCIAEoDRIbChNleHBlY3RlZF9nZW5lcmF0aW9uGAMgASgNEgwKBGtpbmQYBCABKAkSDgoGaGVhZGVyGAUgASgJEg0KBXZhbHVlGAYgASgJIpQBChNNY3BDcmVkZW50aWFsU3RhdHVzEg4KBnN0YXR1cxgBIAEoCRIRCglyZWZlcmVuY2UYAiABKAkSDAoEa2luZBgDIAEoCRIbChNjcmVkZW50aWFsX2lkZW50aXR5GAQgASgJEg8KB3ZlcnNpb24YBSABKAMSHgoWcmVzZXJ2YXRpb25fZ2VuZXJhdGlvbhgGIAEoAyKMAQoYUHV0TWNwQ3JlZGVudGlhbFJlc3BvbnNlEjgKCmNvbm5lY3Rpb24YASABKAsyJC5lbmdyYW0uYXBwLnYxLkludGVncmF0aW9uQ29ubmVjdGlvbhI2CgpjcmVkZW50aWFsGAIgASgLMiIuZW5ncmFtLmFwcC52MS5NY3BDcmVkZW50aWFsU3RhdHVzIpQBCiBDb21wbGV0ZU1jcEF1dGhvcml6YXRpb25SZXNwb25zZRI4Cgpjb25uZWN0aW9uGAEgASgLMiQuZW5ncmFtLmFwcC52MS5JbnRlZ3JhdGlvbkNvbm5lY3Rpb24SNgoKY3JlZGVudGlhbBgCIAEoCzIiLmVuZ3JhbS5hcHAudjEuTWNwQ3JlZGVudGlhbFN0YXR1cyKQAQocUmVjb3Zlck1jcENyZWRlbnRpYWxSZXNwb25zZRI4Cgpjb25uZWN0aW9uGAEgASgLMiQuZW5ncmFtLmFwcC52MS5JbnRlZ3JhdGlvbkNvbm5lY3Rpb24SNgoKY3JlZGVudGlhbBgCIAEoCzIiLmVuZ3JhbS5hcHAudjEuTWNwQ3JlZGVudGlhbFN0YXR1cyKGAQoSTWNwQ3JlZGVudGlhbFN0YXRlEjgKCmNvbm5lY3Rpb24YASABKAsyJC5lbmdyYW0uYXBwLnYxLkludGVncmF0aW9uQ29ubmVjdGlvbhI2CgpjcmVkZW50aWFsGAIgASgLMiIuZW5ncmFtLmFwcC52MS5NY3BDcmVkZW50aWFsU3RhdHVzIjsKFk1jcEFwcHJvdmVkRGVzdGluYXRpb24SDwoHem9uZV9pZBgBIAEoCRIQCghlbmRwb2ludBgCIAEoCSK7AQoVTWNwQ2xpZW50UmVnaXN0cmF0aW9uEgoKAmlkGAEgASgJEhEKCWNsaWVudF9pZBgCIAEoCRIUCgxjb25maWRlbnRpYWwYAyABKAgSFAoMcmVkaXJlY3RfdXJpGAQgASgJEg4KBmlzc3VlchgFIAEoCRI3CghyZXNvdXJjZRgGIAEoCzIlLmVuZ3JhbS5hcHAudjEuTWNwQXBwcm92ZWREZXN0aW5hdGlvbhIOCgZzY29wZXMYByADKAkiKwoSR2V0TWNwU2V0dXBSZXF1ZXN0EhUKDWNvbm5lY3Rpb25faWQYASABKAki9QIKE0dldE1jcFNldHVwUmVzcG9uc2USDwoHZW5hYmxlZBgBIAEoCBI7Cg1yZWdpc3RyYXRpb25zGAIgAygLMiQuZW5ncmFtLmFwcC52MS5NY3BDbGllbnRSZWdpc3RyYXRpb24SOwoMZGVzdGluYXRpb25zGAMgAygLMiUuZW5ncmFtLmFwcC52MS5NY3BBcHByb3ZlZERlc3RpbmF0aW9uEjIKB2N1cnJlbnQYBCABKAsyIS5lbmdyYW0uYXBwLnYxLk1jcENyZWRlbnRpYWxTdGF0ZRIaChJ0cmFuc2l0aW9uX3BlbmRpbmcYBSABKAgSDwoHZmxvd19pZBgGIAEoCRIXCg9mbG93X2V4cGlyZXNfYXQYByABKAkSIAoYY2xpZW50X3NlY3JldF9jb25maWd1cmVkGAggASgIEhsKE2Rpc2NvdmVyeV9hdmFpbGFibGUYCSABKAgSGgoScmVjb3ZlcnlfYXZhaWxhYmxlGAogASgIItQBCh9Db21wbGV0ZU1jcEF1dGhvcml6YXRpb25SZXF1ZXN0EhUKDWNvbm5lY3Rpb25faWQYASABKAkSGQoRZXhwZWN0ZWRfcmV2aXNpb24YAiABKA0SGwoTZXhwZWN0ZWRfZ2VuZXJhdGlvbhgDIAEoDRIPCgdmbG93X2lkGAQgASgJEg0KBXN0YXRlGAUgASgJEgwKBGNvZGUYBiABKAkSEwoGaXNzdWVyGAcgASgJSACIAQESFAoMcmVkaXJlY3RfdXJpGAggASgJQgkKB19pc3N1ZXIibAobUmVjb3Zlck1jcENyZWRlbnRpYWxSZXF1ZXN0EhUKDWNvbm5lY3Rpb25faWQYASABKAkSGQoRZXhwZWN0ZWRfcmV2aXNpb24YAiABKA0SGwoTZXhwZWN0ZWRfZ2VuZXJhdGlvbhgDIAEoDSLqAwoYTWNwQXBwcm92YWxSZXZpZXdDb250ZXh0EhQKDGV4ZWN1dGlvbl9pZBgBIAEoCRISCgpzZXNzaW9uX2lkGAIgASgJEg4KBnJ1bl9pZBgDIAEoCRIUCgx0b29sX2NhbGxfaWQYBCABKAkSFgoOcmVxdWVzdF9kaWdlc3QYBSABKAkSFQoNYXJndW1lbnRfaGFzaBgGIAEoCRIUCgxiaW5kaW5nX2hhc2gYByABKAkSFQoNY29ubmVjdGlvbl9pZBgIIAEoCRISCgpjYXRhbG9nX2lkGAkgASgJEhgKEGNhdGFsb2dfcmV2aXNpb24YCiABKA0SFwoPY29uZmlnX3JldmlzaW9uGAsgASgNEhcKD2F1dGhfZ2VuZXJhdGlvbhgMIAEoDRIbChNjcmVkZW50aWFsX2lkZW50aXR5GA0gASgJEhUKDXVwc3RyZWFtX25hbWUYDiABKAkSGAoQdG9vbF9maW5nZXJwcmludBgPIAEoCRIWCg5jb250ZXh0X2RpZ2VzdBgQIAEoCRIYChBqb3VybmFsX3JldmlzaW9uGBEgASgNEhIKCmV4cGlyZXNfYXQYEiABKAkSFwoPcG9saWN5X3JldmlzaW9uGBMgASgJEhUKDWJpbmRpbmdfZXBvY2gYFCABKAkivwEKEk1jcEFwcHJvdmFsUmVjZWlwdBIUCgxleGVjdXRpb25faWQYASABKAkSEAoIZGVjaXNpb24YAiABKAkSEAoIYWN0b3JfaWQYAyABKAkSEgoKZGVjaWRlZF9hdBgEIAEoCRIYChBqb3VybmFsX3JldmlzaW9uGAUgASgNEkEKEHJldmlld2VkX2NvbnRleHQYBiABKAsyJy5lbmdyYW0uYXBwLnYxLk1jcEFwcHJvdmFsUmV2aWV3Q29udGV4dCLhBAoSTWNwRXhlY3V0aW9uUmV2aWV3EhQKDGV4ZWN1dGlvbl9pZBgBIAEoCRIVCg1jb25uZWN0aW9uX2lkGAIgASgJEg0KBXN0YXRlGAMgASgJEhgKEGpvdXJuYWxfcmV2aXNpb24YBCABKA0SGgoScmVzdWx0X3VuYXZhaWxhYmxlGAUgASgIEioKBnJldmlldxgGIAEoCzIaLmVuZ3JhbS5hcHAudjEuTWNwQXBwcm92YWwSFwoPYWxsb3dlZF9hY3Rpb25zGAcgAygJEg4KBnJlYXNvbhgIIAEoCRIUCgx0b29sX2NhbGxfaWQYCSABKAkSDgoGcnVuX2lkGAogASgJEhYKDnJlcXVlc3RfZGlnZXN0GAsgASgJEhUKDXVwc3RyZWFtX25hbWUYDCABKAkSGQoRYXBwcm92YWxfZGVjaXNpb24YDSABKAkSGgoSb3JpZ2luYWxfYXJndW1lbnRzGA4gASgJEi0KDXJldmlld2VkX3Rvb2wYDyABKAsyFi5lbmdyYW0uYXBwLnYxLk1jcFRvb2wSKgoiYXJndW1lbnRfY29uc3RyYWludF9kb2N1bWVudHNfanNvbhgQIAMoCRIUCgxiaW5kaW5nX2hhc2gYESABKAkSQgoYZXhwZWN0ZWRfdW5rbm93bl9jb250ZXh0GBIgASgLMiAuZW5ncmFtLmFwcC52MS5NY3BVbmtub3duQ29udGV4dBJDChRjYW5jZWxsYXRpb25fcmVjZWlwdBgTIAEoCzIlLmVuZ3JhbS5hcHAudjEuTWNwQ2FuY2VsbGF0aW9uUmVjZWlwdCKEAQoRTWNwVW5rbm93bkNvbnRleHQSFAoMZXhlY3V0aW9uX2lkGAEgASgJEhgKEGpvdXJuYWxfcmV2aXNpb24YAiABKA0SFgoOcmVxdWVzdF9kaWdlc3QYAyABKAkSFAoMYmluZGluZ19oYXNoGAQgASgJEhEKCWludGVudF9pZBgFIAEoCSKvAQoWTWNwQ2FuY2VsbGF0aW9uUmVjZWlwdBIUCgxleGVjdXRpb25faWQYASABKAkSEAoIYWN0b3JfaWQYAiABKAkSDQoFY2F1c2UYAyABKAkSIQoZZXhwZWN0ZWRfam91cm5hbF9yZXZpc2lvbhgEIAEoDRIYChBqb3VybmFsX3JldmlzaW9uGAUgASgNEg0KBXN0YXRlGAYgASgJEhIKCmNyZWF0ZWRfYXQYByABKAkinwEKFE1jcFJlc29sdXRpb25SZWNlaXB0EhQKDGV4ZWN1dGlvbl9pZBgBIAEoCRIQCghhY3Rvcl9pZBgCIAEoCRIPCgdvdXRjb21lGAMgASgJEhIKCmNyZWF0ZWRfYXQYBCABKAkSOgoQZXhwZWN0ZWRfY29udGV4dBgFIAEoCzIgLmVuZ3JhbS5hcHAudjEuTWNwVW5rbm93bkNvbnRleHQigQEKEU1jcEJpbmRpbmdSZWNlaXB0EhQKDGJpbmRpbmdfaGFzaBgBIAEoCRIVCg1iaW5kaW5nX2Vwb2NoGAIgASgJEg8KB2N1cnJlbnQYAyABKAgSFQoNc3RhbGVfcmVhc29ucxgEIAMoCRIXCg9hbGxvd2VkX2FjdGlvbnMYBSADKAkigAIKGUdldFNlc3Npb25NY3BTdGF0ZVJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCRIaChJhZnRlcl9leGVjdXRpb25faWQYAiABKAkSEQoJcGFnZV9zaXplGAMgASgNEh4KEXBsYW5fdG9vbF9jYWxsX2lkGAQgASgJSACIAQESHwoXYWZ0ZXJfYmluZGluZ190b29sX25hbWUYBSABKAkSGQoRYmluZGluZ19wYWdlX3NpemUYBiABKA0SFQoNb21pdF9iaW5kaW5ncxgHIAEoCBIXCg9vbWl0X2V4ZWN1dGlvbnMYCCABKAhCFAoSX3BsYW5fdG9vbF9jYWxsX2lkIr8EChpHZXRTZXNzaW9uTWNwU3RhdGVSZXNwb25zZRI5Cg9iaW5kaW5nX3JlY2VpcHQYASABKAsyIC5lbmdyYW0uYXBwLnYxLk1jcEJpbmRpbmdSZWNlaXB0EjUKCmV4ZWN1dGlvbnMYAiADKAsyIS5lbmdyYW0uYXBwLnYxLk1jcEV4ZWN1dGlvblJldmlldxItCglhcHByb3ZhbHMYAyADKAsyGi5lbmdyYW0uYXBwLnYxLk1jcEFwcHJvdmFsEhwKFHVua25vd25fd3JpdGVfYmxvY2tzGAQgAygJEkAKE3Jlc29sdXRpb25fcmVjZWlwdHMYBSADKAsyIy5lbmdyYW0uYXBwLnYxLk1jcFJlc29sdXRpb25SZWNlaXB0EhcKD2FsbG93ZWRfYWN0aW9ucxgGIAMoCRIUCgxhdmFpbGFiaWxpdHkYByABKAkSGQoRbmV4dF9leGVjdXRpb25faWQYCCABKAkSPgoMd3JpdGVfcG9saWN5GAkgASgLMiguZW5ncmFtLmFwcC52MS5HZXRNY3BXcml0ZVBvbGljeVJlc3BvbnNlEjkKDWJpbmRpbmdfdG9vbHMYCiADKAsyIi5lbmdyYW0uYXBwLnYxLk1jcEJvdW5kVG9vbFN1bW1hcnkSHgoWbmV4dF9iaW5kaW5nX3Rvb2xfbmFtZRgLIAEoCRIeChZoYXNfbW9yZV9iaW5kaW5nX3Rvb2xzGAwgASgIEhsKE2hhc19tb3JlX2V4ZWN1dGlvbnMYDSABKAgiSAocR2V0TWNwRXhlY3V0aW9uUmVzdWx0UmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEhQKDGV4ZWN1dGlvbl9pZBgCIAEoCSKwAQodR2V0TWNwRXhlY3V0aW9uUmVzdWx0UmVzcG9uc2USDQoFc3RhdGUYASABKAkSGgoScmVzdWx0X3VuYXZhaWxhYmxlGAIgASgIEhgKC3Jlc3VsdF9qc29uGAMgASgJSACIAQESOgoKc2V0dGxlbWVudBgEIAEoDjImLmVuZ3JhbS5hcHAudjEuTWNwQ29tcGxldGlvblNldHRsZW1lbnRCDgoMX3Jlc3VsdF9qc29uIsoCChNNY3BCb3VuZFRvb2xTdW1tYXJ5EhUKDWNvbm5lY3Rpb25faWQYASABKAkSGAoQY29ubmVjdGlvbl9hbGlhcxgCIAEoCRIXCg9tb2RlbF90b29sX25hbWUYAyABKAkSLQoNcmV2aWV3ZWRfdG9vbBgEIAEoCzIWLmVuZ3JhbS5hcHAudjEuTWNwVG9vbBIqCiJhcmd1bWVudF9jb25zdHJhaW50X2RvY3VtZW50c19qc29uGAUgAygJEhgKEGF1dG9tYXRpb25fd3JpdGUYBiABKAgSFAoMdW5yZXN0cmljdGVkGAcgASgIEhIKCmNhdGFsb2dfaWQYCCABKAkSGAoQY2F0YWxvZ19yZXZpc2lvbhgJIAEoDRIXCg9jb25maWdfcmV2aXNpb24YCiABKA0SFwoPYXV0aF9nZW5lcmF0aW9uGAsgASgNKsgBChRNY3BSZWJpbmREcmFpblJlYXNvbhInCiNNQ1BfUkVCSU5EX0RSQUlOX1JFQVNPTl9VTlNQRUNJRklFRBAAEjAKLE1DUF9SRUJJTkRfRFJBSU5fUkVBU09OX0RJU1BBVENIX0lOX1BST0dSRVNTEAESKgomTUNQX1JFQklORF9EUkFJTl9SRUFTT05fVU5LTk9XTl9JTlRFTlQQAhIpCiVNQ1BfUkVCSU5EX0RSQUlOX1JFQVNPTl9MRUdBQ1lfSU5URU5UEAMqzwEKF01jcENvbXBsZXRpb25TZXR0bGVtZW50EikKJU1DUF9DT01QTEVUSU9OX1NFVFRMRU1FTlRfVU5TUEVDSUZJRUQQABIlCiFNQ1BfQ09NUExFVElPTl9TRVRUTEVNRU5UX1BFTkRJTkcQARI0CjBNQ1BfQ09NUExFVElPTl9TRVRUTEVNRU5UX1NFVFRMRURfT1JfVEFSR0VUX0dPTkUQAhIsCihNQ1BfQ09NUExFVElPTl9TRVRUTEVNRU5UX05PVF9BUFBMSUNBQkxFEAMyqBkKEkludGVncmF0aW9uU2VydmljZRJdCg5MaXN0Q29ubmVjdG9ycxIkLmVuZ3JhbS5hcHAudjEuTGlzdENvbm5lY3RvcnNSZXF1ZXN0GiUuZW5ncmFtLmFwcC52MS5MaXN0Q29ubmVjdG9yc1Jlc3BvbnNlEmAKD1Vwc2VydENvbm5lY3RvchIlLmVuZ3JhbS5hcHAudjEuVXBzZXJ0Q29ubmVjdG9yUmVxdWVzdBomLmVuZ3JhbS5hcHAudjEuVXBzZXJ0Q29ubmVjdG9yUmVzcG9uc2USYAoPRGVsZXRlQ29ubmVjdG9yEiUuZW5ncmFtLmFwcC52MS5EZWxldGVDb25uZWN0b3JSZXF1ZXN0GiYuZW5ncmFtLmFwcC52MS5EZWxldGVDb25uZWN0b3JSZXNwb25zZRJyChVHZXRJbnRlZ3JhdGlvbkNhdGFsb2cSKy5lbmdyYW0uYXBwLnYxLkdldEludGVncmF0aW9uQ2F0YWxvZ1JlcXVlc3QaLC5lbmdyYW0uYXBwLnYxLkdldEludGVncmF0aW9uQ2F0YWxvZ1Jlc3BvbnNlEmYKEVNldE1pbnRDcmVkZW50aWFsEicuZW5ncmFtLmFwcC52MS5TZXRNaW50Q3JlZGVudGlhbFJlcXVlc3QaKC5lbmdyYW0uYXBwLnYxLlNldE1pbnRDcmVkZW50aWFsUmVzcG9uc2USbAoTVXBsb2FkQ29ubmVjdG9yTG9nbxIpLmVuZ3JhbS5hcHAudjEuVXBsb2FkQ29ubmVjdG9yTG9nb1JlcXVlc3QaKi5lbmdyYW0uYXBwLnYxLlVwbG9hZENvbm5lY3RvckxvZ29SZXNwb25zZRJaCg1UZXN0Q29ubmVjdG9yEiMuZW5ncmFtLmFwcC52MS5UZXN0Q29ubmVjdG9yUmVxdWVzdBokLmVuZ3JhbS5hcHAudjEuVGVzdENvbm5lY3RvclJlc3BvbnNlEm8KFFNldENvbm5lY3RvclNldHRpbmdzEiouZW5ncmFtLmFwcC52MS5TZXRDb25uZWN0b3JTZXR0aW5nc1JlcXVlc3QaKy5lbmdyYW0uYXBwLnYxLlNldENvbm5lY3RvclNldHRpbmdzUmVzcG9uc2USYAoPTGlzdENvbm5lY3Rpb25zEiUuZW5ncmFtLmFwcC52MS5MaXN0Q29ubmVjdGlvbnNSZXF1ZXN0GiYuZW5ncmFtLmFwcC52MS5MaXN0Q29ubmVjdGlvbnNSZXNwb25zZRJjChBDcmVhdGVDb25uZWN0aW9uEiYuZW5ncmFtLmFwcC52MS5DcmVhdGVDb25uZWN0aW9uUmVxdWVzdBonLmVuZ3JhbS5hcHAudjEuQ3JlYXRlQ29ubmVjdGlvblJlc3BvbnNlEmMKEFVwZGF0ZUNvbm5lY3Rpb24SJi5lbmdyYW0uYXBwLnYxLlVwZGF0ZUNvbm5lY3Rpb25SZXF1ZXN0GicuZW5ncmFtLmFwcC52MS5VcGRhdGVDb25uZWN0aW9uUmVzcG9uc2USYwoQRGVsZXRlQ29ubmVjdGlvbhImLmVuZ3JhbS5hcHAudjEuRGVsZXRlQ29ubmVjdGlvblJlcXVlc3QaJy5lbmdyYW0uYXBwLnYxLkRlbGV0ZUNvbm5lY3Rpb25SZXNwb25zZRJdCg5UZXN0Q29ubmVjdGlvbhIkLmVuZ3JhbS5hcHAudjEuVGVzdENvbm5lY3Rpb25SZXF1ZXN0GiUuZW5ncmFtLmFwcC52MS5UZXN0Q29ubmVjdGlvblJlc3BvbnNlEm8KFFNldENvbm5lY3Rpb25FbmFibGVkEiouZW5ncmFtLmFwcC52MS5TZXRDb25uZWN0aW9uRW5hYmxlZFJlcXVlc3QaKy5lbmdyYW0uYXBwLnYxLlNldENvbm5lY3Rpb25FbmFibGVkUmVzcG9uc2USbAoTR2V0R29vZ2xlQ2xvdWRTZXR1cBIpLmVuZ3JhbS5hcHAudjEuR2V0R29vZ2xlQ2xvdWRTZXR1cFJlcXVlc3QaKi5lbmdyYW0uYXBwLnYxLkdldEdvb2dsZUNsb3VkU2V0dXBSZXNwb25zZRJdCg5HZXRNY3BDYXRhbG9ncxIkLmVuZ3JhbS5hcHAudjEuR2V0TWNwQ2F0YWxvZ3NSZXF1ZXN0GiUuZW5ncmFtLmFwcC52MS5HZXRNY3BDYXRhbG9nc1Jlc3BvbnNlElQKC0Rpc2NvdmVyTWNwEiEuZW5ncmFtLmFwcC52MS5EaXNjb3Zlck1jcFJlcXVlc3QaIi5lbmdyYW0uYXBwLnYxLkRpc2NvdmVyTWNwUmVzcG9uc2USUQoKUHVibGlzaE1jcBIgLmVuZ3JhbS5hcHAudjEuUHVibGlzaE1jcFJlcXVlc3QaIS5lbmdyYW0uYXBwLnYxLlB1Ymxpc2hNY3BSZXNwb25zZRJjChBQdXRNY3BDcmVkZW50aWFsEiYuZW5ncmFtLmFwcC52MS5QdXRNY3BDcmVkZW50aWFsUmVxdWVzdBonLmVuZ3JhbS5hcHAudjEuUHV0TWNwQ3JlZGVudGlhbFJlc3BvbnNlElQKC0dldE1jcFNldHVwEiEuZW5ncmFtLmFwcC52MS5HZXRNY3BTZXR1cFJlcXVlc3QaIi5lbmdyYW0uYXBwLnYxLkdldE1jcFNldHVwUmVzcG9uc2USewoYQ29tcGxldGVNY3BBdXRob3JpemF0aW9uEi4uZW5ncmFtLmFwcC52MS5Db21wbGV0ZU1jcEF1dGhvcml6YXRpb25SZXF1ZXN0Gi8uZW5ncmFtLmFwcC52MS5Db21wbGV0ZU1jcEF1dGhvcml6YXRpb25SZXNwb25zZRJvChRSZWNvdmVyTWNwQ3JlZGVudGlhbBIqLmVuZ3JhbS5hcHAudjEuUmVjb3Zlck1jcENyZWRlbnRpYWxSZXF1ZXN0GisuZW5ncmFtLmFwcC52MS5SZWNvdmVyTWNwQ3JlZGVudGlhbFJlc3BvbnNlElcKDEF1dGhvcml6ZU1jcBIiLmVuZ3JhbS5hcHAudjEuQXV0aG9yaXplTWNwUmVxdWVzdBojLmVuZ3JhbS5hcHAudjEuQXV0aG9yaXplTWNwUmVzcG9uc2USWgoNRGlzY29ubmVjdE1jcBIjLmVuZ3JhbS5hcHAudjEuRGlzY29ubmVjdE1jcFJlcXVlc3QaJC5lbmdyYW0uYXBwLnYxLkRpc2Nvbm5lY3RNY3BSZXNwb25zZRJpChJHZXRTZXNzaW9uTWNwU3RhdGUSKC5lbmdyYW0uYXBwLnYxLkdldFNlc3Npb25NY3BTdGF0ZVJlcXVlc3QaKS5lbmdyYW0uYXBwLnYxLkdldFNlc3Npb25NY3BTdGF0ZVJlc3BvbnNlEnIKFUdldE1jcEV4ZWN1dGlvblJlc3VsdBIrLmVuZ3JhbS5hcHAudjEuR2V0TWNwRXhlY3V0aW9uUmVzdWx0UmVxdWVzdBosLmVuZ3JhbS5hcHAudjEuR2V0TWNwRXhlY3V0aW9uUmVzdWx0UmVzcG9uc2USYwoQTGlzdE1jcEFwcHJvdmFscxImLmVuZ3JhbS5hcHAudjEuTGlzdE1jcEFwcHJvdmFsc1JlcXVlc3QaJy5lbmdyYW0uYXBwLnYxLkxpc3RNY3BBcHByb3ZhbHNSZXNwb25zZRJmChFEZWNpZGVNY3BBcHByb3ZhbBInLmVuZ3JhbS5hcHAudjEuRGVjaWRlTWNwQXBwcm92YWxSZXF1ZXN0GiguZW5ncmFtLmFwcC52MS5EZWNpZGVNY3BBcHByb3ZhbFJlc3BvbnNlEloKDUNhbmNlbE1jcENhbGwSIy5lbmdyYW0uYXBwLnYxLkNhbmNlbE1jcENhbGxSZXF1ZXN0GiQuZW5ncmFtLmFwcC52MS5DYW5jZWxNY3BDYWxsUmVzcG9uc2USZgoRUmVzb2x2ZU1jcE91dGNvbWUSJy5lbmdyYW0uYXBwLnYxLlJlc29sdmVNY3BPdXRjb21lUmVxdWVzdBooLmVuZ3JhbS5hcHAudjEuUmVzb2x2ZU1jcE91dGNvbWVSZXNwb25zZRJjChBQcmVwYXJlTWNwUmViaW5kEiYuZW5ncmFtLmFwcC52MS5QcmVwYXJlTWNwUmViaW5kUmVxdWVzdBonLmVuZ3JhbS5hcHAudjEuUHJlcGFyZU1jcFJlYmluZFJlc3BvbnNlEk4KCVJlYmluZE1jcBIfLmVuZ3JhbS5hcHAudjEuUmViaW5kTWNwUmVxdWVzdBogLmVuZ3JhbS5hcHAudjEuUmViaW5kTWNwUmVzcG9uc2ViBnByb3RvMw", [file_engram_app_v1_session]); /** * @generated from message engram.app.v1.GoogleCloudConnectionConfig @@ -106,6 +108,36 @@ export type IntegrationConnection = Message<"engram.app.v1.IntegrationConnection * @generated from field: bool is_default = 10; */ isDefault: boolean; + + /** + * @generated from field: engram.app.v1.McpConnectionConfig mcp = 11; + */ + mcp?: McpConnectionConfig; + + /** + * @generated from field: uint32 config_revision = 12; + */ + configRevision: number; + + /** + * @generated from field: uint32 auth_generation = 13; + */ + authGeneration: number; + + /** + * @generated from field: string credential_identity = 14; + */ + credentialIdentity: string; + + /** + * @generated from field: uint32 catalog_revision = 15; + */ + catalogRevision: number; + + /** + * @generated from field: bool revoked = 16; + */ + revoked: boolean; }; /** @@ -168,6 +200,11 @@ export type CreateConnectionRequest = Message<"engram.app.v1.CreateConnectionReq * @generated from field: engram.app.v1.GoogleCloudConnectionConfig google_cloud = 4; */ googleCloud?: GoogleCloudConnectionConfig; + + /** + * @generated from field: engram.app.v1.McpConnectionConfig mcp = 5; + */ + mcp?: McpConnectionConfig; }; /** @@ -198,6 +235,16 @@ export const CreateConnectionResponseSchema: GenMessage & { + /** + * @generated from field: uint32 expected_generation = 7; + */ + expectedGeneration: number; + + /** + * @generated from field: uint32 expected_revision = 6; + */ + expectedRevision: number; + /** * @generated from field: string id = 1; */ @@ -217,6 +264,11 @@ export type UpdateConnectionRequest = Message<"engram.app.v1.UpdateConnectionReq * @generated from field: engram.app.v1.GoogleCloudConnectionConfig google_cloud = 4; */ googleCloud?: GoogleCloudConnectionConfig; + + /** + * @generated from field: engram.app.v1.McpConnectionConfig mcp = 5; + */ + mcp?: McpConnectionConfig; }; /** @@ -247,6 +299,16 @@ export const UpdateConnectionResponseSchema: GenMessage & { + /** + * @generated from field: uint32 expected_revision = 2; + */ + expectedRevision: number; + + /** + * @generated from field: uint32 expected_generation = 3; + */ + expectedGeneration: number; + /** * @generated from field: string id = 1; */ @@ -329,6 +391,11 @@ export type SetConnectionEnabledRequest = Message<"engram.app.v1.SetConnectionEn * @generated from field: bool enabled = 2; */ enabled: boolean; + + /** + * @generated from field: uint32 expected_revision = 3; + */ + expectedRevision: number; }; /** @@ -1035,172 +1102,2298 @@ export const SetConnectorSettingsResponseSchema: GenMessage & { /** - * Built-in seeds (read-only) ∪ admin-authored connectors. Admin-only; each row - * carries the derived connected/available `status`. - * - * @generated from rpc engram.app.v1.IntegrationService.ListConnectors + * @generated from field: string endpoint = 1; */ - listConnectors: { - methodKind: "unary"; - input: typeof ListConnectorsRequestSchema; - output: typeof ListConnectorsResponseSchema; - }, + endpoint: string; + /** - * Create or replace a custom connector. Rejects a built-in provider id and - * any config `parseConnector` refuses. Invalidates the registry cache. - * - * @generated from rpc engram.app.v1.IntegrationService.UpsertConnector + * @generated from field: string zone = 2; */ - upsertConnector: { - methodKind: "unary"; - input: typeof UpsertConnectorRequestSchema; - output: typeof UpsertConnectorResponseSchema; - }, + zone: string; + /** - * Delete a custom connector (idempotent). Rejects a built-in provider id. - * - * @generated from rpc engram.app.v1.IntegrationService.DeleteConnector + * @generated from field: string auth_kind = 3; */ - deleteConnector: { - methodKind: "unary"; - input: typeof DeleteConnectorRequestSchema; - output: typeof DeleteConnectorResponseSchema; - }, + authKind: string; + /** - * Member-readable provider catalog: per-connector display identity + the - * powers it grants + the hosts it opens, derived from the merged registry. - * Carries NO secret material (no secretRef / header / template / mint kind) — - * safe for the Launch receipt + in-session provider icons (redesign). - * - * @generated from rpc engram.app.v1.IntegrationService.GetIntegrationCatalog + * @generated from field: string api_key_secret_ref = 4; */ - getIntegrationCatalog: { - methodKind: "unary"; - input: typeof GetIntegrationCatalogRequestSchema; - output: typeof GetIntegrationCatalogResponseSchema; - }, + apiKeySecretRef: string; + /** - * Admin-only: store a mint kind's credentials, sealing each field as an org - * secret `.` (the coordinator resolves them by that name). The - * GitHub App private key is a secret; the App ID is config — both are sealed. - * - * @generated from rpc engram.app.v1.IntegrationService.SetMintCredential + * @generated from field: string oauth_client_registration_id = 5; */ - setMintCredential: { - methodKind: "unary"; - input: typeof SetMintCredentialRequestSchema; - output: typeof SetMintCredentialResponseSchema; - }, + oauthClientRegistrationId: string; + /** - * Admin-only: upload (or replace) a connector's logo — orchestrator-owned - * (the coordinator never sees connectors). Bytes land in the connector_logo - * table (≤512 KB, SVG or square PNG, sniffed) and are served at - * GET /api/v1/integrations//logo. Empty `data` clears the logo. - * - * @generated from rpc engram.app.v1.IntegrationService.UploadConnectorLogo + * @generated from field: bool share_with_org_sessions = 6; */ - uploadConnectorLogo: { - methodKind: "unary"; - input: typeof UploadConnectorLogoRequestSchema; - output: typeof UploadConnectorLogoResponseSchema; - }, + shareWithOrgSessions: boolean; +}; + +/** + * Describes the message engram.app.v1.McpConnectionConfig. + * Use `create(McpConnectionConfigSchema)` to create a new message. + */ +export const McpConnectionConfigSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 38); + +/** + * @generated from message engram.app.v1.McpTool + */ +export type McpTool = Message<"engram.app.v1.McpTool"> & { /** - * Admin-only: test a connector's credential. Builds the resolved spec from the - * registry and delegates the actual unseal/mint + benign GET to the coordinator - * (MintService.RunConnectorTest). `draft_values` carries an about-to-be-saved - * credential to test before sealing (inject: the single value under any key; - * mint: the field map); empty tests the stored credential. - * - * @generated from rpc engram.app.v1.IntegrationService.TestConnector + * @generated from field: string upstream_name = 1; */ - testConnector: { - methodKind: "unary"; - input: typeof TestConnectorRequestSchema; - output: typeof TestConnectorResponseSchema; - }, + upstreamName: string; + /** - * Admin-only: store the values of a connector's declared `settings` (non- - * secret parameters such as the API host a regional or self-hosted provider - * is reached on). Values live on the provider's default connection and feed - * the connector's effective hosts, the test probe, and the CLI env. - * - * @generated from rpc engram.app.v1.IntegrationService.SetConnectorSettings + * @generated from field: string description = 2; */ - setConnectorSettings: { - methodKind: "unary"; - input: typeof SetConnectorSettingsRequestSchema; - output: typeof SetConnectorSettingsResponseSchema; - }, + description: string; + /** - * ADR 0109: configured credential instances. Existing providers expose an - * ordinary default connection; the first provider with user-created named - * connections is Google Cloud, whose config contains no private key. + * @generated from field: string input_schema_json = 3; + */ + inputSchemaJson: string; + + /** + * @generated from field: string output_schema_json = 4; + */ + outputSchemaJson: string; + + /** + * Empty in discovery drafts. Publication requires explicit read or write. * - * @generated from rpc engram.app.v1.IntegrationService.ListConnections + * @generated from field: string classification = 5; */ - listConnections: { - methodKind: "unary"; - input: typeof ListConnectionsRequestSchema; - output: typeof ListConnectionsResponseSchema; - }, + classification: string; + /** - * @generated from rpc engram.app.v1.IntegrationService.CreateConnection + * @generated from field: bool enabled = 6; */ - createConnection: { - methodKind: "unary"; - input: typeof CreateConnectionRequestSchema; - output: typeof CreateConnectionResponseSchema; - }, + enabled: boolean; + /** - * @generated from rpc engram.app.v1.IntegrationService.UpdateConnection + * @generated from field: string fingerprint = 7; */ - updateConnection: { - methodKind: "unary"; - input: typeof UpdateConnectionRequestSchema; - output: typeof UpdateConnectionResponseSchema; - }, + fingerprint: string; +}; + +/** + * Describes the message engram.app.v1.McpTool. + * Use `create(McpToolSchema)` to create a new message. + */ +export const McpToolSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 39); + +/** + * @generated from message engram.app.v1.McpCatalog + */ +export type McpCatalog = Message<"engram.app.v1.McpCatalog"> & { /** - * @generated from rpc engram.app.v1.IntegrationService.DeleteConnection + * @generated from field: string id = 1; */ - deleteConnection: { - methodKind: "unary"; - input: typeof DeleteConnectionRequestSchema; - output: typeof DeleteConnectionResponseSchema; - }, + id: string; + /** - * @generated from rpc engram.app.v1.IntegrationService.TestConnection + * @generated from field: string connection_id = 2; */ - testConnection: { - methodKind: "unary"; - input: typeof TestConnectionRequestSchema; - output: typeof TestConnectionResponseSchema; - }, + connectionId: string; + /** - * @generated from rpc engram.app.v1.IntegrationService.SetConnectionEnabled + * @generated from field: string kind = 3; */ - setConnectionEnabled: { - methodKind: "unary"; - input: typeof SetConnectionEnabledRequestSchema; - output: typeof SetConnectionEnabledResponseSchema; - }, + kind: string; + /** - * @generated from rpc engram.app.v1.IntegrationService.GetGoogleCloudSetup + * @generated from field: uint32 revision = 4; */ - getGoogleCloudSetup: { - methodKind: "unary"; - input: typeof GetGoogleCloudSetupRequestSchema; - output: typeof GetGoogleCloudSetupResponseSchema; + revision: number; + + /** + * @generated from field: uint32 config_revision = 5; + */ + configRevision: number; + + /** + * @generated from field: uint32 auth_generation = 6; + */ + authGeneration: number; + + /** + * @generated from field: repeated engram.app.v1.McpTool tools = 7; + */ + tools: McpTool[]; + + /** + * @generated from field: string fingerprint = 8; + */ + fingerprint: string; +}; + +/** + * Describes the message engram.app.v1.McpCatalog. + * Use `create(McpCatalogSchema)` to create a new message. + */ +export const McpCatalogSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 40); + +/** + * @generated from message engram.app.v1.GetMcpCatalogsRequest + */ +export type GetMcpCatalogsRequest = Message<"engram.app.v1.GetMcpCatalogsRequest"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; +}; + +/** + * Describes the message engram.app.v1.GetMcpCatalogsRequest. + * Use `create(GetMcpCatalogsRequestSchema)` to create a new message. + */ +export const GetMcpCatalogsRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 41); + +/** + * @generated from message engram.app.v1.GetMcpCatalogsResponse + */ +export type GetMcpCatalogsResponse = Message<"engram.app.v1.GetMcpCatalogsResponse"> & { + /** + * @generated from field: uint32 config_revision = 1; + */ + configRevision: number; + + /** + * @generated from field: uint32 catalog_revision = 2; + */ + catalogRevision: number; + + /** + * @generated from field: engram.app.v1.McpCatalog draft = 3; + */ + draft?: McpCatalog; + + /** + * @generated from field: engram.app.v1.McpCatalog published = 4; + */ + published?: McpCatalog; +}; + +/** + * Describes the message engram.app.v1.GetMcpCatalogsResponse. + * Use `create(GetMcpCatalogsResponseSchema)` to create a new message. + */ +export const GetMcpCatalogsResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 42); + +/** + * @generated from message engram.app.v1.DiscoverMcpRequest + */ +export type DiscoverMcpRequest = Message<"engram.app.v1.DiscoverMcpRequest"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: uint32 expected_revision = 2; + */ + expectedRevision: number; +}; + +/** + * Describes the message engram.app.v1.DiscoverMcpRequest. + * Use `create(DiscoverMcpRequestSchema)` to create a new message. + */ +export const DiscoverMcpRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 43); + +/** + * @generated from message engram.app.v1.DiscoverMcpResponse + */ +export type DiscoverMcpResponse = Message<"engram.app.v1.DiscoverMcpResponse"> & { + /** + * @generated from field: engram.app.v1.McpCatalog catalog = 1; + */ + catalog?: McpCatalog; +}; + +/** + * Describes the message engram.app.v1.DiscoverMcpResponse. + * Use `create(DiscoverMcpResponseSchema)` to create a new message. + */ +export const DiscoverMcpResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 44); + +/** + * @generated from message engram.app.v1.PublishMcpRequest + */ +export type PublishMcpRequest = Message<"engram.app.v1.PublishMcpRequest"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: string draft_id = 2; + */ + draftId: string; + + /** + * @generated from field: uint32 expected_revision = 3; + */ + expectedRevision: number; + + /** + * @generated from field: repeated engram.app.v1.McpTool tools = 4; + */ + tools: McpTool[]; +}; + +/** + * Describes the message engram.app.v1.PublishMcpRequest. + * Use `create(PublishMcpRequestSchema)` to create a new message. + */ +export const PublishMcpRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 45); + +/** + * @generated from message engram.app.v1.PublishMcpResponse + */ +export type PublishMcpResponse = Message<"engram.app.v1.PublishMcpResponse"> & { + /** + * @generated from field: engram.app.v1.McpCatalog catalog = 1; + */ + catalog?: McpCatalog; +}; + +/** + * Describes the message engram.app.v1.PublishMcpResponse. + * Use `create(PublishMcpResponseSchema)` to create a new message. + */ +export const PublishMcpResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 46); + +/** + * @generated from message engram.app.v1.AuthorizeMcpRequest + */ +export type AuthorizeMcpRequest = Message<"engram.app.v1.AuthorizeMcpRequest"> & { + /** + * @generated from field: uint32 expected_revision = 4; + */ + expectedRevision: number; + + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: uint32 expected_generation = 2; + */ + expectedGeneration: number; + + /** + * @generated from field: bool share_with_org_sessions = 3; + */ + shareWithOrgSessions: boolean; +}; + +/** + * Describes the message engram.app.v1.AuthorizeMcpRequest. + * Use `create(AuthorizeMcpRequestSchema)` to create a new message. + */ +export const AuthorizeMcpRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 47); + +/** + * @generated from message engram.app.v1.AuthorizeMcpResponse + */ +export type AuthorizeMcpResponse = Message<"engram.app.v1.AuthorizeMcpResponse"> & { + /** + * @generated from field: uint32 config_revision = 4; + */ + configRevision: number; + + /** + * @generated from field: uint32 auth_generation = 5; + */ + authGeneration: number; + + /** + * @generated from field: string flow_id = 2; + */ + flowId: string; + + /** + * @generated from field: string expires_at = 3; + */ + expiresAt: string; + + /** + * @generated from field: string authorization_url = 1; + */ + authorizationUrl: string; +}; + +/** + * Describes the message engram.app.v1.AuthorizeMcpResponse. + * Use `create(AuthorizeMcpResponseSchema)` to create a new message. + */ +export const AuthorizeMcpResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 48); + +/** + * @generated from message engram.app.v1.DisconnectMcpRequest + */ +export type DisconnectMcpRequest = Message<"engram.app.v1.DisconnectMcpRequest"> & { + /** + * @generated from field: uint32 expected_revision = 3; + */ + expectedRevision: number; + + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: uint32 expected_generation = 2; + */ + expectedGeneration: number; +}; + +/** + * Describes the message engram.app.v1.DisconnectMcpRequest. + * Use `create(DisconnectMcpRequestSchema)` to create a new message. + */ +export const DisconnectMcpRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 49); + +/** + * @generated from message engram.app.v1.DisconnectMcpResponse + */ +export type DisconnectMcpResponse = Message<"engram.app.v1.DisconnectMcpResponse"> & { + /** + * @generated from field: engram.app.v1.IntegrationConnection connection = 1; + */ + connection?: IntegrationConnection; +}; + +/** + * Describes the message engram.app.v1.DisconnectMcpResponse. + * Use `create(DisconnectMcpResponseSchema)` to create a new message. + */ +export const DisconnectMcpResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 50); + +/** + * @generated from message engram.app.v1.McpApproval + */ +export type McpApproval = Message<"engram.app.v1.McpApproval"> & { + /** + * @generated from field: string execution_id = 1; + */ + executionId: string; + + /** + * @generated from field: string session_id = 2; + */ + sessionId: string; + + /** + * @generated from field: string run_id = 3; + */ + runId: string; + + /** + * @generated from field: string tool_call_id = 4; + */ + toolCallId: string; + + /** + * @generated from field: string connection_id = 5; + */ + connectionId: string; + + /** + * @generated from field: string upstream_name = 6; + */ + upstreamName: string; + + /** + * @generated from field: string request_digest = 7; + */ + requestDigest: string; + + /** + * @generated from field: string argument_hash = 8; + */ + argumentHash: string; + + /** + * @generated from field: string expires_at = 9; + */ + expiresAt: string; + + /** + * @generated from field: string original_arguments = 10; + */ + originalArguments: string; + + /** + * @generated from field: engram.app.v1.McpTool reviewed_tool = 11; + */ + reviewedTool?: McpTool; + + /** + * @generated from field: repeated string argument_constraint_documents_json = 12; + */ + argumentConstraintDocumentsJson: string[]; + + /** + * @generated from field: engram.app.v1.McpApprovalReviewContext expected_review_context = 13; + */ + expectedReviewContext?: McpApprovalReviewContext; +}; + +/** + * Describes the message engram.app.v1.McpApproval. + * Use `create(McpApprovalSchema)` to create a new message. + */ +export const McpApprovalSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 51); + +/** + * @generated from message engram.app.v1.ListMcpApprovalsRequest + */ +export type ListMcpApprovalsRequest = Message<"engram.app.v1.ListMcpApprovalsRequest"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string after_execution_id = 2; + */ + afterExecutionId: string; +}; + +/** + * Describes the message engram.app.v1.ListMcpApprovalsRequest. + * Use `create(ListMcpApprovalsRequestSchema)` to create a new message. + */ +export const ListMcpApprovalsRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 52); + +/** + * @generated from message engram.app.v1.ListMcpApprovalsResponse + */ +export type ListMcpApprovalsResponse = Message<"engram.app.v1.ListMcpApprovalsResponse"> & { + /** + * @generated from field: repeated engram.app.v1.McpApproval approvals = 1; + */ + approvals: McpApproval[]; + + /** + * @generated from field: string next_execution_id = 2; + */ + nextExecutionId: string; +}; + +/** + * Describes the message engram.app.v1.ListMcpApprovalsResponse. + * Use `create(ListMcpApprovalsResponseSchema)` to create a new message. + */ +export const ListMcpApprovalsResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 53); + +/** + * @generated from message engram.app.v1.DecideMcpApprovalRequest + */ +export type DecideMcpApprovalRequest = Message<"engram.app.v1.DecideMcpApprovalRequest"> & { + /** + * @generated from field: string execution_id = 1; + */ + executionId: string; + + /** + * @generated from field: bool approve = 2; + */ + approve: boolean; + + /** + * @generated from field: engram.app.v1.McpApprovalReviewContext expected_review_context = 3; + */ + expectedReviewContext?: McpApprovalReviewContext; +}; + +/** + * Describes the message engram.app.v1.DecideMcpApprovalRequest. + * Use `create(DecideMcpApprovalRequestSchema)` to create a new message. + */ +export const DecideMcpApprovalRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 54); + +/** + * @generated from message engram.app.v1.DecideMcpApprovalResponse + */ +export type DecideMcpApprovalResponse = Message<"engram.app.v1.DecideMcpApprovalResponse"> & { + /** + * @generated from field: string state = 1; + */ + state: string; + + /** + * @generated from field: engram.app.v1.McpApprovalReceipt receipt = 2; + */ + receipt?: McpApprovalReceipt; +}; + +/** + * Describes the message engram.app.v1.DecideMcpApprovalResponse. + * Use `create(DecideMcpApprovalResponseSchema)` to create a new message. + */ +export const DecideMcpApprovalResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 55); + +/** + * @generated from message engram.app.v1.CancelMcpCallRequest + */ +export type CancelMcpCallRequest = Message<"engram.app.v1.CancelMcpCallRequest"> & { + /** + * @generated from field: string execution_id = 1; + */ + executionId: string; + + /** + * @generated from field: uint32 expected_journal_revision = 2; + */ + expectedJournalRevision: number; +}; + +/** + * Describes the message engram.app.v1.CancelMcpCallRequest. + * Use `create(CancelMcpCallRequestSchema)` to create a new message. + */ +export const CancelMcpCallRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 56); + +/** + * @generated from message engram.app.v1.CancelMcpCallResponse + */ +export type CancelMcpCallResponse = Message<"engram.app.v1.CancelMcpCallResponse"> & { + /** + * @generated from field: string state = 1; + */ + state: string; + + /** + * @generated from field: engram.app.v1.McpCancellationReceipt receipt = 2; + */ + receipt?: McpCancellationReceipt; +}; + +/** + * Describes the message engram.app.v1.CancelMcpCallResponse. + * Use `create(CancelMcpCallResponseSchema)` to create a new message. + */ +export const CancelMcpCallResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 57); + +/** + * @generated from message engram.app.v1.ResolveMcpOutcomeRequest + */ +export type ResolveMcpOutcomeRequest = Message<"engram.app.v1.ResolveMcpOutcomeRequest"> & { + /** + * @generated from field: string execution_id = 1; + */ + executionId: string; + + /** + * @generated from field: string outcome = 2; + */ + outcome: string; + + /** + * @generated from field: engram.app.v1.McpUnknownContext expected_unknown_context = 3; + */ + expectedUnknownContext?: McpUnknownContext; +}; + +/** + * Describes the message engram.app.v1.ResolveMcpOutcomeRequest. + * Use `create(ResolveMcpOutcomeRequestSchema)` to create a new message. + */ +export const ResolveMcpOutcomeRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 58); + +/** + * @generated from message engram.app.v1.ResolveMcpOutcomeResponse + */ +export type ResolveMcpOutcomeResponse = Message<"engram.app.v1.ResolveMcpOutcomeResponse"> & { + /** + * @generated from field: string outcome = 1; + */ + outcome: string; + + /** + * @generated from field: engram.app.v1.McpResolutionReceipt receipt = 2; + */ + receipt?: McpResolutionReceipt; +}; + +/** + * Describes the message engram.app.v1.ResolveMcpOutcomeResponse. + * Use `create(ResolveMcpOutcomeResponseSchema)` to create a new message. + */ +export const ResolveMcpOutcomeResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 59); + +/** + * @generated from message engram.app.v1.PrepareMcpRebindRequest + */ +export type PrepareMcpRebindRequest = Message<"engram.app.v1.PrepareMcpRebindRequest"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string expected_binding_hash = 2; + */ + expectedBindingHash: string; +}; + +/** + * Describes the message engram.app.v1.PrepareMcpRebindRequest. + * Use `create(PrepareMcpRebindRequestSchema)` to create a new message. + */ +export const PrepareMcpRebindRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 60); + +/** + * @generated from message engram.app.v1.PrepareMcpRebindResponse + */ +export type PrepareMcpRebindResponse = Message<"engram.app.v1.PrepareMcpRebindResponse"> & { + /** + * @generated from field: string binding_hash = 1; + */ + bindingHash: string; + + /** + * @generated from field: string binding_epoch = 2; + */ + bindingEpoch: string; + + /** + * @generated from field: repeated engram.app.v1.McpBoundToolSummary binding_tools = 3; + */ + bindingTools: McpBoundToolSummary[]; + + /** + * @generated from field: repeated string deny_hosts = 4; + */ + denyHosts: string[]; + + /** + * @generated from field: string deny_revision = 5; + */ + denyRevision: string; + + /** + * @generated from field: string deny_digest = 6; + */ + denyDigest: string; + + /** + * @generated from field: repeated engram.app.v1.McpRebindConnection connections = 7; + */ + connections: McpRebindConnection[]; + + /** + * @generated from field: engram.app.v1.McpRebindDrain drain = 8; + */ + drain?: McpRebindDrain; +}; + +/** + * Describes the message engram.app.v1.PrepareMcpRebindResponse. + * Use `create(PrepareMcpRebindResponseSchema)` to create a new message. + */ +export const PrepareMcpRebindResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 61); + +/** + * @generated from message engram.app.v1.McpRebindDrain + */ +export type McpRebindDrain = Message<"engram.app.v1.McpRebindDrain"> & { + /** + * @generated from field: engram.app.v1.McpRebindDrainReason reason = 1; + */ + reason: McpRebindDrainReason; + + /** + * @generated from field: string retry_after = 2; + */ + retryAfter: string; +}; + +/** + * Describes the message engram.app.v1.McpRebindDrain. + * Use `create(McpRebindDrainSchema)` to create a new message. + */ +export const McpRebindDrainSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 62); + +/** + * @generated from message engram.app.v1.McpRebindConnection + */ +export type McpRebindConnection = Message<"engram.app.v1.McpRebindConnection"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: string alias = 2; + */ + alias: string; + + /** + * @generated from field: string endpoint = 3; + */ + endpoint: string; + + /** + * @generated from field: string zone = 4; + */ + zone: string; + + /** + * @generated from field: string credential_identity = 5; + */ + credentialIdentity: string; + + /** + * @generated from field: uint32 config_revision = 6; + */ + configRevision: number; + + /** + * @generated from field: uint32 auth_generation = 7; + */ + authGeneration: number; + + /** + * @generated from field: uint32 catalog_revision = 8; + */ + catalogRevision: number; +}; + +/** + * Describes the message engram.app.v1.McpRebindConnection. + * Use `create(McpRebindConnectionSchema)` to create a new message. + */ +export const McpRebindConnectionSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 63); + +/** + * @generated from message engram.app.v1.RebindMcpRequest + */ +export type RebindMcpRequest = Message<"engram.app.v1.RebindMcpRequest"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string expected_binding_hash = 2; + */ + expectedBindingHash: string; + + /** + * @generated from field: string expected_candidate_hash = 3; + */ + expectedCandidateHash: string; +}; + +/** + * Describes the message engram.app.v1.RebindMcpRequest. + * Use `create(RebindMcpRequestSchema)` to create a new message. + */ +export const RebindMcpRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 64); + +/** + * @generated from message engram.app.v1.RebindMcpResponse + */ +export type RebindMcpResponse = Message<"engram.app.v1.RebindMcpResponse"> & { + /** + * @generated from field: string binding_hash = 1; + */ + bindingHash: string; +}; + +/** + * Describes the message engram.app.v1.RebindMcpResponse. + * Use `create(RebindMcpResponseSchema)` to create a new message. + */ +export const RebindMcpResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 65); + +/** + * Entered values are request-only; responses contain metadata and references. + * + * @generated from message engram.app.v1.PutMcpCredentialRequest + */ +export type PutMcpCredentialRequest = Message<"engram.app.v1.PutMcpCredentialRequest"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: uint32 expected_revision = 2; + */ + expectedRevision: number; + + /** + * @generated from field: uint32 expected_generation = 3; + */ + expectedGeneration: number; + + /** + * @generated from field: string kind = 4; + */ + kind: string; + + /** + * @generated from field: string header = 5; + */ + header: string; + + /** + * @generated from field: string value = 6; + */ + value: string; +}; + +/** + * Describes the message engram.app.v1.PutMcpCredentialRequest. + * Use `create(PutMcpCredentialRequestSchema)` to create a new message. + */ +export const PutMcpCredentialRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 66); + +/** + * @generated from message engram.app.v1.McpCredentialStatus + */ +export type McpCredentialStatus = Message<"engram.app.v1.McpCredentialStatus"> & { + /** + * @generated from field: string status = 1; + */ + status: string; + + /** + * @generated from field: string reference = 2; + */ + reference: string; + + /** + * @generated from field: string kind = 3; + */ + kind: string; + + /** + * @generated from field: string credential_identity = 4; + */ + credentialIdentity: string; + + /** + * @generated from field: int64 version = 5; + */ + version: bigint; + + /** + * @generated from field: int64 reservation_generation = 6; + */ + reservationGeneration: bigint; +}; + +/** + * Describes the message engram.app.v1.McpCredentialStatus. + * Use `create(McpCredentialStatusSchema)` to create a new message. + */ +export const McpCredentialStatusSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 67); + +/** + * @generated from message engram.app.v1.PutMcpCredentialResponse + */ +export type PutMcpCredentialResponse = Message<"engram.app.v1.PutMcpCredentialResponse"> & { + /** + * @generated from field: engram.app.v1.IntegrationConnection connection = 1; + */ + connection?: IntegrationConnection; + + /** + * @generated from field: engram.app.v1.McpCredentialStatus credential = 2; + */ + credential?: McpCredentialStatus; +}; + +/** + * Describes the message engram.app.v1.PutMcpCredentialResponse. + * Use `create(PutMcpCredentialResponseSchema)` to create a new message. + */ +export const PutMcpCredentialResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 68); + +/** + * @generated from message engram.app.v1.CompleteMcpAuthorizationResponse + */ +export type CompleteMcpAuthorizationResponse = Message<"engram.app.v1.CompleteMcpAuthorizationResponse"> & { + /** + * @generated from field: engram.app.v1.IntegrationConnection connection = 1; + */ + connection?: IntegrationConnection; + + /** + * @generated from field: engram.app.v1.McpCredentialStatus credential = 2; + */ + credential?: McpCredentialStatus; +}; + +/** + * Describes the message engram.app.v1.CompleteMcpAuthorizationResponse. + * Use `create(CompleteMcpAuthorizationResponseSchema)` to create a new message. + */ +export const CompleteMcpAuthorizationResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 69); + +/** + * @generated from message engram.app.v1.RecoverMcpCredentialResponse + */ +export type RecoverMcpCredentialResponse = Message<"engram.app.v1.RecoverMcpCredentialResponse"> & { + /** + * @generated from field: engram.app.v1.IntegrationConnection connection = 1; + */ + connection?: IntegrationConnection; + + /** + * @generated from field: engram.app.v1.McpCredentialStatus credential = 2; + */ + credential?: McpCredentialStatus; +}; + +/** + * Describes the message engram.app.v1.RecoverMcpCredentialResponse. + * Use `create(RecoverMcpCredentialResponseSchema)` to create a new message. + */ +export const RecoverMcpCredentialResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 70); + +/** + * @generated from message engram.app.v1.McpCredentialState + */ +export type McpCredentialState = Message<"engram.app.v1.McpCredentialState"> & { + /** + * @generated from field: engram.app.v1.IntegrationConnection connection = 1; + */ + connection?: IntegrationConnection; + + /** + * @generated from field: engram.app.v1.McpCredentialStatus credential = 2; + */ + credential?: McpCredentialStatus; +}; + +/** + * Describes the message engram.app.v1.McpCredentialState. + * Use `create(McpCredentialStateSchema)` to create a new message. + */ +export const McpCredentialStateSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 71); + +/** + * @generated from message engram.app.v1.McpApprovedDestination + */ +export type McpApprovedDestination = Message<"engram.app.v1.McpApprovedDestination"> & { + /** + * @generated from field: string zone_id = 1; + */ + zoneId: string; + + /** + * @generated from field: string endpoint = 2; + */ + endpoint: string; +}; + +/** + * Describes the message engram.app.v1.McpApprovedDestination. + * Use `create(McpApprovedDestinationSchema)` to create a new message. + */ +export const McpApprovedDestinationSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 72); + +/** + * @generated from message engram.app.v1.McpClientRegistration + */ +export type McpClientRegistration = Message<"engram.app.v1.McpClientRegistration"> & { + /** + * @generated from field: string id = 1; + */ + id: string; + + /** + * @generated from field: string client_id = 2; + */ + clientId: string; + + /** + * @generated from field: bool confidential = 3; + */ + confidential: boolean; + + /** + * @generated from field: string redirect_uri = 4; + */ + redirectUri: string; + + /** + * @generated from field: string issuer = 5; + */ + issuer: string; + + /** + * @generated from field: engram.app.v1.McpApprovedDestination resource = 6; + */ + resource?: McpApprovedDestination; + + /** + * @generated from field: repeated string scopes = 7; + */ + scopes: string[]; +}; + +/** + * Describes the message engram.app.v1.McpClientRegistration. + * Use `create(McpClientRegistrationSchema)` to create a new message. + */ +export const McpClientRegistrationSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 73); + +/** + * @generated from message engram.app.v1.GetMcpSetupRequest + */ +export type GetMcpSetupRequest = Message<"engram.app.v1.GetMcpSetupRequest"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; +}; + +/** + * Describes the message engram.app.v1.GetMcpSetupRequest. + * Use `create(GetMcpSetupRequestSchema)` to create a new message. + */ +export const GetMcpSetupRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 74); + +/** + * @generated from message engram.app.v1.GetMcpSetupResponse + */ +export type GetMcpSetupResponse = Message<"engram.app.v1.GetMcpSetupResponse"> & { + /** + * @generated from field: bool enabled = 1; + */ + enabled: boolean; + + /** + * @generated from field: repeated engram.app.v1.McpClientRegistration registrations = 2; + */ + registrations: McpClientRegistration[]; + + /** + * @generated from field: repeated engram.app.v1.McpApprovedDestination destinations = 3; + */ + destinations: McpApprovedDestination[]; + + /** + * @generated from field: engram.app.v1.McpCredentialState current = 4; + */ + current?: McpCredentialState; + + /** + * @generated from field: bool transition_pending = 5; + */ + transitionPending: boolean; + + /** + * @generated from field: string flow_id = 6; + */ + flowId: string; + + /** + * @generated from field: string flow_expires_at = 7; + */ + flowExpiresAt: string; + + /** + * @generated from field: bool client_secret_configured = 8; + */ + clientSecretConfigured: boolean; + + /** + * @generated from field: bool discovery_available = 9; + */ + discoveryAvailable: boolean; + + /** + * Credential reconciliation and disconnect, independent of new dispatch. + * + * @generated from field: bool recovery_available = 10; + */ + recoveryAvailable: boolean; +}; + +/** + * Describes the message engram.app.v1.GetMcpSetupResponse. + * Use `create(GetMcpSetupResponseSchema)` to create a new message. + */ +export const GetMcpSetupResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 75); + +/** + * @generated from message engram.app.v1.CompleteMcpAuthorizationRequest + */ +export type CompleteMcpAuthorizationRequest = Message<"engram.app.v1.CompleteMcpAuthorizationRequest"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: uint32 expected_revision = 2; + */ + expectedRevision: number; + + /** + * @generated from field: uint32 expected_generation = 3; + */ + expectedGeneration: number; + + /** + * @generated from field: string flow_id = 4; + */ + flowId: string; + + /** + * @generated from field: string state = 5; + */ + state: string; + + /** + * @generated from field: string code = 6; + */ + code: string; + + /** + * @generated from field: optional string issuer = 7; + */ + issuer?: string; + + /** + * @generated from field: string redirect_uri = 8; + */ + redirectUri: string; +}; + +/** + * Describes the message engram.app.v1.CompleteMcpAuthorizationRequest. + * Use `create(CompleteMcpAuthorizationRequestSchema)` to create a new message. + */ +export const CompleteMcpAuthorizationRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 76); + +/** + * @generated from message engram.app.v1.RecoverMcpCredentialRequest + */ +export type RecoverMcpCredentialRequest = Message<"engram.app.v1.RecoverMcpCredentialRequest"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: uint32 expected_revision = 2; + */ + expectedRevision: number; + + /** + * @generated from field: uint32 expected_generation = 3; + */ + expectedGeneration: number; +}; + +/** + * Describes the message engram.app.v1.RecoverMcpCredentialRequest. + * Use `create(RecoverMcpCredentialRequestSchema)` to create a new message. + */ +export const RecoverMcpCredentialRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 77); + +/** + * Exact server review. Clients must return it unchanged when they decide. + * + * @generated from message engram.app.v1.McpApprovalReviewContext + */ +export type McpApprovalReviewContext = Message<"engram.app.v1.McpApprovalReviewContext"> & { + /** + * @generated from field: string execution_id = 1; + */ + executionId: string; + + /** + * @generated from field: string session_id = 2; + */ + sessionId: string; + + /** + * @generated from field: string run_id = 3; + */ + runId: string; + + /** + * @generated from field: string tool_call_id = 4; + */ + toolCallId: string; + + /** + * @generated from field: string request_digest = 5; + */ + requestDigest: string; + + /** + * @generated from field: string argument_hash = 6; + */ + argumentHash: string; + + /** + * @generated from field: string binding_hash = 7; + */ + bindingHash: string; + + /** + * @generated from field: string connection_id = 8; + */ + connectionId: string; + + /** + * @generated from field: string catalog_id = 9; + */ + catalogId: string; + + /** + * @generated from field: uint32 catalog_revision = 10; + */ + catalogRevision: number; + + /** + * @generated from field: uint32 config_revision = 11; + */ + configRevision: number; + + /** + * @generated from field: uint32 auth_generation = 12; + */ + authGeneration: number; + + /** + * @generated from field: string credential_identity = 13; + */ + credentialIdentity: string; + + /** + * @generated from field: string upstream_name = 14; + */ + upstreamName: string; + + /** + * @generated from field: string tool_fingerprint = 15; + */ + toolFingerprint: string; + + /** + * @generated from field: string context_digest = 16; + */ + contextDigest: string; + + /** + * @generated from field: uint32 journal_revision = 17; + */ + journalRevision: number; + + /** + * @generated from field: string expires_at = 18; + */ + expiresAt: string; + + /** + * @generated from field: string policy_revision = 19; + */ + policyRevision: string; + + /** + * @generated from field: string binding_epoch = 20; + */ + bindingEpoch: string; +}; + +/** + * Describes the message engram.app.v1.McpApprovalReviewContext. + * Use `create(McpApprovalReviewContextSchema)` to create a new message. + */ +export const McpApprovalReviewContextSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 78); + +/** + * @generated from message engram.app.v1.McpApprovalReceipt + */ +export type McpApprovalReceipt = Message<"engram.app.v1.McpApprovalReceipt"> & { + /** + * @generated from field: string execution_id = 1; + */ + executionId: string; + + /** + * @generated from field: string decision = 2; + */ + decision: string; + + /** + * @generated from field: string actor_id = 3; + */ + actorId: string; + + /** + * @generated from field: string decided_at = 4; + */ + decidedAt: string; + + /** + * @generated from field: uint32 journal_revision = 5; + */ + journalRevision: number; + + /** + * @generated from field: engram.app.v1.McpApprovalReviewContext reviewed_context = 6; + */ + reviewedContext?: McpApprovalReviewContext; +}; + +/** + * Describes the message engram.app.v1.McpApprovalReceipt. + * Use `create(McpApprovalReceiptSchema)` to create a new message. + */ +export const McpApprovalReceiptSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 79); + +/** + * @generated from message engram.app.v1.McpExecutionReview + */ +export type McpExecutionReview = Message<"engram.app.v1.McpExecutionReview"> & { + /** + * @generated from field: string execution_id = 1; + */ + executionId: string; + + /** + * @generated from field: string connection_id = 2; + */ + connectionId: string; + + /** + * @generated from field: string state = 3; + */ + state: string; + + /** + * @generated from field: uint32 journal_revision = 4; + */ + journalRevision: number; + + /** + * @generated from field: bool result_unavailable = 5; + */ + resultUnavailable: boolean; + + /** + * @generated from field: engram.app.v1.McpApproval review = 6; + */ + review?: McpApproval; + + /** + * @generated from field: repeated string allowed_actions = 7; + */ + allowedActions: string[]; + + /** + * @generated from field: string reason = 8; + */ + reason: string; + + /** + * @generated from field: string tool_call_id = 9; + */ + toolCallId: string; + + /** + * @generated from field: string run_id = 10; + */ + runId: string; + + /** + * @generated from field: string request_digest = 11; + */ + requestDigest: string; + + /** + * @generated from field: string upstream_name = 12; + */ + upstreamName: string; + + /** + * @generated from field: string approval_decision = 13; + */ + approvalDecision: string; + + /** + * @generated from field: string original_arguments = 14; + */ + originalArguments: string; + + /** + * @generated from field: engram.app.v1.McpTool reviewed_tool = 15; + */ + reviewedTool?: McpTool; + + /** + * @generated from field: repeated string argument_constraint_documents_json = 16; + */ + argumentConstraintDocumentsJson: string[]; + + /** + * @generated from field: string binding_hash = 17; + */ + bindingHash: string; + + /** + * @generated from field: engram.app.v1.McpUnknownContext expected_unknown_context = 18; + */ + expectedUnknownContext?: McpUnknownContext; + + /** + * @generated from field: engram.app.v1.McpCancellationReceipt cancellation_receipt = 19; + */ + cancellationReceipt?: McpCancellationReceipt; +}; + +/** + * Describes the message engram.app.v1.McpExecutionReview. + * Use `create(McpExecutionReviewSchema)` to create a new message. + */ +export const McpExecutionReviewSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 80); + +/** + * @generated from message engram.app.v1.McpUnknownContext + */ +export type McpUnknownContext = Message<"engram.app.v1.McpUnknownContext"> & { + /** + * @generated from field: string execution_id = 1; + */ + executionId: string; + + /** + * @generated from field: uint32 journal_revision = 2; + */ + journalRevision: number; + + /** + * @generated from field: string request_digest = 3; + */ + requestDigest: string; + + /** + * @generated from field: string binding_hash = 4; + */ + bindingHash: string; + + /** + * @generated from field: string intent_id = 5; + */ + intentId: string; +}; + +/** + * Describes the message engram.app.v1.McpUnknownContext. + * Use `create(McpUnknownContextSchema)` to create a new message. + */ +export const McpUnknownContextSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 81); + +/** + * @generated from message engram.app.v1.McpCancellationReceipt + */ +export type McpCancellationReceipt = Message<"engram.app.v1.McpCancellationReceipt"> & { + /** + * @generated from field: string execution_id = 1; + */ + executionId: string; + + /** + * @generated from field: string actor_id = 2; + */ + actorId: string; + + /** + * @generated from field: string cause = 3; + */ + cause: string; + + /** + * @generated from field: uint32 expected_journal_revision = 4; + */ + expectedJournalRevision: number; + + /** + * @generated from field: uint32 journal_revision = 5; + */ + journalRevision: number; + + /** + * @generated from field: string state = 6; + */ + state: string; + + /** + * @generated from field: string created_at = 7; + */ + createdAt: string; +}; + +/** + * Describes the message engram.app.v1.McpCancellationReceipt. + * Use `create(McpCancellationReceiptSchema)` to create a new message. + */ +export const McpCancellationReceiptSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 82); + +/** + * @generated from message engram.app.v1.McpResolutionReceipt + */ +export type McpResolutionReceipt = Message<"engram.app.v1.McpResolutionReceipt"> & { + /** + * @generated from field: string execution_id = 1; + */ + executionId: string; + + /** + * @generated from field: string actor_id = 2; + */ + actorId: string; + + /** + * @generated from field: string outcome = 3; + */ + outcome: string; + + /** + * @generated from field: string created_at = 4; + */ + createdAt: string; + + /** + * @generated from field: engram.app.v1.McpUnknownContext expected_context = 5; + */ + expectedContext?: McpUnknownContext; +}; + +/** + * Describes the message engram.app.v1.McpResolutionReceipt. + * Use `create(McpResolutionReceiptSchema)` to create a new message. + */ +export const McpResolutionReceiptSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 83); + +/** + * @generated from message engram.app.v1.McpBindingReceipt + */ +export type McpBindingReceipt = Message<"engram.app.v1.McpBindingReceipt"> & { + /** + * @generated from field: string binding_hash = 1; + */ + bindingHash: string; + + /** + * @generated from field: string binding_epoch = 2; + */ + bindingEpoch: string; + + /** + * @generated from field: bool current = 3; + */ + current: boolean; + + /** + * @generated from field: repeated string stale_reasons = 4; + */ + staleReasons: string[]; + + /** + * @generated from field: repeated string allowed_actions = 5; + */ + allowedActions: string[]; +}; + +/** + * Describes the message engram.app.v1.McpBindingReceipt. + * Use `create(McpBindingReceiptSchema)` to create a new message. + */ +export const McpBindingReceiptSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 84); + +/** + * @generated from message engram.app.v1.GetSessionMcpStateRequest + */ +export type GetSessionMcpStateRequest = Message<"engram.app.v1.GetSessionMcpStateRequest"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string after_execution_id = 2; + */ + afterExecutionId: string; + + /** + * @generated from field: uint32 page_size = 3; + */ + pageSize: number; + + /** + * @generated from field: optional string plan_tool_call_id = 4; + */ + planToolCallId?: string; + + /** + * @generated from field: string after_binding_tool_name = 5; + */ + afterBindingToolName: string; + + /** + * @generated from field: uint32 binding_page_size = 6; + */ + bindingPageSize: number; + + /** + * @generated from field: bool omit_bindings = 7; + */ + omitBindings: boolean; + + /** + * @generated from field: bool omit_executions = 8; + */ + omitExecutions: boolean; +}; + +/** + * Describes the message engram.app.v1.GetSessionMcpStateRequest. + * Use `create(GetSessionMcpStateRequestSchema)` to create a new message. + */ +export const GetSessionMcpStateRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 85); + +/** + * @generated from message engram.app.v1.GetSessionMcpStateResponse + */ +export type GetSessionMcpStateResponse = Message<"engram.app.v1.GetSessionMcpStateResponse"> & { + /** + * @generated from field: engram.app.v1.McpBindingReceipt binding_receipt = 1; + */ + bindingReceipt?: McpBindingReceipt; + + /** + * @generated from field: repeated engram.app.v1.McpExecutionReview executions = 2; + */ + executions: McpExecutionReview[]; + + /** + * @generated from field: repeated engram.app.v1.McpApproval approvals = 3; + */ + approvals: McpApproval[]; + + /** + * @generated from field: repeated string unknown_write_blocks = 4; + */ + unknownWriteBlocks: string[]; + + /** + * @generated from field: repeated engram.app.v1.McpResolutionReceipt resolution_receipts = 5; + */ + resolutionReceipts: McpResolutionReceipt[]; + + /** + * @generated from field: repeated string allowed_actions = 6; + */ + allowedActions: string[]; + + /** + * @generated from field: string availability = 7; + */ + availability: string; + + /** + * @generated from field: string next_execution_id = 8; + */ + nextExecutionId: string; + + /** + * @generated from field: engram.app.v1.GetMcpWritePolicyResponse write_policy = 9; + */ + writePolicy?: GetMcpWritePolicyResponse; + + /** + * @generated from field: repeated engram.app.v1.McpBoundToolSummary binding_tools = 10; + */ + bindingTools: McpBoundToolSummary[]; + + /** + * @generated from field: string next_binding_tool_name = 11; + */ + nextBindingToolName: string; + + /** + * @generated from field: bool has_more_binding_tools = 12; + */ + hasMoreBindingTools: boolean; + + /** + * @generated from field: bool has_more_executions = 13; + */ + hasMoreExecutions: boolean; +}; + +/** + * Describes the message engram.app.v1.GetSessionMcpStateResponse. + * Use `create(GetSessionMcpStateResponseSchema)` to create a new message. + */ +export const GetSessionMcpStateResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 86); + +/** + * @generated from message engram.app.v1.GetMcpExecutionResultRequest + */ +export type GetMcpExecutionResultRequest = Message<"engram.app.v1.GetMcpExecutionResultRequest"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string execution_id = 2; + */ + executionId: string; +}; + +/** + * Describes the message engram.app.v1.GetMcpExecutionResultRequest. + * Use `create(GetMcpExecutionResultRequestSchema)` to create a new message. + */ +export const GetMcpExecutionResultRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 87); + +/** + * @generated from message engram.app.v1.GetMcpExecutionResultResponse + */ +export type GetMcpExecutionResultResponse = Message<"engram.app.v1.GetMcpExecutionResultResponse"> & { + /** + * @generated from field: string state = 1; + */ + state: string; + + /** + * @generated from field: bool result_unavailable = 2; + */ + resultUnavailable: boolean; + + /** + * @generated from field: optional string result_json = 3; + */ + resultJson?: string; + + /** + * @generated from field: engram.app.v1.McpCompletionSettlement settlement = 4; + */ + settlement: McpCompletionSettlement; +}; + +/** + * Describes the message engram.app.v1.GetMcpExecutionResultResponse. + * Use `create(GetMcpExecutionResultResponseSchema)` to create a new message. + */ +export const GetMcpExecutionResultResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 88); + +/** + * Immutable session grant. Schemas remain independent documents with their own roots. + * + * @generated from message engram.app.v1.McpBoundToolSummary + */ +export type McpBoundToolSummary = Message<"engram.app.v1.McpBoundToolSummary"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: string connection_alias = 2; + */ + connectionAlias: string; + + /** + * @generated from field: string model_tool_name = 3; + */ + modelToolName: string; + + /** + * @generated from field: engram.app.v1.McpTool reviewed_tool = 4; + */ + reviewedTool?: McpTool; + + /** + * @generated from field: repeated string argument_constraint_documents_json = 5; + */ + argumentConstraintDocumentsJson: string[]; + + /** + * @generated from field: bool automation_write = 6; + */ + automationWrite: boolean; + + /** + * @generated from field: bool unrestricted = 7; + */ + unrestricted: boolean; + + /** + * @generated from field: string catalog_id = 8; + */ + catalogId: string; + + /** + * @generated from field: uint32 catalog_revision = 9; + */ + catalogRevision: number; + + /** + * @generated from field: uint32 config_revision = 10; + */ + configRevision: number; + + /** + * @generated from field: uint32 auth_generation = 11; + */ + authGeneration: number; +}; + +/** + * Describes the message engram.app.v1.McpBoundToolSummary. + * Use `create(McpBoundToolSummarySchema)` to create a new message. + */ +export const McpBoundToolSummarySchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_integration, 89); + +/** + * @generated from enum engram.app.v1.McpRebindDrainReason + */ +export enum McpRebindDrainReason { + /** + * @generated from enum value: MCP_REBIND_DRAIN_REASON_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: MCP_REBIND_DRAIN_REASON_DISPATCH_IN_PROGRESS = 1; + */ + DISPATCH_IN_PROGRESS = 1, + + /** + * @generated from enum value: MCP_REBIND_DRAIN_REASON_UNKNOWN_INTENT = 2; + */ + UNKNOWN_INTENT = 2, + + /** + * @generated from enum value: MCP_REBIND_DRAIN_REASON_LEGACY_INTENT = 3; + */ + LEGACY_INTENT = 3, +} + +/** + * Describes the enum engram.app.v1.McpRebindDrainReason. + */ +export const McpRebindDrainReasonSchema: GenEnum = /*@__PURE__*/ + enumDesc(file_engram_app_v1_integration, 0); + +/** + * Settlement does not establish guest delivery or an upstream outcome. + * + * @generated from enum engram.app.v1.McpCompletionSettlement + */ +export enum McpCompletionSettlement { + /** + * @generated from enum value: MCP_COMPLETION_SETTLEMENT_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: MCP_COMPLETION_SETTLEMENT_PENDING = 1; + */ + PENDING = 1, + + /** + * @generated from enum value: MCP_COMPLETION_SETTLEMENT_SETTLED_OR_TARGET_GONE = 2; + */ + SETTLED_OR_TARGET_GONE = 2, + + /** + * @generated from enum value: MCP_COMPLETION_SETTLEMENT_NOT_APPLICABLE = 3; + */ + NOT_APPLICABLE = 3, +} + +/** + * Describes the enum engram.app.v1.McpCompletionSettlement. + */ +export const McpCompletionSettlementSchema: GenEnum = /*@__PURE__*/ + enumDesc(file_engram_app_v1_integration, 1); + +/** + * ADR 0057 C3: orchestrator-native connector catalog CRUD (Plane B). Connectors + * live in the orchestrator DB (the `connector` table, C1); the built-ins + * (github/datadog) are read-only file seeds surfaced alongside custom ones. + * NOT compiled into the coordinator (orchestrator-native, like ProfileService / + * TaskService). Admin-gated; `config_json` is the raw connector JSON, validated + * by `parseConnector` (the admin-trust boundary) on write. + * + * @generated from service engram.app.v1.IntegrationService + */ +export const IntegrationService: GenService<{ + /** + * Built-in seeds (read-only) ∪ admin-authored connectors. Admin-only; each row + * carries the derived connected/available `status`. + * + * @generated from rpc engram.app.v1.IntegrationService.ListConnectors + */ + listConnectors: { + methodKind: "unary"; + input: typeof ListConnectorsRequestSchema; + output: typeof ListConnectorsResponseSchema; + }, + /** + * Create or replace a custom connector. Rejects a built-in provider id and + * any config `parseConnector` refuses. Invalidates the registry cache. + * + * @generated from rpc engram.app.v1.IntegrationService.UpsertConnector + */ + upsertConnector: { + methodKind: "unary"; + input: typeof UpsertConnectorRequestSchema; + output: typeof UpsertConnectorResponseSchema; + }, + /** + * Delete a custom connector (idempotent). Rejects a built-in provider id. + * + * @generated from rpc engram.app.v1.IntegrationService.DeleteConnector + */ + deleteConnector: { + methodKind: "unary"; + input: typeof DeleteConnectorRequestSchema; + output: typeof DeleteConnectorResponseSchema; + }, + /** + * Member-readable provider catalog: per-connector display identity + the + * powers it grants + the hosts it opens, derived from the merged registry. + * Carries NO secret material (no secretRef / header / template / mint kind) — + * safe for the Launch receipt + in-session provider icons (redesign). + * + * @generated from rpc engram.app.v1.IntegrationService.GetIntegrationCatalog + */ + getIntegrationCatalog: { + methodKind: "unary"; + input: typeof GetIntegrationCatalogRequestSchema; + output: typeof GetIntegrationCatalogResponseSchema; + }, + /** + * Admin-only: store a mint kind's credentials, sealing each field as an org + * secret `.` (the coordinator resolves them by that name). The + * GitHub App private key is a secret; the App ID is config — both are sealed. + * + * @generated from rpc engram.app.v1.IntegrationService.SetMintCredential + */ + setMintCredential: { + methodKind: "unary"; + input: typeof SetMintCredentialRequestSchema; + output: typeof SetMintCredentialResponseSchema; + }, + /** + * Admin-only: upload (or replace) a connector's logo — orchestrator-owned + * (the coordinator never sees connectors). Bytes land in the connector_logo + * table (≤512 KB, SVG or square PNG, sniffed) and are served at + * GET /api/v1/integrations//logo. Empty `data` clears the logo. + * + * @generated from rpc engram.app.v1.IntegrationService.UploadConnectorLogo + */ + uploadConnectorLogo: { + methodKind: "unary"; + input: typeof UploadConnectorLogoRequestSchema; + output: typeof UploadConnectorLogoResponseSchema; + }, + /** + * Admin-only: test a connector's credential. Builds the resolved spec from the + * registry and delegates the actual unseal/mint + benign GET to the coordinator + * (MintService.RunConnectorTest). `draft_values` carries an about-to-be-saved + * credential to test before sealing (inject: the single value under any key; + * mint: the field map); empty tests the stored credential. + * + * @generated from rpc engram.app.v1.IntegrationService.TestConnector + */ + testConnector: { + methodKind: "unary"; + input: typeof TestConnectorRequestSchema; + output: typeof TestConnectorResponseSchema; + }, + /** + * Admin-only: store the values of a connector's declared `settings` (non- + * secret parameters such as the API host a regional or self-hosted provider + * is reached on). Values live on the provider's default connection and feed + * the connector's effective hosts, the test probe, and the CLI env. + * + * @generated from rpc engram.app.v1.IntegrationService.SetConnectorSettings + */ + setConnectorSettings: { + methodKind: "unary"; + input: typeof SetConnectorSettingsRequestSchema; + output: typeof SetConnectorSettingsResponseSchema; + }, + /** + * ADR 0109: configured credential instances. Existing providers expose an + * ordinary default connection; the first provider with user-created named + * connections is Google Cloud, whose config contains no private key. + * + * @generated from rpc engram.app.v1.IntegrationService.ListConnections + */ + listConnections: { + methodKind: "unary"; + input: typeof ListConnectionsRequestSchema; + output: typeof ListConnectionsResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.CreateConnection + */ + createConnection: { + methodKind: "unary"; + input: typeof CreateConnectionRequestSchema; + output: typeof CreateConnectionResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.UpdateConnection + */ + updateConnection: { + methodKind: "unary"; + input: typeof UpdateConnectionRequestSchema; + output: typeof UpdateConnectionResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.DeleteConnection + */ + deleteConnection: { + methodKind: "unary"; + input: typeof DeleteConnectionRequestSchema; + output: typeof DeleteConnectionResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.TestConnection + */ + testConnection: { + methodKind: "unary"; + input: typeof TestConnectionRequestSchema; + output: typeof TestConnectionResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.SetConnectionEnabled + */ + setConnectionEnabled: { + methodKind: "unary"; + input: typeof SetConnectionEnabledRequestSchema; + output: typeof SetConnectionEnabledResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.GetGoogleCloudSetup + */ + getGoogleCloudSetup: { + methodKind: "unary"; + input: typeof GetGoogleCloudSetupRequestSchema; + output: typeof GetGoogleCloudSetupResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.GetMcpCatalogs + */ + getMcpCatalogs: { + methodKind: "unary"; + input: typeof GetMcpCatalogsRequestSchema; + output: typeof GetMcpCatalogsResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.DiscoverMcp + */ + discoverMcp: { + methodKind: "unary"; + input: typeof DiscoverMcpRequestSchema; + output: typeof DiscoverMcpResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.PublishMcp + */ + publishMcp: { + methodKind: "unary"; + input: typeof PublishMcpRequestSchema; + output: typeof PublishMcpResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.PutMcpCredential + */ + putMcpCredential: { + methodKind: "unary"; + input: typeof PutMcpCredentialRequestSchema; + output: typeof PutMcpCredentialResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.GetMcpSetup + */ + getMcpSetup: { + methodKind: "unary"; + input: typeof GetMcpSetupRequestSchema; + output: typeof GetMcpSetupResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.CompleteMcpAuthorization + */ + completeMcpAuthorization: { + methodKind: "unary"; + input: typeof CompleteMcpAuthorizationRequestSchema; + output: typeof CompleteMcpAuthorizationResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.RecoverMcpCredential + */ + recoverMcpCredential: { + methodKind: "unary"; + input: typeof RecoverMcpCredentialRequestSchema; + output: typeof RecoverMcpCredentialResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.AuthorizeMcp + */ + authorizeMcp: { + methodKind: "unary"; + input: typeof AuthorizeMcpRequestSchema; + output: typeof AuthorizeMcpResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.DisconnectMcp + */ + disconnectMcp: { + methodKind: "unary"; + input: typeof DisconnectMcpRequestSchema; + output: typeof DisconnectMcpResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.GetSessionMcpState + */ + getSessionMcpState: { + methodKind: "unary"; + input: typeof GetSessionMcpStateRequestSchema; + output: typeof GetSessionMcpStateResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.GetMcpExecutionResult + */ + getMcpExecutionResult: { + methodKind: "unary"; + input: typeof GetMcpExecutionResultRequestSchema; + output: typeof GetMcpExecutionResultResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.ListMcpApprovals + */ + listMcpApprovals: { + methodKind: "unary"; + input: typeof ListMcpApprovalsRequestSchema; + output: typeof ListMcpApprovalsResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.DecideMcpApproval + */ + decideMcpApproval: { + methodKind: "unary"; + input: typeof DecideMcpApprovalRequestSchema; + output: typeof DecideMcpApprovalResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.CancelMcpCall + */ + cancelMcpCall: { + methodKind: "unary"; + input: typeof CancelMcpCallRequestSchema; + output: typeof CancelMcpCallResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.ResolveMcpOutcome + */ + resolveMcpOutcome: { + methodKind: "unary"; + input: typeof ResolveMcpOutcomeRequestSchema; + output: typeof ResolveMcpOutcomeResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.PrepareMcpRebind + */ + prepareMcpRebind: { + methodKind: "unary"; + input: typeof PrepareMcpRebindRequestSchema; + output: typeof PrepareMcpRebindResponseSchema; + }, + /** + * @generated from rpc engram.app.v1.IntegrationService.RebindMcp + */ + rebindMcp: { + methodKind: "unary"; + input: typeof RebindMcpRequestSchema; + output: typeof RebindMcpResponseSchema; }, }> = /*@__PURE__*/ serviceDesc(file_engram_app_v1_integration, 0); diff --git a/orchestrator/src/gen/engram/app/v1/profile_pb.ts b/orchestrator/src/gen/engram/app/v1/profile_pb.ts index 1dfa02b2f..0ab89ac6d 100644 --- a/orchestrator/src/gen/engram/app/v1/profile_pb.ts +++ b/orchestrator/src/gen/engram/app/v1/profile_pb.ts @@ -10,7 +10,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file engram/app/v1/profile.proto. */ export const file_engram_app_v1_profile: GenFile = /*@__PURE__*/ - fileDesc("ChtlbmdyYW0vYXBwL3YxL3Byb2ZpbGUucHJvdG8SDWVuZ3JhbS5hcHAudjEi6QUKB1Byb2ZpbGUSCgoCaWQYASABKAkSDAoEbmFtZRgCIAEoCRITCgtkZXNjcmlwdGlvbhgDIAEoCRIMCgRpY29uGAQgASgJEhAKCGltYWdlX2lkGAUgASgJEhsKE2luY2x1ZGVfdXNlcl90b2tlbnMYBiABKAgSNQoIZW52X3ZhcnMYByADKAsyIy5lbmdyYW0uYXBwLnYxLlByb2ZpbGUuRW52VmFyc0VudHJ5EhAKCGFyY2hpdmVkGAggASgIEhIKCmNyZWF0ZWRfYXQYCSABKAkSEgoKdXBkYXRlZF9hdBgKIAEoCRIOCgZza2lsbHMYCyADKAkSLgoHbmV0d29yaxgNIAEoCzIdLmVuZ3JhbS5hcHAudjEuUHJvZmlsZU5ldHdvcmsSLQoHc2VjcmV0cxgOIAMoCzIcLmVuZ3JhbS5hcHAudjEuUHJvZmlsZVNlY3JldBIUCgdoYXJuZXNzGBAgASgJSACIAQESEgoFbW9kZWwYESABKAlIAYgBARITCgZlZmZvcnQYEiABKAlIAogBARJCChJpbnRlZ3JhdGlvbl9ncmFudHMYFSADKAsyJi5lbmdyYW0uYXBwLnYxLlByb2ZpbGVJbnRlZ3JhdGlvbkdyYW50EikKBXJlcG9zGBYgAygLMhouZW5ncmFtLmFwcC52MS5Qcm9maWxlUmVwbxIZCgxtb2RlbF9yb3V0ZXIYFyABKAlIA4gBARInCgRhcHBzGBggAygLMhkuZW5ncmFtLmFwcC52MS5TZXNzaW9uQXBwEhkKEXByb2dyYW1tYXRpY19vbmx5GBkgASgIGi4KDEVudlZhcnNFbnRyeRILCgNrZXkYASABKAkSDQoFdmFsdWUYAiABKAk6AjgBQgoKCF9oYXJuZXNzQggKBl9tb2RlbEIJCgdfZWZmb3J0Qg8KDV9tb2RlbF9yb3V0ZXJKBAgMEA1KBAgPEBBKBAgTEBRKBAgUEBVSC2Rlc2lnbmF0aW9uIigKClNlc3Npb25BcHASDAoEbmFtZRgBIAEoCRIMCgRwb3J0GAIgASgNInEKC1Byb2ZpbGVSZXBvEgwKBHBhdGgYASABKAkSEgoKcmVtb3RlX3VybBgCIAEoCRI1CgZyZW1vdGUYAyABKAsyIC5lbmdyYW0uYXBwLnYxLlByb2ZpbGVSZXBvUmVtb3RlSACIAQFCCQoHX3JlbW90ZSI+ChFQcm9maWxlUmVwb1JlbW90ZRIMCgRob3N0GAEgASgJEg0KBW93bmVyGAIgASgJEgwKBG5hbWUYAyABKAkiMQobRGlzY292ZXJQcm9maWxlUmVwb3NSZXF1ZXN0EhIKCnByb2ZpbGVfaWQYASABKAkiUAoORGlzY292ZXJlZFJlcG8SDAoEcGF0aBgBIAEoCRIwCgdyZW1vdGVzGAIgAygLMh8uZW5ncmFtLmFwcC52MS5EaXNjb3ZlcmVkUmVtb3RlIm8KEERpc2NvdmVyZWRSZW1vdGUSDAoEbmFtZRgBIAEoCRILCgN1cmwYAiABKAkSNQoGcGFyc2VkGAMgASgLMiAuZW5ncmFtLmFwcC52MS5Qcm9maWxlUmVwb1JlbW90ZUgAiAEBQgkKB19wYXJzZWQiTAocRGlzY292ZXJQcm9maWxlUmVwb3NSZXNwb25zZRIsCgVyZXBvcxgBIAMoCzIdLmVuZ3JhbS5hcHAudjEuRGlzY292ZXJlZFJlcG8iewoXUHJvZmlsZUludGVncmF0aW9uR3JhbnQSFQoNY29ubmVjdGlvbl9pZBgBIAEoCRIRCglvcGVyYXRpb24YAiABKAkSHAoUcmVzb3VyY2VfY29uc3RyYWludHMYAyADKAkSGAoQY3JlZGVudGlhbF9zY29wZRgEIAEoCSJTCg5Qcm9maWxlTmV0d29yaxIPCgdkZWZhdWx0GAEgASgJEhMKC2FsbG93X2hvc3RzGAIgAygJEhsKE2FsbG93X2hvc3RfcGF0dGVybnMYAyADKAkibQoNUHJvZmlsZVNlY3JldBILCgNyZWYYASABKAkSDwoHZW52X3ZhchgCIAEoCRIMCgRtb2RlGAMgASgJEhMKC2FsbG93X2hvc3RzGAQgAygJEhsKE2FsbG93X2hvc3RfcGF0dGVybnMYBSADKAkibgoPUHJvZmlsZVNuYXBzaG90EgoKAmlkGAEgASgJEgwKBG5hbWUYAiABKAkSDAoEaWNvbhgDIAEoCRIQCghhcmNoaXZlZBgEIAEoCBIRCglpbWFnZV91cmkYBSABKAkSDgoGc2tpbGxzGAYgAygJIksKE0xpc3RQcm9maWxlc1JlcXVlc3QSGAoQaW5jbHVkZV9hcmNoaXZlZBgBIAEoCBIaChJmb3Jfc2Vzc2lvbl9waWNrZXIYAiABKAgiQAoUTGlzdFByb2ZpbGVzUmVzcG9uc2USKAoIcHJvZmlsZXMYASADKAsyFi5lbmdyYW0uYXBwLnYxLlByb2ZpbGUiHwoRR2V0UHJvZmlsZVJlcXVlc3QSCgoCaWQYASABKAkiPQoSR2V0UHJvZmlsZVJlc3BvbnNlEicKB3Byb2ZpbGUYASABKAsyFi5lbmdyYW0uYXBwLnYxLlByb2ZpbGUisAUKFENyZWF0ZVByb2ZpbGVSZXF1ZXN0EgwKBG5hbWUYASABKAkSEwoLZGVzY3JpcHRpb24YAiABKAkSDAoEaWNvbhgDIAEoCRIQCghpbWFnZV9pZBgEIAEoCRIbChNpbmNsdWRlX3VzZXJfdG9rZW5zGAUgASgIEkIKCGVudl92YXJzGAYgAygLMjAuZW5ncmFtLmFwcC52MS5DcmVhdGVQcm9maWxlUmVxdWVzdC5FbnZWYXJzRW50cnkSDgoGc2tpbGxzGAcgAygJEi4KB25ldHdvcmsYCSABKAsyHS5lbmdyYW0uYXBwLnYxLlByb2ZpbGVOZXR3b3JrEi0KB3NlY3JldHMYCiADKAsyHC5lbmdyYW0uYXBwLnYxLlByb2ZpbGVTZWNyZXQSFAoHaGFybmVzcxgMIAEoCUgAiAEBEhIKBW1vZGVsGA0gASgJSAGIAQESEwoGZWZmb3J0GA4gASgJSAKIAQESQgoSaW50ZWdyYXRpb25fZ3JhbnRzGBEgAygLMiYuZW5ncmFtLmFwcC52MS5Qcm9maWxlSW50ZWdyYXRpb25HcmFudBIpCgVyZXBvcxgSIAMoCzIaLmVuZ3JhbS5hcHAudjEuUHJvZmlsZVJlcG8SGQoMbW9kZWxfcm91dGVyGBMgASgJSAOIAQESJwoEYXBwcxgUIAMoCzIZLmVuZ3JhbS5hcHAudjEuU2Vzc2lvbkFwcBIZChFwcm9ncmFtbWF0aWNfb25seRgVIAEoCBouCgxFbnZWYXJzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4AUIKCghfaGFybmVzc0IICgZfbW9kZWxCCQoHX2VmZm9ydEIPCg1fbW9kZWxfcm91dGVySgQICBAJSgQICxAMSgQIDxAQSgQIEBARIkAKFUNyZWF0ZVByb2ZpbGVSZXNwb25zZRInCgdwcm9maWxlGAEgASgLMhYuZW5ncmFtLmFwcC52MS5Qcm9maWxlItcFChRVcGRhdGVQcm9maWxlUmVxdWVzdBIKCgJpZBgBIAEoCRIMCgRuYW1lGAIgASgJEhMKC2Rlc2NyaXB0aW9uGAMgASgJEgwKBGljb24YBCABKAkSEAoIaW1hZ2VfaWQYBSABKAkSGwoTaW5jbHVkZV91c2VyX3Rva2VucxgGIAEoCBJCCghlbnZfdmFycxgHIAMoCzIwLmVuZ3JhbS5hcHAudjEuVXBkYXRlUHJvZmlsZVJlcXVlc3QuRW52VmFyc0VudHJ5Eg4KBnNraWxscxgIIAMoCRIuCgduZXR3b3JrGAogASgLMh0uZW5ncmFtLmFwcC52MS5Qcm9maWxlTmV0d29yaxItCgdzZWNyZXRzGAsgAygLMhwuZW5ncmFtLmFwcC52MS5Qcm9maWxlU2VjcmV0EhQKB2hhcm5lc3MYDSABKAlIAIgBARISCgVtb2RlbBgOIAEoCUgBiAEBEhMKBmVmZm9ydBgPIAEoCUgCiAEBEkIKEmludGVncmF0aW9uX2dyYW50cxgSIAMoCzImLmVuZ3JhbS5hcHAudjEuUHJvZmlsZUludGVncmF0aW9uR3JhbnQSKQoFcmVwb3MYEyADKAsyGi5lbmdyYW0uYXBwLnYxLlByb2ZpbGVSZXBvEhkKDG1vZGVsX3JvdXRlchgUIAEoCUgDiAEBEicKBGFwcHMYFSADKAsyGS5lbmdyYW0uYXBwLnYxLlNlc3Npb25BcHASHgoRcHJvZ3JhbW1hdGljX29ubHkYFiABKAhIBIgBARouCgxFbnZWYXJzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4AUIKCghfaGFybmVzc0IICgZfbW9kZWxCCQoHX2VmZm9ydEIPCg1fbW9kZWxfcm91dGVyQhQKEl9wcm9ncmFtbWF0aWNfb25seUoECAkQCkoECAwQDUoECBAQEUoECBEQEiJAChVVcGRhdGVQcm9maWxlUmVzcG9uc2USJwoHcHJvZmlsZRgBIAEoCzIWLmVuZ3JhbS5hcHAudjEuUHJvZmlsZSIiChREZWxldGVQcm9maWxlUmVxdWVzdBIKCgJpZBgBIAEoCSIXChVEZWxldGVQcm9maWxlUmVzcG9uc2UywQQKDlByb2ZpbGVTZXJ2aWNlElcKDExpc3RQcm9maWxlcxIiLmVuZ3JhbS5hcHAudjEuTGlzdFByb2ZpbGVzUmVxdWVzdBojLmVuZ3JhbS5hcHAudjEuTGlzdFByb2ZpbGVzUmVzcG9uc2USUQoKR2V0UHJvZmlsZRIgLmVuZ3JhbS5hcHAudjEuR2V0UHJvZmlsZVJlcXVlc3QaIS5lbmdyYW0uYXBwLnYxLkdldFByb2ZpbGVSZXNwb25zZRJaCg1DcmVhdGVQcm9maWxlEiMuZW5ncmFtLmFwcC52MS5DcmVhdGVQcm9maWxlUmVxdWVzdBokLmVuZ3JhbS5hcHAudjEuQ3JlYXRlUHJvZmlsZVJlc3BvbnNlEloKDVVwZGF0ZVByb2ZpbGUSIy5lbmdyYW0uYXBwLnYxLlVwZGF0ZVByb2ZpbGVSZXF1ZXN0GiQuZW5ncmFtLmFwcC52MS5VcGRhdGVQcm9maWxlUmVzcG9uc2USbwoURGlzY292ZXJQcm9maWxlUmVwb3MSKi5lbmdyYW0uYXBwLnYxLkRpc2NvdmVyUHJvZmlsZVJlcG9zUmVxdWVzdBorLmVuZ3JhbS5hcHAudjEuRGlzY292ZXJQcm9maWxlUmVwb3NSZXNwb25zZRJaCg1EZWxldGVQcm9maWxlEiMuZW5ncmFtLmFwcC52MS5EZWxldGVQcm9maWxlUmVxdWVzdBokLmVuZ3JhbS5hcHAudjEuRGVsZXRlUHJvZmlsZVJlc3BvbnNlYgZwcm90bzM"); + fileDesc("ChtlbmdyYW0vYXBwL3YxL3Byb2ZpbGUucHJvdG8SDWVuZ3JhbS5hcHAudjEihQYKB1Byb2ZpbGUSCgoCaWQYASABKAkSDAoEbmFtZRgCIAEoCRITCgtkZXNjcmlwdGlvbhgDIAEoCRIMCgRpY29uGAQgASgJEhAKCGltYWdlX2lkGAUgASgJEhsKE2luY2x1ZGVfdXNlcl90b2tlbnMYBiABKAgSNQoIZW52X3ZhcnMYByADKAsyIy5lbmdyYW0uYXBwLnYxLlByb2ZpbGUuRW52VmFyc0VudHJ5EhAKCGFyY2hpdmVkGAggASgIEhIKCmNyZWF0ZWRfYXQYCSABKAkSEgoKdXBkYXRlZF9hdBgKIAEoCRIOCgZza2lsbHMYCyADKAkSLgoHbmV0d29yaxgNIAEoCzIdLmVuZ3JhbS5hcHAudjEuUHJvZmlsZU5ldHdvcmsSLQoHc2VjcmV0cxgOIAMoCzIcLmVuZ3JhbS5hcHAudjEuUHJvZmlsZVNlY3JldBIUCgdoYXJuZXNzGBAgASgJSACIAQESEgoFbW9kZWwYESABKAlIAYgBARITCgZlZmZvcnQYEiABKAlIAogBARJCChJpbnRlZ3JhdGlvbl9ncmFudHMYFSADKAsyJi5lbmdyYW0uYXBwLnYxLlByb2ZpbGVJbnRlZ3JhdGlvbkdyYW50EikKBXJlcG9zGBYgAygLMhouZW5ncmFtLmFwcC52MS5Qcm9maWxlUmVwbxIZCgxtb2RlbF9yb3V0ZXIYFyABKAlIA4gBARInCgRhcHBzGBggAygLMhkuZW5ncmFtLmFwcC52MS5TZXNzaW9uQXBwEhkKEXByb2dyYW1tYXRpY19vbmx5GBkgASgIEhoKEm1jcF9ncmFudF9yZXZpc2lvbhgaIAEoAxouCgxFbnZWYXJzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4AUIKCghfaGFybmVzc0IICgZfbW9kZWxCCQoHX2VmZm9ydEIPCg1fbW9kZWxfcm91dGVySgQIDBANSgQIDxAQSgQIExAUSgQIFBAVUgtkZXNpZ25hdGlvbiIoCgpTZXNzaW9uQXBwEgwKBG5hbWUYASABKAkSDAoEcG9ydBgCIAEoDSJxCgtQcm9maWxlUmVwbxIMCgRwYXRoGAEgASgJEhIKCnJlbW90ZV91cmwYAiABKAkSNQoGcmVtb3RlGAMgASgLMiAuZW5ncmFtLmFwcC52MS5Qcm9maWxlUmVwb1JlbW90ZUgAiAEBQgkKB19yZW1vdGUiPgoRUHJvZmlsZVJlcG9SZW1vdGUSDAoEaG9zdBgBIAEoCRINCgVvd25lchgCIAEoCRIMCgRuYW1lGAMgASgJIjEKG0Rpc2NvdmVyUHJvZmlsZVJlcG9zUmVxdWVzdBISCgpwcm9maWxlX2lkGAEgASgJIlAKDkRpc2NvdmVyZWRSZXBvEgwKBHBhdGgYASABKAkSMAoHcmVtb3RlcxgCIAMoCzIfLmVuZ3JhbS5hcHAudjEuRGlzY292ZXJlZFJlbW90ZSJvChBEaXNjb3ZlcmVkUmVtb3RlEgwKBG5hbWUYASABKAkSCwoDdXJsGAIgASgJEjUKBnBhcnNlZBgDIAEoCzIgLmVuZ3JhbS5hcHAudjEuUHJvZmlsZVJlcG9SZW1vdGVIAIgBAUIJCgdfcGFyc2VkIkwKHERpc2NvdmVyUHJvZmlsZVJlcG9zUmVzcG9uc2USLAoFcmVwb3MYASADKAsyHS5lbmdyYW0uYXBwLnYxLkRpc2NvdmVyZWRSZXBvIqUBChdQcm9maWxlSW50ZWdyYXRpb25HcmFudBIVCg1jb25uZWN0aW9uX2lkGAEgASgJEhEKCW9wZXJhdGlvbhgCIAEoCRIcChRyZXNvdXJjZV9jb25zdHJhaW50cxgDIAMoCRIYChBjcmVkZW50aWFsX3Njb3BlGAQgASgJEigKA21jcBgFIAEoCzIbLmVuZ3JhbS5hcHAudjEuTWNwVG9vbEdyYW50IlMKDlByb2ZpbGVOZXR3b3JrEg8KB2RlZmF1bHQYASABKAkSEwoLYWxsb3dfaG9zdHMYAiADKAkSGwoTYWxsb3dfaG9zdF9wYXR0ZXJucxgDIAMoCSJtCg1Qcm9maWxlU2VjcmV0EgsKA3JlZhgBIAEoCRIPCgdlbnZfdmFyGAIgASgJEgwKBG1vZGUYAyABKAkSEwoLYWxsb3dfaG9zdHMYBCADKAkSGwoTYWxsb3dfaG9zdF9wYXR0ZXJucxgFIAMoCSJuCg9Qcm9maWxlU25hcHNob3QSCgoCaWQYASABKAkSDAoEbmFtZRgCIAEoCRIMCgRpY29uGAMgASgJEhAKCGFyY2hpdmVkGAQgASgIEhEKCWltYWdlX3VyaRgFIAEoCRIOCgZza2lsbHMYBiADKAkiSwoTTGlzdFByb2ZpbGVzUmVxdWVzdBIYChBpbmNsdWRlX2FyY2hpdmVkGAEgASgIEhoKEmZvcl9zZXNzaW9uX3BpY2tlchgCIAEoCCJAChRMaXN0UHJvZmlsZXNSZXNwb25zZRIoCghwcm9maWxlcxgBIAMoCzIWLmVuZ3JhbS5hcHAudjEuUHJvZmlsZSIfChFHZXRQcm9maWxlUmVxdWVzdBIKCgJpZBgBIAEoCSI9ChJHZXRQcm9maWxlUmVzcG9uc2USJwoHcHJvZmlsZRgBIAEoCzIWLmVuZ3JhbS5hcHAudjEuUHJvZmlsZSL6BQoUQ3JlYXRlUHJvZmlsZVJlcXVlc3QSDAoEbmFtZRgBIAEoCRITCgtkZXNjcmlwdGlvbhgCIAEoCRIMCgRpY29uGAMgASgJEhAKCGltYWdlX2lkGAQgASgJEhsKE2luY2x1ZGVfdXNlcl90b2tlbnMYBSABKAgSQgoIZW52X3ZhcnMYBiADKAsyMC5lbmdyYW0uYXBwLnYxLkNyZWF0ZVByb2ZpbGVSZXF1ZXN0LkVudlZhcnNFbnRyeRIOCgZza2lsbHMYByADKAkSLgoHbmV0d29yaxgJIAEoCzIdLmVuZ3JhbS5hcHAudjEuUHJvZmlsZU5ldHdvcmsSLQoHc2VjcmV0cxgKIAMoCzIcLmVuZ3JhbS5hcHAudjEuUHJvZmlsZVNlY3JldBIUCgdoYXJuZXNzGAwgASgJSACIAQESEgoFbW9kZWwYDSABKAlIAYgBARITCgZlZmZvcnQYDiABKAlIAogBARJCChJpbnRlZ3JhdGlvbl9ncmFudHMYESADKAsyJi5lbmdyYW0uYXBwLnYxLlByb2ZpbGVJbnRlZ3JhdGlvbkdyYW50EikKBXJlcG9zGBIgAygLMhouZW5ncmFtLmFwcC52MS5Qcm9maWxlUmVwbxIZCgxtb2RlbF9yb3V0ZXIYEyABKAlIA4gBARInCgRhcHBzGBQgAygLMhkuZW5ncmFtLmFwcC52MS5TZXNzaW9uQXBwEhkKEXByb2dyYW1tYXRpY19vbmx5GBUgASgIEigKG2V4cGVjdGVkX21jcF9ncmFudF9yZXZpc2lvbhgWIAEoA0gEiAEBGi4KDEVudlZhcnNFbnRyeRILCgNrZXkYASABKAkSDQoFdmFsdWUYAiABKAk6AjgBQgoKCF9oYXJuZXNzQggKBl9tb2RlbEIJCgdfZWZmb3J0Qg8KDV9tb2RlbF9yb3V0ZXJCHgocX2V4cGVjdGVkX21jcF9ncmFudF9yZXZpc2lvbkoECAgQCUoECAsQDEoECA8QEEoECBAQESJAChVDcmVhdGVQcm9maWxlUmVzcG9uc2USJwoHcHJvZmlsZRgBIAEoCzIWLmVuZ3JhbS5hcHAudjEuUHJvZmlsZSKhBgoUVXBkYXRlUHJvZmlsZVJlcXVlc3QSCgoCaWQYASABKAkSDAoEbmFtZRgCIAEoCRITCgtkZXNjcmlwdGlvbhgDIAEoCRIMCgRpY29uGAQgASgJEhAKCGltYWdlX2lkGAUgASgJEhsKE2luY2x1ZGVfdXNlcl90b2tlbnMYBiABKAgSQgoIZW52X3ZhcnMYByADKAsyMC5lbmdyYW0uYXBwLnYxLlVwZGF0ZVByb2ZpbGVSZXF1ZXN0LkVudlZhcnNFbnRyeRIOCgZza2lsbHMYCCADKAkSLgoHbmV0d29yaxgKIAEoCzIdLmVuZ3JhbS5hcHAudjEuUHJvZmlsZU5ldHdvcmsSLQoHc2VjcmV0cxgLIAMoCzIcLmVuZ3JhbS5hcHAudjEuUHJvZmlsZVNlY3JldBIUCgdoYXJuZXNzGA0gASgJSACIAQESEgoFbW9kZWwYDiABKAlIAYgBARITCgZlZmZvcnQYDyABKAlIAogBARJCChJpbnRlZ3JhdGlvbl9ncmFudHMYEiADKAsyJi5lbmdyYW0uYXBwLnYxLlByb2ZpbGVJbnRlZ3JhdGlvbkdyYW50EikKBXJlcG9zGBMgAygLMhouZW5ncmFtLmFwcC52MS5Qcm9maWxlUmVwbxIZCgxtb2RlbF9yb3V0ZXIYFCABKAlIA4gBARInCgRhcHBzGBUgAygLMhkuZW5ncmFtLmFwcC52MS5TZXNzaW9uQXBwEh4KEXByb2dyYW1tYXRpY19vbmx5GBYgASgISASIAQESKAobZXhwZWN0ZWRfbWNwX2dyYW50X3JldmlzaW9uGBcgASgDSAWIAQEaLgoMRW52VmFyc0VudHJ5EgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCToCOAFCCgoIX2hhcm5lc3NCCAoGX21vZGVsQgkKB19lZmZvcnRCDwoNX21vZGVsX3JvdXRlckIUChJfcHJvZ3JhbW1hdGljX29ubHlCHgocX2V4cGVjdGVkX21jcF9ncmFudF9yZXZpc2lvbkoECAkQCkoECAwQDUoECBAQEUoECBEQEiJAChVVcGRhdGVQcm9maWxlUmVzcG9uc2USJwoHcHJvZmlsZRgBIAEoCzIWLmVuZ3JhbS5hcHAudjEuUHJvZmlsZSIiChREZWxldGVQcm9maWxlUmVxdWVzdBIKCgJpZBgBIAEoCSIXChVEZWxldGVQcm9maWxlUmVzcG9uc2UiogEKDE1jcFRvb2xHcmFudBIVCg11cHN0cmVhbV9uYW1lGAEgASgJEhgKEGF1dG9tYXRpb25fd3JpdGUYAiABKAgSFAoMdW5yZXN0cmljdGVkGAQgASgIEioKImFyZ3VtZW50X2NvbnN0cmFpbnRfZG9jdW1lbnRzX2pzb24YBSADKAlKBAgDEARSGWFyZ3VtZW50X2NvbnN0cmFpbnRzX2pzb24ywQQKDlByb2ZpbGVTZXJ2aWNlElcKDExpc3RQcm9maWxlcxIiLmVuZ3JhbS5hcHAudjEuTGlzdFByb2ZpbGVzUmVxdWVzdBojLmVuZ3JhbS5hcHAudjEuTGlzdFByb2ZpbGVzUmVzcG9uc2USUQoKR2V0UHJvZmlsZRIgLmVuZ3JhbS5hcHAudjEuR2V0UHJvZmlsZVJlcXVlc3QaIS5lbmdyYW0uYXBwLnYxLkdldFByb2ZpbGVSZXNwb25zZRJaCg1DcmVhdGVQcm9maWxlEiMuZW5ncmFtLmFwcC52MS5DcmVhdGVQcm9maWxlUmVxdWVzdBokLmVuZ3JhbS5hcHAudjEuQ3JlYXRlUHJvZmlsZVJlc3BvbnNlEloKDVVwZGF0ZVByb2ZpbGUSIy5lbmdyYW0uYXBwLnYxLlVwZGF0ZVByb2ZpbGVSZXF1ZXN0GiQuZW5ncmFtLmFwcC52MS5VcGRhdGVQcm9maWxlUmVzcG9uc2USbwoURGlzY292ZXJQcm9maWxlUmVwb3MSKi5lbmdyYW0uYXBwLnYxLkRpc2NvdmVyUHJvZmlsZVJlcG9zUmVxdWVzdBorLmVuZ3JhbS5hcHAudjEuRGlzY292ZXJQcm9maWxlUmVwb3NSZXNwb25zZRJaCg1EZWxldGVQcm9maWxlEiMuZW5ncmFtLmFwcC52MS5EZWxldGVQcm9maWxlUmVxdWVzdBokLmVuZ3JhbS5hcHAudjEuRGVsZXRlUHJvZmlsZVJlc3BvbnNlYgZwcm90bzM"); /** * An admin-curated bundle of session-launch inputs (ADR §1). @@ -169,6 +169,13 @@ export type Profile = Message<"engram.app.v1.Profile"> & { * @generated from field: bool programmatic_only = 25; */ programmaticOnly: boolean; + + /** + * Revision for explicit MCP grant publication; zero means no publication. + * + * @generated from field: int64 mcp_grant_revision = 26; + */ + mcpGrantRevision: bigint; }; /** @@ -394,6 +401,11 @@ export type ProfileIntegrationGrant = Message<"engram.app.v1.ProfileIntegrationG * @generated from field: string credential_scope = 4; */ credentialScope: string; + + /** + * @generated from field: engram.app.v1.McpToolGrant mcp = 5; + */ + mcp?: McpToolGrant; }; /** @@ -717,6 +729,13 @@ export type CreateProfileRequest = Message<"engram.app.v1.CreateProfileRequest"> * @generated from field: bool programmatic_only = 21; */ programmaticOnly: boolean; + + /** + * Presence requests explicit MCP grant publication. First publication expects zero. + * + * @generated from field: optional int64 expected_mcp_grant_revision = 22; + */ + expectedMcpGrantRevision?: bigint; }; /** @@ -847,6 +866,13 @@ export type UpdateProfileRequest = Message<"engram.app.v1.UpdateProfileRequest"> * @generated from field: optional bool programmatic_only = 22; */ programmaticOnly?: boolean; + + /** + * Presence requests publication or republication of the exact MCP grants. + * + * @generated from field: optional int64 expected_mcp_grant_revision = 23; + */ + expectedMcpGrantRevision?: bigint; }; /** @@ -903,6 +929,41 @@ export type DeleteProfileResponse = Message<"engram.app.v1.DeleteProfileResponse export const DeleteProfileResponseSchema: GenMessage = /*@__PURE__*/ messageDesc(file_engram_app_v1_profile, 21); +/** + * @generated from message engram.app.v1.McpToolGrant + */ +export type McpToolGrant = Message<"engram.app.v1.McpToolGrant"> & { + /** + * @generated from field: string upstream_name = 1; + */ + upstreamName: string; + + /** + * @generated from field: bool automation_write = 2; + */ + automationWrite: boolean; + + /** + * @generated from field: bool unrestricted = 4; + */ + unrestricted: boolean; + + /** + * Every document must pass in its own dialect and resource scope. + * An empty list requires an explicit unrestricted choice. + * + * @generated from field: repeated string argument_constraint_documents_json = 5; + */ + argumentConstraintDocumentsJson: string[]; +}; + +/** + * Describes the message engram.app.v1.McpToolGrant. + * Use `create(McpToolGrantSchema)` to create a new message. + */ +export const McpToolGrantSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_profile, 22); + /** * Session profiles (ADR 0053) — admin-curated session starting points. * ORCHESTRATOR-NATIVE: the control plane neither implements nor knows about diff --git a/orchestrator/src/gen/engram/app/v1/session_pb.ts b/orchestrator/src/gen/engram/app/v1/session_pb.ts index b5b3a07b1..e31f3289f 100644 --- a/orchestrator/src/gen/engram/app/v1/session_pb.ts +++ b/orchestrator/src/gen/engram/app/v1/session_pb.ts @@ -12,7 +12,7 @@ import type { Message } from "@bufbuild/protobuf"; * Describes the file engram/app/v1/session.proto. */ export const file_engram_app_v1_session: GenFile = /*@__PURE__*/ - fileDesc("ChtlbmdyYW0vYXBwL3YxL3Nlc3Npb24ucHJvdG8SDWVuZ3JhbS5hcHAudjEiFQoTTGlzdFNlc3Npb25zUmVxdWVzdCJIChRMaXN0U2Vzc2lvbnNSZXNwb25zZRIwCghzZXNzaW9ucxgBIAMoCzIeLmVuZ3JhbS5hcHAudjEuU2Vzc2lvbkxpc3RJdGVtIrQBCg9TZXNzaW9uTGlzdEl0ZW0SJwoHc2Vzc2lvbhgBIAEoCzIWLmVuZ3JhbS5hcHAudjEuU2Vzc2lvbhIYCgtvd25lcl9lbWFpbBgCIAEoCUgAiAEBEhcKCm93bmVyX25hbWUYAyABKAlIAYgBARIXCgpvd25lcl9raW5kGAQgASgJSAKIAQFCDgoMX293bmVyX2VtYWlsQg0KC19vd25lcl9uYW1lQg0KC19vd25lcl9raW5kIpgCCgdTZXNzaW9uEgoKAmlkGAEgASgJEg4KBnN0YXR1cxgCIAEoCRIUCgdob3N0X2lkGAMgASgJSACIAQESFwoKc2FuZGJveF9pZBgEIAEoCUgBiAEBEg0KBWltYWdlGAUgASgJEgwKBG1vZGUYBiABKAkSEgoKY3JlYXRlZF9hdBgHIAEoCRIWCg5sYXN0X2FjdGl2ZV9hdBgIIAEoCRIcCg9zdWdnZXN0ZWRfdGl0bGUYCSABKAlIAogBARIaCg1sYXN0X2V2ZW50X2F0GAogASgJSAOIAQFCCgoIX2hvc3RfaWRCDQoLX3NhbmRib3hfaWRCEgoQX3N1Z2dlc3RlZF90aXRsZUIQCg5fbGFzdF9ldmVudF9hdCK4BQoUQ3JlYXRlU2Vzc2lvblJlcXVlc3QSEQoJaW1hZ2VfdXJpGAEgASgJEgwKBG1vZGUYAiABKAkSEwoGcHJvbXB0GAMgASgJSACIAQESFgoJcHJvbXB0X2lkGAcgASgJSAGIAQESQQoHc2VjcmV0cxgFIAMoCzIwLmVuZ3JhbS5hcHAudjEuQ3JlYXRlU2Vzc2lvblJlcXVlc3QuU2VjcmV0c0VudHJ5EkgKC2hhcm5lc3NfZW52GAYgAygLMjMuZW5ncmFtLmFwcC52MS5DcmVhdGVTZXNzaW9uUmVxdWVzdC5IYXJuZXNzRW52RW50cnkSFwoPc2VsZWN0ZWRfc2tpbGxzGAggAygJEhQKDGNhcGFiaWxpdGllcxgJIAMoCRIfChdpbnRlZ3JhdGlvbl9wb2xpY3lfanNvbhgKIAEoCRIUCgdoYXJuZXNzGAsgASgJSAKIAQESPwoQb2F1dGhfY3JlZGVudGlhbBgMIAEoCzIlLmVuZ3JhbS5hcHAudjEuT0F1dGhDcmVkZW50aWFsQmluZGluZxIZCgxoYXJuZXNzX21vZGUYDSABKAlIA4gBARIhChRyZXF1ZXN0ZWRfc2Vzc2lvbl9pZBgOIAEoCUgEiAEBEigKBGFwcHMYDyADKAsyGi5lbmdyYW0uYXBwLnYxLkFwcEVuZHBvaW50Gi4KDFNlY3JldHNFbnRyeRILCgNrZXkYASABKAkSDQoFdmFsdWUYAiABKAk6AjgBGjEKD0hhcm5lc3NFbnZFbnRyeRILCgNrZXkYASABKAkSDQoFdmFsdWUYAiABKAk6AjgBQgkKB19wcm9tcHRCDAoKX3Byb21wdF9pZEIKCghfaGFybmVzc0IPCg1faGFybmVzc19tb2RlQhcKFV9yZXF1ZXN0ZWRfc2Vzc2lvbl9pZEoECAQQBSItCgtBcHBFbmRwb2ludBIQCghob3N0bmFtZRgBIAEoCRIMCgRwb3J0GAIgASgNIlgKFk9BdXRoQ3JlZGVudGlhbEJpbmRpbmcSLAoHc3ViamVjdBgBIAEoCzIbLmVuZ3JhbS5hcHAudjEuT0F1dGhTdWJqZWN0EhAKCHByb3ZpZGVyGAIgASgJImAKFUNyZWF0ZVNlc3Npb25SZXNwb25zZRISCgpzZXNzaW9uX2lkGAEgASgJEg4KBnN0YXR1cxgCIAEoCRIVCg1pbWFnZV92ZXJzaW9uGAMgASgJEgwKBGtpbmQYBCABKAkiJwoRR2V0U2Vzc2lvblJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCSI9ChJHZXRTZXNzaW9uUmVzcG9uc2USJwoHc2Vzc2lvbhgBIAEoCzIWLmVuZ3JhbS5hcHAudjEuU2Vzc2lvbiIqChREZWxldGVTZXNzaW9uUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJIhcKFURlbGV0ZVNlc3Npb25SZXNwb25zZSJ0ChFTZW5kUHJvbXB0UmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEgwKBHRleHQYAiABKAkSEQoJcHJvbXB0X2lkGAMgASgJEhkKDGhhcm5lc3NfbW9kZRgEIAEoCUgAiAEBQg8KDV9oYXJuZXNzX21vZGUiNgoSU2VuZFByb21wdFJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSDAoEbm90ZRgCIAEoCSI2ChBJbnRlcnJ1cHRSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkSDgoGc291cmNlGAIgASgJIjUKEUludGVycnVwdFJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSDAoEbm90ZRgCIAEoCSJOChdFZGl0UXVldWVkUHJvbXB0UmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEhEKCXByb21wdF9pZBgCIAEoCRIMCgR0ZXh0GAMgASgJIjwKGEVkaXRRdWV1ZWRQcm9tcHRSZXNwb25zZRISCgpzZXNzaW9uX2lkGAEgASgJEgwKBG5vdGUYAiABKAkiQwoaRGVxdWV1ZVF1ZXVlZFByb21wdFJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCRIRCglwcm9tcHRfaWQYAiABKAkiPwobRGVxdWV1ZVF1ZXVlZFByb21wdFJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSDAoEbm90ZRgCIAEoCSJYChdDb21wbGV0ZVRvb2xDYWxsUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEhQKDHRvb2xfY2FsbF9pZBgCIAEoCRITCgtyZXN1bHRfanNvbhgDIAEoCSI8ChhDb21wbGV0ZVRvb2xDYWxsUmVzcG9uc2USEgoKc2Vzc2lvbl9pZBgBIAEoCRIMCgRub3RlGAIgASgJIl0KE1N0cmVhbUV2ZW50c1JlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCRISCgVzaW5jZRgCIAEoA0gAiAEBEhQKDGR1cmFibGVfb25seRgDIAEoCEIICgZfc2luY2UiTAoMU2Vzc2lvbkV2ZW50EhAKA2lkeBgBIAEoA0gAiAEBEgwKBGtpbmQYAiABKAkSFAoMcGF5bG9hZF9qc29uGAMgASgJQgYKBF9pZHgivQEKGExpc3RTZXNzaW9uRXZlbnRzUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEhYKCWFmdGVyX2lkeBgCIAEoA0gAiAEBEhIKBWxpbWl0GAMgASgDSAGIAQESFwoKYmVmb3JlX2lkeBgEIAEoA0gCiAEBEg0KBWtpbmRzGAUgAygJEhIKCnRvb2xfbmFtZXMYBiADKAlCDAoKX2FmdGVyX2lkeEIICgZfbGltaXRCDQoLX2JlZm9yZV9pZHgiYAoZTGlzdFNlc3Npb25FdmVudHNSZXNwb25zZRIrCgZldmVudHMYASADKAsyGy5lbmdyYW0uYXBwLnYxLlNlc3Npb25FdmVudBIWCg5uZXh0X2FmdGVyX2lkeBgCIAEoAyKXAwoLRXhlY1JlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCRIUCgdjb21tYW5kGAIgASgJSACIAQESDAoEYXJndhgDIAMoCRIwCgNlbnYYBCADKAsyIy5lbmdyYW0uYXBwLnYxLkV4ZWNSZXF1ZXN0LkVudkVudHJ5EhQKB3dvcmtkaXIYBSABKAlIAYgBARIZCgx0aW1lb3V0X3NlY3MYBiABKARIAogBARIUCgdleGVjX2lkGAcgASgJSAOIAQESGgoNc3Rkb3V0X29mZnNldBgIIAEoBEgEiAEBEhoKDXN0ZGVycl9vZmZzZXQYCSABKARIBYgBARIRCgR3YWtlGAogASgISAaIAQEaKgoIRW52RW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4AUIKCghfY29tbWFuZEIKCghfd29ya2RpckIPCg1fdGltZW91dF9zZWNzQgoKCF9leGVjX2lkQhAKDl9zdGRvdXRfb2Zmc2V0QhAKDl9zdGRlcnJfb2Zmc2V0QgcKBV93YWtlIjgKEUNhbmNlbEV4ZWNSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkSDwoHZXhlY19pZBgCIAEoCSIUChJDYW5jZWxFeGVjUmVzcG9uc2UidQoRV3JpdGVGaWxlTWV0YWRhdGESEgoKc2Vzc2lvbl9pZBgBIAEoCRIMCgRwYXRoGAIgASgJEhIKCnNpemVfYnl0ZXMYAyABKAQSDgoGc2hhMjU2GAQgASgJEhEKBG1vZGUYBSABKA1IAIgBAUIHCgVfbW9kZSJiChBXcml0ZUZpbGVSZXF1ZXN0EjQKCG1ldGFkYXRhGAEgASgLMiAuZW5ncmFtLmFwcC52MS5Xcml0ZUZpbGVNZXRhZGF0YUgAEg8KBWNodW5rGAIgASgMSABCBwoFZnJhbWUiRQoRV3JpdGVGaWxlUmVzcG9uc2USDAoEcGF0aBgBIAEoCRISCgpzaXplX2J5dGVzGAIgASgEEg4KBnNoYTI1NhgDIAEoCSIzCg9SZWFkRmlsZVJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCRIMCgRwYXRoGAIgASgJIlcKEFJlYWRGaWxlTWV0YWRhdGESDAoEcGF0aBgBIAEoCRISCgpzaXplX2J5dGVzGAIgASgEEg4KBnNoYTI1NhgDIAEoCRIRCglmaWxlX25hbWUYBCABKAkiYQoQUmVhZEZpbGVSZXNwb25zZRIzCghtZXRhZGF0YRgBIAEoCzIfLmVuZ3JhbS5hcHAudjEuUmVhZEZpbGVNZXRhZGF0YUgAEg8KBWNodW5rGAIgASgMSABCBwoFZnJhbWUiVwoQQ29weUZpbGVzUmVxdWVzdBIZChFzb3VyY2Vfc2Vzc2lvbl9pZBgBIAEoCRIZChF0YXJnZXRfc2Vzc2lvbl9pZBgCIAEoCRINCgVwYXRocxgDIAMoCSJCCg5Db3B5RmlsZVJlc3VsdBIMCgRwYXRoGAEgASgJEhIKCnNpemVfYnl0ZXMYAiABKAQSDgoGc2hhMjU2GAMgASgJIkEKEUNvcHlGaWxlc1Jlc3BvbnNlEiwKBWZpbGVzGAEgAygLMh0uZW5ncmFtLmFwcC52MS5Db3B5RmlsZVJlc3VsdCKRAQoKRXhlY091dHB1dBItCgdzdGFydGVkGAEgASgLMhouZW5ncmFtLmFwcC52MS5FeGVjU3RhcnRlZEgAEhAKBnN0ZG91dBgCIAEoDEgAEhAKBnN0ZGVychgDIAEoDEgAEicKBGV4aXQYBCABKAsyFy5lbmdyYW0uYXBwLnYxLkV4ZWNFeGl0SABCBwoFZXZlbnQiHgoLRXhlY1N0YXJ0ZWQSDwoHZXhlY19pZBgBIAEoCSJfCghFeGVjRXhpdBIYCgtleGl0X3N0YXR1cxgBIAEoBUgAiAEBEikKBnJ1c2FnZRgCIAEoCzIZLmVuZ3JhbS5hcHAudjEuRXhlY1J1c2FnZUIOCgxfZXhpdF9zdGF0dXMimQEKCkV4ZWNSdXNhZ2USDwoHd2FsbF9tcxgBIAEoBBIYCgtwZWFrX3Jzc19rYhgCIAEoBEgAiAEBEhgKC3VzZXJfY3B1X21zGAMgASgESAGIAQESFwoKc3lzX2NwdV9tcxgEIAEoBEgCiAEBQg4KDF9wZWFrX3Jzc19rYkIOCgxfdXNlcl9jcHVfbXNCDQoLX3N5c19jcHVfbXMieQoNR2V0TG9nUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEhEKBGtpbmQYAiABKAlIAIgBARISCgVsaW1pdBgDIAEoA0gBiAEBEhEKBHRhaWwYBCABKAhIAogBAUIHCgVfa2luZEIICgZfbGltaXRCBwoFX3RhaWwiZAoOR2V0TG9nUmVzcG9uc2USEgoKc2Vzc2lvbl9pZBgBIAEoCRIMCgRraW5kGAIgASgJEjAKBmV2ZW50cxgDIAMoCzIgLmVuZ3JhbS5hcHAudjEuQ29udmVyc2F0aW9uRW50cnkiUAoRQ29udmVyc2F0aW9uRW50cnkSCwoDaWR4GAEgASgDEgwKBGtpbmQYAiABKAkSCgoCYXQYAyABKAkSFAoMcGF5bG9hZF9qc29uGAQgASgJIiUKD1NuYXBzaG90UmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJIoYBChBTbmFwc2hvdFJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSGAoLc25hcHNob3RfaWQYAiABKAlIAIgBARIXCgpzaXplX2J5dGVzGAMgASgESAGIAQESDAoEbm90ZRgEIAEoCUIOCgxfc25hcHNob3RfaWRCDQoLX3NpemVfYnl0ZXMiKgoURW5zdXJlQnJvd3NlclJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCSIlChVFbnN1cmVCcm93c2VyUmVzcG9uc2USDAoEcG9ydBgBIAEoDSImChBFbnN1cmVJZGVSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkiIQoRRW5zdXJlSWRlUmVzcG9uc2USDAoEcG9ydBgBIAEoDSIjCg1SZXN1bWVSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkihAEKDlJlc3VtZVJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSGAoLc25hcHNob3RfaWQYAiABKAlIAIgBARIXCgpzaXplX2J5dGVzGAMgASgESAGIAQESDAoEbm90ZRgEIAEoCUIOCgxfc25hcHNob3RfaWRCDQoLX3NpemVfYnl0ZXMiJwoRRXZpY3RMb2NhbFJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCSIUChJFdmljdExvY2FsUmVzcG9uc2UiJgoQRXZpY3RJZGxlUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJIjcKEUV2aWN0SWRsZVJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSDgoGc3RhdHVzGAIgASgJIiwKFkxpc3RDaGVja3BvaW50c1JlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCSJkChdMaXN0Q2hlY2twb2ludHNSZXNwb25zZRISCgpzZXNzaW9uX2lkGAEgASgJEjUKC2NoZWNrcG9pbnRzGAIgAygLMiAuZW5ncmFtLmFwcC52MS5DaGVja3BvaW50U3VtbWFyeSKmAQoRQ2hlY2twb2ludFN1bW1hcnkSEwoLc25hcHNob3RfaWQYASABKAkSEgoKY3JlYXRlZF9hdBgCIAEoCRISCgpzaXplX2J5dGVzGAMgASgEEhoKDWV2ZW50c19jdXJzb3IYBCABKANIAIgBARITCgtyZWNvdmVyYWJsZRgFIAEoCBIRCglpc19sYXRlc3QYBiABKAhCEAoOX2V2ZW50c19jdXJzb3IiPQoSR2V0QXJ0aWZhY3RSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkSEwoLYXJ0aWZhY3RfaWQYAiABKAkiYgoTR2V0QXJ0aWZhY3RSZXNwb25zZRIzCghtZXRhZGF0YRgBIAEoCzIfLmVuZ3JhbS5hcHAudjEuQXJ0aWZhY3RNZXRhZGF0YUgAEg8KBWNodW5rGAIgASgMSABCBQoDbXNnIk0KEEFydGlmYWN0TWV0YWRhdGESEgoKbWVkaWFfdHlwZRgBIAEoCRISCgpzaXplX2J5dGVzGAIgASgEEhEKCWZpbGVfbmFtZRgDIAEoCSJ7Ch1DcmVhdGVBcnRpZmFjdEZyb21QYXRoUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEgwKBHBhdGgYAiABKAkSFAoHY2FwdGlvbhgDIAEoCUgAiAEBEhYKDnN1cHByZXNzX2V2ZW50GAQgASgIQgoKCF9jYXB0aW9uIl0KHkNyZWF0ZUFydGlmYWN0RnJvbVBhdGhSZXNwb25zZRITCgthcnRpZmFjdF9pZBgBIAEoCRISCgptZWRpYV90eXBlGAIgASgJEhIKCnNpemVfYnl0ZXMYAyABKAQitAEKEVJlbGF5U2hlbGxSZXF1ZXN0EigKBG9wZW4YASABKAsyGC5lbmdyYW0uYXBwLnYxLlNoZWxsT3BlbkgAEg4KBHRleHQYAiABKAlIABIQCgZiaW5hcnkYAyABKAxIABIOCgRwaW5nGAQgASgMSAASDgoEcG9uZxgFIAEoDEgAEioKBWNsb3NlGAYgASgLMhkuZW5ncmFtLmFwcC52MS5TaGVsbENsb3NlSABCBwoFZnJhbWUiiwEKElJlbGF5U2hlbGxSZXNwb25zZRIOCgR0ZXh0GAEgASgJSAASEAoGYmluYXJ5GAIgASgMSAASDgoEcGluZxgDIAEoDEgAEg4KBHBvbmcYBCABKAxIABIqCgVjbG9zZRgFIAEoCzIZLmVuZ3JhbS5hcHAudjEuU2hlbGxDbG9zZUgAQgcKBWZyYW1lIh8KCVNoZWxsT3BlbhISCgpzZXNzaW9uX2lkGAEgASgJIioKClNoZWxsQ2xvc2USDAoEY29kZRgBIAEoDRIOCgZyZWFzb24YAiABKAkifwoQUmVsYXlQb3J0UmVxdWVzdBInCgRvcGVuGAEgASgLMhcuZW5ncmFtLmFwcC52MS5Qb3J0T3BlbkgAEg4KBGRhdGEYAiABKAxIABIpCgVjbG9zZRgDIAEoCzIYLmVuZ3JhbS5hcHAudjEuUG9ydENsb3NlSABCBwoFZnJhbWUiVwoRUmVsYXlQb3J0UmVzcG9uc2USDgoEZGF0YRgBIAEoDEgAEikKBWNsb3NlGAIgASgLMhguZW5ncmFtLmFwcC52MS5Qb3J0Q2xvc2VIAEIHCgVmcmFtZSIsCghQb3J0T3BlbhISCgpzZXNzaW9uX2lkGAEgASgJEgwKBHBvcnQYAiABKA0iCwoJUG9ydENsb3NlMuoRCg5TZXNzaW9uU2VydmljZRJXCgxMaXN0U2Vzc2lvbnMSIi5lbmdyYW0uYXBwLnYxLkxpc3RTZXNzaW9uc1JlcXVlc3QaIy5lbmdyYW0uYXBwLnYxLkxpc3RTZXNzaW9uc1Jlc3BvbnNlEloKDUNyZWF0ZVNlc3Npb24SIy5lbmdyYW0uYXBwLnYxLkNyZWF0ZVNlc3Npb25SZXF1ZXN0GiQuZW5ncmFtLmFwcC52MS5DcmVhdGVTZXNzaW9uUmVzcG9uc2USUQoKR2V0U2Vzc2lvbhIgLmVuZ3JhbS5hcHAudjEuR2V0U2Vzc2lvblJlcXVlc3QaIS5lbmdyYW0uYXBwLnYxLkdldFNlc3Npb25SZXNwb25zZRJaCg1EZWxldGVTZXNzaW9uEiMuZW5ncmFtLmFwcC52MS5EZWxldGVTZXNzaW9uUmVxdWVzdBokLmVuZ3JhbS5hcHAudjEuRGVsZXRlU2Vzc2lvblJlc3BvbnNlElEKClNlbmRQcm9tcHQSIC5lbmdyYW0uYXBwLnYxLlNlbmRQcm9tcHRSZXF1ZXN0GiEuZW5ncmFtLmFwcC52MS5TZW5kUHJvbXB0UmVzcG9uc2USTgoJSW50ZXJydXB0Eh8uZW5ncmFtLmFwcC52MS5JbnRlcnJ1cHRSZXF1ZXN0GiAuZW5ncmFtLmFwcC52MS5JbnRlcnJ1cHRSZXNwb25zZRJjChBFZGl0UXVldWVkUHJvbXB0EiYuZW5ncmFtLmFwcC52MS5FZGl0UXVldWVkUHJvbXB0UmVxdWVzdBonLmVuZ3JhbS5hcHAudjEuRWRpdFF1ZXVlZFByb21wdFJlc3BvbnNlEmwKE0RlcXVldWVRdWV1ZWRQcm9tcHQSKS5lbmdyYW0uYXBwLnYxLkRlcXVldWVRdWV1ZWRQcm9tcHRSZXF1ZXN0GiouZW5ncmFtLmFwcC52MS5EZXF1ZXVlUXVldWVkUHJvbXB0UmVzcG9uc2USYwoQQ29tcGxldGVUb29sQ2FsbBImLmVuZ3JhbS5hcHAudjEuQ29tcGxldGVUb29sQ2FsbFJlcXVlc3QaJy5lbmdyYW0uYXBwLnYxLkNvbXBsZXRlVG9vbENhbGxSZXNwb25zZRJRCgxTdHJlYW1FdmVudHMSIi5lbmdyYW0uYXBwLnYxLlN0cmVhbUV2ZW50c1JlcXVlc3QaGy5lbmdyYW0uYXBwLnYxLlNlc3Npb25FdmVudDABEmYKEUxpc3RTZXNzaW9uRXZlbnRzEicuZW5ncmFtLmFwcC52MS5MaXN0U2Vzc2lvbkV2ZW50c1JlcXVlc3QaKC5lbmdyYW0uYXBwLnYxLkxpc3RTZXNzaW9uRXZlbnRzUmVzcG9uc2USPwoERXhlYxIaLmVuZ3JhbS5hcHAudjEuRXhlY1JlcXVlc3QaGS5lbmdyYW0uYXBwLnYxLkV4ZWNPdXRwdXQwARJRCgpDYW5jZWxFeGVjEiAuZW5ncmFtLmFwcC52MS5DYW5jZWxFeGVjUmVxdWVzdBohLmVuZ3JhbS5hcHAudjEuQ2FuY2VsRXhlY1Jlc3BvbnNlElAKCVdyaXRlRmlsZRIfLmVuZ3JhbS5hcHAudjEuV3JpdGVGaWxlUmVxdWVzdBogLmVuZ3JhbS5hcHAudjEuV3JpdGVGaWxlUmVzcG9uc2UoARJNCghSZWFkRmlsZRIeLmVuZ3JhbS5hcHAudjEuUmVhZEZpbGVSZXF1ZXN0Gh8uZW5ncmFtLmFwcC52MS5SZWFkRmlsZVJlc3BvbnNlMAESTgoJQ29weUZpbGVzEh8uZW5ncmFtLmFwcC52MS5Db3B5RmlsZXNSZXF1ZXN0GiAuZW5ncmFtLmFwcC52MS5Db3B5RmlsZXNSZXNwb25zZRJFCgZHZXRMb2cSHC5lbmdyYW0uYXBwLnYxLkdldExvZ1JlcXVlc3QaHS5lbmdyYW0uYXBwLnYxLkdldExvZ1Jlc3BvbnNlEksKCFNuYXBzaG90Eh4uZW5ncmFtLmFwcC52MS5TbmFwc2hvdFJlcXVlc3QaHy5lbmdyYW0uYXBwLnYxLlNuYXBzaG90UmVzcG9uc2USWgoNRW5zdXJlQnJvd3NlchIjLmVuZ3JhbS5hcHAudjEuRW5zdXJlQnJvd3NlclJlcXVlc3QaJC5lbmdyYW0uYXBwLnYxLkVuc3VyZUJyb3dzZXJSZXNwb25zZRJOCglFbnN1cmVJZGUSHy5lbmdyYW0uYXBwLnYxLkVuc3VyZUlkZVJlcXVlc3QaIC5lbmdyYW0uYXBwLnYxLkVuc3VyZUlkZVJlc3BvbnNlEkUKBlJlc3VtZRIcLmVuZ3JhbS5hcHAudjEuUmVzdW1lUmVxdWVzdBodLmVuZ3JhbS5hcHAudjEuUmVzdW1lUmVzcG9uc2USUQoKRXZpY3RMb2NhbBIgLmVuZ3JhbS5hcHAudjEuRXZpY3RMb2NhbFJlcXVlc3QaIS5lbmdyYW0uYXBwLnYxLkV2aWN0TG9jYWxSZXNwb25zZRJOCglFdmljdElkbGUSHy5lbmdyYW0uYXBwLnYxLkV2aWN0SWRsZVJlcXVlc3QaIC5lbmdyYW0uYXBwLnYxLkV2aWN0SWRsZVJlc3BvbnNlEmAKD0xpc3RDaGVja3BvaW50cxIlLmVuZ3JhbS5hcHAudjEuTGlzdENoZWNrcG9pbnRzUmVxdWVzdBomLmVuZ3JhbS5hcHAudjEuTGlzdENoZWNrcG9pbnRzUmVzcG9uc2USVgoLR2V0QXJ0aWZhY3QSIS5lbmdyYW0uYXBwLnYxLkdldEFydGlmYWN0UmVxdWVzdBoiLmVuZ3JhbS5hcHAudjEuR2V0QXJ0aWZhY3RSZXNwb25zZTABEnUKFkNyZWF0ZUFydGlmYWN0RnJvbVBhdGgSLC5lbmdyYW0uYXBwLnYxLkNyZWF0ZUFydGlmYWN0RnJvbVBhdGhSZXF1ZXN0Gi0uZW5ncmFtLmFwcC52MS5DcmVhdGVBcnRpZmFjdEZyb21QYXRoUmVzcG9uc2UyZQoRU2hlbGxSZWxheVNlcnZpY2USUAoFUmVsYXkSIC5lbmdyYW0uYXBwLnYxLlJlbGF5U2hlbGxSZXF1ZXN0GiEuZW5ncmFtLmFwcC52MS5SZWxheVNoZWxsUmVzcG9uc2UoATABMmIKEFBvcnRSZWxheVNlcnZpY2USTgoFUmVsYXkSHy5lbmdyYW0uYXBwLnYxLlJlbGF5UG9ydFJlcXVlc3QaIC5lbmdyYW0uYXBwLnYxLlJlbGF5UG9ydFJlc3BvbnNlKAEwAWIGcHJvdG8z", [file_engram_app_v1_oauth]); + fileDesc("ChtlbmdyYW0vYXBwL3YxL3Nlc3Npb24ucHJvdG8SDWVuZ3JhbS5hcHAudjEiFQoTTGlzdFNlc3Npb25zUmVxdWVzdCJIChRMaXN0U2Vzc2lvbnNSZXNwb25zZRIwCghzZXNzaW9ucxgBIAMoCzIeLmVuZ3JhbS5hcHAudjEuU2Vzc2lvbkxpc3RJdGVtIrQBCg9TZXNzaW9uTGlzdEl0ZW0SJwoHc2Vzc2lvbhgBIAEoCzIWLmVuZ3JhbS5hcHAudjEuU2Vzc2lvbhIYCgtvd25lcl9lbWFpbBgCIAEoCUgAiAEBEhcKCm93bmVyX25hbWUYAyABKAlIAYgBARIXCgpvd25lcl9raW5kGAQgASgJSAKIAQFCDgoMX293bmVyX2VtYWlsQg0KC19vd25lcl9uYW1lQg0KC19vd25lcl9raW5kIpgCCgdTZXNzaW9uEgoKAmlkGAEgASgJEg4KBnN0YXR1cxgCIAEoCRIUCgdob3N0X2lkGAMgASgJSACIAQESFwoKc2FuZGJveF9pZBgEIAEoCUgBiAEBEg0KBWltYWdlGAUgASgJEgwKBG1vZGUYBiABKAkSEgoKY3JlYXRlZF9hdBgHIAEoCRIWCg5sYXN0X2FjdGl2ZV9hdBgIIAEoCRIcCg9zdWdnZXN0ZWRfdGl0bGUYCSABKAlIAogBARIaCg1sYXN0X2V2ZW50X2F0GAogASgJSAOIAQFCCgoIX2hvc3RfaWRCDQoLX3NhbmRib3hfaWRCEgoQX3N1Z2dlc3RlZF90aXRsZUIQCg5fbGFzdF9ldmVudF9hdCK4BQoUQ3JlYXRlU2Vzc2lvblJlcXVlc3QSEQoJaW1hZ2VfdXJpGAEgASgJEgwKBG1vZGUYAiABKAkSEwoGcHJvbXB0GAMgASgJSACIAQESFgoJcHJvbXB0X2lkGAcgASgJSAGIAQESQQoHc2VjcmV0cxgFIAMoCzIwLmVuZ3JhbS5hcHAudjEuQ3JlYXRlU2Vzc2lvblJlcXVlc3QuU2VjcmV0c0VudHJ5EkgKC2hhcm5lc3NfZW52GAYgAygLMjMuZW5ncmFtLmFwcC52MS5DcmVhdGVTZXNzaW9uUmVxdWVzdC5IYXJuZXNzRW52RW50cnkSFwoPc2VsZWN0ZWRfc2tpbGxzGAggAygJEhQKDGNhcGFiaWxpdGllcxgJIAMoCRIfChdpbnRlZ3JhdGlvbl9wb2xpY3lfanNvbhgKIAEoCRIUCgdoYXJuZXNzGAsgASgJSAKIAQESPwoQb2F1dGhfY3JlZGVudGlhbBgMIAEoCzIlLmVuZ3JhbS5hcHAudjEuT0F1dGhDcmVkZW50aWFsQmluZGluZxIZCgxoYXJuZXNzX21vZGUYDSABKAlIA4gBARIhChRyZXF1ZXN0ZWRfc2Vzc2lvbl9pZBgOIAEoCUgEiAEBEigKBGFwcHMYDyADKAsyGi5lbmdyYW0uYXBwLnYxLkFwcEVuZHBvaW50Gi4KDFNlY3JldHNFbnRyeRILCgNrZXkYASABKAkSDQoFdmFsdWUYAiABKAk6AjgBGjEKD0hhcm5lc3NFbnZFbnRyeRILCgNrZXkYASABKAkSDQoFdmFsdWUYAiABKAk6AjgBQgkKB19wcm9tcHRCDAoKX3Byb21wdF9pZEIKCghfaGFybmVzc0IPCg1faGFybmVzc19tb2RlQhcKFV9yZXF1ZXN0ZWRfc2Vzc2lvbl9pZEoECAQQBSItCgtBcHBFbmRwb2ludBIQCghob3N0bmFtZRgBIAEoCRIMCgRwb3J0GAIgASgNIlgKFk9BdXRoQ3JlZGVudGlhbEJpbmRpbmcSLAoHc3ViamVjdBgBIAEoCzIbLmVuZ3JhbS5hcHAudjEuT0F1dGhTdWJqZWN0EhAKCHByb3ZpZGVyGAIgASgJImAKFUNyZWF0ZVNlc3Npb25SZXNwb25zZRISCgpzZXNzaW9uX2lkGAEgASgJEg4KBnN0YXR1cxgCIAEoCRIVCg1pbWFnZV92ZXJzaW9uGAMgASgJEgwKBGtpbmQYBCABKAkiJwoRR2V0U2Vzc2lvblJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCSI9ChJHZXRTZXNzaW9uUmVzcG9uc2USJwoHc2Vzc2lvbhgBIAEoCzIWLmVuZ3JhbS5hcHAudjEuU2Vzc2lvbiIqChREZWxldGVTZXNzaW9uUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJIhcKFURlbGV0ZVNlc3Npb25SZXNwb25zZSJ0ChFTZW5kUHJvbXB0UmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEgwKBHRleHQYAiABKAkSEQoJcHJvbXB0X2lkGAMgASgJEhkKDGhhcm5lc3NfbW9kZRgEIAEoCUgAiAEBQg8KDV9oYXJuZXNzX21vZGUiNgoSU2VuZFByb21wdFJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSDAoEbm90ZRgCIAEoCSI2ChBJbnRlcnJ1cHRSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkSDgoGc291cmNlGAIgASgJIjUKEUludGVycnVwdFJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSDAoEbm90ZRgCIAEoCSJOChdFZGl0UXVldWVkUHJvbXB0UmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEhEKCXByb21wdF9pZBgCIAEoCRIMCgR0ZXh0GAMgASgJIjwKGEVkaXRRdWV1ZWRQcm9tcHRSZXNwb25zZRISCgpzZXNzaW9uX2lkGAEgASgJEgwKBG5vdGUYAiABKAkiQwoaRGVxdWV1ZVF1ZXVlZFByb21wdFJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCRIRCglwcm9tcHRfaWQYAiABKAkiPwobRGVxdWV1ZVF1ZXVlZFByb21wdFJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSDAoEbm90ZRgCIAEoCSJXChFNY3BXcml0ZVBvbGljeVJlZhISCgpzZXNzaW9uX2lkGAEgASgJEhcKD3BvbGljeV9yZXZpc2lvbhgCIAEoAxIVCg1iaW5kaW5nX2Vwb2NoGAMgASgDIscDChVNY3BUcnVzdGVkV3JpdGVQb2xpY3kSLQoDcmVmGAEgASgLMiAuZW5ncmFtLmFwcC52MS5NY3BXcml0ZVBvbGljeVJlZhIZChF3cml0ZV9lbGlnaWJpbGl0eRgCIAEoCRINCgVjYXVzZRgDIAEoCRIZChFhY2NlcHRlZF9pbnB1dF9pZBgEIAEoCRIjChZhdXRoZW50aWNhdGVkX2FjdG9yX2lkGAUgASgJSACIAQESFgoJcHJvbXB0X2lkGAYgASgJSAGIAQESHgoRcGxhbl90b29sX2NhbGxfaWQYByABKAlIAogBARIgChNwbGFuX3JlcXVlc3RfZGlnZXN0GAggASgJSAOIAQESPgoUYW5jZXN0b3JfcG9saWN5X3JlZnMYCSADKAsyIC5lbmdyYW0uYXBwLnYxLk1jcFdyaXRlUG9saWN5UmVmEg8KB2N1cnJlbnQYCiABKAgSEwoLaW5pdGlhbGl6ZWQYCyABKAhCGQoXX2F1dGhlbnRpY2F0ZWRfYWN0b3JfaWRCDAoKX3Byb21wdF9pZEIUChJfcGxhbl90b29sX2NhbGxfaWRCFgoUX3BsYW5fcmVxdWVzdF9kaWdlc3QiqQEKDU1jcFBsYW5SZXZpZXcSNAoKcG9saWN5X3JlZhgBIAEoCzIgLmVuZ3JhbS5hcHAudjEuTWNwV3JpdGVQb2xpY3lSZWYSFAoMdG9vbF9jYWxsX2lkGAIgASgJEg4KBnJ1bl9pZBgDIAEoCRIRCglldmVudF9pZHgYBCABKAMSFgoOcmVxdWVzdF9kaWdlc3QYBSABKAkSEQoJYXJnc19qc29uGAYgASgJImQKGEdldE1jcFdyaXRlUG9saWN5UmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEh4KEXBsYW5fdG9vbF9jYWxsX2lkGAIgASgJSACIAQFCFAoSX3BsYW5fdG9vbF9jYWxsX2lkIq0BChlHZXRNY3BXcml0ZVBvbGljeVJlc3BvbnNlEjQKBnBvbGljeRgBIAEoCzIkLmVuZ3JhbS5hcHAudjEuTWNwVHJ1c3RlZFdyaXRlUG9saWN5EjEKC3BsYW5fcmV2aWV3GAIgASgLMhwuZW5ncmFtLmFwcC52MS5NY3BQbGFuUmV2aWV3Eg4KBnJlYXNvbhgDIAEoCRIXCg9hbGxvd2VkX2FjdGlvbnMYBCADKAkimAEKF0NvbXBsZXRlVG9vbENhbGxSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkSFAoMdG9vbF9jYWxsX2lkGAIgASgJEhMKC3Jlc3VsdF9qc29uGAMgASgJEj4KGGV4cGVjdGVkX21jcF9wbGFuX3JldmlldxgEIAEoCzIcLmVuZ3JhbS5hcHAudjEuTWNwUGxhblJldmlldyI8ChhDb21wbGV0ZVRvb2xDYWxsUmVzcG9uc2USEgoKc2Vzc2lvbl9pZBgBIAEoCRIMCgRub3RlGAIgASgJIl0KE1N0cmVhbUV2ZW50c1JlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCRISCgVzaW5jZRgCIAEoA0gAiAEBEhQKDGR1cmFibGVfb25seRgDIAEoCEIICgZfc2luY2UiTAoMU2Vzc2lvbkV2ZW50EhAKA2lkeBgBIAEoA0gAiAEBEgwKBGtpbmQYAiABKAkSFAoMcGF5bG9hZF9qc29uGAMgASgJQgYKBF9pZHgivQEKGExpc3RTZXNzaW9uRXZlbnRzUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEhYKCWFmdGVyX2lkeBgCIAEoA0gAiAEBEhIKBWxpbWl0GAMgASgDSAGIAQESFwoKYmVmb3JlX2lkeBgEIAEoA0gCiAEBEg0KBWtpbmRzGAUgAygJEhIKCnRvb2xfbmFtZXMYBiADKAlCDAoKX2FmdGVyX2lkeEIICgZfbGltaXRCDQoLX2JlZm9yZV9pZHgiYAoZTGlzdFNlc3Npb25FdmVudHNSZXNwb25zZRIrCgZldmVudHMYASADKAsyGy5lbmdyYW0uYXBwLnYxLlNlc3Npb25FdmVudBIWCg5uZXh0X2FmdGVyX2lkeBgCIAEoAyKXAwoLRXhlY1JlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCRIUCgdjb21tYW5kGAIgASgJSACIAQESDAoEYXJndhgDIAMoCRIwCgNlbnYYBCADKAsyIy5lbmdyYW0uYXBwLnYxLkV4ZWNSZXF1ZXN0LkVudkVudHJ5EhQKB3dvcmtkaXIYBSABKAlIAYgBARIZCgx0aW1lb3V0X3NlY3MYBiABKARIAogBARIUCgdleGVjX2lkGAcgASgJSAOIAQESGgoNc3Rkb3V0X29mZnNldBgIIAEoBEgEiAEBEhoKDXN0ZGVycl9vZmZzZXQYCSABKARIBYgBARIRCgR3YWtlGAogASgISAaIAQEaKgoIRW52RW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4AUIKCghfY29tbWFuZEIKCghfd29ya2RpckIPCg1fdGltZW91dF9zZWNzQgoKCF9leGVjX2lkQhAKDl9zdGRvdXRfb2Zmc2V0QhAKDl9zdGRlcnJfb2Zmc2V0QgcKBV93YWtlIjgKEUNhbmNlbEV4ZWNSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkSDwoHZXhlY19pZBgCIAEoCSIUChJDYW5jZWxFeGVjUmVzcG9uc2UidQoRV3JpdGVGaWxlTWV0YWRhdGESEgoKc2Vzc2lvbl9pZBgBIAEoCRIMCgRwYXRoGAIgASgJEhIKCnNpemVfYnl0ZXMYAyABKAQSDgoGc2hhMjU2GAQgASgJEhEKBG1vZGUYBSABKA1IAIgBAUIHCgVfbW9kZSJiChBXcml0ZUZpbGVSZXF1ZXN0EjQKCG1ldGFkYXRhGAEgASgLMiAuZW5ncmFtLmFwcC52MS5Xcml0ZUZpbGVNZXRhZGF0YUgAEg8KBWNodW5rGAIgASgMSABCBwoFZnJhbWUiRQoRV3JpdGVGaWxlUmVzcG9uc2USDAoEcGF0aBgBIAEoCRISCgpzaXplX2J5dGVzGAIgASgEEg4KBnNoYTI1NhgDIAEoCSIzCg9SZWFkRmlsZVJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCRIMCgRwYXRoGAIgASgJIlcKEFJlYWRGaWxlTWV0YWRhdGESDAoEcGF0aBgBIAEoCRISCgpzaXplX2J5dGVzGAIgASgEEg4KBnNoYTI1NhgDIAEoCRIRCglmaWxlX25hbWUYBCABKAkiYQoQUmVhZEZpbGVSZXNwb25zZRIzCghtZXRhZGF0YRgBIAEoCzIfLmVuZ3JhbS5hcHAudjEuUmVhZEZpbGVNZXRhZGF0YUgAEg8KBWNodW5rGAIgASgMSABCBwoFZnJhbWUiVwoQQ29weUZpbGVzUmVxdWVzdBIZChFzb3VyY2Vfc2Vzc2lvbl9pZBgBIAEoCRIZChF0YXJnZXRfc2Vzc2lvbl9pZBgCIAEoCRINCgVwYXRocxgDIAMoCSJCCg5Db3B5RmlsZVJlc3VsdBIMCgRwYXRoGAEgASgJEhIKCnNpemVfYnl0ZXMYAiABKAQSDgoGc2hhMjU2GAMgASgJIkEKEUNvcHlGaWxlc1Jlc3BvbnNlEiwKBWZpbGVzGAEgAygLMh0uZW5ncmFtLmFwcC52MS5Db3B5RmlsZVJlc3VsdCKRAQoKRXhlY091dHB1dBItCgdzdGFydGVkGAEgASgLMhouZW5ncmFtLmFwcC52MS5FeGVjU3RhcnRlZEgAEhAKBnN0ZG91dBgCIAEoDEgAEhAKBnN0ZGVychgDIAEoDEgAEicKBGV4aXQYBCABKAsyFy5lbmdyYW0uYXBwLnYxLkV4ZWNFeGl0SABCBwoFZXZlbnQiHgoLRXhlY1N0YXJ0ZWQSDwoHZXhlY19pZBgBIAEoCSJfCghFeGVjRXhpdBIYCgtleGl0X3N0YXR1cxgBIAEoBUgAiAEBEikKBnJ1c2FnZRgCIAEoCzIZLmVuZ3JhbS5hcHAudjEuRXhlY1J1c2FnZUIOCgxfZXhpdF9zdGF0dXMimQEKCkV4ZWNSdXNhZ2USDwoHd2FsbF9tcxgBIAEoBBIYCgtwZWFrX3Jzc19rYhgCIAEoBEgAiAEBEhgKC3VzZXJfY3B1X21zGAMgASgESAGIAQESFwoKc3lzX2NwdV9tcxgEIAEoBEgCiAEBQg4KDF9wZWFrX3Jzc19rYkIOCgxfdXNlcl9jcHVfbXNCDQoLX3N5c19jcHVfbXMieQoNR2V0TG9nUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEhEKBGtpbmQYAiABKAlIAIgBARISCgVsaW1pdBgDIAEoA0gBiAEBEhEKBHRhaWwYBCABKAhIAogBAUIHCgVfa2luZEIICgZfbGltaXRCBwoFX3RhaWwiZAoOR2V0TG9nUmVzcG9uc2USEgoKc2Vzc2lvbl9pZBgBIAEoCRIMCgRraW5kGAIgASgJEjAKBmV2ZW50cxgDIAMoCzIgLmVuZ3JhbS5hcHAudjEuQ29udmVyc2F0aW9uRW50cnkiUAoRQ29udmVyc2F0aW9uRW50cnkSCwoDaWR4GAEgASgDEgwKBGtpbmQYAiABKAkSCgoCYXQYAyABKAkSFAoMcGF5bG9hZF9qc29uGAQgASgJIiUKD1NuYXBzaG90UmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJIoYBChBTbmFwc2hvdFJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSGAoLc25hcHNob3RfaWQYAiABKAlIAIgBARIXCgpzaXplX2J5dGVzGAMgASgESAGIAQESDAoEbm90ZRgEIAEoCUIOCgxfc25hcHNob3RfaWRCDQoLX3NpemVfYnl0ZXMiKgoURW5zdXJlQnJvd3NlclJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCSIlChVFbnN1cmVCcm93c2VyUmVzcG9uc2USDAoEcG9ydBgBIAEoDSImChBFbnN1cmVJZGVSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkiIQoRRW5zdXJlSWRlUmVzcG9uc2USDAoEcG9ydBgBIAEoDSIjCg1SZXN1bWVSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkihAEKDlJlc3VtZVJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSGAoLc25hcHNob3RfaWQYAiABKAlIAIgBARIXCgpzaXplX2J5dGVzGAMgASgESAGIAQESDAoEbm90ZRgEIAEoCUIOCgxfc25hcHNob3RfaWRCDQoLX3NpemVfYnl0ZXMiJwoRRXZpY3RMb2NhbFJlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCSIUChJFdmljdExvY2FsUmVzcG9uc2UiJgoQRXZpY3RJZGxlUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJIjcKEUV2aWN0SWRsZVJlc3BvbnNlEhIKCnNlc3Npb25faWQYASABKAkSDgoGc3RhdHVzGAIgASgJIiwKFkxpc3RDaGVja3BvaW50c1JlcXVlc3QSEgoKc2Vzc2lvbl9pZBgBIAEoCSJkChdMaXN0Q2hlY2twb2ludHNSZXNwb25zZRISCgpzZXNzaW9uX2lkGAEgASgJEjUKC2NoZWNrcG9pbnRzGAIgAygLMiAuZW5ncmFtLmFwcC52MS5DaGVja3BvaW50U3VtbWFyeSKmAQoRQ2hlY2twb2ludFN1bW1hcnkSEwoLc25hcHNob3RfaWQYASABKAkSEgoKY3JlYXRlZF9hdBgCIAEoCRISCgpzaXplX2J5dGVzGAMgASgEEhoKDWV2ZW50c19jdXJzb3IYBCABKANIAIgBARITCgtyZWNvdmVyYWJsZRgFIAEoCBIRCglpc19sYXRlc3QYBiABKAhCEAoOX2V2ZW50c19jdXJzb3IiPQoSR2V0QXJ0aWZhY3RSZXF1ZXN0EhIKCnNlc3Npb25faWQYASABKAkSEwoLYXJ0aWZhY3RfaWQYAiABKAkiYgoTR2V0QXJ0aWZhY3RSZXNwb25zZRIzCghtZXRhZGF0YRgBIAEoCzIfLmVuZ3JhbS5hcHAudjEuQXJ0aWZhY3RNZXRhZGF0YUgAEg8KBWNodW5rGAIgASgMSABCBQoDbXNnIk0KEEFydGlmYWN0TWV0YWRhdGESEgoKbWVkaWFfdHlwZRgBIAEoCRISCgpzaXplX2J5dGVzGAIgASgEEhEKCWZpbGVfbmFtZRgDIAEoCSJ7Ch1DcmVhdGVBcnRpZmFjdEZyb21QYXRoUmVxdWVzdBISCgpzZXNzaW9uX2lkGAEgASgJEgwKBHBhdGgYAiABKAkSFAoHY2FwdGlvbhgDIAEoCUgAiAEBEhYKDnN1cHByZXNzX2V2ZW50GAQgASgIQgoKCF9jYXB0aW9uIl0KHkNyZWF0ZUFydGlmYWN0RnJvbVBhdGhSZXNwb25zZRITCgthcnRpZmFjdF9pZBgBIAEoCRISCgptZWRpYV90eXBlGAIgASgJEhIKCnNpemVfYnl0ZXMYAyABKAQitAEKEVJlbGF5U2hlbGxSZXF1ZXN0EigKBG9wZW4YASABKAsyGC5lbmdyYW0uYXBwLnYxLlNoZWxsT3BlbkgAEg4KBHRleHQYAiABKAlIABIQCgZiaW5hcnkYAyABKAxIABIOCgRwaW5nGAQgASgMSAASDgoEcG9uZxgFIAEoDEgAEioKBWNsb3NlGAYgASgLMhkuZW5ncmFtLmFwcC52MS5TaGVsbENsb3NlSABCBwoFZnJhbWUiiwEKElJlbGF5U2hlbGxSZXNwb25zZRIOCgR0ZXh0GAEgASgJSAASEAoGYmluYXJ5GAIgASgMSAASDgoEcGluZxgDIAEoDEgAEg4KBHBvbmcYBCABKAxIABIqCgVjbG9zZRgFIAEoCzIZLmVuZ3JhbS5hcHAudjEuU2hlbGxDbG9zZUgAQgcKBWZyYW1lIh8KCVNoZWxsT3BlbhISCgpzZXNzaW9uX2lkGAEgASgJIioKClNoZWxsQ2xvc2USDAoEY29kZRgBIAEoDRIOCgZyZWFzb24YAiABKAkifwoQUmVsYXlQb3J0UmVxdWVzdBInCgRvcGVuGAEgASgLMhcuZW5ncmFtLmFwcC52MS5Qb3J0T3BlbkgAEg4KBGRhdGEYAiABKAxIABIpCgVjbG9zZRgDIAEoCzIYLmVuZ3JhbS5hcHAudjEuUG9ydENsb3NlSABCBwoFZnJhbWUiVwoRUmVsYXlQb3J0UmVzcG9uc2USDgoEZGF0YRgBIAEoDEgAEikKBWNsb3NlGAIgASgLMhguZW5ncmFtLmFwcC52MS5Qb3J0Q2xvc2VIAEIHCgVmcmFtZSIsCghQb3J0T3BlbhISCgpzZXNzaW9uX2lkGAEgASgJEgwKBHBvcnQYAiABKA0iCwoJUG9ydENsb3NlMtISCg5TZXNzaW9uU2VydmljZRJXCgxMaXN0U2Vzc2lvbnMSIi5lbmdyYW0uYXBwLnYxLkxpc3RTZXNzaW9uc1JlcXVlc3QaIy5lbmdyYW0uYXBwLnYxLkxpc3RTZXNzaW9uc1Jlc3BvbnNlEloKDUNyZWF0ZVNlc3Npb24SIy5lbmdyYW0uYXBwLnYxLkNyZWF0ZVNlc3Npb25SZXF1ZXN0GiQuZW5ncmFtLmFwcC52MS5DcmVhdGVTZXNzaW9uUmVzcG9uc2USUQoKR2V0U2Vzc2lvbhIgLmVuZ3JhbS5hcHAudjEuR2V0U2Vzc2lvblJlcXVlc3QaIS5lbmdyYW0uYXBwLnYxLkdldFNlc3Npb25SZXNwb25zZRJaCg1EZWxldGVTZXNzaW9uEiMuZW5ncmFtLmFwcC52MS5EZWxldGVTZXNzaW9uUmVxdWVzdBokLmVuZ3JhbS5hcHAudjEuRGVsZXRlU2Vzc2lvblJlc3BvbnNlElEKClNlbmRQcm9tcHQSIC5lbmdyYW0uYXBwLnYxLlNlbmRQcm9tcHRSZXF1ZXN0GiEuZW5ncmFtLmFwcC52MS5TZW5kUHJvbXB0UmVzcG9uc2USTgoJSW50ZXJydXB0Eh8uZW5ncmFtLmFwcC52MS5JbnRlcnJ1cHRSZXF1ZXN0GiAuZW5ncmFtLmFwcC52MS5JbnRlcnJ1cHRSZXNwb25zZRJjChBFZGl0UXVldWVkUHJvbXB0EiYuZW5ncmFtLmFwcC52MS5FZGl0UXVldWVkUHJvbXB0UmVxdWVzdBonLmVuZ3JhbS5hcHAudjEuRWRpdFF1ZXVlZFByb21wdFJlc3BvbnNlEmwKE0RlcXVldWVRdWV1ZWRQcm9tcHQSKS5lbmdyYW0uYXBwLnYxLkRlcXVldWVRdWV1ZWRQcm9tcHRSZXF1ZXN0GiouZW5ncmFtLmFwcC52MS5EZXF1ZXVlUXVldWVkUHJvbXB0UmVzcG9uc2USYwoQQ29tcGxldGVUb29sQ2FsbBImLmVuZ3JhbS5hcHAudjEuQ29tcGxldGVUb29sQ2FsbFJlcXVlc3QaJy5lbmdyYW0uYXBwLnYxLkNvbXBsZXRlVG9vbENhbGxSZXNwb25zZRJmChFHZXRNY3BXcml0ZVBvbGljeRInLmVuZ3JhbS5hcHAudjEuR2V0TWNwV3JpdGVQb2xpY3lSZXF1ZXN0GiguZW5ncmFtLmFwcC52MS5HZXRNY3BXcml0ZVBvbGljeVJlc3BvbnNlElEKDFN0cmVhbUV2ZW50cxIiLmVuZ3JhbS5hcHAudjEuU3RyZWFtRXZlbnRzUmVxdWVzdBobLmVuZ3JhbS5hcHAudjEuU2Vzc2lvbkV2ZW50MAESZgoRTGlzdFNlc3Npb25FdmVudHMSJy5lbmdyYW0uYXBwLnYxLkxpc3RTZXNzaW9uRXZlbnRzUmVxdWVzdBooLmVuZ3JhbS5hcHAudjEuTGlzdFNlc3Npb25FdmVudHNSZXNwb25zZRI/CgRFeGVjEhouZW5ncmFtLmFwcC52MS5FeGVjUmVxdWVzdBoZLmVuZ3JhbS5hcHAudjEuRXhlY091dHB1dDABElEKCkNhbmNlbEV4ZWMSIC5lbmdyYW0uYXBwLnYxLkNhbmNlbEV4ZWNSZXF1ZXN0GiEuZW5ncmFtLmFwcC52MS5DYW5jZWxFeGVjUmVzcG9uc2USUAoJV3JpdGVGaWxlEh8uZW5ncmFtLmFwcC52MS5Xcml0ZUZpbGVSZXF1ZXN0GiAuZW5ncmFtLmFwcC52MS5Xcml0ZUZpbGVSZXNwb25zZSgBEk0KCFJlYWRGaWxlEh4uZW5ncmFtLmFwcC52MS5SZWFkRmlsZVJlcXVlc3QaHy5lbmdyYW0uYXBwLnYxLlJlYWRGaWxlUmVzcG9uc2UwARJOCglDb3B5RmlsZXMSHy5lbmdyYW0uYXBwLnYxLkNvcHlGaWxlc1JlcXVlc3QaIC5lbmdyYW0uYXBwLnYxLkNvcHlGaWxlc1Jlc3BvbnNlEkUKBkdldExvZxIcLmVuZ3JhbS5hcHAudjEuR2V0TG9nUmVxdWVzdBodLmVuZ3JhbS5hcHAudjEuR2V0TG9nUmVzcG9uc2USSwoIU25hcHNob3QSHi5lbmdyYW0uYXBwLnYxLlNuYXBzaG90UmVxdWVzdBofLmVuZ3JhbS5hcHAudjEuU25hcHNob3RSZXNwb25zZRJaCg1FbnN1cmVCcm93c2VyEiMuZW5ncmFtLmFwcC52MS5FbnN1cmVCcm93c2VyUmVxdWVzdBokLmVuZ3JhbS5hcHAudjEuRW5zdXJlQnJvd3NlclJlc3BvbnNlEk4KCUVuc3VyZUlkZRIfLmVuZ3JhbS5hcHAudjEuRW5zdXJlSWRlUmVxdWVzdBogLmVuZ3JhbS5hcHAudjEuRW5zdXJlSWRlUmVzcG9uc2USRQoGUmVzdW1lEhwuZW5ncmFtLmFwcC52MS5SZXN1bWVSZXF1ZXN0Gh0uZW5ncmFtLmFwcC52MS5SZXN1bWVSZXNwb25zZRJRCgpFdmljdExvY2FsEiAuZW5ncmFtLmFwcC52MS5FdmljdExvY2FsUmVxdWVzdBohLmVuZ3JhbS5hcHAudjEuRXZpY3RMb2NhbFJlc3BvbnNlEk4KCUV2aWN0SWRsZRIfLmVuZ3JhbS5hcHAudjEuRXZpY3RJZGxlUmVxdWVzdBogLmVuZ3JhbS5hcHAudjEuRXZpY3RJZGxlUmVzcG9uc2USYAoPTGlzdENoZWNrcG9pbnRzEiUuZW5ncmFtLmFwcC52MS5MaXN0Q2hlY2twb2ludHNSZXF1ZXN0GiYuZW5ncmFtLmFwcC52MS5MaXN0Q2hlY2twb2ludHNSZXNwb25zZRJWCgtHZXRBcnRpZmFjdBIhLmVuZ3JhbS5hcHAudjEuR2V0QXJ0aWZhY3RSZXF1ZXN0GiIuZW5ncmFtLmFwcC52MS5HZXRBcnRpZmFjdFJlc3BvbnNlMAESdQoWQ3JlYXRlQXJ0aWZhY3RGcm9tUGF0aBIsLmVuZ3JhbS5hcHAudjEuQ3JlYXRlQXJ0aWZhY3RGcm9tUGF0aFJlcXVlc3QaLS5lbmdyYW0uYXBwLnYxLkNyZWF0ZUFydGlmYWN0RnJvbVBhdGhSZXNwb25zZTJlChFTaGVsbFJlbGF5U2VydmljZRJQCgVSZWxheRIgLmVuZ3JhbS5hcHAudjEuUmVsYXlTaGVsbFJlcXVlc3QaIS5lbmdyYW0uYXBwLnYxLlJlbGF5U2hlbGxSZXNwb25zZSgBMAEyYgoQUG9ydFJlbGF5U2VydmljZRJOCgVSZWxheRIfLmVuZ3JhbS5hcHAudjEuUmVsYXlQb3J0UmVxdWVzdBogLmVuZ3JhbS5hcHAudjEuUmVsYXlQb3J0UmVzcG9uc2UoATABYgZwcm90bzM", [file_engram_app_v1_oauth]); /** * @generated from message engram.app.v1.ListSessionsRequest @@ -684,6 +684,196 @@ export type DequeueQueuedPromptResponse = Message<"engram.app.v1.DequeueQueuedPr export const DequeueQueuedPromptResponseSchema: GenMessage = /*@__PURE__*/ messageDesc(file_engram_app_v1_session, 19); +/** + * @generated from message engram.app.v1.McpWritePolicyRef + */ +export type McpWritePolicyRef = Message<"engram.app.v1.McpWritePolicyRef"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: int64 policy_revision = 2; + */ + policyRevision: bigint; + + /** + * @generated from field: int64 binding_epoch = 3; + */ + bindingEpoch: bigint; +}; + +/** + * Describes the message engram.app.v1.McpWritePolicyRef. + * Use `create(McpWritePolicyRefSchema)` to create a new message. + */ +export const McpWritePolicyRefSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_session, 20); + +/** + * @generated from message engram.app.v1.McpTrustedWritePolicy + */ +export type McpTrustedWritePolicy = Message<"engram.app.v1.McpTrustedWritePolicy"> & { + /** + * @generated from field: engram.app.v1.McpWritePolicyRef ref = 1; + */ + ref?: McpWritePolicyRef; + + /** + * @generated from field: string write_eligibility = 2; + */ + writeEligibility: string; + + /** + * @generated from field: string cause = 3; + */ + cause: string; + + /** + * @generated from field: string accepted_input_id = 4; + */ + acceptedInputId: string; + + /** + * @generated from field: optional string authenticated_actor_id = 5; + */ + authenticatedActorId?: string; + + /** + * @generated from field: optional string prompt_id = 6; + */ + promptId?: string; + + /** + * @generated from field: optional string plan_tool_call_id = 7; + */ + planToolCallId?: string; + + /** + * @generated from field: optional string plan_request_digest = 8; + */ + planRequestDigest?: string; + + /** + * @generated from field: repeated engram.app.v1.McpWritePolicyRef ancestor_policy_refs = 9; + */ + ancestorPolicyRefs: McpWritePolicyRef[]; + + /** + * @generated from field: bool current = 10; + */ + current: boolean; + + /** + * @generated from field: bool initialized = 11; + */ + initialized: boolean; +}; + +/** + * Describes the message engram.app.v1.McpTrustedWritePolicy. + * Use `create(McpTrustedWritePolicySchema)` to create a new message. + */ +export const McpTrustedWritePolicySchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_session, 21); + +/** + * @generated from message engram.app.v1.McpPlanReview + */ +export type McpPlanReview = Message<"engram.app.v1.McpPlanReview"> & { + /** + * @generated from field: engram.app.v1.McpWritePolicyRef policy_ref = 1; + */ + policyRef?: McpWritePolicyRef; + + /** + * @generated from field: string tool_call_id = 2; + */ + toolCallId: string; + + /** + * @generated from field: string run_id = 3; + */ + runId: string; + + /** + * @generated from field: int64 event_idx = 4; + */ + eventIdx: bigint; + + /** + * @generated from field: string request_digest = 5; + */ + requestDigest: string; + + /** + * @generated from field: string args_json = 6; + */ + argsJson: string; +}; + +/** + * Describes the message engram.app.v1.McpPlanReview. + * Use `create(McpPlanReviewSchema)` to create a new message. + */ +export const McpPlanReviewSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_session, 22); + +/** + * @generated from message engram.app.v1.GetMcpWritePolicyRequest + */ +export type GetMcpWritePolicyRequest = Message<"engram.app.v1.GetMcpWritePolicyRequest"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: optional string plan_tool_call_id = 2; + */ + planToolCallId?: string; +}; + +/** + * Describes the message engram.app.v1.GetMcpWritePolicyRequest. + * Use `create(GetMcpWritePolicyRequestSchema)` to create a new message. + */ +export const GetMcpWritePolicyRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_session, 23); + +/** + * @generated from message engram.app.v1.GetMcpWritePolicyResponse + */ +export type GetMcpWritePolicyResponse = Message<"engram.app.v1.GetMcpWritePolicyResponse"> & { + /** + * @generated from field: engram.app.v1.McpTrustedWritePolicy policy = 1; + */ + policy?: McpTrustedWritePolicy; + + /** + * @generated from field: engram.app.v1.McpPlanReview plan_review = 2; + */ + planReview?: McpPlanReview; + + /** + * @generated from field: string reason = 3; + */ + reason: string; + + /** + * @generated from field: repeated string allowed_actions = 4; + */ + allowedActions: string[]; +}; + +/** + * Describes the message engram.app.v1.GetMcpWritePolicyResponse. + * Use `create(GetMcpWritePolicyResponseSchema)` to create a new message. + */ +export const GetMcpWritePolicyResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_app_v1_session, 24); + /** * @generated from message engram.app.v1.CompleteToolCallRequest */ @@ -708,6 +898,13 @@ export type CompleteToolCallRequest = Message<"engram.app.v1.CompleteToolCallReq * @generated from field: string result_json = 3; */ resultJson: string; + + /** + * Optional exact server review. Only the authenticated human handler consumes it. + * + * @generated from field: engram.app.v1.McpPlanReview expected_mcp_plan_review = 4; + */ + expectedMcpPlanReview?: McpPlanReview; }; /** @@ -715,7 +912,7 @@ export type CompleteToolCallRequest = Message<"engram.app.v1.CompleteToolCallReq * Use `create(CompleteToolCallRequestSchema)` to create a new message. */ export const CompleteToolCallRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 20); + messageDesc(file_engram_app_v1_session, 25); /** * @generated from message engram.app.v1.CompleteToolCallResponse @@ -737,7 +934,7 @@ export type CompleteToolCallResponse = Message<"engram.app.v1.CompleteToolCallRe * Use `create(CompleteToolCallResponseSchema)` to create a new message. */ export const CompleteToolCallResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 21); + messageDesc(file_engram_app_v1_session, 26); /** * @generated from message engram.app.v1.StreamEventsRequest @@ -774,7 +971,7 @@ export type StreamEventsRequest = Message<"engram.app.v1.StreamEventsRequest"> & * Use `create(StreamEventsRequestSchema)` to create a new message. */ export const StreamEventsRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 22); + messageDesc(file_engram_app_v1_session, 27); /** * The typed event ENVELOPE. payload_json is NORMATIVELY the JSON object @@ -812,7 +1009,7 @@ export type SessionEvent = Message<"engram.app.v1.SessionEvent"> & { * Use `create(SessionEventSchema)` to create a new message. */ export const SessionEventSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 23); + messageDesc(file_engram_app_v1_session, 28); /** * @generated from message engram.app.v1.ListSessionEventsRequest @@ -878,7 +1075,7 @@ export type ListSessionEventsRequest = Message<"engram.app.v1.ListSessionEventsR * Use `create(ListSessionEventsRequestSchema)` to create a new message. */ export const ListSessionEventsRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 24); + messageDesc(file_engram_app_v1_session, 29); /** * @generated from message engram.app.v1.ListSessionEventsResponse @@ -907,7 +1104,7 @@ export type ListSessionEventsResponse = Message<"engram.app.v1.ListSessionEvents * Use `create(ListSessionEventsResponseSchema)` to create a new message. */ export const ListSessionEventsResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 25); + messageDesc(file_engram_app_v1_session, 30); /** * @generated from message engram.app.v1.ExecRequest @@ -977,7 +1174,7 @@ export type ExecRequest = Message<"engram.app.v1.ExecRequest"> & { * Use `create(ExecRequestSchema)` to create a new message. */ export const ExecRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 26); + messageDesc(file_engram_app_v1_session, 31); /** * @generated from message engram.app.v1.CancelExecRequest @@ -999,7 +1196,7 @@ export type CancelExecRequest = Message<"engram.app.v1.CancelExecRequest"> & { * Use `create(CancelExecRequestSchema)` to create a new message. */ export const CancelExecRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 27); + messageDesc(file_engram_app_v1_session, 32); /** * @generated from message engram.app.v1.CancelExecResponse @@ -1012,7 +1209,7 @@ export type CancelExecResponse = Message<"engram.app.v1.CancelExecResponse"> & { * Use `create(CancelExecResponseSchema)` to create a new message. */ export const CancelExecResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 28); + messageDesc(file_engram_app_v1_session, 33); /** * @generated from message engram.app.v1.WriteFileMetadata @@ -1055,7 +1252,7 @@ export type WriteFileMetadata = Message<"engram.app.v1.WriteFileMetadata"> & { * Use `create(WriteFileMetadataSchema)` to create a new message. */ export const WriteFileMetadataSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 29); + messageDesc(file_engram_app_v1_session, 34); /** * @generated from message engram.app.v1.WriteFileRequest @@ -1084,7 +1281,7 @@ export type WriteFileRequest = Message<"engram.app.v1.WriteFileRequest"> & { * Use `create(WriteFileRequestSchema)` to create a new message. */ export const WriteFileRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 30); + messageDesc(file_engram_app_v1_session, 35); /** * @generated from message engram.app.v1.WriteFileResponse @@ -1111,7 +1308,7 @@ export type WriteFileResponse = Message<"engram.app.v1.WriteFileResponse"> & { * Use `create(WriteFileResponseSchema)` to create a new message. */ export const WriteFileResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 31); + messageDesc(file_engram_app_v1_session, 36); /** * @generated from message engram.app.v1.ReadFileRequest @@ -1133,7 +1330,7 @@ export type ReadFileRequest = Message<"engram.app.v1.ReadFileRequest"> & { * Use `create(ReadFileRequestSchema)` to create a new message. */ export const ReadFileRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 32); + messageDesc(file_engram_app_v1_session, 37); /** * @generated from message engram.app.v1.ReadFileMetadata @@ -1165,7 +1362,7 @@ export type ReadFileMetadata = Message<"engram.app.v1.ReadFileMetadata"> & { * Use `create(ReadFileMetadataSchema)` to create a new message. */ export const ReadFileMetadataSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 33); + messageDesc(file_engram_app_v1_session, 38); /** * @generated from message engram.app.v1.ReadFileResponse @@ -1194,7 +1391,7 @@ export type ReadFileResponse = Message<"engram.app.v1.ReadFileResponse"> & { * Use `create(ReadFileResponseSchema)` to create a new message. */ export const ReadFileResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 34); + messageDesc(file_engram_app_v1_session, 39); /** * @generated from message engram.app.v1.CopyFilesRequest @@ -1221,7 +1418,7 @@ export type CopyFilesRequest = Message<"engram.app.v1.CopyFilesRequest"> & { * Use `create(CopyFilesRequestSchema)` to create a new message. */ export const CopyFilesRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 35); + messageDesc(file_engram_app_v1_session, 40); /** * @generated from message engram.app.v1.CopyFileResult @@ -1248,7 +1445,7 @@ export type CopyFileResult = Message<"engram.app.v1.CopyFileResult"> & { * Use `create(CopyFileResultSchema)` to create a new message. */ export const CopyFileResultSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 36); + messageDesc(file_engram_app_v1_session, 41); /** * @generated from message engram.app.v1.CopyFilesResponse @@ -1265,7 +1462,7 @@ export type CopyFilesResponse = Message<"engram.app.v1.CopyFilesResponse"> & { * Use `create(CopyFilesResponseSchema)` to create a new message. */ export const CopyFilesResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 37); + messageDesc(file_engram_app_v1_session, 42); /** * engram_core::types::ExecEvent is authoritative for the stream body: @@ -1317,7 +1514,7 @@ export type ExecOutput = Message<"engram.app.v1.ExecOutput"> & { * Use `create(ExecOutputSchema)` to create a new message. */ export const ExecOutputSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 38); + messageDesc(file_engram_app_v1_session, 43); /** * @generated from message engram.app.v1.ExecStarted @@ -1334,7 +1531,7 @@ export type ExecStarted = Message<"engram.app.v1.ExecStarted"> & { * Use `create(ExecStartedSchema)` to create a new message. */ export const ExecStartedSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 39); + messageDesc(file_engram_app_v1_session, 44); /** * @generated from message engram.app.v1.ExecExit @@ -1362,7 +1559,7 @@ export type ExecExit = Message<"engram.app.v1.ExecExit"> & { * Use `create(ExecExitSchema)` to create a new message. */ export const ExecExitSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 40); + messageDesc(file_engram_app_v1_session, 45); /** * engram_core::types::ExecRusage. wall_ms is coordinator-measured wall @@ -1398,7 +1595,7 @@ export type ExecRusage = Message<"engram.app.v1.ExecRusage"> & { * Use `create(ExecRusageSchema)` to create a new message. */ export const ExecRusageSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 41); + messageDesc(file_engram_app_v1_session, 46); /** * @generated from message engram.app.v1.GetLogRequest @@ -1441,7 +1638,7 @@ export type GetLogRequest = Message<"engram.app.v1.GetLogRequest"> & { * Use `create(GetLogRequestSchema)` to create a new message. */ export const GetLogRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 42); + messageDesc(file_engram_app_v1_session, 47); /** * @generated from message engram.app.v1.GetLogResponse @@ -1468,7 +1665,7 @@ export type GetLogResponse = Message<"engram.app.v1.GetLogResponse"> & { * Use `create(GetLogResponseSchema)` to create a new message. */ export const GetLogResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 43); + messageDesc(file_engram_app_v1_session, 48); /** * @generated from message engram.app.v1.ConversationEntry @@ -1504,7 +1701,7 @@ export type ConversationEntry = Message<"engram.app.v1.ConversationEntry"> & { * Use `create(ConversationEntrySchema)` to create a new message. */ export const ConversationEntrySchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 44); + messageDesc(file_engram_app_v1_session, 49); /** * @generated from message engram.app.v1.SnapshotRequest @@ -1521,7 +1718,7 @@ export type SnapshotRequest = Message<"engram.app.v1.SnapshotRequest"> & { * Use `create(SnapshotRequestSchema)` to create a new message. */ export const SnapshotRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 45); + messageDesc(file_engram_app_v1_session, 50); /** * @generated from message engram.app.v1.SnapshotResponse @@ -1555,7 +1752,7 @@ export type SnapshotResponse = Message<"engram.app.v1.SnapshotResponse"> & { * Use `create(SnapshotResponseSchema)` to create a new message. */ export const SnapshotResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 46); + messageDesc(file_engram_app_v1_session, 51); /** * @generated from message engram.app.v1.EnsureBrowserRequest @@ -1572,7 +1769,7 @@ export type EnsureBrowserRequest = Message<"engram.app.v1.EnsureBrowserRequest"> * Use `create(EnsureBrowserRequestSchema)` to create a new message. */ export const EnsureBrowserRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 47); + messageDesc(file_engram_app_v1_session, 52); /** * @generated from message engram.app.v1.EnsureBrowserResponse @@ -1592,7 +1789,7 @@ export type EnsureBrowserResponse = Message<"engram.app.v1.EnsureBrowserResponse * Use `create(EnsureBrowserResponseSchema)` to create a new message. */ export const EnsureBrowserResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 48); + messageDesc(file_engram_app_v1_session, 53); /** * @generated from message engram.app.v1.EnsureIdeRequest @@ -1609,7 +1806,7 @@ export type EnsureIdeRequest = Message<"engram.app.v1.EnsureIdeRequest"> & { * Use `create(EnsureIdeRequestSchema)` to create a new message. */ export const EnsureIdeRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 49); + messageDesc(file_engram_app_v1_session, 54); /** * @generated from message engram.app.v1.EnsureIdeResponse @@ -1629,7 +1826,7 @@ export type EnsureIdeResponse = Message<"engram.app.v1.EnsureIdeResponse"> & { * Use `create(EnsureIdeResponseSchema)` to create a new message. */ export const EnsureIdeResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 50); + messageDesc(file_engram_app_v1_session, 55); /** * @generated from message engram.app.v1.ResumeRequest @@ -1646,7 +1843,7 @@ export type ResumeRequest = Message<"engram.app.v1.ResumeRequest"> & { * Use `create(ResumeRequestSchema)` to create a new message. */ export const ResumeRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 51); + messageDesc(file_engram_app_v1_session, 56); /** * Same shape as SnapshotResponse (the HTTP resume handler reuses it); @@ -1681,7 +1878,7 @@ export type ResumeResponse = Message<"engram.app.v1.ResumeResponse"> & { * Use `create(ResumeResponseSchema)` to create a new message. */ export const ResumeResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 52); + messageDesc(file_engram_app_v1_session, 57); /** * @generated from message engram.app.v1.EvictLocalRequest @@ -1698,7 +1895,7 @@ export type EvictLocalRequest = Message<"engram.app.v1.EvictLocalRequest"> & { * Use `create(EvictLocalRequestSchema)` to create a new message. */ export const EvictLocalRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 53); + messageDesc(file_engram_app_v1_session, 58); /** * Today's HTTP handler returns 202 Accepted with no body. @@ -1713,7 +1910,7 @@ export type EvictLocalResponse = Message<"engram.app.v1.EvictLocalResponse"> & { * Use `create(EvictLocalResponseSchema)` to create a new message. */ export const EvictLocalResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 54); + messageDesc(file_engram_app_v1_session, 59); /** * @generated from message engram.app.v1.EvictIdleRequest @@ -1730,7 +1927,7 @@ export type EvictIdleRequest = Message<"engram.app.v1.EvictIdleRequest"> & { * Use `create(EvictIdleRequestSchema)` to create a new message. */ export const EvictIdleRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 55); + messageDesc(file_engram_app_v1_session, 60); /** * Mirrors api/admin.rs EvictIdleResponse. The synchronous pipeline has @@ -1757,7 +1954,7 @@ export type EvictIdleResponse = Message<"engram.app.v1.EvictIdleResponse"> & { * Use `create(EvictIdleResponseSchema)` to create a new message. */ export const EvictIdleResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 56); + messageDesc(file_engram_app_v1_session, 61); /** * @generated from message engram.app.v1.ListCheckpointsRequest @@ -1774,7 +1971,7 @@ export type ListCheckpointsRequest = Message<"engram.app.v1.ListCheckpointsReque * Use `create(ListCheckpointsRequestSchema)` to create a new message. */ export const ListCheckpointsRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 57); + messageDesc(file_engram_app_v1_session, 62); /** * @generated from message engram.app.v1.ListCheckpointsResponse @@ -1799,7 +1996,7 @@ export type ListCheckpointsResponse = Message<"engram.app.v1.ListCheckpointsResp * Use `create(ListCheckpointsResponseSchema)` to create a new message. */ export const ListCheckpointsResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 58); + messageDesc(file_engram_app_v1_session, 63); /** * ADR 0028 A.log: one checkpoint in a session's chain. @@ -1851,7 +2048,7 @@ export type CheckpointSummary = Message<"engram.app.v1.CheckpointSummary"> & { * Use `create(CheckpointSummarySchema)` to create a new message. */ export const CheckpointSummarySchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 59); + messageDesc(file_engram_app_v1_session, 64); /** * @generated from message engram.app.v1.GetArtifactRequest @@ -1873,7 +2070,7 @@ export type GetArtifactRequest = Message<"engram.app.v1.GetArtifactRequest"> & { * Use `create(GetArtifactRequestSchema)` to create a new message. */ export const GetArtifactRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 60); + messageDesc(file_engram_app_v1_session, 65); /** * @generated from message engram.app.v1.GetArtifactResponse @@ -1904,7 +2101,7 @@ export type GetArtifactResponse = Message<"engram.app.v1.GetArtifactResponse"> & * Use `create(GetArtifactResponseSchema)` to create a new message. */ export const GetArtifactResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 61); + messageDesc(file_engram_app_v1_session, 66); /** * @generated from message engram.app.v1.ArtifactMetadata @@ -1934,7 +2131,7 @@ export type ArtifactMetadata = Message<"engram.app.v1.ArtifactMetadata"> & { * Use `create(ArtifactMetadataSchema)` to create a new message. */ export const ArtifactMetadataSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 62); + messageDesc(file_engram_app_v1_session, 67); /** * ADR 0026 trusted operator file pull: capture ANY file in the session @@ -1973,7 +2170,7 @@ export type CreateArtifactFromPathRequest = Message<"engram.app.v1.CreateArtifac * Use `create(CreateArtifactFromPathRequestSchema)` to create a new message. */ export const CreateArtifactFromPathRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 63); + messageDesc(file_engram_app_v1_session, 68); /** * @generated from message engram.app.v1.CreateArtifactFromPathResponse @@ -2000,7 +2197,7 @@ export type CreateArtifactFromPathResponse = Message<"engram.app.v1.CreateArtifa * Use `create(CreateArtifactFromPathResponseSchema)` to create a new message. */ export const CreateArtifactFromPathResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 64); + messageDesc(file_engram_app_v1_session, 69); /** * Frame kinds mirror the WS frames / host ProxyShellMessage 1:1. Request @@ -2059,7 +2256,7 @@ export type RelayShellRequest = Message<"engram.app.v1.RelayShellRequest"> & { * Use `create(RelayShellRequestSchema)` to create a new message. */ export const RelayShellRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 65); + messageDesc(file_engram_app_v1_session, 70); /** * @generated from message engram.app.v1.RelayShellResponse @@ -2106,7 +2303,7 @@ export type RelayShellResponse = Message<"engram.app.v1.RelayShellResponse"> & { * Use `create(RelayShellResponseSchema)` to create a new message. */ export const RelayShellResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 66); + messageDesc(file_engram_app_v1_session, 71); /** * @generated from message engram.app.v1.ShellOpen @@ -2123,7 +2320,7 @@ export type ShellOpen = Message<"engram.app.v1.ShellOpen"> & { * Use `create(ShellOpenSchema)` to create a new message. */ export const ShellOpenSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 67); + messageDesc(file_engram_app_v1_session, 72); /** * @generated from message engram.app.v1.ShellClose @@ -2145,7 +2342,7 @@ export type ShellClose = Message<"engram.app.v1.ShellClose"> & { * Use `create(ShellCloseSchema)` to create a new message. */ export const ShellCloseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 68); + messageDesc(file_engram_app_v1_session, 73); /** * Raw opaque bytes; no frame-kind taxonomy (cf. RelayShell*). Request @@ -2186,7 +2383,7 @@ export type RelayPortRequest = Message<"engram.app.v1.RelayPortRequest"> & { * Use `create(RelayPortRequestSchema)` to create a new message. */ export const RelayPortRequestSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 69); + messageDesc(file_engram_app_v1_session, 74); /** * @generated from message engram.app.v1.RelayPortResponse @@ -2215,7 +2412,7 @@ export type RelayPortResponse = Message<"engram.app.v1.RelayPortResponse"> & { * Use `create(RelayPortResponseSchema)` to create a new message. */ export const RelayPortResponseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 70); + messageDesc(file_engram_app_v1_session, 75); /** * @generated from message engram.app.v1.PortOpen @@ -2239,7 +2436,7 @@ export type PortOpen = Message<"engram.app.v1.PortOpen"> & { * Use `create(PortOpenSchema)` to create a new message. */ export const PortOpenSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 71); + messageDesc(file_engram_app_v1_session, 76); /** * Clean teardown sentinel (see ProxyPortClose in host_service.proto). @@ -2254,7 +2451,7 @@ export type PortClose = Message<"engram.app.v1.PortClose"> & { * Use `create(PortCloseSchema)` to create a new message. */ export const PortCloseSchema: GenMessage = /*@__PURE__*/ - messageDesc(file_engram_app_v1_session, 72); + messageDesc(file_engram_app_v1_session, 77); /** * The orchestrator-facing app contract (ADR 0051 §2.3, rev 2026-06-10). @@ -2347,6 +2544,14 @@ export const SessionService: GenService<{ input: typeof CompleteToolCallRequestSchema; output: typeof CompleteToolCallResponseSchema; }, + /** + * @generated from rpc engram.app.v1.SessionService.GetMcpWritePolicy + */ + getMcpWritePolicy: { + methodKind: "unary"; + input: typeof GetMcpWritePolicyRequestSchema; + output: typeof GetMcpWritePolicyResponseSchema; + }, /** * Streams the event envelope itself, not a wrapped Response message. * buf:lint:ignore RPC_RESPONSE_STANDARD_NAME diff --git a/orchestrator/src/gen/engram/mcp/v1/control_pb.ts b/orchestrator/src/gen/engram/mcp/v1/control_pb.ts new file mode 100644 index 000000000..31061b23b --- /dev/null +++ b/orchestrator/src/gen/engram/mcp/v1/control_pb.ts @@ -0,0 +1,912 @@ +// @generated by protoc-gen-es v2.2.5 with parameter "target=ts" +// @generated from file engram/mcp/v1/control.proto (package engram.mcp.v1, syntax proto3) +/* eslint-disable */ + +import type { GenEnum, GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv1"; +import { enumDesc, fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv1"; +import type { Message } from "@bufbuild/protobuf"; + +/** + * Describes the file engram/mcp/v1/control.proto. + */ +export const file_engram_mcp_v1_control: GenFile = /*@__PURE__*/ + fileDesc("ChtlbmdyYW0vbWNwL3YxL2NvbnRyb2wucHJvdG8SDWVuZ3JhbS5tY3AudjEiMAoLRGVzdGluYXRpb24SDwoHem9uZV9pZBgBIAEoCRIQCghlbmRwb2ludBgCIAEoCSKYAQoNQ2hhbmdlQ29udGV4dBIVCg1jb25uZWN0aW9uX2lkGAEgASgJEhcKD2NvbmZpZ19yZXZpc2lvbhgCIAEoAxIpCiFwcmV2aW91c19hdXRob3JpemF0aW9uX2dlbmVyYXRpb24YAyABKAMSLAoIcmVzb3VyY2UYBCABKAsyGi5lbmdyYW0ubWNwLnYxLkRlc3RpbmF0aW9uIt8BChBBdXRob3JpdHlDb250ZXh0EhUKDWNvbm5lY3Rpb25faWQYASABKAkSFwoPY29uZmlnX3JldmlzaW9uGAIgASgDEiAKGGF1dGhvcml6YXRpb25fZ2VuZXJhdGlvbhgDIAEoAxIbChNjcmVkZW50aWFsX2lkZW50aXR5GAQgASgJEiwKCHJlc291cmNlGAUgASgLMhouZW5ncmFtLm1jcC52MS5EZXN0aW5hdGlvbhIVCg1wb2xpY3lfZGlnZXN0GAYgASgJEhcKD3BvbGljeV9yZXZpc2lvbhgHIAEoCSKUAQoTQ3JlZGVudGlhbFJlZmVyZW5jZRIrCgRraW5kGAEgASgOMh0uZW5ncmFtLm1jcC52MS5DcmVkZW50aWFsS2luZBIMCgRuYW1lGAIgASgJEhUKDWNvbm5lY3Rpb25faWQYAyABKAkSEgoKZ2VuZXJhdGlvbhgEIAEoAxIXCg9yZWdpc3RyYXRpb25faWQYBSABKAkirgEKEENyZWRlbnRpYWxTdGF0dXMSEAoIcHJvdG9jb2wYASABKA0SMAoHY29udGV4dBgCIAEoCzIfLmVuZ3JhbS5tY3AudjEuQXV0aG9yaXR5Q29udGV4dBI1CglyZWZlcmVuY2UYAyABKAsyIi5lbmdyYW0ubWNwLnYxLkNyZWRlbnRpYWxSZWZlcmVuY2USDwoHdmVyc2lvbhgEIAEoAxIOCgZzdGF0dXMYBSABKAki5wEKEFB1dFNlY3JldFJlcXVlc3QSEAoIcHJvdG9jb2wYASABKA0SLQoHY29udGV4dBgCIAEoCzIcLmVuZ3JhbS5tY3AudjEuQ2hhbmdlQ29udGV4dBIrCgRraW5kGAMgASgOMh0uZW5ncmFtLm1jcC52MS5DcmVkZW50aWFsS2luZBI0CghleHBlY3RlZBgEIAEoCzIiLmVuZ3JhbS5tY3AudjEuQ3JlZGVudGlhbFJlZmVyZW5jZRIOCgZoZWFkZXIYBSABKAkSDQoFdmFsdWUYBiABKAkSEAoIYWN0b3JfaWQYByABKAkijwIKGUJlZ2luQXV0aG9yaXphdGlvblJlcXVlc3QSEAoIcHJvdG9jb2wYASABKA0SLQoHY29udGV4dBgCIAEoCzIcLmVuZ3JhbS5tY3AudjEuQ2hhbmdlQ29udGV4dBIXCg9yZWdpc3RyYXRpb25faWQYAyABKAkSOQoNY2xpZW50X3NlY3JldBgEIAEoCzIiLmVuZ3JhbS5tY3AudjEuQ3JlZGVudGlhbFJlZmVyZW5jZRIdChBleHBlY3RlZF92ZXJzaW9uGAUgASgDSACIAQESEAoIYWN0b3JfaWQYBiABKAkSFwoPc2hhcmluZ19jb25zZW50GAcgASgIQhMKEV9leHBlY3RlZF92ZXJzaW9uIqABChpCZWdpbkF1dGhvcml6YXRpb25SZXNwb25zZRIQCghwcm90b2NvbBgBIAEoDRIPCgdmbG93X2lkGAIgASgJEjAKB2NvbnRleHQYAyABKAsyHy5lbmdyYW0ubWNwLnYxLkF1dGhvcml0eUNvbnRleHQSGQoRYXV0aG9yaXphdGlvbl91cmwYBCABKAkSEgoKZXhwaXJlc19hdBgFIAEoAyLxAQocQ29tcGxldGVBdXRob3JpemF0aW9uUmVxdWVzdBIQCghwcm90b2NvbBgBIAEoDRIwCgdjb250ZXh0GAIgASgLMh8uZW5ncmFtLm1jcC52MS5BdXRob3JpdHlDb250ZXh0EhcKD3JlZ2lzdHJhdGlvbl9pZBgDIAEoCRIPCgdmbG93X2lkGAQgASgJEg0KBXN0YXRlGAUgASgJEhQKDHJlZGlyZWN0X3VyaRgGIAEoCRITCgZpc3N1ZXIYByABKAlIAIgBARIMCgRjb2RlGAggASgJEhAKCGFjdG9yX2lkGAkgASgJQgkKB19pc3N1ZXIicgoQR2V0U3RhdHVzUmVxdWVzdBIQCghwcm90b2NvbBgBIAEoDRIVCg1jb25uZWN0aW9uX2lkGAIgASgJEjUKCXJlZmVyZW5jZRgDIAEoCzIiLmVuZ3JhbS5tY3AudjEuQ3JlZGVudGlhbFJlZmVyZW5jZSJmCg1SZXZva2VSZXF1ZXN0EhAKCHByb3RvY29sGAEgASgNEjEKCGV4cGVjdGVkGAIgASgLMh8uZW5ncmFtLm1jcC52MS5DcmVkZW50aWFsU3RhdHVzEhAKCGFjdG9yX2lkGAMgASgJInQKDldvcmtlcklkZW50aXR5EhAKCHByb3RvY29sGAEgASgNEg8KB2Jvb3RfaWQYAiABKAkSDwoHem9uZV9pZBgDIAEoCRIVCg1wb2xpY3lfZGlnZXN0GAQgASgJEhcKD3BvbGljeV9yZXZpc2lvbhgFIAEoCSJCCglBZG1pc3Npb24SCgoCaWQYASABKAkSFQoNZGVwbG95bWVudF9pZBgCIAEoCRISCgpleHBpcmVzX2F0GAMgASgDIjsKE0FkbWluaXN0cmF0aW9uU2NvcGUSEgoKcmVxdWVzdF9pZBgBIAEoCRIQCghhY3Rvcl9pZBgCIAEoCSJgCg5FeGVjdXRpb25TY29wZRISCgpzZXNzaW9uX2lkGAEgASgJEg4KBnJ1bl9pZBgCIAEoCRIUCgx0b29sX2NhbGxfaWQYAyABKAkSFAoMYmluZGluZ19oYXNoGAQgASgJItADCgxMZWFzZVJlcXVlc3QSEAoIcHJvdG9jb2wYASABKA0SMAoHY29udGV4dBgCIAEoCzIfLmVuZ3JhbS5tY3AudjEuQXV0aG9yaXR5Q29udGV4dBI1CglyZWZlcmVuY2UYAyABKAsyIi5lbmdyYW0ubWNwLnYxLkNyZWRlbnRpYWxSZWZlcmVuY2USLQoGd29ya2VyGAQgASgLMh0uZW5ncmFtLm1jcC52MS5Xb3JrZXJJZGVudGl0eRIRCglpbnRlbnRfaWQYBSABKAkSFgoOcmVxdWVzdF9kaWdlc3QYBiABKAkSEQoJb3BlcmF0aW9uGAcgASgJEhIKCmV4cGlyZXNfYXQYCCABKAMSHAoUb3BlcmF0aW9uX2V4cGlyZXNfYXQYDCABKAMSKwoJYWRtaXNzaW9uGAkgASgLMhguZW5ncmFtLm1jcC52MS5BZG1pc3Npb24SPAoOYWRtaW5pc3RyYXRpb24YCiABKAsyIi5lbmdyYW0ubWNwLnYxLkFkbWluaXN0cmF0aW9uU2NvcGVIABIyCglleGVjdXRpb24YCyABKAsyHS5lbmdyYW0ubWNwLnYxLkV4ZWN1dGlvblNjb3BlSABCBwoFc2NvcGUihQEKDUxlYXNlUmVzcG9uc2USEAoIcHJvdG9jb2wYASABKA0SDgoGaGVhZGVyGAIgASgJEg0KBXZhbHVlGAMgASgJEhIKCmV4cGlyZXNfYXQYBCABKAMSEQoJaW50ZW50X2lkGAUgASgJEhwKFG9wZXJhdGlvbl9leHBpcmVzX2F0GAYgASgDIoEBChpDYW5jZWxBdXRob3JpemF0aW9uUmVxdWVzdBIQCghwcm90b2NvbBgBIAEoDRIPCgdmbG93X2lkGAIgASgJEhUKDWNvbm5lY3Rpb25faWQYAyABKAkSFwoPcmVnaXN0cmF0aW9uX2lkGAQgASgJEhAKCGFjdG9yX2lkGAUgASgJInAKG0NhbmNlbEF1dGhvcml6YXRpb25SZXNwb25zZRIQCghwcm90b2NvbBgBIAEoDRIPCgdmbG93X2lkGAIgASgJEhUKDWNvbm5lY3Rpb25faWQYAyABKAkSFwoPcmVnaXN0cmF0aW9uX2lkGAQgASgJIkQKEVB1dFNlY3JldFJlc3BvbnNlEi8KBnN0YXR1cxgBIAEoCzIfLmVuZ3JhbS5tY3AudjEuQ3JlZGVudGlhbFN0YXR1cyJQCh1Db21wbGV0ZUF1dGhvcml6YXRpb25SZXNwb25zZRIvCgZzdGF0dXMYASABKAsyHy5lbmdyYW0ubWNwLnYxLkNyZWRlbnRpYWxTdGF0dXMiRAoRR2V0U3RhdHVzUmVzcG9uc2USLwoGc3RhdHVzGAEgASgLMh8uZW5ncmFtLm1jcC52MS5DcmVkZW50aWFsU3RhdHVzIkEKDlJldm9rZVJlc3BvbnNlEi8KBnN0YXR1cxgBIAEoCzIfLmVuZ3JhbS5tY3AudjEuQ3JlZGVudGlhbFN0YXR1cyqMAQoOQ3JlZGVudGlhbEtpbmQSHwobQ1JFREVOVElBTF9LSU5EX1VOU1BFQ0lGSUVEEAASGwoXQ1JFREVOVElBTF9LSU5EX0FQSV9LRVkQARIhCh1DUkVERU5USUFMX0tJTkRfQ0xJRU5UX1NFQ1JFVBACEhkKFUNSRURFTlRJQUxfS0lORF9PQVVUSBADMpIFChhDcmVkZW50aWFsQ29udHJvbFNlcnZpY2USTgoJUHV0U2VjcmV0Eh8uZW5ncmFtLm1jcC52MS5QdXRTZWNyZXRSZXF1ZXN0GiAuZW5ncmFtLm1jcC52MS5QdXRTZWNyZXRSZXNwb25zZRJpChJCZWdpbkF1dGhvcml6YXRpb24SKC5lbmdyYW0ubWNwLnYxLkJlZ2luQXV0aG9yaXphdGlvblJlcXVlc3QaKS5lbmdyYW0ubWNwLnYxLkJlZ2luQXV0aG9yaXphdGlvblJlc3BvbnNlEmwKE0NhbmNlbEF1dGhvcml6YXRpb24SKS5lbmdyYW0ubWNwLnYxLkNhbmNlbEF1dGhvcml6YXRpb25SZXF1ZXN0GiouZW5ncmFtLm1jcC52MS5DYW5jZWxBdXRob3JpemF0aW9uUmVzcG9uc2UScgoVQ29tcGxldGVBdXRob3JpemF0aW9uEisuZW5ncmFtLm1jcC52MS5Db21wbGV0ZUF1dGhvcml6YXRpb25SZXF1ZXN0GiwuZW5ncmFtLm1jcC52MS5Db21wbGV0ZUF1dGhvcml6YXRpb25SZXNwb25zZRJOCglHZXRTdGF0dXMSHy5lbmdyYW0ubWNwLnYxLkdldFN0YXR1c1JlcXVlc3QaIC5lbmdyYW0ubWNwLnYxLkdldFN0YXR1c1Jlc3BvbnNlEkUKBlJldm9rZRIcLmVuZ3JhbS5tY3AudjEuUmV2b2tlUmVxdWVzdBodLmVuZ3JhbS5tY3AudjEuUmV2b2tlUmVzcG9uc2USQgoFTGVhc2USGy5lbmdyYW0ubWNwLnYxLkxlYXNlUmVxdWVzdBocLmVuZ3JhbS5tY3AudjEuTGVhc2VSZXNwb25zZWIGcHJvdG8z"); + +/** + * @generated from message engram.mcp.v1.Destination + */ +export type Destination = Message<"engram.mcp.v1.Destination"> & { + /** + * @generated from field: string zone_id = 1; + */ + zoneId: string; + + /** + * @generated from field: string endpoint = 2; + */ + endpoint: string; +}; + +/** + * Describes the message engram.mcp.v1.Destination. + * Use `create(DestinationSchema)` to create a new message. + */ +export const DestinationSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 0); + +/** + * @generated from message engram.mcp.v1.ChangeContext + */ +export type ChangeContext = Message<"engram.mcp.v1.ChangeContext"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: int64 config_revision = 2; + */ + configRevision: bigint; + + /** + * @generated from field: int64 previous_authorization_generation = 3; + */ + previousAuthorizationGeneration: bigint; + + /** + * @generated from field: engram.mcp.v1.Destination resource = 4; + */ + resource?: Destination; +}; + +/** + * Describes the message engram.mcp.v1.ChangeContext. + * Use `create(ChangeContextSchema)` to create a new message. + */ +export const ChangeContextSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 1); + +/** + * @generated from message engram.mcp.v1.AuthorityContext + */ +export type AuthorityContext = Message<"engram.mcp.v1.AuthorityContext"> & { + /** + * @generated from field: string connection_id = 1; + */ + connectionId: string; + + /** + * @generated from field: int64 config_revision = 2; + */ + configRevision: bigint; + + /** + * @generated from field: int64 authorization_generation = 3; + */ + authorizationGeneration: bigint; + + /** + * @generated from field: string credential_identity = 4; + */ + credentialIdentity: string; + + /** + * @generated from field: engram.mcp.v1.Destination resource = 5; + */ + resource?: Destination; + + /** + * @generated from field: string policy_digest = 6; + */ + policyDigest: string; + + /** + * @generated from field: string policy_revision = 7; + */ + policyRevision: string; +}; + +/** + * Describes the message engram.mcp.v1.AuthorityContext. + * Use `create(AuthorityContextSchema)` to create a new message. + */ +export const AuthorityContextSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 2); + +/** + * @generated from message engram.mcp.v1.CredentialReference + */ +export type CredentialReference = Message<"engram.mcp.v1.CredentialReference"> & { + /** + * @generated from field: engram.mcp.v1.CredentialKind kind = 1; + */ + kind: CredentialKind; + + /** + * @generated from field: string name = 2; + */ + name: string; + + /** + * @generated from field: string connection_id = 3; + */ + connectionId: string; + + /** + * @generated from field: int64 generation = 4; + */ + generation: bigint; + + /** + * @generated from field: string registration_id = 5; + */ + registrationId: string; +}; + +/** + * Describes the message engram.mcp.v1.CredentialReference. + * Use `create(CredentialReferenceSchema)` to create a new message. + */ +export const CredentialReferenceSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 3); + +/** + * @generated from message engram.mcp.v1.CredentialStatus + */ +export type CredentialStatus = Message<"engram.mcp.v1.CredentialStatus"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.AuthorityContext context = 2; + */ + context?: AuthorityContext; + + /** + * @generated from field: engram.mcp.v1.CredentialReference reference = 3; + */ + reference?: CredentialReference; + + /** + * @generated from field: int64 version = 4; + */ + version: bigint; + + /** + * @generated from field: string status = 5; + */ + status: string; +}; + +/** + * Describes the message engram.mcp.v1.CredentialStatus. + * Use `create(CredentialStatusSchema)` to create a new message. + */ +export const CredentialStatusSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 4); + +/** + * @generated from message engram.mcp.v1.PutSecretRequest + */ +export type PutSecretRequest = Message<"engram.mcp.v1.PutSecretRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ChangeContext context = 2; + */ + context?: ChangeContext; + + /** + * @generated from field: engram.mcp.v1.CredentialKind kind = 3; + */ + kind: CredentialKind; + + /** + * @generated from field: engram.mcp.v1.CredentialReference expected = 4; + */ + expected?: CredentialReference; + + /** + * @generated from field: string header = 5; + */ + header: string; + + /** + * @generated from field: string value = 6; + */ + value: string; + + /** + * @generated from field: string actor_id = 7; + */ + actorId: string; +}; + +/** + * Describes the message engram.mcp.v1.PutSecretRequest. + * Use `create(PutSecretRequestSchema)` to create a new message. + */ +export const PutSecretRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 5); + +/** + * @generated from message engram.mcp.v1.BeginAuthorizationRequest + */ +export type BeginAuthorizationRequest = Message<"engram.mcp.v1.BeginAuthorizationRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ChangeContext context = 2; + */ + context?: ChangeContext; + + /** + * @generated from field: string registration_id = 3; + */ + registrationId: string; + + /** + * @generated from field: engram.mcp.v1.CredentialReference client_secret = 4; + */ + clientSecret?: CredentialReference; + + /** + * @generated from field: optional int64 expected_version = 5; + */ + expectedVersion?: bigint; + + /** + * @generated from field: string actor_id = 6; + */ + actorId: string; + + /** + * @generated from field: bool sharing_consent = 7; + */ + sharingConsent: boolean; +}; + +/** + * Describes the message engram.mcp.v1.BeginAuthorizationRequest. + * Use `create(BeginAuthorizationRequestSchema)` to create a new message. + */ +export const BeginAuthorizationRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 6); + +/** + * @generated from message engram.mcp.v1.BeginAuthorizationResponse + */ +export type BeginAuthorizationResponse = Message<"engram.mcp.v1.BeginAuthorizationResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string flow_id = 2; + */ + flowId: string; + + /** + * @generated from field: engram.mcp.v1.AuthorityContext context = 3; + */ + context?: AuthorityContext; + + /** + * @generated from field: string authorization_url = 4; + */ + authorizationUrl: string; + + /** + * @generated from field: int64 expires_at = 5; + */ + expiresAt: bigint; +}; + +/** + * Describes the message engram.mcp.v1.BeginAuthorizationResponse. + * Use `create(BeginAuthorizationResponseSchema)` to create a new message. + */ +export const BeginAuthorizationResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 7); + +/** + * @generated from message engram.mcp.v1.CompleteAuthorizationRequest + */ +export type CompleteAuthorizationRequest = Message<"engram.mcp.v1.CompleteAuthorizationRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.AuthorityContext context = 2; + */ + context?: AuthorityContext; + + /** + * @generated from field: string registration_id = 3; + */ + registrationId: string; + + /** + * @generated from field: string flow_id = 4; + */ + flowId: string; + + /** + * @generated from field: string state = 5; + */ + state: string; + + /** + * @generated from field: string redirect_uri = 6; + */ + redirectUri: string; + + /** + * @generated from field: optional string issuer = 7; + */ + issuer?: string; + + /** + * @generated from field: string code = 8; + */ + code: string; + + /** + * @generated from field: string actor_id = 9; + */ + actorId: string; +}; + +/** + * Describes the message engram.mcp.v1.CompleteAuthorizationRequest. + * Use `create(CompleteAuthorizationRequestSchema)` to create a new message. + */ +export const CompleteAuthorizationRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 8); + +/** + * @generated from message engram.mcp.v1.GetStatusRequest + */ +export type GetStatusRequest = Message<"engram.mcp.v1.GetStatusRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string connection_id = 2; + */ + connectionId: string; + + /** + * @generated from field: engram.mcp.v1.CredentialReference reference = 3; + */ + reference?: CredentialReference; +}; + +/** + * Describes the message engram.mcp.v1.GetStatusRequest. + * Use `create(GetStatusRequestSchema)` to create a new message. + */ +export const GetStatusRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 9); + +/** + * @generated from message engram.mcp.v1.RevokeRequest + */ +export type RevokeRequest = Message<"engram.mcp.v1.RevokeRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.CredentialStatus expected = 2; + */ + expected?: CredentialStatus; + + /** + * @generated from field: string actor_id = 3; + */ + actorId: string; +}; + +/** + * Describes the message engram.mcp.v1.RevokeRequest. + * Use `create(RevokeRequestSchema)` to create a new message. + */ +export const RevokeRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 10); + +/** + * @generated from message engram.mcp.v1.WorkerIdentity + */ +export type WorkerIdentity = Message<"engram.mcp.v1.WorkerIdentity"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string boot_id = 2; + */ + bootId: string; + + /** + * @generated from field: string zone_id = 3; + */ + zoneId: string; + + /** + * @generated from field: string policy_digest = 4; + */ + policyDigest: string; + + /** + * @generated from field: string policy_revision = 5; + */ + policyRevision: string; +}; + +/** + * Describes the message engram.mcp.v1.WorkerIdentity. + * Use `create(WorkerIdentitySchema)` to create a new message. + */ +export const WorkerIdentitySchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 11); + +/** + * @generated from message engram.mcp.v1.Admission + */ +export type Admission = Message<"engram.mcp.v1.Admission"> & { + /** + * @generated from field: string id = 1; + */ + id: string; + + /** + * @generated from field: string deployment_id = 2; + */ + deploymentId: string; + + /** + * @generated from field: int64 expires_at = 3; + */ + expiresAt: bigint; +}; + +/** + * Describes the message engram.mcp.v1.Admission. + * Use `create(AdmissionSchema)` to create a new message. + */ +export const AdmissionSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 12); + +/** + * @generated from message engram.mcp.v1.AdministrationScope + */ +export type AdministrationScope = Message<"engram.mcp.v1.AdministrationScope"> & { + /** + * @generated from field: string request_id = 1; + */ + requestId: string; + + /** + * @generated from field: string actor_id = 2; + */ + actorId: string; +}; + +/** + * Describes the message engram.mcp.v1.AdministrationScope. + * Use `create(AdministrationScopeSchema)` to create a new message. + */ +export const AdministrationScopeSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 13); + +/** + * @generated from message engram.mcp.v1.ExecutionScope + */ +export type ExecutionScope = Message<"engram.mcp.v1.ExecutionScope"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string run_id = 2; + */ + runId: string; + + /** + * @generated from field: string tool_call_id = 3; + */ + toolCallId: string; + + /** + * @generated from field: string binding_hash = 4; + */ + bindingHash: string; +}; + +/** + * Describes the message engram.mcp.v1.ExecutionScope. + * Use `create(ExecutionScopeSchema)` to create a new message. + */ +export const ExecutionScopeSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 14); + +/** + * @generated from message engram.mcp.v1.LeaseRequest + */ +export type LeaseRequest = Message<"engram.mcp.v1.LeaseRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.AuthorityContext context = 2; + */ + context?: AuthorityContext; + + /** + * @generated from field: engram.mcp.v1.CredentialReference reference = 3; + */ + reference?: CredentialReference; + + /** + * @generated from field: engram.mcp.v1.WorkerIdentity worker = 4; + */ + worker?: WorkerIdentity; + + /** + * @generated from field: string intent_id = 5; + */ + intentId: string; + + /** + * @generated from field: string request_digest = 6; + */ + requestDigest: string; + + /** + * @generated from field: string operation = 7; + */ + operation: string; + + /** + * Short credential delivery deadline, at most ten seconds. + * + * @generated from field: int64 expires_at = 8; + */ + expiresAt: bigint; + + /** + * Frozen worker operation deadline, at most five minutes. + * + * @generated from field: int64 operation_expires_at = 12; + */ + operationExpiresAt: bigint; + + /** + * @generated from field: engram.mcp.v1.Admission admission = 9; + */ + admission?: Admission; + + /** + * @generated from oneof engram.mcp.v1.LeaseRequest.scope + */ + scope: { + /** + * @generated from field: engram.mcp.v1.AdministrationScope administration = 10; + */ + value: AdministrationScope; + case: "administration"; + } | { + /** + * @generated from field: engram.mcp.v1.ExecutionScope execution = 11; + */ + value: ExecutionScope; + case: "execution"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message engram.mcp.v1.LeaseRequest. + * Use `create(LeaseRequestSchema)` to create a new message. + */ +export const LeaseRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 15); + +/** + * Transient operation memory only. Never queue, cache, persist, log, or return + * this message to a browser or guest. Worker dispatch consumes the bound intent. + * + * @generated from message engram.mcp.v1.LeaseResponse + */ +export type LeaseResponse = Message<"engram.mcp.v1.LeaseResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string header = 2; + */ + header: string; + + /** + * @generated from field: string value = 3; + */ + value: string; + + /** + * @generated from field: int64 expires_at = 4; + */ + expiresAt: bigint; + + /** + * @generated from field: string intent_id = 5; + */ + intentId: string; + + /** + * @generated from field: int64 operation_expires_at = 6; + */ + operationExpiresAt: bigint; +}; + +/** + * Describes the message engram.mcp.v1.LeaseResponse. + * Use `create(LeaseResponseSchema)` to create a new message. + */ +export const LeaseResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 16); + +/** + * Exact attempt only. Cancellation persists in the existing flow row. + * + * @generated from message engram.mcp.v1.CancelAuthorizationRequest + */ +export type CancelAuthorizationRequest = Message<"engram.mcp.v1.CancelAuthorizationRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string flow_id = 2; + */ + flowId: string; + + /** + * @generated from field: string connection_id = 3; + */ + connectionId: string; + + /** + * @generated from field: string registration_id = 4; + */ + registrationId: string; + + /** + * @generated from field: string actor_id = 5; + */ + actorId: string; +}; + +/** + * Describes the message engram.mcp.v1.CancelAuthorizationRequest. + * Use `create(CancelAuthorizationRequestSchema)` to create a new message. + */ +export const CancelAuthorizationRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 17); + +/** + * @generated from message engram.mcp.v1.CancelAuthorizationResponse + */ +export type CancelAuthorizationResponse = Message<"engram.mcp.v1.CancelAuthorizationResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string flow_id = 2; + */ + flowId: string; + + /** + * @generated from field: string connection_id = 3; + */ + connectionId: string; + + /** + * @generated from field: string registration_id = 4; + */ + registrationId: string; +}; + +/** + * Describes the message engram.mcp.v1.CancelAuthorizationResponse. + * Use `create(CancelAuthorizationResponseSchema)` to create a new message. + */ +export const CancelAuthorizationResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 18); + +/** + * @generated from message engram.mcp.v1.PutSecretResponse + */ +export type PutSecretResponse = Message<"engram.mcp.v1.PutSecretResponse"> & { + /** + * @generated from field: engram.mcp.v1.CredentialStatus status = 1; + */ + status?: CredentialStatus; +}; + +/** + * Describes the message engram.mcp.v1.PutSecretResponse. + * Use `create(PutSecretResponseSchema)` to create a new message. + */ +export const PutSecretResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 19); + +/** + * @generated from message engram.mcp.v1.CompleteAuthorizationResponse + */ +export type CompleteAuthorizationResponse = Message<"engram.mcp.v1.CompleteAuthorizationResponse"> & { + /** + * @generated from field: engram.mcp.v1.CredentialStatus status = 1; + */ + status?: CredentialStatus; +}; + +/** + * Describes the message engram.mcp.v1.CompleteAuthorizationResponse. + * Use `create(CompleteAuthorizationResponseSchema)` to create a new message. + */ +export const CompleteAuthorizationResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 20); + +/** + * @generated from message engram.mcp.v1.GetStatusResponse + */ +export type GetStatusResponse = Message<"engram.mcp.v1.GetStatusResponse"> & { + /** + * @generated from field: engram.mcp.v1.CredentialStatus status = 1; + */ + status?: CredentialStatus; +}; + +/** + * Describes the message engram.mcp.v1.GetStatusResponse. + * Use `create(GetStatusResponseSchema)` to create a new message. + */ +export const GetStatusResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 21); + +/** + * @generated from message engram.mcp.v1.RevokeResponse + */ +export type RevokeResponse = Message<"engram.mcp.v1.RevokeResponse"> & { + /** + * @generated from field: engram.mcp.v1.CredentialStatus status = 1; + */ + status?: CredentialStatus; +}; + +/** + * Describes the message engram.mcp.v1.RevokeResponse. + * Use `create(RevokeResponseSchema)` to create a new message. + */ +export const RevokeResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_control, 22); + +/** + * @generated from enum engram.mcp.v1.CredentialKind + */ +export enum CredentialKind { + /** + * @generated from enum value: CREDENTIAL_KIND_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: CREDENTIAL_KIND_API_KEY = 1; + */ + API_KEY = 1, + + /** + * @generated from enum value: CREDENTIAL_KIND_CLIENT_SECRET = 2; + */ + CLIENT_SECRET = 2, + + /** + * @generated from enum value: CREDENTIAL_KIND_OAUTH = 3; + */ + OAUTH = 3, +} + +/** + * Describes the enum engram.mcp.v1.CredentialKind. + */ +export const CredentialKindSchema: GenEnum = /*@__PURE__*/ + enumDesc(file_engram_mcp_v1_control, 0); + +/** + * Internal protocol 1. Served only on the separate mTLS listener. + * + * @generated from service engram.mcp.v1.CredentialControlService + */ +export const CredentialControlService: GenService<{ + /** + * @generated from rpc engram.mcp.v1.CredentialControlService.PutSecret + */ + putSecret: { + methodKind: "unary"; + input: typeof PutSecretRequestSchema; + output: typeof PutSecretResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.CredentialControlService.BeginAuthorization + */ + beginAuthorization: { + methodKind: "unary"; + input: typeof BeginAuthorizationRequestSchema; + output: typeof BeginAuthorizationResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.CredentialControlService.CancelAuthorization + */ + cancelAuthorization: { + methodKind: "unary"; + input: typeof CancelAuthorizationRequestSchema; + output: typeof CancelAuthorizationResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.CredentialControlService.CompleteAuthorization + */ + completeAuthorization: { + methodKind: "unary"; + input: typeof CompleteAuthorizationRequestSchema; + output: typeof CompleteAuthorizationResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.CredentialControlService.GetStatus + */ + getStatus: { + methodKind: "unary"; + input: typeof GetStatusRequestSchema; + output: typeof GetStatusResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.CredentialControlService.Revoke + */ + revoke: { + methodKind: "unary"; + input: typeof RevokeRequestSchema; + output: typeof RevokeResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.CredentialControlService.Lease + */ + lease: { + methodKind: "unary"; + input: typeof LeaseRequestSchema; + output: typeof LeaseResponseSchema; + }, +}> = /*@__PURE__*/ + serviceDesc(file_engram_mcp_v1_control, 0); + diff --git a/orchestrator/src/gen/engram/mcp/v1/operator_pb.ts b/orchestrator/src/gen/engram/mcp/v1/operator_pb.ts new file mode 100644 index 000000000..b20d56761 --- /dev/null +++ b/orchestrator/src/gen/engram/mcp/v1/operator_pb.ts @@ -0,0 +1,127 @@ +// @generated by protoc-gen-es v2.2.5 with parameter "target=ts" +// @generated from file engram/mcp/v1/operator.proto (package engram.mcp.v1, syntax proto3) +/* eslint-disable */ + +import type { GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv1"; +import { fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv1"; +import type { Message } from "@bufbuild/protobuf"; + +/** + * Describes the file engram/mcp/v1/operator.proto. + */ +export const file_engram_mcp_v1_operator: GenFile = /*@__PURE__*/ + fileDesc("ChxlbmdyYW0vbWNwL3YxL29wZXJhdG9yLnByb3RvEg1lbmdyYW0ubWNwLnYxIpsBCiZPcGVyYXRvclB1YmxpY2F0aW9uU2VydmljZVN0YWdlUmVxdWVzdBIQCghwcm90b2NvbBgBIAEoDRIgChNleHBlY3RlZF9nZW5lcmF0aW9uGAIgASgESACIAQESJQodY29uZmlndXJlZF9jYW5kaWRhdGVfaWRlbnRpdHkYAyABKAlCFgoUX2V4cGVjdGVkX2dlbmVyYXRpb24iOQolT3BlcmF0b3JQdWJsaWNhdGlvblNlcnZpY2VSZWFkUmVxdWVzdBIQCghwcm90b2NvbBgBIAEoDSJVCidPcGVyYXRvclB1YmxpY2F0aW9uU2VydmljZVN0YWdlUmVzcG9uc2USEAoIcHJvdG9jb2wYASABKA0SGAoQcHVibGljYXRpb25fanNvbhgCIAEoDCJUCiZPcGVyYXRvclB1YmxpY2F0aW9uU2VydmljZVJlYWRSZXNwb25zZRIQCghwcm90b2NvbBgBIAEoDRIYChBwdWJsaWNhdGlvbl9qc29uGAIgASgMMokCChpPcGVyYXRvclB1YmxpY2F0aW9uU2VydmljZRJ2CgVTdGFnZRI1LmVuZ3JhbS5tY3AudjEuT3BlcmF0b3JQdWJsaWNhdGlvblNlcnZpY2VTdGFnZVJlcXVlc3QaNi5lbmdyYW0ubWNwLnYxLk9wZXJhdG9yUHVibGljYXRpb25TZXJ2aWNlU3RhZ2VSZXNwb25zZRJzCgRSZWFkEjQuZW5ncmFtLm1jcC52MS5PcGVyYXRvclB1YmxpY2F0aW9uU2VydmljZVJlYWRSZXF1ZXN0GjUuZW5ncmFtLm1jcC52MS5PcGVyYXRvclB1YmxpY2F0aW9uU2VydmljZVJlYWRSZXNwb25zZWIGcHJvdG8z"); + +/** + * @generated from message engram.mcp.v1.OperatorPublicationServiceStageRequest + */ +export type OperatorPublicationServiceStageRequest = Message<"engram.mcp.v1.OperatorPublicationServiceStageRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: optional uint64 expected_generation = 2; + */ + expectedGeneration?: bigint; + + /** + * @generated from field: string configured_candidate_identity = 3; + */ + configuredCandidateIdentity: string; +}; + +/** + * Describes the message engram.mcp.v1.OperatorPublicationServiceStageRequest. + * Use `create(OperatorPublicationServiceStageRequestSchema)` to create a new message. + */ +export const OperatorPublicationServiceStageRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_operator, 0); + +/** + * @generated from message engram.mcp.v1.OperatorPublicationServiceReadRequest + */ +export type OperatorPublicationServiceReadRequest = Message<"engram.mcp.v1.OperatorPublicationServiceReadRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; +}; + +/** + * Describes the message engram.mcp.v1.OperatorPublicationServiceReadRequest. + * Use `create(OperatorPublicationServiceReadRequestSchema)` to create a new message. + */ +export const OperatorPublicationServiceReadRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_operator, 1); + +/** + * @generated from message engram.mcp.v1.OperatorPublicationServiceStageResponse + */ +export type OperatorPublicationServiceStageResponse = Message<"engram.mcp.v1.OperatorPublicationServiceStageResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: bytes publication_json = 2; + */ + publicationJson: Uint8Array; +}; + +/** + * Describes the message engram.mcp.v1.OperatorPublicationServiceStageResponse. + * Use `create(OperatorPublicationServiceStageResponseSchema)` to create a new message. + */ +export const OperatorPublicationServiceStageResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_operator, 2); + +/** + * @generated from message engram.mcp.v1.OperatorPublicationServiceReadResponse + */ +export type OperatorPublicationServiceReadResponse = Message<"engram.mcp.v1.OperatorPublicationServiceReadResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: bytes publication_json = 2; + */ + publicationJson: Uint8Array; +}; + +/** + * Describes the message engram.mcp.v1.OperatorPublicationServiceReadResponse. + * Use `create(OperatorPublicationServiceReadResponseSchema)` to create a new message. + */ +export const OperatorPublicationServiceReadResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_operator, 3); + +/** + * Only the dedicated operator mTLS listener registers this mutation service. + * + * @generated from service engram.mcp.v1.OperatorPublicationService + */ +export const OperatorPublicationService: GenService<{ + /** + * @generated from rpc engram.mcp.v1.OperatorPublicationService.Stage + */ + stage: { + methodKind: "unary"; + input: typeof OperatorPublicationServiceStageRequestSchema; + output: typeof OperatorPublicationServiceStageResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.OperatorPublicationService.Read + */ + read: { + methodKind: "unary"; + input: typeof OperatorPublicationServiceReadRequestSchema; + output: typeof OperatorPublicationServiceReadResponseSchema; + }, +}> = /*@__PURE__*/ + serviceDesc(file_engram_mcp_v1_operator, 0); + diff --git a/orchestrator/src/gen/engram/mcp/v1/session_policy_pb.ts b/orchestrator/src/gen/engram/mcp/v1/session_policy_pb.ts new file mode 100644 index 000000000..88ccb6895 --- /dev/null +++ b/orchestrator/src/gen/engram/mcp/v1/session_policy_pb.ts @@ -0,0 +1,1260 @@ +// @generated by protoc-gen-es v2.2.5 with parameter "target=ts" +// @generated from file engram/mcp/v1/session_policy.proto (package engram.mcp.v1, syntax proto3) +/* eslint-disable */ + +import type { GenEnum, GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv1"; +import { enumDesc, fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv1"; +import type { CreateSessionRequest, CreateSessionResponse, GetMcpWritePolicyResponse, McpPlanReview, McpTrustedWritePolicy, McpWritePolicyRef } from "../../app/v1/session_pb"; +import { file_engram_app_v1_session } from "../../app/v1/session_pb"; +import type { Message } from "@bufbuild/protobuf"; + +/** + * Describes the file engram/mcp/v1/session_policy.proto. + */ +export const file_engram_mcp_v1_session_policy: GenFile = /*@__PURE__*/ + fileDesc("CiJlbmdyYW0vbWNwL3YxL3Nlc3Npb25fcG9saWN5LnByb3RvEg1lbmdyYW0ubWNwLnYxIr0BChpDcmVhdGVQb2xpY3lTZXNzaW9uUmVxdWVzdBIQCghwcm90b2NvbBgBIAEoDRIzCgZjcmVhdGUYAiABKAsyIy5lbmdyYW0uYXBwLnYxLkNyZWF0ZVNlc3Npb25SZXF1ZXN0EhAKCGFjdG9yX2lkGAMgASgJEhAKCGlucHV0X2lkGAQgASgJEh4KEXBhcmVudF9zZXNzaW9uX2lkGAUgASgJSACIAQFCFAoSX3BhcmVudF9zZXNzaW9uX2lkInQKFlJlYWRXcml0ZVBvbGljeVJlcXVlc3QSEAoIcHJvdG9jb2wYASABKA0SEgoKc2Vzc2lvbl9pZBgCIAEoCRIeChFwbGFuX3Rvb2xfY2FsbF9pZBgDIAEoCUgAiAEBQhQKEl9wbGFuX3Rvb2xfY2FsbF9pZCKAAQoVQWNjZXB0UGxhbkV4aXRSZXF1ZXN0EhAKCHByb3RvY29sGAEgASgNEi4KCGV4cGVjdGVkGAIgASgLMhwuZW5ncmFtLmFwcC52MS5NY3BQbGFuUmV2aWV3EhAKCGFjdG9yX2lkGAMgASgJEhMKC3Jlc3VsdF9qc29uGAQgASgJIsICChxWYWxpZGF0ZURpc3BhdGNoRmVuY2VSZXF1ZXN0EhAKCHByb3RvY29sGAEgASgNEhIKCnNlc3Npb25faWQYAiABKAkSDgoGcnVuX2lkGAMgASgJEhQKDHRvb2xfY2FsbF9pZBgEIAEoCRI9ChNhY2NlcHRlZF9wb2xpY3lfcmVmGAUgASgLMiAuZW5ncmFtLmFwcC52MS5NY3BXcml0ZVBvbGljeVJlZhIiChphY2NlcHRlZF93cml0ZV9lbGlnaWJpbGl0eRgGIAEoCRI+ChRhbmNlc3Rvcl9wb2xpY3lfcmVmcxgHIAMoCzIgLmVuZ3JhbS5hcHAudjEuTWNwV3JpdGVQb2xpY3lSZWYSFwoPbW9kZWxfdG9vbF9uYW1lGAggASgJEhoKEm9yaWdpbmFsX2FyZ3VtZW50cxgJIAEoCSJBCh1WYWxpZGF0ZURpc3BhdGNoRmVuY2VSZXNwb25zZRIQCghwcm90b2NvbBgBIAEoDRIOCgZzdGF0dXMYAiABKAkiVAobQ3JlYXRlUG9saWN5U2Vzc2lvblJlc3BvbnNlEjUKB3Nlc3Npb24YASABKAsyJC5lbmdyYW0uYXBwLnYxLkNyZWF0ZVNlc3Npb25SZXNwb25zZSJSChdSZWFkV3JpdGVQb2xpY3lSZXNwb25zZRI3CgVzdGF0ZRgBIAEoCzIoLmVuZ3JhbS5hcHAudjEuR2V0TWNwV3JpdGVQb2xpY3lSZXNwb25zZSJOChZBY2NlcHRQbGFuRXhpdFJlc3BvbnNlEjQKBnBvbGljeRgBIAEoCzIkLmVuZ3JhbS5hcHAudjEuTWNwVHJ1c3RlZFdyaXRlUG9saWN5IsYCCiBWYWxpZGF0ZUV4ZWN1dGlvbklkZW50aXR5UmVxdWVzdBIQCghwcm90b2NvbBgBIAEoDRISCgpzZXNzaW9uX2lkGAIgASgJEg4KBnJ1bl9pZBgDIAEoCRIUCgx0b29sX2NhbGxfaWQYBCABKAkSPQoTYWNjZXB0ZWRfcG9saWN5X3JlZhgFIAEoCzIgLmVuZ3JhbS5hcHAudjEuTWNwV3JpdGVQb2xpY3lSZWYSIgoaYWNjZXB0ZWRfd3JpdGVfZWxpZ2liaWxpdHkYBiABKAkSPgoUYW5jZXN0b3JfcG9saWN5X3JlZnMYByADKAsyIC5lbmdyYW0uYXBwLnYxLk1jcFdyaXRlUG9saWN5UmVmEhcKD21vZGVsX3Rvb2xfbmFtZRgIIAEoCRIaChJvcmlnaW5hbF9hcmd1bWVudHMYCSABKAkiRQohVmFsaWRhdGVFeGVjdXRpb25JZGVudGl0eVJlc3BvbnNlEhAKCHByb3RvY29sGAEgASgNEg4KBnN0YXR1cxgCIAEoCSJlChNNYW5hZ2VkUG9saWN5VGFyZ2V0EhIKCnNlc3Npb25faWQYASABKAkSEgoKc2FuZGJveF9pZBgCIAEoCRIPCgdob3N0X2lkGAMgASgJEhUKDWJpbmRpbmdfZXBvY2gYBCABKAQi4AEKFE1hbmFnZWRQb2xpY3lDb250ZXh0EjIKBnRhcmdldBgBIAEoCzIiLmVuZ3JhbS5tY3AudjEuTWFuYWdlZFBvbGljeVRhcmdldBIbChNvcGVyYXRvcl9nZW5lcmF0aW9uGAIgASgEEhkKEW9wZXJhdG9yX2lkZW50aXR5GAMgASgJEjAKBXBoYXNlGAQgASgOMiEuZW5ncmFtLm1jcC52MS5NYW5hZ2VkUG9saWN5UGhhc2USGQoMYmluZGluZ19oYXNoGAUgASgJSACIAQFCDwoNX2JpbmRpbmdfaGFzaCJVChFNYW5hZ2VkRGVueVBvbGljeRIPCgd2ZXJzaW9uGAEgASgNEhAKCHJldmlzaW9uGAIgASgEEg0KBWhvc3RzGAMgAygJEg4KBmRpZ2VzdBgEIAEoCSKWAgoTTWFuYWdlZEFwcGxpZWRTdGF0ZRI0Cgdjb250ZXh0GAEgASgLMiMuZW5ncmFtLm1jcC52MS5NYW5hZ2VkUG9saWN5Q29udGV4dBIwCgZwb2xpY3kYAiABKAsyIC5lbmdyYW0ubWNwLnYxLk1hbmFnZWREZW55UG9saWN5EhgKEGhvc3RfaW5jYXJuYXRpb24YAyABKAkSGQoRcHJveHlfaW5jYXJuYXRpb24YBCABKAkSEAoIc3RhdGVfaWQYBSABKAkSPgoNbmV0d29ya19wcm9vZhgGIAEoCzIiLmVuZ3JhbS5tY3AudjEuTWFuYWdlZE5ldHdvcmtQcm9vZkgAiAEBQhAKDl9uZXR3b3JrX3Byb29mImAKGFJlYWRNYW5hZ2VkUG9saWN5UmVxdWVzdBIQCghwcm90b2NvbBgBIAEoDRIyCgZ0YXJnZXQYAiABKAsyIi5lbmdyYW0ubWNwLnYxLk1hbmFnZWRQb2xpY3lUYXJnZXQitAEKGVJlYWRNYW5hZ2VkUG9saWN5UmVzcG9uc2USEAoIcHJvdG9jb2wYASABKA0SOAoHYXBwbGllZBgCIAEoCzIiLmVuZ3JhbS5tY3AudjEuTWFuYWdlZEFwcGxpZWRTdGF0ZUgAiAEBEj8KEHJldGFpbmVkX2Rlc2lyZWQYAyABKAsyJS5lbmdyYW0ubWNwLnYxLk1hbmFnZWRSZXRhaW5lZERlc2lyZWRCCgoIX2FwcGxpZWQijgEKFk1hbmFnZWRSZXRhaW5lZERlc2lyZWQSMgoGdGFyZ2V0GAEgASgLMiIuZW5ncmFtLm1jcC52MS5NYW5hZ2VkUG9saWN5VGFyZ2V0EjUKBnBvbGljeRgCIAEoCzIgLmVuZ3JhbS5tY3AudjEuTWFuYWdlZERlbnlQb2xpY3lIAIgBAUIJCgdfcG9saWN5It0BChlBcHBseU1hbmFnZWRQb2xpY3lSZXF1ZXN0EhAKCHByb3RvY29sGAEgASgNEjQKB2NvbnRleHQYAiABKAsyIy5lbmdyYW0ubWNwLnYxLk1hbmFnZWRQb2xpY3lDb250ZXh0EjAKBnBvbGljeRgDIAEoCzIgLmVuZ3JhbS5tY3AudjEuTWFuYWdlZERlbnlQb2xpY3kSOQoIZXhwZWN0ZWQYBCABKAsyIi5lbmdyYW0ubWNwLnYxLk1hbmFnZWRBcHBsaWVkU3RhdGVIAIgBAUILCglfZXhwZWN0ZWQiYwoaQXBwbHlNYW5hZ2VkUG9saWN5UmVzcG9uc2USEAoIcHJvdG9jb2wYASABKA0SMwoHYXBwbGllZBgCIAEoCzIiLmVuZ3JhbS5tY3AudjEuTWFuYWdlZEFwcGxpZWRTdGF0ZSJ1ChNNYW5hZ2VkTmV0d29ya1Byb29mEjIKBnRhcmdldBgBIAEoCzIiLmVuZ3JhbS5tY3AudjEuTWFuYWdlZFBvbGljeVRhcmdldBIQCghndWVzdF9pcBgCIAEoCRIYChBuZXR3b3JrX2lkZW50aXR5GAMgASgJIp8BChtSZXNlcnZlTWFuYWdlZENyZWF0ZVJlcXVlc3QSEAoIcHJvdG9jb2wYASABKA0SGAoQY3JlYXRpb25fcmVxdWVzdBgCIAEoDBIPCgd0YXNrX2lkGAMgASgJEhYKDm5hdGl2ZV92ZXJzaW9uGAQgASgNEhUKDW5hdGl2ZV9kaWdlc3QYBSABKAkSFAoMaW5wdXRfZGlnZXN0GAYgASgJIscBChVNYW5hZ2VkQ3JlYXRlSWRlbnRpdHkSEgoKc2Vzc2lvbl9pZBgBIAEoCRIPCgd0YXNrX2lkGAIgASgJEhAKCGlucHV0X2lkGAMgASgJEhAKCGFjdG9yX2lkGAQgASgJEh4KEXBhcmVudF9zZXNzaW9uX2lkGAUgASgJSACIAQESGQoRd3JpdGVfZWxpZ2liaWxpdHkYBiABKAkSFAoMaW5wdXRfZGlnZXN0GAcgASgJQhQKEl9wYXJlbnRfc2Vzc2lvbl9pZCJ3ChxSZXNlcnZlTWFuYWdlZENyZWF0ZVJlc3BvbnNlEhAKCHByb3RvY29sGAEgASgNEjYKCGlkZW50aXR5GAIgASgLMiQuZW5ncmFtLm1jcC52MS5NYW5hZ2VkQ3JlYXRlSWRlbnRpdHkSDQoFc3RhdGUYAyABKAkiZQoZUmVhZE5hdGl2ZU1hbmlmZXN0UmVxdWVzdBIQCghwcm90b2NvbBgBIAEoDRI2CghpZGVudGl0eRgCIAEoCzIkLmVuZ3JhbS5tY3AudjEuTWFuYWdlZENyZWF0ZUlkZW50aXR5IqsBChpSZWFkTmF0aXZlTWFuaWZlc3RSZXNwb25zZRIQCghwcm90b2NvbBgBIAEoDRI2CghpZGVudGl0eRgCIAEoCzIkLmVuZ3JhbS5tY3AudjEuTWFuYWdlZENyZWF0ZUlkZW50aXR5EhYKDm5hdGl2ZV92ZXJzaW9uGAMgASgNEhQKDG5hdGl2ZV9ieXRlcxgEIAEoDBIVCg1uYXRpdmVfZGlnZXN0GAUgASgJImIKFlJlYWRNYW5hZ2VkQm9vdFJlcXVlc3QSEAoIcHJvdG9jb2wYASABKA0SNgoIaWRlbnRpdHkYAiABKAsyJC5lbmdyYW0ubWNwLnYxLk1hbmFnZWRDcmVhdGVJZGVudGl0eSJRChdSZWFkTWFuYWdlZEJvb3RSZXNwb25zZRIQCghwcm90b2NvbBgBIAEoDRIWCglib290X2pzb24YAiABKAxIAIgBAUIMCgpfYm9vdF9qc29uIkIKGkZpbmFsaXplTWFuYWdlZEJvb3RSZXF1ZXN0EhAKCHByb3RvY29sGAEgASgNEhIKCmlucHV0X2pzb24YAiABKAwiVQobRmluYWxpemVNYW5hZ2VkQm9vdFJlc3BvbnNlEhAKCHByb3RvY29sGAEgASgNEhYKCWJvb3RfanNvbhgCIAEoDEgAiAEBQgwKCl9ib290X2pzb24iMAocUmVhZFB1Ymxpc2hlZE9wZXJhdG9yUmVxdWVzdBIQCghwcm90b2NvbBgBIAEoDSJLCh1SZWFkUHVibGlzaGVkT3BlcmF0b3JSZXNwb25zZRIQCghwcm90b2NvbBgBIAEoDRIYChBwdWJsaWNhdGlvbl9qc29uGAIgASgMIpoBCh9SZWFkTWFuYWdlZENyZWF0aW9uU3RhdGVSZXF1ZXN0EhAKCHByb3RvY29sGAEgASgNEjYKCGlkZW50aXR5GAIgASgLMiQuZW5ncmFtLm1jcC52MS5NYW5hZ2VkQ3JlYXRlSWRlbnRpdHkSFgoObmF0aXZlX3ZlcnNpb24YAyABKA0SFQoNbmF0aXZlX2RpZ2VzdBgEIAEoCSJICiBSZWFkTWFuYWdlZENyZWF0aW9uU3RhdGVSZXNwb25zZRIQCghwcm90b2NvbBgBIAEoDRISCgpzdGF0ZV9qc29uGAIgASgMIpQBChlQcmVwYXJlTWFuYWdlZEJvb3RSZXF1ZXN0EhAKCHByb3RvY29sGAEgASgNEjYKCGlkZW50aXR5GAIgASgLMiQuZW5ncmFtLm1jcC52MS5NYW5hZ2VkQ3JlYXRlSWRlbnRpdHkSFgoObmF0aXZlX3ZlcnNpb24YAyABKA0SFQoNbmF0aXZlX2RpZ2VzdBgEIAEoCSJCChpQcmVwYXJlTWFuYWdlZEJvb3RSZXNwb25zZRIQCghwcm90b2NvbBgBIAEoDRISCgpzdGF0ZV9qc29uGAIgASgMIssBChlSZWNvdmVyTWFuYWdlZEJvb3RSZXF1ZXN0EhAKCHByb3RvY29sGAEgASgNEjYKCGlkZW50aXR5GAIgASgLMiQuZW5ncmFtLm1jcC52MS5NYW5hZ2VkQ3JlYXRlSWRlbnRpdHkSFgoObmF0aXZlX3ZlcnNpb24YAyABKA0SFQoNbmF0aXZlX2RpZ2VzdBgEIAEoCRIgChhleHBlY3RlZF9zb3VyY2Vfa2V5X2pzb24YBSABKAwSEwoLcmVjb3ZlcnlfaWQYBiABKAkiQgoaUmVjb3Zlck1hbmFnZWRCb290UmVzcG9uc2USEAoIcHJvdG9jb2wYASABKA0SEgoKc3RhdGVfanNvbhgCIAEoDCp+ChJNYW5hZ2VkUG9saWN5UGhhc2USJAogTUFOQUdFRF9QT0xJQ1lfUEhBU0VfVU5TUEVDSUZJRUQQABIiCh5NQU5BR0VEX1BPTElDWV9QSEFTRV9CT09UU1RSQVAQARIeChpNQU5BR0VEX1BPTElDWV9QSEFTRV9GSU5BTBACMv8MChRTZXNzaW9uUG9saWN5U2VydmljZRJ7ChhSZWFkTWFuYWdlZENyZWF0aW9uU3RhdGUSLi5lbmdyYW0ubWNwLnYxLlJlYWRNYW5hZ2VkQ3JlYXRpb25TdGF0ZVJlcXVlc3QaLy5lbmdyYW0ubWNwLnYxLlJlYWRNYW5hZ2VkQ3JlYXRpb25TdGF0ZVJlc3BvbnNlEmkKElByZXBhcmVNYW5hZ2VkQm9vdBIoLmVuZ3JhbS5tY3AudjEuUHJlcGFyZU1hbmFnZWRCb290UmVxdWVzdBopLmVuZ3JhbS5tY3AudjEuUHJlcGFyZU1hbmFnZWRCb290UmVzcG9uc2USaQoSUmVjb3Zlck1hbmFnZWRCb290EiguZW5ncmFtLm1jcC52MS5SZWNvdmVyTWFuYWdlZEJvb3RSZXF1ZXN0GikuZW5ncmFtLm1jcC52MS5SZWNvdmVyTWFuYWdlZEJvb3RSZXNwb25zZRJyChVSZWFkUHVibGlzaGVkT3BlcmF0b3ISKy5lbmdyYW0ubWNwLnYxLlJlYWRQdWJsaXNoZWRPcGVyYXRvclJlcXVlc3QaLC5lbmdyYW0ubWNwLnYxLlJlYWRQdWJsaXNoZWRPcGVyYXRvclJlc3BvbnNlEmAKD1JlYWRNYW5hZ2VkQm9vdBIlLmVuZ3JhbS5tY3AudjEuUmVhZE1hbmFnZWRCb290UmVxdWVzdBomLmVuZ3JhbS5tY3AudjEuUmVhZE1hbmFnZWRCb290UmVzcG9uc2USbAoTRmluYWxpemVNYW5hZ2VkQm9vdBIpLmVuZ3JhbS5tY3AudjEuRmluYWxpemVNYW5hZ2VkQm9vdFJlcXVlc3QaKi5lbmdyYW0ubWNwLnYxLkZpbmFsaXplTWFuYWdlZEJvb3RSZXNwb25zZRJvChRSZXNlcnZlTWFuYWdlZENyZWF0ZRIqLmVuZ3JhbS5tY3AudjEuUmVzZXJ2ZU1hbmFnZWRDcmVhdGVSZXF1ZXN0GisuZW5ncmFtLm1jcC52MS5SZXNlcnZlTWFuYWdlZENyZWF0ZVJlc3BvbnNlEmkKElJlYWROYXRpdmVNYW5pZmVzdBIoLmVuZ3JhbS5tY3AudjEuUmVhZE5hdGl2ZU1hbmlmZXN0UmVxdWVzdBopLmVuZ3JhbS5tY3AudjEuUmVhZE5hdGl2ZU1hbmlmZXN0UmVzcG9uc2USZgoRUmVhZE1hbmFnZWRQb2xpY3kSJy5lbmdyYW0ubWNwLnYxLlJlYWRNYW5hZ2VkUG9saWN5UmVxdWVzdBooLmVuZ3JhbS5tY3AudjEuUmVhZE1hbmFnZWRQb2xpY3lSZXNwb25zZRJpChJBcHBseU1hbmFnZWRQb2xpY3kSKC5lbmdyYW0ubWNwLnYxLkFwcGx5TWFuYWdlZFBvbGljeVJlcXVlc3QaKS5lbmdyYW0ubWNwLnYxLkFwcGx5TWFuYWdlZFBvbGljeVJlc3BvbnNlEmwKE0NyZWF0ZVBvbGljeVNlc3Npb24SKS5lbmdyYW0ubWNwLnYxLkNyZWF0ZVBvbGljeVNlc3Npb25SZXF1ZXN0GiouZW5ncmFtLm1jcC52MS5DcmVhdGVQb2xpY3lTZXNzaW9uUmVzcG9uc2USYAoPUmVhZFdyaXRlUG9saWN5EiUuZW5ncmFtLm1jcC52MS5SZWFkV3JpdGVQb2xpY3lSZXF1ZXN0GiYuZW5ncmFtLm1jcC52MS5SZWFkV3JpdGVQb2xpY3lSZXNwb25zZRJdCg5BY2NlcHRQbGFuRXhpdBIkLmVuZ3JhbS5tY3AudjEuQWNjZXB0UGxhbkV4aXRSZXF1ZXN0GiUuZW5ncmFtLm1jcC52MS5BY2NlcHRQbGFuRXhpdFJlc3BvbnNlEn4KGVZhbGlkYXRlRXhlY3V0aW9uSWRlbnRpdHkSLy5lbmdyYW0ubWNwLnYxLlZhbGlkYXRlRXhlY3V0aW9uSWRlbnRpdHlSZXF1ZXN0GjAuZW5ncmFtLm1jcC52MS5WYWxpZGF0ZUV4ZWN1dGlvbklkZW50aXR5UmVzcG9uc2UScgoVVmFsaWRhdGVEaXNwYXRjaEZlbmNlEisuZW5ncmFtLm1jcC52MS5WYWxpZGF0ZURpc3BhdGNoRmVuY2VSZXF1ZXN0GiwuZW5ncmFtLm1jcC52MS5WYWxpZGF0ZURpc3BhdGNoRmVuY2VSZXNwb25zZWIGcHJvdG8z", [file_engram_app_v1_session]); + +/** + * @generated from message engram.mcp.v1.CreatePolicySessionRequest + */ +export type CreatePolicySessionRequest = Message<"engram.mcp.v1.CreatePolicySessionRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.app.v1.CreateSessionRequest create = 2; + */ + create?: CreateSessionRequest; + + /** + * @generated from field: string actor_id = 3; + */ + actorId: string; + + /** + * @generated from field: string input_id = 4; + */ + inputId: string; + + /** + * @generated from field: optional string parent_session_id = 5; + */ + parentSessionId?: string; +}; + +/** + * Describes the message engram.mcp.v1.CreatePolicySessionRequest. + * Use `create(CreatePolicySessionRequestSchema)` to create a new message. + */ +export const CreatePolicySessionRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 0); + +/** + * @generated from message engram.mcp.v1.ReadWritePolicyRequest + */ +export type ReadWritePolicyRequest = Message<"engram.mcp.v1.ReadWritePolicyRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string session_id = 2; + */ + sessionId: string; + + /** + * @generated from field: optional string plan_tool_call_id = 3; + */ + planToolCallId?: string; +}; + +/** + * Describes the message engram.mcp.v1.ReadWritePolicyRequest. + * Use `create(ReadWritePolicyRequestSchema)` to create a new message. + */ +export const ReadWritePolicyRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 1); + +/** + * @generated from message engram.mcp.v1.AcceptPlanExitRequest + */ +export type AcceptPlanExitRequest = Message<"engram.mcp.v1.AcceptPlanExitRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.app.v1.McpPlanReview expected = 2; + */ + expected?: McpPlanReview; + + /** + * @generated from field: string actor_id = 3; + */ + actorId: string; + + /** + * @generated from field: string result_json = 4; + */ + resultJson: string; +}; + +/** + * Describes the message engram.mcp.v1.AcceptPlanExitRequest. + * Use `create(AcceptPlanExitRequestSchema)` to create a new message. + */ +export const AcceptPlanExitRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 2); + +/** + * @generated from message engram.mcp.v1.ValidateDispatchFenceRequest + */ +export type ValidateDispatchFenceRequest = Message<"engram.mcp.v1.ValidateDispatchFenceRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string session_id = 2; + */ + sessionId: string; + + /** + * @generated from field: string run_id = 3; + */ + runId: string; + + /** + * @generated from field: string tool_call_id = 4; + */ + toolCallId: string; + + /** + * @generated from field: engram.app.v1.McpWritePolicyRef accepted_policy_ref = 5; + */ + acceptedPolicyRef?: McpWritePolicyRef; + + /** + * @generated from field: string accepted_write_eligibility = 6; + */ + acceptedWriteEligibility: string; + + /** + * @generated from field: repeated engram.app.v1.McpWritePolicyRef ancestor_policy_refs = 7; + */ + ancestorPolicyRefs: McpWritePolicyRef[]; + + /** + * @generated from field: string model_tool_name = 8; + */ + modelToolName: string; + + /** + * @generated from field: string original_arguments = 9; + */ + originalArguments: string; +}; + +/** + * Describes the message engram.mcp.v1.ValidateDispatchFenceRequest. + * Use `create(ValidateDispatchFenceRequestSchema)` to create a new message. + */ +export const ValidateDispatchFenceRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 3); + +/** + * @generated from message engram.mcp.v1.ValidateDispatchFenceResponse + */ +export type ValidateDispatchFenceResponse = Message<"engram.mcp.v1.ValidateDispatchFenceResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string status = 2; + */ + status: string; +}; + +/** + * Describes the message engram.mcp.v1.ValidateDispatchFenceResponse. + * Use `create(ValidateDispatchFenceResponseSchema)` to create a new message. + */ +export const ValidateDispatchFenceResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 4); + +/** + * @generated from message engram.mcp.v1.CreatePolicySessionResponse + */ +export type CreatePolicySessionResponse = Message<"engram.mcp.v1.CreatePolicySessionResponse"> & { + /** + * @generated from field: engram.app.v1.CreateSessionResponse session = 1; + */ + session?: CreateSessionResponse; +}; + +/** + * Describes the message engram.mcp.v1.CreatePolicySessionResponse. + * Use `create(CreatePolicySessionResponseSchema)` to create a new message. + */ +export const CreatePolicySessionResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 5); + +/** + * @generated from message engram.mcp.v1.ReadWritePolicyResponse + */ +export type ReadWritePolicyResponse = Message<"engram.mcp.v1.ReadWritePolicyResponse"> & { + /** + * @generated from field: engram.app.v1.GetMcpWritePolicyResponse state = 1; + */ + state?: GetMcpWritePolicyResponse; +}; + +/** + * Describes the message engram.mcp.v1.ReadWritePolicyResponse. + * Use `create(ReadWritePolicyResponseSchema)` to create a new message. + */ +export const ReadWritePolicyResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 6); + +/** + * @generated from message engram.mcp.v1.AcceptPlanExitResponse + */ +export type AcceptPlanExitResponse = Message<"engram.mcp.v1.AcceptPlanExitResponse"> & { + /** + * @generated from field: engram.app.v1.McpTrustedWritePolicy policy = 1; + */ + policy?: McpTrustedWritePolicy; +}; + +/** + * Describes the message engram.mcp.v1.AcceptPlanExitResponse. + * Use `create(AcceptPlanExitResponseSchema)` to create a new message. + */ +export const AcceptPlanExitResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 7); + +/** + * @generated from message engram.mcp.v1.ValidateExecutionIdentityRequest + */ +export type ValidateExecutionIdentityRequest = Message<"engram.mcp.v1.ValidateExecutionIdentityRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string session_id = 2; + */ + sessionId: string; + + /** + * @generated from field: string run_id = 3; + */ + runId: string; + + /** + * @generated from field: string tool_call_id = 4; + */ + toolCallId: string; + + /** + * @generated from field: engram.app.v1.McpWritePolicyRef accepted_policy_ref = 5; + */ + acceptedPolicyRef?: McpWritePolicyRef; + + /** + * @generated from field: string accepted_write_eligibility = 6; + */ + acceptedWriteEligibility: string; + + /** + * @generated from field: repeated engram.app.v1.McpWritePolicyRef ancestor_policy_refs = 7; + */ + ancestorPolicyRefs: McpWritePolicyRef[]; + + /** + * @generated from field: string model_tool_name = 8; + */ + modelToolName: string; + + /** + * @generated from field: string original_arguments = 9; + */ + originalArguments: string; +}; + +/** + * Describes the message engram.mcp.v1.ValidateExecutionIdentityRequest. + * Use `create(ValidateExecutionIdentityRequestSchema)` to create a new message. + */ +export const ValidateExecutionIdentityRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 8); + +/** + * @generated from message engram.mcp.v1.ValidateExecutionIdentityResponse + */ +export type ValidateExecutionIdentityResponse = Message<"engram.mcp.v1.ValidateExecutionIdentityResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: string status = 2; + */ + status: string; +}; + +/** + * Describes the message engram.mcp.v1.ValidateExecutionIdentityResponse. + * Use `create(ValidateExecutionIdentityResponseSchema)` to create a new message. + */ +export const ValidateExecutionIdentityResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 9); + +/** + * @generated from message engram.mcp.v1.ManagedPolicyTarget + */ +export type ManagedPolicyTarget = Message<"engram.mcp.v1.ManagedPolicyTarget"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string sandbox_id = 2; + */ + sandboxId: string; + + /** + * @generated from field: string host_id = 3; + */ + hostId: string; + + /** + * @generated from field: uint64 binding_epoch = 4; + */ + bindingEpoch: bigint; +}; + +/** + * Describes the message engram.mcp.v1.ManagedPolicyTarget. + * Use `create(ManagedPolicyTargetSchema)` to create a new message. + */ +export const ManagedPolicyTargetSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 10); + +/** + * @generated from message engram.mcp.v1.ManagedPolicyContext + */ +export type ManagedPolicyContext = Message<"engram.mcp.v1.ManagedPolicyContext"> & { + /** + * @generated from field: engram.mcp.v1.ManagedPolicyTarget target = 1; + */ + target?: ManagedPolicyTarget; + + /** + * @generated from field: uint64 operator_generation = 2; + */ + operatorGeneration: bigint; + + /** + * @generated from field: string operator_identity = 3; + */ + operatorIdentity: string; + + /** + * @generated from field: engram.mcp.v1.ManagedPolicyPhase phase = 4; + */ + phase: ManagedPolicyPhase; + + /** + * @generated from field: optional string binding_hash = 5; + */ + bindingHash?: string; +}; + +/** + * Describes the message engram.mcp.v1.ManagedPolicyContext. + * Use `create(ManagedPolicyContextSchema)` to create a new message. + */ +export const ManagedPolicyContextSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 11); + +/** + * @generated from message engram.mcp.v1.ManagedDenyPolicy + */ +export type ManagedDenyPolicy = Message<"engram.mcp.v1.ManagedDenyPolicy"> & { + /** + * @generated from field: uint32 version = 1; + */ + version: number; + + /** + * @generated from field: uint64 revision = 2; + */ + revision: bigint; + + /** + * @generated from field: repeated string hosts = 3; + */ + hosts: string[]; + + /** + * @generated from field: string digest = 4; + */ + digest: string; +}; + +/** + * Describes the message engram.mcp.v1.ManagedDenyPolicy. + * Use `create(ManagedDenyPolicySchema)` to create a new message. + */ +export const ManagedDenyPolicySchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 12); + +/** + * @generated from message engram.mcp.v1.ManagedAppliedState + */ +export type ManagedAppliedState = Message<"engram.mcp.v1.ManagedAppliedState"> & { + /** + * @generated from field: engram.mcp.v1.ManagedPolicyContext context = 1; + */ + context?: ManagedPolicyContext; + + /** + * @generated from field: engram.mcp.v1.ManagedDenyPolicy policy = 2; + */ + policy?: ManagedDenyPolicy; + + /** + * @generated from field: string host_incarnation = 3; + */ + hostIncarnation: string; + + /** + * @generated from field: string proxy_incarnation = 4; + */ + proxyIncarnation: string; + + /** + * @generated from field: string state_id = 5; + */ + stateId: string; + + /** + * @generated from field: optional engram.mcp.v1.ManagedNetworkProof network_proof = 6; + */ + networkProof?: ManagedNetworkProof; +}; + +/** + * Describes the message engram.mcp.v1.ManagedAppliedState. + * Use `create(ManagedAppliedStateSchema)` to create a new message. + */ +export const ManagedAppliedStateSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 13); + +/** + * @generated from message engram.mcp.v1.ReadManagedPolicyRequest + */ +export type ReadManagedPolicyRequest = Message<"engram.mcp.v1.ReadManagedPolicyRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ManagedPolicyTarget target = 2; + */ + target?: ManagedPolicyTarget; +}; + +/** + * Describes the message engram.mcp.v1.ReadManagedPolicyRequest. + * Use `create(ReadManagedPolicyRequestSchema)` to create a new message. + */ +export const ReadManagedPolicyRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 14); + +/** + * @generated from message engram.mcp.v1.ReadManagedPolicyResponse + */ +export type ReadManagedPolicyResponse = Message<"engram.mcp.v1.ReadManagedPolicyResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: optional engram.mcp.v1.ManagedAppliedState applied = 2; + */ + applied?: ManagedAppliedState; + + /** + * Required presence: absence means an unsupported read contract, not no policy. + * + * @generated from field: engram.mcp.v1.ManagedRetainedDesired retained_desired = 3; + */ + retainedDesired?: ManagedRetainedDesired; +}; + +/** + * Describes the message engram.mcp.v1.ReadManagedPolicyResponse. + * Use `create(ReadManagedPolicyResponseSchema)` to create a new message. + */ +export const ReadManagedPolicyResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 15); + +/** + * Coordinator persistence at the checked read target. This is never an applied ACK. + * + * @generated from message engram.mcp.v1.ManagedRetainedDesired + */ +export type ManagedRetainedDesired = Message<"engram.mcp.v1.ManagedRetainedDesired"> & { + /** + * @generated from field: engram.mcp.v1.ManagedPolicyTarget target = 1; + */ + target?: ManagedPolicyTarget; + + /** + * Absence inside a present wrapper means no retained policy at this read. + * + * @generated from field: optional engram.mcp.v1.ManagedDenyPolicy policy = 2; + */ + policy?: ManagedDenyPolicy; +}; + +/** + * Describes the message engram.mcp.v1.ManagedRetainedDesired. + * Use `create(ManagedRetainedDesiredSchema)` to create a new message. + */ +export const ManagedRetainedDesiredSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 16); + +/** + * @generated from message engram.mcp.v1.ApplyManagedPolicyRequest + */ +export type ApplyManagedPolicyRequest = Message<"engram.mcp.v1.ApplyManagedPolicyRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ManagedPolicyContext context = 2; + */ + context?: ManagedPolicyContext; + + /** + * @generated from field: engram.mcp.v1.ManagedDenyPolicy policy = 3; + */ + policy?: ManagedDenyPolicy; + + /** + * @generated from field: optional engram.mcp.v1.ManagedAppliedState expected = 4; + */ + expected?: ManagedAppliedState; +}; + +/** + * Describes the message engram.mcp.v1.ApplyManagedPolicyRequest. + * Use `create(ApplyManagedPolicyRequestSchema)` to create a new message. + */ +export const ApplyManagedPolicyRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 17); + +/** + * @generated from message engram.mcp.v1.ApplyManagedPolicyResponse + */ +export type ApplyManagedPolicyResponse = Message<"engram.mcp.v1.ApplyManagedPolicyResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ManagedAppliedState applied = 2; + */ + applied?: ManagedAppliedState; +}; + +/** + * Describes the message engram.mcp.v1.ApplyManagedPolicyResponse. + * Use `create(ApplyManagedPolicyResponseSchema)` to create a new message. + */ +export const ApplyManagedPolicyResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 18); + +/** + * @generated from message engram.mcp.v1.ManagedNetworkProof + */ +export type ManagedNetworkProof = Message<"engram.mcp.v1.ManagedNetworkProof"> & { + /** + * @generated from field: engram.mcp.v1.ManagedPolicyTarget target = 1; + */ + target?: ManagedPolicyTarget; + + /** + * @generated from field: string guest_ip = 2; + */ + guestIp: string; + + /** + * @generated from field: string network_identity = 3; + */ + networkIdentity: string; +}; + +/** + * Describes the message engram.mcp.v1.ManagedNetworkProof. + * Use `create(ManagedNetworkProofSchema)` to create a new message. + */ +export const ManagedNetworkProofSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 19); + +/** + * Exact bytes of CreatePolicySessionRequest, retained by the caller for retries. + * The coordinator stores only the digest of this payload, which can hold secrets. + * + * @generated from message engram.mcp.v1.ReserveManagedCreateRequest + */ +export type ReserveManagedCreateRequest = Message<"engram.mcp.v1.ReserveManagedCreateRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: bytes creation_request = 2; + */ + creationRequest: Uint8Array; + + /** + * @generated from field: string task_id = 3; + */ + taskId: string; + + /** + * @generated from field: uint32 native_version = 4; + */ + nativeVersion: number; + + /** + * @generated from field: string native_digest = 5; + */ + nativeDigest: string; + + /** + * @generated from field: string input_digest = 6; + */ + inputDigest: string; +}; + +/** + * Describes the message engram.mcp.v1.ReserveManagedCreateRequest. + * Use `create(ReserveManagedCreateRequestSchema)` to create a new message. + */ +export const ReserveManagedCreateRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 20); + +/** + * @generated from message engram.mcp.v1.ManagedCreateIdentity + */ +export type ManagedCreateIdentity = Message<"engram.mcp.v1.ManagedCreateIdentity"> & { + /** + * @generated from field: string session_id = 1; + */ + sessionId: string; + + /** + * @generated from field: string task_id = 2; + */ + taskId: string; + + /** + * @generated from field: string input_id = 3; + */ + inputId: string; + + /** + * @generated from field: string actor_id = 4; + */ + actorId: string; + + /** + * @generated from field: optional string parent_session_id = 5; + */ + parentSessionId?: string; + + /** + * @generated from field: string write_eligibility = 6; + */ + writeEligibility: string; + + /** + * @generated from field: string input_digest = 7; + */ + inputDigest: string; +}; + +/** + * Describes the message engram.mcp.v1.ManagedCreateIdentity. + * Use `create(ManagedCreateIdentitySchema)` to create a new message. + */ +export const ManagedCreateIdentitySchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 21); + +/** + * @generated from message engram.mcp.v1.ReserveManagedCreateResponse + */ +export type ReserveManagedCreateResponse = Message<"engram.mcp.v1.ReserveManagedCreateResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ManagedCreateIdentity identity = 2; + */ + identity?: ManagedCreateIdentity; + + /** + * A held queued reservation is not permission to start a sandbox. + * + * @generated from field: string state = 3; + */ + state: string; +}; + +/** + * Describes the message engram.mcp.v1.ReserveManagedCreateResponse. + * Use `create(ReserveManagedCreateResponseSchema)` to create a new message. + */ +export const ReserveManagedCreateResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 22); + +/** + * @generated from message engram.mcp.v1.ReadNativeManifestRequest + */ +export type ReadNativeManifestRequest = Message<"engram.mcp.v1.ReadNativeManifestRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ManagedCreateIdentity identity = 2; + */ + identity?: ManagedCreateIdentity; +}; + +/** + * Describes the message engram.mcp.v1.ReadNativeManifestRequest. + * Use `create(ReadNativeManifestRequestSchema)` to create a new message. + */ +export const ReadNativeManifestRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 23); + +/** + * @generated from message engram.mcp.v1.ReadNativeManifestResponse + */ +export type ReadNativeManifestResponse = Message<"engram.mcp.v1.ReadNativeManifestResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ManagedCreateIdentity identity = 2; + */ + identity?: ManagedCreateIdentity; + + /** + * @generated from field: uint32 native_version = 3; + */ + nativeVersion: number; + + /** + * @generated from field: bytes native_bytes = 4; + */ + nativeBytes: Uint8Array; + + /** + * @generated from field: string native_digest = 5; + */ + nativeDigest: string; +}; + +/** + * Describes the message engram.mcp.v1.ReadNativeManifestResponse. + * Use `create(ReadNativeManifestResponseSchema)` to create a new message. + */ +export const ReadNativeManifestResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 24); + +/** + * @generated from message engram.mcp.v1.ReadManagedBootRequest + */ +export type ReadManagedBootRequest = Message<"engram.mcp.v1.ReadManagedBootRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ManagedCreateIdentity identity = 2; + */ + identity?: ManagedCreateIdentity; +}; + +/** + * Describes the message engram.mcp.v1.ReadManagedBootRequest. + * Use `create(ReadManagedBootRequestSchema)` to create a new message. + */ +export const ReadManagedBootRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 25); + +/** + * @generated from message engram.mcp.v1.ReadManagedBootResponse + */ +export type ReadManagedBootResponse = Message<"engram.mcp.v1.ReadManagedBootResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: optional bytes boot_json = 2; + */ + bootJson?: Uint8Array; +}; + +/** + * Describes the message engram.mcp.v1.ReadManagedBootResponse. + * Use `create(ReadManagedBootResponseSchema)` to create a new message. + */ +export const ReadManagedBootResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 26); + +/** + * Exact typed ManagedBootFinalizeInput JSON. No guest or profile input is read. + * + * @generated from message engram.mcp.v1.FinalizeManagedBootRequest + */ +export type FinalizeManagedBootRequest = Message<"engram.mcp.v1.FinalizeManagedBootRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: bytes input_json = 2; + */ + inputJson: Uint8Array; +}; + +/** + * Describes the message engram.mcp.v1.FinalizeManagedBootRequest. + * Use `create(FinalizeManagedBootRequestSchema)` to create a new message. + */ +export const FinalizeManagedBootRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 27); + +/** + * @generated from message engram.mcp.v1.FinalizeManagedBootResponse + */ +export type FinalizeManagedBootResponse = Message<"engram.mcp.v1.FinalizeManagedBootResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: optional bytes boot_json = 2; + */ + bootJson?: Uint8Array; +}; + +/** + * Describes the message engram.mcp.v1.FinalizeManagedBootResponse. + * Use `create(FinalizeManagedBootResponseSchema)` to create a new message. + */ +export const FinalizeManagedBootResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 28); + +/** + * @generated from message engram.mcp.v1.ReadPublishedOperatorRequest + */ +export type ReadPublishedOperatorRequest = Message<"engram.mcp.v1.ReadPublishedOperatorRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; +}; + +/** + * Describes the message engram.mcp.v1.ReadPublishedOperatorRequest. + * Use `create(ReadPublishedOperatorRequestSchema)` to create a new message. + */ +export const ReadPublishedOperatorRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 29); + +/** + * @generated from message engram.mcp.v1.ReadPublishedOperatorResponse + */ +export type ReadPublishedOperatorResponse = Message<"engram.mcp.v1.ReadPublishedOperatorResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: bytes publication_json = 2; + */ + publicationJson: Uint8Array; +}; + +/** + * Describes the message engram.mcp.v1.ReadPublishedOperatorResponse. + * Use `create(ReadPublishedOperatorResponseSchema)` to create a new message. + */ +export const ReadPublishedOperatorResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 30); + +/** + * @generated from message engram.mcp.v1.ReadManagedCreationStateRequest + */ +export type ReadManagedCreationStateRequest = Message<"engram.mcp.v1.ReadManagedCreationStateRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ManagedCreateIdentity identity = 2; + */ + identity?: ManagedCreateIdentity; + + /** + * @generated from field: uint32 native_version = 3; + */ + nativeVersion: number; + + /** + * @generated from field: string native_digest = 4; + */ + nativeDigest: string; +}; + +/** + * Describes the message engram.mcp.v1.ReadManagedCreationStateRequest. + * Use `create(ReadManagedCreationStateRequestSchema)` to create a new message. + */ +export const ReadManagedCreationStateRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 31); + +/** + * @generated from message engram.mcp.v1.ReadManagedCreationStateResponse + */ +export type ReadManagedCreationStateResponse = Message<"engram.mcp.v1.ReadManagedCreationStateResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: bytes state_json = 2; + */ + stateJson: Uint8Array; +}; + +/** + * Describes the message engram.mcp.v1.ReadManagedCreationStateResponse. + * Use `create(ReadManagedCreationStateResponseSchema)` to create a new message. + */ +export const ReadManagedCreationStateResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 32); + +/** + * @generated from message engram.mcp.v1.PrepareManagedBootRequest + */ +export type PrepareManagedBootRequest = Message<"engram.mcp.v1.PrepareManagedBootRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ManagedCreateIdentity identity = 2; + */ + identity?: ManagedCreateIdentity; + + /** + * @generated from field: uint32 native_version = 3; + */ + nativeVersion: number; + + /** + * @generated from field: string native_digest = 4; + */ + nativeDigest: string; +}; + +/** + * Describes the message engram.mcp.v1.PrepareManagedBootRequest. + * Use `create(PrepareManagedBootRequestSchema)` to create a new message. + */ +export const PrepareManagedBootRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 33); + +/** + * @generated from message engram.mcp.v1.PrepareManagedBootResponse + */ +export type PrepareManagedBootResponse = Message<"engram.mcp.v1.PrepareManagedBootResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: bytes state_json = 2; + */ + stateJson: Uint8Array; +}; + +/** + * Describes the message engram.mcp.v1.PrepareManagedBootResponse. + * Use `create(PrepareManagedBootResponseSchema)` to create a new message. + */ +export const PrepareManagedBootResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 34); + +/** + * @generated from message engram.mcp.v1.RecoverManagedBootRequest + */ +export type RecoverManagedBootRequest = Message<"engram.mcp.v1.RecoverManagedBootRequest"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: engram.mcp.v1.ManagedCreateIdentity identity = 2; + */ + identity?: ManagedCreateIdentity; + + /** + * @generated from field: uint32 native_version = 3; + */ + nativeVersion: number; + + /** + * @generated from field: string native_digest = 4; + */ + nativeDigest: string; + + /** + * @generated from field: bytes expected_source_key_json = 5; + */ + expectedSourceKeyJson: Uint8Array; + + /** + * @generated from field: string recovery_id = 6; + */ + recoveryId: string; +}; + +/** + * Describes the message engram.mcp.v1.RecoverManagedBootRequest. + * Use `create(RecoverManagedBootRequestSchema)` to create a new message. + */ +export const RecoverManagedBootRequestSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 35); + +/** + * @generated from message engram.mcp.v1.RecoverManagedBootResponse + */ +export type RecoverManagedBootResponse = Message<"engram.mcp.v1.RecoverManagedBootResponse"> & { + /** + * @generated from field: uint32 protocol = 1; + */ + protocol: number; + + /** + * @generated from field: bytes state_json = 2; + */ + stateJson: Uint8Array; +}; + +/** + * Describes the message engram.mcp.v1.RecoverManagedBootResponse. + * Use `create(RecoverManagedBootResponseSchema)` to create a new message. + */ +export const RecoverManagedBootResponseSchema: GenMessage = /*@__PURE__*/ + messageDesc(file_engram_mcp_v1_session_policy, 36); + +/** + * These messages describe a host receipt, not permission to expose guest tools. + * + * @generated from enum engram.mcp.v1.ManagedPolicyPhase + */ +export enum ManagedPolicyPhase { + /** + * @generated from enum value: MANAGED_POLICY_PHASE_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: MANAGED_POLICY_PHASE_BOOTSTRAP = 1; + */ + BOOTSTRAP = 1, + + /** + * @generated from enum value: MANAGED_POLICY_PHASE_FINAL = 2; + */ + FINAL = 2, +} + +/** + * Describes the enum engram.mcp.v1.ManagedPolicyPhase. + */ +export const ManagedPolicyPhaseSchema: GenEnum = /*@__PURE__*/ + enumDesc(file_engram_mcp_v1_session_policy, 0); + +/** + * Dedicated pinned mTLS only. Create reuses the ordinary creation core. + * + * @generated from service engram.mcp.v1.SessionPolicyService + */ +export const SessionPolicyService: GenService<{ + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.ReadManagedCreationState + */ + readManagedCreationState: { + methodKind: "unary"; + input: typeof ReadManagedCreationStateRequestSchema; + output: typeof ReadManagedCreationStateResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.PrepareManagedBoot + */ + prepareManagedBoot: { + methodKind: "unary"; + input: typeof PrepareManagedBootRequestSchema; + output: typeof PrepareManagedBootResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.RecoverManagedBoot + */ + recoverManagedBoot: { + methodKind: "unary"; + input: typeof RecoverManagedBootRequestSchema; + output: typeof RecoverManagedBootResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.ReadPublishedOperator + */ + readPublishedOperator: { + methodKind: "unary"; + input: typeof ReadPublishedOperatorRequestSchema; + output: typeof ReadPublishedOperatorResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.ReadManagedBoot + */ + readManagedBoot: { + methodKind: "unary"; + input: typeof ReadManagedBootRequestSchema; + output: typeof ReadManagedBootResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.FinalizeManagedBoot + */ + finalizeManagedBoot: { + methodKind: "unary"; + input: typeof FinalizeManagedBootRequestSchema; + output: typeof FinalizeManagedBootResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.ReserveManagedCreate + */ + reserveManagedCreate: { + methodKind: "unary"; + input: typeof ReserveManagedCreateRequestSchema; + output: typeof ReserveManagedCreateResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.ReadNativeManifest + */ + readNativeManifest: { + methodKind: "unary"; + input: typeof ReadNativeManifestRequestSchema; + output: typeof ReadNativeManifestResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.ReadManagedPolicy + */ + readManagedPolicy: { + methodKind: "unary"; + input: typeof ReadManagedPolicyRequestSchema; + output: typeof ReadManagedPolicyResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.ApplyManagedPolicy + */ + applyManagedPolicy: { + methodKind: "unary"; + input: typeof ApplyManagedPolicyRequestSchema; + output: typeof ApplyManagedPolicyResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.CreatePolicySession + */ + createPolicySession: { + methodKind: "unary"; + input: typeof CreatePolicySessionRequestSchema; + output: typeof CreatePolicySessionResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.ReadWritePolicy + */ + readWritePolicy: { + methodKind: "unary"; + input: typeof ReadWritePolicyRequestSchema; + output: typeof ReadWritePolicyResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.AcceptPlanExit + */ + acceptPlanExit: { + methodKind: "unary"; + input: typeof AcceptPlanExitRequestSchema; + output: typeof AcceptPlanExitResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.ValidateExecutionIdentity + */ + validateExecutionIdentity: { + methodKind: "unary"; + input: typeof ValidateExecutionIdentityRequestSchema; + output: typeof ValidateExecutionIdentityResponseSchema; + }, + /** + * @generated from rpc engram.mcp.v1.SessionPolicyService.ValidateDispatchFence + */ + validateDispatchFence: { + methodKind: "unary"; + input: typeof ValidateDispatchFenceRequestSchema; + output: typeof ValidateDispatchFenceResponseSchema; + }, +}> = /*@__PURE__*/ + serviceDesc(file_engram_mcp_v1_session_policy, 0); + diff --git a/orchestrator/src/http-log.ts b/orchestrator/src/http-log.ts new file mode 100644 index 000000000..76daf7d2d --- /dev/null +++ b/orchestrator/src/http-log.ts @@ -0,0 +1,23 @@ +import type { MiddlewareHandler } from "hono"; +export interface HttpLogRecord { + method: string; + path: string; + status: number; + durationMs: number; +} +/** Query strings can contain authorization codes. Never log URLs or bodies. */ +export function makeHttpLog( + write: (record: HttpLogRecord) => void, + now: () => number = Date.now, +): MiddlewareHandler { + return async (context, next) => { + const start = now(); + await next(); + write({ + method: context.req.method, + path: context.req.path, + status: context.res.status, + durationMs: Math.max(0, now() - start), + }); + }; +} diff --git a/orchestrator/src/index.ts b/orchestrator/src/index.ts index f964f5c50..136ac8c92 100644 --- a/orchestrator/src/index.ts +++ b/orchestrator/src/index.ts @@ -1,6 +1,7 @@ +import { makeSessionPolicyOverrides } from "./mcp/session-policy.ts"; import { randomUUID } from "node:crypto"; import { Hono } from "hono"; -import { logger as honoLogger } from "hono/logger"; +import { makeHttpLog } from "./http-log.ts"; import { config } from "./config.ts"; import { log } from "./log.ts"; import { buildServer } from "./server.ts"; @@ -70,6 +71,13 @@ import type { ConnectRouter } from "@connectrpc/connect"; // ABOVE the initDbos() call below so their workflows/steps are registered // before DBOS.launch(). Imports below register the finite Slack-thread and // tool-execution workflows. +import "./workflows/mcp-exec.ts"; +import { makeMcpRecoveryScanner } from "./mcp/recovery-scanner.ts"; +import { installMcpRecovery } from "./mcp/recovery.ts"; +import { makeMcpRuntime } from "./mcp/runtime.ts"; +import { loadMcpRuntimeConfiguration, mcpStartupSettings } from "./mcp/runtime-config.ts"; +import { installMcpSessionCapabilities } from "./mcp/session-capabilities.ts"; +import { makeManagedSessionScanner } from "./mcp/managed-scanner.ts"; import { initDbos, shutdownDbos } from "./workflows/dbos.ts"; import { startSpecTicketSyncWorkflow } from "./workflows/spec-ticket-sync.ts"; import { makeProductionListenerManager } from "./listeners/manager.ts"; @@ -267,10 +275,9 @@ const specSyncHub = new SpecSyncHub({ }); setSpecPresence(specSyncHub); -// Request logging (hono/logger) routed through pino, so every HTTP leg — the -// Slack webhooks included — logs `<-- METHOD path` / `--> METHOD path status ms`. +// Request metadata only. OAuth codes and state in query strings are not logged. const httpLog = log.child({ component: "http" }); -app.use(honoLogger((message) => httpLog.info(message))); +app.use(makeHttpLog((record) => httpLog.info(record, "HTTP request"))); // ADR 0064 P2b: live-host preview reverse-proxy. Mounted FIRST so a request to // `.` is resolved + tunneled to the guest port before @@ -467,6 +474,24 @@ app.route("/", ideApp); // Default 404 for unmatched Hono paths. app.notFound((c) => c.json({ error: "not found" }, 404)); +// Keep durable bookkeeping and completion recovery installed on disabled deployments. +const mcpSettings = mcpStartupSettings(process.env); +const mcpRuntime = makeMcpRuntime({ + configuration: mcpSettings.path ? await loadMcpRuntimeConfiguration(mcpSettings.path) : undefined, + deploymentId: config.deploymentId, + enabled: () => mcpSettings.enabled, + bridge: (control) => ({ + readManagedCreationState: (request) => control.readManagedCreationState(request), + prepareManagedBoot: (request) => control.prepareManagedBoot(request), + recoverManagedBoot: (request) => control.recoverManagedBoot(request), + }), +}); +const mcpServices = mcpRuntime.services; +installMcpRecovery(mcpRuntime.recovery); +if (mcpRuntime.capabilities) installMcpSessionCapabilities(mcpRuntime.capabilities); +const managedSessionScanner = mcpRuntime.lifecycle + ? makeManagedSessionScanner({ store: mcpRuntime.store, runtime: mcpRuntime.lifecycle }) + : undefined; const server = buildServer( app, (router: ConnectRouter) => { @@ -514,7 +539,7 @@ const server = buildServer( // Native IntegrationService (ADR 0057 C3): admin-gated connector catalog CRUD // (Plane B) over the orchestrator's own DB; built-ins are read-only seeds. - registerIntegration(router); + registerIntegration(router, { mcp: mcpServices }); // Native AutomationService + WebhookRegistrationService (ADR 0102): // admin-authored triggers, render preview, run/sample history, and sealed @@ -570,16 +595,24 @@ const server = buildServer( // to the control plane with per-method CASL authz gate (ADR 0051 Task 18). // ADR 0053 §3: a DisableImage pre-flight blocks disabling an image that // any active profile still references (the coordinator only knows sessions). - registerPassthrough(router, SURFACE, controlPlaneTransport, undefined, undefined, { - "ImageService.DisableImage": makeDisableImageGuard(), - "SessionService.CompleteToolCall": makeExternalToolCompletionGuard(), - "SessionService.SendPrompt": async (request) => { - const sessionId = (request as { sessionId?: unknown }).sessionId; - if (typeof sessionId !== "string") return; - const response = await controlPlaneSessions.getSession({ sessionId }); - await productionSpecProjection.preparePrompt(sessionId, response.session?.status ?? ""); + registerPassthrough( + router, + SURFACE, + controlPlaneTransport, + undefined, + undefined, + { + "ImageService.DisableImage": makeDisableImageGuard(), + "SessionService.CompleteToolCall": makeExternalToolCompletionGuard(), + "SessionService.SendPrompt": async (request) => { + const sessionId = (request as { sessionId?: unknown }).sessionId; + if (typeof sessionId !== "string") return; + const response = await controlPlaneSessions.getSession({ sessionId }); + await productionSpecProjection.preparePrompt(sessionId, response.session?.status ?? ""); + }, }, - }); + makeSessionPolicyOverrides({ control: mcpRuntime.control }), + ); }, // Raw WS-upgrade hooks, tried in order: the preview proxy first // (Host-keyed — a preview host is a different origin, so it wins @@ -660,6 +693,13 @@ void retireProviderWebhookSchemes({ }).catch((err) => log.error({ err }, "provider webhook scheme retirement failed")); if (process.env.ENGRAM_DEV_TOOLS === "1") registerDevTools(); await initDbos(); +const mcpRecoveryScanner = makeMcpRecoveryScanner({ + deploymentId: config.deploymentId, + journal: mcpRuntime.services.journal, + runtime: () => mcpRuntime.recovery, +}); +await mcpRecoveryScanner.start(); +await managedSessionScanner?.start(); assertSweepPoliciesExhaustive(); // ADR 0119 D1: the block registry is static and must be populated before the // first RPC validates a definition — registration was lazy (first interpreter @@ -731,6 +771,8 @@ process.on("SIGTERM", () => { modelRouterRefresher.stop(); await automationScheduler.stop(); await integrationEventSweeper.stop(); + await managedSessionScanner?.stop(); + await mcpRecoveryScanner.stop(); await listenerManager.stop(); await specPublishScanner.stop(); await draftingSeedScanner.stop(); diff --git a/orchestrator/src/integrations/grants.ts b/orchestrator/src/integrations/grants.ts index 6c4869b95..11218d057 100644 --- a/orchestrator/src/integrations/grants.ts +++ b/orchestrator/src/integrations/grants.ts @@ -1,5 +1,6 @@ /** Connection-aware integration grant validation and policy projection (ADR 0109). */ +import { validateMcpGrant } from "../mcp/types.ts"; import { ConnectError, Code } from "@connectrpc/connect"; import { createHash } from "node:crypto"; @@ -22,6 +23,7 @@ export interface IntegrationSnapshot { profileId: string | null; integrationGrants: readonly ProfileIntegrationGrant[]; integrationConnections: readonly IntegrationConnectionSnapshot[]; + mcpBindingEnvelope?: import("../mcp/binding-store.ts").McpBindingEnvelope | null; } /** Recursively sort object keys so the hash survives a JSONB round-trip @@ -51,6 +53,7 @@ export function integrationSnapshotHash(snapshot: IntegrationSnapshot): string { profileId: snapshot.profileId, integrationGrants: snapshot.integrationGrants, integrationConnections: snapshot.integrationConnections, + ...(snapshot.mcpBindingEnvelope ? { mcpBindingEnvelope: snapshot.mcpBindingEnvelope } : {}), }))) .digest("hex"); return `sha256:${digest}`; @@ -106,6 +109,15 @@ export async function resolveIntegrationGrants( Code.InvalidArgument, ); } + if (connection.provider === "mcp") { + try { + validateMcpGrant(grant); + } catch (error) { + throw new ConnectError((error as Error).message, Code.InvalidArgument); + } + } else if (grant.mcp) { + throw new ConnectError("MCP tool grants require an MCP connection", Code.InvalidArgument); + } if (!OPERATION_RE.test(grant.operation)) { throw new ConnectError( `invalid integration operation "${grant.operation}"`, @@ -141,6 +153,8 @@ export function grantsToCapabilities( ): string[] { const capabilities = new Set(); for (const { grant, connection } of resolved) { + // MCP authority cannot become a generic guest credential or egress grant. + if (connection.provider === "mcp") continue; if (grant.resourceConstraints.length === 0) { capabilities.add(`${connection.provider}:${grant.operation}`); continue; diff --git a/orchestrator/src/listeners/__tests__/tool-consumer.test.ts b/orchestrator/src/listeners/__tests__/tool-consumer.test.ts index 81edf7391..38cca90a2 100644 --- a/orchestrator/src/listeners/__tests__/tool-consumer.test.ts +++ b/orchestrator/src/listeners/__tests__/tool-consumer.test.ts @@ -3,14 +3,8 @@ import { z } from "zod"; import { registerBuiltinTools } from "../../tools/builtin.ts"; import { createToolRegistry } from "../../tools/registry.ts"; -import type { - PendingToolCallInput, - PendingToolCallStore, -} from "../../tools/pending-tool-calls.ts"; -import { - makeToolConsumer, - type ToolWorkflowStarter, -} from "../tool-consumer.ts"; +import type { PendingToolCallInput, PendingToolCallStore } from "../../tools/pending-tool-calls.ts"; +import { makeToolConsumer, type ToolWorkflowStarter } from "../tool-consumer.ts"; function pendingRecorder() { const requested: PendingToolCallInput[] = []; @@ -185,22 +179,28 @@ describe("tool consumer", () => { }); await consumer.handle(requested("ask_user_question"), { sessionId: "session-1" }); - await consumer.handle({ - idx: 2n, - kind: "tool_result_submitted", - payloadJson: JSON.stringify({ - tool_call_id: "call-1", - at: "2026-07-13T12:01:00.000Z", - }), - }, { sessionId: "session-1" }); - await consumer.handle({ - idx: 3n, - kind: "tool_call_completed", - payloadJson: JSON.stringify({ - tool_call_id: "call-1", - at: "2026-07-13T12:02:00.000Z", - }), - }, { sessionId: "session-1" }); + await consumer.handle( + { + idx: 2n, + kind: "tool_result_submitted", + payloadJson: JSON.stringify({ + tool_call_id: "call-1", + at: "2026-07-13T12:01:00.000Z", + }), + }, + { sessionId: "session-1" }, + ); + await consumer.handle( + { + idx: 3n, + kind: "tool_call_completed", + payloadJson: JSON.stringify({ + tool_call_id: "call-1", + at: "2026-07-13T12:02:00.000Z", + }), + }, + { sessionId: "session-1" }, + ); expect(pending.requested[0]).toEqual({ sessionId: "session-1", @@ -225,3 +225,88 @@ describe("tool consumer", () => { ]); }); }); + +test("MCP requests persist the exact event reference before cursor success", async () => { + const pending = pendingRecorder(); + const accepted: unknown[] = []; + let fail = true; + const consumer = makeToolConsumer({ + registry: registryFixture(), + pendingCalls: pending.store, + now: () => new Date(0), + startWorkflow: async () => { + throw Error("static path used"); + }, + startMcpRequest: async (input) => { + accepted.push(input); + if (fail) throw Error("acceptance unavailable"); + }, + }); + const event = { + idx: 7n, + kind: "tool_call_requested", + payloadJson: JSON.stringify({ + tool_call_id: "call-1", + run_id: "run-1", + name: "mcp_bound", + args_json: '{ "x": 1 }', + }), + }; + await expect(consumer.handle(event, { sessionId: "s1" })).rejects.toThrow( + "MCP request unavailable", + ); + fail = false; + await consumer.handle(event, { sessionId: "s1" }); + expect(accepted).toEqual( + Array(2).fill({ + sessionId: "s1", + runId: "run-1", + toolCallId: "call-1", + requestEventIdx: "7", + deadlineMs: 930000, + }), + ); +}); + +test("authenticated MCP cancel event reaches durable cancellation before cursor success", async () => { + const pending = pendingRecorder(), + requests: unknown[] = []; + let fail = true; + const consumer = makeToolConsumer({ + registry: registryFixture(), + pendingCalls: pending.store, + now: () => new Date(0), + startWorkflow: async () => { + throw Error("generic execution used"); + }, + cancelMcpRequest: async (input: unknown) => { + requests.push(input); + if (fail) throw Error("unavailable"); + }, + }); + const event = { + idx: 8n, + kind: "tool_call_cancel_requested", + payloadJson: JSON.stringify({ + run_id: "run-1", + tool_call_id: "call-1", + name: "mcp_bound", + }), + }; + expect(consumer.interestedIn(event.kind)).toBe(true); + await expect(consumer.handle(event, { sessionId: "s1" })).rejects.toThrow( + "MCP cancellation unavailable", + ); + fail = false; + await consumer.handle(event, { sessionId: "s1" }); + expect(requests).toEqual( + Array(2).fill({ + sessionId: "s1", + runId: "run-1", + toolCallId: "call-1", + modelToolName: "mcp_bound", + }), + ); + expect(pending.completed).toEqual([]); + expect(pending.submitted).toEqual([]); +}); diff --git a/orchestrator/src/listeners/tool-consumer.ts b/orchestrator/src/listeners/tool-consumer.ts index dabd97e20..97c05a293 100644 --- a/orchestrator/src/listeners/tool-consumer.ts +++ b/orchestrator/src/listeners/tool-consumer.ts @@ -6,20 +6,26 @@ import { task as taskTable, taskSession as taskSessionTable } from "../db/schema import type { CuratedEvent } from "../control-plane/session-events.ts"; import { log as rootLog } from "../log.ts"; -import { makePendingToolCallStore, type PendingToolCallStore } from "../tools/pending-tool-calls.ts"; +import { + makePendingToolCallStore, + type PendingToolCallStore, +} from "../tools/pending-tool-calls.ts"; import { tools as productionTools, type ToolRegistry } from "../tools/registry.ts"; import type { ToolExecInput } from "../tools/exec.ts"; +import { MCP_LIMITS } from "../mcp/limits.ts"; +import { mcpRecovery } from "../mcp/recovery.ts"; +import { startMcpRequestWorkflow } from "../workflows/mcp-exec.ts"; import { toolExecWorkflow } from "../workflows/tool-exec.ts"; import type { SessionConsumer } from "./consumer.ts"; const log = rootLog.child({ component: "tool-consumer" }); -export type ToolWorkflowStarter = ( - input: ToolExecInput, - workflowId: string, -) => Promise; +export type ToolWorkflowStarter = (input: ToolExecInput, workflowId: string) => Promise; export interface ToolConsumerDeps { + cancelMcpRequest?: (input: import("../mcp/recovery.ts").McpCancelRequest) => Promise; + /** Persist the exact request-event reference before the listener cursor advances. */ + startMcpRequest?: (input: import("../mcp/recovery.ts").McpRequestMarker) => Promise; registry: ToolRegistry; pendingCalls: PendingToolCallStore; startWorkflow: ToolWorkflowStarter; @@ -28,51 +34,40 @@ export interface ToolConsumerDeps { * human owner (an automation). Absent = never auto-approve. */ sessionIsOwnerless?: (sessionId: string) => Promise; /** The internal completer (`tools.complete`) the auto-approve rides. */ - completeSessionTool?: ( - sessionId: string, - toolCallId: string, - result: unknown, - ) => Promise; + completeSessionTool?: (sessionId: string, toolCallId: string, result: unknown) => Promise; } interface ToolCallRequestedPayload { toolCallId: string; name: string; argsJson?: string; + runId?: string; } -export function parseToolCallRequested( - event: CuratedEvent, -): ToolCallRequestedPayload | undefined { +export function parseToolCallRequested(event: CuratedEvent): ToolCallRequestedPayload | undefined { if (event.kind !== "tool_call_requested") return undefined; try { const payload = JSON.parse(event.payloadJson) as { tool_call_id?: unknown; name?: unknown; args_json?: unknown; + run_id?: unknown; }; - if ( - typeof payload.tool_call_id !== "string" || - typeof payload.name !== "string" - ) { + if (typeof payload.tool_call_id !== "string" || typeof payload.name !== "string") { return undefined; } return { toolCallId: payload.tool_call_id, name: payload.name, - ...(typeof payload.args_json === "string" - ? { argsJson: payload.args_json } - : {}), + ...(typeof payload.run_id === "string" ? { runId: payload.run_id } : {}), + ...(typeof payload.args_json === "string" ? { argsJson: payload.args_json } : {}), }; } catch { return undefined; } } -function eventTimestamp( - payload: Record, - now: () => Date, -): Date { +function eventTimestamp(payload: Record, now: () => Date): Date { if (typeof payload.at === "string") { const parsed = new Date(payload.at); if (!Number.isNaN(parsed.getTime())) return parsed; @@ -126,13 +121,73 @@ export function makeToolConsumer(deps: ToolConsumerDeps): SessionConsumer { name: "tool-dispatch", appliesTo: async () => true, interestedIn: (kind) => + kind === "tool_call_cancel_requested" || kind === "tool_call_requested" || kind === "tool_result_submitted" || kind === "tool_call_completed", async handle(event, ctx) { + if (event.kind === "tool_call_cancel_requested") { + const payload: unknown = JSON.parse(event.payloadJson); + if ( + !payload || + typeof payload !== "object" || + !("name" in payload) || + typeof payload.name !== "string" || + !payload.name.startsWith("mcp_") + ) + return; + if ( + !("run_id" in payload) || + !("tool_call_id" in payload) || + ![payload.name, payload.run_id, payload.tool_call_id].every( + (value) => + typeof value === "string" && + value.length > 0 && + value.length <= 256 && + !/[\x00-\x1f]/.test(value), + ) || + typeof payload.run_id !== "string" || + typeof payload.tool_call_id !== "string" + ) + throw Error("MCP cancellation identity invalid"); + try { + if (!deps.cancelMcpRequest) throw Error("unavailable"); + await deps.cancelMcpRequest({ + sessionId: ctx.sessionId, + runId: payload.run_id, + toolCallId: payload.tool_call_id, + modelToolName: payload.name, + }); + } catch { + throw Error("MCP cancellation unavailable"); + } + return; + } await bookkeepToolEvent(ctx.sessionId, event, deps); const requested = parseToolCallRequested(event); if (!requested) return; + if (requested.name.startsWith("mcp_")) { + // Fail closed without acknowledging this event. The durable listener retries it. + if (!deps.startMcpRequest || !requested.runId || requested.argsJson === undefined) + throw Error("MCP request unavailable"); + try { + await deps.startMcpRequest({ + sessionId: ctx.sessionId, + runId: requested.runId, + toolCallId: requested.toolCallId, + requestEventIdx: String(event.idx), + deadlineMs: + deps.now().getTime() + + MCP_LIMITS.approvalMs + + MCP_LIMITS.queueMs + + MCP_LIMITS.maxToolMs, + }); + } catch { + // Listener errors can reach logs. Never retain a raw argument or driver error here. + throw Error("MCP request unavailable"); + } + return; + } const tool = deps.registry.get(requested.name); // ADR 0107 headless policy: a plan proposed in an OWNERLESS session // (an automation — nobody is watching) auto-approves immediately, so @@ -170,15 +225,10 @@ export function makeToolConsumer(deps: ToolConsumerDeps): SessionConsumer { sessionId: ctx.sessionId, toolCallId: requested.toolCallId, toolName: requested.name, - ...(requested.argsJson !== undefined - ? { argsJson: requested.argsJson } - : {}), + ...(requested.argsJson !== undefined ? { argsJson: requested.argsJson } : {}), }; try { - await deps.startWorkflow( - input, - `toolexec:${ctx.sessionId}:${requested.toolCallId}`, - ); + await deps.startWorkflow(input, `toolexec:${ctx.sessionId}:${requested.toolCallId}`); } catch (err) { log.error( { @@ -198,6 +248,15 @@ export function makeToolConsumer(deps: ToolConsumerDeps): SessionConsumer { export function makeProductionToolConsumer(): SessionConsumer { const pendingCalls = makePendingToolCallStore(); return makeToolConsumer({ + cancelMcpRequest: async (input) => { + const runtime = mcpRecovery(); + if (!runtime) throw Error("MCP runtime unavailable"); + await runtime.cancelRequest(input); + }, + startMcpRequest: async (input) => { + if (!mcpRecovery()) throw Error("MCP runtime unavailable"); + await startMcpRequestWorkflow(input); + }, registry: productionTools, pendingCalls, now: () => new Date(), diff --git a/orchestrator/src/mcp/active-watch.ts b/orchestrator/src/mcp/active-watch.ts new file mode 100644 index 000000000..d00bbd44a --- /dev/null +++ b/orchestrator/src/mcp/active-watch.ts @@ -0,0 +1,54 @@ +/** Replica-local abort signals. Authority remains in durable state, not this registry. */ +export function makeMcpActiveWatch(deps: { check(id: string): Promise; intervalMs: number }) { + const active = new Map>(); + return { + register(id: string) { + const controller = new AbortController(); + const controllers = active.get(id) ?? new Set(); + controllers.add(controller); + active.set(id, controllers); + let closed = false; + let failure: unknown; + let timer: ReturnType | undefined; + const check = async (): Promise => { + if (closed || controller.signal.aborted) return; + let timeout: ReturnType | undefined; + try { + await Promise.race([ + deps.check(id), + new Promise((_, reject) => { + timeout = setTimeout( + () => reject(Error("MCP authority check timed out")), + deps.intervalMs, + ); + }), + ]); + } catch (error) { + failure = error; + controller.abort(); + } finally { + if (timeout) clearTimeout(timeout); + } + if (!closed && !controller.signal.aborted) timer = setTimeout(check, deps.intervalMs); + }; + // Defer the first read until registration is visible to local cancellation. + const ready = Promise.resolve().then(check); + return { + signal: controller.signal, + ready, + throwIfFailed() { + if (failure !== undefined) throw failure; + }, + close() { + closed = true; + if (timer) clearTimeout(timer); + controllers.delete(controller); + if (!controllers.size && active.get(id) === controllers) active.delete(id); + }, + }; + }, + cancel(id: string) { + for (const controller of active.get(id) ?? []) controller.abort(); + }, + }; +} diff --git a/orchestrator/src/mcp/admission.ts b/orchestrator/src/mcp/admission.ts new file mode 100644 index 000000000..e7e3fac9d --- /dev/null +++ b/orchestrator/src/mcp/admission.ts @@ -0,0 +1,178 @@ +import { and, eq, inArray, or } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { + mcpAdmission as admission, + mcpAdmissionScope as scope, + mcpExecution as execution, + integrationConnection as connection, +} from "../db/schema.ts"; +import { MCP_LIMITS } from "./limits.ts"; +import type { JournalTx } from "./journal-review.ts"; + +export async function lockAdmissionScope(tx: JournalTx, id: string) { + // Serialize all zone deployments before their local scope and execution locks. + // ponytail: one short global lock; split into ordered session/connection scopes if measured load requires it. + for (const key of [...new Set(["mcp:global-admission", id])]) { + await tx.insert(scope).values({ id: key }).onConflictDoNothing(); + await tx.select().from(scope).where(eq(scope.id, key)).for("update"); + } +} +/** Both phases use the same durable capacity and retain their first queue deadline. */ +export function makeMcpAdmission(db = getDb()) { + return { + async reserve( + id: string, + deploymentId: string, + phase: "schema" | "execute", + now: Date, + toolMs = MCP_LIMITS.toolMs, + ): Promise<"queued" | "admitted"> { + if (!Number.isSafeInteger(toolMs) || toolMs < 1 || toolMs > MCP_LIMITS.maxToolMs) + throw new Error("invalid tool deadline"); + return db.transaction(async (tx) => { + await lockAdmissionScope(tx, deploymentId); + const [identity] = await tx.select().from(execution).where(eq(execution.id, id)); + if (!identity) throw new Error("call is not ready for admission"); + // The first admission INSERT also locks this parent for its FK check. + // Match cancellation: scope -> connection -> execution -> admission. + await tx + .select() + .from(connection) + .where(eq(connection.id, identity.connectionId)) + .for("share"); + const [call] = await tx.select().from(execution).where(eq(execution.id, id)).for("update"); + if ( + !call || + call.connectionId !== identity.connectionId || + call.sessionId !== identity.sessionId || + call.requestDigest !== identity.requestDigest || + call.configRevision !== identity.configRevision || + call.authGeneration !== identity.authGeneration || + !call.requestContext || + call.originalArguments === null || + call.deadline <= now || + (phase === "execute" + ? call.state !== "authorized" + : !["waiting_approval", "authorized"].includes(call.state)) + ) + throw new Error("call is not ready for admission"); + const [existing] = await tx.select().from(admission).where(eq(admission.executionId, id)); + if (existing && existing.deploymentId !== deploymentId) + throw new Error("admission deployment changed"); + if (phase === "execute" && existing?.schemaDigest !== call.requestDigest) + throw new Error("verified schema is required"); + if (phase === "schema" && existing?.schemaDigest) + throw new Error("schema already verified"); + if (existing?.state === "admitted") { + if (existing.phase !== phase || existing.expiresAt <= now) + throw new Error("admission lease unavailable"); + return "admitted"; + } + const queueExpiresAt = + (phase === "schema" ? existing?.schemaQueueExpiresAt : existing?.executeQueueExpiresAt) ?? + new Date(Math.min(call.deadline.getTime(), now.getTime() + MCP_LIMITS.queueMs)); + if (queueExpiresAt <= now) throw new Error("queue entry expired"); + const rows = await tx + .select() + .from(admission) + .where( + and( + or( + eq(admission.deploymentId, deploymentId), + eq(admission.sessionId, call.sessionId), + eq(admission.connectionId, call.connectionId), + ), + inArray(admission.state, ["queued", "admitted"]), + ), + ); + const active = rows.filter((row) => row.state === "admitted"); + const available = + active.filter((row) => row.deploymentId === deploymentId).length < + MCP_LIMITS.deploymentConcurrency && + active.filter((row) => row.connectionId === call.connectionId).length < + MCP_LIMITS.connectionConcurrency && + active.filter((row) => row.sessionId === call.sessionId).length < + MCP_LIMITS.sessionConcurrency && + (phase !== "schema" || + active.filter((row) => row.deploymentId === deploymentId && row.phase === "schema") + .length < 4); + if (!available && existing?.state === "queued") return "queued"; + if ( + !available && + rows.filter((row) => row.deploymentId === deploymentId && row.state === "queued") + .length >= MCP_LIMITS.queueSize + ) + throw new Error("MCP queue is full"); + const state = available ? "admitted" : "queued"; + const toolExpiresAt = + phase === "execute" && available + ? (existing?.executeExpiresAt ?? + new Date(Math.min(call.deadline.getTime(), now.getTime() + toolMs))) + : undefined; + if (toolExpiresAt && toolExpiresAt <= now) throw new Error("tool deadline expired"); + const values = { + phase, + state, + // Every newly admitted operation has a single-use generation, including schema retries. + reservationId: + existing?.state === "queued" ? existing.reservationId : crypto.randomUUID(), + expiresAt: available + ? (toolExpiresAt ?? new Date(Math.min(call.deadline.getTime(), now.getTime() + 10_000))) + : queueExpiresAt, + ...(toolExpiresAt ? { executeExpiresAt: toolExpiresAt } : {}), + ...(phase === "schema" + ? { schemaQueueExpiresAt: queueExpiresAt } + : { executeQueueExpiresAt: queueExpiresAt }), + } as const; + if (existing) await tx.update(admission).set(values).where(eq(admission.executionId, id)); + else + await tx.insert(admission).values({ + ...values, + executionId: id, + deploymentId, + sessionId: call.sessionId, + connectionId: call.connectionId, + createdAt: now, + }); + return state; + }); + }, + /** Only a verified worker response may supply success. No public route exposes this method. */ + async finishSchema( + id: string, + reservationId: string, + expiresAt: Date, + requestDigest: string, + success: boolean, + now: Date, + ) { + const [observed] = await db.select().from(admission).where(eq(admission.executionId, id)); + if (!observed) throw new Error("schema admission missing"); + return db.transaction(async (tx) => { + await lockAdmissionScope(tx, observed.deploymentId); + const [call] = await tx.select().from(execution).where(eq(execution.id, id)).for("update"); + const [lease] = await tx + .select() + .from(admission) + .where(eq(admission.executionId, id)) + .for("update"); + if ( + !call || + !["waiting_approval", "authorized"].includes(call.state) || + call.requestDigest !== requestDigest || + !lease || + lease.phase !== "schema" || + lease.state !== "admitted" || + lease.reservationId !== reservationId || + lease.expiresAt.getTime() !== expiresAt.getTime() || + (success && (lease.expiresAt <= now || call.deadline <= now)) + ) + throw new Error("stale schema admission"); + await tx + .update(admission) + .set({ state: "released", schemaDigest: success ? requestDigest : null }) + .where(and(eq(admission.executionId, id), eq(admission.reservationId, reservationId))); + }); + }, + }; +} diff --git a/orchestrator/src/mcp/approvals.ts b/orchestrator/src/mcp/approvals.ts new file mode 100644 index 000000000..1536e34d9 --- /dev/null +++ b/orchestrator/src/mcp/approvals.ts @@ -0,0 +1,8 @@ +export function assertOwnerOrAdmin( + ownerId: string | null, + actorId: string, + isAdmin: boolean, +): void { + if (!actorId || (!isAdmin && (!ownerId || ownerId !== actorId))) + throw new Error("session owner or administrator required"); +} diff --git a/orchestrator/src/mcp/arguments.ts b/orchestrator/src/mcp/arguments.ts new file mode 100644 index 000000000..39fa11322 --- /dev/null +++ b/orchestrator/src/mcp/arguments.ts @@ -0,0 +1,34 @@ +/** Exact decimal form, with bounded scale arithmetic and no expanded exponent. */ +function decimal(token: string, value: number): string { + const match = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(token); + if (!match) throw Error("invalid numeric argument"); + const fraction = match[3] ?? ""; + const digits = (match[2]! + fraction).replace(/^0+/, ""); + // Zero notation and sign are equivalent, including -0 serialized as 0. + if (!digits) return "0"; + if (value === 0) throw Error("numeric argument underflow"); + const exponent = Number(match[4] ?? "0"); + // A finite nonzero value cannot need a larger exponent after its digits cancel. + if (!Number.isSafeInteger(exponent) || Math.abs(exponent) > token.length + 400) + throw Error("numeric argument exponent exceeds limit"); + let end = digits.length; + while (end > 0 && digits[end - 1] === "0") end--; + const coefficient = digits.slice(0, end); + return `${match[1]}${coefficient}e${exponent - fraction.length + digits.length - coefficient.length}`; +} + +/** Preserve the decimal value that the worker will serialize, not binary exactness. */ +export function parseArgumentJson(raw: string): unknown { + return JSON.parse(raw, (_key, value: unknown, context?: { source?: string }) => { + if (typeof value === "number") { + if ( + typeof context?.source !== "string" || + !Number.isFinite(value) || + (Number.isInteger(value) && !Number.isSafeInteger(value)) || + decimal(context.source, value) !== decimal(JSON.stringify(value), value) + ) + throw Error("numeric argument loses precision"); + } + return value; + }); +} diff --git a/orchestrator/src/mcp/authority-error.ts b/orchestrator/src/mcp/authority-error.ts new file mode 100644 index 000000000..b809a64e0 --- /dev/null +++ b/orchestrator/src/mcp/authority-error.ts @@ -0,0 +1,2 @@ +/** An authoritative local mismatch, distinct from transport or storage failure. */ +export class McpAuthorityStale extends Error {} diff --git a/orchestrator/src/mcp/authority-notification.ts b/orchestrator/src/mcp/authority-notification.ts new file mode 100644 index 000000000..3b9b1efe5 --- /dev/null +++ b/orchestrator/src/mcp/authority-notification.ts @@ -0,0 +1,17 @@ +import { log } from "../log.ts"; + +// Notifications request a recheck; committed authority rows remain the source of truth. +const listeners = new Set<() => void>(); +export function observeMcpAuthorityChanges(listener: () => void): () => void { + listeners.add(listener); + return () => listeners.delete(listener); +} +export function notifyMcpAuthorityChanged(): void { + for (const listener of listeners) { + try { + listener(); + } catch { + log.warn("MCP authority notification unavailable"); + } + } +} diff --git a/orchestrator/src/mcp/binding-store.ts b/orchestrator/src/mcp/binding-store.ts new file mode 100644 index 000000000..e1d156de7 --- /dev/null +++ b/orchestrator/src/mcp/binding-store.ts @@ -0,0 +1,185 @@ +import { McpAuthorityStale } from "./authority-error.ts"; +import { makeSessionBindingResolver } from "./session-resolver.ts"; +import { makeSessionWritePolicy } from "./session-policy.ts"; +import { and, eq } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { + task, + taskSession, + profile, + integrationConnection, + mcpCatalog, + type IntegrationConnectionSnapshot, +} from "../db/schema.ts"; +import { canonicalDigest } from "./hash.ts"; +import { reviewCatalog } from "./catalog.ts"; +import { validateMcpConfig } from "./config.ts"; +import type { BoundSessionAuthority } from "./session-resolver.ts"; +import type { + McpSessionBinding, + McpSessionBindingSnapshot, + McpOperatorPolicy, + McpLiveAuthority, +} from "./session-bindings.ts"; + +/** Bindings remain in integration_connections; this envelope contains only session provenance. */ +export type McpBindingEnvelope = Omit; +export function bindingSnapshotColumns( + snapshot: McpSessionBindingSnapshot, + connections: readonly IntegrationConnectionSnapshot[], +) { + const { immutableBindings, ...mcpBindingEnvelope } = structuredClone(snapshot); + const integrationConnections = connections + .map((connection) => { + const binding = immutableBindings.find((binding) => binding.connectionId === connection.id); + if (connection.provider === "mcp" && !binding) return null; + return { ...structuredClone(connection), ...(binding ? { mcp: binding } : {}) }; + }) + .filter((connection) => connection !== null); + if ( + integrationConnections.filter((connection) => connection.mcp).length !== + immutableBindings.length + ) + throw new Error("MCP snapshot connection identity missing"); + return { mcpBindingEnvelope, integrationConnections }; +} +export function restoreBindingSnapshot( + envelope: McpBindingEnvelope, + connections: readonly IntegrationConnectionSnapshot[], +): McpSessionBindingSnapshot { + const immutableBindings: McpSessionBinding[] = []; + for (const connection of connections) { + if (connection.provider !== "mcp") continue; + const binding = connection.mcp; + if ( + !binding || + !("alias" in binding) || + !("operatorPolicy" in binding) || + binding.connectionId !== connection.id || + binding.alias !== connection.alias || + canonicalDigest(binding.config) !== canonicalDigest(connection.config) + ) + throw new McpAuthorityStale("MCP stored binding identity missing"); + immutableBindings.push(binding); + } + immutableBindings.sort((a, b) => + a.connectionId < b.connectionId ? -1 : a.connectionId > b.connectionId ? 1 : 0, + ); + const snapshot = { + ...structuredClone(envelope), + immutableBindings: structuredClone(immutableBindings), + }; + const { bindingHash, ...body } = snapshot; + if (canonicalDigest(body) !== bindingHash) + throw new McpAuthorityStale("MCP stored binding hash mismatch"); + return snapshot; +} +export async function readStoredBindingSession( + sessionId: string, + db = getDb(), +): Promise { + const rows = await db + .select({ + envelope: taskSession.mcpBindingEnvelope, + connections: taskSession.integrationConnections, + parentSessionId: task.spawningSessionId, + status: task.status, + }) + .from(taskSession) + .innerJoin(task, eq(task.id, taskSession.taskId)) + .where(eq(taskSession.sessionId, sessionId)) + .limit(2); + if (rows.length !== 1 || !rows[0]?.envelope) return null; + const row = rows[0]; + return { + snapshot: restoreBindingSnapshot(row.envelope!, row.connections ?? []), + parentSessionId: row.parentSessionId, + active: !["done", "failed", "cancelled"].includes(row.status), + }; +} + +/** Reads existing connection/catalog/profile authority. Operator policy has no admin-write path. */ +export async function readLiveBindingAuthority( + profileId: string, + operatorPolicies: readonly McpOperatorPolicy[], + db: ReturnType | import("./journal-review.ts").JournalTx = getDb(), +): Promise { + const [currentProfile] = await db.select().from(profile).where(eq(profile.id, profileId)); + if (!currentProfile || currentProfile.deletedAt) + throw new McpAuthorityStale("MCP profile unavailable"); + const rows = await db + .select({ connection: integrationConnection, catalog: mcpCatalog }) + .from(integrationConnection) + .leftJoin( + mcpCatalog, + and( + eq(mcpCatalog.connectionId, integrationConnection.id), + eq(mcpCatalog.kind, "published"), + eq(mcpCatalog.revision, integrationConnection.catalogRevision), + ), + ) + .where(eq(integrationConnection.provider, "mcp")); + const connections: McpLiveAuthority["connections"] = []; + for (const { connection: row, catalog } of rows) { + const available = + catalog && + catalog.configRevision === row.configRevision && + catalog.authGeneration === row.authGeneration && + catalog.credentialIdentity === row.credentialIdentity; + const tools = available ? reviewCatalog(catalog.tools) : []; + if (available && canonicalDigest(tools) !== canonicalDigest(catalog.tools)) + throw new McpAuthorityStale("MCP catalog fingerprint mismatch"); + connections.push({ + id: row.id, + catalogId: catalog?.id ?? "", + alias: row.alias, + config: validateMcpConfig(row.config), + enabled: row.enabled && Boolean(available) && !row.mcpCredential?.transition, + state: { + configRevision: row.configRevision, + authGeneration: row.authGeneration, + credentialIdentity: row.credentialIdentity, + catalogRevision: row.catalogRevision, + revoked: row.revoked, + }, + tools, + }); + } + return { + profileId, + profileGrants: currentProfile.integrationGrants, + profilePublication: currentProfile.mcpGrantPublication, + connections, + operatorPolicies: structuredClone([...operatorPolicies]), + }; +} + +/** Production composition must supply both operator policy and the real host enforcement fence. */ +export function makeStoredSessionBindingResolver( + deps: { + control: import("./control-client.ts").SessionPolicyControlClient; + readOperatorPolicies(): Promise; + enforceManagedDeny?: import("./session-resolver.ts").SessionBindingResolverDeps["enforceManagedDeny"]; + }, + db = getDb(), +) { + const policy = makeSessionWritePolicy(deps.control); + return makeSessionBindingResolver({ + readSession: (sessionId) => readStoredBindingSession(sessionId, db), + readAuthority: async (profileId) => + readLiveBindingAuthority(profileId, await deps.readOperatorPolicies(), db), + readBindingEpoch: async (sessionId) => { + const current = await policy.readForRequest({ sessionId }); + const epoch = current.ref?.bindingEpoch; + if ( + !current.current || + epoch === undefined || + epoch < 0n || + epoch > BigInt(Number.MAX_SAFE_INTEGER) + ) + throw new Error("MCP current binding epoch unavailable"); + return Number(epoch); + }, + ...(deps.enforceManagedDeny ? { enforceManagedDeny: deps.enforceManagedDeny } : {}), + }); +} diff --git a/orchestrator/src/mcp/bindings.ts b/orchestrator/src/mcp/bindings.ts new file mode 100644 index 000000000..8c0bb4ede --- /dev/null +++ b/orchestrator/src/mcp/bindings.ts @@ -0,0 +1,133 @@ +import type { ProfileIntegrationGrant } from "../db/schema.ts"; +import { validateMcpConfig } from "./config.ts"; +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import { MCP_LIMITS } from "./limits.ts"; +import { + validateMcpGrant, + type McpBinding, + type McpConnectionState, + type McpReviewedTool, +} from "./types.ts"; + +export function compileBindings( + connections: readonly { + id: string; + alias: string; + config: unknown; + state: McpConnectionState; + enabled: boolean; + tools: readonly McpReviewedTool[]; + grants: readonly ProfileIntegrationGrant[]; + }[], +): McpBinding[] { + if (connections.length > MCP_LIMITS.connectionsPerSession) + throw new Error("too many MCP connections"); + const ids = new Set(); + const names = new Set(); + const bindings = connections.map((connection) => { + if (ids.has(connection.id)) throw new Error("duplicate MCP connection"); + ids.add(connection.id); + const config = validateMcpConfig(connection.config); + if ( + (config.auth.kind === "api_key" && !config.auth.secretRef) || + !connection.enabled || + connection.state.revoked || + !connection.state.credentialIdentity || + connection.state.catalogRevision < 1 + ) + throw new Error("MCP connection is unavailable"); + if (!/^[a-z][a-z0-9-]{1,62}$/.test(connection.alias)) + throw new Error("invalid connection alias"); + const granted = new Set(); + const tools = connection.grants.map((grant) => { + validateMcpGrant(grant); + if (grant.connectionId !== connection.id || granted.has(grant.mcp!.upstreamName)) + throw new Error("invalid or duplicate bound grant"); + granted.add(grant.mcp!.upstreamName); + const tool = connection.tools.find((t) => t.upstreamName === grant.mcp!.upstreamName); + if (!tool?.enabled) throw new Error("reviewed tool is unavailable"); + const name = `mcp_${connection.alias.replaceAll("-", "_").slice(0, 24)}_${canonicalDigest([connection.id, tool.upstreamName]).slice(7, 27)}`; + if (names.has(name)) throw new Error("tool name collision"); + names.add(name); + return { name, tool, grant: grant.mcp! }; + }); + const binding = { + connectionId: connection.id, + ...connection.state, + config, + tools, + policyHash: canonicalDigest(connection.grants), + }; + return structuredClone({ ...binding, hash: canonicalDigest(binding) }); + }); + if ( + names.size > MCP_LIMITS.toolsPerSession || + Buffer.byteLength(canonicalJson(bindings)) > MCP_LIMITS.manifestBytes + ) + throw new Error("MCP manifest exceeds session limits"); + return bindings; +} + +export function assertBindingCurrent( + binding: McpBinding, + live: McpConnectionState & { enabled: boolean }, +): void { + if ( + !live.enabled || + live.revoked || + binding.configRevision !== live.configRevision || + binding.authGeneration !== live.authGeneration || + binding.credentialIdentity !== live.credentialIdentity || + binding.catalogRevision !== live.catalogRevision + ) + throw new Error("MCP binding is stale; explicit rebind is required"); +} + +/** Child authority can only narrow the parent's effective tool grant. */ +export function intersectMcpGrants( + parent: readonly ProfileIntegrationGrant[], + requested: readonly ProfileIntegrationGrant[], +): ProfileIntegrationGrant[] { + return requested.map((child) => { + validateMcpGrant(child); + const inherited = parent.find( + (p) => + p.connectionId === child.connectionId && p.mcp?.upstreamName === child.mcp!.upstreamName, + ); + if (!inherited) throw new Error("child tool grant exceeds parent authority"); + validateMcpGrant(inherited); + const unrestricted = inherited.mcp!.unrestricted && child.mcp!.unrestricted; + // Keep each root intact: allOf wrapping changes local references and resource scope. + const seen = new Set(); + const argumentConstraintDocuments = [ + ...inherited.mcp!.argumentConstraintDocuments, + ...child.mcp!.argumentConstraintDocuments, + ].filter((document) => { + const digest = canonicalDigest(document); + if (seen.has(digest)) return false; + seen.add(digest); + return true; + }); + const effective = structuredClone({ + ...child, + mcp: { + ...child.mcp!, + automationWrite: inherited.mcp!.automationWrite && child.mcp!.automationWrite, + unrestricted, + argumentConstraintDocuments, + }, + }); + validateMcpGrant(effective); + return effective; + }); +} + +/** Compare only authorization data, so profile display edits do not revoke tools. */ +export function assertBindingGrantsCurrent( + binding: McpBinding, + liveGrants: readonly ProfileIntegrationGrant[], +): void { + const grants = liveGrants.filter((grant) => grant.connectionId === binding.connectionId); + if (canonicalDigest(grants) !== binding.policyHash) + throw new Error("MCP grant policy changed; explicit rebind is required"); +} diff --git a/orchestrator/src/mcp/catalog.ts b/orchestrator/src/mcp/catalog.ts new file mode 100644 index 000000000..998252735 --- /dev/null +++ b/orchestrator/src/mcp/catalog.ts @@ -0,0 +1,110 @@ +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import { MCP_LIMITS } from "./limits.ts"; +import type { McpReviewedTool } from "./types.ts"; + +/** Bound schema traversal before disposable-worker compilation. No remote resolution. */ +export function assertSchemaShape(schema: unknown): asserts schema is Record { + if (!schema || typeof schema !== "object" || Array.isArray(schema)) + throw new Error("schema must be an object"); + if (Buffer.byteLength(canonicalJson(schema)) > MCP_LIMITS.argumentBytes) + throw new Error("schema is too large"); + const schemaMaps = new Set([ + "properties", + "patternProperties", + "$defs", + "definitions", + "dependentSchemas", + ]); + const schemaLists = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]); + const schemaValues = new Set([ + "additionalProperties", + "unevaluatedProperties", + "propertyNames", + "contains", + "not", + "if", + "then", + "else", + "additionalItems", + "unevaluatedItems", + "contentSchema", + ]); + let nodes = 0; + function walk(v: unknown, depth: number, schemaNode: boolean): void { + if (++nodes > 10_000 || depth > 64) throw new Error("schema is too complex"); + if (!v || typeof v !== "object") return; + if (Array.isArray(v)) { + for (const child of v) walk(child, depth + 1, false); + return; + } + for (const [key, child] of Object.entries(v)) { + if ( + schemaNode && + ["$ref", "$dynamicRef", "$recursiveRef"].includes(key) && + (typeof child !== "string" || !child.startsWith("#")) + ) + throw new Error("schema references must be local"); + if ( + schemaNode && + key === "$schema" && + ![ + "http://json-schema.org/draft-07/schema#", + "https://json-schema.org/draft/2020-12/schema", + ].includes(String(child)) + ) + throw new Error("unsupported schema dialect"); + // Schema-map keys are property names. Annotation values are data. + if ( + schemaNode && + (schemaMaps.has(key) || key === "dependencies") && + child && + typeof child === "object" && + !Array.isArray(child) + ) { + for (const sub of Object.values(child)) walk(sub, depth + 2, !Array.isArray(sub)); + } else if (schemaNode && (schemaLists.has(key) || key === "items") && Array.isArray(child)) { + for (const sub of child) walk(sub, depth + 2, true); + } else walk(child, depth + 1, schemaNode && (schemaValues.has(key) || key === "items")); + } + } + walk(schema, 0, true); +} + +export function reviewCatalog(input: readonly unknown[]): McpReviewedTool[] { + if (input.length > MCP_LIMITS.toolsPerSession) throw new Error("too many tools"); + const names = new Set(); + const tools = input.map((raw) => { + if (!raw || typeof raw !== "object") throw new Error("invalid reviewed tool"); + const t = raw as Record; + if ( + typeof t.upstreamName !== "string" || + !t.upstreamName || + t.upstreamName.length > 256 || + /[\u0000-\u001f]/.test(t.upstreamName) || + names.has(t.upstreamName) + ) + throw new Error("invalid or duplicate tool name"); + names.add(t.upstreamName); + if ( + typeof t.description !== "string" || + t.description.length > 16_384 || + !["read", "write"].includes(String(t.classification)) || + typeof t.enabled !== "boolean" + ) + throw new Error("each tool needs explicit review"); + assertSchemaShape(t.inputSchema); + if (t.outputSchema !== undefined) assertSchemaShape(t.outputSchema); + const tool = { + upstreamName: t.upstreamName, + description: t.description, + inputSchema: t.inputSchema, + ...(t.outputSchema === undefined ? {} : { outputSchema: t.outputSchema }), + classification: t.classification as "read" | "write", + enabled: t.enabled, + }; + return structuredClone({ ...tool, fingerprint: canonicalDigest(tool) }); + }); + if (Buffer.byteLength(canonicalJson(tools)) > MCP_LIMITS.manifestBytes) + throw new Error("catalog is too large"); + return tools; +} diff --git a/orchestrator/src/mcp/config.ts b/orchestrator/src/mcp/config.ts new file mode 100644 index 000000000..e92f8976c --- /dev/null +++ b/orchestrator/src/mcp/config.ts @@ -0,0 +1,47 @@ +import { isIP } from "node:net"; +import { z } from "zod"; + +// Naming convention only. Coordinator reservation metadata protects the sealed secret. +const reference = z.string().regex(/^mcp[.-][a-zA-Z0-9_.-]{1,160}$/); +const configSchema = z + .object({ + endpoint: z.string().max(2048), + zone: z.string().regex(/^[a-z][a-z0-9-]{0,62}$/), + auth: z.discriminatedUnion("kind", [ + z + .object({ kind: z.literal("api_key"), secretRef: z.union([reference, z.literal("")]) }) + .strict(), + z + .object({ + kind: z.literal("oauth"), + clientRegistrationId: z.string().min(1).max(128), + shareWithOrgSessions: z.literal(true), + }) + .strict(), + ]), + }) + .strict(); + +export type McpConfig = z.infer; + +/** This validates stored shape. The worker also enforces operator DNS/zone policy. */ +export function validateMcpConfig(input: unknown): McpConfig { + const config = configSchema.parse(input); + const url = new URL(config.endpoint); + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash || + isIP(url.hostname.replace(/^\[|\]$/g, "")) !== 0 || + url.href !== config.endpoint || + url.hostname.length > 253 || + !url.hostname.split(".").every((label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label)) + ) { + throw new Error( + "MCP endpoint must be a canonical HTTPS URL with a DNS hostname and no credentials, query, or fragment", + ); + } + return config; +} diff --git a/orchestrator/src/mcp/contracts.ts b/orchestrator/src/mcp/contracts.ts new file mode 100644 index 000000000..96a6dc2ba --- /dev/null +++ b/orchestrator/src/mcp/contracts.ts @@ -0,0 +1,612 @@ +import { sessionMcpState, type McpReviewProducers } from "./review-state.ts"; +import { create, toBinary, toJson } from "@bufbuild/protobuf"; +import { + McpApprovalReviewContextSchema, + McpCompletionSettlement, + GetMcpExecutionResultResponseSchema, + McpRebindDrainSchema, + McpRebindDrainReason, +} from "../gen/engram/app/v1/integration_pb.ts"; +import { McpPublicationRequired, McpRebindDraining, type McpRebindDrain } from "./rebind.ts"; +import { policyReviewCurrent, type JournalReviewContext } from "./journal-review.ts"; +import { credentialHandlers, requireMcpHumanAdmin } from "./credential-rpc.ts"; +import type { CredentialAdmin } from "./credential-admin.ts"; +import { Code, ConnectError, type ServiceImpl } from "@connectrpc/connect"; +import type { IntegrationService } from "../gen/engram/app/v1/integration_pb.ts"; +import type { + IntegrationConnectionRow, + IntegrationConnectionStore, +} from "../db/integration-connections.ts"; +import { requireUser, type GetSession } from "../rpc/require.ts"; +import { validateMcpConfig, type McpConfig } from "./config.ts"; +import type { McpRepository } from "./repository.ts"; +import type { McpJournal, McpUnknownContext } from "./journal.ts"; +import type { McpDiscoveredTool, McpReviewedTool } from "./types.ts"; +import { assertSchemaShape } from "./catalog.ts"; +import { MCP_LIMITS } from "./limits.ts"; +import { canonicalJson } from "./hash.ts"; +import { verifiedExecutionResult } from "./effect-driver.ts"; +import { parseArgumentJson } from "./arguments.ts"; + +/** Internal services use trusted connection identities, never guest destinations. */ +export interface McpServices { + enabled: boolean; + repository: McpRepository; + journal: McpJournal; + reviewProducers?: McpReviewProducers; + sessionOwner?: (sessionId: string) => Promise; + validateDestination?: (config: McpConfig) => Promise; + validateSchemas?: ( + tools: readonly McpReviewedTool[], + connection: IntegrationConnectionRow, + actorId: string, + ) => Promise; + discovery?: { + discover(connection: IntegrationConnectionRow, actorId: string): Promise; + }; + credentials?: CredentialAdmin; + destinations?: { zoneId: string; endpoint: string }[]; + runtime?: { + cancel(executionId: string): Promise; + /** Both operations authorize a fresh owner/admin; commit requires the reviewed candidate. */ + prepareRebind( + sessionId: string, + expectedBindingHash: string, + actorId: string, + ): Promise< + import("./session-bindings.ts").McpSessionBindingSnapshot & { drain?: McpRebindDrain } + >; + rebind( + sessionId: string, + expectedBindingHash: string, + expectedCandidateHash: string, + actorId: string, + ): Promise<{ bindingHash: string }>; + }; +} +export function requireMcpServices(services: McpServices | undefined): McpServices { + return requireMcpHook(services); +} +export function requireMcp(services: McpServices | undefined): McpServices { + if (!services?.enabled) + throw new ConnectError("external MCP is disabled", Code.FailedPrecondition); + return services; +} +export function requireMcpHook(hook: T | undefined): T { + if (!hook) + throw new ConnectError("external MCP service is not configured", Code.FailedPrecondition); + return hook; +} +function drainProto(drain: McpRebindDrain) { + const reason = + drain.reason === "dispatch_in_progress" + ? McpRebindDrainReason.DISPATCH_IN_PROGRESS + : drain.reason === "unknown_intent" + ? McpRebindDrainReason.UNKNOWN_INTENT + : McpRebindDrainReason.LEGACY_INTENT; + return { reason, retryAfter: drain.retryAfter }; +} +export async function mcpOperation(run: () => Promise): Promise { + try { + return await run(); + } catch (error) { + if (error instanceof ConnectError) throw error; + if (error instanceof McpPublicationRequired) + throw new ConnectError( + "An administrator must publish MCP grants before rebind", + Code.FailedPrecondition, + ); + if (error instanceof McpRebindDraining) + throw new ConnectError("MCP calls are still draining", Code.FailedPrecondition, undefined, [ + { desc: McpRebindDrainSchema, value: drainProto(error.drain) }, + ]); + const message = error instanceof Error ? error.message : "MCP operation failed"; + const code = /owner or administrator/.test(message) + ? Code.PermissionDenied + : /stale|already/.test(message) + ? Code.Aborted + : Code.FailedPrecondition; + // Internal worker errors can contain upstream metadata. Return a stable error. + throw new ConnectError( + code === Code.PermissionDenied + ? "session owner or administrator required" + : code === Code.Aborted + ? "MCP state changed; reload and try again" + : "MCP operation could not be completed", + code, + ); + } +} +export interface McpProtoConfig { + endpoint: string; + zone: string; + authKind: string; + apiKeySecretRef: string; + oauthClientRegistrationId: string; + shareWithOrgSessions: boolean; +} +export function mcpConfigFromProto(value: McpProtoConfig | undefined): McpConfig { + if (!value) throw new ConnectError("MCP config is required", Code.InvalidArgument); + if ( + (value.authKind === "api_key" && + (value.oauthClientRegistrationId || value.shareWithOrgSessions)) || + (value.authKind === "oauth" && value.apiKeySecretRef) + ) + throw new ConnectError("conflicting MCP credential configuration", Code.InvalidArgument); + try { + return validateMcpConfig({ + endpoint: value.endpoint, + zone: value.zone, + auth: + value.authKind === "api_key" + ? { kind: value.authKind, secretRef: value.apiKeySecretRef } + : { + kind: value.authKind, + clientRegistrationId: value.oauthClientRegistrationId, + shareWithOrgSessions: value.shareWithOrgSessions, + }, + }); + } catch { + throw new ConnectError("invalid MCP configuration", Code.InvalidArgument); + } +} +export function mcpConfigToProto(raw: unknown): McpProtoConfig { + const value = validateMcpConfig(raw); + return { + endpoint: value.endpoint, + zone: value.zone, + authKind: value.auth.kind, + apiKeySecretRef: value.auth.kind === "api_key" ? value.auth.secretRef : "", + oauthClientRegistrationId: value.auth.kind === "oauth" ? value.auth.clientRegistrationId : "", + shareWithOrgSessions: value.auth.kind === "oauth" && value.auth.shareWithOrgSessions, + }; +} +function catalogToProto(value: Awaited>) { + if ( + !Array.isArray(value.tools) || + value.tools.length > MCP_LIMITS.toolsPerSession || + Buffer.byteLength(JSON.stringify(value.tools)) > MCP_LIMITS.manifestBytes + ) + throw new Error("catalog is too large"); + for (const tool of value.tools) { + if (typeof tool.upstreamName !== "string" || typeof tool.description !== "string") + throw new Error("invalid catalog"); + assertSchemaShape(tool.inputSchema); + if (tool.outputSchema !== undefined) assertSchemaShape(tool.outputSchema); + } + return { + id: value.id, + connectionId: value.connectionId, + kind: value.kind, + revision: value.revision, + configRevision: value.configRevision, + authGeneration: value.authGeneration, + fingerprint: value.fingerprint, + tools: value.tools.map((t) => ({ + upstreamName: t.upstreamName, + description: t.description, + inputSchemaJson: JSON.stringify(t.inputSchema), + outputSchemaJson: t.outputSchema ? JSON.stringify(t.outputSchema) : "", + classification: "classification" in t ? String(t.classification) : "", + enabled: "enabled" in t && t.enabled === true, + fingerprint: "fingerprint" in t ? String(t.fingerprint) : "", + })), + }; +} +function parseSchema(value: string): Record { + try { + if (Buffer.byteLength(value) > MCP_LIMITS.argumentBytes) throw new Error("schema is too large"); + const schema = parseArgumentJson(value); + if (!schema || typeof schema !== "object" || Array.isArray(schema)) throw new Error(); + return schema as Record; + } catch { + throw new ConnectError("schema must be a JSON object", Code.InvalidArgument); + } +} +type Methods = + | "putMcpCredential" + | "getMcpSetup" + | "completeMcpAuthorization" + | "recoverMcpCredential" + | "getMcpCatalogs" + | "discoverMcp" + | "publishMcp" + | "authorizeMcp" + | "disconnectMcp" + | "getSessionMcpState" + | "getMcpExecutionResult" + | "listMcpApprovals" + | "decideMcpApproval" + | "cancelMcpCall" + | "resolveMcpOutcome" + | "prepareMcpRebind" + | "rebindMcp"; +export function mcpHandlers(deps: { + services?: McpServices; + connections: IntegrationConnectionStore; + getSession: GetSession; + now: () => Date; + connectionToProto: (row: IntegrationConnectionRow) => { id: string }; +}): Pick, Methods> { + const services = () => requireMcp(deps.services); + const durableServices = () => requireMcpServices(deps.services); + async function connection(id: string) { + const row = await deps.connections.get(id); + if (!row || row.provider !== "mcp" || !row.mcp) + throw new ConnectError("MCP connection not found", Code.NotFound); + return row; + } + return { + ...credentialHandlers(deps), + async getMcpCatalogs(req, ctx) { + await requireMcpHumanAdmin(ctx, deps.getSession); + const repository = requireMcpHook(deps.services?.repository); + return mcpOperation(async () => { + const row = await connection(req.connectionId); + const [draft, published] = await Promise.all([ + repository.getLatestDraft(row.id), + row.mcp!.catalogRevision ? repository.getCatalog(row.id, row.mcp!.catalogRevision) : null, + ]); + const result = { + configRevision: row.mcp!.configRevision, + catalogRevision: row.mcp!.catalogRevision, + ...(draft ? { draft: catalogToProto(draft) } : {}), + ...(published ? { published: catalogToProto(published) } : {}), + }; + if (Buffer.byteLength(JSON.stringify(result)) > MCP_LIMITS.responseBytes) + throw new Error("catalog response is too large"); + return result; + }); + }, + async discoverMcp(req, ctx) { + const actor = await requireMcpHumanAdmin(ctx, deps.getSession); + const service = services(); + const discover = requireMcpHook(service.discovery); + return mcpOperation(async () => { + const row = await connection(req.connectionId); + if (row.mcp!.configRevision !== req.expectedRevision) throw new Error("stale connection"); + if (row.mcp!.revoked || !row.mcp!.credentialIdentity) + throw new Error("credential setup required"); + const tools = await discover.discover(row, actor.id); + return { + catalog: catalogToProto( + await service.repository.saveDraft(row.id, row.mcp!, tools, actor.id, deps.now()), + ), + }; + }); + }, + async publishMcp(req, ctx) { + const actor = await requireMcpHumanAdmin(ctx, deps.getSession); + const service = services(); + const validate = requireMcpHook(service.validateSchemas); + const { reviewCatalog } = await import("./catalog.ts"); + let tools: McpReviewedTool[]; + try { + tools = reviewCatalog( + req.tools.map((t) => ({ + upstreamName: t.upstreamName, + description: t.description, + inputSchema: parseSchema(t.inputSchemaJson), + ...(t.outputSchemaJson ? { outputSchema: parseSchema(t.outputSchemaJson) } : {}), + classification: t.classification, + enabled: t.enabled, + })), + ); + } catch { + throw new ConnectError("invalid reviewed tools", Code.InvalidArgument); + } + return mcpOperation(async () => { + await validate(tools, await connection(req.connectionId), actor.id); + return { + catalog: catalogToProto( + await service.repository.publish( + req.connectionId, + req.draftId, + req.expectedRevision, + tools, + actor.id, + deps.now(), + ), + ), + }; + }); + }, + async getMcpExecutionResult(req, ctx) { + const actor = await requireUser(ctx, deps.getSession); + const service = durableServices(); + return mcpOperation(async () => { + const call = await service.journal.readExecutionResult( + req.sessionId, + req.executionId, + actor.id, + ); + let resultJson: string | undefined; + if (call.result !== null) { + resultJson = canonicalJson(call.result); + if (Buffer.byteLength(resultJson) > MCP_LIMITS.resultBytes) + throw new Error("result exceeds limit"); + if (call.result.source === "local_validation") { + const expected = { + source: "local_validation", + delivery: "available", + output: { + content: [ + { type: "text", text: "Tool arguments do not satisfy the approved schemas." }, + ], + isError: true, + }, + }; + if ( + resultJson !== canonicalJson(expected) || + call.state !== "failed" || + call.resultUnavailable + ) + throw new Error("invalid local result"); + } else { + const verified = verifiedExecutionResult(call.result); + if ( + verified.state !== call.state || + verified.resultUnavailable !== call.resultUnavailable + ) + throw new Error("invalid result state"); + } + } + const terminal = ["succeeded", "failed", "canceled", "outcome_unknown"].includes( + call.state, + ); + const response = { + state: call.state, + resultUnavailable: call.resultUnavailable, + resultJson, + settlement: !terminal + ? McpCompletionSettlement.NOT_APPLICABLE + : call.completionEnqueuedVersion >= call.version + ? McpCompletionSettlement.SETTLED_OR_TARGET_GONE + : McpCompletionSettlement.PENDING, + }; + const message = create(GetMcpExecutionResultResponseSchema, response); + const size = Math.max( + toBinary(GetMcpExecutionResultResponseSchema, message).byteLength, + Buffer.byteLength(JSON.stringify(toJson(GetMcpExecutionResultResponseSchema, message))), + ); + if (size > MCP_LIMITS.responseBytes) throw new Error("result response exceeds limit"); + return response; + }); + }, + async getSessionMcpState(req, ctx) { + const actor = await requireUser(ctx, deps.getSession); + const service = durableServices(); + return mcpOperation(() => + sessionMcpState({ + journal: service.journal, + sessionId: req.sessionId, + actorId: actor.id, + now: deps.now(), + after: req.afterExecutionId || undefined, + limit: req.pageSize || undefined, + planToolCallId: req.planToolCallId, + producers: service.reviewProducers, + dispatchEnabled: service.enabled, + canCancel: true, + canRebind: + service.enabled && Boolean(service.runtime?.prepareRebind && service.runtime?.rebind), + afterBindingToolName: req.afterBindingToolName || undefined, + bindingPageSize: req.bindingPageSize || undefined, + omitBindings: req.omitBindings, + omitExecutions: req.omitExecutions, + }), + ); + }, + async listMcpApprovals(req, ctx) { + const actor = await requireUser(ctx, deps.getSession); + const service = durableServices(); + // Use the same bounded read; a continuation is available from GetSessionMcpState. + return mcpOperation(async () => { + const state = await sessionMcpState({ + journal: service.journal, + sessionId: req.sessionId, + actorId: actor.id, + now: deps.now(), + after: req.afterExecutionId || undefined, + producers: service.reviewProducers, + dispatchEnabled: service.enabled, + canCancel: true, + omitBindings: true, + }); + return { approvals: state.approvals, nextExecutionId: state.nextExecutionId }; + }); + }, + async decideMcpApproval(req, ctx) { + const actor = await requireUser(ctx, deps.getSession); + const service = services(); + if (!req.expectedReviewContext) + throw new ConnectError("exact review context is required", Code.InvalidArgument); + // Copy known fields only. Protobuf runtime metadata is not review identity. + const expected = toJson(McpApprovalReviewContextSchema, req.expectedReviewContext, { + alwaysEmitImplicit: true, + }) as JournalReviewContext; + return mcpOperation(async () => { + await service.journal.readSession(expected.sessionId, actor.id, { limit: 1 }); + const producers = requireMcpHook(service.reviewProducers); + const current = await producers.bindings.readCurrent(expected.sessionId); + const policy = await requireMcpHook(producers.policy.readWritePolicy)({ + protocol: 1, + sessionId: expected.sessionId, + }); + if ( + current.bindingHash !== expected.bindingHash || + !policy.state?.policy?.ref || + policy.state.policy.ref.policyRevision?.toString() !== expected.policyRevision || + policy.state.policy.ref.bindingEpoch?.toString() !== expected.bindingEpoch + ) + throw new Error("stale review policy"); + const call = await service.journal.get(req.executionId); + if (!call?.requestContext || !policyReviewCurrent(call.requestContext, policy.state.policy)) + throw new Error("stale review policy lineage"); + if ( + req.approve && + (!policy.state.policy.current || + !policy.state.policy.initialized || + policy.state.policy.writeEligibility !== "write_eligible") + ) + throw new Error("current policy denies writes"); + const result = await service.journal.decide( + req.executionId, + actor.id, + req.approve, + expected, + deps.now(), + ); + return { + state: result.call.state, + receipt: { + executionId: result.call.id, + decision: result.approval.decision, + actorId: result.approval.actorId!, + decidedAt: result.approval.decidedAt!.toISOString(), + journalRevision: result.call.version, + reviewedContext: result.reviewedContext, + }, + }; + }); + }, + async cancelMcpCall(req, ctx) { + const actor = await requireUser(ctx, deps.getSession); + const service = durableServices(); + return mcpOperation(async () => { + if (!req.expectedJournalRevision) + throw new ConnectError("expected journal revision is required", Code.InvalidArgument); + const call = await service.journal.cancel( + req.executionId, + actor.id, + deps.now(), + req.expectedJournalRevision, + ); + // The committed cancellation fences other replicas even if local I/O already ended. + // This receipt proves durable cancellation only. Local I/O abort is best effort. + await service.runtime?.cancel(call.id); + const receipt = await service.journal.cancellationReceipt(call.id); + if (!receipt) throw Error("cancellation receipt unavailable"); + return { + state: call.state, + receipt: { + ...receipt, + actorId: receipt.actorId ?? "", + createdAt: receipt.createdAt.toISOString(), + }, + }; + }); + }, + async resolveMcpOutcome(req, ctx) { + const actor = await requireUser(ctx, deps.getSession); + const service = durableServices(); + if (actor.serviceAccount) + throw new ConnectError("a human decision is required", Code.PermissionDenied); + if (req.outcome !== "succeeded" && req.outcome !== "failed" && req.outcome !== "acknowledged") + throw new ConnectError("invalid outcome resolution", Code.InvalidArgument); + const outcome = req.outcome; + if (!req.expectedUnknownContext) + throw new ConnectError("exact unknown context is required", Code.InvalidArgument); + const context = req.expectedUnknownContext; + const expected: McpUnknownContext = { + executionId: context.executionId, + journalRevision: context.journalRevision, + requestDigest: context.requestDigest, + bindingHash: context.bindingHash, + intentId: context.intentId, + }; + return mcpOperation(async () => { + const receipt = await service.journal.resolveUnknown( + req.executionId, + actor.id, + outcome, + deps.now(), + expected, + ); + return { + outcome: receipt.outcome, + receipt: { + ...receipt, + createdAt: receipt.createdAt.toISOString(), + expectedContext: receipt.expectedContext ?? undefined, + }, + }; + }); + }, + async prepareMcpRebind(req, ctx) { + const actor = await requireUser(ctx, deps.getSession); + if (actor.serviceAccount) + throw new ConnectError("a human decision is required", Code.PermissionDenied); + const runtime = requireMcpHook(services().runtime); + if (!req.expectedBindingHash) + throw new ConnectError("expected binding hash is required", Code.InvalidArgument); + return mcpOperation(async () => { + const snapshot = await runtime.prepareRebind( + req.sessionId, + req.expectedBindingHash, + actor.id, + ); + const bindingTools = snapshot.immutableBindings + .flatMap((binding) => + binding.tools.map((bound) => ({ + connectionId: binding.connectionId, + connectionAlias: binding.alias, + modelToolName: bound.name, + reviewedTool: { + ...bound.tool, + inputSchemaJson: JSON.stringify(bound.tool.inputSchema), + outputSchemaJson: bound.tool.outputSchema + ? JSON.stringify(bound.tool.outputSchema) + : "", + }, + argumentConstraintDocumentsJson: bound.grant.argumentConstraintDocuments.map( + (document) => JSON.stringify(document), + ), + automationWrite: bound.grant.automationWrite, + unrestricted: bound.grant.unrestricted, + catalogId: binding.catalogId, + catalogRevision: binding.catalogRevision, + configRevision: binding.configRevision, + authGeneration: binding.authGeneration, + })), + ) + .sort((a, b) => + a.modelToolName < b.modelToolName ? -1 : a.modelToolName > b.modelToolName ? 1 : 0, + ); + return { + bindingHash: snapshot.bindingHash, + bindingEpoch: String(snapshot.bindingEpoch), + bindingTools, + drain: snapshot.drain ? drainProto(snapshot.drain) : undefined, + denyHosts: snapshot.managedDenyPolicy.hosts, + denyRevision: String(snapshot.managedDenyPolicy.revision), + denyDigest: snapshot.managedDenyPolicy.digest, + connections: snapshot.immutableBindings.map((binding) => ({ + connectionId: binding.connectionId, + alias: binding.alias, + endpoint: binding.config.endpoint, + zone: binding.config.zone, + credentialIdentity: binding.credentialIdentity, + configRevision: binding.configRevision, + authGeneration: binding.authGeneration, + catalogRevision: binding.catalogRevision, + })), + }; + }); + }, + async rebindMcp(req, ctx) { + const actor = await requireUser(ctx, deps.getSession); + if (actor.serviceAccount) + throw new ConnectError("a human decision is required", Code.PermissionDenied); + const runtime = requireMcpHook(services().runtime); + if (!req.expectedBindingHash || !req.expectedCandidateHash) + throw new ConnectError( + "binding and reviewed candidate hashes are required", + Code.InvalidArgument, + ); + return mcpOperation(() => + runtime.rebind(req.sessionId, req.expectedBindingHash, req.expectedCandidateHash, actor.id), + ); + }, + }; +} diff --git a/orchestrator/src/mcp/control-client.ts b/orchestrator/src/mcp/control-client.ts new file mode 100644 index 000000000..8d5f35479 --- /dev/null +++ b/orchestrator/src/mcp/control-client.ts @@ -0,0 +1,100 @@ +import { createClient, ConnectError, Code, type Client } from "@connectrpc/connect"; +import { createGrpcTransport } from "@connectrpc/connect-node"; +import { checkServerIdentity } from "node:tls"; +import { + CredentialControlService, + type CredentialStatus, +} from "../gen/engram/mcp/v1/control_pb.ts"; + +import { SessionPolicyService } from "../gen/engram/mcp/v1/session_policy_pb.ts"; + +type CredentialWireClient = Client; +type StatusMethod = "putSecret" | "completeAuthorization" | "getStatus" | "revoke"; +export type CredentialControlClient = Omit & { + [K in StatusMethod]: (...args: Parameters) => Promise; +}; +/** Keep one status DTO at the lifecycle boundary; missing wire status is never ready. */ +export function unwrapCredentialControlClient(wire: CredentialWireClient): CredentialControlClient { + function status(response: { status?: CredentialStatus }): CredentialStatus { + if (!response.status) + throw new ConnectError("credential control status unavailable", Code.DataLoss); + return response.status; + } + return { + ...wire, + putSecret: async (...args) => status(await wire.putSecret(...args)), + completeAuthorization: async (...args) => status(await wire.completeAuthorization(...args)), + getStatus: async (...args) => status(await wire.getStatus(...args)), + revoke: async (...args) => status(await wire.revoke(...args)), + }; +} +export type SessionPolicyControlClient = Client; +export interface CredentialControlTls { + url: string; + ca: string; + cert: string; + key: string; + serverFingerprint: string; +} +/** Dedicated transport: explicit trust roots, both certificate identities, no retries. */ +export function makeCredentialControlClient(tls: CredentialControlTls): CredentialControlClient { + return unwrapCredentialControlClient( + createClient(CredentialControlService, makeControlTransport(tls, 32768)), + ); +} +export function makeSessionPolicyControlClient( + tls: CredentialControlTls, +): Client { + // Match the coordinator policy service: bounded JSON commands can occupy 8 MiB. + return createClient(SessionPolicyService, makeControlTransport(tls, 9 * 1024 * 1024)); +} +function makeControlTransport(tls: CredentialControlTls, maximumBytes: number) { + const url = new URL(tls.url); + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash || + url.pathname !== "/" || + !tls.ca || + !tls.cert || + !tls.key || + !/^(?:[0-9A-F]{2}:){31}[0-9A-F]{2}$/.test(tls.serverFingerprint) + ) + throw new Error("MCP credential control requires pinned mTLS"); + return createGrpcTransport({ + baseUrl: tls.url, + readMaxBytes: maximumBytes, + writeMaxBytes: maximumBytes, + defaultTimeoutMs: 30000, + idleConnectionTimeoutMs: 1000, + nodeOptions: { + ca: tls.ca, + cert: tls.cert, + key: tls.key, + minVersion: "TLSv1.3", + rejectUnauthorized: true, + checkServerIdentity(host, cert) { + return ( + checkServerIdentity(host, cert) ?? + (cert.fingerprint256 === tls.serverFingerprint + ? undefined + : new Error("MCP control certificate mismatch")) + ); + }, + }, + interceptors: [ + (next) => async (request) => { + try { + return await next(request); + } catch (error) { + throw new ConnectError( + "MCP credential control unavailable", + error instanceof ConnectError ? error.code : Code.Unavailable, + ); + } + }, + ], + }); +} diff --git a/orchestrator/src/mcp/credential-admin.ts b/orchestrator/src/mcp/credential-admin.ts new file mode 100644 index 000000000..8567a68fc --- /dev/null +++ b/orchestrator/src/mcp/credential-admin.ts @@ -0,0 +1,485 @@ +import { Code, ConnectError } from "@connectrpc/connect"; +import { create, equals, fromJson, toJson, type JsonObject } from "@bufbuild/protobuf"; +import { + AuthorityContextSchema, + CredentialKind, + CredentialStatusSchema, + type CredentialStatus, + type AuthorityContext, +} from "../gen/engram/mcp/v1/control_pb.ts"; +import type { CredentialControlClient } from "./control-client.ts"; +import { + type CredentialMetadataStore, + type CredentialClaim, + statusMetadata, + McpConnectionHasHistory, +} from "./credential-store.ts"; +import { validateMcpConfig, type McpConfig } from "./config.ts"; + +export interface McpRegistrationMetadata { + id: string; + clientId: string; + confidential: boolean; + redirectUri: string; + issuer: string; + resource: { zoneId: string; endpoint: string }; + scopes: string[]; +} +function decode(value: JsonObject | undefined) { + return value ? fromJson(CredentialStatusSchema, value) : undefined; +} +function sameAuthority(a: CredentialStatus, b: CredentialStatus) { + return ( + a.protocol === 1 && + b.protocol === 1 && + a.context && + b.context && + a.reference && + b.reference && + equals(AuthorityContextSchema, a.context, b.context) && + a.reference.kind === b.reference.kind && + a.reference.connectionId === b.reference.connectionId && + a.reference.name === b.reference.name && + a.reference.registrationId === b.reference.registrationId + ); +} +function revokedReceipt(old: CredentialStatus, receipt: CredentialStatus) { + if ( + !old.context || + !receipt.context || + !old.reference || + !receipt.reference || + receipt.protocol !== 1 || + receipt.status !== "revoked" || + receipt.version !== old.version + 1n || + receipt.reference.kind !== old.reference.kind || + receipt.reference.name !== old.reference.name || + receipt.reference.registrationId !== old.reference.registrationId || + receipt.reference.connectionId !== old.reference.connectionId + ) + throw new Error("stale revocation receipt"); + if (old.reference.kind === CredentialKind.OAUTH) { + if (!equals(AuthorityContextSchema, old.context, receipt.context)) + throw new Error("stale revocation receipt"); + } else if ( + receipt.reference.generation !== old.reference.generation + 1n || + receipt.context.authorizationGeneration !== old.context.authorizationGeneration + 1n || + receipt.context.credentialIdentity === old.context.credentialIdentity || + !equals( + AuthorityContextSchema, + create(AuthorityContextSchema, { + ...receipt.context, + authorizationGeneration: old.context.authorizationGeneration, + credentialIdentity: old.context.credentialIdentity, + }), + old.context, + ) + ) + throw new Error("stale revocation receipt"); +} +export function makeCredentialAdmin(deps: { + store: CredentialMetadataStore; + control: CredentialControlClient; + registrations: readonly McpRegistrationMetadata[]; + now: () => Date; +}) { + const { store, control, now } = deps; + // Copy only operator metadata. No secret-bearing operator object is retained here. + const registrations = deps.registrations.map((r) => ({ + id: r.id, + clientId: r.clientId, + confidential: r.confidential, + redirectUri: r.redirectUri, + issuer: r.issuer, + resource: { ...r.resource }, + scopes: [...r.scopes], + })); + function registration(config: McpConfig) { + if (config.auth.kind !== "oauth") throw new Error("OAuth connection required"); + const registrationId = config.auth.clientRegistrationId; + const r = registrations.find((r) => r.id === registrationId); + if (!r || r.resource.zoneId !== config.zone || r.resource.endpoint !== config.endpoint) + throw new Error("approved registration required"); + return r; + } + function change(claim: CredentialClaim) { + return { + connectionId: claim.id, + configRevision: BigInt(claim.revision), + previousAuthorizationGeneration: BigInt(claim.generation), + resource: { zoneId: claim.config.zone, endpoint: claim.config.endpoint }, + }; + } + async function revoke(claim: CredentialClaim, field: "status" | "clientSecret", actorId: string) { + const saved = decode(claim.metadata[field]); + if (!saved) return; + const current = await control.getStatus({ + protocol: 1, + connectionId: claim.id, + reference: saved.reference, + }); + if (saved.status === "revoked") { + if (!equals(CredentialStatusSchema, current, saved)) + throw new Error("stale revoked authority"); + return current; + } + // A lost reply can be recovered from the durable receipt, never by replay. + if (current.status === "revoked") { + revokedReceipt(saved, current); + await store.stage(claim, { ...claim.metadata, [field]: statusMetadata(current) }, now()); + return current; + } + if (!sameAuthority(saved, current)) throw new Error("stale credential authority"); + // Normal refresh may change version/status while its identity stays fixed. + await store.stage(claim, { ...claim.metadata, [field]: statusMetadata(current) }, now()); + let receipt: CredentialStatus; + try { + receipt = await control.revoke({ protocol: 1, expected: current, actorId }); + } catch { + receipt = await control.getStatus({ + protocol: 1, + connectionId: claim.id, + reference: current.reference, + }); + } + revokedReceipt(current, receipt); + await store.stage(claim, { ...claim.metadata, [field]: statusMetadata(receipt) }, now()); + return receipt; + } + async function revokeTargetOAuth( + claim: CredentialClaim, + actorId: string, + recover = false, + cancelledContext?: AuthorityContext, + ) { + if (claim.config.auth.kind !== "oauth") return undefined; + const registrationId = claim.config.auth.clientRegistrationId; + const saved = decode(claim.metadata.status); + if ( + !recover && + saved?.reference?.kind === CredentialKind.OAUTH && + saved.reference.registrationId === registrationId + ) + return revoke(claim, "status", actorId); + let current: CredentialStatus; + try { + current = await control.getStatus({ + protocol: 1, + connectionId: claim.id, + reference: { kind: CredentialKind.OAUTH, registrationId }, + }); + } catch (error) { + if ( + error instanceof ConnectError && + error.code === Code.NotFound && + (!saved || + saved.reference?.kind !== CredentialKind.OAUTH || + saved.reference.registrationId !== registrationId) + ) + return undefined; + throw error; + } + if ( + current.context?.connectionId !== claim.id || + current.reference?.kind !== CredentialKind.OAUTH || + current.reference.registrationId !== registrationId + ) + throw new Error("stale OAuth authority"); + if ( + recover && + !( + (saved?.context && + saved.reference?.kind === CredentialKind.OAUTH && + saved.reference.registrationId === registrationId && + equals(AuthorityContextSchema, current.context!, saved.context)) || + (cancelledContext && equals(AuthorityContextSchema, current.context!, cancelledContext)) + ) + ) + throw new Error("stale OAuth recovery authority"); + // Accept only the saved authority or this canceled attempt's exact context. + // A newer same-registration authority is not evidence for old recovery. + await store.stage(claim, { ...claim.metadata, status: statusMetadata(current) }, now()); + return revoke(claim, "status", actorId); + } + async function cancelPending(claim: CredentialClaim, actorId: string) { + const flow = claim.metadata.flow; + if (!flow) return; + const receipt = await control.cancelAuthorization({ + protocol: 1, + flowId: flow.id, + connectionId: claim.id, + registrationId: flow.registrationId, + actorId, + }); + if ( + receipt.protocol !== 1 || + receipt.flowId !== flow.id || + receipt.connectionId !== claim.id || + receipt.registrationId !== flow.registrationId + ) + throw new Error("stale cancellation receipt"); + // Keep the attempt proof durable until the whole transition finishes. + // A later recovery may need it to identify a callback that won publication. + return flow; + } + async function claimCurrent( + id: string, + revision: number, + generation: number, + actor: string, + config?: McpConfig, + ) { + const row = await store.get(id); + const claim = await store.claim(id, revision, generation, actor, config ?? row.config, now()); + await cancelPending(claim, actor); + delete claim.metadata.flow; + return claim; + } + return { + registrations: () => structuredClone(registrations), + async status(id: string) { + const row = await store.get(id), + stored = decode(row.mcpCredential.status); + const status = stored + ? await control.getStatus({ protocol: 1, connectionId: id, reference: stored.reference }) + : undefined; + return { row, status }; + }, + async putSecret( + id: string, + revision: number, + generation: number, + actorId: string, + kind: CredentialKind, + header: string, + value: string, + ) { + if ( + !value || + value.length > 8192 || + /[\r\n]/.test(value) || + !["authorization", "x-api-key", "api-key"].includes(header) + ) + throw new Error("invalid credential input"); + const row = await store.get(id), + config = validateMcpConfig(row.config); + if ( + kind === CredentialKind.API_KEY + ? config.auth.kind !== "api_key" + : kind !== CredentialKind.CLIENT_SECRET || !registration(config).confidential + ) + throw new Error("credential kind does not match connection"); + const claim = await claimCurrent(id, revision, generation, actorId); + const resource = await revoke(claim, "status", actorId); + const old = + kind === CredentialKind.CLIENT_SECRET + ? await revoke(claim, "clientSecret", actorId) + : resource; + const expected = + old?.reference?.kind === kind && + old.context?.authorizationGeneration === BigInt(claim.generation) + ? old.reference + : undefined; + const result = await control.putSecret({ + protocol: 1, + context: change(claim), + kind, + expected, + header, + value, + actorId, + }); + // Retain the returned reference before publication, so a crash can revoke it. + const field = kind === CredentialKind.API_KEY ? "status" : "clientSecret"; + await store.stage(claim, { ...claim.metadata, [field]: statusMetadata(result) }, now()); + if (kind === CredentialKind.API_KEY) { + if (result.reference?.kind !== CredentialKind.API_KEY || !result.reference.name) + throw new Error("invalid credential receipt"); + await store.publish( + claim, + { ...config, auth: { kind: "api_key", secretRef: result.reference.name } }, + statusMetadata(result), + now(), + ); + } else { + if ( + result.reference?.kind !== CredentialKind.CLIENT_SECRET || + result.context?.connectionId !== id || + result.context.configRevision !== BigInt(claim.revision) || + result.context.authorizationGeneration !== BigInt(claim.generation + 1) + ) + throw new Error("invalid client credential receipt"); + await store.publish(claim, config, undefined, now()); + } + }, + async begin( + id: string, + revision: number, + generation: number, + actorId: string, + consent: boolean, + ) { + const row = await store.get(id), + config = validateMcpConfig(row.config), + r = registration(config); + if (!consent || config.auth.kind !== "oauth" || !config.auth.shareWithOrgSessions) + throw new Error("organization sharing consent required"); + const claim = await claimCurrent(id, revision, generation, actorId); + await revoke(claim, "status", actorId); + const old = await revokeTargetOAuth(claim, actorId); + const client = decode(claim.metadata.clientSecret); + if ( + r.confidential && + (!client || + client.status !== "connected" || + client.reference?.kind !== CredentialKind.CLIENT_SECRET) + ) + throw new Error("client credential required"); + const response = await control.beginAuthorization({ + protocol: 1, + context: change(claim), + registrationId: r.id, + clientSecret: r.confidential ? client!.reference : undefined, + expectedVersion: old?.reference?.registrationId === r.id ? old.version : undefined, + actorId, + sharingConsent: consent, + }); + if ( + response.protocol !== 1 || + !response.context || + response.context.connectionId !== id || + response.context.configRevision !== BigInt(claim.revision) || + response.context.authorizationGeneration !== BigInt(claim.generation + 1) || + response.expiresAt <= BigInt(now().getTime()) || + response.expiresAt > BigInt(now().getTime() + 600000) + ) + throw new Error("invalid authorization receipt"); + const url = new URL(response.authorizationUrl); + if (url.protocol !== "https:" || url.username || url.password || url.hash) + throw new Error("invalid authorization URL"); + await store.stage( + claim, + { + ...claim.metadata, + flow: { + id: response.flowId, + actorId, + registrationId: r.id, + context: toJson(AuthorityContextSchema, response.context) as JsonObject, + expiresAt: Number(response.expiresAt), + }, + }, + now(), + ); + await store.publish(claim, config, undefined, now()); + return { + configRevision: claim.revision, + authGeneration: claim.generation, + authorizationUrl: response.authorizationUrl, + flowId: response.flowId, + expiresAt: new Date(Number(response.expiresAt)).toISOString(), + }; + }, + async complete( + id: string, + revision: number, + generation: number, + actorId: string, + input: { flowId: string; state: string; code: string; issuer?: string; redirectUri: string }, + ) { + const row = await store.get(id), + config = validateMcpConfig(row.config), + r = registration(config); + if ( + input.redirectUri !== r.redirectUri || + (input.issuer !== undefined && input.issuer !== r.issuer) + ) + throw new Error("invalid authorization callback"); + const claim = await store.claim(id, revision, generation, actorId, config, now(), { + kind: "callback", + flowId: input.flowId, + }); + const flow = claim.metadata.flow!; + if (flow.registrationId !== r.id) throw new Error("stale authorization registration"); + const result = await control.completeAuthorization({ + protocol: 1, + context: fromJson(AuthorityContextSchema, flow.context), + registrationId: r.id, + flowId: flow.id, + state: input.state, + code: input.code, + issuer: input.issuer, + redirectUri: r.redirectUri, + actorId, + }); + await store.stage(claim, { ...claim.metadata, status: statusMetadata(result) }, now()); + await store.publish(claim, config, statusMetadata(result), now()); + }, + async disconnect(id: string, revision: number, generation: number, actorId: string) { + const claim = await claimCurrent(id, revision, generation, actorId); + await revoke(claim, "status", actorId); + await store.publish(claim, claim.config, undefined, now()); + }, + async configure( + id: string, + revision: number, + generation: number, + actorId: string, + config: McpConfig, + names?: { alias: string; displayName: string }, + ) { + if (config.auth.kind === "oauth") registration(config); + const row = await store.get(id), + current = validateMcpConfig(row.config); + if ( + config.auth.kind === "api_key" && + config.auth.secretRef && + (current.auth.kind !== "api_key" || config.auth.secretRef !== current.auth.secretRef) + ) + throw new Error("fresh credential ingestion required"); + const claim = await claimCurrent(id, revision, generation, actorId, config); + await revoke(claim, "status", actorId); + await revoke(claim, "clientSecret", actorId); + await store.publish(claim, config, undefined, now(), names); + }, + async remove(id: string, revision: number, generation: number, actorId: string) { + const row = await store.get(id); + let claim: CredentialClaim; + try { + claim = await store.claim(id, revision, generation, actorId, row.config, now(), { + kind: "remove", + }); + } catch (error) { + if (error instanceof McpConnectionHasHistory) + throw new ConnectError(error.message, Code.FailedPrecondition); + throw error; + } + await cancelPending(claim, actorId); + delete claim.metadata.flow; + await revoke(claim, "status", actorId); + await revoke(claim, "clientSecret", actorId); + return store.remove(claim); + }, + async recover(id: string, revision: number, generation: number, actorId: string) { + const claim = await store.recover(id, revision, generation, actorId, now()); + const cancelled = await cancelPending(claim, actorId); + const cancelledContext = cancelled + ? fromJson(AuthorityContextSchema, cancelled.context) + : undefined; + if ( + claim.config.auth.kind === "oauth" && + decode(claim.metadata.status)?.reference?.kind === CredentialKind.OAUTH + ) + await revokeTargetOAuth(claim, actorId, true, cancelledContext); + else { + await revoke(claim, "status", actorId); + await revokeTargetOAuth(claim, actorId, true, cancelledContext); + } + // Client-secret writes also need a durable fence after interrupted publication. + await revoke(claim, "clientSecret", actorId); + delete claim.metadata.flow; + await store.publish(claim, claim.config, undefined, now()); + }, + }; +} +export type CredentialAdmin = ReturnType; diff --git a/orchestrator/src/mcp/credential-broker.ts b/orchestrator/src/mcp/credential-broker.ts new file mode 100644 index 000000000..b2f7046c8 --- /dev/null +++ b/orchestrator/src/mcp/credential-broker.ts @@ -0,0 +1,144 @@ +import { createHash } from "node:crypto"; +import type { + WorkerIdentity, + WorkerIntent, + WorkerOperation, + WorkerRequest, + WorkerResponse, +} from "../../../mcp-worker/src/contracts.ts"; +import { canonical, parseWorkerRequest } from "../../../mcp-worker/src/index.ts"; +import type { CredentialStatus } from "../gen/engram/mcp/v1/control_pb.ts"; +import type { CredentialControlClient } from "./control-client.ts"; + +export interface McpResourceWorker { + identity(): Promise; + /** Fixed send only. Put awaited policy checks in the broker beforeDispatch hook. */ + dispatch(request: WorkerRequest): Promise; +} +type ResourceOperation = Extract; +export interface AdmittedCredentialOperation { + status: CredentialStatus; + identity: WorkerIdentity; + intent: Omit; + operation: ResourceOperation; +} +/** The runtime must check live authority and its durable global admission here. */ +export interface McpResourceAdmission { + assertAdmitted(operation: Readonly): Promise; +} +export function makeCredentialBroker(deps: { + control: CredentialControlClient; + worker: McpResourceWorker; + admission: McpResourceAdmission; + now: () => number; + /** Final authority fence; receives no credentials and runs before the last delivery-expiry check. */ + beforeDispatch?(): Promise; +}) { + if (!deps.admission?.assertAdmitted) throw new Error("MCP global admission is required"); + return { + async dispatch(input: AdmittedCredentialOperation): Promise { + const frozen = structuredClone(input), + { status, identity, intent, operation } = frozen; + const context = status.context, + reference = status.reference, + now = deps.now(); + if ( + !context || + !reference || + status.protocol !== 1 || + status.status !== "connected" || + context.connectionId !== intent.connectionId || + context.configRevision !== BigInt(intent.connectionRevision) || + context.credentialIdentity !== intent.credentialIdentity || + context.resource?.endpoint !== intent.destination || + context.resource.zoneId !== intent.zoneId || + identity.protocol !== 1 || + identity.bootId !== intent.bootId || + identity.zoneId !== intent.zoneId || + identity.policyDigest !== intent.policyDigest || + context.policyDigest !== identity.policyDigest || + context.policyRevision !== identity.policyRevision || + operation.endpoint !== intent.destination || + operation.kind !== intent.operation || + "credential" in operation || + intent.expiresAt <= now || + intent.expiresAt > now + 300000 || + intent.lease.expiresAt < intent.expiresAt || + intent.lease.expiresAt > now + 300000 + ) + throw new Error("stale credential dispatch context"); + if (intent.requestDigest.length !== 71 || !/^sha256:[0-9a-f]{64}$/.test(intent.requestDigest)) + throw new Error("MCP request digest invalid"); + parseWorkerRequest({ + protocol: 1, + intent: { ...intent, operationDigest: "0".repeat(64) }, + operation, + }); + await deps.admission.assertAdmitted(structuredClone(frozen)); + const actual = await deps.worker.identity(); + if (canonical(actual) !== canonical(identity)) throw new Error("stale worker identity"); + const deliveryExpiry = Math.min(deps.now() + 10000, intent.expiresAt); + if (deliveryExpiry <= deps.now()) throw new Error("credential delivery expired"); + const lease = await deps.control.lease({ + protocol: 1, + context, + reference, + worker: identity, + intentId: intent.id, + requestDigest: intent.requestDigest.slice("sha256:".length), + operation: operation.kind, + expiresAt: BigInt(deliveryExpiry), + operationExpiresAt: BigInt(intent.expiresAt), + admission: { ...intent.lease, expiresAt: BigInt(intent.lease.expiresAt) }, + scope: + intent.scope.kind === "execution" + ? { case: "execution", value: intent.scope } + : { case: "administration", value: intent.scope }, + }); + if ( + lease.protocol !== 1 || + lease.intentId !== intent.id || + lease.expiresAt !== BigInt(deliveryExpiry) || + lease.operationExpiresAt !== BigInt(intent.expiresAt) || + !["authorization", "x-api-key", "api-key"].includes(lease.header) || + !lease.value || + /[\r\n]/.test(lease.value) + ) + throw new Error("invalid credential delivery"); + if (deps.now() >= deliveryExpiry) throw new Error("credential delivery expired"); + // parseWorkerRequest validates the credential header union at the wire boundary. + const withCredential = { + ...operation, + credential: { header: lease.header, value: lease.value }, + }; + const operationDigest = createHash("sha256").update(canonical(withCredential)).digest("hex"); + const request = parseWorkerRequest({ + protocol: 1, + intent: { ...intent, operationDigest }, + operation: withCredential, + }); + if (deps.now() >= deliveryExpiry) throw new Error("credential delivery expired"); + await deps.admission.assertAdmitted(structuredClone(frozen)); + if (deps.beforeDispatch) await deps.beforeDispatch(); + if (deps.now() >= deliveryExpiry) throw new Error("credential delivery expired"); + let response: WorkerResponse; + try { + response = await deps.worker.dispatch(request); + } catch { + throw new Error("MCP worker operation unavailable"); + } + // No retry on transport loss or mismatch. Task 5 records uncertain writes. + if ( + !response.ok || + canonical(response.identity) !== canonical(identity) || + canonical(response.intent) !== canonical(request.intent) || + response.intentId !== intent.id || + response.requestDigest !== intent.requestDigest || + response.operationDigest !== operationDigest || + response.leaseId !== intent.lease.id + ) + throw new Error("MCP worker response mismatch"); + return response; + }, + }; +} diff --git a/orchestrator/src/mcp/credential-rpc.ts b/orchestrator/src/mcp/credential-rpc.ts new file mode 100644 index 000000000..8e1463294 --- /dev/null +++ b/orchestrator/src/mcp/credential-rpc.ts @@ -0,0 +1,189 @@ +import { Code, ConnectError, type HandlerContext, type ServiceImpl } from "@connectrpc/connect"; +import { fromJson } from "@bufbuild/protobuf"; +import type { IntegrationService } from "../gen/engram/app/v1/integration_pb.ts"; +import { + CredentialKind, + CredentialStatusSchema, + type CredentialStatus, +} from "../gen/engram/mcp/v1/control_pb.ts"; +import { requireAdmin, type GetSession } from "../rpc/require.ts"; +import type { + IntegrationConnectionRow, + IntegrationConnectionStore, +} from "../db/integration-connections.ts"; +import { + mcpOperation, + requireMcp, + requireMcpHook, + requireMcpServices, + type McpServices, +} from "./contracts.ts"; + +export async function requireMcpHumanAdmin(ctx: HandlerContext, getSession: GetSession) { + const actor = await requireAdmin(ctx, getSession); + if (actor.serviceAccount) + throw new ConnectError("a human administrator is required", Code.PermissionDenied); + return actor; +} +function statusToProto(status: CredentialStatus | undefined, revoked: boolean) { + return { + status: revoked ? "disconnected" : (status?.status ?? "not_configured"), + reference: status?.reference?.name || status?.reference?.registrationId || "", + kind: + status?.reference?.kind === CredentialKind.API_KEY + ? "api_key" + : status?.reference?.kind === CredentialKind.OAUTH + ? "oauth" + : "", + credentialIdentity: status?.context?.credentialIdentity ?? "", + version: status?.version ?? 0n, + reservationGeneration: status?.reference?.generation ?? 0n, + }; +} +type Methods = + | "putMcpCredential" + | "getMcpSetup" + | "completeMcpAuthorization" + | "recoverMcpCredential" + | "authorizeMcp" + | "disconnectMcp"; +export function credentialHandlers(deps: { + services?: McpServices; + connections: IntegrationConnectionStore; + getSession: GetSession; + connectionToProto: (row: IntegrationConnectionRow) => { id: string }; +}): Pick, Methods> { + function admin() { + return requireMcpHook(requireMcp(deps.services).credentials); + } + function recoveryAdmin() { + return requireMcpHook(requireMcpServices(deps.services).credentials); + } + async function result(id: string) { + const row = await deps.connections.get(id); + if (!row || row.provider !== "mcp") + throw new ConnectError("MCP connection not found", Code.NotFound); + const { status } = await recoveryAdmin().status(id); + return { + connection: deps.connectionToProto(row), + credential: statusToProto(status, row.mcp?.revoked ?? true), + }; + } + return { + async putMcpCredential(req, ctx) { + const actor = await requireMcpHumanAdmin(ctx, deps.getSession); + const service = admin(); + const kind = + req.kind === "api_key" + ? CredentialKind.API_KEY + : req.kind === "client_secret" + ? CredentialKind.CLIENT_SECRET + : CredentialKind.UNSPECIFIED; + return mcpOperation(async () => { + await service.putSecret( + req.connectionId, + req.expectedRevision, + req.expectedGeneration, + actor.id, + kind, + req.header, + req.value, + ); + return result(req.connectionId); + }); + }, + async authorizeMcp(req, ctx) { + const actor = await requireMcpHumanAdmin(ctx, deps.getSession); + const service = admin(); + return mcpOperation(() => + service.begin( + req.connectionId, + req.expectedRevision, + req.expectedGeneration, + actor.id, + req.shareWithOrgSessions, + ), + ); + }, + async completeMcpAuthorization(req, ctx) { + const actor = await requireMcpHumanAdmin(ctx, deps.getSession); + const service = admin(); + return mcpOperation(async () => { + await service.complete( + req.connectionId, + req.expectedRevision, + req.expectedGeneration, + actor.id, + { + flowId: req.flowId, + state: req.state, + code: req.code, + issuer: req.issuer, + redirectUri: req.redirectUri, + }, + ); + return result(req.connectionId); + }); + }, + async disconnectMcp(req, ctx) { + const actor = await requireMcpHumanAdmin(ctx, deps.getSession); + const service = recoveryAdmin(); + return mcpOperation(async () => { + await service.disconnect( + req.connectionId, + req.expectedRevision, + req.expectedGeneration, + actor.id, + ); + return result(req.connectionId); + }); + }, + async recoverMcpCredential(req, ctx) { + const actor = await requireMcpHumanAdmin(ctx, deps.getSession); + const service = recoveryAdmin(); + return mcpOperation(async () => { + await service.recover( + req.connectionId, + req.expectedRevision, + req.expectedGeneration, + actor.id, + ); + return result(req.connectionId); + }); + }, + async getMcpSetup(req, ctx) { + await requireMcpHumanAdmin(ctx, deps.getSession); + const service = deps.services; + return mcpOperation(async () => { + const base = { + enabled: service?.enabled ?? false, + recoveryAvailable: !!service?.credentials, + discoveryAvailable: !!service?.enabled && !!service.discovery, + registrations: service?.credentials?.registrations() ?? [], + destinations: service?.destinations ?? [], + }; + if (!req.connectionId) return base; + const credentials = requireMcpHook(service?.credentials), + { row, status } = await credentials.status(req.connectionId); + const connection = await deps.connections.get(row.id); + if (!connection) throw new Error("connection not found"); + const client = row.mcpCredential.clientSecret + ? fromJson(CredentialStatusSchema, row.mcpCredential.clientSecret) + : undefined; + return { + ...base, + current: { + connection: deps.connectionToProto(connection), + credential: statusToProto(status, row.revoked), + }, + transitionPending: !!row.mcpCredential.transition, + flowId: row.mcpCredential.flow?.id ?? "", + flowExpiresAt: row.mcpCredential.flow + ? new Date(row.mcpCredential.flow.expiresAt).toISOString() + : "", + clientSecretConfigured: client?.status === "connected", + }; + }); + }, + }; +} diff --git a/orchestrator/src/mcp/credential-store.ts b/orchestrator/src/mcp/credential-store.ts new file mode 100644 index 000000000..603796e3e --- /dev/null +++ b/orchestrator/src/mcp/credential-store.ts @@ -0,0 +1,299 @@ +import { notifyMcpAuthorityChanged } from "./authority-notification.ts"; +import { and, eq, sql } from "drizzle-orm"; +import { create, fromJson, toJson, type JsonObject } from "@bufbuild/protobuf"; +import { getDb } from "../db/client.ts"; +import { + integrationConnection as connection, + mcpCatalog, + mcpExecution, + mcpAdmission, + mcpDiscoveryAdmission, +} from "../db/schema.ts"; +import { + CredentialKind, + CredentialStatusSchema, + AuthorityContextSchema, +} from "../gen/engram/mcp/v1/control_pb.ts"; +import { validateMcpConfig, type McpConfig } from "./config.ts"; + +export class McpConnectionHasHistory extends Error { + constructor() { + super("MCP connection has retained history; disable or disconnect it instead"); + } +} + +/** Only coordinator metadata. Secret values, callback codes and state never enter this type. */ +export interface CredentialMetadata { + status?: JsonObject; + clientSecret?: JsonObject; + flow?: { + id: string; + actorId: string; + registrationId: string; + context: JsonObject; + expiresAt: number; + }; + transition?: { id: string; actorId: string; startedAt: number }; +} +export interface CredentialClaim { + id: string; + token: string; + revision: number; + generation: number; + metadata: CredentialMetadata; + config: McpConfig; +} +export function statusMetadata( + value: ReturnType>, +): JsonObject { + // Protobuf JSON keeps int64 values exact and excludes transport/request fields. + return toJson(CredentialStatusSchema, value) as JsonObject; +} +function checkedMetadata(value: CredentialMetadata): CredentialMetadata { + return { + ...(value.status + ? { status: statusMetadata(fromJson(CredentialStatusSchema, value.status)) } + : {}), + ...(value.clientSecret + ? { clientSecret: statusMetadata(fromJson(CredentialStatusSchema, value.clientSecret)) } + : {}), + ...(value.flow + ? { + flow: { + id: value.flow.id, + actorId: value.flow.actorId, + registrationId: value.flow.registrationId, + expiresAt: value.flow.expiresAt, + context: toJson( + AuthorityContextSchema, + fromJson(AuthorityContextSchema, value.flow.context), + ) as JsonObject, + }, + } + : {}), + ...(value.transition + ? { + transition: { + id: value.transition.id, + actorId: value.transition.actorId, + startedAt: value.transition.startedAt, + }, + } + : {}), + }; +} +export function makeCredentialMetadataStore(db = getDb()) { + async function get(id: string) { + const [row] = await db + .select() + .from(connection) + .where(and(eq(connection.id, id), eq(connection.provider, "mcp"))); + if (!row) throw new Error("MCP connection not found"); + return row; + } + async function claim( + id: string, + revision: number, + generation: number, + actorId: string, + config: unknown, + now: Date, + intent?: { kind: "callback"; flowId: string } | { kind: "remove" }, + ) { + const callbackId = intent?.kind === "callback" ? intent.flowId : undefined; + const target = validateMcpConfig(config); + const result = await db.transaction(async (tx) => { + const [row] = await tx + .select() + .from(connection) + .where(and(eq(connection.id, id), eq(connection.provider, "mcp"))) + .for("update"); + if ( + !row || + row.configRevision !== revision || + row.authGeneration !== generation || + row.mcpCredential.transition + ) + throw new Error("stale credential transition"); + if ( + callbackId && + (row.mcpCredential.flow?.id !== callbackId || + row.mcpCredential.flow.actorId !== actorId || + row.mcpCredential.flow.expiresAt <= now.getTime()) + ) + throw new Error("stale authorization flow"); + if (intent?.kind === "remove") { + // Producers lock this connection before inserting history. The claim then + // advances its revision and revokes it, so later producers cannot insert. + for (const history of [mcpCatalog, mcpExecution, mcpAdmission, mcpDiscoveryAdmission]) { + const retained = await tx + .select({ connectionId: history.connectionId }) + .from(history) + .where(eq(history.connectionId, id)) + .limit(1); + if (retained.length) throw new McpConnectionHasHistory(); + } + } + const metadata = checkedMetadata(row.mcpCredential); + metadata.transition = { id: crypto.randomUUID(), actorId, startedAt: now.getTime() }; + // Keep the exact pending attempt until the lifecycle adapter has durably + // canceled it. Clearing it here would lose the coordinator fence subject. + const nextRevision = revision + (callbackId ? 0 : 1), + nextGeneration = generation + (callbackId ? 0 : 1); + await tx + .update(connection) + .set({ + mcpCredential: metadata, + configRevision: nextRevision, + authGeneration: nextGeneration, + revoked: true, + enabled: false, + testedAt: null, + updatedAt: now, + }) + .where(eq(connection.id, id)); + return { + id, + token: metadata.transition.id, + revision: nextRevision, + generation: nextGeneration, + metadata, + config: target, + }; + }); + notifyMcpAuthorityChanged(); + return result; + } + async function stage(claim: CredentialClaim, metadata: CredentialMetadata, now: Date) { + const safe = checkedMetadata({ ...metadata, transition: claim.metadata.transition }); + const [row] = await db + .update(connection) + .set({ mcpCredential: safe, updatedAt: now }) + .where( + and( + eq(connection.id, claim.id), + eq(connection.configRevision, claim.revision), + eq(connection.authGeneration, claim.generation), + sql`${connection.mcpCredential}->'transition'->>'id' = ${claim.token}`, + ), + ) + .returning(); + if (!row) throw new Error("stale credential transition"); + claim.metadata = safe; + } + async function publish( + claim: CredentialClaim, + config: unknown, + status: JsonObject | undefined, + now: Date, + names?: { alias: string; displayName: string }, + ) { + const target = validateMcpConfig(config); + const receipt = status ? fromJson(CredentialStatusSchema, status) : undefined; + if ( + receipt && + (receipt.protocol !== 1 || + receipt.status !== "connected" || + !receipt.context || + receipt.context.connectionId !== claim.id || + receipt.context.configRevision !== BigInt(claim.revision) || + receipt.context.authorizationGeneration !== BigInt(claim.generation + 1) || + receipt.context.resource?.endpoint !== target.endpoint || + receipt.context.resource.zoneId !== target.zone || + !receipt.context.credentialIdentity || + !receipt.reference || + (target.auth.kind === "api_key" + ? receipt.reference.kind !== CredentialKind.API_KEY || + receipt.reference.connectionId !== claim.id || + receipt.reference.name !== target.auth.secretRef + : receipt.reference.kind !== CredentialKind.OAUTH || + receipt.reference.registrationId !== target.auth.clientRegistrationId)) + ) + throw new Error("stale authority receipt"); + const metadata = checkedMetadata({ ...claim.metadata, ...(status ? { status } : {}) }); + delete metadata.transition; + if (receipt) delete metadata.flow; + const [row] = await db + .update(connection) + .set({ + ...names, + config: target, + mcpCredential: metadata, + ...(receipt + ? { + authGeneration: claim.generation + 1, + credentialIdentity: receipt.context!.credentialIdentity, + revoked: false, + } + : {}), + enabled: false, + testedAt: null, + updatedAt: now, + }) + .where( + and( + eq(connection.id, claim.id), + eq(connection.configRevision, claim.revision), + eq(connection.authGeneration, claim.generation), + sql`${connection.mcpCredential}->'transition'->>'id' = ${claim.token}`, + ), + ) + .returning(); + if (!row) throw new Error("stale credential transition"); + notifyMcpAuthorityChanged(); + return row; + } + return { + get, + claim, + stage, + publish, + async remove(claim: CredentialClaim) { + const rows = await db + .delete(connection) + .where( + and( + eq(connection.id, claim.id), + eq(connection.configRevision, claim.revision), + eq(connection.authGeneration, claim.generation), + sql`${connection.mcpCredential}->'transition'->>'id' = ${claim.token}`, + ), + ) + .returning({ id: connection.id }); + if (!rows.length) throw new Error("stale credential transition"); + notifyMcpAuthorityChanged(); + return true; + }, + async recover(id: string, revision: number, generation: number, actorId: string, now: Date) { + const result = await db.transaction(async (tx) => { + const [row] = await tx.select().from(connection).where(eq(connection.id, id)).for("update"); + if ( + !row || + row.provider !== "mcp" || + row.configRevision !== revision || + row.authGeneration !== generation || + !row.mcpCredential.transition || + row.mcpCredential.transition.startedAt + 35000 > now.getTime() + ) + throw new Error("stale or active credential transition"); + const metadata = checkedMetadata(row.mcpCredential); + metadata.transition = { id: crypto.randomUUID(), actorId, startedAt: now.getTime() }; + await tx + .update(connection) + .set({ mcpCredential: metadata, updatedAt: now }) + .where(eq(connection.id, id)); + return { + id, + token: metadata.transition.id, + revision, + generation, + metadata, + config: validateMcpConfig(row.config), + }; + }); + notifyMcpAuthorityChanged(); + return result; + }, + }; +} +export type CredentialMetadataStore = ReturnType; diff --git a/orchestrator/src/mcp/discovery-admission.ts b/orchestrator/src/mcp/discovery-admission.ts new file mode 100644 index 000000000..ed2e64d13 --- /dev/null +++ b/orchestrator/src/mcp/discovery-admission.ts @@ -0,0 +1,294 @@ +import { fromJson, toJson } from "@bufbuild/protobuf"; +import { and, eq, gt, inArray, or } from "drizzle-orm"; +import type { WorkerIdentity, WorkerIntent } from "../../../mcp-worker/src/contracts.ts"; +import { getDb } from "../db/client.ts"; +import { integrationConnection, mcpDiscoveryAdmission as admission, user } from "../db/schema.ts"; +import { CredentialKind, CredentialStatusSchema } from "../gen/engram/mcp/v1/control_pb.ts"; +import { isServiceAccountEmail } from "../rpc/api-key.ts"; +import { lockAdmissionScope } from "./admission.ts"; +import { validateMcpConfig } from "./config.ts"; +import type { AdmittedCredentialOperation } from "./credential-broker.ts"; +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import { assertSchemaShape } from "./catalog.ts"; +import type { JournalTx } from "./journal-review.ts"; + +export type McpAdministrativeOperation = + | { kind: "discover" } + | { kind: "validateSchemas"; documentsDigest: string }; +export interface McpSchemaReservation extends Omit { + documents: Record[]; +} +export interface AdmittedSchemaOperation { + identity: WorkerIdentity; + intent: Omit; + documents: Record[]; +} +function schemaOperation(documents: Record[]): McpAdministrativeOperation { + if (documents.length > 384 || Buffer.byteLength(canonicalJson({ documents })) > 1024 * 1024) + throw Error("MCP schema limit"); + for (const document of documents) assertSchemaShape(document); + return { kind: "validateSchemas", documentsDigest: canonicalDigest(documents) }; +} +export interface McpDiscoveryReservation { + requestId: string; + actorId: string; + connectionId: string; + configRevision: number; + authGeneration: number; + credentialIdentity: string; + deploymentId: string; + identity: WorkerIdentity; + destination: string; +} +type Row = typeof admission.$inferSelect; +function intentFor(row: Row): Omit { + return { + id: row.intentId, + bootId: row.identity.bootId, + zoneId: row.identity.zoneId, + policyDigest: row.identity.policyDigest, + connectionId: row.connectionId, + connectionRevision: row.configRevision, + credentialIdentity: row.credentialIdentity, + scope: { kind: "administration", requestId: row.requestId, actorId: row.actorId }, + operation: row.operation.kind, + destination: row.destination, + requestDigest: row.requestDigest, + expiresAt: row.expiresAt.getTime(), + lease: { id: row.intentId, deploymentId: row.deploymentId, expiresAt: row.expiresAt.getTime() }, + }; +} +/** Uses existing human, connection and credential authority; no credential values are stored. */ +export function makeMcpDiscoveryAdmission(db = getDb()) { + async function live( + tx: JournalTx, + input: McpDiscoveryReservation & { operation?: McpAdministrativeOperation }, + ) { + // Scope -> human -> connection -> discovery row. No lock survives a network operation. + const [actor] = await tx.select().from(user).where(eq(user.id, input.actorId)).for("share"); + if (!actor || actor.role !== "admin" || actor.banned || isServiceAccountEmail(actor.email)) + throw new Error("human administrator required for discovery"); + const [connection] = await tx + .select() + .from(integrationConnection) + .where(eq(integrationConnection.id, input.connectionId)) + .for("share"); + if ( + !connection || + connection.provider !== "mcp" || + connection.revoked || + connection.mcpCredential.transition || + !connection.mcpCredential.status || + connection.configRevision !== input.configRevision || + connection.authGeneration !== input.authGeneration || + connection.credentialIdentity !== input.credentialIdentity + ) + throw new Error("stale discovery authority"); + const config = validateMcpConfig(connection.config); + const status = fromJson(CredentialStatusSchema, connection.mcpCredential.status); + const context = status.context, + reference = status.reference; + if ( + status.protocol !== 1 || + status.status !== "connected" || + !context || + !reference || + context.connectionId !== input.connectionId || + context.configRevision !== BigInt(input.configRevision) || + context.authorizationGeneration !== BigInt(input.authGeneration) || + context.credentialIdentity !== input.credentialIdentity || + context.resource?.endpoint !== config.endpoint || + context.resource.zoneId !== input.identity.zoneId || + context.policyDigest !== input.identity.policyDigest || + context.policyRevision !== input.identity.policyRevision || + (input.operation?.kind === "validateSchemas" + ? input.destination !== "" + : config.endpoint !== input.destination) || + config.zone !== input.identity.zoneId || + input.identity.protocol !== 1 || + !input.identity.bootId || + !input.identity.policyDigest || + !input.identity.policyRevision || + (config.auth.kind === "api_key" + ? reference.kind !== CredentialKind.API_KEY || + reference.connectionId !== input.connectionId || + reference.name !== config.auth.secretRef + : reference.kind !== CredentialKind.OAUTH || + reference.registrationId !== config.auth.clientRegistrationId) + ) + throw new Error("stale discovery credential context"); + return canonicalDigest(toJson(CredentialStatusSchema, status)); + } + async function locked( + requestId: string, + intentId: string, + now: Date, + action: "dispatch" | "finish", + ) { + now = new Date(now.getTime()); + const [observed] = await db.select().from(admission).where(eq(admission.requestId, requestId)); + if (!observed) throw new Error("discovery admission unavailable"); + return db.transaction(async (tx) => { + await lockAdmissionScope(tx, observed.deploymentId); + if (action === "dispatch") await live(tx, observed); + const [row] = await tx + .select() + .from(admission) + .where(eq(admission.requestId, requestId)) + .for("update"); + if ( + !row || + row.intentId !== intentId || + (action === "dispatch" && (row.state !== "reserved" || row.expiresAt <= now)) + ) + throw new Error("discovery intent unavailable"); + await tx + .update(admission) + .set({ state: action === "dispatch" ? "dispatched" : "finished" }) + .where(eq(admission.requestId, requestId)); + }); + } + async function reserve( + input: McpDiscoveryReservation & { operation: McpAdministrativeOperation }, + now: Date, + ): Promise> { + input = structuredClone(input); + now = new Date(now.getTime()); + if ( + ![ + input.requestId, + input.actorId, + input.connectionId, + input.deploymentId, + input.credentialIdentity, + ].every(Boolean) || + !Number.isSafeInteger(input.configRevision) || + input.configRevision < 1 || + !Number.isSafeInteger(input.authGeneration) || + input.authGeneration < 1 || + !Number.isFinite(now.getTime()) + ) + throw new Error("invalid discovery request"); + return db.transaction(async (tx) => { + await lockAdmissionScope(tx, input.deploymentId); + const statusDigest = await live(tx, input); + const [prior] = await tx + .select() + .from(admission) + .where(eq(admission.requestId, input.requestId)); + if (prior) throw new Error("discovery request already consumed"); + // Expired rows retain their identity forever; only their fixed lease stops owning capacity. + const active = await tx + .select() + .from(admission) + .where( + and( + inArray(admission.state, ["reserved", "dispatched"]), + gt(admission.expiresAt, now), + or( + eq(admission.deploymentId, input.deploymentId), + eq(admission.connectionId, input.connectionId), + ), + ), + ); + if ( + active.filter( + (row) => + row.operation.kind === input.operation.kind && row.deploymentId === input.deploymentId, + ).length >= 4 || + (input.operation.kind === "discover" && + active.some( + (row) => row.operation.kind === "discover" && row.connectionId === input.connectionId, + )) + ) + throw new Error("discovery capacity unavailable"); + const row: Row = { + ...input, + intentId: crypto.randomUUID(), + statusDigest, + requestDigest: canonicalDigest({ ...input, statusDigest }), + state: "reserved", + createdAt: now, + expiresAt: new Date(now.getTime() + 30_000), + }; + await tx.insert(admission).values(row); + return intentFor(row); + }); + } + async function assertSchemaAdmitted(operation: AdmittedSchemaOperation, now: Date) { + operation = structuredClone(operation); + const purpose = schemaOperation(operation.documents), + scope = operation.intent.scope; + if (scope.kind !== "administration") throw Error("MCP schema scope required"); + const [observed] = await db + .select() + .from(admission) + .where(eq(admission.requestId, scope.requestId)); + if (!observed) throw Error("MCP schema admission unavailable"); + await db.transaction(async (tx) => { + await lockAdmissionScope(tx, observed.deploymentId); + const statusDigest = await live(tx, observed); + const [row] = await tx + .select() + .from(admission) + .where(eq(admission.requestId, scope.requestId)) + .for("share"); + if ( + !row || + row.state !== "dispatched" || + row.expiresAt <= now || + row.statusDigest !== statusDigest || + canonicalDigest(row.operation) !== canonicalDigest(purpose) || + canonicalDigest(operation.identity) !== canonicalDigest(row.identity) || + canonicalDigest(operation.intent) !== canonicalDigest(intentFor(row)) + ) + throw Error("stale MCP schema admission"); + }); + } + return { + reserve: (input: McpDiscoveryReservation, now: Date) => + reserve({ ...input, operation: { kind: "discover" } }, now), + reserveSchema: (input: McpSchemaReservation, now: Date) => { + const { documents, ...identity } = input; + return reserve({ ...identity, destination: "", operation: schemaOperation(documents) }, now); + }, + assertSchemaAdmitted, + beginDispatch: (requestId: string, intentId: string, now: Date) => + locked(requestId, intentId, now, "dispatch"), + finish: (requestId: string, intentId: string, now: Date) => + locked(requestId, intentId, now, "finish"), + async assertAdmitted(operation: Readonly, now: Date) { + operation = structuredClone(operation); + now = new Date(now.getTime()); + const scope = operation.intent.scope; + if (scope.kind !== "administration") throw new Error("discovery scope required"); + const [observed] = await db + .select() + .from(admission) + .where(eq(admission.requestId, scope.requestId)); + if (!observed) throw new Error("discovery admission unavailable"); + await db.transaction(async (tx) => { + await lockAdmissionScope(tx, observed.deploymentId); + const statusDigest = await live(tx, observed); + const [row] = await tx + .select() + .from(admission) + .where(eq(admission.requestId, scope.requestId)) + .for("share"); + if ( + !row || + row.state !== "dispatched" || + row.operation.kind !== "discover" || + row.expiresAt <= now || + row.statusDigest !== statusDigest || + canonicalDigest(operation.identity) !== canonicalDigest(row.identity) || + canonicalDigest(operation.intent) !== canonicalDigest(intentFor(row)) || + canonicalDigest(operation.operation) !== + canonicalDigest({ kind: "discover", endpoint: row.destination }) || + canonicalDigest(toJson(CredentialStatusSchema, operation.status)) !== row.statusDigest + ) + throw new Error("stale discovery admission"); + }); + }, + }; +} diff --git a/orchestrator/src/mcp/discovery.ts b/orchestrator/src/mcp/discovery.ts new file mode 100644 index 000000000..ff2c070fb --- /dev/null +++ b/orchestrator/src/mcp/discovery.ts @@ -0,0 +1,127 @@ +import { fromJson } from "@bufbuild/protobuf"; +import { z } from "zod"; +import type { WorkerIdentity, WorkerIntent } from "../../../mcp-worker/src/contracts.ts"; +import type { IntegrationConnectionRow } from "../db/integration-connections.ts"; +import { CredentialStatusSchema } from "../gen/engram/mcp/v1/control_pb.ts"; +import { assertSchemaShape } from "./catalog.ts"; +import { validateMcpConfig } from "./config.ts"; +import type { CredentialControlClient } from "./control-client.ts"; +import { makeCredentialBroker, type AdmittedCredentialOperation } from "./credential-broker.ts"; +import type { CredentialMetadataStore } from "./credential-store.ts"; +import { canonicalJson } from "./hash.ts"; +import { MCP_LIMITS } from "./limits.ts"; +import type { makeMcpOperatorRuntime } from "./operator-runtime.ts"; +import { makeResourceWorkerClient } from "./worker-client.ts"; +import type { McpDiscoveredTool } from "./types.ts"; + +export interface DiscoveryAdmission { + reserve( + input: { + requestId: string; + actorId: string; + connectionId: string; + configRevision: number; + authGeneration: number; + credentialIdentity: string; + deploymentId: string; + identity: WorkerIdentity; + destination: string; + }, + now: Date, + ): Promise>; + beginDispatch(requestId: string, intentId: string, now: Date): Promise; + assertAdmitted(operation: Readonly, now: Date): Promise; + finish(requestId: string, intentId: string, now: Date): Promise; +} +const tool = z + .object({ + name: z + .string() + .min(1) + .max(256) + .regex(/^[^\u0000-\u001f]+$/), + description: z.string().max(16384).optional(), + inputSchema: z.record(z.string(), z.unknown()), + outputSchema: z.record(z.string(), z.unknown()).optional(), + annotations: z.record(z.string(), z.unknown()).optional(), + }) + .strict(); +export function discoveredCatalog(value: unknown): McpDiscoveredTool[] { + if (Buffer.byteLength(canonicalJson(value)) > MCP_LIMITS.manifestBytes) + throw Error("MCP catalog exceeds limit"); + const tools = z.array(tool).max(MCP_LIMITS.toolsPerSession).parse(value); + if (new Set(tools.map((t) => t.name)).size !== tools.length) throw Error("MCP duplicate tool"); + return tools.map((t) => { + assertSchemaShape(t.inputSchema); + if (t.outputSchema) assertSchemaShape(t.outputSchema); + return { + upstreamName: t.name, + description: t.description ?? "", + inputSchema: t.inputSchema, + ...(t.outputSchema ? { outputSchema: t.outputSchema } : {}), + }; + }); +} +/** Discovery owns one durable administrative intent. No retry path can resend it. */ +export function makeMcpDiscovery(deps: { + admission: DiscoveryAdmission; + credentials: Pick; + control: CredentialControlClient; + operator: ReturnType; + now(): Date; +}) { + return { + async discover(connection: IntegrationConnectionRow, actorId: string) { + if (connection.provider !== "mcp" || !connection.mcp) + throw Error("MCP connection unavailable"); + const config = validateMcpConfig(connection.config); + const selected = await deps.operator.select(config); + const worker = makeResourceWorkerClient({ tls: selected.tls, policy: selected.policy }); + const requestId = crypto.randomUUID(); + let intent: Omit | undefined; + try { + const identity = await worker.identity(); + intent = await deps.admission.reserve( + { + requestId, + actorId, + connectionId: connection.id, + configRevision: connection.mcp.configRevision, + authGeneration: connection.mcp.authGeneration, + credentialIdentity: connection.mcp.credentialIdentity, + deploymentId: selected.deploymentId, + identity, + destination: config.endpoint, + }, + deps.now(), + ); + const row = await deps.credentials.get(connection.id); + if (!row.mcpCredential.status || row.mcpCredential.transition) + throw Error("MCP credential unavailable"); + const status = fromJson(CredentialStatusSchema, row.mcpCredential.status); + await deps.admission.beginDispatch(requestId, intent.id, deps.now()); + const broker = makeCredentialBroker({ + control: deps.control, + worker, + admission: { + assertAdmitted: (operation) => deps.admission.assertAdmitted(operation, deps.now()), + }, + beforeDispatch: () => deps.operator.assertCurrent(config, selected.policy), + now: () => deps.now().getTime(), + }); + const response = await broker.dispatch({ + status, + identity, + intent, + operation: { kind: "discover", endpoint: config.endpoint }, + }); + if (!response.ok) throw Error("MCP discovery unavailable"); + await deps.operator.assertCurrent(config, selected.policy); + return discoveredCatalog(response.result); + } finally { + worker.close(); + if (intent) await deps.admission.finish(requestId, intent.id, deps.now()); + } + }, + }; +} diff --git a/orchestrator/src/mcp/effect-driver.ts b/orchestrator/src/mcp/effect-driver.ts new file mode 100644 index 000000000..d5b248842 --- /dev/null +++ b/orchestrator/src/mcp/effect-driver.ts @@ -0,0 +1,597 @@ +import { parseArgumentJson } from "./arguments.ts"; +import { McpAuthorityStale } from "./authority-error.ts"; +import { fromJson } from "@bufbuild/protobuf"; +import { and, eq } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { integrationConnection, mcpExecution, taskSession } from "../db/schema.ts"; +import { CredentialStatusSchema } from "../gen/engram/mcp/v1/control_pb.ts"; +import { digest } from "../../../mcp-worker/src/index.ts"; +import type { + WorkerIdentity, + WorkerIntent, + WorkerOperation, +} from "../../../mcp-worker/src/contracts.ts"; +import type { + CredentialControlClient, + CredentialControlTls, + SessionPolicyControlClient, +} from "./control-client.ts"; +import { makeCredentialBroker } from "./credential-broker.ts"; +import { makeMcpActiveWatch } from "./active-watch.ts"; +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import { makeMcpJournal, McpUnknownWrite, type McpLiveDispatchAuthorization } from "./journal.ts"; +import { requestPolicyContext } from "./journal-review.ts"; +import { MCP_LIMITS } from "./limits.ts"; +import { makeSessionWritePolicy, type RequestPolicyContext } from "./session-policy.ts"; +import { + resolveSnapshotTool, + type SessionBindingResolver, + type ResolvedBoundTool, +} from "./session-resolver.ts"; +import { hasPublishedLeaf, type McpSessionBindingSnapshot } from "./session-bindings.ts"; +import { makeResourceWorkerClient } from "./worker-client.ts"; +import { toWorkerBindingHash } from "./worker-encoding.ts"; + +type Call = typeof mcpExecution.$inferSelect; +type Lease = NonNullable["getAdmission"]>>>; +export interface McpToolRequest { + sessionId: string; + runId: string; + toolCallId: string; + modelToolName: string; + /** Exact first coordinator tool_call_requested.args_json, not parsed and serialized again. */ + originalArguments: string; +} +const requestDenials = { + invalid_arguments: "MCP arguments invalid or exceed limit", + tool_unavailable: "MCP tool unavailable", + publication_required: "MCP profile grants require administrator publication", + disabled: "external MCP is disabled", + session_unavailable: "MCP session unavailable", + authority_changed: "MCP execution authority changed", + principal_unavailable: "MCP principal unavailable", + write_not_allowed: "MCP write authority unavailable", + deadline_expired: "MCP request deadline expired", +} as const; +export type McpRequestDenialCode = keyof typeof requestDenials; +/** A known local rejection, before any execution effect. No raw values or underlying error. */ +export class McpRequestDenied extends Error { + constructor(readonly code: McpRequestDenialCode) { + super(requestDenials[code]); + } +} +export class McpRequestConflict extends Error { + readonly code = "existing_request"; + constructor() { + super("MCP call identity changed"); + } +} +class InvalidArguments extends Error { + constructor(readonly lease: Lease) { + super("MCP arguments do not satisfy approved schemas"); + } +} +function object(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} +function argumentsValue(raw: string): Record { + try { + if (Buffer.byteLength(raw) > MCP_LIMITS.argumentBytes) + throw new McpRequestDenied("invalid_arguments"); + const value: unknown = parseArgumentJson(raw); + if (!object(value)) throw new McpRequestDenied("invalid_arguments"); + return value; + } catch { + throw new McpRequestDenied("invalid_arguments"); + } +} +function publication(snapshot: McpSessionBindingSnapshot) { + if (!hasPublishedLeaf(snapshot)) throw new McpRequestDenied("publication_required"); +} + +/** The authenticated worker envelope is verified by the client and broker first. */ +export function verifiedExecutionResult(value: unknown): { + state: "succeeded" | "failed" | "outcome_unknown"; + result: Record; + resultUnavailable: boolean; +} { + if ( + !object(value) || + Object.keys(value).some((key) => !["outcome", "delivery", "output", "reason"].includes(key)) || + !["succeeded", "failed", "outcome_unknown"].includes(String(value.outcome)) || + !["available", "unavailable"].includes(String(value.delivery)) + ) + throw Error("MCP result invalid"); + const state = value.outcome; + if (state !== "succeeded" && state !== "failed" && state !== "outcome_unknown") + throw Error("MCP result invalid"); + if (value.delivery === "available") { + const output = value.output; + if ( + state === "outcome_unknown" || + value.reason !== undefined || + !object(output) || + Object.keys(output).some( + (key) => !["content", "structuredContent", "isError"].includes(key), + ) || + !Array.isArray(output.content) || + output.content.some( + (item: unknown) => + !object(item) || + Object.keys(item).some((key) => !["type", "text"].includes(key)) || + item.type !== "text" || + typeof item.text !== "string", + ) || + (output.isError !== undefined && typeof output.isError !== "boolean") || + (state === "failed" ? output.isError !== true : output.isError === true) + ) + throw Error("MCP result invalid"); + if (Buffer.byteLength(canonicalJson(value)) > MCP_LIMITS.resultBytes) { + return { + state, + result: { outcome: state, delivery: "unavailable", reason: "result_limit" }, + resultUnavailable: state === "succeeded", + }; + } + } else if ( + value.output !== undefined || + !["unsupported_content", "result_limit", "output_schema", "upstream_failure"].includes( + String(value.reason), + ) || + (state === "outcome_unknown" && value.reason !== "upstream_failure") + ) + throw Error("MCP result invalid"); + return { + state, + result: structuredClone(value), + resultUnavailable: state === "succeeded" && value.delivery === "unavailable", + }; +} + +/** Record verified truth independently of guest delivery and local cancellation. */ +export async function persistVerifiedOutcome( + journal: ReturnType, + dispatched: Call, + outcome: ReturnType, + now: () => Date, + verified = true, +): Promise { + // Cancellation can change dispatching to unknown once. Only the result CAS is retried. + for (let attempt = 0; attempt < 2; attempt++) { + const latest = await journal.get(dispatched.id); + if (!latest || latest.intentId !== dispatched.intentId) + throw Error("MCP result identity changed"); + if (!["dispatching", "outcome_unknown"].includes(latest.state)) return latest; + if (latest.state === "outcome_unknown" && (!verified || outcome.state === "outcome_unknown")) + return latest; + try { + return await journal.finish( + dispatched.id, + latest.version, + outcome.state, + outcome.result, + outcome.resultUnavailable, + now(), + ); + } catch (error) { + if (attempt === 1) throw error; + } + } + throw Error("MCP result commit unavailable"); +} + +/** Trusted effect driver. Call directly, never inside a replayable DBOS/SQL step. + * Task 6 supplies operator/host enforcement and trusted zone TLS configuration. */ +export function makeMcpEffectDriver(deps: { + db?: ReturnType; + deploymentId: string | ((zoneId: string) => string); + control: CredentialControlClient; + policy: SessionPolicyControlClient; + resolver: SessionBindingResolver; + workerTls(zoneId: string): CredentialControlTls; + /** Fresh global and zone kill switches. Missing configuration stays disabled. */ + enabled?(zoneId: string): boolean; + now(): Date; +}) { + const db = deps.db ?? getDb(), + journal = makeMcpJournal(db), + policy = makeSessionWritePolicy(deps.policy); + function enabled(resolved: ResolvedBoundTool, signal?: AbortSignal) { + signal?.throwIfAborted(); + if (!deps.enabled?.(resolved.binding.config.zone)) throw new McpRequestDenied("disabled"); + } + function deployment(resolved: ResolvedBoundTool): string { + const id = + typeof deps.deploymentId === "function" + ? deps.deploymentId(resolved.binding.config.zone) + : deps.deploymentId; + if (!id || id.length > 256) throw Error("MCP worker deployment unavailable"); + return id; + } + async function lookup(input: McpToolRequest) { + const [call] = await db + .select() + .from(mcpExecution) + .where( + and( + eq(mcpExecution.sessionId, input.sessionId), + eq(mcpExecution.toolCallId, input.toolCallId), + ), + ); + if ( + call && + (call.runId !== input.runId || + call.originalArguments !== input.originalArguments || + call.requestContext?.resolvedTool.modelToolName !== input.modelToolName) + ) + throw new McpRequestConflict(); + return call; + } + async function fence(context: RequestPolicyContext, resolved: ResolvedBoundTool, raw: string) { + const expected = { modelToolName: resolved.modelToolName, originalArguments: raw }; + const status = await (resolved.reviewedTool.classification === "write" + ? policy.validateDispatchFence(context, expected) + : policy.validateExecutionIdentity(context, expected)); + if (status === "stale") throw new McpRequestDenied("authority_changed"); + if (status !== "current") throw Error("MCP policy fence unavailable"); + } + async function current(call: Call, signal?: AbortSignal) { + const resolved = call.requestContext?.resolvedTool; + if (!resolved || call.originalArguments === null || call.deadline <= deps.now()) + throw Error("MCP request unavailable"); + enabled(resolved, signal); + const snapshot = await deps.resolver.readCurrent(call.sessionId); + publication(snapshot); + if (snapshot.bindingHash !== call.bindingHash) + throw new McpAuthorityStale("MCP binding changed"); + await journal.assertCurrent(call, deps.now()); + await fence(requestPolicyContext(call.requestContext!), resolved, call.originalArguments); + enabled(resolved, signal); + return resolved; + } + function client(resolved: ResolvedBoundTool, signal?: AbortSignal) { + return makeResourceWorkerClient({ + tls: deps.workerTls(resolved.binding.config.zone), + policy: resolved.binding.operatorPolicy, + signal, + }); + } + function intent( + call: Call, + lease: Lease, + identity: WorkerIdentity, + operation: WorkerOperation, + ): Omit { + const bound = call.requestContext!.resolvedTool.binding; + return { + id: crypto.randomUUID(), + bootId: identity.bootId, + zoneId: identity.zoneId, + policyDigest: identity.policyDigest, + connectionId: call.connectionId, + connectionRevision: call.configRevision, + credentialIdentity: bound.credentialIdentity, + scope: { + kind: "execution", + sessionId: call.sessionId, + runId: call.runId, + toolCallId: call.toolCallId, + bindingHash: toWorkerBindingHash(call.bindingHash), + }, + operation: operation.kind, + destination: "endpoint" in operation ? operation.endpoint : "", + requestDigest: call.requestDigest, + expiresAt: lease.expiresAt.getTime(), + lease: { + id: lease.reservationId, + deploymentId: lease.deploymentId, + expiresAt: lease.expiresAt.getTime(), + }, + }; + } + async function schema(call: Call, resolved: ResolvedBoundTool, signal?: AbortSignal) { + if ((await journal.getAdmission(call.id))?.schemaDigest === call.requestDigest) return true; + const operations: Extract[] = [ + ...(resolved.reviewedTool.outputSchema + ? [{ kind: "validateSchemas" as const, documents: [resolved.reviewedTool.outputSchema] }] + : []), + { + kind: "validateSchemas", + documents: [ + resolved.reviewedTool.inputSchema, + ...resolved.effectiveGrant.argumentConstraintDocuments, + ], + value: argumentsValue(call.originalArguments!), + }, + ]; + for (let i = 0; i < operations.length; i++) { + if ((await journal.admitSchema(call.id, deployment(resolved), deps.now())) === "queued") + return false; + const lease = (await journal.getAdmission(call.id))!; + const worker = client(resolved, signal); + let success = false; + try { + await current(call, signal); + const identity = await worker.identity(), + operation = operations[i]!; + const response = await worker.dispatch({ + protocol: 1, + intent: { + ...intent(call, lease, identity, operation), + operationDigest: digest(operation), + }, + operation, + }); + if (response.ok && response.result === false) throw new InvalidArguments(lease); + if (!response.ok || response.result !== true) throw Error("MCP schema validation failed"); + success = true; + } finally { + worker.close(); + // Compile-only success releases capacity without claiming final argument proof. + await journal.finishSchema( + call.id, + lease.reservationId, + lease.expiresAt, + call.requestDigest, + success && i === operations.length - 1, + deps.now(), + ); + } + } + return true; + } + const activeWatch = makeMcpActiveWatch({ + intervalMs: 1_000, + async check(id) { + const call = await journal.get(id); + if (!call || !["waiting_approval", "authorized", "dispatching"].includes(call.state)) + throw Error("MCP execution no longer active"); + await current(call); + }, + }); + return { + cancel(id: string) { + activeWatch.cancel(id); + }, + async reconcile(id: string, signal?: AbortSignal) { + signal?.throwIfAborted(); + const call = await journal.get(id); + if (!call || !["waiting_approval", "authorized", "dispatching"].includes(call.state)) return; + if (call.deadline <= deps.now()) { + signal?.throwIfAborted(); + activeWatch.cancel(id); + await journal.cancelInternal(id, "deadline", deps.now()); + return; + } + try { + await current(call, signal); + } catch (error) { + activeWatch.cancel(id); + signal?.throwIfAborted(); + if (error instanceof McpRequestDenied || error instanceof McpAuthorityStale) { + await journal.cancelInternal(id, "authority_changed", deps.now()); + return; + } + throw error; + } + }, + async acceptToolRequest(input: McpToolRequest, trustedDeadline?: Date) { + input = structuredClone(input); + const existing = await lookup(input); + if (existing) return existing; + const parsed = argumentsValue(input.originalArguments); + // A transport failure must remain retryable, not become a durable missing-tool denial. + const snapshot = await deps.resolver.readCurrent(input.sessionId); + const resolved = resolveSnapshotTool(snapshot, input.modelToolName); + if (!resolved) throw new McpRequestDenied("tool_unavailable"); + enabled(resolved); + publication(snapshot); + const currentPolicy = await policy.readForRequest(input); + if (!currentPolicy.current || !currentPolicy.ref) + throw new McpRequestDenied("session_unavailable"); + const context: RequestPolicyContext = { + sessionId: input.sessionId, + runId: input.runId, + toolCallId: input.toolCallId, + acceptedPolicyRef: currentPolicy.ref, + acceptedWriteEligibility: currentPolicy.writeEligibility, + ancestorPolicyRefs: currentPolicy.ancestorPolicyRefs, + }; + if ( + resolved.reviewedTool.classification === "write" && + context.acceptedWriteEligibility !== "write_eligible" + ) + throw new McpRequestDenied("write_not_allowed"); + await fence(context, resolved, input.originalArguments); + const [session] = await db + .select() + .from(taskSession) + .where(eq(taskSession.sessionId, input.sessionId)); + if (!session?.integrationPrincipalId) throw new McpRequestDenied("principal_unavailable"); + const write = resolved.reviewedTool.classification === "write"; + const authorization = !write + ? { kind: "read" as const } + : session.integrationPrincipalId.startsWith("automation:") + ? { kind: "automation" as const, grant: resolved.effectiveGrant } + : { kind: "approval" as const }; + if (authorization.kind === "automation" && !authorization.grant.automationWrite) + throw new McpRequestDenied("write_not_allowed"); + const now = deps.now(); + // Both phase queues and approval share this first immutable outer cap. + const cap = + now.getTime() + + (authorization.kind === "approval" ? MCP_LIMITS.approvalMs : 0) + + MCP_LIMITS.queueMs + + MCP_LIMITS.maxToolMs; + const deadline = new Date(Math.min(cap, trustedDeadline?.getTime() ?? cap)); + if (!Number.isFinite(deadline.getTime()) || deadline <= now) + throw new McpRequestDenied("deadline_expired"); + let call: Call; + try { + call = await journal.create( + { + taskId: session.taskId, + ...input, + principalId: session.integrationPrincipalId, + connectionId: resolved.binding.connectionId, + catalogId: resolved.binding.catalogId, + bindingHash: resolved.bindingHash, + argumentHash: canonicalDigest(parsed), + resolvedTool: resolved, + requestPolicyContext: context, + upstreamName: resolved.reviewedTool.upstreamName, + configRevision: resolved.binding.configRevision, + authGeneration: resolved.binding.authGeneration, + classification: resolved.reviewedTool.classification, + deadline, + }, + authorization, + now, + ); + } catch (error) { + // A concurrent first acceptance owns the deadline and immutable context. + const winner = await lookup(input); + if (winner) return winner; + if (error instanceof McpUnknownWrite) throw new McpRequestDenied("write_not_allowed"); + throw error; + } + await current(call); + return call; + }, + async driveOnce(id: string, signal?: AbortSignal): Promise { + const watch = activeWatch.register(id); + signal = signal ? AbortSignal.any([signal, watch.signal]) : watch.signal; + try { + let call = await journal.get(id); + if (!call) throw Error("MCP call unavailable"); + if (!["waiting_approval", "authorized"].includes(call.state)) return call; + await watch.ready; + watch.throwIfFailed(); + signal.throwIfAborted(); + const resolved = await current(call, signal); + try { + if (!(await schema(call, resolved, signal))) return (await journal.get(id))!; + } catch (error) { + if (!(error instanceof InvalidArguments)) throw error; + return journal.rejectBeforeDispatch( + id, + { + requestDigest: call.requestDigest, + version: call.version, + reservation: { + reservationId: error.lease.reservationId, + expiresAt: error.lease.expiresAt, + }, + }, + deps.now(), + ); + } + if (call.state === "waiting_approval") { + await journal.requestApproval(id, deployment(resolved), deps.now()); + return (await journal.get(id))!; + } + await current(call, signal); + const [connection] = await db + .select() + .from(integrationConnection) + .where(eq(integrationConnection.id, call.connectionId)); + if (!connection?.mcpCredential.status || connection.mcpCredential.transition) + throw Error("MCP credential authority unavailable"); + const status = fromJson(CredentialStatusSchema, connection.mcpCredential.status); + if (status.context?.authorizationGeneration !== BigInt(call.authGeneration)) + throw Error("MCP credential authority changed"); + if ((await journal.admit(id, deployment(resolved), deps.now())) === "queued") + return (await journal.get(id))!; + const lease = (await journal.getAdmission(id))!; + const worker = client(resolved, signal); + try { + const identity = await worker.identity(); + await current(call, signal); + const operation: Extract = { + kind: "execute", + endpoint: resolved.binding.config.endpoint, + tool: resolved.reviewedTool.upstreamName, + arguments: argumentsValue(call.originalArguments!), + inputSchema: resolved.reviewedTool.inputSchema, + ...(resolved.reviewedTool.outputSchema + ? { outputSchema: resolved.reviewedTool.outputSchema } + : {}), + argumentConstraintDocuments: resolved.effectiveGrant.argumentConstraintDocuments, + classification: resolved.reviewedTool.classification, + }; + const dispatchIntent = intent(call, lease, identity, operation); + const proof: McpLiveDispatchAuthorization = { + executionId: id, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + principalId: call.principalId, + expectedJournalRevision: call.version, + reservationId: lease.reservationId, + reservationExpiresAt: lease.expiresAt, + }; + call = await journal.dispatch( + id, + { bootId: identity.bootId, intentId: dispatchIntent.id, expiresAt: lease.expiresAt }, + deps.now(), + proof, + ); + const dispatched = call; + const broker = makeCredentialBroker({ + control: deps.control, + now: () => deps.now().getTime(), + admission: { + async assertAdmitted() { + await current(dispatched, signal); + await journal.assertDispatchCurrent(id, proof, deps.now()); + }, + }, + worker, + async beforeDispatch() { + enabled(resolved, signal); + await fence( + requestPolicyContext(dispatched.requestContext!), + resolved, + dispatched.originalArguments!, + ); + // The broker checks delivery expiry next, then sends on the retained socket. + enabled(resolved, signal); + }, + }); + let outcome: ReturnType; + let verified = true; + try { + const response = await broker.dispatch({ + status, + identity, + intent: dispatchIntent, + operation, + }); + if (!response.ok) throw Error("MCP result unavailable"); + outcome = verifiedExecutionResult(response.result); + } catch { + verified = false; + const latest = await journal.get(call.id); + if (!latest || latest.intentId !== dispatched.intentId) + throw Error("MCP result identity changed"); + const interrupted = signal?.aborted || latest.state === "outcome_unknown"; + const unknown = interrupted || call.classification === "write"; + outcome = { + state: unknown ? "outcome_unknown" : "failed", + result: { + outcome: unknown ? "outcome_unknown" : "failed", + delivery: "unavailable", + reason: interrupted ? "execution_interrupted" : "upstream_failure", + }, + resultUnavailable: false, + }; + } + return persistVerifiedOutcome(journal, dispatched, outcome, deps.now, verified); + } finally { + worker.close(); + } + } finally { + watch.close(); + } + }, + }; +} diff --git a/orchestrator/src/mcp/hash.ts b/orchestrator/src/mcp/hash.ts new file mode 100644 index 000000000..4cffea471 --- /dev/null +++ b/orchestrator/src/mcp/hash.ts @@ -0,0 +1,31 @@ +import { createHash } from "node:crypto"; + +/** Strict JSON canonicalization: no silent removal of unsupported values. */ +export function canonicalJson(value: unknown): string { + const seen = new Set(); + function visit(v: unknown): unknown { + if (v === null || typeof v === "string" || typeof v === "boolean") return v; + if (typeof v === "number" && Number.isFinite(v)) return v; + if (typeof v !== "object" || v === null || seen.has(v)) + throw new Error("expected finite, acyclic JSON"); + seen.add(v); + let result: unknown; + if (Array.isArray(v)) result = v.map(visit); + else { + if (Object.getPrototypeOf(v) !== Object.prototype && Object.getPrototypeOf(v) !== null) + throw new Error("expected plain JSON object"); + result = Object.fromEntries( + Object.keys(v) + .sort() + .map((k) => [k, visit((v as Record)[k])]), + ); + } + seen.delete(v); + return result; + } + return JSON.stringify(visit(value)); +} + +export function canonicalDigest(value: unknown): string { + return `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`; +} diff --git a/orchestrator/src/mcp/journal-review.ts b/orchestrator/src/mcp/journal-review.ts new file mode 100644 index 000000000..1aa811bed --- /dev/null +++ b/orchestrator/src/mcp/journal-review.ts @@ -0,0 +1,357 @@ +import { parseArgumentJson } from "./arguments.ts"; +import { McpAuthorityStale } from "./authority-error.ts"; +import { + create, + fromJson, + toJson, + type JsonValue, + type MessageInitShape, +} from "@bufbuild/protobuf"; +import { and, eq } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { + user, + task, + taskSession, + profile, + mcpExecution, + mcpApproval, + integrationConnection, + mcpCatalog, +} from "../db/schema.ts"; +import { isServiceAccountEmail } from "../rpc/api-key.ts"; +import { + McpWritePolicyRefSchema, + McpTrustedWritePolicySchema, +} from "../gen/engram/app/v1/session_pb.ts"; +import type { RequestPolicyContext } from "./session-policy.ts"; +import type { ResolvedBoundTool } from "./session-resolver.ts"; +import { restoreBindingSnapshot } from "./binding-store.ts"; +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import { MCP_LIMITS } from "./limits.ts"; + +type Db = ReturnType; +export type JournalTx = Parameters[0]>[0]; +export type FrozenRequestContext = { + resolvedTool: ResolvedBoundTool; + requestPolicyContext: Omit & { + acceptedPolicyRef: JsonValue; + ancestorPolicyRefs: JsonValue[]; + }; +}; +export type JournalReviewContext = { + executionId: string; + sessionId: string; + runId: string; + toolCallId: string; + requestDigest: string; + argumentHash: string; + bindingHash: string; + connectionId: string; + catalogId: string; + catalogRevision: number; + configRevision: number; + authGeneration: number; + credentialIdentity: string; + upstreamName: string; + toolFingerprint: string; + contextDigest: string; + journalRevision: number; + expiresAt: string; + policyRevision: string; + bindingEpoch: string; +}; +export function freezeRequestContext( + resolvedTool: ResolvedBoundTool, + policy: RequestPolicyContext, +): FrozenRequestContext { + const result = { + resolvedTool: structuredClone(resolvedTool), + requestPolicyContext: { + ...policy, + acceptedPolicyRef: toJson(McpWritePolicyRefSchema, policy.acceptedPolicyRef), + ancestorPolicyRefs: policy.ancestorPolicyRefs.map((ref) => + toJson(McpWritePolicyRefSchema, ref), + ), + }, + }; + if (Buffer.byteLength(canonicalJson(result)) > 2 * MCP_LIMITS.manifestBytes) + throw new Error("request context exceeds limit"); + return JSON.parse(canonicalJson(result)); +} +export function requestPolicyContext(context: FrozenRequestContext): RequestPolicyContext { + const policy = context.requestPolicyContext; + return { + ...policy, + acceptedPolicyRef: fromJson(McpWritePolicyRefSchema, policy.acceptedPolicyRef), + ancestorPolicyRefs: policy.ancestorPolicyRefs.map((ref) => + fromJson(McpWritePolicyRefSchema, ref), + ), + }; +} +/** Compare the complete frozen mode-policy lineage; a leaf revision alone is not enough. */ +export function policyReviewCurrent( + context: FrozenRequestContext, + current: MessageInitShape, +): boolean { + const policy = create(McpTrustedWritePolicySchema, current); + return ( + Boolean(policy.ref) && + canonicalDigest(context.requestPolicyContext.acceptedPolicyRef) === + canonicalDigest(toJson(McpWritePolicyRefSchema, policy.ref!)) && + canonicalDigest(context.requestPolicyContext.ancestorPolicyRefs) === + canonicalDigest(policy.ancestorPolicyRefs.map((ref) => toJson(McpWritePolicyRefSchema, ref))) + ); +} +export function assertOriginalArguments(original: string, expectedHash: string) { + if ( + typeof original !== "string" || + Buffer.byteLength(original) > MCP_LIMITS.argumentBytes || + Buffer.from(original).toString("utf8") !== original + ) + throw new Error("invalid argument bytes"); + let parsed: unknown; + try { + parsed = parseArgumentJson(original); + } catch { + throw new Error("invalid argument JSON"); + } + if ( + !parsed || + typeof parsed !== "object" || + Array.isArray(parsed) || + canonicalDigest(parsed) !== expectedHash + ) + throw new Error("argument digest mismatch"); +} +/** Shared locks serialize authorization with owner, role, ban and session changes. */ +export async function lockHumanSession( + tx: JournalTx, + sessionId: string, + actorId: string, + includeAncestors = false, + sessionLock: "share" | "update" = "share", +) { + const [actor] = await tx.select().from(user).where(eq(user.id, actorId)).for("share"); + if (!actor || actor.banned || isServiceAccountEmail(actor.email)) + throw new McpAuthorityStale("human session owner or administrator required"); + const { owner, session, ancestors } = await lockSessionChain( + tx, + sessionId, + includeAncestors, + sessionLock, + ); + if (actor.role !== "admin" && owner.createdByUserId !== actorId) + throw new McpAuthorityStale("human session owner or administrator required"); + return { owner, session, actor, ancestors }; +} +export async function lockSessionChain( + tx: JournalTx, + sessionId: string, + includeAncestors = true, + sessionLock: "share" | "update" = "share", +) { + const chain: { sessionId: string; taskId: string; parentId: string | null }[] = []; + let id: string | null = sessionId; + const seen = new Set(); + while (id) { + if (seen.has(id) || chain.length >= 32) + throw new McpAuthorityStale("stale MCP review ancestry"); + seen.add(id); + const refs = await tx + .select({ taskId: taskSession.taskId, parentId: task.spawningSessionId }) + .from(taskSession) + .innerJoin(task, eq(task.id, taskSession.taskId)) + .where(eq(taskSession.sessionId, id)) + .limit(2); + if (refs.length !== 1) + throw new McpAuthorityStale("human session owner or administrator required"); + chain.push({ sessionId: id, ...refs[0]! }); + id = includeAncestors ? refs[0]!.parentId : null; + } + // Global order: user -> sorted tasks -> sorted sessions -> sorted profiles/connections -> execution. + const tasks = new Map(); + for (const taskId of [...new Set(chain.map((ref) => ref.taskId))].sort()) { + const [row] = await tx.select().from(task).where(eq(task.id, taskId)).for("share"); + if (!row) throw new McpAuthorityStale("human session owner or administrator required"); + tasks.set(taskId, row); + } + const sessions = new Map(); + for (const ref of [...chain].sort((a, b) => a.sessionId.localeCompare(b.sessionId))) { + const [row] = await tx + .select() + .from(taskSession) + .where(and(eq(taskSession.sessionId, ref.sessionId), eq(taskSession.taskId, ref.taskId))) + .for(sessionLock); + // Revalidate the discovered chain after all relevant writes are blocked. + if (!row || tasks.get(ref.taskId)?.spawningSessionId !== ref.parentId) + throw new McpAuthorityStale("stale MCP review ancestry"); + sessions.set(ref.sessionId, row); + } + const owner = tasks.get(chain[0]!.taskId)!, + session = sessions.get(sessionId)!; + return { + owner, + session, + ancestors: chain + .slice(1) + .map((ref) => ({ session: sessions.get(ref.sessionId)!, owner: tasks.get(ref.taskId)! })), + }; +} +export type McpReviewCall = Pick< + typeof mcpExecution.$inferSelect, + "requestContext" | "originalArguments" | "bindingHash" +>; +export async function assertLocalReviewCurrent( + tx: JournalTx, + call: McpReviewCall, + session: typeof taskSession.$inferSelect, + ancestors: { session: typeof taskSession.$inferSelect; owner: typeof task.$inferSelect }[], + connectionLock: "share" | "update" = "share", +) { + if (!call.requestContext || call.originalArguments === null || !session.mcpBindingEnvelope) + throw new McpAuthorityStale("original request context unavailable"); + const stored = restoreBindingSnapshot( + session.mcpBindingEnvelope, + session.integrationConnections ?? [], + ); + const bound = call.requestContext.resolvedTool; + const policy = requestPolicyContext(call.requestContext); + if ( + stored.bindingHash !== call.bindingHash || + canonicalDigest(stored.profilePublication ?? null) !== + canonicalDigest(bound.profilePublication ?? null) || + BigInt(stored.bindingEpoch) !== policy.acceptedPolicyRef.bindingEpoch || + canonicalDigest(stored.ancestorAuthorityRefs) !== + canonicalDigest(bound.ancestorAuthorityRefs) || + !stored.immutableBindings.some( + (binding) => canonicalDigest(binding) === canonicalDigest(bound.binding), + ) + ) + throw new McpAuthorityStale("stale MCP review binding"); + const snapshots = [stored]; + const [leafTask] = await tx.select().from(task).where(eq(task.id, session.taskId)); + let parentId = leafTask?.spawningSessionId; + const seen = new Set([session.sessionId]); + if (stored.ancestorAuthorityRefs.length >= 32) + throw new McpAuthorityStale("stale MCP review ancestry"); + for (let i = 0; i < stored.ancestorAuthorityRefs.length; i++) { + const expected = stored.ancestorAuthorityRefs[i]!; + if (parentId !== expected.sessionId || seen.has(parentId)) + throw new McpAuthorityStale("stale MCP review ancestry"); + seen.add(parentId); + const ancestor = ancestors[i]?.session, + ancestorTask = ancestors[i]?.owner; + if ( + !ancestor?.mcpBindingEnvelope || + !ancestorTask || + ancestor.sessionId !== parentId || + ["done", "failed", "cancelled"].includes(ancestorTask.status) + ) + throw new McpAuthorityStale("stale MCP review ancestry"); + const snapshot = restoreBindingSnapshot( + ancestor.mcpBindingEnvelope, + ancestor.integrationConnections ?? [], + ); + if ( + snapshot.bindingHash !== expected.bindingHash || + snapshot.bindingEpoch !== expected.bindingEpoch || + canonicalDigest(snapshot.ancestorAuthorityRefs) !== + canonicalDigest(stored.ancestorAuthorityRefs.slice(i + 1)) + ) + throw new McpAuthorityStale("stale MCP review ancestry"); + snapshots.push(snapshot); + parentId = ancestorTask.spawningSessionId; + } + if (parentId || ancestors.length !== stored.ancestorAuthorityRefs.length) + throw new McpAuthorityStale("stale MCP review ancestry"); + const profileIds = [...new Set(snapshots.map((snapshot) => snapshot.profileId))].sort(); + for (const id of profileIds) { + const [currentProfile] = await tx.select().from(profile).where(eq(profile.id, id)).for("share"); + if ( + !currentProfile || + currentProfile.deletedAt || + snapshots + .filter((snapshot) => snapshot.profileId === id) + .some( + (snapshot) => + canonicalDigest( + (currentProfile.integrationGrants ?? []).filter((grant) => grant.mcp !== undefined), + ) !== snapshot.profileGrantsHash || + canonicalDigest(currentProfile.mcpGrantPublication) !== + canonicalDigest(snapshot.profilePublication ?? null), + ) + ) + throw new McpAuthorityStale("stale MCP review profile"); + } + const bindings = snapshots.flatMap((snapshot) => snapshot.immutableBindings); + for (const id of [...new Set(bindings.map((binding) => binding.connectionId))].sort()) { + const [live] = await tx + .select() + .from(integrationConnection) + .where(eq(integrationConnection.id, id)) + .for(connectionLock); + for (const binding of bindings.filter((binding) => binding.connectionId === id)) { + const [catalog] = await tx + .select() + .from(mcpCatalog) + .where(eq(mcpCatalog.id, binding.catalogId)); + if ( + !live || + live.provider !== "mcp" || + !live.enabled || + live.mcpCredential?.transition || + live.revoked || + live.alias !== binding.alias || + canonicalDigest(live.config) !== canonicalDigest(binding.config) || + live.configRevision !== binding.configRevision || + live.authGeneration !== binding.authGeneration || + live.credentialIdentity !== binding.credentialIdentity || + live.catalogRevision !== binding.catalogRevision || + !catalog || + catalog.kind !== "published" || + catalog.connectionId !== id || + catalog.revision !== binding.catalogRevision || + catalog.configRevision !== binding.configRevision || + catalog.authGeneration !== binding.authGeneration || + catalog.credentialIdentity !== binding.credentialIdentity || + binding.tools.some( + (bound) => + !catalog.tools.some((tool) => canonicalDigest(tool) === canonicalDigest(bound.tool)), + ) + ) + throw new McpAuthorityStale("stale MCP review connection"); + } + } +} +export function reviewContext( + call: typeof mcpExecution.$inferSelect, + approval: typeof mcpApproval.$inferSelect, +): JournalReviewContext { + if (!call.requestContext || call.originalArguments === null) + throw new McpAuthorityStale("original request context unavailable"); + const { resolvedTool } = call.requestContext, + policy = requestPolicyContext(call.requestContext); + return { + executionId: call.id, + sessionId: call.sessionId, + runId: call.runId, + toolCallId: call.toolCallId, + requestDigest: call.requestDigest, + argumentHash: call.argumentHash, + bindingHash: call.bindingHash, + connectionId: call.connectionId, + catalogId: call.catalogId, + catalogRevision: resolvedTool.binding.catalogRevision, + configRevision: call.configRevision, + authGeneration: call.authGeneration, + credentialIdentity: resolvedTool.binding.credentialIdentity, + upstreamName: call.upstreamName, + toolFingerprint: resolvedTool.reviewedTool.fingerprint, + contextDigest: canonicalDigest(call.requestContext), + journalRevision: call.version, + expiresAt: approval.expiresAt.toISOString(), + policyRevision: policy.acceptedPolicyRef.policyRevision.toString(), + bindingEpoch: policy.acceptedPolicyRef.bindingEpoch.toString(), + }; +} diff --git a/orchestrator/src/mcp/journal.ts b/orchestrator/src/mcp/journal.ts new file mode 100644 index 000000000..5e66006bb --- /dev/null +++ b/orchestrator/src/mcp/journal.ts @@ -0,0 +1,1126 @@ +import { lockRuntimeAuthority } from "./runtime-authority.ts"; +import { makeMcpAdmission, lockAdmissionScope as lockScope } from "./admission.ts"; +import { and, asc, eq, gt, inArray, isNull, lte, or, sql } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { + integrationConnection as connection, + mcpCatalog as catalog, + mcpExecution as execution, + mcpApproval as approval, + mcpResolution as resolution, + mcpCancellation as cancellation, + mcpAdmission as admission, + taskSession, +} from "../db/schema.ts"; +import { + assertOriginalArguments, + freezeRequestContext, + lockHumanSession, + lockSessionChain, + assertLocalReviewCurrent, + reviewContext, + requestPolicyContext, + type JournalReviewContext, +} from "./journal-review.ts"; +import type { RequestPolicyContext } from "./session-policy.ts"; +import type { ResolvedBoundTool } from "./session-resolver.ts"; +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import { MCP_LIMITS } from "./limits.ts"; +import { validateMcpGrant, type McpToolGrant, type McpExecutionState } from "./types.ts"; + +type Db = ReturnType; +type Tx = Parameters[0]>[0]; +export type McpCallAuthorization = + | { kind: "read" } + | { kind: "approval" } + | { kind: "automation"; grant: McpToolGrant }; +export type McpCallInput = Pick< + typeof execution.$inferInsert, + | "taskId" + | "sessionId" + | "runId" + | "toolCallId" + | "connectionId" + | "catalogId" + | "principalId" + | "bindingHash" + | "argumentHash" + | "upstreamName" + | "configRevision" + | "authGeneration" + | "classification" + | "deadline" +> & { + originalArguments: string; + resolvedTool: ResolvedBoundTool; + requestPolicyContext: RequestPolicyContext; +}; + +function unknownBlocks(now: Date) { + return or( + isNull(resolution.executionId), + isNull(execution.intentId), + isNull(execution.intentExpiresAt), + gt(execution.intentExpiresAt, now), + ); +} +async function unknown( + tx: Tx | Db, + sessionIds: string[], + connectionId: string, + now: Date, +): Promise { + const rows = await tx + .select({ id: execution.id }) + .from(execution) + .leftJoin(resolution, eq(resolution.executionId, execution.id)) + .where( + and( + inArray(execution.sessionId, sessionIds), + eq(execution.connectionId, connectionId), + eq(execution.classification, "write"), + eq(execution.state, "outcome_unknown"), + unknownBlocks(now), + ), + ) + .limit(1); + return rows.length > 0; +} + +export class McpUnknownWrite extends Error { + constructor() { + super("unknown write outcome blocks this connection"); + } +} + +/** Only the chain revalidated under the runtime locks can extend the write restriction. */ +async function assertNoUnknownWrite( + tx: Tx, + call: Pick & { connectionId: string }, + authority: Awaited>, + now: Date, +) { + if ( + call.classification === "write" && + (await unknown( + tx, + [authority.session.sessionId, ...authority.ancestors.map(({ session }) => session.sessionId)], + call.connectionId, + now, + )) + ) + throw new McpUnknownWrite(); +} + +/** Internal runtime proof; no public or guest route accepts this value. The runtime obtains it + * from its fresh principal, full binding/ancestry, schema, policy and kill checks. */ +export interface McpLiveDispatchAuthorization { + executionId: string; + requestDigest: string; + bindingHash: string; + principalId: string; + expectedJournalRevision: number; + reservationId: string; + reservationExpiresAt: Date; +} +export type McpUnknownContext = { + executionId: string; + journalRevision: number; + requestDigest: string; + bindingHash: string; + intentId: string; +}; +export function unknownContext(call: typeof execution.$inferSelect): McpUnknownContext { + if (call.state !== "outcome_unknown" || !call.intentId) + throw Error("call has no unknown outcome"); + return { + executionId: call.id, + journalRevision: call.version, + requestDigest: call.requestDigest, + bindingHash: call.bindingHash, + intentId: call.intentId, + }; +} +export type McpInternalCancellationCause = + | "interrupt" + | "end" + | "revoke" + | "kill" + | "deadline" + | "resume" + | "authority_changed"; +export function makeMcpJournal(db = getDb()) { + const phasedAdmission = makeMcpAdmission(db); + async function cancelExecution( + id: string, + authority: + | { kind: "human"; actorId: string; expectedJournalRevision: number } + | { kind: "internal"; cause: McpInternalCancellationCause }, + now: Date, + ) { + return db.transaction(async (tx) => { + const [identity] = await tx.select().from(execution).where(eq(execution.id, id)); + if (!identity) throw Error("call not found"); + if (authority.kind === "human") + await lockHumanSession(tx, identity.sessionId, authority.actorId); + await tx + .select() + .from(connection) + .where(eq(connection.id, identity.connectionId)) + .for("update"); + const [call] = await tx.select().from(execution).where(eq(execution.id, id)).for("update"); + if (!call) throw Error("call not found"); + const [prior] = await tx.select().from(cancellation).where(eq(cancellation.executionId, id)); + if (authority.kind === "human") { + if ( + prior?.actorId === authority.actorId && + prior.expectedJournalRevision === authority.expectedJournalRevision + ) + return call; + if ( + !Number.isSafeInteger(authority.expectedJournalRevision) || + call.version !== authority.expectedJournalRevision + ) + throw Error("stale cancellation context"); + } + if (!["waiting_approval", "authorized", "dispatching"].includes(call.state)) { + if (authority.kind === "human") throw Error("stale cancellation context"); + return call; + } + const state = call.state === "dispatching" ? "outcome_unknown" : "canceled"; + const [updated] = await tx + .update(execution) + .set({ state, version: call.version + 1, updatedAt: now }) + .where(eq(execution.id, id)) + .returning(); + await tx.insert(cancellation).values({ + executionId: id, + actorId: authority.kind === "human" ? authority.actorId : null, + cause: authority.kind === "human" ? "human" : authority.cause, + expectedJournalRevision: call.version, + journalRevision: call.version + 1, + state, + createdAt: now, + }); + await tx.delete(admission).where(eq(admission.executionId, id)); + await tx + .update(approval) + .set({ + decision: "denied", + actorId: authority.kind === "human" ? authority.actorId : null, + decidedAt: now, + }) + .where(and(eq(approval.executionId, id), eq(approval.decision, "pending"))); + return updated!; + }); + } + + return { + admitSchema: (id: string, deploymentId: string, now: Date) => + phasedAdmission.reserve(id, deploymentId, "schema", now), + finishSchema: phasedAdmission.finishSchema, + async get(id: string) { + return (await db.select().from(execution).where(eq(execution.id, id)))[0] ?? null; + }, + async getAdmission(id: string) { + return (await db.select().from(admission).where(eq(admission.executionId, id)))[0] ?? null; + }, + hasUnknown(sessionId: string, connectionId: string, now: Date = new Date()) { + return unknown(db, [sessionId], connectionId, now); + }, + async assertCurrent(call: typeof execution.$inferSelect, now: Date) { + return db.transaction(async (tx) => { + const authority = await lockRuntimeAuthority(tx, call, "share"); + await assertNoUnknownWrite(tx, call, authority, now); + }); + }, + async create(input: McpCallInput, authorization: McpCallAuthorization, now: Date) { + input = structuredClone(input); + authorization = structuredClone(authorization); + assertOriginalArguments(input.originalArguments, input.argumentHash); + const context = freezeRequestContext(input.resolvedTool, input.requestPolicyContext); + const { resolvedTool, requestPolicyContext: policy, ...storedInput } = input; + const bound = resolvedTool.binding; + if ( + policy.sessionId !== input.sessionId || + policy.runId !== input.runId || + policy.toolCallId !== input.toolCallId || + policy.acceptedPolicyRef.sessionId !== input.sessionId || + policy.acceptedPolicyRef.bindingEpoch < 0n || + policy.acceptedPolicyRef.bindingEpoch > BigInt(Number.MAX_SAFE_INTEGER) || + policy.ancestorPolicyRefs.length !== resolvedTool.ancestorAuthorityRefs.length || + policy.ancestorPolicyRefs.some( + (ref, index) => + ref.sessionId !== resolvedTool.ancestorAuthorityRefs[index]?.sessionId || + ref.bindingEpoch !== BigInt(resolvedTool.ancestorAuthorityRefs[index]!.bindingEpoch), + ) || + resolvedTool.bindingHash !== input.bindingHash || + bound.connectionId !== input.connectionId || + bound.catalogId !== input.catalogId || + bound.configRevision !== input.configRevision || + bound.authGeneration !== input.authGeneration || + resolvedTool.reviewedTool.upstreamName !== input.upstreamName || + resolvedTool.reviewedTool.classification !== input.classification || + !bound.tools.some( + (tool) => + tool.name === resolvedTool.modelToolName && + canonicalDigest(tool.tool) === canonicalDigest(resolvedTool.reviewedTool) && + canonicalDigest(tool.grant) === canonicalDigest(resolvedTool.effectiveGrant), + ) + ) + throw new Error("request review identity mismatch"); + if ( + ![input.argumentHash, input.bindingHash].every((hash) => /^sha256:[0-9a-f]{64}$/.test(hash)) + ) + throw new Error("canonical binding and argument hashes are required"); + if ( + input.deadline <= now || + input.deadline.getTime() - now.getTime() > + MCP_LIMITS.approvalMs + MCP_LIMITS.queueMs + MCP_LIMITS.maxToolMs || + !input.runId || + !input.toolCallId || + !input.principalId + ) + throw new Error("invalid call identity or deadline"); + if (authorization.kind === "read" && input.classification !== "read") + throw new Error("write requires approval or an explicit automation grant"); + if (authorization.kind === "automation") { + validateMcpGrant({ + connectionId: input.connectionId, + operation: "invoke", + resourceConstraints: [], + credentialScope: "org", + mcp: authorization.grant, + }); + if ( + input.classification !== "write" || + !authorization.grant.automationWrite || + authorization.grant.upstreamName !== input.upstreamName || + canonicalDigest(authorization.grant) !== canonicalDigest(resolvedTool.effectiveGrant) + ) + throw new Error("explicit automation write grant is required"); + } + const state = authorization.kind === "approval" ? "waiting_approval" : "authorized"; + const authorizationKind = authorization.kind; + const automationGrantHash = + authorization.kind === "automation" ? canonicalDigest(authorization.grant) : null; + const requestDigest = canonicalDigest({ + ...storedInput, + requestContext: context, + deadline: input.deadline.toISOString(), + authorizationKind, + automationGrantHash, + }); + return db.transaction(async (tx) => { + async function existing() { + const [row] = await tx + .select() + .from(execution) + .where( + and( + eq(execution.sessionId, input.sessionId), + eq(execution.toolCallId, input.toolCallId), + ), + ); + if (row && row.requestDigest !== requestDigest) throw new Error("call digest mismatch"); + return row; + } + // Duplicates retain the original request and outcome even after authority changes. + const prior = await existing(); + if (prior) return prior; + const authority = await lockSessionChain(tx, input.sessionId); + if (authority.session.taskId !== input.taskId) + throw new Error("stored binding unavailable"); + await assertLocalReviewCurrent( + tx, + { ...storedInput, requestContext: context }, + authority.session, + authority.ancestors, + "update", + ); + // A concurrent acceptance can commit while this request waits for the connection. + const winner = await existing(); + if (winner) return winner; + await assertNoUnknownWrite(tx, input, authority, now); + await tx + .insert(execution) + .values({ + ...storedInput, + requestContext: context, + id: crypto.randomUUID(), + state, + authorizationKind, + automationGrantHash, + requestDigest, + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing({ target: [execution.sessionId, execution.toolCallId] }); + const [row] = await tx + .select() + .from(execution) + .where( + and( + eq(execution.sessionId, input.sessionId), + eq(execution.toolCallId, input.toolCallId), + ), + ); + if (!row || row.requestDigest !== requestDigest) throw new Error("call digest mismatch"); + return row; + }); + }, + async requestApproval(id: string, deploymentId: string, now: Date) { + return db.transaction(async (tx) => { + await lockScope(tx, deploymentId); + const [call] = await tx.select().from(execution).where(eq(execution.id, id)).for("update"); + if (!call || call.state !== "waiting_approval" || call.deadline <= now) + throw new Error("call is not pending approval"); + const [schema] = await tx.select().from(admission).where(eq(admission.executionId, id)); + if (!schema || schema.schemaDigest !== call.requestDigest || schema.state !== "released") + throw new Error("verified released schema is required"); + const [existing] = await tx.select().from(approval).where(eq(approval.executionId, id)); + if (existing) return existing; + const pending = await tx + .select() + .from(approval) + .where( + and( + or(eq(approval.deploymentId, deploymentId), eq(approval.sessionId, call.sessionId)), + eq(approval.decision, "pending"), + gt(approval.expiresAt, now), + ), + ); + if ( + pending.filter((a) => a.deploymentId === deploymentId).length >= + MCP_LIMITS.deploymentApprovals || + pending.filter((a) => a.sessionId === call.sessionId).length >= + MCP_LIMITS.sessionApprovals + ) + throw new Error("approval capacity reached"); + const [row] = await tx + .insert(approval) + .values({ + executionId: id, + deploymentId, + sessionId: call.sessionId, + requestDigest: call.requestDigest, + decision: "pending", + expiresAt: new Date( + Math.min(call.deadline.getTime(), now.getTime() + MCP_LIMITS.approvalMs), + ), + }) + .returning(); + return row!; + }); + }, + async pending(sessionId: string, actorId: string, now: Date) { + return db.transaction(async (tx) => { + await lockHumanSession(tx, sessionId, actorId); + const rows = await tx + .select({ call: execution, approval }) + .from(execution) + .innerJoin(approval, eq(approval.executionId, execution.id)) + .where( + and( + eq(execution.sessionId, sessionId), + eq(approval.decision, "pending"), + gt(approval.expiresAt, now), + ), + ) + .orderBy(asc(execution.id)) + .limit(MCP_LIMITS.sessionApprovals); + return rows.filter((row) => row.call.requestContext && row.call.originalArguments !== null); + }); + }, + async readExecutionResult(sessionId: string, executionId: string, actorId: string) { + return db.transaction(async (tx) => { + await lockHumanSession(tx, sessionId, actorId); + const [call] = await tx + .select() + .from(execution) + .where(and(eq(execution.sessionId, sessionId), eq(execution.id, executionId))); + if (!call) throw new Error("call not found"); + return call; + }); + }, + async readSession( + sessionId: string, + actorId: string, + page: { after?: string; limit?: number } = {}, + now: Date = new Date(), + ) { + const limit = page.limit ?? 16; + if ( + !Number.isSafeInteger(limit) || + limit < 1 || + limit > 16 || + (page.after?.length ?? 0) > 128 + ) + throw new Error("invalid state page"); + return db.transaction(async (tx) => { + const { session } = await lockHumanSession(tx, sessionId, actorId); + const rows = await tx + .select({ call: execution, approval, resolution, cancellation }) + .from(execution) + .leftJoin(approval, eq(approval.executionId, execution.id)) + .leftJoin(resolution, eq(resolution.executionId, execution.id)) + .leftJoin(cancellation, eq(cancellation.executionId, execution.id)) + .where( + and( + eq(execution.sessionId, sessionId), + page.after ? gt(execution.id, page.after) : undefined, + ), + ) + .orderBy(asc(execution.id)) + .limit(limit + 1); + const blocks = await tx + .selectDistinct({ connectionId: execution.connectionId }) + .from(execution) + .leftJoin(resolution, eq(resolution.executionId, execution.id)) + .where( + and( + eq(execution.sessionId, sessionId), + eq(execution.classification, "write"), + eq(execution.state, "outcome_unknown"), + unknownBlocks(now), + ), + ) + .limit(17); + if (blocks.length > 16) throw new Error("unknown connection state exceeds limit"); + return { + session, + rows: rows.slice(0, limit), + hasMore: rows.length > limit, + unknownWriteBlocks: blocks.map((row) => row.connectionId), + }; + }); + }, + async decide( + id: string, + actorId: string, + approve: boolean, + expected: JournalReviewContext, + now: Date, + ) { + return db.transaction(async (tx) => { + const [identity] = await tx.select().from(execution).where(eq(execution.id, id)); + if (!identity) throw new Error("call not found"); + const { session, owner, ancestors } = await lockHumanSession( + tx, + identity.sessionId, + actorId, + true, + ); + // Lock profile before connection/execution; no network call is made in this transaction. + await assertLocalReviewCurrent(tx, identity, session, ancestors); + const [live] = await tx + .select() + .from(connection) + .where(eq(connection.id, identity.connectionId)) + .for("share"); + const [call] = await tx.select().from(execution).where(eq(execution.id, id)).for("update"); + const [item] = await tx + .select() + .from(approval) + .where(eq(approval.executionId, id)) + .for("update"); + if ( + !call || + call.requestDigest !== identity.requestDigest || + call.taskId !== owner.id || + call.state !== "waiting_approval" || + !item || + item.decision !== "pending" || + item.expiresAt <= now || + call.deadline <= now + ) + throw new Error("approval expired or already decided"); + if (!expected || canonicalDigest(expected) !== canonicalDigest(reviewContext(call, item))) + throw new Error("stale review context"); + if ( + approve && + call.classification === "write" && + requestPolicyContext(call.requestContext!).acceptedWriteEligibility !== "write_eligible" + ) + throw new Error("request policy denies writes"); + const frozen = call.requestContext!.resolvedTool; + const [review] = await tx.select().from(catalog).where(eq(catalog.id, call.catalogId)); + if ( + ["done", "failed", "cancelled"].includes(owner.status) || + !live?.enabled || + live.revoked || + live.alias !== frozen.binding.alias || + canonicalDigest(live.config) !== canonicalDigest(frozen.binding.config) || + live.configRevision !== call.configRevision || + live.authGeneration !== call.authGeneration || + live.credentialIdentity !== frozen.binding.credentialIdentity || + live.catalogRevision !== frozen.binding.catalogRevision || + !review || + review.kind !== "published" || + review.connectionId !== call.connectionId || + review.revision !== frozen.binding.catalogRevision || + !review.tools.some( + (tool) => canonicalDigest(tool) === canonicalDigest(frozen.reviewedTool), + ) + ) + throw new Error("stale MCP review authority"); + const [decision] = await tx + .update(approval) + .set({ decision: approve ? "approved" : "denied", actorId, decidedAt: now }) + .where(eq(approval.executionId, id)) + .returning(); + const [updated] = await tx + .update(execution) + .set({ + state: approve ? "authorized" : "canceled", + version: call.version + 1, + updatedAt: now, + }) + .where(eq(execution.id, id)) + .returning(); + return { call: updated!, approval: decision!, reviewedContext: expected }; + }); + }, + admit: (id: string, deploymentId: string, now: Date, toolMs = MCP_LIMITS.toolMs) => + phasedAdmission.reserve(id, deploymentId, "execute", now, toolMs), + /** Irreversible dispatch barrier. The upstream effect runs after this commits. */ + async dispatch( + id: string, + intent: { bootId: string; intentId: string; expiresAt: Date }, + now: Date, + liveAuthorization: McpLiveDispatchAuthorization, + ) { + liveAuthorization = structuredClone(liveAuthorization); + intent = structuredClone(intent); + const [reservation] = await db.select().from(admission).where(eq(admission.executionId, id)); + if (!reservation) throw new Error("valid admission is required"); + try { + return await db.transaction(async (tx) => { + await lockScope(tx, reservation.deploymentId); + const [identity] = await tx.select().from(execution).where(eq(execution.id, id)); + if (!identity) throw new Error("call not found"); + // Scope -> principal -> tasks/sessions -> profiles -> sorted connections -> call. + const authority = await lockRuntimeAuthority(tx, identity); + const [live] = await tx + .select() + .from(connection) + .where(eq(connection.id, identity.connectionId)) + .for("update"); + const [call] = await tx + .select() + .from(execution) + .where(eq(execution.id, id)) + .for("update"); + if (!call || call.state !== "authorized" || call.deadline <= now) + throw new Error("call cannot dispatch"); + if ( + !call.requestContext || + call.originalArguments === null || + !liveAuthorization || + liveAuthorization.executionId !== id || + liveAuthorization.requestDigest !== call.requestDigest || + liveAuthorization.bindingHash !== call.bindingHash || + liveAuthorization.principalId !== call.principalId || + liveAuthorization.expectedJournalRevision !== call.version + ) + throw new Error("stale dispatch authorization"); + const frozenPolicy = requestPolicyContext(call.requestContext); + if ( + call.classification === "write" && + frozenPolicy.acceptedWriteEligibility !== "write_eligible" + ) + throw new Error("request policy denies writes"); + const [review] = await tx.select().from(catalog).where(eq(catalog.id, call.catalogId)); + if ( + !live || + live.provider !== "mcp" || + !live.enabled || + live.revoked || + live.configRevision !== call.configRevision || + live.authGeneration !== call.authGeneration || + !review || + review.connectionId !== live.id || + review.kind !== "published" || + review.revision !== live.catalogRevision || + review.configRevision !== call.configRevision || + review.authGeneration !== call.authGeneration || + review.credentialIdentity !== live.credentialIdentity + ) + throw new Error("stale MCP authority"); + const reviewedTool = review.tools.find((t) => t.upstreamName === call.upstreamName); + if ( + !reviewedTool || + !("classification" in reviewedTool) || + reviewedTool.classification !== call.classification || + !("enabled" in reviewedTool) || + !reviewedTool.enabled + ) + throw new Error("tool is not approved for this operation"); + await assertNoUnknownWrite(tx, call, authority, now); + const [lease] = await tx + .select() + .from(admission) + .where( + and( + eq(admission.executionId, id), + eq(admission.state, "admitted"), + gt(admission.expiresAt, now), + ), + ); + if ( + !lease || + lease.phase !== "execute" || + lease.schemaDigest !== call.requestDigest || + lease.reservationId !== reservation.reservationId || + lease.reservationId !== liveAuthorization.reservationId || + lease.expiresAt.getTime() !== liveAuthorization.reservationExpiresAt.getTime() || + !intent.bootId || + !intent.intentId || + intent.expiresAt <= now || + intent.expiresAt > lease.expiresAt + ) + throw new Error("valid admission and intent are required"); + const [decision] = await tx.select().from(approval).where(eq(approval.executionId, id)); + if (call.authorizationKind === "approval" && !decision) + throw new Error("approval is required"); + if (decision) { + const consumed = await tx + .update(approval) + .set({ consumedAt: now }) + .where( + and( + eq(approval.executionId, id), + eq(approval.requestDigest, call.requestDigest), + eq(approval.decision, "approved"), + isNull(approval.consumedAt), + gt(approval.expiresAt, now), + ), + ) + .returning(); + if (!consumed.length) throw new Error("approval expired or consumed"); + } + const [updated] = await tx + .update(execution) + .set({ + state: "dispatching", + version: call.version + 1, + bootId: intent.bootId, + intentId: intent.intentId, + intentExpiresAt: intent.expiresAt, + updatedAt: now, + }) + .where(eq(execution.id, id)) + .returning(); + return updated!; + }); + } catch (error) { + // Release only a pre-barrier reservation. A concurrent committed dispatch keeps its lease. + await db.transaction(async (tx) => { + await lockScope(tx, reservation.deploymentId); + const [call] = await tx + .select() + .from(execution) + .where(eq(execution.id, id)) + .for("update"); + if ( + call?.state === "authorized" && + reservation.reservationId === liveAuthorization?.reservationId && + reservation.expiresAt.getTime() === liveAuthorization.reservationExpiresAt.getTime() + ) + await tx + .update(admission) + .set({ state: "released" }) + .where( + and( + eq(admission.executionId, id), + eq(admission.reservationId, reservation.reservationId), + eq(admission.expiresAt, reservation.expiresAt), + ), + ); + }); + throw error; + } + }, + /** Both credential-broker checks use the same committed generation and barrier revision. */ + async assertDispatchCurrent(id: string, proof: McpLiveDispatchAuthorization, now: Date) { + const [observed] = await db.select().from(admission).where(eq(admission.executionId, id)); + if (!observed) throw new Error("dispatch admission unavailable"); + return db.transaction(async (tx) => { + await lockScope(tx, observed.deploymentId); + const [identity] = await tx.select().from(execution).where(eq(execution.id, id)); + if (!identity) throw new Error("call not found"); + const authority = await lockRuntimeAuthority(tx, identity, "share"); + const [call] = await tx.select().from(execution).where(eq(execution.id, id)).for("share"); + const [lease] = await tx + .select() + .from(admission) + .where(eq(admission.executionId, id)) + .for("share"); + if ( + !call || + call.state !== "dispatching" || + call.deadline <= now || + !call.intentExpiresAt || + call.intentExpiresAt <= now || + proof.executionId !== id || + proof.requestDigest !== call.requestDigest || + proof.bindingHash !== call.bindingHash || + proof.principalId !== call.principalId || + proof.expectedJournalRevision + 1 !== call.version || + !lease || + lease.phase !== "execute" || + lease.state !== "admitted" || + lease.schemaDigest !== call.requestDigest || + lease.reservationId !== proof.reservationId || + lease.expiresAt.getTime() !== proof.reservationExpiresAt.getTime() || + lease.expiresAt <= now + ) + throw new Error("stale dispatch admission"); + if (call.authorizationKind === "approval") { + const [decision] = await tx + .select() + .from(approval) + .where(eq(approval.executionId, id)) + .for("share"); + if ( + !decision || + decision.decision !== "approved" || + !decision.consumedAt || + decision.requestDigest !== call.requestDigest || + decision.expiresAt <= now + ) + throw new Error("dispatch approval unavailable"); + } + await assertNoUnknownWrite(tx, call, authority, now); + return call; + }); + }, + /** Trusted local validation failure, before any irreversible dispatch. */ + async rejectBeforeDispatch( + id: string, + expected: { + requestDigest: string; + version: number; + reservation: { reservationId: string; expiresAt: Date } | null; + }, + now: Date, + ) { + expected = structuredClone(expected); + const [observed] = await db.select().from(admission).where(eq(admission.executionId, id)); + return db.transaction(async (tx) => { + if (observed) await lockScope(tx, observed.deploymentId); + const [identity] = await tx.select().from(execution).where(eq(execution.id, id)); + if (!identity) throw new Error("call not found"); + await tx + .select() + .from(connection) + .where(eq(connection.id, identity.connectionId)) + .for("update"); + const [call] = await tx.select().from(execution).where(eq(execution.id, id)).for("update"); + const [lease] = await tx + .select() + .from(admission) + .where(eq(admission.executionId, id)) + .for("update"); + if ( + !call || + !["waiting_approval", "authorized"].includes(call.state) || + call.version !== expected.version || + call.requestDigest !== expected.requestDigest || + call.intentId || + (expected.reservation + ? !observed || + !lease || + lease.deploymentId !== observed.deploymentId || + lease.reservationId !== expected.reservation.reservationId || + lease.expiresAt.getTime() !== expected.reservation.expiresAt.getTime() + : Boolean(lease)) + ) + throw new Error("stale pre-dispatch rejection"); + const [updated] = await tx + .update(execution) + .set({ + state: "failed", + version: call.version + 1, + result: { + source: "local_validation", + delivery: "available", + output: { + content: [ + { type: "text", text: "Tool arguments do not satisfy the approved schemas." }, + ], + isError: true, + }, + }, + resultUnavailable: false, + updatedAt: now, + }) + .where(eq(execution.id, id)) + .returning(); + await tx.delete(admission).where(eq(admission.executionId, id)); + await tx + .update(approval) + .set({ decision: "denied", decidedAt: now }) + .where( + and( + eq(approval.executionId, id), + isNull(approval.consumedAt), + inArray(approval.decision, ["pending", "approved"]), + ), + ); + return updated!; + }); + }, + async finish( + id: string, + version: number, + state: "succeeded" | "failed" | "outcome_unknown", + result: Record | null, + resultUnavailable: boolean, + now: Date, + ) { + if (result && Buffer.byteLength(canonicalJson(result)) > MCP_LIMITS.resultBytes) + throw new Error("result exceeds limit"); + if (state === "succeeded" && result === null && !resultUnavailable) + throw new Error("success needs a verified result or an explicit unavailable result"); + if (state !== "succeeded" && resultUnavailable) + throw new Error("unavailable result requires known success"); + return db.transaction(async (tx) => { + const [identity] = await tx + .select({ connectionId: execution.connectionId }) + .from(execution) + .where(eq(execution.id, id)); + if (!identity) throw new Error("call not found"); + await tx + .select() + .from(connection) + .where(eq(connection.id, identity.connectionId)) + .for("update"); + const [row] = await tx + .update(execution) + .set({ state, result, resultUnavailable, version: version + 1, updatedAt: now }) + .where( + and( + eq(execution.id, id), + eq(execution.version, version), + state === "outcome_unknown" + ? eq(execution.state, "dispatching") + : inArray(execution.state, ["dispatching", "outcome_unknown"]), + ), + ) + .returning(); + if (!row) throw new Error("stale execution transition"); + await tx.delete(admission).where(eq(admission.executionId, id)); + return row; + }); + }, + cancel(id: string, actorId: string, now: Date, expectedJournalRevision: number) { + return cancelExecution(id, { kind: "human", actorId, expectedJournalRevision }, now); + }, + /** Trusted server lifecycle only; never supplied through a public request. */ + cancelInternal(id: string, cause: McpInternalCancellationCause, now: Date) { + return cancelExecution(id, { kind: "internal", cause }, now); + }, + async cancellationReceipt(id: string) { + return ( + (await db.select().from(cancellation).where(eq(cancellation.executionId, id)))[0] ?? null + ); + }, + async resolveUnknown( + id: string, + actorId: string, + outcome: "succeeded" | "failed" | "acknowledged", + now: Date, + expected: McpUnknownContext, + ) { + return db.transaction(async (tx) => { + const [identity] = await tx.select().from(execution).where(eq(execution.id, id)); + if (!identity) throw new Error("call not found"); + await lockHumanSession(tx, identity.sessionId, actorId); + await tx + .select() + .from(connection) + .where(eq(connection.id, identity.connectionId)) + .for("update"); + const [call] = await tx.select().from(execution).where(eq(execution.id, id)).for("update"); + const [prior] = await tx.select().from(resolution).where(eq(resolution.executionId, id)); + if ( + prior && + prior.actorId === actorId && + prior.outcome === outcome && + canonicalDigest(prior.expectedContext) === canonicalDigest(expected) + ) + return prior; + if ( + !call || + call.state !== "outcome_unknown" || + !expected || + canonicalDigest(expected) !== canonicalDigest(unknownContext(call)) + ) + throw new Error("stale unknown outcome context"); + + const [row] = await tx + .insert(resolution) + .values({ executionId: id, actorId, outcome, expectedContext: expected, createdAt: now }) + .onConflictDoNothing() + .returning(); + if (!row) throw new Error("outcome already resolved"); + // Human resolution unblocks the connection but never invents an upstream result. + return row; + }); + }, + /** Recover a crash between call insertion and its approval/admission transaction. */ + async expireUnadmitted(now: Date, limit = MCP_LIMITS.queueSize) { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > MCP_LIMITS.queueSize) + throw new Error("invalid expiry batch size"); + const candidates = await db + .select({ id: execution.id }) + .from(execution) + .leftJoin(admission, eq(admission.executionId, execution.id)) + .where( + and( + inArray(execution.state, ["waiting_approval", "authorized"]), + lte(execution.deadline, now), + sql`(${admission.executionId} IS NULL OR ${admission.state} = 'released')`, + ), + ) + .orderBy(asc(execution.deadline)) + .limit(limit); + let expired = 0; + // One call lock per transaction avoids lock-order cycles with the lease sweeper. + for (const candidate of candidates) { + const changed = await db.transaction(async (tx) => { + const [call] = await tx + .select() + .from(execution) + .where(eq(execution.id, candidate.id)) + .for("update"); + if ( + !call || + !["waiting_approval", "authorized"].includes(call.state) || + call.deadline > now + ) + return false; + // Re-read after the row lock. Admission can have committed while this step waited. + const [lease] = await tx + .select() + .from(admission) + .where(eq(admission.executionId, call.id)); + if (lease && lease.state !== "released") return false; + await tx + .update(execution) + .set({ state: "canceled", version: call.version + 1, updatedAt: now }) + .where(eq(execution.id, call.id)); + await tx + .update(approval) + .set({ decision: "expired", decidedAt: now }) + .where( + and( + eq(approval.executionId, call.id), + inArray(approval.decision, ["pending", "approved"]), + isNull(approval.consumedAt), + ), + ); + return true; + }); + if (changed) expired++; + } + return expired; + }, + /** Stored scopes survive worker removal and disabled dispatch. Scan only due work. */ + async expiryScopes(now: Date, after = "") { + const rows = await db.execute<{ deployment_id: string }>(sql` + SELECT deployment_id FROM ( + SELECT ${admission.deploymentId} AS deployment_id FROM ${admission} + WHERE ${admission.state} IN ('queued', 'admitted') AND ${admission.expiresAt} <= ${now} + UNION + SELECT ${approval.deploymentId} AS deployment_id FROM ${approval} + WHERE ${approval.decision} IN ('pending', 'approved') + AND ${approval.consumedAt} IS NULL AND ${approval.expiresAt} <= ${now} + ) AS due WHERE deployment_id > ${after} ORDER BY deployment_id LIMIT 32 + `); + return rows.rows.map((row) => row.deployment_id); + }, + async expire(deploymentId: string, now: Date) { + // Lock order is scope -> connection -> execution, shared with admission/dispatch. + return db.transaction(async (tx) => { + await lockScope(tx, deploymentId); + const expired = await tx + .select() + .from(admission) + .where( + and( + eq(admission.deploymentId, deploymentId), + inArray(admission.state, ["queued", "admitted"]), + lte(admission.expiresAt, now), + ), + ); + for (const lease of expired.sort((a, b) => a.connectionId.localeCompare(b.connectionId))) { + await tx + .select() + .from(connection) + .where(eq(connection.id, lease.connectionId)) + .for("update"); + const [call] = await tx + .select() + .from(execution) + .where(eq(execution.id, lease.executionId)) + .for("update"); + if ( + lease.phase === "schema" && + lease.schemaQueueExpiresAt && + lease.schemaQueueExpiresAt > now + ) { + await tx + .update(admission) + .set({ state: "released" }) + .where(eq(admission.executionId, lease.executionId)); + continue; + } + if (call && ["waiting_approval", "authorized", "dispatching"].includes(call.state)) + await tx + .update(execution) + .set({ + state: call.state === "dispatching" ? "outcome_unknown" : "canceled", + version: call.version + 1, + updatedAt: now, + }) + .where(eq(execution.id, call.id)); + await tx + .update(admission) + .set({ state: "released" }) + .where(eq(admission.executionId, lease.executionId)); + } + const pending = await tx + .select() + .from(approval) + .where( + and( + eq(approval.deploymentId, deploymentId), + inArray(approval.decision, ["pending", "approved"]), + isNull(approval.consumedAt), + lte(approval.expiresAt, now), + ), + ); + for (const item of pending) { + await tx.select().from(execution).where(eq(execution.id, item.executionId)).for("update"); + await tx + .update(approval) + .set({ decision: "expired", decidedAt: now }) + .where(and(eq(approval.executionId, item.executionId), isNull(approval.consumedAt))); + await tx + .update(execution) + .set({ state: "canceled", version: sql`${execution.version} + 1`, updatedAt: now }) + .where( + and( + eq(execution.id, item.executionId), + inArray(execution.state, ["waiting_approval", "authorized"]), + ), + ); + } + return expired.length + pending.length; + }); + }, + }; +} +export type McpJournal = ReturnType; diff --git a/orchestrator/src/mcp/lifecycle-events.ts b/orchestrator/src/mcp/lifecycle-events.ts new file mode 100644 index 000000000..462adf060 --- /dev/null +++ b/orchestrator/src/mcp/lifecycle-events.ts @@ -0,0 +1,60 @@ +import { and, eq } from "drizzle-orm"; +import type { getDb } from "../db/client.ts"; +import { consumerCursor } from "../db/schema.ts"; +import { parseStatusChangeTo } from "../control-plane/session-events.ts"; + +export type LifecycleEvent = { idx: bigint; kind: string; payloadJson: string }; +const consumer = "mcp-lifecycle"; + +/** One ordered page. Current-authority reconciliation commits before the cursor moves to the next event. */ +export function makeMcpLifecycleEvents(deps: { + db: ReturnType; + now(): Date; + list(sessionId: string, after: bigint): Promise; + reconcile(sessionId: string, event: LifecycleEvent): Promise; +}) { + return { + async runOnce(sessionId: string) { + await deps.db + .insert(consumerCursor) + .values({ sessionId, consumer, lastIdx: -1n, updatedAt: deps.now() }) + .onConflictDoNothing(); + const [cursor] = await deps.db + .select() + .from(consumerCursor) + .where(and(eq(consumerCursor.sessionId, sessionId), eq(consumerCursor.consumer, consumer))); + let previous = cursor!.lastIdx; + const events = await deps.list(sessionId, previous); + if (events.length > 200) throw Error("MCP lifecycle page too large"); + let last = previous; + for (const event of events) { + if (event.idx <= 0n || event.idx <= last) throw Error("invalid MCP lifecycle event order"); + last = event.idx; + } + for (const event of events) { + const status = + event.kind === "status_changed" ? parseStatusChangeTo(event.payloadJson) : undefined; + if (event.kind === "status_changed" && !status) throw Error("invalid MCP lifecycle status"); + if ( + event.kind === "run_interrupted" || + event.kind === "resumed" || + (status && ["completed", "failed", "dead"].includes(status)) + ) + await deps.reconcile(sessionId, event); + const moved = await deps.db + .update(consumerCursor) + .set({ lastIdx: event.idx, updatedAt: deps.now() }) + .where( + and( + eq(consumerCursor.sessionId, sessionId), + eq(consumerCursor.consumer, consumer), + eq(consumerCursor.lastIdx, previous), + ), + ) + .returning({ lastIdx: consumerCursor.lastIdx }); + if (!moved.length) return; + previous = event.idx; + } + }, + }; +} diff --git a/orchestrator/src/mcp/lifecycle-interceptor.ts b/orchestrator/src/mcp/lifecycle-interceptor.ts new file mode 100644 index 000000000..2e0ddd0fd --- /dev/null +++ b/orchestrator/src/mcp/lifecycle-interceptor.ts @@ -0,0 +1,25 @@ +import type { Interceptor } from "@connectrpc/connect"; +import { SessionService } from "../gen/engram/app/v1/session_pb.ts"; +import type { McpInternalCancellationCause } from "./journal.ts"; + +const causes: Readonly> = { + Interrupt: "interrupt", + DeleteSession: "end", + Resume: "resume", + EvictLocal: "kill", + EvictIdle: "kill", +}; + +/** Trusted session transport: commit cancellation before the coordinator changes lifecycle. */ +export function makeMcpLifecycleInterceptor( + invalidate: (id: string, cause: McpInternalCancellationCause) => Promise, +): Interceptor { + return (next) => async (req) => { + if (!req.stream && req.service.typeName === SessionService.typeName) { + const cause = causes[req.method.name]; + const id = (req.message as { sessionId?: unknown } | null)?.sessionId; + if (cause && typeof id === "string" && id) await invalidate(id, cause); + } + return next(req); + }; +} diff --git a/orchestrator/src/mcp/lifecycle-invalidation.ts b/orchestrator/src/mcp/lifecycle-invalidation.ts new file mode 100644 index 000000000..76e5626b5 --- /dev/null +++ b/orchestrator/src/mcp/lifecycle-invalidation.ts @@ -0,0 +1,37 @@ +import { and, asc, eq, gt, inArray } from "drizzle-orm"; +import type { getDb } from "../db/client.ts"; +import { mcpExecution } from "../db/schema.ts"; +import { makeMcpJournal, type McpInternalCancellationCause } from "./journal.ts"; +import { MCP_LIMITS } from "./limits.ts"; + +/** Internal causes carry no human actor. Each cancellation is durable before local abort. */ +export function makeMcpLifecycleInvalidator(deps: { + db: ReturnType; + abort(id: string): Promise; + now?: () => Date; +}) { + const journal = makeMcpJournal(deps.db); + return async (sessionId: string, cause: McpInternalCancellationCause) => { + let after = ""; + for (;;) { + const rows = await deps.db + .select({ id: mcpExecution.id }) + .from(mcpExecution) + .where( + and( + eq(mcpExecution.sessionId, sessionId), + gt(mcpExecution.id, after), + inArray(mcpExecution.state, ["waiting_approval", "authorized", "dispatching"]), + ), + ) + .orderBy(asc(mcpExecution.id)) + .limit(MCP_LIMITS.queueSize); + for (const row of rows) { + await journal.cancelInternal(row.id, cause, (deps.now ?? (() => new Date()))()); + await deps.abort(row.id); + after = row.id; + } + if (rows.length < MCP_LIMITS.queueSize) return; + } + }; +} diff --git a/orchestrator/src/mcp/lifecycle-reconciler.ts b/orchestrator/src/mcp/lifecycle-reconciler.ts new file mode 100644 index 000000000..862dbefd8 --- /dev/null +++ b/orchestrator/src/mcp/lifecycle-reconciler.ts @@ -0,0 +1,137 @@ +import { and, asc, desc, eq, gt, lte, sql } from "drizzle-orm"; +import type { getDb } from "../db/client.ts"; +import { mcpExecution } from "../db/schema.ts"; +import type { makeMcpEffectDriver } from "./effect-driver.ts"; +import { MCP_LIMITS } from "./limits.ts"; + +const active = sql`${mcpExecution.state} IN ('waiting_approval','authorized','dispatching')`; + +/** Independently revisits all active executions; failures do not starve later rows. */ +export function makeMcpLifecycleReconciler(deps: { + db: ReturnType; + driver: Pick, "reconcile" | "cancel">; + timeoutMs?: number; + onSession?(sessionId: string): Promise; +}) { + let after = ""; + const pending = new Map; controller: AbortController }>(); + async function check(id: string) { + let operation = pending.get(id); + if (!operation) { + const controller = new AbortController(); + const promise = deps.driver.reconcile(id, controller.signal); + operation = { promise, controller }; + pending.set(id, operation); + void promise.then( + () => pending.delete(id), + () => pending.delete(id), + ); + } + let timer: ReturnType | undefined; + try { + await Promise.race([ + operation.promise, + new Promise((_, reject) => { + timer = setTimeout(() => { + operation!.controller.abort(); + deps.driver.cancel(id); + reject(Error("MCP authority check timed out")); + }, deps.timeoutMs ?? 5_000); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + } + return { + async reconcileAuthorityChange() { + const [last] = await deps.db + .select({ id: mcpExecution.id }) + .from(mcpExecution) + .where(active) + .orderBy(desc(mcpExecution.id)) + .limit(1); + if (!last) return; + let cursor = ""; + let failed = false; + for (;;) { + const rows = await deps.db + .select({ id: mcpExecution.id }) + .from(mcpExecution) + .where(and(active, gt(mcpExecution.id, cursor), lte(mcpExecution.id, last.id))) + .orderBy(asc(mcpExecution.id)) + .limit(MCP_LIMITS.queueSize); + for (const row of rows) { + try { + await check(row.id); + } catch { + failed = true; + } + cursor = row.id; + } + if (rows.length < MCP_LIMITS.queueSize) break; + } + if (failed) throw Error("MCP authority reconciliation unavailable"); + }, + async runOnce() { + const rows = await deps.db + .select({ id: mcpExecution.id, sessionId: mcpExecution.sessionId }) + .from(mcpExecution) + .where(and(active, gt(mcpExecution.id, after))) + .orderBy(asc(mcpExecution.id)) + .limit(MCP_LIMITS.queueSize); + let failed = false; + const sessions = new Set(); + for (const row of rows) { + if (!sessions.has(row.sessionId)) { + sessions.add(row.sessionId); + try { + await deps.onSession?.(row.sessionId); + } catch { + failed = true; + } + } + try { + await check(row.id); + } catch { + failed = true; + } + after = row.id; + } + if (rows.length < MCP_LIMITS.queueSize) after = ""; + if (failed) throw Error("MCP lifecycle reconciliation unavailable"); + return rows.length === MCP_LIMITS.queueSize; + }, + /** A fixed upper key bounds one event's scan even as new calls arrive. */ + async reconcileSession(sessionId: string) { + const [last] = await deps.db + .select({ id: mcpExecution.id }) + .from(mcpExecution) + .where(and(eq(mcpExecution.sessionId, sessionId), active)) + .orderBy(desc(mcpExecution.id)) + .limit(1); + if (!last) return; + let cursor = ""; + for (;;) { + const rows = await deps.db + .select({ id: mcpExecution.id }) + .from(mcpExecution) + .where( + and( + eq(mcpExecution.sessionId, sessionId), + active, + gt(mcpExecution.id, cursor), + lte(mcpExecution.id, last.id), + ), + ) + .orderBy(asc(mcpExecution.id)) + .limit(MCP_LIMITS.queueSize); + for (const row of rows) { + await check(row.id); + cursor = row.id; + } + if (rows.length < MCP_LIMITS.queueSize) return; + } + }, + }; +} diff --git a/orchestrator/src/mcp/limits.ts b/orchestrator/src/mcp/limits.ts new file mode 100644 index 000000000..7617f0c3f --- /dev/null +++ b/orchestrator/src/mcp/limits.ts @@ -0,0 +1,22 @@ +/** Operator ceilings. Per-zone policy can reduce these values. */ +export const MCP_LIMITS = Object.freeze({ + connectionsPerSession: 16, + toolsPerSession: 128, + manifestBytes: 1024 * 1024, + argumentBytes: 256 * 1024, + resultBytes: 1024 * 1024, + responseBytes: 2 * 1024 * 1024, + connectMs: 5000, + toolMs: 60_000, + maxToolMs: 300_000, + sessionConcurrency: 4, + connectionConcurrency: 8, + deploymentConcurrency: 32, + queueSize: 128, + queueMs: 30_000, + sessionApprovals: 16, + deploymentApprovals: 128, + approvalMs: 600_000, + schemaMs: 1000, + schemaHeapMiB: 128, +}); diff --git a/orchestrator/src/mcp/managed-create-input.ts b/orchestrator/src/mcp/managed-create-input.ts new file mode 100644 index 000000000..5fd0bbb3d --- /dev/null +++ b/orchestrator/src/mcp/managed-create-input.ts @@ -0,0 +1,111 @@ +import { createHash } from "node:crypto"; +import { create, toBinary } from "@bufbuild/protobuf"; +import { z } from "zod"; +import { CreatePolicySessionRequestSchema } from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import type { SessionCreateInput } from "../rpc/task-create.ts"; +import type { ManagedCreationIdentity } from "./managed-lifecycle.ts"; + +export const bytesDigest = (bytes: Uint8Array) => + `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +export function managedCreateInputDigest(input: { + protocol: number; + nativeVersion: number; + taskId: string; + nativeDigest: string; + creationRequest: Uint8Array; +}) { + const hash = createHash("sha256").update("engram-managed-create-v1\0"); + for (const value of [input.protocol, input.nativeVersion]) { + const bytes = Buffer.alloc(4); + bytes.writeUInt32BE(value); + hash.update(bytes); + } + for (const bytes of [ + Buffer.from(input.taskId), + Buffer.from(input.nativeDigest), + input.creationRequest, + ]) { + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(bytes.length)); + hash.update(length).update(bytes); + } + return `sha256:${hash.digest("hex")}`; +} +const descriptor = z + .object({ + name: z + .string() + .min(1) + .max(256) + .refine((name) => !name.startsWith("mcp_")), + description: z.string(), + inputSchema: z.record(z.string(), z.unknown()), + execution: z.enum(["sync", "deferred"]), + nativeBindings: z.record(z.string(), z.string().nullable()).optional(), + resultFormat: z.literal("json").optional(), + }) + .strict(); +export function frozenNativeManifest(raw: string | undefined) { + const text = raw ?? "[]", + bytes = new TextEncoder().encode(text); + if (bytes.length > 1024 * 1024) throw Error("MCP native manifest exceeds limit"); + const descriptors = z.array(descriptor).max(128).parse(JSON.parse(text)); + if (new Set(descriptors.map((d) => d.name)).size !== descriptors.length) + throw Error("MCP duplicate native tool"); + return { bytes, digest: bytesDigest(bytes), text }; +} +/** Exact wire bytes are transient: they can contain native harness credentials. Persist + * only the returned identity and native digest, never the request body. */ +export function managedCreationInput( + input: SessionCreateInput, + source: { + taskId: string; + actorId: string; + inputId: string; + parentSessionId?: string; + }, +) { + if (!input.requestedSessionId || ![undefined, "", "default", "plan"].includes(input.harnessMode)) + throw Error("MCP creation context unavailable"); + const native = frozenNativeManifest(input.harnessEnv?.ENGRAM_TOOLS); + const { + mcpBindingEnvelope: _binding, + integrationConnections: _connections, + integrationGrants: _grants, + ...wire + } = input; + const creationRequest = toBinary( + CreatePolicySessionRequestSchema, + create(CreatePolicySessionRequestSchema, { + protocol: 1, + actorId: source.actorId, + inputId: source.inputId, + parentSessionId: source.parentSessionId, + create: { ...wire, harnessEnv: { ...wire.harnessEnv, ENGRAM_TOOLS: native.text } }, + }), + ); + if (creationRequest.length > 3 * 1024 * 1024) throw Error("MCP creation input exceeds limit"); + const request = { + protocol: 1, + nativeVersion: 1, + taskId: source.taskId, + nativeDigest: native.digest, + creationRequest, + }; + const inputDigest = managedCreateInputDigest(request); + const identity: ManagedCreationIdentity = { + sessionId: input.requestedSessionId, + taskId: source.taskId, + inputId: source.inputId, + actorId: source.actorId, + ...(source.parentSessionId ? { parentSessionId: source.parentSessionId } : {}), + writeEligibility: input.harnessMode === "plan" ? "read_only" : "write_eligible", + inputDigest, + }; + return { + request: { ...request, inputDigest }, + identity, + nativeVersion: 1 as const, + nativeDigest: native.digest, + }; +} diff --git a/orchestrator/src/mcp/managed-lifecycle-store.ts b/orchestrator/src/mcp/managed-lifecycle-store.ts new file mode 100644 index 000000000..5fe449c80 --- /dev/null +++ b/orchestrator/src/mcp/managed-lifecycle-store.ts @@ -0,0 +1,205 @@ +import { and, asc, eq, gt, isNotNull, isNull, sql } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { task, taskSession } from "../db/schema.ts"; +import { canonicalJson } from "./hash.ts"; +import { managedSourceSchema, rustCreationIdentity } from "./managed-state.ts"; +import { compileMcpManifest } from "./session-bindings.ts"; +import type { ManagedLifecycle } from "./managed-lifecycle.ts"; + +function checked(value: ManagedLifecycle): ManagedLifecycle { + if ( + value.version !== 1 || + value.nativeVersion !== 1 || + !/^sha256:[0-9a-f]{64}$/.test(value.nativeDigest) || + !/^sha256:[0-9a-f]{64}$/.test(value.identity.inputDigest) || + !/^sha256:[0-9a-f]{64}$/.test(value.bindingHash) || + !["reserving", "held", "preparing", "finalizing", "started", "recovering", "terminal"].includes( + value.phase, + ) || + Object.keys(value).some( + (key) => + ![ + "version", + "identity", + "nativeVersion", + "nativeDigest", + "phase", + "bindingHash", + "sourceKey", + "recovery", + "pending", + ].includes(key), + ) || + Object.keys(value.identity).some( + (key) => + ![ + "sessionId", + "taskId", + "inputId", + "actorId", + "parentSessionId", + "writeEligibility", + "inputDigest", + ].includes(key), + ) || + !value.identity.sessionId || + !value.identity.taskId || + !value.identity.actorId || + !value.identity.inputId || + !["read_only", "write_eligible"].includes(value.identity.writeEligibility) || + Buffer.byteLength(canonicalJson(value)) > 2 * 1024 * 1024 + ) + throw Error("MCP lifecycle identity unavailable"); + const identity = rustCreationIdentity(value.identity); + for (const key of [value.sourceKey, value.recovery?.sourceKey]) { + if (!key) continue; + managedSourceSchema.parse(key); + if ( + canonicalJson(key.creation) !== canonicalJson(identity) || + key.native_digest !== value.nativeDigest || + key.context.target.session_id !== value.identity.sessionId + ) + throw Error("MCP lifecycle source changed"); + } + if (value.pending) { + const pending = value.pending; + if ( + !value.sourceKey || + pending.snapshot.bindingHash !== value.bindingHash || + pending.snapshot.bindingEpoch !== value.sourceKey.context.target.binding_epoch || + !/^sha256:[0-9a-f]{64}$/.test(pending.combinedManifestDigest) || + !/^sha256:[0-9a-f]{64}$/.test(pending.preparedReceiptId) || + Object.keys(pending).some( + (k) => !["snapshot", "combinedManifestDigest", "preparedReceiptId"].includes(k), + ) + ) + throw Error("MCP lifecycle finalization changed"); + compileMcpManifest(pending.snapshot); + } + if (value.recovery) { + const recovery = value.recovery; + if ( + !/^[a-zA-Z0-9-]{1,128}$/.test(recovery.id) || + !recovery.actorId || + !/^sha256:[0-9a-f]{64}$/.test(recovery.expectedBindingHash) || + recovery.candidateHash !== recovery.candidate.bindingHash || + recovery.candidate.bindingEpoch !== recovery.sourceKey.context.target.binding_epoch + 1 || + Object.keys(recovery).some( + (k) => + ![ + "id", + "sourceKey", + "expectedBindingHash", + "candidateHash", + "actorId", + "candidate", + ].includes(k), + ) + ) + throw Error("MCP lifecycle recovery changed"); + compileMcpManifest(recovery.candidate); + } + return structuredClone(value); +} +/** Exact JSON and binding CAS; no transaction spans a coordinator or worker call. */ +export function makeManagedLifecycleStore(db = getDb()) { + async function read(sessionId: string) { + const rows = await db + .select({ session: taskSession, owner: task }) + .from(taskSession) + .innerJoin(task, eq(task.id, taskSession.taskId)) + .where(eq(taskSession.sessionId, sessionId)) + .limit(2); + if (rows.length !== 1) throw Error("MCP session reservation unavailable"); + const row = rows[0]!; + if (row.session.mcpManagedLifecycle) checked(row.session.mcpManagedLifecycle); + return row; + } + return { + read, + async initialize(value: ManagedLifecycle) { + value = checked(value); + if (value.phase !== "reserving" || value.sourceKey || value.recovery || value.pending) + throw Error("MCP initial lifecycle unavailable"); + const { session, owner } = await read(value.identity.sessionId); + if ( + session.taskId !== value.identity.taskId || + session.integrationPrincipalId !== value.identity.actorId || + owner.spawningSessionId !== (value.identity.parentSessionId ?? null) || + session.mcpBindingEnvelope?.bindingHash !== value.bindingHash || + session.mcpBindingEnvelope.bindingEpoch !== 0 + ) + throw Error("MCP reservation identity changed"); + if (session.mcpManagedLifecycle) { + if (canonicalJson(session.mcpManagedLifecycle) !== canonicalJson(value)) + throw Error("MCP creation already reserved"); + return; + } + const rows = await db + .update(taskSession) + .set({ mcpManagedLifecycle: value }) + .where( + and( + eq(taskSession.taskId, value.identity.taskId), + eq(taskSession.sessionId, value.identity.sessionId), + eq(taskSession.integrationPrincipalId, value.identity.actorId), + isNull(taskSession.mcpManagedLifecycle), + sql`${taskSession.mcpBindingEnvelope}->>'bindingHash' = ${value.bindingHash}`, + ), + ) + .returning({ id: taskSession.sessionId }); + if (rows.length !== 1) throw Error("MCP creation reservation changed"); + }, + async compareAndSet( + previous: ManagedLifecycle, + next: ManagedLifecycle, + columns?: Pick< + typeof taskSession.$inferInsert, + | "mcpBindingEnvelope" + | "integrationConnections" + | "integrationGrants" + | "integrationSnapshotHash" + >, + ) { + previous = checked(previous); + next = checked(next); + if ( + canonicalJson(previous.identity) !== canonicalJson(next.identity) || + previous.nativeDigest !== next.nativeDigest + ) + throw Error("MCP immutable creation changed"); + if ( + (!columns && previous.bindingHash !== next.bindingHash) || + (columns && columns.mcpBindingEnvelope?.bindingHash !== next.bindingHash) + ) + throw Error("MCP final binding changed"); + const rows = await db + .update(taskSession) + .set({ ...columns, mcpManagedLifecycle: next }) + .where( + and( + eq(taskSession.taskId, previous.identity.taskId), + eq(taskSession.sessionId, previous.identity.sessionId), + sql`${taskSession.mcpManagedLifecycle} = ${JSON.stringify(previous)}::jsonb`, + sql`${taskSession.mcpBindingEnvelope}->>'bindingHash' = ${previous.bindingHash}`, + ), + ) + .returning({ id: taskSession.sessionId }); + if (rows.length !== 1) throw Error("MCP lifecycle changed"); + }, + async pending(after?: string) { + return db + .select({ sessionId: taskSession.sessionId }) + .from(taskSession) + .where( + and( + isNotNull(taskSession.mcpManagedLifecycle), + sql`${taskSession.mcpManagedLifecycle}->>'phase' NOT IN ('started','terminal')`, + ...(after ? [gt(taskSession.sessionId, after)] : []), + ), + ) + .orderBy(asc(taskSession.sessionId)) + .limit(32); + }, + }; +} diff --git a/orchestrator/src/mcp/managed-lifecycle.ts b/orchestrator/src/mcp/managed-lifecycle.ts new file mode 100644 index 000000000..ded70accc --- /dev/null +++ b/orchestrator/src/mcp/managed-lifecycle.ts @@ -0,0 +1,56 @@ +import type { McpSessionBindingSnapshot } from "./session-bindings.ts"; + +/** Non-secret immutable creation association, copied from trusted creation input. */ +export interface ManagedCreationIdentity { + sessionId: string; + taskId: string; + inputId: string; + actorId: string; + parentSessionId?: string; + writeEligibility: string; + inputDigest: string; +} +export interface ManagedBootSourceKey { + creation: { + session_id: string; + task_id: string; + policy: { + actorId: string; + inputId: string; + parentSessionId: string | null; + eligibility: string; + }; + input_digest: string; + }; + context: { + target: { session_id: string; sandbox_id: string; host_id: string; binding_epoch: number }; + operator_generation: number; + operator_identity: string; + binding_hash: null; + }; + boot_attempt: string; + native_digest: string; +} +/** This is progress metadata, never host, capture, placement or effect authority. */ +export interface ManagedLifecycle { + version: 1; + identity: ManagedCreationIdentity; + nativeVersion: 1; + nativeDigest: string; + phase: "reserving" | "held" | "preparing" | "finalizing" | "started" | "recovering" | "terminal"; + bindingHash: string; + sourceKey?: ManagedBootSourceKey; + recovery?: { + id: string; + sourceKey: ManagedBootSourceKey; + expectedBindingHash: string; + candidateHash: string; + actorId: string; + candidate: McpSessionBindingSnapshot; + }; + pending?: { + snapshot: McpSessionBindingSnapshot; + combinedManifestDigest: string; + preparedReceiptId: string; + }; +} diff --git a/orchestrator/src/mcp/managed-policy.ts b/orchestrator/src/mcp/managed-policy.ts new file mode 100644 index 000000000..b7194bb21 --- /dev/null +++ b/orchestrator/src/mcp/managed-policy.ts @@ -0,0 +1,235 @@ +import { createHash } from "node:crypto"; +import { isIP } from "node:net"; +import { equals } from "@bufbuild/protobuf"; +import { + ManagedPolicyTargetSchema, + ManagedPolicyPhase, + type ManagedAppliedState, + type ManagedDenyPolicy as WirePolicy, + type ManagedPolicyTarget, +} from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import type { SessionPolicyControlClient } from "./control-client.ts"; +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import type { PublishedOperator } from "./operator.ts"; +import type { ManagedDenyPolicy } from "./session-bindings.ts"; +import type { ManagedDenyAck } from "./session-resolver.ts"; + +function number(value: bigint) { + if (value < 0n || value > BigInt(Number.MAX_SAFE_INTEGER)) + throw Error("MCP managed integer unavailable"); + return Number(value); +} +export function managedPolicy(value: WirePolicy): ManagedDenyPolicy { + const body = { version: 1 as const, revision: number(value.revision), hosts: [...value.hosts] }; + if ( + value.version !== 1 || + body.hosts.length > 4096 || + body.hosts.some( + (host, i) => + host.length > 253 || + !host.includes(".") || + isIP(host) || + !host.split(".").every((label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label)) || + (i > 0 && body.hosts[i - 1]! >= host), + ) || + canonicalDigest(body) !== value.digest + ) + throw Error("MCP managed policy unavailable"); + return { ...body, digest: value.digest }; +} +function targetJson(target: ManagedPolicyTarget) { + if (!target.sessionId || !target.sandboxId || !target.hostId || target.bindingEpoch <= 0n) + throw Error("MCP managed target unavailable"); + return { + session_id: target.sessionId, + sandbox_id: target.sandboxId, + host_id: target.hostId, + binding_epoch: number(target.bindingEpoch), + }; +} +/** Exact Rust field order for a verified receipt and finalize request. */ +export function managedAppliedJson(applied: ManagedAppliedState) { + const context = applied.context, + proof = applied.networkProof; + if ( + !context?.target || + !applied.policy || + !applied.hostIncarnation || + !applied.proxyIncarnation || + applied.hostIncarnation.length > 128 || + applied.proxyIncarnation.length > 128 || + context.operatorGeneration <= 0n || + !/^sha256:[0-9a-f]{64}$/.test(context.operatorIdentity) || + (context.phase === ManagedPolicyPhase.FINAL + ? !context.bindingHash || !proof + : context.phase !== ManagedPolicyPhase.BOOTSTRAP || context.bindingHash !== undefined) || + (context.bindingHash !== undefined && !/^sha256:[0-9a-f]{64}$/.test(context.bindingHash)) + ) + throw Error("MCP managed receipt unavailable"); + const target = targetJson(context.target); + if ( + proof && + (!proof.target || + !equals(ManagedPolicyTargetSchema, context.target, proof.target) || + isIP(proof.guestIp) !== 4 || + proof.guestIp === "0.0.0.0" || + !/^sha256:[0-9a-f]{64}$/.test(proof.networkIdentity)) + ) + throw Error("MCP network proof unavailable"); + const policy = managedPolicy(applied.policy); + const value = { + context: { + target, + operator_generation: number(context.operatorGeneration), + operator_identity: context.operatorIdentity, + binding_hash: context.bindingHash ?? null, + }, + policy: { + version: policy.version, + revision: policy.revision, + hosts: policy.hosts, + digest: policy.digest, + }, + host_incarnation: applied.hostIncarnation, + proxy_incarnation: applied.proxyIncarnation, + state_id: "", + network_proof: proof + ? { + target: targetJson(proof.target!), + guest_ip: proof.guestIp, + network_identity: proof.networkIdentity, + } + : null, + }; + const digest = `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex")}`; + if (applied.stateId !== digest) throw Error("MCP managed receipt digest changed"); + return { ...value, state_id: digest }; +} +export function unionManagedPolicies(policies: readonly ManagedDenyPolicy[]): ManagedDenyPolicy { + if (!policies.length) throw Error("MCP managed policy unavailable"); + for (const p of policies) + managedPolicy({ + $typeName: "engram.mcp.v1.ManagedDenyPolicy", + ...p, + revision: BigInt(p.revision), + }); + const revision = Math.max(...policies.map((p) => p.revision)) + 1; + if (!Number.isSafeInteger(revision)) throw Error("MCP managed revision exhausted"); + const body = { + version: 1 as const, + revision, + hosts: [...new Set(policies.flatMap((p) => p.hosts))].sort(), + }; + return { ...body, digest: canonicalDigest(body) }; +} +type ManagedPlacement = { + target: ManagedPolicyTarget; + networkIdentity: string; + guestIp: string; + hostIncarnation: string; +}; +function matchesPlacement(applied: ManagedAppliedState, placement: ManagedPlacement) { + const proof = applied.networkProof; + return ( + !!proof?.target && + equals(ManagedPolicyTargetSchema, proof.target, placement.target) && + proof.networkIdentity === placement.networkIdentity && + proof.guestIp === placement.guestIp && + applied.hostIncarnation === placement.hostIncarnation + ); +} +export function makeManagedPolicyAuthority(deps: { + control: Pick; + readPlacement(sessionId: string): Promise; + readPublished(): Promise; +}) { + async function read(target: ManagedPolicyTarget) { + targetJson(target); + const result = await deps.control.readManagedPolicy({ protocol: 1, target }); + if ( + result.protocol !== 1 || + !result.retainedDesired?.target || + !equals(ManagedPolicyTargetSchema, result.retainedDesired.target, target) + ) + throw Error("MCP retained policy presence unavailable"); + const retained = result.retainedDesired.policy + ? managedPolicy(result.retainedDesired.policy) + : undefined; + if (result.applied) { + managedAppliedJson(result.applied); + if ( + !result.applied.context?.target || + !equals(ManagedPolicyTargetSchema, result.applied.context.target, target) + ) + throw Error("MCP applied target changed"); + } + return { applied: result.applied, retained }; + } + return { + read, + async enforce(expected: ManagedDenyAck, policy: ManagedDenyPolicy): Promise { + const placement = await deps.readPlacement(expected.sessionId), + target = placement.target; + if ( + target.sessionId !== expected.sessionId || + number(target.bindingEpoch) !== expected.bindingEpoch + ) + throw Error("MCP managed binding changed"); + const [state, publication] = await Promise.all([read(target), deps.readPublished()]); + const applied = state.applied; + if ( + !applied?.context || + !applied.policy || + applied.context.phase !== ManagedPolicyPhase.FINAL || + applied.context.bindingHash !== expected.bindingHash || + applied.context.operatorGeneration !== BigInt(publication.generation) || + applied.context.operatorIdentity !== publication.identity || + !matchesPlacement(applied, placement) || + canonicalJson(managedPolicy(applied.policy)) !== canonicalJson(policy) || + !state.retained || + canonicalJson(state.retained) !== canonicalJson(policy) || + expected.denyDigest !== policy.digest || + expected.denyRevision !== policy.revision + ) + throw Error("MCP final managed acknowledgement unavailable"); + return { ...expected }; + }, + async apply( + target: ManagedPolicyTarget, + bindingHash: string, + policy: ManagedDenyPolicy, + previous: ManagedAppliedState | undefined, + publication: PublishedOperator, + ) { + const result = await deps.control.applyManagedPolicy({ + protocol: 1, + context: { + target, + operatorGeneration: BigInt(publication.generation), + operatorIdentity: publication.identity, + phase: ManagedPolicyPhase.FINAL, + bindingHash, + }, + policy: { ...policy, revision: BigInt(policy.revision) }, + expected: previous, + }); + if (result.protocol !== 1 || !result.applied) throw Error("MCP managed apply unavailable"); + managedAppliedJson(result.applied); + const context = result.applied.context!, + placement = await deps.readPlacement(target.sessionId); + if ( + !context.target || + !equals(ManagedPolicyTargetSchema, context.target, target) || + context.bindingHash !== bindingHash || + context.phase !== ManagedPolicyPhase.FINAL || + !equals(ManagedPolicyTargetSchema, placement.target, target) || + !matchesPlacement(result.applied, placement) || + context.operatorGeneration !== BigInt(publication.generation) || + context.operatorIdentity !== publication.identity || + canonicalJson(managedPolicy(result.applied.policy!)) !== canonicalJson(policy) + ) + throw Error("MCP managed apply changed"); + return result.applied; + }, + }; +} diff --git a/orchestrator/src/mcp/managed-rebind.ts b/orchestrator/src/mcp/managed-rebind.ts new file mode 100644 index 000000000..5abd16ed6 --- /dev/null +++ b/orchestrator/src/mcp/managed-rebind.ts @@ -0,0 +1,272 @@ +import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; +import { create } from "@bufbuild/protobuf"; +import { getDb } from "../db/client.ts"; +import { integrationConnection, mcpExecution, profile, taskSession } from "../db/schema.ts"; +import { ManagedPolicyTargetSchema } from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import { lockHumanSession } from "./journal-review.ts"; +import { readLiveBindingAuthority, restoreBindingSnapshot } from "./binding-store.ts"; +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import { + compileSessionBindings, + checkBindingCurrent, + hasPublishedLeaf, + type McpOperatorPolicy, + type McpSessionBindingSnapshot, +} from "./session-bindings.ts"; +import { makeManagedLifecycleStore } from "./managed-lifecycle-store.ts"; +import { makeManagedSessionRuntime } from "./managed-runtime.ts"; +import { + makeManagedPolicyAuthority, + managedPolicy, + unionManagedPolicies, +} from "./managed-policy.ts"; +import type { ManagedLifecycle } from "./managed-lifecycle.ts"; +import { McpPublicationRequired, McpRebindDraining, type McpRebindDrain } from "./rebind.ts"; +import { appendMcpManifest } from "../tools/manifest.ts"; + +/** Prepare outside locks; commit only a durable recovery intent under local authority locks. */ +export function makeManagedRebind(deps: { + db?: ReturnType; + store: ReturnType; + lifecycle: ReturnType; + managed: ReturnType; + readOperatorPolicies(): Promise; + readParentCurrent(id: string): Promise; + now(): Date; +}) { + const db = deps.db ?? getDb(); + async function drain(tx: Parameters[0]>[0], sessionId: string) { + const [pending] = await tx + .select() + .from(mcpExecution) + .where( + and( + eq(mcpExecution.sessionId, sessionId), + or( + eq(mcpExecution.state, "dispatching"), + and( + eq(mcpExecution.state, "outcome_unknown"), + or( + isNull(mcpExecution.intentId), + isNull(mcpExecution.intentExpiresAt), + gt(mcpExecution.intentExpiresAt, deps.now()), + ), + ), + ), + ), + ) + .limit(1); + return pending + ? ({ + reason: + !pending.intentId || !pending.intentExpiresAt + ? "legacy_intent" + : pending.state === "dispatching" + ? "dispatch_in_progress" + : "unknown_intent", + retryAfter: + pending.intentId && pending.intentExpiresAt + ? pending.intentExpiresAt.toISOString() + : "", + } satisfies McpRebindDrain) + : undefined; + } + async function prepare(sessionId: string, oldHash: string, actorId: string) { + const local = await db.transaction(async (tx) => ({ + ...(await lockHumanSession(tx, sessionId, actorId, true)), + drain: await drain(tx, sessionId), + })); + const old = + local.session.mcpBindingEnvelope && + restoreBindingSnapshot( + local.session.mcpBindingEnvelope, + local.session.integrationConnections ?? [], + ); + const previous = local.session.mcpManagedLifecycle; + if ( + !old || + old.bindingHash !== oldHash || + !previous?.sourceKey || + previous.phase !== "started" || + ["done", "failed", "cancelled"].includes(local.owner.status) + ) + throw Error("MCP rebind source unavailable"); + const state = await deps.lifecycle.read(previous); + const key = state.boot?.request.key; + if ( + !key || + state.state === "terminal" || + canonicalJson(key) !== canonicalJson(previous.sourceKey) + ) + throw Error("MCP rebind source changed"); + const policies = await deps.readOperatorPolicies(); + const live = await readLiveBindingAuthority(old.profileId, policies, db); + const parentAuthority = local.owner.spawningSessionId + ? { + sessionId: local.owner.spawningSessionId, + snapshot: await deps.readParentCurrent(local.owner.spawningSessionId), + } + : undefined; + const parentTools = parentAuthority?.snapshot.immutableBindings.flatMap((binding) => + binding.tools.map((tool) => ({ + connectionId: binding.connectionId, + upstreamName: tool.grant.upstreamName, + })), + ); + const requestedGrants = parentTools + ? live.profileGrants.filter( + (grant) => + grant.mcp && + parentTools.some( + (tool) => + tool.connectionId === grant.connectionId && + tool.upstreamName === grant.mcp!.upstreamName, + ), + ) + : live.profileGrants; + const next = compileSessionBindings({ + launchPolicy: { profileId: old.profileId, integrationGrants: live.profileGrants }, + liveAuthority: live, + bindingEpoch: key.context.target.binding_epoch + 1, + requestedGrants, + parentAuthority, + }); + if (!hasPublishedLeaf(next)) throw new McpPublicationRequired(); + const target = create(ManagedPolicyTargetSchema, { + sessionId, + sandboxId: key.context.target.sandbox_id, + hostId: key.context.target.host_id, + bindingEpoch: BigInt(key.context.target.binding_epoch), + }); + const deny = [ + old.managedDenyPolicy, + next.managedDenyPolicy, + state.boot!.request.bootstrap_policy, + ]; + if (state.boot!.retired) { + // Exact durable retirement is only a deny superset, never an Applied acknowledgement. + deny.push(state.boot!.retired.retained_policy); + } else { + const applied = await deps.managed.read(target); + if (!applied.retained) throw Error("MCP retained source policy unavailable"); + deny.push(applied.retained); + if (applied.applied?.policy) deny.push(managedPolicy(applied.applied.policy)); + } + const { bindingHash: _hash, ...body } = next; + const candidateBody = { ...body, managedDenyPolicy: unionManagedPolicies(deny) }; + const candidate = { ...candidateBody, bindingHash: canonicalDigest(candidateBody) }; + appendMcpManifest(await deps.lifecycle.native(previous), candidate); + return { candidate, previous, policies, drain: local.drain }; + } + return { + async prepare(sessionId: string, oldHash: string, actorId: string) { + const result = await prepare(sessionId, oldHash, actorId); + return { ...result.candidate, ...(result.drain ? { drain: result.drain } : {}) }; + }, + async commit(sessionId: string, oldHash: string, candidateHash: string, actorId: string) { + const current = await db.transaction( + async (tx) => + (await lockHumanSession(tx, sessionId, actorId, true)).session.mcpManagedLifecycle, + ); + if (!current) throw Error("MCP rebind source unavailable"); + let recoveryId = current.recovery?.id; + if ( + !( + current.recovery?.expectedBindingHash === oldHash && + current.recovery.candidateHash === candidateHash && + current.recovery.actorId === actorId + ) + ) { + const prepared = await prepare(sessionId, oldHash, actorId); + if (prepared.candidate.bindingHash !== candidateHash) + throw Error("stale MCP rebind candidate"); + if (prepared.drain) throw new McpRebindDraining(prepared.drain); + const { pending: _pending, ...prior } = prepared.previous; + recoveryId = crypto.randomUUID(); + const next: ManagedLifecycle = { + ...prior, + phase: "recovering", + recovery: { + id: recoveryId, + actorId, + sourceKey: prior.sourceKey!, + expectedBindingHash: oldHash, + candidateHash, + candidate: prepared.candidate, + }, + }; + await db.transaction(async (tx) => { + const local = await lockHumanSession(tx, sessionId, actorId, true, "update"); + if ( + canonicalJson(local.session.mcpManagedLifecycle) !== canonicalJson(prepared.previous) || + local.session.mcpBindingEnvelope?.bindingHash !== oldHash + ) + throw Error("MCP rebind reservation changed"); + const ids = [ + ...new Set([ + prepared.candidate.profileId, + ...local.ancestors.flatMap((a) => (a.session.profileId ? [a.session.profileId] : [])), + ]), + ].sort(); + await tx + .select() + .from(profile) + .where(inArray(profile.id, ids)) + .orderBy(asc(profile.id)) + .for("share"); + await tx + .select() + .from(integrationConnection) + .where(eq(integrationConnection.provider, "mcp")) + .orderBy(asc(integrationConnection.id)) + .for("share"); + const live = await readLiveBindingAuthority( + prepared.candidate.profileId, + prepared.policies, + tx, + ); + if ( + !checkBindingCurrent({ snapshot: prepared.candidate, liveAuthority: live }).current || + prepared.candidate.ancestorAuthorityRefs.some( + (ref, i) => + ref.sessionId !== local.ancestors[i]?.session.sessionId || + ref.bindingHash !== local.ancestors[i]?.session.mcpBindingEnvelope?.bindingHash || + ref.bindingEpoch !== local.ancestors[i]?.session.mcpBindingEnvelope?.bindingEpoch, + ) + ) + throw Error("MCP rebind authority changed"); + const pending = await drain(tx, sessionId); + if (pending) throw new McpRebindDraining(pending); + const rows = await tx + .update(taskSession) + .set({ mcpManagedLifecycle: next }) + .where( + and( + eq(taskSession.sessionId, sessionId), + sql`${taskSession.mcpManagedLifecycle} = ${JSON.stringify(prepared.previous)}::jsonb`, + sql`${taskSession.mcpBindingEnvelope}->>'bindingHash' = ${oldHash}`, + ), + ) + .returning({ id: taskSession.sessionId }); + if (rows.length !== 1) throw Error("MCP rebind reservation changed"); + }); + } + const value = (await deps.store.read(sessionId)).session.mcpManagedLifecycle; + if ( + !value?.recovery || + value.recovery.id !== recoveryId || + value.recovery.expectedBindingHash !== oldHash || + value.recovery.candidateHash !== candidateHash || + value.recovery.actorId !== actorId || + !["started", "recovering", "finalizing"].includes(value.phase) + ) + throw Error("MCP rebind reservation changed"); + if (value.phase !== "started") + throw new McpRebindDraining({ + reason: "dispatch_in_progress", + retryAfter: new Date(deps.now().getTime() + 1000).toISOString(), + }); + return { bindingHash: value.bindingHash }; + }, + }; +} diff --git a/orchestrator/src/mcp/managed-runtime.ts b/orchestrator/src/mcp/managed-runtime.ts new file mode 100644 index 000000000..8c1c052ee --- /dev/null +++ b/orchestrator/src/mcp/managed-runtime.ts @@ -0,0 +1,372 @@ +import { create, equals } from "@bufbuild/protobuf"; +import { + ManagedCreateIdentitySchema, + ManagedPolicyTargetSchema, +} from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import type { SessionPolicyControlClient } from "./control-client.ts"; +import type { SessionCreateInput } from "../rpc/task-create.ts"; +import type { TrustedCreationInput } from "./session-create.ts"; +import type { ManagedCreationIdentity, ManagedLifecycle } from "./managed-lifecycle.ts"; +import { makeManagedLifecycleStore } from "./managed-lifecycle-store.ts"; +import { decodeManagedState, type ManagedCreationState } from "./managed-state.ts"; +import { bytesDigest, frozenNativeManifest, managedCreationInput } from "./managed-create-input.ts"; +import { bindingSnapshotColumns, restoreBindingSnapshot } from "./binding-store.ts"; +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import { + managedAppliedJson, + makeManagedPolicyAuthority, + unionManagedPolicies, +} from "./managed-policy.ts"; +import type { McpSessionBindingSnapshot } from "./session-bindings.ts"; +import type { PublishedOperator } from "./operator.ts"; +import { appendMcpManifest } from "../tools/manifest.ts"; +import { integrationSnapshotHash } from "../integrations/grants.ts"; + +type Query = { + protocol: number; + identity: ManagedCreationIdentity; + nativeVersion: number; + nativeDigest: string; +}; +/** Signed identity-only producer contract. Runtime installation also requires its actual handlers. */ +export interface ManagedBootBridge { + readManagedCreationState(query: Query): Promise<{ protocol: number; stateJson: Uint8Array }>; + prepareManagedBoot(query: Query): Promise<{ protocol: number; stateJson: Uint8Array }>; + recoverManagedBoot( + query: Query & { expectedSourceKeyJson: Uint8Array; recoveryId: string }, + ): Promise<{ protocol: number; stateJson: Uint8Array }>; +} +/** The immutable reservation must survive an uncertain coordinator outcome. */ +export class McpManagedCreationPending extends Error { + constructor() { + super("MCP session creation is pending"); + } +} +export function makeManagedSessionRuntime(deps: { + store: ReturnType; + control: Pick< + SessionPolicyControlClient, + "reserveManagedCreate" | "readNativeManifest" | "finalizeManagedBoot" + >; + bridge: ManagedBootBridge; + managed: ReturnType; + readPublished(): Promise; + validate(snapshot: McpSessionBindingSnapshot, sessionId: string): Promise; + authorize(source: TrustedCreationInput, sessionId: string): Promise; + enroll(sessionId: string): Promise; + enabled(): boolean; +}) { + const query = (v: ManagedLifecycle): Query => ({ + protocol: 1, + identity: v.identity, + nativeVersion: v.nativeVersion, + nativeDigest: v.nativeDigest, + }); + async function read(v: ManagedLifecycle) { + return decodeManagedState(await deps.bridge.readManagedCreationState(query(v)), v); + } + function publicationMatches(state: ManagedCreationState, publication: PublishedOperator) { + const context = state.boot?.request.key.context; + if ( + !context || + context.operator_generation !== publication.generation || + context.operator_identity !== publication.identity + ) + throw Error("MCP boot publication changed"); + } + async function native(v: ManagedLifecycle) { + const result = await deps.control.readNativeManifest({ protocol: 1, identity: v.identity }); + if ( + result.protocol !== 1 || + !result.identity || + !equals( + ManagedCreateIdentitySchema, + result.identity, + create(ManagedCreateIdentitySchema, v.identity), + ) || + result.nativeVersion !== v.nativeVersion || + result.nativeDigest !== v.nativeDigest || + bytesDigest(result.nativeBytes) !== v.nativeDigest + ) + throw Error("MCP frozen native manifest changed"); + return frozenNativeManifest( + new TextDecoder("utf-8", { fatal: true }).decode(result.nativeBytes), + ).text; + } + async function runOnce(sessionId: string) { + const row = await deps.store.read(sessionId); + let value = row.session.mcpManagedLifecycle; + if (!value || value.phase === "terminal") return; + let state = await read(value); + if (state.state === "absent") return; // Cannot fence an in-flight reserve; never resend lost bytes. + if (state.state === "terminal") { + await deps.store.compareAndSet(value, { ...value, phase: "terminal" }); + return; + } + await deps.enroll(sessionId); + if (!deps.enabled()) return; + if (state.state === "held") { + state = decodeManagedState(await deps.bridge.prepareManagedBoot(query(value)), value); + if (!state.boot) return; + } + if (value.recovery) { + const recovery = value.recovery; + state = decodeManagedState( + await deps.bridge.recoverManagedBoot({ + ...query(value), + expectedSourceKeyJson: new TextEncoder().encode(JSON.stringify(recovery.sourceKey)), + recoveryId: recovery.id, + }), + value, + ); + if ( + state.operation?.recovery_id !== recovery.id || + canonicalJson(state.operation.source_key) !== canonicalJson(recovery.sourceKey) + ) + throw Error("MCP recovery operation changed"); + if ( + !state.operation.completed || + !state.boot || + canonicalJson(state.boot.request.key) === canonicalJson(recovery.sourceKey) + ) + return; + } + const boot = state.boot; + if (!boot || ["claimed", "interrupted"].includes(boot.phase)) return; + const key = boot.request.key; + if (!boot.prepared) throw Error("MCP prepared receipt unavailable"); + const publication = await deps.readPublished(); + publicationMatches(state, publication); + const target = create(ManagedPolicyTargetSchema, { + sessionId, + sandboxId: key.context.target.sandbox_id, + hostId: key.context.target.host_id, + bindingEpoch: BigInt(key.context.target.binding_epoch), + }); + const old = restoreBindingSnapshot( + row.session.mcpBindingEnvelope!, + row.session.integrationConnections ?? [], + ); + const desired = value.recovery?.candidate ?? old; + const exactNative = await native(value); + if (!value.pending) { + if (boot.phase !== "prepared") throw Error("MCP final binding reservation missing"); + await deps.validate(desired, sessionId); + const policyState = await deps.managed.read(target); + const policies = [ + old.managedDenyPolicy, + desired.managedDenyPolicy, + boot.request.bootstrap_policy, + ]; + if (policyState.retained) policies.push(policyState.retained); + if (policyState.applied?.policy) + policies.push({ + ...policyState.applied.policy, + revision: Number(policyState.applied.policy.revision), + version: 1, + }); + const publishedHosts = [ + ...new Set( + publication.policies.flatMap((p) => [ + ...p.authorities.map((a) => new URL(`https://${a}/`).hostname), + ...p.aliases, + ]), + ), + ].sort(); + const publishedBody = { version: 1 as const, revision: 0, hosts: publishedHosts }; + policies.push({ ...publishedBody, digest: canonicalDigest(publishedBody) }); + const { bindingHash: _hash, ...body } = desired; + const candidate = { + ...body, + bindingEpoch: key.context.target.binding_epoch, + managedDenyPolicy: unionManagedPolicies(policies), + }; + const snapshot = { ...candidate, bindingHash: canonicalDigest(candidate) }; + const manifest = new TextEncoder().encode( + JSON.stringify(appendMcpManifest(exactNative, snapshot)), + ); + const next: ManagedLifecycle = { + ...value, + phase: "finalizing", + sourceKey: key, + bindingHash: snapshot.bindingHash, + pending: { + snapshot, + preparedReceiptId: boot.prepared.receipt_id, + combinedManifestDigest: bytesDigest(manifest), + }, + }; + const columns = bindingSnapshotColumns(snapshot, [ + ...(row.session.integrationConnections ?? []).filter((c) => c.provider !== "mcp"), + ...snapshot.immutableBindings.map((b) => ({ + id: b.connectionId, + alias: b.alias, + provider: "mcp", + displayName: b.alias, + config: b.config, + })), + ]); + const integrationGrants = [ + ...(row.session.integrationGrants ?? []).filter((g) => !g.mcp), + ...snapshot.immutableBindings.flatMap((b) => + b.tools.map((t) => ({ + connectionId: b.connectionId, + operation: "invoke", + credentialScope: "org" as const, + resourceConstraints: [], + mcp: t.grant, + })), + ), + ]; + await deps.store.compareAndSet(value, next, { + ...columns, + integrationGrants, + integrationSnapshotHash: integrationSnapshotHash({ + profileId: snapshot.profileId, + integrationGrants, + ...columns, + }), + }); + value = next; + } + const pending = value.pending!; + if ( + canonicalJson(value.sourceKey) !== canonicalJson(key) || + pending.preparedReceiptId !== boot.prepared.receipt_id || + pending.snapshot.bindingEpoch !== key.context.target.binding_epoch || + pending.snapshot.bindingHash !== value.bindingHash + ) + throw Error("MCP pending boot changed"); + await deps.validate(pending.snapshot, sessionId); + const manifest = new TextEncoder().encode( + JSON.stringify(appendMcpManifest(exactNative, pending.snapshot)), + ); + if (bytesDigest(manifest) !== pending.combinedManifestDigest) + throw Error("MCP final manifest changed"); + if (boot.finalization) { + if (boot.finalization.combined_manifest_digest !== pending.combinedManifestDigest) + throw Error("MCP finalized manifest changed"); + await deps.managed.enforce( + { + sessionId, + bindingEpoch: pending.snapshot.bindingEpoch, + bindingHash: value.bindingHash, + denyRevision: pending.snapshot.managedDenyPolicy.revision, + denyDigest: pending.snapshot.managedDenyPolicy.digest, + }, + pending.snapshot.managedDenyPolicy, + ); + if (boot.phase === "started" && value.phase !== "started") + await deps.store.compareAndSet(value, { ...value, phase: "started" }); + return; + } + const policyState = await deps.managed.read(target); + let applied = policyState.applied; + const wanted = pending.snapshot.managedDenyPolicy; + if ( + !applied || + applied.context?.bindingHash !== value.bindingHash || + applied.policy?.digest !== wanted.digest + ) { + applied = await deps.managed.apply(target, value.bindingHash, wanted, applied, publication); + } + // Fresh publication and binding checks immediately precede the durable finalization. + publicationMatches(state, await deps.readPublished()); + await deps.validate(pending.snapshot, sessionId); + const latest = (await deps.store.read(sessionId)).session.mcpManagedLifecycle; + if (canonicalJson(latest) !== canonicalJson(value)) + throw Error("MCP finalization reservation changed"); + const response = await deps.control.finalizeManagedBoot({ + protocol: 1, + inputJson: new TextEncoder().encode( + JSON.stringify({ + key, + prepared_receipt_id: pending.preparedReceiptId, + policy: managedAppliedJson(applied), + native_version: value.nativeVersion, + combined_manifest: [...manifest], + combined_manifest_digest: pending.combinedManifestDigest, + }), + ), + }); + if (response.protocol !== 1 || !response.bootJson) + throw Error("MCP finalization response unavailable"); + // A fresh read on the next pass verifies finalization and exact Started. The reply alone cannot open dispatch. + } + return { + runOnce, + read, + native, + async create(input: SessionCreateInput, source: TrustedCreationInput) { + const id = input.requestedSessionId; + if (!id || !input.mcpBindingEnvelope) throw Error("MCP creation unavailable"); + try { + if (!deps.enabled()) throw Error("MCP creation unavailable"); + await deps.authorize(source, id); + const row = await deps.store.read(id); + const snapshot = restoreBindingSnapshot( + input.mcpBindingEnvelope, + input.integrationConnections ?? [], + ); + await deps.validate(snapshot, id); + const existing = row.session.mcpManagedLifecycle; + if (existing) { + if ( + existing.identity.actorId !== source.actorId || + existing.identity.inputId !== source.inputId + ) + throw Error("MCP creation input changed"); + const state = await read(existing); + if (state.state === "absent" || state.state === "terminal") + throw new McpManagedCreationPending(); + await deps.enroll(id); + return { sessionId: id }; + } + const inputState = managedCreationInput(input, { + ...source, + taskId: row.session.taskId, + ...(row.owner.spawningSessionId ? { parentSessionId: row.owner.spawningSessionId } : {}), + }); + const value: ManagedLifecycle = { + version: 1, + identity: inputState.identity, + nativeVersion: 1, + nativeDigest: inputState.nativeDigest, + bindingHash: snapshot.bindingHash, + phase: "reserving", + }; + await deps.store.initialize(value); + try { + const response = await deps.control.reserveManagedCreate(inputState.request); + if ( + response.protocol !== 1 || + response.state !== "held" || + !response.identity || + !equals( + ManagedCreateIdentitySchema, + response.identity, + create(ManagedCreateIdentitySchema, value.identity), + ) + ) + throw Error("MCP reserve response changed"); + await deps.store.compareAndSet(value, { ...value, phase: "held" }); + await deps.enroll(id); + return { sessionId: id }; + } catch { + throw new McpManagedCreationPending(); + } + } catch (error) { + // A concurrent creator or an earlier attempt can own the durable identity. + // Failure to read it cannot justify deleting its reservation or coordinator session. + let retained: boolean; + try { + retained = Boolean((await deps.store.read(id)).session.mcpManagedLifecycle); + } catch { + throw new McpManagedCreationPending(); + } + if (retained) throw new McpManagedCreationPending(); + throw error; + } + }, + }; +} diff --git a/orchestrator/src/mcp/managed-scanner.ts b/orchestrator/src/mcp/managed-scanner.ts new file mode 100644 index 000000000..70366cc1e --- /dev/null +++ b/orchestrator/src/mcp/managed-scanner.ts @@ -0,0 +1,104 @@ +import { Code, ConnectError } from "@connectrpc/connect"; +import type { makeManagedLifecycleStore } from "./managed-lifecycle-store.ts"; +import type { makeManagedSessionRuntime } from "./managed-runtime.ts"; +import { log } from "../log.ts"; + +// Only fixed local failures may select a reason. Never log the caught error. +const reconciliationReasons = new Map([ + ["MCP prepared receipt unavailable", "prepared_receipt_unavailable"], + ["MCP boot publication changed", "boot_publication_changed"], + ["MCP frozen native manifest changed", "frozen_native_manifest_changed"], + ["MCP final binding reservation missing", "final_binding_reservation_missing"], + ["MCP retained policy presence unavailable", "retained_policy_presence_unavailable"], + ["MCP pending boot changed", "pending_boot_changed"], + ["MCP final manifest changed", "final_manifest_changed"], + ["MCP managed apply unavailable", "managed_apply_unavailable"], + ["MCP managed apply changed", "managed_apply_changed"], + ["MCP finalization reservation changed", "finalization_reservation_changed"], + ["MCP recovery operation changed", "recovery_operation_changed"], + ["MCP finalized manifest changed", "finalized_manifest_changed"], + ["MCP finalization response unavailable", "finalization_response_unavailable"], + ["MCP managed integer unavailable", "managed_integer_unavailable"], + ["MCP managed policy unavailable", "managed_policy_unavailable"], + ["MCP managed target unavailable", "managed_target_unavailable"], + ["MCP managed receipt unavailable", "managed_receipt_unavailable"], + ["MCP network proof unavailable", "network_proof_unavailable"], + ["MCP managed receipt digest changed", "managed_receipt_digest_changed"], + ["MCP managed revision exhausted", "managed_revision_exhausted"], + ["MCP applied target changed", "applied_target_changed"], + ["MCP managed binding changed", "managed_binding_changed"], + ["MCP final managed acknowledgement unavailable", "final_managed_acknowledgement_unavailable"], +]); +const rpcReasons = new Map([ + [Code.Canceled, "rpc_canceled"], + [Code.Unknown, "rpc_unknown"], + [Code.InvalidArgument, "rpc_invalid_argument"], + [Code.DeadlineExceeded, "rpc_deadline_exceeded"], + [Code.NotFound, "rpc_not_found"], + [Code.AlreadyExists, "rpc_already_exists"], + [Code.PermissionDenied, "rpc_permission_denied"], + [Code.ResourceExhausted, "rpc_resource_exhausted"], + [Code.FailedPrecondition, "rpc_failed_precondition"], + [Code.Aborted, "rpc_aborted"], + [Code.OutOfRange, "rpc_out_of_range"], + [Code.Unimplemented, "rpc_unimplemented"], + [Code.Internal, "rpc_internal"], + [Code.Unavailable, "rpc_unavailable"], + [Code.DataLoss, "rpc_data_loss"], + [Code.Unauthenticated, "rpc_unauthenticated"], +]); + +/** Durable keyset scan. A failed row cannot starve later reservations. */ +export function makeManagedSessionScanner(deps: { + store: Pick, "pending">; + runtime: Pick, "runOnce">; +}) { + let after: string | undefined; + let stopped = true; + let timer: ReturnType | undefined; + let running: Promise | undefined; + async function runOnce(stopBetweenRows = false) { + const rows = await deps.store.pending(after); + for (const row of rows) { + if (stopBetweenRows && stopped) break; + try { + await deps.runtime.runOnce(row.sessionId); + } catch (error) { + const reason = + error instanceof ConnectError + ? (rpcReasons.get(error.code) ?? "unclassified") + : error instanceof Error + ? (reconciliationReasons.get(error.message) ?? "unclassified") + : "unclassified"; + log.warn({ reason }, "MCP managed session reconciliation unavailable"); + } + after = row.sessionId; + } + if (rows.length === 0) after = undefined; + } + async function tick() { + try { + await runOnce(true); + } catch { + log.warn("MCP managed session scan unavailable"); + } + if (!stopped) + timer = setTimeout(() => { + running = tick(); + }, 1000); + } + return { + runOnce: () => runOnce(), + async start() { + if (!stopped) return; + stopped = false; + running = tick(); + await running; + }, + async stop() { + stopped = true; + clearTimeout(timer); + await running; + }, + }; +} diff --git a/orchestrator/src/mcp/managed-state.ts b/orchestrator/src/mcp/managed-state.ts new file mode 100644 index 000000000..c50da10b3 --- /dev/null +++ b/orchestrator/src/mcp/managed-state.ts @@ -0,0 +1,196 @@ +import { z } from "zod"; +import { canonicalJson } from "./hash.ts"; +import type { ManagedCreationIdentity, ManagedLifecycle } from "./managed-lifecycle.ts"; +import { bytesDigest } from "./managed-create-input.ts"; + +const id = z + .string() + .min(1) + .max(128) + .regex(/^[a-zA-Z0-9-]+$/); +const actor = z + .string() + .min(1) + .max(256) + .regex(/^[^\u0000-\u001f]+$/); +const hash = z.string().regex(/^sha256:[0-9a-f]{64}$/); +const integer = z.number().int().safe().nonnegative(); +export const managedIdentitySchema = z + .object({ + session_id: id, + task_id: actor, + policy: z + .object({ + actorId: actor, + inputId: actor, + parentSessionId: id.nullable(), + eligibility: z.enum(["read_only", "write_eligible"]), + }) + .strict(), + input_digest: hash, + }) + .strict(); +export const managedSourceSchema = z + .object({ + creation: managedIdentitySchema, + context: z + .object({ + target: z + .object({ + session_id: id, + sandbox_id: id, + host_id: id, + binding_epoch: integer.positive(), + }) + .strict(), + operator_generation: integer.positive(), + operator_identity: hash, + binding_hash: z.null(), + }) + .strict(), + boot_attempt: id, + native_digest: hash, + }) + .strict(); +const deny = z + .object({ + version: z.literal(1), + revision: integer, + hosts: z.array(z.string()).max(4096), + digest: hash, + }) + .strict(); +const request = z.object({ key: managedSourceSchema, bootstrap_policy: deny }); +const prepared = z.object({ + request, + host_incarnation: id, + network_identity: hash, + guest_ip: z.string(), + receipt_id: hash, +}); +const boot = z.object({ + request, + revision: integer.positive(), + phase: z.enum([ + "claimed", + "prepared", + "finalizing", + "finalized", + "start_issued", + "started", + "interrupted", + ]), + prepared: prepared.nullable(), + sealed_agent: z.null(), + retired: z.object({ prepared, retained_policy: deny, receipt_id: hash }).nullable(), + finalization: z + .object({ + prepared, + native_version: z.literal(1), + combined_manifest: z.array(z.number().int().min(0).max(255)).max(1024 * 1024), + combined_manifest_digest: hash, + }) + .nullable(), +}); +const state = z + .object({ + query: z + .object({ + identity: managedIdentitySchema, + native_version: z.literal(1), + native_digest: hash, + }) + .strict(), + state: z.enum(["absent", "held", "preparing", "boot", "recovering", "terminal"]), + boot: boot.nullable(), + operation: z + .object({ + intent_id: id, + revision: integer.positive(), + recovery_id: id.nullable(), + source_key: managedSourceSchema.nullable(), + capture_id: id.nullable(), + completed: z.boolean(), + }) + .strict() + .nullable(), + terminal_reason: z.string().max(256).nullable(), + }) + .strict(); +export type ManagedCreationState = z.infer; +export function rustCreationIdentity(value: ManagedCreationIdentity) { + return managedIdentitySchema.parse({ + session_id: value.sessionId, + task_id: value.taskId, + policy: { + actorId: value.actorId, + inputId: value.inputId, + parentSessionId: value.parentSessionId ?? null, + eligibility: value.writeEligibility, + }, + input_digest: value.inputDigest, + }); +} +export function managedQuery(value: ManagedLifecycle) { + return { + identity: rustCreationIdentity(value.identity), + native_version: value.nativeVersion, + native_digest: value.nativeDigest, + }; +} +/** Decode only the sanitized producer view. Missing or mismatched state is never an ACK. */ +export function decodeManagedState( + response: { protocol: number; stateJson: Uint8Array }, + expected: ManagedLifecycle, +) { + if ( + response.protocol !== 1 || + !response.stateJson.length || + response.stateJson.length > 8 * 1024 * 1024 + ) + throw Error("MCP creation state unavailable"); + const value = state.parse( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(response.stateJson)), + ); + if (canonicalJson(value.query) !== canonicalJson(managedQuery(expected))) + throw Error("MCP creation identity changed"); + if (value.state === "absent" && (value.boot || value.operation || value.terminal_reason)) + throw Error("MCP absent state changed"); + const key = value.boot?.request.key; + if ( + key && + (canonicalJson(key.creation) !== canonicalJson(value.query.identity) || + key.native_digest !== expected.nativeDigest || + key.context.target.session_id !== expected.identity.sessionId) + ) + throw Error("MCP boot identity changed"); + for (const receipt of [ + value.boot?.prepared, + value.boot?.finalization?.prepared, + value.boot?.retired?.prepared, + ]) { + if (receipt && canonicalJson(receipt.request.key) !== canonicalJson(key)) + throw Error("MCP prepared identity changed"); + } + const finalization = value.boot?.finalization; + if ( + finalization && + bytesDigest(new Uint8Array(finalization.combined_manifest)) !== + finalization.combined_manifest_digest + ) + throw Error("MCP final manifest changed"); + const operation = value.operation; + if ( + operation && + (Boolean(operation.recovery_id) !== Boolean(operation.source_key) || + Boolean(operation.source_key) !== Boolean(operation.capture_id)) + ) + throw Error("MCP recovery association unavailable"); + if ( + operation?.source_key && + (canonicalJson(operation.source_key.creation) !== canonicalJson(value.query.identity) || + operation.source_key.native_digest !== expected.nativeDigest) + ) + throw Error("MCP recovery source changed"); + return value; +} diff --git a/orchestrator/src/mcp/operator-runtime.ts b/orchestrator/src/mcp/operator-runtime.ts new file mode 100644 index 000000000..39dc2640a --- /dev/null +++ b/orchestrator/src/mcp/operator-runtime.ts @@ -0,0 +1,50 @@ +import type { McpConfig } from "./config.ts"; +import { validateMcpConfig } from "./config.ts"; +import { canonicalJson } from "./hash.ts"; +import type { PublishedOperator } from "./operator.ts"; +import type { McpRuntimeConfiguration } from "./runtime-config.ts"; +import type { McpOperatorPolicy } from "./session-bindings.ts"; + +/** A configured transport is not authority. Read publication on every selection and + * again at the final send fence; never keep a successful permission read as a cache. */ +export function makeMcpOperatorRuntime(deps: { + configuration: McpRuntimeConfiguration; + read(): Promise; + enabled(): boolean; +}) { + function enabled(zoneId: string) { + return deps.enabled() && deps.configuration.workers.get(zoneId)?.enabled === true; + } + function worker(zoneId: string) { + const worker = deps.configuration.workers.get(zoneId); + if (!enabled(zoneId) || !worker) throw Error("MCP worker is disabled"); + return worker; + } + async function select(raw: McpConfig) { + const config = validateMcpConfig(raw); + const configured = worker(config.zone); + const published = await deps.read(); + const policy = published.policies.find((policy) => policy.zoneId === config.zone); + const endpoint = new URL(config.endpoint); + if (!policy || !policy.authorities.includes(endpoint.host)) + throw Error("MCP destination is not published"); + worker(config.zone); + return { policy, ...configured }; + } + return { + enabled, + select, + workerTls: (zoneId: string) => ({ ...worker(zoneId).tls }), + deploymentId: (zoneId: string) => worker(zoneId).deploymentId, + async assertCurrent(config: McpConfig, expected: McpOperatorPolicy) { + if (canonicalJson((await select(config)).policy) !== canonicalJson(expected)) + throw Error("MCP operator authority changed"); + }, + async policies() { + return (await deps.read()).policies; + }, + async validateDestination(config: McpConfig) { + await select(config); + }, + }; +} diff --git a/orchestrator/src/mcp/operator.ts b/orchestrator/src/mcp/operator.ts new file mode 100644 index 000000000..b4c649bc8 --- /dev/null +++ b/orchestrator/src/mcp/operator.ts @@ -0,0 +1,192 @@ +import { createHash } from "node:crypto"; +import { z } from "zod"; +import { ZonePolicy } from "../../../mcp-worker/src/network.ts"; +import { canonicalJson } from "./hash.ts"; +import type { McpOperatorPolicy, McpOperatorPublicationIdentity } from "./session-bindings.ts"; + +const digest = z.string().regex(/^sha256:[0-9a-f]{64}$/); +const id = z.string().min(1).max(128); +const integer = z.number().int().min(1).max(Number.MAX_SAFE_INTEGER); +const digests = z.record(id, digest).refine((value) => Object.keys(value).length > 0); +const configuration = z + .object({ + id, + revision: id, + mode: z.enum(["public", "internal"]), + authorities: z.array(z.string()).min(1).max(4096), + aliases: z.array(z.string()).max(4096), + cidrs: z.array(z.string()).max(4096), + denyCidrs: z.array(z.string()).max(4096), + ca: z + .string() + .max(1024 * 1024) + .optional(), + limits: z + .object({ + execute: integer.optional(), + discover: integer.optional(), + oauth: integer.optional(), + schema: integer.optional(), + toolMs: integer.optional(), + }) + .strict() + .optional(), + }) + .strict(); +const zone = z + .object({ + revision: id, + worker_digest: digest, + authorities: z.array(z.string()).min(1).max(4096), + aliases: z.array(z.string()).max(4096), + configuration, + }) + .strict(); +const candidate = z + .object({ + version: z.literal(1), + zones: z.record(id, zone), + deployment_identity: digest, + network_identity: digest, + trust_identity: digest, + ca_content_digests: digests, + }) + .strict(); +const generation = z + .object({ + generation: integer, + expected_generation: integer.nullable(), + identity: digest, + candidate, + }) + .strict(); +const publication = z + .object({ published: generation, pending: z.null(), admission_open: z.literal(true) }) + .strict(); +export type OperatorInfrastructure = Omit< + McpOperatorPublicationIdentity, + "generation" | "identity" +>; +export interface PublishedOperator { + generation: number; + identity: string; + policies: McpOperatorPolicy[]; +} +const hash = (value: string) => `sha256:${createHash("sha256").update(value).digest("hex")}`; +function sorted(value: Record): Record { + return Object.fromEntries(Object.entries(value).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))); +} +function canonicalSet(values: string[]): string[] { + const result = [...new Set(values)].sort(); + if (canonicalJson(result) !== canonicalJson(values)) + throw Error("operator projection is not canonical"); + return result; +} +/** Decode only the pinned coordinator publication. Configuration files specify expected + * infrastructure identities; they cannot supply a replacement publication. */ +export function parsePublishedOperator( + bytes: Uint8Array, + expected: OperatorInfrastructure, +): PublishedOperator { + if (!bytes.length || bytes.length > 2 * 1024 * 1024) + throw Error("operator publication unavailable"); + const parsed = publication.parse( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)), + ); + const p = parsed.published, + c = p.candidate; + if ( + p.generation !== (p.expected_generation ?? 0) + 1 || + !Object.keys(c.zones).length || + Object.keys(c.zones).length > 128 + ) + throw Error("operator generation unavailable"); + // Rust serializes struct fields in declaration order and map/object keys in sorted order. + // The worker digest uses its own canonical encoding and is checked independently below. + const encoded = { + version: c.version, + zones: sorted( + Object.fromEntries( + Object.entries(c.zones).map(([name, value]) => [ + name, + { + revision: value.revision, + worker_digest: value.worker_digest, + authorities: value.authorities, + aliases: value.aliases, + configuration: JSON.parse(canonicalJson(value.configuration)), + }, + ]), + ), + ), + deployment_identity: c.deployment_identity, + network_identity: c.network_identity, + trust_identity: c.trust_identity, + ca_content_digests: sorted(c.ca_content_digests), + }; + if ( + hash(JSON.stringify(encoded)) !== p.identity || + c.deployment_identity !== expected.deploymentIdentity || + c.network_identity !== expected.networkIdentity || + c.trust_identity !== expected.trustIdentity || + canonicalJson(c.ca_content_digests) !== canonicalJson(expected.caContentDigests) + ) + throw Error("operator infrastructure identity changed"); + const identity: McpOperatorPublicationIdentity = { + ...structuredClone(expected), + generation: p.generation, + identity: p.identity, + }; + const hosts = new Set(); + const policies = Object.entries(c.zones) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([zoneId, value]) => { + const config = value.configuration; + if ( + config.id !== zoneId || + config.revision !== value.revision || + canonicalJson(canonicalSet(value.authorities)) !== + canonicalJson([...new Set(config.authorities)].sort()) || + canonicalJson(canonicalSet(value.aliases)) !== + canonicalJson([...new Set(config.aliases)].sort()) || + (config.ca && + (!config.ca.startsWith("-----BEGIN CERTIFICATE-----") || + config.ca.includes("PRIVATE KEY") || + !Object.values(c.ca_content_digests).includes(hash(config.ca)))) + ) + throw Error("operator zone projection changed"); + const worker = new ZonePolicy(config); + if (`sha256:${worker.digest}` !== value.worker_digest) + throw Error("operator worker configuration changed"); + for (const authority of value.authorities) + hosts.add(new URL(`https://${authority}/`).hostname); + for (const alias of value.aliases) hosts.add(alias); + return { + publication: structuredClone(identity), + zoneId, + policyRevision: value.revision, + policyDigest: value.worker_digest, + authorities: [...value.authorities], + aliases: [...value.aliases], + }; + }); + if (hosts.size > 4096) throw Error("operator authority set exceeds limit"); + return { generation: p.generation, identity: p.identity, policies }; +} +export function makeOperatorReader( + control: { + readPublishedOperator(input: { + protocol: number; + }): Promise<{ protocol: number; publicationJson: Uint8Array }>; + }, + expected: OperatorInfrastructure, +) { + const trusted = structuredClone(expected); + return { + async read() { + const response = await control.readPublishedOperator({ protocol: 1 }); + if (response.protocol !== 1) throw Error("operator read protocol unavailable"); + return parsePublishedOperator(response.publicationJson, trusted); + }, + }; +} diff --git a/orchestrator/src/mcp/published-credentials.ts b/orchestrator/src/mcp/published-credentials.ts new file mode 100644 index 000000000..07b6eb896 --- /dev/null +++ b/orchestrator/src/mcp/published-credentials.ts @@ -0,0 +1,36 @@ +import type { CredentialAdmin } from "./credential-admin.ts"; +import type { CredentialMetadataStore } from "./credential-store.ts"; +import { validateMcpConfig } from "./config.ts"; +import type { makeMcpOperatorRuntime } from "./operator-runtime.ts"; + +/** New credential authority requires a published destination. Recovery and revocation + * remain available while publication or new dispatch is disabled. */ +export function makePublishedCredentialAdmin(deps: { + admin: CredentialAdmin; + store: Pick; + operator: ReturnType; +}): CredentialAdmin { + async function current(id: string) { + const row = await deps.store.get(id); + await deps.operator.validateDestination(validateMcpConfig(row.config)); + } + return { + ...deps.admin, + async putSecret(...args) { + await current(args[0]); + return deps.admin.putSecret(...args); + }, + async begin(...args) { + await current(args[0]); + return deps.admin.begin(...args); + }, + async complete(...args) { + await current(args[0]); + return deps.admin.complete(...args); + }, + async configure(...args) { + await deps.operator.validateDestination(args[4]); + return deps.admin.configure(...args); + }, + }; +} diff --git a/orchestrator/src/mcp/rebind.ts b/orchestrator/src/mcp/rebind.ts new file mode 100644 index 000000000..36c89b4e7 --- /dev/null +++ b/orchestrator/src/mcp/rebind.ts @@ -0,0 +1,14 @@ +export class McpPublicationRequired extends Error { + constructor() { + super("MCP publication required"); + } +} +export type McpRebindDrain = { + reason: "dispatch_in_progress" | "unknown_intent" | "legacy_intent"; + retryAfter: string; +}; +export class McpRebindDraining extends Error { + constructor(readonly drain: McpRebindDrain) { + super("MCP dispatch in progress"); + } +} diff --git a/orchestrator/src/mcp/recovery-scanner.ts b/orchestrator/src/mcp/recovery-scanner.ts new file mode 100644 index 000000000..327b44515 --- /dev/null +++ b/orchestrator/src/mcp/recovery-scanner.ts @@ -0,0 +1,84 @@ +import { makeMcpJournal } from "./journal.ts"; +import { type McpRecovery, type McpExecutionMarker, mcpRecovery } from "./recovery.ts"; +import { startMcpWorkflow } from "../workflows/mcp-exec.ts"; +import { log } from "../log.ts"; + +/** Bounded database maintenance, with a keyset cursor so parked calls cannot starve later work. */ +export function makeMcpRecoveryScanner(deps: { + deploymentId: string; + journal?: ReturnType; + runtime?: () => McpRecovery | undefined; + startWorkflow?: (marker: McpExecutionMarker, workflowId: string) => Promise; + now?: () => Date; +}) { + let after: string | undefined; + let expiryAfter: string | undefined; + let timer: ReturnType | undefined; + let running: Promise | undefined; + let stopped = true; + let lifecycleTimer: ReturnType | undefined; + let lifecycleRunning: Promise | undefined; + async function runOnce() { + const journal = deps.journal ?? makeMcpJournal(); + const now = (deps.now ?? (() => new Date()))(); + // These producers recover distinct crash states. Disabling dispatch never disables expiry. + const scopes = await journal.expiryScopes(now, expiryAfter); + for (const scope of scopes) { + await journal.expire(scope, now); + expiryAfter = scope; + } + if (scopes.length === 0) expiryAfter = undefined; + await journal.expireUnadmitted(now); + const runtime = (deps.runtime ?? mcpRecovery)(); + if (!runtime) return; + const rows = await runtime.pending(after); + for (const { workflowId, ...marker } of rows) { + await (deps.startWorkflow ?? startMcpWorkflow)(marker, workflowId); + after = marker.executionId; + } + if (rows.length === 0) after = undefined; + } + async function tick() { + try { + await runOnce(); + } catch { + log.warn("MCP recovery scan unavailable"); + } + if (!stopped) + timer = setTimeout(() => { + running = tick(); + }, 1_000); + } + async function runLifecycleOnce() { + await (deps.runtime ?? mcpRecovery)()?.reconcileLifecycle(); + } + async function lifecycleTick() { + try { + await runLifecycleOnce(); + } catch { + log.warn("MCP lifecycle reconciliation unavailable"); + } + if (!stopped) + lifecycleTimer = setTimeout(() => { + lifecycleRunning = lifecycleTick(); + }, 1_000); + } + return { + runOnce, + runLifecycleOnce, + async start() { + if (!stopped) return; + stopped = false; + running = tick(); + lifecycleRunning = lifecycleTick(); + await running; + }, + async stop() { + stopped = true; + clearTimeout(timer); + clearTimeout(lifecycleTimer); + await running; + await lifecycleRunning; + }, + }; +} diff --git a/orchestrator/src/mcp/recovery.ts b/orchestrator/src/mcp/recovery.ts new file mode 100644 index 000000000..fdbeb94b5 --- /dev/null +++ b/orchestrator/src/mcp/recovery.ts @@ -0,0 +1,406 @@ +import { Code, ConnectError } from "@connectrpc/connect"; +import { log } from "../log.ts"; +import { and, asc, eq, gt, lt, sql } from "drizzle-orm"; +import { sessions } from "../control-plane/client.ts"; +import { getDb } from "../db/client.ts"; +import { mcpExecution, taskSession } from "../db/schema.ts"; +import { requestPolicyContext } from "./journal-review.ts"; +import { McpAuthorityStale } from "./authority-error.ts"; +import type { SessionPolicyControlClient } from "./control-client.ts"; +import type { ToolCallCompleter } from "../tools/complete.ts"; +import { + McpRequestDenied, + type McpRequestDenialCode, + type makeMcpEffectDriver, + type McpToolRequest, +} from "./effect-driver.ts"; +import type { CuratedEvent } from "../control-plane/session-events.ts"; +import { makeMcpJournal } from "./journal.ts"; +import { MCP_LIMITS } from "./limits.ts"; +import { makeMcpLifecycleEvents, type LifecycleEvent } from "./lifecycle-events.ts"; +import { observeMcpAuthorityChanges } from "./authority-notification.ts"; +import { makeMcpLifecycleReconciler } from "./lifecycle-reconciler.ts"; + +/** Only references cross the durable workflow boundary. */ +export interface McpExecutionMarker { + executionId: string; + sessionId: string; + runId: string; + toolCallId: string; +} +export interface McpCancelRequest { + sessionId: string; + runId: string; + toolCallId: string; + modelToolName: string; +} +/** Durable pointer to the first authenticated coordinator request, not a guest projection. */ +export interface McpRequestMarker { + sessionId: string; + runId: string; + toolCallId: string; + requestEventIdx: string; + /** Trusted consumer clock plus the fixed maximum preacceptance budget. Never renewed. */ + deadlineMs: number; +} +export type McpRequestDecision = + | { kind: "accepted"; execution: McpExecutionMarker } + | { kind: "denied"; code: McpRequestDenialCode }; + +function marker(call: typeof mcpExecution.$inferSelect): McpExecutionMarker { + return { + executionId: call.id, + sessionId: call.sessionId, + runId: call.runId, + toolCallId: call.toolCallId, + }; +} +const active = new Set(["waiting_approval", "authorized", "dispatching"]); +// These exact CompleteToolCall dispositions prove that delivery can no longer occur. +// A host loss, generic precondition failure or transport outage does not prove deletion. +function targetGone(error: unknown): boolean { + return ( + error instanceof ConnectError && + (error.code === Code.NotFound || + (error.code === Code.FailedPrecondition && + error.metadata.get("engram-error-slug") === "snapshot_invalidated")) + ); +} +type CompletionDisposition = "enqueued" | "target_gone"; + +/** Plain runtime operations: effects and completion must never run in a DBOS step. */ +export function makeMcpRecovery(deps: { + db?: ReturnType; + /** Durable completion remains available when dispatch configuration is absent. */ + driver?: ReturnType; + cancellationPolicy?: Pick; + dispatchEnabled?: () => boolean; + deploymentId: string; + completer?: ToolCallCompleter; + now(): Date; + readRequestEvent?: (sessionId: string, index: bigint) => Promise; + listLifecycleEvents?: (sessionId: string, after: bigint) => Promise; +}) { + const db = deps.db ?? getDb(); + const journal = makeMcpJournal(db); + const dispatchEnabled = () => Boolean(deps.driver) && (deps.dispatchEnabled?.() ?? true); + function requireDispatch() { + if (!dispatchEnabled() || !deps.driver) throw new McpRequestDenied("disabled"); + return deps.driver; + } + async function reconcile(id: string, signal?: AbortSignal) { + signal?.throwIfAborted(); + if (!dispatchEnabled()) { + deps.driver?.cancel(id); + await journal.cancelInternal(id, "kill", deps.now()); + return; + } + await requireDispatch().reconcile(id, signal); + } + const reconciler = makeMcpLifecycleReconciler({ + db, + driver: { + reconcile, + cancel: (id) => deps.driver?.cancel(id), + }, + onSession: (id) => lifecycle.runOnce(id), + }); + const lifecycle = makeMcpLifecycleEvents({ + db, + now: deps.now, + async list(sessionId, after) { + if (deps.listLifecycleEvents) return deps.listLifecycleEvents(sessionId, after); + const page = await sessions.listSessionEvents( + { sessionId, afterIdx: after, limit: 200n }, + { signal: AbortSignal.timeout(5_000) }, + ); + return page.events.map((event) => { + if (event.idx === undefined) throw Error("MCP lifecycle event index unavailable"); + return { idx: event.idx, kind: event.kind, payloadJson: event.payloadJson }; + }); + }, + reconcile: (sessionId) => reconciler.reconcileSession(sessionId), + }); + async function existing(ref: Pick) { + return ( + await db + .select() + .from(mcpExecution) + .where( + and( + eq(mcpExecution.sessionId, ref.sessionId), + eq(mcpExecution.toolCallId, ref.toolCallId), + ), + ) + )[0]; + } + return { + /** Advisory authenticated harness event. Only the exact current accepted call may cancel. */ + async cancelRequest(input: McpCancelRequest) { + if ( + !Object.values(input).every( + (value) => + typeof value === "string" && + value.length > 0 && + value.length <= 256 && + !/[\x00-\x1f]/.test(value), + ) + ) + throw Error("MCP cancellation identity invalid"); + const call = await existing(input); + // The persisted native cancellation fences delayed acceptance; reconciliation handles races. + if (!call) return; + if (!active.has(call.state)) return; + if ( + call.runId !== input.runId || + call.requestContext?.resolvedTool.modelToolName !== input.modelToolName || + call.originalArguments === null + ) + return; + const [session] = await db + .select() + .from(taskSession) + .where(eq(taskSession.sessionId, input.sessionId)); + if (session?.mcpBindingEnvelope?.bindingHash !== call.bindingHash) return; + if (!deps.cancellationPolicy) throw Error("MCP cancellation authority unavailable"); + const result = await deps.cancellationPolicy.validateExecutionIdentity({ + protocol: 1, + ...requestPolicyContext(call.requestContext), + modelToolName: input.modelToolName, + originalArguments: call.originalArguments, + }); + if (result.protocol !== 1 || result.status === "unknown") + throw Error("MCP cancellation identity unavailable"); + if (result.status !== "current") return; + await journal.cancelInternal(call.id, "interrupt", deps.now()); + deps.driver?.cancel(call.id); + }, + reconcileLifecycle: async () => { + await reconciler.runOnce(); + }, + async reconcileAuthorityChange() { + await reconciler.reconcileAuthorityChange(); + }, + abortExecution: (id: string) => { + if (!deps.driver) throw Error("MCP cancellation unavailable"); + deps.driver.cancel(id); + }, + /** The durable step records only this decision. It never runs a provider operation. */ + async decideRequest(ref: McpRequestMarker): Promise { + try { + const first = await existing(ref); + if (first) return { kind: "accepted", execution: marker(first) }; + if (!dispatchEnabled()) return { kind: "denied", code: "disabled" }; + if (!Number.isSafeInteger(ref.deadlineMs) || ref.deadlineMs <= deps.now().getTime()) + return { kind: "denied", code: "deadline_expired" }; + if (!/^[1-9][0-9]{0,18}$/.test(ref.requestEventIdx)) throw Error("invalid event reference"); + const index = BigInt(ref.requestEventIdx); + if (index > 9223372036854775807n) throw Error("invalid event reference"); + const event = deps.readRequestEvent + ? await deps.readRequestEvent(ref.sessionId, index) + : ( + await sessions.listSessionEvents({ + sessionId: ref.sessionId, + afterIdx: index - 1n, + limit: 1n, + }) + ).events[0]; + if (!event || event.idx !== index || event.kind !== "tool_call_requested") + throw Error("request reference unavailable"); + const payload: unknown = JSON.parse(event.payloadJson); + if ( + !payload || + typeof payload !== "object" || + !("tool_call_id" in payload) || + payload.tool_call_id !== ref.toolCallId || + !("run_id" in payload) || + payload.run_id !== ref.runId || + !("name" in payload) || + typeof payload.name !== "string" || + !payload.name.startsWith("mcp_") || + !("args_json" in payload) || + typeof payload.args_json !== "string" + ) + throw Error("request reference unavailable"); + try { + const call = await requireDispatch().acceptToolRequest( + { + sessionId: ref.sessionId, + runId: ref.runId, + toolCallId: ref.toolCallId, + modelToolName: payload.name, + originalArguments: payload.args_json, + }, + new Date(ref.deadlineMs), + ); + return { kind: "accepted", execution: marker(call) }; + } catch (error) { + // Acceptance may have committed before a later check failed. That original row owns completion. + const winner = await existing(ref); + if (winner) return { kind: "accepted", execution: marker(winner) }; + if (error instanceof McpRequestDenied) return { kind: "denied", code: error.code }; + if (error instanceof McpAuthorityStale) + return { kind: "denied", code: "authority_changed" }; + throw error; + } + } catch { + throw Error("MCP request acceptance unavailable"); + } + }, + async deliverDenial(ref: McpRequestMarker, code: McpRequestDenialCode): Promise { + let disposition: CompletionDisposition = "enqueued"; + try { + await (deps.completer ?? sessions).completeToolCall({ + sessionId: ref.sessionId, + toolCallId: ref.toolCallId, + resultJson: JSON.stringify({ + source: "runtime", + denial: code, + delivery: "available", + output: { + isError: true, + content: [{ type: "text", text: "External tool request denied." }], + }, + }), + }); + } catch (error) { + if (!targetGone(error)) throw Error("MCP denial completion unavailable"); + disposition = "target_gone"; + } + log.info( + { sessionId: ref.sessionId, toolCallId: ref.toolCallId, denial: code, disposition }, + "MCP negative completion settled", + ); + }, + async accept(input: McpToolRequest): Promise { + return marker(await requireDispatch().acceptToolRequest(input)); + }, + async expire() { + await journal.expire(deps.deploymentId, deps.now()); + await journal.expireUnadmitted(deps.now()); + }, + async pending(after?: string) { + const rows = await db + .select({ + id: mcpExecution.id, + sessionId: mcpExecution.sessionId, + runId: mcpExecution.runId, + toolCallId: mcpExecution.toolCallId, + version: mcpExecution.version, + state: mcpExecution.state, + }) + .from(mcpExecution) + .where( + and( + lt(mcpExecution.completionEnqueuedVersion, mcpExecution.version), + after ? gt(mcpExecution.id, after) : undefined, + ), + ) + .orderBy(asc(mcpExecution.id)) + .limit(MCP_LIMITS.queueSize); + return rows.map(({ id, state, version, ...refs }) => ({ + ...refs, + executionId: id, + // One live driver per execution; a later committed outcome gets its own delivery marker. + workflowId: active.has(state) ? `mcp:${id}` : `mcp:${id}:result:${version}`, + })); + }, + async runOnce(ref: McpExecutionMarker, signal?: AbortSignal): Promise<"pending" | "done"> { + let call = await journal.get(ref.executionId); + if (!call) return "done"; // Existing session retention deletes its journal too. + if ( + call.sessionId !== ref.sessionId || + call.runId !== ref.runId || + call.toolCallId !== ref.toolCallId + ) + throw Error("MCP execution reference changed"); + if (active.has(call.state) && !dispatchEnabled()) { + await reconcile(call.id, signal); + const retained = await journal.get(call.id); + if (!retained) return "done"; + call = retained; + } else if (call.state === "waiting_approval" || call.state === "authorized") + call = await requireDispatch().driveOnce(call.id, signal); + if (active.has(call.state)) return "pending"; + if (call.completionEnqueuedVersion >= call.version) return "done"; + // A scanner can prove interruption, not an upstream outcome. Preserve that distinction. + const result = + call.result ?? + (call.state === "outcome_unknown" || call.state === "canceled" + ? { + source: "runtime", + outcome: call.state, + delivery: "unavailable", + reason: "execution_interrupted", + } + : undefined); + if (!result) throw Error("MCP committed result unavailable"); + let disposition: CompletionDisposition = "enqueued"; + try { + await (deps.completer ?? sessions).completeToolCall({ + sessionId: call.sessionId, + toolCallId: call.toolCallId, + resultJson: JSON.stringify({ ...result, resultUnavailable: call.resultUnavailable }), + }); + } catch (error) { + if (!targetGone(error)) throw Error("MCP completion unavailable"); + disposition = "target_gone"; + } + // This legacy field records delivery settlement: enqueue ACK OR authoritative target loss. + // It proves neither upstream outcome nor guest ACK. A newer result remains pending. + const settled = await db + .update(mcpExecution) + .set({ completionEnqueuedVersion: call.version }) + .where( + and( + eq(mcpExecution.id, call.id), + lt(mcpExecution.completionEnqueuedVersion, call.version), + sql`${mcpExecution.version} >= ${call.version}`, + ), + ) + .returning({ id: mcpExecution.id }); + if (settled.length) + log.info( + { + executionId: call.id, + sessionId: call.sessionId, + toolCallId: call.toolCallId, + version: call.version, + disposition, + }, + "MCP completion settled", + ); + return "done"; + }, + }; +} +export type McpRecovery = ReturnType; + +// Trusted operator/host composition is installed before DBOS launch. Absence stays closed. +let runtime: McpRecovery | undefined; +export function installMcpRecovery(value: McpRecovery): void { + if (runtime) throw Error("MCP runtime already installed"); + runtime = value; + let pending = false; + let running = false; + observeMcpAuthorityChanges(() => { + pending = true; + if (running) return; + running = true; + void (async () => { + try { + while (pending) { + pending = false; + await value.reconcileAuthorityChange(); + } + } catch { + log.warn("MCP authority reconciliation unavailable"); + } finally { + running = false; + } + })(); + }); +} +export function mcpRecovery(): McpRecovery | undefined { + return runtime; +} diff --git a/orchestrator/src/mcp/repository.ts b/orchestrator/src/mcp/repository.ts new file mode 100644 index 000000000..601ed0082 --- /dev/null +++ b/orchestrator/src/mcp/repository.ts @@ -0,0 +1,218 @@ +import { notifyMcpAuthorityChanged } from "./authority-notification.ts"; +import { lockMcpPublisher } from "../authz/publisher.ts"; +import { and, desc, eq } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { integrationConnection as connection, mcpCatalog as catalog } from "../db/schema.ts"; +import { assertSchemaShape, reviewCatalog } from "./catalog.ts"; +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import { MCP_LIMITS } from "./limits.ts"; +import type { McpConnectionState, McpDiscoveredTool } from "./types.ts"; + +export function makeMcpRepository(db = getDb()) { + return { + async setEnabled(id: string, expectedRevision: number, enabled: boolean, now: Date) { + const result = await db.transaction(async (tx) => { + const [row] = await tx + .select() + .from(connection) + .where(and(eq(connection.id, id), eq(connection.provider, "mcp"))) + .for("update"); + if (!row || row.configRevision !== expectedRevision) + throw new Error("stale connection revision"); + if (enabled) { + const [active] = await tx + .select() + .from(catalog) + .where( + and( + eq(catalog.connectionId, id), + eq(catalog.kind, "published"), + eq(catalog.revision, row.catalogRevision), + ), + ); + if ( + row.revoked || + !row.credentialIdentity || + !active || + active.configRevision !== row.configRevision || + active.authGeneration !== row.authGeneration || + active.credentialIdentity !== row.credentialIdentity + ) + throw new Error("current reviewed catalog and authorization required"); + } + const [updated] = await tx + .update(connection) + .set({ enabled, updatedAt: now }) + .where(eq(connection.id, id)) + .returning(); + return updated!; + }); + notifyMcpAuthorityChanged(); + return result; + }, + async saveDraft( + id: string, + expected: McpConnectionState, + tools: McpDiscoveredTool[], + actorId: string, + now: Date, + ) { + expected = structuredClone(expected); + tools = structuredClone(tools); + if ( + tools.length > MCP_LIMITS.toolsPerSession || + Buffer.byteLength(canonicalJson(tools)) > MCP_LIMITS.manifestBytes + ) + throw new Error("catalog is too large"); + const names = new Set(); + for (const t of tools) { + if ( + !t.upstreamName || + t.upstreamName.length > 256 || + names.has(t.upstreamName) || + typeof t.description !== "string" + ) + throw new Error("invalid discovery"); + names.add(t.upstreamName); + assertSchemaShape(t.inputSchema); + if (t.outputSchema !== undefined) assertSchemaShape(t.outputSchema); + } + // Pick only untrusted metadata. Discovery never grants read or write authority. + const discovered = tools.map((t) => ({ + upstreamName: t.upstreamName, + description: t.description, + inputSchema: t.inputSchema, + ...(t.outputSchema === undefined ? {} : { outputSchema: t.outputSchema }), + })); + return db.transaction(async (tx) => { + const [row] = await tx + .select() + .from(connection) + .where(and(eq(connection.id, id), eq(connection.provider, "mcp"))) + .for("update"); + if ( + !row || + row.revoked || + row.configRevision !== expected.configRevision || + row.authGeneration !== expected.authGeneration || + row.credentialIdentity !== expected.credentialIdentity + ) + throw new Error("stale discovery"); + const [draft] = await tx + .insert(catalog) + .values({ + id: crypto.randomUUID(), + connectionId: id, + kind: "draft", + revision: 0, + configRevision: row.configRevision, + authGeneration: row.authGeneration, + credentialIdentity: row.credentialIdentity, + tools: discovered, + fingerprint: canonicalDigest(discovered), + actorId, + createdAt: now, + }) + .returning(); + return draft!; + }); + }, + async publish( + id: string, + draftId: string, + expectedRevision: number, + input: readonly unknown[], + actorId: string, + now: Date, + ) { + const tools = reviewCatalog(input); + const result = await db.transaction(async (tx) => { + await lockMcpPublisher(tx, actorId); + const [row] = await tx + .select() + .from(connection) + .where(and(eq(connection.id, id), eq(connection.provider, "mcp"))) + .for("update"); + const [draft] = await tx + .select() + .from(catalog) + .where( + and(eq(catalog.id, draftId), eq(catalog.connectionId, id), eq(catalog.kind, "draft")), + ); + if ( + !row || + !draft || + row.revoked || + !row.credentialIdentity || + row.catalogRevision !== expectedRevision || + draft.configRevision !== row.configRevision || + draft.authGeneration !== row.authGeneration || + draft.credentialIdentity !== row.credentialIdentity + ) + throw new Error("stale catalog revision or discovery"); + const originals = new Map(draft.tools.map((t) => [t.upstreamName, t])); + for (const t of tools) { + const original = originals.get(t.upstreamName); + const metadata = { + upstreamName: t.upstreamName, + description: t.description, + inputSchema: t.inputSchema, + ...(t.outputSchema === undefined ? {} : { outputSchema: t.outputSchema }), + }; + if (!original || canonicalDigest(original) !== canonicalDigest(metadata)) + throw new Error("review cannot change discovered metadata"); + } + const revision = expectedRevision + 1; + const [published] = await tx + .insert(catalog) + .values({ + id: crypto.randomUUID(), + connectionId: id, + kind: "published", + revision, + configRevision: row.configRevision, + authGeneration: row.authGeneration, + credentialIdentity: row.credentialIdentity, + tools, + fingerprint: canonicalDigest(tools), + actorId, + createdAt: now, + }) + .returning(); + await tx + .update(connection) + .set({ catalogRevision: revision, enabled: false, testedAt: now, updatedAt: now }) + .where(eq(connection.id, id)); + return published!; + }); + notifyMcpAuthorityChanged(); + return result; + }, + async getCatalog(id: string, revision: number) { + return ( + ( + await db + .select() + .from(catalog) + .where( + and( + eq(catalog.connectionId, id), + eq(catalog.kind, "published"), + eq(catalog.revision, revision), + ), + ) + )[0] ?? null + ); + }, + async getLatestDraft(id: string) { + const [draft] = await db + .select() + .from(catalog) + .where(and(eq(catalog.connectionId, id), eq(catalog.kind, "draft"))) + .orderBy(desc(catalog.createdAt), desc(catalog.id)) + .limit(1); + return draft ?? null; + }, + }; +} +export type McpRepository = ReturnType; diff --git a/orchestrator/src/mcp/review-state.ts b/orchestrator/src/mcp/review-state.ts new file mode 100644 index 000000000..c6a4c960f --- /dev/null +++ b/orchestrator/src/mcp/review-state.ts @@ -0,0 +1,313 @@ +import { hasPublishedLeaf } from "./session-bindings.ts"; +import { create, toBinary, toJson, type MessageInitShape } from "@bufbuild/protobuf"; +import { GetSessionMcpStateResponseSchema } from "../gen/engram/app/v1/integration_pb.ts"; +import { unknownContext, type McpJournal } from "./journal.ts"; +import { reviewContext, policyReviewCurrent } from "./journal-review.ts"; +import { restoreBindingSnapshot } from "./binding-store.ts"; +import type { SessionBindingResolver } from "./session-resolver.ts"; +import type { SessionPolicyControl } from "./session-policy.ts"; + +class PublicationRequired extends Error {} + +function stateSize(value: MessageInitShape): number { + const message = create(GetSessionMcpStateResponseSchema, value); + return Math.max( + toBinary(GetSessionMcpStateResponseSchema, message).byteLength, + Buffer.byteLength(JSON.stringify(toJson(GetSessionMcpStateResponseSchema, message))), + ); +} + +type SessionRows = Awaited>; +export interface McpReviewProducers { + bindings: Pick; + policy: SessionPolicyControl; +} +export function approvalToProto(row: { + call: SessionRows["rows"][number]["call"]; + approval: NonNullable; +}) { + const { call, approval } = row; + if (!call.requestContext || call.originalArguments === null) + throw new Error("original request context unavailable"); + const tool = call.requestContext.resolvedTool.reviewedTool; + return { + executionId: call.id, + sessionId: call.sessionId, + runId: call.runId, + toolCallId: call.toolCallId, + connectionId: call.connectionId, + upstreamName: call.upstreamName, + requestDigest: call.requestDigest, + argumentHash: call.argumentHash, + expiresAt: approval.expiresAt.toISOString(), + originalArguments: call.originalArguments, + reviewedTool: { + ...tool, + inputSchemaJson: JSON.stringify(tool.inputSchema), + outputSchemaJson: tool.outputSchema ? JSON.stringify(tool.outputSchema) : "", + }, + argumentConstraintDocumentsJson: + call.requestContext.resolvedTool.effectiveGrant.argumentConstraintDocuments.map((document) => + JSON.stringify(document), + ), + expectedReviewContext: reviewContext(call, approval), + }; +} +/** Compose external freshness after the protected database read. No network inside PG locks. */ +export async function sessionMcpState(input: { + journal: McpJournal; + sessionId: string; + actorId: string; + now: Date; + after?: string; + limit?: number; + planToolCallId?: string; + producers?: McpReviewProducers; + /** Dispatch availability is independent of access to persisted history. */ + dispatchEnabled?: boolean; + canCancel?: boolean; + canRebind?: boolean; + afterBindingToolName?: string; + bindingPageSize?: number; + omitBindings?: boolean; + omitExecutions?: boolean; +}) { + const bindingLimit = input.bindingPageSize ?? 16; + if ( + !Number.isSafeInteger(bindingLimit) || + bindingLimit < 1 || + bindingLimit > 16 || + (input.afterBindingToolName?.length ?? 0) > 128 + ) + throw new Error("invalid binding page"); + const data = await input.journal.readSession( + input.sessionId, + input.actorId, + { + after: input.after, + limit: input.limit, + }, + input.now, + ); + const response: MessageInitShape = { + executions: [], + bindingTools: [], + approvals: [], + unknownWriteBlocks: data.unknownWriteBlocks, + resolutionReceipts: [], + allowedActions: [], + availability: "unavailable", + }; + let current = false; + const envelope = data.session.mcpBindingEnvelope; + let snapshot: ReturnType | undefined; + if (envelope) { + response.bindingReceipt = { + bindingHash: envelope.bindingHash, + bindingEpoch: String(envelope.bindingEpoch), + current: false, + staleReasons: ["live_authority_unavailable"], + allowedActions: [], + }; + try { + snapshot = restoreBindingSnapshot(envelope, data.session.integrationConnections ?? []); + if (!hasPublishedLeaf(snapshot)) { + response.bindingReceipt.staleReasons = ["publication_required"]; + throw new PublicationRequired(); + } + if (!input.producers?.policy.readWritePolicy) throw new Error("live producers missing"); + const live = await input.producers.bindings.readCurrent(input.sessionId); + const policy = await input.producers.policy.readWritePolicy({ + protocol: 1, + sessionId: input.sessionId, + planToolCallId: input.planToolCallId, + }); + if ( + !policy.state?.policy?.ref || + policy.state.policy.ref.sessionId !== input.sessionId || + live.bindingHash !== envelope.bindingHash + ) + throw new Error("live authority changed"); + response.writePolicy = policy.state; + response.writePolicy.allowedActions = policy.state.planReview ? ["approve_current_plan"] : []; + current = true; + response.bindingReceipt.current = true; + response.bindingReceipt.staleReasons = []; + response.availability = input.dispatchEnabled === false ? "unavailable" : "available"; + } catch (error) { + if (!(error instanceof PublicationRequired)) + response.bindingReceipt.staleReasons = ["binding_or_policy_unavailable"]; + } + } + if (input.dispatchEnabled !== false && input.canRebind && snapshot && response.bindingReceipt) + response.bindingReceipt.allowedActions = ["rebind"]; + let last = input.after ?? ""; + for (const row of input.omitExecutions ? [] : data.rows) { + const { call, approval, resolution, cancellation } = row; + const actions: string[] = []; + const review = + approval && call.requestContext && call.originalArguments !== null + ? approvalToProto({ call, approval }) + : undefined; + let reason = call.requestContext ? "" : "original_request_unavailable"; + if ( + input.dispatchEnabled !== false && + current && + review && + call.requestContext && + response.writePolicy?.policy && + policyReviewCurrent(call.requestContext, response.writePolicy.policy) && + approval?.decision === "pending" && + call.state === "waiting_approval" && + approval.expiresAt > input.now && + call.deadline > input.now && + response.writePolicy?.policy?.ref?.policyRevision?.toString() === + review.expectedReviewContext.policyRevision && + response.writePolicy.policy.ref.bindingEpoch?.toString() === + review.expectedReviewContext.bindingEpoch + ) { + actions.push("reject"); + if ( + call.requestContext?.requestPolicyContext.acceptedWriteEligibility === "write_eligible" && + response.writePolicy.policy.current && + response.writePolicy.policy.initialized && + response.writePolicy.policy.writeEligibility === "write_eligible" + ) + actions.push("approve"); + } else if (call.state === "waiting_approval") + reason = "review_expired_or_authority_unavailable"; + if (input.canCancel && ["waiting_approval", "authorized", "dispatching"].includes(call.state)) + actions.push("cancel"); + if (call.state === "outcome_unknown" && !resolution) actions.push("resolve_unknown"); + const frozenTool = call.requestContext?.resolvedTool.reviewedTool; + const content = + !review && frozenTool + ? { + originalArguments: call.originalArguments ?? "", + reviewedTool: { + ...frozenTool, + inputSchemaJson: JSON.stringify(frozenTool.inputSchema), + outputSchemaJson: frozenTool.outputSchema + ? JSON.stringify(frozenTool.outputSchema) + : "", + }, + argumentConstraintDocumentsJson: + call.requestContext!.resolvedTool.effectiveGrant.argumentConstraintDocuments.map( + (document) => JSON.stringify(document), + ), + } + : {}; + const entry = { + ...content, + expectedUnknownContext: + call.state === "outcome_unknown" && call.intentId && !resolution + ? unknownContext(call) + : undefined, + cancellationReceipt: cancellation + ? { + ...cancellation, + actorId: cancellation.actorId ?? "", + createdAt: cancellation.createdAt.toISOString(), + } + : undefined, + bindingHash: call.bindingHash, + executionId: call.id, + connectionId: call.connectionId, + state: call.state, + journalRevision: call.version, + resultUnavailable: call.resultUnavailable, + review: actions.includes("approve") ? undefined : review, + allowedActions: actions, + reason, + toolCallId: call.toolCallId, + runId: call.runId, + requestDigest: call.requestDigest, + upstreamName: call.upstreamName, + approvalDecision: approval?.decision ?? "", + }; + const candidate = { + ...response, + executions: [...response.executions!, entry], + approvals: [ + ...response.approvals!, + ...(review && + (actions.includes("approve") || + (input.dispatchEnabled === false && + approval?.decision === "pending" && + call.state === "waiting_approval")) + ? [review] + : []), + ], + resolutionReceipts: [ + ...response.resolutionReceipts!, + ...(resolution + ? [ + { + executionId: resolution.executionId, + actorId: resolution.actorId, + outcome: resolution.outcome, + createdAt: resolution.createdAt.toISOString(), + expectedContext: resolution.expectedContext ?? undefined, + }, + ] + : []), + ], + }; + if (stateSize(candidate) > 2 * 1024 * 1024 - 256) { + if (last === (input.after ?? "")) throw new Error("review exceeds state page limit"); + response.nextExecutionId = last; + response.hasMoreExecutions = true; + break; + } + Object.assign(response, candidate); + last = call.id; + } + if (!input.omitExecutions && data.hasMore) { + response.nextExecutionId = last; + response.hasMoreExecutions = true; + } + if (snapshot && !input.omitBindings) { + const tools = snapshot.immutableBindings + .flatMap((binding) => binding.tools.map((bound) => ({ binding, bound }))) + .sort((a, b) => (a.bound.name < b.bound.name ? -1 : a.bound.name > b.bound.name ? 1 : 0)) + .filter( + (item) => !input.afterBindingToolName || item.bound.name > input.afterBindingToolName, + ); + let lastTool = input.afterBindingToolName ?? "", + added = 0; + for (const { binding, bound } of tools) { + const summary = { + connectionId: binding.connectionId, + connectionAlias: binding.alias, + modelToolName: bound.name, + reviewedTool: { + ...bound.tool, + inputSchemaJson: JSON.stringify(bound.tool.inputSchema), + outputSchemaJson: bound.tool.outputSchema ? JSON.stringify(bound.tool.outputSchema) : "", + }, + argumentConstraintDocumentsJson: bound.grant.argumentConstraintDocuments.map((document) => + JSON.stringify(document), + ), + automationWrite: bound.grant.automationWrite, + unrestricted: bound.grant.unrestricted, + catalogId: binding.catalogId, + catalogRevision: binding.catalogRevision, + configRevision: binding.configRevision, + authGeneration: binding.authGeneration, + }; + const candidate = { ...response, bindingTools: [...response.bindingTools!, summary] }; + if (added === bindingLimit || stateSize(candidate) > 2 * 1024 * 1024 - 256) { + if (input.omitExecutions && added === 0) + throw new Error("binding review exceeds page limit"); + response.hasMoreBindingTools = true; + response.nextBindingToolName = lastTool; + break; + } + Object.assign(response, candidate); + lastTool = bound.name; + added++; + } + } + if (stateSize(response) > 2 * 1024 * 1024) throw new Error("state exceeds limit"); + return response; +} diff --git a/orchestrator/src/mcp/runtime-authority.ts b/orchestrator/src/mcp/runtime-authority.ts new file mode 100644 index 000000000..8190560d6 --- /dev/null +++ b/orchestrator/src/mcp/runtime-authority.ts @@ -0,0 +1,91 @@ +import { McpAuthorityStale } from "./authority-error.ts"; +import { subject } from "@casl/ability"; +import { eq } from "drizzle-orm"; +import { automation, automationRun, automationSession, mcpExecution } from "../db/schema.ts"; +import { abilityFor } from "../authz/ability.ts"; +import { canonicalDigest } from "./hash.ts"; +import { + assertLocalReviewCurrent, + lockHumanSession, + lockSessionChain, + type JournalTx, +} from "./journal-review.ts"; + +/** Local authorization stays in the journal transaction. No network or cached role is accepted. */ +export async function lockRuntimeAuthority( + tx: JournalTx, + call: typeof mcpExecution.$inferSelect, + connectionLock: "share" | "update" = "update", +) { + let authority: Awaited>; + if (call.principalId.startsWith("automation:")) { + const id = call.principalId.slice("automation:".length); + const [linked] = await tx + .select() + .from(automationSession) + .where(eq(automationSession.sessionId, call.sessionId)); + if (!linked || !id) throw new McpAuthorityStale("automation identity unavailable"); + const [owner] = await tx.select().from(automation).where(eq(automation.id, id)).for("share"); + const [run] = await tx + .select() + .from(automationRun) + .where(eq(automationRun.id, linked.runId)) + .for("share"); + const [session] = await tx + .select() + .from(automationSession) + .where(eq(automationSession.sessionId, call.sessionId)) + .for("share"); + if ( + !owner?.enabled || + owner.archivedAt || + !run || + run.automationId !== id || + run.status !== "running" || + run.dryRun || + run.endedAt || + session?.runId !== run.id || + (session.role === "primary" && + (run.sessionId !== call.sessionId || run.taskId !== call.taskId)) + ) + throw new McpAuthorityStale("automation authority unavailable"); + authority = await lockSessionChain(tx, call.sessionId); + if (call.classification === "write") { + const resolved = call.requestContext?.resolvedTool; + const publication = authority.session.mcpBindingEnvelope?.profilePublication; + if ( + call.authorizationKind !== "automation" || + !resolved?.effectiveGrant.automationWrite || + !publication || + !Number.isSafeInteger(publication.revision) || + publication.revision < 1 || + !publication.actorId || + publication.grantsHash !== authority.session.mcpBindingEnvelope?.profileGrantsHash || + canonicalDigest(resolved.effectiveGrant) !== call.automationGrantHash + ) + throw new McpAuthorityStale("explicit published automation write grant required"); + } + } else { + const human = await lockHumanSession(tx, call.sessionId, call.principalId, true); + const ability = abilityFor({ id: human.actor.id, role: human.actor.role ?? "user" }); + const owner = { createdByUserId: human.owner.createdByUserId }; + if ( + !ability.can("update", subject("Task", { ...owner })) || + !ability.can("prompt", subject("Session", { ...owner })) || + call.authorizationKind === "automation" + ) + throw new McpAuthorityStale("current session permission required"); + authority = human; + } + if ( + (authority.session.mcpManagedLifecycle != null && + authority.session.mcpManagedLifecycle.phase !== "started") || + authority.session.profileId !== authority.session.mcpBindingEnvelope?.profileId || + authority.session.integrationPrincipalId !== call.principalId || + authority.session.taskId !== call.taskId || + ["done", "failed", "cancelled"].includes(authority.owner.status) + ) + throw new McpAuthorityStale("current session principal required"); + await assertLocalReviewCurrent(tx, call, authority.session, authority.ancestors, connectionLock); + return authority; +} diff --git a/orchestrator/src/mcp/runtime-config.ts b/orchestrator/src/mcp/runtime-config.ts new file mode 100644 index 000000000..0109b2555 --- /dev/null +++ b/orchestrator/src/mcp/runtime-config.ts @@ -0,0 +1,119 @@ +import { readFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { z } from "zod"; +import type { CredentialControlTls } from "./control-client.ts"; +import type { McpRegistrationMetadata } from "./credential-admin.ts"; +import type { OperatorInfrastructure } from "./operator.ts"; + +const hash = z.string().regex(/^sha256:[0-9a-f]{64}$/); +const tlsSchema = z + .object({ + url: z.string().url(), + caName: z.string().min(1), + certFile: z.string().min(1), + keyFile: z.string().min(1), + serverFingerprint: z.string().regex(/^(?:[0-9A-F]{2}:){31}[0-9A-F]{2}$/), + }) + .strict(); +const configSchema = z + .object({ + version: z.literal(1), + infrastructure: z + .object({ deploymentIdentity: hash, networkIdentity: hash, trustIdentity: hash }) + .strict(), + caFiles: z.record(z.string().min(1).max(128), z.string().min(1)), + control: tlsSchema, + workers: z + .array( + z + .object({ + zoneId: z.string().min(1).max(128), + deploymentId: z.string().min(1).max(256), + enabled: z.boolean().default(false), + tls: tlsSchema, + }) + .strict(), + ) + .min(1) + .max(128), + registrations: z + .array( + z + .object({ + id: z.string().min(1), + clientId: z.string().min(1), + confidential: z.boolean(), + redirectUri: z.string().url(), + issuer: z.string().url(), + resource: z.object({ zoneId: z.string().min(1), endpoint: z.string().url() }).strict(), + scopes: z.array(z.string()), + }) + .strict(), + ) + .max(128) + .default([]), + }) + .strict(); +export interface McpRuntimeConfiguration { + infrastructure: OperatorInfrastructure; + control: CredentialControlTls; + workers: Map; + registrations: McpRegistrationMetadata[]; +} +/** Only operator-owned files supply transport configuration. Published coordinator + * authority must independently match it before a worker is selected. */ +export async function loadMcpRuntimeConfiguration(path: string): Promise { + const raw = await readFile(path); + if (raw.length > 2 * 1024 * 1024) throw Error("MCP runtime configuration exceeds limit"); + const config = configSchema.parse(JSON.parse(raw.toString("utf8"))); + const ca = new Map(); + const caContentDigests: Record = {}; + for (const [name, file] of Object.entries(config.caFiles)) { + const bytes = await readFile(file); + if (!bytes.length || bytes.length > 1024 * 1024) throw Error("MCP trust roots unavailable"); + ca.set(name, bytes.toString("utf8")); + caContentDigests[name] = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + } + if (!ca.size) throw Error("MCP trust roots unavailable"); + async function tls(input: z.infer): Promise { + const root = ca.get(input.caName); + if (!root) throw Error("MCP configured trust root unavailable"); + const [cert, key] = await Promise.all([ + readFile(input.certFile, "utf8"), + readFile(input.keyFile, "utf8"), + ]); + if ( + !cert || + !key || + Buffer.byteLength(cert) > 1024 * 1024 || + Buffer.byteLength(key) > 1024 * 1024 + ) + throw Error("MCP transport identity unavailable"); + return { url: input.url, ca: root, cert, key, serverFingerprint: input.serverFingerprint }; + } + const workers: McpRuntimeConfiguration["workers"] = new Map(); + for (const worker of config.workers) { + if (workers.has(worker.zoneId)) throw Error("MCP worker zone duplicated"); + workers.set(worker.zoneId, { + enabled: worker.enabled, + deploymentId: worker.deploymentId, + tls: await tls(worker.tls), + }); + } + return { + infrastructure: { ...config.infrastructure, caContentDigests }, + control: await tls(config.control), + workers, + registrations: config.registrations, + }; +} + +/** Both global and per-zone admission are opt-in. A file can remain configured on rollback. */ +export function mcpStartupSettings(env: Record) { + const enabled = env.MCP_ENABLED === "true"; + if (env.MCP_ENABLED !== undefined && !["true", "false"].includes(env.MCP_ENABLED)) + throw Error("MCP_ENABLED must be true or false"); + const path = env.MCP_RUNTIME_CONFIG?.trim() || undefined; + if (enabled && !path) throw Error("MCP runtime configuration required"); + return { enabled, path }; +} diff --git a/orchestrator/src/mcp/runtime.ts b/orchestrator/src/mcp/runtime.ts new file mode 100644 index 000000000..c35bc1f70 --- /dev/null +++ b/orchestrator/src/mcp/runtime.ts @@ -0,0 +1,351 @@ +import { create, fromJson } from "@bufbuild/protobuf"; +import { and, eq } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { + automation, + automationRun, + integrationConnection, + sessionListener, + user, +} from "../db/schema.ts"; +import { CredentialStatusSchema } from "../gen/engram/mcp/v1/control_pb.ts"; +import { ManagedPolicyTargetSchema } from "../gen/engram/mcp/v1/session_policy_pb.ts"; +import { + makeCredentialControlClient, + makeSessionPolicyControlClient, + type SessionPolicyControlClient, +} from "./control-client.ts"; +import { makeOperatorReader } from "./operator.ts"; +import { makeMcpOperatorRuntime } from "./operator-runtime.ts"; +import type { McpRuntimeConfiguration } from "./runtime-config.ts"; +import { makeCredentialMetadataStore } from "./credential-store.ts"; +import { makeCredentialAdmin } from "./credential-admin.ts"; +import { makePublishedCredentialAdmin } from "./published-credentials.ts"; +import { makeMcpDiscoveryAdmission } from "./discovery-admission.ts"; +import { makeMcpDiscovery } from "./discovery.ts"; +import { makeMcpSchemaValidator } from "./schema-validator.ts"; +import { makeMcpJournal } from "./journal.ts"; +import { makeMcpRepository } from "./repository.ts"; +import { makeMcpRecovery } from "./recovery.ts"; +import { makeMcpEffectDriver } from "./effect-driver.ts"; +import { makeManagedLifecycleStore } from "./managed-lifecycle-store.ts"; +import { makeManagedSessionRuntime, type ManagedBootBridge } from "./managed-runtime.ts"; +import { makeManagedPolicyAuthority } from "./managed-policy.ts"; +import { decodeManagedState } from "./managed-state.ts"; +import { readLiveBindingAuthority, readStoredBindingSession } from "./binding-store.ts"; +import { checkBindingCurrent, type McpSessionBindingSnapshot } from "./session-bindings.ts"; +import { makeSessionBindingResolver } from "./session-resolver.ts"; +import { makeSessionWritePolicy } from "./session-policy.ts"; +import { makeMcpSessionCompiler } from "./session-compiler.ts"; +import { assertMcpDirectPolicyJson } from "./session-conflicts.ts"; +import { canonicalJson } from "./hash.ts"; +import type { ToolCallCompleter } from "../tools/complete.ts"; +import type { McpServices } from "./contracts.ts"; +import { makeManagedRebind } from "./managed-rebind.ts"; + +/** A single composition is shared by RPC, compilation, execution and recovery. */ +export function makeMcpRuntime(deps: { + configuration?: McpRuntimeConfiguration; + deploymentId: string; + enabled(): boolean; + bridge(control: SessionPolicyControlClient): ManagedBootBridge; + db?: ReturnType; + now?: () => Date; + completer?: ToolCallCompleter; +}) { + const db = deps.db ?? getDb(), + now = deps.now ?? (() => new Date()); + const journal = makeMcpJournal(db), + repository = makeMcpRepository(db); + const store = makeManagedLifecycleStore(db); + if (!deps.configuration) + return { + services: { enabled: false, journal, repository } satisfies McpServices, + recovery: makeMcpRecovery({ + db, + deploymentId: deps.deploymentId, + now, + completer: deps.completer, + }), + store, + }; + const configuration = deps.configuration; + const control = makeSessionPolicyControlClient(configuration.control); + const credentialsControl = makeCredentialControlClient(configuration.control); + const bridge = deps.bridge(control); + const reader = makeOperatorReader(control, configuration.infrastructure); + const operator = makeMcpOperatorRuntime({ + configuration, + read: reader.read, + enabled: deps.enabled, + }); + const readAuthority = async (profileId: string) => + readLiveBindingAuthority(profileId, await operator.policies(), db); + async function state(sessionId: string) { + const row = await store.read(sessionId), + value = row.session.mcpManagedLifecycle; + if (!value) throw Error("MCP managed creation unavailable"); + const result = decodeManagedState( + await bridge.readManagedCreationState({ + protocol: 1, + identity: value.identity, + nativeVersion: value.nativeVersion, + nativeDigest: value.nativeDigest, + }), + value, + ); + return { row, value, result }; + } + const managed = makeManagedPolicyAuthority({ + control, + readPublished: reader.read, + async readPlacement(sessionId) { + const { value, result } = await state(sessionId); + const key = result.boot?.request.key, + prepared = result.boot?.prepared; + if ( + !key || + !prepared || + result.state === "terminal" || + !value.sourceKey || + canonicalJson(key) !== canonicalJson(value.sourceKey) + ) + throw Error("MCP managed placement unavailable"); + return { + target: create(ManagedPolicyTargetSchema, { + sessionId, + sandboxId: key.context.target.sandbox_id, + hostId: key.context.target.host_id, + bindingEpoch: BigInt(key.context.target.binding_epoch), + }), + networkIdentity: prepared.network_identity, + guestIp: prepared.guest_ip, + hostIncarnation: prepared.host_incarnation, + }; + }, + }); + const policy = makeSessionWritePolicy(control); + const resolver = makeSessionBindingResolver({ + async readSession(id) { + const { value, result } = await state(id); + if ( + value.phase === "finalizing" && + result.state === "boot" && + result.boot?.phase === "started" && + value.sourceKey && + canonicalJson(result.boot.request.key) === canonicalJson(value.sourceKey) + ) + throw Error("MCP managed session start is pending"); + if ( + value.phase !== "started" || + result.state !== "boot" || + result.boot?.phase !== "started" || + canonicalJson(result.boot.request.key) !== canonicalJson(value.sourceKey) + ) + return null; + return readStoredBindingSession(id, db); + }, + readAuthority, + async readBindingEpoch(id) { + const current = await policy.readForRequest({ sessionId: id }); + const epoch = current.ref?.bindingEpoch; + if ( + !current.current || + epoch === undefined || + epoch < 1n || + epoch > BigInt(Number.MAX_SAFE_INTEGER) + ) + throw Error("MCP binding epoch unavailable"); + return Number(epoch); + }, + enforceManagedDeny: managed.enforce, + }); + async function reservedSecretReferences() { + const rows = await db + .select({ metadata: integrationConnection.mcpCredential }) + .from(integrationConnection) + .where(eq(integrationConnection.provider, "mcp")); + const refs = new Set(); + for (const row of rows) + for (const raw of [row.metadata.status, row.metadata.clientSecret]) { + if (raw) { + const name = fromJson(CredentialStatusSchema, raw).reference?.name; + if (name) refs.add(name); + } + } + return refs; + } + async function authorize(source: { actorId: string; inputId: string }, id: string) { + const { session, owner } = await store.read(id); + if ( + source.inputId !== id || + session.integrationPrincipalId !== source.actorId || + ["done", "failed", "cancelled"].includes(owner.status) + ) + throw Error("MCP creation principal unavailable"); + if (source.actorId.startsWith("automation:")) { + const automationId = source.actorId.slice("automation:".length); + const rows = await db + .select({ definition: automation, run: automationRun }) + .from(automationRun) + .innerJoin(automation, eq(automation.id, automationRun.automationId)) + .where( + and( + eq(automation.id, automationId), + eq(automationRun.taskId, owner.rootTaskId ?? owner.id), + ), + ) + .limit(2); + if ( + rows.length !== 1 || + !rows[0]!.definition.enabled || + rows[0]!.definition.archivedAt || + rows[0]!.run.status !== "running" || + rows[0]!.run.dryRun || + rows[0]!.run.endedAt + ) + throw Error("MCP automation creation unavailable"); + return; + } + const [principal] = await db.select().from(user).where(eq(user.id, source.actorId)); + if ( + !principal || + principal.banned || + source.inputId !== id || + session.integrationPrincipalId !== source.actorId || + owner.createdByUserId !== source.actorId || + ["done", "failed", "cancelled"].includes(owner.status) + ) + throw Error("MCP creation principal unavailable"); + } + async function validate(snapshot: McpSessionBindingSnapshot, sessionId: string) { + const { session, owner } = await store.read(sessionId); + if (session.mcpManagedLifecycle) + await authorize(session.mcpManagedLifecycle.identity, sessionId); + if ( + ["done", "failed", "cancelled"].includes(owner.status) || + !checkBindingCurrent({ snapshot, liveAuthority: await readAuthority(snapshot.profileId) }) + .current + ) + throw Error("MCP creation authority changed"); + for (const binding of snapshot.immutableBindings) + await operator.assertCurrent(binding.config, binding.operatorPolicy); + if (owner.spawningSessionId) { + const parent = await resolver.readCurrent(owner.spawningSessionId), + ref = snapshot.ancestorAuthorityRefs[0]; + if ( + !ref || + ref.sessionId !== owner.spawningSessionId || + ref.bindingHash !== parent.bindingHash || + ref.bindingEpoch !== parent.bindingEpoch + ) + throw Error("MCP parent authority changed"); + } else if (snapshot.ancestorAuthorityRefs.length) throw Error("MCP parent authority missing"); + if (session.profileId !== snapshot.profileId || !owner.launchPolicy) + throw Error("MCP launch authority unavailable"); + if (owner.launchPolicy.integrationPolicyJson) + assertMcpDirectPolicyJson( + owner.launchPolicy.integrationPolicyJson, + snapshot.managedDenyPolicy.hosts, + await reservedSecretReferences(), + ); + } + const lifecycle = makeManagedSessionRuntime({ + store, + control, + bridge, + managed, + readPublished: reader.read, + validate, + enabled: deps.enabled, + authorize, + async enroll(sessionId) { + await db.insert(sessionListener).values({ sessionId }).onConflictDoNothing(); + }, + }); + const driver = makeMcpEffectDriver({ + db, + deploymentId: operator.deploymentId, + control: credentialsControl, + policy: control, + resolver, + workerTls: operator.workerTls, + enabled: operator.enabled, + now, + }); + const recovery = makeMcpRecovery({ + db, + deploymentId: deps.deploymentId, + driver, + cancellationPolicy: control, + completer: deps.completer, + dispatchEnabled: deps.enabled, + now, + }); + const metadata = makeCredentialMetadataStore(db), + admission = makeMcpDiscoveryAdmission(db); + const rebind = makeManagedRebind({ + db, + store, + lifecycle, + managed, + readOperatorPolicies: operator.policies, + readParentCurrent: resolver.readCurrent, + now, + }); + const services: McpServices = { + get enabled() { + return deps.enabled(); + }, + journal, + repository, + credentials: makePublishedCredentialAdmin({ + store: metadata, + operator, + admin: makeCredentialAdmin({ + store: metadata, + control: credentialsControl, + registrations: configuration.registrations, + now, + }), + }), + discovery: makeMcpDiscovery({ + admission, + credentials: metadata, + control: credentialsControl, + operator, + now, + }), + validateSchemas: makeMcpSchemaValidator({ admission, operator, now }), + validateDestination: operator.validateDestination, + reviewProducers: { bindings: resolver, policy: control }, + runtime: { + cancel: async (id) => { + recovery.abortExecution(id); + }, + prepareRebind: rebind.prepare, + rebind: rebind.commit, + }, + }; + return { + services, + recovery, + store, + control, + lifecycle, + operator, + managed, + resolver, + readAuthority, + validate, + capabilities: { + compile: makeMcpSessionCompiler({ + operator, + readAuthority, + readParentCurrent: resolver.readCurrent, + reservedSecretReferences, + }), + create: lifecycle.create, + }, + }; +} diff --git a/orchestrator/src/mcp/schema-validator.ts b/orchestrator/src/mcp/schema-validator.ts new file mode 100644 index 000000000..6ac86dc9b --- /dev/null +++ b/orchestrator/src/mcp/schema-validator.ts @@ -0,0 +1,74 @@ +import { canonical } from "../../../mcp-worker/src/index.ts"; +import { digest } from "../../../mcp-worker/src/index.ts"; +import type { WorkerIntent, WorkerOperation } from "../../../mcp-worker/src/contracts.ts"; +import type { IntegrationConnectionRow } from "../db/integration-connections.ts"; +import { validateMcpConfig } from "./config.ts"; +import type { makeMcpDiscoveryAdmission } from "./discovery-admission.ts"; +import type { makeMcpOperatorRuntime } from "./operator-runtime.ts"; +import type { McpReviewedTool } from "./types.ts"; +import { makeResourceWorkerClient } from "./worker-client.ts"; + +/** Compilation runs in the isolated worker's disposable schema child. The O process + * sends no credentials and never starts a schema process beside application secrets. */ +export function makeMcpSchemaValidator(deps: { + admission: Pick< + ReturnType, + "reserveSchema" | "beginDispatch" | "assertSchemaAdmitted" | "finish" + >; + operator: ReturnType; + now(): Date; +}) { + return async function validate( + tools: readonly McpReviewedTool[], + connection: IntegrationConnectionRow, + actorId: string, + ) { + if (connection.provider !== "mcp" || !connection.mcp) throw Error("MCP connection unavailable"); + const config = validateMcpConfig(connection.config), + selected = await deps.operator.select(config); + const operation: Extract = { + kind: "validateSchemas", + documents: tools.flatMap((t) => + t.outputSchema ? [t.inputSchema, t.outputSchema] : [t.inputSchema], + ), + }; + const worker = makeResourceWorkerClient({ tls: selected.tls, policy: selected.policy }); + const requestId = crypto.randomUUID(); + let intent: Omit | undefined; + try { + const identity = await worker.identity(); + intent = await deps.admission.reserveSchema( + { + requestId, + actorId, + connectionId: connection.id, + configRevision: connection.mcp.configRevision, + authGeneration: connection.mcp.authGeneration, + credentialIdentity: connection.mcp.credentialIdentity, + deploymentId: selected.deploymentId, + identity, + documents: operation.documents, + }, + deps.now(), + ); + await deps.admission.beginDispatch(requestId, intent.id, deps.now()); + if (canonical(await worker.identity()) !== canonical(identity)) + throw Error("MCP schema worker changed"); + await deps.operator.assertCurrent(config, selected.policy); + await deps.admission.assertSchemaAdmitted( + { identity, intent, documents: operation.documents }, + deps.now(), + ); + const response = await worker.dispatch({ + protocol: 1, + intent: { ...intent, operationDigest: digest(operation) }, + operation, + }); + if (!response.ok || response.result !== true) throw Error("MCP schema validation failed"); + await deps.operator.assertCurrent(config, selected.policy); + } finally { + worker.close(); + if (intent) await deps.admission.finish(requestId, intent.id, deps.now()); + } + }; +} diff --git a/orchestrator/src/mcp/session-bindings.ts b/orchestrator/src/mcp/session-bindings.ts new file mode 100644 index 000000000..a762625c5 --- /dev/null +++ b/orchestrator/src/mcp/session-bindings.ts @@ -0,0 +1,383 @@ +import { isIP } from "node:net"; +import type { ProfileIntegrationGrant } from "../db/schema.ts"; +import type { ToolManifestEntry } from "../tools/manifest.ts"; +import { compileBindings, intersectMcpGrants } from "./bindings.ts"; +import { reviewCatalog } from "./catalog.ts"; +import { validateMcpConfig } from "./config.ts"; +import { canonicalDigest, canonicalJson } from "./hash.ts"; +import { MCP_LIMITS } from "./limits.ts"; +import type { McpBinding, McpConnectionState, McpReviewedTool } from "./types.ts"; + +/** Operator-owned projection of the worker's approved zone policy. Never profile input. */ +export interface McpOperatorPublicationIdentity { + generation: number; + identity: string; + deploymentIdentity: string; + networkIdentity: string; + trustIdentity: string; + caContentDigests: Record; +} +export interface McpOperatorPolicy { + publication: McpOperatorPublicationIdentity; + zoneId: string; + policyRevision: string; + policyDigest: string; + authorities: string[]; + aliases: string[]; +} +export interface McpLiveAuthority { + profilePublication?: import("../db/profiles.ts").McpGrantPublication | null; + profileId: string; + profileGrants: ProfileIntegrationGrant[]; + connections: { + id: string; + catalogId: string; + alias: string; + enabled: boolean; + config: unknown; + state: McpConnectionState; + tools: McpReviewedTool[]; + }[]; + operatorPolicies: McpOperatorPolicy[]; +} +export interface McpSessionBinding extends McpBinding { + catalogId: string; + alias: string; + operatorPolicy: McpOperatorPolicy; +} +export interface McpAncestorAuthorityRef { + sessionId: string; + bindingHash: string; + bindingEpoch: number; +} +export interface ManagedDenyPolicy { + version: 1; + revision: number; + hosts: string[]; + digest: string; +} +export interface McpSessionBindingSnapshot { + version: 1; + bindingEpoch: number; + profileId: string; + profileGrantsHash: string; + profilePublication?: import("../db/profiles.ts").McpGrantPublication | null; + managedAuthorityHash: string; + immutableBindings: McpSessionBinding[]; + ancestorAuthorityRefs: McpAncestorAuthorityRef[]; + managedDenyPolicy: ManagedDenyPolicy; + bindingHash: string; +} +/** Supplied only from the actual parent resolver after its full currentness check. */ +export interface McpParentAuthority { + sessionId: string; + snapshot: McpSessionBindingSnapshot; +} + +function mcpGrants(grants: readonly ProfileIntegrationGrant[]) { + return grants.filter((grant) => grant.mcp !== undefined); +} +function profileHash(grants: readonly ProfileIntegrationGrant[]) { + return canonicalDigest(mcpGrants(grants)); +} +function normalizedPolicy(policy: McpOperatorPolicy): McpOperatorPolicy { + const publication = policy.publication; + if ( + !policy.zoneId || + !policy.policyRevision || + !/^sha256:[0-9a-f]{64}$/.test(policy.policyDigest) || + !publication || + !Number.isSafeInteger(publication.generation) || + publication.generation < 1 || + ![ + publication.identity, + publication.deploymentIdentity, + publication.networkIdentity, + publication.trustIdentity, + ...Object.values(publication.caContentDigests), + ].every((value) => /^sha256:[0-9a-f]{64}$/.test(value)) || + !Object.keys(publication.caContentDigests).length + ) + throw new Error("operator MCP policy identity unavailable"); + const authorities = [...new Set(policy.authorities)].sort(); + const aliases = [...new Set(policy.aliases)].sort(); + for (const authority of authorities) { + const url = new URL(`https://${authority}/`); + if (url.host !== authority || url.username || url.password || url.pathname !== "/") + throw new Error("operator authority must be canonical"); + assertHostname(url.hostname); + } + for (const alias of aliases) assertHostname(alias); + return { + publication: structuredClone(publication), + zoneId: policy.zoneId, + policyRevision: policy.policyRevision, + policyDigest: policy.policyDigest, + authorities, + aliases, + }; +} +function assertHostname(host: string) { + if ( + host.length > 253 || + isIP(host) || + !host.split(".").every((label) => /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label)) + ) + throw new Error("operator hostname must be canonical"); +} +function policyFor(config: ReturnType, live: McpLiveAuthority) { + const matches = live.operatorPolicies.filter((policy) => policy.zoneId === config.zone); + if (matches.length !== 1) throw new Error("operator MCP zone policy unavailable"); + const policy = normalizedPolicy(matches[0]!); + if (!policy.authorities.includes(new URL(config.endpoint).host)) + throw new Error("MCP authority is not operator approved"); + return policy; +} +export function managedAuthorityHash(live: McpLiveAuthority) { + const policies = live.operatorPolicies + .map(normalizedPolicy) + .sort((a, b) => a.zoneId.localeCompare(b.zoneId)); + if (!policies.length || new Set(policies.map((policy) => policy.zoneId)).size !== policies.length) + throw new Error("operator MCP zone policy unavailable"); + return canonicalDigest({ + policies, + connections: live.connections + .map((connection) => { + const config = validateMcpConfig(connection.config); + return { + id: connection.id, + endpoint: config.endpoint, + operatorPolicy: policyFor(config, live), + }; + }) + .sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)), + }); +} +export function hasPublishedLeaf(snapshot: McpSessionBindingSnapshot): boolean { + const value = snapshot.profilePublication; + return ( + !!value && + Number.isSafeInteger(value.revision) && + value.revision >= 1 && + !!value.actorId && + value.grantsHash === snapshot.profileGrantsHash + ); +} +function effectiveGrants(snapshot: McpSessionBindingSnapshot): ProfileIntegrationGrant[] { + return snapshot.immutableBindings.flatMap((binding) => + binding.tools.map((tool) => ({ + connectionId: binding.connectionId, + operation: "invoke", + credentialScope: "org" as const, + resourceConstraints: [], + mcp: structuredClone(tool.grant), + })), + ); +} +function assertSnapshot(snapshot: McpSessionBindingSnapshot) { + const { bindingHash, ...body } = snapshot; + const { digest, ...deny } = snapshot.managedDenyPolicy; + if ( + snapshot.version !== 1 || + canonicalDigest(body) !== bindingHash || + canonicalDigest(deny) !== digest + ) + throw new Error("MCP snapshot integrity failed"); +} + +/** Compile a candidate only. This does not install or acknowledge guest denies. */ +export function compileSessionBindings(input: { + launchPolicy: { profileId: string; integrationGrants: readonly ProfileIntegrationGrant[] }; + liveAuthority: McpLiveAuthority; + bindingEpoch: number; + /** New sessions start at zero. Rebind must supply its monotonic policy revision. */ + denyRevision?: number; + requestedGrants?: readonly ProfileIntegrationGrant[]; + parentAuthority?: McpParentAuthority; +}): McpSessionBindingSnapshot { + const { launchPolicy, liveAuthority: live, parentAuthority: parent } = input; + const denyRevision = input.denyRevision ?? 0; + if (!Number.isSafeInteger(denyRevision) || denyRevision < 0) + throw new Error("invalid managed deny revision"); + if ( + !Number.isSafeInteger(input.bindingEpoch) || + input.bindingEpoch < 0 || + launchPolicy.profileId !== live.profileId + ) + throw new Error("MCP binding context unavailable"); + if (profileHash(launchPolicy.integrationGrants) !== profileHash(live.profileGrants)) + throw new Error("MCP profile changed during compilation"); + const publication = live.profilePublication; + if ( + publication && + (!Number.isSafeInteger(publication.revision) || + publication.revision < 1 || + !publication.actorId || + publication.grantsHash !== profileHash(live.profileGrants)) + ) + throw new Error("MCP grant publication identity unavailable"); + let grants = mcpGrants(launchPolicy.integrationGrants); + if (input.requestedGrants !== undefined) + grants = intersectMcpGrants(grants, mcpGrants(input.requestedGrants)); + if (parent) { + assertSnapshot(parent.snapshot); + // No request means no child tools. A parent snapshot is never an implicit grant. + grants = intersectMcpGrants( + effectiveGrants(parent.snapshot), + input.requestedGrants === undefined ? [] : grants, + ); + } + const sources = [...new Set(grants.map((grant) => grant.connectionId))].sort().map((id) => { + const matches = live.connections.filter((connection) => connection.id === id); + if (matches.length !== 1) throw new Error("MCP connection authority unavailable"); + const source = matches[0]!; + if (!source.catalogId) throw new Error("MCP catalog identity unavailable"); + if (canonicalDigest(reviewCatalog(source.tools)) !== canonicalDigest(source.tools)) + throw new Error("MCP catalog review fingerprint changed"); + return { ...source, grants: grants.filter((grant) => grant.connectionId === id) }; + }); + const immutableBindings = compileBindings(sources).map((binding) => { + const source = sources.find((source) => source.id === binding.connectionId)!; + const extended = { + ...binding, + catalogId: source.catalogId, + alias: source.alias, + operatorPolicy: policyFor(binding.config, live), + }; + const { hash: _hash, ...body } = extended; + return { ...body, hash: canonicalDigest(body) }; + }); + // Keep managed destinations even when the request has no tool grants. + const hosts = new Set(parent?.snapshot.managedDenyPolicy.hosts ?? []); + // The published set includes unused zones and authorities before a connection exists. + for (const raw of live.operatorPolicies) { + const policy = normalizedPolicy(raw); + for (const authority of policy.authorities) + hosts.add(new URL(`https://${authority}/`).hostname); + for (const alias of policy.aliases) hosts.add(alias); + } + for (const connection of live.connections) { + const config = validateMcpConfig(connection.config); + const policy = policyFor(config, live); + hosts.add(new URL(config.endpoint).hostname); + for (const alias of policy.aliases) hosts.add(alias); + } + const deny = { version: 1 as const, revision: denyRevision, hosts: [...hosts].sort() }; + const ancestorAuthorityRefs = parent + ? [ + { + sessionId: parent.sessionId, + bindingHash: parent.snapshot.bindingHash, + bindingEpoch: parent.snapshot.bindingEpoch, + }, + ...parent.snapshot.ancestorAuthorityRefs, + ] + : []; + if ( + ancestorAuthorityRefs.length >= 32 || + new Set(ancestorAuthorityRefs.map((ref) => ref.sessionId)).size !== ancestorAuthorityRefs.length + ) + throw new Error("invalid MCP ancestry"); + const body = { + version: 1 as const, + bindingEpoch: input.bindingEpoch, + profileId: live.profileId, + profileGrantsHash: profileHash(live.profileGrants), + profilePublication: live.profilePublication ?? null, + immutableBindings, + ancestorAuthorityRefs, + managedAuthorityHash: managedAuthorityHash(live), + managedDenyPolicy: { ...deny, digest: canonicalDigest(deny) }, + }; + const snapshot = structuredClone({ ...body, bindingHash: canonicalDigest(body) }); + compileMcpManifest(snapshot); + return snapshot; +} + +/** Exact identity checks. Token bytes/expiry are deliberately absent from this authority. */ +export function checkBindingCurrent(input: { + snapshot: McpSessionBindingSnapshot; + liveAuthority: McpLiveAuthority; +}): { current: true } | { current: false; staleReasons: string[] } { + const staleReasons: string[] = []; + const { snapshot, liveAuthority: live } = input; + try { + assertSnapshot(snapshot); + } catch { + staleReasons.push("snapshot_integrity"); + } + if ( + snapshot.profileId !== live.profileId || + snapshot.profileGrantsHash !== profileHash(live.profileGrants) || + canonicalDigest(snapshot.profilePublication ?? null) !== + canonicalDigest(live.profilePublication ?? null) + ) + staleReasons.push("profile_grants"); + try { + if (snapshot.managedAuthorityHash !== managedAuthorityHash(live)) + staleReasons.push("managed_destinations"); + } catch { + staleReasons.push("managed_destinations_unavailable"); + } + for (const binding of snapshot.immutableBindings) { + const matches = live.connections.filter((connection) => connection.id === binding.connectionId); + const connection = matches[0]; + if (matches.length !== 1 || !connection || !connection.enabled || connection.state.revoked) { + staleReasons.push("connection_unavailable"); + continue; + } + for (const field of [ + "configRevision", + "authGeneration", + "credentialIdentity", + "catalogRevision", + ] as const) + if (binding[field] !== connection.state[field]) staleReasons.push(field); + if (binding.catalogId !== connection.catalogId) staleReasons.push("catalog_identity"); + if ( + binding.alias !== connection.alias || + canonicalDigest(binding.config) !== canonicalDigest(connection.config) + ) + staleReasons.push("connection_identity"); + try { + if ( + canonicalDigest(binding.operatorPolicy) !== canonicalDigest(policyFor(binding.config, live)) + ) + staleReasons.push("operator_policy"); + const reviewed = reviewCatalog(connection.tools); + if (canonicalDigest(reviewed) !== canonicalDigest(connection.tools)) + staleReasons.push("catalog_fingerprint"); + for (const bound of binding.tools) { + const tool = reviewed.find((tool) => tool.upstreamName === bound.tool.upstreamName); + if (!tool?.enabled || canonicalDigest(tool) !== canonicalDigest(bound.tool)) + staleReasons.push("reviewed_tool"); + } + } catch { + staleReasons.push("operator_or_catalog_unavailable"); + } + } + return staleReasons.length + ? { current: false, staleReasons: [...new Set(staleReasons)] } + : { current: true }; +} + +/** Raw JSON Schema stays in its original dialect and resource scope. */ +export function compileMcpManifest(snapshot: McpSessionBindingSnapshot): ToolManifestEntry[] { + assertSnapshot(snapshot); + const manifest = snapshot.immutableBindings.flatMap((binding) => + binding.tools.map((bound) => ({ + name: bound.name, + description: bound.tool.description, + inputSchema: structuredClone(bound.tool.inputSchema), + execution: bound.tool.classification === "write" ? ("deferred" as const) : ("sync" as const), + nativeBindings: {}, + resultFormat: "mcp" as const, + })), + ); + if ( + manifest.length > MCP_LIMITS.toolsPerSession || + Buffer.byteLength(canonicalJson(manifest)) > MCP_LIMITS.manifestBytes + ) + throw new Error("MCP manifest exceeds session limits"); + return manifest; +} diff --git a/orchestrator/src/mcp/session-capabilities.ts b/orchestrator/src/mcp/session-capabilities.ts new file mode 100644 index 000000000..6d23c54ec --- /dev/null +++ b/orchestrator/src/mcp/session-capabilities.ts @@ -0,0 +1,26 @@ +import type { IntegrationPolicyJson } from "../connectors/registry.ts"; +import type { ProfileIntegrationGrant } from "../db/schema.ts"; +import type { SessionCreateInput } from "../rpc/task-create.ts"; +import type { TrustedCreationInput } from "./session-create.ts"; + +export interface McpSessionCapabilities { + compile( + input: SessionCreateInput, + context: { + profileId: string; + profileGrants: readonly ProfileIntegrationGrant[]; + grants: readonly ProfileIntegrationGrant[]; + directPolicy: IntegrationPolicyJson; + spawningSessionId?: string; + }, + ): Promise; + create(input: SessionCreateInput, source: TrustedCreationInput): Promise<{ sessionId: string }>; +} +let installed: McpSessionCapabilities | undefined; +export function installMcpSessionCapabilities(value: McpSessionCapabilities) { + if (installed) throw Error("MCP session runtime already installed"); + installed = value; +} +export function mcpSessionCapabilities() { + return installed; +} diff --git a/orchestrator/src/mcp/session-child.ts b/orchestrator/src/mcp/session-child.ts new file mode 100644 index 000000000..679daba15 --- /dev/null +++ b/orchestrator/src/mcp/session-child.ts @@ -0,0 +1,45 @@ +import { and, eq } from "drizzle-orm"; +import { getDb } from "../db/client.ts"; +import { taskSession, type ProfileIntegrationGrant } from "../db/schema.ts"; +import { compileSessionBindings, type McpSessionBindingSnapshot } from "./session-bindings.ts"; +import { mcpSessionCapabilities } from "./session-capabilities.ts"; + +/** The caller supplies the actual tool-call session ID, never a task launch snapshot. */ +export async function compileChildSessionBindings( + input: Omit[0], "parentAuthority"> & { + spawningSessionId: string; + readParentCurrent(sessionId: string): Promise; + }, +) { + const snapshot = await input.readParentCurrent(input.spawningSessionId); + return compileSessionBindings({ + ...input, + requestedGrants: input.requestedGrants ?? [], + parentAuthority: { sessionId: input.spawningSessionId, snapshot }, + }); +} +/** A managed parent requires the installed trusted creation path even with zero child tools. */ +export async function assertChildMcpCreationBlocked( + taskId: string, + spawningSessionId: string, + launchGrants: readonly ProfileIntegrationGrant[], + db = getDb(), +) { + const [parent] = await db + .select({ + grants: taskSession.integrationGrants, + connections: taskSession.integrationConnections, + envelope: taskSession.mcpBindingEnvelope, + }) + .from(taskSession) + .where(and(eq(taskSession.taskId, taskId), eq(taskSession.sessionId, spawningSessionId))); + if (!parent) throw new Error("spawning session authority unavailable"); + const managed = Boolean( + parent.envelope || + [...(parent.grants ?? []), ...launchGrants].some((grant) => grant.mcp) || + parent.connections?.some((connection) => connection.provider === "mcp"), + ); + if (managed && !mcpSessionCapabilities()) + throw new Error("external MCP session activation is not configured"); + return managed; +} diff --git a/orchestrator/src/mcp/session-compiler.ts b/orchestrator/src/mcp/session-compiler.ts new file mode 100644 index 000000000..120cc0651 --- /dev/null +++ b/orchestrator/src/mcp/session-compiler.ts @@ -0,0 +1,50 @@ +import type { McpSessionCapabilities } from "./session-capabilities.ts"; +import type { makeMcpOperatorRuntime } from "./operator-runtime.ts"; +import type { McpLiveAuthority, McpSessionBindingSnapshot } from "./session-bindings.ts"; +import { compileBoundSessionCandidate } from "./session-create.ts"; +import { assertMcpDirectConflicts } from "./session-conflicts.ts"; +import { restoreBindingSnapshot } from "./binding-store.ts"; +import { appendMcpManifest } from "../tools/manifest.ts"; +import { frozenNativeManifest } from "./managed-create-input.ts"; + +export function makeMcpSessionCompiler(deps: { + operator: ReturnType; + readAuthority(profileId: string): Promise; + readParentCurrent(sessionId: string): Promise; + reservedSecretReferences(): Promise>; +}): McpSessionCapabilities["compile"] { + return async (input, context) => { + const live = await deps.readAuthority(context.profileId); + const parent = context.spawningSessionId + ? { + sessionId: context.spawningSessionId, + snapshot: await deps.readParentCurrent(context.spawningSessionId), + } + : undefined; + const candidate = compileBoundSessionCandidate(input, { + launchPolicy: { + profileId: context.profileId, + integrationGrants: parent ? live.profileGrants : context.profileGrants, + }, + liveAuthority: live, + requestedGrants: context.grants, + bindingEpoch: 0, + ...(parent ? { parentAuthority: parent } : {}), + }); + const snapshot = restoreBindingSnapshot( + candidate.mcpBindingEnvelope!, + candidate.integrationConnections ?? [], + ); + for (const binding of snapshot.immutableBindings) + await deps.operator.assertCurrent(binding.config, binding.operatorPolicy); + assertMcpDirectConflicts( + context.directPolicy, + snapshot.managedDenyPolicy.hosts, + await deps.reservedSecretReferences(), + ); + const native = frozenNativeManifest(candidate.harnessEnv?.ENGRAM_TOOLS); + // Validate the combined limits now, but keep only native descriptors in the held request. + appendMcpManifest(native.text, snapshot); + return { ...candidate, harnessEnv: { ...candidate.harnessEnv, ENGRAM_TOOLS: native.text } }; + }; +} diff --git a/orchestrator/src/mcp/session-conflicts.ts b/orchestrator/src/mcp/session-conflicts.ts new file mode 100644 index 000000000..06b520e92 --- /dev/null +++ b/orchestrator/src/mcp/session-conflicts.ts @@ -0,0 +1,73 @@ +import type { IntegrationPolicyJson } from "../connectors/registry.ts"; +import { z } from "zod"; + +function hostname(value: string) { + const parsed = new URL(`https://${value}/`); + if (parsed.username || parsed.password || parsed.pathname !== "/" || parsed.search || parsed.hash) + throw Error("MCP direct authority invalid"); + return parsed.hostname.toLowerCase().replace(/\.$/, ""); +} +function matches(pattern: string, host: string) { + const parts = pattern.split("*").map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")); + return new RegExp(`^${parts.join(".*")}$`, "i").test(host); +} +/** Called after provider, native-harness and model-router policy contributions. */ +export function assertMcpDirectConflicts( + policy: IntegrationPolicyJson, + managedHosts: readonly string[], + reservedReferences: ReadonlySet, +) { + const hosts = new Set(managedHosts); + const direct = [ + ...policy.network.allow_hosts, + ...policy.injects.flatMap((i) => i.hosts), + ...policy.secrets.flatMap((s) => s.allow_hosts), + ]; + const patterns = [ + ...policy.network.allow_host_patterns, + ...policy.secrets.flatMap((s) => s.allow_host_patterns), + ]; + if ( + direct.some((value) => hosts.has(hostname(value))) || + patterns.some((pattern) => managedHosts.some((host) => matches(pattern, host))) + ) + throw Error("direct integration authority conflicts with managed MCP"); + if ( + policy.secrets.some((secret) => reservedReferences.has(secret.secret_ref)) || + policy.injects.some((inject) => reservedReferences.has(inject.secret_ref)) + ) + throw Error("managed MCP credential cannot be injected into a session"); +} +/** Validate the final frozen child policy after all launch-policy substitutions. */ +export function assertMcpDirectPolicyJson( + raw: string, + hosts: readonly string[], + reservedReferences: ReadonlySet = new Set(), +) { + const strings = z.array(z.string()); + const policy = z + .object({ + network: z.object({ allow_hosts: strings, allow_host_patterns: strings }), + injects: z.array(z.object({ hosts: strings, secret_ref: z.string() })), + secrets: z.array( + z.object({ secret_ref: z.string(), allow_hosts: strings, allow_host_patterns: strings }), + ), + }) + .parse(JSON.parse(raw)); + const direct = [ + ...policy.network.allow_hosts, + ...policy.injects.flatMap((i) => i.hosts), + ...policy.secrets.flatMap((s) => s.allow_hosts), + ]; + const patterns = [ + ...policy.network.allow_host_patterns, + ...policy.secrets.flatMap((s) => s.allow_host_patterns), + ]; + if ( + direct.some((value) => hosts.includes(hostname(value))) || + patterns.some((pattern) => hosts.some((host) => matches(pattern, host))) || + policy.injects.some((i) => reservedReferences.has(i.secret_ref)) || + policy.secrets.some((s) => reservedReferences.has(s.secret_ref)) + ) + throw Error("direct integration authority conflicts with managed MCP"); +} diff --git a/orchestrator/src/mcp/session-create.ts b/orchestrator/src/mcp/session-create.ts new file mode 100644 index 000000000..7cab65067 --- /dev/null +++ b/orchestrator/src/mcp/session-create.ts @@ -0,0 +1,85 @@ +import { mcpSessionCapabilities } from "./session-capabilities.ts"; +import type { IntegrationConnectionSnapshot } from "../db/schema.ts"; +import type { SessionCreateInput } from "../rpc/task-create.ts"; +import { bindingSnapshotColumns } from "./binding-store.ts"; +import { compileSessionBindings } from "./session-bindings.ts"; +import type { McpSessionCapabilities } from "./session-capabilities.ts"; + +/** Candidate preparation is separate from activation. It cannot put tools into guest env. */ +export function compileBoundSessionCandidate( + input: SessionCreateInput, + authority: Parameters[0], +): SessionCreateInput { + const snapshot = compileSessionBindings(authority); + const connections: IntegrationConnectionSnapshot[] = authority.liveAuthority.connections.map( + (connection) => ({ + id: connection.id, + alias: connection.alias, + displayName: connection.alias, + provider: "mcp", + config: structuredClone( + snapshot.immutableBindings.find((binding) => binding.connectionId === connection.id) + ?.config ?? {}, + ), + }), + ); + const direct = (input.integrationConnections ?? []).filter( + (connection) => connection.provider !== "mcp", + ); + const grants = snapshot.immutableBindings.flatMap((binding) => + binding.tools.map((bound) => ({ + connectionId: binding.connectionId, + operation: "invoke", + credentialScope: "org" as const, + resourceConstraints: [], + mcp: structuredClone(bound.grant), + })), + ); + return { + ...input, + ...bindingSnapshotColumns(snapshot, [...direct, ...connections]), + integrationGrants: [ + ...(input.integrationGrants ?? []).filter((grant) => !grant.mcp), + ...grants, + ], + }; +} +export interface TrustedCreationInput { + actorId: string; + inputId: string; +} +/** Retain the actual created session so callers can compensate a rejected generic response. */ +export class McpSessionCreationMismatchError extends Error { + constructor(readonly createdSessionId: string) { + super("coordinator returned a different reserved session ID"); + this.name = "McpSessionCreationMismatchError"; + } +} + +/** The normal task path cannot route MCP authority through legacy/generic creation. */ +export async function createReservedTaskSession( + deps: { + sessions: { createSession(input: SessionCreateInput): Promise<{ sessionId: string }> }; + boundSessionCreator?: Pick; + }, + input: SessionCreateInput, + source: TrustedCreationInput, +) { + const hasMcp = + input.mcpBindingEnvelope || + input.integrationGrants?.some((grant) => grant.mcp) || + input.integrationConnections?.some((connection) => connection.provider === "mcp"); + if (!hasMcp) { + const created = await deps.sessions.createSession(input); + if (input.requestedSessionId && created.sessionId !== input.requestedSessionId) + throw new McpSessionCreationMismatchError(created.sessionId); + return created; + } + const creator = deps.boundSessionCreator ?? mcpSessionCapabilities(); + if (!input.mcpBindingEnvelope || !creator) + throw new Error("external MCP session activation is not configured"); + const created = await creator.create(input, source); + if (created.sessionId !== input.requestedSessionId) + throw new McpSessionCreationMismatchError(created.sessionId); + return created; +} diff --git a/orchestrator/src/mcp/session-policy.ts b/orchestrator/src/mcp/session-policy.ts new file mode 100644 index 000000000..c5817534f --- /dev/null +++ b/orchestrator/src/mcp/session-policy.ts @@ -0,0 +1,189 @@ +import { Code, ConnectError } from "@connectrpc/connect"; +import { eq } from "drizzle-orm"; +import { create, type MessageInitShape } from "@bufbuild/protobuf"; +import { getDb } from "../db/client.ts"; +import { task, taskSession, user } from "../db/schema.ts"; +import { getSessionFromHeaders } from "../auth/session.ts"; +import { isServiceAccountEmail } from "../rpc/api-key.ts"; +import type { GetSession } from "../rpc/require.ts"; +import type { UnaryOverride } from "../rpc/passthrough.ts"; +import { + CompleteToolCallResponseSchema, + GetMcpWritePolicyResponseSchema, + type CompleteToolCallRequest, + type GetMcpWritePolicyRequest, +} from "../gen/engram/app/v1/session_pb.ts"; +import type { + AcceptPlanExitRequestSchema, + ReadWritePolicyRequestSchema, + AcceptPlanExitResponseSchema, + ReadWritePolicyResponseSchema, +} from "../gen/engram/mcp/v1/session_policy_pb.ts"; + +export interface SessionPolicyControl { + acceptPlanExit?( + request: MessageInitShape, + ): Promise>; + readWritePolicy?( + request: MessageInitShape, + ): Promise>; +} +/** Fresh principal and owner lookup. Cached session roles and owner projections do not authorize. */ +export async function authorizePolicyActor( + sessionId: string, + actorId: string, +): Promise<{ id: string; isAdmin: boolean }> { + const [row] = await getDb() + .select({ + ownerId: task.createdByUserId, + role: user.role, + email: user.email, + banned: user.banned, + }) + .from(taskSession) + .innerJoin(task, eq(taskSession.taskId, task.id)) + .innerJoin(user, eq(user.id, actorId)) + .where(eq(taskSession.sessionId, sessionId)) + .limit(1); + if ( + !row || + row.banned || + isServiceAccountEmail(row.email) || + (row.role !== "admin" && row.ownerId !== actorId) + ) + throw new ConnectError("human session owner or administrator required", Code.PermissionDenied); + return { id: actorId, isAdmin: row.role === "admin" }; +} +/** Overrides run after ordinary permission and tool-output guards, never as preflights. */ +export function makeSessionPolicyOverrides(deps?: { + control?: SessionPolicyControl; + getSession?: GetSession; + authorize?: typeof authorizePolicyActor; +}): Record { + const getSession = deps?.getSession ?? getSessionFromHeaders; + const authorize = deps?.authorize ?? authorizePolicyActor; + return { + async "SessionService.CompleteToolCall"(request, ctx) { + const req = request as CompleteToolCallRequest; + if (!req.expectedMcpPlanReview) return undefined; + let result: unknown; + try { + result = JSON.parse(req.resultJson); + } catch { + throw new ConnectError("invalid plan result", Code.InvalidArgument); + } + if ( + !result || + typeof result !== "object" || + !("decision" in result) || + result.decision !== "approve" + ) { + // Generic rejection uses the existing delivery path and cannot change policy. + req.expectedMcpPlanReview = undefined; + return undefined; + } + const review = req.expectedMcpPlanReview; + if (review.policyRef?.sessionId !== req.sessionId || review.toolCallId !== req.toolCallId) + throw new ConnectError("plan review changed; reload the plan", Code.FailedPrecondition); + const session = await getSession(ctx.requestHeader); + if (!session) throw new ConnectError("unauthenticated", Code.Unauthenticated); + const actor = await authorize(req.sessionId, session.user.id); + const accept = deps?.control?.acceptPlanExit; + if (!accept) + throw new ConnectError("session policy control unavailable", Code.FailedPrecondition); + try { + await accept({ + protocol: 1, + expected: review, + actorId: actor.id, + resultJson: req.resultJson, + }); + } catch { + throw new ConnectError( + "session policy acceptance unavailable; reload the plan", + Code.FailedPrecondition, + ); + } + return { + response: create(CompleteToolCallResponseSchema, { + sessionId: req.sessionId, + note: "tool result queued", + }), + }; + }, + async "SessionService.GetMcpWritePolicy"(request, ctx) { + const req = request as GetMcpWritePolicyRequest; + const session = await getSession(ctx.requestHeader); + if (!session) throw new ConnectError("unauthenticated", Code.Unauthenticated); + await authorize(req.sessionId, session.user.id); + const read = deps?.control?.readWritePolicy; + if (!read) + throw new ConnectError("session policy control unavailable", Code.FailedPrecondition); + try { + const result = await read({ + protocol: 1, + sessionId: req.sessionId, + planToolCallId: req.planToolCallId, + }); + if (!result.state) throw new Error("policy state missing"); + const response = create(GetMcpWritePolicyResponseSchema, result.state); + response.allowedActions = response.planReview + ? ["approve_current_plan"] + : !response.policy?.initialized + ? ["start_supported_session"] + : response.policy.writeEligibility !== "write_eligible" && + response.policy.ancestorPolicyRefs.length + ? ["inspect_ancestor_policy"] + : []; + return { response }; + } catch { + throw new ConnectError("session policy read unavailable", Code.FailedPrecondition); + } + }, + }; +} + +export interface RequestPolicyContext { + sessionId: string; + runId: string; + toolCallId: string; + acceptedPolicyRef: import("../gen/engram/app/v1/session_pb.ts").McpWritePolicyRef; + acceptedWriteEligibility: string; + ancestorPolicyRefs: import("../gen/engram/app/v1/session_pb.ts").McpWritePolicyRef[]; +} +/** The journal persists the returned context before any authorization or dispatch. */ +export function makeSessionWritePolicy( + control: import("./control-client.ts").SessionPolicyControlClient, +) { + return { + async readForRequest(input: { sessionId: string }) { + const response = await control.readWritePolicy({ protocol: 1, sessionId: input.sessionId }); + if (!response.state?.policy?.ref || response.state.policy.ref.sessionId !== input.sessionId) + throw new ConnectError("session policy unavailable", Code.FailedPrecondition); + return response.state.policy; + }, + async validateExecutionIdentity( + context: RequestPolicyContext, + expected: { modelToolName: string; originalArguments: string }, + ) { + const result = await control.validateExecutionIdentity({ + protocol: 1, + ...context, + ...expected, + }); + if (result.protocol !== 1 || !["current", "stale", "unknown"].includes(result.status)) + throw new ConnectError("session identity fence unavailable", Code.FailedPrecondition); + return result.status; + }, + /** After the durable barrier, immediately before dispatch. Never cache a success. */ + async validateDispatchFence( + context: RequestPolicyContext, + expected: { modelToolName: string; originalArguments: string }, + ) { + const result = await control.validateDispatchFence({ protocol: 1, ...context, ...expected }); + if (result.protocol !== 1 || !["current", "stale", "unknown"].includes(result.status)) + throw new ConnectError("session policy fence unavailable", Code.FailedPrecondition); + return result.status; + }, + }; +} diff --git a/orchestrator/src/mcp/session-resolver.ts b/orchestrator/src/mcp/session-resolver.ts new file mode 100644 index 000000000..559ffa1ac --- /dev/null +++ b/orchestrator/src/mcp/session-resolver.ts @@ -0,0 +1,144 @@ +import { McpAuthorityStale } from "./authority-error.ts"; +import type { McpBoundTool } from "./types.ts"; +import { + checkBindingCurrent, + type McpLiveAuthority, + type McpSessionBindingSnapshot, + type McpSessionBinding, + type McpAncestorAuthorityRef, + type ManagedDenyPolicy, +} from "./session-bindings.ts"; + +/** Task 6 must obtain this from the trusted host after durable apply and connection closure. + * A stored receipt, echo, or registry replacement is not an enforcement implementation. */ +export interface ManagedDenyAck { + sessionId: string; + bindingEpoch: number; + bindingHash: string; + denyRevision: number; + denyDigest: string; +} +export function expectedManagedDenyAck( + sessionId: string, + snapshot: McpSessionBindingSnapshot, +): ManagedDenyAck { + return { + sessionId, + bindingEpoch: snapshot.bindingEpoch, + bindingHash: snapshot.bindingHash, + denyRevision: snapshot.managedDenyPolicy.revision, + denyDigest: snapshot.managedDenyPolicy.digest, + }; +} +export function assertManagedDenyAck(expected: ManagedDenyAck, actual: ManagedDenyAck) { + if ( + Object.keys(expected).some( + (key) => expected[key as keyof ManagedDenyAck] !== actual[key as keyof ManagedDenyAck], + ) + ) + throw new McpAuthorityStale("MCP managed deny enforcement changed"); +} +export interface BoundSessionAuthority { + snapshot: McpSessionBindingSnapshot; + parentSessionId: string | null; + active: boolean; +} +export interface SessionBindingResolverDeps { + readSession(sessionId: string): Promise; + readAuthority(profileId: string): Promise; + /** Fresh coordinator policy reference. Missing or terminal identity must reject; write eligibility is separate. */ + readBindingEpoch(sessionId: string): Promise; + /** Mandatory trusted host read/apply fence. Do not implement with a stored JSON receipt. */ + enforceManagedDeny?(expected: ManagedDenyAck, policy: ManagedDenyPolicy): Promise; +} +export interface ResolvedBoundTool { + profilePublication?: import("../db/profiles.ts").McpGrantPublication | null; + bindingHash: string; + binding: McpSessionBinding; + modelToolName: string; + reviewedTool: McpBoundTool["tool"]; + effectiveGrant: McpBoundTool["grant"]; + ancestorAuthorityRefs: McpAncestorAuthorityRef[]; + execution: "sync" | "deferred"; +} +/** Pure exact-name lookup after the caller has verified this snapshot's live authority. */ +export function resolveSnapshotTool( + snapshot: McpSessionBindingSnapshot, + modelToolName: string, +): ResolvedBoundTool | undefined { + for (const binding of snapshot.immutableBindings) { + const bound = binding.tools.find((tool) => tool.name === modelToolName); + if (bound) + return structuredClone({ + bindingHash: snapshot.bindingHash, + profilePublication: snapshot.profilePublication ?? null, + binding, + modelToolName: bound.name, + reviewedTool: bound.tool, + effectiveGrant: bound.grant, + ancestorAuthorityRefs: snapshot.ancestorAuthorityRefs, + execution: bound.tool.classification === "write" ? "deferred" : "sync", + }); + } +} +/** Session-local resolution never changes the process-wide registry or accepts a destination. */ +export function makeSessionBindingResolver(deps: SessionBindingResolverDeps) { + async function readCurrent(sessionId: string): Promise { + const leaf = await deps.readSession(sessionId); + if (!leaf?.active) throw new McpAuthorityStale("MCP session unavailable"); + const refs = leaf.snapshot.ancestorAuthorityRefs; + if (refs.length >= 32) throw new McpAuthorityStale("MCP ancestry unavailable"); + const seen = new Set(); + let id: string | null = sessionId; + for (let index = 0; index <= refs.length; index++) { + if (!id || seen.has(id)) throw new McpAuthorityStale("MCP ancestry unavailable"); + seen.add(id); + const row: BoundSessionAuthority | null = index === 0 ? leaf : await deps.readSession(id); + if (!row?.active) throw new McpAuthorityStale("MCP ancestor unavailable"); + const snapshot = row.snapshot; + const expected = index === 0 ? undefined : refs[index - 1]; + if ( + expected && + (expected.sessionId !== id || + expected.bindingHash !== snapshot.bindingHash || + expected.bindingEpoch !== snapshot.bindingEpoch) + ) + throw new McpAuthorityStale("MCP ancestor binding changed"); + const suffix = refs.slice(index); + if (JSON.stringify(snapshot.ancestorAuthorityRefs) !== JSON.stringify(suffix)) + throw new McpAuthorityStale("MCP ancestry changed"); + const live = await deps.readAuthority(snapshot.profileId); + if ( + !checkBindingCurrent({ snapshot, liveAuthority: live }).current || + (await deps.readBindingEpoch(id)) !== snapshot.bindingEpoch + ) + throw new McpAuthorityStale("MCP authority changed"); + id = row.parentSessionId; + } + if (id !== null) throw new McpAuthorityStale("MCP ancestry truncated"); + if (!deps.enforceManagedDeny) throw new Error("MCP managed deny enforcement unavailable"); + const expected = expectedManagedDenyAck(sessionId, leaf.snapshot); + assertManagedDenyAck( + expected, + await deps.enforceManagedDeny(expected, leaf.snapshot.managedDenyPolicy), + ); + return leaf.snapshot; + } + return { + readCurrent, + async resolveBoundTool(input: { + sessionId: string; + modelToolName: string; + }): Promise { + try { + const snapshot = await readCurrent(input.sessionId); + return resolveSnapshotTool(snapshot, input.modelToolName); + } catch { + /* Missing authority, enforcement and malformed old snapshots all deny. */ + } + return undefined; + }, + }; +} + +export type SessionBindingResolver = ReturnType; diff --git a/orchestrator/src/mcp/types.ts b/orchestrator/src/mcp/types.ts new file mode 100644 index 000000000..8c981bcb3 --- /dev/null +++ b/orchestrator/src/mcp/types.ts @@ -0,0 +1,82 @@ +import type { ProfileIntegrationGrant } from "../db/schema.ts"; +import type { McpConfig } from "./config.ts"; +import { assertSchemaShape } from "./catalog.ts"; +import { canonicalJson } from "./hash.ts"; +import { MCP_LIMITS } from "./limits.ts"; + +export interface McpToolGrant { + upstreamName: string; + automationWrite: boolean; + /** Conjunction of independent documents. Compile each in its own dialect/resource scope. */ + argumentConstraintDocuments: Record[]; + /** Required affirmative choice when constraints are empty. */ + unrestricted: boolean; +} +export interface McpConnectionState { + configRevision: number; + authGeneration: number; + credentialIdentity: string; + catalogRevision: number; + revoked: boolean; +} +export interface McpDiscoveredTool { + upstreamName: string; + description: string; + inputSchema: Record; + outputSchema?: Record; +} +export interface McpReviewedTool extends McpDiscoveredTool { + classification: "read" | "write"; + enabled: boolean; + fingerprint: string; +} +export interface McpBoundTool { + name: string; + tool: McpReviewedTool; + grant: McpToolGrant; +} +export interface McpBinding extends McpConnectionState { + connectionId: string; + config: McpConfig; + tools: McpBoundTool[]; + policyHash: string; + hash: string; +} +export type McpExecutionState = + | "waiting_approval" + | "authorized" + | "dispatching" + | "succeeded" + | "failed" + | "canceled" + | "outcome_unknown"; + +export function validateMcpGrant(grant: ProfileIntegrationGrant): void { + if ( + grant.operation !== "invoke" || + grant.credentialScope !== "org" || + grant.resourceConstraints.length !== 0 || + !grant.mcp + ) { + throw new Error("MCP grants require invoke, explicit org credentials, and a tool grant"); + } + const tool = grant.mcp; + if ( + !tool.upstreamName || + tool.upstreamName.length > 256 || + /[\u0000-\u001f]/.test(tool.upstreamName) || + typeof tool.automationWrite !== "boolean" || + typeof tool.unrestricted !== "boolean" + ) + throw new Error("invalid MCP tool grant"); + const documents = tool.argumentConstraintDocuments; + if (!Array.isArray(documents)) throw new Error("MCP constraints must be schema documents"); + if (Buffer.byteLength(canonicalJson(documents)) > MCP_LIMITS.argumentBytes) + throw new Error("MCP constraints are too large"); + for (const document of documents) assertSchemaShape(document); + const empty = documents.length === 0; + if (!empty && documents.every((document) => Object.keys(document).length === 0)) + throw new Error("empty constraints require an explicit unrestricted choice"); + if (empty !== tool.unrestricted) + throw new Error("empty constraints require an explicit unrestricted choice"); +} diff --git a/orchestrator/src/mcp/worker-client.ts b/orchestrator/src/mcp/worker-client.ts new file mode 100644 index 000000000..2e92f99d5 --- /dev/null +++ b/orchestrator/src/mcp/worker-client.ts @@ -0,0 +1,316 @@ +import { Agent, request as httpsRequest } from "node:https"; +import { checkServerIdentity, TLSSocket } from "node:tls"; +import type { IncomingMessage } from "node:http"; +import type { + WorkerIdentity, + WorkerRequest, + WorkerResponse, +} from "../../../mcp-worker/src/contracts.ts"; +import { canonical, digest, parseWorkerRequest } from "../../../mcp-worker/src/index.ts"; +import { assertWorkerPolicy, type ExpectedWorkerPolicy } from "./worker-encoding.ts"; +import type { CredentialControlTls } from "./control-client.ts"; + +const REQUEST_BYTES = 1536 * 1024; +const RESPONSE_BYTES = 2 * 1024 * 1024; +const IDENTITY_BYTES = 4096; +const CHANNEL_MS = 35000; +const failure = () => new Error("MCP worker operation unavailable"); +function object(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} +function identityFrom(value: unknown): WorkerIdentity { + if ( + !object(value) || + value.protocol !== 1 || + typeof value.bootId !== "string" || + !value.bootId || + value.bootId.length > 8192 || + typeof value.zoneId !== "string" || + !value.zoneId || + typeof value.policyRevision !== "string" || + !value.policyRevision || + typeof value.policyDigest !== "string" || + !/^[0-9a-f]{64}$/.test(value.policyDigest) + ) + throw failure(); + return { + protocol: 1, + bootId: value.bootId, + zoneId: value.zoneId, + policyRevision: value.policyRevision, + policyDigest: value.policyDigest, + }; +} + +/** Create one client per operation. Never share this mutable channel across calls. + * An unused identity check can be replaced before credential acquisition. Dispatch is + * one-use, and cannot reconnect or move the credential body to another TLS socket. */ +export function makeResourceWorkerClient(options: { + tls: CredentialControlTls; + policy: ExpectedWorkerPolicy; + signal?: AbortSignal; +}) { + const tls = { ...options.tls }, + policy = { ...options.policy }; + const operationSignal = options.signal; + let url: URL; + try { + url = new URL(tls.url); + } catch { + throw new Error("MCP worker requires pinned mTLS and policy"); + } + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.search || + url.hash || + url.pathname !== "/" || + !tls.ca || + !tls.cert || + !tls.key || + !/^(?:[0-9A-F]{2}:){31}[0-9A-F]{2}$/.test(tls.serverFingerprint) || + !policy.zoneId || + !policy.policyRevision || + !/^sha256:[0-9a-f]{64}$/.test(policy.policyDigest) + ) + throw new Error("MCP worker requires pinned mTLS and policy"); + + let agent: Agent | undefined; + let channel: { socket: TLSSocket; identity: WorkerIdentity } | undefined; + let idleTimer: ReturnType | undefined; + let state: "unused" | "handshake" | "ready" | "dispatch" | "closed" = "unused"; + function discardChannel() { + clearTimeout(idleTimer); + channel = undefined; + agent?.destroy(); + agent = undefined; + } + function close() { + state = "closed"; + discardChannel(); + } + + async function exchange( + path: "/identity" | "/rpc", + body: string | undefined, + maximumBytes: number, + deadline: number, + signal?: AbortSignal, + requiredSocket?: TLSSocket, + ): Promise<{ value: unknown; socket: TLSSocket }> { + if (!agent || signal?.aborted || deadline <= Date.now()) throw failure(); + return new Promise((resolve, reject) => { + let finished = false; + let response: IncomingMessage | undefined; + const req = httpsRequest(url, { + agent, + path, + method: body === undefined ? "GET" : "POST", + headers: { + accept: "application/json", + ...(body === undefined + ? {} + : { + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + }), + }, + maxHeaderSize: 8192, + }); + const fail = () => { + if (finished) return; + finished = true; + cleanup(); + response?.destroy(); + req.destroy(); + reject(failure()); + }; + const timer = setTimeout(fail, Math.max(1, deadline - Date.now())); + const connectTimer = setTimeout(fail, Math.min(5000, Math.max(1, deadline - Date.now()))); + function cleanup() { + clearTimeout(timer); + clearTimeout(connectTimer); + signal?.removeEventListener("abort", fail); + } + signal?.addEventListener("abort", fail, { once: true }); + req.once("error", fail); + req.once("socket", (socket) => { + if ( + !(socket instanceof TLSSocket) || + (requiredSocket && (socket !== requiredSocket || socket.destroyed || !socket.authorized)) + ) { + fail(); + return; + } + if (requiredSocket) { + clearTimeout(connectTimer); + // No credential bytes are written until socket identity has been checked. + req.end(body); + } else { + socket.once("secureConnect", () => { + if ( + !socket.authorized || + socket.getPeerCertificate().fingerprint256 !== tls.serverFingerprint + ) { + fail(); + return; + } + clearTimeout(connectTimer); + req.end(); + }); + } + }); + req.once("response", (res) => { + response = res; + const socket = res.socket; + if ( + !(socket instanceof TLSSocket) || + res.statusCode !== 200 || + !/^application\/json(?:\s*;|$)/i.test(res.headers["content-type"] ?? "") || + (res.headers["content-encoding"] && res.headers["content-encoding"] !== "identity") || + Number(res.headers["content-length"] ?? 0) > maximumBytes + ) { + fail(); + return; + } + const chunks: Buffer[] = []; + let size = 0; + res.on("data", (chunk: Buffer) => { + size += chunk.length; + if (size > maximumBytes) { + fail(); + return; + } + chunks.push(chunk); + }); + res.once("error", fail); + res.once("aborted", fail); + res.once("end", () => { + if (finished) return; + try { + const value: unknown = JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode(Buffer.concat(chunks)), + ); + finished = true; + cleanup(); + resolve({ value, socket }); + } catch { + fail(); + } + }); + }); + if (signal?.aborted) fail(); + }); + } + return { + async identity(): Promise { + if (state !== "unused" && state !== "ready") throw failure(); + discardChannel(); + state = "handshake"; + try { + agent = new Agent({ + keepAlive: true, + maxSockets: 1, + maxFreeSockets: 1, + maxCachedSessions: 0, + ca: tls.ca, + cert: tls.cert, + key: tls.key, + minVersion: "TLSv1.3", + rejectUnauthorized: true, + checkServerIdentity(host, cert) { + return ( + checkServerIdentity(host, cert) ?? + (cert.fingerprint256 === tls.serverFingerprint ? undefined : failure()) + ); + }, + }); + const response = await exchange( + "/identity", + undefined, + IDENTITY_BYTES, + Date.now() + 10000, + operationSignal, + ); + const identity = identityFrom(response.value); + assertWorkerPolicy(identity, policy); + if (response.socket.destroyed) throw failure(); + channel = { socket: response.socket, identity }; + state = "ready"; + idleTimer = setTimeout(close, CHANNEL_MS); + idleTimer.unref(); + return { ...identity }; + } catch { + close(); + throw failure(); + } + }, + async dispatch(input: WorkerRequest): Promise { + if (state !== "ready" || !channel) throw failure(); + const verified = channel; + state = "dispatch"; + clearTimeout(idleTimer); + try { + const request = parseWorkerRequest(structuredClone(input)); + const { intent, operation } = request; + const now = Date.now(); + if ( + !["execute", "discover", "validateSchemas", "validateDestination", "cancel"].includes( + operation.kind, + ) || + intent.bootId !== verified.identity.bootId || + intent.zoneId !== verified.identity.zoneId || + intent.policyDigest !== verified.identity.policyDigest || + intent.operationDigest !== digest(operation) || + intent.destination !== ("endpoint" in operation ? operation.endpoint : "") || + intent.expiresAt <= now || + intent.expiresAt > now + 300000 || + intent.lease.expiresAt < intent.expiresAt || + intent.lease.expiresAt > now + 300000 || + verified.socket.destroyed + ) + throw failure(); + const body = JSON.stringify(request); + if (Buffer.byteLength(body) > REQUEST_BYTES) throw failure(); + const maximumMs = + operation.kind === "execute" ? 300000 : operation.kind === "cancel" ? 5000 : 10000; + const { value } = await exchange( + "/rpc", + body, + RESPONSE_BYTES, + Math.min(intent.expiresAt, now + maximumMs), + operationSignal, + verified.socket, + ); + if ( + !object(value) || + value.ok !== true || + canonical(value.identity) !== canonical(verified.identity) || + canonical(value.intent) !== canonical(intent) || + value.intentId !== intent.id || + value.requestDigest !== intent.requestDigest || + value.operationDigest !== intent.operationDigest || + value.leaseId !== intent.lease.id || + !("result" in value) + ) + throw failure(); + return { + ok: true, + identity: verified.identity, + intent, + intentId: intent.id, + requestDigest: intent.requestDigest, + operationDigest: intent.operationDigest, + leaseId: intent.lease.id, + result: value.result, + }; + } catch { + throw failure(); + } finally { + close(); + } + }, + close, + }; +} diff --git a/orchestrator/src/mcp/worker-encoding.ts b/orchestrator/src/mcp/worker-encoding.ts new file mode 100644 index 000000000..207d1e5af --- /dev/null +++ b/orchestrator/src/mcp/worker-encoding.ts @@ -0,0 +1,27 @@ +import type { WorkerIdentity } from "../../../mcp-worker/src/contracts.ts"; + +/** The journal keeps its prefixed digest. Only the worker scope uses bare hex. */ +export function toWorkerBindingHash(value: string): string { + if (!/^sha256:[0-9a-f]{64}$/.test(value)) throw new Error("MCP binding digest invalid"); + return value.slice("sha256:".length); +} + +export interface ExpectedWorkerPolicy { + zoneId: string; + policyRevision: string; + /** Exact frozen operator identity, in the binding's prefixed representation. */ + policyDigest: string; +} + +/** Compare representations, without changing the worker or coordinator receipt. */ +export function assertWorkerPolicy(identity: WorkerIdentity, expected: ExpectedWorkerPolicy): void { + if ( + identity.protocol !== 1 || + !/^[0-9a-f]{64}$/.test(identity.policyDigest) || + !/^sha256:[0-9a-f]{64}$/.test(expected.policyDigest) || + `sha256:${identity.policyDigest}` !== expected.policyDigest || + identity.zoneId !== expected.zoneId || + identity.policyRevision !== expected.policyRevision + ) + throw new Error("MCP worker policy changed"); +} diff --git a/orchestrator/src/routes/connection-credential-broker.ts b/orchestrator/src/routes/connection-credential-broker.ts index 87825032b..bca8122bc 100644 --- a/orchestrator/src/routes/connection-credential-broker.ts +++ b/orchestrator/src/routes/connection-credential-broker.ts @@ -54,6 +54,7 @@ interface CachedToken { } export interface CredentialBrokerSession { + mcpBindingEnvelope?: import("../mcp/binding-store.ts").McpBindingEnvelope | null; userId: string | null; principalId: string | null; profileId: string | null; @@ -135,6 +136,7 @@ export function makeConnectionCredentialBrokerRoute(deps: { profileId: taskSession.profileId, integrationGrants: taskSession.integrationGrants, integrationConnections: taskSession.integrationConnections, + mcpBindingEnvelope: taskSession.mcpBindingEnvelope, }).from(taskSession).innerJoin(task, eq(taskSession.taskId, task.id)).where( eq(taskSession.sessionId, sessionId), ).limit(1); @@ -178,6 +180,7 @@ export function makeConnectionCredentialBrokerRoute(deps: { profileId: session.profileId, integrationGrants: grants, integrationConnections: connections, + mcpBindingEnvelope: session.mcpBindingEnvelope, }); let outcome = "denied"; diff --git a/orchestrator/src/routes/session-files.ts b/orchestrator/src/routes/session-files.ts index a2a878d11..d2656a9fd 100644 --- a/orchestrator/src/routes/session-files.ts +++ b/orchestrator/src/routes/session-files.ts @@ -62,7 +62,16 @@ function errorResponse(error: unknown): Response { : error.code === Code.AlreadyExists || error.code === Code.FailedPrecondition ? 409 : 400; - return Response.json({ error: error.rawMessage }, { status }); + return Response.json( + { error: error.rawMessage }, + { + status, + headers: + Number.isInteger(error.code) && error.code >= 1 && error.code <= 16 + ? { "x-engram-rpc-code": String(error.code) } + : {}, + }, + ); } return Response.json( { error: error instanceof Error ? error.message : String(error) }, diff --git a/orchestrator/src/rpc/integration.ts b/orchestrator/src/rpc/integration.ts index 79ed128aa..64e9c8b75 100644 --- a/orchestrator/src/rpc/integration.ts +++ b/orchestrator/src/rpc/integration.ts @@ -10,7 +10,12 @@ * * Injectable deps (getSession, connectors) for tests. */ +import { requireMcpHumanAdmin } from "../mcp/credential-rpc.ts"; +import { + mcpHandlers, mcpConfigFromProto, mcpConfigToProto, + requireMcp, requireMcpHook, mcpOperation, type McpServices, +} from "../mcp/contracts.ts"; import { ConnectError, Code } from "@connectrpc/connect"; import type { ConnectRouter } from "@connectrpc/connect"; @@ -125,6 +130,7 @@ export interface IntegrationOpAccess { } export interface IntegrationDeps { + mcp?: McpServices; getSession?: GetSession; connectors?: ConnectorStore; connectorLogos?: ConnectorLogoStore; @@ -305,6 +311,7 @@ function connectionToProto(row: IntegrationConnectionRow) { createdAt: row.createdAt.toISOString(), updatedAt: row.updatedAt.toISOString(), ...config, + ...(row.provider === "mcp" ? { mcp: mcpConfigToProto(row.config), ...row.mcp } : {}), }; } @@ -400,6 +407,7 @@ export function registerIntegration(router: ConnectRouter, deps?: IntegrationDep const requireConnection = (id: string) => requireConnectionIn(providers, connections, id); router.service(IntegrationService, { + ...mcpHandlers({ services: deps?.mcp, connections, getSession, now, connectionToProto }), async listConnectors(_req, ctx) { await requireAdmin(ctx, getSession); const statusInputs = await fetchStatusInputs(orgSecret, mint, oauthCredential); @@ -729,6 +737,25 @@ export function registerIntegration(router: ConnectRouter, deps?: IntegrationDep async createConnection(req, ctx) { await requireAdmin(ctx, getSession); + if (req.provider === "mcp") { + await requireMcpHumanAdmin(ctx, getSession); + if (req.mcp?.apiKeySecretRef) + throw new ConnectError( + "use fresh credential setup after creating the connection", + Code.InvalidArgument, + ); + const service = requireMcp(deps?.mcp); + const validate = requireMcpHook(service.validateDestination); + assertConnectionNames(req.alias, req.displayName); + const config = mcpConfigFromProto(req.mcp); + return mcpOperation(async () => { + await validate(config); + const row = await connections.create({ + alias: req.alias, provider: "mcp", displayName: req.displayName.trim(), config, + }); + return { connection: connectionToProto(row) }; + }); + } const provider = requireProvider(req.provider); provider.assertDeploymentReady?.({ issuer, deploymentId }); assertConnectionNames(req.alias, req.displayName); @@ -745,6 +772,28 @@ export function registerIntegration(router: ConnectRouter, deps?: IntegrationDep async updateConnection(req, ctx) { await requireAdmin(ctx, getSession); assertConnectionNames(req.alias, req.displayName); + const current = await connections.get(req.id); + if (current?.provider === "mcp") { + const actor = await requireMcpHumanAdmin(ctx, getSession); + const service = requireMcp(deps?.mcp); + const validate = requireMcpHook(service.validateDestination); + const credentials = requireMcpHook(service.credentials); + if (req.mcp?.apiKeySecretRef) + throw new ConnectError("credential references cannot be entered", Code.InvalidArgument); + const config = mcpConfigFromProto(req.mcp); + return mcpOperation(async () => { + await validate(config); + await credentials.configure( + req.id, + req.expectedRevision, + req.expectedGeneration, + actor.id, + config, + { alias: req.alias, displayName: req.displayName.trim() }, + ); + return { connection: connectionToProto((await connections.get(req.id))!) }; + }); + } const { provider } = await requireConnection(req.id); provider.assertDeploymentReady?.({ issuer, deploymentId }); const config = configFromProto(provider, req); @@ -771,6 +820,18 @@ export function registerIntegration(router: ConnectRouter, deps?: IntegrationDep if (referenced) { throw new ConnectError("connection is still granted to a profile", Code.FailedPrecondition); } + if (connection?.provider === "mcp") { + const actor = await requireMcpHumanAdmin(ctx, getSession); + const credentials = requireMcpHook(requireMcp(deps?.mcp).credentials); + return mcpOperation(async () => ({ + deleted: await credentials.remove( + req.id, + req.expectedRevision, + req.expectedGeneration, + actor.id, + ), + })); + } return { deleted: await connections.delete(req.id) }; }, @@ -799,6 +860,13 @@ export function registerIntegration(router: ConnectRouter, deps?: IntegrationDep await requireAdmin(ctx, getSession); const current = await connections.get(req.id); if (!current) throw new ConnectError("connection not found", Code.NotFound); + if (current.provider === "mcp") { + const service = requireMcp(deps?.mcp); + return mcpOperation(async () => { + await service.repository.setEnabled(req.id, req.expectedRevision, req.enabled, now()); + return { connection: connectionToProto((await connections.get(req.id))!) }; + }); + } if (req.enabled) { providers.get(current.provider)?.assertDeploymentReady?.({ issuer, deploymentId }); } diff --git a/orchestrator/src/rpc/passthrough.ts b/orchestrator/src/rpc/passthrough.ts index 201e0291e..65a28314d 100644 --- a/orchestrator/src/rpc/passthrough.ts +++ b/orchestrator/src/rpc/passthrough.ts @@ -50,9 +50,7 @@ export interface PassthroughSpec { * Injected getSession implementation. The default is getSessionFromHeaders * (auth/session.ts); tests inject a stub. */ -export type GetSession = ( - headers: Headers, -) => Promise<{ +export type GetSession = (headers: Headers) => Promise<{ user: { id: string; role?: string | null; email?: string | null }; } | null>; @@ -62,8 +60,13 @@ export type GetSession = ( */ export type ResolveOwner = (sessionId: string) => Promise; -/** Optional per-method pre-flight, keyed by policyKey ("ImageService.DisableImage"). - * Runs AFTER the authz gate and BEFORE the upstream forward. Throws to block. */ +/** Optional handler after authorization and preflight; undefined forwards upstream. */ +export type UnaryOverride = ( + req: unknown, + ctx: HandlerContext, +) => Promise<{ response: unknown } | undefined>; + +/** Optional per-method preflight after authorization and before dispatch. */ export type Preflight = (req: unknown, ctx: HandlerContext) => Promise; /** Unary methods that log one line per call with the time of each stage. */ @@ -90,11 +93,11 @@ function elapsedMs(start: number): number { * meaningful (and actively harmful) on a Connect response. */ const BLOCKED_RESPONSE_HEADERS = new Set([ - "content-type", // ConnectRouter owns this — forwarding application/grpc breaks browser parsing - "grpc-status", // gRPC trailer, not a Connect response header - "grpc-message", // gRPC trailer + "content-type", // ConnectRouter owns this — forwarding application/grpc breaks browser parsing + "grpc-status", // gRPC trailer, not a Connect response header + "grpc-message", // gRPC trailer "grpc-status-details-bin", // gRPC trailer - "transfer-encoding", // hop-by-hop, must not be forwarded + "transfer-encoding", // hop-by-hop, must not be forwarded ]); /** @@ -193,10 +196,9 @@ export function registerPassthrough( getSession?: GetSession, resolveOwner?: ResolveOwner, preflight?: Record, + overrides?: Record, ): void { - const resolveSession: GetSession = - getSession ?? - getSessionFromHeaders; + const resolveSession: GetSession = getSession ?? getSessionFromHeaders; const ownerResolver: ResolveOwner = resolveOwner ?? resolveSessionOwner; @@ -210,10 +212,7 @@ export function registerPassthrough( // Only support unary and server-streaming (bidi/client-streaming live // on dedicated routes). - if ( - m.methodKind !== "unary" && - m.methodKind !== "server_streaming" - ) { + if (m.methodKind !== "unary" && m.methodKind !== "server_streaming") { continue; } @@ -238,10 +237,7 @@ export function registerPassthrough( // 2. Look up policy (fail-closed: no entry → denied). const entry = POLICY[key]; if (!entry) { - throw new ConnectError( - `no policy for ${key}`, - Code.PermissionDenied, - ); + throw new ConnectError(`no policy for ${key}`, Code.PermissionDenied); } // 3. Build ability. @@ -275,12 +271,7 @@ export function registerPassthrough( // Anti-enumeration: return NotFound even when the session exists // but is not owned by the caller — don't confirm existence. - if ( - !ability.can( - entry.action, - subject("Session", { createdByUserId: ownerId }), - ) - ) { + if (!ability.can(entry.action, subject("Session", { createdByUserId: ownerId }))) { throw new ConnectError("not found", Code.NotFound); } } else { @@ -299,6 +290,8 @@ export function registerPassthrough( const pf = preflight?.[key]; const preflightStart = performance.now(); if (pf) await pf(req, ctx); + const handled = await overrides?.[key]?.(req, ctx); + if (handled) return handled.response; const preflightMs = elapsedMs(preflightStart); const forwardStart = performance.now(); let ok = false; @@ -356,7 +349,9 @@ export function registerPassthrough( // Same header-scrubbing rationale as the unary case above. upstreamHeaders(ctx.requestHeader), // eslint-disable-next-line @typescript-eslint/no-explicit-any - (async function* () { yield req as any; })(), + (async function* () { + yield req as any; + })(), ctx.values, ); copyHeaders(res.header, ctx.responseHeader); diff --git a/orchestrator/src/rpc/profiles.ts b/orchestrator/src/rpc/profiles.ts index d9ff7a070..c2285df18 100644 --- a/orchestrator/src/rpc/profiles.ts +++ b/orchestrator/src/rpc/profiles.ts @@ -72,6 +72,8 @@ import { import { sessions as defaultSessions } from "../control-plane/client.ts"; import { makeModelRouterStore, type ModelRouterStore } from "../db/model-routers.ts"; import { getModelRouterDefinition, selectRouterProtocol } from "../model-routers/registry.ts"; +import { parseArgumentJson } from "../mcp/arguments.ts"; +import { MCP_LIMITS } from "../mcp/limits.ts"; /** Subset of ImageService client used here (catalog validation). */ export interface ImagesClient { @@ -112,11 +114,18 @@ export interface ProfileDeps { discoverRepos?: (profileId: string) => Promise; } +function publication(expected: bigint | undefined, actorId: string) { + if (expected === undefined) return undefined; + if (expected < 0n || expected > BigInt(Number.MAX_SAFE_INTEGER)) + throw new ConnectError("invalid MCP grant revision", Code.InvalidArgument); + return { actorId, expectedRevision: Number(expected) }; +} /** Map a ProfileRow to the proto Profile. env_vars included only when admin. */ function toProto(row: ProfileRow, isAdmin: boolean): Profile { return { id: row.id, + mcpGrantRevision: BigInt(row.mcpGrantPublication?.revision ?? 0), name: row.name, description: row.description, icon: row.icon, @@ -132,6 +141,12 @@ function toProto(row: ProfileRow, isAdmin: boolean): Profile { operation: grant.operation, resourceConstraints: grant.resourceConstraints, credentialScope: grant.credentialScope ?? "", + mcp: grant.mcp ? { + upstreamName: grant.mcp.upstreamName, + automationWrite: grant.mcp.automationWrite, + argumentConstraintDocumentsJson: grant.mcp.argumentConstraintDocuments.map((document) => JSON.stringify(document)), + unrestricted: grant.mcp.unrestricted, + } : undefined, })), // ADR 0057: network + secrets describe access/config (the secret VALUES // live in the org store, never here), so they're member-visible like skills. @@ -486,6 +501,12 @@ export function registerProfiles(router: ConnectRouter, deps?: ProfileDeps): voi operation?: string; resourceConstraints?: string[]; credentialScope?: string; + mcp?: { + upstreamName: string; + automationWrite: boolean; + argumentConstraintDocumentsJson: string[]; + unrestricted: boolean; + }; }>, ): ProfileIntegrationGrant[] { return grants.map((grant) => { @@ -496,12 +517,35 @@ export function registerProfiles(router: ConnectRouter, deps?: ProfileDeps): voi Code.InvalidArgument, ); } + let mcp: ProfileIntegrationGrant["mcp"]; + if (grant.mcp) { + const documents = grant.mcp.argumentConstraintDocumentsJson.map((json) => { + let document: unknown; + try { + if (Buffer.byteLength(json) > MCP_LIMITS.argumentBytes) + throw new Error("schema is too large"); + document = parseArgumentJson(json); + } catch { + throw new ConnectError("MCP constraints must be valid JSON", Code.InvalidArgument); + } + if (!document || typeof document !== "object" || Array.isArray(document)) + throw new ConnectError("MCP constraints must be JSON objects", Code.InvalidArgument); + return document as Record; + }); + mcp = { + upstreamName: grant.mcp.upstreamName, + automationWrite: grant.mcp.automationWrite, + argumentConstraintDocuments: documents, + unrestricted: grant.mcp.unrestricted, + }; + } return { + ...(mcp ? { mcp } : {}), connectionId: (grant.connectionId ?? "").trim(), operation: (grant.operation ?? "").trim(), resourceConstraints: [...new Set(grant.resourceConstraints ?? [])].sort(), // Stored only when user-scoped; absent means org (the default). - ...(scope === "user" ? { credentialScope: "user" as const } : {}), + ...(scope === "user" ? { credentialScope: "user" as const } : scope === "org" ? { credentialScope: "org" as const } : {}), }; }); } @@ -540,88 +584,99 @@ export function registerProfiles(router: ConnectRouter, deps?: ProfileDeps): voi async createProfile(req, ctx) { const user = await requireUser(ctx, getSession); const ability = abilityFor(user); - if (!ability.can("manage", "Profile")) throw new ConnectError("forbidden", Code.PermissionDenied); + if (!ability.can("manage", "Profile")) + throw new ConnectError("forbidden", Code.PermissionDenied); if (!req.name.trim()) throw new ConnectError("name is required", Code.InvalidArgument); await assertImageEnabled(req.imageId); const model = catalogOptionId(req.model); const modelRouter = catalogOptionId(req.modelRouter); const effort = catalogOptionId(req.effort); - const harness = await assertHarnessValid(catalogOptionId(req.harness), modelRouter, model, effort); + const harness = await assertHarnessValid( + catalogOptionId(req.harness), + modelRouter, + model, + effort, + ); await assertSkillsValid(req.skills ?? []); const integrationGrants = normalizeIntegrationGrants(req.integrationGrants ?? []); const resolvedGrants = await resolveIntegrationGrants(integrationGrants, connections); - assertResolvedGrantsValid( - resolvedGrants, - await loadRegistry(connectors), - toolCapabilities(), - ); + assertResolvedGrantsValid(resolvedGrants, await loadRegistry(connectors), toolCapabilities()); const network = normalizeNetwork(req.network); const secrets = normalizeSecrets(req.secrets ?? []); assertNetworkValid(network); assertSecretsValid(secrets); - const row = await store.create({ - name: req.name, - description: req.description, - icon: req.icon || "Bot", - imageId: req.imageId, - programmaticOnly: req.programmaticOnly, - includeUserTokens: req.includeUserTokens, - envVars: req.envVars ?? {}, - skills: req.skills ?? [], - integrationGrants, - network, - secrets, - repos: normalizeReposChecked(req.repos ?? []), - harness, - modelRouter, - model, - effort, - apps: normalizeAppsChecked(req.apps ?? []), - }); + const row = await store.create( + { + name: req.name, + description: req.description, + icon: req.icon || "Bot", + imageId: req.imageId, + programmaticOnly: req.programmaticOnly, + includeUserTokens: req.includeUserTokens, + envVars: req.envVars ?? {}, + skills: req.skills ?? [], + integrationGrants, + network, + secrets, + repos: normalizeReposChecked(req.repos ?? []), + harness, + modelRouter, + model, + effort, + apps: normalizeAppsChecked(req.apps ?? []), + }, + publication(req.expectedMcpGrantRevision, user.id), + ); return { profile: toProto(row, true) }; }, async updateProfile(req, ctx) { const user = await requireUser(ctx, getSession); const ability = abilityFor(user); - if (!ability.can("manage", "Profile")) throw new ConnectError("forbidden", Code.PermissionDenied); + if (!ability.can("manage", "Profile")) + throw new ConnectError("forbidden", Code.PermissionDenied); if (!req.name.trim()) throw new ConnectError("name is required", Code.InvalidArgument); await assertImageEnabled(req.imageId); const model = catalogOptionId(req.model); const modelRouter = catalogOptionId(req.modelRouter); const effort = catalogOptionId(req.effort); - const harness = await assertHarnessValid(catalogOptionId(req.harness), modelRouter, model, effort); + const harness = await assertHarnessValid( + catalogOptionId(req.harness), + modelRouter, + model, + effort, + ); await assertSkillsValid(req.skills ?? []); const integrationGrants = normalizeIntegrationGrants(req.integrationGrants ?? []); const resolvedGrants = await resolveIntegrationGrants(integrationGrants, connections); - assertResolvedGrantsValid( - resolvedGrants, - await loadRegistry(connectors), - toolCapabilities(), - ); + assertResolvedGrantsValid(resolvedGrants, await loadRegistry(connectors), toolCapabilities()); const network = normalizeNetwork(req.network); const secrets = normalizeSecrets(req.secrets ?? []); assertNetworkValid(network); assertSecretsValid(secrets); - const row = await store.update(req.id, { - name: req.name, - description: req.description, - icon: req.icon || "Bot", - imageId: req.imageId, - programmaticOnly: req.programmaticOnly, - includeUserTokens: req.includeUserTokens, - envVars: req.envVars ?? {}, - skills: req.skills ?? [], - integrationGrants, - network, - secrets, - repos: normalizeReposChecked(req.repos ?? []), - harness, - modelRouter, - model, - effort, - apps: normalizeAppsChecked(req.apps ?? []), - }); + const row = await store.update( + req.id, + { + name: req.name, + description: req.description, + icon: req.icon || "Bot", + imageId: req.imageId, + programmaticOnly: req.programmaticOnly, + includeUserTokens: req.includeUserTokens, + envVars: req.envVars ?? {}, + skills: req.skills ?? [], + integrationGrants, + network, + secrets, + repos: normalizeReposChecked(req.repos ?? []), + harness, + modelRouter, + model, + effort, + apps: normalizeAppsChecked(req.apps ?? []), + }, + publication(req.expectedMcpGrantRevision, user.id), + ); if (!row) throw new ConnectError("not found", Code.NotFound); return { profile: toProto(row, true) }; }, diff --git a/orchestrator/src/rpc/task-create.ts b/orchestrator/src/rpc/task-create.ts index de0b22549..b3f9f7f2f 100644 --- a/orchestrator/src/rpc/task-create.ts +++ b/orchestrator/src/rpc/task-create.ts @@ -17,6 +17,11 @@ * policy. */ +import { + createReservedTaskSession, + McpSessionCreationMismatchError, +} from "../mcp/session-create.ts"; + import { resolveIntegrationHost } from "../integrations/run-op.ts"; import { ConnectError, Code } from "@connectrpc/connect"; import { and, eq, isNull } from "drizzle-orm"; @@ -53,6 +58,11 @@ import { type IntegrationSecretJson, } from "../connectors/registry.ts"; import { compileToolManifest } from "../tools/manifest.ts"; +import { + mcpSessionCapabilities, + type McpSessionCapabilities, +} from "../mcp/session-capabilities.ts"; +import { McpManagedCreationPending } from "../mcp/managed-runtime.ts"; import { tools as productionTools, type ToolRegistry } from "../tools/registry.ts"; import { systemPromptForTaskType } from "../prompts/base.ts"; import type { SpecPromptContext } from "../prompts/spec-mode.ts"; @@ -80,10 +90,7 @@ import { providerGuestServices, } from "../integrations/providers/index.ts"; import { makeModelRouterStore, type ModelRouterStore } from "../db/model-routers.ts"; -import { - getModelRouterDefinition, - selectRouterProtocol, -} from "../model-routers/registry.ts"; +import { getModelRouterDefinition, selectRouterProtocol } from "../model-routers/registry.ts"; const log = rootLog.child({ component: "task" }); @@ -116,6 +123,8 @@ export type Db = NodePgDatabase; /** The control-plane CreateSession request the orchestrator compiles. Mirrors * the `SessionsClient.createSession` request shape (rpc/tasks.ts). */ export interface SessionCreateInput { + /** Candidate metadata only; production MCP creation remains guarded. */ + mcpBindingEnvelope?: import("../mcp/binding-store.ts").McpBindingEnvelope; /** ADR 0109: reserved before boot so the authorization snapshot exists. */ requestedSessionId?: string; /** ADR 0118: the session's apps, already resolved to public hostnames. The @@ -198,6 +207,7 @@ export interface OrgSecretNameClient { } export interface SessionCompileDeps { + mcpCompiler?: McpSessionCapabilities["compile"]; images: ImagesClient; connectors: CustomConnectorSource; harnessCatalog: HarnessCatalogClient; @@ -228,6 +238,9 @@ export interface SessionCompileDeps { } export interface SessionCompileOpts { + /** Actual trusted spawning session, never a launch snapshot. */ + spawningSessionId?: string; + inheritMcp?: boolean; prompt?: string; /** Orchestrator task context used to select prompt and tool surfaces. This * value is not sent to the sandbox. */ @@ -331,18 +344,28 @@ export async function compileSessionCreateInput( const selectedRouterId = opts.modelRouter ?? profile.modelRouter ?? undefined; const descriptor = harnesses.find((h) => h.name === selectedHarness)?.descriptor; if (!descriptor) { - throw new ConnectError(`harness \`${selectedHarness}\` is not in the catalog`, Code.FailedPrecondition); + throw new ConnectError( + `harness \`${selectedHarness}\` is not in the catalog`, + Code.FailedPrecondition, + ); } const router = selectedRouterId ? getModelRouterDefinition(selectedRouterId) : null; if (selectedRouterId && !router) { - throw new ConnectError(`model router \`${selectedRouterId}\` is not registered`, Code.InvalidArgument); + throw new ConnectError( + `model router \`${selectedRouterId}\` is not registered`, + Code.InvalidArgument, + ); } - const routerHost = router ? await resolveIntegrationHost(router.catalog.provider, { - connectors: deps.connectors, - settingsFor: async (provider) => - storedSettingsOf((await deps.connections.getDefault(provider))?.config), - }) : undefined; - const routerProtocol = router ? selectRouterProtocol(router, descriptor.routerProtocols ?? []) : null; + const routerHost = router + ? await resolveIntegrationHost(router.catalog.provider, { + connectors: deps.connectors, + settingsFor: async (provider) => + storedSettingsOf((await deps.connections.getDefault(provider))?.config), + }) + : undefined; + const routerProtocol = router + ? selectRouterProtocol(router, descriptor.routerProtocols ?? []) + : null; if (router && !routerProtocol) { throw new ConnectError( `harness \`${selectedHarness}\` has no protocol adapter for model router \`${router.id}\``, @@ -369,19 +392,26 @@ export async function compileSessionCreateInput( const secretClient: OrgSecretNameClient = deps.orgSecret ?? defaultOrgSecret; const [routed, availableSecrets] = await Promise.all([ (deps.modelRouters ?? makeModelRouterStore()).getModel(router.id, modelId), - secretClient - .listSecrets({}) - .then((r) => new Set(r.secrets.map((secret) => secret.name))), + secretClient.listSecrets({}).then((r) => new Set(r.secrets.map((secret) => secret.name))), ]); routedModel = routed; if (!routedModel || !routedModel.available) { throw new ConnectError(`router model \`${modelId}\` is unavailable`, Code.FailedPrecondition); } if (!routedModel.enabled || (isHumanPrincipal(opts) && !routedModel.userEnabled)) { - log.warn({ routerId: router.id, modelId, human: isHumanPrincipal(opts) }, "model router policy rejected launch"); - throw new ConnectError(`router model \`${modelId}\` is not enabled for this principal`, Code.PermissionDenied); + log.warn( + { routerId: router.id, modelId, human: isHumanPrincipal(opts) }, + "model router policy rejected launch", + ); + throw new ConnectError( + `router model \`${modelId}\` is not enabled for this principal`, + Code.PermissionDenied, + ); } - if (!routedModel.supportedParameters.includes("reasoning") && !routedModel.supportedParameters.includes("reasoning_effort")) { + if ( + !routedModel.supportedParameters.includes("reasoning") && + !routedModel.supportedParameters.includes("reasoning_effort") + ) { effortId = undefined; } if (!availableSecrets.has(router.credentialSecret)) { @@ -400,7 +430,10 @@ export async function compileSessionCreateInput( "model router launch accepted", ); } else if (modelId && !modelOption) { - throw new ConnectError(`model \`${modelId}\` is not valid for harness \`${selectedHarness}\``, Code.InvalidArgument); + throw new ConnectError( + `model \`${modelId}\` is not valid for harness \`${selectedHarness}\``, + Code.InvalidArgument, + ); } const effortOption = descriptor.effort.find((e) => e.id === effortId); @@ -490,6 +523,15 @@ export async function compileSessionCreateInput( ? overrideGrants : [...profile.integrationGrants, ...overrideGrants]; const resolvedEffectiveGrants = await resolveIntegrationGrants(effectiveGrants, connections); + const hasMcp = + opts.inheritMcp === true || + resolvedEffectiveGrants.some(({ connection }) => connection.provider === "mcp"); + const mcpCompiler = deps.mcpCompiler ?? mcpSessionCapabilities()?.compile; + if (hasMcp && !mcpCompiler) + throw new ConnectError( + "external MCP session activation is not configured", + Code.FailedPrecondition, + ); const disabledConnection = resolvedEffectiveGrants.find(({ connection }) => !connection.enabled); if (disabledConnection) { throw new ConnectError( @@ -587,12 +629,7 @@ export async function compileSessionCreateInput( baseToolRegistry.complete(sessionId, toolCallId, result), } : baseToolRegistry; - const toolManifest = compileToolManifest( - manifestRegistry, - surfacedCapabilities, - opts.taskType, - ); - if (toolManifest.length > 0) harness.ENGRAM_TOOLS = JSON.stringify(toolManifest); + const toolManifest = compileToolManifest(manifestRegistry, surfacedCapabilities, opts.taskType); if (!opts.dropProfileSecretsAndEnv) { for (const [k, v] of Object.entries(profile.envVars)) harness[k] = v; } @@ -649,7 +686,12 @@ export async function compileSessionCreateInput( // The human credential remains the final, principal-authoritative env write, // so no descriptor or trigger can replace it. - if (isHuman && !router && userEnv && humanUserToken !== undefined) harness[userEnv] = humanUserToken; + if (isHuman && !router && userEnv && humanUserToken !== undefined) + harness[userEnv] = humanUserToken; + // Only the trusted registry supplies the native manifest. Freeze it after every + // environment contribution, before managed creation stores its exact bytes. + if (toolManifest.length > 0) harness.ENGRAM_TOOLS = JSON.stringify(toolManifest); + else delete harness.ENGRAM_TOOLS; const harnessEnv = Object.keys(harness).length > 0 ? harness : undefined; // Per-session integration policy (caps + network + secrets), shipped only @@ -715,7 +757,7 @@ export async function compileSessionCreateInput( } const integrationPolicyJson = policyHasContent(policy) ? JSON.stringify(policy) : undefined; - return { + const compiled: SessionCreateInput = { imageUri: image.imageUri, mode: "agent", harness: selectedHarness, @@ -747,6 +789,14 @@ export async function compileSessionCreateInput( ).values(), ], }; + if (!hasMcp) return compiled; + return mcpCompiler!(compiled, { + profileId: profile.id, + profileGrants: profile.integrationGrants, + grants: activeResolvedGrants.map(({ grant }) => grant), + directPolicy: policy, + ...(opts.spawningSessionId ? { spawningSessionId: opts.spawningSessionId } : {}), + }); } function isHumanPrincipal(opts: SessionCompileOpts): boolean { @@ -766,6 +816,7 @@ export interface TaskSessionsClient { } export interface CreateTaskDeps { + boundSessionCreator?: Pick; profiles: ProfileStore; images: ImagesClient; connectors: CustomConnectorSource; @@ -877,9 +928,13 @@ export interface CreatedTask { } /** Make an already-persisted session discoverable by the listener scanner. - * Callers with consumer-specific bindings must persist those bindings first. */ + * Callers with consumer-specific bindings must persist those bindings first. + * Managed creation can enroll first; keep its existing lease and terminal state. */ export async function registerSessionListener(db: Db, sessionId: string): Promise { - await db.insert(sessionListenerTable).values({ sessionId }); + await db + .insert(sessionListenerTable) + .values({ sessionId }) + .onConflictDoNothing({ target: sessionListenerTable.sessionId }); } /** @@ -910,6 +965,9 @@ export function buildLaunchPolicy( integrationPolicyJson: sessionInput.integrationPolicyJson ?? "", integrationGrants: [...(sessionInput.integrationGrants ?? [])], integrationConnections: [...(sessionInput.integrationConnections ?? [])], + ...(sessionInput.mcpBindingEnvelope + ? { mcpBindingEnvelope: structuredClone(sessionInput.mcpBindingEnvelope) } + : {}), network: profile.network, secrets: [...profile.secrets], repos: [...profile.repos], @@ -1029,10 +1087,16 @@ export async function createSessionForExistingTask( capabilities: sessionInput.capabilities ?? [], integrationGrants: sessionInput.integrationGrants ?? [], integrationConnections: sessionInput.integrationConnections ?? [], + ...(sessionInput.mcpBindingEnvelope + ? { mcpBindingEnvelope: sessionInput.mcpBindingEnvelope } + : {}), integrationSnapshotHash: integrationSnapshotHash({ profileId: profile.id, integrationGrants: sessionInput.integrationGrants ?? [], integrationConnections: sessionInput.integrationConnections ?? [], + ...(sessionInput.mcpBindingEnvelope + ? { mcpBindingEnvelope: sessionInput.mcpBindingEnvelope } + : {}), }), ...(integrationPrincipalId ? { integrationPrincipalId } : {}), }); @@ -1042,17 +1106,25 @@ export async function createSessionForExistingTask( try { // When prompt is omitted (as it is for the finder), the session boots idle // so deterministic bootstrap can finish before SendPrompt wakes it. - const created = await deps.sessions.createSession(sessionInput); + const created = await createReservedTaskSession(deps, sessionInput, { + actorId: integrationPrincipalId ?? "", + inputId: sessionId, + }); createdSessionId = created.sessionId; if (created.sessionId !== sessionId) { throw new Error("coordinator returned a different reserved session ID"); } if (params.registerListener === true) { await deps.db.transaction(async (tx) => { - await tx.insert(sessionListenerTable).values({ sessionId }); + await tx + .insert(sessionListenerTable) + .values({ sessionId }) + .onConflictDoNothing({ target: sessionListenerTable.sessionId }); }); } } catch (err) { + if (err instanceof McpManagedCreationPending) throw err; + if (err instanceof McpSessionCreationMismatchError) createdSessionId = err.createdSessionId; // Compensation must never mask the original failure: guard every step and // log what it could not undo. On an ID mismatch, the session that leaks is // the one the coordinator ACTUALLY created, so delete that one. @@ -1221,10 +1293,16 @@ export async function createTaskWithSession( capabilities: sessionInput.capabilities ?? [], integrationGrants: sessionInput.integrationGrants ?? [], integrationConnections: sessionInput.integrationConnections ?? [], + ...(sessionInput.mcpBindingEnvelope + ? { mcpBindingEnvelope: sessionInput.mcpBindingEnvelope } + : {}), integrationSnapshotHash: integrationSnapshotHash({ profileId: profile.id, integrationGrants: sessionInput.integrationGrants ?? [], integrationConnections: sessionInput.integrationConnections ?? [], + ...(sessionInput.mcpBindingEnvelope + ? { mcpBindingEnvelope: sessionInput.mcpBindingEnvelope } + : {}), }), integrationPrincipalId: params.ownerUserId, }); @@ -1264,15 +1342,23 @@ export async function createTaskWithSession( let createdSessionId: string | undefined; try { - const created = await deps.sessions.createSession(sessionInput); + const created = await createReservedTaskSession(deps, sessionInput, { + actorId: params.ownerUserId, + inputId: sessionId, + }); createdSessionId = created.sessionId; if (created.sessionId !== sessionId) { throw new Error("coordinator returned a different reserved session ID"); } await deps.db.transaction(async (tx) => { - await tx.insert(sessionListenerTable).values({ sessionId }); + await tx + .insert(sessionListenerTable) + .values({ sessionId }) + .onConflictDoNothing({ target: sessionListenerTable.sessionId }); }); } catch (err) { + if (err instanceof McpManagedCreationPending) throw err; + if (err instanceof McpSessionCreationMismatchError) createdSessionId = err.createdSessionId; // Compensation must never mask the original failure: guard every step and // log what it could not undo. On an ID mismatch, the session that leaks is // the one the coordinator ACTUALLY created, so delete that one. diff --git a/orchestrator/src/sweep/__tests__/sweeper.test.ts b/orchestrator/src/sweep/__tests__/sweeper.test.ts index 7179a1dbf..cfda7e4a0 100644 --- a/orchestrator/src/sweep/__tests__/sweeper.test.ts +++ b/orchestrator/src/sweep/__tests__/sweeper.test.ts @@ -86,10 +86,8 @@ async function fixture( const status = makeInMemoryDbosStatusStore(rows, now, { isVersionLive: async (applicationVersion, graceMs) => (await heartbeats.liveVersions(graceMs)).includes(applicationVersion), - isPodLive: async (podName, graceMs) => - (await heartbeats.livePods(graceMs)).includes(podName), - recordSweep: (workflowUuid, workflowName) => - ledger.recordSweep(workflowUuid, workflowName), + isPodLive: async (podName, graceMs) => (await heartbeats.livePods(graceMs)).includes(podName), + recordSweep: (workflowUuid, workflowName) => ledger.recordSweep(workflowUuid, workflowName), }); const cancelled: string[] = []; const deps: SweepTickDeps = { @@ -116,7 +114,9 @@ describe("runSweepTick retention", () => { const settings = makeInMemoryOrgSettingStore({ [RETENTION_KEY]: { runDetailDays: 7 } }); const store = makeInMemoryRetentionStore({ runs: [{ id: "run-old", endedAt: new Date(NOW.getTime() - 8 * DAY_MS), steps: 3 }], - workflows: [{ id: "wf-old", status: "SUCCESS", createdAt: new Date(NOW.getTime() - 8 * DAY_MS) }], + workflows: [ + { id: "wf-old", status: "SUCCESS", createdAt: new Date(NOW.getTime() - 8 * DAY_MS) }, + ], }); const f = await fixture([], { retention: { settings, store, now: () => new Date(NOW) } }); @@ -137,7 +137,9 @@ describe("runSweepTick retention", () => { test("does not prune when the lease is not held", async () => { const store = makeInMemoryRetentionStore({ - workflows: [{ id: "wf-old", status: "SUCCESS", createdAt: new Date(NOW.getTime() - 60 * DAY_MS) }], + workflows: [ + { id: "wf-old", status: "SUCCESS", createdAt: new Date(NOW.getTime() - 60 * DAY_MS) }, + ], }); const f = await fixture([], { retention: { settings: makeInMemoryOrgSettingStore(), store, now: () => new Date(NOW) }, @@ -235,6 +237,26 @@ describe("runSweepTick", () => { }); }); + test("adopts MCP delivery workflows after 48 hours or missing heartbeat history", async () => { + for (const lastSeen of [new Date(NOW.getTime() - 72 * HOUR_MS), null]) { + const f = await fixture( + [ + row("mcp-request", { name: "McpRequestWorkflow" }), + row("mcp-execution", { name: "McpExecWorkflow" }), + row("ordinary-stale"), + ], + {}, + {}, + { deadVersionLastSeen: lastSeen }, + ); + const result = await runSweepTick(f.deps); + expect(result.decisions.filter((x) => x.name.startsWith("Mcp")).map((x) => x.action)).toEqual( + ["adopted", "adopted"], + ); + expect(f.cancelled).toEqual(["ordinary-stale"]); + } + }); + test("cancels a stale adopt-policy workflow instead of adopting it", async () => { // Staleness = version abandonment: DEAD_VERSION last heartbeat 2h ago, // past ToolExecWorkflow's 1h window. The workflow's own age is irrelevant. @@ -269,9 +291,7 @@ describe("runSweepTick", () => { expect(result.decisions[0]?.action).toBe("adopted"); expect(f.cancelled).toEqual([]); - expect( - f.status.inspect("wf-long-lived-thread")?.applicationVersion, - ).toBeNull(); + expect(f.status.inspect("wf-long-lived-thread")?.applicationVersion).toBeNull(); }); test("a version with no heartbeat history is abandoned forever", async () => { @@ -319,9 +339,7 @@ describe("runSweepTick", () => { }); test("reports an unknown workflow name without changing it", async () => { - const f = await fixture([ - row("wf-unknown", { name: "DeletedWorkflowName" }), - ]); + const f = await fixture([row("wf-unknown", { name: "DeletedWorkflowName" })]); const result = await runSweepTick(f.deps); @@ -330,17 +348,17 @@ describe("runSweepTick", () => { expect(await f.deps.ledger.get("wf-unknown")).toBeNull(); }); - test.each([ - "temp_workflow-send-message-123", - "_dbos_debouncer_workflow", - ])("adopts a fresh prefix-covered workflow: %s", async (name) => { - const f = await fixture([row("wf-prefix", { name })]); + test.each(["temp_workflow-send-message-123", "_dbos_debouncer_workflow"])( + "adopts a fresh prefix-covered workflow: %s", + async (name) => { + const f = await fixture([row("wf-prefix", { name })]); - const result = await runSweepTick(f.deps); + const result = await runSweepTick(f.deps); - expect(result.decisions[0]?.action).toBe("adopted"); - expect(f.status.inspect("wf-prefix")?.applicationVersion).toBeNull(); - }); + expect(result.decisions[0]?.action).toBe("adopted"); + expect(f.status.inspect("wf-prefix")?.applicationVersion).toBeNull(); + }, + ); test("caps mutating actions and stops examining rows at the cap", async () => { const rows = [ @@ -358,10 +376,7 @@ describe("runSweepTick", () => { // scanned counts examined rows only: the row after the cap is left for // the next tick's cursor, not silently counted as covered. expect(result.scanned).toBe(2); - expect(result.decisions.map((decision) => decision.action)).toEqual([ - "alert_only", - "adopted", - ]); + expect(result.decisions.map((decision) => decision.action)).toEqual(["alert_only", "adopted"]); expect(f.status.inspect("wf-1")?.applicationVersion).toBeNull(); expect(f.status.inspect("wf-2")?.applicationVersion).toBe(DEAD_VERSION); }); @@ -428,9 +443,7 @@ describe("runSweepTick", () => { expect(result.scanned).toBe(40); expect(result.decisions).toHaveLength(40); - expect(f.status.inspect("wf-beyond-budget")?.applicationVersion).toBe( - DEAD_VERSION, - ); + expect(f.status.inspect("wf-beyond-budget")?.applicationVersion).toBe(DEAD_VERSION); expect(warnings).toEqual([ "DBOS orphan sweep scan budget exhausted; resuming from the cursor next cycle", ]); @@ -461,18 +474,12 @@ describe("runSweepTick", () => { const first = await runSweepTick(f.deps); expect(first.scanned).toBe(40); - expect(f.status.inspect("wf-beyond-budget")?.applicationVersion).toBe( - DEAD_VERSION, - ); + expect(f.status.inspect("wf-beyond-budget")?.applicationVersion).toBe(DEAD_VERSION); expect(scanCursor.value?.workflowUuid).toBe("wf-unknown-39"); const second = await runSweepTick(f.deps); - expect( - second.decisions.map((decision) => decision.action), - ).toEqual(["adopted"]); - expect( - f.status.inspect("wf-beyond-budget")?.applicationVersion, - ).toBeNull(); + expect(second.decisions.map((decision) => decision.action)).toEqual(["adopted"]); + expect(f.status.inspect("wf-beyond-budget")?.applicationVersion).toBeNull(); // The pass reached the end of the backlog, so the next tick restarts // from the oldest row. expect(scanCursor.value).toBeUndefined(); @@ -516,14 +523,10 @@ describe("runSweepTick", () => { const sweeper = new Sweeper(f.deps); await sweeper.runOnce(); - expect(f.status.inspect("wf-beyond-budget")?.applicationVersion).toBe( - DEAD_VERSION, - ); + expect(f.status.inspect("wf-beyond-budget")?.applicationVersion).toBe(DEAD_VERSION); await sweeper.runOnce(); - expect( - f.status.inspect("wf-beyond-budget")?.applicationVersion, - ).toBeNull(); + expect(f.status.inspect("wf-beyond-budget")?.applicationVersion).toBeNull(); }); test("a persistently failing stale cancel retries every tick until it lands", async () => { @@ -632,19 +635,13 @@ describe("runSweepTick", () => { () => new Date(NOW), { isVersionLive: async (applicationVersion, graceMs) => - (await f.heartbeats.liveVersions(graceMs)).includes( - applicationVersion, - ), + (await f.heartbeats.liveVersions(graceMs)).includes(applicationVersion), }, ); f.deps.status = { ...underlying, async listNonTerminalOnVersionsNotIn(liveVersions, limit, after) { - const listed = await underlying.listNonTerminalOnVersionsNotIn( - liveVersions, - limit, - after, - ); + const listed = await underlying.listNonTerminalOnVersionsNotIn(liveVersions, limit, after); await f.heartbeats.beat(DEAD_VERSION, "pod-returned"); return listed; }, @@ -856,7 +853,11 @@ describe("runSweepTick — stranded executors (per-pod recovery)", () => { // owner is a pod: "pod-current" beats now, "pod-dead" last beat 30 min // ago (the fixture's dead version), "local" (the SDK default id every // pod once shared) never beats at all. - function stranded(workflowUuid: string, executorId: string, overrides: Partial = {}) { + function stranded( + workflowUuid: string, + executorId: string, + overrides: Partial = {}, + ) { return row(workflowUuid, { applicationVersion: CURRENT_VERSION, name: "AutomationRunWorkflow", @@ -879,7 +880,7 @@ describe("runSweepTick — stranded executors (per-pod recovery)", () => { expect((await f.deps.ledger.get("wf-stranded"))?.sweepCount).toBe(1); }); - test("the legacy executor id \"local\" is a pod that never beats: its work is re-enqueued once", async () => { + test('the legacy executor id "local" is a pod that never beats: its work is re-enqueued once', async () => { const f = await fixture([stranded("wf-local", "local")]); const result = await runSweepTick(f.deps); expect(result.decisions.map((d) => d.action)).toEqual(["requeued"]); @@ -967,6 +968,8 @@ describe("sweep policy exhaustiveness", () => { assertSweepPoliciesExhaustive([ "ToolExecWorkflow", "AutomationRunWorkflow", + "McpRequestWorkflow", + "McpExecWorkflow", ]), ).not.toThrow(); }); @@ -1020,9 +1023,7 @@ describe("VersionHeartbeat", () => { await heartbeat.start(); - expect(beats).toEqual([ - { appVersion: CURRENT_VERSION, podName: "pod-a" }, - ]); + expect(beats).toEqual([{ appVersion: CURRENT_VERSION, podName: "pod-a" }]); expect(intervalScheduled).toBe(true); await heartbeat.stop(); }); diff --git a/orchestrator/src/sweep/policy.ts b/orchestrator/src/sweep/policy.ts index 70cf1bb9b..e0ffa1144 100644 --- a/orchestrator/src/sweep/policy.ts +++ b/orchestrator/src/sweep/policy.ts @@ -8,10 +8,13 @@ import { getAllRegisteredFunctions } from "../../node_modules/@dbos-inc/dbos-sdk */ export interface SweepPolicy { mode: "adopt"; - staleAfterHours: number; + /** Null preserves durable delivery obligations across an arbitrarily long outage. */ + staleAfterHours: number | null; } export const SWEEP_POLICIES: Record = { + McpRequestWorkflow: { mode: "adopt", staleAfterHours: null }, + McpExecWorkflow: { mode: "adopt", staleAfterHours: null }, ToolExecWorkflow: { mode: "adopt", staleAfterHours: 1 }, // ADR 0119: interpreter runs park on recv (waits, session phases) for as // long as a review or a thread takes, so the launch-scale 1h staleness @@ -50,18 +53,12 @@ export function registeredWorkflowNames(): string[] { .map((registration) => registration.name); } -export function assertSweepPoliciesExhaustive( - names: string[] = registeredWorkflowNames(), -): void { +export function assertSweepPoliciesExhaustive(names: string[] = registeredWorkflowNames()): void { const missing = names.filter( (name) => - !SWEEP_POLICIES[name] && - !name.startsWith("_dbos_") && - !name.startsWith("temp_workflow-"), + !SWEEP_POLICIES[name] && !name.startsWith("_dbos_") && !name.startsWith("temp_workflow-"), ); if (missing.length > 0) { - throw new Error( - `DBOS workflows missing sweep policies: ${missing.join(", ")}`, - ); + throw new Error(`DBOS workflows missing sweep policies: ${missing.join(", ")}`); } } diff --git a/orchestrator/src/sweep/sweeper.ts b/orchestrator/src/sweep/sweeper.ts index e74bac09c..173867079 100644 --- a/orchestrator/src/sweep/sweeper.ts +++ b/orchestrator/src/sweep/sweeper.ts @@ -26,7 +26,7 @@ export const ALERT_ONLY_STALE_AFTER_HOURS = 48; // dbos_version_heartbeats accrues one row per (version, pod) across deploys // and nothing else deletes them. Rows older than the grace window are already // dead for liveness, so pruning at a comfortable multiple only bounds growth. -// INVARIANT: retention must exceed every policy's staleAfterHours — the +// INVARIANT: retention must exceed every finite policy's staleAfterHours — the // heartbeat history is also the staleness clock (a version with no surviving // rows reads as abandoned forever), so pruning too aggressively would make a // freshly-stranded workflow look ancient. @@ -115,11 +115,7 @@ const MUTATING_ACTIONS = new Set([ "cancelled_stale", ]); -const ALERT_ACTIONS = new Set([ - "alert_only", - "cancelled_stale", - "error", -]); +const ALERT_ACTIONS = new Set(["alert_only", "cancelled_stale", "error"]); function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); @@ -143,19 +139,14 @@ function logCycle(log: Logger, result: SweepTickResult): void { ); } -export async function runSweepTick( - deps: SweepTickDeps, -): Promise { +export async function runSweepTick(deps: SweepTickDeps): Promise { const result: SweepTickResult = { leaseHeld: false, liveVersions: [], scanned: 0, decisions: [], }; - const leaseHeld = await deps.lease.tryAcquire( - deps.owner, - 2 * deps.config.sweepIntervalMs, - ); + const leaseHeld = await deps.lease.tryAcquire(deps.owner, 2 * deps.config.sweepIntervalMs); if (!leaseHeld) { logCycle(deps.log, result); return result; @@ -163,9 +154,7 @@ export async function runSweepTick( result.leaseHeld = true; try { - const liveVersions = await deps.heartbeats.liveVersions( - deps.config.graceMs, - ); + const liveVersions = await deps.heartbeats.liveVersions(deps.config.graceMs); result.liveVersions = liveVersions; const selfVersion = deps.appVersion(); if (!liveVersions.includes(selfVersion)) { @@ -188,16 +177,10 @@ export async function runSweepTick( Math.max(HEARTBEAT_RETENTION_MS, 2 * deps.config.graceMs), ); if (pruned > 0) { - deps.log.info( - { component: "dbos-sweep", pruned }, - "pruned stale version-heartbeat rows", - ); + deps.log.info({ component: "dbos-sweep", pruned }, "pruned stale version-heartbeat rows"); } } catch (error) { - deps.log.warn( - { error }, - "failed to prune stale version-heartbeat rows", - ); + deps.log.warn({ error }, "failed to prune stale version-heartbeat rows"); } // Staleness is measured from ABANDONMENT — when the workflow's owning @@ -242,7 +225,11 @@ export async function runSweepTick( }; } else { const requeued = await deps.status.requeueStrandedPendingRecording( - { workflowUuid: row.workflowUuid, executorId: row.executorId, workflowName: row.name }, + { + workflowUuid: row.workflowUuid, + executorId: row.executorId, + workflowName: row.name, + }, deps.config.podGraceMs, ); decision = requeued.flipped @@ -271,10 +258,7 @@ export async function runSweepTick( if (MUTATING_ACTIONS.has(decision.action)) actions++; } } catch (error) { - deps.log.warn( - { error }, - "DBOS stranded-executor scan failed; version scan continues", - ); + deps.log.warn({ error }, "DBOS stranded-executor scan failed; version scan continues"); } const abandonedMsByVersion = await deps.heartbeats.abandonedMsByVersion(); @@ -285,21 +269,11 @@ export async function runSweepTick( let after = cursor?.value; let exhausted = false; - scan: while ( - actions < deps.config.batchCap && - result.scanned < scanBudget - ) { - const rows = await deps.status.listNonTerminalOnVersionsNotIn( - liveVersions, - pageSize, - after, - ); + scan: while (actions < deps.config.batchCap && result.scanned < scanBudget) { + const rows = await deps.status.listNonTerminalOnVersionsNotIn(liveVersions, pageSize, after); for (const row of rows) { - if ( - actions >= deps.config.batchCap || - result.scanned >= scanBudget - ) { + if (actions >= deps.config.batchCap || result.scanned >= scanBudget) { break scan; } result.scanned++; @@ -325,18 +299,14 @@ export async function runSweepTick( // ERROR and is never re-swept; the rare silent zombie is an // operator call via the ledger's `suppressed` flag, with // sweep_count kept as the evidence trail. - const abandonedMs = abandonedMsByVersion.get( - row.applicationVersion!, - ); + const abandonedMs = abandonedMsByVersion.get(row.applicationVersion!); const abandonedHours = abandonedMs === undefined ? Number.POSITIVE_INFINITY : abandonedMs / (60 * 60 * 1_000); const staleAfterHours = - policy.mode === "alert-only" - ? ALERT_ONLY_STALE_AFTER_HOURS - : policy.staleAfterHours; - if (abandonedHours > staleAfterHours) { + policy.mode === "alert-only" ? ALERT_ONLY_STALE_AFTER_HOURS : policy.staleAfterHours; + if (staleAfterHours !== null && abandonedHours > staleAfterHours) { await deps.ledger.recordSweep(row.workflowUuid, row.name); await deps.cancelWorkflow(row.workflowUuid); decision = { @@ -345,8 +315,7 @@ export async function runSweepTick( action: "cancelled_stale", ...(policy.mode === "alert-only" ? { - reason: - "unregistered workflow name past the stale window", + reason: "unregistered workflow name past the stale window", } : {}), }; @@ -379,15 +348,14 @@ export async function runSweepTick( reason: "owner became live or row changed", }; } else if (row.status === "ENQUEUED") { - const cleared = - await deps.status.clearVersionOnEnqueuedRecording( - { - workflowUuid: row.workflowUuid, - expectedVersion: row.applicationVersion!, - workflowName: row.name, - }, - deps.config.graceMs, - ); + const cleared = await deps.status.clearVersionOnEnqueuedRecording( + { + workflowUuid: row.workflowUuid, + expectedVersion: row.applicationVersion!, + workflowName: row.name, + }, + deps.config.graceMs, + ); decision = cleared.flipped ? { workflowUuid: row.workflowUuid, @@ -429,11 +397,7 @@ export async function runSweepTick( } } - if ( - !exhausted && - actions < deps.config.batchCap && - result.scanned >= scanBudget - ) { + if (!exhausted && actions < deps.config.batchCap && result.scanned >= scanBudget) { deps.log.warn( { component: "dbos-sweep", @@ -452,20 +416,13 @@ export async function runSweepTick( ALERT_ACTIONS.has(decision.action), ).length; } catch (error) { - deps.log.warn( - { error }, - "DBOS sweep decision alerting failed", - ); + deps.log.warn({ error }, "DBOS sweep decision alerting failed"); } try { - result.failureScan = - await deps.alerter.scanTerminalFailures(); + result.failureScan = await deps.alerter.scanTerminalFailures(); } catch (error) { - deps.log.warn( - { error }, - "DBOS terminal failure scan failed", - ); + deps.log.warn({ error }, "DBOS terminal failure scan failed"); } } @@ -490,10 +447,7 @@ export async function runSweepTick( } export interface SweeperDeps extends SweepTickDeps { - setInterval?: ( - fn: () => void, - ms: number, - ) => ReturnType; + setInterval?: (fn: () => void, ms: number) => ReturnType; clearInterval?: (timer: ReturnType) => void; } @@ -550,10 +504,7 @@ export interface VersionHeartbeatDeps { heartbeats: HeartbeatStore; intervalMs: number; log: Logger; - setInterval?: ( - fn: () => void, - ms: number, - ) => ReturnType; + setInterval?: (fn: () => void, ms: number) => ReturnType; clearInterval?: (timer: ReturnType) => void; } diff --git a/orchestrator/src/tools/__tests__/exec.test.ts b/orchestrator/src/tools/__tests__/exec.test.ts index dc47b60d3..0a7f2ae34 100644 --- a/orchestrator/src/tools/__tests__/exec.test.ts +++ b/orchestrator/src/tools/__tests__/exec.test.ts @@ -212,3 +212,22 @@ describe("tool exec plain functions", () => { expect(f.submitted).toEqual(["session-1:call-1"]); }); }); + +test("MCP resolution is session-local and absent runtime never calls a static handler", async () => { + const f = fixture(); + let resolvedSession: string | undefined; + const deps: ToolExecDeps = { + ...f.deps, + resolveMcpBoundTool: async ({ sessionId }) => { + resolvedSession = sessionId; + return undefined; + }, + }; + const outcome = await executeToolCall({ ...f.input, toolName: "mcp_docs_unknown" }, deps); + expect(resolvedSession).toBe(f.input.sessionId); + expect(outcome).toEqual({ + kind: "submit", + result: { error: "external MCP tool is unavailable" }, + }); + expect(f.handlerCalls).toEqual([]); +}); diff --git a/orchestrator/src/tools/coordination.ts b/orchestrator/src/tools/coordination.ts index 0c8450e1c..838852a8c 100644 --- a/orchestrator/src/tools/coordination.ts +++ b/orchestrator/src/tools/coordination.ts @@ -1,5 +1,13 @@ /** Recursive session coordination tools (ADR 0113). */ +import { + createReservedTaskSession, + McpSessionCreationMismatchError, +} from "../mcp/session-create.ts"; +import { McpManagedCreationPending } from "../mcp/managed-runtime.ts"; +import { assertMcpDirectPolicyJson } from "../mcp/session-conflicts.ts"; +import { assertChildMcpCreationBlocked } from "../mcp/session-child.ts"; + import { createHash } from "node:crypto"; import { Code, ConnectError } from "@connectrpc/connect"; import { and, eq, ne, sql } from "drizzle-orm"; @@ -214,6 +222,7 @@ function snapshotConnectionStore(policy: TaskLaunchPolicy): IntegrationConnectio async function compileChildInput( parent: TaskRow, + spawningSessionId: string, args: z.output, sessionId: string, ): Promise { @@ -225,6 +234,11 @@ async function compileChildInput( // unattended sessions never carry human credentials extends to spawns. // The frozen policy already stamped org authority for such a parent (no // oauth_user mint sources), so no user seam below would fire anyway. + const inheritMcp = await assertChildMcpCreationBlocked( + parent.id, + spawningSessionId, + policy.integrationGrants, + ); const ownerId = parent.createdByUserId; const db = getDb(); const secrets = makeUserSecretStore(db); @@ -286,7 +300,8 @@ async function compileChildInput( connectors: { list: () => makeConnectorStore(db).list() }, harnessCatalog: harnessCatalog as unknown as HarnessCatalogClient, toolRegistry: productionTools, - resolveUserToken: (envVar) => (ownerId ? secrets.get(ownerId, envVar) : Promise.resolve(null)), + resolveUserToken: (envVar) => + ownerId ? secrets.get(ownerId, envVar) : Promise.resolve(null), resolveAllUserTokens: () => (ownerId ? secrets.getAll(ownerId) : Promise.resolve({})), ...(ownerId ? { @@ -320,8 +335,15 @@ async function compileChildInput( ...(identity ? { owner: identity } : {}), ...(ownerId ? {} : { programmatic: true }), excludeHumanInteractionTools: true, + spawningSessionId, + inheritMcp, }, ); + if (input.mcpBindingEnvelope) + assertMcpDirectPolicyJson( + policy.integrationPolicyJson, + input.mcpBindingEnvelope.managedDenyPolicy.hosts, + ); return { ...input, requestedSessionId: sessionId, @@ -329,8 +351,12 @@ async function compileChildInput( selectedSkills: [...policy.skills], capabilities: [...policy.capabilities], integrationPolicyJson: policy.integrationPolicyJson, - integrationGrants: structuredClone(policy.integrationGrants), - integrationConnections: structuredClone(policy.integrationConnections), + integrationGrants: input.mcpBindingEnvelope + ? input.integrationGrants + : structuredClone(policy.integrationGrants), + integrationConnections: input.mcpBindingEnvelope + ? input.integrationConnections + : structuredClone(policy.integrationConnections), prompt: undefined, harnessMode: undefined, }; @@ -536,10 +562,23 @@ export function registerCoordinationTools(registry: ToolRegistry): void { throw new Error(`sub-session depth exceeds ${config.subSessionMaxDepth}`); } const rootTaskId = parent.rootTaskId ?? parent.id; - const input = await compileChildInput(parent, args, sessionId); + const input = await compileChildInput(parent, ctx.sessionId, args, sessionId); + const [spawning] = await getDb() + .select({ principal: taskSession.integrationPrincipalId }) + .from(taskSession) + .where(and(eq(taskSession.taskId, parent.id), eq(taskSession.sessionId, ctx.sessionId))); + const childPrincipal = input.mcpBindingEnvelope + ? spawning?.principal + : parent.createdByUserId; + if (input.mcpBindingEnvelope && !childPrincipal) + throw Error("MCP parent principal unavailable"); const policy = parent.launchPolicy!; + const { mcpBindingEnvelope: _parentEnvelope, ...basePolicy } = structuredClone(policy); const childPolicy: TaskLaunchPolicy = { - ...structuredClone(policy), + ...basePolicy, + integrationGrants: input.integrationGrants ?? [], + integrationConnections: input.integrationConnections ?? [], + ...(input.mcpBindingEnvelope ? { mcpBindingEnvelope: input.mcpBindingEnvelope } : {}), harness: input.harness ?? policy.harness, ...(input.modelRouter ? { modelRouter: input.modelRouter } : {}), ...(input.model ? { model: input.model } : {}), @@ -613,13 +652,17 @@ export function registerCoordinationTools(registry: ToolRegistry): void { role: "primary", profileId: policy.profileId, capabilities: [...policy.capabilities], - integrationGrants: structuredClone(policy.integrationGrants), - integrationConnections: structuredClone(policy.integrationConnections), - integrationPrincipalId: parent.createdByUserId, + integrationGrants: structuredClone(input.integrationGrants ?? []), + integrationConnections: structuredClone(input.integrationConnections ?? []), + ...(input.mcpBindingEnvelope ? { mcpBindingEnvelope: input.mcpBindingEnvelope } : {}), + integrationPrincipalId: childPrincipal, integrationSnapshotHash: integrationSnapshotHash({ profileId: policy.profileId, - integrationGrants: policy.integrationGrants, - integrationConnections: policy.integrationConnections, + integrationGrants: input.integrationGrants ?? [], + integrationConnections: input.integrationConnections ?? [], + ...(input.mcpBindingEnvelope + ? { mcpBindingEnvelope: input.mcpBindingEnvelope } + : {}), }), }) .onConflictDoNothing(); @@ -651,8 +694,17 @@ export function registerCoordinationTools(registry: ToolRegistry): void { } }); try { - await sessions.createSession(input); + await createReservedTaskSession({ sessions }, input, { + actorId: childPrincipal ?? "", + inputId: sessionId, + }); } catch (error) { + if (error instanceof McpManagedCreationPending) throw error; + if (error instanceof McpSessionCreationMismatchError) { + // Compensate the actual created identity, never the requested reservation. + await sessions.deleteSession({ sessionId: error.createdSessionId }).catch(() => {}); + throw error; + } try { await sessions.getSession({ sessionId }); } catch { @@ -669,10 +721,7 @@ export function registerCoordinationTools(registry: ToolRegistry): void { // listener permanently terminal (review finding on this PR). A // crash inside this window leaves the session unenrolled — the same // narrow exposure the sibling paths accept. - await getDb() - .insert(sessionListenerTable) - .values({ sessionId }) - .onConflictDoNothing(); + await getDb().insert(sessionListenerTable).values({ sessionId }).onConflictDoNothing(); if ((args.file_paths?.length ?? 0) > 0) { await sessions.copyFiles({ sourceSessionId: ctx.sessionId, diff --git a/orchestrator/src/tools/exec.ts b/orchestrator/src/tools/exec.ts index 760d738a8..53390cbf9 100644 --- a/orchestrator/src/tools/exec.ts +++ b/orchestrator/src/tools/exec.ts @@ -60,6 +60,7 @@ export type ToolExecutionOutcome = | { kind: "submit"; result: unknown }; export interface ToolExecDeps { + resolveMcpBoundTool?: import("../mcp/session-resolver.ts").SessionBindingResolver["resolveBoundTool"]; registry: ToolRegistry; resolveContext(sessionId: string): Promise; pendingCalls: PendingToolCallStore; @@ -113,6 +114,27 @@ export async function executeToolCall( "tool exec start", ); try { + if (input.toolName.startsWith("mcp_")) { + // Raw remote schemas never enter Zod or the global registry. Task 5d owns effects. + try { + const context = await deps.resolveContext(input.sessionId); + if (context.sessionId !== input.sessionId) throw new Error("session changed"); + const bound = await deps.resolveMcpBoundTool?.({ + sessionId: input.sessionId, + modelToolName: input.toolName, + }); + return { + kind: "submit", + result: { + error: bound + ? "external MCP execution is not configured" + : "external MCP tool is unavailable", + }, + }; + } catch { + return { kind: "submit", result: { error: "external MCP tool is unavailable" } }; + } + } const tool = deps.registry.get(input.toolName); if (!tool || tool.handling !== "handled") { throw new Error(`handled tool is not registered: ${input.toolName}`); diff --git a/orchestrator/src/tools/manifest.ts b/orchestrator/src/tools/manifest.ts index 976fe66d8..795ce3acb 100644 --- a/orchestrator/src/tools/manifest.ts +++ b/orchestrator/src/tools/manifest.ts @@ -1,3 +1,5 @@ +import { compileMcpManifest, type McpSessionBindingSnapshot } from "../mcp/session-bindings.ts"; +import { MCP_LIMITS } from "../mcp/limits.ts"; import { z } from "zod"; import { @@ -12,6 +14,7 @@ export interface ToolManifestEntry { inputSchema: object; execution: "sync" | "deferred"; nativeBindings: NativeBindings; + resultFormat?: "mcp"; } /** Compile the model-facing manifest from a registry without side effects. */ @@ -35,3 +38,36 @@ export function compileToolManifest( nativeBindings: tool.nativeBindings ?? {}, })); } + +/** Preserve native descriptors and raw remote schemas, with one total wire-size bound. */ +export function appendMcpManifest( + nativeJson: string | undefined, + snapshot: McpSessionBindingSnapshot, +): ToolManifestEntry[] { + const native: unknown = nativeJson === undefined ? [] : JSON.parse(nativeJson); + if (!Array.isArray(native)) throw new Error("invalid native tool manifest"); + const entries: ToolManifestEntry[] = []; + for (const entry of native) { + if ( + !entry || + typeof entry !== "object" || + typeof entry.name !== "string" || + typeof entry.description !== "string" || + !entry.inputSchema || + typeof entry.inputSchema !== "object" || + !["sync", "deferred"].includes(entry.execution) || + !entry.nativeBindings || + typeof entry.nativeBindings !== "object" + ) + throw new Error("invalid native tool descriptor"); + entries.push(entry); + } + entries.push(...compileMcpManifest(snapshot)); + if ( + entries.length > MCP_LIMITS.toolsPerSession || + new Set(entries.map((entry) => entry.name)).size !== entries.length || + Buffer.byteLength(JSON.stringify(entries)) > MCP_LIMITS.manifestBytes + ) + throw new Error("session tool manifest exceeds limits or contains duplicate names"); + return structuredClone(entries); +} diff --git a/orchestrator/src/workflows/mcp-exec.ts b/orchestrator/src/workflows/mcp-exec.ts new file mode 100644 index 000000000..6b3c7eff2 --- /dev/null +++ b/orchestrator/src/workflows/mcp-exec.ts @@ -0,0 +1,73 @@ +import { DBOS } from "@dbos-inc/dbos-sdk"; +import { + mcpRecovery, + type McpExecutionMarker, + type McpRequestMarker, + type McpRequestDecision, +} from "../mcp/recovery.ts"; + +async function mcpExecWorkflowEntry(marker: McpExecutionMarker): Promise { + for (;;) { + try { + const runtime = mcpRecovery(); + // Plain calls deliberately run outside DBOS steps: only the journal authorizes effects. + if (runtime && (await runtime.runOnce(marker)) === "done") return; + } catch { + // All errors stay transient and private. Original journal deadlines are never renewed. + } + await DBOS.sleep(1_000); + } +} +export const mcpExecWorkflow = DBOS.registerWorkflow(mcpExecWorkflowEntry, { + name: "McpExecWorkflow", +}); +export async function startMcpWorkflow( + marker: McpExecutionMarker, + workflowId = `mcp:${marker.executionId}`, +): Promise { + await DBOS.startWorkflow(mcpExecWorkflow, { workflowID: workflowId })(marker); +} + +/** The first decision is durable before either delivery or child execution starts. */ +async function mcpRequestWorkflowEntry(ref: McpRequestMarker): Promise { + let decision: McpRequestDecision; + for (;;) { + try { + decision = await DBOS.runStep( + async () => { + const runtime = mcpRecovery(); + if (!runtime) throw Error("MCP request acceptance unavailable"); + return runtime.decideRequest(ref); + }, + { name: "mcp-request-decision", retriesAllowed: false }, + ); + break; + } catch { + await DBOS.sleep(1_000); + } + } + if (decision.kind === "accepted") { + await startMcpWorkflow(decision.execution); + return; + } + for (;;) { + try { + const runtime = mcpRecovery(); + if (runtime) { + await runtime.deliverDenial(ref, decision.code); + return; + } + } catch { + /* Retry only the fixed negative completion, never acceptance. */ + } + await DBOS.sleep(1_000); + } +} +export const mcpRequestWorkflow = DBOS.registerWorkflow(mcpRequestWorkflowEntry, { + name: "McpRequestWorkflow", +}); +export async function startMcpRequestWorkflow(ref: McpRequestMarker): Promise { + await DBOS.startWorkflow(mcpRequestWorkflow, { + workflowID: `mcp-request:${ref.sessionId}:${ref.toolCallId}`, + })(ref); +} diff --git a/orchestrator/tsconfig.json b/orchestrator/tsconfig.json index 2a5deb6f4..f507c03a8 100644 --- a/orchestrator/tsconfig.json +++ b/orchestrator/tsconfig.json @@ -7,7 +7,7 @@ "skipLibCheck": true, "types": ["bun", "node"], "outDir": "dist", - "rootDir": "src", + "rootDir": "..", "esModuleInterop": true, "resolveJsonModule": true, "allowImportingTsExtensions": true, diff --git a/sandbox-netstack/README.md b/sandbox-netstack/README.md new file mode 100644 index 000000000..d05d973a7 --- /dev/null +++ b/sandbox-netstack/README.md @@ -0,0 +1,205 @@ +# Private sandbox network helper + +This artifact provides one IPv4 netstack per VM. It has no host IP listener, +route mutation, external resolver, destination dial, switch, or NAT. It does not +activate a backend or supply a final managed-policy proof. Managed creation also +requires the backend and durable boot checks. + +Build with Go **1.26.2**, `GOTOOLCHAIN=local`. The exact module closure is in +`go.mod` and `go.sum`. The gVisor pin is +`v0.0.0-20260413194555-9680d69bf798`; its module hash is +`h1:7h21uRYtzPJjXp9W/BZLx68z8TwW8+h5BPdI4P5ZkJ0=` and its module-file hash is +`h1:xQ2PWgHmWJA/Ph4i1q1jBm39BKhc3W0DXqWoDSyuBOY=`. DNS decoding uses +`golang.org/x/net/dns/dnsmessage` v0.44.0. No floating dependency is allowed. + +From the repository root: + +```sh +sandbox-netstack/build.sh test +sandbox-netstack/build.sh race +sandbox-netstack/build.sh build +``` + +The optional second argument selects an owned output directory. The default is +`var/mcp-netstack-build`. The script rejects cache symlinks and uses that directory +for every Go cache and temporary file. The build writes +`engram-sandbox-netstack` there, with `CGO_ENABLED=0` and `-trimpath`. The race gate +uses `CGO_ENABLED=1` and needs a native C compiler. The tests build and execute the +real helper, so the Go tool must be available when tests run. The `netstack` CI +lane runs these commands on Linux and macOS and publishes checksummed binaries. +Cargo does not build this module. + +## Descriptor and startup contract + +Start the executable with no arguments. Supply only these three descriptors; +use the process API's stdin/stdout/stderr descriptor mapping. These descriptors +are channels, **not logs**. Clear the child environment. All other descriptors +must be close-on-exec. Retain the parent control peer and child process handle. + +| Descriptor | Type | Owner and use | +|---|---|---| +| 0 | Connected AF_UNIX datagram | One VM Ethernet attachment; one datagram per frame | +| 1 | Connected AF_UNIX stream | Parent lifetime and explicit OPEN/CLOSE control | +| 2 | Connected AF_UNIX stream | Registered private proxy allocation capability | + +The helper duplicates each inherited descriptor into Go's poller and closes the +original. The parent must close its child-side copies immediately after spawn. +The parent must terminate and reap the helper on every failed startup, socket +loss, VM teardown, or invalidated generation. Do not infer lifetime from datagram +EOF. Do not pass the capability through argv, environment, disk, guest data, or +operational logs. Do not retain a second copy across host process generations. + +Register descriptor 2 first through `ProxydHandle::register_private_network`, or +`engram_egress_proto::private::register` on a dedicated host UDS connection. The +host-only upgrade is `ToProxyd::PrivateNetwork`, followed by the bounded +`private::Allocation`. It binds host ID/incarnation, proxy incarnation, +session/sandbox, boot attempt, attachment generation, exact guest/gateway IPv4, +and the current managed policy revision/digest. Registration requires a live +managed policy, starts CLOSED, and has no placement epoch. The returned +`RegisteredPrivateNetwork` owns the exact allocation and the socket whose CLOSED +reply was consumed. There is no unchecked constructor. A backend reads +`allocation()` and consumes `into_stream()` only when it transfers that same +capability to its helper; it must not accept separate caller-asserted metadata. +While pinning files before a helper exists, it must watch reliable proxy stop +and close the capability promptly. Disk-copy time must not delay the barrier. + +The host sends this configuration on descriptor 1 within five seconds: + +```json +{"version":1,"guest":"192.0.2.2","gateway":"192.0.2.1","mac":"02:00:00:00:00:02"} +``` + +Addresses must match the registered allocation and actual attachment. The host +must record and check that exact shape in its backend proof. The gateway MAC is +`02:00:00:00:00:01`. The guest MAC must be a different unicast MAC. MTU is exactly +1500. Only Ethernet IPv4 and gateway ARP are accepted. The helper returns: + +```json +{"version":1,"sequence":0,"epoch":0,"state":"closed"} +``` + +After obtaining the actual placement epoch, the parent sends: + +```json +{"version":1,"kind":"open","epoch":7} +``` + +The helper binds that epoch once on the proxy channel, drains closed-phase +packets, then opens packet admission. The response is sequence 1, the same epoch, +and state `ready`. This is **transport readiness only**. It cannot satisfy the +backend network proof, durable boot release, or final tool-exposure ACK. + +CLOSE uses `{"version":1,"kind":"close","epoch":7}`. A successful response has +sequence 2 and state `closed`. CLOSE is terminal. OPEN replay, wrong epoch, +malformed control, or either reliable channel's loss invalidates the allocation. +There is no auto-reopen or reconnect. Use a new allocation after loss. + +## Private flow contract + +Frames are a 4-byte big-endian length followed by strict JSON, at most 2048 bytes. +The first byte may wait while idle. A partial frame and each response write have +a five-second budget. Every request has version 1, a strictly increasing sequence +starting at 1, and the exact bound epoch. Operations are nested: + +```json +{"version":1,"sequence":1,"epoch":7,"operation":{"kind":"bind"}} +{"version":1,"sequence":2,"epoch":7,"operation":{"kind":"tcp","destination":"198.51.100.7","port":8443}} +``` + +`bind` returns `ready`. `tcp` and `dns` require one connected AF_UNIX stream via +SCM_RIGHTS on a single byte `0xED`, immediately after the header. The receiver +sets close-on-exec and rejects extra descriptors, wrong socket types, unknown +fields, missing ports, sequence replay, wrong epochs, and oversized input. +`accepted` on the allocation channel means the flow is owned by the session +lease; it is not an allow decision. The handed-off TCP stream contains guest TLS +bytes and goes through the existing Rust policy dispatcher on every port. + +DNS uses `kind: "dns"`, the exact allocation gateway, and port 53. Its stream +contains one two-byte-length-prefixed DNS request and one matching response. +The existing Rust DNS name policy and resolver own the decision. The helper +has no external DNS socket. TCP 53 on any other address goes to the ordinary TLS +dispatcher and plaintext DNS fails. UDP to any other port/address is dropped. + +## Bounds and cancellation + +Each helper has 64 active flows, at most 8 DNS exchanges, 16 pending TCP +handshakes, a 64-packet link queue, and 64 KiB TCP send/receive buffers. The pinned +gVisor IPv4 reassembly cache trims at 4 MiB to 3 MiB. DNS bodies are limited to +4096 bytes; UDP DNS must also fit the MTU and cannot be fragmented. TCP fragments +are reassembled by gVisor. Other UDP, ICMP input, IPv6, VLAN, spoofed MAC/IPv4/ARP, +short frames, oversized frames, and `MSG_TRUNC` are rejected. + +DNS exchanges have a ten-second budget. TCP flow lifetime is capped at five +minutes. Packet output waits at most 250 ms. Gateway response writes wait at +most one second. OPEN drains at most 512 queued datagrams; an input flood fails +activation. A closed packet is never stored for later replay. + +Allocation identity comes only from the inherited capability. A guest IP, MAC, +header, or payload cannot select another session. The proxy holds the existing +stable session lease before reading policy or TLS. Replacement and revocation +cancel and join owned Rust TCP/DNS/HTTP2 work. Allocation tombstones are bounded +at 4096 per proxy incarnation; exhaustion fails closed. Attachment generations +must increase within a host incarnation. Retired host incarnations cannot return. +An unrelated session has a different admission/drain lock. + +Revocation uses the reliable private stream, never datagram EOF. The proxy +first cancels and joins its TCP/DNS work, closes its write half, and waits up to +five seconds for helper EOF. On that stop signal the helper closes its gate, +closes the packet descriptor, and joins all packet readers/writers, stack work, +flow/DNS work, and control readers before it closes the private descriptor. +The stable session lease remains held through this barrier. Pending descriptor +carriers are drained with an OS-maximum ancillary buffer and closed. + +If cleanup EOF is absent, malformed, or fails, the proxy poisons that session's +stable gate. Apply, full replacement, and removal cannot acknowledge success; +a fresh allocation or ordinary policy replay cannot reopen it. Other sessions +remain usable. Recovery requires a fresh proxy incarnation plus the existing +full policy replay and quarantine checks. The host adapter must treat any +failed cleanup as a failed enforcement receipt and replace the proxy before +recovery. A helper OPEN or CLOSE response is never the final enforcement ACK. +Bytes already delivered before the cleanup barrier cannot be revoked. + +## Managed VZ lifecycle + +`VzBackend::prepare_managed_cold` accepts this registered owner plus exact trusted +cold disk and boot inputs and a selected helper artifact path/digest. It pins +private copies, builds a fresh stopped machine with a file-handle attachment, +and returns an opaque `PreparedManagedVz` outside the ordinary sandbox map. +Preparation does not load saved memory, start the machine, open the helper, +publish a network enforcement proof, or return a final policy acknowledgement. +The owner can release the helper for the exact placement epoch, check current +readiness, and start the harness once in the fresh VM. After start, it supports +authenticated execution, file transfer and execution cancellation. Disk +retirement stops the VM, closes the bridge and helper, and copies and syncs the +root disk to the capture destination. It does not capture memory or device state. + +The owner also closes and reaps its resources asynchronously. Drop starts cleanup +without waiting. A thread reserved before helper creation owns cleanup through +reaping, so an unresponsive child cannot block other allocations on the async +executor. Canceling a close wait does not cancel cleanup or discard its result +while the owner remains live. + +The helper inherits only descriptors 0, 1 and 2 through native close-by-default +spawn actions, an empty environment and no arguments. The selected native +attachment is retained and checked again on the VM queue. A helper cleanup +failure remains failed after forced reaping. Only the proxy owns the stable +session poison and final drain acknowledgement. + +The host's managed boot path validates capture eligibility, pinned artifacts, +placement, final policy and manifest identity, and durable finalization before +start. The host records the retired disk's hash and size before it closes the +capture owner. Helper readiness alone does not establish these checks or prove +guest DNS and packet enforcement. That proof requires actual VM acceptance. + +The backend-local process fixture uses the packaged helper explicitly: + +```sh +ENGRAM_SANDBOX_NETSTACK_TEST_BIN=/path/to/verified/engram-sandbox-netstack \ + cargo test -p engram-sandbox-vz --lib \ + actual_helper_closed_lifetime_and_descriptor_isolation -- --ignored +``` + +A missing fixture binary fails. The `vz` CI lane verifies the native artifact +from `netstack`, then invokes all three packaged-helper tests through +`.github/scripts/mcp-acceptance-helper.sh`. These process tests do not establish +physical VM acceptance or a current-head CI result. diff --git a/sandbox-netstack/build.sh b/sandbox-netstack/build.sh new file mode 100755 index 000000000..846067fef --- /dev/null +++ b/sandbox-netstack/build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail +mode=${1:-test} +root=$(cd "$(dirname "$0")/.." && pwd -P) +out=${2:-"$root/var/mcp-netstack-build"} +case "$mode" in build|test|race) ;; *) echo 'usage: build.sh [build|test|race] [owned-output-directory]' >&2; exit 2;; esac +# Cache/output ownership is explicit. Never follow a cache symlink. +[[ ! -L "$out" ]] || exit 1 +if [[ ! -e "$out" ]]; then + mkdir -m 700 "$out" + printf '%s\n' 'sandbox-netstack build outputs and Go caches' > "$out/ownership.txt" +fi +[[ -f "$out/ownership.txt" && ! -L "$out/ownership.txt" ]] || exit 1 +out=$(cd "$out" && pwd -P) +for dir in go-path go-mod go-cache tmp; do + [[ ! -L "$out/$dir" ]] || exit 1 + mkdir -p "$out/$dir" +done +export GOTOOLCHAIN=local GOMAXPROCS=2 CGO_ENABLED=0 +export GOPATH="$out/go-path" GOMODCACHE="$out/go-mod" GOCACHE="$out/go-cache" TMPDIR="$out/tmp" +[[ $(go env GOVERSION) == go1.26.2 ]] || { echo 'Go 1.26.2 is required' >&2; exit 1; } +cd "$root/sandbox-netstack" +case "$mode" in + build) go build -mod=readonly -trimpath -o "$out/engram-sandbox-netstack" . ;; + test) go test -mod=readonly -timeout 30s ./...; go vet ./...; go mod verify ;; + race) CGO_ENABLED=1 go test -mod=readonly -race -timeout 45s ./... ;; +esac diff --git a/sandbox-netstack/frame.go b/sandbox-netstack/frame.go new file mode 100644 index 000000000..08710fbe4 --- /dev/null +++ b/sandbox-netstack/frame.go @@ -0,0 +1,92 @@ +package main + +import ( + "bytes" + "errors" + "gvisor.dev/gvisor/pkg/tcpip/header" + "net" + "net/netip" + "syscall" +) + +const mtu = 1500 + +var gatewayMAC = []byte{2, 0, 0, 0, 0, 1} + +type config struct { + Version uint32 `json:"version"` + Guest string `json:"guest"` + Gateway string `json:"gateway"` + MAC string `json:"mac"` +} +type network struct { + guest, gateway netip.Addr + mac [6]byte +} + +func parseNetwork(c config) (network, error) { + var n network + var err error + n.guest, err = netip.ParseAddr(c.Guest) + if err != nil { + return n, err + } + n.gateway, err = netip.ParseAddr(c.Gateway) + if err != nil { + return n, err + } + mac, err := net.ParseMAC(c.MAC) + if err != nil || len(mac) != 6 || mac[0]&1 != 0 || bytes.Equal(mac, gatewayMAC) || bytes.Equal(mac, make([]byte, 6)) || c.Version != 1 || !n.guest.Is4() || !n.gateway.Is4() || !n.guest.IsGlobalUnicast() || !n.gateway.IsGlobalUnicast() || n.guest == n.gateway { + return n, errors.New("invalid network shape") + } + copy(n.mac[:], mac) + return n, nil +} +func (n network) accepts(frame []byte, flags int) bool { + if flags&(syscall.MSG_TRUNC|syscall.MSG_CTRUNC) != 0 || len(frame) < header.EthernetMinimumSize || len(frame) > mtu+header.EthernetMinimumSize { + return false + } + eth := header.Ethernet(frame) + if !bytes.Equal([]byte(eth.SourceAddress()), n.mac[:]) { + return false + } + body := frame[header.EthernetMinimumSize:] + switch eth.Type() { + case header.IPv4ProtocolNumber: + if !bytes.Equal([]byte(eth.DestinationAddress()), gatewayMAC) || len(body) < header.IPv4MinimumSize { + return false + } + ip := header.IPv4(body) + if !ip.IsValid(len(body)) || ip.SourceAddress().As4() != n.guest.As4() || int(ip.TotalLength()) != len(body) { + return false + } + switch ip.Protocol() { + case 6: + return true // TCP reassembly belongs to gVisor. + case 17: + // DNS messages are bounded; fragmented UDP is not an accepted transport. + if ip.FragmentOffset() != 0 || ip.Flags()&header.IPv4FlagMoreFragments != 0 || ip.DestinationAddress().As4() != n.gateway.As4() || len(ip.Payload()) < header.UDPMinimumSize { + return false + } + u := header.UDP(ip.Payload()) + return u.DestinationPort() == 53 && int(u.Length()) == len(ip.Payload()) + default: + return false + } + case header.ARPProtocolNumber: + if len(body) < header.ARPSize { + return false + } + a := header.ARP(body) + guest, gateway := n.guest.As4(), n.gateway.As4() + if !a.IsValid() || !bytes.Equal(a.HardwareAddressSender(), n.mac[:]) || !bytes.Equal(a.ProtocolAddressSender(), guest[:]) || !bytes.Equal(a.ProtocolAddressTarget(), gateway[:]) { + return false + } + if a.Op() == header.ARPRequest { + return bytes.Equal([]byte(eth.DestinationAddress()), gatewayMAC) || eth.DestinationAddress() == header.EthernetBroadcastAddress + } + return a.Op() == header.ARPReply && bytes.Equal([]byte(eth.DestinationAddress()), gatewayMAC) && bytes.Equal(a.HardwareAddressTarget(), gatewayMAC) + default: + return false + } +} diff --git a/sandbox-netstack/frame_test.go b/sandbox-netstack/frame_test.go new file mode 100644 index 000000000..feea90253 --- /dev/null +++ b/sandbox-netstack/frame_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/header" + "net/netip" + "syscall" + "testing" +) + +func testNetwork() network { + return network{netip.MustParseAddr("192.0.2.2"), netip.MustParseAddr("192.0.2.1"), [6]byte{2, 0, 0, 0, 0, 2}} +} +func ipv4Frame(src [4]byte, protocol uint8, destination [4]byte, port uint16) []byte { + b := make([]byte, 14+20+20) + header.Ethernet(b).Encode(&header.EthernetFields{SrcAddr: tcpip.LinkAddress("\x02\x00\x00\x00\x00\x02"), DstAddr: tcpip.LinkAddress("\x02\x00\x00\x00\x00\x01"), Type: header.IPv4ProtocolNumber}) + header.IPv4(b[14:]).Encode(&header.IPv4Fields{TotalLength: 40, Protocol: protocol, SrcAddr: tcpip.AddrFrom4(src), DstAddr: tcpip.AddrFrom4(destination)}) + if protocol == 17 { + header.UDP(b[34:]).Encode(&header.UDPFields{SrcPort: 10000, DstPort: port, Length: 20}) + } + return b +} +func TestPacketAdmission(t *testing.T) { + n := testNetwork() + valid := ipv4Frame([4]byte{192, 0, 2, 2}, 6, [4]byte{198, 51, 100, 7}, 8443) + if !n.accepts(valid, 0) { + t.Fatal("valid all-port TCP rejected") + } + spoof := append([]byte(nil), valid...) + spoof[6] = 4 + ipv6 := append([]byte(nil), valid...) + ipv6[12] = 0x86 + ipv6[13] = 0xdd + vlan := append([]byte(nil), valid...) + vlan[12] = 0x81 + vlan[13] = 0 + for name, b := range map[string][]byte{ + "short": {1, 2, 3}, "oversized": make([]byte, 1515), "mac": spoof, "ipv6": ipv6, "vlan": vlan, + "ip": ipv4Frame([4]byte{192, 0, 2, 3}, 6, [4]byte{198, 51, 100, 7}, 443), + "other_udp": ipv4Frame([4]byte{192, 0, 2, 2}, 17, [4]byte{192, 0, 2, 1}, 9000), + "other_dns": ipv4Frame([4]byte{192, 0, 2, 2}, 17, [4]byte{198, 51, 100, 53}, 53), + "icmp": ipv4Frame([4]byte{192, 0, 2, 2}, 1, [4]byte{198, 51, 100, 7}, 0), + } { + t.Run(name, func(t *testing.T) { + if n.accepts(b, 0) { + t.Fatal("unsafe frame admitted") + } + }) + } + if n.accepts(valid, syscall.MSG_TRUNC) { + t.Fatal("truncated datagram admitted") + } + if !n.accepts(ipv4Frame([4]byte{192, 0, 2, 2}, 17, [4]byte{192, 0, 2, 1}, 53), 0) { + t.Fatal("controlled DNS rejected") + } +} + +func TestARPSourceAndGatewayAreExact(t *testing.T) { + n := testNetwork() + b := make([]byte, header.EthernetMinimumSize+header.ARPSize) + header.Ethernet(b).Encode(&header.EthernetFields{SrcAddr: tcpip.LinkAddress(n.mac[:]), DstAddr: header.EthernetBroadcastAddress, Type: header.ARPProtocolNumber}) + a := header.ARP(b[14:]) + a.SetIPv4OverEthernet() + a.SetOp(header.ARPRequest) + copy(a.HardwareAddressSender(), n.mac[:]) + copy(a.ProtocolAddressSender(), []byte{192, 0, 2, 2}) + copy(a.ProtocolAddressTarget(), []byte{192, 0, 2, 1}) + if !n.accepts(b, 0) { + t.Fatal("valid gateway ARP rejected") + } + for _, offset := range []int{14 + 8, 14 + 14, 14 + 24} { + forged := append([]byte(nil), b...) + forged[offset] ^= 1 + if n.accepts(forged, 0) { + t.Fatalf("forged ARP field at %d accepted", offset) + } + } +} diff --git a/sandbox-netstack/go.mod b/sandbox-netstack/go.mod new file mode 100644 index 000000000..09a2af056 --- /dev/null +++ b/sandbox-netstack/go.mod @@ -0,0 +1,15 @@ +module engrams/sandbox-netstack + +go 1.26.2 + +require ( + golang.org/x/net v0.44.0 + gvisor.dev/gvisor v0.0.0-20260413194555-9680d69bf798 +) + +require ( + github.com/google/btree v1.1.2 // indirect + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/time v0.12.0 // indirect +) diff --git a/sandbox-netstack/go.sum b/sandbox-netstack/go.sum new file mode 100644 index 000000000..27255c920 --- /dev/null +++ b/sandbox-netstack/go.sum @@ -0,0 +1,12 @@ +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= +golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +gvisor.dev/gvisor v0.0.0-20260413194555-9680d69bf798 h1:7h21uRYtzPJjXp9W/BZLx68z8TwW8+h5BPdI4P5ZkJ0= +gvisor.dev/gvisor v0.0.0-20260413194555-9680d69bf798/go.mod h1:xQ2PWgHmWJA/Ph4i1q1jBm39BKhc3W0DXqWoDSyuBOY= diff --git a/sandbox-netstack/halfclose_test.go b/sandbox-netstack/halfclose_test.go new file mode 100644 index 000000000..3d1444865 --- /dev/null +++ b/sandbox-netstack/halfclose_test.go @@ -0,0 +1,196 @@ +package main + +import ( + "context" + "io" + "net" + "testing" + "time" + + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/network/ipv4" +) + +// Use run's TCP forwarder, packet path, and descriptor handoff for each flow. +func halfCloseFlow(t *testing.T) (*gonet.TCPConn, *net.UnixConn, *net.UnixConn) { + t.Helper() + packet, wire := datagramPair(t) + parent, control, err := socketPair() + if err != nil { + t.Fatal(err) + } + proxy, private, err := socketPair() + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- run(packet, control, private) }() + flows := make(chan *net.UnixConn, 1) + fixtureDone := make(chan struct{}) + go func() { + defer close(fixtureDone) + for { + var r request + if readFrame(proxy, &r) != nil { + return + } + state := "accepted" + switch r.Operation.Kind { + case "bind": + state = "ready" + case "close": + state = "closed" + case "tcp": + c, err := receiveFD(proxy) + if err != nil { + return + } + flows <- c + default: + return + } + if writeFrame(proxy, reply{1, r.Sequence, r.Epoch, state}) != nil || state == "closed" { + return + } + } + }() + t.Cleanup(func() { + parent.Close() + proxy.Close() + wire.Close() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Error("helper did not join") + } + select { + case <-fixtureDone: + case <-time.After(time.Second): + t.Error("proxy fixture did not join") + } + select { + case c := <-flows: + c.Close() + default: + } + }) + parent.SetDeadline(time.Now().Add(2 * time.Second)) + if err = writeFrame(parent, config{1, "192.0.2.2", "192.0.2.1", "02:00:00:00:00:02"}); err != nil { + t.Fatal(err) + } + var response reply + if err = readFrame(parent, &response); err != nil || response.State != "closed" { + t.Fatalf("readiness: %v %v", response, err) + } + if err = writeFrame(parent, command{1, "open", 7}); err != nil { + t.Fatal(err) + } + if err = readFrame(parent, &response); err != nil || response.State != "ready" { + t.Fatalf("open: %v %v", response, err) + } + s, stopGuest := guestStack(t, wire) + t.Cleanup(stopGuest) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + guest, err := gonet.DialContextTCP(ctx, s, tcpip.FullAddress{NIC: 1, Addr: tcpip.AddrFrom4([4]byte{198, 51, 100, 7}), Port: 8443}, ipv4.ProtocolNumber) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { guest.Close() }) + guest.SetDeadline(time.Now().Add(5 * time.Second)) + select { + case upstream := <-flows: + t.Cleanup(func() { upstream.Close() }) + upstream.SetDeadline(time.Now().Add(5 * time.Second)) + return guest, upstream, parent + case <-ctx.Done(): + t.Fatal("no TCP descriptor handoff") + return nil, nil, nil + } +} + +func TestUpstreamHalfClosePreservesGuestUpload(t *testing.T) { + guest, upstream, _ := halfCloseFlow(t) + if _, err := guest.Write([]byte("request-start")); err != nil { + t.Fatal(err) + } + initial := make([]byte, len("request-start")) + if _, err := io.ReadFull(upstream, initial); err != nil || string(initial) != "request-start" { + t.Fatalf("initial upload: %q %v", initial, err) + } + if _, err := upstream.Write([]byte("response-complete")); err != nil { + t.Fatal(err) + } + if err := upstream.CloseWrite(); err != nil { + t.Fatal(err) + } + if response, err := io.ReadAll(guest); err != nil || string(response) != "response-complete" { + t.Fatalf("response and EOF: %q %v", response, err) + } + if _, err := guest.Write([]byte("distinct-trailing-upload")); err != nil { + t.Fatalf("upload after response EOF: %v", err) + } + if err := guest.CloseWrite(); err != nil { + t.Fatal(err) + } + if trailing, err := io.ReadAll(upstream); err != nil || string(trailing) != "distinct-trailing-upload" { + t.Fatalf("trailing upload and EOF: %q %v", trailing, err) + } +} + +func TestGuestHalfClosePreservesUpstreamResponse(t *testing.T) { + guest, upstream, _ := halfCloseFlow(t) + if _, err := guest.Write([]byte("complete-request")); err != nil { + t.Fatal(err) + } + if _, err := upstream.Write([]byte("response-start")); err != nil { + t.Fatal(err) + } + initial := make([]byte, len("response-start")) + if _, err := io.ReadFull(guest, initial); err != nil || string(initial) != "response-start" { + t.Fatalf("initial response: %q %v", initial, err) + } + if err := guest.CloseWrite(); err != nil { + t.Fatal(err) + } + if request, err := io.ReadAll(upstream); err != nil || string(request) != "complete-request" { + t.Fatalf("request and EOF: %q %v", request, err) + } + if _, err := upstream.Write([]byte("distinct-trailing-response")); err != nil { + t.Fatal(err) + } + if err := upstream.CloseWrite(); err != nil { + t.Fatal(err) + } + if trailing, err := io.ReadAll(guest); err != nil || string(trailing) != "distinct-trailing-response" { + t.Fatalf("trailing response and EOF: %q %v", trailing, err) + } +} + +func TestCloseJoinsActiveHalfClosedTCP(t *testing.T) { + guest, upstream, parent := halfCloseFlow(t) + if err := upstream.CloseWrite(); err != nil { + t.Fatal(err) + } + if response, err := io.ReadAll(guest); err != nil || len(response) != 0 { + t.Fatalf("response EOF: %q %v", response, err) + } + if _, err := guest.Write([]byte("still-active")); err != nil { + t.Fatal(err) + } + active := make([]byte, len("still-active")) + if _, err := io.ReadFull(upstream, active); err != nil || string(active) != "still-active" { + t.Fatalf("active upload before close: %q %v", active, err) + } + if err := writeFrame(parent, command{1, "close", 7}); err != nil { + t.Fatal(err) + } + var response reply + if err := readFrame(parent, &response); err != nil || response.State != "closed" { + t.Fatalf("close did not join active flow: %v %v", response, err) + } + if data, err := io.ReadAll(upstream); err != nil || len(data) != 0 { + t.Fatalf("revoked upstream: %q %v", data, err) + } +} diff --git a/sandbox-netstack/helper_test.go b/sandbox-netstack/helper_test.go new file mode 100644 index 000000000..2365c3340 --- /dev/null +++ b/sandbox-netstack/helper_test.go @@ -0,0 +1,540 @@ +package main + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "golang.org/x/net/dns/dnsmessage" + "gvisor.dev/gvisor/pkg/buffer" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/header" + "gvisor.dev/gvisor/pkg/tcpip/link/channel" + "gvisor.dev/gvisor/pkg/tcpip/link/ethernet" + "gvisor.dev/gvisor/pkg/tcpip/network/arp" + "gvisor.dev/gvisor/pkg/tcpip/network/ipv4" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" + "io" + "net" + "os/exec" + "path/filepath" + "sync" + "syscall" + "testing" + "time" +) + +func datagramPair(t *testing.T) (*net.UnixConn, *net.UnixConn) { + t.Helper() + fd, e := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_DGRAM, 0) + if e != nil { + t.Fatal(e) + } + a, e := ownedUnix(fd[0]) + if e != nil { + t.Fatal(e) + } + b, e := ownedUnix(fd[1]) + if e != nil { + t.Fatal(e) + } + return a, b +} +func TestParentLossClosesClosedHelper(t *testing.T) { + packet, guest := datagramPair(t) + defer guest.Close() + parent, control, e := socketPair() + if e != nil { + t.Fatal(e) + } + defer parent.Close() + proxy, private, e := socketPair() + if e != nil { + t.Fatal(e) + } + defer proxy.Close() + done := make(chan error, 1) + go func() { done <- run(packet, control, private) }() + if e = writeFrame(parent, config{1, "192.0.2.2", "192.0.2.1", "02:00:00:00:00:02"}); e != nil { + t.Fatal(e) + } + parent.SetReadDeadline(time.Now().Add(2 * time.Second)) + var r reply + if e = readFrame(parent, &r); e != nil || r.State != "closed" { + t.Fatalf("closed readiness: %v %v", r, e) + } + parent.Close() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("parent loss left helper alive") + } + proxy.SetReadDeadline(time.Now().Add(time.Second)) + var b [1]byte + if n, _ := proxy.Read(b[:]); n != 0 { + t.Fatal("private data after parent loss") + } +} + +// The fixture uses only inherited Unix streams. It is a transport peer, not a +// replacement for the Rust policy tests in engram-egress-proxyd. +func receiveFD(c *net.UnixConn) (*net.UnixConn, error) { + var carrier [1]byte + control := make([]byte, syscall.CmsgSpace(4)) + n, o, flags, _, e := c.ReadMsgUnix(carrier[:], control) + if e != nil { + return nil, e + } + if n != 1 || carrier[0] != 0xED || flags != 0 { + return nil, fmt.Errorf("bad descriptor carrier") + } + messages, e := syscall.ParseSocketControlMessage(control[:o]) + if e != nil { + return nil, e + } + fds, e := syscall.ParseUnixRights(&messages[0]) + if e != nil { + return nil, e + } + if len(fds) != 1 { + return nil, fmt.Errorf("bad descriptor count") + } + return ownedUnix(fds[0]) +} +func TestNetstackAllTCPAndControlledDNS(t *testing.T) { + packet, guest := datagramPair(t) + parent, control, e := socketPair() + if e != nil { + t.Fatal(e) + } + proxy, private, e := socketPair() + if e != nil { + t.Fatal(e) + } + done := make(chan error, 1) + go func() { done <- run(packet, control, private) }() + t.Cleanup(func() { + parent.Close() + proxy.Close() + guest.Close() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Error("helper did not join") + } + }) + events := make(chan operation, 16) + fixtureDone := make(chan struct{}) + go func() { + defer close(fixtureDone) + defer proxy.Close() + var flows sync.WaitGroup + defer flows.Wait() + for { + var r request + if readFrame(proxy, &r) != nil { + return + } + state := "accepted" + if r.Operation.Kind == "bind" { + state = "ready" + } else if r.Operation.Kind == "close" { + state = "closed" + } else { + c, e := receiveFD(proxy) + if e != nil { + return + } + events <- r.Operation + flows.Go(func() { + defer c.Close() + c.SetDeadline(time.Now().Add(2 * time.Second)) + if r.Operation.Kind == "tcp" { + _, _ = io.Copy(c, c) + return + } + var length [2]byte + if _, e := io.ReadFull(c, length[:]); e != nil { + return + } + data := make([]byte, binary.BigEndian.Uint16(length[:])) + if _, e := io.ReadFull(c, data); e != nil { + return + } + var q dnsmessage.Message + if q.Unpack(data) != nil { + return + } + q.Response = true + q.RecursionAvailable = true + q.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Questions[0].Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET, TTL: 1}, Body: &dnsmessage.AResource{A: [4]byte{198, 51, 100, 7}}}} + out, e := q.Pack() + if e != nil { + return + } + binary.BigEndian.PutUint16(length[:], uint16(len(out))) + _, _ = c.Write(append(length[:], out...)) + }) + } + if writeFrame(proxy, reply{1, r.Sequence, r.Epoch, state}) != nil { + return + } + if state == "closed" { + return + } + } + }() + t.Cleanup(func() { + proxy.Close() + select { + case <-fixtureDone: + case <-time.After(3 * time.Second): + t.Error("fixture did not join") + } + }) + if e = writeFrame(parent, config{1, "192.0.2.2", "192.0.2.1", "02:00:00:00:00:02"}); e != nil { + t.Fatal(e) + } + var response reply + if e = readFrame(parent, &response); e != nil { + t.Fatal(e) + } + // Queue a valid TCP SYN while closed. The helper must discard it before OPEN. + closedSYN := ipv4Frame([4]byte{192, 0, 2, 2}, 6, [4]byte{198, 51, 100, 7}, 8443) + ip := header.IPv4(closedSYN[14:]) + ip.SetTTL(64) + ip.SetChecksum(^ip.CalculateChecksum()) + segment := header.TCP(closedSYN[34:]) + segment.Encode(&header.TCPFields{SrcPort: 12345, DstPort: 8443, SeqNum: 1, DataOffset: 20, Flags: header.TCPFlagSyn, WindowSize: 65535}) + segment.SetChecksum(^segment.CalculateChecksum(header.PseudoHeaderChecksum(tcp.ProtocolNumber, ip.SourceAddress(), ip.DestinationAddress(), 20))) + for range 8 { + guest.Write(closedSYN) + } + if e = writeFrame(parent, command{1, "open", 7}); e != nil { + t.Fatal(e) + } + if e = readFrame(parent, &response); e != nil || response.State != "ready" { + t.Fatalf("open: %v %v", response, e) + } + select { + case op := <-events: + t.Fatalf("closed traffic replayed: %v", op) + default: + } + guest.SetReadDeadline(time.Now().Add(40 * time.Millisecond)) + var unexpected [1514]byte + if size, e := guest.Read(unexpected[:]); e == nil { + t.Fatalf("closed SYN replay produced %d output bytes", size) + } + guest.SetReadDeadline(time.Time{}) + s, stopGuest := guestStack(t, guest) + defer stopGuest() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + for _, port := range []uint16{443, 8443, 80, 53} { + c, e := gonet.DialContextTCP(ctx, s, tcpip.FullAddress{NIC: 1, Addr: tcpip.AddrFrom4([4]byte{198, 51, 100, 7}), Port: port}, ipv4.ProtocolNumber) + if e != nil { + t.Fatalf("TCP %d: %v", port, e) + } + c.SetDeadline(time.Now().Add(2 * time.Second)) + sentinel := []byte("session=forged") + if _, e = c.Write(sentinel); e != nil { + t.Fatal(e) + } + got := make([]byte, len(sentinel)) + _, e = io.ReadFull(c, got) + c.Close() + if e != nil || !bytes.Equal(got, sentinel) { + t.Fatalf("TCP echo: %q %v", got, e) + } + select { + case op := <-events: + if op.Kind != "tcp" || op.Port != port || op.Destination != "198.51.100.7" { + t.Fatalf("tuple: %v", op) + } + case <-time.After(2 * time.Second): + t.Fatalf("no handoff for completed TCP echo on port %d", port) + } + } + q := dnsmessage.Message{Header: dnsmessage.Header{ID: 19, RecursionDesired: true}, Questions: []dnsmessage.Question{{Name: dnsmessage.MustNewName("fixture.example."), Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}}} + query, _ := q.Pack() + for _, transport := range []string{"udp", "tcp"} { + address := tcpip.FullAddress{NIC: 1, Addr: tcpip.AddrFrom4([4]byte{192, 0, 2, 1}), Port: 53} + var c net.Conn + if transport == "udp" { + c, e = gonet.DialUDP(s, nil, &address, ipv4.ProtocolNumber) + } else { + c, e = gonet.DialContextTCP(ctx, s, address, ipv4.ProtocolNumber) + } + if e != nil { + t.Fatal(e) + } + c.SetDeadline(time.Now().Add(2 * time.Second)) + out := query + if transport == "tcp" { + out = append([]byte{byte(len(query) >> 8), byte(len(query))}, query...) + } + if _, e = c.Write(out); e != nil { + t.Fatal(e) + } + data := make([]byte, 4096) + var count int + if transport == "tcp" { + var size [2]byte + _, e = io.ReadFull(c, size[:]) + if e == nil { + count = int(binary.BigEndian.Uint16(size[:])) + _, e = io.ReadFull(c, data[:count]) + } + } else { + count, e = c.Read(data) + } + c.Close() + if e != nil { + t.Fatalf("DNS %s: %v", transport, e) + } + var answer dnsmessage.Message + if e = answer.Unpack(data[:count]); e != nil || len(answer.Answers) != 1 { + t.Fatalf("DNS %s: %v %v", transport, answer, e) + } + select { + case op := <-events: + if op.Kind != "dns" || op.Destination != "192.0.2.1" { + t.Fatalf("DNS tuple: %v", op) + } + case <-time.After(2 * time.Second): + t.Fatal("no DNS handoff") + } + } + if e = writeFrame(parent, command{1, "close", 7}); e != nil { + t.Fatal(e) + } + if e = readFrame(parent, &response); e != nil || response.State != "closed" { + t.Fatalf("close: %v %v", response, e) + } +} +func guestStack(t *testing.T, wire *net.UnixConn) (*stack.Stack, func()) { + t.Helper() + ep := channel.New(64, 1500, tcpip.LinkAddress("\x02\x00\x00\x00\x00\x02")) + s := stack.New(stack.Options{NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, arp.NewProtocol}, TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol}}) + if e := s.CreateNIC(1, ethernet.New(ep)); e != nil { + t.Fatal(e) + } + if e := s.AddProtocolAddress(1, tcpip.ProtocolAddress{Protocol: ipv4.ProtocolNumber, AddressWithPrefix: tcpip.AddressWithPrefix{Address: tcpip.AddrFrom4([4]byte{192, 0, 2, 2}), PrefixLen: 24}}, stack.AddressProperties{}); e != nil { + t.Fatal(e) + } + s.SetRouteTable([]tcpip.Route{{Destination: header.IPv4EmptySubnet, Gateway: tcpip.AddrFrom4([4]byte{192, 0, 2, 1}), NIC: 1}}) + ctx, cancel := context.WithCancel(context.Background()) + var work sync.WaitGroup + work.Go(func() { + for { + p := ep.ReadContext(ctx) + if p == nil { + return + } + v := p.ToView() + _, e := wire.Write(v.AsSlice()) + v.Release() + p.DecRef() + if e != nil { + return + } + } + }) + work.Go(func() { + b := make([]byte, 1515) + for { + n, e := wire.Read(b) + if e != nil { + return + } + p := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buffer.MakeWithData(append([]byte(nil), b[:n]...))}) + ep.InjectInbound(header.Ethernet(b[:n]).Type(), p) + p.DecRef() + } + }) + return s, func() { + cancel() + wire.Close() + ep.Close() + s.Close() + work.Wait() + s.Wait() + } +} + +func TestInheritedProcessLosesParentAndProxy(t *testing.T) { + binary := filepath.Join(t.TempDir(), "netstack") + build := exec.Command("go", "build", "-mod=readonly", "-trimpath", "-o", binary, ".") + if output, e := build.CombinedOutput(); e != nil { + t.Fatalf("build helper: %v %s", e, output) + } + for _, lost := range []string{"parent", "proxy"} { + t.Run(lost, func(t *testing.T) { + packet, guest := datagramPair(t) + defer guest.Close() + parent, control, e := socketPair() + if e != nil { + t.Fatal(e) + } + defer parent.Close() + proxy, private, e := socketPair() + if e != nil { + t.Fatal(e) + } + defer proxy.Close() + cmd := exec.Command(binary) + cmd.Env = []string{} + f0, e := packet.File() + if e != nil { + t.Fatal(e) + } + f1, e := control.File() + if e != nil { + t.Fatal(e) + } + f2, e := private.File() + if e != nil { + t.Fatal(e) + } + cmd.Stdin = f0 + cmd.Stdout = f1 + cmd.Stderr = f2 + e = cmd.Start() + f0.Close() + f1.Close() + f2.Close() + packet.Close() + control.Close() + private.Close() + if e != nil { + t.Fatal(e) + } + waited := make(chan error, 1) + go func() { waited <- cmd.Wait() }() + t.Cleanup(func() { + cmd.Process.Kill() + select { + case <-waited: + case <-time.After(2 * time.Second): + t.Error("child was not reaped") + } + }) + if e = writeFrame(parent, config{1, "192.0.2.2", "192.0.2.1", "02:00:00:00:00:02"}); e != nil { + t.Fatal(e) + } + parent.SetReadDeadline(time.Now().Add(2 * time.Second)) + var ready reply + if e = readFrame(parent, &ready); e != nil || ready.State != "closed" { + t.Fatalf("child ready: %v %v", ready, e) + } + if lost == "parent" { + parent.Close() + } else { + proxy.Close() + } + select { + case e = <-waited: + if e == nil { + t.Fatal("lost channel exit must be nonzero") + } + waited <- e + case <-time.After(2 * time.Second): + t.Fatal("lost channel left helper running") + } + }) + } +} + +func TestCloseJoinsHalfOpenTCP(t *testing.T) { + packet, guest := datagramPair(t) + defer guest.Close() + parent, control, e := socketPair() + if e != nil { + t.Fatal(e) + } + defer parent.Close() + proxy, private, e := socketPair() + if e != nil { + t.Fatal(e) + } + defer proxy.Close() + done := make(chan error, 1) + go func() { done <- run(packet, control, private) }() + t.Cleanup(func() { + parent.Close() + proxy.Close() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Error("half-open helper did not join") + } + }) + fixture := make(chan struct{}) + go func() { + defer close(fixture) + defer proxy.Close() + for { + var r request + if readFrame(proxy, &r) != nil { + return + } + state := "ready" + if r.Operation.Kind == "close" { + state = "closed" + } + if r.Operation.Kind != "bind" && r.Operation.Kind != "close" { + return + } + if writeFrame(proxy, reply{1, r.Sequence, r.Epoch, state}) != nil || state == "closed" { + return + } + } + }() + t.Cleanup(func() { + proxy.Close() + select { + case <-fixture: + case <-time.After(time.Second): + t.Error("fixture not joined") + } + }) + if e = writeFrame(parent, config{1, "192.0.2.2", "192.0.2.1", "02:00:00:00:00:02"}); e != nil { + t.Fatal(e) + } + var response reply + if readFrame(parent, &response) != nil { + t.Fatal("no readiness") + } + if e = writeFrame(parent, command{1, "open", 7}); e != nil { + t.Fatal(e) + } + if readFrame(parent, &response) != nil { + t.Fatal("no open ack") + } + syn := ipv4Frame([4]byte{192, 0, 2, 2}, 6, [4]byte{198, 51, 100, 7}, 8443) + ip := header.IPv4(syn[14:]) + ip.SetTTL(64) + ip.SetChecksum(^ip.CalculateChecksum()) + segment := header.TCP(syn[34:]) + segment.Encode(&header.TCPFields{SrcPort: 12345, DstPort: 8443, SeqNum: 1, DataOffset: 20, Flags: header.TCPFlagSyn, WindowSize: 65535}) + segment.SetChecksum(^segment.CalculateChecksum(header.PseudoHeaderChecksum(tcp.ProtocolNumber, ip.SourceAddress(), ip.DestinationAddress(), 20))) + if _, e = guest.Write(syn); e != nil { + t.Fatal(e) + } + guest.SetReadDeadline(time.Now().Add(time.Second)) + var packetOut [1514]byte + if _, e = guest.Read(packetOut[:]); e != nil { + t.Fatalf("no handshake or neighbor request: %v", e) + } + if e = writeFrame(parent, command{1, "close", 7}); e != nil { + t.Fatal(e) + } + parent.SetReadDeadline(time.Now().Add(3 * time.Second)) + if e = readFrame(parent, &response); e != nil || response.State != "closed" { + t.Fatalf("close did not join half-open TCP: %v %v", response, e) + } +} diff --git a/sandbox-netstack/main.go b/sandbox-netstack/main.go new file mode 100644 index 000000000..2cc20dcc0 --- /dev/null +++ b/sandbox-netstack/main.go @@ -0,0 +1,500 @@ +// The only inherited descriptors are 0=packet, 1=parent control, 2=proxy +// capability. None is a log. No arguments, environment config, or listener. +package main + +import ( + "context" + "encoding/binary" + "errors" + "golang.org/x/net/dns/dnsmessage" + "gvisor.dev/gvisor/pkg/buffer" + "gvisor.dev/gvisor/pkg/tcpip" + "gvisor.dev/gvisor/pkg/tcpip/adapters/gonet" + "gvisor.dev/gvisor/pkg/tcpip/header" + "gvisor.dev/gvisor/pkg/tcpip/link/channel" + "gvisor.dev/gvisor/pkg/tcpip/link/ethernet" + "gvisor.dev/gvisor/pkg/tcpip/network/arp" + "gvisor.dev/gvisor/pkg/tcpip/network/ipv4" + "gvisor.dev/gvisor/pkg/tcpip/stack" + "gvisor.dev/gvisor/pkg/tcpip/transport/tcp" + "gvisor.dev/gvisor/pkg/tcpip/transport/udp" + "gvisor.dev/gvisor/pkg/waiter" + "io" + "net" + "os" + "sync" + "syscall" + "time" +) + +const maxFlows = 64 +const maxDNS = 8 + +type gate struct { + mu sync.Mutex + open, dead bool + active, dns int + conns map[net.Conn]bool + work sync.WaitGroup +} + +func (g *gate) enter(dns bool) bool { + g.mu.Lock() + defer g.mu.Unlock() + if !g.open || g.dead || g.active >= maxFlows || (dns && g.dns >= maxDNS) { + return false + } + g.active++ + if dns { + g.dns++ + } + g.work.Add(1) + return true +} +func (g *gate) leave(dns bool) { + g.mu.Lock() + g.active-- + if dns { + g.dns-- + } + g.mu.Unlock() + g.work.Done() +} +func (g *gate) track(c net.Conn) bool { + g.mu.Lock() + defer g.mu.Unlock() + if !g.open || g.dead { + c.Close() + return false + } + g.conns[c] = true + return true +} +func (g *gate) untrack(c net.Conn) { c.Close(); g.mu.Lock(); delete(g.conns, c); g.mu.Unlock() } +func (g *gate) close() { + g.mu.Lock() + defer g.mu.Unlock() + g.open = false + g.dead = true + for c := range g.conns { + c.Close() + } +} +func main() { + if len(os.Args) != 1 { + os.Exit(1) + } + packet, e := ownedUnix(0) + if e != nil { + os.Exit(1) + } + control, e := ownedUnix(1) + if e != nil { + packet.Close() + os.Exit(1) + } + private, e := ownedUnix(2) + if e != nil { + packet.Close() + control.Close() + os.Exit(1) + } + if run(packet, control, private) != nil { + os.Exit(1) + } +} +func run(packet, control, private *net.UnixConn) error { + defer packet.Close() + defer control.Close() + defer private.Close() + var cfg config + control.SetReadDeadline(time.Now().Add(handoffTimeout)) + if err := readFrame(control, &cfg); err != nil { + return err + } + n, err := parseNetwork(cfg) + if err != nil { + return err + } + // Socket types are part of the attachment contract. + for conn, want := range map[*net.UnixConn]int{packet: syscall.SOCK_DGRAM, control: syscall.SOCK_STREAM, private: syscall.SOCK_STREAM} { + raw, e := conn.SyscallConn() + if e != nil { + return e + } + var actual int + var inner error + if e = raw.Control(func(fd uintptr) { actual, inner = syscall.GetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_TYPE) }); e != nil { + return e + } + if inner != nil || actual != want { + return errors.New("wrong private descriptor type") + } + } + g := &gate{conns: make(map[net.Conn]bool)} + stop := make(chan struct{}) + var once sync.Once + fail := func() { + once.Do(func() { + g.close() + close(stop) + packet.Close() + control.Close() + // Interrupt a pending handoff write without sending cleanup EOF. + private.SetWriteDeadline(time.Now()) + }) + } + defer fail() + i := &ingress{conn: private, replies: make(chan reply), stop: stop, fail: fail} + var channels sync.WaitGroup + defer func() { + fail() + // This defer runs after stack, packet, and flow cleanup below. EOF on + // this reliable stream is the proxy's cleanup acknowledgement. + private.SetReadDeadline(time.Now()) + channels.Wait() + private.Close() + }() + channels.Go(i.read) + commands := make(chan command) + channels.Go(func() { + for { + var c command + if readFrame(control, &c) != nil { + fail() + return + } + select { + case commands <- c: + case <-stop: + return + } + } + }) + ep := channel.New(64, mtu, tcpip.LinkAddress(gatewayMAC)) + s := stack.New(stack.Options{NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, arp.NewProtocol}, TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol}}) + defer func() { s.Close(); s.Wait() }() + if e := s.CreateNIC(1, ethernet.New(ep)); e != nil { + return errors.New(e.String()) + } + address := tcpip.AddrFrom4(n.gateway.As4()) + if e := s.AddProtocolAddress(1, tcpip.ProtocolAddress{Protocol: ipv4.ProtocolNumber, AddressWithPrefix: address.WithPrefix()}, stack.AddressProperties{}); e != nil { + return errors.New(e.String()) + } + if e := s.SetPromiscuousMode(1, true); e != nil { + return errors.New(e.String()) + } + if e := s.SetSpoofing(1, true); e != nil { + return errors.New(e.String()) + } + s.SetRouteTable([]tcpip.Route{{Destination: header.IPv4EmptySubnet, NIC: 1}}) + // Conservative per-flow buffers and bounded SYN retries. There is no load + // tuning knob at this boundary. + for _, option := range []tcpip.SettableTransportProtocolOption{ + &tcpip.TCPSendBufferSizeRangeOption{Min: 4096, Default: 65536, Max: 65536}, + &tcpip.TCPReceiveBufferSizeRangeOption{Min: 4096, Default: 65536, Max: 65536}, + } { + if e := s.SetTransportProtocolOption(tcp.ProtocolNumber, option); e != nil { + return errors.New(e.String()) + } + } + retries := tcpip.TCPSynRetriesOption(1) + if e := s.SetTransportProtocolOption(tcp.ProtocolNumber, &retries); e != nil { + return errors.New(e.String()) + } + exchangeDNS := func(data []byte) ([]byte, error) { + var m dnsmessage.Message + if len(data) == 0 || len(data) > dnsLimit { + return nil, errors.New("DNS size") + } + if e := m.Unpack(data); e != nil { + return nil, e + } + if m.Response || len(m.Questions) != 1 { + return nil, errors.New("DNS query shape") + } + c, e := i.flow("dns", n.gateway.String(), 53) + if e != nil { + return nil, e + } + if !g.track(c) { + return nil, errors.New("closed") + } + defer g.untrack(c) + c.SetDeadline(time.Now().Add(10 * time.Second)) + var size [2]byte + binary.BigEndian.PutUint16(size[:], uint16(len(data))) + if _, e = c.Write(append(size[:], data...)); e != nil { + return nil, e + } + if _, e = io.ReadFull(c, size[:]); e != nil { + return nil, e + } + length := int(binary.BigEndian.Uint16(size[:])) + if length == 0 || length > dnsLimit { + return nil, errors.New("DNS reply size") + } + out := make([]byte, length) + if _, e = io.ReadFull(c, out); e != nil { + return nil, e + } + var response dnsmessage.Message + if e = response.Unpack(out); e != nil { + return nil, e + } + if !response.Response || response.ID != m.ID || len(response.Questions) != 1 || response.Questions[0] != m.Questions[0] { + return nil, errors.New("DNS reply mismatch") + } + return out, nil + } + forward := tcp.NewForwarder(s, 65536, 16, func(r *tcp.ForwarderRequest) { + id := r.ID() + dns := id.LocalAddress == address && id.LocalPort == 53 + if id.RemoteAddress != tcpip.AddrFrom4(n.guest.As4()) || !g.enter(dns) { + r.Complete(true) + return + } + defer g.leave(dns) + var queue waiter.Queue + endpoint, tcpErr := r.CreateEndpoint(&queue) + r.Complete(false) + if tcpErr != nil { + return + } + c := gonet.NewTCPConn(&queue, endpoint) + if !g.track(c) { + return + } + defer g.untrack(c) + c.SetDeadline(time.Now().Add(5 * time.Minute)) + if dns { + c.SetDeadline(time.Now().Add(10 * time.Second)) + var size [2]byte + if _, e := io.ReadFull(c, size[:]); e != nil { + return + } + length := int(binary.BigEndian.Uint16(size[:])) + if length == 0 || length > dnsLimit { + return + } + data := make([]byte, length) + if _, e := io.ReadFull(c, data); e != nil { + return + } + out, e := exchangeDNS(data) + if e != nil { + return + } + binary.BigEndian.PutUint16(size[:], uint16(len(out))) + // Close sets the gate under this same lock; no response can start after it. + g.mu.Lock() + if g.open && !g.dead { + c.SetWriteDeadline(time.Now().Add(time.Second)) + _, _ = c.Write(append(size[:], out...)) + } + g.mu.Unlock() + return + } + upstream, e := i.flow("tcp", id.LocalAddress.String(), id.LocalPort) + if e != nil { + return + } + if !g.track(upstream) { + return + } + defer g.untrack(upstream) + upstream.SetDeadline(time.Now().Add(5 * time.Minute)) + finished := make(chan struct{}) + go func() { _, _ = io.Copy(upstream, c); upstream.CloseWrite(); close(finished) }() + _, _ = io.Copy(c, upstream) + c.CloseWrite() + <-finished + }) + s.SetTransportProtocolHandler(tcp.ProtocolNumber, forward.HandlePacket) + udpConn, e := gonet.DialUDP(s, &tcpip.FullAddress{NIC: 1, Addr: address, Port: 53}, nil, ipv4.ProtocolNumber) + if e != nil { + return e + } + var workers sync.WaitGroup + workers.Go(func() { + b := make([]byte, dnsLimit+1) + for { + length, peer, e := udpConn.ReadFrom(b) + if e != nil { + return + } + if length > dnsLimit || !g.enter(true) { + continue + } + data := append([]byte(nil), b[:length]...) + go func() { + defer g.leave(true) + out, e := exchangeDNS(data) + if e != nil { + return + } + g.mu.Lock() + defer g.mu.Unlock() + if g.open && !g.dead { + udpConn.SetWriteDeadline(time.Now().Add(time.Second)) + _, _ = udpConn.WriteTo(out, peer) + } + }() + } + }) + workers.Go(func() { + for { + p := ep.ReadContext(context.Background()) + if p == nil { + return + } + v := p.ToView() + var writeErr error + g.mu.Lock() + if g.open && !g.dead { + packet.SetWriteDeadline(time.Now().Add(250 * time.Millisecond)) + _, writeErr = packet.Write(v.AsSlice()) + } + g.mu.Unlock() + v.Release() + p.DecRef() + if writeErr != nil { + fail() + return + } + } + }) + activate := make(chan chan error) + workers.Go(func() { + b := make([]byte, mtu+15) + for { + select { + case result := <-activate: + // The receiver owns this drain. No pre-OPEN datagram remains queued, and + // an input flood fails activation instead of extending an unbounded drain. + raw, e := packet.SyscallConn() + if e == nil { + controlErr := raw.Control(func(fd uintptr) { + for count := 0; count < 512; count++ { + _, _, _, _, x := syscall.Recvmsg(int(fd), b, nil, syscall.MSG_DONTWAIT) + if x == syscall.EAGAIN || x == syscall.EWOULDBLOCK { + return + } + if x != nil { + e = x + return + } + } + e = errors.New("closed packet queue did not drain") + }) + if controlErr != nil { + e = controlErr + } + } + g.mu.Lock() + if e == nil && !g.dead { + g.open = true + } else if e == nil { + e = errors.New("closed") + } + g.mu.Unlock() + result <- e + default: + } + packet.SetReadDeadline(time.Now().Add(25 * time.Millisecond)) + length, _, flags, _, e := packet.ReadMsgUnix(b, nil) + if e != nil { + if timeout, ok := e.(net.Error); ok && timeout.Timeout() { + continue + } + g.mu.Lock() + dead := g.dead + g.mu.Unlock() + if !dead { + fail() + } + return + } + g.mu.Lock() + open := g.open && !g.dead + g.mu.Unlock() + if !open || !n.accepts(b[:length], flags) { + continue + } + p := stack.NewPacketBuffer(stack.PacketBufferOptions{Payload: buffer.MakeWithData(append([]byte(nil), b[:length]...))}) + ep.InjectInbound(header.Ethernet(b[:length]).Type(), p) + p.DecRef() + } + }) + var cleanOnce sync.Once + cleanup := func() { + cleanOnce.Do(func() { + g.close() + packet.Close() + udpConn.Close() + ep.Close() + s.Close() + workers.Wait() + g.work.Wait() + s.Close() + s.Wait() + }) + } + defer func() { cleanup(); fail() }() + if e := writeFrame(control, reply{1, 0, 0, "closed"}); e != nil { + return e + } + opened := false + for { + select { + case <-stop: + return errors.New("private channel closed") + case c := <-commands: + if c.Version != 1 || c.Epoch == 0 || c.Epoch > 9007199254740991 { + return errors.New("invalid command") + } + switch c.Kind { + case "open": + if opened { + return errors.New("allocation already used") + } + opened = true + i.epoch = c.Epoch + if e := i.call(operation{Kind: "bind"}, nil); e != nil { + return e + } + result := make(chan error, 1) + select { + case activate <- result: + case <-stop: + return errors.New("closed") + } + select { + case e := <-result: + if e != nil { + return e + } + case <-stop: + return errors.New("closed") + } + if e := writeFrame(control, reply{1, 1, c.Epoch, "ready"}); e != nil { + return e + } + case "close": + if !opened || c.Epoch != i.epoch { + return errors.New("wrong epoch") + } + cleanup() + if e := i.call(operation{Kind: "close"}, nil); e != nil { + return e + } + if e := writeFrame(control, reply{1, 2, c.Epoch, "closed"}); e != nil { + return e + } + return nil + default: + return errors.New("unknown command") + } + } + } +} diff --git a/sandbox-netstack/protocol.go b/sandbox-netstack/protocol.go new file mode 100644 index 000000000..ffdeb9194 --- /dev/null +++ b/sandbox-netstack/protocol.go @@ -0,0 +1,225 @@ +package main + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "errors" + "io" + "net" + "os" + "sync" + "syscall" + "time" +) + +const headerLimit = 2048 +const dnsLimit = 4096 +const handoffTimeout = 5 * time.Second + +type operation struct { + Kind string `json:"kind"` + Destination string `json:"destination,omitempty"` + Port uint16 `json:"port,omitempty"` +} +type request struct { + Version uint32 `json:"version"` + Sequence uint64 `json:"sequence"` + Epoch uint64 `json:"epoch"` + Operation operation `json:"operation"` +} +type reply struct { + Version uint32 `json:"version"` + Sequence uint64 `json:"sequence"` + Epoch uint64 `json:"epoch"` + State string `json:"state"` +} +type command struct { + Version uint32 `json:"version"` + Kind string `json:"kind"` + Epoch uint64 `json:"epoch"` +} + +func readFrame(c *net.UnixConn, v any) error { + var head [4]byte + // Idle reliable channels stay live; partial frames have a fixed budget. + if _, err := io.ReadFull(c, head[:1]); err != nil { + return err + } + if err := c.SetReadDeadline(time.Now().Add(handoffTimeout)); err != nil { + return err + } + defer c.SetReadDeadline(time.Time{}) + if _, err := io.ReadFull(c, head[1:]); err != nil { + return err + } + length := binary.BigEndian.Uint32(head[:]) + if length == 0 || length > headerLimit { + return errors.New("invalid private header") + } + data := make([]byte, length) + if _, err := io.ReadFull(c, data); err != nil { + return err + } + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + if err := d.Decode(v); err != nil { + return err + } + if d.Decode(new(any)) != io.EOF { + return errors.New("extra private data") + } + return nil +} +func writeFrame(c *net.UnixConn, v any) error { + body, err := json.Marshal(v) + if err != nil { + return err + } + if len(body) > headerLimit { + return errors.New("private header too large") + } + data := make([]byte, 4, len(body)+4) + binary.BigEndian.PutUint32(data, uint32(len(body))) + data = append(data, body...) + if err = c.SetWriteDeadline(time.Now().Add(handoffTimeout)); err != nil { + return err + } + defer c.SetWriteDeadline(time.Time{}) + for len(data) > 0 { + n, e := c.Write(data) + if e != nil { + return e + } + data = data[n:] + } + return nil +} +func socketPair() (*net.UnixConn, *net.UnixConn, error) { + fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0) + if err != nil { + return nil, nil, err + } + syscall.CloseOnExec(fds[0]) + syscall.CloseOnExec(fds[1]) + a, e := ownedUnix(fds[0]) + if e != nil { + syscall.Close(fds[1]) + return nil, nil, e + } + b, e := ownedUnix(fds[1]) + if e != nil { + a.Close() + return nil, nil, e + } + return a, b, nil +} +func ownedUnix(fd int) (*net.UnixConn, error) { + f := os.NewFile(uintptr(fd), "private-network") + defer f.Close() + c, err := net.FileConn(f) + if err != nil { + return nil, err + } + unix, ok := c.(*net.UnixConn) + if !ok { + c.Close() + return nil, errors.New("expected Unix descriptor") + } + return unix, nil +} +func sendFD(c *net.UnixConn, fd *net.UnixConn) error { + raw, err := fd.SyscallConn() + if err != nil { + return err + } + var writeErr error + err = raw.Control(func(value uintptr) { + _, _, writeErr = c.WriteMsgUnix([]byte{0xED}, syscall.UnixRights(int(value)), nil) + }) + if err != nil { + return err + } + return writeErr +} + +type ingress struct { + conn *net.UnixConn + mu sync.Mutex + sequence, epoch uint64 + replies chan reply + stop <-chan struct{} + fail func() +} + +func (i *ingress) read() { + for { + var r reply + if readFrame(i.conn, &r) != nil { + i.fail() + return + } + select { + case i.replies <- r: + case <-i.stop: + return + } + if r.State == "closed" && r.Sequence > 0 { + return + } + } +} +func (i *ingress) call(op operation, fd *net.UnixConn) error { + i.mu.Lock() + defer i.mu.Unlock() + select { + case <-i.stop: + return errors.New("closed") + default: + } + i.sequence++ + if err := writeFrame(i.conn, request{1, i.sequence, i.epoch, op}); err != nil { + i.fail() + return err + } + if fd != nil { + i.conn.SetWriteDeadline(time.Now().Add(handoffTimeout)) + defer i.conn.SetWriteDeadline(time.Time{}) + if err := sendFD(i.conn, fd); err != nil { + i.fail() + return err + } + } + timer := time.NewTimer(handoffTimeout) + defer timer.Stop() + select { + case r := <-i.replies: + want := "accepted" + if op.Kind == "bind" { + want = "ready" + } + if op.Kind == "close" { + want = "closed" + } + if r.Version == 1 && r.Sequence == i.sequence && r.Epoch == i.epoch && r.State == want { + return nil + } + case <-timer.C: + case <-i.stop: + } + i.fail() + return errors.New("private handoff failed") +} +func (i *ingress) flow(kind, destination string, port uint16) (*net.UnixConn, error) { + a, b, err := socketPair() + if err != nil { + i.fail() + return nil, err + } + defer b.Close() + if err = i.call(operation{kind, destination, port}, b); err != nil { + a.Close() + return nil, err + } + return a, nil +} diff --git a/tests/fixtures/mcp-model-cli/Cargo.lock b/tests/fixtures/mcp-model-cli/Cargo.lock new file mode 100644 index 000000000..eef9b3f94 --- /dev/null +++ b/tests/fixtures/mcp-model-cli/Cargo.lock @@ -0,0 +1,191 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "engram-mcp-model-fixture" +version = "0.1.0" +dependencies = [ + "libc", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tests/fixtures/mcp-model-cli/Cargo.toml b/tests/fixtures/mcp-model-cli/Cargo.toml new file mode 100644 index 000000000..b15cfa29e --- /dev/null +++ b/tests/fixtures/mcp-model-cli/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "engram-mcp-model-fixture" +version = "0.1.0" +edition = "2021" +publish = false + +[workspace] + +[dependencies] +serde_json = { version = "=1.0.154", features = ["unbounded_depth"] } +serde = "=1.0.229" +sha2 = "=0.10.9" +libc = "=0.2.191" + +[[bin]] +name = "fixture-mock-bridge" +path = "tests/support/mock_bridge.rs" diff --git a/tests/fixtures/mcp-model-cli/README.md b/tests/fixtures/mcp-model-cli/README.md new file mode 100644 index 000000000..29306e062 --- /dev/null +++ b/tests/fixtures/mcp-model-cli/README.md @@ -0,0 +1,315 @@ +# MCP model CLI fixture + +This test-only program replaces the model CLI. It does not replace either +harness or authorize an MCP call. Claude uses the generated strict MCP config +and hook settings. Codex uses app-server JSON-RPC. By default, both use the call ID +`fixture-call` and send `{"fixture":""}` as tool arguments. The selected +manifest tool must accept that input and have no native binding. Use a neutral +fixture tool name, description, and data. + +Set these variables for each fixture process: + +- `ENGRAM_FIXTURE_TOOL`: the exact manifest tool name. +- `ENGRAM_FIXTURE_MODE`: `read`, `write`, `denied`, `refire`, `held-call`, or + `forged` (call an unlisted name and require wrapper rejection). + The mode is an input sentinel. Policy and the harness determine the outcome. +- `ENGRAM_FIXTURE_ARGUMENTS_JSON`: optional JSON object, at most 4096 UTF-8 + bytes and 16 nested containers (the root object has depth 1). Duplicate keys + at any depth, trailing data, and non-object values are rejected. The exact + parsed value is used in the assistant frame, hook, bridge, and dynamic call. +- `ENGRAM_FIXTURE_CALL_ID`: optional ASCII `[A-Za-z0-9_-]{1,64}`. All call and + proof locations use it, including a resumed or repeated call. The default + remains `fixture-call`. +- `ENGRAM_FIXTURE_EXPECTED_RESULT_JSON`: optional literal native result object, + at most 1 MiB. Duplicate keys and excessive parser depth are rejected. + Compare the actual native object, as described below. Do not put private + data in the environment. The operating system can impose a smaller total + environment limit than this parser limit. +- `ENGRAM_FIXTURE_STATE_DIR`: an absolute, private mode-0700 directory owned by + the test. No component may be a symlink. +- `ENGRAM_FIXTURE_PROOF`: a new direct child of that directory. The fixture + creates it exclusively. An optional single `{launch}` in the basename expands + to `-<32 lowercase hex characters>`. The hex suffix uses + 16 fresh bytes from the operating system's `/dev/urandom`. Failure to open or + read all 16 bytes fails closed. For example, `proof-{launch}.json` produces + `proof-42-000102030405060708090a0b0c0d0e0f.json`. A new launch can retain + earlier proof files even when a cold boot reuses the process ID. + The existing optional single `{pid}` still expands to the decimal fixture + process ID; a literal basename is unchanged. PID-only names can collide after + a cold boot. Tokens cannot mix or repeat. Unknown or unmatched braces are + rejected before protocol output. The parent is checked before expansion. + A collision fails closed; the fixture never retries, replaces, or deletes an + existing proof. Launch names are local observations, not credentials or + session, call, run, or binding identities. No environment variable can supply + the PID, random bytes, or random source. +- `ENGRAM_FIXTURE_BARRIER`: optional direct child of that directory. While it is + absent, a call waits for up to 30 seconds; an interrupt can end the wait. + Create a regular file at that path to release it. A symlink is rejected. + +The default proof contains only the call ID, the fixture tool name, booleans for +the neutral `sentinel`, `second sentinel`, and `{"fixture":true}` result +values, and bounded native outcome and delivery fields. It records a CLI-side +observation. It is not evidence that the journal or an upstream service +committed a result. Empty proof means no result was recorded. Never reuse a +proof path. Proof and diagnostic output do not contain arguments or +tool payloads. Native protocol stdout carries the actual call and result. `--fixture-version` prints `engram-mcp-model-fixture 1`. + +With any of the three optional scalar variables set, proof also records +`selectedToolName`, `selectedSchemaSha256`, and `exposedMcpToolNamesSha256`. +The first two come from the single matching +member of the actual Claude `tools/list` or Codex `dynamicTools` array. The caller +must verify the alias and schema against the public session binding before the +prompt. The exposure hash covers the sorted array of actual exposed names that +start with `mcp_`. It rejects invalid names, duplicates, and more than 128 MCP +names. The caller compares this hash against the complete public binding set. +This fixture does not select tools by description or infer their alias. + +When an expected object is configured and native output is available, proof adds +`resultMatchesExpected`, `resultSha256`, and `expectedResultSha256`. A mismatch +records `false`. SHA-256 values have the prefix `sha256:` and 64 lowercase hex +digits. Hash input is compact JSON with recursively sorted object keys. Array +order, absent fields, and text strings are preserved. No arguments, expected +values, or received output bytes are stored in proof or error output. + +Claude compares the JSON-RPC `result` object, including `_meta`. Codex compares +the complete `item/tool/call` response `result` object, including `success` and +all `contentItems`. JSON text inside those items remains literal text. For +example, a count of 1 with a successful receipt and `resultUnavailable=false` +has these two expected objects: + +```json +{"content":[{"type":"text","text":"count=1"}],"structuredContent":{"count":1},"isError":false,"_meta":{"engrams/toolResult":{"outcome":"succeeded","delivery":"available","resultUnavailable":false,"isError":false}}} +``` + +```json +{"success":true,"contentItems":[{"type":"inputText","text":"count=1"},{"type":"inputText","text":"{\"structuredContent\":{\"count\":1}}"},{"type":"inputText","text":"{\"delivery\":\"available\",\"isError\":false,\"outcome\":\"succeeded\",\"resultUnavailable\":false}"}]} +``` + +Only fields actually sent by the runtime belong in the expected value. In +particular, do not add an omitted `resultUnavailable` or `isError` field. +An unavailable delivery has no comparison fields. A transport error does not +become a tool error. The existing receipt fields remain separate, and a hash +match does not prove an upstream effect. The legacy `resultReceived` flag says +that a native completion was observed, including an unavailable receipt; it +does not establish that tool output is available. Codex receipt flags use the +last text item, which the native encoder reserves for receipt metadata. + +A scalar configuration has one fixed call ID and one fixed tool. For a bounded +sequence, set `ENGRAM_FIXTURE_CALL_PLAN_JSON` instead of TOOL, ARGUMENTS_JSON, +CALL_ID and EXPECTED_RESULT_JSON. These scalar variables must be absent in plan +mode. The plan is at most 65536 UTF-8 bytes and has exactly a `steps` array of +1 to 16 objects. Each step has `marker`, `tool`, `arguments`, `callId`, +`resultFormat` (`mcp` or `json`), and optional `expectedResult`, `expectExposure`, +and `completionProjection`. Unknown fields +are rejected. Markers and call IDs use the call-ID grammar. Markers must be +unique. Call IDs must be unique except for the fixed rebind pair below. +Arguments have the scalar size and depth bounds. The whole-plan size limit also +bounds the optional literal expected native objects. + +The current user prompt must contain exactly one `[[fixture:MARKER]]` token. +Claude reads user `message.content` text; Codex reads current `turn/start` input +text items. A missing, unknown, repeated, or ambiguous token fails closed. Tool +result blocks cannot select a step. The same marker always selects the same +fixed tool, arguments, and call ID. A new marker can select a new call ID in the +same session. A fixed `spawn_session` message can contain a child marker, so the +child selects its MCP read step rather than the spawn step. No argument or +command interpolation is supported. + +```json +{"steps":[{"marker":"read","tool":"mcp_example_01234567890123456789","arguments":{},"callId":"read-1","resultFormat":"mcp"},{"marker":"spawn","tool":"spawn_session","arguments":{"task_name":"fixture-child","message":"[[fixture:read]]","idempotency_key":"child-1"},"callId":"spawn-1","resultFormat":"json"}]} +``` + +Plan proof includes `planSha256`, the canonical digest of the entire immutable +plan. A native JSON step hashes its literal native output even without an +expected object. With an expected object, it also records the comparison and +expected hash. Claude native JSON output is `{"content":[{"type":"text", +"text":""}]}`. Codex uses `{"success":true,"contentItems": +[{"type":"inputText","text":""}]}`. Native JSON proof has +no MCP receipt fields; Codex retains its actual `success` flag. The fixture does +not decode those text strings or assert that a child was actually created. + +Each actual response appends one JSON line to the same process proof file and +flushes it before the native tool-complete frame, then the turn-complete frame. +Barrier and adversarial observations use separate lines. A live reader must +parse only complete newline-terminated records. Use `{launch}` to claim a new +file for each process launch when the state directory survives a cold boot. +For `proof-{launch}.json`, discovery must match +`^proof-[0-9]+-[a-f0-9]{32}\.json$` with no PID-only fallback. A filename alone +cannot select an observation or advance a plan. Read proof as JSONL when there +are multiple turns; use the call ID and plan digest to select the required +observation. Never merge proof files. + +A direct child of the private state directory, `plan-selection.json`, contains +only planSha256, harness, mode, marker and pending. The fixture writes this +receipt atomically before it emits a call. It marks the selection completed +after an actual native response or after the fixed adversarial event is emitted +and flushed. A denied absent-alias response also completes the selection. It stores no arguments or result bytes. +A deferred Claude `--resume` in write/refire mode has no current prompt; it can +repeat only a pending selection for the exact same plan, harness and mode. +Missing, completed, malformed, mismatched or symlink receipts fail closed. +A replay cannot advance to another step. A new ID requires a current marked +prompt. This is a test-client retry; the production journal remains responsible +for authorization, idempotency and effects. A session needs its own state +directory, including when a parent and child use the same fixed plan. + + +A plan step can set `expectExposure` to `present` (the default) or `absent`. +The absent case requires an `mcp_` alias, `resultFormat: "mcp"`, no expected +result, and no completion projection. The fixture checks that the exact alias +is absent from the actual current tool list, then sends that fixed alias through +the native request path. Claude must return JSON-RPC error `-32602` with message +`Unknown tool`. Codex must return `success: false` with the single native text +`undeclared dynamic tool: `. Any other response fails closed. This does +not relax the positive tool-list checks. + +Exactly two ordered steps may share a call ID for a rebind probe. The first +expects present exposure (explicitly or by default); the second explicitly +expects absent exposure. Both must have the same `mcp_` tool, exact canonical +arguments and MCP result format, with no completion projection. The first may +have an expected result; the second must not. All other duplicate IDs fail. +The runner must establish the actual rebind, then use the new marked prompt +and actual current exposed list. Select the new negative proof by its position +and `expectExposure`, so the prior positive proof cannot satisfy the check. + +Absent proof has `expectExposure: "absent"`, `requestAttempted: true`, and +`protocolDenialCode: "unknown_tool"`, plus the existing call, tool, plan and +exposure identities. Claude also records `nativeDenialCode: -32602`. There is +no selected schema, result receipt, outcome or effect claim. After this proof, +the fixture emits a normal native failure completion from the actual denial. + +For normal results, Claude emits a streamed `user` message with one `tool_result` +block. It carries the exact call ID, actual content, and actual `isError` when +present. JSON-RPC errors use their actual message and `is_error: true`. Codex +emits `item/completed` with `type: "dynamicToolCall"`, the exact call ID/tool, +actual success, and the complete actual native result. Native JSON responses +remain native JSON; no MCP receipt is added. These native events let the real +wrapper report tool completion. Only the actual coordinator outbox can prove ACK. + +`ENGRAM_FIXTURE_COMPLETION_BARRIER`, when set, must equal +`STATE_DIR/completion-release`. The fixture first validates the native result +and flushes its proof. It then records `observation: "completionBarrierHeld"` +and waits before it emits native completion. The runner can inspect the real +unacknowledged outbox and restart its services while this barrier is held. + +The release must be a newly created empty regular file, mode 0600, with one link +and the same owner as the private state directory. Create it exclusively. The +fixture rejects a preexisting release at startup and before each call. It opens +without following symlinks, checks file identity and metadata, then removes only +that release file. The fixed timeout is 120 seconds. No timeout environment +setting exists. Input EOF, interrupt, unexpected input, invalid release or timeout +fails closed without native completion. A valid interrupt receives the native +interrupt reply before the process exits with failure. A release records +`observation: "completionBarrierReleased"` before normal completion. + +The optional plan field `completionProjection: "premature"` selects one fixed +adversarial probe. It requires a present `mcp_` alias, MCP result format, no +expected result, and `ENGRAM_FIXTURE_PREMATURE_BARRIER` equal to +`STATE_DIR/premature-release`. This barrier uses the same release rules and +120-second bound. The legacy `forged` mode cannot combine with either new probe. + +Claude must first observe an actual `defer` hook decision for the known call. +Codex first emits the actual dynamic request and has no native response. The +fixture records `observation: "prematureBarrierHeld"`. The runner must observe +actual waiting approval and zero upstream effects before release. A Codex +response received while held fails closed. After release, the fixture records +`prematureBarrierReleased`, then emits the fixed native success projection: +Claude has empty content and `is_error: false`; Codex has `success: true` and +no result field. No caller can supply event fields or a result payload. + +After the adversarial event is emitted and flushed, proof records only +`observation: "prematureCompletionEmitted"`, its canonical `eventSha256`, and +the call/tool/plan identities. It clears the local pending selection to prevent +promptless replay. This proves emission only. Interrupted or unreleased probes +do not clear pending selection. The real journal, outbox and upstream counter +must establish whether authorization or an effect occurred. A projected +completion can retire a call or run; a later approval for that call may fail. +Use a distinct valid call to test subsequent approved execution. + +All barrier observations contain only the observation name, call/tool identity, +and plan digest when configured. They do not establish result receipt or an +upstream effect. Use one private state directory per session. + +Build this standalone workspace and run its nonignored protocol tests: + +```sh +export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$PWD/target}" +cargo test --locked --manifest-path tests/fixtures/mcp-model-cli/Cargo.toml +cargo build --locked --release --target aarch64-unknown-linux-musl \ + --manifest-path tests/fixtures/mcp-model-cli/Cargo.toml --bin engram-mcp-model-fixture +file "$CARGO_TARGET_DIR/aarch64-unknown-linux-musl/release/engram-mcp-model-fixture" +shasum -a 256 "$CARGO_TARGET_DIR/aarch64-unknown-linux-musl/release/engram-mcp-model-fixture" +``` + +Run these commands from the repository root and set `CARGO_TARGET_DIR` to a +different owned directory if the root target is shared. +Stage the exact build and verify its digest. This workspace is excluded from +the main workspace, so normal workspace nextest does not run its tests. CI must +trigger and invoke this standalone target for changes in this directory and for +changes to either harness's CLI protocol. It must fail if the test count is +zero or the expected artifact is missing. + +The existing Codex adapter has an explicit fixture gate. Build the fixture for +the host first, then invoke the gate once with `ENGRAM_FIXTURE_MODE=read` and +again with `ENGRAM_FIXTURE_MODE=held-call`. For each run, set all required +variables above, `ENGRAM_FIXTURE_BIN` to the absolute path of that host binary, +and `ENGRAM_FIXTURE_TOOL=remote_inventory`. Use a fresh proof path and state +directory each time. Run: + +```sh +cargo test --locked -p engram-harness-codex --bin engram-harness-codex \ + tests::external_mcp_static_fixture_adapter_gate -- --ignored --exact +``` + +Require exactly one selected and passed test in each run. The gate checks the +fixture version, real adapter call and result, or cancellation event and +interrupted turn. It does not boot a guest or call a provider. Claude's local +fixture tests use a mock subprocess; they do not prove the production bridge. +The Linux guest gate must stage the current production Claude wrapper and this +fixture, set `ENGRAM_CLAUDE_BIN` to the staged fixture, supply the immutable +manifest and the state variables above, and drive the real harness transport. +It must verify the actual hook, bridge, result and cancellation paths, then +repeat the required cold and full-stack cases. A local fixture pass cannot +replace that gate. Remove only the test-owned state and staged files after all +children have exited. + +The additive fixture contract still uses version 1. Claude emits its neutral +native assistant tool-use frame before the hook. In `write` and `refire` modes, +`--resume` makes it reissue the same call once without a new stdin prompt, +as the production deferred-result path expects. Other modes still wait for a +prompt. A harness test bounds launch count to four and uses private state. + +On Linux, `.github/scripts/mcp-acceptance-linux.sh` builds both current shipping +wrapper binaries and the fixture, checks executable ELF artifacts and fixture +version, then runs exactly eight integration tests in `linux_mcp_entrypoint`. +Each test exercises both wrappers through TCP `HarnessAttach`, sequenced events, +and event ACKs. It uses a fresh environment and owned state. The refire case +kills and observes the reaped direct model child; the wrapper remains alive. +Claude resumes the deferred call; Codex resumes its thread and a new turn +reissues the same ID. The latter emits a second host request; server journal +idempotency is outside this test. The result sender is the test host, so denial +receipt delivery is not evidence of server authorization. Forged names are +rejected by the actual wrappers before a host request. Cancellation checks the +signal and absent model result, without a late result or an external effect. + +This process test closes the shipping Linux entrypoint gap. Real guest, +Firecracker/VZ isolation, cold restore, authenticated server policy, worker +network enforcement, journal/outbox durability, and pinned vendor CLI behavior +remain separate acceptance gates. `HarnessAttach` does not negotiate the global +wire version. CI builds the test and wrappers from the same source revision. + +The Linux gate runs under the test-only `linux_mcp_supervisor` Cargo example. +Before starting the test driver, it becomes a Linux child subreaper. A forced +100 ms deadline check starts both real wrappers with held calls, kills the test +process with SIGKILL, and confirms that its four observed wrapper/model children +remain alive in separate groups. The supervisor then kills only its own +kernel-reported direct children, reaps adopted descendants until `ECHILD`, and +checks that the observed processes are gone before it removes state. It repeats +this cleanup after the ordinary eight-case matrix, including timeout/failure. + +The matrix has a 100 second internal deadline; cleanup waits are bounded. +A 240 second hard outer deadline bounds supervisor failure. The shell removes +state only after the supervisor writes its cleanup-complete marker. If cleanup +cannot be confirmed, the gate fails and retains state. The supervisor is an +explicitly built example, not a production binary or a libtest discovery target. +The deliberately held `supervisor_timeout_fixture` test is ignored and excluded +from the eight-case matrix; only the supervisor runs it. diff --git a/tests/fixtures/mcp-model-cli/src/evidence.rs b/tests/fixtures/mcp-model-cli/src/evidence.rs new file mode 100644 index 000000000..9b5e2cee9 --- /dev/null +++ b/tests/fixtures/mcp-model-cli/src/evidence.rs @@ -0,0 +1,230 @@ +//! Fixed native completion probes. These observations are not execution evidence. +use crate::{digest, emit, Config, Result}; +use serde_json::{json, Value}; +use std::fs::{self, OpenOptions}; +use std::io::{self, Write}; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +pub fn barrier(root: &Path, env: &str, name: &str) -> Result> { + let Some(raw) = std::env::var_os(env) else { + return Ok(None); + }; + let path = PathBuf::from(raw); + if path != root.join(name) { + return Err("invalid evidence barrier path".into()); + } + require_absent(&path)?; + Ok(Some(path)) +} +pub fn require_absent(path: &Path) -> Result<()> { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + _ => Err("evidence release already exists".into()), + } +} +fn consume_release(path: &Path, owner: u32) -> Result { + let before = match fs::symlink_metadata(path) { + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error.into()), + Ok(metadata) => metadata, + }; + let valid = |m: &fs::Metadata| { + m.file_type().is_file() + && m.len() == 0 + && m.permissions().mode() & 0o7777 == 0o600 + && m.nlink() == 1 + && m.uid() == owner + }; + if !valid(&before) { + return Err("invalid evidence release".into()); + } + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(path)?; + let opened = file.metadata()?; + let after = fs::symlink_metadata(path)?; + if !valid(&opened) + || !valid(&after) + || before.ino() != opened.ino() + || before.dev() != opened.dev() + || after.ino() != opened.ino() + || after.dev() != opened.dev() + { + return Err("evidence release changed".into()); + } + fs::remove_file(path)?; + Ok(true) +} +fn wait_until( + path: &Path, + owner: u32, + input: &mpsc::Receiver>, + deadline: Instant, +) -> Result> { + loop { + // A queued interrupt or native result wins over release. + match input.try_recv() { + Ok(line) => return Ok(Some(serde_json::from_str(&line?)?)), + Err(mpsc::TryRecvError::Disconnected) => { + return Err("input closed at evidence barrier".into()) + } + Err(mpsc::TryRecvError::Empty) => {} + } + if Instant::now() >= deadline { + return Err("evidence barrier deadline exceeded".into()); + } + if consume_release(path, owner)? { + return Ok(None); + } + match input.recv_timeout(Duration::from_millis(20)) { + Ok(line) => return Ok(Some(serde_json::from_str(&line?)?)), + Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err("input closed at evidence barrier".into()) + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + } +} +impl Config { + pub fn evidence_observation(&mut self, observation: &str, event: Option<&Value>) -> Result<()> { + let mut proof = json!({"observation":observation,"callId":self.call_id,"tool":self.tool}); + if let Some(plan) = &self.plan { + proof["planSha256"] = json!(plan.hash); + } + if let Some(event) = event { + proof["eventSha256"] = json!(digest(event)?); + } + writeln!(self.proof, "{proof}")?; + self.proof.flush()?; + Ok(()) + } + pub fn prepare_evidence(&self) -> Result<()> { + if (self.absent || self.premature) && self.mode == "forged" { + return Err("conflicting evidence mode".into()); + } + if self.premature && self.premature_barrier.is_none() { + return Err("premature barrier required".into()); + } + for path in [&self.completion_barrier, &self.premature_barrier] + .into_iter() + .flatten() + { + require_absent(path)?; + } + Ok(()) + } + fn evidence_wait( + &mut self, + premature: bool, + input: &mpsc::Receiver>, + ) -> Result<()> { + let path = if premature { + &self.premature_barrier + } else { + &self.completion_barrier + }; + let Some(path) = path.clone() else { + return Ok(()); + }; + self.evidence_observation( + if premature { + "prematureBarrierHeld" + } else { + "completionBarrierHeld" + }, + None, + )?; + if let Some(value) = wait_until( + &path, + fs::metadata(&self.root)?.uid(), + input, + Instant::now() + Duration::from_secs(120), + )? { + if self.harness == "claude" { + if value["type"] == "control_request" + && value.pointer("/request/subtype") == Some(&json!("interrupt")) + && value["request_id"].is_string() + { + emit( + json!({"type":"control_response","response":{"subtype":"success","request_id":value["request_id"]}}), + )?; + emit( + json!({"type":"result","subtype":"error_during_execution","session_id":"fixture-session"}), + )?; + } + } else if value["method"] == "turn/interrupt" && value.get("id").is_some() { + emit(json!({"id":value["id"],"result":{}}))?; + emit( + json!({"method":"turn/completed","params":{"threadId":"fixture-thread","turn":{"id":"fixture-turn","status":"interrupted"}}}), + )?; + } + return Err("input interrupted evidence barrier".into()); + } + self.evidence_observation( + if premature { + "prematureBarrierReleased" + } else { + "completionBarrierReleased" + }, + None, + ) + } + pub fn native_completion( + &mut self, + response: &Value, + input: &mpsc::Receiver>, + ) -> Result<()> { + let event = if self.harness == "claude" { + let mut block = json!({"type":"tool_result","tool_use_id":self.call_id}); + if let Some(result) = response.get("result") { + block["content"] = result + .get("content") + .ok_or("missing native content")? + .clone(); + if let Some(error) = result.get("isError") { + block["is_error"] = error.clone(); + } + } else { + block["content"] = json!(response + .pointer("/error/message") + .and_then(Value::as_str) + .ok_or("missing native error message")?); + block["is_error"] = json!(true); + } + json!({"type":"user","message":{"role":"user","content":[block]}}) + } else { + let result = response.get("result").ok_or("missing native result")?; + json!({"method":"item/completed","params":{"threadId":"fixture-thread","turnId":"fixture-turn","item":{"id":self.call_id,"type":"dynamicToolCall","tool":self.tool,"status":"completed","success":result["success"],"result":result}}}) + }; + self.evidence_wait(false, input)?; + emit(event) + } + pub fn premature_completion( + &mut self, + input: &mpsc::Receiver>, + ) -> Result<()> { + self.evidence_wait(true, input)?; + let event = if self.harness == "claude" { + json!({"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":self.call_id,"content":[],"is_error":false}]}}) + } else { + json!({"method":"item/completed","params":{"threadId":"fixture-thread","turnId":"fixture-turn","item":{"id":self.call_id,"type":"dynamicToolCall","tool":self.tool,"status":"completed","success":true}}}) + }; + emit(event.clone())?; + self.evidence_observation("prematureCompletionEmitted", Some(&event))?; + self.persist_selection(false) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn expired_deadline_cannot_consume_a_release() { + let (_tx, rx) = mpsc::channel(); + assert!(wait_until(Path::new("unused-release"), 0, &rx, Instant::now()).is_err()); + } +} diff --git a/tests/fixtures/mcp-model-cli/src/main.rs b/tests/fixtures/mcp-model-cli/src/main.rs new file mode 100644 index 000000000..14b6dc19a --- /dev/null +++ b/tests/fixtures/mcp-model-cli/src/main.rs @@ -0,0 +1,1038 @@ +mod evidence; +mod plan; +mod scalar; +use scalar::{configured_object, digest}; +use serde_json::{json, Value}; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, BufRead, BufReader, Write}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Component, Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +type Result = std::result::Result>; + +fn optional_bool(value: &Value, name: &str) -> Result> { + match value.get(name) { + Some(value) => Ok(Some(value.as_bool().ok_or("invalid boolean")?)), + None => Ok(None), + } +} +struct Config { + tool: String, + mode: String, + proof: File, + barrier: Option, + completion_barrier: Option, + premature_barrier: Option, + absent: bool, + premature: bool, + arguments: Value, + call_id: String, + expected: Option, + extended_proof: bool, + selected_schema: Option, + exposed_hash: Option, + exposed_tools: Vec, + plan: Option, + root: PathBuf, + marker: Option, + native_json: bool, + harness: &'static str, +} + +fn direct_child(root: &Path, path: &Path) -> bool { + path.parent() == Some(root) + && matches!(path.file_name(), Some(name) if !name.is_empty() && name != "." && name != "..") +} +fn proof_file( + root: &Path, + path: &Path, + pid: u32, + entropy: impl FnOnce() -> io::Result, +) -> Result { + if !direct_child(root, path) { + return Err("proof must be a direct child of the state directory".into()); + } + let basename = path + .file_name() + .and_then(|s| s.to_str()) + .ok_or("invalid proof name")?; + let token = ["{launch}", "{pid}"] + .into_iter() + .find(|token| basename.contains(token)); + let literal = token.map_or_else( + || basename.to_owned(), + |token| basename.replacen(token, "", 1), + ); + if literal.contains(['{', '}']) { + return Err("invalid proof token".into()); + } + // Launch-local observation only, never a session, call, or run identity. + let basename = match token { + Some("{launch}") => { + let mut bytes = [0; 16]; + entropy()?.read_exact(&mut bytes)?; + basename.replace( + "{launch}", + &format!("{pid}-{:032x}", u128::from_be_bytes(bytes)), + ) + } + Some(_) => basename.replace("{pid}", &pid.to_string()), + None => basename.to_owned(), + }; + Ok(OpenOptions::new() + .write(true) + .create_new(true) + .open(root.join(basename))?) +} +fn config() -> Result { + let root = PathBuf::from(std::env::var("ENGRAM_FIXTURE_STATE_DIR")?); + if !root.is_absolute() { + return Err("state directory must be absolute".into()); + } + let mut current = PathBuf::new(); + for component in root.components() { + if !matches!(component, Component::RootDir | Component::Normal(_)) { + return Err("invalid state directory".into()); + } + current.push(component); + if fs::symlink_metadata(¤t)?.file_type().is_symlink() { + return Err("state directory contains a symlink".into()); + } + } + let metadata = fs::metadata(&root)?; + if !metadata.is_dir() || metadata.permissions().mode() & 0o777 != 0o700 { + return Err("state directory must be private".into()); + } + let proof_path = PathBuf::from(std::env::var("ENGRAM_FIXTURE_PROOF")?); + let barrier = std::env::var_os("ENGRAM_FIXTURE_BARRIER").map(PathBuf::from); + if barrier + .as_ref() + .is_some_and(|path| !direct_child(&root, path)) + { + return Err("barrier must be a direct child of the state directory".into()); + } + let completion_barrier = evidence::barrier( + &root, + "ENGRAM_FIXTURE_COMPLETION_BARRIER", + "completion-release", + )?; + let premature_barrier = evidence::barrier( + &root, + "ENGRAM_FIXTURE_PREMATURE_BARRIER", + "premature-release", + )?; + let plan = plan::Plan::from_env()?; + let tool = if plan.is_some() { + "plan_pending".into() + } else { + std::env::var("ENGRAM_FIXTURE_TOOL")? + }; + if !plan::tool_name(&tool) { + return Err("invalid fixture tool".into()); + } + let mode = std::env::var("ENGRAM_FIXTURE_MODE")?; + if !["read", "write", "denied", "refire", "held-call", "forged"].contains(&mode.as_str()) { + return Err("unsupported fixture mode".into()); + } + let arguments = configured_object("ENGRAM_FIXTURE_ARGUMENTS_JSON", 4096, 16)?; + let expected = configured_object("ENGRAM_FIXTURE_EXPECTED_RESULT_JSON", 1024 * 1024, 128)?; + let configured_id = match std::env::var("ENGRAM_FIXTURE_CALL_ID") { + Ok(value) => Some(value), + Err(std::env::VarError::NotPresent) => None, + Err(_) => return Err("invalid fixture call ID".into()), + }; + let extended_proof = + plan.is_some() || arguments.is_some() || expected.is_some() || configured_id.is_some(); + let call_id = configured_id.unwrap_or_else(|| "fixture-call".into()); + if !plan::identifier(&call_id) { + return Err("invalid fixture call ID".into()); + } + let arguments = arguments.unwrap_or_else(|| json!({"fixture":mode})); + let proof = proof_file(&root, &proof_path, std::process::id(), || { + File::open("/dev/urandom") + })?; + Ok(Config { + tool, + mode, + proof, + barrier, + completion_barrier, + premature_barrier, + absent: false, + premature: false, + arguments, + call_id, + expected, + extended_proof, + selected_schema: None, + exposed_hash: None, + exposed_tools: Vec::new(), + plan, + root, + marker: None, + native_json: false, + harness: "unset", + }) +} +impl Config { + fn bind_tools(&mut self, tools: &[Value], harness: &'static str) -> Result<()> { + self.harness = harness; + if self.extended_proof { + self.exposed_hash = Some(plan::exposed_mcp_hash(tools)?); + } + self.exposed_tools = tools.to_vec(); + if self.plan.is_none() { + self.validate_selected()?; + } + Ok(()) + } + fn validate_selected(&mut self) -> Result<()> { + let selected: Vec<_> = self + .exposed_tools + .iter() + .filter(|tool| tool.get("name") == Some(&json!(self.tool))) + .collect(); + if self.absent { + if !selected.is_empty() { + return Err("negative alias is exposed".into()); + } + self.selected_schema = None; + return Ok(()); + } + if selected.len() != 1 + || !selected[0]["inputSchema"].is_object() + || (self.harness == "codex" && !selected[0]["description"].is_string()) + { + return Err("expected tool missing, duplicated, or malformed".into()); + } + self.selected_schema = Some(digest(&selected[0]["inputSchema"])?); + Ok(()) + } + fn select(&mut self, prompt: Option<&Value>) -> Result<()> { + let Some(plan) = &self.plan else { + return Ok(()); + }; + let step = match prompt { + Some(value) => plan.select(&plan::prompt_text(value, self.harness == "claude")?)?, + None => plan.resume(&self.root, self.harness, &self.mode)?, + }; + self.tool = step.tool; + self.arguments = step.arguments; + self.call_id = step.call_id; + self.expected = step.expected; + self.native_json = step.native_json; + self.absent = step.absent; + self.premature = step.premature; + self.marker = Some(step.marker); + self.validate_selected()?; + Ok(()) + } + fn persist_selection(&self, pending: bool) -> Result<()> { + if let Some(plan) = &self.plan { + plan.persist( + &self.root, + self.harness, + &self.mode, + self.marker.as_deref().ok_or("missing selected marker")?, + pending, + )?; + } + Ok(()) + } +} + +fn record(cfg: &mut Config, mut proof: Value, native: Option<&Value>) -> Result<()> { + if cfg.extended_proof { + proof["selectedToolName"] = json!(cfg.tool); + if !cfg.absent { + proof["selectedSchemaSha256"] = json!(cfg + .selected_schema + .as_ref() + .ok_or("missing selected schema")?); + } + proof["exposedMcpToolNamesSha256"] = json!(cfg + .exposed_hash + .as_ref() + .ok_or("missing exposed MCP hash")?); + } + if let Some(plan) = &cfg.plan { + proof["planSha256"] = json!(plan.hash); + } + if let Some(native) = native.filter(|_| cfg.expected.is_some() || cfg.native_json) { + if !native.is_object() || serde_json::to_vec(native)?.len() > 1024 * 1024 { + return Err("invalid native result size or shape".into()); + } + let actual_hash = digest(native)?; + if let Some(expected) = &cfg.expected { + let expected_hash = digest(expected)?; + proof["resultMatchesExpected"] = json!(actual_hash == expected_hash); + proof["expectedResultSha256"] = json!(expected_hash); + } + proof["resultSha256"] = json!(actual_hash); + } + writeln!(cfg.proof, "{proof}")?; + cfg.proof.flush()?; + cfg.persist_selection(false)?; + Ok(()) +} + +fn stdin_lines() -> mpsc::Receiver> { + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in io::stdin().lock().lines() { + if tx.send(line).is_err() { + break; + } + } + }); + rx +} +fn wait_barrier( + path: Option<&Path>, + input: &mpsc::Receiver>, +) -> Result> { + let Some(path) = path else { return Ok(None) }; + let deadline = Instant::now() + Duration::from_secs(30); + loop { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_file() => return Ok(None), + Ok(_) => return Err("invalid barrier".into()), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + if Instant::now() >= deadline { + return Err("barrier deadline exceeded".into()); + } + match input.recv_timeout(Duration::from_millis(20)) { + Ok(line) => return Ok(Some(serde_json::from_str(&line?)?)), + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => { + return Err("input closed at barrier".into()) + } + } + } +} +fn emit(value: Value) -> Result<()> { + let mut out = io::stdout().lock(); + writeln!(out, "{value}")?; + out.flush()?; + Ok(()) +} +fn codex(mut cfg: Config) -> Result<()> { + let mut initialized = false; + let mut thread_ready = false; + let mut pending = false; + let input = stdin_lines(); + while let Ok(line) = input.recv() { + let value: Value = serde_json::from_str(&line?)?; + if value.get("method").is_none() && value.get("id") == Some(&json!(77)) && pending { + if value.get("error").is_some() { + return Err("dynamic transport error".into()); + } + let result = value + .get("result") + .and_then(Value::as_object) + .ok_or("missing dynamic result")?; + let success = result + .get("success") + .and_then(Value::as_bool) + .ok_or("missing success")?; + if cfg.absent { + if success + || result.get("contentItems") + != Some( + &json!([{"type":"inputText","text":format!("undeclared dynamic tool: {}", cfg.tool)}]), + ) + { + return Err("absent alias was not rejected".into()); + } + let proof = json!({"callId":cfg.call_id,"tool":cfg.tool,"expectExposure":"absent","requestAttempted":true,"protocolDenialCode":"unknown_tool"}); + record(&mut cfg, proof, None)?; + cfg.native_completion(&value, &input)?; + emit( + json!({"method":"turn/completed","params":{"threadId":"fixture-thread","turn":{"id":"fixture-turn","status":"completed"}}}), + )?; + pending = false; + continue; + } + if cfg.mode == "forged" { + if success + || result.get("contentItems") + != Some( + &json!([{"type":"inputText","text":"undeclared dynamic tool: fixture_forged"}]), + ) + { + return Err("forged tool was not rejected".into()); + } + let proof = + json!({"callId":cfg.call_id,"tool":"fixture_forged","forgedRejected":true}); + record(&mut cfg, proof, None)?; + emit( + json!({"method":"turn/completed","params":{"threadId":"fixture-thread","turn":{"id":"fixture-turn","status":"completed"}}}), + )?; + pending = false; + continue; + } + let items = result + .get("contentItems") + .and_then(Value::as_array) + .ok_or("missing content items")?; + if items.iter().any(|item| { + item.get("type") != Some(&json!("inputText")) + || !item.get("text").is_some_and(Value::is_string) + }) { + return Err("unsupported content item".into()); + } + if cfg.native_json { + let proof = json!({"callId":cfg.call_id,"tool":cfg.tool,"resultReceived":true,"success":success}); + record(&mut cfg, proof, value.get("result"))?; + cfg.native_completion(&value, &input)?; + emit( + json!({"method":"turn/completed","params":{"threadId":"fixture-thread","turn":{"id":"fixture-turn","status":"completed"}}}), + )?; + pending = false; + continue; + } + let text_sentinel = items + .iter() + .any(|item| item == &json!({"type":"inputText","text":"sentinel"})); + let text_sequence = items.first() + == Some(&json!({"type":"inputText","text":"sentinel"})) + && items.get(1) == Some(&json!({"type":"inputText","text":"second sentinel"})); + let parsed: Vec = items + .iter() + .filter_map(|item| item.get("text").and_then(Value::as_str)) + .filter_map(|text| serde_json::from_str(text).ok()) + .collect(); + let structured = parsed + .iter() + .any(|item| item == &json!({"structuredContent":{"fixture":true}})); + let receipt: Value = serde_json::from_str( + items + .last() + .and_then(|item| item.get("text")) + .and_then(Value::as_str) + .ok_or("missing receipt")?, + )?; + let receipt = &receipt; + let delivery = receipt["delivery"] + .as_str() + .filter(|s| ["available", "unavailable"].contains(s)) + .ok_or("invalid delivery")?; + let outcome = receipt.get("outcome").and_then(Value::as_str); + if outcome.is_some_and(|s| { + !["succeeded", "failed", "canceled", "outcome_unknown"].contains(&s) + }) { + return Err("invalid outcome".into()); + } + let is_error = optional_bool(receipt, "isError")?; + let unavailable = optional_bool(receipt, "resultUnavailable")?; + let denial_plan = receipt.get("denial") == Some(&json!("plan_mode")); + let proof = json!({"callId":cfg.call_id,"tool":cfg.tool,"resultReceived":true,"success":success,"textSentinel":text_sentinel,"textSequence":text_sequence,"structuredSentinel":structured,"isError":is_error,"outcome":outcome,"delivery":delivery,"resultUnavailable":unavailable,"denialPlanMode":denial_plan}); + let native = value.get("result").filter(|_| delivery == "available"); + record(&mut cfg, proof, native)?; + cfg.native_completion(&value, &input)?; + emit( + json!({"method":"turn/completed","params":{"threadId":"fixture-thread","turn":{"id":"fixture-turn","status":"completed"}}}), + )?; + pending = false; + continue; + } + let method = value + .get("method") + .and_then(Value::as_str) + .ok_or("missing method")?; + let id = value.get("id").cloned(); + match method { + "initialized" if initialized && id.is_none() => {} + "initialize" if !initialized => { + initialized = true; + emit(json!({"id":id.ok_or("missing id")?,"result":{}}))?; + } + "account/login/start" if initialized => { + emit(json!({"id":id.ok_or("missing id")?,"result":{}}))? + } + "account/read" if initialized => { + emit(json!({"id":id.ok_or("missing id")?,"result":{"account":{"type":"chatgpt"}}}))? + } + "thread/start" | "thread/resume" if initialized && !thread_ready => { + if method == "thread/resume" + && value.pointer("/params/threadId") != Some(&json!("fixture-thread")) + { + return Err("invalid resumed thread".into()); + } + let tools = value + .pointer("/params/dynamicTools") + .and_then(Value::as_array) + .ok_or("missing dynamic tools")?; + cfg.bind_tools(tools, "codex")?; + thread_ready = true; + emit( + json!({"id":id.ok_or("missing id")?,"result":{"thread":{"id":"fixture-thread","turns":[]}}}), + )?; + } + "turn/start" if thread_ready && !pending => { + cfg.select(Some(&value))?; + cfg.prepare_evidence()?; + emit( + json!({"id":id.ok_or("missing id")?,"result":{"turn":{"id":"fixture-turn","status":"inProgress"}}}), + )?; + if let Some(interrupt) = wait_barrier(cfg.barrier.as_deref(), &input)? { + if interrupt["method"] != "turn/interrupt" { + return Err("unexpected barrier input".into()); + } + emit( + json!({"id":interrupt.get("id").ok_or("missing interrupt id")?,"result":{}}), + )?; + emit( + json!({"method":"turn/completed","params":{"threadId":"fixture-thread","turn":{"id":"fixture-turn","status":"interrupted"}}}), + )?; + continue; + } + cfg.persist_selection(true)?; + emit( + json!({"id":77,"method":"item/tool/call","params":{"callId":cfg.call_id,"tool":if cfg.mode == "forged" { "fixture_forged" } else { &cfg.tool },"arguments":cfg.arguments,"threadId":"fixture-thread","turnId":"fixture-turn"}}), + )?; + if cfg.premature { + cfg.premature_completion(&input)?; + emit( + json!({"method":"turn/completed","params":{"threadId":"fixture-thread","turn":{"id":"fixture-turn","status":"completed"}}}), + )?; + } else { + pending = true; + } + } + "turn/interrupt" if pending => { + emit(json!({"id":id.ok_or("missing id")?,"result":{}}))?; + emit( + json!({"method":"turn/completed","params":{"threadId":"fixture-thread","turn":{"id":"fixture-turn","status":"interrupted"}}}), + )?; + pending = false; + } + _ => return Err("unsupported app-server message".into()), + } + } + if pending { + return Err("dynamic result missing".into()); + } + Ok(()) +} + +struct ChildGuard(Child); +impl Drop for ChildGuard { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn read_lines(child: &mut Child) -> Result>> { + let output = child.stdout.take().ok_or("missing child output")?; + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(output).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + Ok(rx) +} + +fn receive(rx: &mpsc::Receiver>) -> Result { + Ok(serde_json::from_str( + &rx.recv_timeout(Duration::from_secs(30))??, + )?) +} + +fn interruptible_receive( + rx: &mpsc::Receiver>, + input: &mpsc::Receiver>, +) -> Result> { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + match rx.recv_timeout(Duration::from_millis(20)) { + Ok(line) => return Ok(Some(serde_json::from_str(&line?)?)), + Err(mpsc::RecvTimeoutError::Disconnected) => return Err("child output closed".into()), + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + match input.try_recv() { + Ok(line) => { + let interrupt: Value = serde_json::from_str(&line?)?; + if interrupt["type"] != "control_request" + || interrupt.pointer("/request/subtype") != Some(&json!("interrupt")) + { + return Err("unexpected turn input".into()); + } + emit( + json!({"type":"control_response","response":{"subtype":"success","request_id":interrupt["request_id"]}}), + )?; + emit( + json!({"type":"result","subtype":"error_during_execution","session_id":"fixture-session"}), + )?; + return Ok(None); + } + Err(mpsc::TryRecvError::Disconnected) => return Err("input closed during call".into()), + Err(mpsc::TryRecvError::Empty) => {} + } + if Instant::now() >= deadline { + return Err("child response deadline exceeded".into()); + } + } +} + +fn child_send(child: &mut Child, value: Value) -> Result<()> { + let input = child.stdin.as_mut().ok_or("missing child input")?; + writeln!(input, "{value}")?; + input.flush()?; + Ok(()) +} + +fn claude(mut cfg: Config) -> Result<()> { + let args: Vec<_> = std::env::args_os().collect(); + let argument = |flag: &str| -> Result { + let pos = args.iter().position(|a| a == flag).ok_or("missing flag")?; + Ok(PathBuf::from( + args.get(pos + 1).ok_or("missing flag value")?, + )) + }; + if !args.iter().any(|a| a == "--strict-mcp-config") { + return Err("missing strict config".into()); + } + let config: Value = serde_json::from_slice(&fs::read(argument("--mcp-config")?)?)?; + let servers = config + .get("mcpServers") + .and_then(Value::as_object) + .ok_or("invalid servers")?; + if servers.len() != 1 { + return Err("unexpected server count".into()); + } + let server = servers.get("engrams").ok_or("missing self bridge")?; + if server["type"] != "stdio" || server["args"] != json!(["mcp-bridge"]) { + return Err("invalid bridge config".into()); + } + let command = server["command"].as_str().ok_or("missing bridge command")?; + if !Path::new(command).is_absolute() { + return Err("bridge command must be absolute".into()); + } + let settings: Value = serde_json::from_slice(&fs::read(argument("--settings")?)?)?; + let hook = settings + .pointer("/hooks/PreToolUse/0/hooks/0/command") + .and_then(Value::as_str) + .ok_or("missing hook")?; + if hook.strip_suffix(" hook-bridge") != Some(command) { + return Err("invalid hook command".into()); + } + let mut bridge = ChildGuard( + Command::new(command) + .arg("mcp-bridge") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn()?, + ); + let bridge_rx = read_lines(&mut bridge.0)?; + child_send( + &mut bridge.0, + json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"fixture","version":"1"}}}), + )?; + let initialized = receive(&bridge_rx)?; + let result = initialized.get("result"); + if initialized["jsonrpc"] != "2.0" + || initialized["id"] != 1 + || initialized.get("error").is_some() + || result + .and_then(|v| v.get("protocolVersion")) + .and_then(Value::as_str) + != Some("2025-06-18") + || !result + .and_then(|v| v.get("capabilities")) + .is_some_and(Value::is_object) + || !["name", "version"].iter().all(|field| { + result + .and_then(|v| v.get("serverInfo")) + .and_then(|v| v.get(*field)) + .and_then(Value::as_str) + .is_some_and(|value| !value.is_empty()) + }) + { + return Err("invalid initialize response".into()); + } + child_send( + &mut bridge.0, + json!({"jsonrpc":"2.0","method":"notifications/initialized"}), + )?; + child_send( + &mut bridge.0, + json!({"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}), + )?; + let list = receive(&bridge_rx)?; + let tools = list + .pointer("/result/tools") + .and_then(Value::as_array) + .ok_or("missing tools")?; + cfg.bind_tools(tools, "claude")?; + emit(json!({"type":"system","subtype":"init","session_id":"fixture-session"}))?; + let input = stdin_lines(); + let mut call_request_id = 3_u64; + // Deferred calls can re-fire without a prompt. Completed plan selections + // wait for the next current prompt when the conversation resumes. + let mut resumed = ["write", "refire"].contains(&cfg.mode.as_str()) + && args.iter().any(|arg| arg == "--resume") + && match &cfg.plan { + Some(plan) => plan.has_pending_selection(&cfg.root, "claude", &cfg.mode)?, + None => true, + }; + loop { + let no_current_prompt = std::mem::take(&mut resumed); + let prompt: Value = if no_current_prompt { + json!({"type":"user"}) + } else { + match input.recv() { + Ok(line) => serde_json::from_str(&line?)?, + Err(_) => break, + } + }; + if prompt["type"] != "user" { + return Err("unsupported Claude input".into()); + } + cfg.select(if no_current_prompt { + None + } else { + Some(&prompt) + })?; + cfg.prepare_evidence()?; + cfg.persist_selection(true)?; + let mut hook = ChildGuard( + Command::new(command) + .arg("hook-bridge") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn()?, + ); + let hook_rx = read_lines(&mut hook.0)?; + let arguments = cfg.arguments.clone(); + let call_tool = if cfg.mode == "forged" { + "fixture_forged" + } else { + &cfg.tool + }; + emit( + json!({"type":"assistant","message":{"id":"fixture-message","content":[{"type":"tool_use","id":cfg.call_id,"name":format!("mcp__engrams__{call_tool}"),"input":arguments}]}}), + )?; + child_send( + &mut hook.0, + json!({"tool_name":format!("mcp__engrams__{}",call_tool),"tool_use_id":cfg.call_id,"tool_input":arguments,"permission_mode":"default"}), + )?; + drop(hook.0.stdin.take()); + let Some(decision) = interruptible_receive(&hook_rx, &input)? else { + continue; + }; + match decision + .pointer("/hookSpecificOutput/permissionDecision") + .and_then(Value::as_str) + { + Some("defer") if cfg.premature => { + cfg.premature_completion(&input)?; + emit( + json!({"type":"result","subtype":"success","session_id":"fixture-session","result":"fixture"}), + )?; + continue; + } + _ if cfg.premature + || cfg.absent + && decision.pointer("/hookSpecificOutput/permissionDecision") + != Some(&json!("allow")) => + { + return Err("unexpected evidence hook decision".into()); + } + Some("allow") => {} + Some("defer") | Some("deny") => { + emit( + json!({"type":"result","subtype":"success","session_id":"fixture-session","result":"fixture","terminal_reason":"tool_deferred"}), + )?; + continue; + } + _ => return Err("invalid hook decision".into()), + } + if let Some(interrupt) = wait_barrier(cfg.barrier.as_deref(), &input)? { + if interrupt["type"] != "control_request" + || interrupt.pointer("/request/subtype") != Some(&json!("interrupt")) + { + return Err("unexpected barrier input".into()); + } + emit( + json!({"type":"control_response","response":{"subtype":"success","request_id":interrupt["request_id"]}}), + )?; + emit( + json!({"type":"result","subtype":"error_during_execution","session_id":"fixture-session"}), + )?; + continue; + } + child_send( + &mut bridge.0, + json!({"jsonrpc":"2.0","id":call_request_id,"method":"tools/call","params":{"name":call_tool,"arguments":arguments,"_meta":{"claudecode/toolUseId":cfg.call_id}}}), + )?; + let Some(response) = interruptible_receive(&bridge_rx, &input)? else { + child_send( + &mut bridge.0, + json!({"jsonrpc":"2.0","method":"notifications/cancelled","params":{"requestId":call_request_id}}), + )?; + call_request_id += 1; + continue; + }; + if response["id"] != call_request_id { + return Err("invalid call response".into()); + } + if cfg.mode == "forged" { + if response.pointer("/error/code") != Some(&json!(-32602)) { + return Err("forged tool was not rejected".into()); + } + let proof = json!({"callId":cfg.call_id,"tool":"fixture_forged","forgedRejected":true}); + record(&mut cfg, proof, None)?; + emit( + json!({"type":"result","subtype":"success","session_id":"fixture-session","result":"fixture"}), + )?; + break; + } + if response.get("result").is_some() && response.get("error").is_some() { + return Err("ambiguous tool response".into()); + } + let result = response.get("result"); + if cfg.absent { + if result.is_some() + || response.pointer("/error/code") != Some(&json!(-32602)) + || response.pointer("/error/message") != Some(&json!("Unknown tool")) + { + return Err("absent alias was not rejected".into()); + } + let proof = json!({"callId":cfg.call_id,"tool":cfg.tool,"expectExposure":"absent","requestAttempted":true,"protocolDenialCode":"unknown_tool","nativeDenialCode":-32602}); + record(&mut cfg, proof, None)?; + cfg.native_completion(&response, &input)?; + emit( + json!({"type":"result","subtype":"success","session_id":"fixture-session","result":"fixture"}), + )?; + continue; + } + if cfg.native_json { + let native = result.ok_or("missing native JSON result")?; + let content = native + .get("content") + .and_then(Value::as_array) + .ok_or("missing native JSON content")?; + if content.iter().any(|item| { + item.get("type") != Some(&json!("text")) + || !item.get("text").is_some_and(Value::is_string) + }) { + return Err("invalid native JSON content".into()); + } + let proof = json!({"callId":cfg.call_id,"tool":cfg.tool,"resultReceived":true}); + record(&mut cfg, proof, Some(native))?; + cfg.native_completion(&response, &input)?; + emit( + json!({"type":"result","subtype":"success","session_id":"fixture-session","result":"fixture"}), + )?; + continue; + } + let receipt = response + .pointer("/result/_meta/engrams~1toolResult") + .or_else(|| response.pointer("/error/data")) + .ok_or("missing receipt")?; + let delivery = receipt["delivery"] + .as_str() + .filter(|s| ["available", "unavailable"].contains(s)) + .ok_or("invalid delivery")?; + let outcome = receipt.get("outcome").and_then(Value::as_str); + if outcome + .is_some_and(|s| !["succeeded", "failed", "canceled", "outcome_unknown"].contains(&s)) + { + return Err("invalid outcome".into()); + } + let text = result + .and_then(|v| v.get("content")) + .and_then(Value::as_array) + .is_some_and(|content| { + content + .iter() + .any(|item| item == &json!({"type":"text","text":"sentinel"})) + }); + let text_sequence = result + .and_then(|v| v.get("content")) + .and_then(Value::as_array) + .is_some_and(|content| { + content + == &vec![ + json!({"type":"text","text":"sentinel"}), + json!({"type":"text","text":"second sentinel"}), + ] + }); + let structured = + result.and_then(|v| v.get("structuredContent")) == Some(&json!({"fixture":true})); + if (delivery == "available") != result.is_some() { + return Err("inconsistent delivery response".into()); + } + if let Some(value) = result { + if !value.get("content").is_some_and(Value::is_array) { + return Err("missing content".into()); + } + } + let is_error = result + .map(|v| optional_bool(v, "isError")) + .transpose()? + .flatten(); + let unavailable = optional_bool(receipt, "resultUnavailable")?; + let denial_plan = receipt.get("denial") == Some(&json!("plan_mode")); + let proof = json!({"callId":cfg.call_id,"tool":cfg.tool,"resultReceived":true,"textSentinel":text,"textSequence":text_sequence,"structuredSentinel":structured,"isError":is_error,"outcome":outcome,"delivery":delivery,"resultUnavailable":unavailable,"denialPlanMode":denial_plan}); + record(&mut cfg, proof, result)?; + cfg.native_completion(&response, &input)?; + emit( + json!({"type":"result","subtype":"success","session_id":"fixture-session","result":"fixture"}), + )?; + // A completed deferred re-fire must leave this model process idle. + // Exiting here would simulate a new crash and replay the completed + // fixture call on every subsequent --resume startup. + if cfg.plan.is_none() && !["write", "refire"].contains(&cfg.mode.as_str()) { + break; + } + } + Ok(()) +} +fn main() { + if std::env::args_os().nth(1).as_deref() == Some(std::ffi::OsStr::new("--fixture-version")) { + println!("engram-mcp-model-fixture 1"); + return; + } + let result = (|| -> Result<()> { + let cfg = config()?; + if std::env::args_os().nth(1).as_deref() == Some(std::ffi::OsStr::new("app-server")) { + codex(cfg) + } else { + claude(cfg) + } + })(); + if result.is_err() { + eprintln!("fixture protocol failure"); + std::process::exit(1); + } +} + +#[cfg(test)] +mod proof_tests { + use super::*; + use std::io::Read; + + struct State(PathBuf); + impl State { + fn new(name: &str) -> Self { + let path = + std::env::temp_dir().join(format!("proof-unit-{}-{name}", std::process::id())); + fs::create_dir(&path).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o700)).unwrap(); + Self(path) + } + } + impl Drop for State { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).unwrap(); + } + } + + #[test] + fn same_pid_launches_claim_distinct_files_and_collision_preserves_prior_bytes() { + let state = State::new("same-pid"); + let template = state.0.join("proof-{launch}.json"); + let first = state + .0 + .join("proof-42-000102030405060708090a0b0c0d0e0f.json"); + let second = state + .0 + .join("proof-42-ffffffffffffffffffffffffffffffff.json"); + let nonce = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; + let mut initial = + proof_file(&state.0, &template, 42, || Ok(io::Cursor::new(nonce))).unwrap(); + initial.write_all(b"retained prior observation\n").unwrap(); + initial.flush().unwrap(); + drop(initial); + let mut fresh = + proof_file(&state.0, &template, 42, || Ok(io::Cursor::new([255; 16]))).unwrap(); + fresh.write_all(b"fresh observation\n").unwrap(); + fresh.flush().unwrap(); + drop(fresh); + let error = proof_file(&state.0, &template, 42, || Ok(io::Cursor::new(nonce))).unwrap_err(); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + io::ErrorKind::AlreadyExists + ); + assert_eq!(fs::read(&first).unwrap(), b"retained prior observation\n"); + assert_eq!(fs::read(&second).unwrap(), b"fresh observation\n"); + assert_eq!(fs::read_dir(&state.0).unwrap().count(), 2); + } + + #[test] + fn launch_entropy_failure_and_short_reads_claim_no_file() { + let state = State::new("entropy"); + let template = state.0.join("proof-{launch}.json"); + for bytes in 0..16 { + let error = proof_file(&state.0, &template, 42, || { + Ok(io::Cursor::new(vec![0; bytes])) + }) + .unwrap_err(); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + io::ErrorKind::UnexpectedEof + ); + assert_eq!(fs::read_dir(&state.0).unwrap().count(), 0); + } + let error = proof_file::(&state.0, &template, 42, || { + Err(io::Error::other("entropy unavailable")) + }) + .unwrap_err(); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + io::ErrorKind::Other + ); + assert_eq!(fs::read_dir(&state.0).unwrap().count(), 0); + struct FailedRead; + impl Read for FailedRead { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::other("entropy read failed")) + } + } + let error = proof_file(&state.0, &template, 42, || { + Ok(io::repeat(0).take(8).chain(FailedRead)) + }) + .unwrap_err(); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + io::ErrorKind::Other + ); + assert_eq!(fs::read_dir(&state.0).unwrap().count(), 0); + } + + #[test] + fn literal_and_pid_paths_need_no_entropy_and_never_replace_existing_files() { + let state = State::new("legacy"); + for (template, name) in [ + ("proof.json", "proof.json"), + ("proof-{pid}.json", "proof-42.json"), + ] { + let path = state.0.join(template); + let mut file = + proof_file::(&state.0, &path, 42, || panic!("unexpected entropy read")) + .unwrap(); + file.write_all(b"preserve\n").unwrap(); + file.flush().unwrap(); + drop(file); + let error = + proof_file::(&state.0, &path, 42, || panic!("unexpected entropy read")) + .unwrap_err(); + assert_eq!( + error.downcast_ref::().unwrap().kind(), + io::ErrorKind::AlreadyExists + ); + assert_eq!(fs::read(state.0.join(name)).unwrap(), b"preserve\n"); + } + } +} diff --git a/tests/fixtures/mcp-model-cli/src/plan.rs b/tests/fixtures/mcp-model-cli/src/plan.rs new file mode 100644 index 000000000..07a5b5fb3 --- /dev/null +++ b/tests/fixtures/mcp-model-cli/src/plan.rs @@ -0,0 +1,296 @@ +use crate::scalar::{digest, parse_object}; +use crate::Result; +use serde_json::{json, Value}; +use std::collections::BTreeSet; +use std::fs::{self, OpenOptions}; +use std::io::{Read, Write}; +use std::os::unix::fs::OpenOptionsExt; +use std::path::Path; + +#[derive(Clone)] +pub struct Step { + pub marker: String, + pub tool: String, + pub arguments: Value, + pub call_id: String, + pub native_json: bool, + pub expected: Option, + pub absent: bool, + pub premature: bool, +} +pub struct Plan { + pub hash: String, + steps: Vec, +} +pub fn identifier(value: &str) -> bool { + !value.is_empty() + && value.len() <= 64 + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') +} +pub fn tool_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_') +} +fn exact_keys(value: &Value, required: &[&str], optional: &[&str]) -> Result<()> { + let object = value.as_object().ok_or("invalid object")?; + if required.iter().any(|key| !object.contains_key(*key)) + || object + .keys() + .any(|key| !required.contains(&key.as_str()) && !optional.contains(&key.as_str())) + { + return Err("invalid object fields".into()); + } + Ok(()) +} +fn text(value: &Value, key: &str) -> Result { + Ok(value + .get(key) + .and_then(Value::as_str) + .ok_or("missing string")? + .to_owned()) +} +impl Plan { + pub fn from_env() -> Result> { + let raw = match std::env::var("ENGRAM_FIXTURE_CALL_PLAN_JSON") { + Ok(raw) => raw, + Err(std::env::VarError::NotPresent) => return Ok(None), + Err(_) => return Err("invalid plan environment".into()), + }; + for key in [ + "ENGRAM_FIXTURE_TOOL", + "ENGRAM_FIXTURE_ARGUMENTS_JSON", + "ENGRAM_FIXTURE_CALL_ID", + "ENGRAM_FIXTURE_EXPECTED_RESULT_JSON", + ] { + if std::env::var_os(key).is_some() { + return Err("plan conflicts with scalar configuration".into()); + } + } + // The root object, steps array and step object enclose each field. + // Preserve the expected-result depth bound through those three levels. + let value = parse_object(&raw, 65536, 128 + 3)?; + exact_keys(&value, &["steps"], &[])?; + let steps = value["steps"].as_array().ok_or("invalid plan steps")?; + if steps.is_empty() || steps.len() > 16 { + return Err("invalid plan length".into()); + } + let mut markers = BTreeSet::new(); + let mut parsed: Vec = Vec::new(); + for step in steps { + exact_keys( + step, + &["marker", "tool", "arguments", "callId", "resultFormat"], + &["expectedResult", "expectExposure", "completionProjection"], + )?; + let marker = text(step, "marker")?; + let call_id = text(step, "callId")?; + let tool = text(step, "tool")?; + if !identifier(&marker) + || !identifier(&call_id) + || !tool_name(&tool) + || !markers.insert(marker.clone()) + { + return Err("invalid or duplicate plan identity".into()); + } + let arguments = parse_object(&step["arguments"].to_string(), 4096, 16)?; + let expected = step + .get("expectedResult") + .map(|value| parse_object(&value.to_string(), 1024 * 1024, 128)) + .transpose()?; + let native_json = match step["resultFormat"].as_str() { + Some("json") => true, + Some("mcp") => false, + _ => return Err("invalid plan result format".into()), + }; + let absent = match step.get("expectExposure").and_then(Value::as_str) { + Some("absent") => true, + Some("present") => false, + None if step.get("expectExposure").is_none() => false, + _ => return Err("invalid exposure expectation".into()), + }; + let premature = match step.get("completionProjection").and_then(Value::as_str) { + Some("premature") => true, + None if step.get("completionProjection").is_none() => false, + _ => return Err("invalid completion projection".into()), + }; + if (absent || premature) + && (native_json || expected.is_some() || !tool.starts_with("mcp_")) + || absent && premature + { + return Err("invalid evidence step".into()); + } + let duplicates: Vec<_> = parsed + .iter() + .filter(|prior| prior.call_id == call_id) + .collect(); + if !duplicates.is_empty() + && (duplicates.len() != 1 + || !absent + || duplicates[0].absent + || duplicates[0].premature + || duplicates[0].native_json + || duplicates[0].tool != tool + || digest(&duplicates[0].arguments)? != digest(&arguments)?) + { + return Err("invalid repeated call identity".into()); + } + parsed.push(Step { + marker, + tool, + arguments, + call_id, + native_json, + expected, + absent, + premature, + }); + } + Ok(Some(Self { + hash: digest(&value)?, + steps: parsed, + })) + } + pub fn select(&self, prompt: &str) -> Result { + if prompt.len() > 65536 { + return Err("prompt exceeds fixture limit".into()); + } + let mut tokens = prompt.split("[[fixture:").skip(1); + let token = tokens.next().ok_or("missing plan marker")?; + if tokens.next().is_some() { + return Err("ambiguous plan marker".into()); + } + let marker = token.split_once("]]").ok_or("invalid plan marker")?.0; + self.step(marker) + } + fn step(&self, marker: &str) -> Result { + self.steps + .iter() + .find(|step| step.marker == marker) + .cloned() + .ok_or_else(|| "unknown plan marker".into()) + } + fn read_receipt(&self, root: &Path, harness: &str, mode: &str) -> Result> { + let path = root.join("plan-selection.json"); + let metadata = match fs::symlink_metadata(&path) { + Ok(value) => value, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if !metadata.file_type().is_file() || metadata.len() > 4096 { + return Err("invalid plan receipt".into()); + } + let mut raw = String::new(); + fs::File::open(path)?.take(4097).read_to_string(&mut raw)?; + let value = parse_object(&raw, 4096, 4)?; + exact_keys( + &value, + &["planSha256", "harness", "mode", "marker", "pending"], + &[], + )?; + if value["planSha256"] != self.hash + || value["harness"] != harness + || value["mode"] != mode + || !value["pending"].is_boolean() + { + return Err("mismatched plan receipt".into()); + } + self.step(value["marker"].as_str().ok_or("invalid receipt marker")?)?; + Ok(Some(value)) + } + pub fn has_pending_selection(&self, root: &Path, harness: &str, mode: &str) -> Result { + Ok(self + .read_receipt(root, harness, mode)? + .is_some_and(|value| value["pending"] == true)) + } + pub fn resume(&self, root: &Path, harness: &str, mode: &str) -> Result { + let value = self + .read_receipt(root, harness, mode)? + .ok_or("missing pending plan receipt")?; + if value["pending"] != true { + return Err("plan selection already completed".into()); + } + self.step(value["marker"].as_str().ok_or("invalid receipt marker")?) + } + pub fn persist( + &self, + root: &Path, + harness: &str, + mode: &str, + marker: &str, + pending: bool, + ) -> Result<()> { + self.read_receipt(root, harness, mode)?; + let temporary = root.join(format!("plan-selection-{}.tmp", std::process::id())); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary)?; + let result = (|| -> Result<()> { + write!( + file, + "{}", + json!({"planSha256":self.hash,"harness":harness,"mode":mode,"marker":marker,"pending":pending}) + )?; + file.sync_all()?; + fs::rename(&temporary, root.join("plan-selection.json"))?; + fs::File::open(root)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(temporary); + } + result + } +} + +pub fn prompt_text(value: &Value, claude: bool) -> Result { + if claude && value.pointer("/message/role").and_then(Value::as_str) != Some("user") { + return Err("not a current user prompt".into()); + } + let content = if claude { + value.pointer("/message/content") + } else { + value.pointer("/params/input") + } + .ok_or("missing current prompt")?; + if claude { + if let Some(text) = content.as_str() { + return Ok(text.to_owned()); + } + } + let items = content.as_array().ok_or("invalid current prompt")?; + let mut parts = Vec::new(); + for item in items { + if item.get("type").and_then(Value::as_str) != Some("text") { + return Err("nontext current prompt".into()); + } + parts.push( + item.get("text") + .and_then(Value::as_str) + .ok_or("invalid prompt text")?, + ); + } + Ok(parts.join("\n")) +} + +pub fn exposed_mcp_hash(tools: &[Value]) -> Result { + let mut names = BTreeSet::new(); + for tool in tools { + let name = tool + .get("name") + .and_then(Value::as_str) + .ok_or("invalid exposed tool name")?; + if name.starts_with("mcp_") + && (!tool_name(name) || !names.insert(name) || names.len() > 128) + { + return Err("invalid exposed MCP tool set".into()); + } + } + digest(&json!(names.into_iter().collect::>())) +} diff --git a/tests/fixtures/mcp-model-cli/src/scalar.rs b/tests/fixtures/mcp-model-cli/src/scalar.rs new file mode 100644 index 000000000..38fc3f4b5 --- /dev/null +++ b/tests/fixtures/mcp-model-cli/src/scalar.rs @@ -0,0 +1,111 @@ +use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor}; +use serde_json::{Map, Value}; +use sha2::{Digest, Sha256}; +use std::fmt; + +// Parse objects without silently discarding duplicate keys at any depth. +struct JsonSeed(usize); +impl<'de> DeserializeSeed<'de> for JsonSeed { + type Value = Value; + fn deserialize>(self, deserializer: D) -> Result { + deserializer.deserialize_any(self) + } +} +impl<'de> Visitor<'de> for JsonSeed { + type Value = Value; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("bounded JSON without duplicate keys") + } + fn visit_bool(self, value: bool) -> Result { + Ok(value.into()) + } + fn visit_i64(self, value: i64) -> Result { + Ok(value.into()) + } + fn visit_u64(self, value: u64) -> Result { + Ok(value.into()) + } + fn visit_f64(self, value: f64) -> Result { + serde_json::Number::from_f64(value) + .map(Value::Number) + .ok_or_else(|| E::custom("invalid number")) + } + fn visit_str(self, value: &str) -> Result { + Ok(value.into()) + } + fn visit_unit(self) -> Result { + Ok(Value::Null) + } + fn visit_seq>(self, mut seq: A) -> Result { + let remaining = self + .0 + .checked_sub(1) + .ok_or_else(|| de::Error::custom("excessive depth"))?; + let mut values = Vec::new(); + while let Some(value) = seq.next_element_seed(JsonSeed(remaining))? { + values.push(value); + } + Ok(Value::Array(values)) + } + fn visit_map>(self, mut map: A) -> Result { + let remaining = self + .0 + .checked_sub(1) + .ok_or_else(|| de::Error::custom("excessive depth"))?; + let mut values = Map::new(); + while let Some(key) = map.next_key::()? { + if values.contains_key(&key) { + return Err(de::Error::custom("duplicate key")); + } + values.insert(key, map.next_value_seed(JsonSeed(remaining))?); + } + Ok(Value::Object(values)) + } +} + +pub fn configured_object(name: &str, limit: usize, depth: usize) -> super::Result> { + let raw = match std::env::var(name) { + Ok(raw) => raw, + Err(std::env::VarError::NotPresent) => return Ok(None), + Err(_) => return Err("invalid JSON environment".into()), + }; + parse_object(&raw, limit, depth).map(Some) +} + +pub fn parse_object(raw: &str, limit: usize, depth: usize) -> super::Result { + if raw.len() > limit { + return Err("JSON configuration exceeds size limit".into()); + } + let mut deserializer = serde_json::Deserializer::from_str(raw); + // JsonSeed bounds every container. The decoder's separate recursion cap + // must not reject valid fields when a plan adds its enclosing containers. + deserializer.disable_recursion_limit(); + let value = JsonSeed(depth).deserialize(&mut deserializer)?; + deserializer.end()?; + if !value.is_object() { + return Err("JSON configuration must be an object".into()); + } + Ok(value) +} + +pub fn digest(value: &Value) -> super::Result { + fn sorted(value: &Value) -> Value { + match value { + Value::Object(map) => { + let ordered: std::collections::BTreeMap<_, _> = map.iter().collect(); + Value::Object( + ordered + .into_iter() + .map(|(key, value)| (key.clone(), sorted(value))) + .collect(), + ) + } + Value::Array(values) => Value::Array(values.iter().map(sorted).collect()), + value => value.clone(), + } + } + Ok(format!( + "sha256:{:x}", + Sha256::digest(serde_json::to_vec(&sorted(value))?) + )) +} diff --git a/tests/fixtures/mcp-model-cli/tests/protocol.rs b/tests/fixtures/mcp-model-cli/tests/protocol.rs new file mode 100644 index 000000000..2db4e067d --- /dev/null +++ b/tests/fixtures/mcp-model-cli/tests/protocol.rs @@ -0,0 +1,783 @@ +use std::io::{BufRead, BufReader, Write}; +use std::process::{Child, Command, Stdio}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, +}; +use std::time::Duration; + +use serde_json::{json, Value}; + +struct Fixture(Child); +struct State(std::path::PathBuf); + +impl State { + fn new(name: &str) -> Self { + use std::os::unix::fs::DirBuilderExt; + static NEXT_STATE_ID: AtomicUsize = AtomicUsize::new(0); + let path = std::env::var_os("TMPDIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))) + .join(format!( + "fixture-state-{}-{name}-{}", + std::process::id(), + NEXT_STATE_ID.fetch_add(1, Ordering::Relaxed) + )); + std::fs::DirBuilder::new() + .mode(0o700) + .create(&path) + .unwrap(); + Self(path) + } +} + +impl Drop for State { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn send(child: &mut Child, value: Value) { + let input = child.stdin.as_mut().unwrap(); + writeln!(input, "{value}").unwrap(); + input.flush().unwrap(); +} + +fn wait_for_failure(child: &mut Child) { + let deadline = std::time::Instant::now() + Duration::from_secs(3); + loop { + if let Some(status) = child.try_wait().unwrap() { + assert!(!status.success()); + return; + } + assert!(std::time::Instant::now() < deadline, "fixture did not exit"); + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +fn claude_rejects_failed_or_malformed_initialize_even_when_list_works() { + for case in [ + "error", + "missing", + "malformed", + "server-info", + "version", + "both", + ] { + let state = State::new(case); + let mock = env!("CARGO_BIN_EXE_fixture-mock-bridge"); + let config = state.0.join("config.json"); + let settings = state.0.join("settings.json"); + let proof = state.0.join("proof.json"); + std::fs::write(state.0.join("bad-initialize"), case).unwrap(); + std::fs::write( + &config, + json!({"mcpServers":{"engrams":{"type":"stdio","command":mock,"args":["mcp-bridge"]}}}) + .to_string(), + ) + .unwrap(); + std::fs::write( + &settings, + json!({"hooks":{"PreToolUse":[{"hooks":[{"command":format!("{mock} hook-bridge")}]}]}}) + .to_string(), + ) + .unwrap(); + let child = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .args(["--print", "--strict-mcp-config", "--mcp-config"]) + .arg(config) + .arg("--settings") + .arg(settings) + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", "read") + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", &proof) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let mut child = Fixture(child); + send( + &mut child.0, + json!({"type":"user","message":{"role":"user","content":"fixture"}}), + ); + drop(child.0.stdin.take()); + wait_for_failure(&mut child.0); + assert!(std::fs::read(&proof).unwrap().is_empty()); + } +} + +#[test] +fn codex_declares_and_delivers_the_requested_tool() { + let state = State::new("delivery"); + let proof = state.0.join("proof.json"); + let child = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .arg("app-server") + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", "read") + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", &proof) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut child = Fixture(child); + let output = child.0.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(output).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + let receive = || -> Value { + serde_json::from_str(&rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap()).unwrap() + }; + send( + &mut child.0, + json!({"id":1,"method":"initialize","params":{}}), + ); + assert_eq!(receive()["id"], 1); + send( + &mut child.0, + json!({"id":2,"method":"thread/start","params":{"dynamicTools":[{"name":"remote_inventory","description":"test","inputSchema":{"type":"object"}}]}}), + ); + assert_eq!(receive()["result"]["thread"]["id"], "fixture-thread"); + send( + &mut child.0, + json!({"id":3,"method":"turn/start","params":{"threadId":"fixture-thread"}}), + ); + assert_eq!(receive()["id"], 3); + let call = receive(); + assert_eq!(call["method"], "item/tool/call"); + assert_eq!(call["params"]["tool"], "remote_inventory"); + assert_eq!(call["params"]["callId"], "fixture-call"); + send( + &mut child.0, + json!({"id":call["id"],"result":{"success":false,"contentItems":[{"type":"inputText","text":"sentinel"},{"type":"inputText","text":"second sentinel"},{"type":"inputText","text":"{\"structuredContent\":{\"fixture\":true}}"},{"type":"inputText","text":"{\"delivery\":\"available\",\"isError\":true,\"outcome\":\"failed\",\"resultUnavailable\":false}"}]}}), + ); + assert_eq!(receive()["method"], "item/completed"); + assert_eq!(receive()["method"], "turn/completed"); + let recorded: Value = serde_json::from_slice(&std::fs::read(&proof).unwrap()).unwrap(); + assert_eq!( + recorded, + json!({"callId":"fixture-call","tool":"remote_inventory","resultReceived":true,"success":false,"textSentinel":true,"textSequence":true,"structuredSentinel":true,"isError":true,"outcome":"failed","delivery":"available","resultUnavailable":false,"denialPlanMode":false}) + ); +} + +#[test] +fn codex_rejects_a_thread_without_the_expected_tool() { + let state = State::new("missing-tool"); + let child = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .arg("app-server") + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", "read") + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", state.0.join("proof.json")) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut child = Fixture(child); + send( + &mut child.0, + json!({"id":0,"method":"initialize","params":{}}), + ); + send( + &mut child.0, + json!({"id":1,"method":"thread/start","params":{"dynamicTools":[]}}), + ); + drop(child.0.stdin.take()); + wait_for_failure(&mut child.0); + let mut error = String::new(); + std::io::Read::read_to_string(&mut child.0.stderr.take().unwrap(), &mut error).unwrap(); + assert_eq!(error, "fixture protocol failure\n"); +} + +#[test] +fn rejects_proof_outside_the_owned_state_directory() { + let state = State::new("root"); + let other = State::new("outside"); + let proof = other.0.join("proof.json"); + std::fs::write(&proof, b"preserve").unwrap(); + let child = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .arg("app-server") + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", "read") + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", &proof) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut child = Fixture(child); + wait_for_failure(&mut child.0); + let mut error = String::new(); + std::io::Read::read_to_string(&mut child.0.stderr.take().unwrap(), &mut error).unwrap(); + assert_eq!(error, "fixture protocol failure\n"); + assert_eq!(std::fs::read(proof).unwrap(), b"preserve"); +} + +#[test] +fn claude_reads_generated_config_and_exchanges_mcp_and_hook_messages() { + for mode in ["read", "write", "denied", "forged"] { + let state = State::new(&format!("claude-{mode}")); + let mock = env!("CARGO_BIN_EXE_fixture-mock-bridge"); + let config = state.0.join("config.json"); + let settings = state.0.join("settings.json"); + let proof = state.0.join("proof.json"); + std::fs::write(&config, json!({"mcpServers":{"engrams":{"type":"stdio","command":mock,"args":["mcp-bridge"],"alwaysLoad":true}}}).to_string()).unwrap(); + std::fs::write(&settings, json!({"hooks":{"PreToolUse":[{"matcher":"*","hooks":[{"type":"command","command":format!("{mock} hook-bridge")}]}]}}).to_string()).unwrap(); + let child = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .args(["--print", "--strict-mcp-config", "--mcp-config"]) + .arg(&config) + .arg("--settings") + .arg(&settings) + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", mode) + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", &proof) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut child = Fixture(child); + let output = child.0.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(output).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + send( + &mut child.0, + json!({"type":"user","message":{"role":"user","content":"fixture"}}), + ); + let receive = || -> Value { + serde_json::from_str(&rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap()) + .unwrap() + }; + assert_eq!(receive()["subtype"], "init"); + assert_eq!(receive()["type"], "assistant"); + if mode != "forged" { + assert_eq!(receive()["type"], "user"); + } + assert_eq!(receive()["type"], "result"); + let recorded: Value = serde_json::from_slice(&std::fs::read(proof).unwrap()).unwrap(); + if mode == "forged" { + assert_eq!( + recorded, + json!({"callId":"fixture-call","tool":"fixture_forged","forgedRejected":true}) + ); + continue; + } + let denied = mode == "denied"; + assert_eq!( + recorded, + json!({"callId":"fixture-call","tool":"remote_inventory","resultReceived":true,"textSentinel":true,"textSequence":true,"structuredSentinel":true,"isError":denied,"outcome":if denied { Value::Null } else { json!("succeeded") },"delivery":"available","resultUnavailable":false,"denialPlanMode":denied}) + ); + } +} + +#[test] +fn codex_interrupts_a_closed_barrier_without_emitting_a_call() { + let state = State::new("barrier"); + let child = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .arg("app-server") + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", "held-call") + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", state.0.join("proof.json")) + .env("ENGRAM_FIXTURE_BARRIER", state.0.join("release")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut child = Fixture(child); + let output = child.0.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(output).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + let receive = || -> Value { + serde_json::from_str(&rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap()).unwrap() + }; + send( + &mut child.0, + json!({"id":0,"method":"initialize","params":{}}), + ); + assert_eq!(receive()["id"], 0); + send( + &mut child.0, + json!({"id":1,"method":"thread/start","params":{"dynamicTools":[{"name":"remote_inventory","description":"test","inputSchema":{"type":"object"}}]}}), + ); + assert_eq!(receive()["id"], 1); + send( + &mut child.0, + json!({"id":2,"method":"turn/start","params":{}}), + ); + assert_eq!(receive()["id"], 2); + send( + &mut child.0, + json!({"id":3,"method":"turn/interrupt","params":{"threadId":"fixture-thread","turnId":"fixture-turn"}}), + ); + assert_eq!(receive()["id"], 3); + let completed = receive(); + assert_eq!(completed["method"], "turn/completed"); + assert_eq!(completed["params"]["turn"]["status"], "interrupted"); + assert!(std::fs::read(state.0.join("proof.json")) + .unwrap() + .is_empty()); +} + +#[test] +fn codex_interrupts_after_a_tool_call_is_sent() { + let state = State::new("codex-held"); + let child = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .arg("app-server") + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", "held-call") + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", state.0.join("proof.json")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut child = Fixture(child); + let output = child.0.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(output).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + let receive = || -> Value { + serde_json::from_str(&rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap()).unwrap() + }; + send( + &mut child.0, + json!({"id":0,"method":"initialize","params":{}}), + ); + assert_eq!(receive()["id"], 0); + send( + &mut child.0, + json!({"id":1,"method":"thread/start","params":{"dynamicTools":[{"name":"remote_inventory","description":"test","inputSchema":{"type":"object"}}]}}), + ); + assert_eq!(receive()["id"], 1); + send( + &mut child.0, + json!({"id":2,"method":"turn/start","params":{}}), + ); + assert_eq!(receive()["id"], 2); + assert_eq!(receive()["method"], "item/tool/call"); + send( + &mut child.0, + json!({"id":3,"method":"turn/interrupt","params":{"threadId":"fixture-thread","turnId":"fixture-turn"}}), + ); + assert_eq!(receive()["id"], 3); + assert_eq!(receive()["params"]["turn"]["status"], "interrupted"); + assert!(std::fs::read(state.0.join("proof.json")) + .unwrap() + .is_empty()); +} + +#[test] +fn claude_interrupts_an_inflight_bridge_call() { + let state = State::new("claude-held"); + let mock = env!("CARGO_BIN_EXE_fixture-mock-bridge"); + let config = state.0.join("config.json"); + let settings = state.0.join("settings.json"); + std::fs::write( + &config, + json!({"mcpServers":{"engrams":{"type":"stdio","command":mock,"args":["mcp-bridge"]}}}) + .to_string(), + ) + .unwrap(); + std::fs::write( + &settings, + json!({"hooks":{"PreToolUse":[{"hooks":[{"command":format!("{mock} hook-bridge")}]}]}}) + .to_string(), + ) + .unwrap(); + let child = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .args(["--print", "--strict-mcp-config", "--mcp-config"]) + .arg(config) + .arg("--settings") + .arg(settings) + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", "held-call") + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", state.0.join("proof.json")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut child = Fixture(child); + let output = child.0.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(output).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + let receive = || -> Value { + serde_json::from_str(&rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap()).unwrap() + }; + send( + &mut child.0, + json!({"type":"user","message":{"role":"user","content":"fixture"}}), + ); + assert_eq!(receive()["subtype"], "init"); + let deadline = std::time::Instant::now() + Duration::from_secs(3); + while !state.0.join("mock-pending").exists() { + assert!( + std::time::Instant::now() < deadline, + "bridge call did not start" + ); + std::thread::sleep(Duration::from_millis(10)); + } + send( + &mut child.0, + json!({"type":"control_request","request_id":"interrupt-1","request":{"subtype":"interrupt"}}), + ); + assert_eq!(receive()["type"], "assistant"); + assert_eq!(receive()["type"], "control_response"); + assert_eq!(receive()["subtype"], "error_during_execution"); + assert!(std::fs::read(state.0.join("proof.json")) + .unwrap() + .is_empty()); +} + +#[test] +fn claude_refires_the_same_call_after_process_death() { + let state = State::new("refire"); + let mock = env!("CARGO_BIN_EXE_fixture-mock-bridge"); + let config = state.0.join("config.json"); + let settings = state.0.join("settings.json"); + std::fs::write( + &config, + json!({"mcpServers":{"engrams":{"type":"stdio","command":mock,"args":["mcp-bridge"]}}}) + .to_string(), + ) + .unwrap(); + std::fs::write( + &settings, + json!({"hooks":{"PreToolUse":[{"hooks":[{"command":format!("{mock} hook-bridge")}]}]}}) + .to_string(), + ) + .unwrap(); + for attempt in 0..2 { + if attempt == 1 { + std::fs::write(state.0.join("allow-refire"), b"ready").unwrap(); + } + let proof = state.0.join(format!("proof-{attempt}.json")); + let child = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .args(["--print", "--strict-mcp-config", "--mcp-config"]) + .arg(&config) + .arg("--settings") + .arg(&settings) + .args(if attempt == 1 { + vec!["--resume", "fixture-session"] + } else { + vec![] + }) + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", "refire") + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", &proof) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut child = Fixture(child); + let output = child.0.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(output).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + let receive = || -> Value { + serde_json::from_str(&rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap()) + .unwrap() + }; + if attempt == 0 { + send( + &mut child.0, + json!({"type":"user","message":{"role":"user","content":"fixture"}}), + ); + } + assert_eq!(receive()["subtype"], "init"); + assert_eq!(receive()["type"], "assistant"); + if attempt == 1 { + assert_eq!(receive()["type"], "user"); + } + assert_eq!(receive()["type"], "result"); + if attempt == 0 { + assert!(std::fs::read(&proof).unwrap().is_empty()); + } else { + let recorded: Value = serde_json::from_slice(&std::fs::read(&proof).unwrap()).unwrap(); + assert_eq!(recorded["callId"], "fixture-call"); + assert_eq!(recorded["resultReceived"], true); + } + } +} + +#[test] +fn codex_write_denied_and_unavailable_calls_keep_the_native_result() { + for (mode, case) in [ + ("write", "write"), + ("denied", "denied"), + ("read", "unavailable"), + ] { + let state = State::new(case); + let proof = state.0.join("proof.json"); + let child = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .arg("app-server") + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", mode) + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", &proof) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut child = Fixture(child); + let output = child.0.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(output).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + let receive = || -> Value { + serde_json::from_str(&rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap()) + .unwrap() + }; + send( + &mut child.0, + json!({"id":0,"method":"initialize","params":{}}), + ); + assert_eq!(receive()["id"], 0); + send( + &mut child.0, + json!({"id":1,"method":"thread/resume","params":{"threadId":"fixture-thread","dynamicTools":[{"name":"remote_inventory","description":"test","inputSchema":{"type":"object"}}]}}), + ); + assert_eq!(receive()["id"], 1); + send( + &mut child.0, + json!({"id":2,"method":"turn/start","params":{}}), + ); + assert_eq!(receive()["id"], 2); + let call = receive(); + assert_eq!(call["params"]["arguments"]["fixture"], mode); + let success = case == "write"; + let receipt = if success { + json!({"outcome":"succeeded","delivery":"available","isError":false,"resultUnavailable":false}) + } else if case == "unavailable" { + json!({"outcome":"succeeded","delivery":"unavailable","reason":"result_limit","resultUnavailable":true}) + } else { + json!({"source":"runtime","denial":"plan_mode","delivery":"available","isError":true,"resultUnavailable":false}) + }; + send( + &mut child.0, + json!({"id":77,"result":{"success":success,"contentItems":[{"type":"inputText","text":"sentinel"},{"type":"inputText","text":receipt.to_string()}]}}), + ); + assert_eq!(receive()["method"], "item/completed"); + assert_eq!(receive()["method"], "turn/completed"); + let recorded: Value = serde_json::from_slice(&std::fs::read(&proof).unwrap()).unwrap(); + assert_eq!(recorded["success"], success); + assert_eq!( + recorded["isError"], + if case == "unavailable" { + Value::Null + } else { + json!(!success) + } + ); + assert_eq!( + recorded["delivery"], + if case == "unavailable" { + "unavailable" + } else { + "available" + } + ); + assert_eq!(recorded["resultUnavailable"], case == "unavailable"); + assert_eq!(recorded["denialPlanMode"], case == "denied"); + assert_eq!( + recorded["outcome"], + if case == "denied" { + Value::Null + } else { + json!("succeeded") + } + ); + } +} + +#[test] +fn proof_launch_token_claims_distinct_files_with_native_result_observations() { + use scalar_contract::{command, finish, output, receive}; + let state = State::new("proof-launch"); + let mut retained = Vec::new(); + for _ in 0..2 { + let mut child = Fixture( + command(&state, false, "read") + .env("ENGRAM_FIXTURE_PROOF", state.0.join("proof-{launch}.json")) + .spawn() + .unwrap(), + ); + let pid = child.0.id(); + let rx = output(&mut child.0); + send( + &mut child.0, + json!({"id":1,"method":"initialize","params":{}}), + ); + assert_eq!(receive(&rx)["id"], 1); + let files: Vec<_> = std::fs::read_dir(&state.0) + .unwrap() + .map(|entry| entry.unwrap().path()) + .filter(|path| !retained.iter().any(|(old, _)| old == path)) + .collect(); + assert_eq!(files.len(), 1); + let name = files[0].file_name().unwrap().to_str().unwrap(); + let nonce = name + .strip_prefix(&format!("proof-{pid}-")) + .unwrap() + .strip_suffix(".json") + .unwrap(); + assert_eq!(nonce.len(), 32); + assert!(nonce + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))); + send( + &mut child.0, + json!({"id":2,"method":"thread/start","params":{"dynamicTools":[{"name":"remote_inventory","description":"test","inputSchema":{"type":"object"}}]}}), + ); + assert_eq!(receive(&rx)["id"], 2); + send( + &mut child.0, + json!({"id":3,"method":"turn/start","params":{"threadId":"fixture-thread"}}), + ); + assert_eq!(receive(&rx)["id"], 3); + let call = receive(&rx); + assert_eq!(call["method"], "item/tool/call"); + assert_eq!(call["params"]["callId"], "fixture-call"); + send( + &mut child.0, + json!({"id":call["id"],"result":{"success":true,"contentItems":[{"type":"inputText","text":"sentinel"},{"type":"inputText","text":"{\"delivery\":\"available\",\"outcome\":\"succeeded\",\"resultUnavailable\":false,\"isError\":false}"}]}}), + ); + assert_eq!(receive(&rx)["method"], "item/completed"); + assert_eq!(receive(&rx)["method"], "turn/completed"); + finish(&mut child.0); + let bytes = std::fs::read(&files[0]).unwrap(); + assert!(bytes.ends_with(b"\n")); + let record: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(record["resultReceived"], true); + assert_eq!(record["success"], true); + assert_eq!(record["textSentinel"], true); + assert_eq!(record["tool"], "remote_inventory"); + assert_eq!(record["callId"], "fixture-call"); + assert_eq!(record["delivery"], "available"); + assert_eq!(record["outcome"], "succeeded"); + retained.push((files[0].clone(), bytes)); + for (path, bytes) in &retained { + assert_eq!(&std::fs::read(path).unwrap(), bytes); + } + } + assert_eq!(std::fs::read_dir(&state.0).unwrap().count(), 2); +} + +#[test] +fn proof_templates_reject_unknown_mixed_duplicate_and_malformed_tokens_before_protocol() { + for name in [ + "proof-{launch}-{pid}.json", + "proof-{launch}-{launch}.json", + "proof-{pid}-{pid}.json", + "proof-{nonce}.json", + "proof-{launch.json", + "proof-launch}.json", + "proof-{{launch}}.json", + "proof-{PID}.json", + ] { + let state = State::new("proof-invalid-token"); + let mut child = Fixture( + scalar_contract::command(&state, false, "read") + .env("ENGRAM_FIXTURE_PROOF", state.0.join(name)) + .spawn() + .unwrap(), + ); + // Rejection can close stdin before this request is written. + let _ = writeln!( + child.0.stdin.as_mut().unwrap(), + "{}", + json!({"id":1,"method":"initialize","params":{}}) + ); + drop(child.0.stdin.take()); + wait_for_failure(&mut child.0); + let mut error = String::new(); + std::io::Read::read_to_string(&mut child.0.stderr.take().unwrap(), &mut error).unwrap(); + assert_eq!(error, "fixture protocol failure\n", "{name}"); + let mut stdout = String::new(); + std::io::Read::read_to_string(&mut child.0.stdout.take().unwrap(), &mut stdout).unwrap(); + assert!(stdout.is_empty(), "{name}"); + assert_eq!(std::fs::read_dir(&state.0).unwrap().count(), 0, "{name}"); + } +} + +#[test] +fn proof_pid_token_is_expanded_only_in_the_direct_child_basename() { + let state = State::new("proof-pid"); + for _ in 0..2 { + let mut child = Fixture( + Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")) + .arg("app-server") + .env_clear() + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", "read") + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", state.0.join("proof-{pid}.json")) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(), + ); + let pid = child.0.id(); + scalar_contract::finish(&mut child.0); + assert!(state.0.join(format!("proof-{pid}.json")).is_file()); + } + assert_eq!(std::fs::read_dir(&state.0).unwrap().count(), 2); + assert!(!state.0.join("proof-{pid}.json").exists()); +} + +#[path = "support/plan_contract.rs"] +mod plan_contract; +#[path = "support/scalar_contract.rs"] +mod scalar_contract; diff --git a/tests/fixtures/mcp-model-cli/tests/support/mock_bridge.rs b/tests/fixtures/mcp-model-cli/tests/support/mock_bridge.rs new file mode 100644 index 000000000..53cd9f3f0 --- /dev/null +++ b/tests/fixtures/mcp-model-cli/tests/support/mock_bridge.rs @@ -0,0 +1,190 @@ +#[path = "../../src/scalar.rs"] +mod scalar; +type Result = std::result::Result>; + +use serde_json::{json, Value}; +use std::io::{self, BufRead, Read, Write}; + +fn main() { + match std::env::args().nth(1).as_deref() { + Some("hook-bridge") => { + let mut input = String::new(); + io::stdin().read_to_string(&mut input).unwrap(); + let value: Value = serde_json::from_str(&input).unwrap(); + assert!(value["tool_use_id"] == call_id()); + assert!(value["tool_input"] == arguments()); + let forged = std::env::var("ENGRAM_FIXTURE_MODE").as_deref() == Ok("forged"); + assert!( + value["tool_name"] + == format!( + "mcp__engrams__{}", + if forged { + "fixture_forged".to_owned() + } else { + tool() + } + ) + ); + let deferred = std::env::var("ENGRAM_FIXTURE_MODE").as_deref() == Ok("refire") + && !std::path::Path::new(&std::env::var_os("ENGRAM_FIXTURE_STATE_DIR").unwrap()) + .join("allow-refire") + .exists(); + println!( + "{}", + json!({"hookSpecificOutput":{"hookEventName":"PreToolUse","permissionDecision":if deferred { "defer" } else { "allow" }}}) + ); + } + Some("mcp-bridge") => { + for line in io::stdin().lock().lines() { + let v: Value = serde_json::from_str(&line.unwrap()).unwrap(); + let id = v["id"].clone(); + if v["method"] == "initialize" { + let root = std::env::var_os("ENGRAM_FIXTURE_STATE_DIR").unwrap(); + let marker = std::path::Path::new(&root).join("bad-initialize"); + if let Ok(case) = std::fs::read_to_string(marker) { + let response = match case.as_str() { + "error" => { + json!({"jsonrpc":"2.0","id":id,"error":{"code":-32603,"message":"fixture error"}}) + } + "missing" => json!({"jsonrpc":"2.0","id":id}), + "malformed" => { + json!({"jsonrpc":"2.0","id":id,"result":{"protocolVersion":"2025-06-18","capabilities":[],"serverInfo":{"name":"fixture","version":"1"}}}) + } + "server-info" => { + json!({"jsonrpc":"2.0","id":id,"result":{"protocolVersion":"2025-06-18","capabilities":{},"serverInfo":{"name":"","version":"1"}}}) + } + "version" => { + json!({"jsonrpc":"2.0","id":id,"result":{"protocolVersion":"unsupported","capabilities":{},"serverInfo":{"name":"fixture","version":"1"}}}) + } + "both" => { + json!({"jsonrpc":"2.0","id":id,"error":{"code":-32603,"message":"fixture error"},"result":{"protocolVersion":"2025-06-18","capabilities":{},"serverInfo":{"name":"fixture","version":"1"}}}) + } + _ => panic!("invalid case"), + }; + println!("{response}"); + io::stdout().flush().unwrap(); + continue; + } + } + if v["method"] == "tools/call" && v["params"]["name"] == "fixture_forged" { + println!( + "{}", + json!({"jsonrpc":"2.0","id":id,"error":{"code":-32602,"message":"Unknown tool"}}) + ); + io::stdout().flush().unwrap(); + continue; + } + let result = match v["method"].as_str().unwrap() { + "initialize" => { + json!({"protocolVersion":"2025-06-18","capabilities":{"tools":{}},"serverInfo":{"name":"fixture","version":"1"}}) + } + "tools/list" => { + let root = std::env::var_os("ENGRAM_FIXTURE_STATE_DIR").unwrap(); + if let Ok(raw) = + std::fs::read(std::path::Path::new(&root).join("exposed-tools.json")) + { + json!({"tools":serde_json::from_slice::(&raw).unwrap()}) + } else { + json!({"tools":[{"name":"remote_inventory","inputSchema":{"type":"object"}}]}) + } + } + "tools/call" => { + assert!(v["params"]["_meta"]["claudecode/toolUseId"] == call_id()); + assert!(v["params"]["arguments"] == arguments()); + let root = std::env::var_os("ENGRAM_FIXTURE_STATE_DIR").unwrap(); + if std::path::Path::new(&root).join("hold-result").exists() { + std::fs::write( + std::path::Path::new(&root).join("mock-pending"), + b"ready", + ) + .unwrap(); + continue; + } + if let Ok(raw) = + std::fs::read(std::path::Path::new(&root).join("native-responses.json")) + { + let responses: Value = serde_json::from_slice(&raw).unwrap(); + let mut response = responses.get(call_id()).unwrap().clone(); + response["id"] = id; + println!("{response}"); + io::stdout().flush().unwrap(); + continue; + } + if let Ok(raw) = + std::fs::read(std::path::Path::new(&root).join("native-response.json")) + { + let mut response: Value = serde_json::from_slice(&raw).unwrap(); + response["id"] = id; + println!("{response}"); + io::stdout().flush().unwrap(); + continue; + } + if std::env::var("ENGRAM_FIXTURE_MODE").as_deref() == Ok("held-call") { + let root = std::env::var_os("ENGRAM_FIXTURE_STATE_DIR").unwrap(); + std::fs::write( + std::path::Path::new(&root).join("mock-pending"), + b"ready", + ) + .unwrap(); + continue; + } + if std::env::var("ENGRAM_FIXTURE_MODE").as_deref() == Ok("denied") { + json!({"content":[{"type":"text","text":"sentinel"},{"type":"text","text":"second sentinel"}],"structuredContent":{"fixture":true},"isError":true,"_meta":{"engrams/toolResult":{"source":"runtime","denial":"plan_mode","delivery":"available","resultUnavailable":false,"isError":true}}}) + } else { + json!({"content":[{"type":"text","text":"sentinel"},{"type":"text","text":"second sentinel"}],"structuredContent":{"fixture":true},"isError":false,"_meta":{"engrams/toolResult":{"outcome":"succeeded","delivery":"available","resultUnavailable":false,"isError":false}}}) + } + } + "notifications/initialized" => continue, + "notifications/cancelled" => continue, + _ => panic!("unexpected method"), + }; + println!("{}", json!({"jsonrpc":"2.0","id":id,"result":result})); + io::stdout().flush().unwrap(); + } + } + _ => std::process::exit(1), + } +} + +fn active_step() -> Option { + let plan = + scalar::configured_object("ENGRAM_FIXTURE_CALL_PLAN_JSON", 65536, 128 + 3).unwrap()?; + let root = std::env::var_os("ENGRAM_FIXTURE_STATE_DIR").unwrap(); + let receipt: Value = serde_json::from_slice( + &std::fs::read(std::path::Path::new(&root).join("plan-selection.json")).unwrap(), + ) + .unwrap(); + assert!(receipt["planSha256"] == scalar::digest(&plan).unwrap()); + Some( + plan["steps"] + .as_array() + .unwrap() + .iter() + .find(|step| step["marker"] == receipt["marker"]) + .unwrap() + .clone(), + ) +} +fn call_id() -> String { + active_step() + .map(|step| step["callId"].as_str().unwrap().to_owned()) + .unwrap_or_else(|| { + std::env::var("ENGRAM_FIXTURE_CALL_ID").unwrap_or_else(|_| "fixture-call".into()) + }) +} +fn tool() -> String { + active_step() + .map(|step| step["tool"].as_str().unwrap().to_owned()) + .unwrap_or_else(|| std::env::var("ENGRAM_FIXTURE_TOOL").unwrap()) +} +fn arguments() -> Value { + active_step() + .map(|step| step["arguments"].clone()) + .unwrap_or_else(|| { + std::env::var("ENGRAM_FIXTURE_ARGUMENTS_JSON") + .map(|raw| serde_json::from_str(&raw).unwrap()) + .unwrap_or_else( + |_| json!({"fixture":std::env::var("ENGRAM_FIXTURE_MODE").unwrap()}), + ) + }) +} diff --git a/tests/fixtures/mcp-model-cli/tests/support/plan_contract.rs b/tests/fixtures/mcp-model-cli/tests/support/plan_contract.rs new file mode 100644 index 000000000..fc8150cdd --- /dev/null +++ b/tests/fixtures/mcp-model-cli/tests/support/plan_contract.rs @@ -0,0 +1,1239 @@ +use super::scalar_contract::{command, finish, hash, native, output, receive}; +use super::*; + +fn tools() -> Value { + json!([ + {"name":"spawn_session","description":"Spawn a child session","inputSchema":{"type":"object"}}, + {"name":"mcp_write_counter","description":"Write count","inputSchema":{"type":"object"}}, + {"name":"mcp_read_counter","description":"Read count","inputSchema":{"type":"object"}} + ]) +} +fn plan(claude: bool) -> Value { + json!({"steps":[ + {"marker":"old","tool":"mcp_write_counter","arguments":{"delta":1},"callId":"write-old","resultFormat":"mcp","expectedResult":native(claude,false)}, + {"marker":"new","tool":"mcp_write_counter","arguments":{"delta":1},"callId":"write-new","resultFormat":"mcp","expectedResult":native(claude,true)}, + {"marker":"spawn","tool":"spawn_session","arguments":{"task_name":"fixture-child","message":"[[fixture:child]]","idempotency_key":"child-once"},"callId":"spawn-once","resultFormat":"json"}, + {"marker":"child","tool":"mcp_read_counter","arguments":{},"callId":"child-read","resultFormat":"mcp","expectedResult":native(claude,false)} + ]}) +} +fn spawn_result(claude: bool) -> Value { + let text = r#"{"session_id":"neutral-child-id","canonical_task_name":"root/fixture-child"}"#; + if claude { + json!({"content":[{"type":"text","text":text}]}) + } else { + json!({"success":true,"contentItems":[{"type":"inputText","text":text}]}) + } +} +fn plan_command(state: &State, claude: bool, plan: &Value, mode: &str) -> Command { + let mut cmd = command(state, claude, mode); + cmd.env_remove("ENGRAM_FIXTURE_TOOL") + .env("ENGRAM_FIXTURE_CALL_PLAN_JSON", plan.to_string()); + std::fs::write(state.0.join("exposed-tools.json"), tools().to_string()).unwrap(); + cmd +} +fn initialize( + child: &mut Child, + rx: &mpsc::Receiver>, + claude: bool, + exposed: Value, +) { + if claude { + assert_eq!(receive(rx)["type"], "system"); + } else { + send(child, json!({"id":1,"method":"initialize"})); + assert_eq!(receive(rx)["id"], 1); + send( + child, + json!({"id":2,"method":"thread/start","params":{"dynamicTools":exposed}}), + ); + assert_eq!(receive(rx)["id"], 2); + } +} +fn prompt(child: &mut Child, claude: bool, text: &str) { + send( + child, + if claude { + json!({"type":"user","message":{"role":"user","content":[{"type":"text","text":text}]}}) + } else { + json!({"id":3,"method":"turn/start","params":{"input":[{"type":"text","text":text}]}}) + }, + ); +} +fn call(rx: &mpsc::Receiver>, claude: bool) -> Value { + if claude { + receive(rx)["message"]["content"][0].clone() + } else { + assert_eq!(receive(rx)["id"], 3); + receive(rx)["params"].clone() + } +} +fn complete( + child: &mut Child, + rx: &mpsc::Receiver>, + claude: bool, + result: Value, +) { + if !claude { + send(child, json!({"id":77,"result":result})); + } + let native_completed = receive(rx); + assert_eq!( + if claude { + &native_completed["type"] + } else { + &native_completed["method"] + }, + if claude { "user" } else { "item/completed" } + ); + let completed = receive(rx); + assert_eq!( + if claude { + &completed["type"] + } else { + &completed["method"] + }, + if claude { "result" } else { "turn/completed" } + ); +} +fn proof(state: &State) -> Vec { + std::fs::read_to_string(state.0.join("proof.json")) + .unwrap() + .split_inclusive('\n') + .filter(|line| line.ends_with('\n')) + .map(|line| serde_json::from_str(line).unwrap()) + .collect() +} +#[test] +fn fixed_markers_drive_new_and_repeated_ids_native_spawn_and_child_steps() { + for claude in [false, true] { + let state = State::new("plan-turns"); + let plan = plan(claude); + let response = spawn_result(claude); + let unknown = json!({"source":"runtime","outcome":"outcome_unknown","delivery":"unavailable","reason":"execution_interrupted","resultUnavailable":false}); + let old_response = if claude { + json!({"jsonrpc":"2.0","error":{"code":-32603,"message":"External tool output is unavailable.","data":unknown}}) + } else { + json!({"result":{"success":false,"contentItems":[{"type":"inputText","text":unknown.to_string()}]}}) + }; + let responses = json!({ + "write-old":old_response, + "write-new":{"jsonrpc":"2.0","result":native(claude,true)}, + "spawn-once":{"jsonrpc":"2.0","result":response}, + "child-read":{"jsonrpc":"2.0","result":native(claude,false)} + }); + std::fs::write(state.0.join("native-responses.json"), responses.to_string()).unwrap(); + let mut child = Fixture( + plan_command(&state, claude, &plan, "write") + .spawn() + .unwrap(), + ); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, claude, tools()); + for marker in ["old", "old", "new", "spawn", "child"] { + let step = plan["steps"] + .as_array() + .unwrap() + .iter() + .find(|step| step["marker"] == marker) + .unwrap(); + prompt(&mut child.0, claude, &format!("Run [[fixture:{marker}]]")); + let call = call(&rx, claude); + assert_eq!(call[if claude { "id" } else { "callId" }], step["callId"]); + assert_eq!( + call[if claude { "input" } else { "arguments" }], + step["arguments"] + ); + if marker == "child" { + // The exact fixed child message selects the read step, not spawn. + assert_eq!( + call[if claude { "name" } else { "tool" }], + if claude { + "mcp__engrams__mcp_read_counter" + } else { + "mcp_read_counter" + } + ); + } + complete( + &mut child.0, + &rx, + claude, + responses[step["callId"].as_str().unwrap()]["result"].clone(), + ); + } + finish(&mut child.0); + let proof = proof(&state); + assert_eq!(proof.len(), 5); + for entry in &proof { + assert_eq!(entry["planSha256"], hash(&plan)); + assert_eq!( + entry["exposedMcpToolNamesSha256"], + hash(&json!(["mcp_read_counter", "mcp_write_counter"])) + ); + } + assert_eq!(proof[0]["callId"], proof[1]["callId"]); + assert_eq!(proof[0]["outcome"], "outcome_unknown"); + assert_eq!(proof[0]["delivery"], "unavailable"); + assert!(proof[0].get("resultMatchesExpected").is_none()); + assert_ne!(proof[1]["callId"], proof[2]["callId"]); + assert_eq!(proof[2]["isError"], true); + assert_eq!(proof[3]["resultSha256"], hash(&response)); + for key in [ + "resultMatchesExpected", + "expectedResultSha256", + "delivery", + "outcome", + "isError", + "resultUnavailable", + ] { + assert!(proof[3].get(key).is_none()); + } + assert!(!std::fs::read_to_string(state.0.join("proof.json")) + .unwrap() + .contains("neutral-child-id")); + let receipt: Value = + serde_json::from_slice(&std::fs::read(state.0.join("plan-selection.json")).unwrap()) + .unwrap(); + assert_eq!(receipt["pending"], false); + assert_eq!(receipt["marker"], "child"); + } +} +#[test] +fn plan_rejects_missing_unknown_ambiguous_and_tool_result_markers() { + for claude in [false, true] { + for text in [ + "none", + "[[fixture:unknown]]", + "[[fixture:old]] [[fixture:new]]", + "[[fixture:old]] [[fixture:old]]", + "[[fixture:old", + ] { + let state = State::new("invalid-marker"); + let mut child = Fixture( + plan_command(&state, claude, &plan(claude), "read") + .spawn() + .unwrap(), + ); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, claude, tools()); + prompt(&mut child.0, claude, text); + wait_for_failure(&mut child.0); + assert!(proof(&state).is_empty()); + assert!(!state.0.join("plan-selection.json").exists()); + } + let state = State::new("tool-result-marker"); + let mut child = Fixture( + plan_command(&state, claude, &plan(claude), "read") + .spawn() + .unwrap(), + ); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, claude, tools()); + send( + &mut child.0, + if claude { + json!({"type":"user","message":{"content":[{"type":"tool_result","content":"[[fixture:old]]"}]}}) + } else { + json!({"id":3,"method":"turn/start","params":{"input":[{"type":"tool_result","text":"[[fixture:old]]"}]}}) + }, + ); + wait_for_failure(&mut child.0); + assert!(proof(&state).is_empty()); + } +} +#[test] +fn invalid_plan_configuration_is_rejected_before_any_protocol_output() { + let valid = plan(false); + let mut cases = vec![ + json!({"steps":[]}), + json!({"steps":vec![valid["steps"][0].clone();17]}), + ]; + for field in ["marker", "tool", "arguments", "callId", "resultFormat"] { + let mut changed = valid.clone(); + changed["steps"][0].as_object_mut().unwrap().remove(field); + cases.push(changed); + } + for (field, value) in [ + ("extra", json!(true)), + ("marker", json!("bad/id")), + ("arguments", json!([])), + ("resultFormat", json!("guess")), + ("expectedResult", json!(false)), + ] { + let mut changed = valid.clone(); + changed["steps"][0][field] = value; + cases.push(changed); + } + for field in ["marker", "callId"] { + let mut changed = valid.clone(); + changed["steps"][1][field] = changed["steps"][0][field].clone(); + cases.push(changed); + } + for value in cases { + let state = State::new("invalid-plan"); + let mut child = Fixture(plan_command(&state, false, &value, "read").spawn().unwrap()); + wait_for_failure(&mut child.0); + assert!(!state.0.join("proof.json").exists()); + } + let state = State::new("conflicting-plan"); + let mut cmd = plan_command(&state, false, &valid, "read"); + cmd.env("ENGRAM_FIXTURE_CALL_ID", "conflict"); + let mut child = Fixture(cmd.spawn().unwrap()); + wait_for_failure(&mut child.0); + assert!(!state.0.join("proof.json").exists()); +} +#[test] +fn claude_promptless_plan_resume_requires_exact_pending_private_receipt() { + for case in [ + "missing", + "complete", + "changed-plan", + "wrong-harness", + "wrong-mode", + "malformed", + "symlink", + "unknown-marker", + "pending", + ] { + let state = State::new("plan-receipt"); + let plan = plan(true); + let mut receipt = json!({"planSha256":hash(&plan),"harness":"claude","mode":"write","marker":"old","pending":true}); + match case { + "complete" => receipt["pending"] = json!(false), + "changed-plan" => receipt["planSha256"] = json!("sha256:other"), + "wrong-harness" => receipt["harness"] = json!("codex"), + "wrong-mode" => receipt["mode"] = json!("read"), + "unknown-marker" => receipt["marker"] = json!("unknown"), + _ => {} + } + let path = state.0.join("plan-selection.json"); + if case == "symlink" { + std::fs::write(state.0.join("other.json"), receipt.to_string()).unwrap(); + std::os::unix::fs::symlink(state.0.join("other.json"), &path).unwrap(); + } else if case != "missing" { + std::fs::write( + &path, + if case == "malformed" { + "{bad".into() + } else { + receipt.to_string() + }, + ) + .unwrap(); + } + std::fs::write( + state.0.join("native-response.json"), + json!({"jsonrpc":"2.0","result":native(true,false)}).to_string(), + ) + .unwrap(); + let mut cmd = plan_command(&state, true, &plan, "write"); + cmd.args(["--resume", "fixture-session"]); + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, true, tools()); + if case == "pending" { + assert_eq!(call(&rx, true)["id"], "write-old"); + complete(&mut child.0, &rx, true, Value::Null); + finish(&mut child.0); + assert_eq!(proof(&state)[0]["resultMatchesExpected"], true); + let receipt: Value = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + assert_eq!(receipt["pending"], false); + } else if ["missing", "complete"].contains(&case) { + // A conversation resume without pending work waits for a new prompt. + assert!(matches!( + rx.recv_timeout(Duration::from_millis(100)), + Err(mpsc::RecvTimeoutError::Timeout) + )); + assert!(child.0.try_wait().unwrap().is_none()); + assert!(proof(&state).is_empty()); + finish(&mut child.0); + } else { + wait_for_failure(&mut child.0); + assert!(proof(&state).is_empty()); + } + } +} + +#[test] +fn claude_plan_resume_reads_a_new_prompt_after_missing_or_completed_selection() { + for receipt_state in ["missing", "complete"] { + let state = State::new("plan-current-prompt"); + let plan = json!({"steps":[ + {"marker":"allowed","tool":"mcp_write_counter","arguments":{"delta":1},"callId":"policy-allowed","resultFormat":"mcp"}, + {"marker":"plan","tool":"exit_plan_mode","arguments":{"plan":"Run the neutral counter call."},"callId":"policy-plan","resultFormat":"json"} + ]}); + if receipt_state == "complete" { + std::fs::write( + state.0.join("plan-selection.json"), + json!({"planSha256":hash(&plan),"harness":"claude","mode":"write","marker":"allowed","pending":false}).to_string(), + ) + .unwrap(); + } + let expected = json!({"content":[{"type":"text","text":"{\"decision\":\"approve\"}"}]}); + std::fs::write( + state.0.join("native-responses.json"), + json!({"policy-plan":{"jsonrpc":"2.0","result":expected}}).to_string(), + ) + .unwrap(); + let mut command = plan_command(&state, true, &plan, "write"); + command.args(["--resume", "fixture-session"]); + let mut exposed = tools(); + exposed.as_array_mut().unwrap().push(json!({ + "name":"exit_plan_mode","description":"Present a plan", + "inputSchema":{"type":"object"} + })); + std::fs::write(state.0.join("exposed-tools.json"), exposed.to_string()).unwrap(); + let mut child = Fixture(command.spawn().unwrap()); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, true, exposed); + prompt(&mut child.0, true, "[[fixture:plan]]"); + let selected = call(&rx, true); + assert_eq!(selected["id"], "policy-plan"); + assert_eq!(selected["name"], "mcp__engrams__exit_plan_mode"); + assert_eq!(selected["input"], plan["steps"][1]["arguments"]); + complete(&mut child.0, &rx, true, Value::Null); + finish(&mut child.0); + let recorded = proof(&state); + assert_eq!(recorded.len(), 1); + assert_eq!(recorded[0]["callId"], "policy-plan"); + assert_eq!(recorded[0]["tool"], "exit_plan_mode"); + assert_eq!(recorded[0]["planSha256"], hash(&plan)); + assert_eq!(recorded[0]["resultReceived"], true); + assert_eq!(recorded[0]["resultSha256"], hash(&expected)); + let receipt: Value = + serde_json::from_slice(&std::fs::read(state.0.join("plan-selection.json")).unwrap()) + .unwrap(); + assert_eq!(receipt["marker"], "plan"); + assert_eq!(receipt["pending"], false); + } +} + +#[test] +fn lost_native_reply_replays_only_the_pending_marker_after_restart() { + let state = State::new("lost-native-reply"); + let plan = plan(true); + std::fs::write(state.0.join("hold-result"), b"hold").unwrap(); + let mut child = Fixture(plan_command(&state, true, &plan, "write").spawn().unwrap()); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, true, tools()); + prompt(&mut child.0, true, "[[fixture:old]]"); + assert_eq!(call(&rx, true)["id"], "write-old"); + let deadline = std::time::Instant::now() + Duration::from_secs(3); + while !state.0.join("mock-pending").exists() { + assert!(std::time::Instant::now() < deadline); + std::thread::sleep(Duration::from_millis(10)); + } + // Closing input aborts the pending receive and reaps the real mock bridge. + drop(child.0.stdin.take()); + wait_for_failure(&mut child.0); + assert!(proof(&state).is_empty()); + let receipt: Value = + serde_json::from_slice(&std::fs::read(state.0.join("plan-selection.json")).unwrap()) + .unwrap(); + assert_eq!(receipt["pending"], true); + assert_eq!(receipt["marker"], "old"); + assert_eq!(receipt.as_object().unwrap().len(), 5); + std::fs::remove_file(state.0.join("hold-result")).unwrap(); + std::fs::write( + state.0.join("native-response.json"), + json!({"jsonrpc":"2.0","result":native(true,false)}).to_string(), + ) + .unwrap(); + let mut cmd = plan_command(&state, true, &plan, "write"); + cmd.args(["--resume", "fixture-session"]) + .env("ENGRAM_FIXTURE_PROOF", state.0.join("replayed.json")); + let mut replay = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut replay.0); + initialize(&mut replay.0, &rx, true, tools()); + let call = call(&rx, true); + assert_eq!(call["id"], "write-old"); + assert_eq!(call["input"], json!({"delta":1})); + complete(&mut replay.0, &rx, true, Value::Null); + finish(&mut replay.0); + let receipt: Value = + serde_json::from_slice(&std::fs::read(state.0.join("plan-selection.json")).unwrap()) + .unwrap(); + assert_eq!(receipt["pending"], false); + assert_eq!(receipt["marker"], "old"); + let observed: Value = + serde_json::from_slice(&std::fs::read(state.0.join("replayed.json")).unwrap()).unwrap(); + assert_eq!(observed["resultMatchesExpected"], true); +} + +#[test] +fn actual_exposed_mcp_alias_set_is_sorted_bounded_and_validated() { + for claude in [false, true] { + for case in ["valid", "extra", "invalid", "duplicate", "over-limit"] { + let state = State::new("exposure"); + let plan = plan(claude); + let mut exposed = tools().as_array().unwrap().clone(); + match case { + "valid"=>exposed.reverse(), + "extra"=>exposed.push(json!({"name":"mcp_unapproved","description":"extra","inputSchema":{"type":"object"}})), + "invalid"=>exposed.push(json!({"name":"mcp_bad/name"})), + "duplicate"=>exposed.push(exposed[1].clone()), + "over-limit"=>for i in 0..127 {exposed.push(json!({"name":format!("mcp_extra_{i}")}));}, + _=>unreachable!() + } + let mut cmd = plan_command(&state, claude, &plan, "read"); + std::fs::write( + state.0.join("exposed-tools.json"), + json!(exposed).to_string(), + ) + .unwrap(); + std::fs::write( + state.0.join("native-response.json"), + json!({"jsonrpc":"2.0","result":native(claude,false)}).to_string(), + ) + .unwrap(); + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + if ["invalid", "duplicate", "over-limit"].contains(&case) { + if !claude { + send(&mut child.0, json!({"id":1,"method":"initialize"})); + assert_eq!(receive(&rx)["id"], 1); + send( + &mut child.0, + json!({"id":2,"method":"thread/start","params":{"dynamicTools":exposed}}), + ); + } + wait_for_failure(&mut child.0); + assert!(proof(&state).is_empty()); + continue; + } + initialize(&mut child.0, &rx, claude, json!(exposed)); + prompt(&mut child.0, claude, "[[fixture:child]]"); + call(&rx, claude); + complete(&mut child.0, &rx, claude, native(claude, false)); + finish(&mut child.0); + let expected = if case == "extra" { + json!(["mcp_read_counter", "mcp_unapproved", "mcp_write_counter"]) + } else { + json!(["mcp_read_counter", "mcp_write_counter"]) + }; + assert_eq!( + proof(&state)[0]["exposedMcpToolNamesSha256"], + hash(&expected) + ); + } + } +} + +fn nested_object(depth: usize) -> Value { + let mut value = json!(0); + for _ in 0..depth { + value = json!({"nested":value}); + } + value +} + +#[test] +fn expected_result_nested_native_object_matches_in_scalar_and_plan_modes() { + // Claude delivers structuredContent as a nested object. Codex delivers it + // as a literal text string, so it does not have this native nesting depth. + for depth in [34, 126] { + let mut actual = native(true, false); + actual["structuredContent"] = nested_object(depth - 1); + let scalar = super::scalar_contract::exercise( + true, + "read", + json!({}), + "nested-read", + actual.clone(), + actual.clone(), + false, + ); + assert_eq!(scalar["resultMatchesExpected"], true); + let state = State::new("nested-expected-plan"); + let plan = json!({"steps":[{ + "marker":"nested","tool":"mcp_read_counter","arguments":{}, + "callId":"nested-read","resultFormat":"mcp","expectedResult":actual + }]}); + std::fs::write( + state.0.join("native-response.json"), + json!({"jsonrpc":"2.0","result":actual}).to_string(), + ) + .unwrap(); + let mut child = Fixture(plan_command(&state, true, &plan, "read").spawn().unwrap()); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, true, tools()); + prompt(&mut child.0, true, "[[fixture:nested]]"); + assert_eq!(call(&rx, true)["id"], "nested-read"); + complete(&mut child.0, &rx, true, Value::Null); + finish(&mut child.0); + let planned = &proof(&state)[0]; + assert_eq!(planned["resultMatchesExpected"], true); + assert_eq!(planned["resultSha256"], scalar["resultSha256"]); + assert_eq!( + planned["expectedResultSha256"], + scalar["expectedResultSha256"] + ); + } +} + +fn assert_plan_configuration(plan: &Value, valid: bool) { + let state = State::new("expected-result-plan-boundary"); + let mut child = Fixture(plan_command(&state, false, plan, "read").spawn().unwrap()); + let rx = output(&mut child.0); + if valid { + send(&mut child.0, json!({"id":1,"method":"initialize"})); + assert_eq!(receive(&rx)["id"], 1); + finish(&mut child.0); + } else { + wait_for_failure(&mut child.0); + assert!(!state.0.join("proof.json").exists()); + assert!(matches!( + rx.recv_timeout(Duration::from_secs(3)), + Err(mpsc::RecvTimeoutError::Disconnected) + )); + } +} + +#[test] +fn expected_result_plan_depth_and_byte_boundaries_preserve_argument_limits() { + let base = json!({"steps":[{ + "marker":"nested","tool":"mcp_read_counter","arguments":{}, + "callId":"nested-read","resultFormat":"mcp","expectedResult":{} + }]}); + for depth in [127, 128, 129] { + let expected = nested_object(depth); + let scalar = super::scalar_contract::config_parser::parse_object( + &expected.to_string(), + 1024 * 1024, + 128, + ); + assert_eq!(scalar.is_ok(), depth <= 128); + let mut plan = base.clone(); + plan["steps"][0]["expectedResult"] = expected; + assert_plan_configuration(&plan, depth <= 128); + } + for depth in [16, 17] { + let mut plan = base.clone(); + plan["steps"][0]["expectedResult"] = nested_object(128); + plan["steps"][0]["arguments"] = nested_object(depth); + assert_plan_configuration(&plan, depth == 16); + } + for bytes in [4096, 4097] { + let mut plan = base.clone(); + plan["steps"][0]["arguments"] = json!({"a":"x".repeat(bytes - 8)}); + assert_eq!(plan["steps"][0]["arguments"].to_string().len(), bytes); + assert_plan_configuration(&plan, bytes == 4096); + } + let mut plan = base; + plan["steps"][0]["expectedResult"] = json!({"padding":""}); + let padding = 65536 - plan.to_string().len(); + for extra in [0, 1] { + plan["steps"][0]["expectedResult"]["padding"] = json!("x".repeat(padding + extra)); + assert_eq!(plan.to_string().len(), 65536 + extra); + assert_plan_configuration(&plan, extra == 0); + } +} + +fn evidence_plan(absent: bool, premature: bool) -> Value { + let mut step = json!({"marker":"probe","tool":"mcp_read_counter","arguments":{},"callId":"probe-call","resultFormat":"mcp"}); + if absent { + step["expectExposure"] = json!("absent"); + } + if premature { + step["completionProjection"] = json!("premature"); + } + json!({"steps":[step]}) +} +fn release(state: &State, name: &str) { + use std::os::unix::fs::OpenOptionsExt; + std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(state.0.join(name)) + .unwrap(); +} +fn wait_observation(state: &State, observation: &str) -> Vec { + let deadline = std::time::Instant::now() + Duration::from_secs(3); + loop { + let records = proof(state); + if records.iter().any(|v| v["observation"] == observation) { + return records; + } + assert!( + std::time::Instant::now() < deadline, + "missing {observation}" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} +fn interrupt(child: &mut Child, claude: bool) { + send( + child, + if claude { + json!({"type":"control_request","request_id":"stop","request":{"subtype":"interrupt"}}) + } else { + json!({"id":4,"method":"turn/interrupt","params":{"threadId":"fixture-thread","turnId":"fixture-turn"}}) + }, + ); +} +fn assert_completion(event: &Value, claude: bool, native: &Value) { + if claude { + assert_eq!(event["type"], "user"); + assert_eq!(event["message"]["content"][0]["type"], "tool_result"); + assert_eq!(event["message"]["content"][0]["tool_use_id"], "probe-call"); + assert_eq!(event["message"]["content"][0]["content"], native["content"]); + assert_eq!( + event["message"]["content"][0]["is_error"], + native["isError"] + ); + } else { + assert_eq!(event["method"], "item/completed"); + assert_eq!(event["params"]["item"]["id"], "probe-call"); + assert_eq!(event["params"]["item"]["tool"], "mcp_read_counter"); + assert_eq!(event["params"]["item"]["type"], "dynamicToolCall"); + assert_eq!(event["params"]["item"]["success"], native["success"]); + assert_eq!(event["params"]["item"]["result"], *native); + } +} + +#[test] +fn native_completion_is_after_proof_and_private_release_on_both_protocols() { + for claude in [false, true] { + let state = State::new("completion-release"); + let plan = evidence_plan(false, false); + let actual = native(claude, true); + let mut cmd = plan_command(&state, claude, &plan, "read"); + cmd.env( + "ENGRAM_FIXTURE_COMPLETION_BARRIER", + state.0.join("completion-release"), + ); + std::fs::write( + state.0.join("native-response.json"), + json!({"result":actual}).to_string(), + ) + .unwrap(); + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, claude, tools()); + prompt(&mut child.0, claude, "[[fixture:probe]]"); + call(&rx, claude); + if !claude { + send(&mut child.0, json!({"id":77,"result":actual})); + } + let records = wait_observation(&state, "completionBarrierHeld"); + assert_eq!(records[0]["resultReceived"], true); + assert_eq!(records[0]["outcome"], "failed"); + assert!(matches!( + rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + )); + release(&state, "completion-release"); + assert_completion(&receive(&rx), claude, &actual); + assert_eq!( + receive(&rx)[if claude { "type" } else { "method" }], + if claude { "result" } else { "turn/completed" } + ); + finish(&mut child.0); + assert!(!state.0.join("completion-release").exists()); + assert_eq!( + proof(&state).last().unwrap()["observation"], + "completionBarrierReleased" + ); + } +} + +#[test] +fn absent_exposure_calls_the_fixed_alias_and_proves_only_native_denial() { + for claude in [false, true] { + let state = State::new("absent-probe"); + let plan = evidence_plan(true, false); + let mut cmd = plan_command(&state, claude, &plan, "read"); + std::fs::write(state.0.join("exposed-tools.json"), "[]").unwrap(); + std::fs::write( + state.0.join("native-response.json"), + json!({"error":{"code":-32602,"message":"Unknown tool"}}).to_string(), + ) + .unwrap(); + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, claude, json!([])); + prompt(&mut child.0, claude, "[[fixture:probe]]"); + let called = call(&rx, claude); + assert_eq!( + called[if claude { "name" } else { "tool" }], + if claude { + "mcp__engrams__mcp_read_counter" + } else { + "mcp_read_counter" + } + ); + if !claude { + send( + &mut child.0, + json!({"id":77,"result":{"success":false,"contentItems":[{"type":"inputText","text":"undeclared dynamic tool: mcp_read_counter"}]}}), + ); + } + let completed = receive(&rx); + assert_eq!( + completed[if claude { "type" } else { "method" }], + if claude { "user" } else { "item/completed" } + ); + receive(&rx); + finish(&mut child.0); + let records = proof(&state); + assert_eq!(records.len(), 1); + let record = &records[0]; + assert_eq!(record["expectExposure"], "absent"); + assert_eq!(record["requestAttempted"], true); + assert_eq!(record["protocolDenialCode"], "unknown_tool"); + assert_eq!(record["exposedMcpToolNamesSha256"], hash(&json!([]))); + assert_eq!(record["callId"], "probe-call"); + for field in [ + "selectedSchemaSha256", + "resultReceived", + "resultSha256", + "outcome", + "delivery", + "success", + ] { + assert!(record.get(field).is_none()); + } + } +} + +#[test] +fn premature_projection_uses_pending_call_and_records_only_emitted_event() { + for claude in [false, true] { + let state = State::new("premature-probe"); + let plan = evidence_plan(false, true); + let mut cmd = plan_command(&state, claude, &plan, "refire"); + cmd.env( + "ENGRAM_FIXTURE_PREMATURE_BARRIER", + state.0.join("premature-release"), + ); + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, claude, tools()); + prompt(&mut child.0, claude, "[[fixture:probe]]"); + call(&rx, claude); + wait_observation(&state, "prematureBarrierHeld"); + assert!(matches!( + rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + )); + release(&state, "premature-release"); + let event = receive(&rx); + if claude { + assert_eq!( + event, + json!({"type":"user","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"probe-call","content":[],"is_error":false}]}}) + ); + } else { + assert_eq!( + event, + json!({"method":"item/completed","params":{"threadId":"fixture-thread","turnId":"fixture-turn","item":{"id":"probe-call","type":"dynamicToolCall","tool":"mcp_read_counter","status":"completed","success":true}}}) + ); + } + receive(&rx); + finish(&mut child.0); + let records = proof(&state); + assert_eq!( + records.last().unwrap()["observation"], + "prematureCompletionEmitted" + ); + assert_eq!(records.last().unwrap()["eventSha256"], hash(&event)); + for record in records { + for field in [ + "resultReceived", + "resultSha256", + "outcome", + "delivery", + "success", + ] { + assert!(record.get(field).is_none()); + } + } + } +} + +#[test] +fn evidence_plan_rejects_ambiguous_or_unbounded_projection_configuration() { + for (field, value) in [ + ("expectExposure", json!(null)), + ("expectExposure", json!("either")), + ("completionProjection", json!("normal")), + ("completionProjection", json!({"success":true})), + ] { + let mut plan = evidence_plan(false, false); + plan["steps"][0][field] = value; + assert_plan_configuration(&plan, false); + } + for premature in [false, true] { + let mut plan = evidence_plan(!premature, premature); + plan["steps"][0]["expectedResult"] = json!({}); + assert_plan_configuration(&plan, false); + plan["steps"][0] + .as_object_mut() + .unwrap() + .remove("expectedResult"); + plan["steps"][0]["resultFormat"] = json!("json"); + assert_plan_configuration(&plan, false); + } + assert_plan_configuration(&evidence_plan(true, true), false); +} + +#[test] +fn absent_exposure_rejects_present_alias_before_any_request() { + for claude in [false, true] { + let state = State::new("absent-must-be-absent"); + let mut child = Fixture( + plan_command(&state, claude, &evidence_plan(true, false), "read") + .spawn() + .unwrap(), + ); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, claude, tools()); + prompt(&mut child.0, claude, "[[fixture:probe]]"); + wait_for_failure(&mut child.0); + assert!(proof(&state).is_empty()); + assert!(matches!( + rx.recv_timeout(Duration::from_secs(1)), + Err(mpsc::RecvTimeoutError::Disconnected) + )); + } +} + +#[test] +fn held_evidence_barriers_fail_closed_on_interrupt_and_invalid_release() { + for claude in [false, true] { + for premature in [false, true] { + for case in [ + "interrupt", + "symlink", + "nonempty", + "permissions", + "hardlink", + "directory", + "early-result", + "eof", + ] { + if claude && case == "early-result" { + continue; + } + let state = State::new("barrier-failclosed"); + let plan = evidence_plan(false, premature); + let actual = native(claude, false); + let mut cmd = plan_command( + &state, + claude, + &plan, + if premature { "refire" } else { "read" }, + ); + let name = if premature { + "premature-release" + } else { + "completion-release" + }; + cmd.env( + if premature { + "ENGRAM_FIXTURE_PREMATURE_BARRIER" + } else { + "ENGRAM_FIXTURE_COMPLETION_BARRIER" + }, + state.0.join(name), + ); + std::fs::write( + state.0.join("native-response.json"), + json!({"result":actual}).to_string(), + ) + .unwrap(); + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, claude, tools()); + prompt(&mut child.0, claude, "[[fixture:probe]]"); + call(&rx, claude); + if !claude && !premature { + send(&mut child.0, json!({"id":77,"result":actual})); + } + wait_observation( + &state, + if premature { + "prematureBarrierHeld" + } else { + "completionBarrierHeld" + }, + ); + match case { + "interrupt" => interrupt(&mut child.0, claude), + "eof" => { + drop(child.0.stdin.take()); + } + "early-result" => send(&mut child.0, json!({"id":77,"result":actual})), + "symlink" => { + std::os::unix::fs::symlink(state.0.join("proof.json"), state.0.join(name)) + .unwrap() + } + "nonempty" => { + release(&state, "invalid"); + std::fs::write(state.0.join("invalid"), "not-empty").unwrap(); + std::fs::rename(state.0.join("invalid"), state.0.join(name)).unwrap(); + } + "permissions" => { + use std::os::unix::fs::PermissionsExt; + release(&state, "invalid"); + std::fs::set_permissions( + state.0.join("invalid"), + std::fs::Permissions::from_mode(0o644), + ) + .unwrap(); + std::fs::rename(state.0.join("invalid"), state.0.join(name)).unwrap(); + } + "hardlink" => { + release(&state, "other"); + std::fs::hard_link(state.0.join("other"), state.0.join(name)).unwrap(); + } + "directory" => std::fs::create_dir(state.0.join(name)).unwrap(), + _ => unreachable!(), + } + wait_for_failure(&mut child.0); + let frames: Vec = rx + .iter() + .map(|line| serde_json::from_str(&line.unwrap()).unwrap()) + .collect(); + assert!(frames + .iter() + .all(|v| v["type"] != "user" && v["method"] != "item/completed")); + if case == "interrupt" { + assert_eq!(frames.len(), 2); + } + assert!(!proof(&state) + .iter() + .any(|v| v["observation"] == "prematureCompletionEmitted" + || v["observation"] == "completionBarrierReleased")); + let receipt: Value = serde_json::from_slice( + &std::fs::read(state.0.join("plan-selection.json")).unwrap(), + ) + .unwrap(); + assert_eq!(receipt["pending"], premature); + } + } + } +} + +#[test] +fn evidence_release_paths_and_required_barrier_fail_before_request() { + for case in [ + "outside", + "wrong-name", + "preexisting", + "symlink", + "missing-premature", + ] { + let state = State::new("invalid-evidence-path"); + let plan = evidence_plan(false, case == "missing-premature"); + let mut cmd = plan_command(&state, false, &plan, "read"); + if case != "missing-premature" { + let path = match case { + "outside" => state.0.parent().unwrap().join("completion-release"), + "wrong-name" => state.0.join("arbitrary-release"), + _ => state.0.join("completion-release"), + }; + if case == "preexisting" { + release(&state, "completion-release"); + } + if case == "symlink" { + std::os::unix::fs::symlink("missing", &path).unwrap(); + } + cmd.env("ENGRAM_FIXTURE_COMPLETION_BARRIER", path); + } + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + if case == "missing-premature" { + initialize(&mut child.0, &rx, false, tools()); + prompt(&mut child.0, false, "[[fixture:probe]]"); + } + wait_for_failure(&mut child.0); + assert!(matches!( + rx.recv_timeout(Duration::from_secs(1)), + Err(mpsc::RecvTimeoutError::Disconnected) + )); + } +} + +#[test] +fn absent_exposure_does_not_convert_other_responses_into_denial_proof() { + for claude in [false, true] { + for case in ["success", "wrong-message", "wrong-code", "ambiguous"] { + let state = State::new("invalid-absent-response"); + let plan = evidence_plan(true, false); + let mut cmd = plan_command(&state, claude, &plan, "read"); + std::fs::write(state.0.join("exposed-tools.json"), "[]").unwrap(); + let mut response = if claude { + json!({"error":{"code":-32602,"message":"Unknown tool"}}) + } else { + json!({"result":{"success":false,"contentItems":[{"type":"inputText","text":"undeclared dynamic tool: mcp_read_counter"}]}}) + }; + match case { + "success" => response = json!({"result":native(claude,false)}), + "wrong-message" => { + if claude { + response["error"]["message"] = json!("different") + } else { + response["result"]["contentItems"][0]["text"] = + json!("undeclared dynamic tool: other") + } + } + "wrong-code" => { + if claude { + response["error"]["code"] = json!(-32603) + } else { + response["result"]["success"] = json!(true) + } + } + "ambiguous" => { + if claude { + response["result"] = native(claude, false) + } else { + response["error"] = json!({"code":-32603,"message":"error"}) + } + } + _ => unreachable!(), + } + std::fs::write(state.0.join("native-response.json"), response.to_string()).unwrap(); + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, claude, json!([])); + prompt(&mut child.0, claude, "[[fixture:probe]]"); + call(&rx, claude); + if !claude { + response["id"] = json!(77); + send(&mut child.0, response); + } + wait_for_failure(&mut child.0); + assert!(proof(&state).is_empty()); + assert!(matches!( + rx.recv_timeout(Duration::from_secs(1)), + Err(mpsc::RecvTimeoutError::Disconnected) + )); + } + } +} + +fn rebind_plan() -> Value { + json!({"steps":[ + {"marker":"before","tool":"mcp_read_counter","arguments":{},"callId":"probe-call","resultFormat":"mcp"}, + {"marker":"after","tool":"mcp_read_counter","arguments":{},"callId":"probe-call","resultFormat":"mcp","expectExposure":"absent"} + ]}) +} + +#[test] +fn rebind_pair_repeats_only_the_same_immutable_request_after_actual_absence() { + for claude in [false, true] { + let state = State::new("rebind-exposure"); + let plan = rebind_plan(); + for absent in [false, true] { + let mut cmd = plan_command(&state, claude, &plan, "read"); + let proof_path = state.0.join(if absent { + "negative.json" + } else { + "positive.json" + }); + cmd.env("ENGRAM_FIXTURE_PROOF", &proof_path); + let exposed = if absent { json!([]) } else { tools() }; + std::fs::write(state.0.join("exposed-tools.json"), exposed.to_string()).unwrap(); + let response = if absent { + if claude { + json!({"error":{"code":-32602,"message":"Unknown tool"}}) + } else { + json!({"result":{"success":false,"contentItems":[{"type":"inputText","text":"undeclared dynamic tool: mcp_read_counter"}]}}) + } + } else { + json!({"result":native(claude,false)}) + }; + std::fs::write(state.0.join("native-response.json"), response.to_string()).unwrap(); + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + initialize(&mut child.0, &rx, claude, exposed); + prompt( + &mut child.0, + claude, + if absent { + "[[fixture:after]]" + } else { + "[[fixture:before]]" + }, + ); + let request = call(&rx, claude); + assert_eq!(request[if claude { "id" } else { "callId" }], "probe-call"); + assert_eq!( + request[if claude { "input" } else { "arguments" }], + json!({}) + ); + assert_eq!( + request[if claude { "name" } else { "tool" }], + if claude { + "mcp__engrams__mcp_read_counter" + } else { + "mcp_read_counter" + } + ); + complete(&mut child.0, &rx, claude, response["result"].clone()); + finish(&mut child.0); + let record: Value = + serde_json::from_slice(&std::fs::read(&proof_path).unwrap()).unwrap(); + assert_eq!(record["callId"], "probe-call"); + assert_eq!(record["planSha256"], hash(&plan)); + if absent { + assert_eq!(record["expectExposure"], "absent"); + assert_eq!(record["protocolDenialCode"], "unknown_tool"); + assert!(record.get("resultReceived").is_none()); + } else { + assert_eq!(record["resultReceived"], true); + } + } + } +} + +#[test] +fn duplicate_call_ids_require_exact_ordered_present_absent_pair() { + let plan = rebind_plan(); + assert_plan_configuration(&plan, true); + let mut explicit = plan.clone(); + explicit["steps"][0]["expectExposure"] = json!("present"); + explicit["steps"][0]["expectedResult"] = json!({}); + assert_plan_configuration(&explicit, true); + for case in [ + "tool", + "arguments", + "format", + "order", + "third", + "both-present", + "both-absent", + "projection", + "non-mcp", + ] { + let mut bad = plan.clone(); + match case { + "tool" => bad["steps"][1]["tool"] = json!("mcp_write_counter"), + "arguments" => bad["steps"][1]["arguments"] = json!({"different":true}), + "format" => bad["steps"][0]["resultFormat"] = json!("json"), + "order" => bad["steps"].as_array_mut().unwrap().reverse(), + "third" => { + let mut third = bad["steps"][1].clone(); + third["marker"] = json!("third"); + bad["steps"].as_array_mut().unwrap().push(third); + } + "both-present" => bad["steps"][1] + .as_object_mut() + .unwrap() + .remove("expectExposure") + .map(|_| ()) + .unwrap(), + "both-absent" => bad["steps"][0]["expectExposure"] = json!("absent"), + "projection" => bad["steps"][0]["completionProjection"] = json!("premature"), + "non-mcp" => { + bad["steps"][0]["tool"] = json!("native_tool"); + bad["steps"][1]["tool"] = json!("native_tool"); + } + _ => unreachable!(), + } + assert_plan_configuration(&bad, false); + } +} diff --git a/tests/fixtures/mcp-model-cli/tests/support/scalar_contract.rs b/tests/fixtures/mcp-model-cli/tests/support/scalar_contract.rs new file mode 100644 index 000000000..ff9acc05b --- /dev/null +++ b/tests/fixtures/mcp-model-cli/tests/support/scalar_contract.rs @@ -0,0 +1,439 @@ +use super::*; +use sha2::{Digest, Sha256}; + +pub(super) fn hash(value: &Value) -> String { + // serde_json's default object map is sorted; strings remain literal strings. + format!("sha256:{:x}", Sha256::digest(value.to_string().as_bytes())) +} +pub(super) fn native(claude: bool, error: bool) -> Value { + let receipt = json!({"delivery":"available","outcome":if error {"failed"} else {"succeeded"},"resultUnavailable":false,"isError":error}); + if claude { + json!({"content":[{"type":"text","text":"count=1"},{"type":"text","text":"private-result-marker"}],"structuredContent":{"count":1,"nested":{"z":2,"a":1}},"isError":error,"_meta":{"engrams/toolResult":receipt}}) + } else { + json!({"success":!error,"contentItems":[{"type":"inputText","text":"count=1"},{"type":"inputText","text":"private-result-marker"},{"type":"inputText","text":"{\"structuredContent\":{\"count\":1,\"nested\":{\"a\":1,\"z\":2}}}"},{"type":"inputText","text":receipt.to_string()}]}) + } +} +pub(super) fn command(state: &State, claude: bool, mode: &str) -> Command { + let mut cmd = Command::new(env!("CARGO_BIN_EXE_engram-mcp-model-fixture")); + cmd.env_remove("ENGRAM_FIXTURE_CALL_PLAN_JSON") + .env_remove("ENGRAM_FIXTURE_ARGUMENTS_JSON") + .env_remove("ENGRAM_FIXTURE_EXPECTED_RESULT_JSON") + .env_remove("ENGRAM_FIXTURE_CALL_ID") + .env_remove("ENGRAM_FIXTURE_BARRIER") + .env_remove("ENGRAM_FIXTURE_COMPLETION_BARRIER") + .env_remove("ENGRAM_FIXTURE_PREMATURE_BARRIER") + .env("ENGRAM_FIXTURE_TOOL", "remote_inventory") + .env("ENGRAM_FIXTURE_MODE", mode) + .env("ENGRAM_FIXTURE_STATE_DIR", &state.0) + .env("ENGRAM_FIXTURE_PROOF", state.0.join("proof.json")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if claude { + let mock = env!("CARGO_BIN_EXE_fixture-mock-bridge"); + std::fs::write( + state.0.join("config.json"), + json!({"mcpServers":{"engrams":{"type":"stdio","command":mock,"args":["mcp-bridge"]}}}) + .to_string(), + ) + .unwrap(); + std::fs::write( + state.0.join("settings.json"), + json!({"hooks":{"PreToolUse":[{"hooks":[{"command":format!("{mock} hook-bridge")}]}]}}) + .to_string(), + ) + .unwrap(); + cmd.args(["--print", "--strict-mcp-config", "--mcp-config"]) + .arg(state.0.join("config.json")) + .arg("--settings") + .arg(state.0.join("settings.json")); + } else { + cmd.arg("app-server"); + } + cmd +} +pub(super) fn output(child: &mut Child) -> mpsc::Receiver> { + let stdout = child.stdout.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(stdout).lines() { + if tx.send(line).is_err() { + break; + } + } + }); + rx +} +pub(super) fn receive(rx: &mpsc::Receiver>) -> Value { + serde_json::from_str(&rx.recv_timeout(Duration::from_secs(3)).unwrap().unwrap()).unwrap() +} +pub(super) fn finish(child: &mut Child) { + drop(child.stdin.take()); + let deadline = std::time::Instant::now() + Duration::from_secs(3); + loop { + if let Some(status) = child.try_wait().unwrap() { + assert!(status.success()); + break; + } + assert!(std::time::Instant::now() < deadline, "fixture did not exit"); + std::thread::sleep(Duration::from_millis(10)); + } + let mut stderr = String::new(); + std::io::Read::read_to_string(&mut child.stderr.take().unwrap(), &mut stderr).unwrap(); + assert!(stderr.is_empty()); +} +pub(super) fn exercise( + claude: bool, + mode: &str, + args: Value, + id: &str, + actual: Value, + expected: Value, + resume: bool, +) -> Value { + let state = State::new("scalar"); + let mut cmd = command(&state, claude, mode); + cmd.env("ENGRAM_FIXTURE_ARGUMENTS_JSON", args.to_string()) + .env("ENGRAM_FIXTURE_CALL_ID", id) + .env("ENGRAM_FIXTURE_EXPECTED_RESULT_JSON", expected.to_string()); + if claude { + std::fs::write( + state.0.join("native-response.json"), + if actual.get("jsonrpc").is_some() { + actual.clone() + } else { + json!({"jsonrpc":"2.0","result":actual}) + } + .to_string(), + ) + .unwrap(); + if resume { + cmd.args(["--resume", "fixture-session"]); + } + } + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + if claude { + assert_eq!(receive(&rx)["type"], "system"); + if !resume { + send( + &mut child.0, + json!({"type":"user","message":{"role":"user","content":"fixture"}}), + ); + } + let frame = receive(&rx); + assert_eq!(frame.pointer("/message/content/0/id"), Some(&json!(id))); + assert_eq!(frame.pointer("/message/content/0/input"), Some(&args)); + let completion = receive(&rx); + let block = &completion["message"]["content"][0]; + assert_eq!(completion["type"], "user"); + assert_eq!(block["tool_use_id"], id); + if actual.get("error").is_some() { + assert_eq!(block["content"], actual["error"]["message"]); + assert_eq!(block["is_error"], true); + } else { + assert_eq!(block["content"], actual["content"]); + assert_eq!(block.get("is_error"), actual.get("isError")); + } + assert_eq!(receive(&rx)["type"], "result"); + } else { + send(&mut child.0, json!({"id":1,"method":"initialize"})); + assert_eq!(receive(&rx)["id"], 1); + send( + &mut child.0, + json!({"id":2,"method":if resume {"thread/resume"} else {"thread/start"},"params":{"threadId":"fixture-thread","dynamicTools":[{"name":"remote_inventory","description":"test","inputSchema":{"type":"object"}}]}}), + ); + assert_eq!(receive(&rx)["id"], 2); + send(&mut child.0, json!({"id":3,"method":"turn/start"})); + assert_eq!(receive(&rx)["id"], 3); + let call = receive(&rx); + assert_eq!(call["params"]["callId"], id); + assert_eq!(call["params"]["arguments"], args); + send(&mut child.0, json!({"id":77,"result":actual})); + let completion = receive(&rx); + assert_eq!(completion["method"], "item/completed"); + assert_eq!(completion["params"]["item"]["id"], id); + assert_eq!(completion["params"]["item"]["result"], actual); + assert_eq!(receive(&rx)["method"], "turn/completed"); + } + finish(&mut child.0); + let bytes = std::fs::read(state.0.join("proof.json")).unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("private-result-marker")); + assert!(!String::from_utf8_lossy(&bytes).contains("private-argument-marker")); + serde_json::from_slice(&bytes).unwrap() +} +#[test] +fn actual_counter_arguments_native_results_ids_and_resume_on_both_protocols() { + for claude in [false, true] { + for (mode, args, id, resume, error) in [ + ("read", json!({}), "counter-read_1", false, false), + ("write", json!({"delta":1}), "counter-write_1", false, false), + ("write", json!({"delta":1}), "counter-write_1", true, false), + ("write", json!({"delta":1}), "counter-write_2", true, true), + ] { + let actual = native(claude, error); + let proof = exercise( + claude, + mode, + args, + id, + actual.clone(), + actual.clone(), + resume, + ); + assert_eq!(proof["callId"], id); + assert_eq!(proof["selectedToolName"], "remote_inventory"); + assert_eq!( + proof["selectedSchemaSha256"], + hash(&json!({"type":"object"})) + ); + assert_eq!(proof["resultMatchesExpected"], true); + assert_eq!(proof["resultSha256"], hash(&actual)); + assert_eq!(proof["expectedResultSha256"], hash(&actual)); + assert_eq!(proof["isError"], error); + assert_eq!(proof["textSentinel"], false); + } + } +} +#[test] +fn equality_preserves_array_order_presence_and_literal_codex_strings() { + for claude in [false, true] { + for case in 0..4 { + let actual = native(claude, false); + let mut expected = actual.clone(); + if claude { + match case { + 0 => expected["content"].as_array_mut().unwrap().reverse(), + 1 => { + expected.as_object_mut().unwrap().remove("isError"); + } + 2 => expected["structuredContent"]["count"] = json!(2), + _ => expected["isError"] = json!(true), + } + } else { + match case { + 0 => expected["contentItems"].as_array_mut().unwrap().swap(0, 1), + 1 => { + expected + .as_object_mut() + .unwrap() + .insert("absent".into(), Value::Null); + } + 2 => { + expected["contentItems"][2]["text"] = json!( + "{ \"structuredContent\": {\"count\":1,\"nested\":{\"a\":1,\"z\":2}}}" + ) + } + _ => expected["success"] = json!(false), + } + } + let proof = exercise( + claude, + "read", + json!({"marker":"private-argument-marker"}), + "literal", + actual.clone(), + expected.clone(), + false, + ); + assert_eq!(proof["resultMatchesExpected"], false); + assert_eq!(proof["resultSha256"], hash(&actual)); + assert_eq!(proof["expectedResultSha256"], hash(&expected)); + } + } +} +#[test] +fn invalid_scalar_configuration_fails_closed_without_payload_output() { + let nested = format!("{}0{}", "{\"a\":".repeat(17), "}".repeat(17)); + let mut cases = vec![ + ("ENGRAM_FIXTURE_ARGUMENTS_JSON", "[1]".into()), + ( + "ENGRAM_FIXTURE_ARGUMENTS_JSON", + "{broken-private-marker".into(), + ), + ("ENGRAM_FIXTURE_ARGUMENTS_JSON", "{\"a\":1,\"a\":2}".into()), + ( + "ENGRAM_FIXTURE_ARGUMENTS_JSON", + "{\"a\":{\"b\":1,\"b\":2}}".into(), + ), + ("ENGRAM_FIXTURE_ARGUMENTS_JSON", nested), + ( + "ENGRAM_FIXTURE_ARGUMENTS_JSON", + json!({"a":"x".repeat(4096)}).to_string(), + ), + ("ENGRAM_FIXTURE_EXPECTED_RESULT_JSON", "null".into()), + ( + "ENGRAM_FIXTURE_EXPECTED_RESULT_JSON", + "{\"a\":1,\"a\":2}".into(), + ), + ("ENGRAM_FIXTURE_CALL_ID", "".into()), + ("ENGRAM_FIXTURE_CALL_ID", "a".repeat(65)), + ("ENGRAM_FIXTURE_CALL_ID", "private-marker/invalid".into()), + ("ENGRAM_FIXTURE_CALL_ID", "é".into()), + ]; + cases.push(("ENGRAM_FIXTURE_ARGUMENTS_JSON", "{} {}".into())); + for claude in [false, true] { + for (key, value) in &cases { + let state = State::new("invalid-scalar"); + let mut cmd = command(&state, claude, "read"); + cmd.env(key, value); + let mut child = Fixture(cmd.spawn().unwrap()); + drop(child.0.stdin.take()); + wait_for_failure(&mut child.0); + let mut out = String::new(); + std::io::Read::read_to_string(&mut child.0.stdout.take().unwrap(), &mut out).unwrap(); + assert!(out.is_empty()); + let mut err = String::new(); + std::io::Read::read_to_string(&mut child.0.stderr.take().unwrap(), &mut err).unwrap(); + assert_eq!(err, "fixture protocol failure\n"); + assert!(!state.0.join("proof.json").exists()); + } + } +} + +// Compile the exact bounded parser here so the standalone protocol CI target +// covers limits larger than the operating system's environment limit. +type Result = std::result::Result>; +#[path = "../../src/scalar.rs"] +pub(super) mod config_parser; + +#[test] +fn parser_exact_utf8_size_depth_and_canonical_hash_boundaries() { + assert!( + config_parser::configured_object("ENGRAM_FIXTURE_TEST_ABSENT_CONFIGURATION", 1, 1) + .unwrap() + .is_none() + ); + for limit in [4096, 1024 * 1024] { + let raw = format!("{{\"a\":\"{}\"}}", "x".repeat(limit - 8)); + assert_eq!(raw.len(), limit); + assert!(config_parser::parse_object(&raw, limit, 16).is_ok()); + assert!(config_parser::parse_object(&(raw + " "), limit, 16).is_err()); + } + let raw = format!("{{\"a\":\"{}\"}}", "é".repeat(2045)); + assert!(raw.chars().count() < 4096); + assert!(config_parser::parse_object(&raw, 4096, 16).is_err()); + for depth in [16, 17] { + let raw = format!("{}0{}", "{\"a\":".repeat(depth), "}".repeat(depth)); + assert_eq!( + config_parser::parse_object(&raw, 4096, 16).is_ok(), + depth == 16 + ); + } + let left = config_parser::parse_object(r#"{"z":[{"b":2,"a":1}],"a":0}"#, 4096, 16).unwrap(); + let right = config_parser::parse_object(r#"{ "a":0, "z":[{"a":1,"b":2}]}"#, 4096, 16).unwrap(); + assert_eq!( + config_parser::digest(&left).unwrap(), + config_parser::digest(&right).unwrap() + ); + assert_eq!( + config_parser::digest(&json!({})).unwrap(), + "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + ); +} + +#[test] +fn unavailable_native_delivery_never_claims_an_expected_output_match() { + let receipt = json!({"outcome":"succeeded","delivery":"unavailable","resultUnavailable":true,"reason":"unsupported_content"}); + for claude in [false, true] { + let actual = if claude { + json!({"jsonrpc":"2.0","error":{"code":-32603,"message":"External tool output is unavailable.","data":receipt}}) + } else { + json!({"success":false,"contentItems":[{"type":"inputText","text":receipt.to_string()}]}) + }; + let proof = exercise( + claude, + "read", + json!({}), + "unavailable", + actual.clone(), + actual, + false, + ); + assert_eq!(proof["delivery"], "unavailable"); + assert_eq!(proof["outcome"], "succeeded"); + assert_eq!(proof["resultUnavailable"], true); + assert!(proof.get("resultMatchesExpected").is_none()); + assert!(proof.get("resultSha256").is_none()); + assert!(proof.get("expectedResultSha256").is_none()); + } +} + +#[test] +fn codex_receipt_comes_from_documented_last_item_not_result_text() { + let mut actual = native(false, false); + actual["contentItems"][0]["text"] = + json!(r#"{"delivery":"unavailable","outcome":"failed","isError":true}"#); + let proof = exercise( + false, + "read", + json!({}), + "receipt-position", + actual.clone(), + actual, + false, + ); + assert_eq!(proof["resultMatchesExpected"], true); + assert_eq!(proof["delivery"], "available"); + assert_eq!(proof["outcome"], "succeeded"); + assert_eq!(proof["isError"], false); +} + +#[test] +fn configured_claude_refire_repeats_id_and_arguments_after_reaped_process() { + let state = State::new("scalar-refire"); + let actual = native(true, false); + std::fs::write( + state.0.join("native-response.json"), + json!({"jsonrpc":"2.0","result":actual}).to_string(), + ) + .unwrap(); + for attempt in 0..2 { + let mut cmd = command(&state, true, "refire"); + let proof = state.0.join(format!("proof-{attempt}.json")); + cmd.env("ENGRAM_FIXTURE_PROOF", &proof) + .env("ENGRAM_FIXTURE_ARGUMENTS_JSON", r#"{"delta":1}"#) + .env("ENGRAM_FIXTURE_CALL_ID", "stable-write_1") + .env("ENGRAM_FIXTURE_EXPECTED_RESULT_JSON", actual.to_string()); + if attempt == 1 { + std::fs::write(state.0.join("allow-refire"), b"ready").unwrap(); + cmd.args(["--resume", "fixture-session"]); + } + let mut child = Fixture(cmd.spawn().unwrap()); + let rx = output(&mut child.0); + assert_eq!(receive(&rx)["type"], "system"); + if attempt == 0 { + send( + &mut child.0, + json!({"type":"user","message":{"role":"user","content":"fixture"}}), + ); + } + let frame = receive(&rx); + assert_eq!( + frame.pointer("/message/content/0/id"), + Some(&json!("stable-write_1")) + ); + assert_eq!( + frame.pointer("/message/content/0/input"), + Some(&json!({"delta":1})) + ); + let result = receive(&rx); + if attempt == 0 { + assert_eq!(result["terminal_reason"], "tool_deferred"); + assert!(std::fs::read(&proof).unwrap().is_empty()); + // EOF lets the fixture reap its bridge before this process is reaped. + finish(&mut child.0); + } else { + assert_eq!(result["type"], "user"); + assert_eq!( + result["message"]["content"][0]["tool_use_id"], + "stable-write_1" + ); + assert_eq!(receive(&rx)["subtype"], "success"); + finish(&mut child.0); + let proof: Value = serde_json::from_slice(&std::fs::read(proof).unwrap()).unwrap(); + assert_eq!(proof["callId"], "stable-write_1"); + assert_eq!(proof["resultMatchesExpected"], true); + } + } +} diff --git a/tests/fixtures/mcp-stack/README.md b/tests/fixtures/mcp-stack/README.md new file mode 100644 index 000000000..190c9b7a6 --- /dev/null +++ b/tests/fixtures/mcp-stack/README.md @@ -0,0 +1,237 @@ +# Standalone MCP stack fixture + +This test-only Node 24 process reuses the worker's MCP and OAuth fixture. It is +an independent server, with durable effects outside the worker and orchestrator +restart domains. It does not use production credentials or change worker +transport checks. Start it with one absolute config path: + +```sh +node tests/fixtures/mcp-stack/main.mts /absolute/owned-run/config.json +``` + +The launcher supplies the certificates, key, neutral credentials, run directory, +DNS, routes, and process handles. Use one fixture process per run directory. The +launcher must keep the control listener outside guest and worker reach. The +fixture does not create networks or install DNS entries. Local tests bind only +loopback and use ephemeral ports. The [platform adapter](platform/README.md) +starts independent modern and legacy fixture containers on its private network, +supplies their exact listener addresses and files, and checks authenticated +readiness. Native DNS and routes are operator prerequisites. + +## Config version 1 + +All keys below are required, except `control.host` (default `127.0.0.1`) and keys +inside `case`. Unknown keys fail startup. Paths must be absolute, real regular +files. TLS and credential files must be inside the real, private (`0700`) run +root. Symlinks are rejected. Each file has a size limit. Keep the config private. +The JSON below uses neutral example paths and names; a launcher supplies real +values and available ports. + +```json +{ + "version": 1, + "runRoot": "/absolute/owned-run", + "listen": { "host": "192.0.2.20", "port": 8443 }, + "advertised": { + "resource": "https://mcp.fixture.example:8443", + "issuer": "https://issuer.fixture.example:8443", + "authorization": "https://authorization.fixture.example:8443", + "token": "https://token.fixture.example:8443", + "redirectUri": "https://callback.fixture.example:9443/integrations/mcp/callback", + "authorities": [ + "mcp.fixture.example:8443", + "issuer.fixture.example:8443", + "authorization.fixture.example:8443", + "token.fixture.example:8443" + ] + }, + "tls": { + "certificateFile": "/absolute/owned-run/server.pem", + "keyFile": "/absolute/owned-run/server.key", + "caFile": "/absolute/owned-run/ca.pem" + }, + "control": { + "host": "127.0.0.1", + "port": 9444, + "origin": "https://localhost:9444", + "tokenFile": "/absolute/owned-run/observer.token" + }, + "protocol": "2026-07-28", + "case": { + "reply": "json", + "response": "normal", + "holdCalls": false, + "requireAuth": true, + "revision": 1 + }, + "apiKeyFile": "/absolute/owned-run/resource.key" +} +``` + +Origins must be canonical HTTPS DNS origins without a trailing slash, path, +query, fragment, credentials, or IP literal. The four data origins use the exact +listener port; distinct DNS names can reach that one listener. `authorities` +contains exactly their distinct canonical authorities (host and non-default port), +with no extra or duplicate +entry. The server checks both the request authority and the route's origin. +`/mcp` and protected-resource metadata belong to `resource`, authorization-server +metadata belongs to `issuer`, `/authorize` belongs to `authorization`, and +`/token` belongs to `token`. The resource identifier is `/mcp`. +`redirectUri` is the exact HTTPS redirect URL, with a DNS hostname and no +credentials, query, or fragment. It can have a separate authority, port, and +path. Authorization and token exchange require exact equality with this value. +It is not a fixture listener authority. The fixture does not follow the redirect. +Old shared-fixture tests retain their default `/callback` URL. + +Supply a PEM leaf certificate signed directly by the supplied PEM CA, plus its +matching key. The leaf SANs must cover all data DNS names and the control DNS +name. The listener uses the supplied identity. Client code must trust that CA +and use ordinary DNS and HTTPS. Neither the standalone server nor the platform +acceptance transport substitutes a fixture fetch adapter or resolver. + +`protocol` accepts `2026-07-28`, `2025-11-25`, or `2025-06-18` and stays fixed for +the process lifetime. Legacy sessions retain credential binding and explicit +cancellation. The paginated catalog contains `read_counter` and `write_counter`. +The schema and annotation fixtures stay in the shared server. `reply` accepts +`json` or `sse`. `response` accepts `normal`, `malformed`, `oversize`, `binary`, +`image`, `audio`, `resource`, `resource_link`, or `error`. These modes preserve the +write effect. `revision` accepts integers 1 through 100 and changes the catalog +schema and description. The remaining case keys are booleans. Defaults match +the existing fixture: modern protocol, JSON, normal results, no held calls, no +auth requirement, revision 1. Acceptance launchers must set `requireAuth: true`. + +The resource key file contains a neutral 16–128 character ASCII token. The +observer file contains a different neutral 32–128 character ASCII token. Both +use letters, digits, `_`, or `-`, with optional surrounding whitespace. The +resource accepts `Authorization: Bearer ` or its current OAuth +access token. The observer accepts only its own bearer token. Do not use a real +credential in either file or pass credential values in command arguments. + +## Fixed control and readiness contract + +All control routes use the separate HTTPS control listener and exact configured +control authority. All require the observer bearer token. Wrong authentication +returns 401, wrong authority returns 421, unknown routes or methods return 404, +and invalid input returns 400. POST requests use JSON and reject unknown keys. +Bodies are limited to 4 KiB. Responses have `Cache-Control: no-store`. + +| Method and route | Request | Response or effect | +| ------------------- | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- | +| GET `/v1/ready` | No body | `version:1`, `ready`, resource URL, issuer origin | +| GET `/v1/observe` | No body | `version:1`, durability health, effect state, read/held/cancellation counts, OAuth observations | +| POST `/v1/case` | Any subset of the case keys above | Applies checked controls; returns 409 while any call is held | +| POST `/v1/release` | `{}` | Sends all held responses; future calls still use the selected hold mode | +| POST `/v1/cancel` | `{}` | Closes all held responses; each close increments cancellation count | +| POST `/v1/oauth` | Optional `account`, `scope`, `issuer`, and/or `expiresInSeconds` | Sets neutral controls for future token issue or refresh | +| POST `/v1/shutdown` | `{}` | Replies, then closes both listeners and all held connections | + +OAuth accounts are `account-one` or `account-two`. Issuer mode is `configured` +(the configured issuer) or `mismatch` (the existing neutral wrong-issuer case, +`https://other.example`). This changes only the callback `iss` field. Scope controls are +`tools:read`, `tools:write`, or `tools:read tools:write`. The scope control can +produce an account/scope change for the consumer to detect. It does not confer +production authority. Observations contain only the selected account, scope +override (or null), configured `expiresInSeconds`, and counts of authorizations, code exchanges, rotations, +and rejected token/authorization requests. Issued codes/tokens, client secrets, +headers, arguments, and results are absent. OAuth grants are process-local; only +external write effects persist across fixture restarts. + +`expiresInSeconds` accepts integers 1 through 3600 and defaults to 3600. The +configured value applies to each future authorization-code exchange or refresh +response as `expires_in`. Each issued access token keeps its own private deadline. +A control change never shortens or extends a token that was already issued. +Expired bearer access returns 401; the refresh grant can still rotate through +the existing token endpoint. Set `{"expiresInSeconds":2}` before issue or refresh +to exercise expiry in a bounded acceptance run. Invalid values (including +strings, fractions, zero, and values outside the range) or unknown control keys +return 400 before any account, scope, issuer, or lifetime change. The observer +shows the configured value, without token values or issuance deadlines. The +fixture uses real elapsed wall-clock time; it exposes no clock control. + +Existing public/confidential clients, PKCE S256 checks, exact resource and +redirect checks, single-use state/code checks, rotating refresh tokens, and +scope-subset checks are reused. Public client ID is `fixture-public`; +confidential client ID and neutral secret retain the shared fixture values +(`fixture-confidential`, `fixture-client-secret`). The fixed fixture does not +support dynamic clients or arbitrary proxy destinations. + +Startup emits one JSON line with `version:1`, `kind:"ready"`, control origin, +and resource URL only after durable state and both listeners are ready. A +startup failure emits only `{"version":1,"kind":"startup_failed"}` and exits +with code 1. No secret, arguments, raw results, or raw error is printed. The +launcher must also call authenticated `/v1/ready`; an open TCP port alone is not +readiness. `ready` becomes false if a durable write fails. SIGTERM and SIGINT +close both listeners and held connections and wait for closure. The launcher +must retain exact process handles, enforce its deadline, then stop/kill/wait on +all paths. It must not restart the fixture when it restarts the worker or +orchestrator for acceptance. + +Bounds: 64 held calls, 128 legacy sessions, 128 observed protocol requests, +64 data connections, 32 control connections, 64 KiB protocol request bodies, +5 second request/header timeouts, and 1 second idle keepalive. OAuth retains at +most 128 pending code/refresh grants and 1,024 used states per process. Reaching +a bound fails the operation; it never skips an executed write. + +## Durable effect evidence + +`effects.json` holds `{state,sha256}`. `state` is version 1, with `executions`, +`count`, and the last 64 events. Each event has only a monotonic `sequence` and +SHA-256 `callHash` of the neutral transport-session/request-ID tuple. It stores +no raw ID, delta, argument, result, or credential. `executions` counts every +accepted write, including repeated request IDs; `count` is the counter's current +value. Reads and malformed requests do not increment it. Read/cancellation/OAuth +counts in the observer are process-local. + +Each write makes a bounded replacement record in exclusive `effects.next`, +fsyncs that file, renames it to `effects.json`, then fsyncs the directory. Only +then can the server send or hold its reply. A committed held write is observable +through `effects.executions` and `held:1` before its client receives a response. +The observer does not acknowledge writes independently of this barrier. +Duplicate IDs are never suppressed. A reopened process loads the committed +record and continues both counters. Truncated, malformed, oversized, wrong-hash, +invalid-sequence state, or an incomplete `effects.next` fails startup. No reset +is automatic. A durable commit error blocks later commits and marks observer +health false. A failed record requires explicit launcher diagnosis and a new +run directory; do not delete evidence and continue the same acceptance run. + +## Focused local proof + +From the repository root, with Node 24 and existing worker dependencies: + +```sh +mkdir -p var/mcp-stack-fixture +node --test --test-concurrency=1 --test-timeout=30000 \ + mcp-worker/test/fixture.test.ts \ + tests/fixtures/mcp-stack/durable.test.mts \ + tests/fixtures/mcp-stack/process.test.mts +node mcp-worker/node_modules/typescript/bin/tsc \ + -p tests/fixtures/mcp-stack/tsconfig.json --noEmit +``` + +The tests use private real directories below `var/mcp-stack-fixture`, supplied +neutral certificates, one standalone process at a time, only PATH/TMPDIR in its +environment, 29 second process watchdogs, exact stop/kill/wait cleanup, and no +provider. A separate-origin config test uses the local OpenSSL CLI to supply +neutral DNS SANs; it does not create DNS entries or a non-loopback listener. +The SDK smoke uses native fetch with explicit test CA trust. The wire tests use +Node HTTPS with that CA and normal localhost resolution. Tests include a +committed barrier, SIGKILL/reopen persistence, repeated-ID increments, API-key +and OAuth flow, observer authentication, invalid config/state, cancellation, +and graceful shutdown while held. + +The `mcp-worker` CI lane runs the shared fixture tests through `npm test`, then +invokes the standalone `durable.test.mts` and `process.test.mts` command above. +The `orchestrator` lane typechecks and tests the common matrix runner and +platform adapter. + +These are source and process checks. They do not establish guest isolation, +Firecracker, VZ, provider, or full-stack acceptance. The separate +`mcp-physical-fc` and `mcp-physical-vz` jobs use +`.github/scripts/mcp-stack-assets.py` to prepare source-matched assets and invoke +the backend wrapper. The platform adapter supplies network and process +supervision and runs the common 24-case matrix. See +[physical CI prerequisites](platform/ci/README.md). + +Workflow wiring does not establish a successful physical run or a current-head +CI result. Current-source physical acceptance remains unverified. Production +enable defaults remain disabled. diff --git a/tests/fixtures/mcp-stack/config.mts b/tests/fixtures/mcp-stack/config.mts new file mode 100644 index 000000000..56ba8d21c --- /dev/null +++ b/tests/fixtures/mcp-stack/config.mts @@ -0,0 +1,258 @@ +import { X509Certificate } from "node:crypto"; +import { isIP } from "node:net"; +import { lstatSync, readFileSync, realpathSync } from "node:fs"; +import { isAbsolute, relative, resolve } from "node:path"; +import type { FixtureOptions } from "../../../mcp-worker/test/fixtures/server.ts"; + +export function object( + value: unknown, +): asserts value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) + throw new Error("Expected object"); +} +export function keys( + value: Record, + allowed: string[], + required = allowed, +) { + if ( + Object.keys(value).some((k) => !allowed.includes(k)) || + required.some((k) => !(k in value)) + ) + throw new Error("Unknown or missing field"); +} +function text(value: unknown): string { + if (typeof value !== "string" || value.length === 0 || value.length > 4096) + throw new Error("Invalid string"); + return value; +} +function port(value: unknown): number { + if ( + typeof value !== "number" || + !Number.isInteger(value) || + value < 1 || + value > 65535 + ) + throw new Error("Invalid port"); + return value; +} +export function caseOptions(value: unknown): Pick< + FixtureOptions, + "reply" | "response" | "holdCalls" | "requireAuth" +> & { + revision?: number; +} { + object(value); + keys( + value, + ["reply", "response", "holdCalls", "requireAuth", "revision"], + [], + ); + if ( + value.reply !== undefined && + (typeof value.reply !== "string" || !["json", "sse"].includes(value.reply)) + ) + throw new Error("Invalid reply"); + if ( + value.response !== undefined && + (typeof value.response !== "string" || + ![ + "normal", + "malformed", + "oversize", + "binary", + "image", + "audio", + "resource", + "resource_link", + "error", + ].includes(value.response)) + ) + throw new Error("Invalid response"); + for (const k of ["holdCalls", "requireAuth"]) + if (value[k] !== undefined && typeof value[k] !== "boolean") + throw new Error("Invalid boolean"); + if ( + value.revision !== undefined && + (typeof value.revision !== "number" || + !Number.isSafeInteger(value.revision) || + value.revision < 1 || + value.revision > 100) + ) + throw new Error("Invalid revision"); + // These fields have each been checked above; no unchecked input crosses this boundary. + return value as ReturnType; +} +export function loadConfig(path: string) { + const file = (path: string, max: number) => { + if ( + !isAbsolute(path) || + realpathSync(path) !== path || + !lstatSync(path).isFile() || + lstatSync(path).size > max + ) + throw new Error("Invalid file"); + return readFileSync(path, "utf8"); + }; + const raw: unknown = JSON.parse(file(path, 16384)); + object(raw); + keys(raw, [ + "version", + "runRoot", + "listen", + "advertised", + "tls", + "control", + "protocol", + "case", + "apiKeyFile", + ]); + if (raw.version !== 1) throw new Error("Invalid config version"); + const root = text(raw.runRoot); + if ( + !isAbsolute(root) || + resolve(root) !== root || + realpathSync(root) !== root || + !lstatSync(root).isDirectory() || + (lstatSync(root).mode & 0o077) !== 0 + ) + throw new Error("Invalid run root"); + const ownedFile = (value: unknown, max: number) => { + const p = text(value); + const rel = relative(root, p); + if (!rel || rel.startsWith("..") || isAbsolute(rel)) + throw new Error("File outside run root"); + return file(p, max); + }; + const listen = raw.listen; + object(listen); + keys(listen, ["host", "port"]); + const host = text(listen.host); + if (!isIP(host)) throw new Error("Listener must be an exact IP"); + const dataPort = port(listen.port); + const advertised = raw.advertised; + object(advertised); + keys(advertised, [ + "resource", + "issuer", + "authorization", + "token", + "redirectUri", + "authorities", + ]); + const origin = (v: unknown, expectedPort?: number) => { + const s = text(v); + const u = new URL(s); + if ( + u.protocol !== "https:" || + u.origin !== s || + u.username || + u.password || + isIP(u.hostname) || + u.hostname.length > 253 || + u.hostname + .split(".") + .some( + (label) => !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(label), + ) || + (expectedPort !== undefined && Number(u.port || 443) !== expectedPort) + ) + throw new Error("Invalid advertised origin"); + return s; + }; + const resource = origin(advertised.resource, dataPort); + const redirectUri = text(advertised.redirectUri); + const redirect = new URL(redirectUri); + origin(redirect.origin); + if ( + redirect.href !== redirectUri || + redirectUri.includes("?") || + redirectUri.includes("#") || + redirect.username || + redirect.password || + redirect.search || + redirect.hash + ) + throw new Error("Invalid redirect URI"); + const oauth = { + issuer: origin(advertised.issuer, dataPort), + authorization: origin(advertised.authorization, dataPort), + token: origin(advertised.token, dataPort), + redirectUri, + }; + const authorities = [ + ...new Set( + [resource, oauth.issuer, oauth.authorization, oauth.token].map( + (s) => new URL(s).host, + ), + ), + ].sort(); + if ( + !Array.isArray(advertised.authorities) || + JSON.stringify([...advertised.authorities].sort()) !== + JSON.stringify(authorities) + ) + throw new Error("Authority mismatch"); + const tls = raw.tls; + object(tls); + keys(tls, ["certificateFile", "keyFile", "caFile"]); + const cert = ownedFile(tls.certificateFile, 16384); + const key = ownedFile(tls.keyFile, 16384); + const ca = ownedFile(tls.caFile, 16384); + const leaf = new X509Certificate(cert); + const trust = new X509Certificate(ca); + if (!leaf.verify(trust.publicKey)) + throw new Error("Certificate trust mismatch"); + for (const authority of authorities) + if (!leaf.checkHost(new URL(`https://${authority}`).hostname)) + throw new Error("Certificate name mismatch"); + const control = raw.control; + object(control); + keys( + control, + ["port", "origin", "tokenFile", "host"], + ["port", "origin", "tokenFile"], + ); + const controlHost = + control.host === undefined ? "127.0.0.1" : text(control.host); + if (!isIP(controlHost)) throw new Error("Invalid control bind"); + const controlPort = port(control.port); + if (controlPort === dataPort) throw new Error("Listener collision"); + const controlOrigin = origin(control.origin, controlPort); + if (!leaf.checkHost(new URL(controlOrigin).hostname)) + throw new Error("Control certificate name mismatch"); + const token = ownedFile(control.tokenFile, 256).trim(); + const apiKey = ownedFile(raw.apiKeyFile, 256).trim(); + if ( + !/^[A-Za-z0-9_-]{32,128}$/.test(token) || + !/^[A-Za-z0-9_-]{16,128}$/.test(apiKey) || + token === apiKey + ) + throw new Error("Invalid neutral credential"); + if ( + typeof raw.protocol !== "string" || + !["2026-07-28", "2025-11-25", "2025-06-18"].includes(raw.protocol) + ) + throw new Error("Invalid protocol"); + const protocol = raw.protocol as NonNullable; + return { + root, + token, + control: { host: controlHost, port: controlPort, origin: controlOrigin }, + options: { + ...caseOptions(raw.case), + version: protocol, + apiKey, + network: { + host, + port: dataPort, + key, + cert, + ca, + origin: resource, + oauth, + allowedAuthorities: authorities, + }, + }, + }; +} diff --git a/tests/fixtures/mcp-stack/durable.mts b/tests/fixtures/mcp-stack/durable.mts new file mode 100644 index 000000000..7cc55abc8 --- /dev/null +++ b/tests/fixtures/mcp-stack/durable.mts @@ -0,0 +1,115 @@ +import { createHash } from "node:crypto"; +import { + closeSync, + existsSync, + fsyncSync, + lstatSync, + openSync, + readFileSync, + renameSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; + +interface EffectEvent { + sequence: number; + callHash: string; +} +interface EffectState { + version: 1; + executions: number; + count: number; + events: EffectEvent[]; +} +const hash = (value: string) => + createHash("sha256").update(value).digest("hex"); +const exactKeys = (value: object, keys: string[]) => + Object.keys(value).sort().join() === keys.sort().join(); + +// Each execution commits before the protocol fixture can send or hold a response. +// Duplicate IDs deliberately increment again so replay remains visible. +export function openEffects(root: string) { + const path = join(root, "effects.json"); + const next = join(root, "effects.next"); + if (existsSync(next)) throw new Error("Incomplete durable state"); + let state: EffectState = { version: 1, executions: 0, count: 0, events: [] }; + if (existsSync(path)) { + try { + if (!lstatSync(path).isFile() || lstatSync(path).size > 16384) throw 0; + const parsed = JSON.parse(readFileSync(path, "utf8")); + const s = parsed.state; + if ( + !exactKeys(parsed, ["state", "sha256"]) || + !s || + !exactKeys(s, ["version", "executions", "count", "events"]) || + parsed.sha256 !== hash(JSON.stringify(s)) || + s.version !== 1 || + !Number.isSafeInteger(s.executions) || + s.executions < 0 || + !Number.isSafeInteger(s.count) || + !Array.isArray(s.events) || + s.events.length !== Math.min(s.executions, 64) || + s.events.some( + (e: EffectEvent, i: number) => + !e || + !exactKeys(e, ["sequence", "callHash"]) || + e.sequence !== s.executions - s.events.length + i + 1 || + !/^[a-f0-9]{64}$/.test(e.callHash), + ) + ) + throw 0; + state = s; + } catch { + throw new Error("Invalid durable state"); + } + } + let failed = false; + function save(value: EffectState) { + const fd = openSync(next, "wx", 0o600); + try { + writeFileSync( + fd, + JSON.stringify({ state: value, sha256: hash(JSON.stringify(value)) }), + ); + fsyncSync(fd); + } finally { + closeSync(fd); + } + renameSync(next, path); + const dir = openSync(root, "r"); + try { + fsyncSync(dir); + } finally { + closeSync(dir); + } + } + if (!existsSync(path)) save(state); + return { + healthy: () => !failed, + snapshot: () => structuredClone(state), + commit(id: string, delta: number) { + if (failed) throw new Error("Durable state unavailable"); + const count = state.count + delta; + const executions = state.executions + 1; + if (!Number.isSafeInteger(count) || !Number.isSafeInteger(executions)) + throw new Error("Counter limit"); + const value: EffectState = { + version: 1, + executions, + count, + events: [ + ...state.events.slice(-63), + { sequence: executions, callHash: hash(id) }, + ], + }; + try { + save(value); + } catch { + failed = true; + throw new Error("Durable commit failed"); + } + state = value; + return count; + }, + }; +} diff --git a/tests/fixtures/mcp-stack/durable.test.mts b/tests/fixtures/mcp-stack/durable.test.mts new file mode 100644 index 000000000..941d48886 --- /dev/null +++ b/tests/fixtures/mcp-stack/durable.test.mts @@ -0,0 +1,47 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "node:fs"; +import { resolve } from "node:path"; +import { test } from "node:test"; +import { openEffects } from "./durable.mts"; + +const base = resolve("var/mcp-stack-fixture"); +test("every write is committed, including duplicate IDs, and survives reopen", () => { + const root = mkdtempSync(`${base}/durable-`); + try { + const effects = openEffects(root); + assert.equal(effects.commit("same", 2), 2); + assert.equal(effects.commit("same", 2), 4); + const reopened = openEffects(root); + assert.equal(reopened.snapshot().executions, 2); + assert.equal(reopened.commit("same", 2), 6); + assert.equal(openEffects(root).snapshot().executions, 3); + const text = readFileSync(`${root}/effects.json`, "utf8"); + assert.equal(text.includes('"same"'), false); + assert.equal(text.includes('"delta"'), false); + assert.equal(openEffects(root).snapshot().events.length, 3); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("truncated, corrupt, and incomplete durable state fails closed", () => { + const root = mkdtempSync(`${base}/corrupt-`); + try { + const state = openEffects(root); + state.commit("call", 1); + const valid = readFileSync(`${root}/effects.json`, "utf8"); + for (const bad of [ + "", + valid.slice(0, -2), + valid.replace('"executions":1', '"executions":0'), + ]) { + writeFileSync(`${root}/effects.json`, bad); + assert.throws(() => openEffects(root), /Invalid durable state/); + } + writeFileSync(`${root}/effects.json`, valid); + writeFileSync(`${root}/effects.next`, "torn"); + assert.throws(() => openEffects(root), /Incomplete durable state/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/tests/fixtures/mcp-stack/main.mts b/tests/fixtures/mcp-stack/main.mts new file mode 100644 index 000000000..d820bfa8a --- /dev/null +++ b/tests/fixtures/mcp-stack/main.mts @@ -0,0 +1,206 @@ +import { timingSafeEqual } from "node:crypto"; +import { createServer } from "node:https"; +import { caseOptions, keys, loadConfig, object } from "./config.mts"; +import { openEffects } from "./durable.mts"; +import { startMcpFixture } from "../../../mcp-worker/test/fixtures/server.ts"; + +async function main() { + if (process.argv.length !== 3) throw new Error("Expected config path"); + const config = loadConfig(process.argv[2]!); + const effects = openEffects(config.root); + const fixture = await startMcpFixture({ + ...config.options, + initialCount: effects.snapshot().count, + commitWrite: (id, delta) => effects.commit(id, delta), + }); + if (config.options.revision !== undefined) + fixture.setCatalogRevision(config.options.revision); + let closing: Promise | undefined; + const control = createServer( + { key: config.options.network.key, cert: config.options.network.cert }, + (req, res) => { + void (async () => { + const respond = (value: unknown, status = 200) => { + res + .writeHead(status, { + "content-type": "application/json", + "cache-control": "no-store", + }) + .end(JSON.stringify(value)); + }; + const expected = Buffer.from(`Bearer ${config.token}`); + const supplied = Buffer.from(req.headers.authorization ?? ""); + if ( + expected.length !== supplied.length || + !timingSafeEqual(expected, supplied) + ) { + respond({ error: "unauthorized" }, 401); + return; + } + if (req.headers.host !== new URL(config.control.origin).host) { + respond({ error: "authority" }, 421); + return; + } + const route = req.url; + if (req.method === "GET" && route === "/v1/ready") { + respond({ + version: 1, + ready: !closing && effects.healthy(), + resource: fixture.url, + issuer: config.options.network.oauth.issuer, + }); + return; + } + if (req.method === "GET" && route === "/v1/observe") { + respond({ + version: 1, + healthy: effects.healthy(), + effects: effects.snapshot(), + reads: fixture.reads, + held: fixture.heldCalls, + cancellations: fixture.cancellations, + oauth: fixture.oauth.observe(), + }); + return; + } + if ( + req.method !== "POST" || + ![ + "/v1/case", + "/v1/release", + "/v1/cancel", + "/v1/oauth", + "/v1/shutdown", + ].includes(route ?? "") + ) { + respond({ error: "unknown operation" }, 404); + return; + } + let bytes = 0; + const chunks: Buffer[] = []; + for await (const chunk of req) { + bytes += Buffer.byteLength(chunk); + if (bytes > 4096) { + respond({ error: "body limit" }, 413); + return; + } + chunks.push(Buffer.from(chunk)); + } + const body: unknown = JSON.parse( + Buffer.concat(chunks).toString("utf8"), + ); + object(body); + if (route === "/v1/case") { + const value = caseOptions(body); + if (fixture.heldCalls) { + respond({ error: "held calls" }, 409); + return; + } + fixture.setCase(value); + if (value.revision !== undefined) + fixture.setCatalogRevision(value.revision); + } else if (route === "/v1/oauth") { + keys(body, ["account", "scope", "issuer", "expiresInSeconds"], []); + if ( + body.account !== undefined && + (typeof body.account !== "string" || + !["account-one", "account-two"].includes(body.account)) + ) + throw new Error("Invalid account"); + if ( + body.scope !== undefined && + (typeof body.scope !== "string" || + !["tools:read", "tools:write", "tools:read tools:write"].includes( + body.scope, + )) + ) + throw new Error("Invalid scope"); + if ( + body.issuer !== undefined && + body.issuer !== "configured" && + body.issuer !== "mismatch" + ) + throw new Error("Invalid issuer mode"); + if ( + body.expiresInSeconds !== undefined && + (typeof body.expiresInSeconds !== "number" || + !Number.isInteger(body.expiresInSeconds) || + body.expiresInSeconds < 1 || + body.expiresInSeconds > 3600) + ) + throw new Error("Invalid token lifetime"); + const value: { + expiresInSeconds?: number; + account?: string; + scope?: string; + issuer?: "configured" | "mismatch"; + } = {}; + if (typeof body.expiresInSeconds === "number") + value.expiresInSeconds = body.expiresInSeconds; + if (body.issuer === "configured" || body.issuer === "mismatch") + value.issuer = body.issuer; + if (typeof body.account === "string") value.account = body.account; + if (typeof body.scope === "string") value.scope = body.scope; + fixture.oauth.configure(value); + } else { + keys(body, []); + if (route === "/v1/release") fixture.releaseCalls(); + if (route === "/v1/cancel") fixture.cancelCalls(); + } + respond({ ok: true }); + if (route === "/v1/shutdown") + setImmediate(() => { + void stop(); + }); + })().catch(() => { + if (!res.headersSent) + res.writeHead(400, { "content-type": "application/json" }); + res.end('{"error":"invalid operation"}'); + }); + }, + ); + control.requestTimeout = 5000; + control.headersTimeout = 5000; + control.keepAliveTimeout = 1000; + control.maxConnections = 32; + async function stop() { + closing ??= (async () => { + control.closeAllConnections(); + await Promise.all([ + fixture.close(), + new Promise((resolve, reject) => + control.close((error) => (error ? reject(error) : resolve())), + ), + ]); + })(); + await closing; + } + try { + await new Promise((resolve, reject) => { + control.once("error", reject); + control.listen(config.control.port, config.control.host, resolve); + }); + } catch { + await fixture.close(); + throw new Error("Control listener failed"); + } + const shutdown = () => { + void stop().catch(() => { + process.exitCode = 1; + }); + }; + process.once("SIGTERM", shutdown); + process.once("SIGINT", shutdown); + process.stdout.write( + JSON.stringify({ + version: 1, + kind: "ready", + control: config.control.origin, + resource: fixture.url, + }) + "\n", + ); +} +void main().catch(() => { + process.stderr.write('{"version":1,"kind":"startup_failed"}\n'); + process.exitCode = 1; +}); diff --git a/tests/fixtures/mcp-stack/platform/README.md b/tests/fixtures/mcp-stack/platform/README.md new file mode 100644 index 000000000..17239d683 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/README.md @@ -0,0 +1,336 @@ +# Isolated platform matrix + +The two adapters run the same common MCP matrix with real services and a real +Firecracker or VZ host. They create only resources for one fresh run. Source +checks do not establish either physical backend result. + +```sh +bash .github/scripts/mcp-stack-fc.sh --run-root /private/absent-run --artifacts /private/fc-inputs.json +bash .github/scripts/mcp-stack-vz.sh --run-root /private/absent-run --artifacts /private/vz-inputs.json +``` + +The selected Bun must match the manifest's binary hash and `.bun-version`. +The wrappers select a local Bun, clear the inherited environment, and pass only +these arguments. `main.ts` qualifies the platform, creates the stack, enables +the guest image through the public API, writes the strict private context, +starts the authenticated supervisor, validates the complete context against that +real private socket, then invokes the unchanged command: + +```sh +bun orchestrator/scripts/mcp-stack/run.ts --context /private/absent-run/context.json --matrix all +``` + +A missing prerequisite, invalid input, failed readiness check, failed matrix, +or cleanup failure returns nonzero. The wrappers default to the full matrix. +They also accept `--matrix ` or +`--stage infrastructure` through the same runner. A focused run records stage +completion only; full matrix completion and physical acceptance require +`--matrix all`. Deployment defaults remain disabled. + +## Operator inputs + +`preflight.ts` defines the exact version 1 manifest schema. No command, +environment, credential, receipt, extra image, or extra role field is accepted. +Paths must be canonical, owned files with one link. The manifest is private. +The run root must not exist; its parent and the empty engine config directory +must be owned, canonical directories with mode 0700. + +Required binary pins are `node`, `bun`, `docker`, `openssl`, `hostAgent`, +`kernel`, `helper`, `claudeHarness`, `codexHarness`, `modelCli`, `agentdBundle`, +`sentinelBundle`, `guestToolsBundle`, and `guestInit`. Each pin has `path` and +lowercase `sha256`. Node is 24; OpenSSL is 3. The native host, helper, and proxy +must have the selected native architecture. Harness and model binaries target +Linux with that architecture. Guest kernel checks are backend specific. + +`proxy` is required for both backends. FC also requires `firecracker` and the +`hostAgent` image. VZ forbids both FC-only fields, requires arm64, and requires +`nativeProbe`, compiled from `native-process-identity.c` on macOS: + +```sh +cc -std=c11 -Wall -Wextra -Werror -O2 tests/fixtures/mcp-stack/platform/native-process-identity.c -o /private/nativeProbe +``` + +Hash final binary bytes. Sign the VZ host with the virtualization entitlement +before hashing it. The probe reads kernel PID, UID, executable, parent, group, +and birth seconds plus microseconds. It accepts only the run's host, proxy, +and managed netstack paths. It has no signal or arbitrary process API. + +Required image keys are `postgres`, `blob`, `registry`, `dns`, `edge`, `guard`, +`coordinator`, `orchestrator`, `worker`, `fixture`, and `guest`. Each image has +an `archive` file pin, `manifestDigest` for the raw single-platform OCI +manifest, and `id` for its config blob and local engine image ID. Every image +must already be loaded in the selected engine. Application images and the +FC host image must carry the exact `sourceCommit` revision label. + +The input also selects the exact owned Unix engine socket, engine ID, empty +`engineConfig`, isolated private `/24`, nondefault registry port, and actual +`nativeHostIp` outside the fixture subnet. Docker CLI and server must be 26 +or later and support API 1.45 or later. Every engine call passes explicit +`--config` and `--host` arguments, checks config/socket identity before and +after, and excludes parent credentials and default contexts. + +The operator must supply actual private DNS and routes from the native host +to all exact fixture names, including `postgres.fixture.example`, and to the +Linux substrate. For Firecracker, `host.fixture.example` must resolve to the +coordinator bridge address (`.21`). For VZ, it must resolve to +`nativeHostIp`. Native resolution is checked against the same exact map used +by fixture DNS. TLS, database, fixture, host, +and matrix operations test their real routes. No adapter installs a VM +manager or changes host DNS, host routes, or global engine settings. + +## Image and service contracts + +The guard image must provide `/bin/sh`, `/bin/bash`, `/bin/sleep`, `/bin/mkdir`, +`/bin/chown`, `/bin/chmod`, `/usr/bin/find`, `/usr/bin/sha256sum`, +`/usr/bin/skopeo`, `/usr/sbin/nft`, `/usr/bin/dig`, `/usr/bin/nc`, +`/usr/bin/curl`, `/usr/bin/mksquashfs`, `/usr/sbin/ip`, and +`/usr/bin/openssl` (OpenSSL 3). Production bundle scripts also use +coreutils. `nc` must use the OpenBSD exit/timeout behavior; `dig` must use BIND +exit status 9 for unreachable DNS. Tool availability is checked before use. +Guard archive work runs with the caller's UID. Projection staging uses root +with only CHOWN, DAC_OVERRIDE, and FOWNER. Network guards use root with only +NET_ADMIN. All start with the default capabilities dropped. + +The fixture image must provide Node 24 at `/usr/local/bin/node` and the real +fixture entrypoint at `/app/tests/fixtures/mcp-stack/main.mts`, with its source +imports. Modern and legacy fixtures are independent processes with generated +TLS and durable control/effect history. Their writable mounts contain only +their exact run roots. They stay alive across application restarts. + +Postgres must provide UID 999, `/usr/bin/pg_isready`, and its normal initialized +image entrypoint with `PGDATA=/var/lib/postgresql/data`. This acceptance fixture +uses one fresh `mcp_` database for both real production +migration paths. Coordinator and orchestrator have distinct private DSN files. +Their strict read-only observers check each role's real schema. No authority, +image, placement, OAuth, publication, or user row is inserted by the launcher. +SQL source has no shared table, index, or function name across the two migration +sets. Real migration failure remains a hard failure; source inspection cannot +prove migration compatibility. + +Blob uses `/bin/fake-gcs-server` on HTTP 4443 with UID 1000 and `/data` storage. +Registry uses its inspected entrypoint, UID 1000, and `/var/lib/registry`. +CoreDNS uses `/coredns`, UID 1000, and the generated no-forward Corefile. +The fixed `dns` and `denied-dns` fixture roles add NET_BIND_SERVICE after +all default capabilities are dropped. The pinned CoreDNS executable has a file +capability that requires this capability in its bounding set, even when port +53 is unprivileged. Each effective audit requires exactly this one added +capability for these two DNS roles. Both DNS roles retain read-only roots, +no-new-privileges, default seccomp, and private config projections. +The trusted Firecracker host also adds NET_BIND_SERVICE to bind its private +managed DNS listener on port 53 and requires its exact four-capability set. +Every other role rejects NET_BIND_SERVICE. Worker and application capabilities +do not change. +The TLS edge uses `/usr/sbin/nginx`, generated config, and writable `/tmp`. +Select upstream image bytes that meet these explicit user, path, environment, +and volume contracts. Automatic image volumes and unexpected image environment +keys fail the effective audit. Application entrypoints remain the production +image entrypoints. O migrations use its production `bun run migrate` leaf. + +The FC-only `host-agent.Dockerfile` derives from a production host image at an +immutable manifest digest and adds only an independently hashed Firecracker +binary. Pass `HOST_IMAGE=registry/name@sha256:...` and `FIRECRACKER_SHA256=...`, +with `firecracker` in a private build context. It inherits the source revision +and entrypoint. Pin the derivative archive/manifest/config. Extract the final +host, UFFD, and proxy bytes for their binary pins; the launcher checks all four +baked executable hashes before releasing the host entrypoint. + +FC requires one exclusive disposable Linux x86-64/KVM runner. No other host +agent, Firecracker, proxy, or storage job can share its kernel. The operator +must prepare a fresh dedicated dm-thin pool before the job. The strict FC-only +`thinPool` input has exactly `name` and `directory`. Names match +`[A-Za-z][A-Za-z0-9_.-]{0,126}`; physical admission also requires at most 26 +characters so production checkpoint names fit the kernel's 127-byte limit. +The directory is canonical, owned by the job UID, mode 0700, and has no symlink +components. VZ rejects this field. + +Admission checks the real pool table/device, metadata/data loop devices and +backing file inodes/devices, full file allocation, holders and open FDs. It +checks all active and inactive thin IDs before the host can open the pool. +Production host startup deletes inactive orphan IDs; it must never open an +unqualified shared pool. Metadata inventory runs only after kernel process and +device-FD checks prove +that the host and its VMs have stopped. An offline lease blocks every owned +host start until the metadata snapshot is released. A pre-existing reservation +is refused, and an uncertain release stops further inventory. Live observation +and live release checks read tables and FDs without reserving metadata. No job +loads modules, creates or formats a +pool, changes node modes, clears caches, or force-removes devices. + +Only the trusted FC host joins the initial host PID namespace. It retains a +private cgroup and mount namespace, C's owned network namespace, default +seccomp, a read-only root, sysfs and cgroup mounts, and bounded resources. +It has NET_ADMIN, NET_BIND_SERVICE, SYS_ADMIN and MKNOD. Exact KVM, TUN, mapper +control, pool, and read-only metadata nodes are projected. A block-device cgroup rule grants +`rwm` only for the running kernel's DM major; `/dev/engram` binds one private +run-owned node directory. This is global DM control on the exclusive runner. +All other roles, including the worker, retain private PID namespaces and no +DM authority. No broad `/dev`, host root, engine socket, or host network mount +is allowed. Effective container inspection and the real infrastructure smoke +must prove this contract; source tests do not establish installed capability. + +`ENGRAM_THIN_POOL` selects the qualified pool. Admission requires healthy data/metadata fractions, metadata capacity from +the installed data-block count (48 bytes per block plus 25% reserve), fully +allocated backing files, and a separate +16 GiB filesystem reserve. Actual image/root/swap sizes are then checked against +two live roots, two checkpoints, two in-flight roots, corresponding swap overlap, +one staged image, and 25% data reserve before infrastructure can pass. This is +a serial-suite budget, not a volume count based on the 24 cases. A larger sparse +file cannot satisfy it. The production File memory restore mode stays unchanged; +UFFD helper bytes remain pinned without a claim of UFFD runtime qualification. + +Only the Firecracker host container sets `apparmor=unconfined` and +`systempaths=unconfined`. The production host creates private network namespaces +and sets their IPv6 sysctls. Its audit requires the exact inspected AppArmor +profile and empty masked and read-only system-path lists. Docker consumes the +`systempaths` flag into those lists. The audit requires exact no-new-privileges +and AppArmor options, with only Docker's derived `label=disable` also allowed. Other +roles keep the default profile and system-path protections. The host keeps +no-new-privileges, default seccomp, exact capabilities, and all device and mount +bounds. Native qualification reads the effective proc mount and requires writable +`/proc/sys`, while `/sys` and `/sys/fs/cgroup` remain read-only. It checks all +three effective, permitted and bounding masks against the four host capabilities. +Ordinary and VZ role audits reject empty masked or read-only system-path lists, +even when Docker has consumed the systempaths option. + +FC's listener is `0.0.0.0:9101` only inside its owned namespace. C's guard +publishes only the verified `nativeHostIp:9101`. The advertised TLS authority +stays `host.fixture.example:9101` and resolves to C's bridge address because +FC and C share that namespace. The published native port remains exact. +VZ resolves this name to and binds its verified native address directly. + +## Network and startup order + +One internal bridge carries isolated PG, blob, DNS, fixtures, and guards. O, +C, and worker join persistent guard namespaces with exact default-drop nft +rules. The owned registry shares C's guard and is published only on +`127.0.0.1:` for asset bootstrap. Managed MCP destinations +retain strict HTTPS and nonloopback checks. + +C alone has an exact `/32` route to the native host. FC uses the internal +bridge gateway for this native route. C's host RPC uses its own bridge address, +with the same host TLS name and leaf pin. VZ has one additional +run-owned noninternal private IPv4 bridge attached only to C's guard. The +launcher checks its actual IPAM for overlap with fixture and native host +addresses, derives the exact interface from its endpoint IP and MAC, removes +that link's automatic default under the installed nft default-drop policy, +and installs only the native-host `/32`. Docker's internal bridge forwarding +rules block destinations outside its subnet, so the internal bridge alone +cannot reach the native Mac. No global route or firewall is changed. Both +networks have exact ownership and cleanup checks. + +After actual host readiness, a fixed short-lived probe in C's namespace uses +only a private read-only CA/client certificate/key projection, resolver, and +bounded tmpfs. OpenSSL verifies TLS 1.3, SNI, hostname, and the actual first host +leaf fingerprint. Only a sanitized certificate digest is retained. Exact +probe removal, route/no-default checks, network membership, nft policy, and +native mTLS qualification repeat for status and restart. + +Only the FC C guard has `net.ipv4.ip_forward=1` and +`net.ipv4.conf.all.route_localnet=1`; other roles have no configured sysctls. +The actual FC host and guard must share a network namespace and read both +values as 1. Qualification checks the actual production INPUT, FORWARD, +PREROUTING, and POSTROUTING rules and their order after readiness, status, +and restart. Only the trusted FC host requires writable `/proc/sys` in its +private mount namespace; `/sys` and `/sys/fs/cgroup` remain read-only. Other +roles retain their default system-path protections. + +A named volume contains only the fixed `resolv.conf`, with file mode 0644 and +directory mode 0755. A read-only `volume-subpath=resolv.conf,volume-nocopy` +mount supplies each container's actual `/etc/resolv.conf`. No host path is +exposed to the worker. Its other mount is only its exact config/TLS projection; +private bounded tmpfs supplies temporary writes. Secret configuration volumes +retain directories 0700 and files 0600. Only the resolver and guard CA-only +probe projection are public within their isolated volumes. + +The old copy-to-intrinsic-file approach is not used: Docker archive extraction +rejects writes through a read-only root filesystem. Moby's +[file subpath implementation](https://github.com/moby/moby/blob/master/daemon/volume/safepath/join_linux.go) +and [mount setup](https://github.com/moby/moby/blob/master/daemon/volume/mounts/mounts.go) +provide the named file projection. See also [Docker copy semantics](https://docs.docker.com/reference/cli/docker/container/cp/) +and [volume subpaths](https://docs.docker.com/engine/storage/volumes/). + +Before app release, the launcher audits effective mounts, exact subpath, +volume ownership, read-only state, namespace/user, capabilities/devices, +resources, environment, DNS options, publications, and inspected entrypoint. +It reads back actual resolver bytes. A fixed waiting shell then execs the +original entrypoint and arguments. No application code runs before these +checks. The FC host shell also sends the actual host output to its private +0600 observation log. + +Qualification inspects installed nft JSON, resolves and reaches both real +HTTPS fixtures, confirms an owned denial sink is live, and verifies that +sink traffic, the embedded DNS stub, and a separate owned DNS stub are blocked. +Unknown DNS names receive refusal with no forwarding. Checks repeat after +start, status, and restart. A boolean or operator receipt cannot replace them. + +Startup order is dependencies, both fixtures, guards and controls, TLS edges +and registry, worker readiness, coordinator readiness and real migrations, +O's real migration leaf, O readiness, native host readiness, normal bootstrap +human signup, public image enable, actual host image readiness, strict context, +supervisor, then the full common matrix. Only the deterministic model CLI is +replaced in the test harness bundles. Actual harness binaries, descriptors, +agentd, and the exact committed production init shim remain intact. + +## Cleanup and evidence + +The supervisor permits only its fixed role status/restart/stop protocol and +the FC-only infrastructure and disk-read checkpoints. Fixture roles are +status-only. +O disable variants change only O's enable switch; C, worker, host, operator +candidate, PG, blob, fixtures, and namespace identities remain normal. + +Signals and deadlines stop and drain the common runner and supervisor. Final +teardown uses the authenticated public SessionService DeleteSession path for +bounded, verified run-owned sessions while the host is alive. It retains task, +session, operation, journal, capture, and effect history. An API return does +not prove teardown: the launcher verifies that live and transient thin volumes +are removed by the production delete path, VM/device FDs are closed, and no +live/transient mappings or device holders remain before stopping the host. +After reaping the owned processes, it also checks metadata for unknown thin IDs. +Recorded image and +checkpoint volumes can remain with their exact device, table and thin ID and +zero opens. Unknown, busy or changed state fails cleanup. No forced DM remove, +pool erasure, forced adoption, or history deletion manufactures success. + +Host restart retains the admitted pool/backing identity and append-only physical +boot evidence. It inventories inactive IDs before allowing the replacement host +to open the pool. Each physical boot binds its rootfs symlink, exact session +volume, thin table/ID and pool, canonical device number, real Firecracker FD, +process birth/hash/cgroup, and an increase in kernel read sectors sampled during +that boot. Prior-boot counters cannot qualify a new boot. Missing or failed +observation writes fail acceptance. + +The common infrastructure stage performs real guest create/use/delete before +MCP cases. Its fixed authenticated `infrastructure-ready` checkpoint requires +physical boot observations and normal thin release. It then stops the owned +host, verifies process and FD quiescence, inspects metadata, and restarts the host +under the same lifecycle lock. The returned status must show a new host boot and +unchanged identities for every other role. The worker cannot supply these facts. The same path runs in the full matrix; no second runner framework +or mocked PASS replaces it. Final cleanup checks retained processes and pool +state again after owned processes stop. + +Engine CLI cancellation uses the retained process group's TERM/KILL/wait path. +Cleanup recovers exact planned container and network names after a lost create +response, refuses foreign labels, and records survivors as failure. VZ rechecks +stable kernel birth, UID, executable, and PID before signaling exact owned +helper/proxy survivors, including separate groups, then requires none remain. +It sends no pattern kill. Logs, identity snapshots, matrix output, and +cleanup/completion JSON stay private. Named volumes retain inactive database +and effect history; the cleanup receipt names them. No guest result is written +by the launcher. + +## Source checks and remaining physical gates + +```sh +MCP_PLATFORM_TEST_OPENSSL=/absolute/openssl3 bun test tests/fixtures/mcp-stack/platform/platform.test.ts tests/fixtures/mcp-stack/platform/composition.test.ts +node orchestrator/node_modules/typescript/bin/tsc -p tests/fixtures/mcp-stack/platform/tsconfig.json --noEmit +bash -n .github/scripts/mcp-stack-fc.sh .github/scripts/mcp-stack-vz.sh +``` + +These tests use files, controlled subprocesses, synthetic engine replies, +private TLS, and private Unix sockets. They establish argument, identity, +authority, resolver, launch-order, cancellation, and cleanup rejection behavior. +Native C compile/own-process checks establish only local process identity. +No source test proves a real mount, installed policy, image materialization, +service/VM cleanup, public image enable, or matrix success. Both real backend +runs, their assets and capable runners, and current-head CI are still required. diff --git a/tests/fixtures/mcp-stack/platform/assets.ts b/tests/fixtures/mcp-stack/platform/assets.ts new file mode 100644 index 000000000..b554b124e --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/assets.ts @@ -0,0 +1,368 @@ +import { constants } from "node:fs"; +import { chmod, copyFile, mkdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { + check, + childEnvironment, + command, + hash, + put, + readOwned, + verifyPin, + type Pin, +} from "./owned.ts"; +import { Engine } from "./engine.ts"; +import { REPO, type Inputs, type ImagePin } from "./preflight.ts"; + +/** Bind raw archive manifest and config bytes to the two independent image pins. */ +export function verifyArchiveIdentity( + image: ImagePin, + raw: Uint8Array, + config: Uint8Array, +) { + const manifest = JSON.parse(Buffer.from(raw).toString()); + check( + manifest.schemaVersion === 2 && + !manifest.manifests && + manifest.config?.digest === image.id && + `sha256:${hash(raw)}` === image.manifestDigest, + "archive_image_identity", + ); + check(`sha256:${hash(config)}` === image.id, "archive_config_identity"); +} + +export async function stageAssets( + inputs: Inputs, + root: string, + engine: Engine, +) { + const directory = join(root, "assets"), + bundles = join(root, "bundles"); + const sourceFile = async (path: string) => { + const committed = await command( + "/usr/bin/git", + ["-C", REPO, "show", `${inputs.sourceCommit}:${path}`], + REPO, + childEnvironment(root), + ); + check( + committed.equals(await readFile(join(REPO, path))), + "asset_source_changed", + ); + return committed; + }; + await mkdir(directory, { mode: 0o700 }); + await mkdir(bundles, { mode: 0o700 }); + async function copy( + name: string, + source: Pin, + executable = false, + ): Promise { + await verifyPin(source); + const path = join(directory, name); + await copyFile(source.path, path, constants.COPYFILE_EXCL); + await chmod(path, executable ? 0o700 : 0o600); + const result = { path, sha256: source.sha256 }; + await verifyPin(result); + return result; + } + const binaries = { ...inputs.binaries }; + for (const [name, pin] of Object.entries(inputs.binaries)) + binaries[name as keyof typeof binaries] = await copy( + name, + pin, + ![ + "kernel", + "agentdBundle", + "sentinelBundle", + "guestToolsBundle", + ].includes(name), + ); + const proxy = await copy("proxy", inputs.proxy, true); + const nativeProbe = inputs.nativeProbe + ? await copy("nativeProbe", inputs.nativeProbe, true) + : undefined; + const firecracker = inputs.firecracker + ? await copy("firecracker", inputs.firecracker, true) + : undefined; + // MaterializeImage uses this exact built-in shim, with its transport substitution. + const source = ( + await sourceFile("crates/engram-rootfs-materializer/src/inject.rs") + ).toString(); + const match = /pub const DEFAULT_INIT_SHIM: &str = r#"([\s\S]*?)"#;/.exec( + source, + ); + check( + match && + Buffer.from(match[1]!).equals(await readOwned(binaries.guestInit.path)), + "production_init_shim", + ); + const artifacts = { + coordinator: await copy( + "coordinator.tar", + inputs.images.coordinator.archive, + ), + orchestrator: await copy( + "orchestrator.tar", + inputs.images.orchestrator.archive, + ), + worker: await copy("worker.tar", inputs.images.worker.archive), + fixture: await copy("fixture.tar", inputs.images.fixture.archive), + guestImage: await copy("guest.tar", inputs.images.guest.archive), + hostAgent: binaries.hostAgent, + kernel: binaries.kernel, + helper: binaries.helper, + claudeHarness: binaries.claudeHarness, + codexHarness: binaries.codexHarness, + modelCli: binaries.modelCli, + }; + for (const [role, image] of Object.entries(inputs.images)) { + await verifyPin(image.archive); + const stage = `verify-${role.toLowerCase()}`; + const id = await engine.create( + stage, + inputs.images.guard.id, + [ + "--network", + "none", + "--user", + `${process.getuid?.()}:${process.getgid?.()}`, + "--read-only", + "--mount", + `type=bind,src=${image.archive.path},dst=/input/image.tar,readonly`, + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=256m,mode=1777", + ], + ["infinity"], + "/bin/sleep", + ); + try { + const raw = await engine.call([ + "exec", + id, + "/usr/bin/skopeo", + "--tmpdir", + "/tmp", + "inspect", + "--raw", + "oci-archive:/input/image.tar", + ]); + const config = await engine.call([ + "exec", + id, + "/usr/bin/skopeo", + "--tmpdir", + "/tmp", + "inspect", + "--raw", + "--config", + "oci-archive:/input/image.tar", + ]); + verifyArchiveIdentity(image, raw, config); + } finally { + await engine.remove(stage); + } + await verifyPin(image.archive); + } + const stamp: Record = {}; + for (const [logical, input] of [ + ["agentd", binaries.agentdBundle], + ["sentinel", binaries.sentinelBundle], + ["guest-tools", binaries.guestToolsBundle], + ] as const) { + await copyFile( + input.path, + join(bundles, `${input.sha256}.squashfs`), + constants.COPYFILE_EXCL, + ); + await chmod(join(bundles, `${input.sha256}.squashfs`), 0o600); + stamp[logical] = input.sha256; + } + const recipes = join(directory, "recipes"); + await mkdir(recipes, { mode: 0o700 }); + await put( + join(recipes, "_pack.sh"), + await sourceFile("deploy/bundles/_pack.sh"), + ); + for (const name of ["claude", "codex"] as const) { + const path = join(recipes, `harness-${name}`); + await mkdir(path, { mode: 0o700 }); + await put( + join(path, "build.sh"), + await sourceFile(`deploy/bundles/harness-${name}/build.sh`), + ); + } + const packed: Partial> = {}; + for (const harness of ["claude", "codex"] as const) { + const tree = join(directory, `${harness}-tree`); + await mkdir(tree, { mode: 0o700 }); + for (const [name, source] of [ + [ + "harness", + harness === "claude" ? binaries.claudeHarness : binaries.codexHarness, + ], + [harness, binaries.modelCli], + ["engram-mcp-model-fixture", binaries.modelCli], + ] as const) { + await copyFile(source.path, join(tree, name), constants.COPYFILE_EXCL); + await chmod(join(tree, name), 0o755); + } + await put( + join(tree, "harness.toml"), + await sourceFile(`deploy/harness-${harness}/harness.toml`), + ); + const output = join(directory, `${harness}.squashfs`); + const role = `pack-${harness}`; + const id = await engine.create( + role, + inputs.images.guard.id, + [ + "--network", + "none", + "--read-only", + "--user", + `${process.getuid?.()}:${process.getgid?.()}`, + "--mount", + `type=bind,src=${directory},dst=${directory}`, + "--mount", + `type=bind,src=${recipes},dst=/recipes,readonly`, + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=64m", + ], + [`/recipes/harness-${harness}/build.sh`, tree, output], + "/bin/bash", + ); + check( + (await engine.call(["wait", id], undefined, 30000)).toString().trim() === + "0", + "bundle_pack", + ); + await engine.remove(role); + const bytes = await readFile(output); + const sha256 = hash(bytes); + await chmod(output, 0o600); + const pin = { path: output, sha256 }; + await verifyPin(pin); + await copyFile( + output, + join(bundles, `${sha256}.squashfs`), + constants.COPYFILE_EXCL, + ); + await chmod(join(bundles, `${sha256}.squashfs`), 0o600); + stamp[`harness-${harness}`] = sha256; + packed[harness] = pin; + } + // This is the existing host bundle stamp (logical name -> content hash), not a guest tool manifest. + await put(join(bundles, "current.json"), JSON.stringify(stamp)); + check(packed.claude && packed.codex, "bundle_set"); + return { + binaries, + proxy, + nativeProbe, + firecracker, + artifacts: { + ...artifacts, + claudeBundle: packed.claude, + codexBundle: packed.codex, + }, + bundles, + stamp, + }; +} + +/** The immutable archive, config blob and registry manifest must be one image. */ +export async function publishGuest( + inputs: Inputs, + root: string, + engine: Engine, +) { + await verifyPin(inputs.images.guest.archive); + const guard = engine.guardIds.get("coordinator"); + check(guard, "registry_namespace"); + const role = "image-copy"; + const id = await engine.create( + role, + inputs.images.guard.id, + [ + "--network", + `container:${guard}`, + "--user", + `${process.getuid?.()}:${process.getgid?.()}`, + "--read-only", + "--mount", + `type=bind,src=${inputs.images.guest.archive.path},dst=/input/guest.tar,readonly`, + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=256m,mode=1777", + ], + ["infinity"], + "/bin/sleep", + ); + try { + const archive = "oci-archive:/input/guest.tar"; + const manifest = await engine.call([ + "exec", + id, + "/usr/bin/skopeo", + "--tmpdir", + "/tmp", + "inspect", + "--raw", + archive, + ]); + const config = await engine.call([ + "exec", + id, + "/usr/bin/skopeo", + "--tmpdir", + "/tmp", + "inspect", + "--raw", + "--config", + archive, + ]); + verifyArchiveIdentity(inputs.images.guest, manifest, config); + const uri = `127.0.0.1:${inputs.registryPort}/${engine.runId}/guest`; + await engine.call( + [ + "exec", + id, + "/usr/bin/skopeo", + "--tmpdir", + "/tmp", + "copy", + "--preserve-digests", + "--dest-tls-verify=false", + archive, + `docker://${uri}:fixture`, + ], + undefined, + 120000, + ); + const actual = await engine.call([ + "exec", + id, + "/usr/bin/skopeo", + "--tmpdir", + "/tmp", + "inspect", + "--tls-verify=false", + "--raw", + `docker://${uri}@${inputs.images.guest.manifestDigest}`, + ]); + check(hash(actual) === hash(manifest), "registry_manifest_changed"); + await verifyPin(inputs.images.guest.archive); + await put( + join(root, "image-provenance.json"), + JSON.stringify({ + version: 1, + archiveSha256: inputs.images.guest.archive.sha256, + manifestDigest: inputs.images.guest.manifestDigest, + configDigest: inputs.images.guest.id, + imageUri: `${uri}@${inputs.images.guest.manifestDigest}`, + }), + ); + return `${uri}@${inputs.images.guest.manifestDigest}`; + } finally { + await engine.remove(role); + } +} diff --git a/tests/fixtures/mcp-stack/platform/ci/README.md b/tests/fixtures/mcp-stack/platform/ci/README.md new file mode 100644 index 000000000..0f7c842c7 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/ci/README.md @@ -0,0 +1,296 @@ +# Required physical MCP CI + +`mcp-physical-fc` and `mcp-physical-vz` are required through CI Gate. Each +job builds and consumes its own complete assets. Both use the same path +selector. There is no dependency on a path-skipped binary or image job. +A missing setting or capability fails the job. A source test or a skipped +hypervisor test cannot replace either physical result. + +Provision dedicated, disposable, operator-controlled runners with Python3.12+ +and these labels: + +- `[self-hosted, Linux, X64, engram-mcp-firecracker]` +- `[self-hosted, macOS, ARM64, engram-mcp-vz]` + +These labels are a prerequisite, not evidence of a registered runner. The +repository runner inventory had no registered runners when this source was +prepared. Privileged FC public-source jobs require an ephemeral, disposable dedicated +runner VM with no standing platform, cloud or user credentials. Do not run +untrusted jobs on a persistent machine with credentials. +The workflows do not register runners or change repository settings. + +The Linux producer and adapter run as root. The VZ producer and adapter run +as the runner UID. Provision a canonical, already-running Linux Docker engine +of the matching architecture. Its Unix socket must belong to that UID, have +no symlink components, and stay at the same inode/device. Docker CLI/server +must satisfy composition preflight (at least26, API1.45). Install buildx as a +system CLI plugin; the producer uses the engine's default Docker driver with +explicit `--config`, `--host` and `--builder default`. It creates no extra +BuildKit daemon. Provision the classic image store with raw config SHA256 +image IDs. A store that exposes manifest IDs fails the strict image checks. +The default driver must support image loading and the local exporter. +For source images, the producer uses `--load` and retains build metadata in +its private work root. It accepts only `containerimage.config.digest`, checks +the exact loaded image ID, Linux architecture, source revision and run label, +then saves that config ID to the owned Docker archive. Before conversion, +the saved raw config must hash to the build config ID. After conversion to +OCI, the full config must match with JSON object keys sorted; value types +and list order must stay the same. The final OCI image ID and manifest digest +come from its actual raw bytes through the same strict archive and import +checks as other images. +Missing or invalid metadata or image facts stop the producer before save. +The local-directory export path stays unchanged. No tag or new builder is used. +Do not start Docker Desktop, Lima, Colima or another engine from the job. + +For FC reserve the entire dedicated Linux x86-64 kernel. Prepare a fresh, +exclusive dm-thin pool with the production node helper before running the job: + +```sh +install -d -m 0700 /work/mcp-thin/acceptance +POOL_NAME=mcp-acceptance POOL_DIR=/work/mcp-thin/acceptance \ + POOL_DATA_SIZE=96G POOL_META_SIZE=2G POOL_RESERVE=16G \ + sh deploy/helm/engram-host-fleet/files/thin-pool.sh +``` + +This is explicit node preparation, never a preflight action. Use an empty, +disposable directory and no `POOL_CACHE_DIR`. A pool name alone does not isolate +DM control. No other host agent, Firecracker, proxy, or storage job may run. +Install `/usr/sbin/dmsetup`, `/usr/sbin/losetup` and `/usr/sbin/thin_dump`. +The adapter verifies full allocation and actual backing identities, active and +inactive IDs, capacity and holders before the production host can open the pool. +The 96 GiB allocation is an operator estimate. Capacity is checked against +actual installed image/session/swap overlap before infrastructure passes; +a larger image can require more capacity. Keep 16 GiB of filesystem space free +in addition to pool data and metadata. Source tests do not prove these minima +are sufficient for a particular installed image. + +The trusted host alone receives SYS_ADMIN, NET_ADMIN, MKNOD, mapper control, +the admitted pool and read-only metadata devices, and an `rwm` device rule for +the kernel's observed DM major. Only its private node directory binds at +`/dev/engram`; there is no broad `/dev` mount. Other roles retain their existing +device denial and private PID namespaces. The real infrastructure stage must +create, use and release production dm-thin session volumes under this effective +policy before MCP cases start. Missing capability fails without a retry that +adds authority. A device rule cannot isolate global mapper control; kernel +exclusivity is required. + +Cleanup uses normal session deletion while the host is alive, then verifies +closed FDs and removal of live/transient volumes. Recorded images/checkpoints +can remain, with exact table/ID/device and zero opens. Any unexplained survivor, +foreign change or inactive ID fails cleanup. No forced DM removal, pool reset, +cache clearing or history deletion is allowed. Retain restricted evidence for +identity-scoped recovery on a failed run. + +Provision native private DNS and routes for the exact fixture hosts and the +selected bridge subnet. Set `host.fixture.example` to the coordinator bridge +address (`.21`) for Firecracker, or to `nativeHostIp` for VZ. +The native DNS answer must match fixture DNS; the launcher does not change it. +Keep `nativeHostIp` as the real native interface address outside the fixture +subnet. Native host processes must reach container IPs, and containers must +reach the native host address. Provide real KVM and TUN access +on Linux, and real hardware virtualization on native arm64 macOS. Production +Firecracker uses the production dm-thin rootfs path; the platform's reviewed native +capability preflight must qualify that path before a VM starts. No workflow +changes modules, device permissions, sysctl, routes or global DNS. A failed +capability probe is a failure. The VZ host is built and ad-hoc signed with the +production virtualization entitlement before its file hash is calculated. + +Place a bounded nonsecret JSON file at the fixed path: + +- Linux: `/etc/engrams/mcp-physical-firecracker.json` +- macOS: `/Library/Application Support/engrams/mcp-physical-vz.json` + +The file must be canonical, single-link, at most16KiB, owned by root or the +runner UID, and not writable by group/other. Exact schema (replace paths and +addresses with the provisioned facts): + +```json +{ + "version": 1, + "backend": "firecracker", + "engineSocket": "/run/docker.sock", + "nativeHostIp": "192.168.80.2", + "subnet": "10.88.2.0/24", + "registryPort": 30000, + "buildRegistryPort": 31000, + "thinPool": { + "name": "mcp-acceptance", + "directory": "/work/mcp-thin/acceptance" + }, + "tools": { + "node": "/opt/mcp/node/bin/node", + "bun": "/opt/mcp/bun/bin/bun", + "docker": "/opt/mcp/docker/bin/docker", + "openssl": "/opt/mcp/openssl/bin/openssl", + "skopeo": "/opt/mcp/skopeo/bin/skopeo", + "gh": "/opt/mcp/gh/bin/gh", + "oras": "/opt/mcp/oras/bin/oras", + "npm": "/opt/mcp/node/lib/node_modules/npm/bin/npm-cli.js" + } +} +``` + +An administrator can set the optional repository variable `MCP_FC_BOOT_ARTIFACTS` +to an absolute path such as `/etc/engrams/mcp-boot/manifest.json` on the dedicated +FC runner. With this variable absent or empty, CI keeps the strict existing boot +artifact download path. A download failure does not select another source. With +an explicit value, CI passes it as one quoted `--boot-artifacts` argument. The +variable contains a path, never a command, credential, or supplied result. + +Before the job, the operator must install that manifest and its four pinned +files in the same canonical private directory outside the checkout and exported +source. The directory must be owned by root and mode 0700. The manifest and each +file must be root-owned, single-link, mode 0400, and have no symlink components. +Do not generate this custody record in the workflow. The manifest pins the +Firecracker executable, kernel, companion kernel checksum, and build provenance +with exact `{ "path": "...", "sha256": "..." }` objects. Its fixed identity is +version 1, kind `mcp-physical-boot-assets`, backend `firecracker`, architecture +`amd64`, and kernel release `fc-kernel-6.1.102-1`. `forkGitlink` must match +`third_party/firecracker` in the checkout. The provenance must identify a release +build for `x86_64-unknown-linux-musl` from that exact fork revision and bind the +executable hash. The producer checks the actual static x86-64 executable, all +pins and the kernel checksum before the build. It stages verified copies in the +private run directory. It rejects unknown fields, substituted files and invalid +provenance. VZ does not accept this option. + +The repository variable and preinstalled files are administrator-controlled +runner preparation. This source change does not set the variable or provision a +runner. The same serialized FC job, 24 common cases, cleanup checks and `CI Gate` +remain required with either explicit source selection. + +VZ omits `thinPool`, sets backend`vz` and additionally supplies canonical executable paths for +`cargo`, `rustc`, `cc` and `go` under tools. Supply actual Rust1.99.0 toolchain +binaries, not rustup shims. Supply Go1.26.2, Node24 (CI source uses24.22.2), +Bun matching `.bun-version`, OpenSSL3 and the native build prerequisites, +including protoc and Xcode command-line tools. Runtime copies must remain +executable with their normal platform library dependencies. The VZ build uses +fresh private Cargo/Go caches, real production host/proxy source and the actual +netstack build script. It compiles the read-only native process identity probe. + +The native address is a nonloopback RFC1918 address on a real noninternal +interface, separate from the nonoverlapping10.x.y.0/24 subnet. Reserve +registryPort through registryPort+3, native host9101 and the fixture service +ports checked by the adapter. buildRegistryPort is separate. On FC only it +publishes a run-owned registry at127.0.0.1. The default engine must support +its normal loopback registry access without a daemon setting change. The +selected engine's build resolver must reach that same loopback publication. +This registry publishes no external image. Do not add credentials, commands, +scripts, environment maps or PASS booleans to operator JSON. + +Before any asset operation, the producer makes a fresh private checkout owned +by its executing UID and reexecutes the fixed producer there. This also puts the +actual adapter, its Git reads and private dependency installation under the same +principal. The bootstrap reads only the exact canonical original source with +command-local `safe.directory` trust, disables optional index writes, and fetches +the exact HEAD through local Git into a new repository with an empty template. +It verifies HEAD, clean files, every file owner and the small generated Git +config; source credentials, config, hooks and hard links are never copied. No +global trust entry or shared-checkout ownership change is used. The outer work +root retains bootstrap logs and safe summary; the private checkout is at +`checkout/` and actual assets/evidence/private caches are under `execution/`. +The inner stage accepts its private flag only with that owned canonical layout. + +The producer verifies clean exact HEAD and exports only its tracked source +with git archive into a new private directory. It excludes only the known legacy +runtime directory `crates/engram-coordinator/var` before validation and archive +creation. All other secret/var/.git paths fail validation. The archive uses the +same finite list of literal tracked names. Before extraction, the producer checks +all archive member paths and types against that selection, including parent +directories and empty submodule directories. Missing, extra, duplicate, linked, +or unsafe members fail before any file is extracted. Safe tar extraction remains +enabled. It never sends the checkout, node_modules, runtime files, `.env` or Docker +configuration as a build context. Dependency installation uses private cache, +explicit empty npm configuration and XDG Bun configuration, with lifecycle +scripts disabled; no user package-manager config or inherited token is used. It uses actual production Dockerfiles for +coordinator/orchestrator/worker/demo, unchanged reviewed guard/fixture recipes, +and the fixed Linux-musl recipe here. That recipe builds real agentd and both +harnesses, the deterministic model CLI, and production bundle leaves for the +selected Linux architecture. guestInit is the exact DEFAULT_INIT_SHIM source +bytes. Only the model CLI is a replacement. + +The fixed dependency runtime paths and explicit users match composition: + +| Image | Runtime user | Command and writable paths | +| -------- | ------------ | ----------------------------------------------------------------------------------------------------------------------------- | +| Postgres | 999:999 | production entrypoint; `/usr/bin/pg_isready`; owned `/var/lib/postgresql/data` volume and `/var/run/postgresql`, `/tmp` tmpfs | +| Blob | 1000:1000 | `/bin/fake-gcs-server`; owned `/data` volume, `/tmp` tmpfs | +| Registry | 1000:1000 | image entrypoint; owned `/var/lib/registry` volume, `/tmp` tmpfs | +| DNS | 1000:1000 | `/coredns`; read-only `/projection/Corefile` and hosts file | +| Edge | 1000:1000 | `/usr/sbin/nginx`; read-only `/projection`, writable `/tmp` tmpfs | + +The producer-only loopback registry uses its default root user with a private +tmpfs store. These source path assumptions must pass the actual image and +physical consumer gates; no default image user substitutes for explicit +composition users. + +`upstream-images.json` pins five upstream single-platform manifests for each +architecture. Anonymous registry metadata supplied those facts. Actual CI +checks the pinned upstream raw manifest, then converts with skopeo to an OCI +archive. OCI/Docker media type conversion can change the manifest digest; the +final Inputs use the actual OCI manifest and config, never the upstream tag or +a substituted digest. The archive file SHA, raw OCI manifest SHA and raw config +SHA are three separate facts. Skopeo converts the same OCI archive to a Docker +load archive; explicit engine inspect checks config ID/OS/architecture and +current source labels. The actual adapter independently checks all archive +linkage before real public image enable and rootfs materialization. + +For FC the production host Dockerfile builds the base at current HEAD. A +loopback run-owned registry makes that immutable manifest available to the +reviewed host derivative. It adds only the gitlink-keyed production fork. +The final image supplies the exact host/proxy/UFFD/fork pin bytes through +`docker cp` from an unstarted run-owned extraction container. The staged fork +must match the baked file. There is no stock FC fallback or skip. + +The workflow token is permitted only for fixed upstream fork/kernel reads. +The producer can run anonymously when those artifacts are public. Fixed `gh +release download` children receive only GH_TOKEN and a private GH_CONFIG_DIR. +The production kernel release checksum must match. Fixed GHCR oras login uses +password stdin and a private0600 registry config deleted after fetch. The +resolved immutable fork manifest and gitlink are retained. No credential reaches +builds, Inputs, the adapter, CLI arguments or public logs. Missing artifact read +permission remains a hard failure and a release prerequisite. + +The producer invokes the actual strict Inputs writer with Node24 transform +types, then the exact FC/VZ wrapper with a private Inputs file and absent +canonical run root. The unchanged common matrix must produce exactly one +PASS result with selected24 and passed24. The actual adapter completion must +confirm the same backend/HEAD, matrix completion and no cleanup failures. +A zero-case run, missing output, failure, cancellation or cleanup failure fails. + +Work roots under RUNNER_TEMP are0700 and retained on operator-controlled +storage. Logs, journal, evidence, database volumes and effect history can +contain sensitive test material. Limit access and retention there. Public +artifacts contain only the fixed safe status/count summary. Never upload raw +Inputs, context, environment, logs, HTTP bodies, TLS keys, cookies, tokens or +KEK. Producer cleanup removes only exact planned label-owned registry and +extraction containers and checks absence. It also requires the actual adapter +completion/cleanup files and queries that run's labels for container/network +survivors. The adapter owns native/kernel cleanup and its graph cleanup. + +Each command starts a new process session. Its group stays in the ownership +map through success, nonzero exit, +timeout and cancellation. TERM, then KILL and bounded actual group-absence +checks drain descendants even after the leader exits or closes its streams. +Only ESRCH proves group absence; inaccessible groups remain unresolved. A group +that remains fails cleanup, with its ownership and private event log retained. +The adapter keeps its240s graceful cleanup allowance. + +No global prune or foreign process/resource removal occurs. A missing engine +query or survivor is failure. A hard runner termination needs operator recovery +of its exact run-owned resources and private evidence; no source claim can +prove cleanup after loss of the machine. + +Primary implementation references: + +- [Docker buildx build](https://docs.docker.com/reference/cli/docker/buildx/build/) +- [Docker file export](https://docs.docker.com/build/building/export/) +- [Skopeo copy](https://github.com/containers/skopeo/blob/main/docs/skopeo-copy.1.md) +- [OCI destination media types](https://github.com/containers/image/blob/main/oci/layout/oci_dest.go) +- [Container transports](https://github.com/containers/image/blob/main/docs/containers-transports.5.md) +- [Bun global configuration](https://bun.com/docs/runtime/bunfig) +- [Bun npm configuration](https://bun.com/docs/pm/npmrc) + +This source is runnable only after operator provisioning and reviewed +composition/Inputs integration. No local image build, engine call, container, +service, database, helper, VM, physical matrix or remote CI run is established +by the producer's source tests. diff --git a/tests/fixtures/mcp-stack/platform/ci/linux-assets.Dockerfile b/tests/fixtures/mcp-stack/platform/ci/linux-assets.Dockerfile new file mode 100644 index 000000000..93b0b8c8b --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/ci/linux-assets.Dockerfile @@ -0,0 +1,24 @@ +# Test model CLI only. Harnesses, agentd and bundle leaves are production source. +FROM rust:1.99-alpine@sha256:7e9e2d068a107a2adafd19930011178d14a78326053cd4c3a9f7ad3fd049557d AS build +RUN apk add --no-cache build-base protobuf-dev clang-dev openssl-dev openssl-libs-static perl linux-headers bash curl squashfs-tools +WORKDIR /src +COPY . . +ARG TARGETARCH +ENV SQLX_OFFLINE=true \ + CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=gcc \ + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=gcc \ + CC_x86_64_unknown_linux_musl=gcc \ + CC_aarch64_unknown_linux_musl=gcc \ + OPENSSL_STATIC=1 +RUN set -eu; case "$TARGETARCH" in amd64) target=x86_64-unknown-linux-musl ;; arm64) target=aarch64-unknown-linux-musl ;; *) exit 2 ;; esac; \ + cargo build --locked --release --target "$target" -p engram-agentd -p engram-harness-claude -p engram-harness-codex; \ + cargo build --locked --release --target "$target" --manifest-path tests/fixtures/mcp-model-cli/Cargo.toml --bin engram-mcp-model-fixture; \ + mkdir /out; \ + cp "target/$target/release/engram-harness-claude" /out/claudeHarness; \ + cp "target/$target/release/engram-harness-codex" /out/codexHarness; \ + cp "tests/fixtures/mcp-model-cli/target/$target/release/engram-mcp-model-fixture" /out/modelCli; \ + deploy/bundles/agentd/build.sh "target/$target/release/engram-agentd" /out/agentdBundle; \ + deploy/bundles/sentinel/build.sh /out/sentinelBundle; \ + deploy/bundles/guest-tools/build.sh /out/guestToolsBundle "$TARGETARCH" +FROM scratch +COPY --from=build /out/ / diff --git a/tests/fixtures/mcp-stack/platform/ci/upstream-images.json b/tests/fixtures/mcp-stack/platform/ci/upstream-images.json new file mode 100644 index 000000000..7a1df4eea --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/ci/upstream-images.json @@ -0,0 +1,32 @@ +{ + "postgres": { + "repository": "docker.io/library/postgres", + "version": "17.6-bookworm", + "amd64": "sha256:45cd22f8d32e189d245403954882f88e7a8714301fda80dab6da90f1265b25a3", + "arm64": "sha256:5d11ffb37e58a7c9a2285359e50f7674e216c99b9114e47b0e7f21187c11252c" + }, + "registry": { + "repository": "docker.io/library/registry", + "version": "2.8.3", + "amd64": "sha256:46faa9a1ae6813194b53921a370f2f4f8c5e1aae228a89bceafef5847a6a3278", + "arm64": "sha256:fa647fc1e5d5df7d8d923fb6332aab8e78783f8fca1a1394efb4011f68f5a793" + }, + "edge": { + "repository": "docker.io/library/nginx", + "version": "1.28.0-bookworm", + "amd64": "sha256:3b03dfd827515a07ccf4946000f0bbdd2a4fb6aca7499249fcb6b1f9726823d6", + "arm64": "sha256:b869934f513dcd08fc9de959e0bf6175e47ff3553a9c127afce9969722304356" + }, + "blob": { + "repository": "docker.io/fsouza/fake-gcs-server", + "version": "1.52.3", + "amd64": "sha256:29e8b9db5339f370d999dd8ac57986b0b563614dde7ddfcd4e379e4aa3c4e528", + "arm64": "sha256:a3d621d29177b5fc75d297f3ff666576d433340243167d85c2738b75b6e5527a" + }, + "dns": { + "repository": "docker.io/coredns/coredns", + "version": "1.12.3", + "amd64": "sha256:f0f40f86da53ab64fd07a0a4fd337d7707db446a7c94dd5565c2cad93eb55876", + "arm64": "sha256:f23708142ce31de03565029732bdae87a10bd0ca5edad61f554be511df819a98" + } +} diff --git a/tests/fixtures/mcp-stack/platform/composition.test.ts b/tests/fixtures/mcp-stack/platform/composition.test.ts new file mode 100644 index 000000000..68e60e507 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/composition.test.ts @@ -0,0 +1,4479 @@ +import { poolFixture } from "./fc-thin-fixture.ts"; +import { test, expect, spyOn } from "bun:test"; +import { + mkdtemp, + mkdir, + chmod, + writeFile, + readFile, + realpath, + stat, + rm, +} from "node:fs/promises"; +import { createServer } from "node:net"; +import { join } from "node:path"; +import { Engine } from "./engine.ts"; +import { hash, PlatformFailure, put, type Pin } from "./owned.ts"; +import { BINARIES, IMAGES, parseInputs, type Inputs } from "./preflight.ts"; + +test("engine operations pin socket/config and exclude parent credentials", async () => { + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/engine-"), + ); + const config = join(root, "config"); + await mkdir(config, { mode: 0o700 }); + const socket = join(root, "engine.sock"); + const server = createServer(); + await new Promise((r) => server.listen(socket, r)); + const script = join(root, "docker"); + const output = join(root, "args.json"); + await writeFile( + script, + `#!/bin/sh\nexec ${JSON.stringify(process.execPath)} -e 'require("fs").writeFileSync(${JSON.stringify(output)},JSON.stringify({args:process.argv.slice(1),env:process.env}));process.stdout.write(JSON.stringify({ID:"engine-identity-123"}))' -- "$@"\n`, + { mode: 0o700 }, + ); + const inputs = { + engineSocket: socket, + engineConfig: config, + engineId: "engine-identity-123", + binaries: { + docker: { path: script, sha256: hash(await readFile(script)) }, + }, + } as Inputs; + const old = process.env.AWS_SECRET_ACCESS_KEY; + process.env.AWS_SECRET_ACCESS_KEY = "forbidden-parent-value"; + try { + const engine = new Engine(inputs, root, "mcp-123456789012345678901234"); + await engine.call(["info", "--format", "{{json .}}"]); + const got = JSON.parse(await readFile(output, "utf8")); + expect(got.args).toEqual([ + "--config", + config, + "--host", + `unix://${socket}`, + "info", + "--format", + "{{json .}}", + ]); + expect(got.env.AWS_SECRET_ACCESS_KEY).toBeUndefined(); + expect(got.env.HOME).toBeUndefined(); + expect(got.env.DOCKER_HOST).toBeUndefined(); + await writeFile(join(config, "config.json"), "{}"); + await expect(engine.call(["info"])).rejects.toThrow( + "engine_config_not_empty", + ); + } finally { + if (old === undefined) delete process.env.AWS_SECRET_ACCESS_KEY; + else process.env.AWS_SECRET_ACCESS_KEY = old; + await new Promise((r) => server.close(() => r())); + await rm(root, { recursive: true }); + } +}); + +test("private cleanup logs capture stderr with shared bounds and owned cancellation", async () => { + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/private-logs-"), + ); + const config = join(root, "config"); + await mkdir(config, { mode: 0o700 }); + const socket = join(root, "engine.sock"); + const server = createServer(); + await new Promise((resolve) => server.listen(socket, resolve)); + const script = join(root, "docker"), + started = join(root, "started"), + descendant = join(root, "descendant"); + await writeFile( + script, + `#!${Bun.which("node")} +const fs = require("node:fs"); +const args = process.argv.slice(2 + 4); +const mode = args[0] === "fail" ? "d" : args[0] === "logs" ? args[3]?.[0] : "a"; +if (mode !== "e") fs.writeFileSync(${JSON.stringify(started)}, String(process.pid)); +if (mode === "b") { + process.stderr.write(Buffer.alloc(4 * 1024 * 1024 + 1, 120)); +} else if (mode === "c") { + process.stdout.write(Buffer.alloc(2 * 1024 * 1024 + 1, 120)); + process.stderr.write(Buffer.alloc(2 * 1024 * 1024, 120)); +} else if (mode === "e") { + const child = require("node:child_process").spawn("/bin/sleep", ["30"]); + fs.writeFileSync(${JSON.stringify(descendant)}, String(child.pid)); + process.on("SIGTERM", () => child.kill("SIGTERM")); + child.on("exit", () => process.exit(0)); + process.stderr.write("private-stderr-marker"); + setInterval(() => process.stderr.write("private-stderr-marker"), 10); +} else { + process.stdout.write("private-stdout-marker"); + process.stderr.write("private-stderr-marker"); + if (mode === "d") process.exitCode = 1; +} +if (mode === "e") fs.writeFileSync(${JSON.stringify(started)}, String(process.pid)); +`, + { mode: 0o700 }, + ); + const inputs = { + engineSocket: socket, + engineConfig: config, + binaries: { + docker: { path: script, sha256: hash(await readFile(script)) }, + }, + } as Inputs; + const controller = new AbortController(); + try { + const engine = new Engine(inputs, root, "mcp-123456789012345678901234"); + const logs = (mode: string) => ["logs", "--tail", "10000", mode.repeat(64)]; + const bytes = await engine.call(logs("a")); + const artifact = join(root, "retained-probe.log"); + await put(artifact, bytes); + const retained = await readFile(artifact, "utf8"); + expect(retained).toContain("private-stdout-marker"); + expect(retained).toContain("private-stderr-marker"); + expect((await stat(artifact)).mode & 0o777).toBe(0o600); + for (const args of [ + ["info"], + ["exec", "a".repeat(64), "probe"], + ["logs", "--tail", "10000", "A".repeat(64)], + ["logs", "--tail", "9999", "a".repeat(64)], + [...logs("a"), "extra"], + ]) + expect((await engine.call(args)).toString()).toBe( + "private-stdout-marker", + ); + for (const args of [logs("b"), logs("c"), logs("d"), ["fail"], logs("e")]) { + await expect( + engine.call(args, undefined, args[3]?.[0] === "e" ? 500 : 2000), + ).rejects.toThrow(/^INVALID_CONTEXT:engine_command$/); + const pid = Number(await readFile(started, "utf8")); + expect(() => process.kill(pid, 0)).toThrow(); + expect(() => process.kill(-pid, 0)).toThrow(); + } + const deadlineChild = Number(await readFile(descendant, "utf8")); + expect(() => process.kill(deadlineChild, 0)).toThrow(); + await rm(started); + const running = engine.controlled(controller.signal, () => + engine.call(logs("e"), undefined, 30000), + ); + const outcome = running.then( + () => undefined, + (error: unknown) => error, + ); + const deadline = performance.now() + 2000; + while (!(await readFile(started).catch(() => undefined))) { + if (performance.now() >= deadline) + throw new Error("controlled child did not start"); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + const pid = Number(await readFile(started, "utf8")); + controller.abort(); + const error = await outcome; + expect(error).toBeInstanceOf(PlatformFailure); + expect(String(error)).toBe("Error: INVALID_CONTEXT:engine_command"); + expect(String(error)).not.toContain("private-stderr-marker"); + expect(() => process.kill(pid, 0)).toThrow(); + expect(() => process.kill(-pid, 0)).toThrow(); + const childPid = Number(await readFile(descendant, "utf8")); + expect(() => process.kill(childPid, 0)).toThrow(); + await expect( + engine.controlled(new AbortController().signal, () => + engine.call(logs("a")), + ), + ).resolves.toHaveLength(bytes.length); + } finally { + controller.abort(); + await new Promise((resolve) => server.close(() => resolve())); + await rm(root, { recursive: true }); + } +}); + +test("guest manifest identity is mandatory and cannot alias the config id by omission", async () => { + const { parseInputs, BINARIES, IMAGES } = await import("./preflight.ts"); + const pin = { path: "/owned/binary", sha256: "a".repeat(64) }; + const value = { + version: 1, + backend: "vz", + proxy: pin, + nativeProbe: pin, + architecture: "arm64", + sourceCommit: "b".repeat(40), + engineSocket: "/owned/socket", + engineConfig: "/owned/config", + engineId: "engine-identity-123", + registryPort: 32000, + subnet: "10.42.3.0/24", + nativeHostIp: "10.42.4.1", + binaries: Object.fromEntries(BINARIES.map((k) => [k, pin])), + images: Object.fromEntries( + IMAGES.map((k) => [ + k, + { + archive: pin, + id: `sha256:${"c".repeat(64)}`, + manifestDigest: `sha256:${"d".repeat(64)}`, + }, + ]), + ), + }; + expect(parseInputs(value, "vz").images.guest.manifestDigest).toBe( + `sha256:${"d".repeat(64)}`, + ); + expect(() => + parseInputs( + { + ...value, + images: { + ...value.images, + guest: { archive: pin, id: `sha256:${"c".repeat(64)}` }, + }, + }, + "vz", + ), + ).toThrow(); + expect(() => + parseInputs( + { ...value, images: { ...value.images, hostAgent: value.images.guest } }, + "vz", + ), + ).toThrow(); +}); + +test("native ownership rejects foreign executables, UID and incomplete birth identity", async () => { + const { parseNativeIdentities } = await import("./launch.ts"); + const base = { + pid: 1234, + uid: process.getuid!(), + parent: 12, + group: 12, + seconds: "1730000000", + microseconds: "123456", + executable: "/owned/run/assets/hostAgent", + }; + expect(parseNativeIdentities([base], "/owned/run")[0]?.microseconds).toBe( + "123456", + ); + for (const change of [ + { uid: base.uid + 1 }, + { microseconds: "1000000" }, + { seconds: 1730000000 }, + { executable: "/foreign/hostAgent" }, + { executable: "/owned/run/host/../../foreign/netstack" }, + { pid: [1234] }, + ]) + expect(() => + parseNativeIdentities([{ ...base, ...change }], "/owned/run"), + ).toThrow(); +}); + +// Replace the native inventory and signal boundaries. No host or VM is started. +async function nativeCleanupFixture() { + const { Stack, parseNativeIdentities } = await import("./launch.ts"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/native-cleanup-"), + ); + const stack = new Stack( + { backend: "vz" } as Inputs, + root, + {} as ConstructorParameters[2], + { + binaries: { hostAgent: { path: join(root, "assets/hostAgent") } }, + } as ConstructorParameters[3], + {} as ConstructorParameters[4], + {} as ConstructorParameters[5], + ); + const leader = { + pid: 41001, + uid: process.getuid!(), + parent: 1, + group: 41001, + seconds: "1730000000", + microseconds: "123456", + executable: join(root, "assets/hostAgent"), + }; + let identities = [ + leader, + { ...leader, pid: 41002, executable: join(root, "assets/proxy") }, + ]; + let alive = true; + let finish!: () => void; + const events: string[] = []; + const done = new Promise((resolve) => { + finish = resolve; + }); + const complete = () => { + alive = false; + identities = identities.filter((p) => p.pid !== leader.pid); + events.push("host-complete"); + finish(); + }; + Reflect.set(stack, "host", { + child: { pid: leader.pid }, + running: () => alive, + done, + stop: () => { + throw new Error("group stop must not run"); + }, + }); + Reflect.set(stack, "nativeIdentities", new Map([[leader.pid, leader]])); + let scan = () => identities; + Reflect.set(stack, "nativeProcesses", async () => + parseNativeIdentities(scan(), root), + ); + const actualKill = process.kill.bind(process); + const kill = spyOn(process, "kill").mockImplementation((pid, signal) => { + if (pid < 0) return actualKill(pid, signal); + expect(pid).toBeGreaterThan(1); + events.push(`${pid}:${signal}`); + if (pid === leader.pid) complete(); + else identities = identities.filter((p) => p.pid !== pid); + return true; + }); + return { + stack, + leader, + events, + complete, + alive: () => alive, + identities: () => identities, + setIdentities: (next: typeof identities) => { + identities = next; + }, + setScan: (next: typeof scan) => { + scan = next; + }, + kill, + close: async () => { + kill.mockRestore(); + await rm(root, { recursive: true }); + }, + }; +} + +test("native cleanup waits for host completion before scanning and signaling replacement proxies", async () => { + const fixture = await nativeCleanupFixture(); + let timer: ReturnType | undefined; + fixture.kill.mockImplementation((pid, signal) => { + fixture.events.push(`${pid}:${signal}`); + if (pid === fixture.leader.pid) { + // The supervisor replaces its proxy while the leader still exits. + fixture.setIdentities([ + fixture.leader, + { + ...fixture.identities()[1]!, + pid: 41003, + }, + ]); + timer = setTimeout(fixture.complete, 5); + } else { + fixture.events.push( + fixture.alive() ? "proxy-before-completion" : "proxy-after-completion", + ); + fixture.setIdentities(fixture.identities().filter((p) => p.pid !== pid)); + } + return true; + }); + try { + await fixture.stack.cleanupHost(); + expect(fixture.events).toEqual([ + "41001:SIGTERM", + "host-complete", + "41003:SIGTERM", + "proxy-after-completion", + ]); + expect(fixture.identities()).toEqual([]); + } finally { + if (timer) clearTimeout(timer); + await fixture.close(); + } +}); + +test("native cleanup refuses a host inventory without its retained handle", async () => { + const fixture = await nativeCleanupFixture(); + Reflect.set(fixture.stack, "host", undefined); + try { + await expect(fixture.stack.cleanupHost()).rejects.toThrow( + "native_host_handle_missing", + ); + expect(fixture.events).toEqual([]); + } finally { + await fixture.close(); + } +}); + +test("native cleanup does not adopt a reused PID after an uncaptured host exits", async () => { + const fixture = await nativeCleanupFixture(); + fixture.complete(); + fixture.events.length = 0; + Reflect.set(fixture.stack, "nativeIdentities", new Map()); + fixture.setIdentities([ + { ...fixture.leader, seconds: "1730000001" }, + fixture.identities()[0]!, + ]); + try { + await expect(fixture.stack.cleanupHost()).rejects.toThrow( + "native_host_survivor", + ); + expect(fixture.events).toEqual([]); + } finally { + await fixture.close(); + } +}); + +test("native cleanup removes remaining proxies when no host exists", async () => { + const fixture = await nativeCleanupFixture(); + Reflect.set(fixture.stack, "host", undefined); + fixture.setIdentities( + fixture.identities().filter((p) => p.pid !== fixture.leader.pid), + ); + try { + await fixture.stack.cleanupHost(); + expect(fixture.events).toEqual(["41002:SIGTERM"]); + expect(fixture.identities()).toEqual([]); + } finally { + await fixture.close(); + } +}); + +test("native cleanup kills a TERM-resistant host before signaling its proxy", async () => { + const fixture = await nativeCleanupFixture(); + fixture.kill.mockImplementation((pid, signal) => { + fixture.events.push(`${pid}:${signal}`); + if (pid === fixture.leader.pid) { + if (signal === "SIGKILL") fixture.complete(); + } else { + fixture.events.push( + fixture.alive() ? "proxy-before-completion" : "proxy-after-completion", + ); + fixture.setIdentities(fixture.identities().filter((p) => p.pid !== pid)); + } + return true; + }); + try { + await fixture.stack.cleanupHost(); + expect(fixture.events).toEqual([ + "41001:SIGTERM", + "41001:SIGKILL", + "host-complete", + "41002:SIGTERM", + "proxy-after-completion", + ]); + } finally { + await fixture.close(); + } +}); + +for (const change of [ + { seconds: "1730000001" }, + { executable: "/foreign/hostAgent" }, +]) { + test(`native cleanup sends no signal when the retained host changes (${Object.keys(change)[0]})`, async () => { + const fixture = await nativeCleanupFixture(); + let scans = 0; + fixture.setScan(() => + ++scans === 1 + ? fixture.identities() + : [{ ...fixture.leader, ...change }, fixture.identities()[1]!], + ); + try { + await expect(fixture.stack.cleanupHost()).rejects.toThrow(); + expect(fixture.events).toEqual([]); + } finally { + await fixture.close(); + } + }); +} + +test("cancelled engine control runs no command and drains its control lease", async () => { + const engine = new Engine( + {} as Inputs, + "/owned/run", + "mcp-123456789012345678901234", + ); + const abort = new AbortController(); + abort.abort(); + let called = false; + await expect( + engine.controlled(abort.signal, async () => { + called = true; + }), + ).rejects.toThrow(); + expect(called).toBe(false); + await engine.controlled(new AbortController().signal, async () => { + called = true; + }); + expect(called).toBe(true); +}); + +function inspectedContainer( + imageRole: (typeof IMAGES)[number] = "worker", + firecrackerPool?: string, +) { + const pin = { path: "/owned/binary", sha256: "a".repeat(64) }; + const inputs = parseInputs( + { + version: 1, + backend: firecrackerPool ? "firecracker" : "vz", + architecture: firecrackerPool ? "amd64" : "arm64", + sourceCommit: "b".repeat(40), + engineSocket: "/owned/socket", + engineConfig: "/owned/config", + engineId: "engine-identity-123", + registryPort: 32000, + subnet: "10.42.3.0/24", + nativeHostIp: "10.42.4.1", + proxy: pin, + ...(firecrackerPool + ? { + firecracker: pin, + thinPool: { name: "suite", directory: firecrackerPool }, + } + : { nativeProbe: pin }), + binaries: Object.fromEntries(BINARIES.map((role) => [role, pin])), + images: { + ...Object.fromEntries( + IMAGES.map((role, index) => [ + role, + { + archive: pin, + id: `sha256:${index.toString(16).repeat(64)}`, + manifestDigest: `sha256:${"d".repeat(64)}`, + }, + ]), + ), + ...(firecrackerPool + ? { + hostAgent: { + archive: pin, + id: `sha256:${"c".repeat(64)}`, + manifestDigest: `sha256:${"d".repeat(64)}`, + }, + } + : {}), + }, + }, + firecrackerPool ? "firecracker" : "vz", + ); + const id = "a".repeat(64), + image = firecrackerPool + ? inputs.images.hostAgent!.id + : inputs.images[imageRole].id, + runId = "mcp-123456789012345678901234"; + const state = { + Id: id, + AppArmorProfile: "", + Image: image, + Config: { + Labels: { "engrams.mcp.run": runId }, + User: "1000:1000", + Env: ["PATH=/usr/bin:/bin"], + Entrypoint: ["/bin/sleep"], + Cmd: ["infinity"], + }, + HostConfig: { + RestartPolicy: { Name: "no" }, + Privileged: false, + ReadonlyRootfs: true, + PidsLimit: 128, + Memory: 805306368, + NanoCpus: 2000000000, + NetworkMode: "none", + PidMode: "", + IpcMode: "private", + CgroupnsMode: "private", + CapDrop: ["ALL"], + CapAdd: Array(), + SecurityOpt: ["no-new-privileges"], + MaskedPaths: ["/proc/kcore"], + ReadonlyPaths: ["/proc/sys"], + Binds: [], + VolumesFrom: [], + Devices: [], + DeviceRequests: [], + Tmpfs: { "/tmp": "rw,nosuid,nodev,size=4m" }, + }, + Mounts: [], + State: { Running: true, Pid: 42 }, + }; + const imageState = { + Id: image, + Config: { + Env: ["PATH=/usr/bin:/bin"], + Entrypoint: ["/bin/sleep"], + Cmd: ["infinity"], + }, + }; + class Inspector extends Engine { + override async call(args: string[]): Promise { + if (args[0] === "create") return Buffer.from(id); + if (args[0] === "container" && args[1] === "inspect") + return Buffer.from(JSON.stringify([state])); + if (args[0] === "image") return Buffer.from(JSON.stringify([imageState])); + if (args[0] === "start") return Buffer.from(id); + throw new Error("unexpected fixed test operation"); + } + } + const engine = new Inspector(inputs, "/owned/run", runId); + const options = [ + "--network", + "none", + "--user", + "1000:1000", + "--read-only", + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=4m", + ]; + return { engine, state, imageState, id, image, options }; +} + +async function inspectedHost(firecrackerPool?: string) { + const fixture = inspectedContainer("worker", firecrackerPool); + const { engine, state, image, options } = fixture; + Object.assign(state.HostConfig, { + PidsLimit: 1024, + Memory: 8589934592, + NanoCpus: 4000000000, + PidMode: firecrackerPool ? "host" : "", + CapAdd: ["CAP_NET_ADMIN", "CAP_SYS_ADMIN", "CAP_MKNOD"], + }); + if (firecrackerPool) { + state.AppArmorProfile = "unconfined"; + state.HostConfig.SecurityOpt = ["no-new-privileges", "apparmor=unconfined"]; + state.HostConfig.MaskedPaths = []; + state.HostConfig.ReadonlyPaths = []; + state.HostConfig.CapAdd.push("CAP_NET_BIND_SERVICE"); + } + state.Config.User = "0:0"; + options[options.indexOf("--user") + 1] = "0:0"; + let createdOptions: string[] = []; + const call = engine.call.bind(engine); + const intercepted = spyOn(engine, "call").mockImplementation(async (args) => { + if (args[0] === "create") { + createdOptions = args.flatMap((arg, i) => + arg === "--security-opt" ? [args[i + 1]!] : [], + ); + state.Config.Entrypoint = [args[args.indexOf("--entrypoint") + 1]!]; + state.Config.Cmd = args.slice(args.indexOf(image) + 1); + } + if (args[0] === "start") throw Error("test stop before start"); + return call(args); + }); + try { + await expect( + engine.create("host-agent", image, [ + ...options, + ...(firecrackerPool ? ["--pid", "host"] : []), + "--cgroupns", + "private", + ...[ + "NET_ADMIN", + "SYS_ADMIN", + "MKNOD", + ...(firecrackerPool ? ["NET_BIND_SERVICE"] : []), + ].flatMap((capability) => ["--cap-add", capability]), + ]), + ).rejects.toThrow("test stop before start"); + } finally { + intercepted.mockRestore(); + } + expect(createdOptions).toEqual( + firecrackerPool + ? ["no-new-privileges", "apparmor=unconfined", "systempaths=unconfined"] + : ["no-new-privileges"], + ); + return fixture; +} + +test("FC host PID audit accepts only the exact managed namespace posture", async () => { + const directory = await realpath( + await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/host-label-"), + ), + ); + try { + const { engine, state, id } = await inspectedHost(directory); + const required = ["no-new-privileges", "apparmor=unconfined"]; + for (const options of [ + required, + [...required, "label=disable"], + ["label=disable", ...required.slice().reverse()], + ]) { + state.HostConfig.SecurityOpt = options; + expect(await engine.audit(id)).toBeDefined(); + } + for (const options of [ + [...required, "systempaths=unconfined"], + [...required, "systempaths=other"], + [...required, "seccomp=unconfined"], + [...required, "apparmor=other"], + [...required, "label=disable", "label=disable"], + [], + ["no-new-privileges"], + ["apparmor=unconfined"], + ["no-new-privileges", "label=disable"], + ["no-new-privileges", "apparmor=unconfined", "apparmor=unconfined"], + [ + "no-new-privileges", + "apparmor=unconfined", + "label=disable", + "label=disable", + ], + ["no-new-privileges", "apparmor=unconfined", "apparmor=other"], + ["no-new-privileges", "apparmor=unconfined", "seccomp=unconfined"], + ["no-new-privileges", "apparmor=unconfined", "label=type:container_t"], + ]) { + state.HostConfig.SecurityOpt = options; + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + } + state.HostConfig.SecurityOpt = [...required, "label=disable"]; + for (const paths of ["MaskedPaths", "ReadonlyPaths"] as const) { + state.HostConfig[paths] = ["/proc/sys"]; + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + state.HostConfig[paths] = []; + } + const exactCapabilities = [...state.HostConfig.CapAdd]; + for (const capabilities of [ + exactCapabilities.filter((value) => value !== "CAP_NET_BIND_SERVICE"), + [...exactCapabilities, "CAP_NET_BIND_SERVICE"], + [...exactCapabilities, "CAP_SYS_PTRACE"], + ]) { + state.HostConfig.CapAdd = capabilities; + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + } + state.HostConfig.CapAdd = exactCapabilities; + for (const profile of ["", "docker-default", "other"]) { + state.AppArmorProfile = profile; + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + } + state.AppArmorProfile = "unconfined"; + state.HostConfig.CapAdd.push("CAP_SYS_PTRACE"); + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + state.HostConfig.CapAdd.pop(); + state.HostConfig.Privileged = true; + await expect(engine.audit(id)).rejects.toThrow("container_resources"); + state.HostConfig.Privileged = false; + state.HostConfig.PidMode = ""; + await expect(engine.audit(id)).rejects.toThrow("container_namespace"); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test("ordinary containers and VZ hosts reject Docker's host PID label option", async () => { + const directory = await realpath( + await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/nonhost-label-"), + ), + ); + try { + const ordinary = inspectedContainer("worker", directory); + await ordinary.engine.create("probe", ordinary.image, ordinary.options); + const vz = await inspectedHost(); + for (const { engine, state, id } of [ordinary, vz]) { + for (const options of [ + ["no-new-privileges", "label=disable"], + ["no-new-privileges", "systempaths=unconfined"], + ["no-new-privileges", "apparmor=unconfined", "systempaths=unconfined"], + ["no-new-privileges", "apparmor=unconfined"], + ["no-new-privileges", "apparmor=unconfined", "label=disable"], + ]) { + state.HostConfig.SecurityOpt = options; + await expect(engine.audit(id)).rejects.toThrow( + "container_capabilities", + ); + } + state.HostConfig.SecurityOpt = ["no-new-privileges"]; + for (const paths of ["MaskedPaths", "ReadonlyPaths"] as const) { + const protectedPaths = state.HostConfig[paths]; + state.HostConfig[paths] = []; + await expect(engine.audit(id)).rejects.toThrow( + "container_capabilities", + ); + state.HostConfig[paths] = protectedPaths; + } + state.HostConfig.CapAdd.push("CAP_NET_BIND_SERVICE"); + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + state.HostConfig.CapAdd.pop(); + state.AppArmorProfile = "unconfined"; + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + } + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +const pinnedImageDefaults = [ + { + imageRole: "guard", + entry: "HOME=/tmp", + wrong: ["HOME=/root", "HOME=/private/home"], + }, + { + imageRole: "orchestrator", + entry: "BUN_INSTALL_BIN=/usr/local/bin", + wrong: ["BUN_INSTALL_BIN=/tmp/bin"], + }, + { + imageRole: "orchestrator", + entry: "BUN_RUNTIME_TRANSPILER_CACHE_PATH=0", + wrong: [ + "BUN_RUNTIME_TRANSPILER_CACHE_PATH=/tmp/cache", + "BUN_RUNTIME_TRANSPILER_CACHE_PATH=1", + ], + }, + { + imageRole: "dns", + entry: "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt", + wrong: ["SSL_CERT_FILE=/tmp/ca.pem"], + }, +] satisfies { + imageRole: (typeof IMAGES)[number]; + entry: string; + wrong: string[]; +}[]; + +test.each(pinnedImageDefaults)( + "pinned image default $entry passes its exact image audit", + async ({ imageRole, entry }) => { + const { engine, state, imageState, image, options, id } = + inspectedContainer(imageRole); + imageState.Config.Env.push(entry); + state.Config.Env.push(entry); + await engine.create("probe", image, options, ["infinity"], "/bin/sleep"); + await expect(engine.audit(id)).resolves.toBeDefined(); + state.Config.Env.push("DATABASE_URL=forbidden"); + await expect(engine.audit(id)).rejects.toThrow("container_environment"); + state.Config.Env.pop(); + state.Config.Env[1] = entry + "-changed"; + await expect(engine.audit(id)).rejects.toThrow("container_environment"); + }, +); + +test.each(pinnedImageDefaults)( + "pinned image default $entry rejects other images and values", + async ({ imageRole, entry, wrong }) => { + for (const [role, value] of [ + ["worker", entry], + ...wrong.map((value): [(typeof IMAGES)[number], string] => [ + imageRole, + value, + ]), + [imageRole, "AWS_SECRET_ACCESS_KEY=forbidden"], + [imageRole, "HTTPS_PROXY=http://proxy.invalid"], + ] satisfies [(typeof IMAGES)[number], string][]) { + const { engine, state, imageState, image, options } = + inspectedContainer(role); + imageState.Config.Env.push(value); + state.Config.Env.push(value); + if (imageRole === "dns") { + state.HostConfig.CapAdd = ["NET_BIND_SERVICE"]; + options.push("--cap-add", "NET_BIND_SERVICE"); + } + await expect( + engine.create(imageRole, image, options, ["infinity"], "/bin/sleep"), + ).rejects.toThrow("image_environment"); + } + }, +); + +test.each([ + "tool-check", + "pack-claude", + "pack-codex", + "probe", + "host-tls-probe", +])("pinned image default HOME is available to guard reuse %s", async (role) => { + const { engine, state, imageState, image, options } = + inspectedContainer("guard"); + imageState.Config.Env.push("HOME=/tmp"); + state.Config.Env.push("HOME=/tmp"); + await engine.create(role, image, options, ["infinity"], "/bin/sleep"); +}); + +test.each(["dns", "denied-dns"])( + "pinned image default certificate is available to DNS reuse %s", + async (role) => { + const { engine, state, imageState, image, options } = + inspectedContainer("dns"); + imageState.Config.Env.push( + "SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt", + ); + state.Config.Env.push("SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt"); + state.HostConfig.CapAdd = ["NET_BIND_SERVICE"]; + await engine.create( + role, + image, + [...options, "--cap-add", "NET_BIND_SERVICE"], + ["infinity"], + "/bin/sleep", + ); + }, +); + +test("pinned image defaults allow both Bun entries together but preserve image identity", async () => { + const { engine, state, imageState, image, options, id } = + inspectedContainer("orchestrator"); + const defaults = pinnedImageDefaults + .filter(({ imageRole }) => imageRole === "orchestrator") + .map(({ entry }) => entry); + imageState.Config.Env.push(...defaults); + state.Config.Env.push(...defaults); + await engine.create("probe", image, options, ["infinity"], "/bin/sleep"); + imageState.Id = "sha256:" + "f".repeat(64); + await expect(engine.audit(id)).rejects.toThrow("container_image"); +}); + +test("resolver-stage compares equivalent capability names exactly", async () => { + const { engine, state, id, image, options } = inspectedContainer(); + const requested = ["CHOWN", "DAC_OVERRIDE", "FOWNER"]; + state.HostConfig.CapAdd = requested.map((capability) => "CAP_" + capability); + await engine.create( + "resolver-stage", + image, + [ + ...options, + ...requested.flatMap((capability) => ["--cap-add", capability]), + ], + ["infinity"], + "/bin/sleep", + ); + state.HostConfig.CapAdd = requested; + await expect(engine.audit(id)).resolves.toBeDefined(); + for (const capabilities of [ + ["CAP_CHOWN", "CAP_DAC_OVERRIDE"], + ["CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_NET_ADMIN"], + [...requested, "CAP_NET_ADMIN"], + [...requested, "CAP_FOWNER"], + ["CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_FOWNER", "CAP_FOWNER"], + ["CAP_CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_FOWNER"], + ]) { + state.HostConfig.CapAdd = capabilities; + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + } + state.HostConfig.CapAdd = requested.map((capability) => "CAP_" + capability); + for (const dropped of [ + [], + ["CAP_ALL"], + ["ALL", "NET_ADMIN"], + ["ALL", "ALL"], + ]) { + state.HostConfig.CapDrop = dropped; + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + } + state.HostConfig.CapDrop = ["ALL"]; + for (const security of [ + [], + ["no-new-privileges:true"], + ["no-new-privileges", "seccomp=unconfined"], + ]) { + state.HostConfig.SecurityOpt = security; + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + } + state.HostConfig.SecurityOpt = ["no-new-privileges"]; + await expect(engine.audit(id)).resolves.toBeDefined(); +}); + +test.each(["dns", "denied-dns"])( + "%s requires exactly NET_BIND_SERVICE in its effective audit", + async (role) => { + const { engine, state, id, image, options } = inspectedContainer(); + state.HostConfig.CapAdd = ["CAP_NET_BIND_SERVICE"]; + await engine.create( + role, + image, + [...options, "--cap-add", "NET_BIND_SERVICE"], + ["infinity"], + "/bin/sleep", + ); + state.HostConfig.CapAdd = ["NET_BIND_SERVICE"]; + await expect(engine.audit(id)).resolves.toBeDefined(); + for (const capabilities of [ + [], + ["NET_ADMIN"], + ["NET_BIND_SERVICE", "NET_ADMIN"], + ["NET_BIND_SERVICE", "NET_BIND_SERVICE"], + ["CAP_NET_BIND_SERVICE", "CAP_NET_BIND_SERVICE"], + ["NET_BIND_SERVICE", "CAP_NET_BIND_SERVICE"], + ["CAP_CAP_NET_BIND_SERVICE"], + ]) { + state.HostConfig.CapAdd = capabilities; + await expect(engine.audit(id)).rejects.toThrow("container_capabilities"); + } + const missing = inspectedContainer(); + await expect( + missing.engine.create( + role, + missing.image, + missing.options, + ["infinity"], + "/bin/sleep", + ), + ).rejects.toThrow("container_capabilities"); + const extra = inspectedContainer(); + extra.state.HostConfig.CapAdd = ["NET_BIND_SERVICE", "NET_ADMIN"]; + await expect( + extra.engine.create( + role, + extra.image, + [ + ...extra.options, + "--cap-add", + "NET_BIND_SERVICE", + "--cap-add", + "NET_ADMIN", + ], + ["infinity"], + "/bin/sleep", + ), + ).rejects.toThrow("container_capabilities"); + }, +); + +test.each([ + "worker", + "orchestrator", + "coordinator", + "probe", + "dns-other", + "denied-dns-other", +])("%s cannot request the fixture DNS capability", async (role) => { + const { engine, state, image, options } = inspectedContainer(); + state.HostConfig.CapAdd = ["NET_BIND_SERVICE"]; + await expect( + engine.create( + role, + image, + [...options, "--cap-add", "NET_BIND_SERVICE"], + ["infinity"], + "/bin/sleep", + ), + ).rejects.toThrow("container_capabilities"); +}); + +test("effective container audit rejects added authority and environment", async () => { + const { engine, state, id, image } = inspectedContainer(); + await engine.create( + "probe", + image, + [ + "--network", + "none", + "--user", + "1000:1000", + "--read-only", + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=4m", + ], + ["infinity"], + "/bin/sleep", + ); + state.Config.Env.push("DATABASE_URL=forbidden"); + await expect(engine.audit(id)).rejects.toThrow("container_environment"); + state.Config.Env.pop(); + state.HostConfig.Privileged = true; + await expect(engine.audit(id)).rejects.toThrow("container_resources"); + state.HostConfig.Privileged = false; + state.Config.User = "0:0"; + await expect(engine.audit(id)).rejects.toThrow("container_namespace"); + state.Config.User = "1000:1000"; + Reflect.set(state.HostConfig, "PortBindings", { + "8443/tcp": [{ HostIp: "0.0.0.0", HostPort: "8443" }], + }); + await expect(engine.audit(id)).rejects.toThrow("container_exposure"); + Reflect.deleteProperty(state.HostConfig, "PortBindings"); + state.HostConfig.PidMode = "container:foreign"; + await expect(engine.audit(id)).rejects.toThrow("container_namespace"); + state.HostConfig.PidMode = "host"; + await expect(engine.audit(id)).rejects.toThrow("container_namespace"); + state.HostConfig.PidMode = ""; + Reflect.set(state.HostConfig, "Sysctls", { "net.ipv4.ip_forward": "1" }); + await expect(engine.audit(id)).rejects.toThrow("container_exposure"); + Reflect.deleteProperty(state.HostConfig, "Sysctls"); + Reflect.set(state.HostConfig, "CgroupnsMode", "host"); + await expect(engine.audit(id)).rejects.toThrow("container_namespace"); + Reflect.deleteProperty(state.HostConfig, "CgroupnsMode"); + await expect(engine.audit(id)).resolves.toBeDefined(); +}); + +test("platform output keeps only approved runner failure classifications", async () => { + const { platformFailureLabel } = await import("./main.ts"); + const { RunnerFailure } = + await import("../../../../orchestrator/scripts/mcp-stack/failure.ts"); + for (const [code, seam, label] of [ + [ + "INVALID_CONTEXT", + "owned_path", + "FAILED:runner_invalid_context_owned_path", + ], + [ + "INVALID_CONTEXT", + "private_file", + "FAILED:runner_invalid_context_private_file", + ], + [ + "INVALID_CONTEXT", + "request_route", + "FAILED:runner_invalid_context_request_route", + ], + [ + "INVALID_CONTEXT", + "request_size", + "FAILED:runner_invalid_context_request_size", + ], + [ + "ASSERTION_FAILED", + "observer_token", + "FAILED:runner_assertion_failed_observer_token", + ], + [ + "ASSERTION_FAILED", + "fixture_observer_status", + "FAILED:runner_assertion_failed_fixture_observer_status", + ], + [ + "ASSERTION_FAILED", + "fixture_identity", + "FAILED:runner_assertion_failed_fixture_identity", + ], + [ + "TRANSPORT_FAILED", + "https_request", + "FAILED:runner_transport_failed_https_request", + ], + ] as const) { + const error = new RunnerFailure(code, seam); + error.message = "private response body"; + error.stack = "private stack"; + expect(platformFailureLabel(error)).toBe(label); + } + const changedCode = new RunnerFailure("TRANSPORT_FAILED", "https_request"); + Reflect.set(changedCode, "code", "PRIVATE_CODE"); + for (const error of [ + changedCode, + new RunnerFailure("ASSERTION_FAILED", "https_request"), + new RunnerFailure("TRANSPORT_FAILED", "private_value"), + new RunnerFailure("TRANSPORT_FAILED", "https_request\nprivate_value"), + new Error("TRANSPORT_FAILED:https_request private_value"), + { code: "TRANSPORT_FAILED", seam: "https_request" }, + "private_value", + null, + ]) + expect(platformFailureLabel(error)).toBe("FAILED:platform_composition"); + expect( + platformFailureLabel( + new PlatformFailure("NEEDS_CONTEXT", "native_dns_lookup"), + ), + ).toBe("NEEDS_CONTEXT:native_dns_lookup"); +}); + +for (const kind of ["modern", "legacy"] as const) { + test(`fixture ${kind} readiness gives unknown errors a fixed phase label`, async () => { + const { Stack } = await import("./launch.ts"); + const { FixtureObserver } = + await import("../../../../orchestrator/scripts/mcp-stack/observer.ts"); + const { RunnerFailure } = + await import("../../../../orchestrator/scripts/mcp-stack/failure.ts"); + const inputs = { + images: { fixture: { id: "fixture-image" } }, + } as Inputs; + const ctx = { + protocolFixtures: { + modern: { + runRoot: "/owned/modern", + configuration: { path: "/owned/modern/config" }, + }, + legacy: { + runRoot: "/owned/legacy", + configuration: { path: "/owned/legacy/config" }, + }, + }, + } as import("../../../../orchestrator/scripts/mcp-stack/context.ts").Context; + const engine = new Engine(inputs, "/owned", "mcp-123456789012345678901234"); + engine.networkOptions = () => []; + engine.create = async (role) => role; + engine.nativeDns = async () => {}; + const stack = new Stack( + inputs, + "/owned", + {} as ConstructorParameters[2], + {} as ConstructorParameters[3], + {} as ConstructorParameters[4], + engine, + ); + for (const failure of [ + new Error("private path and response"), + new RunnerFailure("TRANSPORT_FAILED", "https_request"), + new PlatformFailure("INVALID_CONTEXT", "private_file"), + ]) { + let index = 0; + const ready = spyOn( + FixtureObserver.prototype, + "ready", + ).mockImplementation(async () => { + if (["modern", "legacy"][index++] === kind) throw failure; + }); + try { + const preparing = stack.fixtures(ctx); + if ( + failure instanceof RunnerFailure || + failure instanceof PlatformFailure + ) + await expect(preparing).rejects.toBe(failure); + else + await expect(preparing).rejects.toMatchObject({ + code: "FAILED", + seam: + kind === "modern" + ? "fixture_modern_readiness" + : "fixture_legacy_readiness", + }); + } finally { + ready.mockRestore(); + } + } + }); +} + +test("entrypoint rejects a foreign backend before creating output or invoking a matrix", async () => { + const { main, matrixArguments } = await import("./main.ts"); + const { BINARIES, IMAGES } = await import("./preflight.ts"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/entry-"), + ); + try { + const output = join(root, "new-run"), + manifest = join(root, "manifest.json"), + pin = { path: "/owned/binary", sha256: "a".repeat(64) }; + await writeFile( + manifest, + JSON.stringify({ + version: 1, + backend: "vz", + architecture: "arm64", + proxy: pin, + nativeProbe: pin, + sourceCommit: "b".repeat(40), + engineSocket: "/owned/engine.sock", + engineConfig: "/owned/config", + engineId: "engine-identity-123", + registryPort: 32000, + subnet: "10.42.3.0/24", + nativeHostIp: "10.42.4.1", + binaries: Object.fromEntries(BINARIES.map((k) => [k, pin])), + images: Object.fromEntries( + IMAGES.map((k) => [ + k, + { + archive: pin, + id: "sha256:" + "c".repeat(64), + manifestDigest: "sha256:" + "d".repeat(64), + }, + ]), + ), + }), + { mode: 0o600 }, + ); + await expect( + main([ + "--backend", + "firecracker", + "--run-root", + output, + "--artifacts", + manifest, + ]), + ).rejects.toThrow("platform_identity"); + await expect(readFile(join(output, "context.json"))).rejects.toThrow(); + expect(matrixArguments("/owned/context.json").slice(1)).toEqual([ + "--context", + "/owned/context.json", + "--matrix", + "all", + ]); + } finally { + await rm(root, { recursive: true }); + } +}); + +for (const failedLookup of [false, true]) { + test(`real startup prefix checks DNS after dependencies and stops on failure (${failedLookup})`, async () => { + const { bootPlatform } = await import("./main.ts"); + const { Stack } = await import("./launch.ts"); + const { FixtureObserver } = + await import("../../../../orchestrator/scripts/mcp-stack/observer.ts"); + const inputs = { + backend: "vz", + images: { fixture: { id: "fixture-image" } }, + } as Inputs; + const ctx = { + protocolFixtures: { + modern: { + runRoot: "/owned/modern", + configuration: { path: "/owned/modern/config" }, + }, + legacy: { + runRoot: "/owned/legacy", + configuration: { path: "/owned/legacy/config" }, + }, + }, + } as import("../../../../orchestrator/scripts/mcp-stack/context.ts").Context; + const steps: string[] = []; + const step = async (name: string) => { + steps.push(name); + }; + const engine = new Engine(inputs, "/owned", "mcp-123456789012345678901234"); + engine.reserveRegistryPort = () => step("registry-port"); + engine.reserveHostPort = () => step("host-port"); + engine.qualifyTools = () => step("tools"); + engine.network = () => step("network"); + engine.networkOptions = () => []; + engine.create = async (role) => { + steps.push(role); + return role; + }; + engine.nativeDns = async () => { + steps.push("native-dns"); + if (!steps.includes("dependencies")) + throw new PlatformFailure("NEEDS_CONTEXT", "dns_before_dependencies"); + if (failedLookup) + throw new PlatformFailure("NEEDS_CONTEXT", "native_dns_lookup"); + }; + engine.guards = () => step("guards"); + engine.verifyNetwork = async (probes) => { + steps.push(`verify-network:${probes}`); + return { + generation: "test-generation", + enforcementSha256: "a".repeat(64), + fixtureOnlyEgress: true, + externalDnsBlocked: true, + roles: ["orchestrator", "coordinator", "worker"], + }; + }; + const stack = new Stack( + inputs, + "/owned", + {} as ConstructorParameters[2], + {} as ConstructorParameters[3], + {} as ConstructorParameters[4], + engine, + ); + stack.dependencies = () => step("dependencies"); + stack.edges = () => step("edges"); + stack.startRole = (_ctx, role) => step(`start:${role}`); + stack.readyRole = (_ctx, role) => step(`ready:${role}`); + stack.migrateOrchestrator = () => step("migrate"); + // Replace only the native I/O boundary; retain the real fixture launch and DNS gate. + const ready = spyOn(FixtureObserver.prototype, "ready").mockImplementation( + () => step("observer"), + ); + try { + const boot = bootPlatform( + engine, + { path: "/owned/ca", sha256: "a".repeat(64) }, + async () => { + steps.push("prepare"); + return { stack, ctx }; + }, + ); + const prefix = [ + "registry-port", + "host-port", + "tools", + "prepare", + "network", + "dependencies", + "fixture-modern", + "fixture-legacy", + "native-dns", + ]; + if (failedLookup) { + await expect(boot).rejects.toMatchObject({ + code: "NEEDS_CONTEXT", + seam: "native_dns_lookup", + }); + expect(steps).toEqual(prefix); + } else { + await expect(boot).resolves.toBe(ctx); + expect(steps).toEqual([ + ...prefix, + "observer", + "observer", + "guards", + "verify-network:true", + "edges", + "start:worker", + "ready:worker", + "start:coordinator", + "ready:coordinator", + "migrate", + "start:orchestrator", + "ready:orchestrator", + "start:host-agent", + "ready:host-agent", + ]); + } + } finally { + ready.mockRestore(); + } + }); +} + +for (const backend of ["firecracker", "vz"] as const) + test(`native DNS accepts only the expected addresses for every fixture name (${backend})`, async () => { + const engine = new Engine( + { backend, subnet: "10.42.3.0/24", nativeHostIp: "10.42.4.1" } as Inputs, + "/owned", + "mcp-123456789012345678901234", + ); + const records: Record = { + "postgres.fixture.example": "10.42.3.10", + "app.fixture.example": "10.42.3.20", + "coordinator.fixture.example": "10.42.3.21", + "operator.fixture.example": "10.42.3.21", + "control.fixture.example": "10.42.3.21", + "worker.fixture.example": "10.42.3.22", + "host.fixture.example": + backend === "firecracker" ? "10.42.3.21" : "10.42.4.1", + "resource.modern.fixture.example": "10.42.3.30", + "issuer.modern.fixture.example": "10.42.3.30", + "authorization.modern.fixture.example": "10.42.3.30", + "token.modern.fixture.example": "10.42.3.30", + "observer.modern.fixture.example": "10.42.3.30", + "resource.legacy.fixture.example": "10.42.3.31", + "issuer.legacy.fixture.example": "10.42.3.31", + "authorization.legacy.fixture.example": "10.42.3.31", + "token.legacy.fixture.example": "10.42.3.31", + "observer.legacy.fixture.example": "10.42.3.31", + }; + const names: string[] = []; + await engine.nativeDns(async (name, options) => { + expect(options).toEqual({ all: true }); + expect(records[name]).toBeDefined(); + names.push(name); + return [{ address: records[name]!, family: 4 }]; + }); + expect(names).toEqual(Object.keys(records)); + if (backend === "firecracker") + await expect( + engine.nativeDns(async (name) => [ + { + address: + name === "host.fixture.example" ? "10.42.4.1" : records[name]!, + family: 4, + }, + ]), + ).rejects.toMatchObject({ + code: "NEEDS_CONTEXT", + seam: "native_dns_route", + }); + }); + +for (const actual of [ + [], + [{ address: "10.42.3.11", family: 4 }], + [ + { address: "10.42.3.10", family: 4 }, + { address: "10.42.3.11", family: 4 }, + ], +]) { + test(`native DNS rejects empty or wrong successful answers (${actual.length})`, async () => { + const engine = new Engine( + { + backend: "vz", + subnet: "10.42.3.0/24", + nativeHostIp: "10.42.4.1", + } as Inputs, + "/owned", + "mcp-123456789012345678901234", + ); + await expect(engine.nativeDns(async () => actual)).rejects.toMatchObject({ + code: "NEEDS_CONTEXT", + seam: "native_dns_route", + }); + }); +} + +test("native DNS lookup rejection exposes only the static safe seam", async () => { + const engine = new Engine( + { + backend: "vz", + subnet: "10.42.3.0/24", + nativeHostIp: "10.42.4.1", + } as Inputs, + "/owned", + "mcp-123456789012345678901234", + ); + let failure: unknown; + try { + await engine.nativeDns(async () => { + throw new Error( + "secret-marker lookup postgres.fixture.example at 10.42.3.10", + ); + }); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(PlatformFailure); + expect(failure).toMatchObject({ + code: "NEEDS_CONTEXT", + seam: "native_dns_lookup", + }); + expect(String(failure)).toBe("Error: NEEDS_CONTEXT:native_dns_lookup"); + expect(JSON.stringify(failure)).not.toMatch( + /secret-marker|fixture\.example|10\.42\.3\.10/, + ); + expect(failure).not.toHaveProperty("cause"); +}); + +test("fixed boot graph waits for worker, coordinator and real migration before app and host", async () => { + const { bootApplications } = await import("./main.ts"); + const ctx = + {} as import("../../../../orchestrator/scripts/mcp-stack/context.ts").Context; + const steps: string[] = []; + const lifecycle = { + startRole: async (_ctx: typeof ctx, role: string, variant: string) => { + steps.push(`start:${role}:${variant}`); + }, + readyRole: async (_ctx: typeof ctx, role: string) => { + steps.push(`ready:${role}`); + }, + migrateOrchestrator: async () => { + steps.push("migrate"); + }, + }; + await bootApplications(lifecycle, ctx); + expect(steps).toEqual([ + "start:worker:normal", + "ready:worker", + "start:coordinator:normal", + "ready:coordinator", + "migrate", + "start:orchestrator:normal", + "ready:orchestrator", + "start:host-agent:normal", + "ready:host-agent", + ]); + steps.length = 0; + await expect( + bootApplications( + { + ...lifecycle, + migrateOrchestrator: async () => { + throw new Error("real migration failed"); + }, + }, + ctx, + ), + ).rejects.toThrow("real migration failed"); + expect(steps).toEqual([ + "start:worker:normal", + "ready:worker", + "start:coordinator:normal", + "ready:coordinator", + ]); +}); + +test("FC bind stays in its private namespace while VZ binds the verified native address", async () => { + const { hostListenAddress } = await import("./launch.ts"); + const { dnsRecords, dnsHosts, address, rules } = await import("./network.ts"); + const inputs = { + backend: "firecracker" as const, + nativeHostIp: "10.42.4.1", + subnet: "10.42.3.0/24", + }; + expect(hostListenAddress(inputs)).toBe("0.0.0.0:9101"); + expect(hostListenAddress({ ...inputs, backend: "vz" })).toBe( + "10.42.4.1:9101", + ); + expect(dnsRecords(inputs)["postgres.fixture.example"]).toBe("10.42.3.10"); + expect(dnsHosts(inputs)).toContain("10.42.3.10 postgres.fixture.example\n"); + const hostRule = (ip: string) => ({ + rule: { + family: "inet", + table: "mcp", + chain: "output", + expr: [ + { + match: { + op: "==", + left: { payload: { protocol: "ip", field: "daddr" } }, + right: ip, + }, + }, + { + match: { + op: "==", + left: { payload: { protocol: "tcp", field: "dport" } }, + right: 9101, + }, + }, + { accept: null }, + ], + }, + }); + for (const backend of ["firecracker", "vz"] as const) { + const selected = backend === "firecracker" ? "10.42.3.21" : "10.42.4.1"; + const other = backend === "firecracker" ? "10.42.4.1" : "10.42.3.21"; + const current = { ...inputs, backend }; + expect(dnsRecords(current)["host.fixture.example"]).toBe(selected); + expect(dnsHosts(current)).toContain(`${selected} host.fixture.example\n`); + expect(dnsHosts(current)).not.toContain(`${other} host.fixture.example\n`); + for (const role of ["coordinator", "orchestrator"] as const) { + expect(rules(current, role)).toContainEqual(hostRule(selected)); + expect(rules(current, role)).not.toContainEqual(hostRule(other)); + } + expect(rules(current, "worker")).not.toContainEqual(hostRule(selected)); + expect(rules(current, "worker")).not.toContainEqual(hostRule(other)); + } + expect(JSON.stringify(rules(inputs, "worker"))).not.toContain( + address(inputs, "postgres"), + ); +}); + +test("abort terminates an in-flight pinned CLI and releases the control lease", async () => { + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/cancel-"), + ); + const config = join(root, "config"); + await mkdir(config, { mode: 0o700 }); + const socket = join(root, "engine.sock"); + const server = createServer(); + await new Promise((resolve) => server.listen(socket, resolve)); + const script = join(root, "docker"), + started = join(root, "started"); + await writeFile( + script, + `#!/bin/sh\nexec ${JSON.stringify(process.execPath)} -e 'require("fs").writeFileSync(${JSON.stringify(started)},String(process.pid));setInterval(()=>{},100)'\n`, + { mode: 0o700 }, + ); + const inputs = { + engineSocket: socket, + engineConfig: config, + binaries: { + docker: { path: script, sha256: hash(await readFile(script)) }, + }, + } as Inputs; + const controller = new AbortController(); + try { + const engine = new Engine(inputs, root, "mcp-123456789012345678901234"); + const running = engine.controlled(controller.signal, () => + engine.call(["info"], undefined, 30000), + ); + const outcome = running.then( + () => false, + () => true, + ); + const deadline = performance.now() + 2000; + while (!(await readFile(started).catch(() => undefined))) { + if (performance.now() >= deadline) + throw new Error("controlled child did not start"); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + const pid = Number(await readFile(started, "utf8")); + controller.abort(); + expect(await outcome).toBe(true); + expect(() => process.kill(pid, 0)).toThrow(); + await expect( + engine.controlled(new AbortController().signal, async () => "drained"), + ).resolves.toBe("drained"); + } finally { + controller.abort(); + await new Promise((resolve) => server.close(() => resolve())); + await rm(root, { recursive: true }); + } +}); + +test("cleanup retains a failure receipt and refuses to delete a substituted network", async () => { + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/cleanup-"), + ); + const runId = "mcp-123456789012345678901234", + id = "c".repeat(64); + const operations: string[][] = []; + let foreign = false; + class CleanupEngine extends Engine { + override async project(): Promise { + return `${runId}-resolver`; + } + override async call(args: string[]): Promise { + operations.push(args); + if (args[0] === "network" && args[1] === "ls") return Buffer.from(""); + if (args[0] === "network" && args[1] === "create") return Buffer.from(id); + if (args[0] === "network" && args[1] === "inspect") + return Buffer.from( + JSON.stringify([ + { + Id: id, + Internal: true, + Driver: "bridge", + Labels: { "engrams.mcp.run": foreign ? "foreign" : runId }, + IPAM: { + Config: [{ Subnet: "10.42.3.0/24", Gateway: "10.42.3.1" }], + }, + Containers: {}, + }, + ]), + ); + if (args[0] === "ps") return Buffer.from(""); + throw new Error("unexpected cleanup command"); + } + } + try { + const engine = new CleanupEngine( + { subnet: "10.42.3.0/24" } as Inputs, + root, + runId, + ); + await engine.network(); + foreign = true; + await expect(engine.cleanup()).rejects.toThrow("engine_cleanup"); + expect( + operations.some((args) => args[0] === "network" && args[1] === "rm"), + ).toBe(false); + const receipt = JSON.parse( + await readFile(join(root, "cleanup.json"), "utf8"), + ); + expect(receipt.success).toBe(false); + expect(receipt.networkRemaining).toBe(true); + } finally { + await rm(root, { recursive: true }); + } +}); + +test("FC launch uses the declared configuration, private namespace and original image entrypoint", async () => { + const { generateIdentity } = await import("./identity.ts"); + const { generateConfiguration } = await import("./configuration.ts"); + const { parseInputs, BINARIES, IMAGES } = await import("./preflight.ts"); + const { Stack } = await import("./launch.ts"); + const { put } = await import("./owned.ts"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/fc-plan-"), + ); + try { + const pin = await put(join(root, "binary"), "controlled test artifact"); + const openssl = await realpath( + process.env.MCP_PLATFORM_TEST_OPENSSL ?? + Bun.which("openssl") ?? + "openssl", + ); + const image = { + archive: pin, + id: `sha256:${"c".repeat(64)}`, + manifestDigest: `sha256:${"d".repeat(64)}`, + }; + const inputs = parseInputs( + { + version: 1, + backend: "firecracker", + architecture: "arm64", + sourceCommit: "b".repeat(40), + engineSocket: "/owned/engine.sock", + engineConfig: "/owned/config", + engineId: "engine-identity-123", + registryPort: 32000, + subnet: "10.42.3.0/24", + nativeHostIp: "10.42.4.1", + proxy: pin, + firecracker: pin, + thinPool: { name: "suite", directory: root }, + binaries: { + ...Object.fromEntries(BINARIES.map((key) => [key, pin])), + openssl: { path: openssl, sha256: hash(await readFile(openssl)) }, + }, + images: { + ...Object.fromEntries(IMAGES.map((key) => [key, image])), + hostAgent: image, + }, + }, + "firecracker", + ); + const identity = await generateIdentity(inputs, root), + configuration = await generateConfiguration(inputs, root, identity); + const artifacts = { + coordinator: pin, + orchestrator: pin, + worker: pin, + fixture: pin, + guestImage: pin, + hostAgent: pin, + kernel: pin, + helper: pin, + claudeHarness: pin, + codexHarness: pin, + modelCli: pin, + claudeBundle: pin, + codexBundle: pin, + }; + const assets = { + binaries: inputs.binaries, + proxy: pin, + nativeProbe: undefined, + firecracker: pin, + artifacts, + bundles: join(root, "bundles"), + stamp: {}, + }; + const launches: { + role: string; + image: string; + options: string[]; + args: string[] | undefined; + entrypoint: string | undefined; + }[] = []; + const projections = new Map>(); + class LaunchEngine extends Engine { + override networkName() { + return "owned-network"; + } + override networkOptions() { + return ["--network", this.networkName()]; + } + override async project( + role: string, + files: Record, + ): Promise { + projections.set(role, files); + return `${this.runId}-${role}`; + } + override async call(args: string[]): Promise { + if (args[0] === "exec" && args[2] === "/usr/bin/dig") + return Buffer.from("10.42.3.40\n"); + if (args[0] === "exec") return Buffer.from(""); + throw new Error("unexpected dependency command"); + } + override async remove() {} + override async create( + role: string, + image: string, + options: string[], + args?: string[], + entrypoint?: string, + ): Promise { + launches.push({ role, image, options, args, entrypoint }); + return "a".repeat(64); + } + } + const engine = new LaunchEngine(inputs, root, identity.runId, { + bootId: "11111111-1111-4111-8111-111111111111", + release: "test-kernel", + pidNamespace: "pid:[1]", + pool: poolFixture(), + }); + engine.guardIds.set("coordinator", "e".repeat(64)); + const stack = new Stack( + inputs, + root, + identity, + assets, + configuration, + engine, + ); + const ctx = await stack.prepare(); + expect(ctx.observer.hostLogFile).toBe( + join(root, "host", "egress-proxyd.log"), + ); + expect((await stat(ctx.observer.hostLogFile)).mode & 0o777).toBe(0o600); + await stack.startRole(ctx, "host-agent", "normal"); + const launch = launches[0]!; + expect(launch.role).toBe("host-agent"); + expect( + launch.options + .flatMap((value, index) => + value === "--cap-add" ? [launch.options[index + 1]!] : [], + ) + .sort(), + ).toEqual(["MKNOD", "NET_ADMIN", "NET_BIND_SERVICE", "SYS_ADMIN"]); + expect(launch.entrypoint).toBeUndefined(); + expect(launch.args).toBeUndefined(); + expect(launch.options).toContain(`container:${"e".repeat(64)}`); + expect(launch.options[launch.options.indexOf("--pid") + 1]).toBe("host"); + expect(launch.options[launch.options.indexOf("--cgroupns") + 1]).toBe( + "private", + ); + expect(launch.options).toContain( + "/dev/mapper/control:/dev/mapper/control:rw", + ); + expect(launch.options).toContain("b 253:* rwm"); + expect(launch.options).toContain("/dev/kvm:/dev/kvm:rw"); + expect(launch.options).toContain("/dev/net/tun:/dev/net/tun:rw"); + const envPath = launch.options[launch.options.indexOf("--env-file") + 1]!; + const actualEnv = Object.fromEntries( + (await readFile(envPath, "utf8")) + .trim() + .split("\n") + .map((line) => { + const i = line.indexOf("="); + return [line.slice(0, i), line.slice(i + 1)]; + }), + ); + const declared = JSON.parse( + await readFile( + ctx.supervisor.configurations["host-agent"].normal.path, + "utf8", + ), + ); + expect(actualEnv).toEqual(declared.env); + expect(actualEnv.ENGRAM_GRPC_LISTEN_ADDR).toBe("0.0.0.0:9101"); + expect(actualEnv.ENGRAM_GRPC_ADVERTISE_ADDR).toBe( + "https://host.fixture.example:9101", + ); + expect(actualEnv.MCP_FIXTURE_HOST_LOG).toBe( + join(root, "host", "host.jsonl"), + ); + expect(actualEnv.ENGRAM_FIRECRACKER_BIN).toBe("/usr/local/bin/firecracker"); + const coordinatorConfiguration = JSON.parse( + await readFile( + ctx.supervisor.configurations.coordinator.normal.path, + "utf8", + ), + ); + const orchestratorConfiguration = JSON.parse( + await readFile( + ctx.supervisor.configurations.orchestrator.normal.path, + "utf8", + ), + ); + expect( + coordinatorConfiguration.env.ENGRAM_AUTH_TOKENS.split(","), + ).toContain(orchestratorConfiguration.env.CONTROL_PLANE_BEARER); + expect( + coordinatorConfiguration.env.ENGRAM_AUTH_TOKENS.split(","), + ).toContain(declared.env.ENGRAM_COORDINATOR_TOKEN); + const workerConfiguration = JSON.parse( + await readFile(ctx.supervisor.configurations.worker.normal.path, "utf8"), + ); + expect(Object.keys(workerConfiguration.env).sort()).toEqual([ + "MCP_WORKER_ENABLED", + "NODE_ENV", + ]); + expect(Object.keys(workerConfiguration.files).sort()).toEqual([ + "config.json", + "tls/ca.crt", + "tls/tls.crt", + "tls/tls.key", + ]); + const coordinatorDsn = await readFile( + ctx.observer.coordinatorDsnFile, + "utf8", + ), + orchestratorDsn = await readFile( + ctx.observer.orchestratorDsnFile, + "utf8", + ); + expect(ctx.observer.coordinatorDsnFile).not.toBe( + ctx.observer.orchestratorDsnFile, + ); + expect(new URL(coordinatorDsn).hostname).toBe("postgres.fixture.example"); + expect(new URL(orchestratorDsn).pathname).toBe( + `/mcp_${identity.runId.replaceAll("-", "_")}`, + ); + expect(coordinatorDsn).toBe(orchestratorDsn); + engine.guardIds.set("worker", "f".repeat(64)); + await stack.startRole(ctx, "worker", "normal"); + const workerLaunch = launches.find((v) => v.role === "worker")!; + expect(workerLaunch.options).not.toContain("--cap-add"); + await stack.dependencies(); + engine.guardIds.set("orchestrator", "d".repeat(64)); + await stack.edges(); + for (const role of ["orchestrator", "coordinator"] as const) { + const files = projections.get(`${role}-edge`)!; + const config = await readFile(files["edge.conf"]!.path, "utf8"); + expect(hash(config)).toBe(files["edge.conf"]!.sha256); + expect( + [...config.matchAll(/(\w+)_temp_path\s+([^;]+);/g)].map( + ([, module, path]) => [module, path], + ), + ).toEqual([ + ["client_body", "/tmp/client"], + ["proxy", "/tmp/proxy"], + ["fastcgi", "/tmp/fastcgi"], + ["uwsgi", "/tmp/uwsgi"], + ["scgi", "/tmp/scgi"], + ]); + expect(config).toContain("pid /tmp/nginx.pid;"); + expect(config).toContain("ssl_protocols TLSv1.3;"); + expect(config).toContain( + `proxy_pass http://127.0.0.1:${role === "orchestrator" ? 8787 : 8080};`, + ); + const edge = launches.find((v) => v.role === `${role}-edge`)!; + expect(edge.image).toBe(inputs.images.edge.id); + expect(edge.entrypoint).toBe("/usr/sbin/nginx"); + expect(edge.args).toEqual([ + "-c", + "/projection/edge.conf", + "-g", + "daemon off;", + ]); + expect(edge.options).toEqual([ + "--network", + `container:${engine.guardIds.get(role)}`, + "--user", + "1000:1000", + "--read-only", + "--mount", + `type=volume,src=${identity.runId}-${role}-edge,dst=/projection,readonly`, + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=16m,mode=1777", + ]); + expect(files["tls.crt"]).toEqual( + identity.leaves[role === "orchestrator" ? "app" : "coordinator"] + .certificate, + ); + expect(files["tls.key"]).toEqual( + identity.leaves[role === "orchestrator" ? "app" : "coordinator"].key, + ); + } + for (const role of ["dns", "denied-dns"]) { + const dnsLaunch = launches.find((v) => v.role === role)!; + expect(dnsLaunch.entrypoint).toBe("/coredns"); + expect(dnsLaunch.options).toContain("--read-only"); + expect(dnsLaunch.options[dnsLaunch.options.indexOf("--user") + 1]).toBe( + "1000:1000", + ); + expect( + dnsLaunch.options.flatMap((v, i) => + v === "--cap-add" ? [dnsLaunch.options[i + 1]] : [], + ), + ).toEqual(["NET_BIND_SERVICE"]); + } + for (const launch of launches.filter( + (v) => !["dns", "denied-dns", "host-agent"].includes(v.role), + )) + expect(launch.options).not.toContain("NET_BIND_SERVICE"); + } finally { + await rm(root, { recursive: true }); + } +}); + +for (const failInspect of [false, true]) { + test(`guest publication uses writable scratch and cleans up after ${failInspect ? "inspect failure" : "success"}`, async () => { + const { publishGuest } = await import("./assets.ts"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/publish-"), + ); + try { + const archive = await put(join(root, "guest.tar"), "owned archive"); + const config = Buffer.from('{"architecture":"arm64","os":"linux"}'); + const image = { + archive, + id: `sha256:${hash(config)}`, + manifestDigest: "", + }; + const manifest = Buffer.from( + JSON.stringify({ + schemaVersion: 2, + config: { digest: image.id }, + layers: [], + }), + ); + image.manifestDigest = `sha256:${hash(manifest)}`; + const inputs = parseInputs( + { + version: 1, + backend: "vz", + architecture: "arm64", + sourceCommit: "b".repeat(40), + engineSocket: "/owned/socket", + engineConfig: "/owned/config", + engineId: "engine-identity-123", + registryPort: 32000, + subnet: "10.42.3.0/24", + nativeHostIp: "10.42.4.1", + proxy: archive, + nativeProbe: archive, + binaries: Object.fromEntries(BINARIES.map((key) => [key, archive])), + images: Object.fromEntries(IMAGES.map((key) => [key, image])), + }, + "vz", + ); + const calls: { + args: string[]; + stdin: string | undefined; + timeout: number | undefined; + }[] = []; + const removed: string[] = []; + const launches: unknown[] = []; + const id = "a".repeat(64), + guard = "e".repeat(64); + const runId = "mcp-123456789012345678901234"; + class PublicationEngine extends Engine { + override async create( + role: string, + image: string, + options: string[], + args?: string[], + entrypoint?: string, + ): Promise { + launches.push({ role, image, options, args, entrypoint }); + return id; + } + override async call( + args: string[], + stdin?: string, + timeout?: number, + ): Promise { + calls.push({ args, stdin, timeout }); + if (failInspect) throw new Error("inspect failed"); + return args.includes("--config") ? config : manifest; + } + override async remove(role: string) { + removed.push(role); + } + } + const engine = new PublicationEngine(inputs, root, runId); + engine.guardIds.set("coordinator", guard); + const uri = `127.0.0.1:32000/${runId}/guest`; + const provenance = join(root, "image-provenance.json"); + if (failInspect) { + await expect(publishGuest(inputs, root, engine)).rejects.toThrow( + "inspect failed", + ); + await expect(stat(provenance)).rejects.toMatchObject({ + code: "ENOENT", + }); + } else { + expect(await publishGuest(inputs, root, engine)).toBe( + `${uri}@${image.manifestDigest}`, + ); + expect(JSON.parse(await readFile(provenance, "utf8"))).toEqual({ + version: 1, + archiveSha256: archive.sha256, + manifestDigest: image.manifestDigest, + configDigest: image.id, + imageUri: `${uri}@${image.manifestDigest}`, + }); + } + expect(removed).toEqual(["image-copy"]); + expect(launches).toEqual([ + { + role: "image-copy", + image: inputs.images.guard.id, + options: [ + "--network", + `container:${guard}`, + "--user", + `${process.getuid?.()}:${process.getgid?.()}`, + "--read-only", + "--mount", + `type=bind,src=${archive.path},dst=/input/guest.tar,readonly`, + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=256m,mode=1777", + ], + args: ["infinity"], + entrypoint: "/bin/sleep", + }, + ]); + const prefix = ["exec", id, "/usr/bin/skopeo", "--tmpdir", "/tmp"]; + const expected = [ + { + args: [...prefix, "inspect", "--raw", "oci-archive:/input/guest.tar"], + stdin: undefined, + timeout: undefined, + }, + { + args: [ + ...prefix, + "inspect", + "--raw", + "--config", + "oci-archive:/input/guest.tar", + ], + stdin: undefined, + timeout: undefined, + }, + { + args: [ + ...prefix, + "copy", + "--preserve-digests", + "--dest-tls-verify=false", + "oci-archive:/input/guest.tar", + `docker://${uri}:fixture`, + ], + stdin: undefined, + timeout: 120000, + }, + { + args: [ + ...prefix, + "inspect", + "--tls-verify=false", + "--raw", + `docker://${uri}@${image.manifestDigest}`, + ], + stdin: undefined, + timeout: undefined, + }, + ]; + expect(calls).toEqual(failInspect ? expected.slice(0, 1) : expected); + } finally { + await rm(root, { recursive: true }); + } + }); +} + +test("archive provenance rejects a swapped config, manifest, or multi-platform index", async () => { + const { verifyArchiveIdentity } = await import("./assets.ts"); + const config = Buffer.from('{"architecture":"arm64","os":"linux"}'); + const id = `sha256:${hash(config)}`; + const manifest = Buffer.from( + JSON.stringify({ schemaVersion: 2, config: { digest: id }, layers: [] }), + ); + const image = { + archive: { path: "/owned/image.tar", sha256: "a".repeat(64) }, + id, + manifestDigest: `sha256:${hash(manifest)}`, + }; + expect(() => verifyArchiveIdentity(image, manifest, config)).not.toThrow(); + expect(() => + verifyArchiveIdentity(image, manifest, Buffer.from("different")), + ).toThrow("archive_config_identity"); + expect(() => + verifyArchiveIdentity( + { ...image, manifestDigest: image.id }, + manifest, + config, + ), + ).toThrow("archive_image_identity"); + const index = Buffer.from( + JSON.stringify({ schemaVersion: 2, manifests: [], config: { digest: id } }), + ); + expect(() => + verifyArchiveIdentity( + { ...image, manifestDigest: `sha256:${hash(index)}` }, + index, + config, + ), + ).toThrow("archive_image_identity"); +}); + +test("resolver uses only a readonly named file projection and rejects source, subpath and byte substitution", async () => { + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/resolver-"), + ); + const runId = "mcp-123456789012345678901234", + image = `sha256:${"b".repeat(64)}`, + id = "a".repeat(64), + network = "e".repeat(64), + volume = `${runId}-resolver`; + const mount = { + Type: "volume", + Source: volume, + Target: "/etc/resolv.conf", + ReadOnly: true, + VolumeOptions: { Subpath: "resolv.conf", NoCopy: true }, + }; + const observed = { + Type: "volume", + Name: volume, + Source: `/engine/volumes/${volume}/_data`, + Destination: "/etc/resolv.conf", + RW: false, + }; + const state = { + Id: id, + Image: image, + Config: { + Labels: { "engrams.mcp.run": runId }, + User: "1000:1000", + Env: ["PATH=/usr/bin:/bin"], + Entrypoint: ["/bin/sleep"], + Cmd: ["infinity"], + }, + HostConfig: { + RestartPolicy: { Name: "no" }, + Privileged: false, + ReadonlyRootfs: true, + PidsLimit: 128, + Memory: 805306368, + NanoCpus: 2000000000, + NetworkMode: "none", + PidMode: "", + IpcMode: "private", + CapDrop: ["ALL"], + CapAdd: [], + SecurityOpt: ["no-new-privileges"], + MaskedPaths: ["/proc/kcore"], + ReadonlyPaths: ["/proc/sys"], + Binds: [], + VolumesFrom: [], + Devices: [], + DeviceRequests: [], + Mounts: [mount], + }, + Mounts: [observed], + }; + const calls: string[][] = []; + let foreignVolume = false, + wrongBytes = false; + class ResolverEngine extends Engine { + override async project(role: string): Promise { + return this.volume(role); + } + override async call(args: string[]): Promise { + calls.push(args); + if ( + (args[0] === "network" && args[1] === "ls") || + (args[0] === "volume" && args[1] === "ls") + ) + return Buffer.from(""); + if (args[0] === "network" && args[1] === "create") + return Buffer.from(network); + if (args[0] === "network" && args[1] === "inspect") + return Buffer.from( + JSON.stringify([ + { + Id: network, + Internal: true, + Driver: "bridge", + Labels: { "engrams.mcp.run": runId }, + IPAM: { + Config: [{ Subnet: "10.42.3.0/24", Gateway: "10.42.3.1" }], + }, + }, + ]), + ); + if (args[0] === "volume" && args[1] === "create") + return Buffer.from(volume); + if (args[0] === "volume" && args[1] === "inspect") + return Buffer.from( + JSON.stringify([ + { + Name: volume, + Mountpoint: `/engine/volumes/${volume}/_data`, + Driver: "local", + Scope: "local", + Labels: { "engrams.mcp.run": foreignVolume ? "foreign" : runId }, + }, + ]), + ); + if (args[0] === "create" || args[0] === "start") return Buffer.from(id); + if (args[0] === "container" && args[1] === "inspect") + return Buffer.from(JSON.stringify([state])); + if (args[0] === "image") + return Buffer.from( + JSON.stringify([ + { + Id: image, + Config: { + Env: ["PATH=/usr/bin:/bin"], + Entrypoint: ["/bin/sleep"], + Cmd: ["infinity"], + }, + }, + ]), + ); + if (args[0] === "cp") { + expect(args[1]).toBe(`${id}:/etc/resolv.conf`); + await writeFile( + args[2]!, + wrongBytes + ? "nameserver 127.0.0.11\n" + : await readFile(join(root, "resolv.conf")), + ); + return Buffer.from(""); + } + throw new Error("unexpected resolver operation"); + } + } + try { + const engine = new ResolverEngine( + { subnet: "10.42.3.0/24", images: { guard: { id: image } } } as Inputs, + root, + runId, + ); + await engine.network(); + await engine.create( + "probe", + image, + ["--network", "none", "--user", "1000:1000", "--read-only"], + ["infinity"], + "/bin/sleep", + ); + expect(calls.find((args) => args[0] === "create")).toContain( + `type=volume,src=${volume},dst=/etc/resolv.conf,volume-subpath=resolv.conf,volume-nocopy,readonly`, + ); + mount.VolumeOptions.Subpath = "foreign"; + await expect(engine.audit(id)).rejects.toThrow( + "container_mount_specification", + ); + mount.VolumeOptions.Subpath = "resolv.conf"; + mount.ReadOnly = false; + await expect(engine.audit(id)).rejects.toThrow( + "container_mount_specification", + ); + mount.ReadOnly = true; + const originalSource = observed.Source; + observed.Source = "/foreign/volume"; + await expect(engine.audit(id)).rejects.toThrow("container_mount"); + observed.Source = originalSource; + observed.Name = "foreign"; + await expect(engine.audit(id)).rejects.toThrow("container_mount"); + observed.Name = volume; + foreignVolume = true; + await expect(engine.audit(id)).rejects.toThrow("volume_identity"); + foreignVolume = false; + wrongBytes = true; + await expect(engine.verifyResolver(id)).rejects.toThrow("resolver_changed"); + } finally { + await rm(root, { recursive: true }); + } +}); + +test("cleanup recovers only a planned owned container after a lost create response", async () => { + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/lost-create-"), + ); + const runId = "mcp-123456789012345678901234", + id = "a".repeat(64); + let removed = false; + const calls: string[][] = []; + class LostResponseEngine extends Engine { + override async call(args: string[]): Promise { + calls.push(args); + if (args[0] === "create") throw new Error("lost create response"); + if (args[0] === "ps") return Buffer.from(removed ? "" : id); + if (args[0] === "container" && args[1] === "inspect") + return Buffer.from( + JSON.stringify([ + { + Id: id, + Name: `/${runId}-probe`, + Config: { Labels: { "engrams.mcp.run": runId } }, + State: { Running: false, Pid: 0 }, + }, + ]), + ); + if (args[0] === "logs") return Buffer.from("controlled retained log"); + if (args[0] === "rm") { + removed = true; + return Buffer.from(id); + } + throw new Error("unexpected cleanup operation"); + } + } + try { + const engine = new LostResponseEngine({} as Inputs, root, runId); + await expect( + engine.create("probe", "sha256:" + "b".repeat(64), ["--network", "none"]), + ).rejects.toThrow("lost create response"); + await engine.cleanup(); + expect(calls.find((args) => args[0] === "rm")).toEqual(["rm", id]); + expect( + JSON.parse(await readFile(join(root, "cleanup.json"), "utf8")).success, + ).toBe(true); + } finally { + await rm(root, { recursive: true }); + } +}); + +async function handoffFixture(root: string) { + const { ContextSchema } = + await import("../../../../orchestrator/scripts/mcp-stack/context.ts"); + const { put } = await import("./owned.ts"); + const pin = await put(join(root, "artifact"), "a".repeat(40)); + const endpoint = (name: string) => ({ + origin: `https://${name}.fixture.example`, + ca: pin, + serverFingerprint: Array(32).fill("AA").join(":"), + }); + const fixture = async (name: "modern" | "legacy") => { + const runRoot = join(root, name); + await mkdir(runRoot, { mode: 0o700 }); + const configuration = await put(join(runRoot, "configuration"), "fixture"); + return { + ...endpoint(`observer.${name}`), + configuration, + runRoot, + tokenFile: configuration.path, + resourceKeyFile: configuration.path, + resourceOrigin: `https://resource.${name}.fixture.example`, + issuerOrigin: `https://issuer.${name}.fixture.example`, + authorizationOrigin: `https://authorization.${name}.fixture.example`, + tokenOrigin: `https://token.${name}.fixture.example`, + redirectUri: "https://app.fixture.example/integrations/mcp/callback", + protocol: name === "modern" ? "2026-07-28" : "2025-06-18", + }; + }; + const human = (name: string) => ({ + email: `${name}@fixture.example`, + passwordFile: pin.path, + cookieFile: join(root, `${name}.cookie`), + }); + return ContextSchema.parse({ + version: 1, + runId: "mcp-controlled-handoff", + runRoot: root, + backend: "vz", + sourceCommit: "b".repeat(40), + hostId: "00000000-0000-4000-8000-000000000001", + imageId: "controlled-image", + imageDigest: "sha256:" + "c".repeat(64), + artifacts: Object.fromEntries( + [ + "coordinator", + "orchestrator", + "worker", + "hostAgent", + "fixture", + "kernel", + "guestImage", + "claudeHarness", + "codexHarness", + "modelCli", + "claudeBundle", + "codexBundle", + "helper", + ].map((name) => [name, pin]), + ), + endpoints: { + app: endpoint("app"), + coordinator: endpoint("coordinator"), + workerControl: endpoint("worker"), + operator: { + ...endpoint("operator"), + certificate: pin, + privateKeyFile: pin.path, + }, + }, + protocolFixtures: { + modern: await fixture("modern"), + legacy: await fixture("legacy"), + }, + publication: { + candidateIdentity: "sha256:" + "d".repeat(64), + expectedGeneration: null, + deploymentIdentity: "sha256:" + "d".repeat(64), + networkIdentity: "sha256:" + "d".repeat(64), + trustIdentity: "sha256:" + "d".repeat(64), + zone: "fixture-zone", + }, + identities: { + admin: human("admin"), + owner: human("owner"), + other: human("other"), + unattendedKeyFile: join(root, "unattended"), + automationKeyFile: join(root, "automation"), + routerSentinelFile: pin.path, + }, + supervisor: { + socket: join(root, "s"), + tokenFile: pin.path, + configurations: { + orchestrator: { normal: pin }, + coordinator: { normal: pin }, + worker: { normal: pin }, + "host-agent": { normal: pin }, + }, + operatorCandidates: { + normal: { file: pin, identity: "sha256:" + "d".repeat(64) }, + }, + }, + observer: { + coordinatorDsnFile: pin.path, + orchestratorDsnFile: pin.path, + hostManagedRoot: join(root, "managed"), + hostLogFile: pin.path, + }, + guest: { + stateDirectory: "/tmp/mcp-stack-fixture", + cliPath: "/opt/engram/dyn/0/engram-mcp-model-fixture", + }, + deadlines: { requestMs: 1000, stepMs: 3000, totalMs: 10000 }, + evidenceFile: join(root, "evidence"), + }); +} + +test("runner handoff validates the complete context with a real private supervisor socket", async () => { + const { startContextSupervisor } = await import("./main.ts"); + const { loadContext } = + await import("../../../../orchestrator/scripts/mcp-stack/context.ts"); + const { lstat } = await import("node:fs/promises"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/h-"), + ); + let supervisor: + Awaited> | undefined; + try { + const ctx = await handoffFixture(root); + await mkdir(ctx.observer.hostManagedRoot, { mode: 0o700 }); + const unexpected = async (): Promise => { + throw new Error("handoff does not need lifecycle access"); + }; + supervisor = await startContextSupervisor(ctx, { + status: unexpected, + restart: unexpected, + stop: unexpected, + }); + expect((await lstat(ctx.supervisor.socket)).isSocket()).toBe(true); + expect((await lstat(ctx.supervisor.socket)).mode & 0o077).toBe(0); + expect(await loadContext(join(root, "context.json"))).toEqual(ctx); + await supervisor.close(); + supervisor = undefined; + await expect(lstat(ctx.supervisor.socket)).rejects.toThrow(); + } finally { + if (supervisor) await supervisor.close(); + await rm(root, { recursive: true }); + } +}); + +test("failed complete context validation drains and removes the real handoff supervisor", async () => { + const { startContextSupervisor } = await import("./main.ts"); + const { lstat } = await import("node:fs/promises"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/h-failed-"), + ); + try { + const ctx = await handoffFixture(root); + const unexpected = async (): Promise => { + throw new Error("handoff does not need lifecycle access"); + }; + await expect( + startContextSupervisor(ctx, { + status: unexpected, + restart: unexpected, + stop: unexpected, + }), + ).rejects.toThrow("owned_path"); + await expect(lstat(ctx.supervisor.socket)).rejects.toThrow(); + expect((await lstat(join(root, "context.json"))).isFile()).toBe(true); + } finally { + await rm(root, { recursive: true }); + } +}); + +test("only the coordinator guard installs the exact native-host route on the internal bridge", async () => { + const calls: string[][] = []; + const inputs = { + backend: "firecracker", + subnet: "10.42.3.0/24", + nativeHostIp: "10.42.4.1", + registryPort: 32000, + images: { guard: { id: "sha256:" + "a".repeat(64) } }, + } as Inputs; + const ids = { + orchestrator: "a".repeat(64), + coordinator: "b".repeat(64), + worker: "c".repeat(64), + }; + class GuardPlan extends Engine { + override async project(): Promise { + return "controlled-probe-trust"; + } + override networkOptions(): string[] { + return ["--network", "controlled-internal"]; + } + override async create(role: string): Promise { + return ids[role.slice(0, -6) as keyof typeof ids]; + } + override async inspect() { + return { + HostConfig: { + PortBindings: { + "32000/tcp": [{ HostIp: "127.0.0.1", HostPort: "32000" }], + "9101/tcp": [{ HostIp: "10.42.4.1", HostPort: "9101" }], + }, + }, + }; + } + override async call(args: string[]): Promise { + calls.push(args); + return Buffer.from(""); + } + override async verifyNetwork() { + return { + generation: "controlled", + enforcementSha256: "e".repeat(64), + fixtureOnlyEgress: true as const, + externalDnsBlocked: true as const, + roles: ["orchestrator", "coordinator", "worker"] as ( + "orchestrator" | "coordinator" | "worker" + )[], + }; + } + } + const engine = new GuardPlan(inputs, "/owned/run", "mcp-controlled-route"); + await engine.guards({ path: "/owned/ca", sha256: "f".repeat(64) }); + expect(calls.filter((args) => args.includes("/usr/sbin/ip"))).toEqual([ + [ + "exec", + ids.coordinator, + "/usr/sbin/ip", + "-4", + "route", + "replace", + "10.42.4.1/32", + "via", + "10.42.3.1", + "dev", + "eth0", + ], + ]); +}); + +test("native-host route qualification rejects default, missing, duplicate and substituted routes", async () => { + const { verifyNativeRouteState } = await import("./engine.ts"); + const inputs = { subnet: "10.42.3.0/24", nativeHostIp: "10.42.4.1" }; + const route = { dst: "10.42.4.1", gateway: "10.42.3.1", dev: "eth0" }; + const selected = { ...route, prefsrc: "10.42.3.21" }; + expect(() => + verifyNativeRouteState([route], [selected], inputs), + ).not.toThrow(); + for (const routes of [ + [], + [route, route], + [route, { dst: "default", gateway: "10.42.3.1" }], + [{ ...route, gateway: "10.42.3.40" }], + [{ ...route, dev: "foreign0" }], + ]) + expect(() => verifyNativeRouteState(routes, [selected], inputs)).toThrow(); + for (const current of [ + { ...selected, gateway: "10.42.3.40" }, + { ...selected, prefsrc: "10.42.3.22" }, + { ...selected, dev: "foreign0" }, + ]) + expect(() => verifyNativeRouteState([route], [current], inputs)).toThrow( + "native_route_selected", + ); +}); + +test("native-host TLS qualification pins the actual first leaf and removes its private probe on failure", async () => { + const { generateIdentity } = await import("./identity.ts"); + const { verifyNativeHostTls } = await import("./engine.ts"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/host-tls-"), + ); + const openssl = await realpath( + process.env.MCP_PLATFORM_TEST_OPENSSL ?? Bun.which("openssl") ?? "openssl", + ); + const inputs = { + subnet: "10.42.3.0/24", + nativeHostIp: "10.42.4.1", + binaries: { + openssl: { path: openssl, sha256: hash(await readFile(openssl)) }, + }, + images: { guard: { id: "sha256:" + "a".repeat(64) } }, + } as Inputs; + try { + const identity = await generateIdentity(inputs, root), + id = "a".repeat(64), + calls: string[][] = []; + const host = identity.leaves.host, + wrong = identity.leaves.worker; + const output = async (certificate: string) => + Buffer.from( + `New, TLSv1.3, Cipher is TLS_AES_256_GCM_SHA384\n${await readFile(certificate, "utf8")}\nVerify return code: 0 (ok)\n`, + ); + let bytes = await output(host.certificate.path), + running = false, + removed = true; + class TlsProbe extends Engine { + override async project( + _role: string, + files: Record, + ): Promise { + expect(Object.keys(files).sort()).toEqual([ + "ca.pem", + "client.crt", + "client.key", + ]); + return "controlled-client-tls"; + } + override async verifyNetwork() { + return { + generation: "controlled", + enforcementSha256: "e".repeat(64), + fixtureOnlyEgress: true as const, + externalDnsBlocked: true as const, + roles: ["orchestrator", "coordinator", "worker"] as ( + "orchestrator" | "coordinator" | "worker" + )[], + }; + } + override async audit(id: string) { + return { Id: id }; + } + override async call(args: string[]): Promise { + calls.push(args); + if (args[0] === "create") { + removed = false; + return Buffer.from(id); + } + if (args[0] === "start") { + running = true; + return Buffer.from(id); + } + if (args[0] === "exec" && args[2] === "/usr/bin/openssl") return bytes; + if (args[0] === "container" && args[1] === "inspect") + return Buffer.from( + JSON.stringify([ + { + Id: id, + Config: { Labels: { "engrams.mcp.run": identity.runId } }, + State: { Running: running, Pid: running ? 123 : 0 }, + }, + ]), + ); + if (args[0] === "stop") { + running = false; + return Buffer.from(id); + } + if (args[0] === "rm") { + removed = true; + return Buffer.from(id); + } + if (args[0] === "ps") return Buffer.from(removed ? "" : id); + throw new Error("unexpected fixed TLS probe command"); + } + } + const engine = new TlsProbe(inputs, root, identity.runId); + engine.guardIds.set("coordinator", "b".repeat(64)); + const tls = { + ca: identity.ca, + clientCertificate: identity.leaves.coordinator.certificate, + clientKey: identity.leaves.coordinator.key, + serverCertificate: host.certificate, + }; + await engine.qualifyNativeHost(tls); + await engine.qualifyNativeHost(tls); + expect(removed).toBe(true); + const created = calls.find((args) => args[0] === "create")!; + expect(created).toContain(`container:${"b".repeat(64)}`); + expect(created).toContain( + "type=volume,src=controlled-client-tls,dst=/etc/mcp-host-probe,readonly", + ); + expect(created).not.toContain("--env-file"); + expect(created).not.toContain("--cap-add"); + expect(created).not.toContain("--device"); + const invoked = calls.find( + (args) => args[0] === "exec" && args[2] === "/usr/bin/openssl", + )!; + for (const flag of [ + "-tls1_3", + "-verify_return_error", + "-verify_hostname", + "-servername", + "-cert", + "-key", + ]) + expect(invoked).toContain(flag); + bytes = await output(wrong.certificate.path); + await expect(engine.qualifyNativeHost(tls)).rejects.toThrow( + "native_host_tls_leaf", + ); + expect(removed).toBe(true); + expect(running).toBe(false); + const correct = await output(host.certificate.path); + expect(() => + verifyNativeHostTls(Buffer.concat([bytes, correct]), host.fingerprint), + ).toThrow("native_host_tls_leaf"); + expect(() => + verifyNativeHostTls( + Buffer.from(correct.toString().replace("TLSv1.3", "TLSv1.2")), + host.fingerprint, + ), + ).toThrow("native_host_tls_verification"); + expect(() => + verifyNativeHostTls( + Buffer.from(correct.toString().replace("0 (ok)", "20 (bad)")), + host.fingerprint, + ), + ).toThrow("native_host_tls_verification"); + } finally { + await rm(root, { recursive: true }); + } +}); + +test("FC sysctls belong only to its C guard and actual firewall order stays production exact", async () => { + const { fcGuardSysctls, fcHostFirewall, verifyFcHostFirewall } = + await import("./engine.ts"); + expect(fcGuardSysctls("firecracker", "coordinator-guard")).toEqual({ + "net.ipv4.ip_forward": "1", + "net.ipv4.conf.all.route_localnet": "1", + }); + for (const backend of ["firecracker", "vz"] as const) + for (const role of [ + "orchestrator-guard", + "worker-guard", + "host-agent", + "worker", + ...(backend === "vz" ? ["coordinator-guard"] : []), + ]) + expect(fcGuardSysctls(backend, role)).toEqual({}); + const expected = fcHostFirewall(26000); + expect(expected.INPUT[0]).toContain("--dport 26003"); + expect(expected.FORWARD.at(-1)).toContain("engram-default-deny"); + expect(expected.PREROUTING[0]).toContain("-i vh-engr-+"); + expect(expected.PREROUTING[2]).toContain("--dport 443"); + const actual = Object.fromEntries( + Object.entries(expected).map(([chain, lines]) => [ + chain, + [ + `-P ${chain} ACCEPT`, + ...lines.map((line) => + line + .replace("-p tcp", "-p tcp -m tcp") + .replace("-p udp", "-p udp -m udp") + .replace("--to-port ", "--to-ports ") + .replace(/--comment (\S+)/, '--comment "$1"'), + ), + ].join("\n") + "\n", + ]), + ); + verifyFcHostFirewall(actual, 26000); + for (const current of [ + { + ...actual, + INPUT: actual.INPUT!.replace("--dport 26003", "--dport 26004"), + }, + { ...actual, INPUT: actual.INPUT!.replace(/-A INPUT[^\n]+\n/, "") }, + { + ...actual, + FORWARD: actual.FORWARD!.replace( + "-P FORWARD ACCEPT\n", + "-P FORWARD ACCEPT\n-A FORWARD -j ACCEPT\n", + ), + }, + { + ...actual, + PREROUTING: actual.PREROUTING!.replace("-i vh-engr-+", "-i tap-engr-+"), + }, + { ...actual, POSTROUTING: "-P POSTROUTING ACCEPT\n" }, + { + ...actual, + INPUT: actual + .INPUT!.split("\n") + .map((line, i, all) => (i === 1 ? all[5] : i === 5 ? all[1] : line)) + .join("\n"), + }, + ]) + expect(() => verifyFcHostFirewall(current, 26000)).toThrow(); +}); + +function dockerDnsFirewall(production: Record): Record< + string, + string +> & { + POSTROUTING: string; + OUTPUT: string; + DOCKER_OUTPUT: string; + DOCKER_POSTROUTING: string; +} { + return { + ...Object.fromEntries( + Object.entries(production).map(([chain, rules]) => [ + chain, + [`-P ${chain} ACCEPT`, ...rules].join("\n") + "\n", + ]), + ), + POSTROUTING: + [ + "-P POSTROUTING ACCEPT", + "-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING", + ...production.POSTROUTING!, + ].join("\n") + "\n", + OUTPUT: "-P OUTPUT ACCEPT\n-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT\n", + DOCKER_OUTPUT: + [ + "-N DOCKER_OUTPUT", + "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p tcp -m tcp --dport 53 -j DNAT --to-destination 127.0.0.11:35285", + "-A DOCKER_OUTPUT -d 127.0.0.11/32 -p udp -m udp --dport 53 -j DNAT --to-destination 127.0.0.11:52990", + ].join("\n") + "\n", + DOCKER_POSTROUTING: + [ + "-N DOCKER_POSTROUTING", + "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p tcp -m tcp --sport 35285 -j SNAT --to-source :53", + "-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p udp -m udp --sport 52990 -j SNAT --to-source :53", + ].join("\n") + "\n", + }; +} + +test("FC firewall accepts the exact Docker loopback DNS graph", async () => { + const { fcHostFirewall, verifyFcHostFirewall } = await import("./engine.ts"); + const actual = dockerDnsFirewall(fcHostFirewall(30000)); + expect(() => verifyFcHostFirewall(actual, 30000)).not.toThrow(); + const otherPorts = Object.fromEntries( + Object.entries(actual).map(([chain, rules]) => [ + chain, + rules.replaceAll("35285", "1").replaceAll("52990", "65535"), + ]), + ); + expect(() => verifyFcHostFirewall(otherPorts, 30000)).not.toThrow(); +}); + +test("FC firewall rejects any changed Docker DNS graph or production rule", async () => { + const { fcHostFirewall, verifyFcHostFirewall } = await import("./engine.ts"); + const actual = dockerDnsFirewall(fcHostFirewall(30000)); + const mutations: Record[] = []; + for (const chain of [ + "OUTPUT", + "DOCKER_OUTPUT", + "DOCKER_POSTROUTING", + ] as const) { + const missing: Record = { ...actual }; + delete missing[chain]; + mutations.push(missing); + mutations.push({ + ...actual, + [chain]: actual[chain].replaceAll("127.0.0.11", "127.0.0.12"), + }); + mutations.push({ + ...actual, + [chain]: actual[chain] + "-A " + chain + " -d 203.0.113.1/32 -j ACCEPT\n", + }); + const lines = actual[chain].trim().split("\n"); + mutations.push({ + ...actual, + [chain]: [...lines, lines[1]].join("\n") + "\n", + }); + } + for (const port of ["0", "65536", "035285", "-1", "1:2"]) + mutations.push({ + ...actual, + DOCKER_OUTPUT: actual.DOCKER_OUTPUT.replace( + "127.0.0.11:35285", + "127.0.0.11:" + port, + ), + }); + mutations.push( + { + ...actual, + DOCKER_OUTPUT: actual.DOCKER_OUTPUT.replace("--dport 53", "--dport 54"), + }, + { + ...actual, + DOCKER_OUTPUT: actual.DOCKER_OUTPUT.replace("-p tcp", "-p udp"), + }, + { + ...actual, + DOCKER_OUTPUT: actual.DOCKER_OUTPUT.replace("-j DNAT", "-j REDIRECT"), + }, + { + ...actual, + DOCKER_POSTROUTING: actual.DOCKER_POSTROUTING.replace( + "--sport 35285", + "--sport 35286", + ), + }, + { + ...actual, + DOCKER_POSTROUTING: actual.DOCKER_POSTROUTING.replace( + "--to-source :53", + "--to-source :54", + ), + }, + { + ...actual, + OUTPUT: actual.OUTPUT.replace("DOCKER_OUTPUT", "OTHER_OUTPUT"), + }, + { + ...actual, + POSTROUTING: actual.POSTROUTING.replace("127.0.0.11/32", "127.0.0.0/8"), + }, + { + ...actual, + POSTROUTING: actual.POSTROUTING.replace( + "DOCKER_POSTROUTING", + "OTHER_POSTROUTING", + ), + }, + { + ...actual, + POSTROUTING: actual.POSTROUTING + "-A POSTROUTING -j MASQUERADE\n", + }, + { + ...actual, + INPUT: actual.INPUT!.replace("--dport 30003", "--dport 30004"), + }, + { ...actual, FORWARD: actual.FORWARD!.split("\n").reverse().join("\n") }, + { ...actual, EXTRA: "-N EXTRA\n" }, + { + ...actual, + DOCKER_OUTPUT: actual.DOCKER_OUTPUT.replace( + /-A DOCKER_OUTPUT[^\n]+\n/, + "", + ), + }, + { + ...actual, + DOCKER_POSTROUTING: actual.DOCKER_POSTROUTING.replace( + /-A DOCKER_POSTROUTING[^\n]+\n/, + "", + ), + }, + ); + const forward = actual.FORWARD!.trim().split("\n"); + mutations.push({ + ...actual, + FORWARD: + [forward[0], forward[2], forward[1], ...forward.slice(3)].join("\n") + + "\n", + }); + mutations.push({ + ...actual, + DOCKER_OUTPUT: actual.DOCKER_OUTPUT.replace( + "--dport 53", + "--dport 53 --dport 53", + ), + }); + mutations.push({ + ...actual, + DOCKER_OUTPUT: actual.DOCKER_OUTPUT.replace( + "127.0.0.11/32", + "127.0.0.11/8", + ), + }); + const swapped = actual.POSTROUTING.trim().split("\n"); + mutations.push({ + ...actual, + POSTROUTING: [swapped[0], swapped[2], swapped[1]].join("\n") + "\n", + }); + mutations.push({ + ...actual, + POSTROUTING: actual.POSTROUTING.replace( + "-P POSTROUTING ACCEPT\n", + "-P POSTROUTING ACCEPT\n-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING\n", + ), + }); + for (const current of mutations) + expect(() => verifyFcHostFirewall(current, 30000)).toThrow(); +}); + +test("FC qualification reads both actual namespace sysctls and production chains", async () => { + const { fcHostFirewall } = await import("./engine.ts"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/fc-net-"), + ); + const calls: string[][] = []; + let badNamespace = false, + badSysctl = false, + dockerDns = false; + class FcProbe extends Engine { + override async call(args: string[]) { + calls.push(args); + if (args[2] === "/usr/bin/readlink") + return Buffer.from( + badNamespace && args[1] === "host" ? "net:[456]\n" : "net:[123]\n", + ); + if (args[2] === "/bin/cat") return Buffer.from(badSysctl ? "0\n" : "1\n"); + if (args[2] === "/usr/sbin/iptables") { + const chain = args[6]!; + if (dockerDns) { + const actual: Record = dockerDnsFirewall( + fcHostFirewall(26000), + ); + const output = actual[chain]; + if (output === undefined) throw Error("unexpected DNS chain"); + return Buffer.from(output); + } + return Buffer.from( + [`-P ${chain} ACCEPT`, ...fcHostFirewall(26000)[chain]!].join("\n") + + "\n", + ); + } + throw new Error("unexpected FC qualification command"); + } + } + try { + const engine = new FcProbe( + { backend: "firecracker", registryPort: 26000 } as Inputs, + root, + "controlled", + ); + engine.guardIds.set("coordinator", "guard"); + await engine.qualifyFcHost("host"); + expect(calls.filter((args) => args[2] === "/bin/cat")).toHaveLength(4); + expect( + calls.filter((args) => args[2] === "/usr/sbin/iptables"), + ).toHaveLength(4); + calls.length = 0; + dockerDns = true; + await engine.qualifyFcHost("host"); + expect( + calls + .filter((args) => args[2] === "/usr/sbin/iptables") + .map((args) => args.slice(4)), + ).toEqual([ + ["filter", "-S", "INPUT"], + ["filter", "-S", "FORWARD"], + ["nat", "-S", "PREROUTING"], + ["nat", "-S", "POSTROUTING"], + ["nat", "-S", "OUTPUT"], + ["nat", "-S", "DOCKER_OUTPUT"], + ["nat", "-S", "DOCKER_POSTROUTING"], + ]); + dockerDns = false; + badNamespace = true; + await expect(engine.qualifyFcHost("host")).rejects.toThrow( + "fc_host_namespace", + ); + badNamespace = false; + badSysctl = true; + await expect(engine.qualifyFcHost("host")).rejects.toThrow( + "fc_host_sysctl", + ); + } finally { + await rm(root, { recursive: true }); + } +}); + +test("VZ link uses private nonoverlapping IPAM and derives its interface from the exact endpoint", async () => { + const { verifyNativeSubnet, nativeInterface, verifyNativeRouteState } = + await import("./engine.ts"); + expect(() => + verifyNativeSubnet("172.25.0.0/16", "172.25.0.1", [ + "10.42.3.0/24", + "192.168.1.5", + ]), + ).not.toThrow(); + for (const [subnet, gateway, forbidden] of [ + ["8.8.8.0/24", "8.8.8.1", []], + ["172.25.0.0/16", "172.26.0.1", []], + ["172.25.0.0/16", "172.25.0.1", ["172.25.3.0/24"]], + ["172.25.0.0/16", "172.25.0.1", ["172.25.3.5"]], + ["172.25.3.0/16", "172.25.3.1", []], + ] as [string, string, string[]][]) + expect(() => verifyNativeSubnet(subnet, gateway, forbidden)).toThrow(); + const value = [ + { + ifname: "owned7", + address: "02:42:ac:19:00:02", + addr_info: [{ family: "inet", local: "172.25.0.2" }], + }, + ]; + expect(nativeInterface(value, "172.25.0.2", "02:42:ac:19:00:02")).toBe( + "owned7", + ); + expect(() => + nativeInterface( + [{ ...value[0], addr_info: [{ family: "inet6", local: "172.25.0.2" }] }], + "172.25.0.2", + "02:42:ac:19:00:02", + ), + ).toThrow("native_link_interface"); + expect(() => + nativeInterface(value, "172.25.0.3", "02:42:ac:19:00:02"), + ).toThrow(); + expect(() => + nativeInterface([...value, ...value], "172.25.0.2", "02:42:ac:19:00:02"), + ).toThrow(); + expect(() => + nativeInterface(value, "172.25.0.2", "02:42:ac:19:00:03"), + ).toThrow(); + const inputs = { subnet: "10.42.3.0/24", nativeHostIp: "192.168.1.5" }; + const link = { gateway: "172.25.0.1", dev: "owned7", source: "172.25.0.2" }; + const route = { + dst: inputs.nativeHostIp, + gateway: link.gateway, + dev: link.dev, + }; + expect(() => + verifyNativeRouteState( + [route], + [{ ...route, prefsrc: link.source }], + inputs, + link, + ), + ).not.toThrow(); + expect(() => + verifyNativeRouteState( + [route, { dst: "default" }], + [{ ...route, prefsrc: link.source }], + inputs, + link, + ), + ).toThrow(); +}); + +test("cleanup selects only authenticated fixture or managed-principal sessions on the exact host", async () => { + const { ownedCleanupSessions } = await import("./main.ts"); + const id = "11111111-1111-4111-8111-111111111111"; + const session = { id, status: "active", hostId: "owned-host" }; + expect( + ownedCleanupSessions( + [{ session, ownerEmail: "owner@fixture.example" }], + "owned-host", + new Map(), + new Set(), + ), + ).toEqual([id]); + expect( + ownedCleanupSessions( + [{ session }], + "owned-host", + new Map([[id, "owned-actor"]]), + new Set(["owned-actor"]), + ), + ).toEqual([id]); + expect(() => + ownedCleanupSessions( + [{ session, ownerEmail: "foreign@example.test" }], + "owned-host", + new Map(), + new Set(), + ), + ).toThrow("cleanup_foreign_session"); + expect(() => + ownedCleanupSessions( + [ + { + session: { ...session, hostId: "foreign-host" }, + ownerEmail: "owner@fixture.example", + }, + ], + "owned-host", + new Map(), + new Set(), + ), + ).toThrow(); + expect(() => + ownedCleanupSessions( + [{ session }, { session }], + "owned-host", + new Map([[id, "owned-actor"]]), + new Set(["owned-actor"]), + ), + ).toThrow("cleanup_session_identity"); + expect( + ownedCleanupSessions( + [{ session: { ...session, status: "completed" } }], + "owned-host", + new Map(), + new Set(), + ), + ).toEqual([]); +}); + +test("session teardown ignores a deleted old automation task and cleans the exact pending session", async () => { + const { create } = + await import("../../../../orchestrator/node_modules/@bufbuild/protobuf/dist/esm/index.js"); + const { WhoAmIResponseSchema } = + await import("../../../../orchestrator/src/gen/engram/app/v1/api_key_pb.ts"); + const { GetAutomationResponseSchema, GetRunResponseSchema } = + await import("../../../../orchestrator/src/gen/engram/app/v1/automation_pb.ts"); + const { GetTaskResponseSchema } = + await import("../../../../orchestrator/src/gen/engram/app/v1/task_pb.ts"); + const { ListSessionsResponseSchema, DeleteSessionResponseSchema } = + await import("../../../../orchestrator/src/gen/engram/app/v1/session_pb.ts"); + const { recordOwnedAutomation } = + await import("../../../../orchestrator/scripts/mcp-stack/automation-ownership.ts"); + const { canonicalJson } = + await import("../../../../orchestrator/scripts/mcp-stack/guest.ts"); + const { publicClient } = + await import("../../../../orchestrator/scripts/mcp-stack/transport.ts"); + const { destroyRunSessions } = await import("./main.ts"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/automation-cleanup-"), + ); + try { + const ctx = await handoffFixture(root); + await mkdir(ctx.observer.hostManagedRoot, { mode: 0o700 }); + const sessionId = "11111111-1111-4111-8111-111111111111"; + const automationId = "fixture-automation"; + const runId = "autorun:fixture-automation:manual:fixture-event"; + const taskId = "fixture-root-task"; + const definition = { + blocks: [ + { + id: "launch", + type: "create_session", + config: { profileId: "fixture-profile" }, + }, + ], + }; + const reference = { + automationId, + definitionActorId: "fixture-admin", + definitionSha256: hash(canonicalJson(definition)), + profileId: "fixture-profile", + }; + await recordOwnedAutomation(ctx, reference); + await recordOwnedAutomation(ctx, { ...reference, runId }); + const oldAutomationId = "fixture-old-automation"; + const oldRunId = "autorun:fixture-old-automation:manual:old-event"; + const oldSessionId = "22222222-2222-4222-8222-222222222222"; + const oldTaskId = "deleted-old-task"; + const oldReference = { ...reference, automationId: oldAutomationId }; + await recordOwnedAutomation(ctx, oldReference); + await recordOwnedAutomation(ctx, { ...oldReference, runId: oldRunId }); + await recordOwnedAutomation(ctx, { + ...oldReference, + runId: oldRunId, + sessionId: oldSessionId, + taskId: oldTaskId, + }); + const admin = await publicClient(ctx, { + kind: "cookie", + value: "fixture=bootstrap", + }); + const destroyed: string[] = []; + const spies = [ + spyOn(admin.identity, "whoAmI").mockResolvedValue( + create(WhoAmIResponseSchema, { + email: "bootstrap@fixture.example", + role: "admin", + userId: "fixture-admin", + }), + ), + spyOn(admin.automations, "getAutomation").mockImplementation( + async (request) => { + const id = request.lookup?.case === "id" ? request.lookup.value : ""; + return create(GetAutomationResponseSchema, { + automation: { + id, + kind: "user", + createdByUserId: "fixture-admin", + currentVersion: 1, + version: { + automationId: id, + number: 1, + definitionJson: JSON.stringify(definition), + createdByUserId: "fixture-admin", + }, + }, + }); + }, + ), + spyOn(admin.automationRuns, "getRun").mockImplementation( + async (request) => { + const old = request.runId === oldRunId; + const sid = old ? oldSessionId : sessionId, + tid = old ? oldTaskId : taskId; + return create(GetRunResponseSchema, { + run: { + brief: { + id: request.runId, + automationId: old ? oldAutomationId : automationId, + version: 1, + triggerSource: "manual", + status: old ? "completed" : "waiting", + }, + steps: [ + { + blockId: "launch", + status: "succeeded", + sessionId: sid, + outputsJson: JSON.stringify({ + session_id: sid, + task_id: tid, + initial_prompt: false, + prompt: "", + }), + }, + ], + sessionIds: [sid], + }, + }); + }, + ), + spyOn(admin.tasks, "getTask").mockImplementation(async (request) => { + if (request.taskId === oldTaskId) + throw new Error("old_task_was_deleted"); + return create(GetTaskResponseSchema, { + task: { + id: taskId, + type: "automation", + sourceJson: JSON.stringify({ + provider: "automation", + automationId, + runId, + }), + sessions: [{ sessionId, role: "primary" }], + }, + }); + }), + spyOn(admin.sessions, "listSessions").mockImplementation(async () => + create(ListSessionsResponseSchema, { + sessions: destroyed.length + ? [] + : [{ session: { id: sessionId, status: "creating" } }], + }), + ), + spyOn(admin.sessions, "deleteSession").mockImplementation( + async (request) => { + if (request.sessionId !== sessionId) + throw new Error("foreign_cleanup_request"); + destroyed.push(request.sessionId); + return create(DeleteSessionResponseSchema); + }, + ), + ]; + try { + await destroyRunSessions(ctx, admin); + expect( + JSON.parse( + await readFile(join(ctx.runRoot, "session-cleanup.json"), "utf8"), + ), + ).toEqual({ destroyed: [sessionId], remaining: [] }); + } finally { + for (const spy of spies) spy.mockRestore(); + } + } finally { + await rm(root, { recursive: true }); + } +}); + +test("cleanup admits only the exact proven automation session before host assignment", async () => { + const { ownedCleanupSessions } = await import("./main.ts"); + const id = "11111111-1111-4111-8111-111111111111"; + const session = { id, status: "active" }; + expect( + ownedCleanupSessions( + [{ session }], + "owned-host", + new Map(), + new Set(), + new Set([id]), + ), + ).toEqual([id]); + for (const row of [ + { session: { ...session, id: "22222222-2222-4222-8222-222222222222" } }, + { session: { ...session, hostId: "foreign-host" } }, + ]) + expect(() => + ownedCleanupSessions( + [row], + "owned-host", + new Map(), + new Set(), + new Set([id]), + ), + ).toThrow("cleanup_foreign_session"); + expect(() => + ownedCleanupSessions( + [{ session }], + "owned-host", + new Map([[id, "automation:foreign"]]), + new Set(), + ), + ).toThrow("cleanup_foreign_session"); +}); + +test("VZ guard acquires and revalidates its exact link with complete address metadata", async () => { + const calls: string[][] = []; + const id = "b".repeat(64), + network = "d".repeat(64), + runId = "mcp-controlled-vz"; + let connected = false, + removed = false; + const endpoint = { + IPv4Address: "172.25.0.2/16", + MacAddress: "02:42:ac:19:00:02", + }; + const interfaces = [ + { + ifname: "owned7", + address: endpoint.MacAddress, + addr_info: [ + { family: "inet6", local: "fe80::42:acff:fe19:2" }, + { family: "inet", local: "172.25.0.2" }, + ], + }, + ]; + class VzPlan extends Engine { + override async project() { + return "controlled-trust"; + } + override networkOptions() { + return ["--network", "controlled-internal"]; + } + override async create(role: string) { + return role === "coordinator-guard" + ? id + : role === "worker-guard" + ? "c".repeat(64) + : "a".repeat(64); + } + override async inspect() { + return { + HostConfig: { + PortBindings: { + "32000/tcp": [{ HostIp: "127.0.0.1", HostPort: "32000" }], + }, + }, + }; + } + override async verifyNetwork() { + return { + generation: "controlled", + enforcementSha256: "e".repeat(64), + fixtureOnlyEgress: true as const, + externalDnsBlocked: true as const, + roles: ["orchestrator", "coordinator", "worker"] as ( + "orchestrator" | "coordinator" | "worker" + )[], + }; + } + override async call(args: string[]) { + calls.push(args); + if (args[0] === "network" && args[1] === "create") + return Buffer.from(network); + if (args[0] === "network" && args[1] === "connect") { + connected = true; + return Buffer.from(""); + } + if (args[0] === "network" && args[1] === "inspect") + return Buffer.from( + JSON.stringify([ + { + Id: network, + Name: `${runId}-native-network`, + Internal: false, + Driver: "bridge", + EnableIPv6: false, + Labels: { "engrams.mcp.run": runId }, + IPAM: { + Config: [{ Subnet: "172.25.0.0/16", Gateway: "172.25.0.1" }], + }, + Containers: connected ? { [id]: endpoint } : {}, + }, + ]), + ); + if (args.includes("address")) + return Buffer.from( + JSON.stringify( + args.includes("-4") + ? interfaces.map(({ ifname, addr_info }) => ({ + ifname, + addr_info: addr_info.filter((info) => info.family === "inet"), + })) + : interfaces, + ), + ); + if ( + args.includes("route") && + (args.includes("table") || args.includes("get")) + ) + return Buffer.from( + JSON.stringify([ + { + dst: "192.168.254.1", + gateway: "172.25.0.1", + dev: "owned7", + ...(args.includes("get") ? { prefsrc: "172.25.0.2" } : {}), + }, + ]), + ); + if (args.at(-1) === "default") + return Buffer.from( + JSON.stringify([ + { dst: "default", gateway: "172.25.0.1", dev: "owned7" }, + ]), + ); + if (args[0] === "network" && args[1] === "rm") { + removed = true; + return Buffer.from(network); + } + return Buffer.from(""); + } + } + const engine = new VzPlan( + { + backend: "vz", + subnet: "10.42.3.0/24", + nativeHostIp: "192.168.254.1", + registryPort: 32000, + images: { guard: { id: "sha256:" + "a".repeat(64) } }, + } as Inputs, + "/owned/run", + runId, + ); + await engine.guards({ path: "/owned/ca", sha256: "f".repeat(64) }); + await engine.verifyNativeRoute(); + const attach = calls.findIndex( + (args) => args[0] === "network" && args[1] === "connect", + ); + expect( + calls.slice(0, attach).filter((args) => args.includes("/usr/sbin/nft")), + ).toHaveLength(3); + expect(calls[attach]).toEqual(["network", "connect", network, id]); + expect(calls).toContainEqual([ + "exec", + id, + "/usr/sbin/ip", + "-4", + "route", + "del", + "default", + "via", + "172.25.0.1", + "dev", + "owned7", + ]); + expect(calls).toContainEqual([ + "exec", + id, + "/usr/sbin/ip", + "-4", + "route", + "replace", + "192.168.254.1/32", + "via", + "172.25.0.1", + "dev", + "owned7", + ]); + expect(removed).toBe(false); +}); + +test("FC process proof rejects a recycled PID, foreign cgroup, UID or executable", async () => { + const { verifyFcProcessIdentity } = await import("./fc-runtime.ts"); + const process = { + pid: 42, + bootId: "fixed-boot", + birth: "100", + uid: 0, + executable: "/usr/local/bin/firecracker", + cgroup: "/owned/container", + sha256: "a".repeat(64), + }; + expect(() => verifyFcProcessIdentity(process, { ...process })).not.toThrow(); + for (const actual of [ + undefined, + { ...process, birth: "101" }, + { ...process, bootId: "foreign-boot" }, + { ...process, uid: 1000 }, + { ...process, executable: "/foreign/firecracker" }, + { ...process, cgroup: "/foreign/container" }, + { ...process, sha256: "b".repeat(64) }, + ]) + expect(() => verifyFcProcessIdentity(process, actual)).toThrow( + "fc_process_identity_changed", + ); +}); + +test("secret projection reads back private file and directory modes and refuses a public key", async () => { + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/private-projection-"), + ); + const { put } = await import("./owned.ts"); + const file = await put(join(root, "client.key"), "controlled-private-key"); + let publicKey = false, + removed = false; + class Projection extends Engine { + override async volume() { + return "controlled-volume"; + } + override async create() { + return "a".repeat(64); + } + override async remove() { + removed = true; + } + override async call(args: string[]) { + if (args.includes("/usr/bin/stat")) + return Buffer.from( + args.includes("%u:%g:%a:%F:%h") + ? `1000:1000:${publicKey ? "644" : "600"}:regular file:1\n` + : "1000:1000:700:directory\n", + ); + if (args.includes("/usr/bin/sha256sum")) + return Buffer.from(file.sha256 + " /projection/client.key\n"); + return Buffer.from(""); + } + } + try { + const engine = new Projection( + { images: { guard: { id: "sha256:" + "a".repeat(64) } } } as Inputs, + root, + "controlled", + ); + await expect( + engine.project("host-tls", { "client.key": file }), + ).resolves.toBe("controlled-volume"); + expect(removed).toBe(true); + publicKey = true; + await expect( + engine.project("host-tls", { "client.key": file }), + ).rejects.toThrow("projection_file_mode"); + } finally { + await rm(root, { recursive: true }); + } +}); + +for (const partial of [false, true]) + test(`FC evidence write failure stays fatal after a later sample (${partial ? "partial append" : "missing append"})`, async () => { + const { FcRuntime } = await import("./fc-runtime.ts"); + const { appendFile } = await import("node:fs/promises"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/evidence-fault-"), + ); + const path = join(root, "fc-physical.jsonl"); + const record = + JSON.stringify({ + sandboxId: "controlled-sandbox", + bootAttempt: "controlled-boot", + readSectors: "1", + }) + "\n"; + let writes = 0; + const runtime = new FcRuntime( + { + bootId: "controlled", + release: "controlled", + pidNamespace: "pid:[1]", + pool: poolFixture(), + }, + {} as Inputs, + root, + async (file, bytes) => { + writes++; + if (writes === 1) { + if (partial) + await appendFile(file, bytes.slice(0, 15), { mode: 0o600 }); + throw new Error("controlled evidence write failure"); + } + await appendFile(file, bytes, { mode: 0o600 }); + }, + ); + try { + await expect( + runtime.retainObservation("controlled-sandbox:controlled-boot", record), + ).rejects.toThrow("controlled evidence write failure"); + // This would be a successful unrelated sample if the failure were lost. + await expect(runtime.observe()).rejects.toThrow( + "fc_physical_evidence_write", + ); + await expect(runtime.requireCoverage()).rejects.toThrow( + "fc_physical_evidence_write", + ); + await expect( + runtime.retainObservation("another-sandbox:another-boot", record), + ).rejects.toThrow("fc_physical_evidence_write"); + expect(writes).toBe(1); + if (partial) { + const retained = await readFile(path, "utf8"); + expect(retained).toBe(record.slice(0, 15)); + expect(() => JSON.parse(retained)).toThrow(); + } else + await expect(readFile(path)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + await rm(root, { recursive: true }); + } + }); + +test("matrix arguments forward the selected stage without replacing focused selectors", async () => { + const { matrixArguments } = await import("./main.ts"); + const key = "codex/catalog-fidelity"; + expect(matrixArguments("/owned/context.json").slice(1)).toEqual([ + "--context", + "/owned/context.json", + "--matrix", + "all", + ]); + expect( + matrixArguments("/owned/context.json", { + stage: "matrix", + matrix: key, + }).slice(1), + ).toEqual(["--context", "/owned/context.json", "--matrix", key]); + expect( + matrixArguments("/owned/context.json", { stage: "infrastructure" }).slice( + 1, + ), + ).toEqual(["--context", "/owned/context.json", "--stage", "infrastructure"]); +}); + +async function receiptFixture( + root: string, + selection: import("../../../../orchestrator/scripts/mcp-stack/manifest.ts").RunSelection, +) { + const { Evidence } = + await import("../../../../orchestrator/scripts/mcp-stack/evidence.ts"); + const { selectedCases } = + await import("../../../../orchestrator/scripts/mcp-stack/manifest.ts"); + const ctx = await handoffFixture(root); + const evidence = await Evidence.create(ctx, selection); + const hash = "a".repeat(64); + try { + await evidence.prerequisites({ + supervisorSha256: hash, + publicationSha256: hash, + publicStateSha256: hash, + }); + await evidence.infrastructure( + ["ordinary", "managed"].map((mode) => ({ + mode, + execSha256: hash, + uploadSha256: hash, + downloadSha256: hash, + sizeBytes: 4096, + unauthenticatedStatus: 401, + nonOwnerStatus: 404, + })), + ); + for (const selected of selectedCases(selection)) { + const digest = await evidence.observe({ + source: "guest-proof", + caseKey: selected.key, + identitySha256: hash, + state: "succeeded", + contentSha256: hash, + }); + await evidence.result({ + key: selected.key, + status: "PASS", + assertions: selected.assertions.map((assertion) => ({ + assertion, + observationHashes: [digest], + })), + }); + } + await evidence.finish(); + } finally { + await evidence.close(); + } + const source = await readFile(ctx.evidenceFile, "utf8"); + return { + ctx, + source, + records: source + .trim() + .split("\n") + .map((line) => JSON.parse(line)), + }; +} +for (const mode of [ + "full", + "filtered", + "infrastructure", + "explicit-all-keys", +] as const) { + test(`adapter completion validates real ${mode} Evidence writer output`, async () => { + const { verifyRunnerCompletion } = await import("./main.ts"); + const { REQUIRED_CASES } = + await import("../../../../orchestrator/scripts/mcp-stack/manifest.ts"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/receipt-"), + ); + try { + const selection = + mode === "infrastructure" + ? { stage: "infrastructure" as const } + : { + stage: "matrix" as const, + matrix: + mode === "full" + ? "all" + : mode === "explicit-all-keys" + ? REQUIRED_CASES.map((c) => c.key).join(",") + : "codex/catalog-fidelity,claude/catalog-fidelity", + }; + const { ctx, source, records } = await receiptFixture(root, selection); + await expect( + verifyRunnerCompletion(ctx, selection), + ).resolves.toBeUndefined(); + expect(records.at(-1).physicalAcceptance).toBe(mode === "full"); + if (mode === "full") { + expect(records.filter((r) => r.kind === "case")).toHaveLength(24); + expect( + records + .filter((r) => r.kind === "case") + .reduce((n, r) => n + r.assertions.length, 0), + ).toBe(138); + expect(Buffer.byteLength(source)).toBeLessThan(16 * 1024 * 1024); + } else + await expect( + verifyRunnerCompletion(ctx, { stage: "matrix", matrix: "all" }), + ).rejects.toThrow("runner_completion"); + } finally { + await rm(root, { recursive: true }); + } + }); +} +for (const malformed of [ + "terminal-only", + "contradictory-stage-flag", + "wrong-stage", + "duplicate-completion", + "copied-after-failed-case", + "missing-infrastructure", + "wrong-infrastructure-order", + "bad-infrastructure-hash", + "missing-case", + "failed-case", + "unrecorded-provenance", + "tampered-observation", + "wrong-run-header", + "wrong-selector", + "trailing-observation", + "unknown-record", + "empty-stream", + "torn-final-line", +] as const) { + test(`adapter completion rejects ${malformed} evidence`, async () => { + const { verifyRunnerCompletion } = await import("./main.ts"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/receipt-"), + ); + try { + const selection = { stage: "matrix" as const, matrix: "all" }; + const { ctx, records } = await receiptFixture(root, selection); + const terminal = records.at(-1), + observation = records.find((r) => r.kind === "observation"), + result = records.find((r) => r.kind === "case"), + infra = records.find((r) => r.kind === "infrastructure"); + let altered = [...records]; + switch (malformed) { + case "terminal-only": + altered = [terminal]; + break; + case "contradictory-stage-flag": + altered[altered.length - 1] = { ...terminal, stageCompleted: true }; + break; + case "wrong-stage": + altered[altered.length - 1] = { + ...terminal, + stage: "infrastructure", + }; + break; + case "duplicate-completion": + altered.push(terminal); + break; + case "copied-after-failed-case": + altered.push({ ...result, status: "FAIL", assertions: [] }, terminal); + break; + case "missing-infrastructure": + altered = altered.filter((r) => r.kind !== "infrastructure"); + break; + case "wrong-infrastructure-order": + infra.rows.reverse(); + break; + case "bad-infrastructure-hash": + infra.rows[0].downloadSha256 = "b".repeat(64); + break; + case "missing-case": + altered = altered.filter((r) => r !== result); + break; + case "failed-case": + result.status = "FAIL"; + result.assertions = []; + break; + case "unrecorded-provenance": + result.assertions[0].observationHashes = ["b".repeat(64)]; + break; + case "tampered-observation": + observation.contentSha256 = "b".repeat(64); + break; + case "wrong-run-header": + altered[0].sourceCommit = "f".repeat(40); + break; + case "wrong-selector": + altered[0].matrix = "claude/catalog-fidelity"; + break; + case "trailing-observation": + altered.push(observation); + break; + case "unknown-record": + altered.splice(1, 0, { version: 1, kind: "unknown" }); + break; + case "empty-stream": + altered = []; + break; + case "torn-final-line": + break; + } + let source = altered.map((r) => JSON.stringify(r)).join("\n") + "\n"; + if (malformed === "torn-final-line") source = source.slice(0, -4); + await writeFile(ctx.evidenceFile, source); + await expect(verifyRunnerCompletion(ctx, selection)).rejects.toThrow( + "runner_completion", + ); + } finally { + await rm(root, { recursive: true }); + } + }); +} + +for (const graceMs of [0, 100]) { + test(`owned process teardown waits for transient group EPERM with ${graceMs}ms grace`, async () => { + const { OwnedProcess, childEnvironment } = await import("./owned.ts"); + const owned = new OwnedProcess( + process.execPath, + ["-e", "setInterval(()=>{},1000)"], + process.cwd(), + childEnvironment(process.cwd()), + ); + const actualKill = process.kill.bind(process); + let retiring = false; + let release = false; + let denied = 0; + const kill = spyOn(process, "kill").mockImplementation((pid, signal) => { + if (pid !== -owned.child.pid!) return actualKill(pid, signal); + if (retiring && !release) { + denied++; + throw Object.assign(new Error("group not yet reaped"), { + code: "EPERM", + }); + } + const result = actualKill(pid, signal); + if (signal === "SIGTERM") retiring = true; + return result; + }); + let settled = false; + const stopping = owned.stop(graceMs).finally(() => { + settled = true; + }); + // Observe rejection before advancing the event loop. + const outcome = stopping.then( + () => undefined, + (error: unknown) => error, + ); + try { + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(denied).toBeGreaterThan(0); + expect(settled).toBe(false); + release = true; + expect(await outcome).toBeUndefined(); + await owned.done; + let absence: unknown; + try { + actualKill(-owned.child.pid!, 0); + } catch (error) { + absence = error; + } + expect(absence).toMatchObject({ code: "ESRCH" }); + } finally { + release = true; + await outcome; + kill.mockRestore(); + await owned.stop(100); + } + }); +} + +test("owned process teardown rejects persistent group EPERM after leader exit", async () => { + const { OwnedProcess, childEnvironment } = await import("./owned.ts"); + const owned = new OwnedProcess( + process.execPath, + ["-e", ""], + process.cwd(), + childEnvironment(process.cwd()), + ); + await owned.done; + const actualKill = process.kill.bind(process); + const kill = spyOn(process, "kill").mockImplementation((pid, signal) => { + if (pid !== -owned.child.pid!) return actualKill(pid, signal); + throw Object.assign(new Error("group permission denied"), { + code: "EPERM", + }); + }); + try { + await expect(owned.stop(0)).rejects.toThrow("FAILED:process_cleanup"); + } finally { + kill.mockRestore(); + await owned.stop(100); + } +}); + +for (const stage of ["infrastructure", "selected-case"] as const) { + for (const started of [false, true]) { + for (const canceled of [false, true]) { + test(`${stage} ${canceled ? "cancellation" : "failure"} ${started ? "after" : "before"} native startup uses shared teardown`, async () => { + const { cleanupPlatform } = await import("./main.ts"); + const { OwnedProcess, childEnvironment } = await import("./owned.ts"); + const fixture = await nativeCleanupFixture(); + const unrelated = new OwnedProcess( + process.execPath, + ["-e", "setInterval(()=>{},1000)"], + process.cwd(), + childEnvironment(process.cwd()), + ); + const runner = new OwnedProcess( + process.execPath, + ["-e", "setInterval(()=>{},1000)"], + process.cwd(), + childEnvironment(process.cwd()), + ); + if (!started) { + Reflect.set(fixture.stack, "host", undefined); + fixture.setIdentities([]); + } + const events: string[] = []; + const abort = new AbortController(); + const failure = new PlatformFailure( + "FAILED", + canceled ? "run_aborted" : `${stage}_failure`, + ); + let failed: string[] = []; + try { + await expect( + (async () => { + try { + if (canceled) abort.abort(); + throw failure; + } finally { + failed = await cleanupPlatform({ + stack: fixture.stack, + runner, + }); + events.push("cleanup"); + } + })(), + ).rejects.toBe(failure); + expect(failed).toEqual([]); + expect(events).toEqual(["cleanup"]); + expect(fixture.events).toEqual( + started ? ["41001:SIGTERM", "host-complete", "41002:SIGTERM"] : [], + ); + expect(fixture.identities()).toEqual([]); + expect(unrelated.running()).toBe(true); + expect(runner.running()).toBe(false); + expect(abort.signal.aborted).toBe(canceled); + } finally { + // Restore the process signal boundary before retiring the separate real child. + await fixture.close(); + await unrelated.stop(100); + await runner.stop(100); + } + }); + } + } +} + +test("shared teardown reports changed native identity without signaling it or an unrelated child", async () => { + const { cleanupPlatform } = await import("./main.ts"); + const fixture = await nativeCleanupFixture(); + fixture.setIdentities([ + { ...fixture.leader, seconds: "1730000001" }, + fixture.identities()[1]!, + ]); + try { + expect(await cleanupPlatform({ stack: fixture.stack })).toEqual(["host"]); + expect(fixture.events).toEqual([]); + expect(fixture.identities()).toHaveLength(2); + } finally { + await fixture.close(); + } +}); + +test("network guard checks run together and drain siblings before releasing control on failure", async () => { + const { GUARDS, address, rules } = await import("./network.ts"); + const inputs = { + backend: "firecracker", + subnet: "10.42.3.0/24", + nativeHostIp: "10.42.4.1", + } as Inputs; + const entered: string[] = [], + finished: string[] = []; + const allEntered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const failure = new Error("controlled guard failure"); + const runId = "mcp-guard-fence"; + class GuardFence extends Engine { + override async verifyNativeRoute() {} + override async audit(id: string) { + entered.push(id); + if (entered.length === 3) allEntered.resolve(); + if (id === "orchestrator") throw failure; + await release.promise; + return { + State: { Running: true }, + NetworkSettings: { + Networks: { + [runId + "-network"]: { + IPAddress: address( + inputs, + id === "worker" ? "worker" : "coordinator", + ), + }, + }, + }, + }; + } + override async verifyResolver() {} + override async call(args: string[]): Promise { + if (args[0] === "network") + return Buffer.from( + JSON.stringify([ + { + Id: "network", + Internal: true, + Labels: { "engrams.mcp.run": runId }, + }, + ]), + ); + const role = args[1] === "worker" ? "worker" : "coordinator"; + if (args.includes("/usr/sbin/nft")) + return Buffer.from(JSON.stringify({ nftables: rules(inputs, role) })); + if (args.includes("+short")) + return Buffer.from(address(inputs, "modern") + "\n"); + if (args.includes("+comments")) + return Buffer.from("status: NXDOMAIN, ANSWER: 0,"); + if (args.includes("/bin/sh")) { + finished.push(role); + return Buffer.from("blocked"); + } + return Buffer.alloc(0); + } + } + const engine = new GuardFence(inputs, "/owned/run", runId); + Reflect.set(engine, "networkId", "network"); + for (const role of GUARDS) engine.guardIds.set(role, role); + const signal = new AbortController().signal; + let settled = false; + const result = engine + .controlled(signal, () => engine.verifyNetwork(true)) + .then( + () => { + settled = true; + return undefined; + }, + (error) => { + settled = true; + return error; + }, + ); + try { + await Promise.race([allEntered.promise, result]); + expect(entered).toEqual(GUARDS); + expect(settled).toBe(false); + await expect(engine.controlled(signal, async () => {})).rejects.toThrow( + "engine_control_busy", + ); + release.resolve(); + expect(await result).toBe(failure); + expect(finished.sort()).toEqual(["coordinator", "worker"]); + await expect( + engine.controlled(signal, async () => {}), + ).resolves.toBeUndefined(); + } finally { + release.resolve(); + await result; + } +}); + +test("role status checks run together and drain siblings before releasing control on failure", async () => { + const { Stack } = await import("./launch.ts"); + const { ROLES } = + await import("../../../../orchestrator/scripts/mcp-stack/supervisor.ts"); + const { GUARDS } = await import("./network.ts"); + const root = await mkdtemp( + join(process.cwd(), "var/mcp-platform-composition/status-fence-"), + ); + const ctx = await handoffFixture(root); + ctx.backend = "firecracker"; + const pin = ctx.artifacts.coordinator; + const inputs = { + backend: "firecracker", + images: Object.fromEntries( + ["coordinator", "orchestrator", "worker", "fixture", "hostAgent"].map( + (role) => [role, { id: role }], + ), + ), + } as Inputs; + const entered: string[] = [], + finished: string[] = []; + const allEntered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const failure = new Error("controlled role failure"); + let fail = true; + class RoleFence extends Engine { + override async audit(id: string) { + entered.push(id); + if (entered.length === ROLES.length) allEntered.resolve(); + if (id === "orchestrator" && fail) throw failure; + await release.promise; + finished.push(id); + return { + State: { Running: true }, + Image: id.startsWith("fixture-") ? "fixture" : id, + }; + } + override async verifyResolver() {} + override async qualifyFcHost() {} + override async qualifyNativeHost() {} + override async inspect() { + return { State: { Running: true, Pid: 123 } }; + } + override async verifyNetwork() { + return { + generation: "controlled", + enforcementSha256: "e".repeat(64), + fixtureOnlyEgress: true as const, + externalDnsBlocked: true as const, + roles: GUARDS, + }; + } + } + const engine = new RoleFence(inputs, root, ctx.runId, { + bootId: "controlled", + release: "controlled", + pidNamespace: "pid:[1]", + pool: poolFixture(), + }); + const kernel = spyOn(engine.fc!, "qualifyKernel").mockResolvedValue( + undefined, + ); + const observe = spyOn(engine.fc!, "observe").mockResolvedValue(undefined); + const identity = { + ca: pin, + leaves: { + coordinator: { certificate: pin, key: pin }, + host: { certificate: pin }, + }, + } as ConstructorParameters[2]; + const stack = new Stack( + inputs, + root, + identity, + {} as ConstructorParameters[3], + {} as ConstructorParameters[4], + engine, + ); + Reflect.set( + stack, + "active", + new Map(ROLES.map((role) => [role, { id: role, configuration: "normal" }])), + ); + Reflect.set( + stack, + "configs", + new Map( + ["orchestrator", "coordinator", "worker", "host-agent"].map((role) => [ + role, + { normal: { env: {}, files: {} } }, + ]), + ), + ); + const signal = new AbortController().signal; + let settled = false; + const result = engine + .controlled(signal, () => stack.status(ctx)) + .then( + () => { + settled = true; + return undefined; + }, + (error) => { + settled = true; + return error; + }, + ); + try { + await Promise.race([allEntered.promise, result]); + expect([...entered].sort()).toEqual([...ROLES].sort()); + expect(settled).toBe(false); + await expect(engine.controlled(signal, async () => {})).rejects.toThrow( + "engine_control_busy", + ); + release.resolve(); + expect(await result).toBe(failure); + expect(finished).toHaveLength(ROLES.length - 1); + fail = false; + const status = await engine.controlled(signal, () => stack.status(ctx)); + expect(status.roles.map((role) => role.role)).toEqual([...ROLES]); + expect(status.roles.every((role) => role.state === "running")).toBe(true); + } finally { + release.resolve(); + await result; + kernel.mockRestore(); + observe.mockRestore(); + await rm(root, { recursive: true }); + } +}); + +test("drained check results keep input order when completion order differs", async () => { + const { drainChecks } = await import("./engine.ts"); + const first = Promise.withResolvers(); + const second = Promise.withResolvers(); + const result = drainChecks([first.promise, second.promise]); + second.resolve("second"); + first.resolve("first"); + expect(await result).toEqual(["first", "second"]); +}); diff --git a/tests/fixtures/mcp-stack/platform/configuration.ts b/tests/fixtures/mcp-stack/platform/configuration.ts new file mode 100644 index 000000000..6962acbdb --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/configuration.ts @@ -0,0 +1,260 @@ +import { join } from "node:path"; +import { ZonePolicy } from "../../../../mcp-worker/src/network.ts"; +import { validateEgressRules } from "../../../../mcp-worker/src/deployment.ts"; +import { canonicalJson } from "../../../../orchestrator/src/mcp/hash.ts"; +import { parsePublishedOperator } from "../../../../orchestrator/src/mcp/operator.ts"; +import { loadMcpRuntimeConfiguration } from "../../../../orchestrator/src/mcp/runtime-config.ts"; +import { hash, newRoot, put, readOwned, type Pin } from "./owned.ts"; +import { address } from "./network.ts"; +import type { Inputs } from "./preflight.ts"; +import type { Identity } from "./identity.ts"; + +export async function generateConfiguration( + inputs: Inputs, + root: string, + identity: Identity, +) { + const configRoot = join(root, "configuration"); + await newRoot(configRoot); + const save = (name: string, value: unknown) => + put(join(configRoot, name), JSON.stringify(value)); + const ca = (await readOwned(identity.ca.path)).toString(); + const authorities = ["legacy", "modern"] + .flatMap((protocol) => + ["resource", "issuer", "authorization", "token"].map( + (role) => `${role}.${protocol}.fixture.example:8443`, + ), + ) + .sort(); + const zone = { + id: "fixture-zone", + revision: "fixture-1", + mode: "internal" as const, + authorities, + aliases: [], + cidrs: [ + address(inputs, "modern") + "/32", + address(inputs, "legacy") + "/32", + ], + denyCidrs: [ + inputs.nativeHostIp + "/32", + address(inputs, "worker") + "/32", + address(inputs, "coordinator") + "/32", + address(inputs, "orchestrator") + "/32", + "169.254.0.0/16", + ], + ca, + limits: { execute: 8, discover: 2, oauth: 2, schema: 2, toolMs: 10000 }, + }; + const policy = new ZonePolicy(zone); + const egressRules = zone.cidrs.map((cidr) => ({ cidr, except: [] })); + validateEgressRules(egressRules, zone.denyCidrs, zone.cidrs, zone.mode); + const infrastructure = { + deploymentIdentity: `sha256:${hash(JSON.stringify({ runId: identity.runId, backend: inputs.backend, sourceCommit: inputs.sourceCommit }))}`, + networkIdentity: `sha256:${hash(JSON.stringify({ subnet: inputs.subnet, nativeHostIp: inputs.nativeHostIp }))}`, + trustIdentity: `sha256:${identity.ca.sha256}`, + caContentDigests: { fixture: `sha256:${identity.ca.sha256}` }, + }; + const candidate = { + version: 1, + zones: { + [zone.id]: { + revision: zone.revision, + worker_digest: `sha256:${policy.digest}`, + authorities, + aliases: [], + configuration: JSON.parse(canonicalJson(zone)), + }, + }, + deployment_identity: infrastructure.deploymentIdentity, + network_identity: infrastructure.networkIdentity, + trust_identity: infrastructure.trustIdentity, + ca_content_digests: infrastructure.caContentDigests, + }; + const candidateIdentity = `sha256:${hash(JSON.stringify(candidate))}`; + // Reuse the production Rust-serialization/ZonePolicy identity verifier. + parsePublishedOperator( + Buffer.from( + JSON.stringify({ + published: { + generation: 1, + expected_generation: null, + identity: candidateIdentity, + candidate, + }, + pending: null, + admission_open: true, + }), + ), + infrastructure, + ); + const candidateFile = await save("operator-candidate.json", candidate); + const tls = (url: string) => ({ + url, + caName: "fixture", + certFile: identity.leaves.orchestrator.certificate.path, + keyFile: identity.leaves.orchestrator.key.path, + serverFingerprint: url.includes("worker.") + ? identity.leaves.worker.fingerprint + : identity.leaves.coordinator.fingerprint, + }); + const registrations = ["modern", "legacy"].map((protocol) => ({ + id: `fixture-${protocol}`, + clientId: "fixture-public", + confidential: false, + redirectUri: "https://app.fixture.example/integrations/mcp/callback", + issuer: `https://issuer.${protocol}.fixture.example:8443`, + resource: { + zoneId: zone.id, + endpoint: `https://resource.${protocol}.fixture.example:8443/mcp`, + }, + scopes: + protocol === "modern" ? ["tools:read"] : ["tools:read", "tools:write"], + })); + const runtime = { + version: 1, + infrastructure: { + deploymentIdentity: infrastructure.deploymentIdentity, + networkIdentity: infrastructure.networkIdentity, + trustIdentity: infrastructure.trustIdentity, + }, + caFiles: { fixture: identity.ca.path }, + control: tls("https://control.fixture.example:8444"), + workers: [ + { + zoneId: zone.id, + deploymentId: identity.runId, + enabled: true, + tls: tls("https://worker.fixture.example:8443"), + }, + ], + registrations, + }; + const runtimeNormal = await save("orchestrator-normal.json", runtime); + await loadMcpRuntimeConfiguration(runtimeNormal.path); + const runtimeZoneDisabled = await save("orchestrator-zone-disabled.json", { + ...runtime, + workers: runtime.workers.map((worker) => ({ ...worker, enabled: false })), + }); + await loadMcpRuntimeConfiguration(runtimeZoneDisabled.path); + const worker = await save("worker.json", { + zone, + callers: [ + { + role: "orchestrator", + fingerprint: identity.leaves.orchestrator.fingerprint, + }, + { + role: "coordinator", + fingerprint: identity.leaves.coordinator.fingerprint, + }, + ], + egressRules, + }); + const clientIdentity = await put( + join(configRoot, "coordinator-client.pem"), + Buffer.concat([ + await readOwned(identity.leaves.coordinator.certificate.path), + await readOwned(identity.leaves.coordinator.key.path), + ]), + ); + const destination = (endpoint: string) => ({ zoneId: zone.id, endpoint }); + const coordinator = await save("coordinator.json", { + registry: { + enabled: true, + deploymentId: identity.runId, + zones: { + [zone.id]: { + policyDigest: policy.digest, + policyRevision: zone.revision, + endpoints: ["legacy", "modern"] + .flatMap((protocol) => [ + `https://resource.${protocol}.fixture.example:8443/mcp`, + `https://resource.${protocol}.fixture.example:8443/.well-known/oauth-protected-resource`, + `https://issuer.${protocol}.fixture.example:8443/.well-known/oauth-authorization-server`, + `https://authorization.${protocol}.fixture.example:8443/authorize`, + `https://token.${protocol}.fixture.example:8443/token`, + ]) + .sort(), + }, + }, + registrations: Object.fromEntries( + registrations.map((registration) => { + const protocol = registration.id.slice("fixture-".length); + return [ + registration.id, + { + ...registration, + resourceMetadata: destination( + `https://resource.${protocol}.fixture.example:8443/.well-known/oauth-protected-resource`, + ), + authorizationMetadata: destination( + `https://issuer.${protocol}.fixture.example:8443/.well-known/oauth-authorization-server`, + ), + authorization: destination( + `https://authorization.${protocol}.fixture.example:8443/authorize`, + ), + token: destination( + `https://token.${protocol}.fixture.example:8443/token`, + ), + issuerParameterRequired: true, + accountField: "fixture_account", + }, + ]; + }), + ), + }, + listenAddr: "0.0.0.0:8444", + serverCertificateFile: identity.leaves.coordinator.certificate.path, + serverPrivateKeyFile: identity.leaves.coordinator.key.path, + orchestratorCaFile: identity.ca.path, + orchestratorFingerprints: [identity.leaves.orchestrator.fingerprint], + workers: { + [zone.id]: { + url: "https://worker.fixture.example:8443", + serverCaFile: identity.ca.path, + clientIdentityFile: clientIdentity.path, + }, + }, + publication: { + listenAddr: "0.0.0.0:8445", + serverCertificateFile: identity.leaves.coordinator.certificate.path, + serverPrivateKeyFile: identity.leaves.coordinator.key.path, + operatorCaFile: identity.ca.path, + operatorFingerprints: [identity.leaves.operator.fingerprint], + candidateFile: candidateFile.path, + caFiles: { fixture: identity.ca.path }, + }, + }); + const hostTls = await save("host-tls.json", { + certificate: identity.leaves.host.certificate.path, + key: identity.leaves.host.key.path, + client_ca: identity.ca.path, + coordinator_pins: [identity.leaves.coordinator.hostPin], + }); + const coordinatorHostTls = await save("coordinator-host-tls.json", { + client_certificate: identity.leaves.coordinator.certificate.path, + client_key: identity.leaves.coordinator.key.path, + hosts: { + [identity.hostId]: { + endpoint: "https://host.fixture.example:9101", + server_name: "host.fixture.example", + server_ca: identity.ca.path, + server_pins: [identity.leaves.host.hostPin], + }, + }, + }); + return { + zone, + infrastructure, + candidateFile, + candidateIdentity, + runtimeNormal, + runtimeZoneDisabled, + worker, + coordinator, + clientIdentity, + hostTls, + coordinatorHostTls, + }; +} diff --git a/tests/fixtures/mcp-stack/platform/engine.ts b/tests/fixtures/mcp-stack/platform/engine.ts new file mode 100644 index 000000000..d92ef1c8d --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/engine.ts @@ -0,0 +1,2168 @@ +import { FcRuntime, type FcInventory } from "./fc-runtime.ts"; +import { randomBytes, X509Certificate } from "node:crypto"; +import { lookup } from "node:dns/promises"; +import type { LookupAddress } from "node:dns"; +import { lstat, mkdir, readFile, rm } from "node:fs/promises"; +import { createServer } from "node:net"; +import { join } from "node:path"; +import { networkInterfaces } from "node:os"; +import { + childEnvironment, + check, + OwnedProcess, + hash, + PlatformFailure, + put, + readOwned, + verifyPin, + ownedPath, + type Pin, +} from "./owned.ts"; +import { + address, + dnsRecords, + effectiveRules, + GUARDS, + rules, + type GuardRole, +} from "./network.ts"; +import { + type Inputs, + REPO, + engineArguments, + validateEngineConfig, +} from "./preflight.ts"; + +// Every check keeps its own command lifetime. Drain all siblings before the +// control scope can release its shared cancellation signal. +export async function drainChecks( + checks: readonly Promise[], +): Promise { + const results = await Promise.allSettled(checks); + return results.map((result) => { + if (result.status === "rejected") throw result.reason; + return result.value; + }); +} + +type NativeRoute = { gateway: string; dev: string; source: string }; +const ipv4 = (value: string) => { + const parts = value.split(".").map(Number); + check( + parts.length === 4 && + parts.every((v) => Number.isInteger(v) && v >= 0 && v <= 255) && + parts.join(".") === value, + "native_link_ip", + ); + return parts.reduce((a, v) => a * 256 + v, 0); +}; +export function verifyNativeSubnet( + subnet: string, + gateway: string, + forbidden: string[], +) { + const [ip, prefix, ...rest] = subnet.split("/"); + check( + ip && prefix && !rest.length && /^(?:[1-9]|[12][0-9]|30)$/.test(prefix), + "native_link_subnet", + ); + const first = ipv4(ip), + count = 2 ** (32 - Number(prefix)); + check( + first % count === 0 && + ((first >= ipv4("10.0.0.0") && first + count <= ipv4("11.0.0.0")) || + (first >= ipv4("172.16.0.0") && first + count <= ipv4("172.32.0.0")) || + (first >= ipv4("192.168.0.0") && first + count <= ipv4("192.169.0.0"))), + "native_link_private", + ); + check( + ipv4(gateway) > first && ipv4(gateway) < first + count - 1, + "native_link_gateway", + ); + for (const entry of forbidden) { + const [address, bits = "32"] = entry.split("/"); + const start = ipv4(address!), + end = start + 2 ** (32 - Number(bits)); + check(first + count <= start || first >= end, "native_link_overlap"); + } +} +export function nativeInterface( + value: unknown, + address: string, + mac: string, +): string { + check( + Array.isArray(value) && /^[a-f0-9]{2}(?::[a-f0-9]{2}){5}$/.test(mac), + "native_link_shape", + ); + const matches = value.filter( + (v) => + v && + v.address === mac && + Array.isArray(v.addr_info) && + v.addr_info.some( + (a: { family?: string; local?: string }) => + a.family === "inet" && a.local === address, + ), + ); + check( + matches.length === 1 && /^[a-zA-Z0-9_-]{1,15}$/.test(matches[0].ifname), + "native_link_interface", + ); + return matches[0].ifname; +} +export function verifyNativeRouteState( + value: unknown, + selected: unknown, + inputs: Pick, + link: NativeRoute = { + gateway: inputs.subnet.slice(0, -4) + "1", + dev: "eth0", + source: address(inputs, "coordinator"), + }, +) { + check( + Array.isArray(value) && Array.isArray(selected) && selected.length === 1, + "native_route_shape", + ); + const objects = value.map((route: unknown) => { + check( + route !== null && typeof route === "object" && !Array.isArray(route), + "native_route_shape", + ); + return route as Record; + }); + check( + !objects.some( + (route) => route.dst === "default" || route.dst === "0.0.0.0/0", + ), + "native_route_default", + ); + const exact = objects.filter( + (route) => + route.dst === inputs.nativeHostIp || + route.dst === `${inputs.nativeHostIp}/32`, + ); + const { gateway } = link; + check( + exact.length === 1 && + exact[0]!.gateway === gateway && + exact[0]!.dev === link.dev && + !exact[0]!.multipath && + (!exact[0]!.type || exact[0]!.type === "unicast"), + "native_route_exact", + ); + const route = selected[0]; + check( + route !== null && + typeof route === "object" && + !Array.isArray(route) && + route.dst === inputs.nativeHostIp && + route.gateway === gateway && + route.dev === link.dev && + route.prefsrc === link.source && + !route.multipath, + "native_route_selected", + ); +} +export function verifyNativeHostTls( + output: Uint8Array, + expectedFingerprint: string, +) { + const text = Buffer.from(output).toString(); + check( + output.length <= 65536 && + /(?:New,|Protocol\s*:)\s*TLSv1\.3\b/.test(text) && + /Verify return code: 0 \(ok\)/.test(text), + "native_host_tls_verification", + ); + const certificate = + /-----BEGIN CERTIFICATE-----[\s\S]*?-----END CERTIFICATE-----/.exec(text); + check( + certificate && + new X509Certificate(certificate[0]).fingerprint256 === + expectedFingerprint, + "native_host_tls_leaf", + ); + return hash(new X509Certificate(certificate[0]).raw); +} +type NativeHostTls = { + ca: Pin; + clientCertificate: Pin; + clientKey: Pin; + serverCertificate: Pin; +}; + +// Match the fixed production host_startup rules in net.rs. Per-VM SNAT +// rules live in each VM's own netns and must not appear in these chains. +export function fcHostFirewall(port: number): Record { + const pool = "10.200.0.0/16"; + const rule = ( + chain: string, + match: string, + action: string, + comment: string, + ) => `-A ${chain} ${match} -j ${action} -m comment --comment ${comment}`; + const input = (protocol: string, p: number, comment: string) => + rule("INPUT", `-s ${pool} -p ${protocol} --dport ${p}`, "ACCEPT", comment); + const redirect = ( + i: string, + protocol: string, + p: number, + to: number, + comment: string, + dest = "", + ) => + rule( + "PREROUTING", + `-i ${i} -p ${protocol} ${dest} --dport ${p}`, + `REDIRECT --to-port ${to}`, + comment, + ); + return { + INPUT: [ + input("tcp", port + 3, "engram-guest-gateway-input"), + input("tcp", port + 2, "engram-proxy-dns-input"), + input("udp", port + 2, "engram-proxy-dns-input"), + input("tcp", port + 1, "engram-proxy-input"), + rule( + "INPUT", + `-s ${pool} -m conntrack --ctstate ESTABLISHED,RELATED`, + "ACCEPT", + "engram-host-input-established", + ), + rule("INPUT", `-s ${pool}`, "DROP", "engram-host-input"), + ], + FORWARD: [ + rule("FORWARD", `-s ${pool} -d ${pool}`, "DROP", "engram-isolate-vm-vm"), + ...[ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "169.254.0.0/16", + "127.0.0.0/8", + ].map((net) => + rule("FORWARD", `-s ${pool} -d ${net}`, "DROP", "engram-host-lan"), + ), + rule("FORWARD", `-s ${pool}`, "DROP", "engram-default-deny"), + ], + PREROUTING: [ + ...["vh-engr-+", "tap-engr-+"].map((i) => + redirect( + i, + "tcp", + 80, + port + 3, + "engram-guest-gateway-redirect", + "-d 169.254.169.254/32", + ), + ), + ...["tap-engr-+", "vh-engr-+"].map((i) => + redirect(i, "tcp", 443, port + 1, "engram-proxy-redirect"), + ), + ...["udp", "tcp"].flatMap((proto) => + ["tap-engr-+", "vh-engr-+"].map((i) => + redirect(i, proto, 53, port + 2, "engram-dns-redirect"), + ), + ), + ], + POSTROUTING: [ + rule( + "POSTROUTING", + `-s ${pool} ! -d ${pool}`, + "MASQUERADE", + "engram-masq", + ), + ], + }; +} +function firewallFields(line: string): Record { + const tokens = line.match(/"[^"\r\n]*"|[^\s]+/g) ?? []; + const fields: Record = {}; + let negate = false; + for (let i = 0; i < tokens.length; i++) { + let key = tokens[i]!; + if (key === "!") { + check(!negate, "fc_firewall_rule"); + negate = true; + continue; + } + const value = tokens[++i]?.replace(/^"|"$/g, ""); + check((value && !negate) || (value && key === "-d"), "fc_firewall_rule"); + if (key === "-m") { + check( + ["comment", "tcp", "udp", "conntrack"].includes(value!), + "fc_firewall_rule", + ); + continue; + } + if (key === "--to-ports") key = "--to-port"; + check( + [ + "-A", + "-s", + "-d", + "-i", + "-p", + "--dport", + "--sport", + "--to-destination", + "--to-source", + "-j", + "--to-port", + "--ctstate", + "--comment", + ].includes(key), + "fc_firewall_rule", + ); + if (negate) key = "!" + key; + check(!(key in fields), "fc_firewall_rule"); + fields[key] = + key === "--ctstate" ? value!.split(",").sort().join(",") : value!; + negate = false; + } + check(!negate, "fc_firewall_rule"); + return fields; +} +function firewallRule(line: string): string { + return JSON.stringify(Object.entries(firewallFields(line)).sort()); +} +function hasDockerDns(output: string): boolean { + const first = output.trim().split("\n")[1]; + return ( + first !== undefined && + firewallRule(first) === + firewallRule("-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING") + ); +} +export function verifyFcHostFirewall( + actual: Record, + port: number, +) { + const expected = fcHostFirewall(port); + if (hasDockerDns(actual.POSTROUTING ?? "")) { + const output = actual.DOCKER_OUTPUT?.trim().split("\n") ?? []; + const ports = ["tcp", "udp"].map((protocol, index) => { + const target = firewallFields(output[index + 1] ?? "")[ + "--to-destination" + ]; + const match = /^127\.0\.0\.11:([1-9][0-9]{0,4})$/.exec(target ?? ""); + check(match && Number(match[1]) <= 65535, "fc_dns_port"); + return { protocol, port: match[1]! }; + }); + expected.POSTROUTING!.unshift( + "-A POSTROUTING -d 127.0.0.11/32 -j DOCKER_POSTROUTING", + ); + expected.OUTPUT = ["-A OUTPUT -d 127.0.0.11/32 -j DOCKER_OUTPUT"]; + expected.DOCKER_OUTPUT = ports.map( + ({ protocol, port }) => + `-A DOCKER_OUTPUT -d 127.0.0.11/32 -p ${protocol} --dport 53 -j DNAT --to-destination 127.0.0.11:${port}`, + ); + expected.DOCKER_POSTROUTING = ports.map( + ({ protocol, port }) => + `-A DOCKER_POSTROUTING -s 127.0.0.11/32 -p ${protocol} --sport ${port} -j SNAT --to-source :53`, + ); + } + check( + Object.keys(actual).sort().join(",") === + Object.keys(expected).sort().join(","), + "fc_firewall_chains", + ); + for (const [chain, lines] of Object.entries(expected)) { + const current = actual[chain]!.trim().split("\n"); + check( + current.shift() === + (chain.startsWith("DOCKER_") ? `-N ${chain}` : `-P ${chain} ACCEPT`), + "fc_firewall_policy", + ); + check( + JSON.stringify(current.map(firewallRule)) === + JSON.stringify(lines.map(firewallRule)), + "fc_firewall_changed", + ); + } +} +export const fcGuardSysctls = ( + backend: Inputs["backend"], + role: string, +): Record => + backend === "firecracker" && role === "coordinator-guard" + ? { "net.ipv4.ip_forward": "1", "net.ipv4.conf.all.route_localnet": "1" } + : {}; + +/** Fixed per-run Linux resources. No application request reaches these commands. */ +export class Engine { + private readonly containers = new Map(); + private readonly plannedContainers = new Map(); + private readonly volumes = new Map(); + private networkId: string | undefined; + private plannedNetworkName: string | undefined; + private nativeNetworkId: string | undefined; + private nativeNetworkName: string | undefined; + private nativeLink: + | (NativeRoute & { subnet: string; mac: string }) + | undefined; + readonly guardIds = new Map(); + private readonly generation = randomBytes(16).toString("hex"); + private baseline: string | undefined; + private configState: + | Awaited> + | undefined; + private socketState: { ino: number; dev: number } | undefined; + private resolver: Pin | undefined; + private resolverVolume: string | undefined; + private nativeHostProjection: + | { volume: string; identity: string } + | undefined; + private readonly specifications = new Map< + string, + { + image: string; + options: string[]; + args: string[]; + entrypoint?: string; + host: boolean; + role: string; + } + >(); + private signal: AbortSignal | undefined; + checkCancellation() { + this.signal?.throwIfAborted(); + } + async controlled( + signal: AbortSignal, + operation: () => Promise, + ): Promise { + check(!this.signal, "engine_control_busy"); + this.signal = signal; + try { + signal.throwIfAborted(); + return await operation(); + } finally { + this.signal = undefined; + } + } + private async engineIdentity() { + this.configState = await validateEngineConfig( + this.inputs.engineConfig, + this.configState, + ); + await ownedPath(this.inputs.engineSocket); + const socket = await lstat(this.inputs.engineSocket); + check(socket.isSocket(), "engine_socket"); + if (this.socketState) + check( + socket.ino === this.socketState.ino && + socket.dev === this.socketState.dev, + "engine_socket_changed", + ); + else this.socketState = { ino: socket.ino, dev: socket.dev }; + } + readonly fc: FcRuntime | undefined; + constructor( + readonly inputs: Inputs, + readonly root: string, + readonly runId: string, + inventory?: FcInventory, + ) { + this.fc = inventory ? new FcRuntime(inventory, inputs, root) : undefined; + } + private label() { + return `engrams.mcp.run=${this.runId}`; + } + private name(role: string) { + check(/^[a-z0-9-]{1,64}$/.test(role), "resource_role"); + return `${this.runId}-${role}`; + } + async call(args: string[], stdin?: string, timeout = 10000): Promise { + this.signal?.throwIfAborted(); + await this.engineIdentity(); + await verifyPin(this.inputs.binaries.docker); + const prefix = engineArguments(this.inputs).slice(0, 4); + try { + const captureStderr = + args.length === 4 && + args[0] === "logs" && + args[1] === "--tail" && + args[2] === "10000" && + /^[a-f0-9]{64}$/.test(args[3]!); + const bytes = await this.runEngine( + [...prefix, ...args], + stdin, + timeout, + captureStderr, + ); + return bytes; + } finally { + await this.engineIdentity(); + } + } + private async runEngine( + args: string[], + input: string | undefined, + timeout: number, + captureStderr = false, + ): Promise { + check( + Number.isInteger(timeout) && timeout > 0 && timeout <= 120000, + "engine_deadline", + ); + this.checkCancellation(); + const process = new OwnedProcess( + this.inputs.binaries.docker.path, + args, + REPO, + childEnvironment(this.root), + captureStderr ? ["pipe", "pipe", "pipe"] : ["pipe", "pipe", "ignore"], + ); + const { stdin, stdout, stderr } = process.child; + check(stdin && stdout, "engine_stdio"); + const chunks: Buffer[] = []; + let size = 0, + failed = false; + let cancel!: () => void; + const cancelled = new Promise((resolve) => { + cancel = resolve; + }); + const interrupt = () => { + failed = true; + cancel(); + }; + const timer = setTimeout(interrupt, timeout); + const signal = this.signal; + signal?.addEventListener("abort", interrupt, { once: true }); + const collect = (bytes: Buffer) => { + size += bytes.length; + if (size > 4 * 1024 * 1024) interrupt(); + else chunks.push(bytes); + }; + stdout.on("data", collect); + stderr?.on("data", collect); + stdout.once("error", interrupt); + stderr?.once("error", interrupt); + stdin.once("error", () => {}); + process.child.once("error", interrupt); + stdin.end(input); + try { + await Promise.race([process.done, cancelled]); + await process.stop(250); + check(!failed && process.child.exitCode === 0, "engine_command"); + signal?.throwIfAborted(); + return Buffer.concat(chunks); + } finally { + clearTimeout(timer); + signal?.removeEventListener("abort", interrupt); + stdin.destroy(); + stdout.destroy(); + stderr?.destroy(); + process.child.unref(); + } + } + async inspect(id: string) { + const objects = JSON.parse( + (await this.call(["container", "inspect", id])).toString(), + ); + check( + Array.isArray(objects) && + objects.length === 1 && + objects[0]?.Id === id && + objects[0]?.Config?.Labels?.["engrams.mcp.run"] === this.runId, + "owned_container", + ); + return objects[0]; + } + async create( + role: string, + image: string, + options: string[], + args: string[] = [], + entrypoint?: string, + ): Promise { + check(!this.containers.has(role), "duplicate_role"); + if (this.resolverVolume) + options = [ + ...options, + "--mount", + `type=volume,src=${this.resolverVolume},dst=/etc/resolv.conf,volume-subpath=resolv.conf,volume-nocopy,readonly`, + ]; + const gated = [ + "orchestrator", + "coordinator", + "worker", + "host-agent", + "orchestrator-migrate", + ].includes(role); + const host = role === "host-agent"; + let selectedArgs = args, + selectedEntrypoint = entrypoint; + if (gated) { + const [imageConfig] = JSON.parse( + (await this.call(["image", "inspect", image])).toString(), + ); + const entry = entrypoint + ? [entrypoint] + : (imageConfig.Config.Entrypoint ?? []); + const commandArgs = args.length ? args : (imageConfig.Config.Cmd ?? []); + check([...entry, ...commandArgs].length > 0, "application_entrypoint"); + selectedEntrypoint = "/bin/sh"; + selectedArgs = [ + "-ec", + 'while [ ! -f /tmp/mcp-start ]; do sleep 0.05; done; if [ -n "${MCP_FIXTURE_HOST_LOG:-}" ]; then exec "$@" >> "$MCP_FIXTURE_HOST_LOG" 2>&1; fi; exec "$@"', + "mcp-start", + ...entry, + ...commandArgs, + ]; + } + this.plannedContainers.set(this.name(role), role); + const id = ( + await this.call([ + "create", + "--pull=never", + "--name", + this.name(role), + "--label", + this.label(), + "--restart=no", + "--pids-limit", + host ? "1024" : "128", + "--memory", + host ? "8g" : "768m", + "--cpus", + host ? "4" : "2", + "--ulimit", + "core=0", + "--security-opt", + "no-new-privileges", + ...(host && this.inputs.backend === "firecracker" + ? [ + "--security-opt", + "apparmor=unconfined", + "--security-opt", + "systempaths=unconfined", + ] + : []), + "--cap-drop", + "ALL", + ...options, + ...(selectedEntrypoint ? ["--entrypoint", selectedEntrypoint] : []), + image, + ...selectedArgs, + ]) + ) + .toString() + .trim(); + check(/^[a-f0-9]{64}$/.test(id), "container_id"); + this.containers.set(role, id); + this.specifications.set(id, { + image, + options, + host, + role, + args: selectedArgs, + ...(selectedEntrypoint ? { entrypoint: selectedEntrypoint } : {}), + }); + await this.audit(id); + if (this.resolverVolume) await this.verifyResolver(id); + await this.call(["start", id]); + if (this.resolverVolume) await this.verifyResolver(id); + if (gated) { + check(this.resolver, "resolver_missing"); + await this.verifyResolver(id); + await this.audit(id); + await this.verifyNetwork(true); + if (host) { + await this.verifyHostImage(id); + check(this.fc, "fc_inventory_missing"); + const actual = await this.inspect(id); + await this.fc.qualifyKernel(actual.State.Pid); + for (const [path, expected] of [ + ["/proc/sys/kernel/random/boot_id", this.fc.inventory.bootId], + ["/proc/sys/kernel/osrelease", this.fc.inventory.release], + ] as const) + check( + (await this.call(["exec", id, "/bin/cat", path])) + .toString() + .trim() === expected, + "fc_engine_kernel_identity", + ); + check( + ( + await this.call([ + "exec", + id, + "/usr/bin/readlink", + "/proc/self/ns/pid", + ]) + ) + .toString() + .trim() === this.fc.inventory.pidNamespace, + "fc_engine_pid_namespace", + ); + } + if (host) await this.fc!.beforeHostStart(); + await this.call([ + "exec", + id, + "/bin/sh", + "-ec", + "umask 077; : > /tmp/mcp-start", + ]); + await this.verifyResolver(id); + } + return id; + } + private async verifyHostImage(id: string) { + const files = [ + ["/usr/local/bin/engram-host-agent", this.inputs.binaries.hostAgent], + ["/usr/local/bin/engram-uffd-handler", this.inputs.binaries.helper], + ["/usr/local/bin/engram-egress-proxyd", this.inputs.proxy], + ["/usr/local/bin/firecracker", this.inputs.firecracker], + ] as const; + for (const [path, pin] of files) { + check(pin, "host_image_pin"); + await verifyPin(pin); + const dir = join( + this.root, + `image-check-${randomBytes(8).toString("hex")}`, + ); + await mkdir(dir, { mode: 0o700 }); + try { + const target = join(dir, "binary"); + await this.call(["cp", `${id}:${path}`, target]); + check( + (await lstat(target)).isFile() && + hash(await readFile(target)) === pin.sha256, + "host_image_binary", + ); + } finally { + await rm(dir, { recursive: true }); + } + } + } + async audit(id: string) { + const spec = this.specifications.get(id); + check(spec, "container_specification"); + const actual = await this.inspect(id), + h = actual.HostConfig, + c = actual.Config; + const values = (flag: string) => + spec.options.flatMap((v, i) => + v === flag ? [spec.options[i + 1]!] : [], + ); + const one = (flag: string) => values(flag)[0]; + check( + actual.Image === spec.image && + h.RestartPolicy.Name === "no" && + h.Privileged === false && + h.ReadonlyRootfs === spec.options.includes("--read-only") && + h.PidsLimit === (spec.host ? 1024 : 128) && + h.Memory === (spec.host ? 8589934592 : 805306368) && + h.NanoCpus === (spec.host ? 4000000000 : 2000000000), + "container_resources", + ); + check( + c.User === (one("--user") ?? "") && + h.NetworkMode === one("--network") && + h.PidMode === + (spec.host && this.inputs.backend === "firecracker" ? "host" : "") && + h.IpcMode === "private" && + !h.UTSMode && + !h.UsernsMode && + (spec.host + ? h.CgroupnsMode === "private" + : !h.CgroupnsMode || h.CgroupnsMode === "private"), + "container_namespace", + ); + const addedCapabilities = values("--cap-add").sort(); + const securityOptions = JSON.stringify([...(h.SecurityOpt ?? [])].sort()); + const fcHost = + spec.role === "host-agent" && + this.inputs.backend === "firecracker" && + h.PidMode === "host"; + // Docker consumes systempaths into path lists and can derive a SELinux label option. + const expectedSecurity = fcHost + ? actual.AppArmorProfile === "unconfined" && + Array.isArray(h.MaskedPaths) && + h.MaskedPaths.length === 0 && + Array.isArray(h.ReadonlyPaths) && + h.ReadonlyPaths.length === 0 && + (securityOptions === '["apparmor=unconfined","no-new-privileges"]' || + securityOptions === + '["apparmor=unconfined","label=disable","no-new-privileges"]') + : actual.AppArmorProfile !== "unconfined" && + Array.isArray(h.MaskedPaths) && + h.MaskedPaths.length > 0 && + Array.isArray(h.ReadonlyPaths) && + h.ReadonlyPaths.length > 0 && + securityOptions === '["no-new-privileges"]'; + check( + (fcHost + ? JSON.stringify(addedCapabilities) === + '["MKNOD","NET_ADMIN","NET_BIND_SERVICE","SYS_ADMIN"]' + : spec.role === "dns" || spec.role === "denied-dns" + ? JSON.stringify(addedCapabilities) === '["NET_BIND_SERVICE"]' + : !addedCapabilities.includes("NET_BIND_SERVICE")) && + JSON.stringify([...(h.CapDrop ?? [])].sort()) === '["ALL"]' && + JSON.stringify( + (h.CapAdd ?? []) + .map((capability: string) => capability.replace(/^CAP_/, "")) + .sort(), + ) === JSON.stringify(addedCapabilities) && + expectedSecurity, + "container_capabilities", + ); + check( + (h.Binds ?? []).length === 0 && + (h.VolumesFrom ?? []).length === 0 && + JSON.stringify( + (h.Devices ?? []) + .map( + (v: { + PathOnHost: string; + PathInContainer: string; + CgroupPermissions: string; + }) => + `${v.PathOnHost}:${v.PathInContainer}:${v.CgroupPermissions}`, + ) + .sort(), + ) === JSON.stringify(values("--device").sort()) && + (h.DeviceRequests ?? []).length === 0 && + JSON.stringify([...(h.DeviceCgroupRules ?? [])].sort()) === + JSON.stringify(values("--device-cgroup-rule").sort()) && + (spec.host || values("--device-cgroup-rule").length === 0), + "container_authority", + ); + const publications: Record = + {}; + for (const value of values("--publish")) { + const [HostIp, HostPort, port, ...rest] = value.split(":"); + check( + HostIp && HostPort && port && rest.length === 0, + "publication_specification", + ); + publications[`${port}/tcp`] = [{ HostIp, HostPort }]; + } + check( + JSON.stringify(Object.entries(h.PortBindings ?? {}).sort()) === + JSON.stringify(Object.entries(publications).sort()) && + !h.PublishAllPorts && + (h.ExtraHosts ?? []).length === 0 && + !h.CgroupParent && + JSON.stringify(Object.entries(h.Sysctls ?? {}).sort()) === + JSON.stringify( + Object.entries( + fcGuardSysctls(this.inputs.backend, spec.role), + ).sort(), + ), + "container_exposure", + ); + check( + JSON.stringify(h.Dns ?? []) === JSON.stringify(values("--dns")) && + JSON.stringify(h.DnsOptions ?? []) === + JSON.stringify(values("--dns-option")), + "container_dns_configuration", + ); + const mounts = values("--mount").map((value) => + Object.fromEntries( + value + .split(",") + .map((part) => + part.includes("=") ? part.split("=", 2) : [part, "true"], + ), + ), + ); + check( + actual.Mounts.filter((m: { Type: string }) => m.Type !== "tmpfs") + .length === mounts.length, + "container_mount_count", + ); + check( + (h.Mounts ?? []).length === mounts.length, + "container_mount_specification_count", + ); + for (const m of mounts) { + let volumeSource: string | undefined; + const configured = (h.Mounts ?? []).filter( + (entry: { Target: string }) => entry.Target === m.dst, + ); + check( + configured.length === 1 && + configured[0].Type === m.type && + configured[0].Source === m.src && + !!configured[0].ReadOnly === !!m.readonly && + (configured[0].VolumeOptions?.Subpath ?? "") === + (m["volume-subpath"] ?? "") && + !!configured[0].VolumeOptions?.NoCopy === !!m["volume-nocopy"], + "container_mount_specification", + ); + if (m.type === "volume") { + check([...this.volumes.values()].includes(m.src!), "volume_scope"); + const [volume] = JSON.parse( + (await this.call(["volume", "inspect", m.src!])).toString(), + ); + check( + volume?.Name === m.src && + volume?.Labels?.["engrams.mcp.run"] === this.runId && + volume.Driver === "local" && + volume.Scope === "local" && + Object.keys(volume.Options ?? {}).length === 0, + "volume_identity", + ); + volumeSource = volume.Mountpoint; + check( + typeof volumeSource === "string" && volumeSource.startsWith("/"), + "volume_source", + ); + } + const found = actual.Mounts.filter( + (entry: { Destination: string }) => entry.Destination === m.dst, + ); + check( + found.length === 1 && + found[0].Type === m.type && + found[0].RW === !m.readonly && + (m.type !== "volume" || found[0].Source === volumeSource) && + (m.type !== "bind" || found[0].Propagation === "rprivate") && + (m.type === "volume" ? found[0].Name : found[0].Source) === m.src, + "container_mount", + ); + } + const expectedTmpfs = Object.fromEntries( + values("--tmpfs").map((v) => { + const i = v.indexOf(":"); + return [v.slice(0, i), v.slice(i + 1)]; + }), + ); + check( + actual.Mounts.filter((m: { Type: string }) => m.Type === "tmpfs").every( + (m: { Destination: string }) => + Object.hasOwn(expectedTmpfs, m.Destination), + ), + "container_tmpfs_mounts", + ); + check( + JSON.stringify(Object.entries(h.Tmpfs ?? {}).sort()) === + JSON.stringify(Object.entries(expectedTmpfs).sort()), + "container_tmpfs", + ); + const [image] = JSON.parse( + (await this.call(["image", "inspect", spec.image])).toString(), + ); + check(image?.Id === spec.image, "container_image"); + const environment = new Map(); + for (const entry of image.Config.Env ?? []) { + const index = entry.indexOf("="), + key = entry.slice(0, index), + value = entry.slice(index + 1); + check( + [ + "PATH", + "NODE_VERSION", + "YARN_VERSION", + "BUN_VERSION", + "NODE_ENV", + "PG_MAJOR", + "PG_VERSION", + "PG_SHA256", + "PGDATA", + "LANG", + "GOSU_VERSION", + "NGINX_VERSION", + "NJS_VERSION", + "NJS_RELEASE", + "PKG_RELEASE", + "DYNPKG_RELEASE", + "ENGRAM_MKE2FS", + ].includes(key) || + (spec.image === this.inputs.images.guard.id && + key === "HOME" && + value === "/tmp") || + (spec.image === this.inputs.images.orchestrator.id && + ((key === "BUN_INSTALL_BIN" && value === "/usr/local/bin") || + (key === "BUN_RUNTIME_TRANSPILER_CACHE_PATH" && + value === "0"))) || + (spec.image === this.inputs.images.dns.id && + key === "SSL_CERT_FILE" && + value === "/etc/ssl/certs/ca-certificates.crt"), + "image_environment", + ); + environment.set(key, value); + } + for (const envFile of values("--env-file")) + for (const line of (await readOwned(envFile)) + .toString() + .split("\n") + .filter(Boolean)) { + const i = line.indexOf("="); + check(i > 0, "environment_line"); + environment.set(line.slice(0, i), line.slice(i + 1)); + } + for (const line of values("--env")) { + const i = line.indexOf("="); + check(i > 0, "environment_line"); + environment.set(line.slice(0, i), line.slice(i + 1)); + } + check( + JSON.stringify([...(c.Env ?? [])].sort()) === + JSON.stringify([...environment].map(([k, v]) => `${k}=${v}`).sort()), + "container_environment", + ); + check( + JSON.stringify(c.Entrypoint ?? []) === + JSON.stringify( + spec.entrypoint ? [spec.entrypoint] : (image.Config.Entrypoint ?? []), + ) && + JSON.stringify(c.Cmd ?? []) === + JSON.stringify( + spec.args.length ? spec.args : (image.Config.Cmd ?? []), + ), + "container_entrypoint", + ); + return actual; + } + async verifyResolver(id: string) { + check(this.resolver, "resolver_missing"); + await verifyPin(this.resolver); + const directory = join( + this.root, + `resolver-${randomBytes(8).toString("hex")}`, + ); + await mkdir(directory, { mode: 0o700 }); + try { + const file = join(directory, "resolv.conf"); + await this.call(["cp", `${id}:/etc/resolv.conf`, file]); + check( + (await lstat(file)).isFile() && + hash(await readFile(file)) === this.resolver.sha256, + "resolver_changed", + ); + } finally { + await rm(directory, { recursive: true }); + } + } + async reserveHostPort() { + return this.reservePort(9101, this.inputs.nativeHostIp); + } + async reserveRegistryPort() { + return this.reservePort(this.inputs.registryPort, "127.0.0.1"); + } + private async reservePort(port: number, address: string) { + const server = createServer(); + try { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, address, resolve); + }); + } catch { + throw new PlatformFailure( + "NEEDS_CONTEXT", + port === 9101 ? "host_port_occupied" : "registry_port_occupied", + ); + } finally { + if (server.listening) + await new Promise((resolve) => server.close(() => resolve())); + } + } + async remove(role: string): Promise { + const id = this.containers.get(role); + check(id, "owned_role"); + await this.inspect(id); + const state = await this.inspect(id); + if (state.State?.Running) + await this.call( + ["stop", "--time", role === "host-tls-probe" ? "0" : "3", id], + undefined, + 5000, + ); + check( + (await this.inspect(id)).State?.Running === false && + (await this.inspect(id)).State?.Pid === 0, + "container_stop", + ); + await this.call(["rm", id]); + const remaining = ( + await this.call(["ps", "-aq", "--no-trunc", "--filter", `id=${id}`]) + ) + .toString() + .trim(); + check(!remaining, "container_survivor"); + this.specifications.delete(id); + this.containers.delete(role); + } + async volume(role: string): Promise { + const name = this.name(role); + const matches = ( + await this.call([ + "volume", + "ls", + "--filter", + `name=${name}`, + "--format", + "{{.Name}}", + ]) + ) + .toString() + .split("\n"); + check(!matches.includes(name), "volume_exists"); + check( + (await this.call(["volume", "create", "--label", this.label(), name])) + .toString() + .trim() === name, + "volume_create", + ); + this.volumes.set(role, name); + return name; + } + async project( + role: string, + files: Record, + uid: 999 | 1000 = 1000, + publicProjection = false, + ): Promise { + for (const file of Object.values(files)) await verifyPin(file); + const volume = await this.volume(role); + const stager = `${role}-stage`; + const id = await this.create( + stager, + this.inputs.images.guard.id, + [ + "--network", + "none", + "--user", + "0:0", + "--cap-add", + "CHOWN", + "--cap-add", + "DAC_OVERRIDE", + "--cap-add", + "FOWNER", + "--mount", + `type=volume,src=${volume},dst=/projection`, + ], + ["infinity"], + "/bin/sleep", + ); + for (const [name, file] of Object.entries(files)) { + check( + /^[a-zA-Z0-9_-][a-zA-Z0-9_./-]{0,160}$/.test(name) && + !name + .split("/") + .some((part) => part === ".." || part === "." || part === ""), + "projection_file", + ); + const parent = name.includes("/") + ? name.slice(0, name.lastIndexOf("/")) + : ""; + if (parent) + await this.call([ + "exec", + id, + "/bin/mkdir", + "-p", + `/projection/${parent}`, + ]); + await this.call(["cp", file.path, `${id}:/projection/${name}`]); + } + await this.call([ + "exec", + id, + "/bin/sh", + "-ec", + `chown -R ${uid}:${uid} /projection; find /projection -type d -exec chmod ${publicProjection ? "755" : "700"} '{}' ';'; find /projection -type f -exec chmod ${publicProjection ? "644" : "600"} '{}' ';'`, + ]); + const directoryMode = publicProjection ? "755" : "700"; + const fileMode = publicProjection ? "644" : "600"; + const directories = new Set(["/projection"]); + for (const name of Object.keys(files)) { + const parts = name.split("/"); + parts.pop(); + for (let i = 1; i <= parts.length; i++) + directories.add(`/projection/${parts.slice(0, i).join("/")}`); + } + for (const directory of directories) + check( + ( + await this.call([ + "exec", + id, + "/usr/bin/stat", + "-c", + "%u:%g:%a:%F", + directory, + ]) + ) + .toString() + .trim() === `${uid}:${uid}:${directoryMode}:directory`, + "projection_directory_mode", + ); + for (const name of Object.keys(files)) + check( + ( + await this.call([ + "exec", + id, + "/usr/bin/stat", + "-c", + "%u:%g:%a:%F:%h", + `/projection/${name}`, + ]) + ) + .toString() + .trim() === `${uid}:${uid}:${fileMode}:regular file:1`, + "projection_file_mode", + ); + for (const [name, file] of Object.entries(files)) { + const digest = ( + await this.call([ + "exec", + id, + "/usr/bin/sha256sum", + `/projection/${name}`, + ]) + ) + .toString() + .split(/\s/)[0]; + check(digest === file.sha256, "projection_digest"); + } + await this.remove(stager); + return volume; + } + async network(): Promise { + this.resolver = await put( + join(this.root, "resolv.conf"), + `nameserver ${address(this.inputs, "dns")}\noptions ndots:1 timeout:1 attempts:1\n`, + ); + const name = this.name("network"); + const matches = ( + await this.call([ + "network", + "ls", + "--filter", + `name=${name}`, + "--format", + "{{.Name}}", + ]) + ) + .toString() + .split("\n"); + check(!matches.includes(name), "network_exists"); + this.plannedNetworkName = name; + this.networkId = ( + await this.call([ + "network", + "create", + "--internal", + "--driver", + "bridge", + "--subnet", + this.inputs.subnet, + "--gateway", + this.inputs.subnet.slice(0, -4) + "1", + "--label", + this.label(), + name, + ]) + ) + .toString() + .trim(); + check(/^[a-f0-9]{64}$/.test(this.networkId), "network_id"); + const [actual] = JSON.parse( + (await this.call(["network", "inspect", this.networkId])).toString(), + ); + check( + actual?.Id === this.networkId && + actual?.Internal === true && + actual?.Driver === "bridge" && + actual?.Labels?.["engrams.mcp.run"] === this.runId && + actual?.IPAM?.Config?.length === 1 && + actual.IPAM.Config[0].Subnet === this.inputs.subnet && + actual.IPAM.Config[0].Gateway === this.inputs.subnet.slice(0, -4) + "1", + "network_state", + ); + this.resolverVolume = await this.project( + "resolver", + { "resolv.conf": this.resolver }, + 1000, + true, + ); + } + networkName() { + check(this.networkId, "network_missing"); + return this.networkId; + } + networkOptions(role: keyof typeof import("./network.ts").ADDRESS): string[] { + check(this.networkId, "network_missing"); + return [ + "--network", + this.networkId, + "--ip", + address(this.inputs, role), + "--dns", + address(this.inputs, "dns"), + "--dns-option", + "attempts:1", + "--dns-option", + "timeout:1", + ]; + } + async guards(ca: Pin): Promise { + const trust = await this.project( + "probe-trust", + { "ca.pem": ca }, + 1000, + true, + ); + for (const role of GUARDS) { + const id = await this.create( + `${role}-guard`, + this.inputs.images.guard.id, + [ + ...this.networkOptions(role), + ...Object.entries( + fcGuardSysctls(this.inputs.backend, `${role}-guard`), + ).flatMap(([key, value]) => ["--sysctl", `${key}=${value}`]), + "--mount", + `type=volume,src=${trust},dst=/etc/mcp-probe,readonly`, + ...(role === "coordinator" + ? [ + "--publish", + `127.0.0.1:${this.inputs.registryPort}:${this.inputs.registryPort}`, + ...(this.inputs.backend === "firecracker" + ? ["--publish", `${this.inputs.nativeHostIp}:9101:9101`] + : []), + ] + : []), + "--read-only", + "--user", + "0:0", + "--cap-add", + "NET_ADMIN", + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=4m", + ], + ["infinity"], + "/bin/sleep", + ); + this.guardIds.set(role, id); + if (role === "coordinator") { + const guard = await this.inspect(id); + const bindings = guard.HostConfig.PortBindings; + check( + Object.keys(bindings).length === + (this.inputs.backend === "firecracker" ? 2 : 1) && + JSON.stringify(bindings[`${this.inputs.registryPort}/tcp`]) === + JSON.stringify([ + { + HostIp: "127.0.0.1", + HostPort: String(this.inputs.registryPort), + }, + ]), + "registry_publication", + ); + if (this.inputs.backend === "firecracker") + check( + JSON.stringify(bindings["9101/tcp"]) === + JSON.stringify([ + { HostIp: this.inputs.nativeHostIp, HostPort: "9101" }, + ]), + "host_publication", + ); + } + const plan = rules(this.inputs, role); + await this.call( + ["exec", "-i", id, "/usr/sbin/nft", "-j", "-f", "-"], + JSON.stringify({ nftables: plan.map((item) => ({ add: item })) }), + ); + if (role === "coordinator" && this.inputs.backend === "firecracker") + await this.call([ + "exec", + id, + "/usr/sbin/ip", + "-4", + "route", + "replace", + `${this.inputs.nativeHostIp}/32`, + "via", + this.inputs.subnet.slice(0, -4) + "1", + "dev", + "eth0", + ]); + } + if (this.inputs.backend === "vz") await this.connectNativeHost(); + await this.verifyNetwork(false); + } + private async connectNativeHost() { + const id = this.guardIds.get("coordinator"); + check(id, "native_route_guard"); + const name = this.name("native-network"); + const existing = ( + await this.call([ + "network", + "ls", + "--filter", + `name=${name}`, + "--format", + "{{.Name}}", + ]) + ) + .toString() + .split("\n"); + check(!existing.includes(name), "native_network_exists"); + this.nativeNetworkName = name; + this.nativeNetworkId = ( + await this.call([ + "network", + "create", + "--driver", + "bridge", + "--ipv6=false", + "--label", + this.label(), + name, + ]) + ) + .toString() + .trim(); + check(/^[a-f0-9]{64}$/.test(this.nativeNetworkId), "native_network_id"); + const [network] = JSON.parse( + ( + await this.call(["network", "inspect", this.nativeNetworkId]) + ).toString(), + ); + check( + network?.Id === this.nativeNetworkId && + network.Internal === false && + network.Driver === "bridge" && + network.EnableIPv6 === false && + network.Labels?.["engrams.mcp.run"] === this.runId && + network.IPAM?.Config?.length === 1 && + Object.keys(network.Containers ?? {}).length === 0, + "native_network_state", + ); + const { Subnet: subnet, Gateway: gateway } = network.IPAM.Config[0]; + check( + typeof subnet === "string" && typeof gateway === "string", + "native_network_ipam", + ); + verifyNativeSubnet(subnet, gateway, [ + this.inputs.subnet, + this.inputs.nativeHostIp, + ...Object.values(networkInterfaces()) + .flat() + .filter((v) => v?.family === "IPv4") + .map((v) => v!.address), + ]); + await this.call(["network", "connect", this.nativeNetworkId, id]); + const [connected] = JSON.parse( + ( + await this.call(["network", "inspect", this.nativeNetworkId]) + ).toString(), + ); + const member = connected.Containers?.[id]; + check( + Object.keys(connected.Containers ?? {}).join(",") === id && + member && + typeof member.IPv4Address === "string" && + typeof member.MacAddress === "string", + "native_network_membership", + ); + const source = member.IPv4Address.split("/")[0]; + const interfaces = JSON.parse( + ( + await this.call(["exec", id, "/usr/sbin/ip", "-j", "address", "show"]) + ).toString(), + ); + const dev = nativeInterface(interfaces, source, member.MacAddress); + this.nativeLink = { subnet, gateway, source, mac: member.MacAddress, dev }; + // nft default-drop is already installed. Remove only this link's default before any app can share the namespace. + const routes = JSON.parse( + ( + await this.call([ + "exec", + id, + "/usr/sbin/ip", + "-j", + "-4", + "route", + "show", + "default", + ]) + ).toString(), + ); + check(Array.isArray(routes) && routes.length <= 1, "native_link_default"); + for (const route of routes) { + check( + route.dst === "default" && + route.gateway === gateway && + route.dev === dev, + "native_link_default", + ); + await this.call([ + "exec", + id, + "/usr/sbin/ip", + "-4", + "route", + "del", + "default", + "via", + gateway, + "dev", + dev, + ]); + } + await this.call([ + "exec", + id, + "/usr/sbin/ip", + "-4", + "route", + "replace", + `${this.inputs.nativeHostIp}/32`, + "via", + gateway, + "dev", + dev, + ]); + } + async verifyNativeRoute() { + const id = this.guardIds.get("coordinator"); + check(id, "native_route_guard"); + const routes = JSON.parse( + ( + await this.call([ + "exec", + id, + "/usr/sbin/ip", + "-j", + "-4", + "route", + "show", + "table", + "all", + ]) + ).toString(), + ); + const selected = JSON.parse( + ( + await this.call([ + "exec", + id, + "/usr/sbin/ip", + "-j", + "-4", + "route", + "get", + this.inputs.nativeHostIp, + ]) + ).toString(), + ); + if (this.inputs.backend === "vz") { + const link = this.nativeLink; + check( + link && this.nativeNetworkId && this.nativeNetworkName, + "native_link_missing", + ); + const [network] = JSON.parse( + ( + await this.call(["network", "inspect", this.nativeNetworkId]) + ).toString(), + ); + check( + network?.Id === this.nativeNetworkId && + network.Name === this.nativeNetworkName && + network.Internal === false && + network.Driver === "bridge" && + network.EnableIPv6 === false && + network.Labels?.["engrams.mcp.run"] === this.runId && + network.IPAM?.Config?.length === 1 && + network.IPAM.Config[0].Subnet === link.subnet && + network.IPAM.Config[0].Gateway === link.gateway && + Object.keys(network.Containers ?? {}).join(",") === id && + network.Containers[id].IPv4Address.split("/")[0] === link.source && + network.Containers[id].MacAddress === link.mac, + "native_network_changed", + ); + const interfaces = JSON.parse( + ( + await this.call(["exec", id, "/usr/sbin/ip", "-j", "address", "show"]) + ).toString(), + ); + check( + nativeInterface(interfaces, link.source, link.mac) === link.dev, + "native_link_changed", + ); + verifyNativeRouteState(routes, selected, this.inputs, link); + } else verifyNativeRouteState(routes, selected, this.inputs); + } + async qualifyFcHost(id: string) { + check(this.inputs.backend === "firecracker", "fc_host_backend"); + const guard = this.guardIds.get("coordinator"); + check(guard, "fc_host_guard"); + const namespace = async (container: string) => + ( + await this.call([ + "exec", + container, + "/usr/bin/readlink", + "/proc/self/ns/net", + ]) + ) + .toString() + .trim(); + const expected = await namespace(guard); + check( + /^net:\[[0-9]+\]$/.test(expected) && (await namespace(id)) === expected, + "fc_host_namespace", + ); + for (const container of [guard, id]) + for (const path of [ + "/proc/sys/net/ipv4/ip_forward", + "/proc/sys/net/ipv4/conf/all/route_localnet", + ]) + check( + ( + await this.call(["exec", container, "/bin/cat", path]) + ).toString() === "1\n", + "fc_host_sysctl", + ); + const actual: Record = {}; + for (const chain of ["INPUT", "FORWARD", "PREROUTING", "POSTROUTING"]) + actual[chain] = ( + await this.call([ + "exec", + id, + "/usr/sbin/iptables", + "-t", + ["INPUT", "FORWARD"].includes(chain) ? "filter" : "nat", + "-S", + chain, + ]) + ).toString(); + if (hasDockerDns(actual.POSTROUTING!)) + for (const chain of ["OUTPUT", "DOCKER_OUTPUT", "DOCKER_POSTROUTING"]) + actual[chain] = ( + await this.call([ + "exec", + id, + "/usr/sbin/iptables", + "-t", + "nat", + "-S", + chain, + ]) + ).toString(); + verifyFcHostFirewall(actual, this.inputs.registryPort); + await put( + join( + this.root, + `fc-host-firewall-${randomBytes(8).toString("hex")}.json`, + ), + JSON.stringify({ + netns: expected, + sysctls: fcGuardSysctls(this.inputs.backend, "coordinator-guard"), + sha256: hash(JSON.stringify(actual)), + }), + ); + } + async qualifyNativeHost(tls: NativeHostTls) { + for (const pin of Object.values(tls)) { + await verifyPin(pin); + await readOwned(pin.path); + } + await this.verifyNetwork(false); + const identity = hash( + Object.values(tls) + .map((pin) => pin.sha256) + .join("\n"), + ); + if (this.nativeHostProjection) + check( + this.nativeHostProjection.identity === identity, + "native_host_probe_identity", + ); + else + this.nativeHostProjection = { + identity, + volume: await this.project("host-probe-tls", { + "ca.pem": tls.ca, + "client.crt": tls.clientCertificate, + "client.key": tls.clientKey, + }), + }; + const role = "host-tls-probe"; + const expected = new X509Certificate( + await readOwned(tls.serverCertificate.path), + ).fingerprint256; + try { + const id = await this.create( + role, + this.inputs.images.guard.id, + [ + "--network", + `container:${this.guardIds.get("coordinator")}`, + "--user", + "1000:1000", + "--read-only", + "--mount", + `type=volume,src=${this.nativeHostProjection.volume},dst=/etc/mcp-host-probe,readonly`, + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=4m", + ], + ["infinity"], + "/bin/sleep", + ); + const output = await this.call( + [ + "exec", + id, + "/usr/bin/openssl", + "s_client", + "-connect", + "host.fixture.example:9101", + "-servername", + "host.fixture.example", + "-verify_hostname", + "host.fixture.example", + "-verify_return_error", + "-tls1_3", + "-no_ign_eof", + "-showcerts", + "-CAfile", + "/etc/mcp-host-probe/ca.pem", + "-cert", + "/etc/mcp-host-probe/client.crt", + "-key", + "/etc/mcp-host-probe/client.key", + ], + undefined, + 5000, + ); + const serverCertificateSha256 = verifyNativeHostTls(output, expected); + await put( + join( + this.root, + `native-host-tls-${randomBytes(8).toString("hex")}.json`, + ), + JSON.stringify({ version: 1, serverCertificateSha256, verified: true }), + ); + } finally { + if (this.containers.has(role)) await this.remove(role); + } + } + async verifyNetwork(probes: boolean) { + check(this.guardIds.size === 3 && this.networkId, "guard_set"); + await this.verifyNativeRoute(); + const [network] = JSON.parse( + (await this.call(["network", "inspect", this.networkId])).toString(), + ); + check( + network?.Id === this.networkId && + network.Internal === true && + network.Labels?.["engrams.mcp.run"] === this.runId, + "network_changed", + ); + const results = await drainChecks( + GUARDS.map(async (role) => { + const id = this.guardIds.get(role)!; + const guard = await this.audit(id); + await this.verifyResolver(id); + check( + guard.State?.Running === true && + guard.NetworkSettings?.Networks?.[this.name("network")] + ?.IPAddress === address(this.inputs, role), + "guard_identity", + ); + check( + JSON.stringify(Object.keys(guard.NetworkSettings.Networks).sort()) === + JSON.stringify( + [ + this.name("network"), + ...(role === "coordinator" && this.inputs.backend === "vz" + ? [this.name("native-network")] + : []), + ].sort(), + ), + "guard_network_membership", + ); + const enforcement = effectiveRules( + JSON.parse( + ( + await this.call([ + "exec", + id, + "/usr/sbin/nft", + "-j", + "list", + "table", + "inet", + "mcp", + ]) + ).toString(), + ), + rules(this.inputs, role), + ); + if (probes) { + // The owned fixture is reachable. The owned denial sink is live but not allowed. + for (const fixture of ["modern", "legacy"] as const) { + await this.call([ + "exec", + id, + "/usr/bin/nc", + "-z", + "-w", + "2", + address(this.inputs, fixture), + "8443", + ]); + await this.call([ + "exec", + id, + "/usr/bin/curl", + "--silent", + "--show-error", + "--max-time", + "2", + "--cacert", + "/etc/mcp-probe/ca.pem", + "--output", + "/dev/null", + `https://resource.${fixture}.fixture.example:8443/`, + ]); + } + // No external packet is sent: both probes target the owned DNS namespace or the blocked embedded stub. + const dns = await this.call([ + "exec", + id, + "/usr/bin/dig", + "+time=1", + "+tries=1", + "+short", + "resource.modern.fixture.example", + "A", + ]); + check( + dns.toString().trim() === address(this.inputs, "modern"), + "fixture_dns", + ); + const unknown = await this.call([ + "exec", + id, + "/usr/bin/dig", + "+time=1", + "+tries=1", + "+comments", + `@${address(this.inputs, "dns")}`, + "unrouted.invalid", + "A", + ]); + check( + /status: (NXDOMAIN|SERVFAIL),/.test(unknown.toString()) && + /ANSWER: 0,/.test(unknown.toString()), + "external_dns", + ); + // Start the three fixed negative probes together, then wait for all of + // them. Their one-second bounds also keep restart controls bounded. + const negative = await this.call([ + "exec", + id, + "/bin/sh", + "-ec", + ` + deny() { if /usr/bin/nc -z -w 1 ${address(this.inputs, "denied")} 8080; then return 1; else code=$?; test "$code" = 1; fi; } + block() { if /usr/bin/dig +time=1 +tries=1 "@$1" unrouted.invalid A >/dev/null 2>&1; then return 1; else code=$?; test "$code" = 9; fi; } + deny & first=$!; block 127.0.0.11 & second=$!; block ${address(this.inputs, "denied")} & third=$! + failed=0; wait "$first" || failed=1; wait "$second" || failed=1; wait "$third" || failed=1 + test "$failed" = 0; printf blocked + `, + ]); + check(negative.toString() === "blocked", "negative_network_probes"); + } + return enforcement; + }), + ); + const enforcementSha256 = hash(results.join("\n")); + if (this.baseline) + check(this.baseline === enforcementSha256, "network_generation"); + else this.baseline = enforcementSha256; + return { + generation: this.generation, + enforcementSha256, + fixtureOnlyEgress: true as const, + externalDnsBlocked: true as const, + roles: GUARDS, + }; + } + async qualifyTools(): Promise { + const versions = JSON.parse( + (await this.call(["version", "--format", "{{json .}}"])).toString(), + ); + for (const selected of [versions.Client, versions.Server]) { + const api = /^1\.([0-9]+)$/.exec(selected?.ApiVersion ?? ""); + check( + /^[0-9]+\./.test(selected?.Version ?? "") && + Number(selected.Version.split(".")[0]) >= 26 && + api && + Number(api[1]) >= 45, + "engine_volume_subpath_support", + ); + } + const id = await this.create( + "tool-check", + this.inputs.images.guard.id, + [ + "--network", + "none", + "--user", + "1000:1000", + "--read-only", + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=4m", + ], + ["infinity"], + "/bin/sleep", + ); + try { + await this.call([ + "exec", + id, + "/bin/sh", + "-ec", + 'for tool in /bin/sh /bin/bash /bin/sleep /bin/mkdir /bin/chown /bin/chmod /usr/bin/find /usr/bin/sha256sum /usr/bin/skopeo /usr/sbin/nft /usr/bin/dig /usr/bin/nc /usr/bin/curl /usr/bin/mksquashfs /usr/sbin/ip /usr/bin/openssl /usr/bin/stat /usr/bin/readlink /bin/cat; do test -x "$tool"; done', + ]); + const version = ( + await this.call(["exec", id, "/usr/bin/skopeo", "--version"]) + ).toString(); + check(/^skopeo version [0-9]+\./.test(version), "guard_skopeo"); + check( + (await this.call(["exec", id, "/usr/bin/openssl", "version"])) + .toString() + .startsWith("OpenSSL 3."), + "guard_openssl3", + ); + } finally { + await this.remove("tool-check"); + } + } + async nativeDns( + resolve: ( + name: string, + options: { all: true }, + ) => Promise = lookup, + ): Promise { + for (const [name, expected] of Object.entries(dnsRecords(this.inputs))) { + let actual: LookupAddress[]; + try { + actual = await resolve(name, { all: true }); + } catch { + throw new PlatformFailure("NEEDS_CONTEXT", "native_dns_lookup"); + } + if (!actual.length || actual.some((entry) => entry.address !== expected)) + throw new PlatformFailure("NEEDS_CONTEXT", "native_dns_route"); + } + } + async cleanup(): Promise { + if (this.fc && this.containers.has("host-agent")) { + try { + await this.fc.requireReleased(); + } catch (error) { + await put( + join(this.root, "fc-cleanup-blocked.json"), + JSON.stringify({ + runId: this.runId, + hostContainer: this.containers.get("host-agent"), + action: "retained_for_identity_scoped_recovery", + reason: + error instanceof PlatformFailure + ? error.seam + : "thin_pool_state_unknown", + }), + ); + throw error; + } + } + let failed = false; + const stopped: string[] = []; + // A cancelled CLI can lose a successful daemon create response. Recover + // only exact planned names with this run's label before stopping anything. + try { + const ids = ( + await this.call([ + "ps", + "-aq", + "--no-trunc", + "--filter", + `label=${this.label()}`, + ]) + ) + .toString() + .trim() + .split("\n") + .filter(Boolean); + for (const id of ids) { + check(/^[a-f0-9]{64}$/.test(id), "cleanup_container_id"); + if ([...this.containers.values()].includes(id)) continue; + const actual = await this.inspect(id); + const role = this.plannedContainers.get( + String(actual.Name).replace(/^\//, ""), + ); + check( + role && !this.containers.has(role), + "cleanup_unplanned_container", + ); + this.containers.set(role, id); + } + } catch { + failed = true; + } + for (const [role] of [...this.containers].reverse()) { + try { + const id = this.containers.get(role)!; + const logs = await this.call(["logs", "--tail", "10000", id]); + await put(join(this.root, `retained-${role}.log`), logs); + } catch { + failed = true; + } + try { + await this.remove(role); + stopped.push(role); + } catch { + failed = true; + } + } + if (!this.nativeNetworkId && this.nativeNetworkName) { + try { + const ids = ( + await this.call([ + "network", + "ls", + "--no-trunc", + "--filter", + `label=${this.label()}`, + "--filter", + `name=${this.nativeNetworkName}`, + "--format", + "{{.ID}}", + ]) + ) + .toString() + .trim() + .split("\n") + .filter(Boolean); + check( + ids.length <= 1 && ids.every((id) => /^[a-f0-9]{64}$/.test(id)), + "cleanup_native_network_identity", + ); + this.nativeNetworkId = ids[0]; + } catch { + failed = true; + } + } + if (this.nativeNetworkId && this.containers.size === 0) { + try { + const [network] = JSON.parse( + ( + await this.call(["network", "inspect", this.nativeNetworkId]) + ).toString(), + ); + check( + network?.Id === this.nativeNetworkId && + network.Name === this.nativeNetworkName && + network.Labels?.["engrams.mcp.run"] === this.runId && + Object.keys(network.Containers ?? {}).length === 0, + "native_network_cleanup", + ); + await this.call(["network", "rm", this.nativeNetworkId]); + this.nativeNetworkId = undefined; + } catch { + failed = true; + } + } + if (!this.networkId && this.plannedNetworkName) { + try { + const ids = ( + await this.call([ + "network", + "ls", + "--no-trunc", + "--filter", + `label=${this.label()}`, + "--filter", + `name=${this.plannedNetworkName}`, + "--format", + "{{.ID}}", + ]) + ) + .toString() + .trim() + .split("\n") + .filter(Boolean); + check( + ids.length <= 1 && ids.every((id) => /^[a-f0-9]{64}$/.test(id)), + "cleanup_network_identity", + ); + this.networkId = ids[0]; + } catch { + failed = true; + } + } + if (this.networkId && this.containers.size === 0) { + try { + const [actual] = JSON.parse( + (await this.call(["network", "inspect", this.networkId])).toString(), + ); + check( + actual?.Id === this.networkId && + actual?.Labels?.["engrams.mcp.run"] === this.runId && + Object.keys(actual.Containers ?? {}).length === 0, + "network_cleanup", + ); + await this.call(["network", "rm", this.networkId]); + this.networkId = undefined; + } catch { + failed = true; + } + } + if (this.fc) { + try { + await this.fc.requireStopped(); + } catch { + failed = true; + } + } + // Named volumes retain database and effect history. The report records them; they are not active resources. + const survivors = ( + await this.call(["ps", "-aq", "--filter", `label=${this.label()}`]).catch( + () => Buffer.from("unknown"), + ) + ) + .toString() + .trim(); + if (survivors || this.networkId || this.nativeNetworkId) failed = true; + if (this.plannedNetworkName) { + const remaining = await this.call([ + "network", + "ls", + "--no-trunc", + "--filter", + `label=${this.label()}`, + "--format", + "{{.ID}}", + ]).catch(() => Buffer.from("unknown")); + if (remaining.toString().trim()) failed = true; + } + await put( + join(this.root, "cleanup.json"), + JSON.stringify({ + version: 1, + runId: this.runId, + stopped, + retainedVolumes: [...this.volumes.values()], + containersRemaining: this.containers.size, + networkRemaining: !!this.networkId || !!this.nativeNetworkId, + success: !failed, + }), + ); + if (failed) throw new PlatformFailure("FAILED", "engine_cleanup"); + } +} diff --git a/tests/fixtures/mcp-stack/platform/fc-runtime.ts b/tests/fixtures/mcp-stack/platform/fc-runtime.ts new file mode 100644 index 000000000..461f9d9ce --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/fc-runtime.ts @@ -0,0 +1,1094 @@ +import { randomBytes } from "node:crypto"; +import { constants } from "node:fs"; +import { + lstat, + readFile, + readdir, + readlink, + stat, + appendFile, + open, +} from "node:fs/promises"; +import { basename, dirname, join } from "node:path"; +import { + check, + hash, + put, + readOwned, + command, + childEnvironment, +} from "./owned.ts"; +import type { Inputs } from "./preflight.ts"; +import { + admitPool, + OfflinePoolAccess, + verifyThinIds, + requirePoolBudget, + verifyBootDevice, + verifyReleasedMappings, + inspectPool, + verifyPoolIdentity, + deviceNode, + type PoolInventory, + type DeviceNode, + type Backing, +} from "./fc-thin.ts"; +export { linuxDevice } from "./fc-thin.ts"; +import { + parseHostRecord, + type HostRecord, +} from "../../../../orchestrator/scripts/mcp-stack/host-observer.ts"; + +const uuid = /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/; +const absent = (error: unknown) => + error !== null && + typeof error === "object" && + "code" in error && + error.code === "ENOENT"; +export function procBirth(value: string, pid: number): string { + const close = value.lastIndexOf(")"); + const fields = value + .slice(close + 1) + .trim() + .split(/\s+/); + check( + value.startsWith(`${pid} (`) && + close > 0 && + /^[0-9]+$/.test(fields[19] ?? ""), + "fc_process_birth", + ); + return fields[19]!; +} +export type FcProcess = { + pid: number; + bootId: string; + birth: string; + uid: number; + executable: string; + cgroup: string; + sha256: string; +}; +export type FcInventory = { + bootId: string; + release: string; + pidNamespace: string; + pool: PoolInventory; +}; +function managedNeedsIo( + record: Pick, +) { + check( + [ + "missing", + "prepared", + "finalized", + "start_pending", + "started", + "failed", + "captured", + "retired", + ].includes(String(record.state.state)), + "fc_managed_boot_state", + ); + // Finalization is persisted before start and is never cleared on failure, + // retirement or capture. Terminal state alone cannot prove never-started. + return ( + record.finalization !== null || + record.capture !== null || + ["finalized", "start_pending", "started", "captured"].includes( + String(record.state.state), + ) + ); +} +export function requireManagedBootCoverage( + required: Set, + observed: ReadonlySet, + key: string, + record: Pick, +) { + if (managedNeedsIo(record)) required.add(key); + check(!required.has(key) || observed.has(key), "fc_physical_boot_unobserved"); +} +export function managedBootPaths( + work: string, + sandbox: string, + attempt: string, +) { + check( + uuid.test(sandbox) && /^[A-Za-z0-9-]{1,128}$/.test(attempt), + "fc_managed_boot_identity", + ); + const directory = join( + work, + `m-${hash(`${sandbox}\0${attempt}`).slice(0, 24)}`, + ); + return { + directory, + apiSocket: join(directory, "firecracker.sock"), + rootfsCanonical: join(directory, "rootfs.ext4"), + volume: `sb-${sandbox.replaceAll("-", "")}-root`, + }; +} +export function selectManagedProcess( + candidates: { process: FcProcess; args: string[] }[], + apiSocket: string, +): FcProcess | undefined { + const matches = candidates.filter((candidate) => + candidate.args.includes(apiSocket), + ); + check(matches.length <= 1, "fc_managed_process_duplicate"); + const match = matches[0]; + if (!match) return undefined; + check( + JSON.stringify(match.args) === + JSON.stringify(["/usr/local/bin/firecracker", "--api-sock", apiSocket]), + "fc_managed_process_arguments", + ); + return match.process; +} +async function processArguments(pid: number) { + const value = await readFile(`/proc/${pid}/cmdline`); + check( + value.length > 0 && value.length <= 4096 && value.at(-1) === 0, + "fc_process_arguments", + ); + return value.subarray(0, -1).toString().split("\0"); +} +type BootCandidate = { + name: string; + process: FcProcess; + rootfsCanonical: string; + boot?: HostRecord; + bootBytes?: Buffer; + sandboxManifest?: string; + apiSocket?: string; +}; +export function verifyFcProcessIdentity( + expected: FcProcess, + actual: FcProcess | undefined, +) { + check( + actual && JSON.stringify(actual) === JSON.stringify(expected), + "fc_process_identity_changed", + ); +} +async function kernelIdentity() { + const bootId = ( + await readFile("/proc/sys/kernel/random/boot_id", "utf8") + ).trim(); + const release = (await readFile("/proc/sys/kernel/osrelease", "utf8")).trim(); + const pidNamespace = await readlink("/proc/1/ns/pid"); + check( + uuid.test(bootId) && + /^[A-Za-z0-9._+-]{1,256}$/.test(release) && + /^pid:\[[0-9]+\]$/.test(pidNamespace) && + (await readlink("/proc/self/ns/pid")) === pidNamespace, + "fc_initial_kernel", + ); + return { bootId, release, pidNamespace }; +} +async function pids(): Promise { + const values = (await readdir("/proc")).filter((v) => + /^[1-9][0-9]*$/.test(v), + ); + check(values.length <= 65536, "fc_process_inventory_bound"); + return values.map(Number); +} +async function processIdentity( + pid: number, + bootId: string, + digest = false, +): Promise { + try { + const path = `/proc/${pid}`; + const before = await readFile(`${path}/stat`, "utf8"); + const executable = await readlink(`${path}/exe`); + const status = await readFile(`${path}/status`, "utf8"); + const uids = /^Uid:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$/m.exec(status); + const cgroup = await readFile(`${path}/cgroup`, "utf8"); + check(uids && /^0::\/[^\n]*\n$/.test(cgroup), "fc_process_identity"); + const sha256 = digest ? hash(await readFile(`${path}/exe`)) : ""; + const birth = procBirth(before, pid); + check( + procBirth(await readFile(`${path}/stat`, "utf8"), pid) === birth && + (await readlink(`${path}/exe`)) === executable, + "fc_process_changed", + ); + return { + pid, + bootId, + birth, + uid: Number(uids[1]), + executable, + cgroup: cgroup.trim().slice(3), + sha256, + }; + } catch (error) { + if (absent(error)) return undefined; + throw error; + } +} +async function deviceFds( + slots: DeviceNode[], + processes: number[], + files: Backing[] = [], +) { + const devices = new Set(slots.map((v) => v.rdev)); + const found: { pid: number; fd: string; rdev: string }[] = []; + for (const pid of processes) { + let fds: string[]; + try { + fds = await readdir(`/proc/${pid}/fd`); + } catch (error) { + if (absent(error)) continue; + throw error; + } + check(fds.length <= 16384, "fc_fd_inventory_bound"); + for (const fd of fds) { + check(/^[0-9]+$/.test(fd), "fc_fd_identity"); + try { + const state = await stat(`/proc/${pid}/fd/${fd}`, { bigint: true }); + if ( + ((state.isBlockDevice() || state.isCharacterDevice()) && + devices.has(String(state.rdev))) || + (state.isFile() && + files.some( + (file) => + String(state.dev) === file.fileDev && + String(state.ino) === file.fileIno, + )) + ) + found.push({ pid, fd, rdev: String(state.rdev) }); + } catch (error) { + if (!absent(error)) throw error; + } + } + } + return found; +} +export async function inspectFcInventory( + inputs: Pick, +): Promise { + const identity = await kernelIdentity(); + check(inputs.thinPool, "fc_pool_required"); + const initial = await inspectPool(inputs.thinPool); + const pool = await new OfflinePoolAccess().inspect( + () => requireStorageQuiescence(inputs, identity.bootId, initial), + (lease) => inspectPool(initial, lease), + ); + pool.fds = await deviceFds( + [pool.data, pool.metadata, await deviceNode(pool.path)], + await pids(), + [pool.data, pool.metadata], + ); + admitPool(pool); + return { ...identity, pool }; +} +async function requireStorageQuiescence( + inputs: Pick, + bootId: string, + pool: PoolInventory, +) { + const names = new Set([ + "engram-host-agent", + "firecracker", + "engram-egress-proxyd", + "engram-uffd-handler", + basename(inputs.binaries.hostAgent.path), + basename(inputs.binaries.helper.path), + basename(inputs.proxy.path), + ...(inputs.firecracker ? [basename(inputs.firecracker.path)] : []), + ]); + const processes = await pids(); + for (const pid of processes) { + const current = await processIdentity(pid, bootId); + check( + !current || !names.has(basename(current.executable)), + "fc_storage_process_live", + ); + } + check( + ( + await deviceFds( + [pool.control, pool.data, pool.metadata, pool.node], + processes, + [pool.data, pool.metadata], + ) + ).length === 0, + "fc_storage_fd_live", + ); +} + +export function verifyFcKernelPosture(mountinfo: string, status: string) { + const mounts = mountinfo + .trim() + .split("\n") + .map((line) => line.split(" ")); + for (const target of ["/sys", "/sys/fs/cgroup"]) + check( + mounts.some( + (fields) => + fields[4] === target && fields[5]?.split(",").includes("ro"), + ), + "fc_host_readonly_kernel_mount", + ); + const procMounts = mounts + .filter((fields) => fields[4] === "/proc" || fields[4] === "/proc/sys") + .sort((left, right) => right[4]!.length - left[4]!.length); + const effectiveProc = procMounts[0]; + check( + effectiveProc && + procMounts.filter((fields) => fields[4] === effectiveProc[4]).length === + 1 && + effectiveProc[5]?.split(",").includes("rw") && + !effectiveProc[5]?.split(",").includes("ro") && + effectiveProc[effectiveProc.indexOf("-") + 1] === "proc", + "fc_host_writable_proc_sys", + ); + const capabilities = (1n << 10n) | (1n << 12n) | (1n << 21n) | (1n << 27n); + for (const field of ["CapEff", "CapPrm", "CapBnd"]) { + const value = new RegExp(`^${field}:\\s+([a-f0-9]+)$`, "m").exec( + status, + )?.[1]; + check( + value && BigInt("0x" + value) === capabilities, + "fc_host_effective_capabilities", + ); + } + check( + /^NoNewPrivs:\s+1$/m.test(status) && /^Seccomp:\s+2$/m.test(status), + "fc_host_effective_confinement", + ); + return mounts; +} +type DiskReadTarget = { + key: string; + identity: string; + process: FcProcess; + node: DeviceNode; + mapping: { name: string; device: string; table: string; id: number }; + deviceFd: { pid: number; fd: string; rdev: string }; +}; +export function requireDiskReadCoverage( + prepared: ReadonlyMap, + current: ReadonlyMap, + observed: ReadonlySet, +) { + check(prepared.size > 0, "fc_disk_read_not_prepared"); + for (const [key, identity] of prepared) { + check(current.get(key) === identity, "fc_disk_read_target"); + check(observed.has(key), "fc_physical_boot_unobserved"); + } +} +/** The runner owns the whole disposable kernel. Device cgroups cannot + * isolate device-mapper control from other host workloads. */ +export class FcRuntime { + private readonly storageAccess = new OfflinePoolAccess(); + private readonly cgroups = new Set(); + private readonly processes = new Map(); + private readonly observations = new Set(); + private readonly requiredBoots = new Set(); + private readonly observationKinds = new Set<"ordinary" | "managed">(); + private evidenceWriteFailed = false; + private observation: Promise = Promise.resolve(); + private host: FcProcess | undefined; + private readonly bootDevices = new Map(); + private readonly bootReads = new Map(); + private readonly ownedSandboxes = new Set(); + private readonly retainedVolumes = new Map(); + private diskReadTargets = new Map(); + private preparedDiskReads = new Map(); + constructor( + readonly inventory: FcInventory, + private readonly inputs: Inputs, + private readonly root: string, + private readonly appendEvidence = (path: string, bytes: string) => + appendFile(path, bytes, { mode: 0o600 }), + ) {} + deviceOptions() { + const p = this.inventory.pool; + return [ + "--device", + "/dev/mapper/control:/dev/mapper/control:rw", + "--device", + `${p.metadata.path}:${p.metadata.path}:r`, + "--device", + `${p.path}:${p.path}:rw`, + "--device-cgroup-rule", + `b ${p.major}:* rwm`, + "--cap-add", + "MKNOD", + "--mount", + `type=bind,src=${join(this.root, "thin-nodes")},dst=/dev/engram`, + ]; + } + environment() { + return { ENGRAM_THIN_POOL: this.inventory.pool.name }; + } + private async proveStorageQuiescent() { + // The offline lease stops new samples. Drain any table-reading subprocess + // before checking control-device FDs, then inspect the actual kernel state. + await this.observation.catch(() => {}); + check(!this.evidenceWriteFailed, "fc_physical_evidence_write"); + await requireStorageQuiescence( + this.inputs, + this.inventory.bootId, + this.inventory.pool, + ); + } + async beforeHostStart() { + const prove = () => this.proveStorageQuiescent(); + await this.storageAccess.stopped(prove); + const current = await this.storageAccess.inspect(prove, (lease) => + inspectPool(this.inventory.pool, lease), + ); + verifyPoolIdentity(this.inventory.pool, current); + if (!this.host) { + current.fds = await deviceFds( + [current.data, current.metadata, await deviceNode(current.path)], + await pids(), + [current.data, current.metadata], + ); + admitPool(current); + } else { + // ThinPoolDisk::open removes inactive orphans even on restart. + // Reject any inactive id before the replacement host can open the pool. + check( + current.thinIds.every((id) => + current.mappings.some((m) => m.id === id), + ) && current.thinIds.length === current.mappings.length, + "fc_restart_inactive_thin", + ); + await this.validateMappings(current); + } + // The engine releases the host's start barrier only after this hook returns. + this.storageAccess.start(); + } + private async validateMappings(pool: PoolInventory) { + for (const m of pool.mappings) { + check(m.name.startsWith(pool.name + "."), "fc_foreign_thin_name"); + const volume = m.name.slice(pool.name.length + 1); + const sandbox = + /^(?:tmp-)?sb-([a-f0-9]{32})-(?:root|swap)(?:-pre|-ckpt-[a-f0-9]{32}-v[0-9]+)?$/.exec( + volume, + ); + const image = /^(?:tmp-)?img-[a-f0-9]{32}-v[0-9]+$/.test(volume); + check( + image || (sandbox && this.ownedSandboxes.has(sandbox[1]!)), + "fc_foreign_thin_owner", + ); + } + } + async qualifyKernel(pid: number) { + verifyPoolIdentity( + this.inventory.pool, + await inspectPool(this.inventory.pool), + ); + const mounts = verifyFcKernelPosture( + await readFile(`/proc/${pid}/mountinfo`, "utf8"), + await readFile(`/proc/${pid}/status`, "utf8"), + ); + check( + JSON.stringify(await kernelIdentity()) === + JSON.stringify({ + bootId: this.inventory.bootId, + release: this.inventory.release, + pidNamespace: this.inventory.pidNamespace, + }), + "fc_kernel_changed", + ); + check( + (await readlink(`/proc/${pid}/ns/pid`)) === this.inventory.pidNamespace && + (await readlink(`/proc/${pid}/ns/cgroup`)) !== + (await readlink("/proc/self/ns/cgroup")), + "fc_host_kernel_namespace", + ); + const current = await processIdentity(pid, this.inventory.bootId); + check( + current && current.uid === 0 && current.cgroup !== "/", + "fc_host_cgroup", + ); + this.cgroups.add(current.cgroup); + this.processes.set(`${pid}:${current.birth}`, current); + const nodeMount = mounts.find((fields) => fields[4] === "/dev/engram"); + check( + nodeMount && + nodeMount[5]?.split(",").includes("rw") && + !nodeMount[5]?.split(",").includes("nodev"), + "fc_host_nodes_mount", + ); + const pool = this.inventory.pool; + for (const expected of [ + pool.control, + pool.metadata, + await deviceNode(pool.path), + ]) { + const projected = await lstat(`/proc/${pid}/root${expected.path}`, { + bigint: true, + }); + check( + String(projected.rdev) === expected.rdev && + (expected.path === pool.control.path + ? projected.isCharacterDevice() + : projected.isBlockDevice()), + "fc_host_device_projection", + ); + } + const nodes = await lstat(`/proc/${pid}/root/dev/engram`, { bigint: true }); + const source = await lstat(join(this.root, "thin-nodes"), { bigint: true }); + check( + nodes.isDirectory() && + nodes.dev === source.dev && + nodes.ino === source.ino, + "fc_host_nodes_projection", + ); + } + + async adoptHost(pid: number) { + await this.qualifyKernel(pid); + const host = await processIdentity(pid, this.inventory.bootId, true); + check( + host && + host.uid === 0 && + host.sha256 === this.inputs.binaries.hostAgent.sha256 && + host.executable === "/usr/local/bin/engram-host-agent" && + host.cgroup !== "/", + "fc_host_process", + ); + const newHost = + !this.host || + this.host.pid !== host.pid || + this.host.birth !== host.birth; + this.host = host; + this.cgroups.add(host.cgroup); + this.processes.set(`${host.pid}:${host.birth}`, host); + if (newHost) + await put( + join(this.root, `fc-host-${host.pid}-${host.birth}.json`), + JSON.stringify({ inventory: this.inventory, host }), + ); + } + private owns(cgroup: string) { + return [...this.cgroups].some( + (value) => cgroup === value || cgroup.startsWith(value + "/"), + ); + } + async observe() { + if (this.storageAccess.inspectionActive) { + await this.observation.catch(() => {}); + check(!this.evidenceWriteFailed, "fc_physical_evidence_write"); + return; + } + return this.withObservation(() => this.observeOnce()); + } + private withObservation(work: () => Promise): Promise { + const current = this.observation.catch(() => {}).then(work); + this.observation = current.then( + () => {}, + () => {}, + ); + return current; + } + async prepareDiskRead() { + check(!this.storageAccess.inspectionActive, "fc_pool_inspection_busy"); + return this.withObservation(async () => { + check(this.preparedDiskReads.size === 0, "fc_disk_read_pending"); + await this.observeOnce(); + const targets = [...this.diskReadTargets.values()]; + check(targets.length > 0, "fc_disk_read_boots"); + for (const target of targets) await this.flushDiskReadTarget(target); + // No fixture FD may remain when the normal foreign-FD check runs. + await this.observeOnce(); + for (const target of targets) { + check( + this.diskReadTargets.get(target.key)?.identity === target.identity, + "fc_disk_read_target", + ); + if (this.requiredBoots.has(target.key)) + this.preparedDiskReads.set(target.key, target.identity); + } + check(this.preparedDiskReads.size > 0, "fc_disk_read_boots"); + }); + } + async verifyDiskRead() { + check(!this.storageAccess.inspectionActive, "fc_pool_inspection_busy"); + return this.withObservation(async () => { + await this.observeOnce(); + requireDiskReadCoverage( + this.preparedDiskReads, + new Map( + [...this.diskReadTargets].map(([key, target]) => [ + key, + target.identity, + ]), + ), + this.observations, + ); + this.preparedDiskReads.clear(); + }); + } + private async flushDiskReadTarget(target: DiskReadTarget) { + const handle = await open( + target.node.path, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + const verify = async () => { + const state = await handle.stat({ bigint: true }); + check( + state.isBlockDevice() && + state.uid === 0n && + String(state.ino) === target.node.ino && + String(state.rdev) === target.node.rdev, + "fc_disk_read_target", + ); + const pool = await inspectPool(this.inventory.pool); + verifyPoolIdentity(this.inventory.pool, pool); + const mapping = pool.mappings.find( + (m) => m.name === target.mapping.name, + ); + check( + mapping && + mapping.device === target.mapping.device && + mapping.id === target.mapping.id && + mapping.table === target.mapping.table, + "fc_disk_read_target", + ); + verifyFcProcessIdentity( + target.process, + await processIdentity( + target.process.pid, + this.inventory.bootId, + true, + ), + ); + const fd = await stat( + `/proc/${target.process.pid}/fd/${target.deviceFd.fd}`, + { bigint: true }, + ); + check( + fd.isBlockDevice() && String(fd.rdev) === target.node.rdev, + "fc_rootfs_device_fd_changed", + ); + }; + await verify(); + // The adapter keeps this descriptor open. The child opens the pinned + // descriptor through proc, never a mutable or caller-selected device path. + await command( + "/usr/sbin/blockdev", + ["--flushbufs", `/proc/${process.pid}/fd/${handle.fd}`], + "/", + childEnvironment("/"), + ); + await verify(); + } finally { + await handle.close(); + } + } + private async observeOnce() { + check(!this.evidenceWriteFailed, "fc_physical_evidence_write"); + if (!this.host) return; + const identities = new Map(); + const all = await pids(); + for (const pid of all) { + const current = await processIdentity(pid, this.inventory.bootId); + if (!current || !this.owns(current.cgroup)) continue; + const full = await processIdentity(pid, this.inventory.bootId, true); + if (!full) continue; + identities.set(pid, full); + this.processes.set(`${pid}:${full.birth}`, full); + } + const pool = await inspectPool(this.inventory.pool); + verifyPoolIdentity(this.inventory.pool, pool); + const nodes = await Promise.all( + pool.mappings.map(async (m) => { + const name = /^DEVNAME=(dm-[0-9]+)$/m.exec( + await readFile(`/sys/dev/block/${m.device}/uevent`, "utf8"), + )?.[1]; + check(name, "fc_thin_device_name"); + return deviceNode(`/dev/${name}`); + }), + ); + const fds = await deviceFds(nodes, all); + check( + fds.every((fd) => identities.has(fd.pid)), + "fc_thin_foreign_fd", + ); + const seenProcesses = new Set(); + const diskReadTargets = new Map(); + const work = join(this.root, "host"); + const candidates: BootCandidate[] = []; + for (const name of (await readdir(work)).filter((v) => uuid.test(v))) { + let raw: string; + try { + raw = ( + await readOwned(join(work, name, "sandbox.json"), 1024 * 1024, false) + ).toString(); + } catch (error) { + if (absent(error)) continue; + throw error; + } + const manifest = JSON.parse(raw); + check( + manifest.schema_version === 3 && + manifest.backend === "firecracker" && + manifest.sandbox_id === name, + "fc_sandbox_manifest", + ); + const fc = manifest.firecracker; + const process = identities.get(fc?.pid); + if (!process) continue; + check( + process.birth === String(fc.start_time_jiffies), + "fc_sandbox_process", + ); + check( + typeof fc.rootfs_canonical === "string" && + dirname(fc.rootfs_canonical) === join(work, "rootfs") && + /^[a-f0-9-]{36}\.dev$/.test(basename(fc.rootfs_canonical)), + "fc_rootfs_path", + ); + candidates.push({ + name, + process, + rootfsCanonical: fc.rootfs_canonical, + sandboxManifest: raw, + }); + } + const commands: { process: FcProcess; args: string[] }[] = []; + for (const process of identities.values()) { + if (process.executable !== "/usr/local/bin/firecracker") continue; + try { + commands.push({ process, args: await processArguments(process.pid) }); + } catch (error) { + if (!absent(error)) throw error; + } + } + const records = await readdir(join(work, "managed-boots")); + check(records.length <= 8192, "fc_managed_record_bound"); + for (const file of records.filter((v) => /^[a-f0-9-]{36}\.boot$/.test(v))) { + const name = file.slice(0, -5); + const bootBytes = await readOwned( + join(work, "managed-boots", file), + 16 * 1024 * 1024, + ); + const boot = parseHostRecord(bootBytes, name).record; + check( + boot.request.key.context.target.host_id === + (await readOwned(join(work, "host_id"))).toString(), + "fc_managed_host_identity", + ); + // A verified durable request owns staging/transient volumes even before + // its Firecracker process is started. + this.ownedSandboxes.add(name.replaceAll("-", "")); + const paths = managedBootPaths(work, name, boot.request.key.boot_attempt); + if (managedNeedsIo(boot)) + this.requiredBoots.add(`${name}:${boot.request.key.boot_attempt}`); + const process = selectManagedProcess(commands, paths.apiSocket); + if (!process) continue; + candidates.push({ + name, + process, + rootfsCanonical: paths.rootfsCanonical, + apiSocket: paths.apiSocket, + boot, + bootBytes, + }); + } + for (const candidate of candidates) { + const { name, process, rootfsCanonical, boot, bootBytes, apiSocket } = + candidate; + check( + this.inputs.firecracker && + process.sha256 === this.inputs.firecracker.sha256 && + process.uid === 0 && + process.executable === "/usr/local/bin/firecracker", + "fc_sandbox_process", + ); + check(!seenProcesses.has(process.pid), "fc_boot_process_duplicate"); + // The symlink target exists inside the trusted host's device mount. + check( + (await lstat(rootfsCanonical)).isSymbolicLink(), + "fc_rootfs_not_thin", + ); + const target = await readlink(rootfsCanonical); + const volume = `sb-${name.replaceAll("-", "")}-root`; + const expectedName = `${pool.name}.${volume}`; + check(target === `/dev/engram/${expectedName}`, "fc_rootfs_volume_name"); + const mapping = pool.mappings.find((m) => m.name === expectedName); + check(mapping, "fc_rootfs_not_thin"); + const node = await deviceNode( + join(this.root, "thin-nodes", expectedName), + ); + check(node.device === mapping.device, "fc_rootfs_device_changed"); + seenProcesses.add(process.pid); + this.ownedSandboxes.add(name.replaceAll("-", "")); + const rootBytes = BigInt(mapping.table.split(" ")[1]!) * 512n; + if (boot) + check( + rootBytes === BigInt(boot.request.cold.disk.size_bytes), + "fc_boot_volume_size", + ); + const largest = (pattern: RegExp) => + pool.mappings + .filter((m) => pattern.test(m.name)) + .map((m) => BigInt(m.table.split(" ")[1]!) * 512n) + .reduce((a, b) => (a > b ? a : b), 0n); + requirePoolBudget( + pool, + rootBytes, + largest(/-swap(?:-|$)/), + largest(/\.(?:tmp-)?img-/), + ); + const identity = JSON.stringify({ + mapping: { + name: mapping.name, + device: mapping.device, + table: mapping.table, + id: mapping.id, + }, + node, + process, + }); + const bootAttempt = + boot?.request.key.boot_attempt ?? `ordinary-${process.birth}`; + const key = `${name}:${bootAttempt}`; + const needsIo = !boot || managedNeedsIo(boot); + if (needsIo) this.requiredBoots.add(key); + const prior = this.bootDevices.get(key); + verifyBootDevice(prior, identity); + this.bootDevices.set(key, identity); + const io = (await readFile(`/sys/dev/block/${node.device}/stat`, "utf8")) + .trim() + .split(/\s+/); + check(io.length >= 11 && /^[0-9]+$/.test(io[2]!), "fc_thin_io_counter"); + const deviceFd = fds.find( + (fd) => fd.pid === process.pid && fd.rdev === node.rdev, + ); + check(deviceFd, "fc_rootfs_device_fd"); + diskReadTargets.set(key, { + key, + identity, + process, + node, + mapping, + deviceFd, + }); + // A newly allocated thin device starts counters at zero. Bind the + // counters to this boot's exact table, thin id and process birth. + const readSectorsBefore = this.bootReads.get(key); + if (readSectorsBefore === undefined) { + this.bootReads.set(key, BigInt(io[2]!)); + continue; + } + if (!needsIo || BigInt(io[2]!) <= readSectorsBefore) continue; + const finalFd = await stat(`/proc/${process.pid}/fd/${deviceFd.fd}`, { + bigint: true, + }); + check( + finalFd.isBlockDevice() && String(finalFd.rdev) === node.rdev, + "fc_rootfs_device_fd_changed", + ); + verifyFcProcessIdentity( + process, + await processIdentity(process.pid, this.inventory.bootId, true), + ); + if (apiSocket) { + const finalArgs = await processArguments(process.pid); + check( + selectManagedProcess([{ process, args: finalArgs }], apiSocket), + "fc_managed_process_arguments_changed", + ); + } + if (this.observations.has(key)) continue; + await this.retainObservation( + key, + JSON.stringify({ + sandboxId: name, + ...(boot ? { sessionId: boot.request.cold.session_id } : {}), + bootAttempt, + kind: boot ? "managed" : "ordinary", + process, + pool: { name: pool.name, device: pool.device, table: pool.table }, + mapping, + deviceFd, + node, + volume, + readSectorsBefore: String(readSectorsBefore), + readSectors: io[2], + rootfsCanonical, + ...(apiSocket ? { apiSocket } : {}), + ...(bootBytes ? { bootRecordSha256: hash(bootBytes) } : {}), + ...(candidate.sandboxManifest + ? { sandboxManifestSha256: hash(candidate.sandboxManifest) } + : {}), + }) + "\n", + ); + this.observationKinds.add(boot ? "managed" : "ordinary"); + } + check( + [...identities.values()] + .filter((p) => p.executable === "/usr/local/bin/firecracker") + .every((p) => seenProcesses.has(p.pid)), + "fc_unobserved_process", + ); + await this.validateMappings(pool); + for (const name of this.retainedVolumes.keys()) + if (!pool.mappings.some((m) => m.name === name)) + this.retainedVolumes.delete(name); + for (const mapping of pool.mappings) { + if (/\.(?:img-|sb-.*-ckpt-)/.test(mapping.name)) { + const previous = this.retainedVolumes.get(mapping.name); + if (previous) { + const prior = JSON.parse(previous); + check( + prior.id === mapping.id && + prior.device === mapping.device && + prior.table === mapping.table, + "fc_retained_volume_replaced", + ); + } + this.retainedVolumes.set(mapping.name, JSON.stringify(mapping)); + } + } + this.diskReadTargets = diskReadTargets; + } + // The path is fixed. This seam permits controlled write-failure tests; + // observe calls it only after all physical checks pass. + async retainObservation(key: string, bytes: string) { + check(!this.evidenceWriteFailed, "fc_physical_evidence_write"); + if (this.observations.has(key)) return; + try { + await this.appendEvidence(join(this.root, "fc-physical.jsonl"), bytes); + } catch (error) { + // A failed append can leave an incomplete record. Preserve that failure + // for the whole run; a later successful sample cannot grant coverage. + this.evidenceWriteFailed = true; + throw error; + } + this.observations.add(key); + } + async requireCoverage() { + check(!this.evidenceWriteFailed, "fc_physical_evidence_write"); + for (const key of this.requiredBoots) + check(this.observations.has(key), "fc_physical_boot_unobserved"); + const files = await readdir(join(this.root, "host/managed-boots")); + const boots = files.filter((v) => /^[a-f0-9-]{36}\.boot$/.test(v)); + check(boots.length > 0 && boots.length <= 4096, "fc_physical_boot_count"); + for (const file of boots) { + const id = file.slice(0, -5); + const boot = parseHostRecord( + await readOwned(join(this.root, "host/managed-boots", file)), + id, + ).record; + requireManagedBootCoverage( + this.requiredBoots, + this.observations, + `${id}:${boot.request.key.boot_attempt}`, + boot, + ); + } + } + async requireInfrastructure(lifecycle: { + stop(): Promise; + restart(): Promise; + }) { + await this.observe(); + check( + this.observationKinds.has("ordinary") && + this.observationKinds.has("managed"), + "fc_infrastructure_boot_evidence", + ); + await this.requireCoverage(); + const deadline = performance.now() + 10000; + for (;;) { + try { + await this.requireReleased(); + break; + } catch (error) { + if (performance.now() >= deadline) throw error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); + await this.observe(); + } + // The live checks above require normal deletion while the host is alive. + // Metadata inspection starts only after that exact host and its VMs stop. + await lifecycle.stop(); + await this.requireStopped(); + await lifecycle.restart(); + await put( + join(this.root, "fc-infrastructure.json"), + JSON.stringify({ + bootKeys: [...this.observations], + pool: this.inventory.pool, + release: "verified", + }), + ); + } + async requireReleased() { + const pool = await inspectPool(this.inventory.pool); + await this.checkReleased(pool, false); + } + private async checkReleased(pool: PoolInventory, metadata: boolean) { + verifyPoolIdentity(this.inventory.pool, pool); + await this.validateMappings(pool); + verifyReleasedMappings(pool, this.retainedVolumes); + for (const m of pool.mappings) { + const path = join(this.root, "thin-nodes", m.name); + const node = await deviceNode(path); + check( + node.device === m.device && + (await readdir(`/sys/dev/block/${m.device}/holders`)).length === 0, + "fc_thin_remaining_holder", + ); + check( + (await deviceFds([node], await pids())).length === 0, + "fc_thin_remaining_fd", + ); + } + const nodeNames = ( + await readdir(join(this.root, "thin-nodes")).catch((error) => { + if (!this.host && absent(error)) return []; + throw error; + }) + ).sort(); + check( + JSON.stringify(nodeNames) === + JSON.stringify(pool.mappings.map((m) => m.name).sort()), + "fc_unreleased_device_node", + ); + await put( + join( + this.root, + `fc-volume-cleanup-${randomBytes(8).toString("hex")}.json`, + ), + JSON.stringify({ + metadataInspected: metadata, + pool, + retained: pool.mappings.map((m) => m.name), + }), + ); + } + async requireStopped() { + const survivors: FcProcess[] = []; + for (const pid of await pids()) { + const current = await processIdentity(pid, this.inventory.bootId); + if (current && this.owns(current.cgroup)) survivors.push(current); + } + await put( + join( + this.root, + `fc-process-cleanup-${randomBytes(8).toString("hex")}.json`, + ), + JSON.stringify({ observed: [...this.processes.values()], survivors }), + ); + check(survivors.length === 0, "fc_owned_process_survivor"); + const prove = () => this.proveStorageQuiescent(); + await this.storageAccess.stopped(prove); + await this.storageAccess.inspect(prove, async (lease) => { + const pool = await inspectPool(this.inventory.pool, lease); + verifyThinIds(pool); + await this.checkReleased(pool, true); + }); + } +} diff --git a/tests/fixtures/mcp-stack/platform/fc-thin-fixture.ts b/tests/fixtures/mcp-stack/platform/fc-thin-fixture.ts new file mode 100644 index 000000000..f7439bd7c --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/fc-thin-fixture.ts @@ -0,0 +1,51 @@ +// Controlled unit-test state; never physical acceptance evidence. +export const poolFixture = () => ({ + name: "suite", + directory: "/owned/pool", + device: "253:0", + path: "/dev/dm-0", + table: "0 268435456 thin-pool 7:0 7:1 128 32768 0", + major: "253", + node: { path: "/dev/dm-0", device: "253:0", ino: "5", rdev: "64768" }, + otherTables: "", + metadata: { + path: "/dev/loop0", + device: "7:0", + ino: "1", + rdev: "1792", + file: "/owned/pool/metadata.img", + fileDev: "4", + fileIno: "11", + bytes: "1073741824", + allocated: "1073741824", + }, + data: { + path: "/dev/loop1", + device: "7:1", + ino: "2", + rdev: "1793", + file: "/owned/pool/data.img", + fileDev: "4", + fileIno: "12", + bytes: "137438953472", + allocated: "137438953472", + }, + control: { + path: "/dev/mapper/control", + ino: "3", + rdev: "2700", + device: "10:236", + }, + status: + "0 268435456 thin-pool 0 10/262144 0/2097152 - rw no_discard_passdown queue_if_no_space - 1024", + thinIds: [] as number[], + mappings: [] as { + name: string; + device: string; + table: string; + id: number; + open: number; + }[], + holders: [] as string[], + fds: [] as { pid: number; fd: string; rdev: string }[], +}); diff --git a/tests/fixtures/mcp-stack/platform/fc-thin.test.ts b/tests/fixtures/mcp-stack/platform/fc-thin.test.ts new file mode 100644 index 000000000..f04d05305 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/fc-thin.test.ts @@ -0,0 +1,530 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + admitPool, + inspectPool, + OfflinePoolAccess, + parseThinPool, + readThinIds as reservedThinIds, + verifyThinIds, + type MetadataReadState, + type OfflinePoolLease, + verifyPoolIdentity, + parsePoolTable, + parsePoolStatus, +} from "./fc-thin.ts"; +import { poolFixture } from "./fc-thin-fixture.ts"; + +test("expanded pool reinspection keeps its name budget and public input stays strict", async () => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), "mcp-pool-reinspection-")), + ); + try { + const input = { name: "suite", directory }; + expect(parseThinPool(input)).toEqual(input); + expect(() => parseThinPool({ ...poolFixture(), directory })).toThrow( + "object_keys", + ); + const inventory = { ...poolFixture(), directory, name: "a".repeat(27) }; + await expect( + new OfflinePoolAccess().inspect( + async () => {}, + (lease) => inspectPool(inventory, lease), + ), + ).rejects.toThrow("fc_pool_name_volume_budget"); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +function readThinIds( + name: string, + path: string, + status: string, + control?: (args: string[]) => Promise, + dump?: () => Promise, + state?: MetadataReadState, +) { + return new OfflinePoolAccess().inspect( + async () => {}, + (lease) => reservedThinIds(name, path, status, lease, control, dump, state), + ); +} +test("thin admission refuses foreign active or inactive volumes, holders and unhealthy capacity", () => { + const good = poolFixture(); + expect(() => admitPool(good)).not.toThrow(); + for (const patch of [ + { thinIds: [1] }, + { holders: ["dm-1"] }, + { fds: [{ pid: 42, fd: "3", rdev: "1" }] }, + { + mappings: [ + { + name: "foreign", + device: "253:1", + table: "0 8 thin 253:0 1", + id: 1, + open: 1, + }, + ], + }, + { status: good.status.replace(" rw ", " ro ") }, + { status: good.status.replace("0/2097152", "2097151/2097152") }, + { status: good.status.replace("10/262144", "262143/262144") }, + { data: { ...good.data, allocated: "1" } }, + ]) + expect(() => admitPool({ ...good, ...patch })).toThrow(); +}); +test("pool identity binds table, backing inode and actual device across launch", () => { + const good = poolFixture(); + expect(() => verifyPoolIdentity(good, good)).not.toThrow(); + for (const patch of [ + { device: "253:1" }, + { table: good.table.replace("7:0", "7:2") }, + { data: { ...good.data, fileIno: "13" } }, + ]) + expect(() => verifyPoolIdentity(good, { ...good, ...patch })).toThrow( + "fc_pool_identity_changed", + ); +}); +test("pool identity permits only fully allocated backing accounting changes", () => { + const good = poolFixture(); + for (const kind of ["data", "metadata"] as const) { + const grown = { + ...good, + [kind]: { + ...good[kind], + allocated: String(BigInt(good[kind].allocated) + 204800n), + }, + }; + expect(() => verifyPoolIdentity(good, grown)).not.toThrow(); + expect(() => verifyPoolIdentity(grown, good)).not.toThrow(); + const sparse = { + ...good, + [kind]: { + ...good[kind], + allocated: String(BigInt(good[kind].bytes) - 1n), + }, + }; + expect(() => verifyPoolIdentity(good, sparse)).toThrow( + "fc_pool_sparse_backing", + ); + expect(() => verifyPoolIdentity(sparse, good)).toThrow( + "fc_pool_sparse_backing", + ); + for (const [field, value] of Object.entries(good[kind])) { + if (field === "allocated") continue; + const changed = { + ...good, + [kind]: { ...good[kind], [field]: value + "1" }, + }; + expect(() => verifyPoolIdentity(good, changed)).toThrow( + "fc_pool_identity_changed", + ); + } + } + for (const patch of [ + { path: "/dev/dm-1" }, + { major: "254" }, + { name: "other" }, + { directory: "/owned/other" }, + { otherTables: "foreign: 0 8 linear 7:2 0" }, + { node: { ...good.node, ino: "99" } }, + { control: { ...good.control, rdev: "99" } }, + ]) + expect(() => verifyPoolIdentity(good, { ...good, ...patch })).toThrow( + "fc_pool_identity_changed", + ); +}); +test("thin table and status reject alternate targets, features and unhealthy metadata", () => { + expect(parsePoolTable(poolFixture().table).blockSectors).toBe(128n); + expect(() => parsePoolTable("0 10 linear 7:0 0")).toThrow(); + expect(() => + parsePoolStatus(poolFixture().status.replace(" - rw", " 2 rw"), 128n), + ).toThrow(); +}); + +test("metadata inspection refuses a foreign reservation and releases only its own snapshot", async () => { + for (const failure of ["none", "dump", "release", "reserve"] as const) { + const calls: string[] = []; + const operation = readThinIds( + "suite", + "/dev/loop0", + poolFixture().status, + async (args) => { + const last = args.at(-1)!; + calls.push(last); + if (last === `${failure}_metadata_snap`) throw Error(failure); + return ""; + }, + async () => { + calls.push("dump"); + if (failure === "dump") throw Error("dump"); + return ''; + }, + { pending: Promise.resolve(), failed: false }, + ); + if (failure === "none") expect(await operation).toEqual([7]); + else await expect(operation).rejects.toThrow(failure); + expect(calls).toEqual( + failure === "reserve" + ? ["reserve_metadata_snap"] + : ["reserve_metadata_snap", "dump", "release_metadata_snap"], + ); + } + let invoked = false; + await expect( + readThinIds( + "suite", + "/dev/loop0", + poolFixture().status.replace(" - rw", " 9 rw"), + async () => { + invoked = true; + return ""; + }, + async () => "", + ), + ).rejects.toThrow("fc_pool_health"); + expect(invoked).toBe(false); +}); + +test("a release timeout poisons later inspections and never releases another reservation", async () => { + const state = { pending: Promise.resolve(), failed: false }; + const calls: string[] = []; + const control = async (args: string[]) => { + const op = args.at(-1)!; + calls.push(op); + if (op === "release_metadata_snap") throw Error("release timeout"); + return ""; + }; + await expect( + readThinIds( + "suite", + "/dev/loop0", + poolFixture().status, + control, + async () => "", + state, + ), + ).rejects.toThrow("release timeout"); + await expect( + readThinIds( + "suite", + "/dev/loop0", + poolFixture().status, + control, + async () => "", + state, + ), + ).rejects.toThrow("fc_pool_metadata_release_uncertain"); + expect(calls).toEqual(["reserve_metadata_snap", "release_metadata_snap"]); +}); +test("live status permits the trusted host's metadata reservation without changing it", () => { + const reserved = poolFixture().status.replace(" - rw", " 9 rw"); + expect(() => parsePoolStatus(reserved, 128n, true)).not.toThrow(); + expect(() => parsePoolStatus(reserved, 128n, false)).toThrow( + "fc_pool_health", + ); +}); + +test("capacity charges actual installed image/session/swap overlap rather than case count", async () => { + const { requirePoolBudget } = await import("./fc-thin.ts"); + expect(() => + requirePoolBudget(poolFixture(), 8n * 1024n ** 3n, 2n * 1024n ** 3n), + ).not.toThrow(); + expect(() => + requirePoolBudget(poolFixture(), 64n * 1024n ** 3n, 2n * 1024n ** 3n), + ).toThrow("fc_pool_overlap_capacity"); +}); +test("per-boot evidence rejects reused minors, thin IDs, volume names and stale process birth", async () => { + const { verifyBootDevice } = await import("./fc-thin.ts"); + const before = { + device: "253:1", + thin: 7, + volume: "sb-owned-root", + birth: "100", + }; + expect(() => + verifyBootDevice(undefined, JSON.stringify(before)), + ).not.toThrow(); + expect(() => + verifyBootDevice(JSON.stringify(before), JSON.stringify(before)), + ).not.toThrow(); + for (const patch of [ + { device: "253:2" }, + { thin: 8 }, + { volume: "sb-other-root" }, + { birth: "101" }, + ]) + expect(() => + verifyBootDevice( + JSON.stringify(before), + JSON.stringify({ ...before, ...patch }), + ), + ).toThrow("fc_boot_device_changed"); +}); + +test("kernel status includes the metadata low-water mark and refuses pressure", () => { + const status = poolFixture().status; + expect(() => parsePoolStatus(status, 128n)).not.toThrow(); + expect(() => + parsePoolStatus(status.replace(" 1024", " 262144"), 128n), + ).toThrow("fc_pool_capacity"); +}); + +test("release keeps only recorded closed image/checkpoint volumes with exact identity", async () => { + const { verifyReleasedMappings } = await import("./fc-thin.ts"); + const mapping = { + name: "suite.img-" + "a".repeat(32) + "-v1", + device: "253:1", + table: "0 16 thin 253:0 7", + id: 7, + open: 0, + }; + const retained = new Map([[mapping.name, JSON.stringify(mapping)]]); + const pool = { ...poolFixture(), mappings: [mapping], thinIds: [7] }; + expect(() => verifyReleasedMappings(pool, retained)).not.toThrow(); + for (const patch of [ + { open: 1 }, + { device: "253:2" }, + { id: 8 }, + { name: "suite.sb-" + "b".repeat(32) + "-root" }, + ]) + expect(() => + verifyReleasedMappings( + { ...pool, mappings: [{ ...mapping, ...patch }] }, + retained, + ), + ).toThrow(); + expect(() => verifyThinIds({ ...pool, thinIds: [7, 8] })).toThrow(); +}); + +test("managed boot selects only the current exact API socket and retains its disk contract", async () => { + const { managedBootPaths, selectManagedProcess } = + await import("./fc-runtime.ts"); + const sandbox = "11111111-1111-4111-8111-111111111111"; + const one = managedBootPaths("/owned/host", sandbox, "boot-1"); + const next = managedBootPaths("/owned/host", sandbox, "boot-2"); + expect(one.directory).toBe("/owned/host/m-548f58de1d3dd34c94a49d74"); + expect(one.directory).not.toBe(next.directory); + expect(one.rootfsCanonical).toBe(one.directory + "/rootfs.ext4"); + expect(one.volume).toBe("sb-11111111111141118111111111111111-root"); + const args = ["/usr/local/bin/firecracker", "--api-sock", one.apiSocket]; + const process = { + pid: 42, + birth: "100", + bootId: "kernel", + uid: 0, + executable: args[0]!, + cgroup: "/owned", + sha256: "a".repeat(64), + }; + expect(selectManagedProcess([{ process, args }], one.apiSocket)).toEqual( + process, + ); + expect( + selectManagedProcess([{ process, args }], next.apiSocket), + ).toBeUndefined(); + expect(() => + selectManagedProcess( + [{ process, args: [...args, "--extra"] }], + one.apiSocket, + ), + ).toThrow("fc_managed_process_arguments"); + expect(() => + selectManagedProcess( + [ + { process, args }, + { process: { ...process, pid: 43 }, args }, + ], + one.apiSocket, + ), + ).toThrow("fc_managed_process_duplicate"); +}); + +test("managed coverage uses durable start prerequisites and never forgets an observed attempt", async () => { + const { requireManagedBootCoverage } = await import("./fc-runtime.ts"); + const required = new Set(); + const proof = new Set(); + requireManagedBootCoverage(required, proof, "prepared", { + finalization: null, + state: { state: "prepared" }, + capture: null, + }); + expect(required.size).toBe(0); + expect(() => + requireManagedBootCoverage(required, proof, "retired", { + finalization: "sha256:abc", + state: { state: "retired" }, + capture: null, + }), + ).toThrow("fc_physical_boot_unobserved"); + expect(required.has("retired")).toBe(true); + expect(() => + requireManagedBootCoverage(required, proof, "retired", { + finalization: null, + state: { state: "failed" }, + capture: null, + }), + ).toThrow("fc_physical_boot_unobserved"); + expect(() => + requireManagedBootCoverage(required, proof, "live", { + finalization: "sha256:abc", + state: { state: "started" }, + capture: null, + }), + ).toThrow("fc_physical_boot_unobserved"); + proof.add("retired"); + proof.add("live"); + expect(() => + requireManagedBootCoverage(required, proof, "retired", { + finalization: "sha256:abc", + state: { state: "retired" }, + capture: null, + }), + ).not.toThrow(); +}); + +test("unknown managed state cannot claim a never-started exemption", async () => { + const { requireManagedBootCoverage } = await import("./fc-runtime.ts"); + expect(() => + requireManagedBootCoverage(new Set(), new Set(), "unknown", { + finalization: null, + state: { state: "unknown" }, + capture: null, + }), + ).toThrow("fc_managed_boot_state"); +}); + +test("metadata inspection cannot release a replacement host reservation", async () => { + const access = new OfflinePoolAccess(); + access.start(); + let owner: "none" | "adapter" | "host" = "none"; + const released: string[] = []; + const read = () => + access.inspect( + async () => {}, + (lease) => + reservedThinIds( + "suite", + "/dev/loop0", + poolFixture().status, + lease, + async (args) => { + if (args.at(-1) === "reserve_metadata_snap") owner = "adapter"; + else { + released.push(owner); + owner = "none"; + } + return ""; + }, + async () => { + // This host reader can replace a reservation only while its process runs. + owner = "none"; + owner = "host"; + return ''; + }, + { pending: Promise.resolve(), failed: false }, + ), + ); + await expect(read()).rejects.toThrow("fc_pool_host_not_quiescent"); + expect(released).not.toContain("host"); + expect(owner).toBe("none"); +}); + +test("offline reservation excludes a host restart until release and invalidates its lease", async () => { + const access = new OfflinePoolAccess(); + access.start(); + let live = true; + await expect( + access.stopped(async () => { + if (live) throw Error("host still live"); + }), + ).rejects.toThrow("host still live"); + await expect( + access.inspect( + async () => {}, + async () => [], + ), + ).rejects.toThrow("fc_pool_host_not_quiescent"); + live = false; + await access.stopped(async () => { + expect(live).toBe(false); + }); + let owner = "none"; + const released: string[] = []; + let expired: OfflinePoolLease | undefined; + const result = await access.inspect( + async () => { + expect(live).toBe(false); + }, + async (lease) => { + expired = lease; + return reservedThinIds( + "suite", + "/dev/loop0", + poolFixture().status, + lease, + async (args) => { + if (args.at(-1) === "reserve_metadata_snap") owner = "adapter"; + else { + released.push(owner); + owner = "none"; + } + return ""; + }, + async () => { + // A replacement host would release and reserve here; the start barrier + // refuses to launch it while the adapter owns the offline lease. + expect(() => access.start()).toThrow( + "fc_metadata_host_start_blocked", + ); + return ''; + }, + { pending: Promise.resolve(), failed: false }, + ); + }, + ); + expect(result).toEqual([7]); + expect(released).toEqual(["adapter"]); + expect(owner).toBe("none"); + expect(() => access.start()).not.toThrow(); + if (!expired) throw Error("lease missing"); + await expect( + reservedThinIds( + "suite", + "/dev/loop0", + poolFixture().status, + expired, + async () => { + throw Error("must not access metadata"); + }, + async () => "", + ), + ).rejects.toThrow("fc_pool_host_not_quiescent"); +}); + +test("disk read acknowledgement requires the same live boot and its measured read proof", async () => { + const { requireDiskReadCoverage } = await import("./fc-runtime.ts"); + const ordinary = "sandbox:ordinary-10"; + const identity = "exact-table-thin-id-device-fd-process-birth"; + const prepared = new Map([[ordinary, identity]]); + expect(() => + requireDiskReadCoverage(prepared, prepared, new Set([ordinary])), + ).not.toThrow(); + expect(() => requireDiskReadCoverage(prepared, prepared, new Set())).toThrow( + "fc_physical_boot_unobserved", + ); + for (const current of [ + new Map(), + new Map([[ordinary, "replacement"]]), + ]) + expect(() => + requireDiskReadCoverage(prepared, current, new Set([ordinary])), + ).toThrow("fc_disk_read_target"); + expect(() => + requireDiskReadCoverage(new Map(), prepared, new Set([ordinary])), + ).toThrow("fc_disk_read_not_prepared"); +}); diff --git a/tests/fixtures/mcp-stack/platform/fc-thin.ts b/tests/fixtures/mcp-stack/platform/fc-thin.ts new file mode 100644 index 000000000..872b4f40e --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/fc-thin.ts @@ -0,0 +1,555 @@ +import { lstat, readFile, readdir, realpath, statfs } from "node:fs/promises"; +import { lstatSync, realpathSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { check, command, childEnvironment, record } from "./owned.ts"; + +export type ThinPoolInput = { name: string; directory: string }; +export function parseThinPool(value: unknown): ThinPoolInput { + const o = record(value, ["name", "directory"]); + check( + typeof o.name === "string" && + /^[A-Za-z][A-Za-z0-9_.-]{0,126}$/.test(o.name), + "fc_pool_name", + ); + check( + typeof o.directory === "string" && + o.directory.startsWith("/") && + resolve(o.directory) === o.directory, + "fc_pool_directory", + ); + for (let p = o.directory; ; p = dirname(p)) { + check(!lstatSync(p).isSymbolicLink(), "fc_pool_directory_symlink"); + if (p === "/") break; + } + const s = lstatSync(o.directory); + check( + s.isDirectory() && + s.uid === process.getuid?.() && + (s.mode & 0o777) === 0o700 && + realpathSync(o.directory) === o.directory, + "fc_pool_directory_owner_mode", + ); + return { name: o.name, directory: o.directory }; +} +export type DeviceNode = { + path: string; + device: string; + ino: string; + rdev: string; +}; +export type Backing = DeviceNode & { + file: string; + fileDev: string; + fileIno: string; + bytes: string; + allocated: string; +}; +export type ThinMapping = { + name: string; + device: string; + table: string; + id: number; + open: number; +}; +export type PoolInventory = ThinPoolInput & { + device: string; + path: string; + table: string; + major: string; + node: DeviceNode; + otherTables: string; + metadata: Backing; + data: Backing; + control: DeviceNode; + status: string; + thinIds: number[]; + mappings: ThinMapping[]; + holders: string[]; + fds: { pid: number; fd: string; rdev: string }[]; +}; +export function linuxDevice(value: bigint): string { + return `${((value >> 8n) & 0xfffn) | ((value >> 32n) & 0xfffff000n)}:${(value & 0xffn) | ((value >> 12n) & 0xffffff00n)}`; +} +export function parsePoolTable(table: string) { + const m = + /^0 ([1-9][0-9]*) thin-pool ([0-9]+:[0-9]+) ([0-9]+:[0-9]+) ([1-9][0-9]*) ([0-9]+) 0$/.exec( + table, + ); + check(m, "fc_pool_table"); + const blockSectors = BigInt(m[4]!); + check(blockSectors === 128n && m[2] !== m[3], "fc_pool_block_size"); + return { sectors: BigInt(m[1]!), metadata: m[2]!, data: m[3]!, blockSectors }; +} +export function parsePoolStatus( + status: string, + blockSectors: bigint, + snapshot = false, +) { + const m = + /^0 ([1-9][0-9]*) thin-pool [0-9]+ ([0-9]+)\/([1-9][0-9]*) ([0-9]+)\/([1-9][0-9]*) (-|[0-9]+) rw (?:discard_passdown|no_discard_passdown) (?:queue_if_no_space|error_if_no_space) - ([0-9]+)$/.exec( + status, + ); + check(m && (snapshot || m[6] === "-"), "fc_pool_health"); + const metaUsed = BigInt(m[2]!), + metaTotal = BigInt(m[3]!); + const dataUsed = BigInt(m[4]!), + dataTotal = BigInt(m[5]!); + check( + metaUsed + BigInt(m[7]!) < metaTotal && dataUsed < dataTotal, + "fc_pool_capacity", + ); + return { + dataFree: (dataTotal - dataUsed) * blockSectors * 512n, + dataTotal: dataTotal * blockSectors * 512n, + metadataFree: (metaTotal - metaUsed) * 4096n, + metadataTotal: metaTotal * 4096n, + }; +} +// Kernel guidance: 48 metadata bytes per data block, plus 25% reserve. +// https://docs.kernel.org/admin-guide/device-mapper/thin-provisioning.html +const metadataBudget = (bytes: bigint, blockSectors: bigint) => + (((bytes + blockSectors * 512n - 1n) / (blockSectors * 512n)) * 48n * 5n + + 3n) / + 4n; +export function admitPool(pool: PoolInventory) { + const table = parsePoolTable(pool.table); + const capacity = parsePoolStatus(pool.status, table.blockSectors); + check( + pool.mappings.length === 0 && + pool.thinIds.length === 0 && + pool.holders.length === 0 && + pool.fds.length === 0, + "fc_pool_not_empty", + ); + check( + capacity.dataFree * 5n >= capacity.dataTotal && + capacity.metadataFree >= + metadataBudget(capacity.dataTotal, table.blockSectors) && + capacity.metadataFree * 10n >= capacity.metadataTotal, + "fc_pool_capacity", + ); + check( + table.sectors * 512n === BigInt(pool.data.bytes) && + capacity.dataTotal <= BigInt(pool.data.bytes), + "fc_pool_backing_size", + ); + for (const backing of [pool.data, pool.metadata]) + check( + BigInt(backing.allocated) >= BigInt(backing.bytes), + "fc_pool_sparse_backing", + ); +} +export function verifyPoolIdentity( + expected: PoolInventory, + actual: PoolInventory, +) { + const backingIdentity = ({ allocated: _allocated, ...identity }: Backing) => + identity; + const identity = (p: PoolInventory) => ({ + name: p.name, + directory: p.directory, + device: p.device, + path: p.path, + table: p.table, + major: p.major, + metadata: backingIdentity(p.metadata), + data: backingIdentity(p.data), + control: p.control, + node: p.node, + otherTables: p.otherTables, + }); + check( + JSON.stringify(identity(expected)) === JSON.stringify(identity(actual)), + "fc_pool_identity_changed", + ); + for (const pool of [expected, actual]) + for (const backing of [pool.data, pool.metadata]) + check( + BigInt(backing.allocated) >= BigInt(backing.bytes), + "fc_pool_sparse_backing", + ); +} +export async function dm(args: string[]) { + return (await command("/usr/sbin/dmsetup", args, "/", childEnvironment("/"))) + .toString() + .trim(); +} +export async function deviceNode( + path: string, + character = false, +): Promise { + const s = await lstat(path, { bigint: true }); + check( + (character ? s.isCharacterDevice() : s.isBlockDevice()) && + s.uid === 0n && + !s.isSymbolicLink() && + (await realpath(path)) === path, + "fc_device_node", + ); + return { + path, + device: linuxDevice(s.rdev), + ino: String(s.ino), + rdev: String(s.rdev), + }; +} +async function backing(device: string, file: string): Promise { + const sys = `/sys/dev/block/${device}`; + const uevent = await readFile(join(sys, "uevent"), "utf8"); + const name = /^DEVNAME=(loop[0-9]+)$/m.exec(uevent)?.[1]; + check(name, "fc_pool_loop_device"); + const node = await deviceNode(`/dev/${name}`); + const linked = ( + await readFile(join(sys, "loop/backing_file"), "utf8") + ).trim(); + check( + node.device === device && + linked === file && + (await realpath(file)) === file, + "fc_pool_backing_path", + ); + const s = await lstat(file, { bigint: true }); + check( + s.isFile() && + s.nlink === 1n && + s.uid === BigInt(process.getuid?.() ?? -1) && + (s.mode & 0o022n) === 0n, + "fc_pool_backing_owner", + ); + // losetup reads the kernel's loop_info64, including backing inode/device. + const loops = JSON.parse( + ( + await command( + "/usr/sbin/losetup", + [ + "--json", + "--output", + "NAME,BACK-FILE,BACK-INO,BACK-MAJ:MIN,OFFSET,SIZELIMIT", + node.path, + ], + "/", + childEnvironment("/"), + ) + ).toString(), + ); + const loop = loops.loopdevices?.[0]; + check( + loops.loopdevices?.length === 1 && + loop.name === node.path && + loop["back-file"] === file && + String(loop["back-ino"]) === String(s.ino) && + loop["back-maj:min"] === linuxDevice(s.dev) && + Number(loop.offset) === 0 && + Number(loop.sizelimit) === 0, + "fc_pool_backing_identity", + ); + return { + ...node, + file, + fileDev: String(s.dev), + fileIno: String(s.ino), + bytes: String(s.size), + allocated: String(s.blocks * 512n), + }; +} +export async function poolMappings(poolDevice: string): Promise { + const raw = await dm([ + "info", + "-c", + "--noheadings", + "--separator", + "|", + "-o", + "name,major,minor,open", + ]); + const found: ThinMapping[] = []; + for (const line of raw.split("\n").filter(Boolean)) { + const [name, major, minor, opened, ...extra] = line + .trim() + .split("|") + .map((v) => v.trim()); + check( + name && + /^[A-Za-z0-9_.+-]+$/.test(name) && + /^[0-9]+$/.test(major!) && + /^[0-9]+$/.test(minor!) && + /^[0-9]+$/.test(opened!) && + extra.length === 0, + "fc_dm_inventory", + ); + const table = await dm(["table", name]); + if (!table.includes(" thin ")) continue; + const m = /^0 ([1-9][0-9]*) thin ([0-9]+:[0-9]+) ([0-9]+)$/.exec(table); + check(m, "fc_thin_table"); + if (m[2] === poolDevice) + found.push({ + name, + device: `${major}:${minor}`, + table, + id: Number(m[3]), + open: Number(opened), + }); + } + return found.sort((a, b) => a.name.localeCompare(b.name)); +} +const offlineLease = Symbol("offline-pool-inspection"); +export type OfflinePoolLease = { readonly [offlineLease]: () => boolean }; + +/** The fixture blocks every host start while an offline pool read owns this gate. + * Stopped state is granted only after the caller proves kernel quiescence. */ +export class OfflinePoolAccess { + private hostMayRun = false; + private inspecting = false; + get inspectionActive() { + return this.inspecting; + } + start() { + check( + !this.inspecting && !this.hostMayRun, + "fc_metadata_host_start_blocked", + ); + this.hostMayRun = true; + } + async stopped(prove: () => Promise) { + check(!this.inspecting, "fc_pool_inspection_busy"); + this.inspecting = true; + try { + await prove(); + this.hostMayRun = false; + } finally { + this.inspecting = false; + } + } + async inspect( + prove: () => Promise, + read: (lease: OfflinePoolLease) => Promise, + ): Promise { + check(!this.hostMayRun && !this.inspecting, "fc_pool_host_not_quiescent"); + this.inspecting = true; + let held = true; + const lease = { + [offlineLease]: () => held && this.inspecting && !this.hostMayRun, + }; + try { + await prove(); + return await read(lease); + } finally { + held = false; + this.inspecting = false; + } + } +} + +export async function inspectPool( + input: ThinPoolInput, + lease?: OfflinePoolLease, +): Promise { + input = parseThinPool({ name: input.name, directory: input.directory }); + check(input.name.length <= 26, "fc_pool_name_volume_budget"); + const table = await dm(["table", input.name]); + const parsed = parsePoolTable(table); + const number = await dm([ + "info", + "-c", + "--noheadings", + "--separator", + ":", + "-o", + "major,minor", + input.name, + ]); + check(/^[0-9]+:[0-9]+$/.test(number), "fc_pool_device"); + const sys = `/sys/dev/block/${number}`; + const devname = /^DEVNAME=(dm-[0-9]+)$/m.exec( + await readFile(join(sys, "uevent"), "utf8"), + )?.[1]; + check( + devname && + (await readFile(join(sys, "dm/name"), "utf8")).trim() === input.name, + "fc_pool_name_identity", + ); + const node = await deviceNode(`/dev/${devname}`); + check(node.device === number, "fc_pool_device_identity"); + const metadata = await backing( + parsed.metadata, + join(input.directory, "metadata.img"), + ); + const data = await backing(parsed.data, join(input.directory, "data.img")); + const status = await dm(["status", input.name]); + parsePoolStatus(status, parsed.blockSectors, !lease); + const thinIds = lease + ? await readThinIds(input.name, metadata.path, status, lease) + : []; + const filesystem = await statfs(input.directory, { bigint: true }); + check( + filesystem.bavail * filesystem.bsize >= 16n * 1024n ** 3n, + "fc_pool_filesystem_reserve", + ); + const mappings = await poolMappings(number); + const holders = await readdir(join(sys, "holders")); + for (const holder of holders) { + const device = ( + await readFile(join(sys, "holders", holder, "dev"), "utf8") + ).trim(); + check( + mappings.some((m) => m.device === device), + "fc_foreign_pool_holder", + ); + } + const ownNames = new Set([input.name, ...mappings.map((m) => m.name)]); + const tables = await dm(["table"]); + const otherTables = tables + .split("\n") + .filter((line) => !ownNames.has(line.split(": ")[0]!)) + .sort() + .join("\n"); + check(!/ thin(?:-pool)? /.test(otherTables), "fc_foreign_thin_pool"); + return { + ...input, + device: number, + path: node.path, + table, + major: number.split(":")[0]!, + node, + otherTables, + metadata, + data, + control: await deviceNode("/dev/mapper/control", true), + status, + thinIds, + mappings, + holders, + fds: [], + }; +} + +export type MetadataReadState = { pending: Promise; failed: boolean }; +const metadataRead: MetadataReadState = { + pending: Promise.resolve(), + failed: false, +}; +export async function readThinIds( + name: string, + path: string, + status: string, + lease: OfflinePoolLease, + control = dm, + dump = async () => + ( + await command( + "/usr/sbin/thin_dump", + ["--skip-mappings", "-m", path], + "/", + childEnvironment("/"), + ) + ).toString(), + state: MetadataReadState = metadataRead, +): Promise { + const previous = state.pending; + let release!: () => void; + state.pending = new Promise((resolve) => { + release = resolve; + }); + await previous; + try { + check(lease[offlineLease](), "fc_pool_host_not_quiescent"); + check(!state.failed, "fc_pool_metadata_release_uncertain"); + parsePoolStatus(status, 128n); + await control(["message", name, "0", "reserve_metadata_snap"]); + try { + const xml = await dump(); + check( + xml.includes(""), + "fc_pool_metadata_dump", + ); + const ids = [...xml.matchAll(/ + Number(m[1]), + ); + check( + (xml.match(/ Number.isSafeInteger(id) && id <= 0xffffff), + "fc_pool_metadata_ids", + ); + return ids; + } finally { + try { + await control(["message", name, "0", "release_metadata_snap"]); + } catch (error) { + state.failed = true; + throw error; + } + } + } finally { + release(); + } +} + +export function requirePoolBudget( + pool: PoolInventory, + rootBytes: bigint, + swapBytes: bigint, + imageBytes = rootBytes, +) { + const capacity = parsePoolStatus( + pool.status, + parsePoolTable(pool.table).blockSectors, + true, + ); + // One staged image plus two live, two checkpoint, and two in-flight root + // volumes; each swap has the same overlap, without a staged image. + const overlap = imageBytes + rootBytes * 6n + swapBytes * 6n; + const reserve = (overlap + 3n) / 4n; + check( + capacity.dataTotal >= overlap + reserve && + capacity.dataFree >= reserve && + capacity.metadataFree >= + metadataBudget( + overlap + reserve, + parsePoolTable(pool.table).blockSectors, + ), + "fc_pool_overlap_capacity", + ); +} +export function verifyBootDevice( + previous: string | undefined, + current: string, +) { + check( + previous === undefined || previous === current, + "fc_boot_device_changed", + ); +} + +export function verifyThinIds(pool: PoolInventory) { + check( + pool.thinIds.length === pool.mappings.length && + pool.thinIds.every((id) => pool.mappings.some((m) => m.id === id)), + "fc_unexplained_inactive_thin", + ); +} +export function verifyReleasedMappings( + pool: PoolInventory, + retained: ReadonlyMap, +) { + for (const m of pool.mappings) { + const previous = retained.get(m.name); + const volume = m.name.slice(pool.name.length + 1); + check( + previous && + m.name.startsWith(pool.name + ".") && + /^(?:img-[a-f0-9]{32}-v[0-9]+|sb-[a-f0-9]{32}-(?:root|swap)-ckpt-[a-f0-9]{32}-v[0-9]+)$/.test( + volume, + ), + "fc_unreleased_volume", + ); + const expected = JSON.parse(previous); + check( + m.id === expected.id && + m.device === expected.device && + m.table === expected.table && + m.open === 0, + "fc_retained_volume_changed_or_busy", + ); + } +} diff --git a/tests/fixtures/mcp-stack/platform/fixture.Dockerfile b/tests/fixtures/mcp-stack/platform/fixture.Dockerfile new file mode 100644 index 000000000..f9ad72a60 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/fixture.Dockerfile @@ -0,0 +1,25 @@ +FROM node:24.22.2-bookworm-slim@sha256:c86ecead9e55a5210ac1727de38a4ca840bf03dcaf34dc9749addbf383b1920c AS source +WORKDIR /app +COPY tests/fixtures/mcp-stack/main.mts tests/fixtures/mcp-stack/config.mts tests/fixtures/mcp-stack/durable.mts ./tests/fixtures/mcp-stack/ +COPY mcp-worker/package.json ./mcp-worker/ +COPY mcp-worker/test/fixtures/server.ts mcp-worker/test/fixtures/http.ts mcp-worker/test/fixtures/oauth.ts ./mcp-worker/test/fixtures/ +# The configured entrypoint requires runtime TLS. Do not package static test keys. +RUN printf '%s\n' \ + 'export const TEST_CA = "";' \ + 'export const TEST_CERT = "";' \ + 'export const TEST_KEY = "";' \ + > mcp-worker/test/fixtures/tls.ts + +FROM node:24.22.2-bookworm-slim@sha256:c86ecead9e55a5210ac1727de38a4ca840bf03dcaf34dc9749addbf383b1920c +ARG SOURCE_COMMIT +LABEL org.opencontainers.image.revision=${SOURCE_COMMIT} +RUN set -eu; \ + test "${#SOURCE_COMMIT}" = 40; \ + case "$SOURCE_COMMIT" in *[!a-f0-9]*) exit 2 ;; esac +ENV NODE_ENV=production +WORKDIR /app +COPY --from=source /app/tests/fixtures/mcp-stack/ ./tests/fixtures/mcp-stack/ +COPY --from=source /app/mcp-worker/package.json ./mcp-worker/ +COPY --from=source /app/mcp-worker/test/fixtures/ ./mcp-worker/test/fixtures/ +USER node +ENTRYPOINT ["/usr/local/bin/node", "/app/tests/fixtures/mcp-stack/main.mts"] diff --git a/tests/fixtures/mcp-stack/platform/guard.Dockerfile b/tests/fixtures/mcp-stack/platform/guard.Dockerfile new file mode 100644 index 000000000..53cf47dfe --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/guard.Dockerfile @@ -0,0 +1,33 @@ +FROM debian:trixie-slim@sha256:f6b08b09a05725f9431502d3b5714f5c4576e82c1a84fa1debe0c24a7e0c3ec4 +ARG SOURCE_COMMIT +LABEL org.opencontainers.image.revision=${SOURCE_COMMIT} +# Retry whole commands to avoid the APT retry-queue memory bug. +# https://bugs.debian.org/1152704 +RUN set -eu; \ + test "${#SOURCE_COMMIT}" = 40; \ + case "$SOURCE_COMMIT" in *[!a-f0-9]*) exit 2 ;; esac; \ + rm -f /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources; \ + printf '%s\n' \ + 'deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/20261102T000000Z trixie main' \ + 'deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/20261102T000000Z trixie-security main' \ + > /etc/apt/sources.list; \ + apt_retry() { \ + for attempt in 1 2 3 4; do \ + if apt-get -o Acquire::Retries=0 -o Acquire::http::Timeout=30 "$@"; then return 0; fi; \ + done; \ + return 1; \ + }; \ + apt_retry --error-on=any update; \ + apt_retry install -y --no-install-recommends \ + bash coreutils findutils ca-certificates nftables bind9-dnsutils \ + netcat-openbsd curl skopeo squashfs-tools iproute2 openssl; \ + rm -rf /var/lib/apt/lists/*; \ + for tool in /bin/sh /bin/bash /bin/sleep /bin/mkdir /bin/chown /bin/chmod \ + /usr/bin/find /usr/bin/sha256sum /usr/bin/skopeo /usr/sbin/nft \ + /usr/bin/dig /usr/bin/nc /usr/bin/curl /usr/bin/mksquashfs \ + /usr/bin/dirname /usr/bin/rm /usr/bin/cut /usr/sbin/ip /usr/bin/openssl; do test -x "$tool"; done +# Numeric packing users need no passwd entry or private home directory. +ENV HOME=/tmp +USER 1000:1000 +ENTRYPOINT ["/bin/sleep"] +CMD ["infinity"] diff --git a/tests/fixtures/mcp-stack/platform/guest.Dockerfile b/tests/fixtures/mcp-stack/platform/guest.Dockerfile new file mode 100644 index 000000000..e29928260 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/guest.Dockerfile @@ -0,0 +1,3 @@ +# Fixture tail appended to the unchanged deploy/demo/Dockerfile, not a standalone recipe. +# The guest runs as root. This on-disk directory survives snapshot and resume. +RUN install -d -m 0700 /tmp/mcp-stack-fixture diff --git a/tests/fixtures/mcp-stack/platform/host-agent.Dockerfile b/tests/fixtures/mcp-stack/platform/host-agent.Dockerfile new file mode 100644 index 000000000..9f2f4132b --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/host-agent.Dockerfile @@ -0,0 +1,16 @@ +# syntax=docker/dockerfile:1.7 +# Test fixture only. The base is the production host image at an immutable +# manifest digest. Its revision label and entrypoint remain unchanged. +ARG HOST_IMAGE +FROM ${HOST_IMAGE} +ARG HOST_IMAGE +ARG FIRECRACKER_SHA256 +RUN set -eu; \ + case "$HOST_IMAGE" in *@sha256:*) digest="${HOST_IMAGE##*@sha256:}" ;; *) exit 2 ;; esac; \ + test "${#digest}" = 64; \ + case "$digest" in *[!a-f0-9]*) exit 2 ;; esac +COPY --chmod=0755 firecracker /usr/local/bin/firecracker +RUN set -eu; \ + test "${#FIRECRACKER_SHA256}" = 64; \ + case "$FIRECRACKER_SHA256" in *[!a-f0-9]*) exit 2 ;; esac; \ + printf '%s /usr/local/bin/firecracker\n' "$FIRECRACKER_SHA256" | sha256sum -c - diff --git a/tests/fixtures/mcp-stack/platform/identity.ts b/tests/fixtures/mcp-stack/platform/identity.ts new file mode 100644 index 000000000..0a157b046 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/identity.ts @@ -0,0 +1,252 @@ +import { + X509Certificate, + createPrivateKey, + createPublicKey, + randomBytes, + randomUUID, +} from "node:crypto"; +import { join } from "node:path"; +import { + check, + childEnvironment, + command, + hash, + newRoot, + privateRoot, + put, + readOwned, + verifyPin, + type Pin, +} from "./owned.ts"; +import type { Inputs } from "./preflight.ts"; + +export interface Leaf { + certificate: Pin; + key: Pin; + fingerprint: string; + hostPin: string; +} +export interface Identity { + runId: string; + hostId: string; + ca: Pin; + caKey: Pin; + leaves: Record< + | "orchestrator" + | "coordinator" + | "operator" + | "host" + | "worker" + | "app" + | "modern" + | "legacy", + Leaf + >; + secrets: Record< + | "supervisor" + | "kek" + | "postgres" + | "coordinator" + | "appGrpc" + | "auth" + | "broker" + | "router" + | "bootstrapPassword" + | "adminPassword" + | "ownerPassword" + | "otherPassword", + Pin + >; +} +export async function generateIdentity( + inputs: Inputs, + root: string, +): Promise { + await privateRoot(root); + await verifyPin(inputs.binaries.openssl); + check( + ( + await command( + inputs.binaries.openssl.path, + ["version"], + root, + childEnvironment(root), + ) + ) + .toString() + .startsWith("OpenSSL 3."), + "openssl3", + ); + const tls = join(root, "tls"), + secretsRoot = join(root, "secrets"); + await newRoot(tls); + await newRoot(secretsRoot); + const openssl = (args: string[]) => + command( + inputs.binaries.openssl.path, + args, + tls, + childEnvironment(root), + undefined, + 10000, + ); + const caPath = join(tls, "ca.pem"), + caKeyPath = join(tls, "ca.key"); + await put(caPath, ""); + await put(caKeyPath, ""); + await openssl([ + "req", + "-x509", + "-newkey", + "ec", + "-pkeyopt", + "ec_paramgen_curve:P-256", + "-nodes", + "-days", + "2", + "-subj", + "/CN=MCP isolated fixture root", + "-keyout", + caKeyPath, + "-out", + caPath, + "-addext", + "basicConstraints=critical,CA:TRUE", + "-addext", + "keyUsage=critical,keyCertSign,cRLSign", + ]); + const pin = async (path: string): Promise => ({ + path, + sha256: hash(await readOwned(path)), + }); + const ca = await pin(caPath), + caKey = await pin(caKeyPath); + const caCert = new X509Certificate(await readOwned(ca.path)); + const leafNames = [ + "orchestrator", + "coordinator", + "operator", + "host", + "worker", + "app", + "modern", + "legacy", + ] as const; + const leaves: Partial = {}; + for (const role of leafNames) { + const names = + role === "modern" || role === "legacy" + ? ["resource", "issuer", "authorization", "token", "observer"].map( + (name) => `${name}.${role}.fixture.example`, + ) + : role === "coordinator" + ? [ + "coordinator.fixture.example", + "control.fixture.example", + "operator.fixture.example", + ] + : [`${role}.fixture.example`]; + const keyPath = join(tls, `${role}.key`), + certPath = join(tls, `${role}.pem`), + csrPath = join(tls, `${role}.csr`); + const extension = await put( + join(tls, `${role}.ext`), + `basicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=serverAuth,clientAuth\nsubjectAltName=${names.map((name) => `DNS:${name}`).join(",")}\n`, + ); + for (const path of [keyPath, certPath, csrPath]) await put(path, ""); + await openssl([ + "req", + "-new", + "-newkey", + "ec", + "-pkeyopt", + "ec_paramgen_curve:P-256", + "-nodes", + "-subj", + `/CN=${names[0]}`, + "-keyout", + keyPath, + "-out", + csrPath, + ]); + await openssl([ + "x509", + "-req", + "-in", + csrPath, + "-CA", + ca.path, + "-CAkey", + caKey.path, + "-set_serial", + `0x${randomBytes(16).toString("hex")}`, + "-days", + "1", + "-extfile", + extension.path, + "-out", + certPath, + ]); + const certificate = await pin(certPath), + key = await pin(keyPath); + const parsed = new X509Certificate(await readOwned(certPath)); + check( + parsed.verify(caCert.publicKey) && + names.every((name) => parsed.checkHost(name) === name), + "leaf_identity", + ); + const publicKey = createPublicKey( + createPrivateKey(await readOwned(keyPath)), + ).export({ type: "spki", format: "der" }); + check( + publicKey.equals( + parsed.publicKey.export({ type: "spki", format: "der" }), + ), + "leaf_key_pair", + ); + leaves[role] = { + certificate, + key, + fingerprint: parsed.fingerprint256, + hostPin: hash(parsed.raw), + }; + } + check( + new Set(Object.values(leaves).map((v) => v.fingerprint)).size === + leafNames.length, + "distinct_leaves", + ); + const secretNames = [ + "supervisor", + "kek", + "postgres", + "coordinator", + "appGrpc", + "auth", + "broker", + "router", + "bootstrapPassword", + "adminPassword", + "ownerPassword", + "otherPassword", + ] as const; + const secrets: Partial = {}; + for (const name of secretNames) { + const bytes = randomBytes(32); + const value = + name === "kek" + ? bytes.toString("base64") + : (name === "router" ? "fixture-neutral-" : "") + + bytes.toString("base64url"); + secrets[name] = await put(join(secretsRoot, name), value); + } + // Every named entry was generated by the fixed exhaustive lists above. + return { + runId: `mcp-${randomBytes(12).toString("hex")}`, + hostId: randomUUID(), + ca, + caKey, + leaves: leaves as Identity["leaves"], + secrets: secrets as Identity["secrets"], + }; +} diff --git a/tests/fixtures/mcp-stack/platform/image-recipes.md b/tests/fixtures/mcp-stack/platform/image-recipes.md new file mode 100644 index 000000000..dd51caf09 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/image-recipes.md @@ -0,0 +1,101 @@ +# Fixed platform image recipes + +These recipes supply the `guard` and `fixture` image inputs for the platform +adapter. They build from the repository root for `linux/amd64` or `linux/arm64`. +They do not build a worker image or install packages in a guest. + +The image producer must use a clean checkout and pass its full current HEAD as +`SOURCE_COMMIT`. Both recipes reject a missing or malformed value and set +`org.opencontainers.image.revision`. The fixture preflight checks that label +against `Inputs.sourceCommit`. The producer must still export a single-platform +OCI archive, record its file hash, manifest digest and config digest, and load +the same image into the selected engine. A recipe is not a physical test result. + +## Bases + +| Recipe | Official immutable image index | +| --- | --- | +| Guard | `debian:trixie-slim@sha256:f6b08b09a05725f9431502d3b5714f5c4576e82c1a84fa1debe0c24a7e0c3ec4` | +| Fixture | `node:24.22.2-bookworm-slim@sha256:c86ecead9e55a5210ac1727de38a4ca840bf03dcaf34dc9749addbf383b1920c` | + +Both [official Debian](https://hub.docker.com/_/debian) and +[official Node](https://hub.docker.com/_/node) indexes were read from the Docker +Registry on 2026-11-02. Each index has Linux amd64 and arm64 manifests. Guard +packages come from the fixed Debian and Debian Security snapshots at +`20261102T000000Z`. APT checks repository signatures; the fixed snapshots disable +only metadata expiry checks. No runtime package installation occurs. + +## Guard consumers + +| Consumer | Required paths and behavior | +| --- | --- | +| `engine.ts` projection | `/bin/sleep`, `/bin/sh`, `/bin/mkdir`, `/bin/chown`, `/bin/chmod`, `/usr/bin/find`, `/usr/bin/sha256sum` | +| `engine.ts` network guards | `/usr/sbin/nft`, `/usr/bin/nc`, `/usr/bin/curl`, `/usr/bin/dig` | +| `assets.ts` archive checks and guest publication | `/bin/sleep`, `/usr/bin/skopeo` | +| `assets.ts` bundle packing | `/bin/bash`, `/usr/bin/mksquashfs`; coreutils supply `dirname`, `rm`, `cut`, and `sha256sum` for the production harness leaves and `_pack.sh` | +| `launch.ts` dependency probes and denied endpoint | `/bin/sleep`, `/usr/bin/nc`, `/usr/bin/dig`, `/usr/bin/curl` | + +The default user is `1000:1000`. The launcher overrides it with the caller's +numeric UID:GID for archive checks, publication, and packing. `HOME=/tmp` permits +these operations without a passwd entry. Packing uses the writable staging bind +at its original absolute path, read-only production recipes at `/recipes`, and +`/tmp` tmpfs. Archive inputs are read-only files under `/input`. The image carries +no engine client, engine socket, host credentials, auth file, or proxy setting. +Skopeo uses its packaged system policy and the explicit archive or registry +arguments from the adapter. + +The engine drops all capabilities. The launcher grants `CHOWN`, `DAC_OVERRIDE`, +and `FOWNER` only to the root file-projection container. Network guards run as +root with `NET_ADMIN` only, a read-only root filesystem, `/tmp` tmpfs, and a +read-only CA-only projection at `/etc/mcp-probe`. The recipe grants no capability. +The denied endpoint and dependency probes run as `1000:1000`. + +## Fixture consumers + +`launch.ts` uses this exact command for both fixture roles: + +```text +/usr/local/bin/node /app/tests/fixtures/mcp-stack/main.mts CONFIG +``` + +`CONFIG` is the absolute path to the launcher's private config. It selects the +modern or legacy protocol. There is no separate protocol command. The launcher +overrides the default `node` user with its current numeric UID:GID, mounts the +private run root writable at the same absolute path, and makes the root +filesystem read-only. `/tmp` is a tmpfs. The private run root contains the config, +dynamic TLS files, neutral API and control credentials, and durable effect state. +It must be accessible to that UID:GID. The image does not contain this data. + +The image has the unchanged `main.mts`, `config.mts`, and `durable.mts` modules at +`/app/tests/fixtures/mcp-stack/`. It also has the unchanged `server.ts`, `http.ts`, +and `oauth.ts` modules at `/app/mcp-worker/test/fixtures/`, and +`/app/mcp-worker/package.json` to retain the module scope. Runtime imports use +Node built-ins and these local files. The only package import is the type-only +`Tool` import from `@modelcontextprotocol/client`; Node 24 erases it. No npm +runtime dependency or install is needed for this entrypoint. + +The recipe generates `tls.ts` with empty-string `TEST_CA`, `TEST_CERT`, and +`TEST_KEY` exports. It never copies the source TLS module into an image stage. +The final image copies only the prepared module closure. No image layer contains +the static test TLS key. `main.mts` requires a config, validates the runtime TLS +and authority fields, and passes them to the fixture. Missing runtime TLS fails +startup. The empty exports provide no valid fallback TLS identity or trust root. + +The existing literal `fixture-client-secret` in `oauth.ts` is a public, +nonproduction protocol test datum. It remains unchanged so the deterministic +OAuth fixture keeps its existing behavior. Dynamic keys, tokens, credentials, +catalog configuration, and durable state remain launcher-owned. + +## Validation boundary + +Static source checks can establish paths, module closure, fixed base identities, +and recipe syntax. They do not establish package installation, final executable +behavior, arbitrary-user packing, TLS service startup, OCI archive linkage, +engine loading, or either physical backend result. Image build and physical +consumer checks must establish those facts before acceptance. + +The guard also carries `/usr/sbin/ip` from `iproute2` for the fixed native-host +/32 route in its own network namespace, and `/usr/bin/openssl` for the short +coordinator-client mutual TLS probe. Both tools use the same fixed APT snapshot +and executable checks as the existing tools. These tools add no host route, +capability, mount or credential to the worker. diff --git a/tests/fixtures/mcp-stack/platform/inputsREADME.md b/tests/fixtures/mcp-stack/platform/inputsREADME.md new file mode 100644 index 000000000..5d0950034 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/inputsREADME.md @@ -0,0 +1,88 @@ +# Staged platform Inputs + +`.github/scripts/mcp-stack-inputs.mts` writes the private Inputs JSON that the +platform adapter consumes. It runs offline and uses the actual `parseInputs`, +`BINARIES`, and `IMAGES` from [preflight.ts](./preflight.ts). A successful write +establishes file hashes and schema validity. It does not qualify an engine, an +image, a host, or a physical backend. + +## Template and final manifest + +The trusted staging job supplies a private JSON template with the current +`Inputs` fields. The only difference is the shape of each file pin: + +| Location | Template | Final Inputs | +| -------------------------------------- | ---------------------------------------------- | ---------------------------------------------------------------------- | +| Each `binaries` entry | `{ "path": "/canonical/staged/file" }` | `{ "path": "/canonical/staged/file", "sha256": "" }` | +| `proxy`, `firecracker`, `nativeProbe` | The same path-only pin | The same final file pin | +| Each `images` entry's `archive` | The same path-only pin | The same final file pin | +| Each image's `id` | Explicit config digest from the producer | Preserved | +| Each image's `manifestDigest` | Explicit raw manifest digest from the producer | Preserved | +| `sourceCommit` and other Inputs fields | Explicit values from the CI stage | Preserved and checked by `parseInputs` | + +Supply every binary role in `BINARIES` and every image role in `IMAGES`. +Firecracker also requires the `firecracker` pin and the `images.hostAgent` entry; +it must omit `nativeProbe`. VZ requires `nativeProbe` and `architecture: "arm64"`; +it must omit `firecracker` and `images.hostAgent`. Both backends require `proxy`. +Missing fields, extra fields, and caller-supplied `sha256` fields fail validation. + +Each image has three separate facts. The writer hashes the OCI archive file to +produce `archive.sha256`. The actual OCI producer supplies `manifestDigest` as +`sha256:` plus the hash of the raw single-platform manifest bytes. It supplies +`id` as `sha256:` plus the config blob hash, which must match the loaded Docker +image ID. The writer does not derive, replace, or validate the linkage of these +two producer facts. Do not substitute the archive hash for either digest. + +## Invocation + +Use Node 24. The transform option is necessary because the shared platform +modules use TypeScript parameter properties. + +```sh +node --experimental-transform-types .github/scripts/mcp-stack-inputs.mts \ + --backend firecracker \ + --template /private/staging/template.json \ + --output /private/staging/inputs.json +``` + +Use `--backend vz` for VZ. The flags have this exact order; no additional flags +are accepted. Both JSON paths must be absolute and canonical. The template must +be a nonempty, owned, single-link regular file of at most 1 MiB, with no group or +other permission bits. Its path and parent components must not be symlinks. + +Every staged artifact must be a nonempty, owned, single-link regular file at a +canonical path with no symlink components. The writer streams each file through +one descriptor and checks its identity, size, and modification/change times. +Make owned copies of downloads and tool binaries; do not use symlinks or hard +links. The writer does not copy files, discover tool paths, load images, execute +commands, or read a home directory or cloud credentials. + +The output parent must already be an owned canonical directory with mode 0700. +The output path must be absent. After all file and schema checks pass, the writer +creates the manifest exclusively with mode 0600. It never replaces an existing +file or link. Success has exit code zero and no standard output. Failure has a +nonzero exit code and does not report platform acceptance. + +## CI staging and remaining gates + +Run the writer as the same UID that runs the platform adapter. Firecracker +preflight requires root: stage actual root-owned file copies, the root-owned +private template, and the root-owned 0700 output parent before running the writer +as root. Running a normal-runner-owned template through `sudo` is insufficient. +For VZ, stage and write as the native arm64 macOS runner UID. Sign the final VZ +host binary before hashing it. Retain executable modes on executable artifacts. + +Real producer jobs must still build or obtain the complete binary and image set, +export single-platform OCI archives, collect their raw manifest/config digests, +and load the same images into the selected engine. The stage must supply the +actual source commit, engine identity/socket/config, private network addresses, +and routing prerequisites. The writer preserves `sourceCommit`; it does not +discover HEAD or certify a clean checkout. + +The existing preflight and asset staging checks must still verify clean matching +HEAD, source revision labels, binary architecture and versions, signatures and +entitlements, image/archive/manifest/config linkage, loaded engine identity, and +the selected backend prerequisites. The actual producer/consumer workflow edges, +native DNS and routes, and required Firecracker and VZ adapter runs remain +necessary. No supplied PASS field replaces those checks. This source tool does +not establish image production or physical acceptance. diff --git a/tests/fixtures/mcp-stack/platform/launch.ts b/tests/fixtures/mcp-stack/platform/launch.ts new file mode 100644 index 000000000..9d65d6fb8 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/launch.ts @@ -0,0 +1,1320 @@ +import { request as httpsRequest } from "node:https"; +import { chmod, lstat, mkdir, open } from "node:fs/promises"; +import { join, relative } from "node:path"; +import { randomBytes } from "node:crypto"; +import type { + Context, + ConfigurationId, +} from "../../../../orchestrator/scripts/mcp-stack/context.ts"; +import { + ROLES, + type MutableRole, + type Role, + type SupervisorStatus, +} from "../../../../orchestrator/scripts/mcp-stack/supervisor.ts"; +import { + jsonRequest, + tlsOptions, +} from "../../../../orchestrator/scripts/mcp-stack/transport.ts"; +import { ZonePolicy } from "../../../../mcp-worker/src/network.ts"; +import { FixtureObserver } from "../../../../orchestrator/scripts/mcp-stack/observer.ts"; +import { RunnerFailure } from "../../../../orchestrator/scripts/mcp-stack/failure.ts"; +import { + check, + childEnvironment, + command, + hash, + OwnedProcess, + PlatformFailure, + put, + readOwned, + verifyPin, + type Pin, +} from "./owned.ts"; +import { address, dnsHosts, dnsRecords } from "./network.ts"; +import { type Inputs, REPO } from "./preflight.ts"; +import type { Identity } from "./identity.ts"; +import { Engine, drainChecks } from "./engine.ts"; +import type { stageAssets } from "./assets.ts"; +import type { generateConfiguration } from "./configuration.ts"; + +type Assets = Awaited>; +type Configuration = Awaited>; +type NativeIdentity = { + pid: number; + uid: number; + parent: number; + group: number; + seconds: string; + microseconds: string; + executable: string; +}; +export function parseNativeIdentities( + value: unknown, + root: string, +): NativeIdentity[] { + check(Array.isArray(value) && value.length < 4096, "native_identity_count"); + return value.map((v: unknown) => { + check( + v !== null && typeof v === "object" && !Array.isArray(v), + "native_identity", + ); + const p = v as Record; + check( + Object.keys(p).sort().join(",") === + "executable,group,microseconds,parent,pid,seconds,uid", + "native_identity_keys", + ); + for (const key of ["pid", "uid", "parent", "group"]) + check( + typeof p[key] === "number" && + Number.isSafeInteger(p[key]) && + Number(p[key]) >= 0, + "native_identity_number", + ); + check( + Number(p.pid) > 1 && + p.uid === process.getuid?.() && + typeof p.seconds === "string" && + /^[0-9]{1,20}$/.test(p.seconds) && + typeof p.microseconds === "string" && + /^[0-9]{1,6}$/.test(p.microseconds) && + Number(p.microseconds) < 1000000, + "native_identity_birth", + ); + check( + typeof p.executable === "string" && + (p.executable === join(root, "assets/hostAgent") || + p.executable === join(root, "assets/proxy") || + (p.executable.startsWith(join(root, "host") + "/") && + p.executable.endsWith("/netstack") && + !p.executable.split("/").includes(".."))), + "native_identity_scope", + ); + // Every field has an exact primitive check above. + return p as NativeIdentity; + }); +} +const nativeKey = (p: NativeIdentity) => + JSON.stringify([p.pid, p.uid, p.seconds, p.microseconds, p.executable]); +export const hostListenAddress = ( + inputs: Pick, +) => + inputs.backend === "firecracker" + ? "0.0.0.0:9101" + : `${inputs.nativeHostIp}:9101`; +type RoleConfig = { env: Record; files: Record }; +export class Stack { + private host: OwnedProcess | undefined; + private hostBoot = "not-started"; + private nativeIdentities = new Map(); + private readonly stopped = new Set(); + private readonly active = new Map< + Role, + { id: string; configuration: ConfigurationId } + >(); + private readonly configs = new Map< + MutableRole, + Partial> + >(); + private readonly projections = new Map(); + private readonly environmentFiles = new Map(); + constructor( + readonly inputs: Inputs, + readonly root: string, + readonly identity: Identity, + readonly assets: Assets, + readonly configuration: Configuration, + readonly engine: Engine, + ) {} + private secret(name: keyof Identity["secrets"]) { + return readOwned(this.identity.secrets[name].path).then((bytes) => + bytes.toString(), + ); + } + private files(pins: Pin[]) { + return Object.fromEntries( + pins.map((pin) => [relative(this.root, pin.path), pin]), + ); + } + async prepare(): Promise { + const work = join(this.root, "host"); + await mkdir(work, { mode: 0o700 }); + if (this.inputs.backend === "firecracker") + await mkdir(join(this.root, "thin-nodes"), { mode: 0o700 }); + await put(join(work, "host_id"), this.identity.hostId); + await put(join(work, "host.jsonl"), ""); + await put(join(work, "egress-proxyd.log"), ""); + await mkdir(join(work, "egress-ca"), { mode: 0o700 }); + await put( + join(work, "egress-ca/ca.pem"), + await readOwned(this.identity.ca.path), + ); + await put( + join(work, "egress-ca/ca.key"), + await readOwned(this.identity.caKey.path), + ); + const dbName = `mcp_${this.identity.runId.replaceAll("-", "_")}`; + const pgPassword = await this.secret("postgres"); + const cDsn = await put( + join(this.root, "coordinator.dsn"), + `postgresql://fixture:${pgPassword}@postgres.fixture.example:5432/${dbName}`, + ); + const oDsn = await put( + join(this.root, "orchestrator.dsn"), + `postgresql://fixture:${pgPassword}@postgres.fixture.example:5432/${dbName}`, + ); + const client = this.identity.leaves; + const c = this.configuration; + const shared = { + NODE_ENV: "production", + SSL_CERT_FILE: this.identity.ca.path, + ENGRAM_KEK_MASTER_KEY: await this.secret("kek"), + }; + const baseO = { + ...shared, + ORCHESTRATOR_DATABASE_URL: (await readOwned(oDsn.path)).toString(), + ORCHESTRATOR_PORT: "8787", + ORCHESTRATOR_PUBLIC_URL: "https://app.fixture.example", + CONTROL_PLANE_GRPC_URL: `http://${address(this.inputs, "coordinator")}:50061`, + CONTROL_PLANE_HTTP_URL: "https://coordinator.fixture.example", + NODE_EXTRA_CA_CERTS: this.identity.ca.path, + CONTROL_PLANE_BEARER: await this.secret("appGrpc"), + CONNECTION_BROKER_BEARER: await this.secret("broker"), + BETTER_AUTH_SECRET: await this.secret("auth"), + ENGRAM_DEPLOYMENT_ID: this.identity.runId, + ORCHESTRATOR_AUTH_MODE: "password", + ORCHESTRATOR_AUTH_PASSWORD_SIGNUP: "true", + ORCHESTRATOR_ADMIN_EMAILS: + "admin@fixture.example,bootstrap@fixture.example", + TRUSTED_ORIGINS: "https://app.fixture.example", + MCP_ENABLED: "true", + MCP_RUNTIME_CONFIG: c.runtimeNormal.path, + }; + const oFiles = this.files([ + this.identity.ca, + client.orchestrator.certificate, + client.orchestrator.key, + c.runtimeNormal, + c.runtimeZoneDisabled, + ]); + this.configs.set("orchestrator", { + normal: { env: baseO, files: oFiles }, + "global-disabled": { + env: { ...baseO, MCP_ENABLED: "false" }, + files: oFiles, + }, + "zone-disabled": { + env: { ...baseO, MCP_RUNTIME_CONFIG: c.runtimeZoneDisabled.path }, + files: oFiles, + }, + }); + this.configs.set("coordinator", { + normal: { + env: { + ...shared, + DATABASE_URL: (await readOwned(cDsn.path)).toString(), + ENGRAM_MODE: "coordinator", + ENGRAM_BIND_ADDR: "0.0.0.0:8080", + ENGRAM_APP_GRPC_ADDR: "0.0.0.0:50061", + ENGRAM_HARNESS_LISTEN_ADDR: "0.0.0.0:50060", + ENGRAM_AUTH_TOKENS: `${await this.secret("coordinator")},${await this.secret("appGrpc")}`, + ENGRAM_APP_GRPC_TOKENS: await this.secret("appGrpc"), + ENGRAM_HOST_CLIENT_TLS_CONFIG: c.coordinatorHostTls.path, + ENGRAM_MCP_CONTROL_CONFIG: c.coordinator.path, + ENGRAM_BLOB_BACKEND: "gcs", + ENGRAM_GCS_BUCKET: this.identity.runId, + STORAGE_EMULATOR_HOST: `http://${address(this.inputs, "blob")}:4443`, + ENGRAM_LOCAL_PATH: "/tmp/coordinator", + ENGRAM_LOG_FORMAT: "json", + ENGRAM_CONNECTION_BROKER_BEARER: await this.secret("broker"), + ENGRAM_ORCHESTRATOR_INTERNAL_URL: "https://app.fixture.example", + }, + files: this.files([ + this.identity.ca, + client.coordinator.certificate, + client.coordinator.key, + c.coordinator, + c.coordinatorHostTls, + c.clientIdentity, + c.candidateFile, + ]), + }, + }); + this.configs.set("worker", { + normal: { + env: { NODE_ENV: "production", MCP_WORKER_ENABLED: "true" }, + files: { + "config.json": c.worker, + "tls/tls.crt": client.worker.certificate, + "tls/tls.key": client.worker.key, + "tls/ca.crt": this.identity.ca, + }, + }, + }); + this.configs.set("host-agent", { + normal: { + env: { + ENGRAM_SANDBOX_BACKEND: this.inputs.backend, + ENGRAM_SANDBOX_WORK_DIR: work, + ENGRAM_DIRTY_ROOT: join(work, "dirty"), + ENGRAM_COORDINATOR_ENDPOINT: + "http://" + address(this.inputs, "coordinator") + ":8080", + ENGRAM_COORDINATOR_TOKEN: await this.secret("coordinator"), + ENGRAM_GRPC_LISTEN_ADDR: hostListenAddress(this.inputs), + ENGRAM_GRPC_ADVERTISE_ADDR: "https://host.fixture.example:9101", + ENGRAM_HOST_MANAGED_TLS_CONFIG: c.hostTls.path, + ENGRAM_BUNDLE_DIR: this.assets.bundles, + ENGRAM_HOST_METRICS_ADDR: "127.0.0.1:0", + ENGRAM_MIGRATE_PEER_LISTEN_ADDR: "disabled", + ENGRAM_BLOB_BACKEND: "gcs", + ENGRAM_GCS_BUCKET: this.identity.runId, + STORAGE_EMULATOR_HOST: `http://${address(this.inputs, "blob")}:4443`, + ENGRAM_LOG_FORMAT: "json", + ENGRAM_EGRESS_CA_SOURCE: "local-disk", + ENGRAM_EGRESS_CA_DIR: join(work, "egress-ca"), + ENGRAM_EGRESS_PROXYD_BIN: this.assets.proxy.path, + ENGRAM_EGRESS_PROXY_PORT: String(this.inputs.registryPort + 1), + ENGRAM_EGRESS_DNS_PORT: String(this.inputs.registryPort + 2), + ENGRAM_GUEST_GATEWAY_PORT: String(this.inputs.registryPort + 3), + ...(this.inputs.backend === "vz" + ? { + ENGRAM_VZ_KERNEL_PATH: this.assets.binaries.kernel.path, + ENGRAM_MANAGED_NETSTACK_PATH: this.assets.binaries.helper.path, + ENGRAM_MANAGED_NETSTACK_SHA256: + this.assets.binaries.helper.sha256, + } + : { + ENGRAM_KERNEL_IMAGE_PATH: this.assets.binaries.kernel.path, + ENGRAM_FIRECRACKER_BIN: "/usr/local/bin/firecracker", + ENGRAM_FC_UFFD_HANDLER_BIN: + "/usr/local/bin/engram-uffd-handler", + ...this.engine.fc?.environment(), + MCP_FIXTURE_HOST_LOG: join(work, "host.jsonl"), + }), + }, + files: this.files([ + this.identity.ca, + client.host.certificate, + client.host.key, + c.hostTls, + ]), + }, + }); + const configurations: Context["supervisor"]["configurations"] = { + orchestrator: { normal: c.runtimeNormal }, + coordinator: { normal: c.coordinator }, + worker: { normal: c.worker }, + "host-agent": { normal: c.hostTls }, + }; + for (const [role, variants] of this.configs) { + for (const [variant, config] of Object.entries(variants)) { + const pin = await put( + join(this.root, `${role}-${variant}.json`), + JSON.stringify(config), + ); + if (role === "orchestrator") { + check( + variant === "normal" || + variant === "global-disabled" || + variant === "zone-disabled", + "role_variant", + ); + configurations.orchestrator[variant] = pin; + } else { + check(variant === "normal", "role_variant"); + configurations[role].normal = pin; + } + if (role !== "host-agent") { + const env = + Object.entries(config.env) + .map(([key, value]) => { + check(!/[\r\n]/.test(value), "environment_line"); + return `${key}=${value}`; + }) + .join("\n") + "\n"; + this.environmentFiles.set( + `${role}-${variant}`, + await put(join(this.root, `${role}-${variant}.env`), env), + ); + } + } + } + const fixture = async (kind: "modern" | "legacy") => { + const runRoot = join(this.root, kind); + await mkdir(runRoot, { mode: 0o700 }); + const copy = async (name: string, pin: Pin) => + put(join(runRoot, name), await readOwned(pin.path)); + const certificate = await copy("server.pem", client[kind].certificate), + key = await copy("server.key", client[kind].key), + ca = await copy("ca.pem", this.identity.ca); + const token = await put( + join(runRoot, "observer.token"), + randomBytes(32).toString("base64url"), + ); + const resource = await put( + join(runRoot, "resource.key"), + randomBytes(32).toString("base64url"), + ); + const origin = (role: string) => + `https://${role}.${kind}.fixture.example:8443`; + const protocol = + kind === "modern" ? ("2026-07-28" as const) : ("2025-06-18" as const); + const configuration = await put( + join(runRoot, "config.json"), + JSON.stringify({ + version: 1, + runRoot, + listen: { host: address(this.inputs, kind), port: 8443 }, + advertised: { + resource: origin("resource"), + issuer: origin("issuer"), + authorization: origin("authorization"), + token: origin("token"), + redirectUri: + "https://app.fixture.example/integrations/mcp/callback", + authorities: ["resource", "issuer", "authorization", "token"].map( + (role) => `${role}.${kind}.fixture.example:8443`, + ), + }, + tls: { + certificateFile: certificate.path, + keyFile: key.path, + caFile: ca.path, + }, + control: { + host: address(this.inputs, kind), + port: 9444, + origin: `https://observer.${kind}.fixture.example:9444`, + tokenFile: token.path, + }, + protocol, + case: { requireAuth: true }, + apiKeyFile: resource.path, + }), + ); + return { + origin: `https://observer.${kind}.fixture.example:9444`, + ca, + serverFingerprint: client[kind].fingerprint, + configuration, + runRoot, + tokenFile: token.path, + resourceKeyFile: resource.path, + resourceOrigin: origin("resource"), + issuerOrigin: origin("issuer"), + authorizationOrigin: origin("authorization"), + tokenOrigin: origin("token"), + redirectUri: "https://app.fixture.example/integrations/mcp/callback", + protocol, + }; + }; + const modern = await fixture("modern"), + legacy = await fixture("legacy"); + const endpoint = (origin: string, leaf: keyof Identity["leaves"]) => ({ + origin, + ca: this.identity.ca, + serverFingerprint: client[leaf].fingerprint, + }); + const human = (role: "admin" | "owner" | "other") => ({ + email: `${role}@fixture.example`, + passwordFile: this.identity.secrets[`${role}Password`].path, + cookieFile: join(this.root, `${role}.cookie`), + }); + return { + version: 1, + runId: this.identity.runId, + runRoot: this.root, + backend: this.inputs.backend, + sourceCommit: this.inputs.sourceCommit, + hostId: this.identity.hostId, + // These fields are filled from the actual public enable result before the context is written or handed to the runner. + imageId: "pending-enable", + imageDigest: `sha256:${"0".repeat(64)}`, + artifacts: this.assets.artifacts, + endpoints: { + app: endpoint("https://app.fixture.example", "app"), + coordinator: endpoint( + "https://coordinator.fixture.example", + "coordinator", + ), + operator: { + ...endpoint("https://operator.fixture.example:8445", "coordinator"), + certificate: client.operator.certificate, + privateKeyFile: client.operator.key.path, + }, + workerControl: endpoint( + "https://worker.fixture.example:8443", + "worker", + ), + }, + protocolFixtures: { + modern: { ...modern, protocol: "2026-07-28" }, + legacy: { ...legacy, protocol: "2025-06-18" }, + }, + publication: { + candidateIdentity: c.candidateIdentity, + expectedGeneration: null, + deploymentIdentity: c.infrastructure.deploymentIdentity, + networkIdentity: c.infrastructure.networkIdentity, + trustIdentity: c.infrastructure.trustIdentity, + zone: c.zone.id, + }, + identities: { + admin: human("admin"), + owner: human("owner"), + other: human("other"), + unattendedKeyFile: join(this.root, "unattended.key"), + automationKeyFile: join(this.root, "automation.key"), + routerSentinelFile: this.identity.secrets.router.path, + }, + supervisor: { + socket: join(this.root, "supervisor.sock"), + tokenFile: this.identity.secrets.supervisor.path, + configurations, + operatorCandidates: { + normal: { file: c.candidateFile, identity: c.candidateIdentity }, + }, + }, + observer: { + orchestratorDsnFile: oDsn.path, + coordinatorDsnFile: cDsn.path, + hostManagedRoot: join(work, "managed-boots"), + hostLogFile: join(work, "egress-proxyd.log"), + }, + guest: { + stateDirectory: "/tmp/mcp-stack-fixture", + cliPath: "/opt/engram/dyn/0/engram-mcp-model-fixture", + }, + deadlines: { requestMs: 30000, stepMs: 120000, totalMs: 5400000 }, + evidenceFile: join(this.root, "evidence.jsonl"), + }; + } + async dependencies() { + const e = this.engine; + const database = `mcp_${this.identity.runId.replaceAll("-", "_")}`; + const env = await put( + join(this.root, "postgres.env"), + `POSTGRES_USER=fixture\nPOSTGRES_PASSWORD=${await this.secret("postgres")}\nPOSTGRES_DB=${database}\nPGDATA=/var/lib/postgresql/data\n`, + ); + const data = await e.project("postgres-data", {}, 999); + const pg = await e.create("postgres", this.inputs.images.postgres.id, [ + ...e.networkOptions("postgres"), + "--user", + "999:999", + "--read-only", + "--env-file", + env.path, + "--mount", + `type=volume,src=${data},dst=/var/lib/postgresql/data`, + "--tmpfs", + "/var/run/postgresql:rw,nosuid,nodev,size=8m,mode=1777", + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=8m,mode=1777", + ]); + await this.poll(async () => { + await e.call([ + "exec", + pg, + "/usr/bin/pg_isready", + "-U", + "fixture", + "-d", + database, + ]); + }); + const blob = await e.project("blob-data", {}); + await e.create( + "blob", + this.inputs.images.blob.id, + [ + ...e.networkOptions("blob"), + "--user", + "1000:1000", + "--read-only", + "--mount", + `type=volume,src=${blob},dst=/data`, + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=16m,mode=1777", + ], + [ + "-scheme", + "http", + "-port", + "4443", + "-filesystem-root", + "/data", + "-public-host", + `${address(this.inputs, "blob")}:4443`, + ], + "/bin/fake-gcs-server", + ); + const core = await put( + join(this.root, "Corefile"), + `.:53 {\n hosts /projection/hosts {\n ttl 1\n }\n }\n`, + ); + const hosts = await put( + join(this.root, "dns-hosts"), + dnsHosts(this.inputs), + ); + const dns = await e.project("dns-config", { Corefile: core, hosts }); + await e.create( + "dns", + this.inputs.images.dns.id, + [ + ...e.networkOptions("dns"), + "--cap-add", + "NET_BIND_SERVICE", + "--user", + "1000:1000", + "--read-only", + "--mount", + `type=volume,src=${dns},dst=/projection,readonly`, + ], + ["-conf", "/projection/Corefile"], + "/coredns", + ); + const sink = await e.create( + "denied", + this.inputs.images.guard.id, + [ + ...e.networkOptions("denied"), + "--user", + "1000:1000", + "--read-only", + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=4m", + ], + ["-l", "-k", "8080"], + "/usr/bin/nc", + ); + const stub = await put( + join(this.root, "denied-Corefile"), + `.:53 {\n template IN A {\n answer "{{ .Name }} 60 IN A ${address(this.inputs, "denied")}"\n }\n }\n`, + ); + const stubVolume = await e.project("denied-dns-config", { Corefile: stub }); + await e.create( + "denied-dns", + this.inputs.images.dns.id, + [ + "--network", + `container:${sink}`, + "--cap-add", + "NET_BIND_SERVICE", + "--user", + "1000:1000", + "--read-only", + "--mount", + `type=volume,src=${stubVolume},dst=/projection,readonly`, + ], + ["-conf", "/projection/Corefile"], + "/coredns", + ); + const probe = await e.create( + "dependency-probe", + this.inputs.images.guard.id, + [ + "--network", + this.engine.networkName(), + "--user", + "1000:1000", + "--read-only", + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=4m", + ], + ["infinity"], + "/bin/sleep", + ); + await this.poll(async () => { + await e.call([ + "exec", + probe, + "/usr/bin/nc", + "-z", + "-w", + "1", + address(this.inputs, "denied"), + "8080", + ]); + }); + const response = await e.call([ + "exec", + probe, + "/usr/bin/dig", + "+short", + `@${address(this.inputs, "denied")}`, + "unrouted.invalid", + "A", + ]); + check( + response.toString().trim() === address(this.inputs, "denied"), + "owned_external_stub_live", + ); + await this.poll(async () => { + await e.call([ + "exec", + probe, + "/usr/bin/curl", + "--fail", + "--silent", + "--max-time", + "2", + `http://${address(this.inputs, "blob")}:4443/storage/v1/b`, + ]); + }); + await e.call([ + "exec", + probe, + "/usr/bin/curl", + "--fail", + "--silent", + "--max-time", + "2", + "-X", + "POST", + "-H", + "Content-Type: application/json", + "--data", + JSON.stringify({ name: this.identity.runId }), + `http://${address(this.inputs, "blob")}:4443/storage/v1/b?project=fixture`, + ]); + await e.remove("dependency-probe"); + } + private async poll(action: () => Promise) { + const deadline = performance.now() + 30000; + while (true) { + this.engine.checkCancellation(); + try { + await action(); + return; + } catch { + if (performance.now() >= deadline) + throw new PlatformFailure("FAILED", "dependency_readiness"); + await new Promise((r) => setTimeout(r, 100)); + } + } + } + async edges() { + for (const role of ["orchestrator", "coordinator"] as const) { + const leaf = + this.identity.leaves[role === "orchestrator" ? "app" : "coordinator"]; + const port = role === "orchestrator" ? 8787 : 8080; + const config = await put( + join(this.root, `${role}-edge.conf`), + `pid /tmp/nginx.pid;\nerror_log /dev/stderr error;\nevents {}\nhttp { access_log off; client_body_temp_path /tmp/client; proxy_temp_path /tmp/proxy; fastcgi_temp_path /tmp/fastcgi; uwsgi_temp_path /tmp/uwsgi; scgi_temp_path /tmp/scgi; server { listen 443 ssl; ssl_certificate /projection/tls.crt; ssl_certificate_key /projection/tls.key; ssl_protocols TLSv1.3; location / { proxy_pass http://127.0.0.1:${port}; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto https; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection upgrade; } } }\n`, + ); + const volume = await this.engine.project(`${role}-edge`, { + "edge.conf": config, + "tls.crt": leaf.certificate, + "tls.key": leaf.key, + }); + await this.engine.create( + `${role}-edge`, + this.inputs.images.edge.id, + [ + "--network", + `container:${this.engine.guardIds.get(role)}`, + "--user", + "1000:1000", + "--read-only", + "--mount", + `type=volume,src=${volume},dst=/projection,readonly`, + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=16m,mode=1777", + ], + ["-c", "/projection/edge.conf", "-g", "daemon off;"], + "/usr/sbin/nginx", + ); + } + const data = await this.engine.project("registry-data", {}); + const registryEnv = await put( + join(this.root, "registry.env"), + `REGISTRY_HTTP_ADDR=0.0.0.0:${this.inputs.registryPort}\nREGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/var/lib/registry\n`, + ); + await this.engine.create("registry", this.inputs.images.registry.id, [ + "--network", + `container:${this.engine.guardIds.get("coordinator")}`, + "--user", + "1000:1000", + "--read-only", + "--env-file", + registryEnv.path, + "--mount", + `type=volume,src=${data},dst=/var/lib/registry`, + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=4m", + ]); + } + async startRole( + ctx: Context, + role: MutableRole, + configuration: ConfigurationId, + ): Promise { + const config = this.configs.get(role)?.[configuration]; + check(config && !this.active.has(role), "role_start"); + check( + role === "orchestrator" || configuration === "normal", + "role_variant", + ); + const configPin = + role === "orchestrator" + ? ctx.supervisor.configurations.orchestrator[configuration] + : ctx.supervisor.configurations[role].normal; + check(configPin, "role_configuration"); + await verifyPin(configPin); + for (const file of Object.values(config.files)) await verifyPin(file); + if (role === "host-agent") { + if (this.inputs.backend === "firecracker") { + check(this.engine.fc, "fc_inventory_missing"); + const image = this.inputs.images.hostAgent; + check(image, "fc_host_image"); + const env = config.env; + const envFile = await put( + join(this.root, `host-env-${randomBytes(6).toString("hex")}`), + Object.entries(env) + .map(([k, v]) => { + check(!/[\r\n]/.test(v), "host_environment_line"); + return `${k}=${v}`; + }) + .join("\n") + "\n", + ); + const id = await this.engine.create("host-agent", image.id, [ + "--network", + `container:${this.engine.guardIds.get("coordinator")}`, + "--pid", + "host", + "--cgroupns", + "private", + "--user", + "0:0", + "--read-only", + "--cap-add", + "NET_ADMIN", + "--cap-add", + "NET_BIND_SERVICE", + "--cap-add", + "SYS_ADMIN", + "--device", + "/dev/kvm:/dev/kvm:rw", + "--device", + "/dev/net/tun:/dev/net/tun:rw", + ...this.engine.fc.deviceOptions(), + "--env-file", + envFile.path, + ...["host", "assets", "bundles", "tls", "configuration"].flatMap( + (directory) => [ + "--mount", + `type=bind,src=${join(this.root, directory)},dst=${join(this.root, directory)}${directory === "host" ? "" : ",readonly"}`, + ], + ), + "--tmpfs", + "/run:rw,nosuid,nodev,size=64m", + "--tmpfs", + "/tmp:rw,nosuid,nodev,size=128m,mode=1777", + ]); + this.hostBoot = id; + this.active.set(role, { id, configuration }); + return; + } + check( + (await this.nativeProcesses()).length === 0, + "native_processes_before_start", + ); + const hostLog = join(this.root, "host", "host.jsonl"); + const log = await open(hostLog, "a", 0o600); + await chmod(hostLog, 0o600); + this.engine.checkCancellation(); + try { + this.host = new OwnedProcess( + this.assets.binaries.hostAgent.path, + [], + this.root, + { ...childEnvironment(this.root), ...config.env }, + ["ignore", log.fd, log.fd], + ); + } finally { + await log.close(); + } + await this.poll(async () => { + const found = (await this.nativeProcesses()).find( + (p) => p.pid === this.host?.child.pid, + ); + check(found, "native_host_identity"); + }); + const live = this.nativeIdentities.get(this.host.child.pid!); + check(live, "native_host_identity"); + this.hostBoot = `${live.pid}-${live.seconds}-${live.microseconds}`; + this.active.set(role, { id: this.hostBoot, configuration }); + return; + } + const key = `${role}-${configuration}`; + let volume = this.projections.get(key); + if (!volume) { + volume = await this.engine.project(key, config.files); + this.projections.set(key, volume); + } + const guard = this.engine.guardIds.get(role); + check(guard, "role_guard"); + const envFile = this.environmentFiles.get(key); + check(envFile, "role_environment"); + await verifyPin(envFile); + const id = await this.engine.create(role, this.inputs.images[role].id, [ + "--network", + `container:${guard}`, + "--user", + "1000:1000", + "--read-only", + "--env-file", + envFile.path, + "--mount", + `type=volume,src=${volume},dst=${role === "worker" ? "/etc/mcp-worker" : this.root},readonly`, + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=128m,mode=1777", + ]); + this.active.set(role, { id, configuration }); + } + async migrateOrchestrator(): Promise { + const config = this.configs.get("orchestrator")?.normal; + check(config, "migration_configuration"); + const volume = await this.engine.project( + "orchestrator-migration", + config.files, + ); + const env = this.environmentFiles.get("orchestrator-normal"); + check(env, "migration_environment"); + await verifyPin(env); + const guard = this.engine.guardIds.get("orchestrator"); + check(guard, "migration_guard"); + const id = await this.engine.create( + "orchestrator-migrate", + this.inputs.images.orchestrator.id, + [ + "--network", + `container:${guard}`, + "--user", + "1000:1000", + "--read-only", + "--env-file", + env.path, + "--mount", + `type=volume,src=${volume},dst=${this.root},readonly`, + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=128m,mode=1777", + ], + ["run", "migrate"], + "/usr/local/bin/bun", + ); + check( + (await this.engine.call(["wait", id], undefined, 30000)) + .toString() + .trim() === "0", + "orchestrator_migration", + ); + await this.engine.remove("orchestrator-migrate"); + } + async stop(role: MutableRole): Promise { + if (role === "host-agent") { + await this.cleanupHost(false); + } else await this.engine.remove(role); + this.stopped.add(role); + // Keep the last immutable identity for status of the stopped role. + } + async restart( + ctx: Context, + role: MutableRole, + configuration: ConfigurationId, + ): Promise { + await this.stop(role); + this.active.delete(role); + this.stopped.delete(role); + await this.startRole(ctx, role, configuration); + await this.readyRole(ctx, role); + } + private nativeHostTls() { + return { + ca: this.identity.ca, + clientCertificate: this.identity.leaves.coordinator.certificate, + clientKey: this.identity.leaves.coordinator.key, + serverCertificate: this.identity.leaves.host.certificate, + }; + } + async readyRole(ctx: Context, role: MutableRole): Promise { + const deadline = performance.now() + 15000; + while (performance.now() < deadline) { + this.engine.checkCancellation(); + try { + if (role === "orchestrator") { + const res = await jsonRequest( + ctx, + ctx.endpoints.app, + "GET", + "/healthz", + ); + check(res.status === 200, "orchestrator_readiness"); + } else if (role === "host-agent") { + check( + this.inputs.backend === "firecracker" + ? (await this.engine.audit(this.hostBoot)).State.Running + : this.host?.running(), + "host_running", + ); + const stat = await lstat(ctx.observer.hostManagedRoot); + check( + stat.isDirectory() && (stat.mode & 0o777) === 0o700, + "host_managed_root", + ); + } else { + if (role === "worker") { + const tls = await tlsOptions(ctx, ctx.endpoints.workerControl); + const certificate = await readOwned( + this.identity.leaves.coordinator.certificate.path, + ); + const key = await readOwned( + this.identity.leaves.coordinator.key.path, + ); + const identity = await new Promise((resolve, reject) => { + const request = httpsRequest( + new URL("/identity", ctx.endpoints.workerControl.origin), + { ...tls, cert: certificate, key, method: "GET", agent: false }, + (response) => { + const chunks: Buffer[] = []; + let bytes = 0; + response.on("data", (chunk: Buffer) => { + bytes += chunk.length; + if (bytes > 4096) request.destroy(); + else chunks.push(chunk); + }); + response.once("error", reject); + response.once("end", () => { + clearTimeout(timer); + try { + check( + response.statusCode === 200, + "worker_readiness_status", + ); + resolve(JSON.parse(Buffer.concat(chunks).toString())); + } catch (error) { + reject(error); + } + }); + }, + ); + const timer = setTimeout(() => request.destroy(), 2000); + request.once("error", () => { + clearTimeout(timer); + reject(new PlatformFailure("FAILED", "worker_readiness_tls")); + }); + request.end(); + }); + check( + identity !== null && + typeof identity === "object" && + "bootId" in identity && + typeof identity.bootId === "string" && + "zoneId" in identity && + identity.zoneId === this.configuration.zone.id && + "policyDigest" in identity && + identity.policyDigest === + new ZonePolicy(this.configuration.zone).digest, + "worker_readiness_identity", + ); + } + if (role === "coordinator") { + const response = await jsonRequest( + ctx, + ctx.endpoints.coordinator, + "GET", + "/readyz", + ); + check(response.status === 200, "coordinator_readiness"); + } + const active = this.active.get(role); + check(active, "role_handle"); + const actual = await this.engine.inspect(active.id); + check(actual.State.Running === true, "role_running"); + } + if (role === "host-agent") { + if (this.inputs.backend === "firecracker") { + await this.engine.qualifyFcHost(this.hostBoot); + check(this.engine.fc, "fc_inventory_missing"); + await this.engine.fc.adoptHost( + (await this.engine.inspect(this.hostBoot)).State.Pid, + ); + await this.engine.fc.observe(); + } + await this.engine.qualifyNativeHost(this.nativeHostTls()); + } + return; + } catch { + await new Promise((r) => setTimeout(r, 100)); + } + } + throw new PlatformFailure("FAILED", "role_readiness"); + } + async status(ctx: Context): Promise { + const states = await drainChecks( + ROLES.map(async (role): Promise => { + const active = this.active.get(role); + check(active, "role_identity"); + const artifact = + role === "host-agent" + ? ctx.artifacts.hostAgent + : role.startsWith("fixture-") + ? ctx.artifacts.fixture + : ctx.artifacts[ + role as "worker" | "orchestrator" | "coordinator" + ]; + await verifyPin(artifact); + const config = + role === "fixture-modern" || role === "fixture-legacy" + ? ctx.protocolFixtures[ + role === "fixture-modern" ? "modern" : "legacy" + ].configuration + : role === "orchestrator" + ? ctx.supervisor.configurations.orchestrator[active.configuration] + : ctx.supervisor.configurations[role].normal; + check(config, "role_configuration"); + await verifyPin(config); + if ( + role === "orchestrator" || + role === "coordinator" || + role === "worker" || + role === "host-agent" + ) { + const current = this.configs.get(role)?.[active.configuration]; + check(current, "role_configuration"); + for (const file of Object.values(current.files)) + await verifyPin(file); + const envFile = this.environmentFiles.get( + `${role}-${active.configuration}`, + ); + if (envFile) await verifyPin(envFile); + } + let running = false; + if (role === "host-agent") { + if (this.inputs.backend === "firecracker") { + if (!this.stopped.has(role)) { + running = + (await this.engine.audit(active.id)).State.Running === true; + await this.engine.verifyResolver(active.id); + } + } else { + const native = await this.nativeProcesses(); + running = + !!this.host?.running() && + native.some((p) => p.pid === this.host?.child.pid); + } + } else if (!this.stopped.has(role)) { + try { + const actual = await this.engine.audit(active.id); + await this.engine.verifyResolver(active.id); + running = actual.State.Running === true; + check( + actual.Image === + this.inputs.images[ + role.startsWith("fixture-") + ? "fixture" + : (role as "worker" | "orchestrator" | "coordinator") + ].id, + "running_artifact", + ); + } catch (error) { + throw error; + } + } + if (role === "host-agent" && running) { + if (this.inputs.backend === "firecracker") { + await this.engine.qualifyFcHost(active.id); + check(this.engine.fc, "fc_inventory_missing"); + await this.engine.fc.qualifyKernel( + (await this.engine.inspect(active.id)).State.Pid, + ); + await this.engine.fc.observe(); + } + await this.engine.qualifyNativeHost(this.nativeHostTls()); + } + return { + role, + state: running ? "running" : "stopped", + bootId: active.id, + artifactSha256: artifact.sha256, + configurationId: active.configuration, + configurationSha256: config.sha256, + }; + }), + ); + return { + version: 1, + runId: ctx.runId, + backend: ctx.backend, + sourceCommit: ctx.sourceCommit, + hostId: ctx.hostId, + roles: states, + network: await this.engine.verifyNetwork(true), + }; + } + async fixtures(ctx: Context): Promise { + for (const kind of ["modern", "legacy"] as const) { + const fixture = ctx.protocolFixtures[kind]; + const role = `fixture-${kind}` as const; + const id = await this.engine.create( + role, + this.inputs.images.fixture.id, + [ + ...this.engine.networkOptions(kind), + "--read-only", + "--user", + `${process.getuid?.()}:${process.getgid?.()}`, + "--mount", + `type=bind,src=${fixture.runRoot},dst=${fixture.runRoot}`, + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=16m", + ], + ["/app/tests/fixtures/mcp-stack/main.mts", fixture.configuration.path], + "/usr/local/bin/node", + ); + this.active.set(role, { id, configuration: "normal" }); + } + await this.engine.nativeDns(); + for (const kind of ["modern", "legacy"] as const) { + try { + await new FixtureObserver(ctx, kind).ready(); + } catch (error) { + if (error instanceof RunnerFailure || error instanceof PlatformFailure) + throw error; + throw new PlatformFailure( + "FAILED", + kind === "modern" + ? "fixture_modern_readiness" + : "fixture_legacy_readiness", + ); + } + } + } + private async nativeProcesses(): Promise { + check(this.assets.nativeProbe, "native_probe"); + await verifyPin(this.assets.nativeProbe); + const bytes = await command( + this.assets.nativeProbe.path, + [this.root], + this.root, + childEnvironment(this.root), + ); + const found = parseNativeIdentities( + JSON.parse(bytes.toString()), + this.root, + ); + for (const p of found) { + const expected = + p.executable === this.assets.binaries.hostAgent.path + ? this.assets.binaries.hostAgent + : p.executable === this.assets.proxy.path + ? this.assets.proxy + : { + path: p.executable, + sha256: this.assets.binaries.helper.sha256, + }; + await verifyPin(expected); + const previous = this.nativeIdentities.get(p.pid); + if (previous) + check(nativeKey(previous) === nativeKey(p), "native_pid_reused"); + this.nativeIdentities.set(p.pid, p); + } + return found; + } + async cleanupHost(requireRelease = true) { + if (this.inputs.backend === "firecracker") { + check(this.engine.fc, "fc_inventory_missing"); + if (requireRelease) await this.engine.fc.requireReleased(); + if (this.active.has("host-agent") && !this.stopped.has("host-agent")) + await this.engine.remove("host-agent"); + if (requireRelease) await this.engine.fc.requireStopped(); + return; + } + const host = this.host; + const leader = host && this.nativeIdentities.get(host.child.pid!); + const before = await this.nativeProcesses(); + await put( + join(this.root, `native-stop-${randomBytes(8).toString("hex")}.json`), + JSON.stringify({ version: 1, identities: before }), + ); + check( + host || + !before.some( + (p) => p.executable === this.assets.binaries.hostAgent.path, + ), + "native_host_handle_missing", + ); + if (host?.running()) + check( + leader && + leader.executable === this.assets.binaries.hostAgent.path && + before.some((p) => nativeKey(p) === nativeKey(leader)), + "native_host_identity_before_stop", + ); + const signalIdentity = async ( + identity: NativeIdentity, + signal: NodeJS.Signals, + ) => { + // Compare the kernel birth time and executable again immediately before signaling. + const current = (await this.nativeProcesses()).find( + (p) => p.pid === identity.pid, + ); + if (!current) return; + check( + nativeKey(identity) === nativeKey(current), + "native_identity_changed", + ); + try { + process.kill(identity.pid, signal); + } catch (error) { + if (!( + error instanceof Error && + "code" in error && + error.code === "ESRCH" + )) + throw error; + } + }; + if (host) { + let closed = false; + void host.done.then(() => { + closed = true; + }); + const leaderPresent = async () => + leader && + (await this.nativeProcesses()).some( + (p) => nativeKey(p) === nativeKey(leader), + ); + // The host restarts its proxy. Stop and reap the leader before selecting children. + for (const signal of ["SIGTERM", "SIGKILL"] as const) { + if (leader) await signalIdentity(leader, signal); + const until = performance.now() + 1500; + while (performance.now() < until) { + const present = await leaderPresent(); + if (closed && !host.running() && !present) break; + await new Promise((r) => setTimeout(r, 20)); + } + if (closed && !host.running() && !(await leaderPresent())) break; + } + check(closed && !host.running(), "native_host_handle_cleanup"); + check(!(await leaderPresent()), "native_host_survivor"); + this.host = undefined; + } + for (const signal of ["SIGTERM", "SIGKILL"] as const) { + const children = await this.nativeProcesses(); + check( + children.every( + (p) => p.executable !== this.assets.binaries.hostAgent.path, + ), + "native_host_survivor", + ); + for (const identity of children) await signalIdentity(identity, signal); + const until = performance.now() + 1500; + while ((await this.nativeProcesses()).length && performance.now() < until) + await new Promise((r) => setTimeout(r, 20)); + } + check((await this.nativeProcesses()).length === 0, "native_survivors"); + this.nativeIdentities.clear(); + } +} diff --git a/tests/fixtures/mcp-stack/platform/main.ts b/tests/fixtures/mcp-stack/platform/main.ts new file mode 100644 index 000000000..2ee847b53 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/main.ts @@ -0,0 +1,748 @@ +import { open, realpath, readFile, readdir } from "node:fs/promises"; +import { join } from "node:path"; +import { networkInterfaces } from "node:os"; +import { randomBytes } from "node:crypto"; +import { cli, preflight, REPO } from "./preflight.ts"; +import { + check, + childEnvironment, + newRoot, + OwnedProcess, + PlatformFailure, + put, + readOwned, + verifyPin, +} from "./owned.ts"; +import { generateIdentity } from "./identity.ts"; +import { generateConfiguration } from "./configuration.ts"; +import { stageAssets, publishGuest } from "./assets.ts"; +import { Engine } from "./engine.ts"; +import { Stack } from "./launch.ts"; +import { inspectFcInventory } from "./fc-runtime.ts"; +import { parseHostRecord } from "../../../../orchestrator/scripts/mcp-stack/host-observer.ts"; +import type { PublicClient } from "../../../../orchestrator/scripts/mcp-stack/transport.ts"; +import { startSupervisor, type Lifecycle } from "./supervisor.ts"; +import { + ContextSchema, + loadContext, + type Context, +} from "../../../../orchestrator/scripts/mcp-stack/context.ts"; +import { + jsonRequest, + publicClient, +} from "../../../../orchestrator/scripts/mcp-stack/transport.ts"; +import { publicPrerequisites } from "../../../../orchestrator/scripts/mcp-stack/preflight.ts"; +import { waitFor } from "../../../../orchestrator/scripts/mcp-stack/bootstrap.ts"; +import type { Identity } from "./identity.ts"; + +import { + EVIDENCE_MAX_BYTES, + verifyEvidence, +} from "../../../../orchestrator/scripts/mcp-stack/evidence.ts"; +import { + FULL_MATRIX, + selectedCases, + type RunSelection, +} from "../../../../orchestrator/scripts/mcp-stack/manifest.ts"; +import { RunnerFailure } from "../../../../orchestrator/scripts/mcp-stack/failure.ts"; +import { + ownedAutomationCleanupSessions, + type ManagedAutomationSession, +} from "../../../../orchestrator/scripts/mcp-stack/automation-ownership.ts"; + +export function platformFailureLabel(error: unknown): string { + if (error instanceof PlatformFailure) return `${error.code}:${error.seam}`; + if (error instanceof RunnerFailure) { + switch (`${error.code}:${error.seam}`) { + case "INVALID_CONTEXT:owned_path": + return "FAILED:runner_invalid_context_owned_path"; + case "INVALID_CONTEXT:private_file": + return "FAILED:runner_invalid_context_private_file"; + case "INVALID_CONTEXT:request_route": + return "FAILED:runner_invalid_context_request_route"; + case "INVALID_CONTEXT:request_size": + return "FAILED:runner_invalid_context_request_size"; + case "ASSERTION_FAILED:observer_token": + return "FAILED:runner_assertion_failed_observer_token"; + case "ASSERTION_FAILED:fixture_observer_status": + return "FAILED:runner_assertion_failed_fixture_observer_status"; + case "ASSERTION_FAILED:fixture_identity": + return "FAILED:runner_assertion_failed_fixture_identity"; + case "TRANSPORT_FAILED:https_request": + return "FAILED:runner_transport_failed_https_request"; + } + } + return "FAILED:platform_composition"; +} + +export const matrixArguments = ( + contextPath: string, + selection: RunSelection = FULL_MATRIX, +) => { + selectedCases(selection); + return [ + join(REPO, "orchestrator/scripts/mcp-stack/run.ts"), + "--context", + contextPath, + ...(selection.stage === "infrastructure" + ? ["--stage", "infrastructure"] + : ["--matrix", selection.matrix]), + ]; +}; + +/** A successful child must leave the completion record for the requested stage. */ +export async function verifyRunnerCompletion( + ctx: Context, + selection: RunSelection, +) { + try { + verifyEvidence( + ctx, + selection, + (await readOwned(ctx.evidenceFile, EVIDENCE_MAX_BYTES)).toString(), + ); + } catch { + throw new PlatformFailure("INVALID_CONTEXT", "runner_completion"); + } +} + +export async function enableGuest( + ctx: Context, + identity: Identity, + uri: string, + expectedDigest: string, +) { + const email = "bootstrap@fixture.example", + password = ( + await readOwned(identity.secrets.bootstrapPassword.path) + ).toString(); + const signup = await jsonRequest( + ctx, + ctx.endpoints.app, + "POST", + "/api/auth/sign-up/email", + { email, password, name: "Fixture bootstrap" }, + { origin: ctx.endpoints.app.origin }, + ); + check(signup.status === 200, "bootstrap_signup"); + const login = await jsonRequest( + ctx, + ctx.endpoints.app, + "POST", + "/api/auth/sign-in/email", + { email, password }, + { origin: ctx.endpoints.app.origin }, + ); + const cookies = login.headers["set-cookie"]; + check( + login.status === 200 && cookies?.length && cookies.length <= 8, + "bootstrap_login", + ); + const cookie = cookies.map((v) => v.split(";", 1)[0]).join("; "); + check(cookie.length <= 8192, "bootstrap_cookie"); + await put(join(ctx.runRoot, "bootstrap.cookie"), cookie); + const admin = await publicClient(ctx, { kind: "cookie", value: cookie }); + const principal = await admin.identity.whoAmI({}); + check( + principal.email === email && + principal.role === "admin" && + !principal.serviceAccount && + principal.userId, + "bootstrap_identity", + ); + const host = await waitFor( + ctx, + "host_enrollment", + () => admin.fleet.getHost({ hostId: ctx.hostId }), + (v) => v.host?.status === "ready", + ); + check(host.host?.id === ctx.hostId, "host_enrollment_identity"); + const job = await admin.images.enableImage({ + imageUri: uri, + config: { + name: "Fixture guest", + resources: { + suggestedMemoryMib: 512, + suggestedVcpus: 1, + suggestedDiskGib: 2, + }, + }, + }); + check( + job.job?.id && + (job.job.manifestDigest === undefined || + job.job.manifestDigest === expectedDigest), + "enable_manifest", + ); + const result = await waitFor( + ctx, + "image_enable_ready", + () => admin.images.getEnableJob({ jobId: job.job!.id }), + (v) => { + check(v.job?.state !== "failed", "image_enable_failed"); + return v.job?.state === "ready"; + }, + ); + check( + result.job?.manifestDigest === undefined || + result.job.manifestDigest === expectedDigest, + "enable_ready_manifest", + ); + check(result.job?.materializeHostId === ctx.hostId, "enable_actual_host"); + const enabled = (await admin.images.listEnabledImages({})).images.filter( + (v) => v.imageUri === uri && v.manifestDigest === expectedDigest, + ); + check(enabled.length === 1, "enabled_image_identity"); + ctx.imageId = enabled[0]!.id; + ctx.imageDigest = expectedDigest; + await publicPrerequisites(ctx, admin); + await put( + join(ctx.runRoot, "enable-observation.json"), + JSON.stringify({ + version: 1, + bootstrapActorId: principal.userId, + jobId: result.job.id, + imageId: ctx.imageId, + imageDigest: ctx.imageDigest, + hostId: ctx.hostId, + state: result.job.state, + }), + ); +} + +/** Prepare dependencies before fixture checks and application startup. */ +export async function bootPlatform( + engine: Pick< + Engine, + | "reserveRegistryPort" + | "reserveHostPort" + | "qualifyTools" + | "network" + | "guards" + | "verifyNetwork" + >, + ca: Identity["ca"], + prepare: () => Promise<{ + stack: Pick< + Stack, + | "dependencies" + | "fixtures" + | "edges" + | "startRole" + | "readyRole" + | "migrateOrchestrator" + >; + ctx: Context; + }>, +) { + await engine.reserveRegistryPort(); + await engine.reserveHostPort(); + await engine.qualifyTools(); + const { stack, ctx } = await prepare(); + await engine.network(); + await stack.dependencies(); + await stack.fixtures(ctx); + await engine.guards(ca); + await engine.verifyNetwork(true); + await stack.edges(); + await bootApplications(stack, ctx); + return ctx; +} + +/** Fixed production boot order. Migration and readiness failures stop the graph. */ +export async function bootApplications( + stack: Pick, + ctx: Context, +) { + await stack.startRole(ctx, "worker", "normal"); + await stack.readyRole(ctx, "worker"); + await stack.startRole(ctx, "coordinator", "normal"); + await stack.readyRole(ctx, "coordinator"); + await stack.migrateOrchestrator(); + await stack.startRole(ctx, "orchestrator", "normal"); + await stack.readyRole(ctx, "orchestrator"); + await stack.startRole(ctx, "host-agent", "normal"); + await stack.readyRole(ctx, "host-agent"); +} +export function ownedCleanupSessions( + rows: { + session?: { id: string; hostId?: string; status: string }; + ownerEmail?: string; + }[], + hostId: string, + managed: Map, + actors: Set, + automationSessions: ReadonlySet = new Set(), +) { + check(rows.length <= 4096, "cleanup_session_bound"); + const seen = new Set(); + const ids: string[] = []; + for (const row of rows) { + const session = row.session; + check( + session && + /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/.test(session.id) && + !seen.has(session.id), + "cleanup_session_identity", + ); + seen.add(session.id); + if (["completed", "failed", "dead"].includes(session.status)) continue; + check( + (!session.hostId || session.hostId === hostId) && + ([ + "admin@fixture.example", + "owner@fixture.example", + "other@fixture.example", + "bootstrap@fixture.example", + ].includes(row.ownerEmail ?? "") || + actors.has(managed.get(session.id) ?? "") || + automationSessions.has(session.id)), + "cleanup_foreign_session", + ); + ids.push(session.id); + } + return ids; +} +export async function destroyRunSessions(ctx: Context, admin: PublicClient) { + const managed = new Map(), + automationManaged = new Map(), + actors = new Set(); + const principal = await admin.identity.whoAmI({}); + check( + principal.email === "bootstrap@fixture.example" && + principal.role === "admin" && + !principal.serviceAccount, + "cleanup_admin_identity", + ); + actors.add(principal.userId); + for (const role of ["admin", "owner", "other"] as const) { + const file = ctx.identities[role].cookieFile; + try { + const client = await publicClient(ctx, { + kind: "cookie", + value: (await readOwned(file)).toString(), + }); + const identity = await client.identity.whoAmI({}); + check( + identity.email === ctx.identities[role].email && + !identity.serviceAccount && + identity.userId, + "cleanup_fixture_identity", + ); + actors.add(identity.userId); + } catch (error) { + if ( + !( + error && + typeof error === "object" && + "code" in error && + error.code === "ENOENT" + ) + ) + throw error; + } + } + for (const file of [ + ctx.identities.unattendedKeyFile, + ctx.identities.automationKeyFile, + ]) { + try { + const client = await publicClient(ctx, { + kind: "api-key", + value: (await readOwned(file)).toString().trim(), + }); + const identity = await client.identity.whoAmI({}); + check(identity.userId, "cleanup_key_identity"); + actors.add(identity.userId); + } catch (error) { + if ( + !( + error && + typeof error === "object" && + "code" in error && + error.code === "ENOENT" + ) + ) + throw error; + } + } + for (const file of await readdir(ctx.observer.hostManagedRoot)) { + if (!/^[a-f0-9-]{36}\.boot$/.test(file)) continue; + const record = parseHostRecord( + await readOwned(join(ctx.observer.hostManagedRoot, file)), + file.slice(0, -5), + ).record; + check( + record.request.key.context.target.host_id === ctx.hostId, + "cleanup_managed_host", + ); + managed.set( + record.request.cold.session_id, + record.request.key.creation.policy.actorId, + ); + automationManaged.set(record.request.cold.session_id, { + sessionId: record.request.key.context.target.session_id, + actorId: record.request.key.creation.policy.actorId, + hostId: record.request.key.context.target.host_id, + taskId: record.request.key.creation.task_id, + }); + } + const destroyed = new Set(); + const deadline = performance.now() + 60000; + while (performance.now() < deadline) { + const rows = (await admin.sessions.listSessions({})).sessions; + check(rows.length <= 4096, "cleanup_session_bound"); + const candidates = new Set( + rows.flatMap(({ session }) => + session && !["completed", "failed", "dead"].includes(session.status) + ? [session.id] + : [], + ), + ); + const automationSessions = candidates.size + ? await ownedAutomationCleanupSessions( + ctx, + admin, + automationManaged, + actors, + candidates, + ) + : new Set(); + const ids = ownedCleanupSessions( + rows, + ctx.hostId, + managed, + actors, + automationSessions, + ); + if (!ids.length) { + await put( + join(ctx.runRoot, "session-cleanup.json"), + JSON.stringify({ destroyed: [...destroyed], remaining: [] }), + ); + return; + } + for (const sessionId of ids) { + await admin.sessions.deleteSession({ sessionId }); + destroyed.add(sessionId); + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + throw new PlatformFailure("FAILED", "cleanup_session_deadline"); +} +/** Write the exact runner context and retain its real control socket. */ +export async function startContextSupervisor( + ctx: Context, + lifecycle: Lifecycle, +) { + ContextSchema.parse(ctx); + const contextFile = await put( + join(ctx.runRoot, "context.json"), + JSON.stringify(ctx), + ); + const supervisor = await startSupervisor(ctx, lifecycle); + try { + await loadContext(contextFile.path); + return supervisor; + } catch (error) { + await supervisor.close(); + throw error; + } +} +/** All stages retire the same bounded owned resources, even after partial boot. */ +export async function cleanupPlatform({ + context, + stack, + engine, + runner, + supervisor, +}: { + context?: Context; + stack?: Stack; + engine?: Engine; + runner?: OwnedProcess; + supervisor?: Awaited>; +}) { + const failed: string[] = []; + for (const [name, action] of [ + [ + "matrix", + async () => { + if (runner) await runner.stop(1500); + }, + ], + [ + "supervisor", + async () => { + if (supervisor) await supervisor.close(); + }, + ], + [ + "sessions", + async () => { + if (!context) return; + let cookie: string; + try { + cookie = ( + await readOwned(join(context.runRoot, "bootstrap.cookie")) + ).toString(); + } catch (error) { + if ( + error && + typeof error === "object" && + "code" in error && + error.code === "ENOENT" + ) + return; + throw error; + } + const admin = await publicClient(context, { + kind: "cookie", + value: cookie, + }); + await destroyRunSessions(context, admin); + if (engine?.fc) { + const deadline = performance.now() + 15000; + let released = false; + while (performance.now() < deadline) { + try { + await engine.fc.requireReleased(); + released = true; + break; + } catch { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + check(released, "cleanup_thin_release_deadline"); + } + }, + ], + [ + "host", + async () => { + if (stack) await stack.cleanupHost(); + }, + ], + [ + "engine", + async () => { + if (engine) await engine.cleanup(); + }, + ], + ] as const) { + try { + await action(); + } catch { + failed.push(name); + } + } + return failed; +} + +export async function main(args: string[]) { + const options = cli(args); + const inputs = await preflight( + options.backend, + options.runRoot, + options.manifest, + ); + await verifyPin({ + path: await realpath(process.execPath), + sha256: inputs.binaries.bun.sha256, + }); + check( + process.versions.bun === + (await readFile(join(REPO, ".bun-version"), "utf8")).trim(), + "launcher_bun_version", + ); + check( + Object.values(networkInterfaces()) + .flat() + .some((v) => v?.address === inputs.nativeHostIp && !v.internal), + "native_host_address", + ); + await newRoot(options.runRoot); + const root = options.runRoot; + const abort = new AbortController(); + let stack: Stack | undefined, + engine: Engine | undefined, + runner: OwnedProcess | undefined; + let context: Context | undefined; + let observing = false; + let observer: Promise | undefined; + let observerFailure: unknown; + let supervisor: Awaited> | undefined; + const signal = () => abort.abort(); + process.once("SIGINT", signal); + process.once("SIGTERM", signal); + let timer: ReturnType | undefined; + let failure: unknown; + try { + const identity = await generateIdentity(inputs, root); + engine = new Engine( + inputs, + root, + identity.runId, + inputs.backend === "firecracker" + ? await inspectFcInventory(inputs) + : undefined, + ); + timer = setTimeout(signal, 7200000); + await engine.controlled(abort.signal, async () => { + const ctx = await bootPlatform(engine!, identity.ca, async () => { + const assets = await stageAssets(inputs, root, engine!); + const configuration = await generateConfiguration( + inputs, + root, + identity, + ); + stack = new Stack( + inputs, + root, + identity, + assets, + configuration, + engine!, + ); + const ctx = await stack.prepare(); + context = ctx; + return { stack, ctx }; + }); + if (engine!.fc) { + observing = true; + observer = (async () => { + let firstFailure: number | undefined; + while (observing) { + try { + await engine!.fc!.observe(); + firstFailure = undefined; + } catch (error) { + firstFailure ??= performance.now(); + if (performance.now() - firstFailure > 3000) { + observerFailure = error; + abort.abort(); + return; + } + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + })(); + } + const uri = await publishGuest(inputs, root, engine!); + await enableGuest(ctx, identity, uri, inputs.images.guest.manifestDigest); + supervisor = await startContextSupervisor(ctx, { + prepareDiskRead: (signal) => + engine!.controlled(signal, async () => { + check(engine!.fc, "fc_inventory_missing"); + await engine!.fc.prepareDiskRead(); + }), + verifyDiskRead: (signal) => + engine!.controlled(signal, async () => { + check(engine!.fc, "fc_inventory_missing"); + await engine!.fc.verifyDiskRead(); + }), + infrastructureReady: (signal) => + engine!.controlled(signal, async () => { + check(engine!.fc, "fc_inventory_missing"); + await engine!.fc.requireInfrastructure({ + stop: () => stack!.stop("host-agent"), + restart: () => stack!.restart(ctx, "host-agent", "normal"), + }); + }), + status: (signal) => + engine!.controlled(signal, () => stack!.status(ctx)), + restart: (role, configuration, signal) => + engine!.controlled(signal, () => + stack!.restart(ctx, role, configuration), + ), + stop: (role, signal) => + engine!.controlled(signal, () => stack!.stop(role)), + }); + await stack!.status(ctx); + }); + const contextPath = join(root, "context.json"); + const logPath = join(root, "matrix.log"); + const log = await open(logPath, "wx", 0o600); + try { + runner = new OwnedProcess( + inputs.binaries.bun.path, + matrixArguments(contextPath, options.selection), + REPO, + childEnvironment(root), + ["ignore", log.fd, log.fd], + ); + } finally { + await log.close(); + } + const aborted = new Promise((_resolve, reject) => { + if (abort.signal.aborted) + reject(new PlatformFailure("FAILED", "run_aborted")); + else + abort.signal.addEventListener( + "abort", + () => reject(new PlatformFailure("FAILED", "run_aborted")), + { once: true }, + ); + }); + await Promise.race([runner.done, aborted]); + check(runner.child.exitCode === 0, "matrix_exit"); + check(context, "runner_context"); + await verifyRunnerCompletion(context, options.selection); + if (observerFailure) throw observerFailure; + await engine?.fc?.observe(); + await engine?.fc?.requireCoverage(); + await runner.stop(1500); + runner = undefined; + } catch (error) { + failure = error; + } finally { + if (timer) clearTimeout(timer); + process.off("SIGINT", signal); + process.off("SIGTERM", signal); + observing = false; + await observer; + const failed = await cleanupPlatform({ + context, + stack, + engine, + runner, + supervisor, + }); + await put( + join(root, `completion-${randomBytes(6).toString("hex")}.json`), + JSON.stringify({ + version: 1, + backend: inputs.backend, + sourceCommit: inputs.sourceCommit, + ...options.selection, + stageCompleted: + !failure && + !failed.length && + !( + options.selection.stage === "matrix" && + options.selection.matrix === "all" + ), + matrixCompleted: + !failure && + !failed.length && + options.selection.stage === "matrix" && + options.selection.matrix === "all", + physicalAcceptance: + !failure && + !failed.length && + options.selection.stage === "matrix" && + options.selection.matrix === "all", + cleanupFailed: failed, + }), + ); + if (failed.length) failure = new PlatformFailure("FAILED", "cleanup"); + } + if (failure) throw failure; +} +if (import.meta.main) { + main(process.argv.slice(2)).catch((error) => { + // Never print external messages, arguments, response bodies, or credentials. + process.stderr.write(platformFailureLabel(error) + "\n"); + process.exitCode = 2; + }); +} diff --git a/tests/fixtures/mcp-stack/platform/native-process-identity.c b/tests/fixtures/mcp-stack/platform/native-process-identity.c new file mode 100644 index 000000000..366b89195 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/native-process-identity.c @@ -0,0 +1,44 @@ +// Read-only identity probe for the native fixture host. It never sends signals. +// Build on macOS: cc -std=c11 -Wall -Wextra -Werror -O2 this-file.c -o nativeProbe +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + if (argc == 2 && strcmp(argv[1], "--version") == 0) { + puts("mcp-native-process-identity-v1"); return 0; + } + if (argc != 2 || argv[1][0] != '/') return 2; + size_t root_len = strlen(argv[1]); + if (root_len < 2 || argv[1][root_len - 1] == '/' || strpbrk(argv[1], "\"\\\n\r\t")) return 2; + int count = proc_listallpids(NULL, 0); + if (count <= 0 || count > 1048576) return 3; + int capacity = count + 1024; + pid_t *pids = calloc((size_t)capacity, sizeof(pid_t)); + if (!pids) return 3; + count = proc_listallpids(pids, capacity * (int)sizeof(pid_t)); + if (count < 0 || count >= capacity) { free(pids); return 3; } + puts("["); int first = 1; + for (int i = 0; i < count; ++i) { + if (pids[i] == getpid()) continue; + struct proc_bsdinfo before, after; + if (proc_pidinfo(pids[i], PROC_PIDTBSDINFO, 0, &before, sizeof(before)) != sizeof(before)) continue; + if (before.pbi_uid != getuid()) continue; + char path[PROC_PIDPATHINFO_MAXSIZE]; + if (proc_pidpath(pids[i], path, sizeof(path)) <= 0) continue; + if (strncmp(path, argv[1], root_len) || path[root_len] != '/') continue; + const char *relative = path + root_len; + size_t path_len = strlen(path); + if (strcmp(relative, "/assets/hostAgent") && strcmp(relative, "/assets/proxy") && !(strncmp(relative, "/host/", 6) == 0 && path_len > 9 && strcmp(path + path_len - 9, "/netstack") == 0)) continue; + if (strpbrk(path, "\"\\\n\r\t")) { free(pids); return 4; } + if (proc_pidinfo(pids[i], PROC_PIDTBSDINFO, 0, &after, sizeof(after)) != sizeof(after)) continue; + if (before.pbi_start_tvsec != after.pbi_start_tvsec || before.pbi_start_tvusec != after.pbi_start_tvusec || before.pbi_uid != after.pbi_uid) { free(pids); return 4; } + printf("%s{\"pid\":%u,\"uid\":%u,\"parent\":%u,\"group\":%u,\"seconds\":\"%llu\",\"microseconds\":\"%llu\",\"executable\":\"%s\"}", first ? "" : ",\n", after.pbi_pid, after.pbi_uid, after.pbi_ppid, after.pbi_pgid, (unsigned long long)after.pbi_start_tvsec, (unsigned long long)after.pbi_start_tvusec, path); + first = 0; + } + puts("\n]"); free(pids); return 0; +} diff --git a/tests/fixtures/mcp-stack/platform/network.ts b/tests/fixtures/mcp-stack/platform/network.ts new file mode 100644 index 000000000..8e905952b --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/network.ts @@ -0,0 +1,170 @@ +import { check, hash, record } from "./owned.ts"; +import type { Inputs } from "./preflight.ts"; + +export const ADDRESS = { + postgres: 10, + blob: 11, + registry: 12, + dns: 13, + orchestrator: 20, + coordinator: 21, + worker: 22, + modern: 30, + legacy: 31, + denied: 40, +} as const; +export type GuardRole = "orchestrator" | "coordinator" | "worker"; +export const GUARDS: GuardRole[] = ["orchestrator", "coordinator", "worker"]; +export function address( + inputs: Pick, + role: keyof typeof ADDRESS, +): string { + return inputs.subnet.slice(0, -4) + ADDRESS[role]; +} +function hostRpcAddress( + inputs: Pick, +): string { + return inputs.backend === "firecracker" + ? address(inputs, "coordinator") + : inputs.nativeHostIp; +} +export function dnsRecords( + inputs: Pick, +): Record { + const records: Record = { + "postgres.fixture.example": address(inputs, "postgres"), + "app.fixture.example": address(inputs, "orchestrator"), + "coordinator.fixture.example": address(inputs, "coordinator"), + "operator.fixture.example": address(inputs, "coordinator"), + "control.fixture.example": address(inputs, "coordinator"), + "worker.fixture.example": address(inputs, "worker"), + "host.fixture.example": hostRpcAddress(inputs), + }; + for (const protocol of ["modern", "legacy"] as const) + for (const name of [ + "resource", + "issuer", + "authorization", + "token", + "observer", + ]) + records[`${name}.${protocol}.fixture.example`] = address( + inputs, + protocol, + ); + return records; +} +export function dnsHosts( + inputs: Pick, +): string { + return ( + Object.entries(dnsRecords(inputs)) + .map(([name, ip]) => `${ip} ${name}`) + .join("\n") + "\n" + ); +} + +// Install these exact nft objects before an app joins its guard namespace. +// No forwarding chain, default route allowance, or external DNS server is installed. +export function rules( + inputs: Pick, + role: GuardRole, +): Record[] { + const result: Record[] = [ + { table: { family: "inet", name: "mcp" } }, + { + chain: { + family: "inet", + table: "mcp", + name: "output", + type: "filter", + hook: "output", + prio: 0, + policy: "drop", + }, + }, + ]; + const rule = (expr: unknown[]) => + result.push({ + rule: { family: "inet", table: "mcp", chain: "output", expr }, + }); + const match = (left: unknown, right: unknown) => ({ + match: { op: "==", left, right }, + }); + const dest = (ip: string) => + match({ payload: { protocol: "ip", field: "daddr" } }, ip); + // Docker's embedded resolver must never bypass the fixture DNS server. + rule([dest("127.0.0.11"), { drop: null }]); + rule([ + match({ ct: { key: "state" } }, { set: ["established", "related"] }), + { accept: null }, + ]); + rule([match({ meta: { key: "oifname" } }, "lo"), { accept: null }]); + const allow = (ip: string, protocol: "tcp" | "udp", port: number) => + rule([ + dest(ip), + match({ payload: { protocol, field: "dport" } }, port), + { accept: null }, + ]); + allow(address(inputs, "dns"), "udp", 53); + allow(address(inputs, "dns"), "tcp", 53); + for (const fixture of ["modern", "legacy"] as const) + allow(address(inputs, fixture), "tcp", 8443); + if (role !== "worker") { + allow(address(inputs, "postgres"), "tcp", 5432); + allow(address(inputs, "worker"), "tcp", 8443); + allow(hostRpcAddress(inputs), "tcp", 9101); + } + if (role === "orchestrator") { + for (const port of [443, 50061, 8444]) + allow(address(inputs, "coordinator"), "tcp", port); + } + if (role === "coordinator") { + allow(address(inputs, "blob"), "tcp", 4443); + allow(address(inputs, "registry"), "tcp", 5000); + allow(address(inputs, "orchestrator"), "tcp", 443); + } + return result; +} +function canonical(value: unknown): string { + if (Array.isArray(value)) return "[" + value.map(canonical).join(",") + "]"; + if (value !== null && typeof value === "object") + return ( + "{" + + Object.entries(value) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => JSON.stringify(k) + ":" + canonical(v)) + .join(",") + + "}" + ); + return JSON.stringify(value); +} +export function effectiveRules( + value: unknown, + expected: Record[], +): string { + const o = record(value, ["nftables"]); + check(Array.isArray(o.nftables), "nft_shape"); + const actual = o.nftables.flatMap((item: unknown) => { + check( + item !== null && typeof item === "object" && !Array.isArray(item), + "nft_object", + ); + const object = item as Record; + if (Object.keys(object).length === 1 && object.metainfo) return []; + check(Object.keys(object).length === 1, "nft_object"); + const [kind, body] = Object.entries(object)[0]!; + check( + ["table", "chain", "rule"].includes(kind) && + body !== null && + typeof body === "object" && + !Array.isArray(body), + "nft_object", + ); + const copy = { ...body }; + delete (copy as Record).handle; + return [{ [kind]: copy }]; + }); + check(canonical(actual) === canonical(expected), "network_rules_changed"); + return hash(canonical(actual)); +} diff --git a/tests/fixtures/mcp-stack/platform/owned.ts b/tests/fixtures/mcp-stack/platform/owned.ts new file mode 100644 index 000000000..b6b98a471 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/owned.ts @@ -0,0 +1,339 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, realpath, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, resolve, sep } from "node:path"; +import { spawn, type ChildProcess } from "node:child_process"; + +export class PlatformFailure extends Error { + constructor( + readonly code: "NEEDS_CONTEXT" | "INVALID_CONTEXT" | "FAILED", + readonly seam: string, + ) { + super(`${code}:${seam}`); + } +} +export function check(value: unknown, seam: string): asserts value { + if (!value) throw new PlatformFailure("INVALID_CONTEXT", seam); +} +export const hash = (bytes: string | Uint8Array) => + createHash("sha256").update(bytes).digest("hex"); +export function sameSecret(a: string, b: string): boolean { + const aa = Buffer.from(a), + bb = Buffer.from(b); + return aa.length === bb.length && timingSafeEqual(aa, bb); +} +export async function ownedPath(path: string, absent = false): Promise { + check(isAbsolute(path) && resolve(path) === path, "absolute_path"); + let current: string = sep; + for (const component of path.split(sep).filter(Boolean)) { + current = join(current, component); + try { + const s = await lstat(current); + check( + !s.isSymbolicLink() && (current === path || s.isDirectory()), + "real_path", + ); + if (current === path) check(s.uid === process.getuid?.(), "path_owner"); + } catch (e) { + if ( + absent && + current === path && + e instanceof Error && + "code" in e && + e.code === "ENOENT" + ) + return; + throw e; + } + } + if (absent) throw new PlatformFailure("INVALID_CONTEXT", "output_exists"); +} +export async function privateRoot(path: string): Promise { + await ownedPath(path); + const s = await lstat(path); + check( + s.isDirectory() && + (s.mode & 0o777) === 0o700 && + (await realpath(path)) === path, + "private_root", + ); +} +export async function newRoot(path: string): Promise { + await privateRoot(dirname(path)); + await ownedPath(path, true); + await mkdir(path, { mode: 0o700 }); + await privateRoot(path); +} +export async function readOwned( + path: string, + limit = 1 << 20, + privateOnly = true, +): Promise { + await ownedPath(path); + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const s = await handle.stat(); + check( + s.isFile() && + s.uid === process.getuid?.() && + s.nlink === 1 && + s.size > 0 && + s.size <= limit, + "file_identity", + ); + check(!privateOnly || (s.mode & 0o077) === 0, "private_file"); + const bytes = await handle.readFile(); + check(bytes.length === s.size, "file_changed"); + const after = await handle.stat(); + check( + after.mtimeMs === s.mtimeMs && + after.ctimeMs === s.ctimeMs && + after.nlink === 1, + "file_changed", + ); + return bytes; + } finally { + await handle.close(); + } +} +export interface Pin { + path: string; + sha256: string; +} +export async function verifyPin(pin: Pin): Promise { + check(/^[a-f0-9]{64}$/.test(pin.sha256), "pin_shape"); + await ownedPath(pin.path); + const h = await open(pin.path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const s = await h.stat(); + check( + s.isFile() && s.nlink === 1 && s.uid === process.getuid?.() && s.size > 0, + "artifact_identity", + ); + const digest = createHash("sha256"); + for await (const bytes of h.createReadStream({ autoClose: false })) + digest.update(bytes); + const after = await h.stat(); + check( + after.mtimeMs === s.mtimeMs && + after.ctimeMs === s.ctimeMs && + after.nlink === 1 && + digest.digest("hex") === pin.sha256, + "artifact_hash", + ); + } finally { + await h.close(); + } +} +export async function put( + path: string, + value: string | Uint8Array, + executable = false, +): Promise { + await privateRoot(dirname(path)); + await ownedPath(path, true); + await writeFile(path, value, { + mode: executable ? 0o700 : 0o600, + flag: "wx", + }); + return { path, sha256: hash(value) }; +} +export function record( + value: unknown, + keys: readonly string[], + optional: readonly string[] = [], +): Record { + check( + value !== null && typeof value === "object" && !Array.isArray(value), + "object", + ); + const data = value as Record; + check( + keys.every((key) => Object.hasOwn(data, key)) && + Object.keys(data).every( + (key) => keys.includes(key) || optional.includes(key), + ), + "object_keys", + ); + return data; +} + +// Only explicit task values cross the child boundary. No shell, HOME, cloud or agent environment is inherited. +export function childEnvironment( + tmp: string, + values: Readonly> = {}, +): NodeJS.ProcessEnv { + check(isAbsolute(tmp), "child_tmp"); + check( + Object.keys(values).every((key) => key === "DOCKER_HOST"), + "child_environment", + ); + if (values.DOCKER_HOST !== undefined) { + const path = values.DOCKER_HOST.slice("unix://".length); + check( + values.DOCKER_HOST.startsWith("unix://") && + isAbsolute(path) && + resolve(path) === path, + "engine_endpoint", + ); + } + return { + PATH: "/usr/bin:/bin:/usr/sbin:/sbin", + TMPDIR: tmp, + LANG: "C", + ...values, + }; +} +export class OwnedProcess { + readonly child: ChildProcess; + readonly done: Promise; + private ended = false; + constructor( + file: string, + args: readonly string[], + cwd: string, + env: NodeJS.ProcessEnv, + stdio: + | "ignore" + | ["ignore", number, number] + | ["pipe", "pipe", "ignore"] + | ["pipe", "pipe", "pipe"] = "ignore", + ) { + check(isAbsolute(file) && isAbsolute(cwd), "process_path"); + this.child = spawn(file, [...args], { cwd, env, detached: true, stdio }); + this.done = new Promise((resolve) => { + this.child.once("error", () => { + this.ended = true; + resolve(); + }); + this.child.once("close", () => { + this.ended = true; + resolve(); + }); + }); + } + running(): boolean { + return ( + !!this.child.pid && + !this.ended && + this.child.exitCode === null && + this.child.signalCode === null + ); + } + private groupExists(): boolean { + if (!this.child.pid) return false; + try { + process.kill(-this.child.pid, 0); + return true; + } catch (e) { + if (e instanceof Error && "code" in e) { + if (e.code === "ESRCH") return false; + // macOS returns EPERM for a group with only unreaped zombies. + // Permission denial leaves group absence unproved on every platform. + if (e.code === "EPERM") return true; + } + throw e; + } + } + async stop(graceMs = 1500): Promise { + check( + Number.isInteger(graceMs) && graceMs >= 0 && graceMs <= 120000, + "process_deadline", + ); + const signal = (s: NodeJS.Signals) => { + if (!this.child.pid) return; + try { + process.kill(-this.child.pid, s); + } catch (e) { + if (!( + e instanceof Error && + "code" in e && + (e.code === "ESRCH" || e.code === "EPERM") + )) + throw e; + } + }; + if (this.groupExists()) signal("SIGTERM"); + const deadline = performance.now() + graceMs; + while (this.groupExists() && performance.now() < deadline) + await new Promise((r) => setTimeout(r, 20)); + if (this.groupExists()) signal("SIGKILL"); + const end = performance.now() + 1500; + while ((!this.ended || this.groupExists()) && performance.now() < end) + await new Promise((r) => setTimeout(r, 20)); + if (!this.ended || this.groupExists()) + throw new PlatformFailure("FAILED", "process_cleanup"); + await this.done; + } +} +export async function command( + file: string, + args: readonly string[], + cwd: string, + env: NodeJS.ProcessEnv, + input?: string, + deadlineMs = 10000, +): Promise { + check(isAbsolute(file) && isAbsolute(cwd), "process_path"); + check( + Number.isInteger(deadlineMs) && deadlineMs > 0 && deadlineMs <= 120000, + "command_deadline", + ); + const owned = new OwnedProcess(file, args, cwd, env, [ + "pipe", + "pipe", + "ignore", + ]); + const { child } = owned; + // These streams are present because this fixed command path selects pipe stdio. + const { stdin, stdout } = child; + check(stdin && stdout, "command_stdio"); + const chunks: Buffer[] = []; + let bytes = 0, + interrupted = false, + unavailable = false; + let interrupt!: () => void; + const interruption = new Promise((resolve) => { + interrupt = resolve; + }); + const timer = setTimeout(() => { + interrupted = true; + interrupt(); + }, deadlineMs); + stdout.on("data", (chunk: Buffer) => { + bytes += chunk.length; + if (bytes > 4 * 1024 * 1024) { + interrupted = true; + interrupt(); + } else chunks.push(chunk); + }); + stdout.on("error", () => { + interrupted = true; + interrupt(); + }); + stdin.on("error", () => {}); + child.once("error", () => { + unavailable = true; + }); + stdin.end(input); + try { + await Promise.race([owned.done, interruption]); + // The leader closing is not group completion. Reuse the retained group's + // TERM -> KILL -> wait path before either success or command failure. + await owned.stop(); + if (unavailable) + throw new PlatformFailure("NEEDS_CONTEXT", "command_unavailable"); + if (interrupted || child.exitCode !== 0) + throw new PlatformFailure("FAILED", "command"); + return Buffer.concat(chunks); + } catch (error) { + if (error instanceof PlatformFailure && error.seam === "process_cleanup") + throw new PlatformFailure("FAILED", "command_cleanup"); + throw error; + } finally { + clearTimeout(timer); + stdin.destroy(); + stdout.destroy(); + child.unref(); + } +} diff --git a/tests/fixtures/mcp-stack/platform/platform.test.ts b/tests/fixtures/mcp-stack/platform/platform.test.ts new file mode 100644 index 000000000..68c3b8734 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/platform.test.ts @@ -0,0 +1,2039 @@ +import { poolFixture } from "./fc-thin-fixture.ts"; +import { afterAll, beforeAll, expect, spyOn, test } from "bun:test"; +import { + chmod, + link, + mkdir, + mkdtemp, + readFile, + realpath, + rename, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { request } from "node:http"; +import { connect } from "node:net"; +import { dirname, join } from "node:path"; +import { + ContextSchema, + type Context, +} from "../../../../orchestrator/scripts/mcp-stack/context.ts"; +import { + ROLES, + type SupervisorStatus, +} from "../../../../orchestrator/scripts/mcp-stack/supervisor.ts"; +import { + childEnvironment, + command, + hash, + newRoot, + OwnedProcess, + PlatformFailure, + privateRoot, + put, + readOwned, + record, + verifyPin, +} from "./owned.ts"; +import { + BINARIES, + binaryArchitecture, + engineArguments, + validateEngineConfig, + validateExecutables, + validateVzEntitlement, + cli, + IMAGES, + parseInputs, + REPO, + requirePlatform, + type Inputs, +} from "./preflight.ts"; +import { effectiveRules, rules } from "./network.ts"; +import { parseOperation, startSupervisor, verifyStatus } from "./supervisor.ts"; +import { generateIdentity } from "./identity.ts"; +import { generateConfiguration } from "./configuration.ts"; + +import { Engine } from "./engine.ts"; +import { verifyFcKernelPosture } from "./fc-runtime.ts"; + +test("FC kernel posture requires writable private proc sys and exactly four capabilities", () => { + const mounts = [ + "31 20 0:27 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw", + "32 20 0:28 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro", + "33 32 0:29 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw", + ].join("\n"); + const status = + "CapEff:\t0000000008201400\nCapPrm:\t0000000008201400\nCapBnd:\t0000000008201400\nNoNewPrivs:\t1\nSeccomp:\t2\n"; + expect(verifyFcKernelPosture(mounts, status)).toHaveLength(3); + expect(() => + verifyFcKernelPosture( + mounts + "\n34 31 0:27 /sys /proc/sys rw - proc proc rw", + status, + ), + ).not.toThrow(); + for (const changed of [ + mounts.replace("/proc rw,", "/proc ro,"), + mounts + "\n34 31 0:27 /sys /proc/sys ro - proc proc rw", + mounts.replace("/sys ro,", "/sys rw,"), + mounts.replace("/sys/fs/cgroup ro,", "/sys/fs/cgroup rw,"), + mounts + .split("\n") + .filter((line) => !line.includes(" /proc ")) + .join("\n"), + ]) + expect(() => verifyFcKernelPosture(changed, status)).toThrow(); + for (const field of ["CapEff", "CapPrm", "CapBnd"]) { + for (const mask of ["0000000008201000", "0000000008281400"]) { + expect(() => + verifyFcKernelPosture( + mounts, + status.replace(field + ":\t0000000008201400", field + ":\t" + mask), + ), + ).toThrow("fc_host_effective_capabilities"); + } + } + for (const changed of [ + status.replace("NoNewPrivs:\t1", "NoNewPrivs:\t0"), + status.replace("Seccomp:\t2", "Seccomp:\t0"), + ]) + expect(() => verifyFcKernelPosture(mounts, changed)).toThrow( + "fc_host_effective_confinement", + ); +}); + +test("the disposable host TLS probe stops without delaying other roles", async () => { + const runId = "mcp-123456789012345678901234"; + const id = "a".repeat(64); + const engine = new Engine(input(), "/owned", runId); + let running = false; + const stops: string[][] = []; + spyOn(engine, "audit").mockImplementation(async () => ({})); + spyOn(engine, "call").mockImplementation(async (args) => { + if (args[0] === "create") return Buffer.from(id); + if (args[0] === "start") running = true; + if (args[0] === "stop") { + stops.push(args); + running = false; + } + if (args[0] === "container" && args[1] === "inspect") + return Buffer.from( + JSON.stringify([ + { + Id: id, + Config: { Labels: { "engrams.mcp.run": runId } }, + State: { Running: running, Pid: running ? 1 : 0 }, + }, + ]), + ); + return Buffer.alloc(0); + }); + await engine.create( + "host-tls-probe", + "guard", + [], + ["infinity"], + "/bin/sleep", + ); + await engine.remove("host-tls-probe"); + await engine.create( + "ordinary-probe", + "guard", + [], + ["infinity"], + "/bin/sleep", + ); + await engine.remove("ordinary-probe"); + expect(stops).toEqual([ + ["stop", "--time", "0", id], + ["stop", "--time", "3", id], + ]); +}); + +let root: string; +let contextCount = 0; +const node = process.execPath; +beforeAll(async () => { + const parent = join(REPO, "var/mcp-platform-bootstrap"); + await mkdir(parent, { mode: 0o700, recursive: true }); + await privateRoot(parent); + root = await mkdtemp(join(parent, "t-")); +}); +afterAll(async () => { + if (root) await rm(root, { recursive: true }); +}); +function input(): Inputs { + const pin = { path: "/owned/artifact", sha256: "a".repeat(64) }; + return parseInputs( + { + version: 1, + backend: "vz", + proxy: pin, + nativeProbe: pin, + architecture: "arm64", + sourceCommit: "b".repeat(40), + engineSocket: "/owned/engine.sock", + engineConfig: "/owned/engine-config", + engineId: "fixture-engine-identity", + registryPort: 24567, + subnet: "10.80.1.0/24", + nativeHostIp: "192.168.50.2", + binaries: Object.fromEntries(BINARIES.map((v) => [v, pin])), + images: Object.fromEntries( + IMAGES.map((v) => [ + v, + { + archive: pin, + id: `sha256:${"c".repeat(64)}`, + manifestDigest: `sha256:${"d".repeat(64)}`, + }, + ]), + ), + }, + "vz", + ); +} +async function context(): Promise { + const ctxRoot = await mkdtemp(join(root, "c-")); + const pin = await put(join(ctxRoot, "normal"), "normal-configuration"); + await put( + join(ctxRoot, "token"), + "fixture-supervisor-token-" + "a".repeat(32), + ); + const endpoint = { + origin: "https://app.fixture.example", + ca: pin, + serverFingerprint: Array(32).fill("AA").join(":"), + }; + const fixture = (kind: string) => ({ + ...endpoint, + origin: `https://observer.${kind}.fixture.example`, + configuration: pin, + runRoot: ctxRoot, + tokenFile: join(ctxRoot, "token"), + resourceKeyFile: join(ctxRoot, "token"), + resourceOrigin: `https://resource.${kind}.fixture.example`, + issuerOrigin: `https://issuer.${kind}.fixture.example`, + authorizationOrigin: `https://authorization.${kind}.fixture.example`, + tokenOrigin: `https://token.${kind}.fixture.example`, + redirectUri: "https://app.fixture.example/integrations/mcp/callback", + }); + const human = (kind: string) => ({ + email: `${kind}@fixture.example`, + passwordFile: join(ctxRoot, `${kind}.password`), + cookieFile: join(ctxRoot, `${kind}.cookie`), + }); + return ContextSchema.parse({ + version: 1, + runId: "fixture-run", + runRoot: root, + backend: "vz", + sourceCommit: "a".repeat(40), + hostId: "00000000-0000-4000-8000-000000000001", + imageId: "fixture-image", + imageDigest: `sha256:${"b".repeat(64)}`, + artifacts: Object.fromEntries( + [ + "coordinator", + "orchestrator", + "worker", + "hostAgent", + "fixture", + "kernel", + "guestImage", + "claudeHarness", + "codexHarness", + "modelCli", + "claudeBundle", + "codexBundle", + "helper", + ].map((v) => [v, pin]), + ), + endpoints: { + app: endpoint, + coordinator: endpoint, + operator: { ...endpoint, certificate: pin, privateKeyFile: pin.path }, + workerControl: endpoint, + }, + protocolFixtures: { + modern: { ...fixture("modern"), protocol: "2026-07-28" }, + legacy: { ...fixture("legacy"), protocol: "2025-06-18" }, + }, + publication: { + candidateIdentity: `sha256:${"b".repeat(64)}`, + expectedGeneration: null, + deploymentIdentity: `sha256:${"b".repeat(64)}`, + networkIdentity: `sha256:${"b".repeat(64)}`, + trustIdentity: `sha256:${"b".repeat(64)}`, + zone: "fixture-zone", + }, + identities: { + admin: human("admin"), + owner: human("owner"), + other: human("other"), + unattendedKeyFile: join(ctxRoot, "unattended"), + automationKeyFile: join(ctxRoot, "automation"), + routerSentinelFile: join(ctxRoot, "router"), + }, + supervisor: { + socket: join(root, String(++contextCount)), + tokenFile: join(ctxRoot, "token"), + configurations: { + orchestrator: { + normal: pin, + "global-disabled": pin, + "zone-disabled": pin, + }, + coordinator: { normal: pin }, + worker: { normal: pin }, + "host-agent": { normal: pin }, + }, + operatorCandidates: { + normal: { file: pin, identity: `sha256:${"b".repeat(64)}` }, + }, + }, + observer: { + orchestratorDsnFile: pin.path, + coordinatorDsnFile: pin.path, + hostManagedRoot: ctxRoot, + hostLogFile: pin.path, + }, + guest: { + stateDirectory: "/tmp/mcp-stack-fixture", + cliPath: "/opt/engram/dyn/0/engram-mcp-model-fixture", + }, + deadlines: { requestMs: 1000, stepMs: 3000, totalMs: 10000 }, + evidenceFile: join(ctxRoot, "evidence"), + }); +} +function status(ctx: Context): SupervisorStatus { + return { + version: 1, + runId: ctx.runId, + backend: ctx.backend, + sourceCommit: ctx.sourceCommit, + hostId: ctx.hostId, + roles: ROLES.map((role) => ({ + role, + state: "running", + bootId: role + "-first", + artifactSha256: ctx.artifacts.fixture.sha256, + configurationId: "normal", + configurationSha256: ctx.artifacts.fixture.sha256, + })), + network: { + generation: "fixture-generation", + enforcementSha256: "e".repeat(64), + fixtureOnlyEgress: true, + externalDnsBlocked: true, + roles: ["orchestrator", "coordinator", "worker"], + }, + }; +} + +test("private input refuses symlink, hardlink, public mode and changed pin", async () => { + const pin = await put(join(root, "input"), "bytes"); + await verifyPin(pin); + await symlink(pin.path, join(root, "symlink")); + await expect(readOwned(join(root, "symlink"))).rejects.toThrow(); + await link(pin.path, join(root, "hardlink")); + await expect(verifyPin(pin)).rejects.toThrow("artifact_identity"); + await rm(join(root, "hardlink")); + await chmod(pin.path, 0o644); + await expect(readOwned(pin.path)).rejects.toThrow("private_file"); + await chmod(pin.path, 0o600); + await writeFile(pin.path, "changed"); + await expect(verifyPin(pin)).rejects.toThrow("artifact_hash"); +}); +test("output root refuses reuse and symlinked parent", async () => { + const fresh = join(root, "fresh"); + await newRoot(fresh); + await expect(newRoot(fresh)).rejects.toThrow("output_exists"); + await symlink(fresh, join(root, "linked")); + await expect(newRoot(join(root, "linked", "child"))).rejects.toThrow( + "real_path", + ); +}); +test("arguments and manifest reject foreign backend, arbitrary commands and mixed architecture", () => { + expect( + cli([ + "--backend", + "vz", + "--run-root", + "/private/run", + "--artifacts", + "/private/input", + ]).backend, + ).toBe("vz"); + expect(() => + cli([ + "--backend", + "vz", + "--run-root", + "relative", + "--artifacts", + "/private/input", + ]), + ).toThrow(); + expect(() => + parseInputs({ ...input(), command: ["sh", "-c", "anything"] }, "vz"), + ).toThrow(); + expect(() => + parseInputs({ ...input(), architecture: "amd64" }, "vz"), + ).toThrow("vz_architecture"); + expect(() => parseInputs(input(), "firecracker")).toThrow( + "platform_identity", + ); + expect(() => + requirePlatform("firecracker", "arm64", "darwin", "arm64"), + ).toThrow("NEEDS_CONTEXT:target_platform"); +}); +test("artifact architecture comes from executable bytes", () => { + const elf = Buffer.alloc(64); + Buffer.from([0x7f, 69, 76, 70, 2, 1]).copy(elf); + elf.writeUInt16LE(183, 18); + expect(binaryArchitecture(elf)).toEqual({ + os: "linux", + architecture: "arm64", + }); + elf.writeUInt16LE(62, 18); + expect(binaryArchitecture(elf)).toEqual({ + os: "linux", + architecture: "amd64", + }); + elf[5] = 2; + expect(() => binaryArchitecture(elf)).toThrow("elf_format"); + const macho = Buffer.alloc(64); + macho.writeUInt32LE(0xfeedfacf); + macho.writeUInt32LE(0x0100000c, 4); + expect(binaryArchitecture(macho)).toEqual({ + os: "darwin", + architecture: "arm64", + }); + expect(() => binaryArchitecture(Buffer.from("a success receipt"))).toThrow( + "binary_header", + ); +}); +test("VZ entitlement verifies first and converts explicit XML through bounded commands", async () => { + const hostAgent = "/owned/host-agent"; + const env = childEnvironment(root); + const xml = + 'com.apple.security.virtualization'; + const calls: Parameters[] = []; + const runCommand: typeof command = async (...args) => { + calls.push(args); + return Buffer.from( + ["", xml, '{"com.apple.security.virtualization":true}'][ + calls.length - 1 + ]!, + ); + }; + await validateVzEntitlement(hostAgent, env, runCommand); + expect(calls).toEqual([ + ["/usr/bin/codesign", ["--verify", "--strict", hostAgent], REPO, env], + [ + "/usr/bin/codesign", + ["--display", "--entitlements", "-", "--xml", hostAgent], + REPO, + env, + ], + [ + "/usr/bin/plutil", + ["-convert", "json", "-o", "-", "--", "-"], + REPO, + env, + xml, + ], + ]); + for (const call of calls) expect(call[3]).toBe(env); +}); +for (const [kind, json] of [ + ["false", '{"com.apple.security.virtualization":false}'], + ["missing", "{}"], + ["string", '{"com.apple.security.virtualization":"true"}'], + ["number", '{"com.apple.security.virtualization":1}'], + ["nested", '{"nested":{"com.apple.security.virtualization":true}}'], + ["array", '[{"com.apple.security.virtualization":true}]'], + ["null", "null"], + ["scalar", "true"], + ["malformed", '{"com.apple.security.virtualization":true'], + ["text", "[Dict]\n[Key] com.apple.security.virtualization\n[Bool] true"], +] as const) { + test(`VZ entitlement rejects ${kind} conversion output`, async () => { + let calls = 0; + const runCommand: typeof command = async () => { + calls++; + // A true XML-looking fragment must not bypass the OS conversion result. + return Buffer.from( + calls === 3 + ? json + : "com.apple.security.virtualization", + ); + }; + await expect( + validateVzEntitlement( + "/owned/host-agent", + childEnvironment(root), + runCommand, + ), + ).rejects.toThrow("INVALID_CONTEXT:vz_entitlement"); + expect(calls).toBe(3); + }); +} +for (const step of [1, 2, 3]) { + test(`VZ entitlement stops after command ${step} fails`, async () => { + const failure = new PlatformFailure("FAILED", "command"); + let calls = 0; + const runCommand: typeof command = async () => { + if (++calls === step) throw failure; + return Buffer.from("com.apple.security.virtualization"); + }; + await expect( + validateVzEntitlement( + "/owned/host-agent", + childEnvironment(root), + runCommand, + ), + ).rejects.toBe(failure); + expect(calls).toBe(step); + }); +} +test("controlled child sees no parent secret or runtime injection", async () => { + process.env.MCP_PLATFORM_TEST_SECRET = "must-not-cross"; + try { + const env = childEnvironment(root, { + DOCKER_HOST: `unix://${root}/engine.sock`, + }); + const result = JSON.parse( + ( + await command( + node, + ["-e", "process.stdout.write(JSON.stringify(process.env))"], + root, + env, + ) + ).toString(), + ); + expect(result.DOCKER_HOST).toBe(`unix://${root}/engine.sock`); + expect(result.MCP_PLATFORM_TEST_SECRET).toBeUndefined(); + expect(result.HOME).toBeUndefined(); + expect(result.NODE_OPTIONS).toBeUndefined(); + expect(() => + childEnvironment(root, { NODE_OPTIONS: "--require=bad" }), + ).toThrow(); + expect(() => + childEnvironment(root, { AWS_ACCESS_KEY_ID: "parent-secret" }), + ).toThrow(); + expect(() => + childEnvironment(root, { DOCKER_HOST: "tcp://remote:2375" }), + ).toThrow(); + } finally { + delete process.env.MCP_PLATFORM_TEST_SECRET; + } +}); +test("owned process stop escalates a TERM-resistant controlled child and waits", async () => { + const marker = join(root, "child-ready"); + const child = new OwnedProcess( + node, + [ + "-e", + "process.on('SIGTERM',()=>{});require('node:fs').writeFileSync(process.argv[1],'ready',{mode:384});setInterval(()=>{},1000)", + marker, + ], + root, + childEnvironment(root), + ); + try { + const deadline = performance.now() + 2000; + while (performance.now() < deadline) { + try { + await readFile(marker); + break; + } catch { + await new Promise((r) => setTimeout(r, 10)); + } + } + expect(child.running()).toBe(true); + await child.stop(50); + expect(child.running()).toBe(false); + expect(child.child.signalCode).toBe("SIGKILL"); + } finally { + await child.stop(50); + } +}); +test("network verifier rejects missing default drop, foreign accept and embedded-DNS allowance", () => { + const plan = rules(input(), "worker"); + const actual = { + nftables: [ + { metainfo: { json_schema_version: 1 } }, + ...plan.map((v) => { + const [kind, body] = Object.entries(v)[0]!; + return { [kind]: { ...(body as Record), handle: 12 } }; + }), + ], + }; + expect(effectiveRules(actual, plan)).toMatch(/^[a-f0-9]{64}$/); + const changed = structuredClone(actual); + const chain = record(changed.nftables[2], ["chain"]).chain; + record(chain, [ + "family", + "table", + "name", + "type", + "hook", + "prio", + "policy", + "handle", + ]).policy = "accept"; + expect(() => effectiveRules(changed, plan)).toThrow("network_rules_changed"); + expect(() => + effectiveRules( + { + nftables: [ + ...plan, + { + rule: { + family: "inet", + table: "mcp", + chain: "output", + expr: [{ accept: null }], + }, + }, + ], + }, + plan, + ), + ).toThrow(); + expect(JSON.stringify(plan)).toContain("127.0.0.11"); + expect(JSON.stringify(plan)).not.toContain('"dport"},"right":5432'); +}); +test("real private TLS leaves and normal/zone configs pass production validators", async () => { + const directory = join(root, "identity"); + await mkdir(directory, { mode: 0o700 }); + const inputs = input(); + const opensslPath = await realpath( + process.env.MCP_PLATFORM_TEST_OPENSSL ?? Bun.which("openssl") ?? "openssl", + ); + inputs.binaries.openssl = { + path: opensslPath, + sha256: hash(await readFile(opensslPath)), + }; + const identities = await generateIdentity(inputs, directory); + expect( + new Set(Object.values(identities.leaves).map((v) => v.fingerprint)).size, + ).toBe(8); + for (const leaf of Object.values(identities.leaves)) + expect(leaf.hostPin).toBe( + leaf.fingerprint.replaceAll(":", "").toLowerCase(), + ); + const configs = await generateConfiguration(inputs, directory, identities); + const normal = JSON.parse( + (await readOwned(configs.runtimeNormal.path)).toString(), + ); + const disabled = JSON.parse( + (await readOwned(configs.runtimeZoneDisabled.path)).toString(), + ); + disabled.workers[0].enabled = true; + expect(disabled).toEqual(normal); + expect(configs.candidateIdentity).toBe( + `sha256:${hash(await readOwned(configs.candidateFile.path))}`, + ); + const c = JSON.parse((await readOwned(configs.coordinator.path)).toString()); + const modern = normal.registrations.find( + (entry: { id: string }) => entry.id === "fixture-modern", + ); + expect(modern.scopes).toEqual(["tools:read"]); + expect(c.registry.registrations["fixture-modern"].scopes).toEqual([ + "tools:read", + ]); + expect(c.registry.registrations["fixture-modern"].accountField).toBe( + "fixture_account", + ); + expect(c.registry.registrations["fixture-legacy"].scopes).toEqual([ + "tools:read", + "tools:write", + ]); + expect(c.registry.registrations["fixture-legacy"].accountField).toBe( + "fixture_account", + ); + expect(c.registry.zones[configs.zone.id].policyDigest).toHaveLength(64); + expect(c.registry.zones[configs.zone.id].endpoints).toContain( + "https://token.modern.fixture.example:8443/token", + ); +}); +test("Unix supervisor authenticates, rejects payload authority and isolates restarts", async () => { + const ctx = await context(); + let current = status(ctx), + mutations = 0, + corruptScope = false; + const server = await startSupervisor(ctx, { + status: async () => structuredClone(current), + restart: async (role, configurationId) => { + mutations++; + const entry = current.roles.find((v) => v.role === role)!; + entry.bootId += "-next"; + entry.configurationId = configurationId; + if (corruptScope) + current.roles.find((v) => v.role === "fixture-modern")!.bootId += + "-changed"; + }, + stop: async (role) => { + mutations++; + current.roles.find((v) => v.role === role)!.state = "stopped"; + }, + }); + const validToken = (await readOwned(ctx.supervisor.tokenFile)).toString(); + async function call(body: unknown, token = validToken) { + const payload = JSON.stringify(body); + return await new Promise<{ code: number; body: unknown }>( + (resolve, reject) => { + const req = request( + { + socketPath: ctx.supervisor.socket, + method: "POST", + path: "/v1/control", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + "content-length": Buffer.byteLength(payload), + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (v) => chunks.push(v)); + res.on("end", () => + resolve({ + code: res.statusCode!, + body: JSON.parse(Buffer.concat(chunks).toString()), + }), + ); + }, + ); + req.on("error", reject); + req.end(payload); + }, + ); + } + const base = { version: 1, runId: ctx.runId }; + try { + expect((await call({ ...base, operation: "status" }, "wrong")).code).toBe( + 401, + ); + expect( + (await call({ ...base, runId: "other", operation: "status" })).code, + ).toBe(400); + expect( + ( + await call({ + ...base, + operation: "restart", + role: "fixture-modern", + configurationId: "normal", + }) + ).code, + ).toBe(400); + expect( + ( + await call({ + ...base, + operation: "restart", + role: "worker", + configurationId: "zone-disabled", + }) + ).code, + ).toBe(400); + expect( + (await call({ ...base, operation: "status", command: "anything" })).code, + ).toBe(400); + expect(mutations).toBe(0); + const before = structuredClone(current); + expect( + ( + await call({ + ...base, + operation: "restart", + role: "orchestrator", + configurationId: "global-disabled", + }) + ).code, + ).toBe(200); + expect(current.roles.find((v) => v.role === "fixture-modern")).toEqual( + before.roles.find((v) => v.role === "fixture-modern"), + ); + expect(() => + verifyStatus(ctx, { ...current, backend: "firecracker" }), + ).toThrow("supervisor_identity"); + expect(() => + verifyStatus(ctx, { ...current, sourceCommit: "0".repeat(40) }), + ).toThrow("supervisor_identity"); + expect(() => + parseOperation( + { + ...base, + operation: "stop", + role: "coordinator", + configurationId: "normal", + }, + ctx, + ), + ).toThrow(); + current = status(ctx); + await writeFile( + ctx.supervisor.configurations.orchestrator.normal.path, + "changed", + ); + const count = mutations; + expect( + ( + await call({ + ...base, + operation: "restart", + role: "orchestrator", + configurationId: "normal", + }) + ).code, + ).toBe(400); + expect(mutations).toBe(count); + await writeFile( + ctx.supervisor.configurations.orchestrator.normal.path, + "normal-configuration", + ); + corruptScope = true; + expect( + ( + await call({ + ...base, + operation: "restart", + role: "orchestrator", + configurationId: "normal", + }) + ).code, + ).toBe(400); + expect((await call({ ...base, operation: "status" })).code).toBe(503); + } finally { + await server.close(); + } +}); + +async function control( + ctx: Context, + body: unknown, + headers: Record = {}, +) { + const payload = JSON.stringify(body); + const token = (await readOwned(ctx.supervisor.tokenFile)).toString(); + return await new Promise<{ code: number; text: string }>( + (resolve, reject) => { + const req = request( + { + socketPath: ctx.supervisor.socket, + method: "POST", + path: "/v1/control", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + "content-length": String(Buffer.byteLength(payload)), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (v) => chunks.push(v)); + res.on("error", reject); + res.on("aborted", () => reject(new Error("response aborted"))); + res.on("end", () => + resolve({ + code: res.statusCode!, + text: Buffer.concat(chunks).toString(), + }), + ); + }, + ); + req.on("error", reject); + req.end(payload); + }, + ); +} + +test("supervisor retains a safe diagnostic for a failed restart", async () => { + const ctx = await context(); + ctx.runRoot = dirname(ctx.supervisor.tokenFile); + ctx.supervisor.socket = join(ctx.runRoot, "s"); + const server = await startSupervisor(ctx, { + status: async () => status(ctx), + restart: async () => { + throw new PlatformFailure("FAILED", "role_readiness"); + }, + stop: async () => {}, + }); + try { + const response = await control(ctx, { + version: 1, + runId: ctx.runId, + operation: "restart", + role: "worker", + configurationId: "normal", + }); + expect(response.code).toBe(400); + const diagnostic = JSON.parse( + await readFile(join(ctx.runRoot, "supervisor-failure-1.json"), "utf8"), + ); + expect(diagnostic).toEqual({ + version: 1, + operation: "restart", + role: "worker", + phase: "lifecycle", + failure: "FAILED", + seam: "role_readiness", + }); + expect( + ( + await control(ctx, { + version: 1, + runId: ctx.runId, + operation: "status", + }) + ).code, + ).toBe(503); + } finally { + await server.close(); + } +}); + +test("supervisor records a disconnected client without its request contents", async () => { + const ctx = await context(); + ctx.runRoot = dirname(ctx.supervisor.tokenFile); + ctx.supervisor.socket = join(ctx.runRoot, "s"); + let entered!: () => void, aborted!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const stopped = new Promise((resolve) => { + aborted = resolve; + }); + const server = await startSupervisor(ctx, { + status: async (signal) => { + entered(); + await new Promise((resolve) => + signal.addEventListener( + "abort", + () => { + aborted(); + resolve(); + }, + { once: true }, + ), + ); + return status(ctx); + }, + restart: async () => {}, + stop: async () => {}, + }); + const token = (await readOwned(ctx.supervisor.tokenFile)).toString(); + const payload = JSON.stringify({ + version: 1, + runId: ctx.runId, + operation: "status", + }); + const req = request({ + socketPath: ctx.supervisor.socket, + method: "POST", + path: "/v1/control", + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + "content-length": Buffer.byteLength(payload), + }, + }); + req.on("error", () => {}); + req.end(payload); + await started; + req.destroy(); + await stopped; + await server.close(); + const diagnostic = JSON.parse( + await readFile(join(ctx.runRoot, "supervisor-failure-1.json"), "utf8"), + ); + expect(diagnostic).toEqual({ + version: 1, + operation: "status", + phase: "before_status", + failure: "disconnect", + seam: "control_aborted", + }); + expect(JSON.stringify(diagnostic)).not.toContain(token); +}); + +test("supervisor records transition validation failure without changing control behavior", async () => { + const ctx = await context(); + ctx.runRoot = dirname(ctx.supervisor.tokenFile); + ctx.supervisor.socket = join(ctx.runRoot, "s"); + const current = status(ctx); + const server = await startSupervisor(ctx, { + status: async () => structuredClone(current), + restart: async (role) => { + current.roles.find((entry) => entry.role === role)!.bootId += "-next"; + current.roles.find((entry) => entry.role === "fixture-modern")!.bootId += + "-wrong"; + }, + stop: async () => {}, + }); + try { + const response = await control(ctx, { + version: 1, + runId: ctx.runId, + operation: "restart", + role: "worker", + configurationId: "normal", + }); + expect(response.code).toBe(400); + expect( + JSON.parse( + await readFile(join(ctx.runRoot, "supervisor-failure-1.json"), "utf8"), + ), + ).toEqual({ + version: 1, + operation: "restart", + role: "worker", + phase: "transition", + failure: "INVALID_CONTEXT", + seam: "restart_scope", + }); + } finally { + await server.close(); + } +}); + +test("supervisor refuses duplicate authentication and oversized bodies before lifecycle access", async () => { + const ctx = await context(); + let reads = 0; + const server = await startSupervisor(ctx, { + status: async () => { + reads++; + return status(ctx); + }, + restart: async () => {}, + stop: async () => {}, + }); + try { + expect( + (await control(ctx, { operation: "status", extra: "x".repeat(4100) })) + .code, + ).toBe(400); + const token = (await readOwned(ctx.supervisor.tokenFile)).toString(); + const response = await new Promise((resolve, reject) => { + const socket = connect(ctx.supervisor.socket); + let result = ""; + socket.on("data", (bytes) => { + result += bytes.toString(); + }); + socket.on("end", () => resolve(result)); + socket.on("error", reject); + socket.once("connect", () => + socket.write( + `POST /v1/control HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ${token}\r\nAuthorization: Bearer ${token}\r\nContent-Type: application/json\r\nContent-Length: 2\r\n\r\n{}`, + ), + ); + }); + expect(response).toContain("401"); + expect(reads).toBe(0); + } finally { + await server.close(); + } +}); + +test("supervisor has no control queue and shutdown aborts and drains the active lifecycle", async () => { + const ctx = await context(); + let entered!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + let drained = false, + mutations = 0; + const server = await startSupervisor(ctx, { + status: async (signal) => { + entered(); + await new Promise((resolve) => + signal.addEventListener( + "abort", + () => { + drained = true; + resolve(); + }, + { once: true }, + ), + ); + return status(ctx); + }, + restart: async () => { + mutations++; + }, + stop: async () => { + mutations++; + }, + }); + const body = { + version: 1, + runId: ctx.runId, + operation: "restart", + role: "worker", + configurationId: "normal", + }; + const pending = control(ctx, body).catch(() => undefined); + await started; + expect((await control(ctx, body)).code).toBe(409); + await server.close(); + await pending; + expect(drained).toBe(true); + expect(mutations).toBe(0); +}); + +test("supervisor reports bounded cleanup failure if a lifecycle ignores cancellation", async () => { + const ctx = await context(); + ctx.deadlines.requestMs = 50; + let entered!: () => void, release!: () => void; + const started = new Promise((resolve) => { + entered = resolve; + }); + const stuck = new Promise((resolve) => { + release = resolve; + }); + let mutations = 0; + const server = await startSupervisor(ctx, { + status: async () => { + entered(); + await stuck; + return status(ctx); + }, + restart: async () => { + mutations++; + }, + stop: async () => { + mutations++; + }, + }); + const pending = control(ctx, { + version: 1, + runId: ctx.runId, + operation: "restart", + role: "worker", + configurationId: "normal", + }).catch(() => undefined); + await started; + try { + await expect(server.close()).rejects.toThrow("FAILED:control_cleanup"); + } finally { + release(); + await pending; + } + await new Promise((resolve) => setImmediate(resolve)); + expect(mutations).toBe(0); +}); + +test("supervisor does not expose lifecycle errors or permit later mutation", async () => { + const ctx = await context(); + ctx.runRoot = dirname(ctx.supervisor.tokenFile); + ctx.supervisor.socket = join(ctx.runRoot, "s"); + let mutations = 0; + const server = await startSupervisor(ctx, { + status: async () => { + throw new Error("private credential must remain private"); + }, + restart: async () => { + mutations++; + }, + stop: async () => { + mutations++; + }, + }); + try { + const body = { + version: 1, + runId: ctx.runId, + operation: "stop", + role: "worker", + }; + const result = await control(ctx, body); + expect(result).toEqual({ code: 400, text: '{"error":"control_failed"}' }); + const diagnostic = JSON.parse( + await readFile(join(ctx.runRoot, "supervisor-failure-1.json"), "utf8"), + ); + expect(diagnostic).toEqual({ + version: 1, + operation: "stop", + role: "worker", + phase: "before_status", + failure: "OTHER", + seam: "other", + }); + expect(JSON.stringify(diagnostic)).not.toContain("private credential"); + expect((await control(ctx, body)).code).toBe(503); + expect(mutations).toBe(0); + } finally { + await server.close(); + } +}); + +test("supervisor cleanup refuses a substituted socket path and preserves the foreign file", async () => { + const ctx = await context(); + const server = await startSupervisor(ctx, { + status: async () => status(ctx), + restart: async () => {}, + stop: async () => {}, + }); + const original = ctx.supervisor.socket + "-original"; + await rename(ctx.supervisor.socket, original); + await put(ctx.supervisor.socket, "foreign file"); + try { + await expect(server.close()).rejects.toThrow("socket_replaced"); + expect((await readOwned(ctx.supervisor.socket)).toString()).toBe( + "foreign file", + ); + } finally { + // Restore the test-owned socket so the retained listener can close safely. + await rm(ctx.supervisor.socket); + await rename(original, ctx.supervisor.socket); + await server.close(); + } +}); + +test("supervisor request deadline cancels status and refuses all later controls", async () => { + const ctx = await context(); + ctx.runRoot = dirname(ctx.supervisor.tokenFile); + ctx.supervisor.socket = join(ctx.runRoot, "s"); + ctx.deadlines.requestMs = 50; + let aborted = false; + const server = await startSupervisor(ctx, { + status: async (signal) => { + await new Promise((resolve) => + signal.addEventListener( + "abort", + () => { + aborted = true; + resolve(); + }, + { once: true }, + ), + ); + return status(ctx); + }, + restart: async () => {}, + stop: async () => {}, + }); + const body = { version: 1, runId: ctx.runId, operation: "status" }; + try { + await expect(control(ctx, body)).rejects.toThrow(); + expect(aborted).toBe(true); + expect((await control(ctx, body)).code).toBe(503); + } finally { + await server.close(); + } + expect( + JSON.parse( + await readFile(join(ctx.runRoot, "supervisor-failure-1.json"), "utf8"), + ), + ).toEqual({ + version: 1, + operation: "status", + phase: "before_status", + failure: "deadline", + seam: "control_aborted", + }); +}); + +test("bounded command rejects failed, oversized and timed-out children without error output", async () => { + const env = childEnvironment(root); + await expect( + command( + node, + ["-e", "process.stderr.write('private-value');process.exit(3)"], + root, + env, + ), + ).rejects.toThrow("FAILED:command"); + await expect( + command( + node, + [ + "-e", + "process.stdout.write(Buffer.alloc(5*1024*1024));setInterval(()=>{},1000)", + ], + root, + env, + ), + ).rejects.toThrow("FAILED:command"); + await expect( + command(node, ["-e", "setInterval(()=>{},1000)"], root, env, undefined, 40), + ).rejects.toThrow("FAILED:command"); + await expect(command("relative", [], root, env)).rejects.toThrow( + "process_path", + ); +}); + +test("engine inspection uses only explicit Unix endpoint and private empty CLI configuration", async () => { + const config = join(root, "docker-client"); + await newRoot(config); + const initial = await validateEngineConfig(config); + const inputs = { + engineConfig: config, + engineSocket: join(root, "engine.sock"), + }; + expect(engineArguments(inputs)).toEqual([ + "--config", + config, + "--host", + `unix://${inputs.engineSocket}`, + "info", + "--format", + "{{json .}}", + ]); + const image = `sha256:${"b".repeat(64)}`; + expect(engineArguments(inputs, image)).toEqual([ + "--config", + config, + "--host", + `unix://${inputs.engineSocket}`, + "image", + "inspect", + image, + ]); + expect(() => engineArguments(inputs, "image; arbitrary-command")).toThrow( + "image_digest", + ); + expect(() => + engineArguments({ ...inputs, engineSocket: "tcp://external" }), + ).toThrow("engine_path"); + expect(Object.keys(childEnvironment(root)).sort()).toEqual([ + "LANG", + "PATH", + "TMPDIR", + ]); + await put(join(config, "config.json"), '{"auths":{"registry.example":{}}}'); + await expect(validateEngineConfig(config, initial)).rejects.toThrow( + "engine_config_not_empty", + ); + await rm(join(config, "config.json")); + await rename(config, config + "-original"); + await newRoot(config); + await expect(validateEngineConfig(config, initial)).rejects.toThrow( + "engine_config_changed", + ); + await rm(config, { recursive: true }); + await symlink(config + "-original", config); + await expect(validateEngineConfig(config, initial)).rejects.toThrow( + "real_path", + ); +}); + +test("supervisor rejects array role and configuration values before lifecycle access", async () => { + const observed: { code: number; reads: number; mutations: number }[] = []; + for (const fields of [ + { operation: "restart", role: ["worker"], configurationId: "normal" }, + { operation: "restart", role: "worker", configurationId: ["normal"] }, + { operation: "stop", role: ["worker"] }, + ]) { + const ctx = await context(); + let reads = 0, + mutations = 0; + const server = await startSupervisor(ctx, { + status: async () => { + reads++; + return status(ctx); + }, + restart: async () => { + mutations++; + }, + stop: async () => { + mutations++; + }, + }); + try { + const result = await control(ctx, { + version: 1, + runId: ctx.runId, + ...fields, + }); + observed.push({ code: result.code, reads, mutations }); + } finally { + await server.close(); + } + } + expect(observed).toEqual( + Array.from({ length: 3 }, () => ({ code: 400, reads: 0, mutations: 0 })), + ); +}); + +test("supervisor rejects declared non-O disable requests before lifecycle access", async () => { + const observed: { code: number; reads: number; mutations: number }[] = []; + for (const role of ["coordinator", "worker", "host-agent"] as const) { + for (const configurationId of [ + "global-disabled", + "zone-disabled", + ] as const) { + const ctx = await context(); + Reflect.set( + ctx.supervisor.configurations[role], + configurationId, + ctx.supervisor.configurations[role].normal, + ); + let reads = 0, + mutations = 0; + const current = status(ctx); + const server = await startSupervisor(ctx, { + status: async () => { + reads++; + return structuredClone(current); + }, + restart: async (selected, variant) => { + mutations++; + const entry = current.roles.find((v) => v.role === selected)!; + entry.bootId += "-next"; + entry.configurationId = variant; + }, + stop: async () => { + mutations++; + }, + }); + try { + const result = await control(ctx, { + version: 1, + runId: ctx.runId, + operation: "restart", + role, + configurationId, + }); + observed.push({ code: result.code, reads, mutations }); + } finally { + await server.close(); + } + } + } + expect(observed).toEqual( + Array.from({ length: 6 }, () => ({ code: 400, reads: 0, mutations: 0 })), + ); +}); + +test("supervisor rejects declared non-O disable status", async () => { + const ctx = await context(); + for (const role of ["coordinator", "worker", "host-agent"] as const) { + for (const configurationId of [ + "global-disabled", + "zone-disabled", + ] as const) { + Reflect.set( + ctx.supervisor.configurations[role], + configurationId, + ctx.supervisor.configurations[role].normal, + ); + const current = status(ctx); + current.roles.find((v) => v.role === role)!.configurationId = + configurationId; + expect(() => verifyStatus(ctx, current)).toThrow( + "supervisor_role_configuration", + ); + } + } +}); + +test("command waits for retained same-group descendants after leader exit", async () => { + const marker = join(root, "command-descendant"); + const descendant = + "process.on('SIGTERM',()=>{});require('node:fs').writeFileSync(process.argv[1],JSON.stringify({group:Number(process.argv[2]),child:process.pid}),{mode:384});setInterval(()=>{},1000)"; + const leader = + "const fs=require('node:fs');const child=require('node:child_process').spawn(process.execPath,['-e',process.argv[1],process.argv[2],String(process.pid)],{stdio:'ignore'});child.unref();const timer=setInterval(()=>{if(fs.existsSync(process.argv[2])){clearInterval(timer);process.exit(0)}},5)"; + let group: number | undefined; + try { + await command( + node, + ["-e", leader, descendant, marker], + root, + childEnvironment(root), + ); + const receipt = JSON.parse((await readOwned(marker)).toString()); + group = receipt.group; + expect(Number.isSafeInteger(group) && Number(group) > 1).toBe(true); + expect(() => process.kill(-Number(group), 0)).toThrow("ESRCH"); + } finally { + if (group === undefined) { + try { + group = JSON.parse((await readOwned(marker)).toString()).group; + } catch {} + } + if (Number.isSafeInteger(group) && Number(group) > 1) { + try { + process.kill(-Number(group), "SIGKILL"); + } catch (error) { + if ( + !(error instanceof Error && "code" in error && error.code === "ESRCH") + ) + throw error; + } + const deadline = performance.now() + 1500; + let gone = false; + while (performance.now() < deadline) { + try { + process.kill(-Number(group), 0); + } catch (error) { + if ( + error instanceof Error && + "code" in error && + error.code === "ESRCH" + ) { + gone = true; + break; + } + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + if (!gone) throw new Error("controlled descendant cleanup failed"); + } + } +}); + +test("FC executable prerequisites reject invalid headers, architecture, platform and mode", async () => { + const elf = Buffer.alloc(64); + Buffer.from([0x7f, 69, 76, 70, 2, 1]).copy(elf); + elf.writeUInt16LE(183, 18); + const valid = await put(join(root, "fc-valid"), elf, true); + const inputs = input(); + inputs.backend = "firecracker"; + for (const name of BINARIES) inputs.binaries[name] = valid; + inputs.firecracker = valid; + inputs.proxy = valid; + await validateExecutables(inputs); + const wrongArchitecture = Buffer.from(elf); + wrongArchitecture.writeUInt16LE(62, 18); + const wrongPlatform = Buffer.alloc(64); + wrongPlatform.writeUInt32LE(0xfeedfacf); + wrongPlatform.writeUInt32LE(0x0100000c, 4); + const rejected: boolean[] = []; + for (const role of ["firecracker", "proxy"] as const) { + for (const [kind, bytes, executable] of [ + ["text", Buffer.from("correctly hashed text is not an executable"), true], + ["architecture", wrongArchitecture, true], + ["platform", wrongPlatform, true], + ["mode", elf, false], + ] as const) { + const pin = await put( + join(root, `fc-${role}-${kind}`), + bytes, + executable, + ); + await verifyPin(pin); + inputs[role] = pin; + rejected.push( + await validateExecutables(inputs).then( + () => false, + () => true, + ), + ); + inputs[role] = valid; + } + } + expect(rejected).toEqual(Array(8).fill(true)); +}); + +test("FC thin device contract uses kernel numbers and process birth", async () => { + const { linuxDevice, procBirth, FcRuntime } = await import("./fc-runtime.ts"); + expect(linuxDevice(11008n)).toBe("43:0"); + const fields = ["S", ...Array.from({ length: 18 }, () => "0"), "123456", "0"]; + expect(procBirth(`42 (a name ) with spaces) ${fields.join(" ")}`, 42)).toBe( + "123456", + ); + expect(() => procBirth(`41 (name) ${fields.join(" ")}`, 42)).toThrow(); + const inventory = { + bootId: "11111111-1111-4111-8111-111111111111", + release: "fixed", + pidNamespace: "pid:[1]", + pool: poolFixture(), + }; + const runtime = new FcRuntime( + inventory, + {} as import("./preflight.ts").Inputs, + "/owned/run", + ); + expect(runtime.deviceOptions()).toContain("b 253:* rwm"); + expect(runtime.deviceOptions()).toContain( + "/dev/mapper/control:/dev/mapper/control:rw", + ); + expect(runtime.environment()).toEqual({ ENGRAM_THIN_POOL: "suite" }); +}); + +test("guest enable accepts an unobserved queued digest and verifies the ready image", async () => { + const { create } = + await import("../../../../orchestrator/node_modules/@bufbuild/protobuf/dist/esm/index.js"); + const { + EnableImageResponseSchema, + GetEnableJobResponseSchema, + ListEnabledImagesResponseSchema, + } = + await import("../../../../orchestrator/src/gen/engram/app/v1/image_pb.ts"); + const { GetHostResponseSchema } = + await import("../../../../orchestrator/src/gen/engram/app/v1/fleet_pb.ts"); + const { WhoAmIResponseSchema } = + await import("../../../../orchestrator/src/gen/engram/app/v1/api_key_pb.ts"); + const transport = + await import("../../../../orchestrator/scripts/mcp-stack/transport.ts"); + const { enableGuest } = await import("./main.ts"); + const directory = await mkdtemp(join(root, "enable-identity-")); + const inputs = input(); + const opensslPath = await realpath( + process.env.MCP_PLATFORM_TEST_OPENSSL ?? Bun.which("openssl") ?? "openssl", + ); + inputs.binaries.openssl = { + path: opensslPath, + sha256: hash(await readFile(opensslPath)), + }; + const identity = await generateIdentity(inputs, directory); + const expectedDigest = `sha256:${"d".repeat(64)}`; + const wrongDigest = `sha256:${"e".repeat(64)}`; + const uri = "registry.fixture.example/guest:fixture"; + for (const scenario of [ + { + name: "unobserved-ready", + id: "job-1", + queuedDigest: undefined, + readyDigest: undefined, + host: "correct", + catalogDigest: expectedDigest, + error: undefined, + }, + { + name: "unobserved", + id: "job-1", + queuedDigest: undefined, + readyDigest: expectedDigest, + host: "correct", + catalogDigest: expectedDigest, + error: undefined, + }, + { + name: "wrong-ready", + id: "job-1", + queuedDigest: undefined, + readyDigest: wrongDigest, + host: "correct", + catalogDigest: expectedDigest, + error: "enable_ready_manifest", + }, + { + name: "missing-id", + id: "", + queuedDigest: undefined, + readyDigest: expectedDigest, + host: "correct", + catalogDigest: expectedDigest, + error: "enable_manifest", + }, + { + name: "wrong-queued", + id: "job-1", + queuedDigest: wrongDigest, + readyDigest: expectedDigest, + host: "correct", + catalogDigest: expectedDigest, + error: "enable_manifest", + }, + { + name: "unobserved-ready-wrong-catalog", + id: "job-1", + queuedDigest: undefined, + readyDigest: undefined, + host: "correct", + catalogDigest: wrongDigest, + error: "enabled_image_identity", + }, + { + name: "wrong-materialization-host", + id: "job-1", + queuedDigest: undefined, + readyDigest: expectedDigest, + host: "wrong", + catalogDigest: expectedDigest, + error: "enable_actual_host", + }, + { + name: "missing-materialization-host", + id: "job-1", + queuedDigest: undefined, + readyDigest: expectedDigest, + host: "missing", + catalogDigest: expectedDigest, + error: "enable_actual_host", + }, + ]) { + const ctx = await context(); + ctx.runRoot = dirname(ctx.endpoints.app.ca.path); + const admin = await transport.publicClient(ctx, { + kind: "cookie", + value: "fixture=cookie", + }); + const responses = [ + create(GetEnableJobResponseSchema, { + job: { id: "job-1", state: "materializing" }, + }), + create(GetEnableJobResponseSchema, { + job: { + id: "job-1", + state: "ready", + manifestDigest: scenario.readyDigest, + materializeHostId: + scenario.host === "correct" + ? ctx.hostId + : scenario.host === "wrong" + ? "00000000-0000-4000-8000-000000000002" + : undefined, + }, + }), + ]; + const spies = [ + spyOn(transport, "jsonRequest").mockResolvedValue({ + status: 200, + headers: { "set-cookie": ["fixture=cookie; HttpOnly"] }, + body: {}, + }), + spyOn(transport, "publicClient").mockResolvedValue(admin), + spyOn(admin.identity, "whoAmI").mockResolvedValue( + create(WhoAmIResponseSchema, { + email: "bootstrap@fixture.example", + role: "admin", + userId: "actor-1", + }), + ), + spyOn(admin.fleet, "getHost").mockResolvedValue( + create(GetHostResponseSchema, { + host: { + id: ctx.hostId, + status: "ready", + capacityTotalMib: 512n, + readyImageDigests: [expectedDigest], + }, + }), + ), + spyOn(admin.images, "enableImage").mockResolvedValue( + create(EnableImageResponseSchema, { + job: { + id: scenario.id, + state: "pending", + manifestDigest: scenario.queuedDigest, + }, + }), + ), + spyOn(admin.images, "getEnableJob").mockImplementation(async () => { + const response = responses.shift(); + if (!response) throw new Error("unexpected enable poll"); + return response; + }), + spyOn(admin.images, "listEnabledImages").mockResolvedValue( + create(ListEnabledImagesResponseSchema, { + images: [ + { + id: "enabled-1", + imageUri: uri, + manifestDigest: scenario.catalogDigest, + }, + ], + }), + ), + ]; + try { + const enabling = enableGuest(ctx, identity, uri, expectedDigest); + if (scenario.error) { + await expect(enabling, scenario.name).rejects.toThrow(scenario.error); + await expect( + readFile(join(ctx.runRoot, "enable-observation.json")), + ).rejects.toThrow(); + } else { + await enabling; + expect( + JSON.parse( + await readFile( + join(ctx.runRoot, "enable-observation.json"), + "utf8", + ), + ), + ).toMatchObject({ + jobId: "job-1", + imageId: "enabled-1", + imageDigest: expectedDigest, + hostId: ctx.hostId, + state: "ready", + }); + expect(ctx.imageId).toBe("enabled-1"); + expect(ctx.imageDigest).toBe(expectedDigest); + } + } finally { + for (const spy of spies) spy.mockRestore(); + } + } +}); + +test("platform validates stage selectors before resource creation and defaults to all", async () => { + const base = [ + "--backend", + "vz", + "--run-root", + "/private/run", + "--artifacts", + "/private/input", + ]; + expect(cli(base).selection).toEqual({ stage: "matrix", matrix: "all" }); + expect(cli([...base, "--stage", "infrastructure"]).selection).toEqual({ + stage: "infrastructure", + }); + const { REQUIRED_CASES } = + await import("../../../../orchestrator/scripts/mcp-stack/manifest.ts"); + expect(cli([...base, "--matrix", REQUIRED_CASES[0]!.key]).selection).toEqual({ + stage: "matrix", + matrix: REQUIRED_CASES[0]!.key, + }); + for (const extra of [ + ["--stage", "unknown"], + ["--matrix", ""], + ["--matrix", "unknown"], + ["--matrix", `${REQUIRED_CASES[0]!.key},${REQUIRED_CASES[0]!.key}`], + ["--stage", "infrastructure", "--matrix", "all"], + ]) + expect(() => cli([...base, ...extra])).toThrow(); +}); + +test("FC inputs require a strict owned thin pool and VZ refuses that field", async () => { + const vz = input(); + const fc = { + ...vz, + backend: "firecracker", + architecture: "amd64", + nativeProbe: undefined, + firecracker: vz.proxy, + images: { ...vz.images, hostAgent: vz.images.guest }, + thinPool: { name: "suite", directory: root }, + }; + expect(parseInputs(fc, "firecracker").thinPool).toEqual(fc.thinPool); + for (const thinPool of [ + undefined, + null, + {}, + { ...fc.thinPool, extra: true }, + { ...fc.thinPool, name: "a".repeat(128) }, + { ...fc.thinPool, name: "../pool" }, + { ...fc.thinPool, name: "pool;true" }, + { ...fc.thinPool, directory: root + "/.." }, + ]) + expect(() => parseInputs({ ...fc, thinPool }, "firecracker")).toThrow(); + expect(() => parseInputs({ ...vz, thinPool: fc.thinPool }, "vz")).toThrow(); + expect(() => parseInputs({ ...vz, thinPool: undefined }, "vz")).toThrow(); + const dir = await mkdtemp(join(root, "pool-mode-")); + await chmod(dir, 0o755); + expect(() => + parseInputs( + { ...fc, thinPool: { name: "suite", directory: dir } }, + "firecracker", + ), + ).toThrow("fc_pool_directory_owner_mode"); +}); + +test("FC infrastructure checkpoint has no caller-selected facts and invokes the physical check", async () => { + const { Supervisor } = + await import("../../../../orchestrator/scripts/mcp-stack/supervisor.ts"); + const ctx = await context(); + ctx.backend = "firecracker"; + const base = { + version: 1, + runId: ctx.runId, + operation: "infrastructure-ready", + } as const; + expect(parseOperation(base, ctx)).toEqual(base); + for (const extra of [ + { role: "host-agent" }, + { PASS: true }, + { pool: "foreign" }, + { configurationId: "normal" }, + ]) + expect(() => parseOperation({ ...base, ...extra }, ctx)).toThrow(); + for (const fail of [false, true]) { + let inspected = 0; + const server = await startSupervisor(ctx, { + status: async () => ({ + ...status(ctx), + roles: status(ctx).roles.map((r) => + r.role === "host-agent" && inspected && !fail + ? { ...r, bootId: "host-restarted" } + : r, + ), + }), + restart: async () => {}, + stop: async () => {}, + infrastructureReady: async () => { + inspected++; + if (fail) throw Error("physical check failed"); + }, + }); + try { + const call = new Supervisor(ctx).infrastructureReady(); + if (fail) await expect(call).rejects.toThrow(); + else { + const after = await call; + expect(after.backend).toBe("firecracker"); + expect(after.roles.find((r) => r.role === "host-agent")?.bootId).toBe( + "host-restarted", + ); + } + expect(inspected).toBe(1); + } finally { + await server.close(); + } + } +}); + +for (const known of [true, false]) + test(`FC infrastructure diagnostics retain only fixed release seams (${known ? "known" : "unknown"})`, async () => { + const { Supervisor } = + await import("../../../../orchestrator/scripts/mcp-stack/supervisor.ts"); + const ctx = await context(); + ctx.backend = "firecracker"; + ctx.runRoot = dirname(ctx.supervisor.tokenFile); + ctx.supervisor.socket = join(ctx.runRoot, "s"); + const seam = known + ? "fc_unreleased_volume" + : "fc_unreleased_volume_private_credential"; + let inspected = 0; + const server = await startSupervisor(ctx, { + status: async () => status(ctx), + restart: async () => {}, + stop: async () => {}, + infrastructureReady: async () => { + inspected++; + throw new PlatformFailure("INVALID_CONTEXT", seam); + }, + }); + try { + await expect(new Supervisor(ctx).infrastructureReady()).rejects.toThrow( + "TRANSPORT_FAILED:supervisor", + ); + expect(inspected).toBe(1); + const diagnostic = await readFile( + join(ctx.runRoot, "supervisor-failure-1.json"), + "utf8", + ); + expect(JSON.parse(diagnostic)).toEqual({ + version: 1, + operation: "infrastructure-ready", + phase: "lifecycle", + failure: known ? "INVALID_CONTEXT" : "OTHER", + seam: known ? "fc_unreleased_volume" : "other", + }); + expect(diagnostic).not.toContain("private_credential"); + } finally { + await server.close(); + } + }); + +for (const operation of ["prepare-disk-read", "verify-disk-read"] as const) + test(`FC disk read control is fixed, authenticated and preserves roles (${operation})`, async () => { + const { Supervisor } = + await import("../../../../orchestrator/scripts/mcp-stack/supervisor.ts"); + const ctx = await context(); + const body = { version: 1, runId: ctx.runId, operation } as const; + expect(() => parseOperation(body, ctx)).toThrow(); + ctx.backend = "firecracker"; + expect(parseOperation(body, ctx)).toEqual(body); + for (const extra of [ + { path: "/dev/foreign" }, + { pid: 1 }, + { role: "host-agent" }, + ]) + expect(() => parseOperation({ ...body, ...extra }, ctx)).toThrow(); + let prepared = 0; + let verified = 0; + const server = await startSupervisor(ctx, { + status: async () => status(ctx), + restart: async () => { + throw Error("unexpected restart"); + }, + stop: async () => { + throw Error("unexpected stop"); + }, + prepareDiskRead: async () => { + prepared++; + }, + verifyDiskRead: async () => { + verified++; + }, + }); + try { + const client = new Supervisor(ctx); + if (operation === "prepare-disk-read") await client.prepareDiskRead(); + else await client.verifyDiskRead(); + expect([prepared, verified]).toEqual( + operation === "prepare-disk-read" ? [1, 0] : [0, 1], + ); + } finally { + await server.close(); + } + }); + +test("FC disk read control rejects an unrelated service restart", async () => { + const { Supervisor } = + await import("../../../../orchestrator/scripts/mcp-stack/supervisor.ts"); + const ctx = await context(); + ctx.backend = "firecracker"; + let changed = false; + const server = await startSupervisor(ctx, { + status: async () => { + const current = status(ctx); + if (changed) + current.roles.find((r) => r.role === "worker")!.bootId = + "foreign-restart"; + return current; + }, + prepareDiskRead: async () => { + changed = true; + }, + restart: async () => {}, + stop: async () => {}, + }); + try { + await expect(new Supervisor(ctx).prepareDiskRead()).rejects.toThrow( + "TRANSPORT_FAILED:supervisor", + ); + } finally { + await server.close(); + } +}); + +for (const change of ["missing-host-restart", "other-role", "network"] as const) + test(`FC infrastructure checkpoint rejects an invalid restart transition (${change})`, async () => { + const { Supervisor } = + await import("../../../../orchestrator/scripts/mcp-stack/supervisor.ts"); + const ctx = await context(); + ctx.backend = "firecracker"; + let inspected = false; + const server = await startSupervisor(ctx, { + status: async () => { + const current = status(ctx); + if (inspected) { + if (change !== "missing-host-restart") + current.roles.find((r) => r.role === "host-agent")!.bootId = + "host-restarted"; + if (change === "other-role") + current.roles.find((r) => r.role === "worker")!.bootId = + "worker-restarted"; + if (change === "network") + current.network.generation = "changed-network"; + } + return current; + }, + restart: async () => {}, + stop: async () => {}, + infrastructureReady: async () => { + inspected = true; + }, + }); + try { + await expect(new Supervisor(ctx).infrastructureReady()).rejects.toThrow(); + } finally { + await server.close(); + } + }); diff --git a/tests/fixtures/mcp-stack/platform/preflight.ts b/tests/fixtures/mcp-stack/platform/preflight.ts new file mode 100644 index 000000000..44ac359e0 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/preflight.ts @@ -0,0 +1,550 @@ +import { + FULL_MATRIX, + selectedCases, + type RunSelection, +} from "../../../../orchestrator/scripts/mcp-stack/manifest.ts"; +import { readFile, access, lstat, open, readdir } from "node:fs/promises"; +import { constants } from "node:fs"; +import { arch, platform } from "node:os"; +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + check, + childEnvironment, + command, + ownedPath, + PlatformFailure, + privateRoot, + readOwned, + record, + verifyPin, + type Pin, +} from "./owned.ts"; + +export const REPO = resolve( + dirname(fileURLToPath(import.meta.url)), + "../../../..", +); +export const BINARIES = [ + "node", + "bun", + "docker", + "openssl", + "hostAgent", + "kernel", + "helper", + "claudeHarness", + "codexHarness", + "modelCli", + "agentdBundle", + "sentinelBundle", + "guestToolsBundle", + "guestInit", +] as const; +export const IMAGES = [ + "postgres", + "blob", + "registry", + "dns", + "edge", + "guard", + "coordinator", + "orchestrator", + "worker", + "fixture", + "guest", +] as const; +export interface ImagePin { + archive: Pin; + id: string; + manifestDigest: string; +} +import { parseThinPool, type ThinPoolInput } from "./fc-thin.ts"; +export interface Inputs { + version: 1; + backend: "firecracker" | "vz"; + architecture: "arm64" | "amd64"; + sourceCommit: string; + engineSocket: string; + engineConfig: string; + engineId: string; + registryPort: number; + subnet: string; + nativeHostIp: string; + binaries: Record<(typeof BINARIES)[number], Pin>; + images: Record<(typeof IMAGES)[number], ImagePin> & { hostAgent?: ImagePin }; + firecracker?: Pin; + thinPool?: ThinPoolInput; + proxy: Pin; + nativeProbe?: Pin; +} +function text(value: unknown, pattern: RegExp, seam: string): string { + check(typeof value === "string" && pattern.test(value), seam); + return value; +} +function pin(value: unknown): Pin { + const p = record(value, ["path", "sha256"]); + const path = text(p.path, /^\//, "artifact_path"); + check(isAbsolute(path) && resolve(path) === path, "artifact_path"); + return { path, sha256: text(p.sha256, /^[a-f0-9]{64}$/, "artifact_hash") }; +} +export function parseInputs( + value: unknown, + backend: Inputs["backend"], +): Inputs { + const o = record( + value, + [ + "version", + "backend", + "architecture", + "sourceCommit", + "engineSocket", + "engineConfig", + "engineId", + "registryPort", + "subnet", + "nativeHostIp", + "binaries", + "images", + "proxy", + ], + ["firecracker", "nativeProbe", "thinPool"], + ); + check( + o.version === 1 && + o.backend === backend && + ["arm64", "amd64"].includes(String(o.architecture)), + "platform_identity", + ); + if (backend === "vz") + check( + o.architecture === "arm64" && + !o.firecracker && + o.nativeProbe && + !("thinPool" in o), + "vz_architecture", + ); + else { + check(o.firecracker && !o.nativeProbe, "fc_artifacts"); + check(o.thinPool, "fc_pool_required"); + } + check( + Number.isInteger(o.registryPort) && + Number(o.registryPort) >= 20000 && + Number(o.registryPort) <= 60000, + "registry_port", + ); + const rawBins = record(o.binaries, BINARIES), + rawImages = record( + o.images, + backend === "firecracker" ? [...IMAGES, "hostAgent"] : IMAGES, + ); + const binaries = Object.fromEntries( + BINARIES.map((key) => [key, pin(rawBins[key])]), + ) as Inputs["binaries"]; + const images = Object.fromEntries( + (backend === "firecracker" ? [...IMAGES, "hostAgent"] : [...IMAGES]).map( + (key) => { + const image = record(rawImages[key], [ + "archive", + "id", + "manifestDigest", + ]); + return [ + key, + { + archive: pin(image.archive), + id: text(image.id, /^sha256:[a-f0-9]{64}$/, "image_digest"), + manifestDigest: text( + image.manifestDigest, + /^sha256:[a-f0-9]{64}$/, + "image_manifest_digest", + ), + }, + ]; + }, + ), + ) as Inputs["images"]; + const subnet = text( + o.subnet, + /^10\.(?:[1-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-4])\.(?:0|[1-9][0-9]?|1[0-9]{2}|2[0-4][0-9]|25[0-4])\.0\/24$/, + "private_subnet", + ); + const nativeHostIp = text( + o.nativeHostIp, + /^(?:10|172|192)\.(?:[0-9]{1,3}\.){2}[0-9]{1,3}$/, + "native_host_ip", + ); + const octets = nativeHostIp.split(".").map(Number); + check( + octets.join(".") === nativeHostIp && + octets.every((v) => v < 256) && + (octets[0] === 10 || + (octets[0] === 172 && octets[1]! >= 16 && octets[1]! <= 31) || + (octets[0] === 192 && octets[1] === 168)), + "native_host_ip", + ); + check(!nativeHostIp.startsWith(subnet.slice(0, -4)), "separate_host_route"); + return { + version: 1, + backend, + architecture: o.architecture as Inputs["architecture"], + sourceCommit: text(o.sourceCommit, /^[a-f0-9]{40}$/, "source_commit"), + engineSocket: text(o.engineSocket, /^\//, "engine_socket"), + engineConfig: text(o.engineConfig, /^\//, "engine_config"), + engineId: text(o.engineId, /^[A-Za-z0-9:_-]{12,128}$/, "engine_identity"), + registryPort: Number(o.registryPort), + subnet, + nativeHostIp, + binaries, + images, + proxy: pin(o.proxy), + ...(backend === "firecracker" + ? { firecracker: pin(o.firecracker), thinPool: parseThinPool(o.thinPool) } + : { nativeProbe: pin(o.nativeProbe) }), + }; +} +export function engineArguments( + inputs: Pick, + image?: string, +): string[] { + for (const path of [inputs.engineSocket, inputs.engineConfig]) + check(isAbsolute(path) && resolve(path) === path, "engine_path"); + if (image !== undefined) + check(/^sha256:[a-f0-9]{64}$/.test(image), "image_digest"); + return [ + "--config", + inputs.engineConfig, + "--host", + `unix://${inputs.engineSocket}`, + ...(image === undefined + ? ["info", "--format", "{{json .}}"] + : ["image", "inspect", image]), + ]; +} +export async function validateEngineConfig( + path: string, + expected?: { ino: number; dev: number; mtimeMs: number; ctimeMs: number }, +) { + await privateRoot(path); + const state = await lstat(path); + check((await readdir(path)).length === 0, "engine_config_not_empty"); + if (expected) + check( + state.ino === expected.ino && + state.dev === expected.dev && + state.mtimeMs === expected.mtimeMs && + state.ctimeMs === expected.ctimeMs, + "engine_config_changed", + ); + return { + ino: state.ino, + dev: state.dev, + mtimeMs: state.mtimeMs, + ctimeMs: state.ctimeMs, + }; +} +export function cli(args: string[]): { + backend: Inputs["backend"]; + runRoot: string; + manifest: string; + selection: RunSelection; +} { + check( + [6, 8].includes(args.length) && + args[0] === "--backend" && + ["firecracker", "vz"].includes(args[1]!) && + args[2] === "--run-root" && + args[4] === "--artifacts", + "arguments", + ); + for (const p of [args[3]!, args[5]!]) + check(isAbsolute(p) && resolve(p) === p, "arguments_path"); + let selection: RunSelection = FULL_MATRIX; + if (args.length === 8) { + check( + args[6] === "--matrix" || + (args[6] === "--stage" && args[7] === "infrastructure"), + "arguments", + ); + selection = + args[6] === "--stage" + ? { stage: "infrastructure" } + : { stage: "matrix", matrix: args[7]! }; + } + selectedCases(selection); + return { + backend: args[1] as Inputs["backend"], + runRoot: args[3]!, + manifest: args[5]!, + selection, + }; +} +export function requirePlatform( + backend: Inputs["backend"], + architecture: Inputs["architecture"], + os = platform(), + cpu = arch(), +): void { + if ( + os !== (backend === "vz" ? "darwin" : "linux") || + cpu !== (architecture === "amd64" ? "x64" : "arm64") + ) + throw new PlatformFailure("NEEDS_CONTEXT", "target_platform"); +} +export function binaryArchitecture(bytes: Uint8Array): { + os: "linux" | "darwin"; + architecture: Inputs["architecture"]; +} { + const data = Buffer.from(bytes); + check(data.length >= 64, "binary_header"); + if (data.subarray(0, 4).equals(Buffer.from([0x7f, 69, 76, 70]))) { + check(data[4] === 2 && data[5] === 1, "elf_format"); + const machine = data.readUInt16LE(18); + check(machine === 62 || machine === 183, "elf_architecture"); + return { os: "linux", architecture: machine === 62 ? "amd64" : "arm64" }; + } + check( + data.readUInt32LE(0) === 0xfeedfacf && data.readUInt32LE(4) === 0x0100000c, + "macho_architecture", + ); + return { os: "darwin", architecture: "arm64" }; +} +async function header(path: string): Promise { + const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const data = Buffer.alloc(1024); + const { bytesRead } = await file.read(data, 0, data.length, 0); + return data.subarray(0, bytesRead); + } finally { + await file.close(); + } +} +export async function validateExecutables(inputs: Inputs): Promise { + for (const name of ["node", "bun", "docker", "openssl"] as const) + await access(inputs.binaries[name].path, constants.X_OK); + const validate = async (pin: Pin, os: "linux" | "darwin") => { + await access(pin.path, constants.X_OK); + const actual = binaryArchitecture(await header(pin.path)); + check( + actual.architecture === inputs.architecture && actual.os === os, + "binary_platform", + ); + }; + for (const role of [ + "hostAgent", + "helper", + "claudeHarness", + "codexHarness", + "modelCli", + ] as const) { + const os = + ["hostAgent", "helper"].includes(role) && inputs.backend === "vz" + ? "darwin" + : "linux"; + await validate(inputs.binaries[role], os); + } + if (inputs.backend === "firecracker") { + check(inputs.firecracker && inputs.proxy, "fc_artifacts"); + await validate(inputs.firecracker, "linux"); + } + await validate(inputs.proxy, inputs.backend === "vz" ? "darwin" : "linux"); + if (inputs.backend === "vz") { + check(inputs.nativeProbe, "native_probe"); + await validate(inputs.nativeProbe, "darwin"); + } +} +export async function validateVzEntitlement( + hostAgent: string, + env: NodeJS.ProcessEnv, + runCommand: typeof command = command, +): Promise { + await runCommand( + "/usr/bin/codesign", + ["--verify", "--strict", hostAgent], + REPO, + env, + ); + const entitlements = await runCommand( + "/usr/bin/codesign", + ["--display", "--entitlements", "-", "--xml", hostAgent], + REPO, + env, + ); + const converted = await runCommand( + "/usr/bin/plutil", + ["-convert", "json", "-o", "-", "--", "-"], + REPO, + env, + entitlements.toString("utf8"), + ); + let value: unknown; + try { + value = JSON.parse(converted.toString("utf8")); + } catch { + throw new PlatformFailure("INVALID_CONTEXT", "vz_entitlement"); + } + check( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + "com.apple.security.virtualization" in value && + value["com.apple.security.virtualization"] === true, + "vz_entitlement", + ); +} +export async function preflight( + backend: Inputs["backend"], + runRoot: string, + filename: string, +): Promise { + const inputs = parseInputs( + JSON.parse((await readOwned(filename)).toString("utf8")), + backend, + ); + requirePlatform(backend, inputs.architecture); + await privateRoot(dirname(runRoot)); + await ownedPath(runRoot, true); + // Never mutate an engine or create the output root when the complete matrix is absent. + try { + await access( + join(REPO, "orchestrator/scripts/mcp-stack/run.ts"), + constants.R_OK, + ); + } catch { + throw new PlatformFailure("NEEDS_CONTEXT", "common_runner"); + } + for (const item of [ + ...Object.values(inputs.binaries), + ...Object.values(inputs.images).map((v) => v.archive), + ...(inputs.firecracker ? [inputs.firecracker] : []), + inputs.proxy, + ...(inputs.nativeProbe ? [inputs.nativeProbe] : []), + ]) + await verifyPin(item); + await validateExecutables(inputs); + const kernel = await header(inputs.binaries.kernel.path); + if (inputs.architecture === "arm64") + check( + kernel.length >= 64 && kernel.readUInt32LE(56) === 0x644d5241, + "arm64_kernel", + ); + else check(binaryArchitecture(kernel).architecture === "amd64", "fc_kernel"); + const env = childEnvironment(dirname(runRoot)); + const head = ( + await command("/usr/bin/git", ["-C", REPO, "rev-parse", "HEAD"], REPO, env) + ) + .toString() + .trim(); + check(head === inputs.sourceCommit, "source_commit"); + const dirty = await command( + "/usr/bin/git", + [ + "--no-optional-locks", + "-C", + REPO, + "status", + "--porcelain", + "--untracked-files=no", + ], + REPO, + env, + ); + check(!dirty.length, "source_dirty"); + check( + (await command(inputs.binaries.node.path, ["--version"], REPO, env)) + .toString() + .startsWith("v24."), + "node24", + ); + check( + (await command(inputs.binaries.openssl.path, ["version"], REPO, env)) + .toString() + .startsWith("OpenSSL 3."), + "openssl3", + ); + await ownedPath(inputs.engineSocket); + check((await lstat(inputs.engineSocket)).isSocket(), "engine_socket"); + const engineState = await lstat(inputs.engineSocket); + const configState = await validateEngineConfig(inputs.engineConfig); + const inspect = async (image?: string) => { + await validateEngineConfig(inputs.engineConfig, configState); + await ownedPath(inputs.engineSocket); + const socket = await lstat(inputs.engineSocket); + check( + socket.isSocket() && + socket.ino === engineState.ino && + socket.dev === engineState.dev, + "engine_socket_changed", + ); + const bytes = await command( + inputs.binaries.docker.path, + engineArguments(inputs, image), + REPO, + childEnvironment(dirname(runRoot)), + ); + await validateEngineConfig(inputs.engineConfig, configState); + return bytes; + }; + const info = JSON.parse((await inspect()).toString()); + check( + info.ID === inputs.engineId && + info.OSType === "linux" && + [ + inputs.architecture, + inputs.architecture === "amd64" ? "x86_64" : "aarch64", + ].includes(info.Architecture), + "engine_platform", + ); + for (const [role, image] of Object.entries(inputs.images)) { + const [actual] = JSON.parse((await inspect(image.id)).toString()); + check( + actual?.Id === image.id && + actual?.Os === "linux" && + actual?.Architecture === inputs.architecture, + "image_platform", + ); + if ( + [ + "coordinator", + "orchestrator", + "worker", + "fixture", + "hostAgent", + ].includes(role) + ) + check( + actual?.Config?.Labels?.["org.opencontainers.image.revision"] === + inputs.sourceCommit, + "image_source", + ); + } + if (backend === "firecracker") { + check( + info.KernelVersion === + (await readFile("/proc/sys/kernel/osrelease", "utf8")).trim(), + "fc_engine_kernel_release", + ); + if (process.getuid?.() !== 0) + throw new PlatformFailure("NEEDS_CONTEXT", "fc_privilege"); + try { + await access("/dev/kvm", constants.R_OK | constants.W_OK); + check((await lstat("/dev/kvm")).isCharacterDevice(), "kvm_device"); + await access("/dev/net/tun", constants.R_OK | constants.W_OK); + } catch { + throw new PlatformFailure("NEEDS_CONTEXT", "linux_kvm_tun"); + } + const { inspectFcInventory } = await import("./fc-runtime.ts"); + await inspectFcInventory(inputs); + } else { + await validateVzEntitlement(inputs.binaries.hostAgent.path, env); + check( + (await command("/usr/sbin/sysctl", ["-n", "kern.hv_support"], REPO, env)) + .toString() + .trim() === "1", + "vz_hypervisor", + ); + } + return inputs; +} diff --git a/tests/fixtures/mcp-stack/platform/supervisor.ts b/tests/fixtures/mcp-stack/platform/supervisor.ts new file mode 100644 index 000000000..0222ad592 --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/supervisor.ts @@ -0,0 +1,649 @@ +import { + createServer, + type IncomingMessage, + type ServerResponse, +} from "node:http"; +import { constants } from "node:fs"; +import { chmod, lstat, open, unlink } from "node:fs/promises"; +import { dirname, join, sep } from "node:path"; +import { + ROLES, + StatusSchema, + type MutableRole, + type SupervisorStatus, +} from "../../../../orchestrator/scripts/mcp-stack/supervisor.ts"; +import type { + ConfigurationId, + Context, +} from "../../../../orchestrator/scripts/mcp-stack/context.ts"; +import { + check, + ownedPath, + PlatformFailure, + privateRoot, + readOwned, + record, + sameSecret, + verifyPin, +} from "./owned.ts"; + +export type Operation = { version: 1; runId: string } & ( + | { operation: "status" } + | { operation: "infrastructure-ready" } + | { operation: "prepare-disk-read" } + | { operation: "verify-disk-read" } + | { operation: "stop"; role: MutableRole } + | { + operation: "restart"; + role: MutableRole; + configurationId: ConfigurationId; + } +); +export function parseOperation(value: unknown, ctx: Context): Operation { + const base = record( + value, + ["version", "runId", "operation"], + ["role", "configurationId"], + ); + check(base.version === 1 && base.runId === ctx.runId, "supervisor_run"); + if ( + base.operation === "prepare-disk-read" || + base.operation === "verify-disk-read" + ) { + record(base, ["version", "runId", "operation"]); + check(ctx.backend === "firecracker", "supervisor_storage_checkpoint"); + return { version: 1, runId: ctx.runId, operation: base.operation }; + } + if ( + base.operation === "status" || + base.operation === "infrastructure-ready" + ) { + record(base, ["version", "runId", "operation"]); + return { version: 1, runId: ctx.runId, operation: base.operation }; + } + check( + base.operation === "restart" || base.operation === "stop", + "supervisor_operation", + ); + check( + base.role === "orchestrator" || + base.role === "worker" || + base.role === "coordinator" || + base.role === "host-agent", + "supervisor_role", + ); + const role = base.role; + if (base.operation === "stop") { + record(base, ["version", "runId", "operation", "role"]); + return { version: 1, runId: ctx.runId, operation: "stop", role }; + } + record(base, ["version", "runId", "operation", "role", "configurationId"]); + check( + base.configurationId === "normal" || + base.configurationId === "global-disabled" || + base.configurationId === "zone-disabled", + "supervisor_configuration", + ); + const configurationId = base.configurationId; + check( + role === "orchestrator" || configurationId === "normal", + "supervisor_role_configuration", + ); + check( + role === "orchestrator" + ? ctx.supervisor.configurations.orchestrator[configurationId] + : ctx.supervisor.configurations[role].normal, + "supervisor_configuration", + ); + return { + version: 1, + runId: ctx.runId, + operation: "restart", + role, + configurationId, + }; +} + +// Only this fixed lifecycle is callable. The adapter supplies owned handles, never request-defined commands. +export type Lifecycle = { + status(signal: AbortSignal): Promise; + infrastructureReady?(signal: AbortSignal): Promise; + prepareDiskRead?(signal: AbortSignal): Promise; + verifyDiskRead?(signal: AbortSignal): Promise; + restart( + role: MutableRole, + configuration: ConfigurationId, + signal: AbortSignal, + ): Promise; + stop(role: MutableRole, signal: AbortSignal): Promise; +}; +export function verifyStatus(ctx: Context, value: unknown): SupervisorStatus { + const status = StatusSchema.parse(value); + check( + status.runId === ctx.runId && + status.backend === ctx.backend && + status.sourceCommit === ctx.sourceCommit && + status.hostId === ctx.hostId, + "supervisor_identity", + ); + check( + new Set(status.roles.map((v) => v.role)).size === ROLES.length && + new Set(status.network.roles).size === 3, + "supervisor_roles", + ); + const artifacts = { + orchestrator: ctx.artifacts.orchestrator, + worker: ctx.artifacts.worker, + coordinator: ctx.artifacts.coordinator, + "host-agent": ctx.artifacts.hostAgent, + "fixture-modern": ctx.artifacts.fixture, + "fixture-legacy": ctx.artifacts.fixture, + }; + for (const role of status.roles) { + check( + role.role === "orchestrator" || role.configurationId === "normal", + "supervisor_role_configuration", + ); + const config = + role.role === "fixture-modern" || role.role === "fixture-legacy" + ? role.configurationId === "normal" + ? ctx.protocolFixtures[ + role.role === "fixture-modern" ? "modern" : "legacy" + ].configuration + : undefined + : role.role === "orchestrator" + ? ctx.supervisor.configurations.orchestrator[role.configurationId] + : ctx.supervisor.configurations[role.role].normal; + check( + config && + config.sha256 === role.configurationSha256 && + artifacts[role.role].sha256 === role.artifactSha256, + "supervisor_hash", + ); + } + return status; +} +export async function startSupervisor(ctx: Context, lifecycle: Lifecycle) { + await privateRoot(ctx.runRoot); + check( + ctx.supervisor.socket.startsWith(ctx.runRoot + sep) && + ctx.supervisor.tokenFile.startsWith(ctx.runRoot + sep), + "supervisor_scope", + ); + await privateRoot(dirname(ctx.supervisor.socket)); + await ownedPath(ctx.supervisor.socket, true); + const token = (await readOwned(ctx.supervisor.tokenFile, 256)) + .toString() + .trim(); + check(/^[A-Za-z0-9_-]{32,128}$/.test(token), "supervisor_token"); + let busy = false, + closing = false; + let active: + | { + controller: AbortController; + settled: Promise; + diagnosed: Promise; + } + | undefined; + const sockets = new Set(); + let failureNumber = 0; + const safeSeams = new Set([ + "role_readiness", + "engine_command", + "engine_deadline", + "container_stop", + "container_survivor", + "role_start", + "role_configuration", + "role_handle", + "role_identity", + "role_running", + "running_artifact", + "artifact_identity", + "artifact_hash", + "supervisor_configuration", + "roles_before_restart", + "restart_network", + "restart_scope", + "restart_identity", + "stop_identity", + "supervisor_identity", + "supervisor_roles", + "supervisor_hash", + "control_aborted", + "worker_readiness_status", + "worker_readiness_tls", + "worker_readiness_identity", + "orchestrator_readiness", + "coordinator_readiness", + // Fixed physical observation, release and stopped-pool diagnostics. + "fc_disk_read_not_prepared", + "fc_disk_read_pending", + "fc_disk_read_target", + "fc_disk_read_boots", + "fc_boot_device_changed", + "fc_boot_process_duplicate", + "fc_boot_volume_size", + "fc_device_node", + "fc_dm_inventory", + "fc_fd_identity", + "fc_fd_inventory_bound", + "fc_foreign_pool_holder", + "fc_foreign_thin_name", + "fc_foreign_thin_owner", + "fc_foreign_thin_pool", + "fc_host_cgroup", + "fc_host_device_projection", + "fc_host_effective_capabilities", + "fc_host_effective_confinement", + "fc_host_kernel_namespace", + "fc_host_nodes_mount", + "fc_host_nodes_projection", + "fc_host_process", + "fc_host_readonly_kernel_mount", + "fc_host_writable_proc_sys", + "fc_infrastructure_boot_evidence", + "fc_initial_kernel", + "fc_kernel_changed", + "fc_managed_boot_identity", + "fc_managed_boot_state", + "fc_managed_host_identity", + "fc_managed_process_arguments", + "fc_managed_process_arguments_changed", + "fc_managed_process_duplicate", + "fc_managed_record_bound", + "fc_metadata_host_start_blocked", + "fc_owned_process_survivor", + "fc_physical_boot_count", + "fc_physical_boot_unobserved", + "fc_physical_evidence_write", + "fc_pool_backing_identity", + "fc_pool_backing_owner", + "fc_pool_backing_path", + "fc_pool_backing_size", + "fc_pool_block_size", + "fc_pool_capacity", + "fc_pool_device", + "fc_pool_device_identity", + "fc_pool_directory", + "fc_pool_directory_owner_mode", + "fc_pool_directory_symlink", + "fc_pool_filesystem_reserve", + "fc_pool_health", + "fc_pool_host_not_quiescent", + "fc_pool_identity_changed", + "fc_pool_inspection_busy", + "fc_pool_loop_device", + "fc_pool_metadata_dump", + "fc_pool_metadata_ids", + "fc_pool_metadata_release_uncertain", + "fc_pool_name", + "fc_pool_name_identity", + "fc_pool_name_volume_budget", + "fc_pool_not_empty", + "fc_pool_overlap_capacity", + "fc_pool_required", + "fc_pool_sparse_backing", + "fc_pool_table", + "fc_process_arguments", + "fc_process_birth", + "fc_process_changed", + "fc_process_identity", + "fc_process_identity_changed", + "fc_process_inventory_bound", + "fc_restart_inactive_thin", + "fc_retained_volume_changed_or_busy", + "fc_retained_volume_replaced", + "fc_rootfs_device_changed", + "fc_rootfs_device_fd", + "fc_rootfs_device_fd_changed", + "fc_rootfs_not_thin", + "fc_rootfs_path", + "fc_rootfs_volume_name", + "fc_sandbox_manifest", + "fc_sandbox_process", + "fc_storage_fd_live", + "fc_storage_process_live", + "fc_thin_device_name", + "fc_thin_foreign_fd", + "fc_thin_io_counter", + "fc_thin_remaining_fd", + "fc_thin_remaining_holder", + "fc_thin_table", + "fc_unexplained_inactive_thin", + "fc_unobserved_process", + "fc_unreleased_device_node", + "fc_unreleased_volume", + ]); + function reply(res: ServerResponse, status: number, body: unknown) { + if (res.destroyed) return; + const bytes = JSON.stringify(body); + if (Buffer.byteLength(bytes) > 65536) { + res.destroy(); + return; + } + res.writeHead(status, { + "content-type": "application/json", + "cache-control": "no-store", + connection: "close", + }); + res.end(bytes); + } + async function serve(req: IncomingMessage, res: ServerResponse) { + if (closing) return reply(res, 503, { error: "closing" }); + if (req.method !== "POST" || req.url !== "/v1/control") + return reply(res, 404, { error: "route" }); + // Duplicate auth/length/type fields are refused before body parsing. + const names = req.rawHeaders + .filter((_v, i) => i % 2 === 0) + .map((v) => v.toLowerCase()); + if ( + names.filter((v) => v === "authorization").length !== 1 || + !sameSecret(req.headers.authorization ?? "", `Bearer ${token}`) + ) + return reply(res, 401, { error: "authentication" }); + if ( + names.filter((v) => v === "content-type").length !== 1 || + names.filter((v) => v === "content-length").length !== 1 || + req.headers["content-type"] !== "application/json" || + req.headers["transfer-encoding"] || + !/^[1-9][0-9]{0,3}$/.test(req.headers["content-length"] ?? "") || + Number(req.headers["content-length"]) > 4096 + ) + return reply(res, 400, { error: "request" }); + if (busy) return reply(res, 409, { error: "busy" }); + busy = true; + const controller = new AbortController(); + const { signal } = controller; + let settle!: () => void; + const settled = new Promise((resolve) => { + settle = resolve; + }); + let finishDiagnostic!: () => void; + const diagnosed = new Promise((resolve) => { + finishDiagnostic = resolve; + }); + active = { controller, settled, diagnosed }; + const aborted = new Promise((_resolve, reject) => { + signal.addEventListener( + "abort", + () => reject(new PlatformFailure("FAILED", "control_aborted")), + { once: true }, + ); + }); + const timer = setTimeout( + () => controller.abort("deadline"), + ctx.deadlines.requestMs, + ); + const disconnect = () => { + if (!res.writableEnded) controller.abort("disconnect"); + }; + res.once("close", disconnect); + let lifecycleStarted = false; + let operation: Operation | undefined; + let phase: + | "request" + | "configuration" + | "before_status" + | "lifecycle" + | "after_status" + | "transition" + | "reply" = "request"; + async function execute() { + const chunks: Buffer[] = []; + let length = 0; + for await (const chunk of req) { + const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + length += bytes.length; + if (length > 4096) + throw new PlatformFailure("INVALID_CONTEXT", "body_size"); + chunks.push(bytes); + } + check(length === Number(req.headers["content-length"]), "body_length"); + operation = parseOperation( + JSON.parse(Buffer.concat(chunks).toString("utf8")), + ctx, + ); + if (operation.operation === "restart") { + phase = "configuration"; + const configuration = + operation.role === "orchestrator" + ? ctx.supervisor.configurations.orchestrator[ + operation.configurationId + ] + : ctx.supervisor.configurations[operation.role].normal; + check(configuration, "supervisor_configuration"); + await verifyPin(configuration); + } + signal.throwIfAborted(); + lifecycleStarted = true; + phase = "before_status"; + const before = verifyStatus(ctx, await lifecycle.status(signal)); + signal.throwIfAborted(); + if (operation.operation === "status") { + phase = "reply"; + return reply(res, 200, before); + } + if (operation.operation === "infrastructure-ready") { + check( + ctx.backend === "firecracker" && lifecycle.infrastructureReady, + "supervisor_storage_checkpoint", + ); + check( + before.roles.every((r) => r.state === "running"), + "roles_before_restart", + ); + phase = "lifecycle"; + await lifecycle.infrastructureReady(signal); + } else if ( + operation.operation === "prepare-disk-read" || + operation.operation === "verify-disk-read" + ) { + const action = + operation.operation === "prepare-disk-read" + ? lifecycle.prepareDiskRead + : lifecycle.verifyDiskRead; + check( + ctx.backend === "firecracker" && action, + "supervisor_storage_checkpoint", + ); + check( + before.roles.every((r) => r.state === "running"), + "roles_before_restart", + ); + phase = "lifecycle"; + await action(signal); + } else if (operation.operation === "restart") { + check( + before.roles.every((r) => r.state === "running"), + "roles_before_restart", + ); + phase = "lifecycle"; + await lifecycle.restart( + operation.role, + operation.configurationId, + signal, + ); + } else { + phase = "lifecycle"; + await lifecycle.stop(operation.role, signal); + } + signal.throwIfAborted(); + phase = "after_status"; + const after = verifyStatus(ctx, await lifecycle.status(signal)); + signal.throwIfAborted(); + phase = "transition"; + check( + before.network.generation === after.network.generation && + before.network.enforcementSha256 === after.network.enforcementSha256, + "restart_network", + ); + for (const prior of before.roles) { + const current = after.roles.find((r) => r.role === prior.role)!; + const changedRole = + operation.operation === "infrastructure-ready" + ? "host-agent" + : operation.operation === "restart" || + operation.operation === "stop" + ? operation.role + : undefined; + if (current.role !== changedRole) + check( + current.bootId === prior.bootId && + current.state === prior.state && + current.configurationId === prior.configurationId && + current.configurationSha256 === prior.configurationSha256, + "restart_scope", + ); + else if (operation.operation !== "stop") + check( + current.state === "running" && + current.bootId !== prior.bootId && + current.configurationId === + (operation.operation === "restart" + ? operation.configurationId + : "normal"), + "restart_identity", + ); + else check(current.state === "stopped", "stop_identity"); + } + phase = "reply"; + reply(res, 200, after); + } + const execution = execute().finally(settle); + try { + await Promise.race([execution, aborted]); + } catch (error) { + // A timed-out or failed lifecycle can have changed state. Refuse further controls. + if (lifecycleStarted) closing = true; + if (signal.aborted) req.destroy(); + if (operation) { + const reason = + signal.aborted && + ["deadline", "disconnect", "shutdown"].includes(signal.reason) + ? (signal.reason as "deadline" | "disconnect" | "shutdown") + : undefined; + const known = + error instanceof PlatformFailure && + ["FAILED", "INVALID_CONTEXT", "NEEDS_CONTEXT"].includes(error.code) && + safeSeams.has(error.seam); + const diagnostic = { + version: 1, + operation: operation.operation, + ...("role" in operation ? { role: operation.role } : {}), + phase, + ...(reason + ? { failure: reason, seam: "control_aborted" } + : { + failure: known ? error.code : "OTHER", + seam: known ? error.seam : "other", + }), + }; + try { + const file = await open( + join(ctx.runRoot, `supervisor-failure-${++failureNumber}.json`), + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600, + ); + try { + await file.writeFile(JSON.stringify(diagnostic) + "\n"); + await file.sync(); + } finally { + await file.close(); + } + } catch { + /* Diagnostics must not change the control response. */ + } + } + if (!signal.aborted) reply(res, 400, { error: "control_failed" }); + } finally { + finishDiagnostic(); + clearTimeout(timer); + res.off("close", disconnect); + if (!lifecycleStarted || !signal.aborted) active = undefined; + busy = false; + } + } + const server = createServer( + { + maxHeaderSize: 4096, + requestTimeout: 5000, + headersTimeout: 5000, + connectionsCheckingInterval: 500, + }, + (req, res) => { + void serve(req, res); + }, + ); + server.maxConnections = 8; + server.keepAliveTimeout = 1; + server.on("connection", (socket) => { + sockets.add(socket); + socket.setTimeout(ctx.deadlines.requestMs, () => socket.destroy()); + socket.once("close", () => sockets.delete(socket)); + }); + server.on("clientError", (_error, socket) => socket.destroy()); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(ctx.supervisor.socket, resolve); + }); + await chmod(ctx.supervisor.socket, 0o600); + const socketStat = await lstat(ctx.supervisor.socket); + return { + async close() { + closing = true; + active?.controller.abort("shutdown"); + for (const socket of sockets) socket.destroy(); + // Closing a Unix listener can unlink its pathname. Check before that operation. + // On substitution retain the descriptor until process exit; report failure, not cleanup. + try { + const current = await lstat(ctx.supervisor.socket); + check( + current.ino === socketStat.ino && + current.dev === socketStat.dev && + current.isSocket(), + "socket_replaced", + ); + } catch (error) { + server.unref(); + throw error; + } + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + if (active) { + let timer: ReturnType | undefined; + try { + await Promise.race([ + Promise.all([active.settled, active.diagnosed]), + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new PlatformFailure("FAILED", "control_cleanup")), + ctx.deadlines.requestMs, + ); + }), + ]); + } finally { + clearTimeout(timer); + } + } + // Node removes its own Unix socket. Never unlink a substituted path. + try { + const current = await lstat(ctx.supervisor.socket); + check( + current.ino === socketStat.ino && + current.dev === socketStat.dev && + current.isSocket(), + "socket_replaced", + ); + await unlink(ctx.supervisor.socket); + } catch (e) { + if (!(e instanceof Error && "code" in e && e.code === "ENOENT")) + throw e; + } + }, + }; +} diff --git a/tests/fixtures/mcp-stack/platform/tsconfig.json b/tests/fixtures/mcp-stack/platform/tsconfig.json new file mode 100644 index 000000000..bac6a1dbe --- /dev/null +++ b/tests/fixtures/mcp-stack/platform/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../../orchestrator/tsconfig.json", + "compilerOptions": { + "noEmit": true, + "typeRoots": ["../../../../orchestrator/node_modules/@types"] + }, + "include": ["./*.ts"] +} diff --git a/tests/fixtures/mcp-stack/process.test.mts b/tests/fixtures/mcp-stack/process.test.mts new file mode 100644 index 000000000..abace7ade --- /dev/null +++ b/tests/fixtures/mcp-stack/process.test.mts @@ -0,0 +1,757 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { once } from "node:events"; +import { + chmodSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { request } from "node:https"; +import { createServer } from "node:net"; +import { resolve } from "node:path"; +import { test } from "node:test"; +import { getCACertificates, setDefaultCACertificates } from "node:tls"; +import { + Client, + StreamableHTTPClientTransport, +} from "../../../mcp-worker/node_modules/@modelcontextprotocol/client/dist/index.mjs"; +import { setTimeout as delay } from "node:timers/promises"; +import { + TEST_CA, + TEST_CERT, + TEST_KEY, +} from "../../../mcp-worker/test/fixtures/tls.ts"; +import { loadConfig } from "./config.mts"; + +const entry = resolve("tests/fixtures/mcp-stack/main.mts"); +const base = resolve("var/mcp-stack-fixture"); +const observerToken = "neutral_observer_token_0000000000000000"; +const apiKey = "neutral_resource_key_0000000000000000"; +async function freePort() { + const server = createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + assert.ok(address && typeof address !== "string"); + const port = address.port; + await new Promise((resolve, reject) => + server.close((e) => (e ? reject(e) : resolve())), + ); + return port; +} +async function setup() { + const root = mkdtempSync(`${base}/process-`); + chmodSync(root, 0o700); + const port = await freePort(); + let controlPort = await freePort(); + while (controlPort === port) controlPort = await freePort(); + const origin = `https://localhost:${port}`; + const control = `https://localhost:${controlPort}`; + for (const [name, value] of Object.entries({ + "cert.pem": TEST_CERT, + "key.pem": TEST_KEY, + "ca.pem": TEST_CA, + "observer.token": observerToken, + "api.key": apiKey, + })) + writeFileSync(`${root}/${name}`, value, { mode: 0o600 }); + const config = { + version: 1, + runRoot: root, + listen: { host: "127.0.0.1", port }, + advertised: { + resource: origin, + issuer: origin, + authorization: origin, + token: origin, + redirectUri: `${origin}/integrations/mcp/callback`, + authorities: [`localhost:${port}`], + }, + tls: { + certificateFile: `${root}/cert.pem`, + keyFile: `${root}/key.pem`, + caFile: `${root}/ca.pem`, + }, + control: { + port: controlPort, + origin: control, + tokenFile: `${root}/observer.token`, + }, + protocol: "2026-07-28", + case: { + reply: "json", + response: "normal", + holdCalls: false, + requireAuth: true, + }, + apiKeyFile: `${root}/api.key`, + }; + const path = `${root}/config.json`; + const save = () => + writeFileSync(path, JSON.stringify(config), { mode: 0o600 }); + save(); + return { root, origin, control, config, path, save }; +} +function wire( + url: string | URL, + options: { + body?: string; + method?: string; + headers?: Record; + signal?: AbortSignal; + } = {}, +) { + return new Promise<{ + status: number; + headers: import("node:http").IncomingHttpHeaders; + text: string; + json: () => any; + }>((resolve, reject) => { + const req = request( + url, + { + ca: TEST_CA, + servername: new URL(url).hostname, + family: 4, + agent: false, + method: options.method ?? "GET", + headers: options.headers ?? {}, + ...(options.signal ? { signal: options.signal } : {}), + }, + (res) => { + const chunks: Buffer[] = []; + let size = 0; + res.on("data", (chunk) => { + size += chunk.length; + if (size > 3 * 1024 * 1024) res.destroy(); + else chunks.push(chunk); + }); + res.on("error", reject); + res.on("end", () => { + const text = Buffer.concat(chunks).toString(); + resolve({ + status: res.statusCode!, + headers: res.headers, + text, + json: () => JSON.parse(text), + }); + }); + }, + ); + req.on("error", reject); + req.end(options.body); + }); +} +async function start( + f: Awaited>, + expectedReady = true, +) { + const child = spawn(process.execPath, [entry, f.path], { + cwd: process.cwd(), + env: { PATH: "/usr/bin:/bin", TMPDIR: f.root }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + if (stdout.length > 4096) child.kill("SIGKILL"); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + if (stderr.length > 4096) child.kill("SIGKILL"); + }); + const exited = once(child, "close"); + const watchdog = setTimeout(() => child.kill("SIGKILL"), 29000); + const stop = async (signal: NodeJS.Signals = "SIGTERM") => { + if (child.exitCode === null && child.signalCode === null) + child.kill(signal); + const escalation = setTimeout(() => child.kill("SIGKILL"), 2000); + try { + await exited; + } finally { + clearTimeout(escalation); + clearTimeout(watchdog); + } + assert.equal( + child.signalCode, + signal === "SIGKILL" ? "SIGKILL" : null, + "graceful exit must not need kill", + ); + if (signal !== "SIGKILL") + assert.equal(child.exitCode, expectedReady ? 0 : 1); + for (const secret of [ + observerToken, + apiKey, + "fixture-access-", + "fixture-refresh-", + "fixture-code-", + ]) + assert.equal((stdout + stderr).includes(secret), false); + }; + try { + if (expectedReady) { + for (let n = 0; !stdout.includes('"kind":"ready"'); ++n) { + if (n === 150 || child.exitCode !== null) + throw new Error(`Fixture did not become ready: ${stderr}`); + await delay(10); + } + } else { + await exited; + assert.equal(stdout, ""); + assert.equal(stderr.trim(), '{"version":1,"kind":"startup_failed"}'); + } + } catch (error) { + await stop(); + throw error; + } + return { child, stop, exited }; +} +function control( + f: Awaited>, + route: string, + body?: object, + auth = observerToken, +) { + return wire(`${f.control}/v1/${route}`, { + headers: { + authorization: `Bearer ${auth}`, + "content-type": "application/json", + }, + ...(body ? { method: "POST", body: JSON.stringify(body) } : {}), + }); +} +async function poll( + f: Awaited>, + test: (value: any) => boolean, +) { + for (let n = 0; n < 100; ++n) { + const value = (await control(f, "observe")).json(); + if (test(value)) return value; + await delay(10); + } + throw new Error("Observer barrier timeout"); +} +function rpc( + f: Awaited>, + name: string, + id: number | string, + auth = apiKey, + signal?: AbortSignal, +) { + return wire(`${f.origin}/mcp`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${auth}`, + "mcp-protocol-version": "2026-07-28", + "mcp-method": "tools/call", + "mcp-name": name, + }, + body: JSON.stringify({ + jsonrpc: "2.0", + id, + method: "tools/call", + params: { + name, + arguments: name === "write_counter" ? { delta: 2 } : {}, + _meta: { "io.modelcontextprotocol/protocolVersion": "2026-07-28" }, + }, + }), + ...(signal ? { signal } : {}), + }); +} + +test("standalone TLS process commits held writes before reply and preserves duplicate executions on reopen", async () => { + const f = await setup(); + let running: Awaited> | undefined; + try { + running = await start(f); + assert.equal((await control(f, "ready")).json().ready, true); + const trust = getCACertificates(); + setDefaultCACertificates([...trust, TEST_CA]); + const client = new Client( + { name: "standalone-fixture-client", version: "1" }, + { versionNegotiation: { mode: "auto" } }, + ); + try { + await client.connect( + new StreamableHTTPClientTransport(new URL(`${f.origin}/mcp`), { + requestInit: { headers: { authorization: `Bearer ${apiKey}` } }, + }), + ); + const first = await client.request({ method: "tools/list", params: {} }); + assert.equal(first.tools[0]?.name, "read_counter"); + assert.equal( + (await client.listTools({ cursor: first.nextCursor! })).tools[0]?.name, + "write_counter", + ); + assert.deepEqual( + (await client.callTool({ name: "read_counter", arguments: {} })) + .structuredContent, + { count: 0 }, + ); + } finally { + await client.close(); + setDefaultCACertificates(trust); + } + assert.equal((await control(f, "observe", undefined, "wrong")).status, 401); + assert.equal((await control(f, "unknown", {})).status, 404); + assert.equal((await control(f, "case", { arbitrary: true })).status, 400); + assert.equal((await rpc(f, "write_counter", "same", "wrong")).status, 401); + assert.equal((await control(f, "case", { holdCalls: true })).status, 200); + let completed = false; + const held = rpc(f, "write_counter", "same").then((value) => { + completed = true; + return value; + }); + const observed = await poll( + f, + (s) => s.effects.executions === 1 && s.held === 1, + ); + assert.equal(completed, false); + assert.equal(observed.effects.count, 2); + assert.equal( + JSON.parse(readFileSync(`${f.root}/effects.json`, "utf8")).state + .executions, + 1, + ); + assert.equal((await control(f, "case", { response: "error" })).status, 409); + await control(f, "release", {}); + assert.equal((await held).json().result.structuredContent.count, 2); + await running.stop("SIGKILL"); + running = undefined; + running = await start(f); + assert.equal((await control(f, "observe")).json().effects.executions, 1); + assert.equal( + (await rpc(f, "write_counter", "same")).json().result.structuredContent + .count, + 4, + ); + assert.equal((await control(f, "observe")).json().effects.executions, 2); + await control(f, "case", { holdCalls: true }); + const canceled = rpc(f, "write_counter", "cancel").catch(() => "canceled"); + await poll(f, (s) => s.held === 1 && s.effects.executions === 3); + await control(f, "cancel", {}); + assert.equal(await canceled, "canceled"); + await poll(f, (s) => s.cancellations === 1); + const stopped = rpc(f, "write_counter", "shutdown").catch(() => "closed"); + await poll(f, (s) => s.held === 1 && s.effects.executions === 4); + await running.stop(); + running = undefined; + assert.equal(await stopped, "closed"); + assert.equal( + JSON.parse(readFileSync(`${f.root}/effects.json`, "utf8")).state + .executions, + 4, + ); + } finally { + if (running) await running.stop(); + rmSync(f.root, { recursive: true, force: true }); + } +}); + +test("standalone OAuth enforces PKCE/resource, rotates refresh and exposes only neutral observations", async () => { + const f = await setup(); + let running: Awaited> | undefined; + try { + running = await start(f); + const metadata = ( + await wire(`${f.origin}/.well-known/oauth-authorization-server`) + ).json(); + const resource = ( + await wire(`${f.origin}/.well-known/oauth-protected-resource`) + ).json(); + assert.equal(metadata.issuer, f.origin); + assert.equal(resource.resource, `${f.origin}/mcp`); + const verifier = "v".repeat(43); + const authorize = new URL(metadata.authorization_endpoint); + authorize.search = new URLSearchParams({ + response_type: "code", + client_id: "fixture-public", + redirect_uri: f.config.advertised.redirectUri, + state: "neutral-state", + code_challenge: createHash("sha256").update(verifier).digest("base64url"), + code_challenge_method: "S256", + resource: `${f.origin}/mcp`, + scope: "tools:read tools:write", + }).toString(); + authorize.searchParams.set( + "redirect_uri", + `${f.config.advertised.redirectUri}/other`, + ); + assert.equal((await wire(authorize)).status, 400); + authorize.searchParams.set("redirect_uri", f.config.advertised.redirectUri); + const authorization = await wire(authorize); + assert.equal(authorization.status, 302); + const callback = new URL(String(authorization.headers.location)); + assert.equal(callback.searchParams.get("iss"), metadata.issuer); + assert.equal(callback.pathname, "/integrations/mcp/callback"); + const token = (body: Record) => + wire(metadata.token_endpoint, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams(body).toString(), + }); + const grant = { + grant_type: "authorization_code", + code: callback.searchParams.get("code")!, + code_verifier: verifier, + client_id: "fixture-public", + redirect_uri: f.config.advertised.redirectUri, + resource: `${f.origin}/mcp`, + }; + assert.equal((await token({ ...grant, code_verifier: "bad" })).status, 400); + assert.equal( + (await token({ ...grant, resource: "https://other.example/mcp" })).status, + 400, + ); + assert.equal( + (await token({ ...grant, redirect_uri: `${grant.redirect_uri}/other` })) + .status, + 400, + ); + const first = (await token(grant)).json(); + assert.equal(first.fixture_account, "account-one"); + assert.equal( + (await rpc(f, "read_counter", 1, first.access_token)).status, + 200, + ); + await control(f, "oauth", { account: "account-two", scope: "tools:read" }); + const refresh = { + grant_type: "refresh_token", + refresh_token: first.refresh_token, + client_id: "fixture-public", + resource: `${f.origin}/mcp`, + }; + const second = (await token(refresh)).json(); + assert.notEqual(second.refresh_token, first.refresh_token); + assert.equal(second.fixture_account, "account-two"); + assert.equal(second.scope, "tools:read"); + assert.equal((await token(refresh)).status, 400); + assert.equal( + (await rpc(f, "read_counter", 2, first.access_token)).status, + 401, + ); + assert.equal( + (await rpc(f, "read_counter", 3, second.access_token)).status, + 200, + ); + const observed = await control(f, "observe"); + assert.deepEqual(observed.json().oauth, { + account: "account-two", + scope: "tools:read", + expiresInSeconds: 3600, + authorizations: 1, + exchanges: 1, + rotations: 1, + rejections: 5, + }); + for (const secret of [ + first.access_token, + first.refresh_token, + second.access_token, + second.refresh_token, + grant.code, + apiKey, + observerToken, + ]) + assert.equal(observed.text.includes(secret), false); + assert.equal((await control(f, "oauth", { scope: "admin" })).status, 400); + await control(f, "oauth", { issuer: "mismatch" }); + authorize.searchParams.set("state", "issuer-mismatch"); + const mismatched = await wire(authorize); + assert.equal( + new URL(String(mismatched.headers.location)).searchParams.get("iss"), + "https://other.example", + ); + } finally { + if (running) await running.stop(); + rmSync(f.root, { recursive: true, force: true }); + } +}); + +test("config, authority, and durable corruption fail closed without secret output", async () => { + const f = await setup(); + let running: Awaited> | undefined; + try { + for (const mutate of [ + (c: typeof f.config & { unknown?: boolean }) => { + c.unknown = true; + }, + (c: typeof f.config) => { + c.version = 2; + }, + (c: typeof f.config) => { + c.advertised.issuer = "http://localhost:1234"; + }, + (c: typeof f.config) => { + c.advertised.token = `https://other.example:${c.listen.port}`; + c.advertised.authorities.push(`other.example:${c.listen.port}`); + }, + (c: typeof f.config) => { + c.advertised.authorities = []; + }, + (c: typeof f.config) => { + c.advertised.redirectUri += "?"; + }, + (c: typeof f.config) => { + c.advertised.redirectUri += "#"; + }, + (c: typeof f.config) => { + c.advertised.redirectUri = "https://user:pass@localhost/callback"; + }, + ]) { + const copy = structuredClone(f.config); + mutate(copy); + writeFileSync(f.path, JSON.stringify(copy)); + assert.throws(() => loadConfig(f.path)); + } + // One real process rejection proves the entrypoint also fails closed. + running = await start(f, false); + await running.stop(); + running = undefined; + f.save(); + running = await start(f); + const wrong = await wire(`${f.origin}/mcp`, { + headers: { host: "other.example" }, + }); + assert.equal(wrong.status, 421); + await running.stop(); + running = undefined; + writeFileSync(`${f.root}/effects.json`, "truncated"); + running = await start(f, false); + await running.stop(); + running = undefined; + } finally { + if (running) await running.stop(); + rmSync(f.root, { recursive: true, force: true }); + } +}); + +test("config accepts separate DNS origins with matching supplied SANs and exact authorities", async () => { + const f = await setup(); + try { + const names = [ + "localhost", + "mcp.fixture.example", + "issuer.fixture.example", + "authorization.fixture.example", + "token.fixture.example", + "callback.fixture.example", + ]; + execFileSync( + "/usr/bin/openssl", + [ + "req", + "-x509", + "-new", + "-key", + `${f.root}/key.pem`, + "-out", + `${f.root}/network.pem`, + "-days", + "2", + "-subj", + "/CN=mcp.fixture.example", + "-addext", + `subjectAltName=${names.map((n) => `DNS:${n}`).join(",")}`, + ], + { + env: { PATH: "/usr/bin:/bin", TMPDIR: f.root }, + stdio: "ignore", + timeout: 2000, + }, + ); + f.config.tls.certificateFile = `${f.root}/network.pem`; + f.config.tls.caFile = `${f.root}/network.pem`; + const advertised = f.config.advertised; + advertised.resource = `https://mcp.fixture.example:${f.config.listen.port}`; + advertised.issuer = `https://issuer.fixture.example:${f.config.listen.port}`; + advertised.authorization = `https://authorization.fixture.example:${f.config.listen.port}`; + advertised.token = `https://token.fixture.example:${f.config.listen.port}`; + advertised.redirectUri = + "https://callback.fixture.example:9443/integrations/mcp/callback"; + advertised.authorities = names + .slice(1, 5) + .map((n) => `${n}:${f.config.listen.port}`); + f.save(); + const loaded = loadConfig(f.path); + assert.equal(loaded.options.network.origin, advertised.resource); + assert.deepEqual(loaded.options.network.oauth, { + issuer: advertised.issuer, + authorization: advertised.authorization, + token: advertised.token, + redirectUri: advertised.redirectUri, + }); + writeFileSync( + f.path, + JSON.stringify({ ...f.config, case: { reply: ["json"] } }), + ); + assert.throws(() => loadConfig(f.path)); + } finally { + rmSync(f.root, { recursive: true, force: true }); + } +}); + +test("OAuth TTL control changes future tokens only and enforces real bearer deadlines", async () => { + const f = await setup(); + let running: Awaited> | undefined; + try { + running = await start(f); + const observed = () => control(f, "observe").then((r) => r.json().oauth); + assert.equal((await observed()).expiresInSeconds, 3600); + const before = await observed(); + for (const expiresInSeconds of [0, -1, 1.5, 3601, "2", null]) { + assert.equal( + ( + await control(f, "oauth", { + account: "account-two", + scope: "tools:read", + issuer: "mismatch", + expiresInSeconds, + }) + ).status, + 400, + ); + assert.deepEqual(await observed(), before); + } + for (const invalid of [ + { unknown: true }, + { account: "bad" }, + { scope: "admin" }, + { issuer: "bad" }, + ]) { + assert.equal( + (await control(f, "oauth", { expiresInSeconds: 2, ...invalid })).status, + 400, + ); + assert.deepEqual(await observed(), before); + } + const verifier = "v".repeat(43); + const token = (body: Record) => + wire(`${f.origin}/token`, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams(body).toString(), + }); + const mint = async (state: string) => { + const url = new URL(`${f.origin}/authorize`); + url.search = new URLSearchParams({ + response_type: "code", + client_id: "fixture-public", + redirect_uri: f.config.advertised.redirectUri, + state, + code_challenge: createHash("sha256") + .update(verifier) + .digest("base64url"), + code_challenge_method: "S256", + resource: `${f.origin}/mcp`, + scope: "tools:read tools:write", + }).toString(); + const authorized = await wire(url); + assert.equal(authorized.status, 302); + const code = new URL( + String(authorized.headers.location), + ).searchParams.get("code")!; + const response = await token({ + grant_type: "authorization_code", + code, + code_verifier: verifier, + client_id: "fixture-public", + redirect_uri: f.config.advertised.redirectUri, + resource: `${f.origin}/mcp`, + }); + assert.equal(response.status, 200); + return response.json(); + }; + const refresh = (refreshToken: string) => + token({ + grant_type: "refresh_token", + refresh_token: refreshToken, + client_id: "fixture-public", + resource: `${f.origin}/mcp`, + }); + assert.equal( + (await control(f, "oauth", { expiresInSeconds: 2 })).status, + 200, + ); + assert.equal((await observed()).expiresInSeconds, 2); + assert.equal((await mint("ttl-runner")).expires_in, 2); + await control(f, "oauth", { expiresInSeconds: 3600 }); + const first = await mint("ttl-default"); + assert.equal(first.expires_in, 3600); + assert.equal( + (await control(f, "oauth", { expiresInSeconds: 1 })).status, + 200, + ); + assert.equal((await observed()).expiresInSeconds, 1); + await delay(1100); + assert.equal( + (await rpc(f, "read_counter", "ttl-earlier", first.access_token)).status, + 200, + ); + const short = await mint("ttl-short"); + assert.equal(short.expires_in, 1); + assert.equal( + (await rpc(f, "read_counter", "ttl-live", short.access_token)).status, + 200, + ); + await control(f, "oauth", { expiresInSeconds: 3600 }); + await delay(1100); + assert.equal( + (await rpc(f, "read_counter", "ttl-expired", short.access_token)).status, + 401, + ); + const renewed = (await refresh(short.refresh_token)).json(); + assert.equal(renewed.expires_in, 3600); + assert.equal( + (await rpc(f, "read_counter", "ttl-renewed", renewed.access_token)) + .status, + 200, + ); + await control(f, "oauth", { expiresInSeconds: 1 }); + const rotated = (await refresh(renewed.refresh_token)).json(); + assert.equal(rotated.expires_in, 1); + assert.equal( + (await rpc(f, "read_counter", "ttl-rotated", rotated.access_token)) + .status, + 200, + ); + assert.equal((await refresh(renewed.refresh_token)).status, 400); + assert.equal( + (await rpc(f, "read_counter", "ttl-old", renewed.access_token)).status, + 401, + ); + await control(f, "oauth", { expiresInSeconds: 3600 }); + await delay(1100); + assert.equal( + ( + await rpc( + f, + "read_counter", + "ttl-refresh-expired", + rotated.access_token, + ) + ).status, + 401, + ); + const recovered = (await refresh(rotated.refresh_token)).json(); + assert.equal(recovered.expires_in, 3600); + assert.equal( + (await rpc(f, "read_counter", "ttl-recovered", recovered.access_token)) + .status, + 200, + ); + const observation = await control(f, "observe"); + assert.equal(observation.json().oauth.expiresInSeconds, 3600); + assert.equal(observation.text.includes("fixture-access-"), false); + assert.equal(observation.text.includes("fixture-refresh-"), false); + } finally { + if (running) await running.stop(); + rmSync(f.root, { recursive: true, force: true }); + } +}); diff --git a/tests/fixtures/mcp-stack/runner-context.md b/tests/fixtures/mcp-stack/runner-context.md new file mode 100644 index 000000000..799c18c09 --- /dev/null +++ b/tests/fixtures/mcp-stack/runner-context.md @@ -0,0 +1,287 @@ +# Shared stack runner context, version 1 + +Status: complete source assembly, pending independent review and real platform +execution. The runner and all twelve driver families are implemented under +`orchestrator/scripts/mcp-stack`. The required-case manifest and unit tests are +not physical results. Platform adapters must not treat them as acceptance evidence. + +The full-matrix command is: + +```sh +bun orchestrator/scripts/mcp-stack/run.ts --context /owned/run/context.json --matrix all +``` + +The parser requires `--context` once. An omitted selector defaults to +`--matrix all`, which selects 12 cases for Claude and the same 12 for Codex. +`--matrix` also accepts unique comma-separated case keys from `manifest.ts`. +`--stage infrastructure` runs only the shared infrastructure checks. The two +selectors are mutually exclusive. Focused runs record stage completion only; +full matrix completion and physical acceptance require `--matrix all` with +every required case and subassertion. The entrypoint validates the exact driver set, +context, fixed supervisor controls, scoped database schema and both fixtures +before publication or signup. Public enabled-image and enrolled-host checks +precede all cases. After an assertion failure with successful case cleanup, +the runner continues only after fresh readiness checks. Transport, cleanup, +prerequisite and unclassified failures stop the run. Any failed case leaves +incomplete evidence. A total watchdog exits 124; the adapter must tear down owned services +on every exit. Do not activate physical CI until the assembly passes review. + +## Context fields + +`ContextSchema` in `context.ts` is the exact schema. It rejects unknown fields. +The context is a private regular file directly inside a private, real run root. +No component can be a symlink. All referenced files, artifacts, outputs and the +supervisor socket belong to that root and current OS user. The loader hashes +actual artifact bytes. Secrets use private file references. Outputs must not +exist before the run, and use exclusive creation. No default endpoint, cookie, +API key, environment file, image, or primary-checkout path is used. + +- `version`: 1; `runId`: a neutral safe identifier; `runRoot`: absolute real path. +- `backend`: `firecracker` or `vz`; `sourceCommit`: exact coherent Git SHA. +- `hostId`: actual enrolled UUID; `imageId`: public enabled-image ID; + `imageDigest`: actual `sha256:` OCI manifest digest. +- `artifacts`: exact `{path,sha256}` for coordinator, orchestrator, worker, + hostAgent, fixture, kernel, guestImage, claudeHarness, codexHarness, modelCli, + claudeBundle, codexBundle, helper. File hashes use bare lowercase hex. + These identify bytes; they do not prove that a guest ran them. +- `endpoints.app`, `coordinator`, `operator`, `workerControl`: canonical HTTPS origins + under `.fixture.example`, CA file/hash, uppercase colon-separated leaf SHA-256 + fingerprint. TLS still verifies the CA and DNS name. Operator additionally + needs a separate certificate file/hash and private key file. No redirects or + resolver overrides are installed by the clients. +- `protocolFixtures.modern` and `protocolFixtures.legacy`: separate private run + roots, immutable `configuration:{path,sha256}`, data/control identities, resource key and observer token files; exact + resource, issuer, authorization, token origins; exact callback URL on app + `/integrations/mcp/callback`. Modern is `2026-07-28`; legacy is `2025-11-25` or + `2025-06-18`. Both use the standalone fixture contract, and remain alive across + all other role restarts. Fidelity subassertions name both actual protocols; + other cases report only the protocol they actually exercised. +- `publication`: exact configured candidate identity, expected generation (null + for new deployment), deployment/network/trust identities and zone. Identities + have `sha256:` prefixes. The generated operator gRPC client sends only the + configured identity and expected generation, then reads actual publication. +- `identities.admin`, `owner`, `other`: distinct neutral email, password file, + new cookie output file. `unattendedKeyFile` is a new output for a real public + role=user service key. `automationKeyFile` is a separate new private output + for the role=admin service key that triggers actual manual automations. It + cannot replace the user key in denial cases. `routerSentinelFile` is a neutral input; each protocol fixture has its own resource key. +- `supervisor`: private Unix socket plus separate token file. +- `observer`: separate private coordinator and orchestrator DSN files. Each + points to a fixture DNS host and database `mcp_`. Only fixed read-only queries are implemented. The two services + can use different database servers with that database name. `hostManagedRoot` + is the actual host-agent `work_dir/managed-boots` directory, owned and mode + 0700. `hostLogFile` is the egress proxy daemon tracing log, owned regular + mode 0600 with one link. Both are direct paths below the run root, not worker mounts + or adapter-created observation files. +- `guest`: owned `/tmp/mcp-stack-fixture` directory and absolute immutable model + CLI path ending in `/engram-mcp-model-fixture`. The image/init owner prepares + this directory as 0700 before CLI launch. The runner cannot create it after a + CLI has already read its startup configuration. +- `deadlines`: request 100–30000 ms, step up to 120000 ms, total up to 7200000 ms; + request <= step <= total. `evidenceFile`: new private output file. + The required TLS probe needs request >= 6000 ms for its fixed five-second + OpenSSL deadline and result framing. + +Before any admin mutation, the final runner must check context artifacts, +supervisor identities/network enforcement, fixture readiness and full source +prerequisites. Signup/login requires the normal isolated password auth mode. +The real public WhoAmI response must establish three distinct human principals +and distinct user/admin service principals. For unattended runtime cases, the +normal public AutomationService creates a manual definition with one +create_session block and an empty initial prompt. The admin service key triggers +RunNow; GetRun supplies actual task/session IDs. Human admin definition author, +service trigger principal and runtime automation actor are recorded separately. +The runner checks Started and guest readiness before the first marked prompt. The normal OrgSecret PutSecret and +migrated router model path provides neutral model auth. No provider refresh is +called by this runner. The launcher must deny provider routing and external DNS +for O, coordinator and worker before boot and across restart. + +## Fixed supervisor interface + +POST `/v1/control` over the private Unix socket. Bearer token comes only from +its file. Request is exactly `{version:1,runId,operation,role?,configurationId?}`. Operations are +`status`, `restart`, `stop`, `infrastructure-ready`, `prepare-disk-read`, and +`verify-disk-read`. The last three are fixed Firecracker checkpoints with no +role or configuration argument. Roles are `orchestrator`, `worker`, `coordinator`, +`host-agent`, `fixture-modern`, `fixture-legacy`. The runner can inspect fixture +status but cannot stop or restart either fixture. The adapter owns fixture cleanup after the runner terminates. + +Response is `StatusSchema` in `supervisor.ts`: exact run/backend/source/host +identity, exactly six role entries `{role,state,bootId,artifactSha256,configurationId,configurationSha256}`, and +`network:{generation,enforcementSha256,fixtureOnlyEgress:true, +externalDnsBlocked:true,roles:[orchestrator,coordinator,worker]}`. A restart changes +only the selected role boot ID, preserves the other five boot IDs and the exact +network identity, and returns after roles run again. Status does not substitute +for public enrollment, actual Started, outbox ACK or guest proof. + +The adapter computes network evidence from actual installed routes, firewall and +DNS policy. The hash is not itself enforcement proof: physical acceptance must +also observe explicit guest probe denial and the corresponding network event. +No arbitrary command, URL, environment override, SQL, process ID or proxy is +accepted. Only the orchestrator has `global-disabled` and `zone-disabled` +configuration IDs. Global changes only `MCP_ENABLED` to false and retains its +runtime configuration. Zone keeps the global flag true and changes only the +matching configured `workers[].enabled` to false. Both fixed files/hashes are +required before the first mutation. Other roles accept only `normal`; their +configuration and process boot identities remain unchanged during these kills. +The adapter validates byte-identical destinations, CIDRs, CAs, identities and +all settings other than the selected enable switch. Status attests the selected +configuration ID and hash. A missing variant fails the prerequisite. + +`supervisor.operatorCandidates` contains only `normal` as +`{file:{path,sha256},identity}`. The runner stages that exact configured identity +once at bootstrap. Global/zone kill and restoration restart only the +orchestrator and read the actual operator publication before and after. Its +identity, generation and admission state must remain unchanged. Neither kill +publishes a new candidate. Coordinator Zone and OperatorCandidate do not have +enable fields, and the runner does not invent one. + +## Read-only observations + +The runner records task/session IDs only after real public creation. Each query +requires an owned session. Coordinator queries join the exact managed creation +(task ID), selected host and boot; a separate query reads only outbox attempts, +relay status and `acked_at` for `tool_result::`. Orchestrator queries +check actual session listener ownership for the same task/session. Pools set +`default_transaction_read_only=on`, one connection and bounded statements. They +never select credentials, raw arguments/results or mutate state. The observer +checks O's managed lifecycle row against the coordinator's exact Started boot +identity. A listener lease alone does not establish Started. + +The public journal result and its settlement field, fixture effect count, +coordinator outbox ACK and actual guest proof are different observations. A +settled-or-target-gone value cannot be called guest delivery. Only actual guest +result proof plus actual ACK establishes that subassertion. The evidence sink +records bounded states, counts, sizes and hashes; it refuses unknown payload +fields and case results that cite observations not collected for that case. +It does not print exceptions, credentials, cookies, codes, arguments or results. + +## Implemented source and remaining physical gates + +1. The runner consumes the implemented CLI contract: bounded + scalar arguments/call IDs/native expected result, or an immutable fixed call + plan with explicit prompt markers. Plans exclude scalar settings. Actual + exposure name-set hashes and native response hashes are verified. The CLI + source is in `tests/fixtures/mcp-model-cli`; actual guest execution remains + a physical acceptance requirement. +2. Guest proof files are JSONL, one flushed completion per line, including + multiple turns in one process. Each record is validated and hashed separately. + Repeated markers retain call IDs; each prompt attempt uses a distinct prompt + ID so the public prompt deduplication layer cannot replace the replay probe. +3. The compiled alias can be derived from the actual public connection ID/alias + and upstream name, then must match paginated public GetSessionMcpState before + prompting. No description-based selector or guessed alias prefix is needed. +4. Modern and legacy MCP protocol is fixed for a standalone fixture lifetime. + The context requires two independent fixture instances. The adapters + implement both lifetimes in `platform/launch.ts`. Real startup and + persistence across restarts remain physical checks. +5. No public global/zone flag setter exists. The fixed orchestrator-only + configuration contract above uses the actual startup switches. Adapters + implement the fixed variants and verify unchanged operator publication and + network trust. Effective deployment behavior remains a physical check. +6. Child grant intersection needs an actual native child-task invocation. + The fixed CLI plan supports an exact native spawn step and a child message + marker. The runner uses that plan through the normal harness transport. + Forged guest completion and direct bypass probes need actual guest transport, + not a human owner's public call labeled as guest traffic. +7. The fixture OAuth control contract implements future-issued `expiresInSeconds` + (integer 1–3600; default 3600), observed under `oauth.expiresInSeconds`. The + runner uses 2 seconds and the actual coordinator clock. Existing token + deadlines do not change when the configuration changes. The standalone + fixture and worker fixture implement this control. +8. Both adapters implement production startup, private configuration and + supervision under `platform/`. Source assembly and unit tests do not prove + coherent service startup or either physical backend result. Real capable + runners, exact assets and current-head CI remain required. + +The fixed call plan can assert `expectExposure: "absent"` for an exact MCP +alias. Such a step must observe a real protocol denial and cannot include an +expected result or premature completion. `completionProjection: "premature"` +is a separate adversarial step for an exposed MCP call and cannot include an +expected result. Barrier records are observations, never result receipts. + +Acceptance profiles freeze `ENGRAM_FIXTURE_PROOF` with a single `{launch}` +token. The CLI expands it to its decimal PID and a fresh 128-bit OS-random +nonce, separated by a hyphen. Discovery accepts only +`--<32 lowercase hex>.json`. It has no PID-only fallback. +Exclusive creation preserves every old proof across disk recovery even when +the new guest reuses a PID. Nonces are observation filenames, not authority. +The CLI implements this token with exclusive file creation and OS entropy; +legacy literal and `{pid}` modes are not used by these acceptance profiles. + +The common runner exposes only two release files under the fixed guest state +root: `completion-release` and `premature-release`. Profile options enable the +matching fixture environment variables. Releases use exclusive creation with +mode 0600. The CLI independently validates and consumes each release. Its fixed +120-second bound is not configurable by the runner. A normal native completion +notification still requires separate actual coordinator outbox ACK evidence. + +Managed resume uses the public `PrepareMcpRebind` and `RebindMcp` review flow. +The runner checks a new sandbox and boot attempt, the next binding epoch, exact +reviewed bindings, coordinator and orchestrator Started state, and guest return. +Ordinary `SessionService.Resume` must remain denied for this managed session. + +A call ID may occur in exactly two ordered plan steps for the rebind test: +first present, then explicit absent. Both steps must use the same MCP alias, +canonical arguments, and MCP result format. Neither can project a premature +completion. All other duplicate IDs fail. The runner counts prior proofs before +recovery and requires a new explicit absent-denial proof after the second step. + +Delivery checks combine the actual native result proof, public +`tool_call_completed` event, and coordinator outbox ACK with the current managed +delivery target. An outbox ACK alone is insufficient: the coordinator also ACKs +an old delivery after managed cold recovery retires its target. + +## Physical observations and platform prerequisites + +The runner reads the sealed host `.boot` record directly. It +requires schema 1, an exact body hash and sandbox identity, a version 1 record, +and the current creation/source key. Reads use no-follow file descriptors, +0600 owner-only single-link files and stable descriptor/path metadata. Envelope +and body bounds are 16 MiB and 8 MiB, with depth at most 64. The completed +capture authority must identify a destroyed source and exactly match the new +cold request. The runner hashes the actual `.disk` bytes with a +16 GiB fixture limit and the step deadline. That limit is not a product limit. + +An exact, read-only coordinator query joins the owned task/session, current +boot attempt and completed managed boot intent. It selects only the nonsecret +asset receipt, start intent, native bytes and combined manifest. It never reads +the sealed agent or credentials. The host Started receipt and incarnation must +match the coordinator start intent and selected backend, kernel, disk and +auxiliary drives. The actual guest harness environment must contain the exact +combined manifest hash, with the unchanged native prefix. Before and after cold +recovery, the guest supplies its real kernel boot ID. An exclusive disk +sentinel must survive, while a sentinel on verified `/dev/shm` tmpfs must be +absent after recovery. This checks loss of the old tmpfs state; it does not +claim that every physical RAM byte was erased. + +The platform adapter must separately establish actual hypervisor availability, +the exact owned process and executable for the selected backend, immutable +source/configuration hashes, and the real guest transport. The managed intent +receipt proves which backend the authenticated production host path selected. +It does not replace that platform evidence. Configuration strings or supplied +success booleans do not establish either layer. + +For managed-destination denial, the runner records the proxy daemon log inode +and byte offset, issues a fixed guest TLS probe to the configured resource +origin, then requires a new exact proxy Reject event for the owned session and +SNI. The log must not rotate or truncate. The reader allows at most 8 MiB of +new bytes and 64 KiB per line, and consumes only newline-terminated records. +Only hashes leave the reader; raw logs are not evidence artifacts. + +`endpoints.workerControl` must name the actual private worker control origin. +The runner makes a TLS 1.3 connection from the trusted fixture network to the +worker control endpoint. It verifies the CA, hostname and exact server leaf, +without a client certificate or an HTTP request. A certificate-required alert +116 is classified as `mtls_authentication_denial`. Timeout, network failure, +close and other TLS errors cannot pass. This verifies the worker's authentication +boundary; it does not establish guest network isolation. + +Adapters must provide both independently owned protocol fixtures, private +databases and normal public auth, actual enrolled host/image/assets, the fixed +supervisor variants and candidate files, the actual host record/log paths, +guest tools/state directory and fixture CLI, and the fixed worker control +endpoint. They own start/stop/reap and cleanup of every launched resource. +The source runner does not build images, launch services, migrate databases, +install routes, mint host credentials or create platform evidence. diff --git a/tests/fixtures/mcp-stack/tsconfig.json b/tests/fixtures/mcp-stack/tsconfig.json new file mode 100644 index 000000000..cb59ab514 --- /dev/null +++ b/tests/fixtures/mcp-stack/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../../mcp-worker/tsconfig.json", + "compilerOptions": { + "typeRoots": ["../../../mcp-worker/node_modules/@types"] + }, + "include": ["./*.mts"] +} diff --git a/web/index.html b/web/index.html index 24d87d52b..bb0b5642f 100644 --- a/web/index.html +++ b/web/index.html @@ -2,6 +2,7 @@ + diff --git a/web/src/App.tsx b/web/src/App.tsx index d23291219..64061591d 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -21,7 +21,8 @@ const queryClient = new QueryClient({ // useQuery/useMutation connect-query hooks without threading a transport prop everywhere. const transport = createConnectTransport({ baseUrl: RPC_BASE, - fetch: (input, init) => fetch(input, { ...init, credentials: API_CREDENTIALS }), + fetch: (input, init) => + fetch(input, { ...init, credentials: API_CREDENTIALS, cache: "no-store" }), }); function InnerApp() { diff --git a/web/src/app-entry.tsx b/web/src/app-entry.tsx new file mode 100644 index 000000000..cc6df679a --- /dev/null +++ b/web/src/app-entry.tsx @@ -0,0 +1,28 @@ +// JetBrains Mono is a multi-axis variable font; Fontsource splits it into one +// CSS file per axis. Import the `wght` axis so the full weight range is +// available to anything using `--font-mono` (code, IDs, tabular numbers). +import "@fontsource-variable/jetbrains-mono/wght.css"; +import "@fontsource-variable/jetbrains-mono/wght-italic.css"; +// Saira carries page titles only (`--font-display`, set 8% wide). The `wdth` +// file ships the weight AND width axes; the sans body face is system-ui, so +// there is nothing to import for it. +import "@fontsource-variable/saira/wdth.css"; +import { createRoot } from "react-dom/client"; +import { App } from "./App"; +import { ThemeProvider } from "./components/theme-provider"; +// index.css pulls in Tailwind once and @imports theme.css (the Lab-Notebook +// partial kept for the not-yet-migrated SessionDetail transcript subtree). +import "./index.css"; + +// StrictMode intentionally double-mounts effects in dev. Useful in +// general, but it interacts badly with ghostty-web's WASM Terminal: +// the first mount opens a WS to ttyd, cleanup closes it, the second +// mount opens a fresh WS — bash sees two SIGWINCH-driven resize +// flurries on top of each other and the rendered output overlaps. +// Until we make TerminalPane fully StrictMode-idempotent, opt out +// at the root. +createRoot(document.getElementById("root")!).render( + + + , +); diff --git a/web/src/components/assistant-ui/tool-fallback-mcp.test.tsx b/web/src/components/assistant-ui/tool-fallback-mcp.test.tsx new file mode 100644 index 000000000..59cd70515 --- /dev/null +++ b/web/src/components/assistant-ui/tool-fallback-mcp.test.tsx @@ -0,0 +1,26 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { cleanup, render, screen } from "@testing-library/react"; +import { ToolFallbackResult } from "./tool-fallback"; + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +test("raw MCP result text and structured content are inert and retain error data", () => { + const fetch = vi.spyOn(globalThis, "fetch"); + const result = { + content: [ + { + type: "text", + text: '![image](https://example.test/image) ', + }, + ], + structuredContent: { url: "https://example.test/resource" }, + isError: true, + }; + const { container } = render(); + expect(screen.getByText(/structuredContent/).textContent).toBe(JSON.stringify(result, null, 2)); + expect(container.querySelector("img, a, iframe")).toBeNull(); + expect(fetch).not.toHaveBeenCalled(); +}); diff --git a/web/src/components/integrations/GoogleCloudSetupPage.tsx b/web/src/components/integrations/GoogleCloudSetupPage.tsx index 8232ed1a4..2317ae84f 100644 --- a/web/src/components/integrations/GoogleCloudSetupPage.tsx +++ b/web/src/components/integrations/GoogleCloudSetupPage.tsx @@ -34,7 +34,18 @@ import { GoogleCloudEndpointDialog } from "./GoogleCloudConnections"; type SetupCodeLanguage = "terraform" | "shellscript"; export function GoogleCloudSetupPage() { - const { connectionId } = useParams({ strict: false }) as { connectionId?: string }; + const { connectionId, provider } = useParams({ strict: false }) as { + connectionId?: string; + provider?: string; + }; + if (provider === "mcp") + return ( +
+ +
+ ); return ; } diff --git a/web/src/components/integrations/IntegrationDetail.tsx b/web/src/components/integrations/IntegrationDetail.tsx index fa1b3dbfe..f967c47f4 100644 --- a/web/src/components/integrations/IntegrationDetail.tsx +++ b/web/src/components/integrations/IntegrationDetail.tsx @@ -70,6 +70,14 @@ export function IntegrationDetail() { window.history.replaceState({}, "", window.location.pathname); }, []); + if (provider === "mcp") { + return ( +
+ +

Manage external MCP connections in Integrations.

+
+ ); + } if (isLoading) return ; if (!view) { return ( diff --git a/web/src/components/integrations/McpCatalogReview.test.tsx b/web/src/components/integrations/McpCatalogReview.test.tsx new file mode 100644 index 000000000..4406f39a0 --- /dev/null +++ b/web/src/components/integrations/McpCatalogReview.test.tsx @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect"; +import { renderWithProviders } from "@/test-utils"; +import { IntegrationService } from "@/gen/engram/app/v1/integration_pb"; +import { McpCatalogReview } from "./McpCatalogReview"; + +describe("MCP catalog review", () => { + it("reloads the stored draft, renders descriptions as text, and requires explicit classification", async () => { + const published: unknown[] = []; + const transport = createRouterTransport((router) => + router.service(IntegrationService, { + getMcpCatalogs: () => ({ + configRevision: 1, + catalogRevision: 2, + draft: { + id: "draft", + connectionId: "c1", + kind: "draft", + revision: 0, + configRevision: 1, + authGeneration: 1, + tools: [ + { + upstreamName: "create.item", + description: '', + inputSchemaJson: '{"type":"object"}', + }, + ], + }, + }), + publishMcp: (req) => { + published.push(req); + return {}; + }, + }), + ); + renderWithProviders( + {}} + />, + { transport }, + ); + const user = userEvent.setup(); + await user.click(await screen.findByRole("button", { name: "Review draft" })); + expect(screen.getByText('')).toBeTruthy(); + expect(document.querySelector('img[src="https://example.test/tracker"]')).toBeNull(); + expect( + (screen.getByRole("button", { name: "Publish reviewed catalog" }) as HTMLButtonElement) + .disabled, + ).toBe(true); + await user.selectOptions(screen.getByLabelText("Classification for create.item"), "write"); + await user.click(screen.getByRole("button", { name: "Publish reviewed catalog" })); + await waitFor(() => expect(published).toHaveLength(1)); + expect(published[0]).toMatchObject({ + connectionId: "c1", + draftId: "draft", + expectedRevision: 2, + tools: [{ upstreamName: "create.item", classification: "write", enabled: false }], + }); + }); + it("drops a stale review and requires a new review after a publication conflict", async () => { + let reads = 0; + const transport = createRouterTransport((router) => + router.service(IntegrationService, { + getMcpCatalogs: () => { + reads++; + return { + configRevision: 1, + catalogRevision: reads, + draft: { + id: "draft", + configRevision: 1, + authGeneration: 1, + tools: [{ upstreamName: "read.item", inputSchemaJson: '{"type":"object"}' }], + }, + }; + }, + publishMcp: () => { + throw new ConnectError("stale", Code.Aborted); + }, + }), + ); + renderWithProviders( + {}} + />, + { transport }, + ); + const user = userEvent.setup(); + await user.click(await screen.findByRole("button", { name: "Review draft" })); + await user.selectOptions(screen.getByLabelText("Classification for read.item"), "read"); + await user.click(screen.getByRole("button", { name: "Publish reviewed catalog" })); + expect(await screen.findByText(/reload and review the draft again/i)).toBeTruthy(); + expect(screen.queryByRole("button", { name: "Publish reviewed catalog" })).toBeNull(); + expect(reads).toBeGreaterThan(1); + }); +}); diff --git a/web/src/components/integrations/McpCatalogReview.tsx b/web/src/components/integrations/McpCatalogReview.tsx new file mode 100644 index 000000000..e83f4d9e3 --- /dev/null +++ b/web/src/components/integrations/McpCatalogReview.tsx @@ -0,0 +1,219 @@ +import { createClient } from "@connectrpc/connect"; +import { useQuery, useTransport } from "@connectrpc/connect-query"; +import { useState } from "react"; +import { Button } from "@/components/ui/button"; +import { getMcpCatalogs } from "@/gen/engram/app/v1/integration-IntegrationService_connectquery"; +import { IntegrationService, type McpTool } from "@/gen/engram/app/v1/integration_pb"; + +export function McpCatalogReview({ + connectionId, + revision, + authGeneration, + discoveryAvailable, + refreshConnection, +}: { + connectionId: string; + revision: number; + authGeneration: number; + discoveryAvailable: boolean; + refreshConnection: () => Promise; +}) { + const client = createClient(IntegrationService, useTransport()); + const catalogs = useQuery(getMcpCatalogs, { connectionId }); + const [review, setReview] = useState(null); + const [draftId, setDraftId] = useState(""); + const [expectedRevision, setExpectedRevision] = useState(0); + const [busy, setBusy] = useState(false); + const [message, setMessage] = useState(""); + async function discover() { + setBusy(true); + setMessage(""); + setReview(null); + try { + await client.discoverMcp({ connectionId, expectedRevision: revision }); + await catalogs.refetch(); + } catch { + setMessage("Discovery could not finish. Reload the connection and try again."); + } finally { + setBusy(false); + } + } + function beginReview() { + if (!catalogs.data?.draft) return; + setDraftId(catalogs.data.draft.id); + setExpectedRevision(catalogs.data.catalogRevision); + setReview( + catalogs.data.draft.tools.map((tool) => ({ ...tool, classification: "", enabled: false })), + ); + setMessage(""); + } + async function publish() { + if ( + !review || + review.some((tool) => tool.classification !== "read" && tool.classification !== "write") + ) + return; + setBusy(true); + setMessage(""); + try { + await client.publishMcp({ connectionId, draftId, expectedRevision, tools: review }); + setReview(null); + await Promise.all([catalogs.refetch(), refreshConnection()]); + setMessage("Catalog published. Review the connection before enabling it."); + } catch { + setReview(null); + await Promise.all([catalogs.refetch(), refreshConnection()]); + setMessage( + "Could not publish. The connection or catalog may have changed. Reload and review the draft again.", + ); + } finally { + setBusy(false); + } + } + const published = catalogs.data?.published; + const draft = catalogs.data?.draft; + const draftNames = new Set(draft?.tools.map((tool) => tool.upstreamName)); + const staleDraft = + !!draft && + (draft.configRevision !== catalogs.data?.configRevision || + draft.authGeneration !== authGeneration); + const previous = new Map(published?.tools.map((tool) => [tool.upstreamName, tool])); + return ( +
+

Tool catalog

+ {catalogs.isError &&

Catalog data is unavailable.

} + {catalogs.data && ( +

+ Published revision {catalogs.data.catalogRevision} · {published?.tools.length ?? 0} tools + · {draft ? "draft available" : "no draft"} +

+ )} + {published && ( +
+ Published tools + {published.tools.map((tool) => ( +

+ {tool.upstreamName} · {tool.classification} ·{" "} + {tool.enabled ? "enabled" : "disabled"} +

+ ))} +
+ )} + {published?.tools + .filter((tool) => draft && !draftNames.has(tool.upstreamName)) + .map((tool) => ( +

+ Removed from draft: {tool.upstreamName} +

+ ))} + {staleDraft && ( +

This draft is stale. Discover the tools again before publishing.

+ )} +
+ + +
+ {review && ( +
+ {review.map((tool, index) => ( +
+ {tool.upstreamName} + + {!previous.has(tool.upstreamName) + ? "New" + : previous.get(tool.upstreamName)?.inputSchemaJson !== tool.inputSchemaJson || + previous.get(tool.upstreamName)?.outputSchemaJson !== tool.outputSchemaJson || + previous.get(tool.upstreamName)?.description !== tool.description + ? "Changed" + : "Unchanged"} + + {previous.has(tool.upstreamName) && ( +
+ Previous review +

+ {previous.get(tool.upstreamName)?.description} +

+
+                    {previous.get(tool.upstreamName)?.inputSchemaJson}
+                  
+
+                    {previous.get(tool.upstreamName)?.outputSchemaJson}
+                  
+

+ {previous.get(tool.upstreamName)?.classification} ·{" "} + {previous.get(tool.upstreamName)?.enabled ? "enabled" : "disabled"} +

+
+ )} +

{tool.description}

+
+ Input schema +
{tool.inputSchemaJson}
+
+ {tool.outputSchemaJson && ( +
+ Output schema +
{tool.outputSchemaJson}
+
+ )} + + +
+ ))} + +
+ )} + {message &&

{message}

} +
+ ); +} diff --git a/web/src/components/integrations/McpConnections.test.tsx b/web/src/components/integrations/McpConnections.test.tsx new file mode 100644 index 000000000..b3c584cab --- /dev/null +++ b/web/src/components/integrations/McpConnections.test.tsx @@ -0,0 +1,306 @@ +import { describe, expect, it } from "vitest"; +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { create } from "@bufbuild/protobuf"; +import { Code, ConnectError, createRouterTransport } from "@connectrpc/connect"; +import { renderWithProviders } from "@/test-utils"; +import { GetMcpSetupResponseSchema, IntegrationService } from "@/gen/engram/app/v1/integration_pb"; +import { McpConnections } from "./McpConnections"; +import { MCP_CALLBACK_PATH, saveMcpFlow } from "@/mcp-callback-bootstrap"; + +describe("external MCP connections", () => { + it("offers only approved endpoints and creates disabled metadata", async () => { + const requests: unknown[] = []; + const transport = createRouterTransport((router) => + router.service(IntegrationService, { + listConnections: () => ({ connections: [] }), + createConnection: (req) => { + requests.push(req); + return {}; + }, + }), + ); + renderWithProviders( + , + { transport }, + ); + const user = userEvent.setup(); + await user.type(await screen.findByLabelText("Connection name"), "Tools"); + await user.type(screen.getByLabelText("Connection alias"), "tools"); + await user.click(screen.getByRole("button", { name: "Create connection" })); + await waitFor(() => expect(requests).toHaveLength(1)); + expect(requests[0]).toMatchObject({ + alias: "tools", + provider: "mcp", + mcp: { zone: "public", endpoint: "https://tools.example/mcp", authKind: "api_key" }, + }); + }); + + it("sends an API key directly and clears the input after failure", async () => { + const requests: string[] = []; + const connection = { + id: "c1", + provider: "mcp", + alias: "tools", + displayName: "Tools", + configRevision: 1, + authGeneration: 1, + mcp: { endpoint: "https://tools.example/mcp", zone: "public", authKind: "api_key" }, + }; + const transport = createRouterTransport((router) => + router.service(IntegrationService, { + listConnections: () => ({ connections: [connection] }), + getMcpSetup: () => ({ + enabled: true, + current: { connection, credential: { status: "not_configured" } }, + }), + getMcpCatalogs: () => ({}), + putMcpCredential: (req) => { + requests.push(req.value); + throw new Error("failed"); + }, + }), + ); + const { queryClient } = renderWithProviders( + , + { transport }, + ); + const user = userEvent.setup(); + const input = (await screen.findByLabelText("API key for Tools")) as HTMLInputElement; + await user.type(input, "key-secret"); + await user.click(screen.getByRole("button", { name: "Save key" })); + await waitFor(() => expect(requests).toEqual(["key-secret"])); + await waitFor(() => expect(input.value).toBe("")); + expect(queryClient.getMutationCache().getAll()).toHaveLength(0); + expect(JSON.stringify(sessionStorage)).not.toContain("key-secret"); + }); + it("does not start another OAuth flow while this tab has one", async () => { + const redirectUri = `${window.location.origin}${MCP_CALLBACK_PATH}`; + saveMcpFlow(sessionStorage, { + connectionId: "other", + flowId: "f1", + configRevision: 1, + authGeneration: 1, + redirectUri, + expiresAt: "2099-01-01T00:00:00Z", + }); + const connection = { + id: "c1", + provider: "mcp", + alias: "tools", + displayName: "Tools", + configRevision: 1, + authGeneration: 1, + mcp: { + endpoint: "https://tools.example/mcp", + zone: "public", + authKind: "oauth", + oauthClientRegistrationId: "reg", + shareWithOrgSessions: true, + }, + }; + let calls = 0; + const transport = createRouterTransport((router) => + router.service(IntegrationService, { + listConnections: () => ({ connections: [connection] }), + getMcpSetup: () => ({ enabled: true, current: { connection } }), + getMcpCatalogs: () => ({}), + authorizeMcp: () => { + calls++; + return {}; + }, + }), + ); + try { + renderWithProviders( + , + { transport }, + ); + const user = userEvent.setup(); + await user.click(await screen.findByRole("button", { name: "Authorize" })); + await screen.findByRole("alert"); + expect(calls).toBe(0); + expect(sessionStorage.length).toBe(1); + } finally { + sessionStorage.clear(); + } + }); + it("keeps existing connections visible and marks unavailable actions when disabled", async () => { + const connection = { + id: "c1", + provider: "mcp", + alias: "tools", + displayName: "Tools", + mcp: { endpoint: "https://tools.example/mcp", zone: "public", authKind: "api_key" }, + }; + const transport = createRouterTransport((router) => + router.service(IntegrationService, { + listConnections: () => ({ connections: [connection] }), + getMcpSetup: () => ({ enabled: false, current: { connection } }), + getMcpCatalogs: () => ({}), + }), + ); + renderWithProviders( + , + { transport }, + ); + expect(await screen.findByRole("region", { name: "Tools MCP connection" })).toBeTruthy(); + expect((screen.getByRole("button", { name: "Recover" }) as HTMLButtonElement).disabled).toBe( + true, + ); + expect(screen.getByText(/new external tool dispatch is disabled/i)).toBeTruthy(); + expect(screen.getByText(/credential recovery is unavailable/i)).toBeTruthy(); + }); +}); + +it("keeps configured recovery and disconnect available while new dispatch is disabled", async () => { + const connection = { + id: "c1", + provider: "mcp", + alias: "tools", + displayName: "Tools", + configRevision: 1, + authGeneration: 1, + mcp: { endpoint: "https://tools.example/mcp", zone: "public", authKind: "api_key" }, + }; + const calls: string[] = []; + const transport = createRouterTransport((router) => + router.service(IntegrationService, { + listConnections: () => ({ connections: [connection] }), + getMcpSetup: () => ({ enabled: false, recoveryAvailable: true, current: { connection } }), + getMcpCatalogs: () => ({}), + recoverMcpCredential: () => { + calls.push("recover"); + return {}; + }, + disconnectMcp: () => { + calls.push("disconnect"); + return {}; + }, + }), + ); + renderWithProviders( + , + { transport }, + ); + const user = userEvent.setup(); + expect( + ((await screen.findByRole("button", { name: "Recover" })) as HTMLButtonElement).disabled, + ).toBe(false); + expect((screen.getByRole("button", { name: "Disconnect" }) as HTMLButtonElement).disabled).toBe( + false, + ); + expect((screen.getByRole("button", { name: "Save key" }) as HTMLButtonElement).disabled).toBe( + true, + ); + expect((screen.getByRole("button", { name: "Enable" }) as HTMLButtonElement).disabled).toBe(true); + await user.click(screen.getByRole("button", { name: "Recover" })); + await waitFor(() => expect(calls).toEqual(["recover"])); + await user.click(screen.getByRole("button", { name: "Disconnect" })); + await waitFor(() => expect(calls).toEqual(["recover", "disconnect"])); +}); + +it.each([false, true])( + "refreshes the connection state after catalog publication (conflict: %s)", + async (conflict) => { + let connection = { + id: "c1", + provider: "mcp", + alias: "tools", + displayName: "Tools", + enabled: true, + configRevision: 1, + authGeneration: 1, + catalogRevision: 2, + mcp: { endpoint: "https://tools.example/mcp", zone: "public", authKind: "api_key" }, + }; + let published = false; + const transport = createRouterTransport((router) => + router.service(IntegrationService, { + listConnections: () => ({ connections: [connection] }), + getMcpSetup: () => ({ current: { connection } }), + getMcpCatalogs: () => ({ + configRevision: connection.configRevision, + catalogRevision: connection.catalogRevision, + draft: + !published || conflict + ? { + id: published ? "new-draft" : "draft", + configRevision: connection.configRevision, + authGeneration: connection.authGeneration, + tools: [{ upstreamName: "read.item", inputSchemaJson: '{"type":"object"}' }], + } + : undefined, + }), + publishMcp: () => { + published = true; + connection = { + ...connection, + enabled: false, + configRevision: 2, + authGeneration: 2, + catalogRevision: 3, + }; + if (conflict) throw new ConnectError("stale", Code.Aborted); + return {}; + }, + setConnectionEnabled: (request) => { + if (request.expectedRevision !== connection.configRevision) { + throw new ConnectError("stale", Code.Aborted); + } + connection = { ...connection, enabled: request.enabled }; + return { connection }; + }, + }), + ); + renderWithProviders( + , + { transport }, + ); + const user = userEvent.setup(); + await screen.findByRole("button", { name: "Disable" }); + await user.click(await screen.findByRole("button", { name: "Review draft" })); + await user.selectOptions(screen.getByLabelText("Classification for read.item"), "read"); + await user.click(screen.getByRole("button", { name: "Publish reviewed catalog" })); + await screen.findByText(conflict ? /reload and review the draft again/i : /catalog published/i); + expect(screen.getByText(/public · https:\/\/tools.example\/mcp · Disabled/)).toBeTruthy(); + expect(screen.getByText(/Published revision 3/)).toBeTruthy(); + const enable = screen.getByRole("button", { name: "Enable" }); + expect((enable as HTMLButtonElement).disabled).toBe(false); + if (conflict) { + expect( + (screen.getByRole("button", { name: "Review draft" }) as HTMLButtonElement).disabled, + ).toBe(false); + expect(screen.queryByText(/this draft is stale/i)).toBeNull(); + } + await user.click(enable); + await screen.findByRole("button", { name: "Disable" }); + expect(screen.getByText(/public · https:\/\/tools.example\/mcp · Enabled/)).toBeTruthy(); + }, +); diff --git a/web/src/components/integrations/McpConnections.tsx b/web/src/components/integrations/McpConnections.tsx new file mode 100644 index 000000000..38c7aab21 --- /dev/null +++ b/web/src/components/integrations/McpConnections.tsx @@ -0,0 +1,618 @@ +import { createClient } from "@connectrpc/connect"; +import { useQuery, useTransport } from "@connectrpc/connect-query"; +import { useRef, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { + getMcpSetup, + listConnections, +} from "@/gen/engram/app/v1/integration-IntegrationService_connectquery"; +import { + IntegrationService, + type GetMcpSetupResponse, + type IntegrationConnection, +} from "@/gen/engram/app/v1/integration_pb"; +import { McpCatalogReview } from "./McpCatalogReview"; +import { + MCP_CALLBACK_PATH, + saveMcpFlow, + assertMcpFlowAvailable, + clearMcpFlowForConnection, + clearMcpFlow, +} from "@/mcp-callback-bootstrap"; + +export function McpConnections({ setup }: { setup: GetMcpSetupResponse }) { + const transport = useTransport(); + const client = createClient(IntegrationService, transport); + const connections = useQuery(listConnections, {}); + const [name, setName] = useState(""); + const [alias, setAlias] = useState(""); + const [destination, setDestination] = useState(0); + const [kind, setKind] = useState("api_key"); + const [registration, setRegistration] = useState(""); + const [sharing, setSharing] = useState(false); + const [message, setMessage] = useState(""); + const [busy, setBusy] = useState(false); + const selected = setup.destinations[destination]; + const redirect = `${window.location.origin}${MCP_CALLBACK_PATH}`; + const registrations = setup.registrations.filter( + (item) => + item.redirectUri === redirect && + item.resource?.zoneId === selected?.zoneId && + item.resource?.endpoint === selected?.endpoint, + ); + async function createConnection() { + if ( + !selected || + !name.trim() || + !alias.trim() || + (kind === "oauth" && (!sharing || !registrations.some((r) => r.id === registration))) + ) + return; + setBusy(true); + setMessage(""); + try { + await client.createConnection({ + alias: alias.trim(), + provider: "mcp", + displayName: name.trim(), + mcp: { + endpoint: selected.endpoint, + zone: selected.zoneId, + authKind: kind, + apiKeySecretRef: "", + oauthClientRegistrationId: kind === "oauth" ? registration : "", + shareWithOrgSessions: kind === "oauth" && sharing, + }, + }); + setName(""); + setAlias(""); + await connections.refetch(); + } catch { + setMessage("Could not create the connection. Check the settings and try again."); + } finally { + setBusy(false); + } + } + return ( +
+ {(connections.data?.connections ?? []) + .filter((c) => c.provider === "mcp") + .map((connection) => ( + connections.refetch()} + /> + ))} + {!setup.discoveryAvailable && ( +

Discovery is unavailable.

+ )} + {setup.enabled && setup.destinations.length > 0 && ( + <> +

Add a connection

+ + + + + {kind === "oauth" && ( + <> + + + + )} + + {message &&

{message}

} + + )} +
+ ); +} + +function McpConnection({ + connection, + setup, + refresh, +}: { + connection: IntegrationConnection; + setup: GetMcpSetupResponse; + refresh: () => unknown; +}) { + const client = createClient(IntegrationService, useTransport()); + const status = useQuery(getMcpSetup, { connectionId: connection.id }); + const secret = useRef(null); + const [header, setHeader] = useState("authorization"); + const [editing, setEditing] = useState(false); + const [confirmDelete, setConfirmDelete] = useState(false); + const [editName, setEditName] = useState(connection.displayName); + const [editAlias, setEditAlias] = useState(connection.alias); + const [editDestination, setEditDestination] = useState(""); + const [editKind, setEditKind] = useState("api_key"); + const [editRegistration, setEditRegistration] = useState(""); + const [editSharing, setEditSharing] = useState(false); + const [busy, setBusy] = useState(false); + const [message, setMessage] = useState(""); + const current = status.data?.current?.connection ?? connection; + async function refreshConnection() { + await Promise.all([status.refetch(), refresh()]); + } + async function run(action: () => Promise) { + setBusy(true); + setMessage(""); + try { + await action(); + await refreshConnection(); + } catch { + setMessage( + "The action could not be completed. Reload the connection and try again or recover it.", + ); + } finally { + if (secret.current) secret.current.value = ""; + setBusy(false); + } + } + const request = { + connectionId: current.id, + expectedRevision: current.configRevision, + expectedGeneration: current.authGeneration, + }; + async function authorize() { + assertMcpFlowAvailable(sessionStorage); + const reg = setup.registrations.find((r) => r.id === current.mcp?.oauthClientRegistrationId); + const redirectUri = `${window.location.origin}${MCP_CALLBACK_PATH}`; + if (!reg || reg.redirectUri !== redirectUri) throw new Error("Callback is not registered"); + const result = await client.authorizeMcp({ ...request, shareWithOrgSessions: true }); + saveMcpFlow(sessionStorage, { + connectionId: current.id, + flowId: result.flowId, + configRevision: result.configRevision, + authGeneration: result.authGeneration, + redirectUri, + expiresAt: result.expiresAt, + }); + window.location.assign(result.authorizationUrl); + } + return ( +
+

{connection.displayName}

+

+ {current.mcp?.zone} · {current.mcp?.endpoint} · {current.enabled ? "Enabled" : "Disabled"} +

+

+ Credential: {status.data?.current?.credential?.status ?? "unavailable"}{" "} + {status.data?.current?.credential?.reference ?? ""} +

+ {!setup.enabled && ( +

New external tool dispatch is disabled.

+ )} + {!setup.recoveryAvailable && ( +

Credential recovery is unavailable.

+ )} + {!editing ? ( + + ) : ( +
+

+ Changing connection settings can require new authorization and catalog review. +

+ + + + + {editKind === "oauth" && ( + <> + + + + )} +
+ + +
+
+ )} + {current.mcp?.authKind === "api_key" && ( + <> + + + + + )} + {current.mcp?.authKind === "oauth" && ( + <> + {setup.registrations.find((item) => item.id === current.mcp?.oauthClientRegistrationId) + ?.confidential && ( + <> +

+ Client secret:{" "} + {status.data?.clientSecretConfigured ? "configured" : "not configured"} +

+ + + + )} + + + )} +
+ + + +
+ {confirmDelete ? ( +
+ + Delete this connection? Profiles that use it must be updated first. + + + +
+ ) : ( + + )} + +
+ If this tab has lost authorization context, recover the connection and clear this tab before + starting again. +
+ + {message &&

{message}

} +
+ ); +} diff --git a/web/src/components/profiles/McpGrantEditor.test.tsx b/web/src/components/profiles/McpGrantEditor.test.tsx new file mode 100644 index 000000000..3bc0bc38b --- /dev/null +++ b/web/src/components/profiles/McpGrantEditor.test.tsx @@ -0,0 +1,108 @@ +import { create } from "@bufbuild/protobuf"; +import { createRouterTransport } from "@connectrpc/connect"; +import { useState } from "react"; +import { expect, it } from "vitest"; +import { fireEvent, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { + IntegrationConnectionSchema, + IntegrationService, +} from "@/gen/engram/app/v1/integration_pb"; +import { renderWithProviders } from "@/test-utils"; +import { McpGrantEditor } from "./McpGrantEditor"; +import type { McpProfileGrant } from "./profileForm"; + +it("requires a constraint or explicit unrestricted choice and keeps documents separate", async () => { + const connection = create(IntegrationConnectionSchema, { + id: "c1", + displayName: "Tools", + provider: "mcp", + configRevision: 1, + authGeneration: 1, + catalogRevision: 1, + }); + const transport = createRouterTransport((router) => + router.service(IntegrationService, { + getMcpCatalogs: () => ({ + catalogRevision: 1, + published: { + id: "p1", + revision: 1, + configRevision: 1, + authGeneration: 1, + tools: [{ upstreamName: "write.item", classification: "write", enabled: true }], + }, + }), + }), + ); + function Fixture() { + const [grants, setGrants] = useState([]); + return ( + <> + {}} + /> + {JSON.stringify(grants)} + + ); + } + renderWithProviders(, { transport }); + const user = userEvent.setup(); + await user.selectOptions(await screen.findByLabelText("Reviewed tool for Tools"), "write.item"); + expect( + (screen.getByRole("button", { name: "Add MCP grant" }) as HTMLButtonElement).disabled, + ).toBe(true); + fireEvent.change(screen.getByLabelText("Argument constraint JSON"), { + target: { value: '{"type":"object"}' }, + }); + await user.click(screen.getByRole("button", { name: "Add MCP grant" })); + await user.click(screen.getByRole("button", { name: "Add constraint document" })); + const output = JSON.parse(screen.getByRole("status").textContent ?? "[]"); + expect(output[0].mcp.argumentConstraintDocumentsJson).toEqual(['{"type":"object"}', ""]); + expect(output[0].mcp.unrestricted).toBe(false); +}); + +it("does not offer tools or publication review for a stale catalog", async () => { + const connection = create(IntegrationConnectionSchema, { + id: "c1", + displayName: "Tools", + provider: "mcp", + configRevision: 2, + authGeneration: 1, + catalogRevision: 1, + }); + const transport = createRouterTransport((router) => + router.service(IntegrationService, { + getMcpCatalogs: () => ({ + catalogRevision: 1, + published: { + id: "p1", + revision: 1, + configRevision: 1, + authGeneration: 1, + tools: [{ upstreamName: "write.item", classification: "write", enabled: true }], + }, + }), + }), + ); + const reports: boolean[] = []; + renderWithProviders( + {}} + onAvailability={(_id, ready) => { + reports.push(ready); + }} + />, + { transport }, + ); + expect((await screen.findByRole("alert")).textContent).toContain( + "current reviewed catalog is unavailable", + ); + expect(screen.queryByLabelText("Reviewed tool for Tools")).toBeNull(); + expect(reports.at(-1)).toBe(false); +}); diff --git a/web/src/components/profiles/McpGrantEditor.tsx b/web/src/components/profiles/McpGrantEditor.tsx new file mode 100644 index 000000000..714bfcb95 --- /dev/null +++ b/web/src/components/profiles/McpGrantEditor.tsx @@ -0,0 +1,254 @@ +import { useQuery } from "@connectrpc/connect-query"; +import { useEffect, useState } from "react"; +import { Button } from "@/components/ui/button"; +import { getMcpCatalogs } from "@/gen/engram/app/v1/integration-IntegrationService_connectquery"; +import type { IntegrationConnection } from "@/gen/engram/app/v1/integration_pb"; +import type { McpProfileGrant } from "./profileForm"; + +export function McpGrantEditor({ + connection, + grants, + onChange, + onAvailability, +}: { + connection: IntegrationConnection; + grants: McpProfileGrant[]; + onChange: (grants: McpProfileGrant[]) => void; + onAvailability: (id: string, available: boolean) => void; +}) { + const catalogs = useQuery(getMcpCatalogs, { connectionId: connection.id }); + const [toolName, setToolName] = useState(""); + const [document, setDocument] = useState(""); + const [unrestricted, setUnrestricted] = useState(false); + const [automation, setAutomation] = useState(false); + const own = grants.filter((g) => g.connectionId === connection.id && g.mcp); + const publication = catalogs.data?.published; + const currentCatalog = + publication && + publication.revision === connection.catalogRevision && + publication.configRevision === connection.configRevision && + publication.authGeneration === connection.authGeneration; + const canReview = + !catalogs.isError && + !catalogs.isFetching && + !!currentCatalog && + own.every((grant) => + publication?.tools.some( + (tool) => tool.enabled && tool.upstreamName === grant.mcp?.upstreamName, + ), + ); + const availabilityKey = `${connection.id}:${connection.configRevision}:${connection.authGeneration}:${connection.catalogRevision}`; + useEffect(() => { + onAvailability(availabilityKey, canReview); + }, [availabilityKey, canReview, onAvailability]); + const available = ( + currentCatalog && !catalogs.isError && !catalogs.isFetching ? publication.tools : [] + ).filter((t) => t.enabled && !own.some((g) => g.mcp?.upstreamName === t.upstreamName)); + function validDocument(value: string) { + try { + const parsed: unknown = JSON.parse(value); + return ( + !!parsed && + typeof parsed === "object" && + !Array.isArray(parsed) && + Object.keys(parsed).length > 0 + ); + } catch { + return false; + } + } + function add() { + if ( + !available.some((tool) => tool.upstreamName === toolName) || + (!unrestricted && !validDocument(document)) + ) + return; + onChange([ + ...grants, + { + connectionId: connection.id, + operation: "invoke", + resourceConstraints: [], + credentialScope: "org", + mcp: { + upstreamName: toolName, + automationWrite: automation, + unrestricted, + argumentConstraintDocumentsJson: unrestricted ? [] : [document], + }, + }, + ]); + setToolName(""); + setDocument(""); + setUnrestricted(false); + setAutomation(false); + } + return ( +
+

{connection.displayName} MCP grants

+ {(catalogs.isError || (catalogs.data && !currentCatalog)) && ( +

The current reviewed catalog is unavailable.

+ )} + {own.map((grant) => { + const mcp = grant.mcp!; + function update(next: typeof mcp) { + onChange(grants.map((g) => (g === grant ? { ...g, mcp: next } : g))); + } + return ( +
+ {mcp.upstreamName} + + {grant.operation} · {grant.credentialScope || "org"} + + {mcp.argumentConstraintDocumentsJson.map((value, index) => ( +