From df77c4083f652f73b9cfc382c5631ee9587d3249 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 11:46:42 +0100 Subject: [PATCH 01/24] =?UTF-8?q?feat(models):=201/4=20=E2=80=94=20add=20A?= =?UTF-8?q?gentKind.PI=20+=20PiAgentConfig=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the Pi harness's config surface: AgentKind.PI enum member, a 7-value PiThinkingLevel literal (strict superset of ThinkingLevel), PiAgentConfig in the AgentConfig discriminated union, and the models/__init__ export. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/coder_eval/models/__init__.py | 2 + src/coder_eval/models/agent_config.py | 50 +++++++++++++++++- src/coder_eval/models/enums.py | 1 + tests/test_pi_agent_config.py | 73 +++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 tests/test_pi_agent_config.py diff --git a/src/coder_eval/models/__init__.py b/src/coder_eval/models/__init__.py index 97d4f64c..e9eab8bc 100644 --- a/src/coder_eval/models/__init__.py +++ b/src/coder_eval/models/__init__.py @@ -14,6 +14,7 @@ LocalPluginConfig, NoneAgentConfig, OpenCodeAgentConfig, + PiAgentConfig, ResolvedAgentConfig, SystemPromptMode, SystemPromptSemantics, @@ -227,6 +228,7 @@ "LocalPluginConfig", "NoneAgentConfig", "OpenCodeAgentConfig", + "PiAgentConfig", "ResolvedAgentConfig", "SystemPromptMode", "SystemPromptSemantics", diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index a22756f7..e666a406 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -381,6 +381,49 @@ class OpenCodeAgentConfig(BaseAgentConfig): ) +# Pi's ``--thinking`` reasoning-effort set. A STRICT SUPERSET of ThinkingLevel +# (adds ``off``/``xhigh``/``max``), so Pi gets its own literal rather than reusing +# the 4-value one — reuse would forbid valid Pi levels. Spike-confirmed against +# ``pi --help`` on Pi 0.84.4. +type PiThinkingLevel = Literal["off", "minimal", "low", "medium", "high", "xhigh", "max"] + + +class PiAgentConfig(BaseAgentConfig): + """Pi agent configuration (the ``pi`` Node coding agent — https://pi.dev/). + + Drives the ``pi`` CLI in JSON print mode + (``pi -p --mode json``), which streams newline-delimited JSON events on + stdout. ``model`` is Pi's provider-prefixed ``provider/model`` form (e.g. + ``openrouter/moonshotai/kimi-k3``) and is passed through verbatim via + ``--model`` (no separate ``--provider`` needed). + + Session continuity (multi-turn / simulation) + -------------------------------------------- + Each ``communicate()`` is one ``pi`` subprocess. A standard task reaches its + solution inside a single ``communicate()`` (Pi runs its own multi-step agent + loop). A simulation/dialog task calls ``communicate()`` once per user turn and + relies on the agent remembering prior turns — so ``PiAgent`` reuses a per-agent + ``--session-dir`` + stable ``--session-id`` on every invocation (create-if-missing + on turn 1, resume after), mirroring OpenCode's ``--session`` continuity. + + Enforced vs unenforced fields + ----------------------------- + ``allowed_tools`` → ``--tools``, ``disallowed_tools`` → ``--exclude-tools`` and + ``system_prompt`` → ``--append-system-prompt`` ARE enforced (a capability win + over OpenCode). ``permission_mode`` is NOT enforced (Pi headless print mode + auto-runs tools; the sandbox driver is the isolation boundary), ``plugins`` are + NOT injected (no activation suites in v1), and ``system_prompt_file`` is NOT + read — all three are warned about at ``start()``. See ``docs/agents/PI.md``. + """ + + type: Literal[AgentKind.PI] # type: ignore[assignment] + + thinking_level: PiThinkingLevel = Field( + default="medium", + description="Pi reasoning effort passed as --thinking (off/minimal/low/medium/high/xhigh/max).", + ) + + class NoneAgentConfig(BaseAgentConfig): """No-op ("agentless") agent configuration. @@ -405,7 +448,12 @@ class NoneAgentConfig(BaseAgentConfig): # Only includes the concrete subclasses (not BaseAgentConfig) since the discriminator # must be a Literal type. BaseAgentConfig is returned by parse_agent_config when type=None. type AgentConfig = Annotated[ - ClaudeCodeAgentConfig | CodexAgentConfig | AntigravityAgentConfig | OpenCodeAgentConfig | NoneAgentConfig, + ClaudeCodeAgentConfig + | CodexAgentConfig + | AntigravityAgentConfig + | OpenCodeAgentConfig + | PiAgentConfig + | NoneAgentConfig, Field(discriminator="type"), ] diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 0cba3650..10ed5c20 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -109,6 +109,7 @@ class AgentKind(StrEnum): CODEX = "codex" ANTIGRAVITY = "antigravity" OPENCODE = "opencode" + PI = "pi" NONE = "none" # Agentless / system task — no coding agent runs; success criteria do all the work. UNKNOWN = "unknown" # Used when agent type cannot be determined (e.g., task loading failure) diff --git a/tests/test_pi_agent_config.py b/tests/test_pi_agent_config.py new file mode 100644 index 00000000..db0cb1d0 --- /dev/null +++ b/tests/test_pi_agent_config.py @@ -0,0 +1,73 @@ +"""Phase 1: ``PiAgentConfig`` model + ``AgentKind.PI`` enum member. + +Registry-dispatch coverage (``parse_agent_config(type="pi")`` returning a +``PiAgentConfig``) lives in ``tests/test_pi_agent.py`` — it needs Phase 2's +registration. Here we exercise the model directly. +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from coder_eval.models import AgentConfig, AgentKind, PiAgentConfig +from coder_eval.models.agent_config import PiThinkingLevel + + +def test_agentkind_pi_value(): + assert AgentKind.PI == "pi" + + +def test_importable_from_models(): + # `from coder_eval.models import PiAgentConfig` must succeed (CE001). + assert PiAgentConfig.__name__ == "PiAgentConfig" + + +def test_validates_and_defaults(): + cfg = PiAgentConfig(type="pi") + assert cfg.type == AgentKind.PI + assert cfg.thinking_level == "medium" + + +def test_round_trips_through_model_dump(): + cfg = PiAgentConfig(type="pi", model="openrouter/moonshotai/kimi-k3", thinking_level="high") + restored = PiAgentConfig.model_validate(cfg.model_dump()) + assert restored == cfg + assert restored.model == "openrouter/moonshotai/kimi-k3" + assert restored.thinking_level == "high" + + +@pytest.mark.parametrize("level", ["off", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_all_seven_thinking_levels_validate(level): + assert PiAgentConfig(type="pi", thinking_level=level).thinking_level == level + + +def test_thinking_level_literal_has_exactly_seven_values(): + # PiThinkingLevel is a strict superset of the 4-value ThinkingLevel. + assert set(PiThinkingLevel.__value__.__args__) == { # type: ignore[attr-defined] + "off", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + } + + +def test_invalid_thinking_level_rejected(): + with pytest.raises(ValidationError): + PiAgentConfig(type="pi", thinking_level="ultra") # type: ignore[arg-type] + + +def test_unknown_extra_field_rejected(): + with pytest.raises(ValidationError): + PiAgentConfig(type="pi", not_a_field=1) # type: ignore[call-arg] + + +def test_member_of_discriminated_union(): + from pydantic import TypeAdapter + + adapter = TypeAdapter(AgentConfig) + cfg = adapter.validate_python({"type": "pi", "model": "openrouter/moonshotai/kimi-k3"}) + assert isinstance(cfg, PiAgentConfig) From 29515acaa0d5d131ba5f53aeb244c2b504665b95 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 11:59:12 +0100 Subject: [PATCH 02/24] =?UTF-8?q?feat(agents):=202/4=20=E2=80=94=20add=20P?= =?UTF-8?q?iAgent=20harness=20(pi=20--mode=20json)=20+=20registration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drive the pi Node CLI in JSON print mode over a subprocess, reducing its nd-JSON stream into the standardized event protocol + TurnRecord via EventCollector (mirrors OpenCodeAgent). Native turn cap off turn_start, cooperative should_stop, turn_timeout, crash/timeout pending_turn capture, per-agent --session-dir/--session-id continuity for simulation, and enforced allowed_tools/disallowed_tools/system_prompt. Registered via register_builtins; pi = [] extra added. Offline replay tests seeded from a byte-real 0.84.4 capture. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 11 + src/coder_eval/agents/__init__.py | 7 +- src/coder_eval/agents/pi_agent.py | 1078 ++++++++++++++++++++++++++ tests/fixtures/pi_happy_stream.jsonl | 45 ++ tests/test_pi_agent.py | 803 +++++++++++++++++++ 5 files changed, 1942 insertions(+), 2 deletions(-) create mode 100644 src/coder_eval/agents/pi_agent.py create mode 100644 tests/fixtures/pi_happy_stream.jsonl create mode 100644 tests/test_pi_agent.py diff --git a/pyproject.toml b/pyproject.toml index 5eea61f9..d382cfdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,6 +145,17 @@ antigravity = [ # fail at start() with a clear hint pointing back here. # See docs/agents/OPENCODE.md. opencode = [] +# Optional extra that enables Pi agent support. +# +# Deliberately EMPTY, exactly like `opencode` above. Pi ships as a Node CLI +# (`npm install -g @earendil-works/pi-coding-agent`), and PiAgent drives that +# binary over a subprocess reading `pi -p --mode json` — it imports no +# third-party Python package. The extra exists to keep the opt-in surface +# uniform across harnesses and to give the `npm install` prerequisite a single +# home in the packaging metadata. Without the CLI on PATH the framework still +# installs and runs; Pi tasks fail at start() with a clear hint pointing here. +# See docs/agents/PI.md. +pi = [] [project.scripts] coder-eval = "coder_eval.cli:app" diff --git a/src/coder_eval/agents/__init__.py b/src/coder_eval/agents/__init__.py index 0bbf0dde..c56ecf39 100644 --- a/src/coder_eval/agents/__init__.py +++ b/src/coder_eval/agents/__init__.py @@ -6,12 +6,13 @@ from coder_eval.agents.codex_agent import CodexAgent from coder_eval.agents.noop_agent import NoOpAgent from coder_eval.agents.opencode_agent import OpenCodeAgent +from coder_eval.agents.pi_agent import PiAgent from coder_eval.agents.registry import AgentRegistry, create_agent from coder_eval.models import AgentKind def register_builtins(registry: type[AgentRegistry]) -> None: - """Register the built-in agents (Claude/Codex/Antigravity/OpenCode/NoOp) onto ``registry``. + """Register the built-in agents (Claude/Codex/Antigravity/OpenCode/Pi/NoOp) onto ``registry``. This is the target of coder-eval's own ``coder_eval.plugins`` entry point, so the built-in agents travel the identical discovery path as any third-party @@ -21,7 +22,7 @@ def register_builtins(registry: type[AgentRegistry]) -> None: """ # Reference the imported classes so the registration side effect is explicit # and a future refactor that drops the top-level imports fails loudly here. - _ = (ClaudeCodeAgent, CodexAgent, AntigravityAgent, OpenCodeAgent, NoOpAgent) + _ = (ClaudeCodeAgent, CodexAgent, AntigravityAgent, OpenCodeAgent, PiAgent, NoOpAgent) # Rot-protection: the decorators fire on import, but assert the built-ins are # actually registered so a future lazy-import refactor (which would leave the # import-cached modules' decorators un-run) fails loudly instead of silently @@ -31,6 +32,7 @@ def register_builtins(registry: type[AgentRegistry]) -> None: AgentKind.CODEX, AgentKind.ANTIGRAVITY, AgentKind.OPENCODE, + AgentKind.PI, AgentKind.NONE, ): if registry.get(kind) is None: @@ -44,6 +46,7 @@ def register_builtins(registry: type[AgentRegistry]) -> None: "CodexAgent", "NoOpAgent", "OpenCodeAgent", + "PiAgent", "create_agent", "register_builtins", ] diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py new file mode 100644 index 00000000..3c7931bf --- /dev/null +++ b/src/coder_eval/agents/pi_agent.py @@ -0,0 +1,1078 @@ +"""Pi agent implementation (the ``pi`` Node coding agent — https://pi.dev/). + +Drives the ``pi`` CLI in JSON print mode:: + + pi -p --mode json --no-context-files --no-approve \ + --session-dir --session-id [--model provider/id] \ + [--thinking L] [--tools ...] [--exclude-tools ...] \ + [--append-system-prompt S] -- + +which streams **newline-delimited JSON events** on stdout. Each line is one +event; this module reduces that stream into the standardized coder_eval event +protocol (``AgentStart`` / ``TurnStart`` / ``ToolStart`` / ``ToolEnd`` / +``TurnEnd`` / ``AgentEnd``) and lets :class:`EventCollector` build the +``TurnRecord`` — so no telemetry is assembled by hand here. The design mirrors +:mod:`coder_eval.agents.opencode_agent` almost verbatim; the differences are +noted inline. + +Event grammar (captured from ``pi`` 0.84.4) +------------------------------------------- +- ``{"type": "session", ...}`` — always line 1 (cwd/id/version). +- ``{"type": "agent_start"}`` — bare; **can appear more than once** per + invocation (Pi auto-retries a transient/provider error internally). +- ``{"type": "turn_start"}`` — bare; one per agent-loop step. This is the unit + ``max_turns`` counts. +- ``{"type": "message_update", "assistantMessageEvent": {...}}`` — streaming + deltas (text/thinking/toolcall). Text deltas drive ``TextChunkEvent``. +- ``{"type": "message_end", "message": {...}}`` — a complete message. Ignored + for token accounting: ``turn_end`` echoes the same assistant usage once per + step, and reading both would double-count. +- ``{"type": "tool_execution_start", "toolCallId": ..., "toolName": ..., + "args": {...}}`` / ``{"type": "tool_execution_end", "toolCallId": ..., + "result": ..., "isError": ...}``. +- ``{"type": "turn_end", "message": {...assistant...}, "toolResults": [...]}`` + — ``message.usage`` is that step's OWN usage (per-generation), summed across + steps for the turn total. +- ``{"type": "agent_end", "messages": [...], "willRetry": }`` — + ``willRetry: true`` means another retry cycle follows in the SAME invocation. +- ``{"type": "agent_settled"}`` — the true terminal event (after all retries); + emit the single ``AgentEndEvent`` here / at EOF, NOT on the first + ``agent_end``. + +Token semantics +--------------- +Per-generation, **SUM** (identical to OpenCode; NOT cumulative). Each +``turn_end.message.usage`` carries ``{input, output, cacheRead, cacheWrite, +[reasoning], totalTokens, cost:{...,total}}`` for that step, where +``totalTokens == input + output + cacheRead + cacheWrite``. Unlike OpenCode, +Pi's ``input`` IS the fresh slice already (no flat/nested arbitration needed), +so it maps straight to ``uncached_input_tokens``. ``reasoning`` bills at the +output rate. ``cost.total`` per step is summed into +``token_usage.total_cost_usd`` (the rate card is only a fallback when the +stream omits cost). + +Session continuity +------------------ +A per-agent ``--session-dir`` + stable ``--session-id`` are assigned in +``start()`` and replayed on every ``communicate()`` — create-if-missing on the +first call, resume after — which is what makes multi-turn (dialog-mode) +evaluation work across separate CLI invocations. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import os +import shutil +import signal +import tempfile +import time +from collections.abc import Callable +from datetime import datetime +from typing import Any, ClassVar, Literal, NoReturn +from uuid import uuid4 + +from coder_eval.agent import Agent +from coder_eval.errors import AgentCrashError, TurnTimeoutError +from coder_eval.isolation.docker_runner import STDOUT_LINE_LIMIT_BYTES +from coder_eval.models import ( + AgentKind, + AgentState, + ApiRoute, + AssistantMessage, + CommandTelemetry, + ContentBlock, + PiAgentConfig, + ResultSummary, + SystemPromptSemantics, + TokenUsage, + TranscriptMessage, + TurnRecord, +) +from coder_eval.pricing import calculate_cost +from coder_eval.streaming.callbacks import StreamCallback, safe_emit +from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentEndStatus, + AgentStartEvent, + StreamEvent, + TextChunkEvent, + ToolEndEvent, + ToolEndStatus, + ToolStartEvent, + TurnEndEvent, + TurnEndStatus, + TurnStartEvent, +) + +from .registry import AgentRegistry + + +logger = logging.getLogger(__name__) + +# Grace period between SIGTERM and SIGKILL when tearing down the CLI subprocess. +# Doubles as the post-EOF exit grace in _settle_turn when no turn deadline is +# configured. Genuinely OpenCode-module-private (not exported), so re-declared +# here at the same value rather than imported (a shared-module hoist is out of +# scope); STDOUT_LINE_LIMIT_BYTES, which IS canonical, is imported above. +_TERM_GRACE_SECONDS = 5.0 + +# SIGKILL does not exist on Windows (where the process-group sweep is a no-op +# anyway); resolve it dynamically so the module imports and typechecks on every +# platform, falling back to SIGTERM for the direct-pid kill_sync path. +_SIGKILL: signal.Signals = getattr(signal, "SIGKILL", signal.SIGTERM) + +# How long to keep draining stdout/stderr after the CLI process has been reaped. +# A print-mode CLI may leave an inherited pipe open, so every post-exit read +# must be bounded. +_DRAIN_SECONDS = 2.0 + +# How many distinct unrecognized event-type strings to retain for the crash +# message when the vocabulary check fails (diagnosis, not an exhaustive list). +_MAX_UNRECOGNIZED_TYPES = 8 + +# pi's native tool names -> the canonical (Claude) vocabulary every criterion is +# written against. Mirrors opencode_agent._TOOL_NAME_MAP: without it a +# `command_executed` with `tool_name: Bash` matches nothing on a Pi run. Unknown +# tools pass through unchanged. +_TOOL_NAME_MAP: dict[str, str] = { + "bash": "Bash", + "read": "Read", + "write": "Write", + "edit": "Edit", + "patch": "Edit", + "multiedit": "Edit", + "glob": "Glob", + "grep": "Grep", + "list": "LS", + "ls": "LS", + "webfetch": "WebFetch", + "todowrite": "TodoWrite", + "todoread": "TodoRead", + "task": "Agent", +} + +# pi per-tool INPUT-arg key -> canonical (Claude) key. Mirrors +# opencode_agent._OPENCODE_ARG_RENAME. The spike's write/read tools used `path`, +# so map it to `file_path` for Read/Write/Edit (the search tools keep `path`, +# which is already Claude's key). Keyed by the canonical tool name (post +# _TOOL_NAME_MAP); unlisted keys pass through. +_PI_ARG_RENAME: dict[str, dict[str, str]] = { + "Read": {"path": "file_path"}, + "Write": {"path": "file_path"}, + "Edit": { + "path": "file_path", + "oldString": "old_string", + "newString": "new_string", + "replaceAll": "replace_all", + }, +} + +# Config fields the Pi CLI has no equivalent knob for (v1). `experiments/default.yaml` +# sets `permission_mode` on every task, so warn once at start() rather than let a +# task believe it constrained the agent. NOTE `allowed_tools`/`disallowed_tools`/ +# `system_prompt` are SUPPORTED (mapped to --tools/--exclude-tools/ +# --append-system-prompt) and thus deliberately NOT here. +_UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ("plugins", "permission_mode", "system_prompt_file") + +# The full recognized Pi vocabulary (from `pi` 0.84.4). A clean exit that +# recognized NOTHING from this set is vocabulary drift and is crashed rather than +# scored as a silent empty success (see _settle_turn). +_RECOGNIZED_EVENTS = frozenset( + { + "session", + "agent_start", + "turn_start", + "message_start", + "message_update", + "message_end", + "tool_execution_start", + "tool_execution_end", + "turn_end", + "agent_end", + "agent_settled", + } +) + +# ToolEndStatus -> CommandTelemetry.result_status (the persisted tri-state). +_RESULT_STATUS: dict[ToolEndStatus, Literal["success", "error", "unknown"]] = { + ToolEndStatus.OK: "success", + ToolEndStatus.ERROR: "error", + ToolEndStatus.PERMISSION_DENIED: "error", + ToolEndStatus.UNRESOLVED: "unknown", +} + + +def _canonical_params(tool_name: str, params: dict[str, Any]) -> dict[str, Any]: + """Rename a tool call's argument keys to the canonical cross-agent vocabulary. + + Order is preserved and unlisted keys pass through untouched. + """ + rename = _PI_ARG_RENAME.get(tool_name) + if not rename: + return params + return {rename.get(key, key): value for key, value in params.items()} + + +def _result_text(result: Any) -> str | None: + """Best-effort flatten of a Pi ``tool_execution_end.result`` to text. + + Pi results are ``{"content": [{"type": "text", "text": "..."}, ...]}`` (spike + verified). Fall back to the string form for any other shape so a summary is + never silently dropped. + """ + if isinstance(result, dict): + content = result.get("content") + if isinstance(content, list): + parts = [str(item.get("text", "")) for item in content if isinstance(item, dict)] + joined = "".join(parts) + if joined: + return joined + if result is None: + return None + return str(result) + + +class _PiTurnState: + """Per-``communicate()`` accumulator: events in, finalization payload out. + + Owns everything the terminal ``AgentEndEvent`` must carry (transcript + messages, summed usage, text output) plus the open-tool bookkeeping needed + to force-close orphans when a turn dies mid-flight. + """ + + def __init__(self, *, task_id: str, iteration: int, user_input: str, model: str | None) -> None: + self.task_id = task_id + self.iteration = iteration + self.user_input = user_input + self.model = model + + self.started_at = time.monotonic() + self.thread_id: str | None = None + + # Cumulative turn totals (summed across every inner step). + self.usage = TokenUsage() + self.cost_usd: float = 0.0 + self.saw_cost = False + + self.messages: list[TranscriptMessage] = [] + self.text_parts: list[str] = [] + + # Pi `turn_start` events counted; this is what max_turns caps. + self.turn_count = 0 + self.turn_id: str = "" + # True between a step's `turn_start` and its `turn_end`. `finalize` needs + # it to close a TurnStartEvent the stream never got to close. + self.turn_open = False + self.turn_started_at: datetime | None = None + self.turn_text_parts: list[str] = [] + self.turn_tool_ids: list[str] = [] + + # toolCallId -> telemetry for tools awaiting a result. + self.open_tools: dict[str, CommandTelemetry] = {} + self.sequence = 0 + self.stop_reason: str | None = None + self.error_message: str | None = None + self.max_turns_exhausted = False + # Guards the one-terminal-event rule; see finalize(). + self.finalized = False + # Count of events matched against the recognized Pi vocabulary (drift check), + # plus a bounded sample of the types that did NOT match — so the drift crash + # message can name what it actually saw. + self.recognized_events = 0 + self.unrecognized_types: set[str] = set() + + self._emit: Callable[[StreamEvent], None] = lambda _e: None + + def bind(self, emit: Callable[[StreamEvent], None]) -> None: + self._emit = emit + + def emit(self, event: StreamEvent) -> None: + self._emit(event) + + @property + def agent_output(self) -> str: + return "".join(self.text_parts) + + # --- event handlers ---------------------------------------------------- + + def on_turn_start(self) -> None: + self.turn_count += 1 + self.turn_open = True + self.turn_id = f"turn_{self.turn_count}" + self.turn_started_at = datetime.now() + self.turn_text_parts = [] + self.turn_tool_ids = [] + self.emit( + TurnStartEvent( + task_id=self.task_id, + thread_id=self.thread_id, + turn_id=self.turn_id, + model=self.model, + ) + ) + + def on_message_update(self, obj: dict[str, Any]) -> None: + """Stream a ``text_delta`` as a ``TextChunkEvent`` (thinking/toolcall ignored).""" + event = obj.get("assistantMessageEvent") + if not isinstance(event, dict) or event.get("type") != "text_delta": + return + delta = event.get("delta") + if not isinstance(delta, str) or not delta: + return + self.text_parts.append(delta) + self.turn_text_parts.append(delta) + self.emit(TextChunkEvent(task_id=self.task_id, thread_id=self.thread_id, turn_id=self.turn_id, text=delta)) + + def on_tool_execution_start(self, obj: dict[str, Any]) -> None: + call_id = str(obj.get("toolCallId") or f"call_{self.sequence + 1}") + if call_id in self.open_tools: + return + self.sequence += 1 + raw_tool = str(obj.get("toolName") or "unknown") + tool_name = _TOOL_NAME_MAP.get(raw_tool.lower(), raw_tool) + args = obj.get("args") + params = args if isinstance(args, dict) else {} + started = datetime.now() + telemetry = CommandTelemetry( + tool_name=tool_name, + tool_id=call_id, + assistant_turn_index=self.turn_count, + timestamp=started, + execution_started_at=started, + parameters=_canonical_params(tool_name, params), + sequence_number=self.sequence, + ) + self.open_tools[call_id] = telemetry + self.turn_tool_ids.append(call_id) + self.emit(ToolStartEvent(task_id=self.task_id, thread_id=self.thread_id, turn_id=self.turn_id, tool=telemetry)) + + def on_tool_execution_end(self, obj: dict[str, Any]) -> None: + call_id = str(obj.get("toolCallId") or "") + summary = _result_text(obj.get("result")) + is_error = bool(obj.get("isError")) + if is_error: + message = summary or "tool failed" + # Best-effort: Pi does not tag permission denials distinctly, so infer + # from the result text. The persisted tri-state folds both to "error" + # (see _RESULT_STATUS), so a misclassified legit "permission denied" in + # output is cosmetic. Mirrors opencode_agent. + denied = "permission" in message.lower() or "denied" in message.lower() + status = ToolEndStatus.PERMISSION_DENIED if denied else ToolEndStatus.ERROR + else: + message = None + status = ToolEndStatus.OK + self._close_tool(call_id, status=status, summary=summary, error=message) + + def _close_tool( + self, + call_id: str, + *, + status: ToolEndStatus, + summary: str | None, + error: str | None, + ) -> None: + telemetry = self.open_tools.pop(call_id, None) + if telemetry is None: + # A result with no matching call (shouldn't happen, but never drop it). + self.sequence += 1 + telemetry = CommandTelemetry( + tool_name="unknown", + tool_id=call_id, + assistant_turn_index=self.turn_count, + timestamp=datetime.now(), + sequence_number=self.sequence, + ) + completed = datetime.now() + telemetry.execution_completed_at = completed + if telemetry.execution_started_at is not None: + telemetry.duration_ms = (completed - telemetry.execution_started_at).total_seconds() * 1000 + telemetry.result_status = _RESULT_STATUS[status] + # Stored untruncated by design (sub-agent returns must survive whole). + telemetry.result_summary = summary + telemetry.error_message = error + self.emit( + ToolEndEvent( + task_id=self.task_id, + thread_id=self.thread_id, + turn_id=self.turn_id, + tool=telemetry, + status=status, + ) + ) + + def _as_int(self, value: Any) -> int: + """Coerce one stream-supplied token count; count a non-number as 0. + + A bare ``int()`` would raise on a non-numeric value, which + ``communicate``'s ``except Exception`` turns into an ``AgentCrashError`` + (categorized ``AGENT_CRASH``, ``max_retries=2``), burning three attempts + on one mistyped bucket. A bool is never a token count (``int(True) == 1``). + """ + if isinstance(value, bool) or not isinstance(value, int | float | str): + return 0 + try: + return int(value) + except (TypeError, ValueError): + return 0 + + def on_turn_end(self, obj: dict[str, Any]) -> None: + """Accumulate this step's usage and append its assistant message. + + Usage is read from ``turn_end`` ONCE per step (not from every + ``message_end``, which echoes the same numbers) so the turn total is the + sum of the per-generation slices. + """ + self.turn_open = False + message = obj.get("message") + message = message if isinstance(message, dict) else {} + usage = message.get("usage") + usage = usage if isinstance(usage, dict) else {} + + step_in = self._as_int(usage.get("input")) + raw_out = self._as_int(usage.get("output")) + step_reasoning = self._as_int(usage.get("reasoning")) + step_cw = self._as_int(usage.get("cacheWrite")) + step_cr = self._as_int(usage.get("cacheRead")) + # Reasoning bills at the output rate but is reported apart from `output`; + # fold it into the turn total (the per-message record keeps it separately). + step_out = raw_out + step_reasoning + + self.usage = TokenUsage( + uncached_input_tokens=self.usage.uncached_input_tokens + step_in, + output_tokens=self.usage.output_tokens + step_out, + cache_creation_input_tokens=self.usage.cache_creation_input_tokens + step_cw, + cache_read_input_tokens=self.usage.cache_read_input_tokens + step_cr, + ) + cost = usage.get("cost") + if isinstance(cost, dict): + total = cost.get("total") + if isinstance(total, int | float) and not isinstance(total, bool): + self.cost_usd += float(total) + self.saw_cost = True + + finish = message.get("stopReason") + if isinstance(finish, str) and finish: + self.stop_reason = finish + + started = self.turn_started_at or datetime.now() + completed = datetime.now() + blocks: list[ContentBlock] = [] + turn_text = "".join(self.turn_text_parts) + if turn_text: + blocks.append(ContentBlock(block_type="text", sequence=0, text=turn_text)) + for i, tool_id in enumerate(self.turn_tool_ids, start=len(blocks)): + blocks.append(ContentBlock(block_type="tool_use", sequence=i, tool_use_id=tool_id)) + + self.messages.append( + AssistantMessage( + started_at=started, + completed_at=completed, + generation_duration_ms=(completed - started).total_seconds() * 1000, + content_blocks=blocks, + tool_use_ids=list(self.turn_tool_ids), + input_tokens=step_in, + output_tokens=step_out, + cache_creation_tokens=step_cw, + cache_read_tokens=step_cr, + reasoning_tokens=step_reasoning, + stop_reason=finish if isinstance(finish, str) else None, + model=self.model, + message_id=str(message.get("responseId") or "") or None, + ) + ) + self.emit( + TurnEndEvent( + task_id=self.task_id, + thread_id=self.thread_id, + turn_id=self.turn_id, + status=TurnEndStatus.COMPLETED, + tokens=TokenUsage( + uncached_input_tokens=step_in, + output_tokens=step_out, + cache_creation_input_tokens=step_cw, + cache_read_input_tokens=step_cr, + ), + ) + ) + + def _rate_card_cost(self) -> float | None: + if not self.model or self.usage.is_empty(): + return None + return calculate_cost( + self.model, + uncached_input_tokens=self.usage.uncached_input_tokens, + output_tokens=self.usage.output_tokens, + cache_creation_tokens=self.usage.cache_creation_input_tokens, + cache_read_tokens=self.usage.cache_read_input_tokens, + ) + + def _resolve_cost(self) -> float | None: + """Decide the turn's cost: the stream's own accounting vs the rate card. + + Pi reports a real per-call ``cost.total`` (spike-verified), which always + wins. The rate card fills only the gap where the stream reported no cost + at all; a genuinely free model resolves to the stream's 0. + """ + rate = self._rate_card_cost() + if not self.saw_cost: + return rate + if self.cost_usd == 0.0 and rate: + logger.warning( + "pi: the stream reported $0 for a turn the rate card prices at $%.6f; using the rate card " + + "so the run total is not understated.", + rate, + ) + return rate + return self.cost_usd + + def close_open_tools(self) -> None: + """Force-close every tool still awaiting a result (crash/timeout orphans).""" + for call_id in list(self.open_tools): + self._close_tool(call_id, status=ToolEndStatus.UNRESOLVED, summary=None, error="no result observed") + + def finalize( + self, + status: AgentEndStatus, + *, + crashed: bool = False, + crash_reason: str | None = None, + ) -> None: + """Close orphaned tools and emit the terminal ``AgentEndEvent`` (idempotent).""" + if self.finalized: + return + self.finalized = True + self.close_open_tools() + usage = self.usage + cost = self._resolve_cost() + if cost is not None: + usage = usage.model_copy(update={"total_cost_usd": cost}) + # A turn still open here never received its `turn_end` — close its + # TurnStartEvent or the one-pair-per-inner-turn contract breaks. + if self.turn_open: + self.turn_open = False + self.emit( + TurnEndEvent( + task_id=self.task_id, + thread_id=self.thread_id, + turn_id=self.turn_id, + status=TurnEndStatus(status.value), + tokens=None, + ) + ) + self.emit( + AgentEndEvent( + task_id=self.task_id, + thread_id=self.thread_id, + status=status, + usage=usage, + iteration=self.iteration, + user_input=self.user_input, + agent_output=self.agent_output, + model_used=self.model, + assistant_turn_count=self.turn_count, + messages=list(self.messages), + num_turns=self.turn_count, + max_turns_exhausted=self.max_turns_exhausted, + result_summary=ResultSummary( + is_error=crashed, + subtype=status.value, + stop_reason=self.stop_reason, + result=crash_reason or self.error_message, + ), + crashed=crashed, + crash_reason=crash_reason, + duration_seconds=time.monotonic() - self.started_at, + ) + ) + + +@AgentRegistry.register(AgentKind.PI, PiAgentConfig) +class PiAgent(Agent[PiAgentConfig]): + """Runs the ``pi`` CLI as a subprocess, one invocation per turn.""" + + # `should_stop` is polled at every event boundary — i.e. tool-call + # granularity — and honored by terminating the CLI subprocess cleanly. + supports_cooperative_stop: ClassVar[bool] = True + + # Pi maps `system_prompt` to `--append-system-prompt`, so it appends to (does + # not replace) the CLI's default prompt. Declared explicitly (mirrors Codex / + # Antigravity, NOT OpenCode's `"unknown"`). + system_prompt_semantics: ClassVar[SystemPromptSemantics] = "append" + + def __init__( + self, + config: PiAgentConfig, + route: ApiRoute | None = None, + *, + task_id: str = "unknown", + ) -> None: + """Every parameter the agent factory can pass is DECLARED, not absorbed. + + ``create_agent`` calls ``agent_class(config, route=route, **kwargs)`` + through a ``cast(Any, ...)``, so a ``**_`` sink would mean nothing checks + the kwargs at runtime either — a mis-gated kwarg must be loud here rather + than silently dropped. + + ``route`` is accepted for factory parity and deliberately unused: the CLI + owns its own provider configuration. ``task_id`` only labels the emitted + event stream. + """ + self.config = config + self.route = route + self.task_id = task_id + self.working_directory: str | None = None + self._env_path_prepend: list[str] = [] + self._plugin_tools_dir: str | None = None + # Per-agent session, reused across communicate() calls for multi-turn / + # simulation continuity (assigned in start(), removed in stop()/kill()). + self._session_id: str | None = None + self._session_dir: str | None = None + self._process: asyncio.subprocess.Process | None = None + # Process-group ids of every invocation this agent spawned, swept on + # kill()/kill_sync()/stop(). + self._spawned_pgids: list[int] = [] + self._state = AgentState.WORKING + + # --- lifecycle --------------------------------------------------------- + + async def start( + self, + working_directory: str, + *, + env_path_prepend: list[str] | None = None, + plugin_tools_dir: str | None = None, + ) -> None: + if shutil.which("pi") is None: + raise RuntimeError( + "The 'pi' CLI was not found on PATH." + + " Install it with `npm install -g @earendil-works/pi-coding-agent` (see https://pi.dev/)." + ) + ignored = [f for f in _UNSUPPORTED_CONFIG_FIELDS if getattr(self.config, f, None)] + if ignored: + logger.warning( + "pi: %s set but NOT enforced — the CLI has no equivalent knob in JSON print mode, so the run is " + + "unconstrained by them; do not rely on them as a boundary (see docs/agents/PI.md).", + ", ".join(ignored), + ) + self.working_directory = working_directory + self._env_path_prepend = list(env_path_prepend or []) + self._plugin_tools_dir = plugin_tools_dir + # A stable pre-assigned id (create-if-missing on turn 1, resume after). + # The tempdir lives OUTSIDE the sandbox working dir and staged reference + # dir, so it never pollutes graded files or trips reference-integrity. + self._session_id = f"coder-eval-{self.task_id}-{uuid4().hex[:8]}" + self._session_dir = tempfile.mkdtemp(prefix="pi-session-") + self._state = AgentState.WORKING + + async def stop(self) -> None: + await self.kill() + self._cleanup_session_dir() + self._mark_stopped() + + def _cleanup_session_dir(self) -> None: + if self._session_dir is not None: + shutil.rmtree(self._session_dir, ignore_errors=True) + self._session_dir = None + + async def kill(self) -> None: + proc = self._process + if proc is not None and proc.returncode is None: + with contextlib.suppress(ProcessLookupError): + proc.terminate() + with contextlib.suppress(TimeoutError, asyncio.TimeoutError): + await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_SECONDS) + if proc.returncode is None: + with contextlib.suppress(ProcessLookupError): + proc.kill() + self._sweep_process_groups() + + def kill_sync(self) -> None: + """SIGKILL the in-flight CLI and its process group (watchdog thread; must not await).""" + proc = self._process + if proc is not None and proc.returncode is None: + with contextlib.suppress(ProcessLookupError, PermissionError): + os.kill(proc.pid, _SIGKILL) + self._sweep_process_groups() + + def _sweep_process_groups(self) -> None: + """SIGKILL every process group this agent spawned (POSIX only). + + Each invocation runs in its own session (``start_new_session``), so its + pgid is the CLI's pid and the group contains ONLY what that invocation + spawned — a lingering child included, a shared daemon we did not start + excluded. + """ + if os.name != "posix": + return + for pgid in self._spawned_pgids: + with contextlib.suppress(ProcessLookupError, PermissionError, OSError): + os.killpg(pgid, _SIGKILL) + self._spawned_pgids.clear() + + def get_environment_info(self) -> dict[str, Any]: + # Spread the base first so the `system_prompt_semantics` run marker is + # always present (CE046). + info: dict[str, Any] = { + **super().get_environment_info(), + "pi_model": self.config.model, + "pi_thinking_level": self.config.thinking_level, + } + if self._session_id: + info["pi_session_id"] = self._session_id + return info + + # --- command construction --------------------------------------------- + + def _build_argv(self, user_input: str) -> list[str]: + # -p exits after the run; --no-context-files + --no-approve isolate the + # sandbox from host AGENTS.md/CLAUDE.md and project-local trust (analogue + # of OpenCode --pure / Claude setting_sources: []). --session-dir + + # --session-id give cross-communicate() continuity (simulation mode) — + # reused every call, created on turn 1, resumed after. NOT --no-session + # (that would defeat continuity). All spike-verified. No --dir flag: the + # working dir is set via the subprocess `cwd`. + assert self._session_dir is not None and self._session_id is not None + argv = [ + "pi", + "-p", + "--mode", + "json", + "--no-context-files", + "--no-approve", + "--session-dir", + self._session_dir, + "--session-id", + self._session_id, + ] + if self.config.model: + argv += ["--model", self.config.model] # provider-prefixed form + if self.config.thinking_level: + argv += ["--thinking", self.config.thinking_level] + if self.config.allowed_tools: + argv += ["--tools", ",".join(self.config.allowed_tools)] # ENFORCED (a win over OpenCode) + if self.config.disallowed_tools: + argv += ["--exclude-tools", ",".join(self.config.disallowed_tools)] + if self.config.system_prompt: + argv += ["--append-system-prompt", self.config.system_prompt] + # user_input is a distinct argv element after `--` (never shell-interpolated). + argv += ["--", user_input] + return argv + + def _build_env(self) -> dict[str, str]: + """The CLI's full environment: the host's, plus the sandbox's contributions. + + The PATH prepend is the mock-shadowing contract (``Agent.start``): the + sandbox's mock CLI directories must resolve BEFORE the real binaries. + ``PLUGIN_TOOLS_DIR`` is advisory and never overrides an inherited value. + Returns the WHOLE environment (seeded from ``os.environ``) so the CLI + keeps the host's provider credentials (``OPENROUTER_API_KEY``, ...). + """ + env = dict(os.environ) + if self._env_path_prepend: + env["PATH"] = os.pathsep.join([*self._env_path_prepend, env.get("PATH", "")]) + if self._plugin_tools_dir and "PLUGIN_TOOLS_DIR" not in env: + env["PLUGIN_TOOLS_DIR"] = self._plugin_tools_dir + return env + + # --- the turn ---------------------------------------------------------- + + async def communicate( + self, + user_input: str, + *, + stream_callback: StreamCallback | None = None, + timeout: float | None = None, + max_turns: int | None = None, + should_stop: Callable[[], bool] | None = None, + ) -> TurnRecord: + if self.working_directory is None: + raise RuntimeError("PiAgent.start() must be called before communicate()") + + self._begin_turn() + collector = EventCollector() + + def emit(event: StreamEvent) -> None: + collector.on_event(event) + if stream_callback is not None: + safe_emit(stream_callback, event) + + state = _PiTurnState( + task_id=self.task_id, + iteration=self._iteration, + user_input=user_input, + model=self.config.model, + ) + state.bind(emit) + + emit( + AgentStartEvent( + task_id=self.task_id, + prompt=user_input, + iteration=self._iteration, + model=self.config.model, + ) + ) + + deadline = None if timeout is None else time.monotonic() + timeout + stopped_early = False + stderr_drain: asyncio.Future[bytes] | None = None + # Bound OUTSIDE the try so the teardown in `finally` can tell "never + # spawned" from "spawned and possibly still running". + proc: asyncio.subprocess.Process | None = None + try: + proc = await asyncio.create_subprocess_exec( + *self._build_argv(user_input), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=self.working_directory, + env=self._build_env(), + # A single nd-JSON event can carry a whole tool result, which blows + # past StreamReader's default 64 KiB line cap and would raise + # ValueError mid-stream, killing the read loop. + limit=STDOUT_LINE_LIMIT_BYTES, + # Own session/process group, so teardown can killpg any lingering + # child without touching anything this invocation didn't spawn. + start_new_session=os.name == "posix", + ) + self._process = proc + if os.name == "posix": + self._spawned_pgids.append(proc.pid) + assert proc.stdout is not None + + # Drain stderr CONCURRENTLY: a child that fills the ~64 KiB stderr pipe + # blocks on write, stops emitting stdout, and never exits — hanging the + # turn to its deadline. It gets its own reader so the nd-JSON on stdout + # stays clean. + if proc.stderr is not None: + stderr_drain = asyncio.ensure_future(proc.stderr.read()) + + # A print-mode CLI may leave an inherited pipe open, so readline() can + # block on an EOF that never comes. Race each read against process + # exit; once the process is gone a bounded drain collects the tail. + exit_waiter = asyncio.ensure_future(proc.wait()) + read_task: asyncio.Future[bytes] | None = None + try: + while True: + remaining = None if deadline is None else deadline - time.monotonic() + if remaining is not None and remaining <= 0: + await self._timeout_turn(state, collector, timeout or 0.0) + + if read_task is None: + read_task = asyncio.ensure_future(proc.stdout.readline()) + done, _pending = await asyncio.wait( + {read_task, exit_waiter}, + timeout=remaining, + return_when=asyncio.FIRST_COMPLETED, + ) + if not done: + await self._timeout_turn(state, collector, timeout or 0.0) + if not read_task.done(): + try: + await asyncio.wait_for(asyncio.shield(read_task), _DRAIN_SECONDS) + except TimeoutError: + break + line = read_task.result() + read_task = None + if not line: + break + + self._handle_line(line, state) + + if max_turns is not None and state.turn_count > max_turns: + state.max_turns_exhausted = True + await self.kill() + break + if should_stop is not None and should_stop(): + stopped_early = True + await self.kill() + break + finally: + if read_task is not None: + read_task.cancel() + exit_waiter.cancel() + + status = await self._settle_turn( + proc, + state, + collector, + stderr_drain, + stopped_early=stopped_early, + deadline=deadline, + timeout=timeout, + ) + state.finalize(status) + # Build BEFORE marking the turn clean: a failure in the reduction is a + # failed turn, and `_end_turn_ok` would clear the rollback flag that + # `discard_pending_turn` needs. + record = collector.build_turn_record() + self._end_turn_ok() + return record + + except (AgentCrashError, TurnTimeoutError): + # Already funneled through finalize by _crash_turn / _timeout_turn. + raise + except asyncio.CancelledError: + self._finalize_external_cancel(state.finalize) + self._capture_partial_turn(collector) + raise + except Exception as e: + # A spawn failure (OSError), a StreamReader ValueError past `limit`, a + # malformed-payload error in a handler, a pydantic error assembling + # telemetry. Funnel to the pending-turn contract like the siblings. + self._crash_turn(state, collector, f"Pi turn failed: {e!s}", cause=e) + raise # unreachable (_crash_turn is NoReturn) — makes the no-fall-through explicit + finally: + if stderr_drain is not None: + stderr_drain.cancel() + self._reap_orphaned_cli(proc) + self._process = None + + def _reap_orphaned_cli(self, proc: asyncio.subprocess.Process | None) -> None: + """Kill a CLI still running as the turn unwinds. No-op otherwise. + + The ``except Exception`` crash and an external cancellation both reach the + ``finally`` with the child possibly alive — neither passes through the + graceful ``kill()``. Abandoning it is not merely a leak: ``AgentCrashError`` + is retried, so attempt 2 would spawn a SECOND ``pi`` editing the very files + the criteria are about to score. Synchronous (no await) so it survives a + ``CancelledError`` in flight. ``proc`` is ``None`` when the spawn failed. + """ + if proc is None or proc.returncode is not None: + return + with contextlib.suppress(ProcessLookupError, PermissionError): + proc.kill() + self._sweep_process_groups() + + async def _settle_turn( + self, + proc: asyncio.subprocess.Process, + state: _PiTurnState, + collector: EventCollector, + stderr_drain: asyncio.Future[bytes] | None, + *, + stopped_early: bool, + deadline: float | None, + timeout: float | None, + ) -> AgentEndStatus: + """Reap the CLI once the read loop is done and decide the turn's end status. + + Raises ``AgentCrashError`` (via :meth:`_crash_turn`) when the process died + with neither an intentional stop nor a recognized event stream. Raises + ``TurnTimeoutError`` when the deadline elapses while waiting for the exit. + """ + remaining = None if deadline is None else max(0.0, deadline - time.monotonic()) + try: + await asyncio.wait_for(proc.wait(), timeout=_TERM_GRACE_SECONDS if remaining is None else remaining) + except TimeoutError: + if remaining is not None: + await self._timeout_turn(state, collector, timeout or 0.0) + await self.kill() + self._crash_turn( + state, + collector, + f"Pi closed its event stream but did not exit within {_TERM_GRACE_SECONDS:.0f}s", + ) + stderr_bytes = b"" + if stderr_drain is not None: + with contextlib.suppress(TimeoutError): + stderr_bytes = await asyncio.wait_for(asyncio.shield(stderr_drain), timeout=_DRAIN_SECONDS) + + # A non-zero exit with no intentional cut means the turn died. + if proc.returncode not in (0, None) and not stopped_early and not state.max_turns_exhausted: + detail = stderr_bytes.decode("utf-8", "replace").strip() or f"exit code {proc.returncode}" + self._crash_turn(state, collector, f"Pi exited non-zero: {detail}") + + # A clean exit that recognized NO events is vocabulary drift — the CLI's + # schema moved, and scoring a silent empty success would be indistinguishable + # from a real pass in every aggregate. Intentional cuts are exempt (a stop + # can land before the first event). + if not stopped_early and not state.max_turns_exhausted and state.recognized_events == 0: + seen = ", ".join(sorted(state.unrecognized_types)) or "none (stdout carried no JSON events)" + self._crash_turn( + state, + collector, + "Pi exited cleanly but the turn captured no recognized events. Unrecognized event types seen: " + + f"{seen}. The CLI's event schema may have changed — see docs/agents/PI.md before trusting any " + + "run from this CLI version.", + ) + + if stopped_early: + return AgentEndStatus.STOPPED_EARLY + if state.max_turns_exhausted: + return AgentEndStatus.MAX_TURNS_EXHAUSTED + return AgentEndStatus.COMPLETED + + def _crash_turn( + self, + state: _PiTurnState, + collector: EventCollector, + message: str, + *, + cause: BaseException | None = None, + ) -> NoReturn: + """Park the crashed partial record and raise ``AgentCrashError``.""" + state.close_open_tools() + try: + self._finalize_and_raise_crash(state.finalize, message, cause=cause) + finally: + self._capture_partial_turn(collector) + + async def _timeout_turn( + self, + state: _PiTurnState, + collector: EventCollector, + timeout: float, + ) -> NoReturn: + """Kill the CLI, park the crashed partial record, raise ``TurnTimeoutError``.""" + await self.kill() + state.close_open_tools() + try: + self._finalize_and_raise_timeout(state.finalize, timeout) + finally: + self._capture_partial_turn(collector) + + def _handle_line(self, line: bytes, state: _PiTurnState) -> None: + """Parse one nd-JSON line and dispatch it. Never raises on bad input. + + Pi may emit multiple ``agent_start``/``turn_*``/``agent_end`` cycles in one + invocation (auto-retry). ``agent_end`` is NOT terminal — only + ``agent_settled`` / stdout EOF is — so it is recognized and otherwise + ignored, and the read loop keeps going. + """ + raw = line.decode("utf-8", "replace").strip() + if not raw: + return + try: + obj = json.loads(raw) + except json.JSONDecodeError: + logger.debug("pi: skipping non-JSON stdout line: %s", raw[:200]) + return + if not isinstance(obj, dict): + return + + event_type = str(obj.get("type") or "") + # `session` is line 1 (cwd/id); `message_start`/`message_end`/`agent_end`/ + # `agent_settled` carry no state we accumulate (usage is read from + # `turn_end`, not the echoing `message_end`). All are recognized vocabulary. + if event_type in _RECOGNIZED_EVENTS: + state.recognized_events += 1 + elif len(state.unrecognized_types) < _MAX_UNRECOGNIZED_TYPES: + state.unrecognized_types.add(event_type or "") + + if event_type == "turn_start": + state.on_turn_start() + elif event_type == "message_update": + state.on_message_update(obj) + elif event_type == "tool_execution_start": + state.on_tool_execution_start(obj) + elif event_type == "tool_execution_end": + state.on_tool_execution_end(obj) + elif event_type == "turn_end": + state.on_turn_end(obj) + else: + logger.debug("pi: unhandled event type %r", event_type) diff --git a/tests/fixtures/pi_happy_stream.jsonl b/tests/fixtures/pi_happy_stream.jsonl new file mode 100644 index 00000000..a6401798 --- /dev/null +++ b/tests/fixtures/pi_happy_stream.jsonl @@ -0,0 +1,45 @@ +{"type":"session","version":3,"id":"01a06be7-fbbf-7641-9680-be47d4693f73","timestamp":"2026-09-04T10:12:40.511Z","cwd":"/private/tmp/claude-501/-Users-carles-pull-request-coder-eval/a0b3ca95-4a6c-49f2-b1df-fbf2641a4a6c/scratchpad/pi_bg"} +{"type":"agent_start"} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"Create a file named hello.txt containing exactly the word hi using the write tool. Then read it back with the read tool and state the contents."}],"timestamp":1788516760585}} +{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"Create a file named hello.txt containing exactly the word hi using the write tool. Then read it back with the read tool and state the contents."}],"timestamp":1788516760585}} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1788516760616}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"thinking_start","contentIndex":0}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"thinking_delta","contentIndex":0,"delta":"Write"}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"thinking_delta","contentIndex":0,"delta":" file then"}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"thinking_delta","contentIndex":0,"delta":" read."}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_start","contentIndex":1,"id":"write:0","toolName":"write"}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_delta","contentIndex":1,"delta":""}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_delta","contentIndex":1,"delta":"{\"path\": \"hello.txt\", \"content\": \"hi\"}"}} +{"type":"message_update","usage":{"input":406,"output":69,"cacheRead":1024,"cacheWrite":0,"reasoning":8,"totalTokens":1499,"cost":{"input":0.0010352999999999998,"output":0.00087975,"cacheRead":0.000262144,"cacheWrite":0,"total":0.002177194}},"assistantMessageEvent":{"type":"thinking_end","contentIndex":0,"content":"Write file then read."}} +{"type":"message_update","usage":{"input":406,"output":69,"cacheRead":1024,"cacheWrite":0,"reasoning":8,"totalTokens":1499,"cost":{"input":0.0010352999999999998,"output":0.00087975,"cacheRead":0.000262144,"cacheWrite":0,"total":0.002177194}},"assistantMessageEvent":{"type":"toolcall_end","contentIndex":1,"toolCall":{"type":"toolCall","id":"write:0","name":"write","arguments":{"path":"hello.txt","content":"hi"}}}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"thinking","thinking":"Write file then read.","thinkingSignature":"[{\"type\":\"reasoning.text\",\"text\":\"Write file then read.\",\"format\":\"unknown\",\"index\":0}]"},{"type":"toolCall","id":"write:0","name":"write","arguments":{"path":"hello.txt","content":"hi"}}],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":406,"output":69,"cacheRead":1024,"cacheWrite":0,"reasoning":8,"totalTokens":1499,"cost":{"input":0.0010352999999999998,"output":0.00087975,"cacheRead":0.000262144,"cacheWrite":0,"total":0.002177194}},"stopReason":"toolUse","timestamp":1788516760616,"responseId":"gen-1788516760-h7AqlA8rocYqvzr5bE9l","rawStopReason":"tool_calls"}} +{"type":"tool_execution_start","toolCallId":"write:0","toolName":"write","args":{"path":"hello.txt","content":"hi"}} +{"type":"tool_execution_end","toolCallId":"write:0","toolName":"write","result":{"content":[{"type":"text","text":"Successfully wrote 2 bytes to hello.txt"}]},"isError":false} +{"type":"message_start","message":{"role":"toolResult","toolCallId":"write:0","toolName":"write","content":[{"type":"text","text":"Successfully wrote 2 bytes to hello.txt"}],"isError":false,"timestamp":1788516762541}} +{"type":"message_end","message":{"role":"toolResult","toolCallId":"write:0","toolName":"write","content":[{"type":"text","text":"Successfully wrote 2 bytes to hello.txt"}],"isError":false,"timestamp":1788516762541}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"thinking","thinking":"Write file then read.","thinkingSignature":"[{\"type\":\"reasoning.text\",\"text\":\"Write file then read.\",\"format\":\"unknown\",\"index\":0}]"},{"type":"toolCall","id":"write:0","name":"write","arguments":{"path":"hello.txt","content":"hi"}}],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":406,"output":69,"cacheRead":1024,"cacheWrite":0,"reasoning":8,"totalTokens":1499,"cost":{"input":0.0010352999999999998,"output":0.00087975,"cacheRead":0.000262144,"cacheWrite":0,"total":0.002177194}},"stopReason":"toolUse","timestamp":1788516760616,"responseId":"gen-1788516760-h7AqlA8rocYqvzr5bE9l","rawStopReason":"tool_calls"},"toolResults":[{"role":"toolResult","toolCallId":"write:0","toolName":"write","content":[{"type":"text","text":"Successfully wrote 2 bytes to hello.txt"}],"isError":false,"timestamp":1788516762541}]} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1788516762544}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_start","contentIndex":0,"id":"read:1","toolName":"read"}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_delta","contentIndex":0,"delta":""}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"toolcall_delta","contentIndex":0,"delta":"{\"path\": \"hello.txt\"}"}} +{"type":"message_update","usage":{"input":512,"output":49,"cacheRead":1024,"cacheWrite":0,"reasoning":3,"totalTokens":1585,"cost":{"input":0.0013055999999999999,"output":0.00062475,"cacheRead":0.000262144,"cacheWrite":0,"total":0.002192494}},"assistantMessageEvent":{"type":"toolcall_end","contentIndex":0,"toolCall":{"type":"toolCall","id":"read:1","name":"read","arguments":{"path":"hello.txt"}}}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"toolCall","id":"read:1","name":"read","arguments":{"path":"hello.txt"}}],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":512,"output":49,"cacheRead":1024,"cacheWrite":0,"reasoning":3,"totalTokens":1585,"cost":{"input":0.0013055999999999999,"output":0.00062475,"cacheRead":0.000262144,"cacheWrite":0,"total":0.002192494}},"stopReason":"toolUse","timestamp":1788516762544,"responseId":"gen-1788516762-S6qNOttzQjEAZuTdBsee","rawStopReason":"tool_calls"}} +{"type":"tool_execution_start","toolCallId":"read:1","toolName":"read","args":{"path":"hello.txt"}} +{"type":"tool_execution_end","toolCallId":"read:1","toolName":"read","result":{"content":[{"type":"text","text":"hi"}]},"isError":false} +{"type":"message_start","message":{"role":"toolResult","toolCallId":"read:1","toolName":"read","content":[{"type":"text","text":"hi"}],"isError":false,"timestamp":1788516764037}} +{"type":"message_end","message":{"role":"toolResult","toolCallId":"read:1","toolName":"read","content":[{"type":"text","text":"hi"}],"isError":false,"timestamp":1788516764037}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"toolCall","id":"read:1","name":"read","arguments":{"path":"hello.txt"}}],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":512,"output":49,"cacheRead":1024,"cacheWrite":0,"reasoning":3,"totalTokens":1585,"cost":{"input":0.0013055999999999999,"output":0.00062475,"cacheRead":0.000262144,"cacheWrite":0,"total":0.002192494}},"stopReason":"toolUse","timestamp":1788516762544,"responseId":"gen-1788516762-S6qNOttzQjEAZuTdBsee","rawStopReason":"tool_calls"},"toolResults":[{"role":"toolResult","toolCallId":"read:1","toolName":"read","content":[{"type":"text","text":"hi"}],"isError":false,"timestamp":1788516764037}]} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"assistant","content":[],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"pending","timestamp":1788516764038}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_start","contentIndex":0}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"Created"}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":" `hello.txt` and"}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":" read it back. Contents"}} +{"type":"message_update","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":": **hi**"}} +{"type":"message_update","usage":{"input":79,"output":28,"cacheRead":1536,"cacheWrite":0,"reasoning":3,"totalTokens":1643,"cost":{"input":0.00020145,"output":0.000357,"cacheRead":0.00039321600000000005,"cacheWrite":0,"total":0.0009516660000000001}},"assistantMessageEvent":{"type":"text_end","contentIndex":0,"content":"Created `hello.txt` and read it back. Contents: **hi**"}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"Created `hello.txt` and read it back. Contents: **hi**"}],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":79,"output":28,"cacheRead":1536,"cacheWrite":0,"reasoning":3,"totalTokens":1643,"cost":{"input":0.00020145,"output":0.000357,"cacheRead":0.00039321600000000005,"cacheWrite":0,"total":0.0009516660000000001}},"stopReason":"stop","timestamp":1788516764038,"responseId":"gen-1788516764-PDS1DE9HRseKaqHTeGIx","rawStopReason":"stop"}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"Created `hello.txt` and read it back. Contents: **hi**"}],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":79,"output":28,"cacheRead":1536,"cacheWrite":0,"reasoning":3,"totalTokens":1643,"cost":{"input":0.00020145,"output":0.000357,"cacheRead":0.00039321600000000005,"cacheWrite":0,"total":0.0009516660000000001}},"stopReason":"stop","timestamp":1788516764038,"responseId":"gen-1788516764-PDS1DE9HRseKaqHTeGIx","rawStopReason":"stop"},"toolResults":[]} +{"type":"agent_end","messages":[{"role":"user","content":[{"type":"text","text":"Create a file named hello.txt containing exactly the word hi using the write tool. Then read it back with the read tool and state the contents."}],"timestamp":1788516760585},{"role":"assistant","content":[{"type":"thinking","thinking":"Write file then read.","thinkingSignature":"[{\"type\":\"reasoning.text\",\"text\":\"Write file then read.\",\"format\":\"unknown\",\"index\":0}]"},{"type":"toolCall","id":"write:0","name":"write","arguments":{"path":"hello.txt","content":"hi"}}],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":406,"output":69,"cacheRead":1024,"cacheWrite":0,"reasoning":8,"totalTokens":1499,"cost":{"input":0.0010352999999999998,"output":0.00087975,"cacheRead":0.000262144,"cacheWrite":0,"total":0.002177194}},"stopReason":"toolUse","timestamp":1788516760616,"responseId":"gen-1788516760-h7AqlA8rocYqvzr5bE9l","rawStopReason":"tool_calls"},{"role":"toolResult","toolCallId":"write:0","toolName":"write","content":[{"type":"text","text":"Successfully wrote 2 bytes to hello.txt"}],"isError":false,"timestamp":1788516762541},{"role":"assistant","content":[{"type":"toolCall","id":"read:1","name":"read","arguments":{"path":"hello.txt"}}],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":512,"output":49,"cacheRead":1024,"cacheWrite":0,"reasoning":3,"totalTokens":1585,"cost":{"input":0.0013055999999999999,"output":0.00062475,"cacheRead":0.000262144,"cacheWrite":0,"total":0.002192494}},"stopReason":"toolUse","timestamp":1788516762544,"responseId":"gen-1788516762-S6qNOttzQjEAZuTdBsee","rawStopReason":"tool_calls"},{"role":"toolResult","toolCallId":"read:1","toolName":"read","content":[{"type":"text","text":"hi"}],"isError":false,"timestamp":1788516764037},{"role":"assistant","content":[{"type":"text","text":"Created `hello.txt` and read it back. Contents: **hi**"}],"api":"openai-completions","provider":"openrouter","model":"moonshotai/kimi-k3","usage":{"input":79,"output":28,"cacheRead":1536,"cacheWrite":0,"reasoning":3,"totalTokens":1643,"cost":{"input":0.00020145,"output":0.000357,"cacheRead":0.00039321600000000005,"cacheWrite":0,"total":0.0009516660000000001}},"stopReason":"stop","timestamp":1788516764038,"responseId":"gen-1788516764-PDS1DE9HRseKaqHTeGIx","rawStopReason":"stop"}],"willRetry":false} +{"type":"agent_settled"} diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py new file mode 100644 index 00000000..265b5a79 --- /dev/null +++ b/tests/test_pi_agent.py @@ -0,0 +1,803 @@ +"""Tests for the Pi agent harness. + +The CLI is never invoked: ``asyncio.create_subprocess_exec`` is patched with a +fake process that replays a newline-delimited JSON event stream, so the whole +reduction path (nd-JSON -> standardized events -> ``TurnRecord``) is exercised +offline and without credentials. + +The happy fixture (``tests/fixtures/pi_happy_stream.jsonl``) is a BYTE-REAL +capture from ``pi -p --mode json`` 0.84.4 (write hello.txt + read it back, 3 +agent-loop turns). The expected token/cost totals below are derived from that +fixture's per-``turn_end`` usages, summed — not free-standing magic numbers. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +from pathlib import Path +from typing import Any + +import pytest + +from coder_eval.agents.pi_agent import PiAgent, _PiTurnState, _result_text +from coder_eval.errors import AgentCrashError, TurnTimeoutError +from coder_eval.models import AgentKind, AssistantMessage, PiAgentConfig +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentEndStatus, + AgentStartEvent, + ToolEndEvent, + ToolEndStatus, + ToolStartEvent, + TurnEndEvent, + TurnEndStatus, + TurnStartEvent, +) + + +_FIXTURE = Path(__file__).parent / "fixtures" / "pi_happy_stream.jsonl" +HAPPY_STREAM = _FIXTURE.read_text(encoding="utf-8").splitlines() + +# Derived from the fixture's three `turn_end` usages (per-generation, summed): +# input 406 + 512 + 79 = 997 +# output (69+8)+(49+3)+(28+3) = 160 (reasoning folds into the output rate) +# cacheRead 1024 + 1024 + 1536 = 3584 +# cost.total 0.002177194 + 0.002192494 + 0.000951666 = 0.005321354 +EXPECTED_INPUT = 997 +EXPECTED_OUTPUT = 160 +EXPECTED_CACHE_READ = 3584 +EXPECTED_COST = 0.005321354 + + +def _turn_start() -> str: + return json.dumps({"type": "turn_start"}) + + +def _turn_end(*, inp: int, out: int, cache_read: int = 0, cache_write: int = 0, reasoning: int = 0, cost: float = 0.0): + """A `turn_end` event carrying that step's own (per-generation) usage.""" + usage: dict[str, Any] = { + "input": inp, + "output": out, + "cacheRead": cache_read, + "cacheWrite": cache_write, + "reasoning": reasoning, + "totalTokens": inp + out + cache_read + cache_write, + "cost": {"total": cost}, + } + return json.dumps( + {"type": "turn_end", "message": {"role": "assistant", "usage": usage, "stopReason": "stop"}, "toolResults": []} + ) + + +def _tool_start(call_id: str, name: str, args: dict[str, Any]) -> str: + return json.dumps({"type": "tool_execution_start", "toolCallId": call_id, "toolName": name, "args": args}) + + +def _tool_end(call_id: str, name: str, text: str, *, is_error: bool = False) -> str: + return json.dumps( + { + "type": "tool_execution_end", + "toolCallId": call_id, + "toolName": name, + "result": {"content": [{"type": "text", "text": text}]}, + "isError": is_error, + } + ) + + +class _FakeProcess: + def __init__(self, lines: list[str], returncode: int = 0, stderr: bytes = b"") -> None: + self._lines = [f"{line}\n".encode() for line in lines] + self.returncode: int | None = None + self._final_returncode = returncode + self._stderr = stderr + self.pid = 4242 + self.terminated = False + self.killed = False + self.stdout = self + + async def readline(self) -> bytes: + if self._lines: + return self._lines.pop(0) + self.returncode = self._final_returncode + return b"" + + async def read(self) -> bytes: + return self._stderr + + async def wait(self) -> int: + self.returncode = self._final_returncode + return self.returncode + + def terminate(self) -> None: + self.terminated = True + self.returncode = self._final_returncode + + def kill(self) -> None: + self.killed = True + self.returncode = self._final_returncode + + +class _RunningProcess(_FakeProcess): + """A process that stays alive until it is explicitly terminated or killed.""" + + def __init__(self, lines: list[str], **kwargs: Any) -> None: + super().__init__(lines, **kwargs) + self._exited = asyncio.Event() + + async def wait(self) -> int: + await self._exited.wait() + self.returncode = self._final_returncode + return self.returncode + + def terminate(self) -> None: + self.terminated = True + self._exited.set() + + def kill(self) -> None: + self.killed = True + self._exited.set() + + +@pytest.fixture +def patch_exec(monkeypatch: pytest.MonkeyPatch): + """Patch subprocess spawn; return a dict capturing the argv used.""" + captured: dict[str, Any] = {"killpg": []} + + def _install(proc: _FakeProcess) -> dict[str, Any]: + async def fake_exec(*argv: str, **kwargs: Any) -> _FakeProcess: + captured["argv"] = list(argv) + captured["kwargs"] = kwargs + proc.stderr = proc # type: ignore[assignment] + return proc + + monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_exec) + monkeypatch.setattr("shutil.which", lambda _name: "/usr/local/bin/pi") + monkeypatch.setattr(os, "killpg", lambda pgid, sig: captured["killpg"].append((pgid, sig)), raising=False) + return captured + + return _install + + +async def _run(agent: PiAgent, tmp_path: Any, prompt: str = "do the thing", **kwargs: Any): + await agent.start(str(tmp_path)) + return await agent.communicate(prompt, **kwargs) + + +def _agent(**overrides: Any) -> PiAgent: + config = PiAgentConfig(type="pi", **{"model": "openrouter/moonshotai/kimi-k3", **overrides}) + return PiAgent(config, task_id="t1") + + +class _EventRecorder: + def __init__(self) -> None: + self.events: list[Any] = [] + + def on_event(self, event: Any) -> None: + self.events.append(event) + + +class TestHappyPath: + async def test_builds_turn_record(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _run(_agent(), tmp_path) + + assert record.crashed is False + # 3 turn_start steps in the fixture (write, read, summarize). + assert record.assistant_turn_count == 3 + assert record.model_used == "openrouter/moonshotai/kimi-k3" + assert "hi" in record.agent_output + + async def test_token_buckets_sum_across_turns(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _run(_agent(), tmp_path) + + usage = record.token_usage + assert usage is not None + assert usage.uncached_input_tokens == EXPECTED_INPUT + assert usage.output_tokens == EXPECTED_OUTPUT + assert usage.cache_read_input_tokens == EXPECTED_CACHE_READ + assert usage.cache_creation_input_tokens == 0 + assert usage.total_cost_usd == pytest.approx(EXPECTED_COST) + + async def test_reconciliation_invariant(self, patch_exec, tmp_path): + """Summing the four buckets across messages must equal token_usage exactly.""" + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _run(_agent(), tmp_path) + + usage = record.token_usage + assert usage is not None + assert sum(m.input_tokens for m in record.messages) == usage.uncached_input_tokens + assert sum(m.output_tokens for m in record.messages) == usage.output_tokens + assert sum(m.cache_read_tokens for m in record.messages) == usage.cache_read_input_tokens + + async def test_tool_calls_captured(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _run(_agent(), tmp_path) + + # write + read, normalized to the canonical vocabulary. + assert [c.tool_name for c in record.commands] == ["Write", "Read"] + write = record.commands[0] + assert write.tool_id == "write:0" + assert write.result_status == "success" + # `path` renamed to Claude's `file_path`. + assert write.parameters == {"file_path": "hello.txt", "content": "hi"} + assert write.result_summary == "Successfully wrote 2 bytes to hello.txt" + + async def test_messages_attributed_to_turns(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _run(_agent(), tmp_path) + + assistants = [m for m in record.messages if isinstance(m, AssistantMessage)] + assert len(assistants) == 3 + assert assistants[0].tool_use_ids == ["write:0"] + assert assistants[1].tool_use_ids == ["read:1"] + assert assistants[2].tool_use_ids == [] # final summary turn + + async def test_event_order_is_balanced(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + recorder = _EventRecorder() + await _run(_agent(), tmp_path, stream_callback=recorder) + + seen = recorder.events + assert len([e for e in seen if isinstance(e, AgentStartEvent)]) == 1 + assert len([e for e in seen if isinstance(e, AgentEndEvent)]) == 1 + starts = len([e for e in seen if isinstance(e, TurnStartEvent)]) + ends = len([e for e in seen if isinstance(e, TurnEndEvent)]) + assert starts == ends == 3 + tstarts = len([e for e in seen if isinstance(e, ToolStartEvent)]) + tends = len([e for e in seen if isinstance(e, ToolEndEvent)]) + assert tstarts == tends == 2 + + +class TestResultTextFlatten: + def test_flattens_content_list(self): + assert _result_text({"content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}) == "ab" + + def test_none_stays_none(self): + assert _result_text(None) is None + + def test_other_shapes_stringify(self): + assert _result_text(42) == "42" + + +class TestToolNormalization: + async def test_bash_maps_to_canonical(self, patch_exec, tmp_path): + stream = [ + _turn_start(), + _tool_start("bash:0", "bash", {"command": "pytest -q"}), + _tool_end("bash:0", "bash", "ok"), + _turn_end(inp=10, out=5), + json.dumps({"type": "agent_settled"}), + ] + patch_exec(_FakeProcess(stream)) + record = await _run(_agent(), tmp_path) + assert record.commands[0].tool_name == "Bash" + assert record.commands[0].parameters == {"command": "pytest -q"} + + async def test_unknown_tool_passes_through(self, patch_exec, tmp_path): + stream = [ + _turn_start(), + _tool_start("x:0", "some_new_tool", {"whatever": 1}), + _tool_end("x:0", "some_new_tool", "ok"), + _turn_end(inp=10, out=5), + ] + patch_exec(_FakeProcess(stream)) + record = await _run(_agent(), tmp_path) + assert record.commands[0].tool_name == "some_new_tool" + assert record.commands[0].parameters == {"whatever": 1} + + async def test_edit_arg_keys_map_to_canonical(self, patch_exec, tmp_path): + stream = [ + _turn_start(), + _tool_start("e:0", "edit", {"path": "a.py", "oldString": "a", "newString": "b", "replaceAll": True}), + _tool_end("e:0", "edit", "ok"), + _turn_end(inp=10, out=5), + ] + patch_exec(_FakeProcess(stream)) + record = await _run(_agent(), tmp_path) + assert record.commands[0].parameters == { + "file_path": "a.py", + "old_string": "a", + "new_string": "b", + "replace_all": True, + } + + +class TestArgvConstruction: + async def test_base_flags_and_prompt(self, patch_exec, tmp_path): + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(), tmp_path) + + argv = captured["argv"] + assert argv[:6] == ["pi", "-p", "--mode", "json", "--no-context-files", "--no-approve"] + assert argv[argv.index("--model") + 1] == "openrouter/moonshotai/kimi-k3" + assert argv[argv.index("--thinking") + 1] == "medium" + assert "--session-dir" in argv and "--session-id" in argv + assert "--no-session" not in argv + assert argv[-2] == "--" + assert argv[-1] == "do the thing" + + async def test_enforced_tool_and_prompt_flags(self, patch_exec, tmp_path): + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run( + _agent(allowed_tools=["Read", "Write"], disallowed_tools=["Bash"], system_prompt="be terse"), + tmp_path, + ) + argv = captured["argv"] + assert argv[argv.index("--tools") + 1] == "Read,Write" + assert argv[argv.index("--exclude-tools") + 1] == "Bash" + assert argv[argv.index("--append-system-prompt") + 1] == "be terse" + + async def test_user_input_is_a_post_dashdash_argv_element(self, patch_exec, tmp_path): + """Shell-safety: the prompt is never interpolated into a shell string.""" + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(), tmp_path, prompt="rm -rf / ; echo $(whoami)") + argv = captured["argv"] + assert argv[-1] == "rm -rf / ; echo $(whoami)" + assert argv[argv.index("--") - 1] != "rm -rf / ; echo $(whoami)" + + async def test_explicit_line_limit_is_passed(self, patch_exec, tmp_path): + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(), tmp_path) + assert captured["kwargs"]["limit"] > 64 * 1024 + + +class TestSessionContinuity: + async def test_successive_calls_reuse_the_same_session(self, patch_exec, tmp_path): + """Multi-turn / simulation stitching: both invocations carry the SAME id+dir.""" + captured1 = patch_exec(_FakeProcess(HAPPY_STREAM)) + agent = _agent() + await _run(agent, tmp_path) + argv1 = captured1["argv"] + sid1 = argv1[argv1.index("--session-id") + 1] + sdir1 = argv1[argv1.index("--session-dir") + 1] + + captured2 = patch_exec(_FakeProcess(HAPPY_STREAM)) + await agent.communicate("follow up") + argv2 = captured2["argv"] + assert argv2[argv2.index("--session-id") + 1] == sid1 + assert argv2[argv2.index("--session-dir") + 1] == sdir1 + + async def test_start_creates_and_stop_removes_the_session_dir(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + agent = _agent() + await agent.start(str(tmp_path)) + session_dir = agent._session_dir + assert session_dir is not None and Path(session_dir).is_dir() + + await agent.stop() + assert not Path(session_dir).exists() + assert agent._session_dir is None + + +class TestEnvironmentInfo: + def test_carries_semantics_and_pi_fields(self): + info = _agent(thinking_level="high").get_environment_info() + assert info["system_prompt_semantics"] == "append" + assert info["pi_model"] == "openrouter/moonshotai/kimi-k3" + assert info["pi_thinking_level"] == "high" + + async def test_session_id_recorded_after_start(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + agent = _agent() + assert "pi_session_id" not in agent.get_environment_info() + await agent.start(str(tmp_path)) + assert agent.get_environment_info()["pi_session_id"].startswith("coder-eval-t1-") + + +class TestSandboxEnvironment: + async def test_prepends_mock_dirs_ahead_of_the_inherited_path(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setenv("PATH", "/parent/bin") + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + agent = _agent() + await agent.start(str(tmp_path), env_path_prepend=["/sandbox/mocks", "/sandbox/bins"]) + await agent.communicate("do the thing") + + assert captured["kwargs"]["env"]["PATH"] == os.pathsep.join(["/sandbox/mocks", "/sandbox/bins", "/parent/bin"]) + + async def test_host_environment_is_inherited_whole(self, patch_exec, tmp_path, monkeypatch): + monkeypatch.setenv("OPENROUTER_API_KEY", "sk-test") + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(), tmp_path) + assert captured["kwargs"]["env"]["OPENROUTER_API_KEY"] == "sk-test" + + +class TestUnsupportedConfigIsAnnounced: + async def test_start_warns_about_unenforced_fields(self, patch_exec, tmp_path, caplog): + patch_exec(_FakeProcess(HAPPY_STREAM)) + with caplog.at_level("WARNING"): + await _agent(plugins=[{"type": "local", "path": "/x"}]).start(str(tmp_path)) + assert "plugins" in caplog.text + assert "NOT enforced" in caplog.text + + async def test_enforced_fields_are_never_named_as_unenforced(self, patch_exec, tmp_path, caplog): + """`permission_mode` always warns (it has a truthy default and is unenforced), + but the ENFORCED fields must never appear in the unenforced-fields warning.""" + patch_exec(_FakeProcess(HAPPY_STREAM)) + with caplog.at_level("WARNING"): + await _agent(allowed_tools=["Read"], disallowed_tools=["Bash"], system_prompt="be terse").start( + str(tmp_path) + ) + # The unenforced-fields warning must not name any ENFORCED field. + warning = "".join(r.message for r in caplog.records if "NOT enforced" in r.message) + assert "allowed_tools" not in warning + assert "disallowed_tools" not in warning + assert "system_prompt" not in warning # neither system_prompt nor system_prompt_file set here + + +class TestAutoRetry: + async def test_two_agent_cycles_reduce_to_one_agent_end(self, patch_exec, tmp_path): + """Pi retries a transient error internally: agent_end(willRetry:true) then a + clean second cycle then agent_settled. The reducer must finalize ONCE.""" + stream = [ + json.dumps({"type": "agent_start"}), + _turn_start(), + _turn_end(inp=100, out=10, cost=0.001), + json.dumps({"type": "agent_end", "messages": [], "willRetry": True}), + json.dumps({"type": "agent_start"}), + _turn_start(), + _tool_start("w:0", "write", {"path": "a.txt", "content": "x"}), + _tool_end("w:0", "write", "ok"), + _turn_end(inp=200, out=20, cost=0.002), + json.dumps({"type": "agent_end", "messages": [], "willRetry": False}), + json.dumps({"type": "agent_settled"}), + ] + patch_exec(_FakeProcess(stream)) + recorder = _EventRecorder() + record = await _run(_agent(), tmp_path, stream_callback=recorder) + + assert record.crashed is False + assert len([e for e in recorder.events if isinstance(e, AgentEndEvent)]) == 1 + # Both cycles' usage merged (SUM): input 100+200, output 10+20. + assert record.token_usage is not None + assert record.token_usage.uncached_input_tokens == 300 + assert record.token_usage.output_tokens == 30 + assert record.assistant_turn_count == 2 + + +class TestCooperativeStop: + def test_capability_flag_is_declared(self): + assert PiAgent.supports_cooperative_stop is True + + async def test_should_stop_ends_turn_cleanly(self, patch_exec, tmp_path): + proc = _RunningProcess(HAPPY_STREAM) + patch_exec(proc) + record = await _run(_agent(), tmp_path, should_stop=lambda: True) + + assert record.crashed is False + assert proc.terminated is True + assert record.assistant_turn_count < 3 + + async def test_partial_record_is_returned_not_raised(self, patch_exec, tmp_path): + proc = _RunningProcess(HAPPY_STREAM) + patch_exec(proc) + record = await _run(_agent(), tmp_path, should_stop=lambda: True) + assert isinstance(record.assistant_turn_count, int) + + +class TestMaxTurns: + async def test_max_turns_marks_exhausted(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _run(_agent(), tmp_path, max_turns=1) + assert record.max_turns_exhausted is True + + async def test_a_cap_the_run_stays_under_is_not_exhausted(self, patch_exec, tmp_path): + """The fixture is exactly 3 turns, so max_turns=3 is the boundary.""" + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _run(_agent(), tmp_path, max_turns=3) + assert record.max_turns_exhausted is False + assert record.assistant_turn_count == 3 + + async def test_the_deciding_turn_is_kept_whole(self, patch_exec, tmp_path): + """max_turns=1 cuts at the START of turn 2, so turn 1 survives complete.""" + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _run(_agent(), tmp_path, max_turns=1) + + assert record.max_turns_exhausted is True + assert len(record.commands) == 1 # turn 1's write + usage = record.token_usage + assert usage is not None + assert usage.uncached_input_tokens == 406 # turn 1's input exactly + assert usage.output_tokens == 77 # 69 + 8 reasoning + + async def test_no_cap_is_uncapped(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _run(_agent(), tmp_path) + assert record.max_turns_exhausted is False + assert record.assistant_turn_count == 3 + + +class _HangingProcess(_FakeProcess): + def __init__(self, lines: list[str], **kwargs: Any) -> None: + super().__init__(lines, **kwargs) + self._exited = asyncio.Event() + + async def readline(self) -> bytes: + if self._lines: + return self._lines.pop(0) + await self._exited.wait() + return b"" + + async def read(self) -> bytes: + await self._exited.wait() + return self._stderr + + async def wait(self) -> int: + await self._exited.wait() + self.returncode = self._final_returncode + return self.returncode + + def terminate(self) -> None: + self.terminated = True + self._exited.set() + + def kill(self) -> None: + self.killed = True + self._exited.set() + + +class TestTimeoutContract: + async def test_deadline_raises_turn_timeout_with_partial_parked(self, patch_exec, tmp_path): + proc = _HangingProcess([_turn_start()]) + patch_exec(proc) + agent = _agent() + recorder = _EventRecorder() + + with pytest.raises(TurnTimeoutError): + await _run(agent, tmp_path, timeout=0.2, stream_callback=recorder) + + partial = agent.pending_turn + assert partial is not None + assert partial.crashed is True + assert proc.terminated is True + ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] + assert len(ends) == 1 + assert ends[0].status is AgentEndStatus.TIMEOUT + + await agent.discard_pending_turn() + assert agent._iteration == 0 + + +class _ExplodingProcess(_FakeProcess): + async def readline(self) -> bytes: + if self._lines: + return self._lines.pop(0) + raise ValueError("Separator is not found, and chunk exceed the limit") + + +class TestFailurePaths: + async def test_nonzero_exit_crashes(self, patch_exec, tmp_path): + patch_exec(_FakeProcess([], returncode=1, stderr=b"boom: bad model")) + with pytest.raises(AgentCrashError, match="boom: bad model"): + await _run(_agent(), tmp_path) + + async def test_empty_clean_exit_crashes_on_no_recognized_events(self, patch_exec, tmp_path): + patch_exec(_FakeProcess([], returncode=0)) + with pytest.raises(AgentCrashError, match="no recognized events"): + await _run(_agent(), tmp_path) + + async def test_drift_crash_names_the_unrecognized_types(self, patch_exec, tmp_path): + """A clean exit whose events are all unrecognized (schema drift) crashes and + names what it saw, for diagnosis.""" + stream = [ + json.dumps({"type": "some.new.event", "foo": 1}), + json.dumps({"type": "another.unknown", "bar": 2}), + ] + patch_exec(_FakeProcess(stream)) + with pytest.raises(AgentCrashError, match=r"another\.unknown, some\.new\.event") as exc: + await _run(_agent(), tmp_path) + assert "no recognized events" in str(exc.value) + + async def test_stream_error_becomes_a_crash_with_partial_parked(self, patch_exec, tmp_path): + stream = [_turn_start(), _tool_start("w:0", "write", {"path": "a.txt", "content": "x"})] + patch_exec(_ExplodingProcess(stream)) + agent = _agent() + + with pytest.raises(AgentCrashError, match="Pi turn failed"): + await _run(agent, tmp_path) + + partial = agent.pending_turn + assert partial is not None + assert partial.crashed is True + # The in-flight tool was force-closed rather than dropped. + assert [c.result_status for c in partial.commands] == ["unknown"] + + async def test_spawn_failure_becomes_a_crash(self, monkeypatch, tmp_path): + async def boom(*_argv: str, **_kwargs: Any): + raise OSError("no fork for you") + + monkeypatch.setattr(asyncio, "create_subprocess_exec", boom) + monkeypatch.setattr("shutil.which", lambda _name: "/usr/local/bin/pi") + + with pytest.raises(AgentCrashError, match="no fork for you"): + await _run(_agent(), tmp_path) + + async def test_malformed_line_is_skipped(self, patch_exec, tmp_path): + stream = ["not json at all", *HAPPY_STREAM] + patch_exec(_FakeProcess(stream)) + record = await _run(_agent(), tmp_path) + assert record.crashed is False + assert record.assistant_turn_count == 3 + + async def test_missing_cli_is_actionable(self, monkeypatch, tmp_path): + monkeypatch.setattr("shutil.which", lambda _name: None) + with pytest.raises(RuntimeError, match="@earendil-works/pi-coding-agent"): + await _agent().start(str(tmp_path)) + + +class TestUnexpectedErrorContract: + async def test_terminal_event_emitted_exactly_once_on_crash(self, patch_exec, tmp_path): + patch_exec(_ExplodingProcess([_turn_start()])) + recorder = _EventRecorder() + + with pytest.raises(AgentCrashError): + await _run(_agent(), tmp_path, stream_callback=recorder) + + ends = [e for e in recorder.events if isinstance(e, AgentEndEvent)] + assert len(ends) == 1 + assert ends[0].crashed is True + assert ends[0].status is AgentEndStatus.CRASHED + + async def test_iteration_rolls_back_after_the_crash(self, patch_exec, tmp_path): + patch_exec(_ExplodingProcess([])) + agent = _agent() + + with pytest.raises(AgentCrashError): + await _run(agent, tmp_path) + assert agent._iteration == 1 + await agent.discard_pending_turn() + assert agent._iteration == 0 + assert agent.pending_turn is None + + +class TestTurnEventsAreBalanced: + @staticmethod + def _pairs(recorder: _EventRecorder) -> tuple[int, int]: + starts = len([e for e in recorder.events if isinstance(e, TurnStartEvent)]) + ends = len([e for e in recorder.events if isinstance(e, TurnEndEvent)]) + return starts, ends + + async def test_a_clean_turn_is_balanced(self, patch_exec, tmp_path): + patch_exec(_FakeProcess(HAPPY_STREAM)) + recorder = _EventRecorder() + await _run(_agent(), tmp_path, stream_callback=recorder) + assert self._pairs(recorder) == (3, 3) + + async def test_a_timeout_closes_the_open_turn(self, patch_exec, tmp_path): + proc = _HangingProcess([_turn_start()]) + patch_exec(proc) + recorder = _EventRecorder() + + with pytest.raises(TurnTimeoutError): + await _run(_agent(), tmp_path, timeout=0.2, stream_callback=recorder) + + assert self._pairs(recorder) == (1, 1) + end = next(e for e in recorder.events if isinstance(e, TurnEndEvent)) + assert end.status is TurnEndStatus.TIMEOUT + + +class TestFactoryContract: + def test_route_accepted_positionally_and_by_keyword(self): + config = PiAgentConfig(type="pi", model="openrouter/moonshotai/kimi-k3") + assert PiAgent(config, route=None).route is None + assert PiAgent(config, None).route is None + + def test_undeclared_kwarg_raises(self): + config = PiAgentConfig(type="pi", model="openrouter/moonshotai/kimi-k3") + assert PiAgent.supports_cost_log_tags is False + with pytest.raises(TypeError): + PiAgent(config, cost_log_tags={"x": "y"}) # type: ignore[call-arg] + + +class TestRegistry: + def test_create_agent_returns_pi_agent(self): + from coder_eval.agents import AgentRegistry, create_agent, register_builtins + + register_builtins(AgentRegistry) + agent = create_agent(AgentKind.PI, PiAgentConfig(type="pi")) + assert isinstance(agent, PiAgent) + + def test_list_kinds_includes_pi(self): + from coder_eval.agents import AgentRegistry, register_builtins + + register_builtins(AgentRegistry) + assert "pi" in AgentRegistry.list_kinds() + + def test_parse_agent_config_dispatches_to_pi(self): + from coder_eval.models import parse_agent_config + + cfg = parse_agent_config(type="pi", model="openrouter/moonshotai/kimi-k3") + assert isinstance(cfg, PiAgentConfig) + assert cfg.model == "openrouter/moonshotai/kimi-k3" + + +@pytest.mark.skipif(os.name != "posix", reason="process-group teardown is POSIX-only by design") +class TestProcessGroupTeardown: + async def test_spawn_uses_its_own_session(self, patch_exec, tmp_path): + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(), tmp_path) + assert captured["kwargs"]["start_new_session"] is (os.name == "posix") + + async def test_stop_sweeps_the_spawned_group(self, patch_exec, tmp_path): + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + agent = _agent() + await _run(agent, tmp_path) + await agent.stop() + assert (4242, signal.SIGKILL) in captured["killpg"] + + async def test_kill_sync_signals_pid_and_group(self, patch_exec, monkeypatch, tmp_path): + killed: list[tuple[int, int]] = [] + monkeypatch.setattr(os, "kill", lambda pid, sig: killed.append((pid, sig))) + captured = patch_exec(_HangingProcess([])) + agent = _agent() + await agent.start(str(tmp_path)) + proc = _HangingProcess([]) + agent._process = proc # type: ignore[assignment] + agent._spawned_pgids = [proc.pid] + + agent.kill_sync() + + assert (4242, signal.SIGKILL) in killed + assert (4242, signal.SIGKILL) in captured["killpg"] + + +class TestToolFailureCapture: + async def test_tool_error_is_captured(self, patch_exec, tmp_path): + stream = [ + _turn_start(), + _tool_start("b:0", "bash", {"command": "ls /root"}), + _tool_end("b:0", "bash", "boom: command exploded", is_error=True), + _turn_end(inp=10, out=5), + ] + patch_exec(_FakeProcess(stream)) + recorder = _EventRecorder() + record = await _run(_agent(), tmp_path, stream_callback=recorder) + + [cmd] = record.commands + assert cmd.result_status == "error" + assert cmd.error_message == "boom: command exploded" + [end] = [e for e in recorder.events if isinstance(e, ToolEndEvent)] + assert end.status is ToolEndStatus.ERROR + + async def test_permission_denial_gets_its_own_status(self, patch_exec, tmp_path): + stream = [ + _turn_start(), + _tool_start("b:0", "bash", {"command": "cat /etc/shadow"}), + _tool_end("b:0", "bash", "Permission denied by policy", is_error=True), + _turn_end(inp=10, out=5), + ] + patch_exec(_FakeProcess(stream)) + recorder = _EventRecorder() + await _run(_agent(), tmp_path, stream_callback=recorder) + [end] = [e for e in recorder.events if isinstance(e, ToolEndEvent)] + assert end.status is ToolEndStatus.PERMISSION_DENIED + + def test_orphan_result_is_never_dropped(self): + state = _PiTurnState(task_id="t", iteration=1, user_input="x", model=None) + events: list[Any] = [] + state.bind(events.append) + state._close_tool("ghost", status=ToolEndStatus.UNRESOLVED, summary=None, error="no result observed") + + [event] = events + assert isinstance(event, ToolEndEvent) + assert event.tool.tool_name == "unknown" + assert event.tool.result_status == "unknown" + + +class TestZeroUsageTurn: + async def test_zero_usage_turn_is_scored_not_crashed(self, patch_exec, tmp_path): + """A provider reporting no usage yields an all-zero token_usage; Pi scores + it (no require_token_telemetry hard-fail) since its multi-provider surface + makes that brittle.""" + stream = [ + _turn_start(), + _turn_end(inp=0, out=0), + json.dumps({"type": "agent_settled"}), + ] + patch_exec(_FakeProcess(stream)) + record = await _run(_agent(), tmp_path) + assert record.crashed is False From 95b0094d1ee266ed057a53b5269d6ed61ba278d1 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 12:03:13 +0100 Subject: [PATCH 03/24] =?UTF-8?q?feat(tasks):=203/4=20=E2=80=94=20add=20lo?= =?UTF-8?q?cal=20pi=5Fsmoke=5Ftest=20task?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror tasks/opencode_smoke_test.yaml for the Pi harness: type pi, the spike-verified kimi-k3 model, three criteria (file_exists / file_contains / run_command), and no smoke-pass tag (run locally — CI E2E lacks the pi CLI + OpenRouter creds). A loader test asserts it resolves to PiAgentConfig. Co-Authored-By: Claude Opus 4.8 (1M context) --- tasks/pi_smoke_test.yaml | 38 +++++++++++++++++++++++++++++++++++++ tests/test_pi_smoke_task.py | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tasks/pi_smoke_test.yaml create mode 100644 tests/test_pi_smoke_task.py diff --git a/tasks/pi_smoke_test.yaml b/tasks/pi_smoke_test.yaml new file mode 100644 index 00000000..f35084de --- /dev/null +++ b/tasks/pi_smoke_test.yaml @@ -0,0 +1,38 @@ +task_id: "pi_smoke_test" +description: "Smoke-test the Pi agent harness: create and run a small Python script." +initial_prompt: "Create a Python file named app.py in the current working directory that prints 'Hello, Pi!' on one line, and today's date in YYYY-MM-DD format on the next line. Use the datetime module. Then run the script with: python app.py" +# No `smoke-pass`: that tag routes a task into the CI E2E bucket, which runs on +# Bedrock runners with no `pi` CLI and no OpenRouter credentials. Run this task +# locally (or in a job that installs both) instead — +# `npm install -g @earendil-works/pi-coding-agent`, then export OPENROUTER_API_KEY. +tags: [smoke, basic, pure-python, pi] + +run_limits: + expected_turns: 5 + # The CLI drives a full agent loop per invocation; give it room but keep the + # smoke bounded so a hung provider fails fast instead of stalling a suite. + task_timeout: 600 + +agent: + type: "pi" + # Pi addresses models as provider/model and speaks OpenRouter natively + # (authenticated by OPENROUTER_API_KEY), so it needs no LiteLLM proxy. Real + # per-call cost lands on the turn: Pi reports `cost` on every turn_end event + # and the harness folds it into token_usage.total_cost_usd (falling back to the + # rate card only when the stream omits it). Spike-verified working model; + # swappable to any Pi-addressable provider/id. + model: "openrouter/moonshotai/kimi-k3" + permission_mode: "acceptEdits" + +success_criteria: + - type: "file_exists" + path: "app.py" + description: "The file app.py must be created." + - type: "file_contains" + path: "app.py" + includes: ["Hello, Pi!", "datetime"] + description: "The script must contain the required string and import." + - type: "run_command" + command: "python app.py" + timeout: 10 + description: "The script must execute successfully." diff --git a/tests/test_pi_smoke_task.py b/tests/test_pi_smoke_task.py new file mode 100644 index 00000000..09bdebe3 --- /dev/null +++ b/tests/test_pi_smoke_task.py @@ -0,0 +1,38 @@ +"""Phase 3: the local Pi smoke task loads, resolves to PiAgentConfig, validates. + +The task is intentionally excluded from the CI E2E bucket (no ``smoke-pass`` +tag — the CI runners have no ``pi`` CLI and no OpenRouter credentials); it is a +local smoke. This test only proves the YAML parses and resolves, offline. +""" + +from __future__ import annotations + +from pathlib import Path + +from coder_eval.models import PiAgentConfig +from coder_eval.orchestration.task_loader import load_task + + +_TASK = Path("tasks/pi_smoke_test.yaml") + + +def test_task_loads_and_resolves_to_pi_agent_config(): + task, _raw = load_task(_TASK) + assert task.task_id == "pi_smoke_test" + assert isinstance(task.agent, PiAgentConfig) + assert task.agent.type == "pi" + assert task.agent.model == "openrouter/moonshotai/kimi-k3" + + +def test_criteria_validate(): + task, _raw = load_task(_TASK) + types = [c.type for c in task.success_criteria] + assert types == ["file_exists", "file_contains", "run_command"] + + +def test_not_in_ci_e2e_bucket(): + """No `smoke-pass` tag: the task must not route into the CI E2E bucket, which + has no `pi` CLI or OpenRouter credentials.""" + task, _raw = load_task(_TASK) + assert "pi" in task.tags + assert "smoke-pass" not in task.tags From 1e4b61a83b30cd397552facb108260a12a7c0174 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 12:11:03 +0100 Subject: [PATCH 04/24] =?UTF-8?q?docs(agents):=204/4=20=E2=80=94=20add=20P?= =?UTF-8?q?i=20harness=20docs=20+=20enumeration-surface=20parity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New docs/agents/PI.md (mirrors OPENCODE.md), a filled pi column + divergence prose in HARNESS_PARITY.md, mkdocs nav + blurb (regenerated index tables), and every prose enumeration of the harness set (README, index, USER_GUIDE, EXTENDING, llms.txt, CLAUDE.md) extended to include Pi. Fix the stale agent.type comment in experiments/default.yaml. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 2 +- README.md | 9 +- docs/EXTENDING.md | 4 +- docs/USER_GUIDE.md | 2 +- docs/agents/HARNESS_PARITY.md | 52 ++++++-- docs/agents/PI.md | 240 ++++++++++++++++++++++++++++++++++ docs/index.md | 9 +- docs/llms.txt | 3 +- experiments/default.yaml | 3 +- mkdocs.yml | 2 + uv.lock | 2 +- 11 files changed, 304 insertions(+), 24 deletions(-) create mode 100644 docs/agents/PI.md diff --git a/CLAUDE.md b/CLAUDE.md index ac00d5ef..bf2286f2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -144,7 +144,7 @@ action.yml # Published composite GitHub Action (coder-ev - **Sub-agent token accounting**: There is NO separate per-sub-agent field. Every sub-agent generation is captured as a `parent_tool_use_id`-tagged `AssistantMessage` in the turn transcript, so per-sub-agent usage is derived by grouping those messages on that id (the evalboard's `aggregateSubAgentUsage` does exactly this). Claude bubbles its sub-agent's intermediate generations into the parent stream natively, and the **terminal** generation (delivered as the Agent tool result, never streamed) is synthesized into one via `_synthesize_subagent_terminal_message` from `tool_use_result.usage`. Codex reconstructs all child generations from the child rollout (`_recover_subagent_tool_calls`). The turn total already includes sub-agent cost — Claude via the SDK's cumulative `model_usage`; Codex via `_fold_subagent_tokens`, which folds the child messages (their real per-generation tokens) into the parent total. `CommandTelemetry.result_summary` is stored **untruncated** (no 200-char cap) so sub-agent returns are preserved whole. Set `CODER_EVAL_RAW_SDK_LOG=1` to dump every raw SDK event to the task log for inspection. - **Reconciliation message (stream self-reconciles to the turn total)**: The per-message stream consistently under-reports the authoritative turn total — a fixed prompt slice (~512 input tokens on Claude) is billed on no SDK-emitted message, and sub-agent input/cache only partially bubbles up. So `EventCollector.build_turn_record` appends one synthetic `ReconciliationMessage` (`role="reconciliation"`, in the `TranscriptMessage` union) per turn, carrying the per-bucket residual = `token_usage` − Σ(assistant message buckets). The invariant: **summing the four token buckets across `TurnRecord.messages` (assistant + reconciliation) equals `token_usage` exactly**, for both Claude and Codex (Codex's stream is already complete after `_recover_subagent_tool_calls`, so its residual is usually 0 and no entry is emitted). This is what lets the evalboard SUM the message stream as the source of truth instead of reading a separate aggregate ("agent tokens"): `selectTokenTotals` returns the stream sum whenever a reconciliation entry is present, and the timeline renders it as its own row. It is agent-agnostic (booked at the single `EventCollector` seam), carries no cost (cost stays on `token_usage`), and is excluded from generation/turn counts and the cost simulator. The LiteLLM open-weight actual-cost join (`litellm_cost.apply_actual_cost`) deliberately writes cost at the TURN level only (`token_usage.total_cost_usd` = the real OpenRouter bill) plus the per-call `TurnRecord.provider_call_costs` audit record; it does NOT touch the message token buckets, so `EventCollector` stays the single writer and this invariant holds on every backend. The Python `token_usage`/`total_token_usage` aggregate is unchanged and still authoritative for budget/judges/reports. - **Reference solutions are directory-only, and shielded (partially) from the agent**: `task.reference` is a single required `directory:` (relative to the task YAML) — the inline `code:` / single-file `file:` forms are gone, because a directory is the only shape that can be permission-gated as a unit; a `model_validator(mode="before")` gives the removed forms a migration error. The orchestrator stages a **per-run private copy** (`orchestration/evaluation.py::stage_reference_dir`, symlinks stripped) into a tempdir, removed in `_cleanup` via `path_utils.rmtree_restrictive` (keyed on `_reference_staging_root`, recorded BEFORE the copy so a failed copy still cleans up; `rmtree(ignore_errors=True)` silently declines on a tree left at 000) and deliberately never preserved into `run_dir/artifacts`. That copy is held at mode `000` for the whole of every `agent.communicate` call via **`Sandbox.set_permissions`**, the driver-aware wrapper over `fs_permissions.py::set_permissions`. Windows **stack**: exiting restores the *enclosing* window's mode, only the outermost exit restores the pre-window mode — that is what makes a mid-turn re-grant (`mode=READ_ONLY_MODE`) expressible, and it covers two windows at the same mode so no refcount is needed. The window is enforced **only inside a docker container** (`Sandbox.enforces_permission_windows`) and is a no-op on the host, where the agent shares our uid. **That gate keys on the `CODER_EVAL_IN_CONTAINER` env var, NOT `sandbox.driver`** — `run_task_internal_command` rewrites `driver: docker` → `tempdir` before building the in-container Orchestrator, so a driver-based gate would silently disable the anti-cheat on exactly the path that needs it (regression-guarded by `TestSandboxDriverGate`); `resolve_reference_dir` gates its `/work/references` branch on the same var for the same reason. The task directory is **not** shielded (`:ro` mount → EROFS, and the same YAML is readable at `/work/input`). Criteria address reference files with the `$REFERENCE_DIR` token (same resolver as `$TASK_DIR`) and the `REFERENCE_DIR` env var for `run_command`; `reference_comparison` names one file via `reference_file`. Docker mounts a throwaway **read-write** copy at `/work/references` (a `:ro` mount cannot be chmod'd — EROFS), masks the in-task-dir original with an empty tmpfs, and drops `DAC_OVERRIDE`/`DAC_READ_SEARCH`. `FOWNER`/`CHOWN` are deliberately **NOT** dropped: the in-container orchestrator that applies the window is the same root process with the same caps, so dropping `FOWNER` breaks *the harness's own* chmod wherever the bind mount preserves a non-root owner (native Linux — verified: `chmod: Operation not permitted`), i.e. exactly where the drop would otherwise bite. A window that cannot be applied is now a hard error, not a warning: `Sandbox.set_permissions` passes `strict=True` whenever it enforces, so an unprotected run fails instead of producing a normal-looking score. **KNOWN GAP — this is defense-in-depth, not a boundary**: (a) `chmod(2)` is gated on owner-or-`CAP_FOWNER` and the container runs as root owning the copy, so a deliberate `chmod 755 /work/references` restores access; (b) the window spans `agent.communicate` only, and nothing reaps agent child processes at turn end, so a backgrounded read loop succeeds once the window closes. The **write** half of (b) is closed — `path_utils.digest_tree` hashes the tree at staging and `Orchestrator._verify_reference_integrity` re-checks before grading, raising `ReferenceTamperedError` (→ `FinalStatus.ERROR`) on a mismatch so an agent cannot overwrite the reference to drive `reference_comparison` to 1.0. Passive reads are blocked; an adversarial agent is not. Full containment requires running the agent as a non-root uid AND holding the window for the agent's whole lifetime — follow-up. `tasks/anti_cheat_reference` probes the passive-read half. -- **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. +- **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. **Pi** is the same shape — the CLI (`pi -p --mode json`) streams a real multi-step loop per `communicate()` (`turn_start`/`turn_end`), so `max_turns: N` counts native `turn_start` steps; Pi retries transient/provider errors INTERNALLY (`agent_end.willRetry`), and the reducer finalizes once at `agent_settled`/EOF (not the first `agent_end`), folding the retry cycles into one turn. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex, Antigravity, and Pi (all run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced) — Pi, in contrast, ENFORCES all three (`--tools`/`--exclude-tools`/`--append-system-prompt`, a capability win over OpenCode) but does NOT read `plugins` (warned+ignored; no activation suites in v1, though it has a native `--skill` flag) nor `system_prompt_file`, and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/README.md b/README.md index 1273b57c..385bfe42 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,12 @@ **evaluating and benchmarking AI coding agents and their skills** — built for CLI and skill builders — with sandboxing, reproducibility, and data-driven analysis. It runs a real agent (**Claude Code**, **Codex**, **Google Antigravity / -Gemini**, or **OpenCode**) in a sandbox against declarative YAML tasks, then scores the files and +Gemini**, **OpenCode**, or **Pi**) in a sandbox against declarative YAML tasks, then scores the files and commands it actually produced. Not an "agentic coding" benchmark: it measures how effective your CLI and skills are when used by coding agents. Reach for it when you want to **test whether a Claude Code skill triggers**, -**A/B-test Claude Code vs. Codex vs. Gemini vs. OpenCode** (or model vs. model, +**A/B-test Claude Code vs. Codex vs. Gemini vs. OpenCode vs. Pi** (or model vs. model, prompt vs. prompt), or **gate CI on coding-agent quality**. Unlike fixed datasets (SWE-bench, SkillsBench) that rank models on a shared leaderboard, Coder Eval evaluates the tasks, skills, and workflows *you* ship — with weighted 0.0–1.0 criteria, a @@ -33,14 +33,14 @@ telemetry. See [How it compares](https://coder-eval.com/docs/comparison). - **Sandboxed execution** in isolated environments with resource limits - **Weighted, continuous scoring** (0.0–1.0) with fractional credit and thresholds - **Many criterion types** — from file checks to code similarity and LLM-graded rubrics -- **Agent abstraction** — Claude Code, Codex, Antigravity (Gemini), and OpenCode today, extensible via a plugin SPI +- **Agent abstraction** — Claude Code, Codex, Antigravity (Gemini), OpenCode, and Pi today, extensible via a plugin SPI - **Experiment layer** — A/B agent configs (models, tools, prompts) side-by-side - **Full telemetry** — every tool call, token counts, and cost, with real-time streaming ## What you can do with it - **Benchmark coding agents** — score an agent across a suite of tasks with weighted scoring and pass/fail thresholds -- **Compare models & configs** — A/B-test Claude vs. Codex vs. Gemini vs. OpenCode, model vs. model, tool-on vs. tool-off, prompt vs. prompt +- **Compare models & configs** — A/B-test Claude vs. Codex vs. Gemini vs. OpenCode vs. Pi, model vs. model, tool-on vs. tool-off, prompt vs. prompt - **Evaluate skills** — verify an agent actually engages a target skill (`skill_triggered`) and score skill-driven suites (SkillsBench-style) - **Keep skills up to date in CI** — re-validate your skills on every change or on a schedule; catch silent regressions when models, prompts, or the skills themselves drift - **Gate CI on agent quality** — run the suite in GitHub Actions and fail the build on regressions @@ -216,6 +216,7 @@ The step's exit code is coder-eval's own: non-zero on any failed task. | [Codex](docs/agents/CODEX.md) | Running the OpenAI Codex agent | | [Antigravity (Gemini)](docs/agents/ANTIGRAVITY.md) | Running the Google Antigravity / Gemini agent | | [OpenCode](docs/agents/OPENCODE.md) | Running the OpenCode agent on open-weight models | +| [Pi](docs/agents/PI.md) | Running the Pi agent on open-weight models | | [Run-Limit Parity](docs/agents/HARNESS_PARITY.md) | What each run_limits field means on every harness | | [A/B Experiments](docs/AB_EXPERIMENTS.md) | Compare models / tools / prompts across the same tasks | | [Bring Your Own Dataset](docs/DATASETS.md) | Fan a single task out over a dataset | diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index 3befae19..401380d9 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -285,8 +285,8 @@ The base package ships **no** plugin rates; only the built-in table. ## See also - [Claude Code](agents/CLAUDE_CODE.md) · [Codex](agents/CODEX.md) · - [Antigravity](agents/ANTIGRAVITY.md) · [OpenCode](agents/OPENCODE.md) — the - built-in agents, each registered via this same SPI + [Antigravity](agents/ANTIGRAVITY.md) · [OpenCode](agents/OPENCODE.md) · + [Pi](agents/PI.md) — the built-in agents, each registered via this same SPI - [Task Definition Guide](TASK_DEFINITION_GUIDE.md) — the criterion catalogue - [CLAUDE.md](https://github.com/UiPath/coder_eval/blob/main/CLAUDE.md) — architecture and extension points in depth diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 1676690d..33e15620 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -39,7 +39,7 @@ coder-eval run tasks/hello_date.yaml --stream full # live LLM output | `-D path=value` / `--set` | Override any resolved task-config field (`agent`/`run_limits`/`sandbox` roots), e.g. `-D run_limits.max_turns=30 -D agent.permission_mode=plan -D agent.sdk_options.effort=high`. Repeatable; schema-validated. This is the way to set permission mode, turn/timeout limits, token/USD budget caps, tools, plugins, and SDK options. | | `--model, -m` | Shorthand alias for `-D agent.model=…` (e.g., `claude-sonnet-5`) | | `--driver` | Shorthand alias for `-D sandbox.driver=…` (`tempdir` or `docker`) | -| `--type, -T` | Override agent type for all tasks (`claude-code`, `codex`, `antigravity`, `opencode`, or a plugin kind). | +| `--type, -T` | Override agent type for all tasks (`claude-code`, `codex`, `antigravity`, `opencode`, `pi`, or a plugin kind). | | `--repeats` | Run each `(task, variant)` N times (≥1); overrides experiment/variant `repeats:`. See [Replicates](#replicates). | | `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. A task with *any* final status (incl. FAILED/ERROR) counts as finalized, so resume does **not** retry failures — delete a task's `task.json` to force a re-run. A config mismatch is warned, not refused. | | `--sample N` | For dataset-backed tasks, run a fixed-seed random N-row sample (reproducible; cheap smoke test). See [Bring Your Own Dataset](DATASETS.md). | diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 8053c47b..1ad5c6a7 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -10,12 +10,12 @@ This page is the contract for what each run limit means per harness, plus the sh ## The table -| Limit | claude-code | codex | antigravity | opencode | -|---|---|---|---|---| -| `run_limits.max_turns` | native SDK cap (agent-loop turns) | visible-turn cap (resolved tool calls) | visible-turn cap (resolved tool calls) | native step cap (the CLI's own agent-loop steps) | -| `run_limits.turn_timeout` | watchdog, SIGKILL on the CLI subprocess | watchdog + cooperative interrupt | watchdog, plus an earlier internal poll deadline at 80% of it (see below) | deadline enforced in-loop and on the final reap; SIGTERM→SIGKILL on the CLI's whole process group | -| `run_limits.task_timeout` | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | -| `run_limits.stop_early` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` (event granularity) | +| Limit | claude-code | codex | antigravity | opencode | pi | +|---|---|---|---|---|---| +| `run_limits.max_turns` | native SDK cap (agent-loop turns) | visible-turn cap (resolved tool calls) | visible-turn cap (resolved tool calls) | native step cap (the CLI's own agent-loop steps) | native turn cap (the CLI's own `turn_start` agent-loop steps) | +| `run_limits.turn_timeout` | watchdog, SIGKILL on the CLI subprocess | watchdog + cooperative interrupt | watchdog, plus an earlier internal poll deadline at 80% of it (see below) | deadline enforced in-loop and on the final reap; SIGTERM→SIGKILL on the CLI's whole process group | deadline enforced in-loop and on the final reap; SIGTERM→SIGKILL on the CLI's whole process group | +| `run_limits.task_timeout` | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | orchestrator-level, agent-agnostic | +| `run_limits.stop_early` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` | cooperative `should_stop` (event granularity) | cooperative `should_stop` (event granularity — Pi streams incrementally) | ## `max_turns` counts visible turns on Codex and Antigravity @@ -50,6 +50,14 @@ when step N+1 begins, with the completed steps' tokens intact. A step is one assistant generation and may carry several tool calls — so, as with claude-code, the same number is a looser tool-call budget than on the visible-turn backends. +**Pi keeps a native unit too — its `turn_start` agent-loop steps.** Like OpenCode, +`pi -p --mode json` runs a real multi-step agent loop per invocation and streams it +(`turn_start` / `turn_end`), so `max_turns: N` allows N complete turns and cuts the +run when turn N+1 begins, with the completed turns' tokens intact. Pi streams +incrementally, so the cut genuinely stops spend mid-run. A Pi turn is one assistant +generation and may carry several tool calls — the same looser budget as claude-code +and OpenCode. + **So holding `max_turns` constant across harnesses does not hold the budget constant.** If you are A/B-ing across backends and the cap is close to binding, that is the number to distrust. @@ -165,15 +173,41 @@ plugin-root shape by lint rule CE045. The rule keys on that variable name only; one, feeds `experiments/plugin-comparison.yaml`, whose default agent is claude-code, so the same requirement applies there and is unlinted. +**OpenCode and Pi diverge further on `plugins`.** OpenCode honors only the *skills* +half of a plugin (mapped to its `skills.paths` — see [OpenCode](OPENCODE.md)). Pi does +**not read `plugins` at all in v1**: it warns and ignores them, so it cannot run +activation suites yet — even though the CLI has a native `--skill ` flag that +a follow-up could wire `agent.plugins → --skill` onto. See +[Pi § Known limitations](PI.md#known-limitations). + +## Pi enforces the tool/prompt knobs OpenCode drops + +Pi is the outlier in the enforcement direction — a capability *win*, not a gap: + +- **`allowed_tools` / `disallowed_tools` are ENFORCED** (`--tools` / `--exclude-tools`), + and **`system_prompt` is ENFORCED** (`--append-system-prompt`, semantics `append`). + OpenCode drops all three; Codex forwards `disallowed_tools` without SDK enforcement. +- **`permission_mode` is NOT enforced** — Pi headless print mode auto-runs tools and + exposes only project-file trust (`--approve` / `--no-approve`), no tool-approval + mode; the sandbox driver is the isolation boundary (same as Codex/Antigravity). +- **`system_prompt_file` is NOT read** (use inline `system_prompt`), matching + Codex/Antigravity. +- **Built-in auto-retry.** Pi retries a transient/provider error *internally* (another + `agent_start` cycle in the same invocation, flagged `willRetry: true`), which the + harness folds into one turn. The internal retry is bounded by + `turn_timeout` / `task_timeout`. + +Full detail: [Pi](PI.md). + ## Reproducing `tasks/run_limits/` holds one fixture per limit: `max_turns_cap.yaml` asks for more sequential work than its cap allows, and `turn_timeout.yaml` runs a command that outlives its watchdog. Run either with `--type claude-code` / `--type codex` / -`--type antigravity` / `--type opencode` to check a backend against the contract -above. +`--type antigravity` / `--type opencode` / `--type pi` to check a backend against the +contract above. ## Related -- [Claude Code](CLAUDE_CODE.md) · [Codex](CODEX.md) · [Antigravity](ANTIGRAVITY.md) · [OpenCode](OPENCODE.md) +- [Claude Code](CLAUDE_CODE.md) · [Codex](CODEX.md) · [Antigravity](ANTIGRAVITY.md) · [OpenCode](OPENCODE.md) · [Pi](PI.md) - [Task Definition Guide](../TASK_DEFINITION_GUIDE.md) — the full `run_limits` schema diff --git a/docs/agents/PI.md b/docs/agents/PI.md new file mode 100644 index 00000000..b8974849 --- /dev/null +++ b/docs/agents/PI.md @@ -0,0 +1,240 @@ +--- +description: >- + Run Pi, the pi Node coding agent, as the agent under evaluation in Coder + Eval — installation, provider authentication, model selection, the enforced + and divergent config fields, and how its event stream maps to sandboxed, + weighted scoring. +--- + +# Running Pi in Coder Eval + +## Overview + +[Pi](https://pi.dev/) is a Node terminal coding agent (the `pi` CLI). Coder Eval +drives it in **JSON print mode**: + +```bash +pi -p --mode json --no-context-files --no-approve \ + --session-dir --session-id -m "" +``` + +`--mode json` streams **newline-delimited JSON events** on stdout, one event per +line. `PiAgent` reduces that stream into the standardized event protocol +(`AgentStart` / `TurnStart` / `ToolStart` / `ToolEnd` / `TurnEnd` / `AgentEnd`) +and lets `EventCollector` build the `TurnRecord`, exactly like every other +harness. + +Because Pi is model-agnostic, this is a cheap way to evaluate a broad set of +open-weight models (Kimi, DeepSeek, GLM, …) through a single agent — and unlike +OpenCode, Pi **enforces** `allowed_tools` / `disallowed_tools` / `system_prompt`. + +## Setup + +### 1. Install the Pi CLI + +Pi is a **Node** CLI, not a Python package: + +```bash +npm install -g @earendil-works/pi-coding-agent +pi --version +``` + +The `coder-eval[pi]` extra exists for symmetry with the other harnesses and +carries **no Python dependencies** — the agent shells out to the binary above and +imports no third-party package: + +```bash +uv sync --extra pi # documents the opt-in; installs no extra packages +``` + +If the binary is missing, the task fails at `start()` with an actionable error +naming the install command, rather than failing obscurely mid-run. + +### 2. Authentication + +Pi addresses a model as `provider/id` and reads the provider key from the +environment, which Coder Eval forwards into the subprocess (it also picks up keys +from `.env`, since `config.py` calls `load_dotenv(override=True)`): + +```bash +export OPENROUTER_API_KEY="sk-or-..." # OpenRouter (any model) +``` + +## Usage + +### Command line + +```bash +uv run coder-eval run tasks/pi_smoke_test.yaml +uv run coder-eval run tasks/my_task.yaml -D agent.type=pi -D agent.model=openrouter/moonshotai/kimi-k3 +``` + +### Task definition (YAML) + +```yaml +agent: + type: "pi" + # provider-prefixed model id; no separate --provider needed. + model: "openrouter/moonshotai/kimi-k3" + permission_mode: "acceptEdits" + thinking_level: "medium" # optional: reasoning effort (see below) +``` + +### Model selection + +`agent.model` is passed through verbatim to `--model`, so it must be Pi's +provider-prefixed `provider/id` form. The provider prefix decides which +credential is used: + +| `agent.model` | Provider | Credential | +|---|---|---| +| `openrouter/moonshotai/kimi-k3` | OpenRouter | `OPENROUTER_API_KEY` | + +Pi speaks OpenRouter natively, so it does **not** need the LiteLLM proxy — that +shim exists to translate Anthropic ↔ OpenAI for the Claude Code SDK. + +> On some OpenRouter accounts a model is blocked by an account **guardrail** +> (`404 "model blocked by guardrail"`). That is an account setting, not a Coder +> Eval problem — adjust it at +> [openrouter.ai/settings/privacy](https://openrouter.ai/settings/privacy), or +> pick another model. `openrouter/moonshotai/kimi-k3` runs cleanly on the dev +> account and is the smoke task's default. + +### `thinking_level` + +Pi's reasoning effort, forwarded as `--thinking`. Accepts the seven-value set +`off` / `minimal` / `low` / `medium` / `high` / `xhigh` / `max` (a strict +superset of the Antigravity `thinking_level`), defaulting to `medium`. + +### Enforced config fields + +Unlike OpenCode, Pi **enforces** the standard tool/prompt knobs (a capability +win): + +| Field | Pi flag | +|---|---| +| `allowed_tools` | `--tools ` | +| `disallowed_tools` | `--exclude-tools ` | +| `system_prompt` | `--append-system-prompt ` (appended, semantics `append`) | +| `thinking_level` | `--thinking ` | +| `model` | `--model ` | + +## Permissions + +Pi headless print mode auto-runs tools; it exposes only project-file trust +(`--approve` / `--no-approve`), not a tool-approval mode. Coder Eval always +passes `--no-approve` so the run never blocks. **`permission_mode` is therefore +not enforced** — the sandbox driver is the isolation boundary (the same posture +as Codex and Antigravity). `start()` logs a warning naming `permission_mode` +(and any other unenforced field it saw) so a task never silently believes it was +constrained. + +## Multi-turn and simulation + +A standard task reaches its solution inside a **single** `communicate()` call — +Pi runs its own multi-step agent loop there (verified: a write+read task ran +three internal `turn_start` steps, all tools executed). A simulation / dialog +task calls `communicate()` once per user turn and relies on the agent remembering +prior turns, so `PiAgent` reuses a per-agent `--session-dir` + stable +`--session-id` on **every** invocation: the first call creates the session, later +calls resume it. The session tempdir lives outside the sandbox working dir and +the staged reference dir, so it never pollutes graded files or trips +reference-integrity, and it is removed in `stop()`. `pi_session_id` is recorded +per task under `environment_info`. + +## Telemetry + +Mapping from the CLI's event vocabulary onto `TurnRecord`: + +| Pi event | Becomes | +|---|---| +| `turn_start` | `TurnStartEvent` (one inner turn; the unit `max_turns` counts) | +| `message_update` (`text_delta`) | `TextChunkEvent` + `agent_output` | +| `tool_execution_start` | `ToolStartEvent` | +| `tool_execution_end` | `ToolEndEvent` | +| `turn_end` | `TurnEndEvent` + per-turn tokens/cost, one `AssistantMessage` | +| `agent_settled` (or stdout EOF) | the single terminal `AgentEndEvent` | + +Token buckets come from `turn_end.message.usage`, read **once per turn** and +**summed** across turns (per-generation, not cumulative). Pi's `input` is already +the fresh input slice, so it maps straight to `uncached_input_tokens`; `cacheRead` +and `cacheWrite` map to the cache buckets. `reasoning` tokens fold into +`output_tokens` (they bill at the output rate) while remaining visible as +`reasoning_tokens` per message. This keeps the reconciliation invariant exact: +summing the four buckets across `TurnRecord.messages` equals `token_usage`. + +Real per-call cost rides on `turn_end.message.usage.cost.total` and lands on +`token_usage.total_cost_usd`, so runs are costed from the provider's own +accounting rather than the static rate card. No `pricing.py` entry is needed for +a Pi model; the rate card (`calculate_cost` over the captured buckets) is only a +fallback for a stream that omits cost entirely. A genuinely free model still +resolves to $0. + +Tool names and argument keys are normalized to the canonical (Claude) vocabulary +on capture — `write` → `Write`, `read` → `Read`, `bash` → `Bash`, and a +`Read`/`Write`/`Edit` call's `path` argument → `file_path` — so one +`command_executed` criterion scores identically whether the run used Claude, +Codex, OpenCode or Pi. An unmapped tool keeps its own name. + +### Auto-retry + +Pi retries a transient/provider error **internally**: it emits another +`agent_start` / `turn_*` / `agent_end` cycle in the same invocation, marking the +first `agent_end` with `willRetry: true`. `PiAgent` treats `agent_end` as +non-terminal and finalizes exactly once at `agent_settled` (or stdout EOF), so a +retried invocation still produces a single `AgentEndEvent` with the cycles' usage +merged. The internal retry is bounded by `turn_timeout` / `task_timeout`. + +### Drift is crashed, not scored + +A turn whose CLI exits cleanly but which captured **no recognized events** (an +upgrade renamed the vocabulary) is failed rather than reported as a clean empty +success — the error names the unrecognized event types it saw. Intentional cuts +(`should_stop`, `max_turns`) are exempt. + +> **Zero-usage turn.** A provider that reports no usage yields an all-zero +> `token_usage`. Pi does **not** hard-fail such a turn (its multi-provider surface +> makes a blanket fail brittle); the turn is scored and a warning is logged. If +> you need strict token accounting, prefer a provider whose stream reports usage. + +## Running in Docker + +**Not supported yet — use the default `tempdir` driver.** Like OpenCode, Pi is a +Node CLI that is not baked into `docker/Dockerfile`, and no Pi credentials are in +the docker driver's `env_passthrough` allowlist. Run Pi tasks under `tempdir` +(the default) on a host that has the CLI and its provider credentials. + +## Known limitations + +- **`permission_mode` is not enforced.** Pi headless print mode auto-runs tools; + the sandbox driver is the isolation boundary (same as Codex/Antigravity). +- **`plugins` / skills are not injected.** Pi has a native `--skill` flag, but v1 + does not wire `agent.plugins → --skill` (two things are still unverified: whether + Pi reads a Claude-style `/skills//SKILL.md` layout, and the + `skill_triggered` engagement-detection branch Pi would need). Declared plugins + are warned about and ignored. **Consequence: Pi cannot run activation suites in + v1.** Follow-up: map each resolved skill dir to a `--skill ` arg and add a + Pi branch to `skill_triggered`. +- **`system_prompt_file` is not read.** Use `system_prompt` (inline) instead — it + is enforced via `--append-system-prompt`. `system_prompt_file` is warned about + at `start()` (matching Codex/Antigravity, which also do not read the file form). +- **`max_turns` counts Pi's native agent-loop turns.** One `turn_start` = one + agent-loop step; `max_turns: N` allows N complete turns, then the run finalizes + cleanly as `max_turns_exhausted`. See + [Run-Limit Parity](HARNESS_PARITY.md) before holding `max_turns` constant across + harnesses. +- **The `docker` sandbox driver is unsupported** — see + [Running in Docker](#running-in-docker). +- **No sub-agent attribution.** Pi's CLI stream does not expose nested agent + generations, so per-sub-agent token grouping (available for Claude and Codex) is + not derivable. +- **Cooperative stop is at event granularity.** `should_stop` is polled between + events and honored by terminating the CLI, so `stop_early` works, but the cut + lands on an event boundary rather than mid-tool. Pi streams incrementally, so + this genuinely cuts spend mid-run. + +## References + +- [Pi](https://pi.dev/) +- [Extending Coder Eval](../EXTENDING.md) — the agent plugin SPI +- [Task Definition Guide](../TASK_DEFINITION_GUIDE.md) diff --git a/docs/index.md b/docs/index.md index 5f22ce7e..68aed62b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ description: >- Coder Eval is an open-source framework to evaluate, benchmark, and A/B-test AI coding agents and Claude Code skills in a sandbox — declarative YAML tasks, weighted scoring, cost/token telemetry, and CI gates for Claude Code, Codex, - Gemini, and OpenCode. + Gemini, OpenCode, and Pi. --- # Evaluate AI coding agents & Claude Code skills — Coder Eval @@ -14,7 +14,7 @@ description: >- builders — with sandboxing, reproducibility, and data-driven analysis. It is not an "agentic coding" benchmark: it measures how effective *your* CLI and skills are when used by coding agents such as **Claude Code**, **Codex**, **Google -Antigravity (Gemini)**, and **OpenCode**. +Antigravity (Gemini)**, **OpenCode**, and **Pi**. If you have ever asked *"how do I test whether my Claude Code skill actually triggers?"*, *"how do I benchmark Claude Code vs. Codex on my own tasks?"*, or @@ -30,14 +30,14 @@ triggers?"*, *"how do I benchmark Claude Code vs. Codex on my own tasks?"*, or - **Sandboxed execution** in isolated environments with resource limits - **Weighted, continuous scoring** (0.0–1.0) with fractional credit and thresholds - **Many criterion types** — from file checks to code similarity and LLM-graded rubrics -- **Agent abstraction** — Claude Code, Codex, Antigravity (Gemini), and OpenCode today, extensible via a plugin SPI +- **Agent abstraction** — Claude Code, Codex, Antigravity (Gemini), OpenCode, and Pi today, extensible via a plugin SPI - **Experiment layer** — A/B agent configs (models, tools, prompts) side-by-side - **Full telemetry** — every tool call, token counts, and cost, with real-time streaming ## Use cases - **Benchmark coding agents** — score an agent across a suite of tasks with weighted pass/fail thresholds -- **Compare models & configs** — A/B-test Claude Code vs. Codex vs. Gemini vs. OpenCode, model vs. model, tool-on vs. tool-off, prompt vs. prompt +- **Compare models & configs** — A/B-test Claude Code vs. Codex vs. Gemini vs. OpenCode vs. Pi, model vs. model, tool-on vs. tool-off, prompt vs. prompt - **Test whether a Claude Code skill triggers** — verify an agent actually engages a target skill (`skill_triggered`) and score skill-driven suites (SkillsBench-style) - **Keep skills fresh in CI** — re-validate skills on every change or on a schedule; catch silent regressions when models, prompts, or the skills themselves drift @@ -82,6 +82,7 @@ New here? Start with **[Tutorial 01 — Your First Evaluation](tutorials/01-firs | [Codex](agents/CODEX.md) | Running the OpenAI Codex agent | | [Antigravity (Gemini)](agents/ANTIGRAVITY.md) | Running the Google Antigravity / Gemini agent | | [OpenCode](agents/OPENCODE.md) | Running the OpenCode agent on open-weight models | +| [Pi](agents/PI.md) | Running the Pi agent on open-weight models | | [Run-Limit Parity](agents/HARNESS_PARITY.md) | What each run_limits field means on every harness | | [A/B Experiments](AB_EXPERIMENTS.md) | Compare models / tools / prompts across the same tasks | | [Bring Your Own Dataset](DATASETS.md) | Fan a single task out over a dataset | diff --git a/docs/llms.txt b/docs/llms.txt index 9dc3865f..333b1b3f 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -4,7 +4,7 @@ > benchmarking, and A/B-testing AI coding agents and their Claude Code skills in a > sandbox. It uses declarative YAML tasks with weighted, continuous scoring > (0.0–1.0), runs real agents (Claude Code, Codex, Google Antigravity/Gemini, -> OpenCode) with +> OpenCode, Pi) with > full tool use, captures per-tool cost/token telemetry, and provides CI-ready > pass/fail gates. It is not a fixed benchmark or leaderboard — it scores your own > tasks, and can verify whether a Claude Code skill actually triggers. @@ -30,6 +30,7 @@ and A/B plumbing. - [Codex](https://coder-eval.com/docs/agents/codex): Running the OpenAI Codex agent - [Antigravity (Gemini)](https://coder-eval.com/docs/agents/antigravity): Running the Google Antigravity / Gemini agent - [OpenCode](https://coder-eval.com/docs/agents/opencode): Running the OpenCode agent on open-weight models +- [Pi](https://coder-eval.com/docs/agents/pi): Running the Pi agent on open-weight models - [Run-Limit Parity](https://coder-eval.com/docs/agents/harness-parity): What each run_limits field means on every harness - [A/B Experiments](https://coder-eval.com/docs/ab-experiments): Compare models / tools / prompts across the same tasks - [Bring Your Own Dataset](https://coder-eval.com/docs/datasets): Fan a single task out over a dataset diff --git a/experiments/default.yaml b/experiments/default.yaml index ebf627b5..b9469a4b 100644 --- a/experiments/default.yaml +++ b/experiments/default.yaml @@ -39,7 +39,8 @@ defaults: # count_cached_input: false agent: - # Agent type (currently only "claude-code" is supported) + # Agent type. Built-in kinds: claude-code, codex, antigravity, opencode, pi, + # none (plus any plugin-registered kind). claude-code is the run-wide default. type: claude-code # Permission mode for agent actions: diff --git a/mkdocs.yml b/mkdocs.yml index db2d2728..05773bff 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -84,6 +84,7 @@ extra: agents/CODEX.md: "Running the OpenAI Codex agent" agents/ANTIGRAVITY.md: "Running the Google Antigravity / Gemini agent" agents/OPENCODE.md: "Running the OpenCode agent on open-weight models" + agents/PI.md: "Running the Pi agent on open-weight models" agents/HARNESS_PARITY.md: "What each run_limits field means on every harness" AB_EXPERIMENTS.md: "Compare models / tools / prompts across the same tasks" DATASETS.md: "Fan a single task out over a dataset" @@ -114,6 +115,7 @@ nav: - Codex: agents/CODEX.md - Antigravity (Gemini): agents/ANTIGRAVITY.md - OpenCode: agents/OPENCODE.md + - Pi: agents/PI.md - Run-Limit Parity: agents/HARNESS_PARITY.md - Advanced: - A/B Experiments: AB_EXPERIMENTS.md diff --git a/uv.lock b/uv.lock index 7ed3a062..c385e28f 100644 --- a/uv.lock +++ b/uv.lock @@ -564,7 +564,7 @@ requires-dist = [ { name = "typer", specifier = ">=0.24.1" }, { name = "uipath", marker = "extra == 'uipath'", specifier = ">=2.10.31" }, ] -provides-extras = ["dev", "uipath", "litellm", "codex", "antigravity", "opencode"] +provides-extras = ["dev", "uipath", "litellm", "codex", "antigravity", "opencode", "pi"] [[package]] name = "colorama" From 95ce2183eb1ac2c629f763453b656adf1ea59dba Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 12:20:09 +0100 Subject: [PATCH 05/24] fix: code review fixes for pi-harness - Harden PiAgent.start() to drop any prior session tempdir before creating a new one, so re-starting the same agent instance cannot leak (defensive; agents are single-use in-tree). Add a regression test. - Correct the PI.md Overview snippet to use --model (matching the code) instead of -m. - Defer a CExxx harness candidate: dead _*TurnState accumulator detection. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/harness-candidates.md | 9 +++++++++ docs/agents/PI.md | 2 +- src/coder_eval/agents/pi_agent.py | 3 +++ tests/test_pi_agent.py | 14 ++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index e067e9ad..c734c88a 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -455,3 +455,12 @@ with the two `action.yml` items above — one considered change to the action's `final_status`, which does not exist in `run.json` and would have made a new assertion dead on arrival. Guard: assert the key set that non-Python consumers depend on, mirroring how CE030 pins doc/schema parity. + +- [ ] A `_*TurnState` (agent turn-state) attribute that is written but never read + outside its own assignment — CE037-class dead accumulator. Surfaced during the + Pi harness port: `_PiTurnState.turns_finished` was copied from OpenCode's + `steps_finished` (which drives that agent's `finished_without_tokens` guard) but + Pi deliberately dropped that guard, leaving the counter dead. Fixed by hand this + run. Guard would need cross-method dataflow over each `Agent`-subclass turn-state + class (write sites vs read sites), which the AST-only CExxx runner can't express + in ~30 min — deferred. Caught in: Pi harness Phase 2 quality review. diff --git a/docs/agents/PI.md b/docs/agents/PI.md index b8974849..9a4f0ec8 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -15,7 +15,7 @@ drives it in **JSON print mode**: ```bash pi -p --mode json --no-context-files --no-approve \ - --session-dir --session-id -m "" + --session-dir --session-id --model -- "" ``` `--mode json` streams **newline-delimited JSON events** on stdout, one event per diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 3c7931bf..6df662f4 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -665,6 +665,9 @@ async def start( # A stable pre-assigned id (create-if-missing on turn 1, resume after). # The tempdir lives OUTSIDE the sandbox working dir and staged reference # dir, so it never pollutes graded files or trips reference-integrity. + # Drop any session dir from a prior start() first so re-starting the same + # agent instance cannot leak a tempdir. + self._cleanup_session_dir() self._session_id = f"coder-eval-{self.task_id}-{uuid4().hex[:8]}" self._session_dir = tempfile.mkdtemp(prefix="pi-session-") self._state = AgentState.WORKING diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 265b5a79..55a163f2 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -373,6 +373,20 @@ async def test_start_creates_and_stop_removes_the_session_dir(self, patch_exec, assert not Path(session_dir).exists() assert agent._session_dir is None + async def test_restart_does_not_leak_the_prior_session_dir(self, patch_exec, tmp_path): + """Re-starting the same agent instance must remove the previous tempdir.""" + patch_exec(_FakeProcess(HAPPY_STREAM)) + agent = _agent() + await agent.start(str(tmp_path)) + first = agent._session_dir + assert first is not None and Path(first).is_dir() + + await agent.start(str(tmp_path)) + second = agent._session_dir + assert second is not None and second != first + assert not Path(first).exists() # the first dir was cleaned up, not leaked + await agent.stop() + class TestEnvironmentInfo: def test_carries_semantics_and_pi_fields(self): From c79f5cb425464c14d354d3f2f1082e1164b986b4 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 12:26:55 +0100 Subject: [PATCH 06/24] fix(pi): do not forward allowed_tools/disallowed_tools to Pi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared config default (experiments/default.yaml) sets Claude-namespaced tool names (Bash/Read/Write/Edit/Glob/Grep/Skill). Pi's built-in tools are lowercase and differently named (bash/read/write/edit/grep/find/ls), so forwarding the PascalCase names to `--tools` allowlisted nonexistent tools and left the agent with ZERO tools ("I don't have tool access") — the end-to-end smoke created no files and scored 0/3. Treat allowed_tools/disallowed_tools as unenforced (add to _UNSUPPORTED_CONFIG_FIELDS, warn at start()), matching OpenCode/Codex/Antigravity, so Pi runs with its full native toolset. system_prompt stays enforced (--append-system-prompt). Updates the two encoding tests and the PI.md / HARNESS_PARITY.md enforcement claims. E2E smoke now passes 3/3 (score 1.000). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/agents/HARNESS_PARITY.md | 19 +++++++++++------- docs/agents/PI.md | 27 +++++++++++++++++++------ src/coder_eval/agents/pi_agent.py | 33 ++++++++++++++++++++++--------- tests/test_pi_agent.py | 24 ++++++++++++---------- 4 files changed, 71 insertions(+), 32 deletions(-) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 1ad5c6a7..7d7754bd 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -180,13 +180,18 @@ activation suites yet — even though the CLI has a native `--skill ` a follow-up could wire `agent.plugins → --skill` onto. See [Pi § Known limitations](PI.md#known-limitations). -## Pi enforces the tool/prompt knobs OpenCode drops - -Pi is the outlier in the enforcement direction — a capability *win*, not a gap: - -- **`allowed_tools` / `disallowed_tools` are ENFORCED** (`--tools` / `--exclude-tools`), - and **`system_prompt` is ENFORCED** (`--append-system-prompt`, semantics `append`). - OpenCode drops all three; Codex forwards `disallowed_tools` without SDK enforcement. +## Pi enforces `system_prompt` but not the tool allowlists + +- **`system_prompt` is ENFORCED** (`--append-system-prompt`, semantics `append`) — a + small win over OpenCode, which drops it. +- **`allowed_tools` / `disallowed_tools` are NOT enforced.** Pi's built-in tools are + lowercase (`bash`/`read`/`write`/`edit`/`grep`/`find`/`ls`), but the shared config + default (`experiments/default.yaml`) sets Claude-namespaced names + (`Bash`/`Read`/`Write`/…). Forwarding those to `--tools` would allowlist tools that + do not exist in Pi and strip the agent of ALL tools — so, like OpenCode (drops them), + Codex (forwards `disallowed_tools` without SDK enforcement), and Antigravity (does not + read them), Pi ignores them and runs with its full native toolset. A task that needs a + restricted Pi toolset would have to name Pi's lowercase tools — a documented follow-up. - **`permission_mode` is NOT enforced** — Pi headless print mode auto-runs tools and exposes only project-file trust (`--approve` / `--no-approve`), no tool-approval mode; the sandbox driver is the isolation boundary (same as Codex/Antigravity). diff --git a/docs/agents/PI.md b/docs/agents/PI.md index 9a4f0ec8..a727998f 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -25,8 +25,14 @@ and lets `EventCollector` build the `TurnRecord`, exactly like every other harness. Because Pi is model-agnostic, this is a cheap way to evaluate a broad set of -open-weight models (Kimi, DeepSeek, GLM, …) through a single agent — and unlike -OpenCode, Pi **enforces** `allowed_tools` / `disallowed_tools` / `system_prompt`. +open-weight models (Kimi, DeepSeek, GLM, …) through a single agent. Pi **enforces +`system_prompt`** (via `--append-system-prompt`) — a small win over OpenCode. It +does **not** enforce `allowed_tools` / `disallowed_tools`: the shared config +default uses Claude-namespaced tool names (`Bash`/`Read`/`Write`/…) that do not +match Pi's lowercase built-ins (`bash`/`read`/`write`/…), so forwarding them would +leave the agent with no tools at all. Like OpenCode / Codex / Antigravity, Pi +therefore runs with its full native toolset and warns that these fields are +unenforced. ## Setup @@ -108,17 +114,26 @@ superset of the Antigravity `thinking_level`), defaulting to `medium`. ### Enforced config fields -Unlike OpenCode, Pi **enforces** the standard tool/prompt knobs (a capability -win): +Pi forwards these config knobs to real CLI flags: | Field | Pi flag | |---|---| -| `allowed_tools` | `--tools ` | -| `disallowed_tools` | `--exclude-tools ` | | `system_prompt` | `--append-system-prompt ` (appended, semantics `append`) | | `thinking_level` | `--thinking ` | | `model` | `--model ` | +### Unenforced config fields + +`allowed_tools` / `disallowed_tools` are **not** forwarded. The shared config +default (`experiments/default.yaml`) sets Claude-namespaced tool names +(`Bash`/`Read`/`Write`/`Edit`/`Glob`/`Grep`/`Skill`), but Pi's built-in tools are +lowercase and differently named (`bash`/`read`/`write`/`edit`/`grep`/`find`/`ls`). +Passing the PascalCase names to `--tools` would allowlist tools that do not exist +in Pi, leaving the agent with **zero** tools. So — like OpenCode, Codex, and +Antigravity — Pi ignores these fields, runs with its full native toolset, and +warns at `start()` that they are unenforced. `permission_mode`, `plugins`, and +`system_prompt_file` are unenforced too (see below). + ## Permissions Pi headless print mode auto-runs tools; it exposes only project-file trust diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 6df662f4..9e31acd8 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -172,12 +172,22 @@ }, } -# Config fields the Pi CLI has no equivalent knob for (v1). `experiments/default.yaml` -# sets `permission_mode` on every task, so warn once at start() rather than let a -# task believe it constrained the agent. NOTE `allowed_tools`/`disallowed_tools`/ -# `system_prompt` are SUPPORTED (mapped to --tools/--exclude-tools/ +# Config fields the Pi CLI has no equivalent knob for (v1), OR that cannot be +# safely forwarded. `experiments/default.yaml` sets `permission_mode` and +# `allowed_tools` on every task, so warn once at start() rather than let a task +# believe it constrained the agent. NOTE `system_prompt` IS supported (mapped to # --append-system-prompt) and thus deliberately NOT here. -_UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ("plugins", "permission_mode", "system_prompt_file") +_UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ( + "plugins", + "permission_mode", + "system_prompt_file", + # Pi's built-in tool names are lowercase (bash/read/write/edit/grep/find/ls) + # and do not match the Claude-namespaced default (Bash/Read/Write/...), so + # forwarding them to --tools would allowlist nonexistent tools and strip the + # agent of ALL tools. Ignored like OpenCode/Codex/Antigravity do. + "allowed_tools", + "disallowed_tools", +) # The full recognized Pi vocabulary (from `pi` 0.84.4). A clean exit that # recognized NOTHING from this set is vocabulary drift and is crashed rather than @@ -756,10 +766,15 @@ def _build_argv(self, user_input: str) -> list[str]: argv += ["--model", self.config.model] # provider-prefixed form if self.config.thinking_level: argv += ["--thinking", self.config.thinking_level] - if self.config.allowed_tools: - argv += ["--tools", ",".join(self.config.allowed_tools)] # ENFORCED (a win over OpenCode) - if self.config.disallowed_tools: - argv += ["--exclude-tools", ",".join(self.config.disallowed_tools)] + # allowed_tools / disallowed_tools are NOT forwarded. The shared config + # default (experiments/default.yaml) sets Claude-namespaced tool names + # (Bash/Read/Write/Edit/Glob/Grep/Skill), but Pi's built-in tools are + # lowercase and differently named (bash/read/write/edit/grep/find/ls). + # Passing the PascalCase names to `--tools` allowlists tools that do not + # exist in Pi, leaving the agent with ZERO tools ("I don't have tool + # access"). So, like OpenCode/Codex/Antigravity, these fields are treated + # as unenforced (see _UNSUPPORTED_CONFIG_FIELDS) and Pi runs with its full + # native toolset. Warned at start(). if self.config.system_prompt: argv += ["--append-system-prompt", self.config.system_prompt] # user_input is a distinct argv element after `--` (never shell-interpolated). diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 55a163f2..5e3bb402 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -321,15 +321,19 @@ async def test_base_flags_and_prompt(self, patch_exec, tmp_path): assert argv[-2] == "--" assert argv[-1] == "do the thing" - async def test_enforced_tool_and_prompt_flags(self, patch_exec, tmp_path): + async def test_tools_are_not_forwarded_but_system_prompt_is(self, patch_exec, tmp_path): + """allowed_tools/disallowed_tools are NOT forwarded: the shared config default + sets Claude-namespaced tool names (Bash/Read/...) that do not exist in Pi + (lowercase bash/read/...), so `--tools` would strip the agent of ALL tools. + Only `system_prompt` (a free-text string with no namespace) is forwarded.""" captured = patch_exec(_FakeProcess(HAPPY_STREAM)) await _run( _agent(allowed_tools=["Read", "Write"], disallowed_tools=["Bash"], system_prompt="be terse"), tmp_path, ) argv = captured["argv"] - assert argv[argv.index("--tools") + 1] == "Read,Write" - assert argv[argv.index("--exclude-tools") + 1] == "Bash" + assert "--tools" not in argv + assert "--exclude-tools" not in argv assert argv[argv.index("--append-system-prompt") + 1] == "be terse" async def test_user_input_is_a_post_dashdash_argv_element(self, patch_exec, tmp_path): @@ -428,19 +432,19 @@ async def test_start_warns_about_unenforced_fields(self, patch_exec, tmp_path, c assert "plugins" in caplog.text assert "NOT enforced" in caplog.text - async def test_enforced_fields_are_never_named_as_unenforced(self, patch_exec, tmp_path, caplog): - """`permission_mode` always warns (it has a truthy default and is unenforced), - but the ENFORCED fields must never appear in the unenforced-fields warning.""" + async def test_unenforced_fields_warn_but_system_prompt_does_not(self, patch_exec, tmp_path, caplog): + """allowed_tools/disallowed_tools are unenforced (Claude-namespaced default cannot + map to Pi's lowercase toolset) and MUST warn when set. `system_prompt` IS enforced + (--append-system-prompt) and must never appear in the unenforced-fields warning.""" patch_exec(_FakeProcess(HAPPY_STREAM)) with caplog.at_level("WARNING"): await _agent(allowed_tools=["Read"], disallowed_tools=["Bash"], system_prompt="be terse").start( str(tmp_path) ) - # The unenforced-fields warning must not name any ENFORCED field. warning = "".join(r.message for r in caplog.records if "NOT enforced" in r.message) - assert "allowed_tools" not in warning - assert "disallowed_tools" not in warning - assert "system_prompt" not in warning # neither system_prompt nor system_prompt_file set here + assert "allowed_tools" in warning + assert "disallowed_tools" in warning + assert "system_prompt" not in warning # the enforced field is never named class TestAutoRetry: From b32721ac885ed485f9ea7afcd74eb329b346d47e Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 12:36:44 +0100 Subject: [PATCH 07/24] docs(pi): fix tool-enforcement claims + cost docstring after review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code review (multi-model) found the tool-forwarding removal (c79f5cb4) left three doc surfaces claiming allowed_tools/disallowed_tools are enforced: - H1: PiAgentConfig docstring (models/agent_config.py) - H2: CLAUDE.md harness-parity bullet ("ENFORCES all three") - M1: pi_agent.py module-docstring command synopsis (--tools/--exclude-tools) All corrected: only system_prompt is enforced; the tool allowlists are ignored (Claude-namespaced default can't map to Pi's lowercase toolset), matching the shipped code + PI.md + HARNESS_PARITY.md. Also: - M2: fix _resolve_cost docstring to match the code (rate-card fallback on a streamed $0 for a priced model is a deliberate conservative choice — never understate max_usd budget gates); demote the expected warning to debug. - L1: correct the session-dir comment (removed in stop(), NOT kill()). - L2: tighten test_should_stop_ends_turn_cleanly to pin the cut at turn 0. No harness correctness bugs were found by the review. Tests 75/75, make lint 389/389, e2e smoke 3/3 (score 1.000). Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 2 +- src/coder_eval/agents/pi_agent.py | 22 +++++++++++++++------- src/coder_eval/models/agent_config.py | 17 +++++++++++------ tests/test_pi_agent.py | 5 ++++- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index bf2286f2..9cd9f92f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -144,7 +144,7 @@ action.yml # Published composite GitHub Action (coder-ev - **Sub-agent token accounting**: There is NO separate per-sub-agent field. Every sub-agent generation is captured as a `parent_tool_use_id`-tagged `AssistantMessage` in the turn transcript, so per-sub-agent usage is derived by grouping those messages on that id (the evalboard's `aggregateSubAgentUsage` does exactly this). Claude bubbles its sub-agent's intermediate generations into the parent stream natively, and the **terminal** generation (delivered as the Agent tool result, never streamed) is synthesized into one via `_synthesize_subagent_terminal_message` from `tool_use_result.usage`. Codex reconstructs all child generations from the child rollout (`_recover_subagent_tool_calls`). The turn total already includes sub-agent cost — Claude via the SDK's cumulative `model_usage`; Codex via `_fold_subagent_tokens`, which folds the child messages (their real per-generation tokens) into the parent total. `CommandTelemetry.result_summary` is stored **untruncated** (no 200-char cap) so sub-agent returns are preserved whole. Set `CODER_EVAL_RAW_SDK_LOG=1` to dump every raw SDK event to the task log for inspection. - **Reconciliation message (stream self-reconciles to the turn total)**: The per-message stream consistently under-reports the authoritative turn total — a fixed prompt slice (~512 input tokens on Claude) is billed on no SDK-emitted message, and sub-agent input/cache only partially bubbles up. So `EventCollector.build_turn_record` appends one synthetic `ReconciliationMessage` (`role="reconciliation"`, in the `TranscriptMessage` union) per turn, carrying the per-bucket residual = `token_usage` − Σ(assistant message buckets). The invariant: **summing the four token buckets across `TurnRecord.messages` (assistant + reconciliation) equals `token_usage` exactly**, for both Claude and Codex (Codex's stream is already complete after `_recover_subagent_tool_calls`, so its residual is usually 0 and no entry is emitted). This is what lets the evalboard SUM the message stream as the source of truth instead of reading a separate aggregate ("agent tokens"): `selectTokenTotals` returns the stream sum whenever a reconciliation entry is present, and the timeline renders it as its own row. It is agent-agnostic (booked at the single `EventCollector` seam), carries no cost (cost stays on `token_usage`), and is excluded from generation/turn counts and the cost simulator. The LiteLLM open-weight actual-cost join (`litellm_cost.apply_actual_cost`) deliberately writes cost at the TURN level only (`token_usage.total_cost_usd` = the real OpenRouter bill) plus the per-call `TurnRecord.provider_call_costs` audit record; it does NOT touch the message token buckets, so `EventCollector` stays the single writer and this invariant holds on every backend. The Python `token_usage`/`total_token_usage` aggregate is unchanged and still authoritative for budget/judges/reports. - **Reference solutions are directory-only, and shielded (partially) from the agent**: `task.reference` is a single required `directory:` (relative to the task YAML) — the inline `code:` / single-file `file:` forms are gone, because a directory is the only shape that can be permission-gated as a unit; a `model_validator(mode="before")` gives the removed forms a migration error. The orchestrator stages a **per-run private copy** (`orchestration/evaluation.py::stage_reference_dir`, symlinks stripped) into a tempdir, removed in `_cleanup` via `path_utils.rmtree_restrictive` (keyed on `_reference_staging_root`, recorded BEFORE the copy so a failed copy still cleans up; `rmtree(ignore_errors=True)` silently declines on a tree left at 000) and deliberately never preserved into `run_dir/artifacts`. That copy is held at mode `000` for the whole of every `agent.communicate` call via **`Sandbox.set_permissions`**, the driver-aware wrapper over `fs_permissions.py::set_permissions`. Windows **stack**: exiting restores the *enclosing* window's mode, only the outermost exit restores the pre-window mode — that is what makes a mid-turn re-grant (`mode=READ_ONLY_MODE`) expressible, and it covers two windows at the same mode so no refcount is needed. The window is enforced **only inside a docker container** (`Sandbox.enforces_permission_windows`) and is a no-op on the host, where the agent shares our uid. **That gate keys on the `CODER_EVAL_IN_CONTAINER` env var, NOT `sandbox.driver`** — `run_task_internal_command` rewrites `driver: docker` → `tempdir` before building the in-container Orchestrator, so a driver-based gate would silently disable the anti-cheat on exactly the path that needs it (regression-guarded by `TestSandboxDriverGate`); `resolve_reference_dir` gates its `/work/references` branch on the same var for the same reason. The task directory is **not** shielded (`:ro` mount → EROFS, and the same YAML is readable at `/work/input`). Criteria address reference files with the `$REFERENCE_DIR` token (same resolver as `$TASK_DIR`) and the `REFERENCE_DIR` env var for `run_command`; `reference_comparison` names one file via `reference_file`. Docker mounts a throwaway **read-write** copy at `/work/references` (a `:ro` mount cannot be chmod'd — EROFS), masks the in-task-dir original with an empty tmpfs, and drops `DAC_OVERRIDE`/`DAC_READ_SEARCH`. `FOWNER`/`CHOWN` are deliberately **NOT** dropped: the in-container orchestrator that applies the window is the same root process with the same caps, so dropping `FOWNER` breaks *the harness's own* chmod wherever the bind mount preserves a non-root owner (native Linux — verified: `chmod: Operation not permitted`), i.e. exactly where the drop would otherwise bite. A window that cannot be applied is now a hard error, not a warning: `Sandbox.set_permissions` passes `strict=True` whenever it enforces, so an unprotected run fails instead of producing a normal-looking score. **KNOWN GAP — this is defense-in-depth, not a boundary**: (a) `chmod(2)` is gated on owner-or-`CAP_FOWNER` and the container runs as root owning the copy, so a deliberate `chmod 755 /work/references` restores access; (b) the window spans `agent.communicate` only, and nothing reaps agent child processes at turn end, so a backgrounded read loop succeeds once the window closes. The **write** half of (b) is closed — `path_utils.digest_tree` hashes the tree at staging and `Orchestrator._verify_reference_integrity` re-checks before grading, raising `ReferenceTamperedError` (→ `FinalStatus.ERROR`) on a mismatch so an agent cannot overwrite the reference to drive `reference_comparison` to 1.0. Passive reads are blocked; an adversarial agent is not. Full containment requires running the agent as a non-root uid AND holding the window for the agent's whole lifetime — follow-up. `tasks/anti_cheat_reference` probes the passive-read half. -- **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. **Pi** is the same shape — the CLI (`pi -p --mode json`) streams a real multi-step loop per `communicate()` (`turn_start`/`turn_end`), so `max_turns: N` counts native `turn_start` steps; Pi retries transient/provider errors INTERNALLY (`agent_end.willRetry`), and the reducer finalizes once at `agent_settled`/EOF (not the first `agent_end`), folding the retry cycles into one turn. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex, Antigravity, and Pi (all run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced) — Pi, in contrast, ENFORCES all three (`--tools`/`--exclude-tools`/`--append-system-prompt`, a capability win over OpenCode) but does NOT read `plugins` (warned+ignored; no activation suites in v1, though it has a native `--skill` flag) nor `system_prompt_file`, and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. +- **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. **Pi** is the same shape — the CLI (`pi -p --mode json`) streams a real multi-step loop per `communicate()` (`turn_start`/`turn_end`), so `max_turns: N` counts native `turn_start` steps; Pi retries transient/provider errors INTERNALLY (`agent_end.willRetry`), and the reducer finalizes once at `agent_settled`/EOF (not the first `agent_end`), folding the retry cycles into one turn. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex, Antigravity, and Pi (all run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), `allowed_tools`/`disallowed_tools` on Pi (its built-in tool names are lowercase — `bash`/`read`/… — and cannot map to the Claude-namespaced config default, so forwarding them would strip the agent of ALL tools; warned+ignored like the three agents above) — Pi DOES enforce `system_prompt` (`--append-system-prompt`, a small win over OpenCode) but does NOT read `plugins` (warned+ignored; no activation suites in v1, though it has a native `--skill` flag) nor `system_prompt_file`, and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 9e31acd8..e6ed3b4f 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -4,8 +4,7 @@ pi -p --mode json --no-context-files --no-approve \ --session-dir --session-id [--model provider/id] \ - [--thinking L] [--tools ...] [--exclude-tools ...] \ - [--append-system-prompt S] -- + [--thinking L] [--append-system-prompt S] -- which streams **newline-delimited JSON events** on stdout. Each line is one event; this module reduces that stream into the standardized coder_eval event @@ -524,15 +523,21 @@ def _rate_card_cost(self) -> float | None: def _resolve_cost(self) -> float | None: """Decide the turn's cost: the stream's own accounting vs the rate card. - Pi reports a real per-call ``cost.total`` (spike-verified), which always - wins. The rate card fills only the gap where the stream reported no cost - at all; a genuinely free model resolves to the stream's 0. + Pi reports a real per-call ``cost.total`` (spike-verified), which wins for + any nonzero total. Two conservative fallbacks to the rate card: + - the stream reported no cost field at all (``saw_cost`` False), or + - it reported a cost field but the turn total came out exactly ``$0`` on a + model the rate card DOES price. A true $0 (free/promo response) and a + provider whose cost field is present-but-always-zero are indistinguishable + from the stream alone, so we prefer the rate card: understating cost would + silently defeat ``max_usd`` budget gates, which is the worse failure. A + genuinely free model (no rate-card entry) still resolves to the stream's 0. """ rate = self._rate_card_cost() if not self.saw_cost: return rate if self.cost_usd == 0.0 and rate: - logger.warning( + logger.debug( "pi: the stream reported $0 for a turn the rate card prices at $%.6f; using the rate card " + "so the run total is not understated.", rate, @@ -639,7 +644,10 @@ def __init__( self._env_path_prepend: list[str] = [] self._plugin_tools_dir: str | None = None # Per-agent session, reused across communicate() calls for multi-turn / - # simulation continuity (assigned in start(), removed in stop()/kill()). + # simulation continuity (assigned in start(), removed in stop() — NOT in + # kill(), which the orchestrator's mid-turn backstop calls; dropping the + # dir there would break resume across a retried turn. _cleanup always + # calls stop() after any kill(), so the tempdir is still reclaimed). self._session_id: str | None = None self._session_dir: str | None = None self._process: asyncio.subprocess.Process | None = None diff --git a/src/coder_eval/models/agent_config.py b/src/coder_eval/models/agent_config.py index e666a406..ca089517 100644 --- a/src/coder_eval/models/agent_config.py +++ b/src/coder_eval/models/agent_config.py @@ -408,12 +408,17 @@ class PiAgentConfig(BaseAgentConfig): Enforced vs unenforced fields ----------------------------- - ``allowed_tools`` → ``--tools``, ``disallowed_tools`` → ``--exclude-tools`` and - ``system_prompt`` → ``--append-system-prompt`` ARE enforced (a capability win - over OpenCode). ``permission_mode`` is NOT enforced (Pi headless print mode - auto-runs tools; the sandbox driver is the isolation boundary), ``plugins`` are - NOT injected (no activation suites in v1), and ``system_prompt_file`` is NOT - read — all three are warned about at ``start()``. See ``docs/agents/PI.md``. + ``system_prompt`` → ``--append-system-prompt`` IS enforced (a small capability + win over OpenCode). ``allowed_tools`` / ``disallowed_tools`` are NOT forwarded: + the shared config default sets Claude-namespaced tool names + (``Bash``/``Read``/…) that do not exist in Pi's lowercase toolset + (``bash``/``read``/…), so forwarding them to ``--tools`` would allowlist + nonexistent tools and strip the agent of ALL tools — like OpenCode/Codex/ + Antigravity, Pi ignores them and runs with its full native toolset. + ``permission_mode`` is NOT enforced (Pi headless print mode auto-runs tools; + the sandbox driver is the isolation boundary), ``plugins`` are NOT injected + (no activation suites in v1), and ``system_prompt_file`` is NOT read — all are + warned about at ``start()``. See ``docs/agents/PI.md``. """ type: Literal[AgentKind.PI] # type: ignore[assignment] diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 5e3bb402..fc58df09 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -488,7 +488,10 @@ async def test_should_stop_ends_turn_cleanly(self, patch_exec, tmp_path): assert record.crashed is False assert proc.terminated is True - assert record.assistant_turn_count < 3 + # should_stop() is True from the first check, which lands after the first + # streamed line (the `session` header) and before any turn completes — so + # the cut is at turn 0, not merely "fewer than the full 3". + assert record.assistant_turn_count == 0 async def test_partial_record_is_returned_not_raised(self, patch_exec, tmp_path): proc = _RunningProcess(HAPPY_STREAM) From 351941ee78443ba8b339676c4f42f291ce17ae60 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 14:19:16 +0100 Subject: [PATCH 08/24] feat(evalboard): show the Pi logo + "Pi" in the harness view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a HARNESS_LOGO entry for agent.type "pi" (AgentKind.PI): the Pi mark (public/harness/pi.png, 128x128, rasterized from pi.dev's favicon.svg) plus short label "Pi" and tooltip "Pi · pi.dev". The badge + selector are already data-driven, so a Pi run now renders the logo and name instead of the raw id. Adds harness-badge test cases for the "pi" short label and alt text. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../_components/__tests__/harness-badge.test.tsx | 11 +++++++++++ evalboard/app/_components/harness-badge.tsx | 1 + evalboard/public/harness/pi.png | Bin 0 -> 1436 bytes 3 files changed, 12 insertions(+) create mode 100644 evalboard/public/harness/pi.png diff --git a/evalboard/app/_components/__tests__/harness-badge.test.tsx b/evalboard/app/_components/__tests__/harness-badge.test.tsx index 32b384c8..f4953575 100644 --- a/evalboard/app/_components/__tests__/harness-badge.test.tsx +++ b/evalboard/app/_components/__tests__/harness-badge.test.tsx @@ -18,6 +18,12 @@ describe("harnessShortLabel", () => { expect(harnessShortLabel("delegate-sdk")).toBe("Delegate"); }); + test("the Pi harness reads as Pi", () => { + // `agent.type` is "pi" (AgentKind.PI) in run.json; the badge shows the + // Pi mark and the short label "Pi". + expect(harnessShortLabel("pi")).toBe("Pi"); + }); + test("an unknown harness falls back to its id rather than a wrong name", () => { expect(harnessShortLabel("some-new-agent")).toBe("some-new-agent"); }); @@ -29,6 +35,11 @@ describe("HarnessBadge", () => { expect(screen.getByAltText("Delegate · UiPath")).toBeInTheDocument(); }); + test("renders the Pi vendor mark with pi.dev in the alt text", () => { + render(); + expect(screen.getByAltText("Pi · pi.dev")).toBeInTheDocument(); + }); + test("renders the id as text when there is no logo for it", () => { // Better a raw id than another vendor's mark on someone else's run. render(); diff --git a/evalboard/app/_components/harness-badge.tsx b/evalboard/app/_components/harness-badge.tsx index 02a7bbaa..4c788e32 100644 --- a/evalboard/app/_components/harness-badge.tsx +++ b/evalboard/app/_components/harness-badge.tsx @@ -17,6 +17,7 @@ const HARNESS_LOGO: Record|{P8Iwh{5d9&9h?$>TgJDGQRE={}ayXzSr6IblvbvKXli3fP;GaMF`c*rkl zUHr=~Y{8-$nHPJCjy<=re<_=`>&@@;J;xn4KCAbbw8TuRHnLPN*CIXp*p9Y^`3KC_ zmgZ)2pH2E0xcc(TEoYf+H5$B93uCA4dpIX@qW3Au?6ZwWHr#wYH}~|@y%Cc7I(EL7 z<9iKFoi{JgRs3`1{rtBlf6iTVX6=)(`8@59WA8mmpPD<(t2p~p#)}h2d-He~1bK$d zk*+T~r&{D`d@Abp##ti*IGvrxvmk1%*`+IQihd?Ku(UQszAoGC`&s@v(}Vec z3>2GYzhen!n&4hnRA*;b6jL`r;NYJ>cQ)0iyg2dn$W#U~u~qYbH0qaZvy>>Af8u1z z)R_~)_lAF3^*zo{v*1(Lj&wnLXT{hSv@ljA{G1VB$iS~7FRaD(pz&$4(@_SS4K+S1 zBpTSK^>d0ce>m{TaA6XIT!fuk2+slL)5lx7$krsmn}6?54j(`NQ=Q%Y^8Y6!mw`>p zsQC6~=g&Wt@#i+Z{=U2DOdrSs30@ute&z%Q9tUQy8>A!}?Oly|%cK@kJs7Pr^5*3n6;pPiM&f6E#^I zuV4NB-S(3{%s^c*ODYcoEs)uq4EAn5&_I|IpYQ@zL0vXu56~=!zMJ4sn`{nr#R)@e zu%kY;0UczSb{rgn_ka#iIQ|Cgo|DpS2F;8za{qGu{H=?o&Aqn15UhK#I*NtHi8xm%v*`~T?17N+@83{(8ECM%$zki*u@o@h`K-1M|4 z^|a+?y>Jm=F5R{^L@a)3=C8PQnntZo54ZXPv${-Q-;-HN!CIoz*Zwj2Bw;b(_<`?5 zUK2N5`*ZdE_b@fa4~7Ercm7rH-MrS5L1BG!u5$faUFV0^ay(V0%6iNi)5YU0roTI- z$$IYAp{V`!+tio;XJh!|r@*$&RH8)s*=lXZ6Vi^r{F!I{eBWemuAZ&mO6uy?T@x&_ zEL3G!5i&{i`7vKN0WqEXw|2UL^St+UriQGow>*M3ALlq!aQoB4V+;-DO$K+i*6c84 zn9$F1vcTfrWA?+5snJU>-RbU|&@Fve@<8kQ^&PW|cBwf3EZ13IB`vPt}Z7X0o@ d?q`Sp410V_o=&*4CyW6IJYD@<);T3K0RYtteOCYg literal 0 HcmV?d00001 From 049eae27326458b9e40d3c8498aa6c442cfba9f1 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 14:53:21 +0100 Subject: [PATCH 09/24] fix(evalboard): show per-row cost for open-weight harnesses (Pi) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-message Cost column priced each row purely from the rate card (messageCostUsd = per-message tokens x list rate). A harness that owns its provider and reports a REAL per-turn total_cost_usd on a model absent from the rate card (Pi on open-weight models like kimi-k3 / deepseek) priced every row null, so the column dashed out ("—") even though the turn had a real bill — while codex (gpt-5.6-terra, in the card) showed numbers. parseMessages now apportions a turn's real token_usage.total_cost_usd across its rows by token share whenever the rate card priced nothing for that turn. The rows then show real numbers that sum EXACTLY to the turn total (last row absorbs the rounding residual), and so to the task total. Priced harnesses (Claude/Codex/Bedrock) already have per-row figures and are untouched — the pass only fills the gap, never overrides. Display-only: the authoritative task total still reads the backend aggregate, so no double-count. Verified on a real Pi run: 3 rows that were "—" now read $0.002915/$0.003074/ $0.003325, summing to exactly the $0.0093141 turn total. Adds 3 parseMessages tests and updates the per-message cost tooltip. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../app/runs/[id]/[...task]/_sections.tsx | 2 +- evalboard/lib/__tests__/parseMessages.test.ts | 71 +++++++++++++++++++ evalboard/lib/runs.ts | 46 ++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx index 617da165..f52ff560 100644 --- a/evalboard/app/runs/[id]/[...task]/_sections.tsx +++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx @@ -279,7 +279,7 @@ const MSG_GRID = // figure, not the SDK's cumulative per-turn cost). Token-column copy is shared // via TOKEN_COLUMN_HELP so the timeline and the run grid stay consistent. const MESSAGE_COST_HELP = - "Per-message cost: this message's recorded tokens priced at list rates — the cost of this single API call. The SDK reports only a cumulative per-turn figure, so these need not sum exactly to the task's total cost (the authoritative SDK number) shown above. Blank when the model is unpriced or no per-message tokens were recorded."; + "Per-message cost: this message's recorded tokens priced at list rates — the cost of this single API call. The SDK reports only a cumulative per-turn figure, so these need not sum exactly to the task's total cost (the authoritative SDK number) shown above. For a harness that reports a real per-turn cost on an unpriced model (e.g. Pi on an open-weight model), the turn's real cost is instead apportioned across its rows by token share, so those rows DO sum to the turn total. Blank only when neither a list price nor a real turn cost is available, or no per-message tokens were recorded."; function messageKind(blockTypes: MessageEvent["blockTypes"]): string { const set = new Set(blockTypes); diff --git a/evalboard/lib/__tests__/parseMessages.test.ts b/evalboard/lib/__tests__/parseMessages.test.ts index ba21f4f7..052ee39f 100644 --- a/evalboard/lib/__tests__/parseMessages.test.ts +++ b/evalboard/lib/__tests__/parseMessages.test.ts @@ -861,3 +861,74 @@ describe("parseMessages — reconciliation entry", () => { expect(recon.costUsd).toBeNull(); }); }); + +describe("parseMessages — open-weight cost apportionment", () => { + // An assistant message carrying token buckets + a model, on a turn that + // reports a real per-turn total_cost_usd. Mirrors the Pi harness shape: + // real stream cost at the turn, models absent from the rate card. + function tokMsg( + model: string, + input: number, + output: number, + // Distinct started_at + message_id so separate generations are NOT + // collapsed into one emission row by the same-emission heuristic. + startSec = 0, + messageId?: string, + ) { + const ss = String(startSec).padStart(2, "0"); + return { + role: "assistant", + started_at: `2026-01-01T00:00:${ss}.000Z`, + completed_at: `2026-01-01T00:00:${ss}.500Z`, + generation_duration_ms: 500, + content_blocks: [{ block_type: "text" as const, text: "x" }], + input_tokens: input, + output_tokens: output, + model, + message_id: messageId ?? `m-${startSec}`, + }; + } + + test("unpriced model + real turn cost: rows are filled and sum EXACTLY to the total", () => { + const turns: TurnEntry[] = [ + { + model_used: "openrouter/unpriced-xyz", + token_usage: { total_cost_usd: 0.009 }, + messages: [ + tokMsg("openrouter/unpriced-xyz", 300, 100, 0), // weight 400 + tokMsg("openrouter/unpriced-xyz", 100, 0, 2), // weight 100 + ], + }, + ]; + const rows = parseMessages(turns); + expect(rows).toHaveLength(2); + for (const r of rows) expect(r.costUsd).not.toBeNull(); + const sum = rows.reduce((a, r) => a + (r.costUsd ?? 0), 0); + expect(sum).toBeCloseTo(0.009, 10); // exact to the cent and far beyond + // Apportioned by token share: first row 400/500, second 100/500. + expect(rows[0].costUsd).toBeCloseTo(0.009 * (400 / 500), 10); + }); + + test("unpriced model + NO turn cost: rows stay blank (—), not a misleading $0", () => { + const turns: TurnEntry[] = [ + { + model_used: "openrouter/unpriced-xyz", + messages: [tokMsg("openrouter/unpriced-xyz", 300, 100)], + }, + ]; + expect(parseMessages(turns)[0].costUsd).toBeNull(); + }); + + test("priced model is left on the rate card (apportionment never overrides)", () => { + const turns: TurnEntry[] = [ + { + model_used: "gpt-5.6-terra", + token_usage: { total_cost_usd: 999 }, // absurd; must NOT leak into rows + messages: [tokMsg("gpt-5.6-terra", 1_000_000, 1_000_000)], + }, + ]; + const r = parseMessages(turns)[0]; + // Rate card: 1M input @ $2 + 1M output @ $12 = $14, not the bogus 999. + expect(r.costUsd).toBeCloseTo(14, 6); + }); +}); diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 010a38b2..a3ba62bb 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -1609,6 +1609,9 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] { const out: MessageEvent[] = []; let order = 0; for (const turn of turns) { + // Index in `out` where this turn's rows begin, so the open-weight + // cost-apportionment pass below can address exactly this turn's rows. + const turnStart = out.length; // Resolve tool_use_id -> CommandEntry per iteration (the id is locally // unique). Used to pull execution time + result preview onto tool_use // blocks. @@ -1998,6 +2001,49 @@ export function parseMessages(turns: TurnEntry[]): MessageEvent[] { note: typeof msg.note === "string" ? msg.note : null, }); } + + // Open-weight cost apportionment. A CLI harness that owns its provider + // (Pi and friends) reports a REAL per-turn `total_cost_usd` from its + // stream but runs models absent from the rate card, so `messageCostUsd` + // priced every row null and the Cost column dashes out even though the + // turn has a real bill. When the rate card priced NOTHING for this turn + // yet the backend reported a real total, distribute that total across + // this turn's rows in proportion to their tokens. The column then shows + // numbers that sum EXACTLY to the turn total (last row absorbs the + // rounding residual), and so to the task total. Priced harnesses + // (Claude / Codex / Bedrock) already have per-row figures and are left + // untouched — this only fills the gap, never overrides. Display-only: + // the authoritative task total still reads the backend aggregate, not a + // sum of these costUsd, so there is no double-count. + const turnRows = out.slice(turnStart); + const realCost = turn.token_usage?.total_cost_usd; + const anyPriced = turnRows.some((r) => r.costUsd != null); + if ( + !anyPriced && + typeof realCost === "number" && + realCost > 0 && + turnRows.length > 0 + ) { + const weight = (r: MessageEvent) => + (r.inputTokens ?? 0) + + (r.outputTokens ?? 0) + + (r.cacheWriteTokens ?? 0) + + (r.cacheReadTokens ?? 0); + const totalWeight = turnRows.reduce((a, r) => a + weight(r), 0); + if (totalWeight > 0) { + let allocated = 0; + turnRows.forEach((r, i) => { + if (i === turnRows.length - 1) { + // Last row takes the residual so the column sums exactly. + r.costUsd = realCost - allocated; + } else { + const share = (realCost * weight(r)) / totalWeight; + r.costUsd = share; + allocated += share; + } + }); + } + } } return out; } From 45f72ae5850347ab2ad520b6696dcc68a792e634 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 15:22:40 +0100 Subject: [PATCH 10/24] =?UTF-8?q?feat(docker):=201/4=20=E2=80=94=20bake=20?= =?UTF-8?q?the=20pinned=20Pi=20CLI=20into=20docker/Dockerfile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ARG PI_VERSION=0.84.4 + `npm install -g @earendil-works/pi-coding-agent` after the claude-code layer, mirroring the CLAUDE_CODE_VERSION pinned-CLI pattern (Node 22 + npm already present). Correct the lines 61-66 comment so Pi is named as baked and OpenCode remains the sole not-baked example. Co-Authored-By: Claude Opus 4.8 --- docker/Dockerfile | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d9cdd586..429df09f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -36,6 +36,12 @@ RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ && rm -rf /var/lib/apt/lists/* \ && npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} +# Pi Node CLI, pinned — same rationale as the Claude Code pin above (the agent +# binary is a dominant non-model driver of results; @latest would freeze +# nondeterministically under layer caching). Node 22 + npm are already present. +ARG PI_VERSION=0.84.4 +RUN npm install -g @earendil-works/pi-coding-agent@${PI_VERSION} + # uv: matches host sandbox.py's `uv venv` + `uv pip install` fast path RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR=/usr/local/bin sh @@ -58,12 +64,15 @@ COPY experiments/default.yaml ./experiments/default.yaml # come from public PyPI, so this needs no private-index credentials. The RUN # below always passes `--extra codex --extra antigravity --extra litellm`. # -# NOT every built-in agent ships here: `opencode` is registered unconditionally -# but its CLI is a Node package (`npm install -g opencode-ai`), absent from this -# image, and no OpenCode credentials are in SandboxConfig.env_passthrough -- so -# `--driver docker` does not support it. Adding it means a pinned version that -# travels with the release tag (as CLAUDE_CODE_VERSION does) plus an -# env_passthrough block; see docs/agents/OPENCODE.md "Running in Docker". +# `pi` IS baked above (pinned PI_VERSION) and its OPENROUTER_API_KEY provider +# credential is in SandboxConfig.env_passthrough, so `--driver docker --type pi` +# is supported. NOT every built-in agent ships here, though: `opencode` is +# registered unconditionally but its CLI is a Node package +# (`npm install -g opencode-ai`), absent from this image, and no OpenCode +# credentials are in SandboxConfig.env_passthrough -- so `--driver docker` does +# not support it. Adding it means a pinned version that travels with the release +# tag (as CLAUDE_CODE_VERSION / PI_VERSION do) plus an env_passthrough block; see +# docs/agents/OPENCODE.md "Running in Docker". # # CODER_EVAL_UV_EXTRAS carries ADDITIONAL opt-in extras on top of those; it # defaults to none. `make docker-image-full` passes `--extra uipath`, which From e5a2562afcecad3eaa2ab2207cc6ce7469d7f79b Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 15:23:52 +0100 Subject: [PATCH 11/24] =?UTF-8?q?feat(sandbox):=202/4=20=E2=80=94=20forwar?= =?UTF-8?q?d=20OPENROUTER=5FAPI=5FKEY=20into=20docker=20containers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add OPENROUTER_API_KEY to DockerDriverConfig.env_passthrough default so a `--driver docker --type pi` run reaches OpenRouter without a per-task env_passthrough_extra, mirroring the CODEX_API_KEY / GEMINI_API_KEY precedent for the other baked agents. Add a membership test that reads the model default rather than a hardcoded list. Co-Authored-By: Claude Opus 4.8 --- src/coder_eval/models/sandbox.py | 5 +++++ tests/test_docker_wildcard_env.py | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/src/coder_eval/models/sandbox.py b/src/coder_eval/models/sandbox.py index d2083d09..94108c47 100644 --- a/src/coder_eval/models/sandbox.py +++ b/src/coder_eval/models/sandbox.py @@ -256,6 +256,11 @@ class DockerDriverConfig(BaseModel): # selects the Gemini model when agent.model is unset. "GEMINI_API_KEY", "ANTIGRAVITY_MODEL", + # Pi agent provider credential — Pi addresses models as `provider/id` + # and reads OpenRouter's key from the env. The baked pi CLI + this + # passthrough make `--driver docker --type pi` work; without it the + # in-container pi has no credential and every turn fails auth. + "OPENROUTER_API_KEY", # User HOME used to keep ~/.claude resolution symmetric with the host. # See docs/DOCKER_ISOLATION.md "HOME is forwarded by default" for the # contract. tl;dr: Path.home() inside the container returns the diff --git a/tests/test_docker_wildcard_env.py b/tests/test_docker_wildcard_env.py index 0c61775f..803ce2ec 100644 --- a/tests/test_docker_wildcard_env.py +++ b/tests/test_docker_wildcard_env.py @@ -44,3 +44,10 @@ def test_env_passthrough_extra_list_persists(self): extras = ["TOKEN_A", "TOKEN_B", "TOKEN_C"] cfg = DockerDriverConfig(env_passthrough_extra=extras) assert cfg.env_passthrough_extra == extras + + def test_openrouter_api_key_in_default_allowlist(self): + """Pi's provider credential ships in the default env_passthrough so a + `--driver docker --type pi` run authenticates without a per-task + env_passthrough_extra. Read the default off the model so the assertion + survives future additions to the allowlist.""" + assert "OPENROUTER_API_KEY" in DockerDriverConfig().env_passthrough From 86c47777266e15e4346146ba7e68ea69c6068e25 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 15:24:54 +0100 Subject: [PATCH 12/24] =?UTF-8?q?docs(pi):=203/4=20=E2=80=94=20document=20?= =?UTF-8?q?docker=20support=20for=20Pi?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite PI.md "Running in Docker" (Pi is supported under --driver docker; baked CLI + OPENROUTER_API_KEY passthrough; docker recommended for untrusted/adversarial runs since permission_mode is unenforced). Drop the now-false "docker sandbox driver is unsupported" limitation bullet. Update the pyproject.toml `pi` extra comment to note the CLI is baked into the docker image. Co-Authored-By: Claude Opus 4.8 --- docs/agents/PI.md | 22 ++++++++++++++++------ pyproject.toml | 8 +++++--- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/agents/PI.md b/docs/agents/PI.md index a727998f..cdfecc5a 100644 --- a/docs/agents/PI.md +++ b/docs/agents/PI.md @@ -214,10 +214,22 @@ success — the error names the unrecognized event types it saw. Intentional cut ## Running in Docker -**Not supported yet — use the default `tempdir` driver.** Like OpenCode, Pi is a -Node CLI that is not baked into `docker/Dockerfile`, and no Pi credentials are in -the docker driver's `env_passthrough` allowlist. Run Pi tasks under `tempdir` -(the default) on a host that has the CLI and its provider credentials. +Pi **is supported under `--driver docker`.** The pinned Pi CLI is baked into +`docker/Dockerfile` (`ARG PI_VERSION`, alongside the claude-code CLI), and +`OPENROUTER_API_KEY` is forwarded into the container by the docker driver's +default `env_passthrough` allowlist — so `--driver docker --type pi` runs +end-to-end with no per-task `env_passthrough_extra`: + +```bash +export OPENROUTER_API_KEY="sk-or-..." +uv run coder-eval run tasks/pi_smoke_test.yaml --driver docker +``` + +**Docker is the recommended driver for untrusted / adversarial Pi runs.** Pi's +`permission_mode` is unenforced — headless print mode auto-runs tools — so the +**container is the confinement boundary**. Under `tempdir` there is no such +boundary; the agent runs with the host's own permissions. Prefer `--driver +docker` whenever the task prompt or workspace is not fully trusted. ## Known limitations @@ -238,8 +250,6 @@ the docker driver's `env_passthrough` allowlist. Run Pi tasks under `tempdir` cleanly as `max_turns_exhausted`. See [Run-Limit Parity](HARNESS_PARITY.md) before holding `max_turns` constant across harnesses. -- **The `docker` sandbox driver is unsupported** — see - [Running in Docker](#running-in-docker). - **No sub-agent attribution.** Pi's CLI stream does not expose nested agent generations, so per-sub-agent token grouping (available for Claude and Codex) is not derivable. diff --git a/pyproject.toml b/pyproject.toml index d382cfdb..c152c793 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,9 +152,11 @@ opencode = [] # binary over a subprocess reading `pi -p --mode json` — it imports no # third-party Python package. The extra exists to keep the opt-in surface # uniform across harnesses and to give the `npm install` prerequisite a single -# home in the packaging metadata. Without the CLI on PATH the framework still -# installs and runs; Pi tasks fail at start() with a clear hint pointing here. -# See docs/agents/PI.md. +# home in the packaging metadata. On `--driver docker` the pinned Pi CLI is baked +# into docker/Dockerfile (like claude-code), so no host install is needed there; +# on `tempdir`/host the `npm install` prerequisite above still applies. Without +# the CLI on PATH the framework still installs and runs; Pi tasks fail at start() +# with a clear hint pointing here. See docs/agents/PI.md. pi = [] [project.scripts] From 2932cc357904bea1b835fab8edcd56dea7501fa0 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 15:26:01 +0100 Subject: [PATCH 13/24] =?UTF-8?q?test(docker):=204/4=20=E2=80=94=20guard?= =?UTF-8?q?=20that=20the=20Dockerfile=20bakes=20a=20pinned=20Pi=20CLI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add test_pi_cli_baked_and_pinned: assert docker/Dockerfile carries an `ARG PI_VERSION=` (never `latest`) and an `npm install -g @earendil-works/pi-coding-agent@${PI_VERSION}` line that references the ARG, so the pin and the install cannot drift. Mirrors the existing claude-code pinned-CLI guard; scoped to the main Dockerfile only (Dockerfile.runtime deliberately does not bake Pi). Co-Authored-By: Claude Opus 4.8 --- tests/test_image_from_dockerfiles.py | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_image_from_dockerfiles.py b/tests/test_image_from_dockerfiles.py index 5ab57d9a..2400aa39 100644 --- a/tests/test_image_from_dockerfiles.py +++ b/tests/test_image_from_dockerfiles.py @@ -507,3 +507,34 @@ def _pin(df: Path) -> str | None: f"CLAUDE_CODE_VERSION drift: docker/Dockerfile pins {fw!r} but docker/Dockerfile.runtime pins {kit!r} — " "inject and rebase tasks would run different Claude Code versions." ) + + +def test_pi_cli_baked_and_pinned() -> None: + """docker/Dockerfile must ship a PINNED Pi CLI, and its install line must + reference the ARG (not a hardcoded literal) so a bump is one edit and the + pin can't drift from the install. Mirrors the claude-code pinned-CLI guard + above — Pi's presence is a positive, mechanically-checkable invariant + (`--driver docker --type pi` depends on it). Fails on @latest or removal. + Scope: main docker/Dockerfile only — Dockerfile.runtime deliberately does + NOT bake Pi (inject-mode is out of scope), so it is not asserted here.""" + df_text = (Path(__file__).resolve().parents[1] / "docker" / "Dockerfile").read_text(encoding="utf-8") + + pin: str | None = None + for ln in df_text.splitlines(): + m = re.match(r"\s*ARG PI_VERSION=(\S+)", ln) + if m: + pin = m.group(1) + break + assert pin, "ARG PI_VERSION= missing from docker/Dockerfile — the Pi CLI is not pinned." + assert pin.lower() != "latest", ( + f"PI_VERSION is pinned to {pin!r} — the Pi CLI must be an exact version, not `latest`, so layer " + "caching cannot freeze it nondeterministically across rebuilds." + ) + + assert re.search( + r"npm install -g @earendil-works/pi-coding-agent@\$\{PI_VERSION\}", + df_text, + ), ( + "docker/Dockerfile must `npm install -g @earendil-works/pi-coding-agent@${PI_VERSION}` — the install " + "must reference the PI_VERSION ARG so the pin and the install cannot drift." + ) From f37b112296db3032cce0c3117f61e23d9dd8df66 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 4 Sep 2026 15:37:49 +0100 Subject: [PATCH 14/24] docs(docker): fix env_passthrough model name + list Pi in baked-toolchain docs Code-review follow-ups (both Low, no correctness bugs found): - The Dockerfile comment (and OPENCODE.md) said the OPENROUTER_API_KEY allowlist is on `SandboxConfig.env_passthrough`; the field is actually on `DockerDriverConfig.env_passthrough` (SandboxConfig has no such field). A maintainer grepping SandboxConfig would find nothing. Corrected both. - docs/DOCKER_ISOLATION.md and docs/tutorials/06 listed the baked toolchain but omitted the now-baked Pi CLI (and the codex/antigravity/litellm SDKs). Made both lists accurate. Comment/docs-only; the built image and the pinned Pi layer are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- docker/Dockerfile | 4 ++-- docs/DOCKER_ISOLATION.md | 2 +- docs/agents/OPENCODE.md | 2 +- docs/tutorials/06-use-docker-isolation.md | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 429df09f..fb06c923 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,11 +65,11 @@ COPY experiments/default.yaml ./experiments/default.yaml # below always passes `--extra codex --extra antigravity --extra litellm`. # # `pi` IS baked above (pinned PI_VERSION) and its OPENROUTER_API_KEY provider -# credential is in SandboxConfig.env_passthrough, so `--driver docker --type pi` +# credential is in DockerDriverConfig.env_passthrough, so `--driver docker --type pi` # is supported. NOT every built-in agent ships here, though: `opencode` is # registered unconditionally but its CLI is a Node package # (`npm install -g opencode-ai`), absent from this image, and no OpenCode -# credentials are in SandboxConfig.env_passthrough -- so `--driver docker` does +# credentials are in DockerDriverConfig.env_passthrough -- so `--driver docker` does # not support it. Adding it means a pinned version that travels with the release # tag (as CLAUDE_CODE_VERSION / PI_VERSION do) plus an env_passthrough block; see # docs/agents/OPENCODE.md "Running in Docker". diff --git a/docs/DOCKER_ISOLATION.md b/docs/DOCKER_ISOLATION.md index 6018f418..14850092 100644 --- a/docs/DOCKER_ISOLATION.md +++ b/docs/DOCKER_ISOLATION.md @@ -15,7 +15,7 @@ Set `sandbox.driver: docker` on a task (or pass `--driver docker` on the CLI — a thin alias for `-D sandbox.driver=docker`) when you want: - **Isolation from the host filesystem/network** — agent-generated code can't reach files outside the sandbox. -- **A pinned toolchain** — the image bakes in Python 3.13, Node 22 LTS, `@anthropic-ai/claude-code`, `uv`, and the matching `coder_eval` version, so results don't drift with host upgrades. +- **A pinned toolchain** — the image bakes in Python 3.13, Node 22 LTS, `@anthropic-ai/claude-code`, the `pi` CLI (`@earendil-works/pi-coding-agent`), the codex/antigravity/litellm agent SDKs, `uv`, and the matching `coder_eval` version, so results don't drift with host upgrades. Aggregation (P/R/F1, suite thresholds, reports) always stays on the host. Each container is a sealed "run one task → emit one `task.json`" worker. diff --git a/docs/agents/OPENCODE.md b/docs/agents/OPENCODE.md index 31921b3d..fb0f03fd 100644 --- a/docs/agents/OPENCODE.md +++ b/docs/agents/OPENCODE.md @@ -296,7 +296,7 @@ Two things are missing, both deliberate rather than overlooked: `CLAUDE_CODE_VERSION` does — a release-process decision, not a one-line edit. (Node 22 is already present in the image, so the change itself is small.) - **No credentials would reach it.** The docker driver forwards host environment - variables through an explicit allowlist (`SandboxConfig.env_passthrough`), + variables through an explicit allowlist (`DockerDriverConfig.env_passthrough`), which carries per-harness blocks for Codex and Antigravity but none for OpenCode — so `OPENROUTER_API_KEY` and friends are not passed through, and `opencode auth login`'s credential file is not mounted. Even a custom image diff --git a/docs/tutorials/06-use-docker-isolation.md b/docs/tutorials/06-use-docker-isolation.md index 2e6b4d25..7e3631d1 100644 --- a/docs/tutorials/06-use-docker-isolation.md +++ b/docs/tutorials/06-use-docker-isolation.md @@ -19,8 +19,8 @@ For the complete reference, see [Docker Isolation](../DOCKER_ISOLATION.md). - **Isolation** — agent-generated code can't touch files or the network outside the sandbox. - **Reproducibility** — the image bakes in Python 3.13, Node 22 LTS, the Claude - CLI, `uv`, and the matching `coder_eval` version, so results don't drift with - host upgrades. + and Pi CLIs, the codex/antigravity/litellm agent SDKs, `uv`, and the matching + `coder_eval` version, so results don't drift with host upgrades. Aggregation (scores, reports, evalboard) still happens on the host — each container is a sealed "run one task → emit one `task.json`" worker. From 91d67230711bafd0a1f4ee0620c550a08bd5da37 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 8 Sep 2026 13:17:20 +0100 Subject: [PATCH 15/24] feat(pi): load agent.plugins skills via --skill + harden turn/token handling - Resolve agent.plugins -> skills dirs via the shared _plugin_skill_dirs resolver (now harness-labeled) and forward each as `pi --skill `; drop `plugins` from the unsupported set. Records pi_skill_paths in environment_info so a report can confirm the skill under test reached the agent. - Close a dangling TurnStartEvent when a new turn_start arrives without a prior turn_end (the willRetry mid-turn abort), preserving the one-pair-per-inner-turn contract renderers depend on. - Surface a terminal provider error (stopReason=error) in the result, and reset it on a recovered turn so an intermediate retry error never leaks. - Warn once per turn on token-accounting drift (bad bucket type / missing usage object) instead of silently zeroing the turn's tokens and cost. Co-Authored-By: Claude Opus 4.8 --- src/coder_eval/agents/opencode_agent.py | 20 ++-- src/coder_eval/agents/pi_agent.py | 86 +++++++++++++++- tests/test_pi_agent.py | 131 +++++++++++++++++++++++- 3 files changed, 223 insertions(+), 14 deletions(-) diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index d0e4c908..32fce4b1 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -306,17 +306,21 @@ def _manifest_skill_dirs(root: Path) -> list[Path]: def _plugin_skill_dirs( plugins: Sequence[Mapping[str, Any]] | None, log: logging.Logger | logging.LoggerAdapter[Any] = logger, + harness: str = "opencode", ) -> list[str]: - """Resolve ``plugins:`` entries to OpenCode ``skills.paths`` directories. - - Every way this can come up empty is logged rather than passed over: a plugin - whose skills never reach the agent still *looks* like a normal run, which is - precisely the failure this function exists to close. + """Resolve ``plugins:`` entries to skill-directory paths for a CLI harness. + + Returns the skills-parent directories (each holding ``/SKILL.md``) that + a ``type: local`` plugin root declares. Shared by OpenCode (``skills.paths`` + in ``OPENCODE_CONFIG_CONTENT``) and Pi (a ``--skill `` argument each); + ``harness`` only labels the diagnostics. Every way this can come up empty is + logged rather than passed over: a plugin whose skills never reach the agent + still *looks* like a normal run, which is precisely the failure this closes. """ resolved: list[str] = [] for plugin in plugins or []: if not isinstance(plugin, Mapping) or plugin.get("type") != "local": - log.warning("opencode: ignoring non-local plugin entry %r — only `type: local` maps to skills.", plugin) + log.warning(f"{harness}: ignoring non-local plugin entry %r — only `type: local` maps to skills.", plugin) continue path_str = plugin.get("path") if not path_str: @@ -326,7 +330,7 @@ def _plugin_skill_dirs( if not root.is_dir(): hint = "env var likely unset" if "$" in expanded else "path does not exist" log.warning( - "opencode: plugin skills path did not resolve: %r -> %r (%s); no skills injected from it", + f"{harness}: plugin skills path did not resolve: %r -> %r (%s); no skills injected from it", path_str, expanded, hint, @@ -344,7 +348,7 @@ def _plugin_skill_dirs( for directory in candidates: if next(directory.glob(f"*/{_SKILL_FILE}"), None) is None: log.warning( - "opencode: no /%s directly under %s (from plugin %r) — the CLI still scans it " + f"{harness}: no /%s directly under %s (from plugin %r) — the CLI still scans it " + "recursively, but check the plugin path points at a skills root", _SKILL_FILE, directory, diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index e6ed3b4f..49f10c61 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -75,6 +75,7 @@ from uuid import uuid4 from coder_eval.agent import Agent +from coder_eval.agents.opencode_agent import _plugin_skill_dirs # shared plugin->skills resolver from coder_eval.errors import AgentCrashError, TurnTimeoutError from coder_eval.isolation.docker_runner import STDOUT_LINE_LIMIT_BYTES from coder_eval.models import ( @@ -175,9 +176,9 @@ # safely forwarded. `experiments/default.yaml` sets `permission_mode` and # `allowed_tools` on every task, so warn once at start() rather than let a task # believe it constrained the agent. NOTE `system_prompt` IS supported (mapped to -# --append-system-prompt) and thus deliberately NOT here. +# --append-system-prompt) and `plugins` IS supported (each resolved skills dir is +# mapped to a `--skill ` argument), so neither is here. _UNSUPPORTED_CONFIG_FIELDS: tuple[str, ...] = ( - "plugins", "permission_mode", "system_prompt_file", # Pi's built-in tool names are lowercase (bash/read/write/edit/grep/find/ls) @@ -294,6 +295,13 @@ def __init__(self, *, task_id: str, iteration: int, user_input: str, model: str # message can name what it actually saw. self.recognized_events = 0 self.unrecognized_types: set[str] = set() + # Warn-once guard for token-accounting drift. The event-vocabulary check + # catches renamed EVENT types, but not a renamed/absent `usage` field or a + # bucket whose type changed — those silently coerce to 0 (see `_as_int`) and + # would zero out the run's tokens/cost, blinding max_total_tokens / max_usd + # gates. Mirrors OpenCode's `_warn_token_shape` (its escape hatch shipped + # with this warning; Pi's earlier cut kept the hatch but dropped the warn). + self.warned_token_shape = False self._emit: Callable[[StreamEvent], None] = lambda _e: None @@ -310,6 +318,23 @@ def agent_output(self) -> str: # --- event handlers ---------------------------------------------------- def on_turn_start(self) -> None: + # A prior step's `turn_start` with no `turn_end` — a generation aborted + # mid-turn (the defining willRetry case: a provider error before the + # assistant message completed). Close its dangling TurnStartEvent before + # opening the next, or the stream carries N starts and N-1 ends, breaking + # the one-pair-per-inner-turn contract renderers depend on. `finalize` + # closes only the LAST open turn, so it cannot cover this. + if self.turn_open: + self.turn_open = False + self.emit( + TurnEndEvent( + task_id=self.task_id, + thread_id=self.thread_id, + turn_id=self.turn_id, + status=TurnEndStatus.CRASHED, + tokens=None, + ) + ) self.turn_count += 1 self.turn_open = True self.turn_id = f"turn_{self.turn_count}" @@ -414,6 +439,13 @@ def _close_tool( ) ) + def _warn_token_shape(self, message: str, *args: Any) -> None: + """Log a token-accounting anomaly at most once per turn (not once per bucket/step).""" + if self.warned_token_shape: + return + self.warned_token_shape = True + logger.warning("pi: unexpected token accounting — " + message, *args) + def _as_int(self, value: Any) -> int: """Coerce one stream-supplied token count; count a non-number as 0. @@ -421,12 +453,20 @@ def _as_int(self, value: Any) -> int: ``communicate``'s ``except Exception`` turns into an ``AgentCrashError`` (categorized ``AGENT_CRASH``, ``max_retries=2``), burning three attempts on one mistyped bucket. A bool is never a token count (``int(True) == 1``). + + ``None`` is a legitimately-absent bucket (silent). Any OTHER unparseable + value is a schema drift and warns once — otherwise a changed bucket type + would silently zero the turn's tokens and cost. """ + if value is None: + return 0 if isinstance(value, bool) or not isinstance(value, int | float | str): + self._warn_token_shape("token count had unexpected type %s (%r); counted as 0", type(value).__name__, value) return 0 try: return int(value) except (TypeError, ValueError): + self._warn_token_shape("token count %r was not parseable as an int; counted as 0", value) return 0 def on_turn_end(self, obj: dict[str, Any]) -> None: @@ -439,8 +479,13 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: self.turn_open = False message = obj.get("message") message = message if isinstance(message, dict) else {} - usage = message.get("usage") - usage = usage if isinstance(usage, dict) else {} + raw_usage = message.get("usage") + if not isinstance(raw_usage, dict) or not raw_usage: + # A completed step that booked no usage object at all — a renamed or + # absent `usage` (which the event-vocabulary check cannot see). Its + # tokens/cost silently resolve to 0; say so once. + self._warn_token_shape("turn_end carried no usage object; this step's tokens/cost counted as 0") + usage = raw_usage if isinstance(raw_usage, dict) else {} step_in = self._as_int(usage.get("input")) raw_out = self._as_int(usage.get("output")) @@ -467,6 +512,15 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: finish = message.get("stopReason") if isinstance(finish, str) and finish: self.stop_reason = finish + # Capture a terminal provider error so a `pi -p` that exits 0 after + # exhausting retries still surfaces WHY (finalize reads error_message into + # result_summary.result). Reset on a non-error turn so an intermediate + # retry error that a later cycle recovered from never leaks into the result. + if finish == "error": + err = message.get("errorMessage") + self.error_message = err if isinstance(err, str) and err else "pi reported stopReason=error" + else: + self.error_message = None started = self.turn_started_at or datetime.now() completed = datetime.now() @@ -643,6 +697,9 @@ def __init__( self.working_directory: str | None = None self._env_path_prepend: list[str] = [] self._plugin_tools_dir: str | None = None + # Skills-parent dirs resolved from `agent.plugins`, passed to `pi --skill` + # (Pi discovers `/SKILL.md` recursively). Assigned in start(). + self._skill_dirs: list[str] = [] # Per-agent session, reused across communicate() calls for multi-turn / # simulation continuity (assigned in start(), removed in stop() — NOT in # kill(), which the orchestrator's mid-turn backstop calls; dropping the @@ -677,6 +734,18 @@ async def start( + "unconstrained by them; do not rely on them as a boundary (see docs/agents/PI.md).", ", ".join(ignored), ) + # Resolve `agent.plugins` -> skills dirs and load them via `pi --skill`. + # Loudly logs when plugins were declared but nothing resolved (the run + # would otherwise measure the model WITHOUT the skill under test). + self._skill_dirs = _plugin_skill_dirs(self.config.plugins, log=logger, harness="pi") + if self._skill_dirs: + logger.info("pi: loading %d skill dir(s) via --skill: %s", len(self._skill_dirs), self._skill_dirs) + elif self.config.plugins: + logger.warning( + "pi: %d plugin(s) declared but 0 skill dir(s) resolved — the agent will run WITHOUT them " + + "(see docs/agents/PI.md).", + len(self.config.plugins), + ) self.working_directory = working_directory self._env_path_prepend = list(env_path_prepend or []) self._plugin_tools_dir = plugin_tools_dir @@ -745,6 +814,10 @@ def get_environment_info(self) -> dict[str, Any]: } if self._session_id: info["pi_session_id"] = self._session_id + if self._skill_dirs: + # Recorded per task so a run's report can confirm the skills under test + # actually reached the agent. + info["pi_skill_paths"] = list(self._skill_dirs) return info # --- command construction --------------------------------------------- @@ -774,6 +847,11 @@ def _build_argv(self, user_input: str) -> list[str]: argv += ["--model", self.config.model] # provider-prefixed form if self.config.thinking_level: argv += ["--thinking", self.config.thinking_level] + for skill_dir in self._skill_dirs: + # Additive skill load (from agent.plugins). Pi lists each skill's + # name+description in the system prompt and the agent `read`s the full + # SKILL.md on demand — the OpenCode/Codex `plugins` mechanism, Pi-native. + argv += ["--skill", skill_dir] # allowed_tools / disallowed_tools are NOT forwarded. The shared config # default (experiments/default.yaml) sets Claude-namespaced tool names # (Bash/Read/Write/Edit/Glob/Grep/Skill), but Pi's built-in tools are diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index fc58df09..645b9862 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -428,10 +428,20 @@ class TestUnsupportedConfigIsAnnounced: async def test_start_warns_about_unenforced_fields(self, patch_exec, tmp_path, caplog): patch_exec(_FakeProcess(HAPPY_STREAM)) with caplog.at_level("WARNING"): - await _agent(plugins=[{"type": "local", "path": "/x"}]).start(str(tmp_path)) - assert "plugins" in caplog.text + await _agent(permission_mode="plan").start(str(tmp_path)) + assert "permission_mode" in caplog.text assert "NOT enforced" in caplog.text + async def test_plugins_that_do_not_resolve_warn_loudly(self, patch_exec, tmp_path, caplog): + """plugins IS supported now (-> --skill), but a path that resolves to no skills + must warn — else the run silently measures the model WITHOUT the skill.""" + patch_exec(_FakeProcess(HAPPY_STREAM)) + with caplog.at_level("WARNING"): + await _agent(plugins=[{"type": "local", "path": "/no/such/dir"}]).start(str(tmp_path)) + assert "0 skill dir(s) resolved" in caplog.text or "did not resolve" in caplog.text + # plugins is no longer named in the "NOT enforced" warning. + assert "plugins" not in "".join(r.message for r in caplog.records if "NOT enforced" in r.message) + async def test_unenforced_fields_warn_but_system_prompt_does_not(self, patch_exec, tmp_path, caplog): """allowed_tools/disallowed_tools are unenforced (Claude-namespaced default cannot map to Pi's lowercase toolset) and MUST warn when set. `system_prompt` IS enforced @@ -822,3 +832,120 @@ async def test_zero_usage_turn_is_scored_not_crashed(self, patch_exec, tmp_path) patch_exec(_FakeProcess(stream)) record = await _run(_agent(), tmp_path) assert record.crashed is False + + +# --- review fixes: token-drift warn (#1), dangling-turn close (#2), error capture (#3) --- + + +def _turn_end_error(msg: str = "404: blocked by guardrail") -> str: + """A `turn_end` whose provider errored (the willRetry / terminal-failure shape).""" + return json.dumps( + { + "type": "turn_end", + "message": { + "role": "assistant", + "usage": {"input": 5, "output": 2, "totalTokens": 7, "cost": {"total": 0.0}}, + "stopReason": "error", + "errorMessage": msg, + }, + "toolResults": [], + } + ) + + +class TestReviewFixes: + async def test_dangling_turn_start_is_closed_on_next_turn_start(self, patch_exec, tmp_path): + """#2: a turn_start with no turn_end (a mid-turn retry abort) must be closed + when the next turn_start arrives, so TurnStart/TurnEnd stay balanced.""" + stream = [ + _turn_start(), # turn 1 opens, then aborts mid-generation (no turn_end) + _turn_start(), # turn 2 opens -> must close turn 1 first + _turn_end(inp=10, out=5), # turn 2 ends cleanly + json.dumps({"type": "agent_settled"}), + ] + patch_exec(_FakeProcess(stream)) + recorder = _EventRecorder() + await _run(_agent(), tmp_path, stream_callback=recorder) + starts = [e for e in recorder.events if isinstance(e, TurnStartEvent)] + ends = [e for e in recorder.events if isinstance(e, TurnEndEvent)] + assert len(starts) == len(ends) == 2 # balanced despite the aborted turn + + async def test_terminal_error_is_surfaced_in_result(self, patch_exec, tmp_path): + """#3: a `pi` run that ends on an error surfaces WHY in result_summary.result.""" + stream = [_turn_start(), _turn_end_error("404: blocked by guardrail"), json.dumps({"type": "agent_settled"})] + patch_exec(_FakeProcess(stream)) + record = await _run(_agent(), tmp_path) + assert record.crashed is False + assert record.result_summary is not None + assert "blocked by guardrail" in (record.result_summary.result or "") + + def test_error_message_resets_on_a_recovered_turn(self): + """#3: an intermediate error a later cycle recovers from must not leak into the result.""" + state = _PiTurnState(task_id="t", iteration=1, user_input="x", model="m") + state.on_turn_end(json.loads(_turn_end_error("transient"))) + assert state.error_message == "transient" + state.on_turn_end(json.loads(_turn_end(inp=5, out=2))) # a later, successful turn + assert state.error_message is None + + async def test_bad_token_bucket_warns_once(self, patch_exec, tmp_path, caplog): + """#1: a bucket whose type drifted (here a dict) coerces to 0 but warns, once.""" + bad = json.dumps( + { + "type": "turn_end", + "message": { + "role": "assistant", + "usage": {"input": {"nested": 1}, "output": 2, "cost": {"total": 0.0}}, + "stopReason": "stop", + }, + "toolResults": [], + } + ) + stream = [_turn_start(), bad, _turn_start(), bad, json.dumps({"type": "agent_settled"})] + patch_exec(_FakeProcess(stream)) + with caplog.at_level("WARNING"): + await _run(_agent(), tmp_path) + warns = [r for r in caplog.records if "unexpected token accounting" in r.getMessage()] + assert len(warns) == 1 # warn-once, not once per bucket/step + + async def test_missing_usage_object_warns(self, patch_exec, tmp_path, caplog): + """#1: a completed turn_end with no `usage` object at all (a rename) warns.""" + no_usage = json.dumps( + {"type": "turn_end", "message": {"role": "assistant", "stopReason": "stop"}, "toolResults": []} + ) + stream = [_turn_start(), no_usage, json.dumps({"type": "agent_settled"})] + patch_exec(_FakeProcess(stream)) + with caplog.at_level("WARNING"): + await _run(_agent(), tmp_path) + assert any("no usage object" in r.getMessage() for r in caplog.records) + + +class TestSkillInjection: + """agent.plugins -> `pi --skill ` (mirrors OpenCode/Codex plugin->skills).""" + + def _plugin_root(self, tmp_path): + # A Claude-plugin root: /skills//SKILL.md (manifest-default layout). + root = tmp_path / "plug" + skill = root / "skills" / "demo-skill" + skill.mkdir(parents=True) + (skill / "SKILL.md").write_text("---\nname: demo-skill\ndescription: demo\n---\n# Demo\n") + return root + + async def test_resolved_plugin_emits_skill_arg(self, patch_exec, tmp_path): + root = self._plugin_root(tmp_path) + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(plugins=[{"type": "local", "path": str(root)}]), tmp_path) + argv = captured["argv"] + assert "--skill" in argv + # Points at the skills-PARENT dir (Pi discovers /SKILL.md recursively). + assert argv[argv.index("--skill") + 1] == str((root / "skills").resolve()) + + async def test_no_plugins_means_no_skill_arg(self, patch_exec, tmp_path): + captured = patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(_agent(), tmp_path) + assert "--skill" not in captured["argv"] + + async def test_skill_paths_recorded_in_environment_info(self, patch_exec, tmp_path): + root = self._plugin_root(tmp_path) + agent = _agent(plugins=[{"type": "local", "path": str(root)}]) + await agent.start(str(tmp_path)) + assert agent.get_environment_info()["pi_skill_paths"] == [str((root / "skills").resolve())] From 0c618cf9de35554636d9d44497364e5226e89815 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 8 Sep 2026 13:29:33 +0100 Subject: [PATCH 16/24] test(pi): scrub personal scratchpad path from happy-stream fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The captured session event embedded the author's machine-specific cwd (UID + scratchpad path). The harness never reads session.cwd, so this is inert, but it should not ship in a public fixture — replace with /work. Co-Authored-By: Claude Opus 4.8 --- tests/fixtures/pi_happy_stream.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/pi_happy_stream.jsonl b/tests/fixtures/pi_happy_stream.jsonl index a6401798..2891a4a0 100644 --- a/tests/fixtures/pi_happy_stream.jsonl +++ b/tests/fixtures/pi_happy_stream.jsonl @@ -1,4 +1,4 @@ -{"type":"session","version":3,"id":"01a06be7-fbbf-7641-9680-be47d4693f73","timestamp":"2026-09-04T10:12:40.511Z","cwd":"/private/tmp/claude-501/-Users-carles-pull-request-coder-eval/a0b3ca95-4a6c-49f2-b1df-fbf2641a4a6c/scratchpad/pi_bg"} +{"type":"session","version":3,"id":"01a06be7-fbbf-7641-9680-be47d4693f73","timestamp":"2026-09-04T10:12:40.511Z","cwd":"/work"} {"type":"agent_start"} {"type":"turn_start"} {"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"Create a file named hello.txt containing exactly the word hi using the write tool. Then read it back with the read tool and state the contents."}],"timestamp":1788516760585}} From 24a006aa1947eb21d33ef9539ba8911d8b2331c2 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 8 Sep 2026 13:48:46 +0100 Subject: [PATCH 17/24] test(pi): install shutil.which patch so the env-info test doesn't need the real CLI test_skill_paths_recorded_in_environment_info declared the patch_exec fixture but never invoked it, so start() hit the real `pi`-on-PATH check and failed on CI runners (Quality Gate + Windows Smoke) that have no pi binary. Call patch_exec() like the sibling skill-injection tests. Co-Authored-By: Claude Opus 4.8 --- tests/test_pi_agent.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 645b9862..1c8617d6 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -945,6 +945,9 @@ async def test_no_plugins_means_no_skill_arg(self, patch_exec, tmp_path): assert "--skill" not in captured["argv"] async def test_skill_paths_recorded_in_environment_info(self, patch_exec, tmp_path): + # Installs the shutil.which("pi") patch so start() doesn't require the real + # CLI on PATH (CI has no pi binary); we only assert on recorded skill paths. + patch_exec(_FakeProcess(HAPPY_STREAM)) root = self._plugin_root(tmp_path) agent = _agent(plugins=[{"type": "local", "path": str(root)}]) await agent.start(str(tmp_path)) From 4602445d090a0148830a0259a60ef02ad0046d70 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Tue, 8 Sep 2026 22:58:53 +0100 Subject: [PATCH 18/24] =?UTF-8?q?fix(pi):=20review=20blockers=20=E2=80=94?= =?UTF-8?q?=20session-id=20sanitize,=20error=20crash,=20telemetry=20warn,?= =?UTF-8?q?=20tool=20map,=20CE047?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses uipreliga review blockers on PR #159: - B8: sanitize task_id before it enters pi's `--session-id` (dataset-row ids are path-shaped "suite/row"; a raw '/' resolved to a non-existent session subdir and failed the row). Keep only [A-Za-z0-9._-]. - B5: crash `_settle_turn` on a terminal provider error (stopReason=error that survived pi's internal retries) so it books as ERROR (retryable / excluded from outcomes) instead of a clean COMPLETED FAILURE. Mirrors opencode_agent. - B6: warn once when a turn_end carried a usage object whose every bucket is 0 (the rename-each-key-to-0 drift shape) — otherwise tokens/cost silently vanish and max_usd / max_total_tokens go blind. - Tool map: `find`→`Glob` (Pi's glob-by-pattern search tool), drop the dead `glob` entry — so command_executed/commands_efficiency criteria written against the canonical `Glob` score on a Pi run that searched. - B2: anchor CE047's roster matcher on word boundaries (the 2-char "Pi" matched "anthropic"/"ci-pipeline"/"copies", hiding a stale roster) + name Pi in pyproject.toml description/keywords, docs/comparison.md, and the pages stub; add a guard test that a short spelling is not satisfied by a substring. Tests: +find→Glob, +all-zero-usage warn, +session-id sanitize, terminal-error now asserts crash; 465 pi + custom-lint tests pass. Co-Authored-By: Claude Opus 4.8 --- .github/pages-stub/index.html | 4 +- docs/comparison.md | 8 ++-- pyproject.toml | 4 +- src/coder_eval/agents/pi_agent.py | 34 +++++++++++++++- tests/lint/agent_roster_parity.py | 16 +++++--- tests/test_custom_lint.py | 14 +++++++ tests/test_pi_agent.py | 66 ++++++++++++++++++++++++++++--- 7 files changed, 125 insertions(+), 21 deletions(-) diff --git a/.github/pages-stub/index.html b/.github/pages-stub/index.html index 4d3677df..39deba00 100644 --- a/.github/pages-stub/index.html +++ b/.github/pages-stub/index.html @@ -12,7 +12,7 @@ -->