diff --git a/CHANGELOG.md b/CHANGELOG.md index fa71ac6..3601cf7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ trace, study, receipt, API and database schema versions. ## [Unreleased] +### Added + +- The Codex and Claude Code runners ship in the package as + `python -m agentloop.runners.codex` and + `python -m agentloop.runners.claude_code`, so they work from a `pip` install + without a repository checkout. `examples/codex_exec_runner.py` and + `examples/claude_code_runner.py` forward to them. The installed-wheel smoke + check starts both runners. + ### Documentation - The README points to the Codex and Claude Code runners, and its standalone diff --git a/README.md b/README.md index 236da5a..ab7dbb1 100644 --- a/README.md +++ b/README.md @@ -107,10 +107,12 @@ baseline, the frozen session, an interleaved before/after capture and the verdict in one command. Pass the unchanged runner as repeated `--before-arg` values and the changed runner after `--`. See [comparing two versions in one command](https://dipeshbabu.github.io/agentloop/GUIDED_OPTIMIZATION/#compare-two-versions-in-one-command). -Example runners for [OpenAI Codex CLI](https://dipeshbabu.github.io/agentloop/CODEX/) -and [Claude Code](https://dipeshbabu.github.io/agentloop/CLAUDE_CODE/) turn each -coding-agent run into a native trace, so the same workflow compares models, -instructions or settings for those agents. +Built-in runners for [OpenAI Codex CLI](https://dipeshbabu.github.io/agentloop/CODEX/) +(`python -m agentloop.runners.codex`) and +[Claude Code](https://dipeshbabu.github.io/agentloop/CLAUDE_CODE/) +(`python -m agentloop.runners.claude_code`) turn each coding-agent run into a +native trace, so the same workflow compares models, instructions or settings +for those agents. To connect your application, run `agentloop improve init --agent my_agent:answer` in its project directory. The generated adapter supports synchronous and async diff --git a/agentloop/runners/__init__.py b/agentloop/runners/__init__.py new file mode 100644 index 0000000..f24895a --- /dev/null +++ b/agentloop/runners/__init__.py @@ -0,0 +1,8 @@ +"""Command-line runners that turn coding-agent runs into native AgentLoop traces. + +Each runner reads one ``agentloop improve capture`` request on stdin, runs the +agent CLI and writes a trace to the requested path: + +- ``python -m agentloop.runners.codex`` runs OpenAI Codex CLI (``codex exec``); +- ``python -m agentloop.runners.claude_code`` runs Claude Code (``claude -p``). +""" diff --git a/agentloop/runners/_common.py b/agentloop/runners/_common.py new file mode 100644 index 0000000..514f1f1 --- /dev/null +++ b/agentloop/runners/_common.py @@ -0,0 +1,35 @@ +"""Helpers shared by the coding-agent runners.""" + +from __future__ import annotations + +import time + +OUTPUT_LIMIT = 4096 + + +def task_prompt(task_input, instructions): + """Return the prompt for a task input, with optional shared instructions first.""" + if isinstance(task_input, dict): + task_input = task_input.get("prompt") or task_input.get("question") + if not isinstance(task_input, str) or not task_input.strip(): + raise ValueError("Task input must be a prompt string or an object with 'prompt'") + return f"{instructions.strip()}\n\n{task_input}" if instructions else task_input + + +def clip(value): + """Bound recorded text so traces stay small.""" + if not isinstance(value, str): + return None + return value if len(value) <= OUTPUT_LIMIT else value[:OUTPUT_LIMIT] + "\n[truncated]" + + +def read_stream(process, lines): + """Timestamp each output line on arrival; agent events carry no timestamps.""" + for raw in process.stdout: + lines.append((time.perf_counter(), raw)) + + +def split_passthrough(argv): + """Split runner options from arguments passed unchanged to the agent CLI.""" + split = argv.index("--") if "--" in argv else len(argv) + return argv[:split], argv[split + 1 :] diff --git a/agentloop/runners/claude_code.py b/agentloop/runners/claude_code.py new file mode 100644 index 0000000..202aea3 --- /dev/null +++ b/agentloop/runners/claude_code.py @@ -0,0 +1,269 @@ +"""Run Claude Code on guided tasks and record a native AgentLoop trace. + +The runner receives the standard `agentloop improve capture` JSON request on +stdin. It runs ``claude -p --output-format stream-json --verbose`` with only +read-only tools allowed, timestamps each JSON line as it arrives, and converts +the stream into AgentLoop events: + +- each Claude API response (one ``message.id``) is one model span. It starts at + the session start or when the last tool result arrived, and ends when the last + part of that response arrives, so text and tool requests in one response are + one model call; +- each ``tool_use`` block is a tool span that ends when its ``tool_result`` + arrives, keeping errors reported by the tool; +- Claude reports usage per response. Input tokens include cache reads and cache + writes, which are also recorded separately, with ``usage_scope = "request"``. + +The ``result`` event's text is the task answer that the frozen checks score. +Arguments after ``--`` are passed to ``claude`` unchanged, so a candidate can +change the model, system prompt or settings without editing this file. + +Example: + + agentloop improve capture --tasks examples/codex_repository_tasks.json \ + --out runs/claude-baseline -- python -m agentloop.runners.claude_code --cd . +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess # nosec B404 - runs the operator's Claude Code CLI with explicit argv +import sys +import threading +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from agentloop.events import AgentEvent +from agentloop.runners._common import clip, read_stream, split_passthrough, task_prompt +from agentloop.tokens import PROVIDER +from agentloop.tracer import AgentTrace + +READ_ONLY_TOOLS = ("Read", "Grep", "Glob") + + +def result_text(content): + """Tool results are a string or a list of content blocks.""" + if isinstance(content, str): + return content + if isinstance(content, list): + return "\n".join( + block.get("text", "") + for block in content + if isinstance(block, dict) and block.get("type") == "text" + ) + return None + + +def convert(lines, *, run_id, root_id, origin, clock, model): + def at(moment): + return (origin + timedelta(seconds=moment - clock)).isoformat() + + events, responses, open_tools = [], {}, {} + output, failure, details, boundary = None, None, {}, clock + for moment, raw in lines: + try: + event = json.loads(raw) + except ValueError: + continue + kind = event.get("type") + if kind == "system" and event.get("subtype") == "init": + boundary = moment + model = model or event.get("model") + details["claude_session_id"] = event.get("session_id") + elif kind == "assistant": + message = event.get("message") or {} + key = message.get("id") or f"anonymous-{len(responses)}" + span = responses.get(key) + if span is None: + span = AgentEvent( + event_id="evt_" + uuid.uuid4().hex, + run_id=run_id, + parent_id=root_id, + event_type="model_call", + name="claude_response", + model=message.get("model") or model, + started_at=at(boundary), + ended_at=at(moment), + duration_ms=(moment - boundary) * 1000, + metadata={"content_types": [], "start": boundary}, + ) + responses[key] = span + events.append(span) + span.ended_at = at(moment) + span.duration_ms = (moment - span.metadata["start"]) * 1000 + usage = message.get("usage") + if isinstance(usage, dict): + # Each part of a response repeats the same usage: set, never add. + cache_read = int(usage.get("cache_read_input_tokens") or 0) + cache_write = int(usage.get("cache_creation_input_tokens") or 0) + span.input_tokens = int(usage.get("input_tokens") or 0) + cache_read + cache_write + span.output_tokens = int(usage.get("output_tokens") or 0) + span.token_provenance = PROVIDER + span.metadata.update( + usage_scope="request", + cached_input_tokens=cache_read, + cache_creation_input_tokens=cache_write, + ) + for block in message.get("content") or []: + if not isinstance(block, dict): + continue + span.metadata["content_types"].append(block.get("type")) + if block.get("type") == "text": + span.output_text = clip(block.get("text")) + elif block.get("type") == "tool_use": + open_tools[block.get("id")] = (moment, block) + elif kind == "user": + message = event.get("message") or {} + for block in message.get("content") or []: + if not isinstance(block, dict) or block.get("type") != "tool_result": + continue + started, request = open_tools.pop(block.get("tool_use_id"), (moment, {})) + failed = block.get("is_error") is True + text = result_text(block.get("content")) + events.append( + AgentEvent( + event_id="evt_" + uuid.uuid4().hex, + run_id=run_id, + parent_id=root_id, + event_type="tool_call", + name=request.get("name") or "tool", + started_at=at(started), + ended_at=at(moment), + duration_ms=(moment - started) * 1000, + input_text=clip(json.dumps(request.get("input"), sort_keys=True)) + if "input" in request + else None, + output_text=clip(text), + status="error" if failed else "ok", + error=(clip(text) or "tool error") if failed else None, + metadata={"claude_tool_use_id": block.get("tool_use_id")}, + ) + ) + boundary = max(boundary, moment) + elif kind == "result": + if isinstance(event.get("result"), str): + output = event["result"] + if event.get("is_error") or event.get("subtype") not in (None, "success"): + failure = ( + event.get("result") + if isinstance(event.get("result"), str) and event.get("result") + else event.get("subtype") or "Claude Code reported an error" + ) + details.update( + claude_num_turns=event.get("num_turns"), + claude_duration_ms=event.get("duration_ms"), + claude_duration_api_ms=event.get("duration_api_ms"), + claude_total_cost_usd=event.get("total_cost_usd"), + ) + for span in responses.values(): + span.metadata.pop("start", None) + usage_seen = any(span.token_provenance == PROVIDER for span in responses.values()) + return events, output, usage_seen, failure, details + + +def run(args, claude_args, request): + claude = shutil.which(args.claude) + if claude is None: + raise SystemExit(f"Cannot find the Claude Code CLI executable {args.claude!r}") + instructions = args.instructions.read_text(encoding="utf-8") if args.instructions else "" + prompt = task_prompt(request["input"], instructions) + command = [ + claude, + "-p", + "--output-format", + "stream-json", + "--verbose", + "--allowedTools", + ",".join(READ_ONLY_TOOLS), + *claude_args, + ] + trace = AgentTrace( + f"claude code: {request['task_id']}", + metadata={ + "integration": "claude_code", + "condition": args.condition, + "task_id": request["task_id"], + "repetition": request["repetition"], + "claude_args": claude_args, + }, + ) + root_id = "evt_" + uuid.uuid4().hex + origin, clock = datetime.now(timezone.utc), time.perf_counter() + process = subprocess.Popen( # nosec B603 - explicit Claude Code argv, no shell + command, + cwd=args.cd.resolve(), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) + lines = [] + reader = threading.Thread(target=read_stream, args=(process, lines), daemon=True) + reader.start() + stderr = threading.Thread(target=lambda: process.stderr.read(), daemon=True) + stderr.start() + process.stdin.write(prompt) + process.stdin.close() + returncode = process.wait() + reader.join() + ended = time.perf_counter() + events, output, usage_seen, failure, details = convert( + lines, run_id=trace.run_id, root_id=root_id, origin=origin, clock=clock, model=args.model + ) + if returncode != 0 and failure is None: + failure = f"claude exited with code {returncode}" + trace.add_event( + AgentEvent( + event_id=root_id, + run_id=trace.run_id, + event_type="agent_step", + name="claude_code", + started_at=origin.isoformat(), + ended_at=(origin + timedelta(seconds=ended - clock)).isoformat(), + duration_ms=(ended - clock) * 1000, + input_text=prompt, + output_text=clip(output), + status="error" if failure else "ok", + error=failure, + metadata={"operation_kind": "agent", "usage_reported": usage_seen, **details}, + ) + ) + for event in events: + trace.add_event(event) + trace.metadata.update( + output=output.strip() if output is not None else None, + success=failure is None and output is not None, + ) + trace.finish() + trace.export_json(request["trace_path"]) + return 1 if failure else 0 + + +def main(): + parser = argparse.ArgumentParser( + prog="python -m agentloop.runners.claude_code", + description=__doc__.splitlines()[0], + ) + parser.add_argument("--cd", type=Path, default=Path("."), help="Directory Claude Code reads.") + parser.add_argument("--claude", default="claude", help="Claude Code CLI executable.") + parser.add_argument("--model", default=None, help="Model passed to Claude Code and recorded.") + parser.add_argument("--condition", default="unlabelled", help="Label recorded in the trace.") + parser.add_argument( + "--instructions", type=Path, default=None, help="Text file prepended to each task prompt." + ) + own, claude_args = split_passthrough(sys.argv[1:]) + args = parser.parse_args(own) + if args.model: + claude_args = ["--model", args.model, *claude_args] + request = json.loads(sys.stdin.readline()) + raise SystemExit(run(args, claude_args, request)) + + +if __name__ == "__main__": + main() diff --git a/agentloop/runners/codex.py b/agentloop/runners/codex.py new file mode 100644 index 0000000..bf5d7af --- /dev/null +++ b/agentloop/runners/codex.py @@ -0,0 +1,256 @@ +"""Run OpenAI Codex CLI on guided tasks and record a native AgentLoop trace. + +The runner receives the standard `agentloop improve capture` JSON request on +stdin. It runs `codex exec --json --ephemeral` in a read-only sandbox, timestamps +each JSONL event as it arrives, and converts the stream into AgentLoop events: + +- each shell command, file change, MCP tool call or web search is a tool span; +- each model round is one model span: it starts at the turn start or when the + last running tool finished, and ends when the model starts a tool or gives + its final message, so messages within a round do not count as extra calls; +- Codex reports token usage once per turn, so the turn total is attached to the + last model span of that turn with ``usage_scope = "turn_total"``. + +The final agent message is the task answer that the frozen checks score. +Arguments after ``--`` are passed to ``codex exec`` unchanged, so a candidate can +change the model, profile or configuration without editing this file. + +Example: + + agentloop improve capture --tasks examples/codex_repository_tasks.json \ + --out runs/codex-baseline -- python -m agentloop.runners.codex --cd . +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess # nosec B404 - runs the operator's Codex CLI with explicit argv +import sys +import threading +import time +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from agentloop.events import AgentEvent +from agentloop.runners._common import clip, read_stream, split_passthrough, task_prompt +from agentloop.tokens import PROVIDER, UNAVAILABLE +from agentloop.tracer import AgentTrace + +TOOL_ITEMS = {"command_execution", "file_change", "mcp_tool_call", "web_search"} +MODEL_ITEMS = {"agent_message", "reasoning"} + + +def convert(lines, *, run_id, root_id, origin, clock, model): + def at(moment): + return (origin + timedelta(seconds=moment - clock)).isoformat() + + events, open_tools, output, usage_seen = [], {}, None, False + failure, boundary, current, current_start = None, clock, None, clock + + def model_round(moment, item_type=None, text=None): + """Open or extend the span for the current model round. + + A round starts at the turn start or when the last running tool finished + and lasts until the model starts a tool or gives its final message. + Messages and reasoning inside one round belong to the same model call. + """ + nonlocal current, current_start + if current is None: + current_start = boundary + current = AgentEvent( + event_id="evt_" + uuid.uuid4().hex, + run_id=run_id, + parent_id=root_id, + event_type="model_call", + name="codex_model_round", + model=model, + started_at=at(boundary), + ended_at=at(moment), + duration_ms=(moment - boundary) * 1000, + token_provenance=UNAVAILABLE, + metadata={"codex_items": []}, + ) + events.append(current) + else: + current.ended_at = at(moment) + current.duration_ms = (moment - current_start) * 1000 + if item_type is not None: + current.metadata["codex_items"].append(item_type) + if text is not None: + current.output_text = clip(text) + + for moment, raw in lines: + try: + event = json.loads(raw) + except ValueError: + continue + kind, item = event.get("type"), event.get("item") or {} + item_type = item.get("type") + if kind == "turn.started": + boundary, current = moment, None + elif kind == "item.started" and item_type in TOOL_ITEMS: + if not open_tools and (current is not None or moment > boundary): + model_round(moment, "tool_call") + # The round ends when its tool calls start running. + current = None + open_tools[item.get("id")] = moment + elif kind == "item.completed" and item_type in TOOL_ITEMS: + # Some tool items arrive only on completion; either way the model + # round that requested them has ended. + current = None + started = open_tools.pop(item.get("id"), moment) + exit_code = item.get("exit_code") + failed = item.get("status") == "failed" or exit_code not in (None, 0) + events.append( + AgentEvent( + event_id="evt_" + uuid.uuid4().hex, + run_id=run_id, + parent_id=root_id, + event_type="tool_call", + name="shell" if item_type == "command_execution" else item_type, + started_at=at(started), + ended_at=at(moment), + duration_ms=(moment - started) * 1000, + input_text=clip(item.get("command") or json.dumps(item.get("changes"))), + output_text=clip(item.get("aggregated_output")), + status="error" if failed else "ok", + error=f"exit code {exit_code}" if failed else None, + metadata={"codex_item_type": item_type, "exit_code": exit_code}, + ) + ) + boundary = max(boundary, moment) + elif kind == "item.completed" and item_type in MODEL_ITEMS: + if open_tools: + # Reported while tools still run: keep the round open after them. + boundary = max([boundary, *open_tools.values()]) + model_round(moment, item_type, item.get("text")) + if item_type == "agent_message": + output = item.get("text") + elif kind == "turn.completed": + usage = event.get("usage") or {} + if current is None and not any(item.event_type == "model_call" for item in events): + model_round(moment) + models = [item for item in events if item.event_type == "model_call"] + last = models[-1] + last.input_tokens = int(usage.get("input_tokens") or 0) + last.output_tokens = int(usage.get("output_tokens") or 0) + last.token_provenance = PROVIDER + last.metadata.update( + usage_scope="turn_total", + cached_input_tokens=int(usage.get("cached_input_tokens") or 0), + reasoning_output_tokens=int(usage.get("reasoning_output_tokens") or 0), + ) + usage_seen = True + elif kind in {"turn.failed", "error"}: + detail = event.get("error") or event.get("message") or kind + failure = detail.get("message", str(detail)) if isinstance(detail, dict) else detail + return events, output, usage_seen, failure + + +def run(args, codex_args, request): + codex = shutil.which(args.codex) + if codex is None: + raise SystemExit(f"Cannot find the Codex CLI executable {args.codex!r}") + instructions = args.instructions.read_text(encoding="utf-8") if args.instructions else "" + prompt = task_prompt(request["input"], instructions) + command = [ + codex, + "exec", + "--json", + "--ephemeral", + "--skip-git-repo-check", + "--sandbox", + "read-only", + "--cd", + str(args.cd.resolve()), + *codex_args, + "-", + ] + trace = AgentTrace( + f"codex exec: {request['task_id']}", + metadata={ + "integration": "codex_exec", + "condition": args.condition, + "task_id": request["task_id"], + "repetition": request["repetition"], + "codex_args": codex_args, + }, + ) + root_id = "evt_" + uuid.uuid4().hex + origin, clock = datetime.now(timezone.utc), time.perf_counter() + process = subprocess.Popen( # nosec B603 - explicit Codex argv, no shell + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding="utf-8", + errors="replace", + ) + lines = [] + reader = threading.Thread(target=read_stream, args=(process, lines), daemon=True) + reader.start() + stderr = threading.Thread(target=lambda: process.stderr.read(), daemon=True) + stderr.start() + process.stdin.write(prompt) + process.stdin.close() + returncode = process.wait() + reader.join() + ended = time.perf_counter() + events, output, usage_seen, failure = convert( + lines, run_id=trace.run_id, root_id=root_id, origin=origin, clock=clock, model=args.model + ) + if returncode != 0 and failure is None: + failure = f"codex exec exited with code {returncode}" + trace.add_event( + AgentEvent( + event_id=root_id, + run_id=trace.run_id, + event_type="agent_step", + name="codex_exec", + started_at=origin.isoformat(), + ended_at=(origin + timedelta(seconds=ended - clock)).isoformat(), + duration_ms=(ended - clock) * 1000, + input_text=prompt, + output_text=clip(output), + status="error" if failure else "ok", + error=failure, + metadata={"operation_kind": "agent", "usage_reported": usage_seen}, + ) + ) + for event in events: + trace.add_event(event) + trace.metadata.update( + output=(output or "").strip() if output is not None else None, + success=failure is None and output is not None, + ) + trace.finish() + trace.export_json(request["trace_path"]) + return 1 if failure else 0 + + +def main(): + parser = argparse.ArgumentParser( + prog="python -m agentloop.runners.codex", + description=__doc__.splitlines()[0], + ) + parser.add_argument("--cd", type=Path, default=Path("."), help="Directory Codex reads.") + parser.add_argument("--codex", default="codex", help="Codex CLI executable.") + parser.add_argument("--model", default=None, help="Model label recorded in the trace.") + parser.add_argument("--condition", default="unlabelled", help="Label recorded in the trace.") + parser.add_argument( + "--instructions", type=Path, default=None, help="Text file prepended to each task prompt." + ) + own, codex_args = split_passthrough(sys.argv[1:]) + args = parser.parse_args(own) + if args.model: + codex_args = ["--model", args.model, *codex_args] + request = json.loads(sys.stdin.readline()) + raise SystemExit(run(args, codex_args, request)) + + +if __name__ == "__main__": + main() diff --git a/docs/CLAUDE_CODE.md b/docs/CLAUDE_CODE.md index 7a4d0ac..d5ddb5e 100644 --- a/docs/CLAUDE_CODE.md +++ b/docs/CLAUDE_CODE.md @@ -1,12 +1,17 @@ # Claude Code AgentLoop can profile and compare [Claude Code](https://docs.claude.com/en/docs/claude-code/overview) -runs through the guided workflow. The example -[runner](https://github.com/dipeshbabu/agentloop/blob/main/examples/claude_code_runner.py) +runs through the guided workflow. The +[runner](https://github.com/dipeshbabu/agentloop/blob/main/agentloop/runners/claude_code.py) runs `claude -p --output-format stream-json --verbose` for each task and converts its event stream into a native AgentLoop trace. Claude Code is not modified and no plugin is installed. +AgentLoop 0.2.6 and newer install the runner with the package: run it as +`python -m agentloop.runners.claude_code`, using the Python environment where +AgentLoop is installed. In 0.2.5 it is only in a repository checkout, as +`python examples/claude_code_runner.py`. + Each task attempt is a real Claude Code run on your Claude plan or API key and counts against that usage. @@ -36,12 +41,12 @@ Tasks use a prompt string or an object with `prompt`. The Codex works unchanged: it asks three metadata questions about this repository with exact expected values. -AgentLoop 0.2.5 and newer compare an unchanged and a changed configuration in -one command. Arguments after a second `--` go to `claude` unchanged, so the +`improve compare` (AgentLoop 0.2.5 and newer) compares an unchanged and a +changed configuration in one command. Arguments after a second `--` go to `claude` unchanged, so the candidate below only lowers the turn limit: ```bash -agentloop improve compare --tasks examples/codex_repository_tasks.json --out runs/claude --timeout 600 --change "Limit lookups to four turns" --before-arg python --before-arg examples/claude_code_runner.py --before-arg --cd --before-arg . -- python examples/claude_code_runner.py --cd . -- --max-turns 4 +agentloop improve compare --tasks examples/codex_repository_tasks.json --out runs/claude --timeout 600 --change "Limit lookups to four turns" --before-arg python --before-arg -m --before-arg agentloop.runners.claude_code --before-arg --cd --before-arg . -- python -m agentloop.runners.claude_code --cd . -- --max-turns 4 ``` Runner options: `--cd` sets the directory Claude Code works in, diff --git a/docs/CODEX.md b/docs/CODEX.md index 166f3e1..7395390 100644 --- a/docs/CODEX.md +++ b/docs/CODEX.md @@ -1,11 +1,16 @@ # OpenAI Codex CLI AgentLoop can profile and compare [Codex CLI](https://github.com/openai/codex) -runs through the guided workflow. The example -[runner](https://github.com/dipeshbabu/agentloop/blob/main/examples/codex_exec_runner.py) +runs through the guided workflow. The +[runner](https://github.com/dipeshbabu/agentloop/blob/main/agentloop/runners/codex.py) runs `codex exec --json` for each task and converts its event stream into a native AgentLoop trace. Codex is not modified and no plugin is installed. +AgentLoop 0.2.6 and newer install the runner with the package: run it as +`python -m agentloop.runners.codex`, using the Python environment where +AgentLoop is installed. Earlier releases need a repository checkout and +`python examples/codex_exec_runner.py`, which still works. + Each task attempt is a real Codex run on your ChatGPT plan or API key and counts against that usage. @@ -34,7 +39,7 @@ Tasks use a prompt string or an object with `prompt`. The checked-in asks three metadata questions about this repository with exact expected values. ```bash -agentloop improve capture --tasks examples/codex_repository_tasks.json --out runs/codex/baseline --timeout 600 -- python examples/codex_exec_runner.py --cd . +agentloop improve capture --tasks examples/codex_repository_tasks.json --out runs/codex/baseline --timeout 600 -- python -m agentloop.runners.codex --cd . agentloop improve start --tasks examples/codex_repository_tasks.json --baseline runs/codex/baseline/capture.json --out runs/codex/session ``` @@ -44,7 +49,7 @@ editing the runner. Capture it with the unchanged runner as an [interleaved control](GUIDED_OPTIMIZATION.md#compare-against-an-interleaved-control): ```bash -agentloop improve capture --tasks examples/codex_repository_tasks.json --out runs/codex/paired --timeout 600 --control-arg python --control-arg examples/codex_exec_runner.py --control-arg --cd --control-arg . -- python examples/codex_exec_runner.py --cd . -- -c model_reasoning_effort="low" +agentloop improve capture --tasks examples/codex_repository_tasks.json --out runs/codex/paired --timeout 600 --control-arg python --control-arg -m --control-arg agentloop.runners.codex --control-arg --cd --control-arg . -- python -m agentloop.runners.codex --cd . -- -c model_reasoning_effort="low" agentloop improve finish --session runs/codex/session/session.json --candidate runs/codex/paired/capture.json --change "Lower reasoning effort for lookups" --out runs/codex/results ``` diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 5344fc0..709d470 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -7,11 +7,13 @@ AgentLoop 0.2.0 also supports 0.22.x and 0.23.x. The [real runtime pilot](HERMES_RUNTIME_PILOT.md) documents exact host pins and deterministic-provider validation scope. -For OpenAI Codex CLI, see [the Codex guide](CODEX.md). Its example runner -converts `codex exec --json` events into native traces for the guided workflow. +For OpenAI Codex CLI, see [the Codex guide](CODEX.md). Its runner, +`python -m agentloop.runners.codex`, converts `codex exec --json` events into +native traces for the guided workflow. -For Claude Code, see [the Claude Code guide](CLAUDE_CODE.md). Its example runner -converts `claude -p --output-format stream-json` events into native traces. +For Claude Code, see [the Claude Code guide](CLAUDE_CODE.md). Its runner, +`python -m agentloop.runners.claude_code`, converts +`claude -p --output-format stream-json` events into native traces. For framework-neutral pipeline and stage annotations, see [workflow tracing](WORKFLOW_TRACING.md). diff --git a/examples/claude_code_runner.py b/examples/claude_code_runner.py index 4972754..50b818f 100644 --- a/examples/claude_code_runner.py +++ b/examples/claude_code_runner.py @@ -1,288 +1,12 @@ -"""Run Claude Code on guided tasks and record a native AgentLoop trace. +"""Run Claude Code on guided tasks; kept for existing commands. -The runner receives the standard `agentloop improve capture` JSON request on -stdin. It runs ``claude -p --output-format stream-json --verbose`` with only -read-only tools allowed, timestamps each JSON line as it arrives, and converts -the stream into AgentLoop events: - -- each Claude API response (one ``message.id``) is one model span. It starts at - the session start or when the last tool result arrived, and ends when the last - part of that response arrives, so text and tool requests in one response are - one model call; -- each ``tool_use`` block is a tool span that ends when its ``tool_result`` - arrives, keeping errors reported by the tool; -- Claude reports usage per response. Input tokens include cache reads and cache - writes, which are also recorded separately, with ``usage_scope = "request"``. - -The ``result`` event's text is the task answer that the frozen checks score. -Arguments after ``--`` are passed to ``claude`` unchanged, so a candidate can -change the model, system prompt or settings without editing this file. - -Example: - - agentloop improve capture --tasks examples/codex_repository_tasks.json \ - --out runs/claude-baseline -- python examples/claude_code_runner.py --cd . +The runner ships in the package as ``python -m agentloop.runners.claude_code``. +This script forwards to it so commands written against a checkout keep working. """ -from __future__ import annotations - -import argparse -import json -import shutil -import subprocess # nosec B404 - runs the operator's Claude Code CLI with explicit argv -import sys -import threading -import time -import uuid -from datetime import datetime, timedelta, timezone -from pathlib import Path - -from agentloop.events import AgentEvent -from agentloop.tokens import PROVIDER -from agentloop.tracer import AgentTrace - -READ_ONLY_TOOLS = ("Read", "Grep", "Glob") -OUTPUT_LIMIT = 4096 - - -def task_prompt(task_input, instructions): - if isinstance(task_input, dict): - task_input = task_input.get("prompt") or task_input.get("question") - if not isinstance(task_input, str) or not task_input.strip(): - raise ValueError("Task input must be a prompt string or an object with 'prompt'") - return f"{instructions.strip()}\n\n{task_input}" if instructions else task_input - - -def clip(value): - if not isinstance(value, str): - return None - return value if len(value) <= OUTPUT_LIMIT else value[:OUTPUT_LIMIT] + "\n[truncated]" - - -def result_text(content): - """Tool results are a string or a list of content blocks.""" - if isinstance(content, str): - return content - if isinstance(content, list): - return "\n".join( - block.get("text", "") - for block in content - if isinstance(block, dict) and block.get("type") == "text" - ) - return None - - -def read_stream(process, lines): - """Timestamp each JSON line on arrival; stream events carry no timestamps.""" - for raw in process.stdout: - lines.append((time.perf_counter(), raw)) - - -def convert(lines, *, run_id, root_id, origin, clock, model): - def at(moment): - return (origin + timedelta(seconds=moment - clock)).isoformat() - - events, responses, open_tools = [], {}, {} - output, failure, details, boundary = None, None, {}, clock - for moment, raw in lines: - try: - event = json.loads(raw) - except ValueError: - continue - kind = event.get("type") - if kind == "system" and event.get("subtype") == "init": - boundary = moment - model = model or event.get("model") - details["claude_session_id"] = event.get("session_id") - elif kind == "assistant": - message = event.get("message") or {} - key = message.get("id") or f"anonymous-{len(responses)}" - span = responses.get(key) - if span is None: - span = AgentEvent( - event_id="evt_" + uuid.uuid4().hex, - run_id=run_id, - parent_id=root_id, - event_type="model_call", - name="claude_response", - model=message.get("model") or model, - started_at=at(boundary), - ended_at=at(moment), - duration_ms=(moment - boundary) * 1000, - metadata={"content_types": [], "start": boundary}, - ) - responses[key] = span - events.append(span) - span.ended_at = at(moment) - span.duration_ms = (moment - span.metadata["start"]) * 1000 - usage = message.get("usage") - if isinstance(usage, dict): - # Each part of a response repeats the same usage: set, never add. - cache_read = int(usage.get("cache_read_input_tokens") or 0) - cache_write = int(usage.get("cache_creation_input_tokens") or 0) - span.input_tokens = int(usage.get("input_tokens") or 0) + cache_read + cache_write - span.output_tokens = int(usage.get("output_tokens") or 0) - span.token_provenance = PROVIDER - span.metadata.update( - usage_scope="request", - cached_input_tokens=cache_read, - cache_creation_input_tokens=cache_write, - ) - for block in message.get("content") or []: - if not isinstance(block, dict): - continue - span.metadata["content_types"].append(block.get("type")) - if block.get("type") == "text": - span.output_text = clip(block.get("text")) - elif block.get("type") == "tool_use": - open_tools[block.get("id")] = (moment, block) - elif kind == "user": - message = event.get("message") or {} - for block in message.get("content") or []: - if not isinstance(block, dict) or block.get("type") != "tool_result": - continue - started, request = open_tools.pop(block.get("tool_use_id"), (moment, {})) - failed = block.get("is_error") is True - text = result_text(block.get("content")) - events.append( - AgentEvent( - event_id="evt_" + uuid.uuid4().hex, - run_id=run_id, - parent_id=root_id, - event_type="tool_call", - name=request.get("name") or "tool", - started_at=at(started), - ended_at=at(moment), - duration_ms=(moment - started) * 1000, - input_text=clip(json.dumps(request.get("input"), sort_keys=True)) - if "input" in request - else None, - output_text=clip(text), - status="error" if failed else "ok", - error=(clip(text) or "tool error") if failed else None, - metadata={"claude_tool_use_id": block.get("tool_use_id")}, - ) - ) - boundary = max(boundary, moment) - elif kind == "result": - if isinstance(event.get("result"), str): - output = event["result"] - if event.get("is_error") or event.get("subtype") not in (None, "success"): - failure = ( - event.get("result") - if isinstance(event.get("result"), str) and event.get("result") - else event.get("subtype") or "Claude Code reported an error" - ) - details.update( - claude_num_turns=event.get("num_turns"), - claude_duration_ms=event.get("duration_ms"), - claude_duration_api_ms=event.get("duration_api_ms"), - claude_total_cost_usd=event.get("total_cost_usd"), - ) - for span in responses.values(): - span.metadata.pop("start", None) - usage_seen = any(span.token_provenance == PROVIDER for span in responses.values()) - return events, output, usage_seen, failure, details - - -def run(args, claude_args, request): - claude = shutil.which(args.claude) - if claude is None: - raise SystemExit(f"Cannot find the Claude Code CLI executable {args.claude!r}") - instructions = args.instructions.read_text(encoding="utf-8") if args.instructions else "" - prompt = task_prompt(request["input"], instructions) - command = [ - claude, - "-p", - "--output-format", - "stream-json", - "--verbose", - "--allowedTools", - ",".join(READ_ONLY_TOOLS), - *claude_args, - ] - trace = AgentTrace( - f"claude code: {request['task_id']}", - metadata={ - "integration": "claude_code", - "condition": args.condition, - "task_id": request["task_id"], - "repetition": request["repetition"], - "claude_args": claude_args, - }, - ) - root_id = "evt_" + uuid.uuid4().hex - origin, clock = datetime.now(timezone.utc), time.perf_counter() - process = subprocess.Popen( # nosec B603 - explicit Claude Code argv, no shell - command, - cwd=args.cd.resolve(), - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - encoding="utf-8", - errors="replace", - ) - lines = [] - reader = threading.Thread(target=read_stream, args=(process, lines), daemon=True) - reader.start() - stderr = threading.Thread(target=lambda: process.stderr.read(), daemon=True) - stderr.start() - process.stdin.write(prompt) - process.stdin.close() - returncode = process.wait() - reader.join() - ended = time.perf_counter() - events, output, usage_seen, failure, details = convert( - lines, run_id=trace.run_id, root_id=root_id, origin=origin, clock=clock, model=args.model - ) - if returncode != 0 and failure is None: - failure = f"claude exited with code {returncode}" - trace.add_event( - AgentEvent( - event_id=root_id, - run_id=trace.run_id, - event_type="agent_step", - name="claude_code", - started_at=origin.isoformat(), - ended_at=(origin + timedelta(seconds=ended - clock)).isoformat(), - duration_ms=(ended - clock) * 1000, - input_text=prompt, - output_text=clip(output), - status="error" if failure else "ok", - error=failure, - metadata={"operation_kind": "agent", "usage_reported": usage_seen, **details}, - ) - ) - for event in events: - trace.add_event(event) - trace.metadata.update( - output=output.strip() if output is not None else None, - success=failure is None and output is not None, - ) - trace.finish() - trace.export_json(request["trace_path"]) - return 1 if failure else 0 - - -def main(): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--cd", type=Path, default=Path("."), help="Directory Claude Code reads.") - parser.add_argument("--claude", default="claude", help="Claude Code CLI executable.") - parser.add_argument("--model", default=None, help="Model passed to Claude Code and recorded.") - parser.add_argument("--condition", default="unlabelled", help="Label recorded in the trace.") - parser.add_argument( - "--instructions", type=Path, default=None, help="Text file prepended to each task prompt." - ) - argv = sys.argv[1:] - split = argv.index("--") if "--" in argv else len(argv) - args = parser.parse_args(argv[:split]) - claude_args = argv[split + 1 :] - if args.model: - claude_args = ["--model", args.model, *claude_args] - request = json.loads(sys.stdin.readline()) - raise SystemExit(run(args, claude_args, request)) +from agentloop.runners.claude_code import convert, main, task_prompt +__all__ = ["convert", "main", "task_prompt"] if __name__ == "__main__": main() diff --git a/examples/codex_exec_runner.py b/examples/codex_exec_runner.py index 105b3d6..c166cae 100644 --- a/examples/codex_exec_runner.py +++ b/examples/codex_exec_runner.py @@ -1,275 +1,12 @@ -"""Run OpenAI Codex CLI on guided tasks and record a native AgentLoop trace. +"""Run OpenAI Codex CLI on guided tasks; kept for existing commands. -The runner receives the standard `agentloop improve capture` JSON request on -stdin. It runs `codex exec --json --ephemeral` in a read-only sandbox, timestamps -each JSONL event as it arrives, and converts the stream into AgentLoop events: - -- each shell command, file change, MCP tool call or web search is a tool span; -- each model round is one model span: it starts at the turn start or when the - last running tool finished, and ends when the model starts a tool or gives - its final message, so messages within a round do not count as extra calls; -- Codex reports token usage once per turn, so the turn total is attached to the - last model span of that turn with ``usage_scope = "turn_total"``. - -The final agent message is the task answer that the frozen checks score. -Arguments after ``--`` are passed to ``codex exec`` unchanged, so a candidate can -change the model, profile or configuration without editing this file. - -Example: - - agentloop improve capture --tasks examples/codex_repository_tasks.json \ - --out runs/codex-baseline -- python examples/codex_exec_runner.py --cd . +The runner ships in the package as ``python -m agentloop.runners.codex``. +This script forwards to it so commands written against a checkout keep working. """ -from __future__ import annotations - -import argparse -import json -import shutil -import subprocess # nosec B404 - runs the operator's Codex CLI with explicit argv -import sys -import threading -import time -import uuid -from datetime import datetime, timedelta, timezone -from pathlib import Path - -from agentloop.events import AgentEvent -from agentloop.tokens import PROVIDER, UNAVAILABLE -from agentloop.tracer import AgentTrace - -TOOL_ITEMS = {"command_execution", "file_change", "mcp_tool_call", "web_search"} -MODEL_ITEMS = {"agent_message", "reasoning"} -OUTPUT_LIMIT = 4096 - - -def task_prompt(task_input, instructions): - if isinstance(task_input, dict): - task_input = task_input.get("prompt") or task_input.get("question") - if not isinstance(task_input, str) or not task_input.strip(): - raise ValueError("Task input must be a prompt string or an object with 'prompt'") - return f"{instructions.strip()}\n\n{task_input}" if instructions else task_input - - -def clip(value): - if not isinstance(value, str): - return None - return value if len(value) <= OUTPUT_LIMIT else value[:OUTPUT_LIMIT] + "\n[truncated]" - - -def read_stream(process, lines): - """Timestamp each JSONL line on arrival; Codex events carry no timestamps.""" - for raw in process.stdout: - lines.append((time.perf_counter(), raw)) - - -def convert(lines, *, run_id, root_id, origin, clock, model): - def at(moment): - return (origin + timedelta(seconds=moment - clock)).isoformat() - - events, open_tools, output, usage_seen = [], {}, None, False - failure, boundary, current, current_start = None, clock, None, clock - - def model_round(moment, item_type=None, text=None): - """Open or extend the span for the current model round. - - A round starts at the turn start or when the last running tool finished - and lasts until the model starts a tool or gives its final message. - Messages and reasoning inside one round belong to the same model call. - """ - nonlocal current, current_start - if current is None: - current_start = boundary - current = AgentEvent( - event_id="evt_" + uuid.uuid4().hex, - run_id=run_id, - parent_id=root_id, - event_type="model_call", - name="codex_model_round", - model=model, - started_at=at(boundary), - ended_at=at(moment), - duration_ms=(moment - boundary) * 1000, - token_provenance=UNAVAILABLE, - metadata={"codex_items": []}, - ) - events.append(current) - else: - current.ended_at = at(moment) - current.duration_ms = (moment - current_start) * 1000 - if item_type is not None: - current.metadata["codex_items"].append(item_type) - if text is not None: - current.output_text = clip(text) - - for moment, raw in lines: - try: - event = json.loads(raw) - except ValueError: - continue - kind, item = event.get("type"), event.get("item") or {} - item_type = item.get("type") - if kind == "turn.started": - boundary, current = moment, None - elif kind == "item.started" and item_type in TOOL_ITEMS: - if not open_tools and (current is not None or moment > boundary): - model_round(moment, "tool_call") - # The round ends when its tool calls start running. - current = None - open_tools[item.get("id")] = moment - elif kind == "item.completed" and item_type in TOOL_ITEMS: - # Some tool items arrive only on completion; either way the model - # round that requested them has ended. - current = None - started = open_tools.pop(item.get("id"), moment) - exit_code = item.get("exit_code") - failed = item.get("status") == "failed" or exit_code not in (None, 0) - events.append( - AgentEvent( - event_id="evt_" + uuid.uuid4().hex, - run_id=run_id, - parent_id=root_id, - event_type="tool_call", - name="shell" if item_type == "command_execution" else item_type, - started_at=at(started), - ended_at=at(moment), - duration_ms=(moment - started) * 1000, - input_text=clip(item.get("command") or json.dumps(item.get("changes"))), - output_text=clip(item.get("aggregated_output")), - status="error" if failed else "ok", - error=f"exit code {exit_code}" if failed else None, - metadata={"codex_item_type": item_type, "exit_code": exit_code}, - ) - ) - boundary = max(boundary, moment) - elif kind == "item.completed" and item_type in MODEL_ITEMS: - if open_tools: - # Reported while tools still run: keep the round open after them. - boundary = max([boundary, *open_tools.values()]) - model_round(moment, item_type, item.get("text")) - if item_type == "agent_message": - output = item.get("text") - elif kind == "turn.completed": - usage = event.get("usage") or {} - if current is None and not any(item.event_type == "model_call" for item in events): - model_round(moment) - models = [item for item in events if item.event_type == "model_call"] - last = models[-1] - last.input_tokens = int(usage.get("input_tokens") or 0) - last.output_tokens = int(usage.get("output_tokens") or 0) - last.token_provenance = PROVIDER - last.metadata.update( - usage_scope="turn_total", - cached_input_tokens=int(usage.get("cached_input_tokens") or 0), - reasoning_output_tokens=int(usage.get("reasoning_output_tokens") or 0), - ) - usage_seen = True - elif kind in {"turn.failed", "error"}: - detail = event.get("error") or event.get("message") or kind - failure = detail.get("message", str(detail)) if isinstance(detail, dict) else detail - return events, output, usage_seen, failure - - -def run(args, codex_args, request): - codex = shutil.which(args.codex) - if codex is None: - raise SystemExit(f"Cannot find the Codex CLI executable {args.codex!r}") - instructions = args.instructions.read_text(encoding="utf-8") if args.instructions else "" - prompt = task_prompt(request["input"], instructions) - command = [ - codex, - "exec", - "--json", - "--ephemeral", - "--skip-git-repo-check", - "--sandbox", - "read-only", - "--cd", - str(args.cd.resolve()), - *codex_args, - "-", - ] - trace = AgentTrace( - f"codex exec: {request['task_id']}", - metadata={ - "integration": "codex_exec", - "condition": args.condition, - "task_id": request["task_id"], - "repetition": request["repetition"], - "codex_args": codex_args, - }, - ) - root_id = "evt_" + uuid.uuid4().hex - origin, clock = datetime.now(timezone.utc), time.perf_counter() - process = subprocess.Popen( # nosec B603 - explicit Codex argv, no shell - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - encoding="utf-8", - errors="replace", - ) - lines = [] - reader = threading.Thread(target=read_stream, args=(process, lines), daemon=True) - reader.start() - stderr = threading.Thread(target=lambda: process.stderr.read(), daemon=True) - stderr.start() - process.stdin.write(prompt) - process.stdin.close() - returncode = process.wait() - reader.join() - ended = time.perf_counter() - events, output, usage_seen, failure = convert( - lines, run_id=trace.run_id, root_id=root_id, origin=origin, clock=clock, model=args.model - ) - if returncode != 0 and failure is None: - failure = f"codex exec exited with code {returncode}" - trace.add_event( - AgentEvent( - event_id=root_id, - run_id=trace.run_id, - event_type="agent_step", - name="codex_exec", - started_at=origin.isoformat(), - ended_at=(origin + timedelta(seconds=ended - clock)).isoformat(), - duration_ms=(ended - clock) * 1000, - input_text=prompt, - output_text=clip(output), - status="error" if failure else "ok", - error=failure, - metadata={"operation_kind": "agent", "usage_reported": usage_seen}, - ) - ) - for event in events: - trace.add_event(event) - trace.metadata.update( - output=(output or "").strip() if output is not None else None, - success=failure is None and output is not None, - ) - trace.finish() - trace.export_json(request["trace_path"]) - return 1 if failure else 0 - - -def main(): - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument("--cd", type=Path, default=Path("."), help="Directory Codex reads.") - parser.add_argument("--codex", default="codex", help="Codex CLI executable.") - parser.add_argument("--model", default=None, help="Model label recorded in the trace.") - parser.add_argument("--condition", default="unlabelled", help="Label recorded in the trace.") - parser.add_argument( - "--instructions", type=Path, default=None, help="Text file prepended to each task prompt." - ) - argv = sys.argv[1:] - split = argv.index("--") if "--" in argv else len(argv) - args = parser.parse_args(argv[:split]) - codex_args = argv[split + 1 :] - if args.model: - codex_args = ["--model", args.model, *codex_args] - request = json.loads(sys.stdin.readline()) - raise SystemExit(run(args, codex_args, request)) +from agentloop.runners.codex import convert, main, task_prompt +__all__ = ["convert", "main", "task_prompt"] if __name__ == "__main__": main() diff --git a/scripts/smoke_installed_wheel.py b/scripts/smoke_installed_wheel.py index f58eb5d..cc6a778 100644 --- a/scripts/smoke_installed_wheel.py +++ b/scripts/smoke_installed_wheel.py @@ -90,6 +90,9 @@ def cli(*args: str) -> None: run("-c", "from agentloop.entrypoint import app; app()", *args) run(str(Path(__file__).resolve().parent / "smoke_guided_wheel.py")) + # The coding-agent runners ship in the wheel and start without a checkout. + for runner in ("agentloop.runners.codex", "agentloop.runners.claude_code"): + run("-m", runner, "--help") fixture = external_fixtures / "harbor" / "atif_v18_multimodal.json" cli("harbor", "import-atif", str(fixture), "--out", "atif", "--synthetic") diff --git a/tests/test_claude_code_runner.py b/tests/test_claude_code_runner.py index bfdc665..e94d839 100644 --- a/tests/test_claude_code_runner.py +++ b/tests/test_claude_code_runner.py @@ -11,8 +11,8 @@ from agentloop.events import AgentEvent from agentloop.findings import build_diagnosis +from agentloop.runners.claude_code import convert, task_prompt from agentloop.tracer import AgentTrace -from examples.claude_code_runner import convert, task_prompt USAGE = { "input_tokens": 12, @@ -243,7 +243,8 @@ def test_runner_drives_the_cli_and_writes_a_successful_trace(tmp_path): completed = subprocess.run( # noqa: S603 [ sys.executable, - "examples/claude_code_runner.py", + "-m", + "agentloop.runners.claude_code", "--claude", str(executable), "--cd", @@ -285,3 +286,20 @@ def test_runner_drives_the_cli_and_writes_a_successful_trace(tmp_path): assert trace.metadata["integration"] == "claude_code" kinds = [event.event_type for event in trace.events] assert kinds.count("model_call") == 2 and kinds.count("tool_call") == 1 + + +@pytest.mark.parametrize( + ("module", "example", "option"), + [ + ("agentloop.runners.codex", "examples/codex_exec_runner.py", "--codex"), + ("agentloop.runners.claude_code", "examples/claude_code_runner.py", "--claude"), + ], +) +def test_packaged_runners_and_example_scripts_share_one_entry_point(module, example, option): + for command in ([sys.executable, "-m", module], [sys.executable, example]): + completed = subprocess.run( # noqa: S603 + [*command, "--help"], capture_output=True, text=True, timeout=60, check=False + ) + assert completed.returncode == 0, completed.stderr + assert option in completed.stdout + assert f"python -m {module}" in completed.stdout diff --git a/tests/test_codex_exec_runner.py b/tests/test_codex_exec_runner.py index de1f8da..03350b5 100644 --- a/tests/test_codex_exec_runner.py +++ b/tests/test_codex_exec_runner.py @@ -7,8 +7,8 @@ from agentloop.events import AgentEvent from agentloop.findings import build_diagnosis +from agentloop.runners.codex import convert, task_prompt from agentloop.tracer import AgentTrace -from examples.codex_exec_runner import convert, task_prompt # Shape recorded from `codex exec --json` (Codex CLI 0.167.1), shortened. STREAM = [