Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,10 @@ trace, study, receipt, API and database schema versions.
(`agentloop_runner.mjs`) for the guided workflow. It imports an agent from a
relative `.mjs`, `.js` or `.cjs` module, calls it as `answer(input, trace)`
with a TypeScript SDK trace, and writes a failed trace when the agent throws.
Without `--agent` it creates an offline example and a `package.json`. CI runs
the complete guided workflow against the packed SDK on Node.js 22 and 24.
It says how to install the npm package when it is missing and names the cause
when the agent returns before a traced call finishes. Without `--agent` it
creates an offline example and a `package.json`. CI runs the complete guided
workflow against the packed SDK on Node.js 22 and 24.
- Evidence-strength notes cover the cost objective. `improve start` and
`improve compare` flag a noisy baseline when estimated cost of the same task
varies between repetitions by at least 10% and more than the required
Expand Down
2 changes: 1 addition & 1 deletion agentloop/improvement_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def init_command(
check = (
"npm install, then node agentloop_runner.mjs --check"
if language == "node" and "package.json" in files
else "node agentloop_runner.mjs --check"
else "npm install agentloop-profiler, then node agentloop_runner.mjs --check"
if language == "node"
else "python agentloop_runner.py --check"
)
Expand Down
30 changes: 25 additions & 5 deletions agentloop/improvement_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,6 @@ def main():
NODE_RUNNER = Template("""// Adapter for $target_label. Inspect this file before running it.
// Requires Node.js 22 or newer and the agentloop-profiler npm package.
import { writeFile } from "node:fs/promises";
import { AgentTrace } from "agentloop-profiler";

const MODULE = $module;
const EXPORT = $export;
Expand All @@ -145,6 +144,18 @@ def main():
}

async function main() {
let AgentTrace;
try {
// Imported here so a missing package gets an actionable message.
({ AgentTrace } = await import("agentloop-profiler"));
} catch (error) {
console.error(
`Cannot load the agentloop-profiler npm package ($${error?.name ?? "Error"}). ` +
"Run npm install agentloop-profiler in this project.",
);
if (options.has("--debug")) console.error(error);
return 2;
}
let answer;
try {
const connected = await import(new URL(MODULE, import.meta.url).href);
Expand All @@ -153,7 +164,7 @@ def main():
} catch (error) {
console.error(
`Cannot connect $${TARGET} ($${error?.name ?? "Error"}). ` +
"Install agentloop-profiler with npm and check the module path in this runner.",
"Check the module path and export name in this runner.",
);
if (options.has("--debug")) console.error(error);
return 2;
Expand Down Expand Up @@ -183,7 +194,16 @@ def main():
trace.setMetadata("success", false);
trace.markOutputUnavailable();
}
trace.finish();
try {
trace.finish();
} catch (error) {
console.error(
"The agent returned while a traced call was still running. " +
"Await every trace.tool(...) and trace.model(...) call before returning.",
);
if (options.has("--debug")) console.error(error);
return 1;
}
await writeFile(request.trace_path, trace.toJSONString());
if (failure !== null) {
console.error(
Expand Down Expand Up @@ -255,10 +275,10 @@ def _node_files(root, target, name):
not module.startswith("/") and not re.match(r"^[A-Za-z]:", module),
"Node agent module must be a relative path inside the project, such as ./src/agent.mjs",
)
if not module.startswith(("./", "../")):
if not module.startswith("./"):
module = "./" + module
require(
bool(NODE_MODULE.match(module)) and ".." not in module.split("/")[1:],
bool(NODE_MODULE.match(module)) and ".." not in module.split("/"),
"Node agent module must be a relative .mjs, .js or .cjs path inside the project, "
"such as ./src/agent.mjs:answer",
)
Expand Down
21 changes: 21 additions & 0 deletions scripts/smoke_node_guided.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,11 +144,32 @@ def smoke(sdk: Path, work: Path) -> None:
],
broken,
)
# Before the SDK is installed, --check says how to install it.
result = run(["node", "agentloop_runner.mjs", "--check"], broken, expect=2)
if "npm install agentloop-profiler" not in result.stderr:
raise SystemExit("A missing SDK package must be reported by --check")
run(["npm", "install", "--no-audit", "--no-fund", str(sdk)], broken)
result = run(["node", "agentloop_runner.mjs", "--check"], broken, expect=2)
if "Cannot connect ./agent.mjs:answer" not in result.stderr:
raise SystemExit("A missing export must be reported by --check")

# An agent that returns before its traced call finishes gets a clear error.
(broken / "agent.mjs").write_text(
"export async function answer(input, trace) {\n"
' trace.tool("slow", () => new Promise((resolve) => setTimeout(resolve, 200)));\n'
" return { ok: true };\n"
"}\n",
encoding="utf-8",
)
result = run(
["node", "agentloop_runner.mjs"],
broken,
expect=1,
stdin=json.dumps({"input": {}, "trace_path": "unawaited.json"}),
)
if "Await every trace.tool" not in result.stderr:
raise SystemExit("An unawaited traced call must be reported")


def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
Expand Down
17 changes: 16 additions & 1 deletion tests/test_improvement_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,9 @@ def test_node_init_generates_an_esm_runner_example_and_package(tmp_path):
"AGENTLOOP.md",
}
runner = (root / "agentloop_runner.mjs").read_text(encoding="utf-8")
assert 'import { AgentTrace } from "agentloop-profiler";' in runner
# Loaded inside main() so a missing package gets an actionable message.
assert '({ AgentTrace } = await import("agentloop-profiler"));' in runner
assert 'from "agentloop-profiler"' not in runner
assert 'const MODULE = "./agentloop_example.mjs";' in runner
assert 'const EXPORT = "answer";' in runner
assert '"synthetic": true' in runner
Expand Down Expand Up @@ -315,6 +317,8 @@ def test_node_init_connects_an_existing_module_without_overwriting_the_project(t
"/abs/agent.mjs:answer",
"C:/abs/agent.mjs:answer",
"./a/../../agent.mjs:answer",
"../agent.mjs:answer",
"src/../../agent.mjs:answer",
"./agent.mjs:not-valid",
"./agent.mjs:class",
"agent.mjs",
Expand All @@ -337,3 +341,14 @@ def test_node_init_cli_points_to_npm_install(tmp_path):
)
assert result.exit_code == 0, result.output
assert "npm install, then node agentloop_runner.mjs --check" in unstyle(result.output)
(tmp_path / "app").mkdir()
(tmp_path / "app" / "agent.mjs").write_text(
"export const answer = () => 1;\n", encoding="utf-8"
)
existing = CliRunner().invoke(
app,
["improve", "init", "--language", "node", "--agent", "agent.mjs:answer"]
+ ["--out", str(tmp_path / "app")],
)
assert existing.exit_code == 0, existing.output
assert "npm install agentloop-profiler, then node" in unstyle(existing.output)
Loading