diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f76dab2..e891470 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,6 +181,12 @@ jobs: working-directory: sdk/typescript - name: Installed artifact interoperability run: uv run --frozen --extra server python scripts/smoke_typescript_sdk.py --module "$RUNNER_TEMP/agentloop-sdk-installed/node_modules/agentloop-profiler/dist/index.js" --out runs/typescript-packed-smoke + - name: Node.js guided workflow from improve init + shell: bash + run: | + package=$(find "$RUNNER_TEMP/agentloop-sdk-package" -name '*.tgz' -print -quit) + test -n "$package" + uv run --frozen python scripts/smoke_node_guided.py --sdk "$package" - name: Upload SDK package if: ${{ inputs.upload_release_artifact || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index ae4f360..211c328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ trace, study, receipt, API and database schema versions. ## [Unreleased] +### Added + +- `agentloop improve init --language node` generates a Node.js adapter + (`agentloop_runner.mjs`) for the guided workflow. It imports an agent from a + relative `.mjs`, `.js` or `.cjs` module, calls it as `answer(input, trace)` + with a TypeScript SDK trace, and writes a failed trace when the agent throws. + Without `--agent` it creates an offline example and a `package.json`. CI runs + the complete guided workflow against the packed SDK on Node.js 22 and 24. + ### Documentation - The README has an Install section covering the PyPI package, the npm diff --git a/README.md b/README.md index b5299d0..2db2d8d 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,9 @@ for those agents. To connect your application, run `agentloop improve init --agent my_agent:answer` in its project directory. The generated adapter supports synchronous and async -Python callables. Replace the example task inputs and expected outputs with +Python callables. For a Node.js agent, run +`agentloop improve init --language node --agent ./src/agent.mjs:answer`; see +[connecting a Node.js agent](https://dipeshbabu.github.io/agentloop/GUIDED_OPTIMIZATION/#connect-an-existing-nodejs-agent). Replace the example task inputs and expected outputs with independent checks for your agent, and keep your model and tool instrumentation. Initialization refuses to overwrite existing files. Read the generated `AGENTLOOP.md` and [onboarding guide](https://dipeshbabu.github.io/agentloop/GUIDED_OPTIMIZATION/). diff --git a/agentloop/improvement_cli.py b/agentloop/improvement_cli.py index 6d04b77..3e3e982 100644 --- a/agentloop/improvement_cli.py +++ b/agentloop/improvement_cli.py @@ -32,19 +32,30 @@ def tasks_path(tasks: Path | None, project: Path) -> Path: @improve_app.command("init") def init_command( out: Path = typer.Option(Path("."), help="Existing project directory or a new directory."), - agent: str | None = typer.Option(None, help="Existing Python package.module:callable."), + agent: str | None = typer.Option( + None, + help="Existing agent: a Python package.module:callable, or with --language node " + "a relative ./module.mjs:exportName.", + ), name: str | None = typer.Option(None, help="Name shown in the trace and reports."), + language: str = typer.Option( + "python", help="Adapter language: python, or node for Node.js 22+ agents." + ), ): - """Create task data, a Python adapter, and an executable offline example.""" + """Create task data, a Python or Node.js adapter, and an executable offline example.""" try: - files = initialize(out, target=agent, name=name) + files = initialize(out, target=agent, name=name, language=language) except (OSError, ValueError, TypeError) as exc: raise typer.BadParameter(str(exc)) from exc console.print(f"Created {len(files)} files in {out.resolve()}", markup=False) - console.print( - "Read AGENTLOOP.md, then run python agentloop_runner.py --check from that directory.", - markup=False, + check = ( + "npm install, then node agentloop_runner.mjs --check" + if language == "node" and "package.json" in files + else "node agentloop_runner.mjs --check" + if language == "node" + else "python agentloop_runner.py --check" ) + console.print(f"Read AGENTLOOP.md, then run {check} from that directory.", markup=False) @improve_app.command("capture") diff --git a/agentloop/improvement_init.py b/agentloop/improvement_init.py index 8993c57..38285f3 100644 --- a/agentloop/improvement_init.py +++ b/agentloop/improvement_init.py @@ -1,9 +1,10 @@ -"""Create a small reviewed Python adapter and data-only guided project config.""" +"""Create a small reviewed Python or Node.js adapter and data-only guided project config.""" from __future__ import annotations import json import keyword +import re from pathlib import Path from string import Template @@ -100,6 +101,241 @@ def main(): ''') +NODE_EXAMPLE = """// Offline tool example. Replace answer() with your existing agent. +import { pbkdf2Sync } from "node:crypto"; + +const NOTES = { tracing: "Record each operation.", quality: "Check the final answer." }; + +function lookupNote(trace, key) { + return trace.tool( + "lookup_note", + () => { + // Local CPU work makes repeated execution visible without a model API. + pbkdf2Sync(key, "agentloop-example", 40000, 32, "sha256"); + return NOTES[key]; + }, + { metadata: { parallel_safe: true } }, + ); +} + +export async function answer(payload, trace) { + const key = payload.key; + // Review the repeated calls, then reuse one lookup result for this request. + const notes = []; + for (let index = 0; index < 3; index++) notes.push(await lookupNote(trace, key)); + return { notes }; +} +""" + +NODE_RUNNER = Template("""// Adapter for $target_label. Inspect this file before running it. +// Requires Node.js 22 or newer and the agentloop-profiler npm package. +import { writeFile } from "node:fs/promises"; +import { AgentTrace } from "agentloop-profiler"; + +const MODULE = $module; +const EXPORT = $export; +const TARGET = $target_json; +const options = new Set(process.argv.slice(2)); + +async function readStdin() { + let text = ""; + process.stdin.setEncoding("utf8"); + for await (const chunk of process.stdin) text += chunk; + return text; +} + +async function main() { + let answer; + try { + const connected = await import(new URL(MODULE, import.meta.url).href); + answer = connected[EXPORT]; + if (typeof answer !== "function") throw new TypeError("The selected export is not a function"); + } catch (error) { + console.error( + `Cannot connect $${TARGET} ($${error?.name ?? "Error"}). ` + + "Install agentloop-profiler with npm and check the module path in this runner.", + ); + if (options.has("--debug")) console.error(error); + return 2; + } + if (options.has("--check")) { + console.log(`Connected $${TARGET}`); + return 0; + } + let request; + try { + request = JSON.parse(await readStdin()); + if (typeof request.trace_path !== "string" || !("input" in request)) throw new TypeError(); + } catch { + console.error( + "Expected an improve capture request on stdin. " + + "Run agentloop improve capture -- node agentloop_runner.mjs.", + ); + return 2; + } + const trace = new AgentTrace($trace_name, { metadata: $metadata }); + let failure = null; + try { + // The agent receives the task input and this trace for recording its spans. + trace.setOutput(await answer(request.input, trace)); + } catch (error) { + failure = error; + trace.setMetadata("success", false); + trace.markOutputUnavailable(); + } + trace.finish(); + await writeFile(request.trace_path, trace.toJSONString()); + if (failure !== null) { + console.error( + `Agent call failed ($${failure?.name ?? "Error"}). Return finite JSON from $${TARGET}. ` + + "Use --show-runner-output with this runner's --debug for details.", + ); + if (options.has("--debug")) console.error(failure); + return 1; + } + return 0; +} + +process.exitCode = await main(); +""") + +NODE_PACKAGE = """{ + "name": "agentloop-node-example", + "private": true, + "type": "module", + "dependencies": { + "agentloop-profiler": "^0.1.2" + } +} +""" + +NODE_MODULE = re.compile(r"^\.{1,2}/[^:\\]+\.(?:mjs|js|cjs)$") + + +def _project_name(root, name): + name = name or root.name + require(isinstance(name, str) and bool(name.strip()), "Project name must be nonempty") + return name + + +def _shared_files(name): + return { + "agentloop.toml": 'schema_version = "1.0"\ntasks = "agentloop_tasks.json"\n', + "agentloop_tasks.json": json.dumps( + { + "schema_version": "1.0", + "name": name, + "repetitions": 1, + "tasks": [ + { + "id": key, + "input": {"key": key}, + "expected": {"notes": [value] * 3}, + "scorer": {"type": "json_subset"}, + } + for key, value in { + "tracing": "Record each operation.", + "quality": "Check the final answer.", + }.items() + ], + }, + indent=2, + ) + + "\n", + } + + +def _node_files(root, target, name): + demo = target is None + target = target or "./agentloop_example.mjs:answer" + require(":" in target, "Use --agent ./path/to/module.mjs:exportName with --language node") + module, export = target.rsplit(":", 1) + module = module.replace("\\", "/") + require( + not module.startswith("/") and not re.match(r"^[A-Za-z]:", module), + "Node agent module must be a relative path inside the project, such as ./src/agent.mjs", + ) + if not module.startswith(("./", "../")): + module = "./" + module + require( + bool(NODE_MODULE.match(module)) and ".." not in module.split("/")[1:], + "Node agent module must be a relative .mjs, .js or .cjs path inside the project, " + "such as ./src/agent.mjs:answer", + ) + require( + export.isidentifier() and not keyword.iskeyword(export), + "Node agent export must be a JavaScript identifier", + ) + target = f"{module}:{export}" + name = _project_name(root, name) + metadata = ( + {"synthetic": True, "source": "improve_init_offline_example"} + if demo + else {"source": "improve_init_node_adapter"} + ) + files = { + **_shared_files(name), + "agentloop_runner.mjs": NODE_RUNNER.substitute( + module=json.dumps(module), + export=json.dumps(export), + target_json=json.dumps(target), + target_label=target, + trace_name=json.dumps(name), + metadata=json.dumps(metadata), + ), + } + if demo: + files["agentloop_example.mjs"] = NODE_EXAMPLE + files["package.json"] = NODE_PACKAGE + install = "npm install" if demo else "npm install agentloop-profiler" + files["AGENTLOOP.md"] = f"""# Optimize {name} + +The included task examples describe the offline tool example. For your existing +agent, replace their inputs and expected answers with independent correctness +criteria before capturing a baseline. The adapter calls `{export}` from +`{module}` as `{export}(input, trace)` with one JSON input value and records its +final JSON output. Record model and tool calls with `trace.model(...)` and +`trace.tool(...)` so they appear in the trace. + +Run from this project directory with Node.js 22 or newer. The runner needs the +`agentloop-profiler` npm package; the `agentloop` command comes from the Python +package (`python -m pip install agentloop-profiler`). + +```bash +{install} +node agentloop_runner.mjs --check +agentloop improve capture --out baseline -- node agentloop_runner.mjs +agentloop improve start --baseline baseline/capture.json --out optimization +``` + +Review `optimization/plan.md` and the original evidence. For the included example, +edit `answer()` to reuse one lookup result within the request. + +```javascript +const value = await lookupNote(trace, key); +return {{ notes: [value, value, value] }}; +``` + +For your own agent, review and apply a change appropriate to its evidence. + +```bash +agentloop improve capture --out candidate -- node agentloop_runner.mjs +agentloop improve finish --session optimization/session.json --candidate candidate/capture.json --change "Reuse the lookup result" --out results +``` + +Open `results/report.html`. A rejected comparison returns exit code 1 and retains +its report. To inspect an agent failure, use `--show-runner-output` on capture +and pass `--debug` after the runner filename. Traces retain final outputs for +quality verification. Review artifacts before sharing them. + +The offline example executes real local CPU work on fictional notes and is marked +synthetic. Its timings illustrate this example. Your agent needs representative +tasks and quality checks. The project config selects task data; the argv after +`--` selects executable code. +""" + return files + + def project_tasks(project: Path) -> Path: """Resolve task data from an explicitly selected project configuration.""" import tomllib @@ -122,8 +358,13 @@ def project_tasks(project: Path) -> Path: return project.resolve().parent / config["tasks"] -def initialize(root: Path, *, target: str | None = None, name: str | None = None): +def initialize( + root: Path, *, target: str | None = None, name: str | None = None, language: str = "python" +): + require(language in {"python", "node"}, "Language must be python or node") root = root.resolve() + if language == "node": + return _write(root, _node_files(root, target, name)) demo = target is None target = target or "agentloop_example:answer" require(target.count(":") == 1, "Use --agent package.module:callable") @@ -136,31 +377,9 @@ def initialize(root: Path, *, target: str | None = None, name: str | None = None ), "Agent module and callable must be Python identifiers separated by dots", ) - name = name or root.name - require(isinstance(name, str) and bool(name.strip()), "Project name must be nonempty") + name = _project_name(root, name) files = { - "agentloop.toml": 'schema_version = "1.0"\ntasks = "agentloop_tasks.json"\n', - "agentloop_tasks.json": json.dumps( - { - "schema_version": "1.0", - "name": name, - "repetitions": 1, - "tasks": [ - { - "id": key, - "input": {"key": key}, - "expected": {"notes": [value] * 3}, - "scorer": {"type": "json_subset"}, - } - for key, value in { - "tracing": "Record each operation.", - "quality": "Check the final answer.", - }.items() - ], - }, - indent=2, - ) - + "\n", + **_shared_files(name), "agentloop_runner.py": RUNNER.substitute( module=module, function=function, @@ -214,6 +433,10 @@ def initialize(root: Path, *, target: str | None = None, name: str | None = None tasks and quality checks. The project config selects task data; the argv after `--` selects executable code. """ + return _write(root, files) + + +def _write(root, files): require(root.is_dir() or not root.exists(), "--out must name a project directory") collisions = [ str(root / filename) diff --git a/docs/GUIDED_OPTIMIZATION.md b/docs/GUIDED_OPTIMIZATION.md index 487e9a6..173541a 100644 --- a/docs/GUIDED_OPTIMIZATION.md +++ b/docs/GUIDED_OPTIMIZATION.md @@ -76,6 +76,35 @@ checking the environment and runner import. A missing trace suggests exporting the finalized trace to `request["trace_path"]`. A timeout suggests inspecting the agent before increasing `--timeout`. +## Connect an existing Node.js agent + +AgentLoop 0.2.7 and newer generate a Node.js adapter for agents written in +JavaScript or compiled TypeScript. It needs Node.js 22 or newer, the +`agentloop-profiler` npm package for the runner, and the Python package for the +`agentloop` command. From your project directory, run: + +```bash +npm install agentloop-profiler +agentloop improve init --language node --agent ./src/agent.mjs:answer +node agentloop_runner.mjs --check +agentloop improve capture --out baseline -- node agentloop_runner.mjs +``` + +`--agent` names a relative `.mjs`, `.js` or `.cjs` module inside the project and +its exported function. The runner calls it as `answer(input, trace)`, where +`trace` is the [TypeScript SDK](../sdk/typescript/README.md) trace for this +attempt. Wrap model and tool calls with `trace.model(...)` and `trace.tool(...)` +so repeated or slow calls appear as spans, and return finite JSON. If the +function throws or returns non-JSON, the runner still writes a failed trace and +capture reports the task as failed. Use `--debug` after the runner filename, with +`--show-runner-output` on capture, to print the error. + +Without `--agent`, `--language node` creates the offline example: an +`agentloop_example.mjs` that repeats a traced lookup and a `package.json` that +depends on `agentloop-profiler`. Run `npm install` in that directory first. The +same capture, start and finish commands then apply, with +`node agentloop_runner.mjs` as the runner. + ## Define tasks and correctness first Save a workload such as this in `tasks.json`. Choose expected values independently diff --git a/scripts/smoke_node_guided.py b/scripts/smoke_node_guided.py new file mode 100644 index 0000000..60e4767 --- /dev/null +++ b/scripts/smoke_node_guided.py @@ -0,0 +1,171 @@ +"""Run the Node.js guided workflow generated by ``improve init --language node``. + +The check installs a TypeScript SDK package (a packed ``.tgz`` or the built +``sdk/typescript`` directory) into a fresh generated project, then captures a +baseline, freezes a session, applies the documented change, and requires the +comparison to pass. It also checks that a failing agent is retained as a failed +attempt and that ``--check`` reports a missing export. +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import subprocess # nosec B404 - fixed Node, npm and Python argv, no shell +import sys +from pathlib import Path +from tempfile import TemporaryDirectory + +ROOT = Path(__file__).resolve().parents[1] +CLI = [sys.executable, "-c", "from agentloop.entrypoint import app; app()"] +CHANGE = ( + """ const notes = []; + for (let index = 0; index < 3; index++) notes.push(await lookupNote(trace, key)); + return { notes };""", + """ const value = await lookupNote(trace, key); + return { notes: [value, value, value] };""", +) + + +def run(argv, cwd, *, expect=0, stdin=None): + executable = shutil.which(argv[0]) or argv[0] + completed = subprocess.run( # nosec B603 - fixed tools and generated local files + [executable, *argv[1:]], + cwd=cwd, + input=stdin, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=600, + check=False, + ) + if completed.returncode != expect: + raise SystemExit( + f"{' '.join(argv)} exited {completed.returncode}, expected {expect}\n" + f"{completed.stdout}\n{completed.stderr}" + ) + return completed + + +def smoke(sdk: Path, work: Path) -> None: + project = work / "node-demo" + run([*CLI, "improve", "init", "--language", "node", "--out", str(project)], work) + run(["npm", "install", "--no-audit", "--no-fund", str(sdk)], project) + run(["node", "agentloop_runner.mjs", "--check"], project) + run( + [*CLI, "improve", "capture", "--out", "baseline", "--", "node", "agentloop_runner.mjs"], + project, + ) + run( + [*CLI, "improve", "start", "--baseline", "baseline/capture.json", "--out", "optimization"], + project, + ) + session = json.loads((project / "optimization/session.json").read_text(encoding="utf-8")) + cases = session["specification"]["cases"] + if not all(case["capture_status"] == "completed" for case in cases): + raise SystemExit("Every Node baseline attempt must complete") + example = project / "agentloop_example.mjs" + source = example.read_text(encoding="utf-8") + if source.count(CHANGE[0]) != 1: + raise SystemExit("The generated example no longer contains the documented change") + example.write_text(source.replace(*CHANGE), encoding="utf-8") + run( + [*CLI, "improve", "capture", "--out", "candidate", "--", "node", "agentloop_runner.mjs"], + project, + ) + run( + [ + *CLI, + "improve", + "finish", + "--session", + "optimization/session.json", + "--candidate", + "candidate/capture.json", + "--change", + "Reuse the lookup result", + "--out", + "results", + ], + project, + ) + report = json.loads((project / "results/report.json").read_text(encoding="utf-8")) + if report["passed"] is not True: + raise SystemExit("The documented Node change must pass the frozen checks") + + # A connected agent that throws is retained as a failed attempt. + failing = work / "node-failing" + failing.mkdir() + (failing / "agent.mjs").write_text( + 'export async function answer() { throw new Error("synthetic failure"); }\n', + encoding="utf-8", + ) + run( + [ + *CLI, + "improve", + "init", + "--language", + "node", + "--agent", + "agent.mjs:answer", + "--out", + ".", + ], + failing, + ) + run(["npm", "install", "--no-audit", "--no-fund", str(sdk)], failing) + run( + [*CLI, "improve", "capture", "--out", "baseline", "--", "node", "agentloop_runner.mjs"], + failing, + expect=1, + ) + manifest = json.loads((failing / "baseline/capture.json").read_text(encoding="utf-8")) + if {row["status"] for row in manifest["runs"]} != {"failed"}: + raise SystemExit("A throwing Node agent must be recorded as failed attempts") + + # A missing export is reported by --check before any task runs. + broken = work / "node-missing-export" + broken.mkdir() + (broken / "agent.mjs").write_text("export const other = 1;\n", encoding="utf-8") + run( + [ + *CLI, + "improve", + "init", + "--language", + "node", + "--agent", + "./agent.mjs:answer", + "--out", + ".", + ], + broken, + ) + run(["npm", "install", "--no-audit", "--no-fund", str(sdk)], broken) + result = run(["node", "agentloop_runner.mjs", "--check"], broken, expect=2) + if "Cannot connect ./agent.mjs:answer" not in result.stderr: + raise SystemExit("A missing export must be reported by --check") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--sdk", + type=Path, + default=ROOT / "sdk/typescript", + help="Packed SDK .tgz or the built sdk/typescript directory.", + ) + args = parser.parse_args() + sdk = args.sdk.resolve() + if not sdk.exists(): + raise SystemExit(f"SDK package not found: {sdk}") + with TemporaryDirectory(prefix="agentloop-node-guided-") as temporary: + smoke(sdk, Path(temporary)) + print("Node guided workflow smoke passed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_improvement_init.py b/tests/test_improvement_init.py index 04fafae..c947978 100644 --- a/tests/test_improvement_init.py +++ b/tests/test_improvement_init.py @@ -264,3 +264,76 @@ def opened(path, *args, **kwargs): with pytest.raises(OSError, match="disk full"): initialize(tmp_path) assert [path.name for path in tmp_path.iterdir()] == ["owned.txt"] + + +def test_node_init_generates_an_esm_runner_example_and_package(tmp_path): + root = tmp_path / "node-project" + names = initialize(root, language="node") + assert set(names) == { + "agentloop.toml", + "agentloop_tasks.json", + "agentloop_runner.mjs", + "agentloop_example.mjs", + "package.json", + "AGENTLOOP.md", + } + runner = (root / "agentloop_runner.mjs").read_text(encoding="utf-8") + assert 'import { AgentTrace } from "agentloop-profiler";' in runner + assert 'const MODULE = "./agentloop_example.mjs";' in runner + assert 'const EXPORT = "answer";' in runner + assert '"synthetic": true' in runner + assert "$" not in runner.replace("${", "") + package = json.loads((root / "package.json").read_text(encoding="utf-8")) + assert package["type"] == "module" + assert "agentloop-profiler" in package["dependencies"] + guide = (root / "AGENTLOOP.md").read_text(encoding="utf-8") + assert "-- node agentloop_runner.mjs" in guide + assert "answer(input, trace)" in guide + tasks = json.loads((root / "agentloop_tasks.json").read_text(encoding="utf-8")) + assert [task["id"] for task in tasks["tasks"]] == ["tracing", "quality"] + + +def test_node_init_connects_an_existing_module_without_overwriting_the_project(tmp_path): + root = tmp_path / "app" + (root / "src").mkdir(parents=True) + (root / "package.json").write_text('{"name": "app"}\n', encoding="utf-8") + names = initialize(root, target=r"src\agent.js:runAgent", name="My agent", language="node") + assert "package.json" not in names and "agentloop_example.mjs" not in names + assert (root / "package.json").read_text(encoding="utf-8") == '{"name": "app"}\n' + runner = (root / "agentloop_runner.mjs").read_text(encoding="utf-8") + assert 'const MODULE = "./src/agent.js";' in runner + assert 'const EXPORT = "runAgent";' in runner + assert 'new AgentTrace("My agent"' in runner + assert '"source": "improve_init_node_adapter"' in runner + assert "npm install agentloop-profiler" in (root / "AGENTLOOP.md").read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + "target", + [ + "agent.ts:answer", + "/abs/agent.mjs:answer", + "C:/abs/agent.mjs:answer", + "./a/../../agent.mjs:answer", + "./agent.mjs:not-valid", + "./agent.mjs:class", + "agent.mjs", + ], +) +def test_node_init_rejects_targets_outside_a_relative_module_export(tmp_path, target): + with pytest.raises(ValueError): + initialize(tmp_path / "node", target=target, language="node") + assert not (tmp_path / "node").exists() + + +def test_init_rejects_unknown_languages(tmp_path): + with pytest.raises(ValueError, match="python or node"): + initialize(tmp_path / "x", language="ruby") + + +def test_node_init_cli_points_to_npm_install(tmp_path): + result = CliRunner().invoke( + app, ["improve", "init", "--language", "node", "--out", str(tmp_path / "demo")] + ) + assert result.exit_code == 0, result.output + assert "npm install, then node agentloop_runner.mjs --check" in unstyle(result.output)