Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,12 @@ jobs:
working-directory: sdk/typescript
- name: Installed artifact interoperability
run: uv run --frozen --extra server python scripts/smoke_typescript_sdk.py --module "$RUNNER_TEMP/agentloop-sdk-installed/node_modules/agentloop-profiler/dist/index.js" --out runs/typescript-packed-smoke
- name: Node.js guided workflow from improve init
shell: bash
run: |
package=$(find "$RUNNER_TEMP/agentloop-sdk-package" -name '*.tgz' -print -quit)
test -n "$package"
uv run --frozen python scripts/smoke_node_guided.py --sdk "$package"
- name: Upload SDK package
if: ${{ inputs.upload_release_artifact || github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
Expand Down
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ trace, study, receipt, API and database schema versions.

## [Unreleased]

### Added

- `agentloop improve init --language node` generates a Node.js adapter
(`agentloop_runner.mjs`) for the guided workflow. It imports an agent from a
relative `.mjs`, `.js` or `.cjs` module, calls it as `answer(input, trace)`
with a TypeScript SDK trace, and writes a failed trace when the agent throws.
Without `--agent` it creates an offline example and a `package.json`. CI runs
the complete guided workflow against the packed SDK on Node.js 22 and 24.

### Documentation

- The README has an Install section covering the PyPI package, the npm
Expand Down
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,9 @@ for those agents.

To connect your application, run `agentloop improve init --agent my_agent:answer`
in its project directory. The generated adapter supports synchronous and async
Python callables. Replace the example task inputs and expected outputs with
Python callables. For a Node.js agent, run
`agentloop improve init --language node --agent ./src/agent.mjs:answer`; see
[connecting a Node.js agent](https://dipeshbabu.github.io/agentloop/GUIDED_OPTIMIZATION/#connect-an-existing-nodejs-agent). Replace the example task inputs and expected outputs with
independent checks for your agent, and keep your model and tool instrumentation.
Initialization refuses to overwrite existing files. Read the generated
`AGENTLOOP.md` and [onboarding guide](https://dipeshbabu.github.io/agentloop/GUIDED_OPTIMIZATION/).
Expand Down
23 changes: 17 additions & 6 deletions agentloop/improvement_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,30 @@ def tasks_path(tasks: Path | None, project: Path) -> Path:
@improve_app.command("init")
def init_command(
out: Path = typer.Option(Path("."), help="Existing project directory or a new directory."),
agent: str | None = typer.Option(None, help="Existing Python package.module:callable."),
agent: str | None = typer.Option(
None,
help="Existing agent: a Python package.module:callable, or with --language node "
"a relative ./module.mjs:exportName.",
),
name: str | None = typer.Option(None, help="Name shown in the trace and reports."),
language: str = typer.Option(
"python", help="Adapter language: python, or node for Node.js 22+ agents."
),
):
"""Create task data, a Python adapter, and an executable offline example."""
"""Create task data, a Python or Node.js adapter, and an executable offline example."""
try:
files = initialize(out, target=agent, name=name)
files = initialize(out, target=agent, name=name, language=language)
except (OSError, ValueError, TypeError) as exc:
raise typer.BadParameter(str(exc)) from exc
console.print(f"Created {len(files)} files in {out.resolve()}", markup=False)
console.print(
"Read AGENTLOOP.md, then run python agentloop_runner.py --check from that directory.",
markup=False,
check = (
"npm install, then node agentloop_runner.mjs --check"
if language == "node" and "package.json" in files
else "node agentloop_runner.mjs --check"
if language == "node"
else "python agentloop_runner.py --check"
)
console.print(f"Read AGENTLOOP.md, then run {check} from that directory.", markup=False)


@improve_app.command("capture")
Expand Down
275 changes: 249 additions & 26 deletions agentloop/improvement_init.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
"""Create a small reviewed Python adapter and data-only guided project config."""
"""Create a small reviewed Python or Node.js adapter and data-only guided project config."""

from __future__ import annotations

import json
import keyword
import re
from pathlib import Path
from string import Template

Expand Down Expand Up @@ -100,6 +101,241 @@ def main():
''')


NODE_EXAMPLE = """// Offline tool example. Replace answer() with your existing agent.
import { pbkdf2Sync } from "node:crypto";

const NOTES = { tracing: "Record each operation.", quality: "Check the final answer." };

function lookupNote(trace, key) {
return trace.tool(
"lookup_note",
() => {
// Local CPU work makes repeated execution visible without a model API.
pbkdf2Sync(key, "agentloop-example", 40000, 32, "sha256");
return NOTES[key];
},
{ metadata: { parallel_safe: true } },
);
}

export async function answer(payload, trace) {
const key = payload.key;
// Review the repeated calls, then reuse one lookup result for this request.
const notes = [];
for (let index = 0; index < 3; index++) notes.push(await lookupNote(trace, key));
return { notes };
}
"""

NODE_RUNNER = Template("""// Adapter for $target_label. Inspect this file before running it.
// Requires Node.js 22 or newer and the agentloop-profiler npm package.
import { writeFile } from "node:fs/promises";
import { AgentTrace } from "agentloop-profiler";

const MODULE = $module;
const EXPORT = $export;
const TARGET = $target_json;
const options = new Set(process.argv.slice(2));

async function readStdin() {
let text = "";
process.stdin.setEncoding("utf8");
for await (const chunk of process.stdin) text += chunk;
return text;
}

async function main() {
let answer;
try {
const connected = await import(new URL(MODULE, import.meta.url).href);
answer = connected[EXPORT];
if (typeof answer !== "function") throw new TypeError("The selected export is not a function");
} catch (error) {
console.error(
`Cannot connect $${TARGET} ($${error?.name ?? "Error"}). ` +
"Install agentloop-profiler with npm and check the module path in this runner.",
);
if (options.has("--debug")) console.error(error);
return 2;
}
if (options.has("--check")) {
console.log(`Connected $${TARGET}`);
return 0;
}
let request;
try {
request = JSON.parse(await readStdin());
if (typeof request.trace_path !== "string" || !("input" in request)) throw new TypeError();
} catch {
console.error(
"Expected an improve capture request on stdin. " +
"Run agentloop improve capture -- node agentloop_runner.mjs.",
);
return 2;
}
const trace = new AgentTrace($trace_name, { metadata: $metadata });
let failure = null;
try {
// The agent receives the task input and this trace for recording its spans.
trace.setOutput(await answer(request.input, trace));
} catch (error) {
failure = error;
trace.setMetadata("success", false);
trace.markOutputUnavailable();
}
trace.finish();
await writeFile(request.trace_path, trace.toJSONString());
if (failure !== null) {
console.error(
`Agent call failed ($${failure?.name ?? "Error"}). Return finite JSON from $${TARGET}. ` +
"Use --show-runner-output with this runner's --debug for details.",
);
if (options.has("--debug")) console.error(failure);
return 1;
}
return 0;
}

process.exitCode = await main();
""")

NODE_PACKAGE = """{
"name": "agentloop-node-example",
"private": true,
"type": "module",
"dependencies": {
"agentloop-profiler": "^0.1.2"
}
}
"""

NODE_MODULE = re.compile(r"^\.{1,2}/[^:\\]+\.(?:mjs|js|cjs)$")


def _project_name(root, name):
name = name or root.name
require(isinstance(name, str) and bool(name.strip()), "Project name must be nonempty")
return name


def _shared_files(name):
return {
"agentloop.toml": 'schema_version = "1.0"\ntasks = "agentloop_tasks.json"\n',
"agentloop_tasks.json": json.dumps(
{
"schema_version": "1.0",
"name": name,
"repetitions": 1,
"tasks": [
{
"id": key,
"input": {"key": key},
"expected": {"notes": [value] * 3},
"scorer": {"type": "json_subset"},
}
for key, value in {
"tracing": "Record each operation.",
"quality": "Check the final answer.",
}.items()
],
},
indent=2,
)
+ "\n",
}


def _node_files(root, target, name):
demo = target is None
target = target or "./agentloop_example.mjs:answer"
require(":" in target, "Use --agent ./path/to/module.mjs:exportName with --language node")
module, export = target.rsplit(":", 1)
module = module.replace("\\", "/")
require(
not module.startswith("/") and not re.match(r"^[A-Za-z]:", module),
"Node agent module must be a relative path inside the project, such as ./src/agent.mjs",
)
if not module.startswith(("./", "../")):
module = "./" + module
require(
bool(NODE_MODULE.match(module)) and ".." not in module.split("/")[1:],
"Node agent module must be a relative .mjs, .js or .cjs path inside the project, "
"such as ./src/agent.mjs:answer",
)
require(
export.isidentifier() and not keyword.iskeyword(export),
"Node agent export must be a JavaScript identifier",
)
target = f"{module}:{export}"
name = _project_name(root, name)
metadata = (
{"synthetic": True, "source": "improve_init_offline_example"}
if demo
else {"source": "improve_init_node_adapter"}
)
files = {
**_shared_files(name),
"agentloop_runner.mjs": NODE_RUNNER.substitute(
module=json.dumps(module),
export=json.dumps(export),
target_json=json.dumps(target),
target_label=target,
trace_name=json.dumps(name),
metadata=json.dumps(metadata),
),
}
if demo:
files["agentloop_example.mjs"] = NODE_EXAMPLE
files["package.json"] = NODE_PACKAGE
install = "npm install" if demo else "npm install agentloop-profiler"
files["AGENTLOOP.md"] = f"""# Optimize {name}

The included task examples describe the offline tool example. For your existing
agent, replace their inputs and expected answers with independent correctness
criteria before capturing a baseline. The adapter calls `{export}` from
`{module}` as `{export}(input, trace)` with one JSON input value and records its
final JSON output. Record model and tool calls with `trace.model(...)` and
`trace.tool(...)` so they appear in the trace.

Run from this project directory with Node.js 22 or newer. The runner needs the
`agentloop-profiler` npm package; the `agentloop` command comes from the Python
package (`python -m pip install agentloop-profiler`).

```bash
{install}
node agentloop_runner.mjs --check
agentloop improve capture --out baseline -- node agentloop_runner.mjs
agentloop improve start --baseline baseline/capture.json --out optimization
```

Review `optimization/plan.md` and the original evidence. For the included example,
edit `answer()` to reuse one lookup result within the request.

```javascript
const value = await lookupNote(trace, key);
return {{ notes: [value, value, value] }};
```

For your own agent, review and apply a change appropriate to its evidence.

```bash
agentloop improve capture --out candidate -- node agentloop_runner.mjs
agentloop improve finish --session optimization/session.json --candidate candidate/capture.json --change "Reuse the lookup result" --out results
```

Open `results/report.html`. A rejected comparison returns exit code 1 and retains
its report. To inspect an agent failure, use `--show-runner-output` on capture
and pass `--debug` after the runner filename. Traces retain final outputs for
quality verification. Review artifacts before sharing them.

The offline example executes real local CPU work on fictional notes and is marked
synthetic. Its timings illustrate this example. Your agent needs representative
tasks and quality checks. The project config selects task data; the argv after
`--` selects executable code.
"""
return files


def project_tasks(project: Path) -> Path:
"""Resolve task data from an explicitly selected project configuration."""
import tomllib
Expand All @@ -122,8 +358,13 @@ def project_tasks(project: Path) -> Path:
return project.resolve().parent / config["tasks"]


def initialize(root: Path, *, target: str | None = None, name: str | None = None):
def initialize(
root: Path, *, target: str | None = None, name: str | None = None, language: str = "python"
):
require(language in {"python", "node"}, "Language must be python or node")
root = root.resolve()
if language == "node":
return _write(root, _node_files(root, target, name))
demo = target is None
target = target or "agentloop_example:answer"
require(target.count(":") == 1, "Use --agent package.module:callable")
Expand All @@ -136,31 +377,9 @@ def initialize(root: Path, *, target: str | None = None, name: str | None = None
),
"Agent module and callable must be Python identifiers separated by dots",
)
name = name or root.name
require(isinstance(name, str) and bool(name.strip()), "Project name must be nonempty")
name = _project_name(root, name)
files = {
"agentloop.toml": 'schema_version = "1.0"\ntasks = "agentloop_tasks.json"\n',
"agentloop_tasks.json": json.dumps(
{
"schema_version": "1.0",
"name": name,
"repetitions": 1,
"tasks": [
{
"id": key,
"input": {"key": key},
"expected": {"notes": [value] * 3},
"scorer": {"type": "json_subset"},
}
for key, value in {
"tracing": "Record each operation.",
"quality": "Check the final answer.",
}.items()
],
},
indent=2,
)
+ "\n",
**_shared_files(name),
"agentloop_runner.py": RUNNER.substitute(
module=module,
function=function,
Expand Down Expand Up @@ -214,6 +433,10 @@ def initialize(root: Path, *, target: str | None = None, name: str | None = None
tasks and quality checks. The project config selects task data; the argv after
`--` selects executable code.
"""
return _write(root, files)


def _write(root, files):
require(root.is_dir() or not root.exists(), "--out must name a project directory")
collisions = [
str(root / filename)
Expand Down
Loading
Loading