From 77f61c5b6cd49e0978a32e23f5153ebc1865b754 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Fri, 20 Nov 2026 03:08:41 -0500 Subject: [PATCH] Send CI traces to an AgentLoop server agentloop upload took a single file, and the reusable comparison workflow could not send traces anywhere. upload now accepts repeated --path values and directories, validates every file before sending any, and adds --label KEY=VALUE pairs under metadata.labels. The reusable workflow takes an optional agentloop_api_url input and agentloop_api_key secret and uploads the compared baseline and candidate traces labelled with condition, repository, commit, ref and workflow run. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/application-performance.yml | 29 ++++++ CHANGELOG.md | 6 ++ agentloop/cli.py | 71 +++++++++++++- docs/CI.md | 37 +++++++ docs/CLOUD_DEPLOY.md | 2 + tests/test_ci_evidence.py | 23 +++++ tests/test_cli_upload.py | 96 +++++++++++++++++++ 7 files changed, 259 insertions(+), 5 deletions(-) create mode 100644 tests/test_cli_upload.py diff --git a/.github/workflows/application-performance.yml b/.github/workflows/application-performance.yml index 1a4d6c6..dc7b0d6 100644 --- a/.github/workflows/application-performance.yml +++ b/.github/workflows/application-performance.yml @@ -32,6 +32,14 @@ on: report_artifact: default: agentloop-application-reports type: string + agentloop_api_url: + description: Optional AgentLoop server URL. When set, the compared traces are uploaded there with CI labels. + default: "" + type: string + secrets: + agentloop_api_key: + description: Project API key for agentloop_api_url. + required: false permissions: contents: read @@ -86,6 +94,27 @@ jobs: --min-cost-improvement-pct "$MIN_COST" \ --github-step-summary \ "${QUALITY_ARGS[@]}" + - name: Send the compared traces to the AgentLoop server + if: ${{ !cancelled() && inputs.agentloop_api_url != '' }} + env: + AGENTLOOP_API_URL: ${{ inputs.agentloop_api_url }} + AGENTLOOP_API_KEY: ${{ secrets.agentloop_api_key }} + run: | + if [ -z "$AGENTLOOP_API_KEY" ]; then + echo "::error::Pass the agentloop_api_key secret to upload traces to $AGENTLOOP_API_URL" >&2 + exit 1 + fi + for condition in baseline candidate; do + if [ "$condition" = baseline ]; then file="$BASELINE"; else file="$CANDIDATE"; fi + uv run --project .agentloop --frozen --no-dev agentloop upload \ + --path "$GITHUB_WORKSPACE/inputs/$file" \ + --api-url "$AGENTLOOP_API_URL" \ + --label "condition=$condition" \ + --label "repository=$GITHUB_REPOSITORY" \ + --label "commit=$GITHUB_SHA" \ + --label "ref=$GITHUB_REF_NAME" \ + --label "workflow_run=$GITHUB_RUN_ID" + done - name: Upload comparison reports if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dd3782..5d12e59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ trace, study, receipt, API and database schema versions. ### Added +- CI can send traces to an AgentLoop server. `agentloop upload` accepts + several `--path` values and directories, validates every file before sending, + and adds `--label KEY=VALUE` pairs under `metadata.labels`. The reusable + `application-performance.yml` workflow takes an optional `agentloop_api_url` + input and `agentloop_api_key` secret, and uploads the compared baseline and + candidate traces labelled with the repository, commit, ref and workflow run. - The container image is published for `linux/arm64` as well as `linux/amd64` under the same tag, so ARM cloud servers can run AgentLoop. Each architecture is built natively, checked, and combined into one multi-architecture tag; CI diff --git a/agentloop/cli.py b/agentloop/cli.py index ed771c8..582866b 100644 --- a/agentloop/cli.py +++ b/agentloop/cli.py @@ -2,6 +2,7 @@ import json import os +import re import sys from pathlib import Path @@ -1270,19 +1271,79 @@ def usage_summary(project_id: str | None = None) -> None: console.print(table) +_LABEL_KEY = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") +_MAX_LABEL_VALUE = 500 + + +def _parse_labels(values: list[str] | None) -> dict[str, str]: + labels: dict[str, str] = {} + for value in values or []: + key, separator, text = value.partition("=") + if not separator or not _LABEL_KEY.match(key): + raise typer.BadParameter( + "Use KEY=VALUE with a key of letters, digits, '.', '_' or '-'", + param_hint="--label", + ) + if len(text) > _MAX_LABEL_VALUE: + raise typer.BadParameter( + f"Label values are limited to {_MAX_LABEL_VALUE} characters", param_hint="--label" + ) + labels[key] = text + return labels + + +def _upload_files(paths: list[Path]) -> list[Path]: + files: list[Path] = [] + for path in paths: + if path.is_dir(): + found = sorted(item for item in path.glob("*.json") if item.is_file()) + if not found: + raise typer.BadParameter( + f"No .json trace files in directory {path}", param_hint="--path" + ) + files.extend(found) + else: + files.append(path) + return files + + @app.command() def upload( - path: Path = Path("runs/research_agent_baseline.json"), + path: list[Path] = typer.Option( + [Path("runs/research_agent_baseline.json")], + "--path", + help="Trace JSON file, or a directory of them; repeat for several.", + ), + label: list[str] | None = typer.Option( + None, + "--label", + help="KEY=VALUE stored under metadata.labels of each uploaded trace, " + "such as commit=abc123; repeat for several.", + ), api_url: str = typer.Option("http://127.0.0.1:8000", help="AgentLoop API base URL."), api_key: str | None = typer.Option( None, help="Optional API key. Defaults to AGENTLOOP_API_KEY." ), ) -> None: - trace = _load_trace(path, param_hint="--path") + """Upload traces to an AgentLoop API, optionally labelled with CI context.""" + labels = _parse_labels(label) + files = _upload_files(path) + traces = [_load_trace(file, param_hint="--path") for file in files] client = _remote_client(api_url, api_key) - response = client.upload_trace(trace) - console.print(f"Uploaded trace {response['run_id']} to {api_url}") - console.print(f"Project: {response.get('project_id', 'default')}") + project = None + for trace in traces: + if labels: + existing = trace.metadata.get("labels") + trace.metadata["labels"] = { + **(existing if isinstance(existing, dict) else {}), + **labels, + } + response = client.upload_trace(trace) + project = response.get("project_id", "default") + console.print(f"Uploaded trace {response['run_id']} to {api_url}", markup=False) + if len(traces) > 1: + console.print(f"Uploaded {len(traces)} traces.", markup=False) + console.print(f"Project: {project}", markup=False) @app.command("remote-delete-trace") diff --git a/docs/CI.md b/docs/CI.md index cd1b6a2..ed9abfc 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -78,6 +78,43 @@ own directory and installs its locked core dependencies. It does not install you application or SDKs. Use built-in quality scorers here; custom Python scorers that need application modules belong in your application's own job using `agentloop ci`. +### Send the traces to your AgentLoop server + +To keep every compared trace on a shared [AgentLoop server](CLOUD_DEPLOY.md), add +its URL and a project API key stored as a repository secret: + +```yaml + compare: + needs: record + uses: dipeshbabu/agentloop/.github/workflows/application-performance.yml@main + permissions: + contents: read + with: + trace_artifact: application-traces + agentloop_api_url: https://agentloop-api.example.com + agentloop_ref: main + secrets: + agentloop_api_key: ${{ secrets.AGENTLOOP_API_KEY }} +``` + +After the comparison runs, whether or not its gates pass, the workflow uploads the +baseline and candidate traces to that key's project. Each is labelled under +`metadata.labels` with `condition` (`baseline` or `candidate`), `repository`, +`commit`, `ref` and `workflow_run`, so the team can find the run behind a trace. +The job fails if the URL is set without the secret. This needs an `agentloop_ref` +that includes AgentLoop 0.2.8 or newer. + +The same command works in any CI system or from a laptop: + +```bash +agentloop upload --path traces/ --label commit="$COMMIT_SHA" \ + --api-url https://agentloop-api.example.com +``` + +`--path` takes files or directories of `.json` traces and can be repeated. +Every file is validated before any is sent. The project key is read from +`AGENTLOOP_API_KEY` or `--api-key`. + The repository smoke workflow exercises artifact upload/download, the reusable workflow, and quality gates using explicitly synthetic fixtures. These smoke results carry the same synthetic-data qualification. diff --git a/docs/CLOUD_DEPLOY.md b/docs/CLOUD_DEPLOY.md index c4270ca..ad05c4e 100644 --- a/docs/CLOUD_DEPLOY.md +++ b/docs/CLOUD_DEPLOY.md @@ -111,6 +111,8 @@ client.upload_trace(trace) or from the TypeScript SDK with `new AgentLoopClient({ baseUrl: "https://agentloop-api.example.com", apiKey: "..." })`. +To send traces from CI automatically, see +[sending traces to your AgentLoop server](CI.md#send-the-traces-to-your-agentloop-server). Sign in to the dashboard with the same key to see that project's traces. Each team can only see its own project; there is no project switcher. diff --git a/tests/test_ci_evidence.py b/tests/test_ci_evidence.py index aa6b346..f0ccee7 100644 --- a/tests/test_ci_evidence.py +++ b/tests/test_ci_evidence.py @@ -91,3 +91,26 @@ def test_reusable_workflow_never_generates_missing_application_traces(): assert scripts.index('test -f "inputs/$CANDIDATE"') < scripts.index("agentloop ci") assert 'test -f "inputs/$QUALITY"' in scripts assert "--no-fail-on-gate" not in scripts + + +def test_reusable_workflow_uploads_only_when_asked_and_only_with_a_project_key(): + text = (ROOT / ".github/workflows/application-performance.yml").read_text(encoding="utf-8") + workflow = yaml.load(text, Loader=yaml.BaseLoader) + call = workflow["on"]["workflow_call"] + assert call["inputs"]["agentloop_api_url"]["default"] == "" + assert call["secrets"]["agentloop_api_key"]["required"] == "false" + steps = workflow["jobs"]["compare"]["steps"] + upload = next(step for step in steps if "AgentLoop server" in step.get("name", "")) + assert "inputs.agentloop_api_url != ''" in upload["if"] + assert "!cancelled()" in upload["if"] + assert upload["env"]["AGENTLOOP_API_KEY"] == "${{ secrets.agentloop_api_key }}" + assert "ADMIN" not in text + script = upload["run"] + assert script.index('-z "$AGENTLOOP_API_KEY"') < script.index("agentloop upload") + for label in ("condition=", "repository=", "commit=", "ref=", "workflow_run="): + assert f'--label "{label}' in script + # The upload runs after the comparison and before the reports are attached. + names = [step.get("name", "") for step in steps] + assert names.index(upload["name"]) > names.index( + "Compare supplied artifacts without generating demo data" + ) diff --git a/tests/test_cli_upload.py b/tests/test_cli_upload.py new file mode 100644 index 0000000..adb9e28 --- /dev/null +++ b/tests/test_cli_upload.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import pytest +from typer.testing import CliRunner + +import agentloop.cli as cli_module +from agentloop.cli import app +from agentloop.tracer import AgentTrace + +runner = CliRunner() + + +@pytest.fixture +def uploads(monkeypatch): + sent: list[AgentTrace] = [] + + class CapturingClient: + def __init__(self, base_url, api_key=None, admin_api_key=None): # type: ignore[no-untyped-def] + self.base_url = base_url + + def upload_trace(self, trace): # type: ignore[no-untyped-def] + sent.append(trace) + return {"run_id": trace.run_id, "project_id": "team"} + + monkeypatch.setattr(cli_module, "AgentLoopClient", CapturingClient) + return sent + + +def _trace(path, name, metadata=None): + trace = AgentTrace(name=name, metadata=metadata or {}) + trace.finish() + trace.export_json(path) + return trace + + +def test_upload_sends_files_and_directories_with_labels(tmp_path, uploads): + single = _trace(tmp_path / "single.json", "single", {"labels": {"owner": "kept"}}) + batch = tmp_path / "batch" + batch.mkdir() + _trace(batch / "b.json", "b") + _trace(batch / "a.json", "a") + (batch / "notes.txt").write_text("ignored", encoding="utf-8") + + result = runner.invoke( + app, + [ + "upload", + "--path", + str(tmp_path / "single.json"), + "--path", + str(batch), + "--label", + "commit=abc123", + "--label", + "branch=feature/x=y", + "--api-url", + "https://agentloop.example", + ], + ) + assert result.exit_code == 0, result.output + assert [trace.name for trace in uploads] == ["single", "a", "b"] + assert uploads[0].run_id == single.run_id + assert uploads[0].metadata["labels"] == { + "owner": "kept", + "commit": "abc123", + "branch": "feature/x=y", + } + assert uploads[1].metadata["labels"] == {"commit": "abc123", "branch": "feature/x=y"} + assert "Uploaded 3 traces." in result.output + assert "Project: team" in result.output + + +@pytest.mark.parametrize("bad", ["no-equals", "=value", "bad key=1", "k=" + "v" * 501]) +def test_upload_rejects_malformed_labels_before_sending(tmp_path, uploads, bad): + _trace(tmp_path / "t.json", "t") + result = runner.invoke(app, ["upload", "--path", str(tmp_path / "t.json"), "--label", bad]) + assert result.exit_code == 2 + assert uploads == [] + + +def test_upload_validates_every_file_before_sending_any(tmp_path, uploads): + _trace(tmp_path / "good.json", "good") + (tmp_path / "broken.json").write_text("{not json", encoding="utf-8") + result = runner.invoke( + app, + ["upload", "--path", str(tmp_path / "good.json"), "--path", str(tmp_path / "broken.json")], + ) + assert result.exit_code == 2 + assert uploads == [] + + +def test_upload_rejects_an_empty_directory(tmp_path, uploads): + (tmp_path / "empty").mkdir() + result = runner.invoke(app, ["upload", "--path", str(tmp_path / "empty")]) + assert result.exit_code == 2 + assert "No .json trace files" in result.output