Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions .github/workflows/application-performance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ on:
report_artifact:
default: agentloop-application-reports
type: string
agentloop_api_url:
description: Optional AgentLoop server URL. When set, the compared traces are uploaded there with CI labels.
default: ""
type: string
secrets:
agentloop_api_key:
description: Project API key for agentloop_api_url.
required: false

permissions:
contents: read
Expand Down Expand Up @@ -86,6 +94,27 @@ jobs:
--min-cost-improvement-pct "$MIN_COST" \
--github-step-summary \
"${QUALITY_ARGS[@]}"
- name: Send the compared traces to the AgentLoop server
if: ${{ !cancelled() && inputs.agentloop_api_url != '' }}
env:
AGENTLOOP_API_URL: ${{ inputs.agentloop_api_url }}
AGENTLOOP_API_KEY: ${{ secrets.agentloop_api_key }}
run: |
if [ -z "$AGENTLOOP_API_KEY" ]; then
echo "::error::Pass the agentloop_api_key secret to upload traces to $AGENTLOOP_API_URL" >&2
exit 1
fi
for condition in baseline candidate; do
if [ "$condition" = baseline ]; then file="$BASELINE"; else file="$CANDIDATE"; fi
uv run --project .agentloop --frozen --no-dev agentloop upload \
--path "$GITHUB_WORKSPACE/inputs/$file" \
--api-url "$AGENTLOOP_API_URL" \
--label "condition=$condition" \
--label "repository=$GITHUB_REPOSITORY" \
--label "commit=$GITHUB_SHA" \
--label "ref=$GITHUB_REF_NAME" \
--label "workflow_run=$GITHUB_RUN_ID"
done
- name: Upload comparison reports
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
Expand Down
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ trace, study, receipt, API and database schema versions.

### Added

- CI can send traces to an AgentLoop server. `agentloop upload` accepts
several `--path` values and directories, validates every file before sending,
and adds `--label KEY=VALUE` pairs under `metadata.labels`. The reusable
`application-performance.yml` workflow takes an optional `agentloop_api_url`
input and `agentloop_api_key` secret, and uploads the compared baseline and
candidate traces labelled with the repository, commit, ref and workflow run.
- The container image is published for `linux/arm64` as well as `linux/amd64`
under the same tag, so ARM cloud servers can run AgentLoop. Each architecture
is built natively, checked, and combined into one multi-architecture tag; CI
Expand Down
71 changes: 66 additions & 5 deletions agentloop/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import json
import os
import re
import sys
from pathlib import Path

Expand Down Expand Up @@ -1270,19 +1271,79 @@ def usage_summary(project_id: str | None = None) -> None:
console.print(table)


_LABEL_KEY = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")
_MAX_LABEL_VALUE = 500


def _parse_labels(values: list[str] | None) -> dict[str, str]:
labels: dict[str, str] = {}
for value in values or []:
key, separator, text = value.partition("=")
if not separator or not _LABEL_KEY.match(key):
raise typer.BadParameter(
"Use KEY=VALUE with a key of letters, digits, '.', '_' or '-'",
param_hint="--label",
)
if len(text) > _MAX_LABEL_VALUE:
raise typer.BadParameter(
f"Label values are limited to {_MAX_LABEL_VALUE} characters", param_hint="--label"
)
labels[key] = text
return labels


def _upload_files(paths: list[Path]) -> list[Path]:
files: list[Path] = []
for path in paths:
if path.is_dir():
found = sorted(item for item in path.glob("*.json") if item.is_file())
if not found:
raise typer.BadParameter(
f"No .json trace files in directory {path}", param_hint="--path"
)
files.extend(found)
else:
files.append(path)
return files


@app.command()
def upload(
path: Path = Path("runs/research_agent_baseline.json"),
path: list[Path] = typer.Option(
[Path("runs/research_agent_baseline.json")],
"--path",
help="Trace JSON file, or a directory of them; repeat for several.",
),
label: list[str] | None = typer.Option(
None,
"--label",
help="KEY=VALUE stored under metadata.labels of each uploaded trace, "
"such as commit=abc123; repeat for several.",
),
api_url: str = typer.Option("http://127.0.0.1:8000", help="AgentLoop API base URL."),
api_key: str | None = typer.Option(
None, help="Optional API key. Defaults to AGENTLOOP_API_KEY."
),
) -> None:
trace = _load_trace(path, param_hint="--path")
"""Upload traces to an AgentLoop API, optionally labelled with CI context."""
labels = _parse_labels(label)
files = _upload_files(path)
traces = [_load_trace(file, param_hint="--path") for file in files]
client = _remote_client(api_url, api_key)
response = client.upload_trace(trace)
console.print(f"Uploaded trace {response['run_id']} to {api_url}")
console.print(f"Project: {response.get('project_id', 'default')}")
project = None
for trace in traces:
if labels:
existing = trace.metadata.get("labels")
trace.metadata["labels"] = {
**(existing if isinstance(existing, dict) else {}),
**labels,
}
response = client.upload_trace(trace)
project = response.get("project_id", "default")
console.print(f"Uploaded trace {response['run_id']} to {api_url}", markup=False)
if len(traces) > 1:
console.print(f"Uploaded {len(traces)} traces.", markup=False)
console.print(f"Project: {project}", markup=False)


@app.command("remote-delete-trace")
Expand Down
37 changes: 37 additions & 0 deletions docs/CI.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,43 @@ own directory and installs its locked core dependencies. It does not install you
application or SDKs. Use built-in quality scorers here; custom Python scorers that
need application modules belong in your application's own job using `agentloop ci`.

### Send the traces to your AgentLoop server

To keep every compared trace on a shared [AgentLoop server](CLOUD_DEPLOY.md), add
its URL and a project API key stored as a repository secret:

```yaml
compare:
needs: record
uses: dipeshbabu/agentloop/.github/workflows/application-performance.yml@main
permissions:
contents: read
with:
trace_artifact: application-traces
agentloop_api_url: https://agentloop-api.example.com
agentloop_ref: main
secrets:
agentloop_api_key: ${{ secrets.AGENTLOOP_API_KEY }}
```

After the comparison runs, whether or not its gates pass, the workflow uploads the
baseline and candidate traces to that key's project. Each is labelled under
`metadata.labels` with `condition` (`baseline` or `candidate`), `repository`,
`commit`, `ref` and `workflow_run`, so the team can find the run behind a trace.
The job fails if the URL is set without the secret. This needs an `agentloop_ref`
that includes AgentLoop 0.2.8 or newer.

The same command works in any CI system or from a laptop:

```bash
agentloop upload --path traces/ --label commit="$COMMIT_SHA" \
--api-url https://agentloop-api.example.com
```

`--path` takes files or directories of `.json` traces and can be repeated.
Every file is validated before any is sent. The project key is read from
`AGENTLOOP_API_KEY` or `--api-key`.

The repository smoke workflow exercises artifact upload/download, the reusable
workflow, and quality gates using explicitly synthetic fixtures. These smoke
results carry the same synthetic-data qualification.
Expand Down
2 changes: 2 additions & 0 deletions docs/CLOUD_DEPLOY.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ client.upload_trace(trace)

or from the TypeScript SDK with
`new AgentLoopClient({ baseUrl: "https://agentloop-api.example.com", apiKey: "..." })`.
To send traces from CI automatically, see
[sending traces to your AgentLoop server](CI.md#send-the-traces-to-your-agentloop-server).
Sign in to the dashboard with the same key to see that project's traces. Each
team can only see its own project; there is no project switcher.

Expand Down
23 changes: 23 additions & 0 deletions tests/test_ci_evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,3 +91,26 @@ def test_reusable_workflow_never_generates_missing_application_traces():
assert scripts.index('test -f "inputs/$CANDIDATE"') < scripts.index("agentloop ci")
assert 'test -f "inputs/$QUALITY"' in scripts
assert "--no-fail-on-gate" not in scripts


def test_reusable_workflow_uploads_only_when_asked_and_only_with_a_project_key():
text = (ROOT / ".github/workflows/application-performance.yml").read_text(encoding="utf-8")
workflow = yaml.load(text, Loader=yaml.BaseLoader)
call = workflow["on"]["workflow_call"]
assert call["inputs"]["agentloop_api_url"]["default"] == ""
assert call["secrets"]["agentloop_api_key"]["required"] == "false"
steps = workflow["jobs"]["compare"]["steps"]
upload = next(step for step in steps if "AgentLoop server" in step.get("name", ""))
assert "inputs.agentloop_api_url != ''" in upload["if"]
assert "!cancelled()" in upload["if"]
assert upload["env"]["AGENTLOOP_API_KEY"] == "${{ secrets.agentloop_api_key }}"
assert "ADMIN" not in text
script = upload["run"]
assert script.index('-z "$AGENTLOOP_API_KEY"') < script.index("agentloop upload")
for label in ("condition=", "repository=", "commit=", "ref=", "workflow_run="):
assert f'--label "{label}' in script
# The upload runs after the comparison and before the reports are attached.
names = [step.get("name", "") for step in steps]
assert names.index(upload["name"]) > names.index(
"Compare supplied artifacts without generating demo data"
)
96 changes: 96 additions & 0 deletions tests/test_cli_upload.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
from __future__ import annotations

import pytest
from typer.testing import CliRunner

import agentloop.cli as cli_module
from agentloop.cli import app
from agentloop.tracer import AgentTrace

runner = CliRunner()


@pytest.fixture
def uploads(monkeypatch):
sent: list[AgentTrace] = []

class CapturingClient:
def __init__(self, base_url, api_key=None, admin_api_key=None): # type: ignore[no-untyped-def]
self.base_url = base_url

def upload_trace(self, trace): # type: ignore[no-untyped-def]
sent.append(trace)
return {"run_id": trace.run_id, "project_id": "team"}

monkeypatch.setattr(cli_module, "AgentLoopClient", CapturingClient)
return sent


def _trace(path, name, metadata=None):
trace = AgentTrace(name=name, metadata=metadata or {})
trace.finish()
trace.export_json(path)
return trace


def test_upload_sends_files_and_directories_with_labels(tmp_path, uploads):
single = _trace(tmp_path / "single.json", "single", {"labels": {"owner": "kept"}})
batch = tmp_path / "batch"
batch.mkdir()
_trace(batch / "b.json", "b")
_trace(batch / "a.json", "a")
(batch / "notes.txt").write_text("ignored", encoding="utf-8")

result = runner.invoke(
app,
[
"upload",
"--path",
str(tmp_path / "single.json"),
"--path",
str(batch),
"--label",
"commit=abc123",
"--label",
"branch=feature/x=y",
"--api-url",
"https://agentloop.example",
],
)
assert result.exit_code == 0, result.output
assert [trace.name for trace in uploads] == ["single", "a", "b"]
assert uploads[0].run_id == single.run_id
assert uploads[0].metadata["labels"] == {
"owner": "kept",
"commit": "abc123",
"branch": "feature/x=y",
}
assert uploads[1].metadata["labels"] == {"commit": "abc123", "branch": "feature/x=y"}
assert "Uploaded 3 traces." in result.output
assert "Project: team" in result.output


@pytest.mark.parametrize("bad", ["no-equals", "=value", "bad key=1", "k=" + "v" * 501])
def test_upload_rejects_malformed_labels_before_sending(tmp_path, uploads, bad):
_trace(tmp_path / "t.json", "t")
result = runner.invoke(app, ["upload", "--path", str(tmp_path / "t.json"), "--label", bad])
assert result.exit_code == 2
assert uploads == []


def test_upload_validates_every_file_before_sending_any(tmp_path, uploads):
_trace(tmp_path / "good.json", "good")
(tmp_path / "broken.json").write_text("{not json", encoding="utf-8")
result = runner.invoke(
app,
["upload", "--path", str(tmp_path / "good.json"), "--path", str(tmp_path / "broken.json")],
)
assert result.exit_code == 2
assert uploads == []


def test_upload_rejects_an_empty_directory(tmp_path, uploads):
(tmp_path / "empty").mkdir()
result = runner.invoke(app, ["upload", "--path", str(tmp_path / "empty")])
assert result.exit_code == 2
assert "No .json trace files" in result.output
Loading