diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d12e59..b6393b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ trace, study, receipt, API and database schema versions. ### Added +- Trace labels are visible and searchable. Paginated trace listings include + each trace's `metadata.labels`, and `GET /v1/traces?label=KEY=VALUE`, + `AgentLoopClient.list_traces(label=...)`, `agentloop list-stored-traces --label` + and a dashboard Traces filter return only traces with that label, in SQLite + and Postgres. - CI can send traces to an AgentLoop server. `agentloop upload` accepts several `--path` values and directories, validates every file before sending, and adds `--label KEY=VALUE` pairs under `metadata.labels`. The reusable diff --git a/agentloop/cli.py b/agentloop/cli.py index 582866b..362f638 100644 --- a/agentloop/cli.py +++ b/agentloop/cli.py @@ -2,7 +2,6 @@ import json import os -import re import sys from pathlib import Path @@ -57,9 +56,12 @@ from agentloop.replay import ReplayGates, build_replay_report, replay_report_to_markdown from agentloop.store import ( DEFAULT_PAGE_SIZE, + LABEL_KEY_PATTERN, + MAX_LABEL_VALUE_LENGTH, FindingNotFoundError, FindingTransitionError, get_store, + parse_label_filter, ) from agentloop.studies import StudyValidationError, study_to_markdown, summarize_study from agentloop.timing import format_duration_ms @@ -1086,14 +1088,24 @@ def list_stored_traces( cursor: str | None = typer.Option( None, help="Continuation cursor from a previous --limit call's printed next-cursor." ), + label: str | None = typer.Option( + None, help="Only traces with this KEY=VALUE label, such as commit=abc123." + ), ) -> None: db = get_store() - if limit is None and cursor is None: + try: + label_filter = parse_label_filter(label) if label is not None else None + except ValueError as exc: + raise typer.BadParameter(str(exc), param_hint="--label") from exc + if limit is None and cursor is None and label_filter is None: traces = db.list_traces(project_id=project_id) next_cursor = None else: page = db.list_traces_page( - project_id=project_id, limit=limit or DEFAULT_PAGE_SIZE, cursor=cursor + project_id=project_id, + limit=limit or DEFAULT_PAGE_SIZE, + cursor=cursor, + label=label_filter, ) traces = page["items"] next_cursor = page["next_cursor"] @@ -1103,8 +1115,11 @@ def list_stored_traces( table.add_column("Name") table.add_column("Runtime ms") table.add_column("Cost") + show_labels = any(item.get("labels") for item in traces) + if show_labels: + table.add_column("Labels") for item in traces: - table.add_row( + row = [ str(item.get("project_id", "")), str(item.get("run_id", "")), str(item.get("name", "")), @@ -1112,7 +1127,11 @@ def list_stored_traces( if item.get("total_runtime_ms") is None else f"{float(item.get('total_runtime_ms', 0)):.2f}", format_cost_usd(item.get("estimated_cost_usd"), item.get("cost_status", "complete")), - ) + ] + if show_labels: + labels = item.get("labels") or {} + row.append(", ".join(f"{key}={value}" for key, value in sorted(labels.items()))) + table.add_row(*row, end_section=False) console.print(table) if next_cursor: console.print(f"More results available. Next cursor: {next_cursor}") @@ -1271,22 +1290,19 @@ def usage_summary(project_id: str | None = None) -> None: console.print(table) -_LABEL_KEY = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") -_MAX_LABEL_VALUE = 500 - - def _parse_labels(values: list[str] | None) -> dict[str, str]: labels: dict[str, str] = {} for value in values or []: key, separator, text = value.partition("=") - if not separator or not _LABEL_KEY.match(key): + if not separator or not LABEL_KEY_PATTERN.match(key): raise typer.BadParameter( "Use KEY=VALUE with a key of letters, digits, '.', '_' or '-'", param_hint="--label", ) - if len(text) > _MAX_LABEL_VALUE: + if len(text) > MAX_LABEL_VALUE_LENGTH: raise typer.BadParameter( - f"Label values are limited to {_MAX_LABEL_VALUE} characters", param_hint="--label" + f"Label values are limited to {MAX_LABEL_VALUE_LENGTH} characters", + param_hint="--label", ) labels[key] = text return labels diff --git a/agentloop/client.py b/agentloop/client.py index 38e2581..5c2098b 100644 --- a/agentloop/client.py +++ b/agentloop/client.py @@ -93,6 +93,7 @@ def list_traces( *, page_size: int | None = None, cursor: str | None = None, + label: str | None = None, ) -> dict[str, Any]: params: dict[str, Any] = {} if project_id: @@ -101,6 +102,8 @@ def list_traces( params["page_size"] = page_size if cursor: params["cursor"] = cursor + if label is not None: + params["label"] = label route = "/traces" if params: route += "?" + urllib.parse.urlencode(params) diff --git a/agentloop/server.py b/agentloop/server.py index 89da4e9..25f3f33 100644 --- a/agentloop/server.py +++ b/agentloop/server.py @@ -29,6 +29,7 @@ TraceProjectConflictError, TraceStore, get_store, + parse_label_filter, ) from agentloop.tracer import AgentTrace from agentloop.value import build_value_report @@ -236,6 +237,7 @@ def list_traces( project_id_filter: str | None = Query(default=None, alias="project_id"), page_size: int | None = Query(default=None, ge=1, le=MAX_PAGE_SIZE), cursor: str | None = Query(default=None), + label: str | None = Query(default=None, description="Only traces with this KEY=VALUE label."), project_id: str = Depends(resolve_project), db: TraceStore = Depends(store), ) -> dict[str, Any]: @@ -247,7 +249,11 @@ def list_traces( fetch the next page; `next_cursor: null` means there are no more rows. """ selected_project = _selected_project(project_id_filter, project_id) - if page_size is None and cursor is None: + try: + label_filter = parse_label_filter(label) if label is not None else None + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from None + if page_size is None and cursor is None and label_filter is None: return { "project_id": selected_project, "traces": db.list_traces(project_id=selected_project), @@ -255,7 +261,10 @@ def list_traces( } try: page = db.list_traces_page( - project_id=selected_project, limit=page_size or DEFAULT_PAGE_SIZE, cursor=cursor + project_id=selected_project, + limit=page_size or DEFAULT_PAGE_SIZE, + cursor=cursor, + label=label_filter, ) except InvalidCursorError as exc: raise HTTPException(status_code=400, detail=str(exc)) from None diff --git a/agentloop/store.py b/agentloop/store.py index 95f7794..0682d2f 100644 --- a/agentloop/store.py +++ b/agentloop/store.py @@ -7,6 +7,7 @@ import json import math import os +import re import secrets import sqlite3 from dataclasses import dataclass @@ -119,6 +120,7 @@ def list_traces_page( project_id: str | None = None, limit: int = DEFAULT_PAGE_SIZE, cursor: str | None = None, + label: tuple[str, str] | None = None, ) -> dict[str, Any]: ... def get_trace(self, run_id: str, project_id: str | None = None) -> AgentTrace | None: ... @@ -257,6 +259,48 @@ def _delete_runs(conn, mark: str, project_id: str, run_ids: list[str]) -> dict[s return counts +LABEL_KEY_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") +MAX_LABEL_VALUE_LENGTH = 500 + + +def validate_label_filter(label: Any) -> tuple[str, str]: + """Check a (key, value) label filter; keys use letters, digits, '.', '_' and '-'.""" + if ( + not isinstance(label, tuple) + or len(label) != 2 + or not all(isinstance(part, str) for part in label) + or not LABEL_KEY_PATTERN.match(label[0]) + or len(label[1]) > MAX_LABEL_VALUE_LENGTH + ): + raise ValueError( + "Label filters are KEY=VALUE with a key of letters, digits, '.', '_' or '-'" + ) + return label + + +def parse_label_filter(text: str) -> tuple[str, str]: + """Parse ``KEY=VALUE`` into a validated label filter.""" + key, separator, value = text.partition("=") if isinstance(text, str) else ("", "", "") + if not separator: + raise ValueError( + "Label filters are KEY=VALUE with a key of letters, digits, '.', '_' or '-'" + ) + return validate_label_filter((key, value)) + + +def _with_labels(item: dict[str, Any]) -> dict[str, Any]: + labels = item.get("labels") + if isinstance(labels, str): + try: + labels = json.loads(labels) + except ValueError: + labels = None + item["labels"] = ( + {str(key): value for key, value in labels.items()} if isinstance(labels, dict) else {} + ) + return item + + def _timestamp_text(value: Any) -> str | None: if value is None: return None @@ -807,26 +851,40 @@ def list_traces_page( project_id: str | None = None, limit: int = DEFAULT_PAGE_SIZE, cursor: str | None = None, + label: tuple[str, str] | None = None, ) -> dict[str, Any]: after_created_at, after_run_id = ( decode_cursor(cursor, expected_parts=2) if cursor else (None, None) ) limit = _validate_page_limit(limit) + label_key, label_value = validate_label_filter(label) if label else (None, None) self.init() with self._connect() as conn: rows = conn.execute( """ SELECT run_id, project_id, name, started_at, total_runtime_ms, estimated_cost_usd, - cost_status, priced_model_call_count, unavailable_model_call_count, created_at + cost_status, priced_model_call_count, unavailable_model_call_count, created_at, + json_extract(payload_json, '$.metadata.labels') AS labels FROM traces WHERE project_id = COALESCE(?, project_id) AND (? = 0 OR (created_at, run_id) < (?, ?)) + AND (? IS NULL OR + json_extract(payload_json, '$.metadata.labels."' || ? || '"') = ?) ORDER BY created_at DESC, run_id DESC LIMIT ? """, - (project_id, 1 if cursor else 0, after_created_at, after_run_id, limit + 1), + ( + project_id, + 1 if cursor else 0, + after_created_at, + after_run_id, + label_key, + label_key, + label_value, + limit + 1, + ), ).fetchall() - items = [dict(row) for row in rows[:limit]] + items = [_with_labels(dict(row)) for row in rows[:limit]] next_cursor = None if len(rows) > limit: last = items[-1] @@ -1311,6 +1369,7 @@ def list_traces_page( project_id: str | None = None, limit: int = DEFAULT_PAGE_SIZE, cursor: str | None = None, + label: tuple[str, str] | None = None, ) -> dict[str, Any]: after_created_at, after_run_id = ( decode_cursor(cursor, expected_parts=2) if cursor else (None, None) @@ -1327,22 +1386,36 @@ def list_traces_page( "priced_model_call_count", "unavailable_model_call_count", "created_at", + "labels", ] + label_key, label_value = validate_label_filter(label) if label else (None, None) self.init() with self._connect() as conn: rows = conn.execute( """ SELECT run_id, project_id, name, started_at, total_runtime_ms, estimated_cost_usd, - cost_status, priced_model_call_count, unavailable_model_call_count, created_at::text + cost_status, priced_model_call_count, unavailable_model_call_count, + created_at::text, payload_json -> 'metadata' -> 'labels' FROM traces WHERE project_id = COALESCE(%s, project_id) AND (%s = 0 OR (created_at, run_id) < (%s, %s)) + AND (%s::text IS NULL OR + payload_json -> 'metadata' -> 'labels' ->> %s::text = %s::text) ORDER BY created_at DESC, run_id DESC LIMIT %s """, - (project_id, 1 if cursor else 0, after_created_at, after_run_id, limit + 1), + ( + project_id, + 1 if cursor else 0, + after_created_at, + after_run_id, + label_key, + label_key, + label_value, + limit + 1, + ), ).fetchall() - items = [dict(zip(keys, row, strict=True)) for row in rows[:limit]] + items = [_with_labels(dict(zip(keys, row, strict=True))) for row in rows[:limit]] next_cursor = None if len(rows) > limit: last = items[-1] diff --git a/dashboard/app.py b/dashboard/app.py index a8d4253..91754ca 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -27,6 +27,7 @@ FindingNotFoundError, FindingTransitionError, get_store, + parse_label_filter, ) from agentloop.tracer import AgentTrace from agentloop.value import build_value_report @@ -234,16 +235,38 @@ def render_gate_table(results: list[dict]) -> None: if notice := st.session_state.pop("trace_deleted_notice", None): st.success(notice) - cursor_key = f"traces_cursor_stack::{project_id}" + label_text = st.text_input( + "Filter by label", + placeholder="KEY=VALUE, for example commit=abc123 or condition=candidate", + ).strip() + label_filter = None + if label_text: + try: + label_filter = parse_label_filter(label_text) + except ValueError as exc: + st.error(f"{exc}. Showing all traces.") + + cursor_key = f"traces_cursor_stack::{project_id}::{label_filter}" if cursor_key not in st.session_state: st.session_state[cursor_key] = [None] # cursor to fetch the *current* page page_index = len(st.session_state[cursor_key]) - 1 page_size = st.number_input("Page size", min_value=5, max_value=200, value=25, step=5) page = store.list_traces_page( - project_id=project_id, limit=int(page_size), cursor=st.session_state[cursor_key][-1] + project_id=project_id, + limit=int(page_size), + cursor=st.session_state[cursor_key][-1], + label=label_filter, ) - traces = page["items"] + traces = [ + { + **item, + "labels": ", ".join( + f"{key}={value}" for key, value in sorted((item.get("labels") or {}).items()) + ), + } + for item in page["items"] + ] nav_cols = st.columns([1, 1, 6]) if nav_cols[0].button("◀ Previous", disabled=page_index == 0): @@ -255,7 +278,11 @@ def render_gate_table(results: list[dict]) -> None: st.caption(f"Page {page_index + 1}") if not traces: - st.info("No traces stored for this project yet.") + st.info( + f"No traces labelled {label_filter[0]}={label_filter[1]} in this project." + if label_filter + else "No traces stored for this project yet." + ) else: df = pd.DataFrame(traces) st.dataframe(df, width="stretch") diff --git a/docs/API_VERSIONING.md b/docs/API_VERSIONING.md index 0731f4e..3e4447c 100644 --- a/docs/API_VERSIONING.md +++ b/docs/API_VERSIONING.md @@ -33,7 +33,7 @@ for larger reports. `timeout_s` controls the underlying socket timeout. | `POST /v1/api-keys` | `POST /api-keys` | | `GET /v1/api-keys?project_id=...` | None (0.2.7 and newer) | | `DELETE /v1/api-keys/{prefix}?project_id=...` | None (0.2.7 and newer) | -| `GET/POST /v1/traces` | `GET/POST /traces` | +| `GET/POST /v1/traces` (`?label=KEY=VALUE` from 0.2.8) | `GET/POST /traces` | | `DELETE /v1/traces/{run_id}` | None (0.2.7 and newer) | | `GET /v1/traces/{run_id}/report` | `GET /traces/{run_id}/report` | | `GET /v1/traces/{run_id}/optimization` | `GET /traces/{run_id}/optimize` | diff --git a/docs/CI.md b/docs/CI.md index ed9abfc..94b3dcd 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -101,6 +101,9 @@ After the comparison runs, whether or not its gates pass, the workflow uploads t baseline and candidate traces to that key's project. Each is labelled under `metadata.labels` with `condition` (`baseline` or `candidate`), `repository`, `commit`, `ref` and `workflow_run`, so the team can find the run behind a trace. +Filter by any label in the dashboard's Traces page (for example +`commit=abc123`), with `agentloop list-stored-traces --label commit=abc123`, or +with `GET /v1/traces?label=commit=abc123`. The job fails if the URL is set without the secret. This needs an `agentloop_ref` that includes AgentLoop 0.2.8 or newer. diff --git a/docs/DASHBOARD.md b/docs/DASHBOARD.md index 5384673..10ece33 100644 --- a/docs/DASHBOARD.md +++ b/docs/DASHBOARD.md @@ -26,8 +26,9 @@ Open the URL printed by Streamlit and select project `demo` in the sidebar. - `Overview`: run count, runtime, cost, token volume, model and tool calls, retries, recent runs, and runtime charts. -- `Traces`: stored trace metadata, event timelines, trace JSON downloads, and - deleting a trace after confirmation. +- `Traces`: stored trace metadata and labels, a `KEY=VALUE` label filter + (such as `commit=abc123` from CI uploads), event timelines, trace JSON + downloads, and deleting a trace after confirmation. - `Optimization Queue`: recurring findings ranked across stored traces. - `Optimization`: bottlenecks, parallelizable groups, optimization findings, and plan downloads. diff --git a/tests/test_cli_store_commands.py b/tests/test_cli_store_commands.py index 77cbc9c..5dc7e03 100644 --- a/tests/test_cli_store_commands.py +++ b/tests/test_cli_store_commands.py @@ -154,3 +154,23 @@ def test_delete_and_purge_traces_from_the_cli(tmp_path, monkeypatch) -> None: monkeypatch.setenv("AGENTLOOP_RETENTION_DAYS", "soon") assert runner.invoke(app, ["purge-traces"]).exit_code == 2 + + +def test_list_stored_traces_filters_by_label(tmp_path, monkeypatch) -> None: + db_path = tmp_path / "labels.db" + monkeypatch.setenv("AGENTLOOP_STORE_BACKEND", "sqlite") + monkeypatch.setenv("AGENTLOOP_SQLITE_PATH", str(db_path)) + store = SQLiteTraceStore(path=str(db_path)) + for name, commit in (("first", "abc"), ("second", "def")): + with trace_agent(name) as trace: + trace.metadata["labels"] = {"commit": commit} + store.save_trace(trace, project_id="acme") + + result = runner.invoke( + app, ["list-stored-traces", "--project-id", "acme", "--label", "commit=abc"] + ) + assert result.exit_code == 0, result.output + assert "first" in result.output and "second" not in result.output + assert "commit=abc" in result.output + bad = runner.invoke(app, ["list-stored-traces", "--label", "no-equals"]) + assert bad.exit_code == 2 diff --git a/tests/test_dashboard_project_access.py b/tests/test_dashboard_project_access.py index c7f1cea..38b3f09 100644 --- a/tests/test_dashboard_project_access.py +++ b/tests/test_dashboard_project_access.py @@ -197,3 +197,28 @@ def test_traces_page_deletes_a_trace_after_confirmation(tmp_path, monkeypatch): store = SQLiteTraceStore(path=str(tmp_path / "dashboard.db")) assert store.list_traces(project_id="alpha") == [] assert len(store.list_traces(project_id="beta")) == 1 + + +def test_traces_page_filters_by_label(tmp_path, monkeypatch): + database = tmp_path / "dashboard.db" + monkeypatch.setenv("AGENTLOOP_STORE_BACKEND", "sqlite") + monkeypatch.setenv("AGENTLOOP_SQLITE_PATH", str(database)) + monkeypatch.delenv("AGENTLOOP_DASHBOARD_REQUIRE_KEY", raising=False) + store = SQLiteTraceStore(path=str(database)) + for name, condition in (("base-run", "baseline"), ("cand-run", "candidate")): + trace = AgentTrace(name=name, metadata={"labels": {"condition": condition}}) + store.save_trace(trace, project_id="default") + app = AppTest.from_file(APP, default_timeout=APPTEST_TIMEOUT) + app.run() + app.sidebar.radio[0].set_value("Traces").run() + assert {"base-run", "cand-run"} <= _shown_names(app) + + next(w for w in app.text_input if w.label == "Filter by label").input( + "condition=candidate" + ).run() + assert not app.exception + names = _shown_names(app) + assert "cand-run" in names and "base-run" not in names + + next(w for w in app.text_input if w.label == "Filter by label").input("oops").run() + assert any("KEY=VALUE" in error.value for error in app.error) diff --git a/tests/test_server.py b/tests/test_server.py index e26c9fd..ac6f022 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -651,3 +651,24 @@ def key(project): assert deleted.json()["deleted"]["traces"] == 1 assert client.get("/v1/traces/run-delete/report", headers=acme).status_code == 404 assert client.delete("/v1/traces/run-delete", headers=acme).status_code == 404 + + +def test_traces_can_be_filtered_by_label(monkeypatch, tmp_path) -> None: + monkeypatch.setenv("AGENTLOOP_STORE_BACKEND", "sqlite") + monkeypatch.setenv("AGENTLOOP_SQLITE_PATH", str(tmp_path / "labels.db")) + monkeypatch.delenv("AGENTLOOP_REQUIRE_API_KEY", raising=False) + client = TestClient(app) + for run_id, commit in (("run-a", "abc"), ("run-b", "def")): + trace = { + "name": run_id, + "run_id": run_id, + "metadata": {"labels": {"commit": commit}}, + "events": [], + } + assert client.post("/v1/traces", json=trace).status_code == 200 + + filtered = client.get("/v1/traces", params={"label": "commit=abc"}).json() + assert [item["run_id"] for item in filtered["traces"]] == ["run-a"] + assert filtered["traces"][0]["labels"] == {"commit": "abc"} + assert client.get("/v1/traces", params={"label": "commit"}).status_code == 422 + assert len(client.get("/v1/traces").json()["traces"]) == 2 diff --git a/tests/test_store_contract.py b/tests/test_store_contract.py index f370d67..7a2e346 100644 --- a/tests/test_store_contract.py +++ b/tests/test_store_contract.py @@ -997,3 +997,50 @@ def test_purge_deletes_only_traces_older_than_the_retention_period(store): for invalid in (0, -1, float("nan"), True, "30"): with pytest.raises(ValueError, match="positive number of days"): store.purge_traces(invalid) + + +def test_traces_list_their_labels_and_filter_by_one(store): + from agentloop.store import parse_label_filter + + def labelled(name, **labels): + trace = _sample_trace(name) + if labels: + trace.metadata["labels"] = labels + store.save_trace(trace, project_id="proj_a") + return trace + + base = labelled("base", condition="baseline", commit="abc", **{"ci.run": "7"}) + cand = labelled("cand", condition="candidate", commit="abc") + other = labelled("other", condition="candidate", commit="def") + plain = labelled("plain") + store.save_trace(_sample_trace("elsewhere"), project_id="proj_b") + + page = store.list_traces_page(project_id="proj_a", limit=10) + by_run = {item["run_id"]: item["labels"] for item in page["items"]} + assert by_run[base.run_id] == {"condition": "baseline", "commit": "abc", "ci.run": "7"} + assert by_run[plain.run_id] == {} + + commit_abc = store.list_traces_page(project_id="proj_a", limit=10, label=("commit", "abc")) + assert {item["run_id"] for item in commit_abc["items"]} == {base.run_id, cand.run_id} + dotted = store.list_traces_page(project_id="proj_a", limit=10, label=("ci.run", "7")) + assert [item["run_id"] for item in dotted["items"]] == [base.run_id] + assert store.list_traces_page(project_id="proj_b", label=("commit", "abc"))["items"] == [] + + # Pagination keeps the filter: two candidates, one per page. + first = store.list_traces_page(project_id="proj_a", limit=1, label=("condition", "candidate")) + assert first["next_cursor"] is not None + second = store.list_traces_page( + project_id="proj_a", limit=1, cursor=first["next_cursor"], label=("condition", "candidate") + ) + assert {first["items"][0]["run_id"], second["items"][0]["run_id"]} == { + cand.run_id, + other.run_id, + } + assert second["next_cursor"] is None + + assert parse_label_filter("commit=a=b") == ("commit", "a=b") + for bad in ("no-equals", "=x", "bad key=1", "k=" + "v" * 501): + with pytest.raises(ValueError, match="KEY=VALUE"): + parse_label_filter(bad) + with pytest.raises(ValueError, match="KEY=VALUE"): + store.list_traces_page(label=("bad key", "x"))