Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ trace, study, receipt, API and database schema versions.

### Added

- Trace labels are visible and searchable. Paginated trace listings include
each trace's `metadata.labels`, and `GET /v1/traces?label=KEY=VALUE`,
`AgentLoopClient.list_traces(label=...)`, `agentloop list-stored-traces --label`
and a dashboard Traces filter return only traces with that label, in SQLite
and Postgres.
- CI can send traces to an AgentLoop server. `agentloop upload` accepts
several `--path` values and directories, validates every file before sending,
and adds `--label KEY=VALUE` pairs under `metadata.labels`. The reusable
Expand Down
40 changes: 28 additions & 12 deletions agentloop/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import json
import os
import re
import sys
from pathlib import Path

Expand Down Expand Up @@ -57,9 +56,12 @@
from agentloop.replay import ReplayGates, build_replay_report, replay_report_to_markdown
from agentloop.store import (
DEFAULT_PAGE_SIZE,
LABEL_KEY_PATTERN,
MAX_LABEL_VALUE_LENGTH,
FindingNotFoundError,
FindingTransitionError,
get_store,
parse_label_filter,
)
from agentloop.studies import StudyValidationError, study_to_markdown, summarize_study
from agentloop.timing import format_duration_ms
Expand Down Expand Up @@ -1086,14 +1088,24 @@ def list_stored_traces(
cursor: str | None = typer.Option(
None, help="Continuation cursor from a previous --limit call's printed next-cursor."
),
label: str | None = typer.Option(
None, help="Only traces with this KEY=VALUE label, such as commit=abc123."
),
) -> None:
db = get_store()
if limit is None and cursor is None:
try:
label_filter = parse_label_filter(label) if label is not None else None
except ValueError as exc:
raise typer.BadParameter(str(exc), param_hint="--label") from exc
if limit is None and cursor is None and label_filter is None:
traces = db.list_traces(project_id=project_id)
next_cursor = None
else:
page = db.list_traces_page(
project_id=project_id, limit=limit or DEFAULT_PAGE_SIZE, cursor=cursor
project_id=project_id,
limit=limit or DEFAULT_PAGE_SIZE,
cursor=cursor,
label=label_filter,
)
traces = page["items"]
next_cursor = page["next_cursor"]
Expand All @@ -1103,16 +1115,23 @@ def list_stored_traces(
table.add_column("Name")
table.add_column("Runtime ms")
table.add_column("Cost")
show_labels = any(item.get("labels") for item in traces)
if show_labels:
table.add_column("Labels")
for item in traces:
table.add_row(
row = [
str(item.get("project_id", "")),
str(item.get("run_id", "")),
str(item.get("name", "")),
"unavailable"
if item.get("total_runtime_ms") is None
else f"{float(item.get('total_runtime_ms', 0)):.2f}",
format_cost_usd(item.get("estimated_cost_usd"), item.get("cost_status", "complete")),
)
]
if show_labels:
labels = item.get("labels") or {}
row.append(", ".join(f"{key}={value}" for key, value in sorted(labels.items())))
table.add_row(*row, end_section=False)
console.print(table)
if next_cursor:
console.print(f"More results available. Next cursor: {next_cursor}")
Expand Down Expand Up @@ -1271,22 +1290,19 @@ def usage_summary(project_id: str | None = None) -> None:
console.print(table)


_LABEL_KEY = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")
_MAX_LABEL_VALUE = 500


def _parse_labels(values: list[str] | None) -> dict[str, str]:
labels: dict[str, str] = {}
for value in values or []:
key, separator, text = value.partition("=")
if not separator or not _LABEL_KEY.match(key):
if not separator or not LABEL_KEY_PATTERN.match(key):
raise typer.BadParameter(
"Use KEY=VALUE with a key of letters, digits, '.', '_' or '-'",
param_hint="--label",
)
if len(text) > _MAX_LABEL_VALUE:
if len(text) > MAX_LABEL_VALUE_LENGTH:
raise typer.BadParameter(
f"Label values are limited to {_MAX_LABEL_VALUE} characters", param_hint="--label"
f"Label values are limited to {MAX_LABEL_VALUE_LENGTH} characters",
param_hint="--label",
)
labels[key] = text
return labels
Expand Down
3 changes: 3 additions & 0 deletions agentloop/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ def list_traces(
*,
page_size: int | None = None,
cursor: str | None = None,
label: str | None = None,
) -> dict[str, Any]:
params: dict[str, Any] = {}
if project_id:
Expand All @@ -101,6 +102,8 @@ def list_traces(
params["page_size"] = page_size
if cursor:
params["cursor"] = cursor
if label is not None:
params["label"] = label
route = "/traces"
if params:
route += "?" + urllib.parse.urlencode(params)
Expand Down
13 changes: 11 additions & 2 deletions agentloop/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
TraceProjectConflictError,
TraceStore,
get_store,
parse_label_filter,
)
from agentloop.tracer import AgentTrace
from agentloop.value import build_value_report
Expand Down Expand Up @@ -236,6 +237,7 @@ def list_traces(
project_id_filter: str | None = Query(default=None, alias="project_id"),
page_size: int | None = Query(default=None, ge=1, le=MAX_PAGE_SIZE),
cursor: str | None = Query(default=None),
label: str | None = Query(default=None, description="Only traces with this KEY=VALUE label."),
project_id: str = Depends(resolve_project),
db: TraceStore = Depends(store),
) -> dict[str, Any]:
Expand All @@ -247,15 +249,22 @@ def list_traces(
fetch the next page; `next_cursor: null` means there are no more rows.
"""
selected_project = _selected_project(project_id_filter, project_id)
if page_size is None and cursor is None:
try:
label_filter = parse_label_filter(label) if label is not None else None
except ValueError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from None
if page_size is None and cursor is None and label_filter is None:
return {
"project_id": selected_project,
"traces": db.list_traces(project_id=selected_project),
"next_cursor": None,
}
try:
page = db.list_traces_page(
project_id=selected_project, limit=page_size or DEFAULT_PAGE_SIZE, cursor=cursor
project_id=selected_project,
limit=page_size or DEFAULT_PAGE_SIZE,
cursor=cursor,
label=label_filter,
)
except InvalidCursorError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from None
Expand Down
85 changes: 79 additions & 6 deletions agentloop/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import json
import math
import os
import re
import secrets
import sqlite3
from dataclasses import dataclass
Expand Down Expand Up @@ -119,6 +120,7 @@ def list_traces_page(
project_id: str | None = None,
limit: int = DEFAULT_PAGE_SIZE,
cursor: str | None = None,
label: tuple[str, str] | None = None,
) -> dict[str, Any]: ...

def get_trace(self, run_id: str, project_id: str | None = None) -> AgentTrace | None: ...
Expand Down Expand Up @@ -257,6 +259,48 @@ def _delete_runs(conn, mark: str, project_id: str, run_ids: list[str]) -> dict[s
return counts


LABEL_KEY_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")
MAX_LABEL_VALUE_LENGTH = 500


def validate_label_filter(label: Any) -> tuple[str, str]:
"""Check a (key, value) label filter; keys use letters, digits, '.', '_' and '-'."""
if (
not isinstance(label, tuple)
or len(label) != 2
or not all(isinstance(part, str) for part in label)
or not LABEL_KEY_PATTERN.match(label[0])
or len(label[1]) > MAX_LABEL_VALUE_LENGTH
):
raise ValueError(
"Label filters are KEY=VALUE with a key of letters, digits, '.', '_' or '-'"
)
return label


def parse_label_filter(text: str) -> tuple[str, str]:
"""Parse ``KEY=VALUE`` into a validated label filter."""
key, separator, value = text.partition("=") if isinstance(text, str) else ("", "", "")
if not separator:
raise ValueError(
"Label filters are KEY=VALUE with a key of letters, digits, '.', '_' or '-'"
)
return validate_label_filter((key, value))


def _with_labels(item: dict[str, Any]) -> dict[str, Any]:
labels = item.get("labels")
if isinstance(labels, str):
try:
labels = json.loads(labels)
except ValueError:
labels = None
item["labels"] = (
{str(key): value for key, value in labels.items()} if isinstance(labels, dict) else {}
)
return item


def _timestamp_text(value: Any) -> str | None:
if value is None:
return None
Expand Down Expand Up @@ -807,26 +851,40 @@ def list_traces_page(
project_id: str | None = None,
limit: int = DEFAULT_PAGE_SIZE,
cursor: str | None = None,
label: tuple[str, str] | None = None,
) -> dict[str, Any]:
after_created_at, after_run_id = (
decode_cursor(cursor, expected_parts=2) if cursor else (None, None)
)
limit = _validate_page_limit(limit)
label_key, label_value = validate_label_filter(label) if label else (None, None)
self.init()
with self._connect() as conn:
rows = conn.execute(
"""
SELECT run_id, project_id, name, started_at, total_runtime_ms, estimated_cost_usd,
cost_status, priced_model_call_count, unavailable_model_call_count, created_at
cost_status, priced_model_call_count, unavailable_model_call_count, created_at,
json_extract(payload_json, '$.metadata.labels') AS labels
FROM traces
WHERE project_id = COALESCE(?, project_id)
AND (? = 0 OR (created_at, run_id) < (?, ?))
AND (? IS NULL OR
json_extract(payload_json, '$.metadata.labels."' || ? || '"') = ?)
ORDER BY created_at DESC, run_id DESC
LIMIT ?
""",
(project_id, 1 if cursor else 0, after_created_at, after_run_id, limit + 1),
(
project_id,
1 if cursor else 0,
after_created_at,
after_run_id,
label_key,
label_key,
label_value,
limit + 1,
),
).fetchall()
items = [dict(row) for row in rows[:limit]]
items = [_with_labels(dict(row)) for row in rows[:limit]]
next_cursor = None
if len(rows) > limit:
last = items[-1]
Expand Down Expand Up @@ -1311,6 +1369,7 @@ def list_traces_page(
project_id: str | None = None,
limit: int = DEFAULT_PAGE_SIZE,
cursor: str | None = None,
label: tuple[str, str] | None = None,
) -> dict[str, Any]:
after_created_at, after_run_id = (
decode_cursor(cursor, expected_parts=2) if cursor else (None, None)
Expand All @@ -1327,22 +1386,36 @@ def list_traces_page(
"priced_model_call_count",
"unavailable_model_call_count",
"created_at",
"labels",
]
label_key, label_value = validate_label_filter(label) if label else (None, None)
self.init()
with self._connect() as conn:
rows = conn.execute(
"""
SELECT run_id, project_id, name, started_at, total_runtime_ms, estimated_cost_usd,
cost_status, priced_model_call_count, unavailable_model_call_count, created_at::text
cost_status, priced_model_call_count, unavailable_model_call_count,
created_at::text, payload_json -> 'metadata' -> 'labels'
FROM traces
WHERE project_id = COALESCE(%s, project_id)
AND (%s = 0 OR (created_at, run_id) < (%s, %s))
AND (%s::text IS NULL OR
payload_json -> 'metadata' -> 'labels' ->> %s::text = %s::text)
ORDER BY created_at DESC, run_id DESC
LIMIT %s
""",
(project_id, 1 if cursor else 0, after_created_at, after_run_id, limit + 1),
(
project_id,
1 if cursor else 0,
after_created_at,
after_run_id,
label_key,
label_key,
label_value,
limit + 1,
),
).fetchall()
items = [dict(zip(keys, row, strict=True)) for row in rows[:limit]]
items = [_with_labels(dict(zip(keys, row, strict=True))) for row in rows[:limit]]
next_cursor = None
if len(rows) > limit:
last = items[-1]
Expand Down
35 changes: 31 additions & 4 deletions dashboard/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
FindingNotFoundError,
FindingTransitionError,
get_store,
parse_label_filter,
)
from agentloop.tracer import AgentTrace
from agentloop.value import build_value_report
Expand Down Expand Up @@ -234,16 +235,38 @@ def render_gate_table(results: list[dict]) -> None:
if notice := st.session_state.pop("trace_deleted_notice", None):
st.success(notice)

cursor_key = f"traces_cursor_stack::{project_id}"
label_text = st.text_input(
"Filter by label",
placeholder="KEY=VALUE, for example commit=abc123 or condition=candidate",
).strip()
label_filter = None
if label_text:
try:
label_filter = parse_label_filter(label_text)
except ValueError as exc:
st.error(f"{exc}. Showing all traces.")

cursor_key = f"traces_cursor_stack::{project_id}::{label_filter}"
if cursor_key not in st.session_state:
st.session_state[cursor_key] = [None] # cursor to fetch the *current* page

page_index = len(st.session_state[cursor_key]) - 1
page_size = st.number_input("Page size", min_value=5, max_value=200, value=25, step=5)
page = store.list_traces_page(
project_id=project_id, limit=int(page_size), cursor=st.session_state[cursor_key][-1]
project_id=project_id,
limit=int(page_size),
cursor=st.session_state[cursor_key][-1],
label=label_filter,
)
traces = page["items"]
traces = [
{
**item,
"labels": ", ".join(
f"{key}={value}" for key, value in sorted((item.get("labels") or {}).items())
),
}
for item in page["items"]
]

nav_cols = st.columns([1, 1, 6])
if nav_cols[0].button("◀ Previous", disabled=page_index == 0):
Expand All @@ -255,7 +278,11 @@ def render_gate_table(results: list[dict]) -> None:
st.caption(f"Page {page_index + 1}")

if not traces:
st.info("No traces stored for this project yet.")
st.info(
f"No traces labelled {label_filter[0]}={label_filter[1]} in this project."
if label_filter
else "No traces stored for this project yet."
)
else:
df = pd.DataFrame(traces)
st.dataframe(df, width="stretch")
Expand Down
2 changes: 1 addition & 1 deletion docs/API_VERSIONING.md
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ for larger reports. `timeout_s` controls the underlying socket timeout.
| `POST /v1/api-keys` | `POST /api-keys` |
| `GET /v1/api-keys?project_id=...` | None (0.2.7 and newer) |
| `DELETE /v1/api-keys/{prefix}?project_id=...` | None (0.2.7 and newer) |
| `GET/POST /v1/traces` | `GET/POST /traces` |
| `GET/POST /v1/traces` (`?label=KEY=VALUE` from 0.2.8) | `GET/POST /traces` |
| `DELETE /v1/traces/{run_id}` | None (0.2.7 and newer) |
| `GET /v1/traces/{run_id}/report` | `GET /traces/{run_id}/report` |
| `GET /v1/traces/{run_id}/optimization` | `GET /traces/{run_id}/optimize` |
Expand Down
Loading
Loading