Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,7 @@ jobs:
echo "POSTGRES_PASSWORD=$db_password"
echo "POSTGRES_IMAGE=$POSTGRES_IMAGE"
echo "CADDY_IMAGE=$CADDY_IMAGE"
echo "AGENTLOOP_RETENTION_DAYS=365"
} > deploy/.env
echo "DASHBOARD_PASSWORD=$dashboard_password" >> "$GITHUB_ENV"
echo "AGENTLOOP_ADMIN_API_KEY=$admin_key" >> "$GITHUB_ENV"
Expand Down Expand Up @@ -693,6 +694,33 @@ jobs:
curl --fail --silent --output /dev/null --cacert caddy-root.crt \
--user "operator:$DASHBOARD_PASSWORD" https://dashboard.localhost/

- name: Verify trace deletion and the retention service
shell: bash
run: |
api() { curl --silent --show-error --cacert caddy-root.crt "$@"; }
key="$(api --fail -X POST https://api.localhost/v1/api-keys \
-H "X-AgentLoop-Admin-Key: $AGENTLOOP_ADMIN_API_KEY" -H 'Content-Type: application/json' \
-d '{"project_id": "retention-check", "name": "ci"}' \
| python3 -c 'import json, sys; print(json.load(sys.stdin)["api_key"])')"
echo "::add-mask::$key"
api --fail -X POST https://api.localhost/v1/traces -H "X-AgentLoop-Key: $key" \
-H 'Content-Type: application/json' \
-d '{"name": "delete-me", "run_id": "ci-delete-me", "metadata": {}, "events": []}' > /dev/null
status="$(api --output /dev/null --write-out '%{http_code}' -X DELETE \
https://api.localhost/v1/traces/ci-delete-me -H "X-AgentLoop-Key: $key")"
test "$status" = 200
status="$(api --output /dev/null --write-out '%{http_code}' \
https://api.localhost/v1/traces/ci-delete-me/report -H "X-AgentLoop-Key: $key")"
test "$status" = 404
for attempt in $(seq 1 20); do
if docker compose logs retention | grep -q "older than 365 days in all projects"; then
break
fi
sleep 3
done
docker compose logs retention | grep "older than 365 days in all projects"
docker compose exec -T retention agentloop purge-traces --older-than-days 3650

- name: Verify only the proxy is published
shell: bash
run: |
Expand Down
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@ trace, study, receipt, API and database schema versions.

### Added

- Stored traces can be deleted, and a retention period enforced. Deleting a
trace also removes its findings, usage records and interventions that compare
it. Use `DELETE /v1/traces/{run_id}` with a project key (own project only),
`AgentLoopClient.delete_trace`, `agentloop delete-trace` /
`remote-delete-trace`, or the dashboard's confirmed Delete control.
`agentloop purge-traces --older-than-days N` (or `AGENTLOOP_RETENTION_DAYS`)
removes older traces, and the cloud stack's new `retention` service runs it
daily.
- Project API keys can be listed and revoked. Admin routes
`GET /v1/api-keys?project_id=` and `DELETE /v1/api-keys/{prefix}?project_id=`,
`AgentLoopClient.list_api_keys` / `revoke_api_key`, and the CLI commands
Expand Down
66 changes: 66 additions & 0 deletions agentloop/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -1014,6 +1014,59 @@ def revoke_api_key(
)


@app.command("delete-trace")
def delete_trace(
run_id: str,
project_id: str = "default",
) -> None:
"""Delete a stored trace with its findings, usage and interventions."""
deleted = get_store().delete_trace(run_id, project_id)
if deleted is None:
console.print(f"No trace {run_id} in project {project_id}.", markup=False)
raise typer.Exit(1)
console.print_json(data={"run_id": run_id, "project_id": project_id, "deleted": deleted})


@app.command("purge-traces")
def purge_traces(
older_than_days: float | None = typer.Option(
None,
min=0.0001,
help="Delete traces stored more than this many days ago. "
"Defaults to AGENTLOOP_RETENTION_DAYS.",
),
project_id: str | None = typer.Option(None, help="Only purge this project."),
) -> None:
"""Apply a retention period: delete traces stored longer ago than allowed."""
if older_than_days is None:
configured = os.getenv("AGENTLOOP_RETENTION_DAYS", "").strip()
if not configured:
console.print(
"Retention is not configured; set --older-than-days or AGENTLOOP_RETENTION_DAYS.",
markup=False,
)
return
try:
older_than_days = float(configured)
except ValueError:
raise typer.BadParameter(
"AGENTLOOP_RETENTION_DAYS must be a number of days",
param_hint="AGENTLOOP_RETENTION_DAYS",
) from None
try:
deleted = get_store().purge_traces(older_than_days, project_id=project_id)
except ValueError as exc:
raise typer.BadParameter(str(exc)) from exc
scope = f"project {project_id}" if project_id else "all projects"
console.print(
f"Deleted {deleted['traces']} trace(s) older than {older_than_days:g} days in {scope} "
f"({deleted['findings']} findings, {deleted['usage_events']} usage events, "
f"{deleted['interventions']} interventions).",
markup=False,
soft_wrap=True,
)


@app.command("store-trace")
def store_trace(
path: Path = Path("runs/research_agent_baseline.json"),
Expand Down Expand Up @@ -1232,6 +1285,19 @@ def upload(
console.print(f"Project: {response.get('project_id', 'default')}")


@app.command("remote-delete-trace")
def remote_delete_trace(
run_id: str,
api_url: str = typer.Option("http://127.0.0.1:8000", help="AgentLoop API base URL."),
api_key: str | None = typer.Option(
None, help="Project API key. Defaults to AGENTLOOP_API_KEY."
),
) -> None:
"""Delete a trace of the key's project on a remote server."""
client = _remote_client(api_url, api_key)
console.print_json(data=client.delete_trace(run_id))


@app.command("remote-optimize")
def remote_optimize(
run_id: str,
Expand Down
5 changes: 5 additions & 0 deletions agentloop/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,11 @@ def usage_summary(self, project_id: str | None = None) -> dict[str, Any]:
route += "?" + urllib.parse.urlencode({"project_id": project_id})
return self._request("GET", route)

def delete_trace(self, run_id: str) -> dict[str, Any]:
"""Delete a trace of this key's project, with its findings, usage and interventions."""
encoded_run_id = urllib.parse.quote(run_id, safe="")
return self._request("DELETE", f"/traces/{encoded_run_id}")

def get_report(self, run_id: str) -> dict[str, Any]:
encoded_run_id = urllib.parse.quote(run_id, safe="")
return self._request("GET", f"/traces/{encoded_run_id}/report")
Expand Down
13 changes: 13 additions & 0 deletions agentloop/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,19 @@ def get_report(
return _load_trace_or_404(db, run_id, project_id).report()


@versioned.delete("/traces/{run_id}")
def delete_trace(
run_id: str,
project_id: str = Depends(resolve_project),
db: TraceStore = Depends(store),
) -> dict[str, Any]:
"""Delete a trace of the caller's project with its findings, usage and interventions."""
deleted = db.delete_trace(run_id, project_id)
if deleted is None:
raise HTTPException(status_code=404, detail="trace not found")
return {"run_id": run_id, "project_id": project_id, "deleted": deleted}


@versioned.get("/traces/{run_id}/optimization")
@legacy.get("/traces/{run_id}/optimize")
def optimize_trace(
Expand Down
121 changes: 121 additions & 0 deletions agentloop/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import secrets
import sqlite3
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Protocol

Expand Down Expand Up @@ -122,6 +123,12 @@ def list_traces_page(

def get_trace(self, run_id: str, project_id: str | None = None) -> AgentTrace | None: ...

def delete_trace(self, run_id: str, project_id: str) -> dict[str, int] | None: ...

def purge_traces(
self, older_than_days: float, project_id: str | None = None
) -> dict[str, int]: ...

def save_diagnosis(self, diagnosis: dict[str, Any], project_id: str = "default") -> None: ...

def list_findings(
Expand Down Expand Up @@ -196,6 +203,60 @@ def _decode_hash_bytes(value: str) -> bytes:
return base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True)


_DELETE_CHUNK = 500


def _retention_cutoff(older_than_days: Any) -> datetime:
if (
isinstance(older_than_days, bool)
or not isinstance(older_than_days, (int, float))
or not math.isfinite(older_than_days)
or older_than_days <= 0
):
raise ValueError("Retention must be a positive number of days")
return datetime.now(timezone.utc) - timedelta(days=float(older_than_days))


def _delete_runs(conn, mark: str, project_id: str, run_ids: list[str]) -> dict[str, int]:
"""Delete traces and everything derived from them, children first.

Interventions that compare a deleted trace go too, since their evidence is
gone. Counts are per table.
"""
counts = {"traces": 0, "findings": 0, "usage_events": 0, "interventions": 0}
for start in range(0, len(run_ids), _DELETE_CHUNK):
chunk = run_ids[start : start + _DELETE_CHUNK]
marks = ", ".join([mark] * len(chunk))
statements = (
(
"interventions",
f"DELETE FROM interventions WHERE project_id = {mark} AND " # nosec B608
f"(baseline_run_id IN ({marks}) OR candidate_run_id IN ({marks}))",
(project_id, *chunk, *chunk),
),
(
"findings",
f"DELETE FROM trace_findings WHERE project_id = {mark} AND run_id IN ({marks})", # nosec B608
(project_id, *chunk),
),
(
"usage_events",
f"DELETE FROM usage_events WHERE project_id = {mark} AND run_id IN ({marks})", # nosec B608
(project_id, *chunk),
),
(
"traces",
f"DELETE FROM traces WHERE project_id = {mark} AND run_id IN ({marks})", # nosec B608
(project_id, *chunk),
),
)
for name, sql, params in statements:
# The statements above interpolate only "?"/"%s" placeholders, never values.
cursor = conn.execute(sql, params)
counts[name] += int(cursor.rowcount or 0)
return counts


def _timestamp_text(value: Any) -> str | None:
if value is None:
return None
Expand Down Expand Up @@ -527,6 +588,36 @@ def revoke_api_key(self, project_id: str, prefix: str) -> int:
)
return int(cursor.rowcount or 0)

def delete_trace(self, run_id: str, project_id: str) -> dict[str, int] | None:
"""Delete one trace of a project and its findings, usage and interventions."""
self.init()
with self._connect() as conn:
exists = conn.execute(
"SELECT 1 FROM traces WHERE run_id = ? AND project_id = ?", (run_id, project_id)
).fetchone()
if exists is None:
return None
return _delete_runs(conn, "?", project_id, [run_id])

def purge_traces(self, older_than_days: float, project_id: str | None = None) -> dict[str, int]:
"""Delete traces stored more than ``older_than_days`` ago, optionally in one project."""
cutoff = _retention_cutoff(older_than_days).strftime("%Y-%m-%d %H:%M:%S")
self.init()
totals = {"traces": 0, "findings": 0, "usage_events": 0, "interventions": 0}
with self._connect() as conn:
query = "SELECT project_id, run_id FROM traces WHERE created_at < ?"
params: tuple[Any, ...] = (cutoff,)
if project_id:
query += " AND project_id = ?"
params += (project_id,)
by_project: dict[str, list[str]] = {}
for row in conn.execute(query, params).fetchall():
by_project.setdefault(row["project_id"], []).append(row["run_id"])
for project, run_ids in by_project.items():
for name, count in _delete_runs(conn, "?", project, run_ids).items():
totals[name] += count
return totals

def _upsert_trace(
self, conn: sqlite3.Connection, trace: AgentTrace, project_id: str
) -> dict[str, Any]:
Expand Down Expand Up @@ -1021,6 +1112,36 @@ def revoke_api_key(self, project_id: str, prefix: str) -> int:
)
return int(cursor.rowcount or 0)

def delete_trace(self, run_id: str, project_id: str) -> dict[str, int] | None:
"""Delete one trace of a project and its findings, usage and interventions."""
self.init()
with self._connect() as conn:
exists = conn.execute(
"SELECT 1 FROM traces WHERE run_id = %s AND project_id = %s", (run_id, project_id)
).fetchone()
if exists is None:
return None
return _delete_runs(conn, "%s", project_id, [run_id])

def purge_traces(self, older_than_days: float, project_id: str | None = None) -> dict[str, int]:
"""Delete traces stored more than ``older_than_days`` ago, optionally in one project."""
cutoff = _retention_cutoff(older_than_days)
self.init()
totals = {"traces": 0, "findings": 0, "usage_events": 0, "interventions": 0}
with self._connect() as conn:
query = "SELECT project_id, run_id FROM traces WHERE created_at < %s"
params: tuple[Any, ...] = (cutoff,)
if project_id:
query += " AND project_id = %s"
params += (project_id,)
by_project: dict[str, list[str]] = {}
for row in conn.execute(query, params).fetchall():
by_project.setdefault(row[0], []).append(row[1])
for project, run_ids in by_project.items():
for name, count in _delete_runs(conn, "%s", project, run_ids).items():
totals[name] += count
return totals

def _upsert_trace(self, conn, trace: AgentTrace, project_id: str) -> dict[str, Any]:
report = trace.report()
cost_status, priced_calls, unavailable_calls = _report_cost_completeness(report)
Expand Down
21 changes: 21 additions & 0 deletions dashboard/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,8 @@ def render_gate_table(results: list[dict]) -> None:

elif page == "Traces":
st.subheader("Stored traces")
if notice := st.session_state.pop("trace_deleted_notice", None):
st.success(notice)

cursor_key = f"traces_cursor_stack::{project_id}"
if cursor_key not in st.session_state:
Expand Down Expand Up @@ -334,6 +336,25 @@ def render_gate_table(results: list[dict]) -> None:
mime="application/json",
)

with st.expander("Delete this trace"):
st.caption(
"Permanently removes the trace with its findings, usage records and any "
"interventions that compare it. This cannot be undone."
)
confirmed = st.checkbox(
f"I want to delete {trace.run_id}", key=f"confirm_delete::{trace.run_id}"
)
if st.button("Delete trace", disabled=not confirmed, type="primary"):
deleted = store.delete_trace(trace.run_id, project_id)
st.session_state[cursor_key] = [None]
st.session_state["trace_deleted_notice"] = (
f"Deleted {trace.run_id} and {deleted['findings']} findings, "
f"{deleted['interventions']} interventions."
if deleted
else f"{trace.run_id} was already deleted."
)
st.rerun()

elif page == "Optimization Queue":
st.subheader("Optimization queue")
dimension = st.selectbox(
Expand Down
4 changes: 4 additions & 0 deletions deploy/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -22,5 +22,9 @@ POSTGRES_DB=agentloop
POSTGRES_USER=agentloop
POSTGRES_PASSWORD=replace-with-a-long-random-db-password

# Delete traces stored more than this many days ago (checked daily).
# Leave empty to keep traces until they are deleted by hand.
AGENTLOOP_RETENTION_DAYS=

# Largest trace upload the proxy accepts.
AGENTLOOP_MAX_UPLOAD=16MB
Loading
Loading