From 8f3689b0465c67dc6474dbc9c34dd21911d4b6e8 Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Fri, 20 Nov 2026 02:32:02 -0500 Subject: [PATCH] Delete stored traces and enforce a retention period A shared server could store traces but never remove them. Both stores gain delete_trace (project-scoped, removing findings, usage records and interventions that compare the trace) and purge_traces (by storage age, optionally per project). They are exposed as DELETE /v1/traces/{run_id} for a project key, AgentLoopClient.delete_trace, delete-trace, remote-delete-trace and purge-traces CLI commands (purge reads AGENTLOOP_RETENTION_DAYS when no age is given), and a confirmed delete control on the dashboard Traces page. The cloud stack adds a retention service that purges daily; CI deletes a trace over HTTPS and checks it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 28 ++++++ CHANGELOG.md | 8 ++ agentloop/cli.py | 66 ++++++++++++++ agentloop/client.py | 5 + agentloop/server.py | 13 +++ agentloop/store.py | 121 +++++++++++++++++++++++++ dashboard/app.py | 21 +++++ deploy/.env.example | 4 + deploy/docker-compose.yml | 25 +++++ docs/API_VERSIONING.md | 1 + docs/CLOUD_DEPLOY.md | 23 ++++- docs/DASHBOARD.md | 3 +- docs/PRODUCTION.md | 23 ++++- docs/TRACE_RETENTION.md | 4 + tests/test_cli_store_commands.py | 32 +++++++ tests/test_client.py | 28 ++++++ tests/test_dashboard_project_access.py | 16 ++++ tests/test_server.py | 28 ++++++ tests/test_store_contract.py | 91 +++++++++++++++++++ 19 files changed, 536 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afdc642..3419068 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -639,6 +639,7 @@ jobs: echo "POSTGRES_PASSWORD=$db_password" echo "POSTGRES_IMAGE=$POSTGRES_IMAGE" echo "CADDY_IMAGE=$CADDY_IMAGE" + echo "AGENTLOOP_RETENTION_DAYS=365" } > deploy/.env echo "DASHBOARD_PASSWORD=$dashboard_password" >> "$GITHUB_ENV" echo "AGENTLOOP_ADMIN_API_KEY=$admin_key" >> "$GITHUB_ENV" @@ -693,6 +694,33 @@ jobs: curl --fail --silent --output /dev/null --cacert caddy-root.crt \ --user "operator:$DASHBOARD_PASSWORD" https://dashboard.localhost/ + - name: Verify trace deletion and the retention service + shell: bash + run: | + api() { curl --silent --show-error --cacert caddy-root.crt "$@"; } + key="$(api --fail -X POST https://api.localhost/v1/api-keys \ + -H "X-AgentLoop-Admin-Key: $AGENTLOOP_ADMIN_API_KEY" -H 'Content-Type: application/json' \ + -d '{"project_id": "retention-check", "name": "ci"}' \ + | python3 -c 'import json, sys; print(json.load(sys.stdin)["api_key"])')" + echo "::add-mask::$key" + api --fail -X POST https://api.localhost/v1/traces -H "X-AgentLoop-Key: $key" \ + -H 'Content-Type: application/json' \ + -d '{"name": "delete-me", "run_id": "ci-delete-me", "metadata": {}, "events": []}' > /dev/null + status="$(api --output /dev/null --write-out '%{http_code}' -X DELETE \ + https://api.localhost/v1/traces/ci-delete-me -H "X-AgentLoop-Key: $key")" + test "$status" = 200 + status="$(api --output /dev/null --write-out '%{http_code}' \ + https://api.localhost/v1/traces/ci-delete-me/report -H "X-AgentLoop-Key: $key")" + test "$status" = 404 + for attempt in $(seq 1 20); do + if docker compose logs retention | grep -q "older than 365 days in all projects"; then + break + fi + sleep 3 + done + docker compose logs retention | grep "older than 365 days in all projects" + docker compose exec -T retention agentloop purge-traces --older-than-days 3650 + - name: Verify only the proxy is published shell: bash run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index caaf633..1b81cb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ trace, study, receipt, API and database schema versions. ### Added +- Stored traces can be deleted, and a retention period enforced. Deleting a + trace also removes its findings, usage records and interventions that compare + it. Use `DELETE /v1/traces/{run_id}` with a project key (own project only), + `AgentLoopClient.delete_trace`, `agentloop delete-trace` / + `remote-delete-trace`, or the dashboard's confirmed Delete control. + `agentloop purge-traces --older-than-days N` (or `AGENTLOOP_RETENTION_DAYS`) + removes older traces, and the cloud stack's new `retention` service runs it + daily. - Project API keys can be listed and revoked. Admin routes `GET /v1/api-keys?project_id=` and `DELETE /v1/api-keys/{prefix}?project_id=`, `AgentLoopClient.list_api_keys` / `revoke_api_key`, and the CLI commands diff --git a/agentloop/cli.py b/agentloop/cli.py index 175811f..ed771c8 100644 --- a/agentloop/cli.py +++ b/agentloop/cli.py @@ -1014,6 +1014,59 @@ def revoke_api_key( ) +@app.command("delete-trace") +def delete_trace( + run_id: str, + project_id: str = "default", +) -> None: + """Delete a stored trace with its findings, usage and interventions.""" + deleted = get_store().delete_trace(run_id, project_id) + if deleted is None: + console.print(f"No trace {run_id} in project {project_id}.", markup=False) + raise typer.Exit(1) + console.print_json(data={"run_id": run_id, "project_id": project_id, "deleted": deleted}) + + +@app.command("purge-traces") +def purge_traces( + older_than_days: float | None = typer.Option( + None, + min=0.0001, + help="Delete traces stored more than this many days ago. " + "Defaults to AGENTLOOP_RETENTION_DAYS.", + ), + project_id: str | None = typer.Option(None, help="Only purge this project."), +) -> None: + """Apply a retention period: delete traces stored longer ago than allowed.""" + if older_than_days is None: + configured = os.getenv("AGENTLOOP_RETENTION_DAYS", "").strip() + if not configured: + console.print( + "Retention is not configured; set --older-than-days or AGENTLOOP_RETENTION_DAYS.", + markup=False, + ) + return + try: + older_than_days = float(configured) + except ValueError: + raise typer.BadParameter( + "AGENTLOOP_RETENTION_DAYS must be a number of days", + param_hint="AGENTLOOP_RETENTION_DAYS", + ) from None + try: + deleted = get_store().purge_traces(older_than_days, project_id=project_id) + except ValueError as exc: + raise typer.BadParameter(str(exc)) from exc + scope = f"project {project_id}" if project_id else "all projects" + console.print( + f"Deleted {deleted['traces']} trace(s) older than {older_than_days:g} days in {scope} " + f"({deleted['findings']} findings, {deleted['usage_events']} usage events, " + f"{deleted['interventions']} interventions).", + markup=False, + soft_wrap=True, + ) + + @app.command("store-trace") def store_trace( path: Path = Path("runs/research_agent_baseline.json"), @@ -1232,6 +1285,19 @@ def upload( console.print(f"Project: {response.get('project_id', 'default')}") +@app.command("remote-delete-trace") +def remote_delete_trace( + run_id: str, + api_url: str = typer.Option("http://127.0.0.1:8000", help="AgentLoop API base URL."), + api_key: str | None = typer.Option( + None, help="Project API key. Defaults to AGENTLOOP_API_KEY." + ), +) -> None: + """Delete a trace of the key's project on a remote server.""" + client = _remote_client(api_url, api_key) + console.print_json(data=client.delete_trace(run_id)) + + @app.command("remote-optimize") def remote_optimize( run_id: str, diff --git a/agentloop/client.py b/agentloop/client.py index 9ea79f6..38e2581 100644 --- a/agentloop/client.py +++ b/agentloop/client.py @@ -112,6 +112,11 @@ def usage_summary(self, project_id: str | None = None) -> dict[str, Any]: route += "?" + urllib.parse.urlencode({"project_id": project_id}) return self._request("GET", route) + def delete_trace(self, run_id: str) -> dict[str, Any]: + """Delete a trace of this key's project, with its findings, usage and interventions.""" + encoded_run_id = urllib.parse.quote(run_id, safe="") + return self._request("DELETE", f"/traces/{encoded_run_id}") + def get_report(self, run_id: str) -> dict[str, Any]: encoded_run_id = urllib.parse.quote(run_id, safe="") return self._request("GET", f"/traces/{encoded_run_id}/report") diff --git a/agentloop/server.py b/agentloop/server.py index 7763689..89da4e9 100644 --- a/agentloop/server.py +++ b/agentloop/server.py @@ -283,6 +283,19 @@ def get_report( return _load_trace_or_404(db, run_id, project_id).report() +@versioned.delete("/traces/{run_id}") +def delete_trace( + run_id: str, + project_id: str = Depends(resolve_project), + db: TraceStore = Depends(store), +) -> dict[str, Any]: + """Delete a trace of the caller's project with its findings, usage and interventions.""" + deleted = db.delete_trace(run_id, project_id) + if deleted is None: + raise HTTPException(status_code=404, detail="trace not found") + return {"run_id": run_id, "project_id": project_id, "deleted": deleted} + + @versioned.get("/traces/{run_id}/optimization") @legacy.get("/traces/{run_id}/optimize") def optimize_trace( diff --git a/agentloop/store.py b/agentloop/store.py index 32f14c3..95f7794 100644 --- a/agentloop/store.py +++ b/agentloop/store.py @@ -10,6 +10,7 @@ import secrets import sqlite3 from dataclasses import dataclass +from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Protocol @@ -122,6 +123,12 @@ def list_traces_page( def get_trace(self, run_id: str, project_id: str | None = None) -> AgentTrace | None: ... + def delete_trace(self, run_id: str, project_id: str) -> dict[str, int] | None: ... + + def purge_traces( + self, older_than_days: float, project_id: str | None = None + ) -> dict[str, int]: ... + def save_diagnosis(self, diagnosis: dict[str, Any], project_id: str = "default") -> None: ... def list_findings( @@ -196,6 +203,60 @@ def _decode_hash_bytes(value: str) -> bytes: return base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True) +_DELETE_CHUNK = 500 + + +def _retention_cutoff(older_than_days: Any) -> datetime: + if ( + isinstance(older_than_days, bool) + or not isinstance(older_than_days, (int, float)) + or not math.isfinite(older_than_days) + or older_than_days <= 0 + ): + raise ValueError("Retention must be a positive number of days") + return datetime.now(timezone.utc) - timedelta(days=float(older_than_days)) + + +def _delete_runs(conn, mark: str, project_id: str, run_ids: list[str]) -> dict[str, int]: + """Delete traces and everything derived from them, children first. + + Interventions that compare a deleted trace go too, since their evidence is + gone. Counts are per table. + """ + counts = {"traces": 0, "findings": 0, "usage_events": 0, "interventions": 0} + for start in range(0, len(run_ids), _DELETE_CHUNK): + chunk = run_ids[start : start + _DELETE_CHUNK] + marks = ", ".join([mark] * len(chunk)) + statements = ( + ( + "interventions", + f"DELETE FROM interventions WHERE project_id = {mark} AND " # nosec B608 + f"(baseline_run_id IN ({marks}) OR candidate_run_id IN ({marks}))", + (project_id, *chunk, *chunk), + ), + ( + "findings", + f"DELETE FROM trace_findings WHERE project_id = {mark} AND run_id IN ({marks})", # nosec B608 + (project_id, *chunk), + ), + ( + "usage_events", + f"DELETE FROM usage_events WHERE project_id = {mark} AND run_id IN ({marks})", # nosec B608 + (project_id, *chunk), + ), + ( + "traces", + f"DELETE FROM traces WHERE project_id = {mark} AND run_id IN ({marks})", # nosec B608 + (project_id, *chunk), + ), + ) + for name, sql, params in statements: + # The statements above interpolate only "?"/"%s" placeholders, never values. + cursor = conn.execute(sql, params) + counts[name] += int(cursor.rowcount or 0) + return counts + + def _timestamp_text(value: Any) -> str | None: if value is None: return None @@ -527,6 +588,36 @@ def revoke_api_key(self, project_id: str, prefix: str) -> int: ) return int(cursor.rowcount or 0) + def delete_trace(self, run_id: str, project_id: str) -> dict[str, int] | None: + """Delete one trace of a project and its findings, usage and interventions.""" + self.init() + with self._connect() as conn: + exists = conn.execute( + "SELECT 1 FROM traces WHERE run_id = ? AND project_id = ?", (run_id, project_id) + ).fetchone() + if exists is None: + return None + return _delete_runs(conn, "?", project_id, [run_id]) + + def purge_traces(self, older_than_days: float, project_id: str | None = None) -> dict[str, int]: + """Delete traces stored more than ``older_than_days`` ago, optionally in one project.""" + cutoff = _retention_cutoff(older_than_days).strftime("%Y-%m-%d %H:%M:%S") + self.init() + totals = {"traces": 0, "findings": 0, "usage_events": 0, "interventions": 0} + with self._connect() as conn: + query = "SELECT project_id, run_id FROM traces WHERE created_at < ?" + params: tuple[Any, ...] = (cutoff,) + if project_id: + query += " AND project_id = ?" + params += (project_id,) + by_project: dict[str, list[str]] = {} + for row in conn.execute(query, params).fetchall(): + by_project.setdefault(row["project_id"], []).append(row["run_id"]) + for project, run_ids in by_project.items(): + for name, count in _delete_runs(conn, "?", project, run_ids).items(): + totals[name] += count + return totals + def _upsert_trace( self, conn: sqlite3.Connection, trace: AgentTrace, project_id: str ) -> dict[str, Any]: @@ -1021,6 +1112,36 @@ def revoke_api_key(self, project_id: str, prefix: str) -> int: ) return int(cursor.rowcount or 0) + def delete_trace(self, run_id: str, project_id: str) -> dict[str, int] | None: + """Delete one trace of a project and its findings, usage and interventions.""" + self.init() + with self._connect() as conn: + exists = conn.execute( + "SELECT 1 FROM traces WHERE run_id = %s AND project_id = %s", (run_id, project_id) + ).fetchone() + if exists is None: + return None + return _delete_runs(conn, "%s", project_id, [run_id]) + + def purge_traces(self, older_than_days: float, project_id: str | None = None) -> dict[str, int]: + """Delete traces stored more than ``older_than_days`` ago, optionally in one project.""" + cutoff = _retention_cutoff(older_than_days) + self.init() + totals = {"traces": 0, "findings": 0, "usage_events": 0, "interventions": 0} + with self._connect() as conn: + query = "SELECT project_id, run_id FROM traces WHERE created_at < %s" + params: tuple[Any, ...] = (cutoff,) + if project_id: + query += " AND project_id = %s" + params += (project_id,) + by_project: dict[str, list[str]] = {} + for row in conn.execute(query, params).fetchall(): + by_project.setdefault(row[0], []).append(row[1]) + for project, run_ids in by_project.items(): + for name, count in _delete_runs(conn, "%s", project, run_ids).items(): + totals[name] += count + return totals + def _upsert_trace(self, conn, trace: AgentTrace, project_id: str) -> dict[str, Any]: report = trace.report() cost_status, priced_calls, unavailable_calls = _report_cost_completeness(report) diff --git a/dashboard/app.py b/dashboard/app.py index 018d877..a8d4253 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -231,6 +231,8 @@ def render_gate_table(results: list[dict]) -> None: elif page == "Traces": st.subheader("Stored traces") + if notice := st.session_state.pop("trace_deleted_notice", None): + st.success(notice) cursor_key = f"traces_cursor_stack::{project_id}" if cursor_key not in st.session_state: @@ -334,6 +336,25 @@ def render_gate_table(results: list[dict]) -> None: mime="application/json", ) + with st.expander("Delete this trace"): + st.caption( + "Permanently removes the trace with its findings, usage records and any " + "interventions that compare it. This cannot be undone." + ) + confirmed = st.checkbox( + f"I want to delete {trace.run_id}", key=f"confirm_delete::{trace.run_id}" + ) + if st.button("Delete trace", disabled=not confirmed, type="primary"): + deleted = store.delete_trace(trace.run_id, project_id) + st.session_state[cursor_key] = [None] + st.session_state["trace_deleted_notice"] = ( + f"Deleted {trace.run_id} and {deleted['findings']} findings, " + f"{deleted['interventions']} interventions." + if deleted + else f"{trace.run_id} was already deleted." + ) + st.rerun() + elif page == "Optimization Queue": st.subheader("Optimization queue") dimension = st.selectbox( diff --git a/deploy/.env.example b/deploy/.env.example index 8ddac02..b269f88 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -22,5 +22,9 @@ POSTGRES_DB=agentloop POSTGRES_USER=agentloop POSTGRES_PASSWORD=replace-with-a-long-random-db-password +# Delete traces stored more than this many days ago (checked daily). +# Leave empty to keep traces until they are deleted by hand. +AGENTLOOP_RETENTION_DAYS= + # Largest trace upload the proxy accepts. AGENTLOOP_MAX_UPLOAD=16MB diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 8ba11e9..1e6df1c 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -88,6 +88,31 @@ services: secrets: - postgres_password + retention: + # Deletes traces stored longer ago than AGENTLOOP_RETENTION_DAYS, once a day. + # Leave the setting empty to keep traces until they are deleted by hand. + image: ${AGENTLOOP_IMAGE} + restart: unless-stopped + command: + [ + "sh", + "-c", + "trap 'exit 0' TERM INT; while true; do agentloop purge-traces; sleep 86400 & wait $$!; done", + ] + environment: + AGENTLOOP_STORE_BACKEND: postgres + PGHOST: db + PGPORT: "5432" + PGDATABASE: ${POSTGRES_DB:-agentloop} + PGUSER: ${POSTGRES_USER:-agentloop} + AGENTLOOP_POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + AGENTLOOP_RETENTION_DAYS: ${AGENTLOOP_RETENTION_DAYS:-} + depends_on: + api: + condition: service_healthy + secrets: + - postgres_password + caddy: image: ${CADDY_IMAGE:-caddy:2} restart: unless-stopped diff --git a/docs/API_VERSIONING.md b/docs/API_VERSIONING.md index 1ff7cda..0731f4e 100644 --- a/docs/API_VERSIONING.md +++ b/docs/API_VERSIONING.md @@ -34,6 +34,7 @@ for larger reports. `timeout_s` controls the underlying socket timeout. | `GET /v1/api-keys?project_id=...` | None (0.2.7 and newer) | | `DELETE /v1/api-keys/{prefix}?project_id=...` | None (0.2.7 and newer) | | `GET/POST /v1/traces` | `GET/POST /traces` | +| `DELETE /v1/traces/{run_id}` | None (0.2.7 and newer) | | `GET /v1/traces/{run_id}/report` | `GET /traces/{run_id}/report` | | `GET /v1/traces/{run_id}/optimization` | `GET /traces/{run_id}/optimize` | | `GET/POST /v1/traces/{run_id}/diagnosis` | `GET/POST /traces/{run_id}/diagnosis` | diff --git a/docs/CLOUD_DEPLOY.md b/docs/CLOUD_DEPLOY.md index c9537a9..2cc632b 100644 --- a/docs/CLOUD_DEPLOY.md +++ b/docs/CLOUD_DEPLOY.md @@ -41,6 +41,8 @@ The `deploy/` directory contains the stack: - `deploy/Caddyfile` serves the API on one hostname and the dashboard on another, with a login in front of the dashboard. Inside, the dashboard also asks for a project API key and shows only that project; +- a `retention` service deletes traces older than `AGENTLOOP_RETENTION_DAYS` + once a day, when that is set; - `deploy/.env.example` lists every setting. Only ports 80 and 443 are published. The API, dashboard and database are @@ -125,6 +127,23 @@ agentloop remote-revoke-api-key --project-id my-team --prefix al_AbCdEfG \ The API rejects the key immediately, and dashboard sessions signed in with it end within a minute. +### Delete traces and set a retention period + +Set `AGENTLOOP_RETENTION_DAYS` in `deploy/.env` to delete traces once they are +older than that many days. The stack's `retention` service applies it once a day; +leave it empty to keep traces until they are deleted by hand. After changing it, +run `docker compose -f deploy/docker-compose.yml up --detach` again. + +To delete one trace, use the dashboard's **Delete this trace** control on the +Traces page, or with the project's key in `AGENTLOOP_API_KEY`: + +```bash +agentloop remote-delete-trace RUN_ID --api-url https://agentloop-api.example.com +``` + +Deleting a trace also removes its findings, usage records and any interventions +that compare it. It cannot be undone except by restoring a backup. + ### Upgrade and back up Back up the database, change `AGENTLOOP_IMAGE` to the new version, then @@ -219,8 +238,8 @@ behind an authenticating ingress. project sign-in enabled so teams only see their own traces. - Keep Postgres off the public internet and back it up. - Traces can contain prompts, tool inputs and outputs. Decide what you capture - and how long you keep it before sending production data; see - [trace retention](TRACE_RETENTION.md). + ([trace retention](TRACE_RETENTION.md)) and how long the server keeps it + (`AGENTLOOP_RETENTION_DAYS`) before sending production data. The [production guide](PRODUCTION.md) covers every setting, health checks and password rotation in detail. diff --git a/docs/DASHBOARD.md b/docs/DASHBOARD.md index 6b85b46..5384673 100644 --- a/docs/DASHBOARD.md +++ b/docs/DASHBOARD.md @@ -26,7 +26,8 @@ Open the URL printed by Streamlit and select project `demo` in the sidebar. - `Overview`: run count, runtime, cost, token volume, model and tool calls, retries, recent runs, and runtime charts. -- `Traces`: stored trace metadata, event timelines, and trace JSON downloads. +- `Traces`: stored trace metadata, event timelines, trace JSON downloads, and + deleting a trace after confirmation. - `Optimization Queue`: recurring findings ranked across stored traces. - `Optimization`: bottlenecks, parallelizable groups, optimization findings, and plan downloads. diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index 84fb74a..681c7a7 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -262,7 +262,28 @@ configured. From 0.2.7 it also publishes the container image to encrypted database connections in hosted environments, and test backups and restores. - Retain traces only as long as needed. Establish a deletion and redaction policy - before ingesting production or customer data. + before ingesting production or customer data, and enforce it with the deletion + and retention tools below. + +## Deleting traces and retention + +From 0.2.7, a stored trace can be deleted together with its findings, usage +records and any interventions that compare it: + +- `DELETE /v1/traces/{run_id}` with a project key deletes a trace of that key's + project; other projects' traces return `404`; +- `AgentLoopClient.delete_trace(run_id)` and `agentloop remote-delete-trace RUN_ID` + call that route; +- `agentloop delete-trace RUN_ID --project-id PROJECT` deletes directly in the + configured store; +- the dashboard's Traces page has a **Delete this trace** control that requires a + confirmation and deletes only within the current project. + +To enforce a retention period, run +`agentloop purge-traces --older-than-days 90` (optionally `--project-id`) on a +schedule. Without `--older-than-days` it reads `AGENTLOOP_RETENTION_DAYS` and does +nothing when that is unset. Age is measured from when the server stored the trace. +Deletion is permanent; restore from a backup if a trace is removed by mistake. The Compose ports bind to `127.0.0.1` by default to keep the local stack off the LAN. Use an explicit production deployment configuration instead of weakening diff --git a/docs/TRACE_RETENTION.md b/docs/TRACE_RETENTION.md index e4f07d4..69b6a7a 100644 --- a/docs/TRACE_RETENTION.md +++ b/docs/TRACE_RETENTION.md @@ -4,6 +4,10 @@ These APIs are included in 0.1.0. Retention acts on completed native traces, before configured finalization destinations. It reduces persisted trace volume; it does not reduce the live tracer's memory use or provide a warehouse. +This page covers what is captured. To delete stored traces or enforce a +retention period on a server, see +[deleting traces and retention](PRODUCTION.md#deleting-traces-and-retention). + ```python from agentloop import RetentionContext, RetentionPolicy, RetentionSession diff --git a/tests/test_cli_store_commands.py b/tests/test_cli_store_commands.py index 5d3b4fb..77cbc9c 100644 --- a/tests/test_cli_store_commands.py +++ b/tests/test_cli_store_commands.py @@ -122,3 +122,35 @@ def test_list_and_revoke_api_keys_from_the_cli(tmp_path, monkeypatch) -> None: assert revoked.exit_code == 0, revoked.output assert "Revoked 1" in revoked.output assert store.verify_api_key(created["api_key"]) is None + + +def test_delete_and_purge_traces_from_the_cli(tmp_path, monkeypatch) -> None: + db_path = tmp_path / "retention.db" + monkeypatch.setenv("AGENTLOOP_STORE_BACKEND", "sqlite") + monkeypatch.setenv("AGENTLOOP_SQLITE_PATH", str(db_path)) + monkeypatch.delenv("AGENTLOOP_RETENTION_DAYS", raising=False) + run_id, _ = _seed_store(db_path, project_id="acme") + store = SQLiteTraceStore(path=str(db_path)) + + assert runner.invoke(app, ["delete-trace", run_id, "--project-id", "other"]).exit_code == 1 + deleted = runner.invoke(app, ["delete-trace", run_id, "--project-id", "acme"]) + assert deleted.exit_code == 0, deleted.output + assert store.get_trace(run_id, project_id="acme") is None + + unconfigured = runner.invoke(app, ["purge-traces"]) + assert unconfigured.exit_code == 0 + assert "not configured" in unconfigured.output + + old_run, _ = _seed_store(db_path, project_id="acme") + with store._connect() as conn: + conn.execute( + "UPDATE traces SET created_at = '2000-01-01 00:00:00' WHERE run_id = ?", (old_run,) + ) + monkeypatch.setenv("AGENTLOOP_RETENTION_DAYS", "30") + purged = runner.invoke(app, ["purge-traces"]) + assert purged.exit_code == 0, purged.output + assert "Deleted 1 trace(s) older than 30 days in all projects" in purged.output + assert store.get_trace(old_run, project_id="acme") is None + + monkeypatch.setenv("AGENTLOOP_RETENTION_DAYS", "soon") + assert runner.invoke(app, ["purge-traces"]).exit_code == 2 diff --git a/tests/test_client.py b/tests/test_client.py index bd9d4b3..ca46bbf 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -269,3 +269,31 @@ def fake_urlopen(request, timeout): # type: ignore[no-untyped-def] for _, _, headers in captured: assert headers["x-agentloop-admin-key"] == "admin" assert "x-agentloop-key" not in headers + + +def test_client_deletes_traces_with_the_project_key(monkeypatch) -> None: + captured: list[tuple[str, str, dict[str, str]]] = [] + + class Response: + def __enter__(self): + return self + + def __exit__(self, *args): # type: ignore[no-untyped-def] + return None + + def read(self, size=-1) -> bytes: + return b"{}" + + def fake_urlopen(request, timeout): # type: ignore[no-untyped-def] + headers = {name.lower(): value for name, value in request.header_items()} + captured.append((request.get_method(), request.full_url, headers)) + return Response() + + monkeypatch.setattr("agentloop.client._open_request", fake_urlopen) + AgentLoopClient( + base_url="https://agentloop.example", api_key="project", admin_api_key="admin" + ).delete_trace("run/1") + method, url, headers = captured[0] + assert (method, url) == ("DELETE", "https://agentloop.example/v1/traces/run%2F1") + assert headers["x-agentloop-key"] == "project" + assert "x-agentloop-admin-key" not in headers diff --git a/tests/test_dashboard_project_access.py b/tests/test_dashboard_project_access.py index a22bb5e..c7f1cea 100644 --- a/tests/test_dashboard_project_access.py +++ b/tests/test_dashboard_project_access.py @@ -181,3 +181,19 @@ def test_shared_ingest_page_offers_only_uploads(tmp_path, monkeypatch): labels = [button.label for button in app.button] assert not any(label.startswith("Generate") for label in labels) assert app.file_uploader, "uploading into the signed-in project stays available" + + +def test_traces_page_deletes_a_trace_after_confirmation(tmp_path, monkeypatch): + app, key = _app(tmp_path, monkeypatch, require_key=True) + _sign_in(app, key) + app.sidebar.radio[0].set_value("Traces").run() + assert not app.exception + delete = next(b for b in app.button if b.label == "Delete trace") + assert delete.disabled, "deleting needs the confirmation checkbox" + next(c for c in app.checkbox if c.label.startswith("I want to delete")).check().run() + next(b for b in app.button if b.label == "Delete trace").click().run() + assert not app.exception + assert any("Deleted" in success.value for success in app.success) + store = SQLiteTraceStore(path=str(tmp_path / "dashboard.db")) + assert store.list_traces(project_id="alpha") == [] + assert len(store.list_traces(project_id="beta")) == 1 diff --git a/tests/test_server.py b/tests/test_server.py index 05d9a27..e26c9fd 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -623,3 +623,31 @@ def test_admin_lists_and_revokes_project_api_keys(monkeypatch, tmp_path) -> None assert revoked.status_code == 200 and revoked.json()["revoked"] == 1 assert client.get("/v1/traces", headers=project).status_code == 401 assert client.delete(delete, params={"project_id": "acme"}, headers=admin).status_code == 404 + + +def test_project_key_deletes_only_its_own_traces(monkeypatch, tmp_path) -> None: + monkeypatch.setenv("AGENTLOOP_STORE_BACKEND", "sqlite") + monkeypatch.setenv("AGENTLOOP_SQLITE_PATH", str(tmp_path / "delete.db")) + monkeypatch.setenv("AGENTLOOP_REQUIRE_API_KEY", "true") + monkeypatch.setenv("AGENTLOOP_ADMIN_API_KEY", "admin-secret") + client = TestClient(app) + admin = {"X-AgentLoop-Admin-Key": "admin-secret"} + + def key(project): + response = client.post( + "/v1/api-keys", json={"project_id": project, "name": "ci"}, headers=admin + ) + return {"X-AgentLoop-Key": response.json()["api_key"]} + + acme, other = key("acme"), key("other") + trace = {"name": "t", "run_id": "run-delete", "metadata": {}, "events": []} + assert client.post("/v1/traces", json=trace, headers=acme).status_code == 200 + + assert client.delete("/v1/traces/run-delete").status_code == 401 + assert client.delete("/v1/traces/run-delete", headers=other).status_code == 404 + assert client.get("/v1/traces/run-delete/report", headers=acme).status_code == 200 + deleted = client.delete("/v1/traces/run-delete", headers=acme) + assert deleted.status_code == 200 + assert deleted.json()["deleted"]["traces"] == 1 + assert client.get("/v1/traces/run-delete/report", headers=acme).status_code == 404 + assert client.delete("/v1/traces/run-delete", headers=acme).status_code == 404 diff --git a/tests/test_store_contract.py b/tests/test_store_contract.py index 705b741..f370d67 100644 --- a/tests/test_store_contract.py +++ b/tests/test_store_contract.py @@ -9,9 +9,11 @@ import copy import os import threading +from datetime import datetime, timedelta, timezone import pytest +from agentloop.findings import build_diagnosis from agentloop.store import ( FindingNotFoundError, FindingTransitionError, @@ -906,3 +908,92 @@ def test_loop_feedback_preserves_schema_and_project_isolation(store): record["reason_code"] == "identical_step_limit" and record["outcome"] == "proposed" for record in expected["decisions"].values() ) + + +def _mark(store): + return "?" if isinstance(store, SQLiteTraceStore) else "%s" + + +def _add_intervention(store, project_id, intervention_id, baseline, candidate): + mark = _mark(store) + with store._connect() as conn: + conn.execute( + "INSERT INTO interventions(project_id, intervention_id, baseline_run_id, " + f"candidate_run_id, payload_json) VALUES ({mark}, {mark}, {mark}, {mark}, {mark})", + (project_id, intervention_id, baseline, candidate, "{}"), + ) + + +def _age(store, run_id, days): + mark = _mark(store) + stamp = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d %H:%M:%S") + with store._connect() as conn: + conn.execute( + f"UPDATE traces SET created_at = {mark} WHERE run_id = {mark}", (stamp, run_id) + ) + + +def _count(store, table, run_id): + mark = _mark(store) + column = "baseline_run_id" if table == "interventions" else "run_id" + with store._connect() as conn: + return conn.execute( + f"SELECT COUNT(*) FROM {table} WHERE {column} = {mark}", (run_id,) + ).fetchone()[0] + + +def _trace_with_findings(name): + with trace_agent(name) as trace: + repeated = "stable context " * 100 + for _ in range(2): + with trace_model_call("summarize", input_text=repeated, output_tokens=10): + pass + return trace + + +def test_delete_trace_removes_derived_records_only_in_its_project(store): + baseline = _trace_with_findings("baseline") + candidate = _sample_trace("candidate") + other = _sample_trace("other-project") + store.save_trace(baseline, project_id="proj_a") + store.save_trace(candidate, project_id="proj_a") + store.save_trace(other, project_id="proj_b") + store.save_diagnosis(build_diagnosis(baseline), project_id="proj_a") + _add_intervention(store, "proj_a", "int_1", baseline.run_id, candidate.run_id) + assert _count(store, "trace_findings", baseline.run_id) > 0 + + # The wrong project cannot delete it. + assert store.delete_trace(baseline.run_id, "proj_b") is None + assert store.get_trace(baseline.run_id, project_id="proj_a") is not None + + deleted = store.delete_trace(baseline.run_id, "proj_a") + assert deleted["traces"] == 1 and deleted["interventions"] == 1 + assert deleted["findings"] >= 1 and deleted["usage_events"] == 1 + assert store.get_trace(baseline.run_id, project_id="proj_a") is None + for table in ("trace_findings", "usage_events", "interventions"): + assert _count(store, table, baseline.run_id) == 0 + assert store.get_trace(candidate.run_id, project_id="proj_a") is not None + assert store.get_trace(other.run_id, project_id="proj_b") is not None + assert store.delete_trace(baseline.run_id, "proj_a") is None + + +def test_purge_deletes_only_traces_older_than_the_retention_period(store): + old_a, new_a, old_b = _sample_trace("old-a"), _sample_trace("new-a"), _sample_trace("old-b") + store.save_trace(old_a, project_id="proj_a") + store.save_trace(new_a, project_id="proj_a") + store.save_trace(old_b, project_id="proj_b") + _age(store, old_a.run_id, 40) + _age(store, old_b.run_id, 40) + + only_b = store.purge_traces(30, project_id="proj_b") + assert only_b["traces"] == 1 + assert store.get_trace(old_a.run_id, project_id="proj_a") is not None + + everywhere = store.purge_traces(30) + assert everywhere["traces"] == 1 and everywhere["usage_events"] == 1 + assert store.get_trace(old_a.run_id, project_id="proj_a") is None + assert store.get_trace(new_a.run_id, project_id="proj_a") is not None + assert store.purge_traces(30)["traces"] == 0 + for invalid in (0, -1, float("nan"), True, "30"): + with pytest.raises(ValueError, match="positive number of days"): + store.purge_traces(invalid)