From 6471c41c60a2fb1d8d31d00d14360fc37aec255f Mon Sep 17 00:00:00 2001 From: Dipesh Babu Date: Thu, 19 Nov 2026 21:58:53 -0500 Subject: [PATCH] Sign dashboard users in by project API key A shared dashboard let anyone past the proxy login type any project name and read its traces. With AGENTLOOP_DASHBOARD_REQUIRE_KEY=true the dashboard asks for a project API key, verifies it like the HTTP API, and binds every page to that key's project. The project field, API-key creation and the host-file patch plan are unavailable to project users, and five failed sign-ins in a session wait a minute. The cloud deploy stack enables it and CI checks the setting. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 1 + CHANGELOG.md | 5 + dashboard/access.py | 71 ++++++++++++ dashboard/app.py | 77 ++++++++++--- deploy/docker-compose.yml | 2 + docs/CLOUD_DEPLOY.md | 25 +++-- docs/DASHBOARD.md | 24 +++- docs/PRODUCTION.md | 8 +- tests/test_dashboard_project_access.py | 148 +++++++++++++++++++++++++ 9 files changed, 325 insertions(+), 36 deletions(-) create mode 100644 dashboard/access.py create mode 100644 tests/test_dashboard_project_access.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32c000e..afdc642 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -677,6 +677,7 @@ jobs: - name: Verify the dashboard requires a login shell: bash run: | + test "$(docker compose exec -T dashboard printenv AGENTLOOP_DASHBOARD_REQUIRE_KEY)" = true status="$(curl --silent --output /dev/null --write-out '%{http_code}' --cacert caddy-root.crt https://dashboard.localhost/)" test "$status" = 401 status="$(curl --silent --output /dev/null --write-out '%{http_code}' --cacert caddy-root.crt \ diff --git a/CHANGELOG.md b/CHANGELOG.md index ce79968..5c0c674 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ trace, study, receipt, API and database schema versions. ### Added +- Shared dashboards can sign users in by project. With + `AGENTLOOP_DASHBOARD_REQUIRE_KEY=true`, the dashboard asks for a project API + key, verifies it like the HTTP API, and shows only that project; the project + field, API-key creation and the host-file patch plan are unavailable, and + repeated failed sign-ins wait a minute. The cloud deployment stack enables it. - Cloud deployment. Releases publish a container image to `ghcr.io/dipeshbabu/agentloop` (version and `latest` tags), built from the validated tag and checked to report the release version as a non-root user. diff --git a/dashboard/access.py b/dashboard/access.py new file mode 100644 index 0000000..320dcde --- /dev/null +++ b/dashboard/access.py @@ -0,0 +1,71 @@ +"""Project-scoped sign-in for a shared, hosted dashboard. + +With ``AGENTLOOP_DASHBOARD_REQUIRE_KEY`` enabled, the dashboard asks for a +project API key, verifies it against the store exactly as the HTTP API does, +and shows only that key's project. Pages that act outside one project are +hidden. Without the setting, the dashboard keeps its local, single-user +behaviour with a free-text project field. +""" + +from __future__ import annotations + +import os +import time + +# Creating keys is an administrator task, and the patch plan reads files on the +# dashboard host; neither belongs to one project's users. +PROJECT_HIDDEN_PAGES = frozenset({"API Keys", "Patch Plan"}) +MAX_FAILURES = 5 +LOCKOUT_SECONDS = 60.0 + + +def require_project_key() -> bool: + """Return whether the dashboard must sign users in with a project API key.""" + return os.getenv("AGENTLOOP_DASHBOARD_REQUIRE_KEY", "").strip().lower() in { + "1", + "true", + "yes", + "on", + } + + +def visible_pages(pages: list[str], *, project_mode: bool) -> list[str]: + """Pages a signed-in project user may open.""" + if not project_mode: + return list(pages) + return [page for page in pages if page not in PROJECT_HIDDEN_PAGES] + + +def lockout_remaining(state, now: float | None = None) -> float: + """Seconds left before another sign-in attempt is allowed.""" + now = time.monotonic() if now is None else now + return max(0.0, float(state.get("signin_locked_until", 0.0)) - now) + + +def sign_in(store, api_key: str, state, now: float | None = None) -> str | None: + """Verify a project API key and record the signed-in project in ``state``. + + Returns the project ID on success. Failed attempts are counted per session, + and after ``MAX_FAILURES`` further attempts wait ``LOCKOUT_SECONDS``. + """ + now = time.monotonic() if now is None else now + if lockout_remaining(state, now) > 0: + return None + key = api_key.strip() if isinstance(api_key, str) else "" + record = store.verify_api_key(key) if key else None + if record is None: + failures = int(state.get("signin_failures", 0)) + 1 + state["signin_failures"] = failures + if failures >= MAX_FAILURES: + state["signin_failures"] = 0 + state["signin_locked_until"] = now + LOCKOUT_SECONDS + return None + state["signin_failures"] = 0 + state["signed_in_project"] = str(record["project_id"]) + state["signed_in_key_prefix"] = str(record.get("prefix") or "") + return state["signed_in_project"] + + +def sign_out(state) -> None: + for name in ("signed_in_project", "signed_in_key_prefix"): + state.pop(name, None) diff --git a/dashboard/app.py b/dashboard/app.py index bc8bd0c..4a4a50e 100644 --- a/dashboard/app.py +++ b/dashboard/app.py @@ -32,9 +32,17 @@ from agentloop.value import build_value_report try: + from dashboard.access import ( + lockout_remaining, + require_project_key, + sign_in, + sign_out, + visible_pages, + ) from dashboard.trace_upload import TraceUploadError, parse_uploaded_trace from dashboard.value_view import assumption_inputs, percentage, render_value_report except ModuleNotFoundError: + from access import lockout_remaining, require_project_key, sign_in, sign_out, visible_pages from trace_upload import TraceUploadError, parse_uploaded_trace from value_view import assumption_inputs, percentage, render_value_report @@ -114,29 +122,62 @@ def render_gate_table(results: list[dict]) -> None: st.sidebar.title("AgentLoop") st.sidebar.caption("Execution profiling and evidence") -project_id = st.sidebar.text_input("Project", value=st.session_state.get("project_id", "default")) -st.session_state["project_id"] = project_id +store = load_store() +project_mode = require_project_key() + +if project_mode: + # A shared dashboard shows exactly one project: the one the API key belongs to. + if "signed_in_project" not in st.session_state: + st.title("AgentLoop Dashboard") + st.subheader("Sign in to your project") + st.caption( + "Enter a project API key. The dashboard shows only the traces and findings " + "of the project that key belongs to." + ) + with st.form("project_sign_in"): + entered_key = st.text_input("Project API key", type="password") + submitted = st.form_submit_button("Sign in") + if submitted: + if sign_in(store, entered_key, st.session_state) is not None: + st.rerun() + remaining = lockout_remaining(st.session_state) + if remaining > 0: + st.error(f"Too many failed attempts. Try again in {int(remaining) + 1} seconds.") + else: + st.error("That API key is not valid for any project.") + st.stop() + project_id = st.session_state["signed_in_project"] + st.sidebar.markdown(f"**Project:** `{project_id}`") + if st.sidebar.button("Sign out"): + sign_out(st.session_state) + st.rerun() +else: + project_id = st.sidebar.text_input( + "Project", value=st.session_state.get("project_id", "default") + ) + st.session_state["project_id"] = project_id page = st.sidebar.radio( "Navigate", - [ - "Overview", - "Traces", - "Optimization Queue", - "Optimization", - "Diagnosis", - "Patch Plan", - "Replay Proof", - "Quality Gates", - "Value & Pricing", - "API Keys", - "Ingest", - "Setup", - ], + visible_pages( + [ + "Overview", + "Traces", + "Optimization Queue", + "Optimization", + "Diagnosis", + "Patch Plan", + "Replay Proof", + "Quality Gates", + "Value & Pricing", + "API Keys", + "Ingest", + "Setup", + ], + project_mode=project_mode, + ), ) -store = load_store() - st.title("AgentLoop Dashboard") st.caption("Cost, latency, and execution-graph optimization for production AI agents") diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index da1156f..8ba11e9 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -80,6 +80,8 @@ services: PGUSER: ${POSTGRES_USER:-agentloop} AGENTLOOP_POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password AGENTLOOP_API_URL: http://api:8000 + # Each user signs in with a project API key and sees only that project. + AGENTLOOP_DASHBOARD_REQUIRE_KEY: "true" depends_on: api: condition: service_healthy diff --git a/docs/CLOUD_DEPLOY.md b/docs/CLOUD_DEPLOY.md index 7ddf3ac..4b08048 100644 --- a/docs/CLOUD_DEPLOY.md +++ b/docs/CLOUD_DEPLOY.md @@ -39,7 +39,8 @@ The `deploy/` directory contains the stack: [Caddy](https://caddyserver.com/), which obtains and renews HTTPS certificates automatically; - `deploy/Caddyfile` serves the API on one hostname and the dashboard on - another, with a login in front of the dashboard; + another, with a login in front of the dashboard. Inside, the dashboard also + asks for a project API key and shows only that project; - `deploy/.env.example` lists every setting. Only ports 80 and 443 are published. The API, dashboard and database are @@ -80,8 +81,9 @@ docker compose -f deploy/docker-compose.yml ps curl https://agentloop-api.example.com/v1/readyz ``` -Open the dashboard hostname in a browser and sign in with the user and password -you chose. +Open the dashboard hostname in a browser and enter the user and password you +chose. The dashboard then asks for a project API key (created in the next +step) and shows only that key's project. ### Create a project key and upload traces @@ -105,7 +107,8 @@ client.upload_trace(trace) or from the TypeScript SDK with `new AgentLoopClient({ baseUrl: "https://agentloop-api.example.com", apiKey: "..." })`. -In the dashboard, set the sidebar's Project field to the same project ID. +Sign in to the dashboard with the same key to see that project's traces. Each +team can only see its own project; there is no project switcher. ### Upgrade and back up @@ -132,8 +135,8 @@ checks that: - the API is ready over HTTPS and stores and returns a trace with a project key; - plain HTTP redirects to HTTPS, and uploads over the size limit are refused; -- the dashboard returns 401 without the login or with a wrong password, and - loads with the right one; +- the dashboard returns 401 without the login or with a wrong password, loads + with the right one, and runs with project sign-in enabled; - the API, dashboard and database ports are not published on the host. ## Option 2: managed container platforms @@ -175,9 +178,10 @@ These deployments are not exercised in CI. The same image and settings apply: 3. Optionally run the dashboard as a second service from the same image with the command `streamlit run dashboard/app.py --server.address=0.0.0.0 - --server.port=8501 --server.headless=true`, the same database settings, and - `AGENTLOOP_API_URL` set to the API's URL. The dashboard has no login of its - own: put it behind the platform's identity layer, such as Identity-Aware + --server.port=8501 --server.headless=true`, the same database settings, + `AGENTLOOP_API_URL` set to the API's URL, and + `AGENTLOOP_DASHBOARD_REQUIRE_KEY=true` so users sign in with a project key. + Also put it behind the platform's identity layer, such as Identity-Aware Proxy on Google Cloud or built-in authentication on Azure Container Apps, or keep it on a private network. Do not expose it publicly without one. @@ -195,7 +199,8 @@ behind an authenticating ingress. - Require API keys (`AGENTLOOP_REQUIRE_API_KEY=true`) and keep the admin key out of clients; give each team or pipeline its own project key. -- Serve only HTTPS, and never expose the dashboard without a login. +- Serve only HTTPS, never expose the dashboard without a login, and keep + project sign-in enabled so teams only see their own traces. - Keep Postgres off the public internet and back it up. - Traces can contain prompts, tool inputs and outputs. Decide what you capture and how long you keep it before sending production data; see diff --git a/docs/DASHBOARD.md b/docs/DASHBOARD.md index d8e205f..912c507 100644 --- a/docs/DASHBOARD.md +++ b/docs/DASHBOARD.md @@ -71,13 +71,27 @@ uv run agentloop upload --path runs/research_agent_baseline.json \ Select project `acme` in a dashboard connected to the same store. +## Project sign-in for shared dashboards + +Set `AGENTLOOP_DASHBOARD_REQUIRE_KEY=true` when several teams share one +dashboard (AgentLoop 0.2.7 and newer). The dashboard then asks for a project API +key, checks it the same way the HTTP API does, and shows only the traces, +findings and reports of the project that key belongs to. The free-text Project +field is replaced by the signed-in project and a Sign out button. The `API Keys` +page (creating keys is an administrator task) and the `Patch Plan` page (it +reads files on the dashboard host) are hidden. After five failed sign-ins in a +browser session, further attempts wait one minute. + +Create project keys with the API's admin key or `agentloop create-api-key`. +Without the setting, the dashboard keeps its local, single-user behaviour. + ## Security boundaries -The dashboard does not provide built-in user authentication. Keep it bound to a -trusted interface for local use. For shared or internet-facing deployments, put -it behind an authenticated reverse proxy and follow -[PRODUCTION.md](PRODUCTION.md), including TLS, explicit CORS, API-key auth, -retention, backups, and database access controls. +Keep a local dashboard bound to a trusted interface. For shared or +internet-facing deployments, enable project sign-in, put the dashboard behind an +authenticated reverse proxy, and follow [PRODUCTION.md](PRODUCTION.md), including +TLS, explicit CORS, API-key auth, retention, backups, and database access +controls. The [cloud deployment stack](CLOUD_DEPLOY.md) does all of this. Quality fixture validation runs only when the Quality Gates form is submitted. Empty suites, missing scorer requirements, and malformed JSON are shown beside diff --git a/docs/PRODUCTION.md b/docs/PRODUCTION.md index 9b2b3f9..4580301 100644 --- a/docs/PRODUCTION.md +++ b/docs/PRODUCTION.md @@ -234,9 +234,11 @@ configured. From 0.2.7 it also publishes the container image to `.env` value alone does not update an existing database role. Use an authenticated Postgres password-change procedure before replacing the secret, and keep the old value only until the recreated clients connect successfully. -- The Streamlit dashboard does not provide end-user authentication. Treat it as - an operator console: keep it on a private network or place it behind an - authenticated reverse proxy. Do not expose it directly to the internet. +- The Streamlit dashboard has no user accounts. For a shared dashboard, set + `AGENTLOOP_DASHBOARD_REQUIRE_KEY=true` so each user signs in with a project API + key and sees only that project, and still keep it on a private network or + behind an authenticated reverse proxy. Do not expose it directly to the + internet. - Terminate TLS at a trusted proxy or load balancer and forward only the API and authenticated dashboard routes that users need. - Apply request-body limits, rate limits, timeouts, and access logging at the diff --git a/tests/test_dashboard_project_access.py b/tests/test_dashboard_project_access.py new file mode 100644 index 0000000..55d46e4 --- /dev/null +++ b/tests/test_dashboard_project_access.py @@ -0,0 +1,148 @@ +"""A shared dashboard signs users in with a project API key and shows only that project.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +st = pytest.importorskip("streamlit") +from dashboard_apptest import APPTEST_TIMEOUT # noqa: E402 +from streamlit.testing.v1 import AppTest # noqa: E402 + +from agentloop.store import SQLiteTraceStore # noqa: E402 +from agentloop.tracer import AgentTrace # noqa: E402 +from dashboard.access import ( # noqa: E402 + LOCKOUT_SECONDS, + MAX_FAILURES, + lockout_remaining, + require_project_key, + sign_in, + sign_out, + visible_pages, +) + +APP = str(Path(__file__).resolve().parents[1] / "dashboard" / "app.py") +PAGES = ["Overview", "Patch Plan", "API Keys", "Ingest"] + + +@pytest.fixture(autouse=True) +def _isolate_the_cached_store(): + st.cache_resource.clear() + yield + st.cache_resource.clear() + + +@pytest.mark.parametrize( + ("value", "expected"), + [("true", True), ("1", True), (" YES ", True), ("on", True), ("false", False), ("", False)], +) +def test_project_key_mode_is_opt_in(monkeypatch, value, expected): + monkeypatch.setenv("AGENTLOOP_DASHBOARD_REQUIRE_KEY", value) + assert require_project_key() is expected + + +def test_project_users_do_not_see_admin_or_host_file_pages(): + assert visible_pages(PAGES, project_mode=False) == PAGES + assert visible_pages(PAGES, project_mode=True) == ["Overview", "Ingest"] + + +def _store(tmp_path): + store = SQLiteTraceStore(path=str(tmp_path / "dashboard.db")) + store.save_trace(AgentTrace(name="alpha-agent"), project_id="alpha") + store.save_trace(AgentTrace(name="beta-agent"), project_id="beta") + return store + + +def test_sign_in_binds_the_project_of_a_valid_key(tmp_path): + store = _store(tmp_path) + key = store.create_api_key(project_id="alpha", name="team")["api_key"] + state = {} + assert sign_in(store, f" {key}\n", state) == "alpha" + assert state["signed_in_project"] == "alpha" + assert key.startswith(state["signed_in_key_prefix"]) + sign_out(state) + assert "signed_in_project" not in state + + +@pytest.mark.parametrize("bad", ["", " ", "al_not-a-real-key", None]) +def test_invalid_keys_are_refused(tmp_path, bad): + state = {} + assert sign_in(_store(tmp_path), bad, state) is None + assert "signed_in_project" not in state + + +def test_repeated_failures_lock_sign_in_for_a_while(tmp_path): + store = _store(tmp_path) + key = store.create_api_key(project_id="alpha", name="team")["api_key"] + state = {} + for _ in range(MAX_FAILURES): + assert sign_in(store, "al_wrong", state, now=100.0) is None + assert lockout_remaining(state, now=100.0) == pytest.approx(LOCKOUT_SECONDS) + # Even the right key waits out the lockout. + assert sign_in(store, key, state, now=100.0 + LOCKOUT_SECONDS - 1) is None + assert sign_in(store, key, state, now=100.0 + LOCKOUT_SECONDS + 1) == "alpha" + + +def _app(tmp_path, monkeypatch, *, require_key): + database = tmp_path / "dashboard.db" + monkeypatch.setenv("AGENTLOOP_STORE_BACKEND", "sqlite") + monkeypatch.setenv("AGENTLOOP_SQLITE_PATH", str(database)) + if require_key: + monkeypatch.setenv("AGENTLOOP_DASHBOARD_REQUIRE_KEY", "true") + else: + monkeypatch.delenv("AGENTLOOP_DASHBOARD_REQUIRE_KEY", raising=False) + store = _store(tmp_path) + key = store.create_api_key(project_id="alpha", name="team")["api_key"] + app = AppTest.from_file(APP, default_timeout=APPTEST_TIMEOUT) + app.run() + return app, key + + +def _sign_in(app, key): + field = next(w for w in app.text_input if w.label == "Project API key") + field.input(key) + next(b for b in app.button if b.label == "Sign in").click() + return app.run() + + +def _shown_names(app): + return { + str(value) + for frame in app.dataframe + for column in frame.value.columns + if column == "name" + for value in frame.value[column] + } + + +def test_hosted_dashboard_shows_only_the_signed_in_project(tmp_path, monkeypatch): + app, key = _app(tmp_path, monkeypatch, require_key=True) + assert not app.exception + assert [w.label for w in app.text_input] == ["Project API key"] + assert not app.sidebar.radio, "no page is reachable before signing in" + + _sign_in(app, "al_wrong-key") + assert any("not valid" in error.value for error in app.error) + assert not app.sidebar.radio + + _sign_in(app, key) + assert not app.exception + assert any("`alpha`" in md.value for md in app.sidebar.markdown) + assert "Project" not in [w.label for w in app.sidebar.text_input] + options = app.sidebar.radio[0].options + assert "API Keys" not in options and "Patch Plan" not in options + names = _shown_names(app) + assert "alpha-agent" in names + assert "beta-agent" not in names + + next(b for b in app.sidebar.button if b.label == "Sign out").click() + app.run() + assert [w.label for w in app.text_input] == ["Project API key"] + + +def test_local_dashboard_keeps_the_free_project_field(tmp_path, monkeypatch): + app, _ = _app(tmp_path, monkeypatch, require_key=False) + assert not app.exception + assert [w.label for w in app.sidebar.text_input] == ["Project"] + assert "API Keys" in app.sidebar.radio[0].options