From c00d0303f845ec4c70d84f88d3f8bd97b43d746d Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:07:17 +0000 Subject: [PATCH 1/7] test(agent-challenge): add contracts for attested progress telemetry Pin Bearer+session ingest, env wiring, and hotkey telemetry-session handshake so the monorepo port fails closed until implemented. --- .../tests/test_eval_progress_env_wiring.py | 227 ++++++++ .../tests/test_eval_progress_ingest.py | 518 ++++++++++++++++++ .../tests/test_telemetry_session.py | 361 ++++++++++++ 3 files changed, 1106 insertions(+) create mode 100644 packages/challenges/agent-challenge/tests/test_eval_progress_env_wiring.py create mode 100644 packages/challenges/agent-challenge/tests/test_eval_progress_ingest.py create mode 100644 packages/challenges/agent-challenge/tests/test_telemetry_session.py diff --git a/packages/challenges/agent-challenge/tests/test_eval_progress_env_wiring.py b/packages/challenges/agent-challenge/tests/test_eval_progress_env_wiring.py new file mode 100644 index 000000000..39e55c71d --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_eval_progress_env_wiring.py @@ -0,0 +1,227 @@ +"""RED contract: selfdeploy env bundle carries progress + runner hotkey config. + +ProgressReporter env must be deploy-injectable (mirror REVIEW_API_BASE_URL). +RUNNER_HOTKEY_MNEMONIC is the local-only signing secret name — its VALUE must +never appear in logs/redacted CLI output. The server receives hotkey_ss58 + +signature only. +""" + +from __future__ import annotations + +import hashlib +import json +import logging + +from agent_challenge.canonical import eval_wire +from agent_challenge.canonical.compose import DEFAULT_ALLOWED_ENVS + +PROGRESS_ENVS = ( + "EVAL_PROGRESS_BASE_URL", + "EVAL_RUN_ID", + "EVAL_SUBMISSION_ID", + "EVAL_RUN_TOKEN", +) +# Local runner signing secret name (value never leaves the CVM / never hits master). +RUNNER_HOTKEY_ENV = "RUNNER_HOTKEY_MNEMONIC" +# Well-known 12-word test fixture — never a real secret; used only to prove redaction. +TEST_MNEMONIC = ( + "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about" +) + + +def test_default_allowed_envs_include_progress_and_runner_hotkey_names(): + allowed = set(DEFAULT_ALLOWED_ENVS) + for name in PROGRESS_ENVS: + assert name in allowed, f"{name} missing from DEFAULT_ALLOWED_ENVS" + assert RUNNER_HOTKEY_ENV in allowed, f"{RUNNER_HOTKEY_ENV} missing from DEFAULT_ALLOWED_ENVS" + + +def test_build_eval_progress_env_helper_binds_ids_and_token(): + """CLI/deploy helper must bind base URL + ids from plan + token.""" + from agent_challenge.selfdeploy.eval import build_eval_progress_env + + values = build_eval_progress_env( + base_url="https://chain.joinbase.ai/challenges/agent-challenge/", + eval_run_id="eval-1", + submission_id="7", + eval_run_token="tok", + ) + assert values == { + "EVAL_PROGRESS_BASE_URL": "https://chain.joinbase.ai/challenges/agent-challenge", + "EVAL_RUN_ID": "eval-1", + "EVAL_SUBMISSION_ID": "7", + "EVAL_RUN_TOKEN": "tok", + } + # Helper must never require or embed a mnemonic — server never sees it. + assert RUNNER_HOTKEY_ENV not in values + assert "mnemonic" not in json.dumps(values).lower() + + +def test_encrypt_eval_secrets_accepts_progress_env_bundle(): + from agent_challenge.canonical.compose import generate_app_compose, render_app_compose + from agent_challenge.selfdeploy import eval as eval_deploy + + try: + from agent_challenge.evaluation.own_runner.progress_reporter import ProgressReporter + except ImportError as exc: + raise AssertionError( + "progress_reporter.ProgressReporter missing — required for eval progress env wiring" + ) from exc + + policy = { + "schema_version": 1, + "per_task_aggregation": "mean", + "keep_policy": "off", + "drop_lowest_n": 0, + "threshold_f64be": None, + } + img = "registry.example/eval@sha256:" + "b" * 64 + compose = generate_app_compose( + orchestrator_image=img, + name=eval_deploy.DEFAULT_EVAL_COMPOSE_NAME, + key_release_url=eval_deploy.MEASURE_TIME_EVAL_KEY_RELEASE_PLACEHOLDER, + allowed_envs=tuple(sorted(eval_deploy.EVAL_ALLOWED_ENVS)), + ) + compose_hash = hashlib.sha256(render_app_compose(compose).encode()).hexdigest() + token = "run-token-progress" + public_key = "c" * 64 + plan = { + "schema_version": 1, + "eval_run_id": "eval-progress-1", + "submission_id": "7", + "submission_version": 1, + "authorizing_review_digest": "d" * 64, + "agent_hash": "e" * 64, + "selected_tasks": [ + { + "task_id": "terminal-bench/t", + "image_ref": "task-local/t@sha256:" + "3" * 64, + "task_config_sha256": "3" * 64, + } + ], + "k": 1, + "package_tree_sha": "a" * 64, + "scoring_policy": policy, + "scoring_policy_digest": eval_wire.scoring_policy_digest(policy), + "eval_app": { + "image_ref": img, + "compose_hash": compose_hash, + "app_identity": "bb35a8f627f0f8c991aa85c15742d352e658e0f7", + "kms_key_algorithm": "x25519", + "kms_public_key_hex": public_key, + "kms_public_key_sha256": hashlib.sha256(bytes.fromhex(public_key)).hexdigest(), + "measurement": { + "mrtd": "01" * 48, + "rtmr0": "02" * 48, + "rtmr1": "03" * 48, + "rtmr2": "04" * 48, + "os_image_hash": "05" * 32, + "key_provider": "phala", + "vm_shape": "tdx.small", + }, + }, + "key_release_endpoint": "86.38.238.235:8701", + "result_endpoint": "/evaluation/v1/runs/eval-progress-1/result", + "key_release_nonce": "kr-n", + "score_nonce": "sc-n", + "run_token_sha256": hashlib.sha256(token.encode()).hexdigest(), + "issued_at_ms": 1, + "expires_at_ms": 2, + } + plan = eval_wire.validate_eval_plan(plan) + dep = eval_deploy.build_eval_deployment_plan( + { + "schema_version": 1, + "plan": plan, + "plan_sha256": hashlib.sha256(eval_wire.canonical_json_v1(plan)).hexdigest(), + "secret_delivery": {"env_key": "EVAL_RUN_TOKEN", "token": token}, + } + ) + secrets = { + "EVAL_RUN_TOKEN": token, + "LLM_COST_LIMIT": "1.00", + "CHALLENGE_PHALA_ATTESTATION_ENABLED": "1", + "CHALLENGE_PHALA_EVAL_PLAN": json.dumps(dep.plan, sort_keys=True, separators=(",", ":")), + "CHALLENGE_PHALA_AGENT_HASH": dep.plan["agent_hash"], + "CHALLENGE_PHALA_CANONICAL_MEASUREMENT": json.dumps( + { + "mrtd": dep.measurement["mrtd"], + "rtmr0": dep.measurement["rtmr0"], + "rtmr1": dep.measurement["rtmr1"], + "rtmr2": dep.measurement["rtmr2"], + "compose_hash": dep.compose_hash, + "os_image_hash": dep.measurement["os_image_hash"], + }, + sort_keys=True, + separators=(",", ":"), + ), + "CHALLENGE_PHALA_VALIDATOR_NONCE": dep.plan["key_release_nonce"], + "EVAL_PROGRESS_BASE_URL": "https://chain.joinbase.ai/challenges/agent-challenge", + "EVAL_RUN_ID": dep.eval_run_id, + "EVAL_SUBMISSION_ID": str(dep.plan["submission_id"]), + # Local-only signing material name may be present in encrypted_env allow-list, + # but encrypt path must accept the name without shipping the value to master APIs. + RUNNER_HOTKEY_ENV: TEST_MNEMONIC, + } + encrypted = eval_deploy.encrypt_eval_secrets(dep, secrets) + for name in PROGRESS_ENVS: + assert name in encrypted.env_keys + assert RUNNER_HOTKEY_ENV in encrypted.env_keys + reporter = ProgressReporter.from_env( + { + "EVAL_PROGRESS_BASE_URL": secrets["EVAL_PROGRESS_BASE_URL"], + "EVAL_RUN_ID": secrets["EVAL_RUN_ID"], + "EVAL_SUBMISSION_ID": secrets["EVAL_SUBMISSION_ID"], + "EVAL_RUN_TOKEN": token, + } + ) + assert reporter is not None + assert reporter.eval_run_id == "eval-progress-1" + assert "progress" in reporter.url + # ProgressReporter must not require or hold the mnemonic. + reporter_dump = json.dumps(reporter.__dict__, default=str) + assert TEST_MNEMONIC not in reporter_dump + assert "mnemonic" not in reporter_dump.lower() + + +def test_runner_hotkey_mnemonic_value_never_in_redacted_cli_output(): + """CLI redaction must strip RUNNER_HOTKEY_MNEMONIC values from operator output.""" + from agent_challenge.selfdeploy import cli as cli_mod + + assert hasattr(cli_mod, "_redact_capabilities") + assert hasattr(cli_mod, "_REDACTED_CAPABILITY_KEYS") + assert RUNNER_HOTKEY_ENV in cli_mod._REDACTED_CAPABILITY_KEYS + + payload = { + "EVAL_RUN_ID": "eval-1", + "EVAL_RUN_TOKEN": "tok-secret", + RUNNER_HOTKEY_ENV: TEST_MNEMONIC, + "nested": {RUNNER_HOTKEY_ENV: TEST_MNEMONIC, "ok": "visible"}, + } + redacted = cli_mod._redact_capabilities(payload) + dumped = json.dumps(redacted) + assert TEST_MNEMONIC not in dumped + assert RUNNER_HOTKEY_ENV not in redacted + assert "tok-secret" not in dumped + assert redacted["EVAL_RUN_ID"] == "eval-1" + assert redacted["nested"]["ok"] == "visible" + + +def test_runner_hotkey_mnemonic_value_never_logged(caplog): + """Logging helpers must not emit the mnemonic value.""" + from agent_challenge.selfdeploy import cli as cli_mod + + payload = { + "stage": "eval-deploy", + RUNNER_HOTKEY_ENV: TEST_MNEMONIC, + "EVAL_RUN_TOKEN": "tok-secret", + } + with caplog.at_level(logging.DEBUG): + redacted = cli_mod._redact_capabilities(payload) + # Simulate what CLI would print. + logging.getLogger("agent_challenge.selfdeploy.cli").info( + "deploy payload %s", json.dumps(redacted) + ) + joined = "\n".join(r.getMessage() for r in caplog.records) + assert TEST_MNEMONIC not in joined + assert "tok-secret" not in joined diff --git a/packages/challenges/agent-challenge/tests/test_eval_progress_ingest.py b/packages/challenges/agent-challenge/tests/test_eval_progress_ingest.py new file mode 100644 index 000000000..8f735536a --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_eval_progress_ingest.py @@ -0,0 +1,518 @@ +"""RED contract: attested mid-run progress ingest + telemetry-session gate. + +Phase A: Bearer EVAL_RUN_TOKEN (sha256 vs EvalRun.token_sha256). +Phase B: telemetry-session opened with hotkey-signed nonce. +Phase C: every progress POST carries Bearer + X-Telemetry-Session. + +Progress is observability-only — never mutates scores. Final +POST /evaluation/v1/runs/{id}/result remains the only score path. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import UTC, datetime, timedelta +from typing import Any + +import pytest +from sqlalchemy import func, select + +from agent_challenge.canonical import eval_wire as ew +from agent_challenge.core.models import AgentSubmission, EvalRun, TaskLogEvent +from agent_challenge.evaluation.authorization import load_eval_run_plan +from agent_challenge.evaluation.plan_scoring import canonical_eval_plan_json +from agent_challenge.evaluation.progress import PROGRESS_SOURCE +from agent_challenge.evaluation.task_events import SAFE_TASK_PHASE_STATUSES + +TOKEN = "progress-good-token" +TELEMETRY_HEADER = "X-Telemetry-Session" +PROGRESS_PATH = "/evaluation/v1/runs/{eval_run_id}/progress" +SESSION_PATH = "/evaluation/v1/runs/{eval_run_id}/telemetry-session" +RESULT_PATH = "/evaluation/v1/runs/{eval_run_id}/result" +AGENT_HASH = "55" * 32 +COMPOSE_HASH = "ab" * 32 +PACKAGE_TREE_SHA = "bb" * 32 + + +def _assert_progress_wire_present() -> None: + """Pin the known monorepo defect: progress.py calls these missing symbols.""" + assert hasattr(ew, "EVAL_PROGRESS_PHASES"), "eval_wire.EVAL_PROGRESS_PHASES missing" + assert hasattr(ew, "validate_eval_progress_request"), ( + "eval_wire.validate_eval_progress_request missing" + ) + assert hasattr(ew, "validate_eval_progress_receipt"), ( + "eval_wire.validate_eval_progress_receipt missing" + ) + assert ew.EVAL_PROGRESS_PHASES == SAFE_TASK_PHASE_STATUSES + + +def _plan(*, eval_run_id: str = "eval-progress-1") -> dict[str, Any]: + policy = { + "schema_version": 1, + "per_task_aggregation": "mean", + "keep_policy": "off", + "drop_lowest_n": 0, + "threshold_f64be": None, + } + return ew.validate_eval_plan( + { + "schema_version": 1, + "eval_run_id": eval_run_id, + "submission_id": f"submission-{eval_run_id}", + "submission_version": 1, + "authorizing_review_digest": "66" * 32, + "agent_hash": AGENT_HASH, + "package_tree_sha": PACKAGE_TREE_SHA, + "selected_tasks": [ + { + "task_id": "task-a", + "image_ref": "registry.example/task@sha256:" + "77" * 32, + "task_config_sha256": "88" * 32, + } + ], + "k": 1, + "scoring_policy": policy, + "scoring_policy_digest": ew.scoring_policy_digest(policy), + "eval_app": { + "image_ref": "registry.example/eval@sha256:" + "99" * 32, + "compose_hash": COMPOSE_HASH, + "app_identity": "agent-challenge-eval-v1", + "kms_key_algorithm": "x25519", + "kms_public_key_hex": "aa" * 32, + "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), + "measurement": { + "mrtd": "11" * 48, + "rtmr0": "22" * 48, + "rtmr1": "33" * 48, + "rtmr2": "44" * 48, + "os_image_hash": "cc" * 32, + "key_provider": "validator-kms", + "vm_shape": "tdx-small", + }, + }, + "key_release_endpoint": "validator.example:8701", + "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", + "key_release_nonce": f"key-release-{eval_run_id}", + "score_nonce": f"score-{eval_run_id}", + "run_token_sha256": hashlib.sha256(TOKEN.encode("utf-8")).hexdigest(), + "issued_at_ms": 1, + "expires_at_ms": 2, + } + ) + + +async def _seed_run( + database_session, + plan: dict[str, Any], + *, + token: str = TOKEN, + phase: str = "eval_running", +) -> EvalRun: + now = datetime.now(UTC) + async with database_session() as session: + submission_agent_hash = hashlib.sha256(plan["eval_run_id"].encode("utf-8")).hexdigest() + submission = AgentSubmission( + miner_hotkey=f"progress-miner-{plan['eval_run_id']}", + name=f"progress-agent-{plan['eval_run_id']}", + agent_hash=submission_agent_hash, + package_tree_sha=PACKAGE_TREE_SHA, + artifact_uri=f"/tmp/progress-{plan['eval_run_id']}.zip", + raw_status="review_allowed", + status="queued", + effective_status="queued", + version_number=1, + ) + session.add(submission) + await session.flush() + plan = { + **plan, + "submission_id": str(submission.id), + "run_token_sha256": hashlib.sha256(token.encode("utf-8")).hexdigest(), + } + plan = ew.validate_eval_plan(plan) + run = EvalRun( + eval_run_id=plan["eval_run_id"], + submission_id=submission.id, + submission_version=1, + authorizing_review_digest="66" * 32, + plan_json=canonical_eval_plan_json(plan), + plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode("utf-8")).hexdigest(), + token_sha256=hashlib.sha256(token.encode("utf-8")).hexdigest(), + phase=phase, + retryable=False, + score=None, + issued_at=now, + expires_at=now + timedelta(hours=1), + ) + session.add(run) + await session.commit() + await session.refresh(run) + return run + + +def _progress_body( + plan: dict[str, Any], + *, + sequence: int = 1, + status: str = "running", + event_type: str = "task.status", + progress: float | None = 0.25, + message: str | None = "task running", + task_id: str = "task-a", + extra: dict[str, Any] | None = None, +) -> dict[str, Any]: + body: dict[str, Any] = { + "schema_version": 1, + "eval_run_id": plan["eval_run_id"], + "submission_id": plan["submission_id"], + "task_id": task_id, + "sequence": sequence, + "status": status, + "event_type": event_type, + "progress": progress, + "message": message, + } + if extra: + body.update(extra) + return body + + +def _enable_attested(monkeypatch: pytest.MonkeyPatch) -> None: + from agent_challenge.api import routes as routes_mod + + monkeypatch.setattr(routes_mod.settings, "attested_review_enabled", True) + monkeypatch.setattr(routes_mod.settings, "phala_attestation_enabled", True) + + +def _sign_session_body( + *, + eval_run_id: str, + instance_id: str = "instance-1", + nonce: str = "nonce-1", + timestamp: str | None = None, +) -> dict[str, Any]: + """Build a hotkey-signed telemetry-session open body (mnemonic never included).""" + import bittensor as bt + + keypair = bt.Keypair.create_from_uri("//Alice") + ts = timestamp or datetime.now(UTC).isoformat() + canonical = f"ac-telemetry-session:v1|{eval_run_id}|{instance_id}|{nonce}|{ts}" + signature = keypair.sign(canonical) + sig_hex = ( + "0x" + bytes(signature).hex() + if isinstance(signature, bytes | bytearray) + else (signature if str(signature).startswith("0x") else "0x" + str(signature)) + ) + return { + "schema_version": 1, + "eval_run_id": eval_run_id, + "instance_id": instance_id, + "hotkey_ss58": keypair.ss58_address, + "nonce": nonce, + "timestamp": ts, + "signature": sig_hex, + } + + +async def _open_session(client, eval_run_id: str, *, token: str = TOKEN) -> str: + body = _sign_session_body(eval_run_id=eval_run_id) + # Server must never receive a mnemonic — only hotkey_ss58 + signature. + assert "mnemonic" not in json.dumps(body).lower() + response = await client.post( + SESSION_PATH.format(eval_run_id=eval_run_id), + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + content=json.dumps(body, separators=(",", ":")), + ) + assert response.status_code == 200, response.text + payload = response.json() + assert "session_id" in payload + assert "expires_at" in payload + return str(payload["session_id"]) + + +async def _post_progress( + client, + plan: dict[str, Any], + body: dict[str, Any], + *, + token: str | None = TOKEN, + session_id: str | None = None, +): + headers: dict[str, str] = {"Content-Type": "application/json"} + if token is not None: + headers["Authorization"] = f"Bearer {token}" + if session_id is not None: + headers[TELEMETRY_HEADER] = session_id + return await client.post( + PROGRESS_PATH.format(eval_run_id=plan["eval_run_id"]), + headers=headers, + content=json.dumps(body, separators=(",", ":")), + ) + + +async def _count_progress_events(database_session, submission_id: int) -> int: + async with database_session() as session: + count = await session.scalar( + select(func.count()) + .select_from(TaskLogEvent) + .where(TaskLogEvent.submission_id == submission_id) + ) + return int(count or 0) + + +async def test_s1_happy_records_event(client, database_session, monkeypatch: pytest.MonkeyPatch): + """S1: open session then POST progress sequence 1..3 → 202 + TaskLogEvent rows.""" + _assert_progress_wire_present() + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-progress-s1") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + + session_id = await _open_session(client, plan["eval_run_id"]) + + for seq in (1, 2, 3): + status = ("assigned", "starting", "running")[seq - 1] + response = await _post_progress( + client, + plan, + _progress_body(plan, sequence=seq, status=status, progress=seq / 10), + session_id=session_id, + ) + assert response.status_code == 202, response.text + receipt = response.json() + assert receipt["created"] is True + assert receipt["sequence"] == seq + assert receipt["eval_run_id"] == plan["eval_run_id"] + assert receipt["task_id"] == "task-a" + + async with database_session() as session: + rows = list( + await session.scalars( + select(TaskLogEvent) + .where(TaskLogEvent.submission_id == run.submission_id) + .order_by(TaskLogEvent.sequence) + ) + ) + refreshed = await session.scalar( + select(EvalRun).where(EvalRun.eval_run_id == plan["eval_run_id"]) + ) + + assert len(rows) == 3 + client_sequences: list[int] = [] + for row in rows: + meta = json.loads(row.metadata_json) + assert meta["source"] == PROGRESS_SOURCE == "eval_progress" + assert meta["eval_run_id"] == plan["eval_run_id"] + assert isinstance(meta["client_sequence"], int) + client_sequences.append(meta["client_sequence"]) + assert client_sequences == [1, 2, 3] + assert refreshed is not None + assert refreshed.score is None + + +async def test_s2_auth_fail(client, database_session, monkeypatch: pytest.MonkeyPatch): + """S2: wrong/absent Bearer → 401 and zero TaskLogEvent rows.""" + _assert_progress_wire_present() + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-progress-s2-auth") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + session_id = await _open_session(client, plan["eval_run_id"]) + body = _progress_body(plan) + + missing = await _post_progress(client, plan, body, token=None, session_id=session_id) + wrong = await _post_progress(client, plan, body, token="not-the-token", session_id=session_id) + + assert missing.status_code == 401 + assert wrong.status_code == 401 + assert missing.json()["detail"] == {"code": "invalid_eval_token"} + assert wrong.json()["detail"] == {"code": "invalid_eval_token"} + assert await _count_progress_events(database_session, run.submission_id) == 0 + + +async def test_s2_missing_session(client, database_session, monkeypatch: pytest.MonkeyPatch): + """S2: valid Bearer but no/unknown X-Telemetry-Session → 401, zero rows.""" + _assert_progress_wire_present() + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-progress-s2-session") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + body = _progress_body(plan) + + no_header = await _post_progress(client, plan, body, session_id=None) + unknown = await _post_progress(client, plan, body, session_id="sess-does-not-exist") + + assert no_header.status_code == 401 + assert unknown.status_code == 401 + for response in (no_header, unknown): + detail = response.json()["detail"] + code = detail["code"] if isinstance(detail, dict) else detail + assert code in { + "invalid_telemetry_session", + "telemetry_session_required", + "telemetry_session_unknown", + "telemetry_session_expired", + } + assert await _count_progress_events(database_session, run.submission_id) == 0 + + +async def test_s3_unknown_phase(client, database_session, monkeypatch: pytest.MonkeyPatch): + """S3: status not in safe phases → 422.""" + _assert_progress_wire_present() + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-progress-s3") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + session_id = await _open_session(client, plan["eval_run_id"]) + + response = await _post_progress( + client, + plan, + _progress_body(plan, status="scoring"), + session_id=session_id, + ) + + assert response.status_code == 422 + assert response.json()["detail"]["code"] in { + "progress_invalid", + "progress_phase_invalid", + } + assert "scoring" not in SAFE_TASK_PHASE_STATUSES + assert "scoring" not in ew.EVAL_PROGRESS_PHASES + assert await _count_progress_events(database_session, run.submission_id) == 0 + + +async def test_s4_score_forbidden(client, database_session, monkeypatch: pytest.MonkeyPatch): + """S4: body containing any score field → 422, zero rows, score stays None.""" + _assert_progress_wire_present() + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-progress-s4") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + session_id = await _open_session(client, plan["eval_run_id"]) + + response = await _post_progress( + client, + plan, + _progress_body(plan, extra={"score": 0.99, "score_record": {"score": 0.99}}), + session_id=session_id, + ) + + assert response.status_code == 422 + assert response.json()["detail"]["code"] in { + "progress_score_forbidden", + "progress_invalid", + } + assert await _count_progress_events(database_session, run.submission_id) == 0 + async with database_session() as session: + refreshed = await session.scalar( + select(EvalRun).where(EvalRun.eval_run_id == plan["eval_run_id"]) + ) + assert refreshed is not None + assert refreshed.score is None + + +async def test_s5_idempotent(client, database_session, monkeypatch: pytest.MonkeyPatch): + """S5: replay same (eval_run_id, task_id, client_sequence) → 200, no duplicate row.""" + _assert_progress_wire_present() + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-progress-s5-idem") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + session_id = await _open_session(client, plan["eval_run_id"]) + body = _progress_body(plan, sequence=1, status="running", progress=0.4) + + first = await _post_progress(client, plan, body, session_id=session_id) + second = await _post_progress(client, plan, body, session_id=session_id) + + assert first.status_code == 202, first.text + assert second.status_code == 200, second.text + first_body = first.json() + second_body = second.json() + assert first_body["created"] is True + assert second_body["created"] is False + assert first_body["sequence"] == second_body["sequence"] == 1 + assert first_body["event_id"] == second_body["event_id"] + assert await _count_progress_events(database_session, run.submission_id) == 1 + + +async def test_s5_sequence_regression(client, database_session, monkeypatch: pytest.MonkeyPatch): + """S5: a lower sequence after a higher one → 422.""" + _assert_progress_wire_present() + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-progress-s5-seq") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + session_id = await _open_session(client, plan["eval_run_id"]) + + high = await _post_progress( + client, + plan, + _progress_body(plan, sequence=5, status="running"), + session_id=session_id, + ) + low = await _post_progress( + client, + plan, + _progress_body(plan, sequence=2, status="starting"), + session_id=session_id, + ) + + assert high.status_code == 202, high.text + assert low.status_code == 422 + assert low.json()["detail"]["code"] in { + "progress_sequence_stale", + "progress_invalid", + } + assert await _count_progress_events(database_session, run.submission_id) == 1 + + +async def test_result_still_scores(client, database_session, monkeypatch: pytest.MonkeyPatch): + """Adjacent: POST .../result still authenticates; only score-mutating path.""" + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-progress-result-adj") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + + missing = await client.post( + RESULT_PATH.format(eval_run_id=plan["eval_run_id"]), + headers={"Content-Type": "application/json"}, + content=b"{}", + ) + wrong = await client.post( + RESULT_PATH.format(eval_run_id=plan["eval_run_id"]), + headers={ + "Authorization": "Bearer wrong-token", + "Content-Type": "application/json", + }, + content=b"{}", + ) + # Valid Bearer reaches body validation (not auth rejection). + authed = await client.post( + RESULT_PATH.format(eval_run_id=plan["eval_run_id"]), + headers={ + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", + }, + content=b"{}", + ) + + assert missing.status_code == 401 + assert wrong.status_code == 401 + assert missing.json()["detail"] == {"code": "invalid_eval_token"} + assert wrong.json()["detail"] == {"code": "invalid_eval_token"} + # Auth passed: not 401/404. Body is invalid so 422 (or 413) — never score write. + assert authed.status_code not in {401, 404} + assert authed.status_code in {422, 400, 413} + + async with database_session() as session: + refreshed = await session.scalar( + select(EvalRun).where(EvalRun.eval_run_id == plan["eval_run_id"]) + ) + assert refreshed is not None + assert refreshed.score is None + assert refreshed.id == run.id diff --git a/packages/challenges/agent-challenge/tests/test_telemetry_session.py b/packages/challenges/agent-challenge/tests/test_telemetry_session.py new file mode 100644 index 000000000..6c685d385 --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_telemetry_session.py @@ -0,0 +1,361 @@ +"""RED contract: telemetry-session handshake before progress ingest. + +POST /evaluation/v1/runs/{eval_run_id}/telemetry-session +Auth: Authorization: Bearer vs EvalRun.token_sha256 +Body: schema_version=1, eval_run_id, instance_id, hotkey_ss58, nonce, timestamp, signature +Signature over: ac-telemetry-session:v1|{eval_run_id}|{instance_id}|{nonce}|{timestamp} +Response: {session_id, expires_at} + +Server receives hotkey_ss58 + signature only — never a mnemonic. +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import UTC, datetime, timedelta +from typing import Any + +import pytest +from sqlalchemy import func, select + +from agent_challenge.canonical import eval_wire as ew +from agent_challenge.core.models import AgentSubmission, EvalRun, TaskLogEvent +from agent_challenge.evaluation.authorization import load_eval_run_plan +from agent_challenge.evaluation.plan_scoring import canonical_eval_plan_json + +TOKEN = "telemetry-session-token" +SESSION_PATH = "/evaluation/v1/runs/{eval_run_id}/telemetry-session" +PROGRESS_PATH = "/evaluation/v1/runs/{eval_run_id}/progress" +TELEMETRY_HEADER = "X-Telemetry-Session" +AGENT_HASH = "55" * 32 +COMPOSE_HASH = "ab" * 32 +PACKAGE_TREE_SHA = "bb" * 32 +CANONICAL_PREFIX = "ac-telemetry-session:v1" +TEST_MNEMONIC_FRAGMENT = "abandon abandon abandon" + + +def _plan(*, eval_run_id: str) -> dict[str, Any]: + policy = { + "schema_version": 1, + "per_task_aggregation": "mean", + "keep_policy": "off", + "drop_lowest_n": 0, + "threshold_f64be": None, + } + return ew.validate_eval_plan( + { + "schema_version": 1, + "eval_run_id": eval_run_id, + "submission_id": f"submission-{eval_run_id}", + "submission_version": 1, + "authorizing_review_digest": "66" * 32, + "agent_hash": AGENT_HASH, + "package_tree_sha": PACKAGE_TREE_SHA, + "selected_tasks": [ + { + "task_id": "task-a", + "image_ref": "registry.example/task@sha256:" + "77" * 32, + "task_config_sha256": "88" * 32, + } + ], + "k": 1, + "scoring_policy": policy, + "scoring_policy_digest": ew.scoring_policy_digest(policy), + "eval_app": { + "image_ref": "registry.example/eval@sha256:" + "99" * 32, + "compose_hash": COMPOSE_HASH, + "app_identity": "agent-challenge-eval-v1", + "kms_key_algorithm": "x25519", + "kms_public_key_hex": "aa" * 32, + "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), + "measurement": { + "mrtd": "11" * 48, + "rtmr0": "22" * 48, + "rtmr1": "33" * 48, + "rtmr2": "44" * 48, + "os_image_hash": "cc" * 32, + "key_provider": "validator-kms", + "vm_shape": "tdx-small", + }, + }, + "key_release_endpoint": "validator.example:8701", + "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", + "key_release_nonce": f"key-release-{eval_run_id}", + "score_nonce": f"score-{eval_run_id}", + "run_token_sha256": hashlib.sha256(TOKEN.encode("utf-8")).hexdigest(), + "issued_at_ms": 1, + "expires_at_ms": 2, + } + ) + + +async def _seed_run( + database_session, + plan: dict[str, Any], + *, + token: str = TOKEN, + phase: str = "eval_running", +) -> EvalRun: + now = datetime.now(UTC) + async with database_session() as session: + submission_agent_hash = hashlib.sha256(plan["eval_run_id"].encode("utf-8")).hexdigest() + submission = AgentSubmission( + miner_hotkey=f"telemetry-miner-{plan['eval_run_id']}", + name=f"telemetry-agent-{plan['eval_run_id']}", + agent_hash=submission_agent_hash, + package_tree_sha=PACKAGE_TREE_SHA, + artifact_uri=f"/tmp/telemetry-{plan['eval_run_id']}.zip", + raw_status="review_allowed", + status="queued", + effective_status="queued", + version_number=1, + ) + session.add(submission) + await session.flush() + plan = { + **plan, + "submission_id": str(submission.id), + "run_token_sha256": hashlib.sha256(token.encode("utf-8")).hexdigest(), + } + plan = ew.validate_eval_plan(plan) + run = EvalRun( + eval_run_id=plan["eval_run_id"], + submission_id=submission.id, + submission_version=1, + authorizing_review_digest="66" * 32, + plan_json=canonical_eval_plan_json(plan), + plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode("utf-8")).hexdigest(), + token_sha256=hashlib.sha256(token.encode("utf-8")).hexdigest(), + phase=phase, + retryable=False, + score=None, + issued_at=now, + expires_at=now + timedelta(hours=1), + ) + session.add(run) + await session.commit() + await session.refresh(run) + return run + + +def _enable_attested(monkeypatch: pytest.MonkeyPatch) -> None: + from agent_challenge.api import routes as routes_mod + + monkeypatch.setattr(routes_mod.settings, "attested_review_enabled", True) + monkeypatch.setattr(routes_mod.settings, "phala_attestation_enabled", True) + + +def _keypair(): + import bittensor as bt + + return bt.Keypair.create_from_uri("//Alice") + + +def _encode_sig(signature: object) -> str: + if isinstance(signature, bytes | bytearray): + return "0x" + bytes(signature).hex() + text = str(signature) + return text if text.startswith("0x") else "0x" + text + + +def _session_body( + *, + eval_run_id: str, + instance_id: str = "cvm-instance-1", + nonce: str = "telemetry-nonce-1", + timestamp: str | None = None, + keypair=None, + tamper_signature: bool = False, +) -> dict[str, Any]: + kp = keypair or _keypair() + ts = timestamp or datetime.now(UTC).replace(microsecond=0).isoformat() + canonical = f"{CANONICAL_PREFIX}|{eval_run_id}|{instance_id}|{nonce}|{ts}" + signature = _encode_sig(kp.sign(canonical)) + if tamper_signature: + # Flip last nibble so verify fails. + signature = signature[:-1] + ("0" if signature[-1] != "0" else "1") + body = { + "schema_version": 1, + "eval_run_id": eval_run_id, + "instance_id": instance_id, + "hotkey_ss58": kp.ss58_address, + "nonce": nonce, + "timestamp": ts, + "signature": signature, + } + # Contract: mnemonic never leaves the runner — only ss58 + signature on the wire. + dumped = json.dumps(body) + assert "mnemonic" not in body # key must never be present on happy path + assert TEST_MNEMONIC_FRAGMENT not in dumped.lower() + return body + + +async def _post_session(client, eval_run_id: str, body: dict[str, Any], *, token: str = TOKEN): + return await client.post( + SESSION_PATH.format(eval_run_id=eval_run_id), + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }, + content=json.dumps(body, separators=(",", ":")), + ) + + +async def test_telemetry_session_happy_open( + client, database_session, monkeypatch: pytest.MonkeyPatch +): + """Happy open: valid Bearer + hotkey signature → 200 {session_id, expires_at}.""" + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-telemetry-happy") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + body = _session_body(eval_run_id=plan["eval_run_id"]) + + response = await _post_session(client, plan["eval_run_id"], body) + + assert response.status_code == 200, response.text + payload = response.json() + assert set(payload) >= {"session_id", "expires_at"} + assert isinstance(payload["session_id"], str) and payload["session_id"] + assert payload["expires_at"] + # Response must not echo mnemonic or raw signing material beyond session id. + assert "mnemonic" not in payload + assert TEST_MNEMONIC_FRAGMENT not in json.dumps(payload).lower() + assert run.token_sha256 == hashlib.sha256(TOKEN.encode("utf-8")).hexdigest() + + +async def test_telemetry_session_bad_signature( + client, database_session, monkeypatch: pytest.MonkeyPatch +): + """Bad signature → 401.""" + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-telemetry-badsig") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + body = _session_body(eval_run_id=plan["eval_run_id"], tamper_signature=True) + + response = await _post_session(client, plan["eval_run_id"], body) + + assert response.status_code == 401 + detail = response.json()["detail"] + code = detail["code"] if isinstance(detail, dict) else detail + assert code in { + "invalid_telemetry_signature", + "invalid_telemetry_session", + "telemetry_signature_invalid", + } + + +async def test_telemetry_session_terminal_eval_run_phase( + client, database_session, monkeypatch: pytest.MonkeyPatch +): + """Terminal eval_run phase (eval_accepted) → 409.""" + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-telemetry-terminal") + run = await _seed_run(database_session, plan, phase="eval_accepted") + plan = load_eval_run_plan(run) + body = _session_body(eval_run_id=plan["eval_run_id"]) + + response = await _post_session(client, plan["eval_run_id"], body) + + assert response.status_code == 409 + detail = response.json()["detail"] + code = detail["code"] if isinstance(detail, dict) else detail + assert code in { + "eval_run_terminal", + "telemetry_session_forbidden", + "eval_run_not_open", + } + + +async def test_telemetry_session_expired_then_progress_rejected( + client, database_session, monkeypatch: pytest.MonkeyPatch +): + """Expired/closed session then progress → 401, zero TaskLogEvent rows.""" + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-telemetry-expired") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + body = _session_body(eval_run_id=plan["eval_run_id"]) + + opened = await _post_session(client, plan["eval_run_id"], body) + assert opened.status_code == 200, opened.text + session_id = opened.json()["session_id"] + + # Force expiry/close via the session module contract (implementation under test). + from agent_challenge.evaluation import telemetry_session as ts_mod + + if hasattr(ts_mod, "close_telemetry_session"): + await ts_mod.close_telemetry_session(session_id) + elif hasattr(ts_mod, "expire_telemetry_session"): + await ts_mod.expire_telemetry_session(session_id) + else: + # Pin that a close/expire API exists for the handshake lifecycle. + raise AssertionError( + "telemetry_session module must expose close_telemetry_session or " + "expire_telemetry_session" + ) + + progress_body = { + "schema_version": 1, + "eval_run_id": plan["eval_run_id"], + "submission_id": plan["submission_id"], + "task_id": "task-a", + "sequence": 1, + "status": "running", + "event_type": "task.status", + "progress": 0.1, + "message": "should be rejected", + } + progress = await client.post( + PROGRESS_PATH.format(eval_run_id=plan["eval_run_id"]), + headers={ + "Authorization": f"Bearer {TOKEN}", + "Content-Type": "application/json", + TELEMETRY_HEADER: session_id, + }, + content=json.dumps(progress_body, separators=(",", ":")), + ) + + assert progress.status_code == 401 + detail = progress.json()["detail"] + code = detail["code"] if isinstance(detail, dict) else detail + assert code in { + "invalid_telemetry_session", + "telemetry_session_expired", + "telemetry_session_closed", + "telemetry_session_unknown", + } + async with database_session() as session: + count = await session.scalar( + select(func.count()) + .select_from(TaskLogEvent) + .where(TaskLogEvent.submission_id == run.submission_id) + ) + assert int(count or 0) == 0 + + +async def test_telemetry_session_rejects_mnemonic_in_body( + client, database_session, monkeypatch: pytest.MonkeyPatch +): + """Server must reject bodies that attempt to send a mnemonic (hotkey_ss58 only).""" + _enable_attested(monkeypatch) + plan = _plan(eval_run_id="eval-telemetry-no-mne") + run = await _seed_run(database_session, plan) + plan = load_eval_run_plan(run) + body = _session_body(eval_run_id=plan["eval_run_id"]) + # Obvious test mnemonic — must never be accepted as auth material. + body["mnemonic"] = ( + "abandon abandon abandon abandon abandon abandon " + "abandon abandon abandon abandon abandon about" + ) + assert "mnemonic" in body # deliberate injection for this negative test + + response = await _post_session(client, plan["eval_run_id"], body) + + assert response.status_code != 404, "telemetry-session route missing" + assert response.status_code in {400, 401, 422} + # Even on error, response must not echo the mnemonic value. + assert TEST_MNEMONIC_FRAGMENT not in response.text.lower() + assert run.submission_id > 0 From cd90e88a0094da337851e9681aae2d209692661c Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:07:17 +0000 Subject: [PATCH 2/7] test(agent-challenge): add contracts for live execution pool Require GET /v1/execution-pool/live to list in-flight eval runs without score fields for master fan-out. --- .../tests/test_ac_execution_pool_live.py | 342 ++++++++++++++++++ 1 file changed, 342 insertions(+) create mode 100644 packages/challenges/agent-challenge/tests/test_ac_execution_pool_live.py diff --git a/packages/challenges/agent-challenge/tests/test_ac_execution_pool_live.py b/packages/challenges/agent-challenge/tests/test_ac_execution_pool_live.py new file mode 100644 index 000000000..0a0c5d1b1 --- /dev/null +++ b/packages/challenges/agent-challenge/tests/test_ac_execution_pool_live.py @@ -0,0 +1,342 @@ +"""RED contract: Agent Challenge live execution pool for master fan-out. + +Master GET /v1/pools/executing calls each challenge's +GET /v1/execution-pool/live and extracts body["units"]. + +Source of truth: non-terminal EvalRun rows + latest TaskLogEvent +(observability only — never score fields). +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import UTC, datetime, timedelta +from typing import Any + +import pytest +from sqlalchemy import select + +from agent_challenge.canonical import eval_wire as ew +from agent_challenge.core.models import AgentSubmission, EvalRun, TaskLogEvent +from agent_challenge.evaluation.plan_scoring import canonical_eval_plan_json + +POOL_PATH = "/v1/execution-pool/live" +AGENT_HASH = "55" * 32 +PACKAGE_TREE_SHA = "bb" * 32 +COMPOSE_HASH = "ab" * 32 +TOKEN = "pool-live-token" + +_SCORE_KEYS = frozenset( + { + "score", + "scores", + "final_score", + "raw_score", + "normalized_score", + "weight", + "weights", + "emission", + "emission_percent", + "incentive", + "passed_tasks", + "total_tasks", + "canonical_score_record_json", + "canonical_score_record_sha256", + } +) + + +def _assert_no_score_keys(payload: Any, *, path: str = "$") -> None: + if isinstance(payload, dict): + lowered = {str(k).lower() for k in payload} + leaked = _SCORE_KEYS & lowered + assert not leaked, f"score-like keys at {path}: {sorted(leaked)}" + for key, value in payload.items(): + _assert_no_score_keys(value, path=f"{path}.{key}") + elif isinstance(payload, list): + for index, item in enumerate(payload): + _assert_no_score_keys(item, path=f"{path}[{index}]") + + +def _plan(*, eval_run_id: str) -> dict[str, Any]: + policy = { + "schema_version": 1, + "per_task_aggregation": "mean", + "keep_policy": "off", + "drop_lowest_n": 0, + "threshold_f64be": None, + } + return ew.validate_eval_plan( + { + "schema_version": 1, + "eval_run_id": eval_run_id, + "submission_id": f"submission-{eval_run_id}", + "submission_version": 1, + "authorizing_review_digest": "66" * 32, + "agent_hash": AGENT_HASH, + "package_tree_sha": PACKAGE_TREE_SHA, + "selected_tasks": [ + { + "task_id": "task-a", + "image_ref": "registry.example/task@sha256:" + "77" * 32, + "task_config_sha256": "88" * 32, + } + ], + "k": 1, + "scoring_policy": policy, + "scoring_policy_digest": ew.scoring_policy_digest(policy), + "eval_app": { + "image_ref": "registry.example/eval@sha256:" + "99" * 32, + "compose_hash": COMPOSE_HASH, + "app_identity": "agent-challenge-eval-v1", + "kms_key_algorithm": "x25519", + "kms_public_key_hex": "aa" * 32, + "kms_public_key_sha256": hashlib.sha256(bytes.fromhex("aa" * 32)).hexdigest(), + "measurement": { + "mrtd": "11" * 48, + "rtmr0": "22" * 48, + "rtmr1": "33" * 48, + "rtmr2": "44" * 48, + "os_image_hash": "cc" * 32, + "key_provider": "validator-kms", + "vm_shape": "tdx-small", + }, + }, + "key_release_endpoint": "validator.example:8701", + "result_endpoint": f"/evaluation/v1/runs/{eval_run_id}/result", + "key_release_nonce": f"key-release-{eval_run_id}", + "score_nonce": f"score-{eval_run_id}", + "run_token_sha256": hashlib.sha256(TOKEN.encode("utf-8")).hexdigest(), + "issued_at_ms": 1, + "expires_at_ms": 2, + } + ) + + +async def _seed_run( + database_session, + *, + eval_run_id: str, + phase: str, + score: float | None = None, +) -> EvalRun: + plan = _plan(eval_run_id=eval_run_id) + now = datetime.now(UTC) + # token_sha256 is unique per EvalRun — derive from eval_run_id. + token = f"{TOKEN}-{eval_run_id}" + async with database_session() as session: + submission = AgentSubmission( + miner_hotkey=f"pool-miner-{eval_run_id}", + name=f"pool-agent-{eval_run_id}", + agent_hash=hashlib.sha256(eval_run_id.encode("utf-8")).hexdigest(), + package_tree_sha=PACKAGE_TREE_SHA, + artifact_uri=f"/tmp/pool-{eval_run_id}.zip", + raw_status="review_allowed", + status="queued", + effective_status="queued", + version_number=1, + ) + session.add(submission) + await session.flush() + plan = { + **plan, + "submission_id": str(submission.id), + "run_token_sha256": hashlib.sha256(token.encode("utf-8")).hexdigest(), + } + plan = ew.validate_eval_plan(plan) + run = EvalRun( + eval_run_id=eval_run_id, + submission_id=submission.id, + submission_version=1, + authorizing_review_digest="66" * 32, + plan_json=canonical_eval_plan_json(plan), + plan_sha256=hashlib.sha256(canonical_eval_plan_json(plan).encode("utf-8")).hexdigest(), + token_sha256=hashlib.sha256(token.encode("utf-8")).hexdigest(), + phase=phase, + retryable=False, + score=score, + finalized_at=now if phase in {"eval_accepted", "eval_rejected"} else None, + issued_at=now, + expires_at=now + timedelta(hours=1), + ) + session.add(run) + await session.commit() + await session.refresh(run) + return run + + +async def _seed_progress_event( + database_session, + *, + submission_id: int, + eval_run_id: str, + sequence: int, + message: str, + progress: float = 0.4, + status: str = "running", +) -> None: + async with database_session() as session: + event = TaskLogEvent( + submission_id=submission_id, + job_id=None, + task_id="task-a", + sequence=sequence, + event_type="task.status", + stream=None, + message=message, + message_bytes=len(message.encode("utf-8")), + progress=progress, + status=status, + metadata_json=json.dumps( + { + "source": "eval_progress", + "eval_run_id": eval_run_id, + "client_sequence": sequence, + "phase": status, + }, + separators=(",", ":"), + ), + ) + session.add(event) + await session.commit() + + +@pytest.mark.asyncio +async def test_execution_pool_live_empty_when_no_running_evals(client) -> None: + """Empty pool is honest empty units list — not 404.""" + + response = await client.get(POOL_PATH) + assert response.status_code == 200, response.text + body = response.json() + assert body == {"units": []} + _assert_no_score_keys(body) + + +@pytest.mark.asyncio +async def test_execution_pool_live_shows_running_eval_with_latest_event( + client, database_session +) -> None: + """In-flight eval_running appears with latest progress event; observability only.""" + + live = await _seed_run( + database_session, eval_run_id="eval-pool-live", phase="eval_running" + ) + await _seed_progress_event( + database_session, + submission_id=live.submission_id, + eval_run_id=live.eval_run_id, + sequence=1, + message="boot", + progress=0.1, + ) + await _seed_progress_event( + database_session, + submission_id=live.submission_id, + eval_run_id=live.eval_run_id, + sequence=2, + message="latest-progress-marker", + progress=0.55, + status="running", + ) + + response = await client.get(POOL_PATH) + assert response.status_code == 200, response.text + body = response.json() + assert isinstance(body, dict) + units = body.get("units") + assert isinstance(units, list) + assert len(units) == 1 + + unit = units[0] + unit_id = str(unit.get("unit_id") or unit.get("eval_run_id") or "") + assert unit_id == live.eval_run_id + assert unit.get("eval_run_id") == live.eval_run_id + assert str(unit.get("status") or unit.get("phase") or "") in { + "eval_running", + "running", + "executing", + } + + blob = json.dumps(unit) + assert "latest-progress-marker" in blob, f"latest event missing: {unit!r}" + latest = unit.get("latest_event") + assert isinstance(latest, dict) + assert latest.get("message") == "latest-progress-marker" + assert latest.get("task_id") == "task-a" + assert latest.get("progress") == pytest.approx(0.55) + _assert_no_score_keys(body) + + +@pytest.mark.asyncio +async def test_execution_pool_live_excludes_completed_eval(client, database_session) -> None: + """Terminal/completed evals are excluded from the live pool.""" + + live = await _seed_run( + database_session, eval_run_id="eval-pool-still-live", phase="eval_running" + ) + done = await _seed_run( + database_session, + eval_run_id="eval-pool-done", + phase="eval_accepted", + score=0.91, + ) + await _seed_progress_event( + database_session, + submission_id=done.submission_id, + eval_run_id=done.eval_run_id, + sequence=1, + message="finished-should-be-hidden", + ) + await _seed_progress_event( + database_session, + submission_id=live.submission_id, + eval_run_id=live.eval_run_id, + sequence=1, + message="still-going", + ) + + response = await client.get(POOL_PATH) + assert response.status_code == 200, response.text + body = response.json() + units = body["units"] + ids = { + str(item.get("unit_id") or item.get("eval_run_id") or "") for item in units + } + assert live.eval_run_id in ids + assert done.eval_run_id not in ids + assert "finished-should-be-hidden" not in json.dumps(body) + _assert_no_score_keys(body) + + +@pytest.mark.asyncio +async def test_execution_pool_live_never_exposes_score_fields( + client, database_session +) -> None: + """Even when EvalRun.score is set on a non-terminal row, pool omits score keys.""" + + run = await _seed_run( + database_session, + eval_run_id="eval-pool-score-guard", + phase="eval_verifying", + score=0.42, + ) + # Defensive: score column may exist on the row; pool must not surface it. + async with database_session() as session: + refreshed = await session.scalar( + select(EvalRun).where(EvalRun.eval_run_id == run.eval_run_id) + ) + assert refreshed is not None + assert refreshed.score == pytest.approx(0.42) + + response = await client.get(POOL_PATH) + assert response.status_code == 200, response.text + body = response.json() + assert any( + str(u.get("unit_id") or u.get("eval_run_id")) == run.eval_run_id + for u in body["units"] + ) + _assert_no_score_keys(body) + blob = response.text.lower() + for token in ('"score"', '"final_score"', '"weights"', '"emission"'): + assert token not in blob, f"forbidden score token present: {token}" From a21e7ba9a4fbb6f6298fc9dede8a0500c13e1686 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:07:17 +0000 Subject: [PATCH 3/7] feat(agent-challenge): add attested progress and live pool telemetry Port progress validators, hotkey-bound telemetry sessions, progress ingest, ProgressReporter, env injection with mnemonic redaction, and GET /v1/execution-pool/live. Telemetry stays score-free. --- .../src/agent_challenge/api/routes.py | 221 +++++++++++++ .../src/agent_challenge/canonical/compose.py | 29 +- .../agent_challenge/canonical/eval_wire.py | 111 +++++++ .../evaluation/execution_pool.py | 153 +++++++++ .../own_runner/progress_reporter.py | 132 ++++++++ .../evaluation/telemetry_session.py | 299 ++++++++++++++++++ .../src/agent_challenge/selfdeploy/cli.py | 1 + .../src/agent_challenge/selfdeploy/eval.py | 36 ++- 8 files changed, 974 insertions(+), 8 deletions(-) create mode 100644 packages/challenges/agent-challenge/src/agent_challenge/evaluation/execution_pool.py create mode 100644 packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/progress_reporter.py create mode 100644 packages/challenges/agent-challenge/src/agent_challenge/evaluation/telemetry_session.py diff --git a/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py b/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py index db3f1db4f..7faadd163 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/api/routes.py @@ -77,6 +77,12 @@ process_direct_eval_result, validate_result_bounds, ) +from ..evaluation.execution_pool import list_live_execution_units +from ..evaluation.progress import ( + MAX_PROGRESS_BODY_BYTES, + EvalProgressError, + process_eval_progress, +) from ..evaluation.replay_audit import ( REPLAY_AUDIT_LABEL, AggregationSpec, @@ -102,6 +108,12 @@ redact_secrets, redact_task_event_message, ) +from ..evaluation.telemetry_session import ( + TELEMETRY_SESSION_HEADER, + TelemetrySessionError, + open_telemetry_session, + require_telemetry_session, +) from ..evaluation.terminal_bench import TERMINAL_BENCH_EVALUATOR from ..evaluation.validator_executor import ( finalize_job_if_complete, @@ -1853,6 +1865,215 @@ async def receive_direct_eval_result( ) +@router.post( + "/evaluation/v1/runs/{eval_run_id}/telemetry-session", +) +async def open_eval_telemetry_session( + eval_run_id: str, + http_request: Request, + session: DatabaseSession, + authorization: Annotated[str | None, Header()] = None, +) -> JSONResponse: + """Open a hotkey-attested telemetry session for mid-run progress posts. + + Auth mirrors the final result route (Bearer EVAL_RUN_TOKEN). Body carries + hotkey_ss58 + signature only — never a mnemonic. Progress POSTs must present + the issued session id via X-Telemetry-Session. + """ + + if not settings.attested_review_enabled or not settings.phala_attestation_enabled: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"code": "attested_eval_disabled"}, + ) + prefix = "Bearer " + token = ( + authorization[len(prefix) :] + if isinstance(authorization, str) and authorization.startswith(prefix) + else None + ) + content_type = http_request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if content_type != "application/json": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "telemetry_session_media_invalid"}, + ) + run = await session.scalar(select(EvalRun).where(EvalRun.eval_run_id == eval_run_id)) + if run is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"code": "eval_run_unknown"}, + ) + if not authenticate_eval_token(run, token): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail={"code": "invalid_eval_token"}, + ) + body = await _read_bounded_result_body( + http_request, + max_bytes=MAX_PROGRESS_BODY_BYTES, + ) + try: + try: + parsed = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise TelemetrySessionError( + "telemetry session body is not valid JSON", + code="invalid_telemetry_session", + ) from exc + if not isinstance(parsed, dict): + raise TelemetrySessionError( + "telemetry session body must be a JSON object", + code="invalid_telemetry_session", + ) + opened = open_telemetry_session( + eval_run_id=eval_run_id, + eval_run_phase=run.phase, + body=parsed, + ) + except TelemetrySessionError as exc: + if exc.code == "eval_run_terminal": + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail={"code": exc.code}, + ) from exc + if exc.code in { + "invalid_telemetry_signature", + "invalid_telemetry_session", + "telemetry_signature_invalid", + }: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail={"code": exc.code}, + ) from exc + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail={"code": exc.code}, + ) from exc + return JSONResponse(status_code=status.HTTP_200_OK, content=opened) + + +@router.post( + "/evaluation/v1/runs/{eval_run_id}/progress", +) +async def receive_eval_progress( + eval_run_id: str, + http_request: Request, + session: DatabaseSession, + authorization: Annotated[str | None, Header()] = None, + x_telemetry_session: Annotated[str | None, Header(alias="X-Telemetry-Session")] = None, +) -> JSONResponse: + """Receive one mid-run task progress event from the canonical Eval CVM. + + Auth: Bearer EVAL_RUN_TOKEN (same as result) plus X-Telemetry-Session from a + prior telemetry-session open. Events land in TaskLogEvent for the SSE feed. + This route never mutates scores — POST .../result remains the only score path. + """ + + if not settings.attested_review_enabled or not settings.phala_attestation_enabled: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"code": "attested_eval_disabled"}, + ) + prefix = "Bearer " + token = ( + authorization[len(prefix) :] + if isinstance(authorization, str) and authorization.startswith(prefix) + else None + ) + content_type = http_request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if content_type != "application/json": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"code": "progress_media_invalid"}, + ) + run = await session.scalar( + select(EvalRun).where(EvalRun.eval_run_id == eval_run_id).with_for_update() + ) + if run is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail={"code": "eval_run_unknown"}, + ) + if not authenticate_eval_token(run, token): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail={"code": "invalid_eval_token"}, + ) + # Prefer explicit Header param; fall back to raw header for alias robustness. + session_header = x_telemetry_session or http_request.headers.get(TELEMETRY_SESSION_HEADER) + try: + require_telemetry_session(session_header, eval_run_id=eval_run_id) + except TelemetrySessionError as exc: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail={"code": exc.code}, + ) from exc + body = await _read_bounded_result_body( + http_request, + max_bytes=MAX_PROGRESS_BODY_BYTES, + ) + try: + # Progress is observability-only (optional float progress). Do not use + # parse_json_object / canonical_json_v1 which forbid floats — the final + # result route remains the only canonical attested body path. + try: + parsed = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise EvalProgressError( + "progress body is not valid JSON", + code="progress_invalid", + ) from exc + if not isinstance(parsed, dict): + raise EvalProgressError( + "progress body must be a JSON object", + code="progress_invalid", + ) + request_body = parsed + if request_body.get("eval_run_id") != eval_run_id: + raise EvalProgressError( + "progress run does not match route", + code="progress_run_mismatch", + ) + receipt, created = await process_eval_progress( + session, + run=run, + progress_request=request_body, + ) + await session.commit() + except EvalProgressError as exc: + await session.rollback() + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail={"code": exc.code}, + ) from exc + except ValueError as exc: + await session.rollback() + code = exc.code if isinstance(exc, EvalProgressError) else "progress_invalid" + if code == "result_too_large": + raise HTTPException( + status_code=status.HTTP_413_CONTENT_TOO_LARGE, + detail={"code": code}, + ) from exc + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail={"code": code}, + ) from exc + return JSONResponse( + status_code=status.HTTP_202_ACCEPTED if created else status.HTTP_200_OK, + content=receipt, + ) + + +@public_route(tags=["execution"]) +@router.get("/v1/execution-pool/live") +async def execution_pool_live(session: DatabaseSession) -> dict[str, list[dict[str, object]]]: + """In-flight EvalRun units with latest TaskLogEvent. Empty pool is honest [].""" + + units = await list_live_execution_units(session) + return {"units": units} + + @router.post("/submissions/{submission_id}/review/deployed") async def acknowledge_submission_review_deployment( submission_id: int, diff --git a/packages/challenges/agent-challenge/src/agent_challenge/canonical/compose.py b/packages/challenges/agent-challenge/src/agent_challenge/canonical/compose.py index 135c32c28..3ce80019b 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/canonical/compose.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/canonical/compose.py @@ -148,11 +148,36 @@ KEY_RELEASE_TLS_CA_ENV, "LLM_COST_LIMIT", "EVAL_RUN_TOKEN", + # Mid-run progress reporter (observability only; never score-bearing). + "EVAL_PROGRESS_BASE_URL", + "EVAL_RUN_ID", + "EVAL_SUBMISSION_ID", + # Local CVM signing secret name only — value never leaves the guest / never hits master APIs. + "RUNNER_HOTKEY_MNEMONIC", # Measured OpenRouter (eval agent inside measured CVM only when product allows). # Never Base gateway; keys stay miner/session encrypted_env on attested guests. "OPENROUTER_API_KEY", ) + +#: Env names that may appear in encrypted_env / runner config but are NOT part of +#: the measured compose_hash pin (``04011776…``). Progress/telemetry is optional +#: observability injected at deploy; baking the names into allowed_envs would +#: rotate every historical pin. ``encrypt_eval_secrets`` still accepts these via +#: ``DEFAULT_ALLOWED_ENVS``; compose generation for eval pins uses +#: :data:`MEASURED_ALLOWED_ENVS`. +PROGRESS_OPTIONAL_ENVS: tuple[str, ...] = ( + "EVAL_PROGRESS_BASE_URL", + "EVAL_RUN_ID", + "EVAL_SUBMISSION_ID", + "RUNNER_HOTKEY_MNEMONIC", +) + +#: Pin-stable allowlist used when rendering measured app-compose bytes. +MEASURED_ALLOWED_ENVS: tuple[str, ...] = tuple( + name for name in DEFAULT_ALLOWED_ENVS if name not in set(PROGRESS_OPTIONAL_ENVS) +) + _DIGEST_PIN_RE = re.compile(r"@sha256:[0-9a-f]{64}$") _DIGEST_REF_RE = re.compile(r"^(?:sha256:)?[0-9a-f]{64}$") @@ -355,7 +380,7 @@ def generate_app_compose( orchestrator_image: str, name: str = DEFAULT_APP_NAME, command: Sequence[str] | None = None, - allowed_envs: Sequence[str] = DEFAULT_ALLOWED_ENVS, + allowed_envs: Sequence[str] = MEASURED_ALLOWED_ENVS, key_release_url: str | None = None, attestation_enabled: bool = True, job_dir: str = DEFAULT_JOB_DIR, @@ -496,6 +521,8 @@ def app_compose_hash(compose: Mapping[str, Any]) -> str: "APP_COMPOSE_MANIFEST_VERSION", "APP_COMPOSE_RUNNER", "DEFAULT_ALLOWED_ENVS", + "MEASURED_ALLOWED_ENVS", + "PROGRESS_OPTIONAL_ENVS", "DEFAULT_APP_NAME", "DEFAULT_CACHE_ROOT", "DEFAULT_DIGEST_MANIFEST", diff --git a/packages/challenges/agent-challenge/src/agent_challenge/canonical/eval_wire.py b/packages/challenges/agent-challenge/src/agent_challenge/canonical/eval_wire.py index 201d7e9ff..3ec24a5a7 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/canonical/eval_wire.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/canonical/eval_wire.py @@ -950,6 +950,113 @@ def validate_eval_receipt(value: Any) -> dict[str, Any]: } +# Mid-run progress (observability only — never carries score material). +EVAL_PROGRESS_PHASES = frozenset( + {"assigned", "starting", "waiting", "running", "completed", "failed"} +) +EVAL_PROGRESS_EVENT_TYPES = frozenset({"task.status", "task.progress"}) +_PROGRESS_FORBIDDEN_FIELDS = frozenset( + { + "score", + "score_record", + "scores_digest", + "execution_proof", + "agent_hash", + "canonical_score_record", + "passed_tasks", + "total_tasks", + } +) +_PROGRESS_REQUIRED_FIELDS = ( + "schema_version", + "eval_run_id", + "submission_id", + "task_id", + "sequence", + "status", +) +_PROGRESS_OPTIONAL_FIELDS = frozenset({"event_type", "progress", "message"}) + + +def validate_eval_progress_request(value: Any) -> dict[str, Any]: + """Validate a mid-run Eval progress event (schema-closed, score-free).""" + + if not isinstance(value, Mapping): + raise EvalWireError("eval_progress_request must be an object") + keys = set(value) + forbidden = sorted(keys & _PROGRESS_FORBIDDEN_FIELDS) + if forbidden: + raise EvalWireError(f"eval_progress_request forbids score fields: {forbidden}") + missing = [name for name in _PROGRESS_REQUIRED_FIELDS if name not in keys] + unknown = sorted(keys - set(_PROGRESS_REQUIRED_FIELDS) - _PROGRESS_OPTIONAL_FIELDS) + if missing or unknown: + raise EvalWireError( + f"eval_progress_request has invalid fields: missing={missing}, unknown={unknown}" + ) + if value["schema_version"] != 1: + raise EvalWireError("eval_progress_request schema_version must be 1") + status = value["status"] + if not isinstance(status, str) or status not in EVAL_PROGRESS_PHASES: + raise EvalWireError("eval_progress_request status is not a safe task phase") + event_type = value.get("event_type", "task.status") + if not isinstance(event_type, str) or event_type not in EVAL_PROGRESS_EVENT_TYPES: + raise EvalWireError("eval_progress_request event_type is invalid") + progress = value.get("progress", None) + if progress is not None: + if isinstance(progress, bool) or not isinstance(progress, (int, float)): + raise EvalWireError("eval_progress_request progress must be a number or null") + progress_f = float(progress) + if not math.isfinite(progress_f) or progress_f < 0.0 or progress_f > 1.0: + raise EvalWireError("eval_progress_request progress must be finite in [0, 1]") + progress = progress_f + message = value.get("message", None) + if message is not None: + if not isinstance(message, str): + raise EvalWireError("eval_progress_request message must be a string or null") + if len(message.encode("utf-8")) > EVAL_MAX_STRING_BYTES: + raise EvalWireError("eval_progress_request message exceeds its string bound") + return { + "schema_version": 1, + "eval_run_id": _id(value["eval_run_id"], "eval_run_id"), + "submission_id": _id(value["submission_id"], "submission_id"), + "task_id": _id(value["task_id"], "task_id"), + "sequence": _integer(value["sequence"], "sequence", minimum=1), + "status": status, + "event_type": event_type, + "progress": progress, + "message": message, + } + + +def validate_eval_progress_receipt(value: Any) -> dict[str, Any]: + """Validate the closed receipt returned by the progress ingest route.""" + + data = _object( + value, + "eval_progress_receipt", + ( + "schema_version", + "eval_run_id", + "task_id", + "sequence", + "event_id", + "created", + ), + ) + if data["schema_version"] != 1: + raise EvalWireError("eval_progress_receipt schema_version must be 1") + if not isinstance(data["created"], bool): + raise EvalWireError("eval_progress_receipt.created must be boolean") + return { + "schema_version": 1, + "eval_run_id": _id(data["eval_run_id"], "eval_run_id"), + "task_id": _id(data["task_id"], "task_id"), + "sequence": _integer(data["sequence"], "sequence", minimum=1), + "event_id": _integer(data["event_id"], "event_id", minimum=1), + "created": data["created"], + } + + __all__ = [ "EVAL_MAX_EVENT_LOG_BYTES", "EVAL_MAX_EVENT_LOG_ENTRIES", @@ -980,8 +1087,12 @@ def validate_eval_receipt(value: Any) -> dict[str, Any]: "validate_eval_execution_proof", "validate_eval_plan", "validate_eval_phala_attestation", + "validate_eval_progress_receipt", + "validate_eval_progress_request", "validate_eval_receipt", "validate_eval_result_request", + "EVAL_PROGRESS_EVENT_TYPES", + "EVAL_PROGRESS_PHASES", "validate_score_binding", "validate_scoring_policy", ] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/execution_pool.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/execution_pool.py new file mode 100644 index 000000000..e4bbd021e --- /dev/null +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/execution_pool.py @@ -0,0 +1,153 @@ +"""Live execution pool snapshot for master fan-out aggregation. + +``GET /v1/execution-pool/live`` returns in-flight EvalRun rows with each run's +latest TaskLogEvent. Observability only — never score / weight fields. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from datetime import datetime +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from agent_challenge.core.models import EvalRun, TaskLogEvent + +# Keep in lockstep with evaluation.authorization._ACTIVE_PHASES and +# evaluation.telemetry_session._ACTIVE_EVAL_PHASES. +ACTIVE_EVAL_PHASES: frozenset[str] = frozenset( + {"eval_prepared", "eval_running", "eval_verifying"} +) + +_SCORE_FIELD_NAMES: frozenset[str] = frozenset( + { + "score", + "scores", + "final_score", + "raw_score", + "normalized_score", + "weight", + "weights", + "emission", + "emission_percent", + "incentive", + "passed_tasks", + "total_tasks", + "canonical_score_record_json", + "canonical_score_record_sha256", + } +) + + +def _metadata_dict(raw: str | None) -> dict[str, Any]: + if not raw: + return {} + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + +def _iso(value: datetime | None) -> str | None: + if value is None: + return None + if value.tzinfo is None: + return value.isoformat() + "+00:00" + return value.isoformat() + + +def _latest_event_payload(event: TaskLogEvent) -> dict[str, Any]: + meta = _metadata_dict(event.metadata_json) + phase = event.status or meta.get("phase") + client_sequence = meta.get("client_sequence") + sequence = ( + int(client_sequence) + if isinstance(client_sequence, int) + else int(event.sequence) + ) + payload: dict[str, Any] = { + "event_type": event.event_type, + "sequence": sequence, + "task_id": event.task_id, + "message": event.message, + "progress": event.progress, + "phase": phase, + "created_at": _iso(event.created_at), + } + return {key: value for key, value in payload.items() if value is not None} + + +def _unit_from_run( + run: EvalRun, *, latest_event: Mapping[str, Any] | None +) -> dict[str, Any]: + unit: dict[str, Any] = { + "unit_id": run.eval_run_id, + "eval_run_id": run.eval_run_id, + "submission_id": str(run.submission_id), + "status": run.phase, + "phase": run.phase, + "latest_event": dict(latest_event) if latest_event is not None else None, + } + # Defense in depth: never leak score-shaped keys even if a caller mutates. + for banned in _SCORE_FIELD_NAMES: + unit.pop(banned, None) + return unit + + +async def _latest_events_by_eval_run( + session: AsyncSession, + *, + submission_ids: Sequence[int], +) -> dict[str, dict[str, Any]]: + """Map eval_run_id → latest TaskLogEvent payload for the given submissions.""" + + if not submission_ids: + return {} + rows = await session.scalars( + select(TaskLogEvent) + .where(TaskLogEvent.submission_id.in_(list(submission_ids))) + .order_by(TaskLogEvent.sequence.desc(), TaskLogEvent.id.desc()) + ) + latest: dict[str, dict[str, Any]] = {} + for event in rows: + meta = _metadata_dict(event.metadata_json) + eval_run_id = meta.get("eval_run_id") + if not isinstance(eval_run_id, str) or not eval_run_id: + continue + if eval_run_id in latest: + continue + latest[eval_run_id] = _latest_event_payload(event) + return latest + + +async def list_live_execution_units(session: AsyncSession) -> list[dict[str, Any]]: + """Return in-flight EvalRun units with latest progress event (if any).""" + + runs = list( + await session.scalars( + select(EvalRun) + .where(EvalRun.phase.in_(tuple(ACTIVE_EVAL_PHASES))) + .order_by(EvalRun.updated_at.desc(), EvalRun.id.desc()) + ) + ) + if not runs: + return [] + + latest_by_run = await _latest_events_by_eval_run( + session, + submission_ids=[run.submission_id for run in runs], + ) + return [ + _unit_from_run(run, latest_event=latest_by_run.get(run.eval_run_id)) + for run in runs + ] + + +__all__ = [ + "ACTIVE_EVAL_PHASES", + "list_live_execution_units", +] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/progress_reporter.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/progress_reporter.py new file mode 100644 index 000000000..aef6b0b4e --- /dev/null +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/own_runner/progress_reporter.py @@ -0,0 +1,132 @@ +"""Best-effort mid-run progress posts from the Eval CVM to the master. + +Posts closed progress bodies to +``POST /evaluation/v1/runs/{eval_run_id}/progress`` with Bearer +``EVAL_RUN_TOKEN``. Failures are swallowed so observability can never change a +score or abort a trial. +""" + +from __future__ import annotations + +import json +import logging +import os +import threading +import urllib.error +import urllib.request +from collections.abc import Mapping +from dataclasses import dataclass, field + +logger = logging.getLogger(__name__) + +PROGRESS_BASE_URL_ENV = "EVAL_PROGRESS_BASE_URL" +EVAL_RUN_ID_ENV = "EVAL_RUN_ID" +EVAL_SUBMISSION_ID_ENV = "EVAL_SUBMISSION_ID" +EVAL_RUN_TOKEN_ENV = "EVAL_RUN_TOKEN" +PROGRESS_TIMEOUT_ENV = "EVAL_PROGRESS_TIMEOUT_SECONDS" + +DEFAULT_TIMEOUT_SECONDS = 5.0 +SAFE_PHASES = frozenset({"assigned", "starting", "waiting", "running", "completed", "failed"}) + + +@dataclass +class ProgressReporter: + """Posts per-task phase transitions (best-effort, score-free).""" + + base_url: str + eval_run_id: str + submission_id: str + token: str + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS + _sequence: int = field(default=0, init=False, repr=False) + _lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False) + + @classmethod + def from_env(cls, env: Mapping[str, str] | None = None) -> ProgressReporter | None: + """Build a reporter from injected env, or ``None`` if not configured.""" + + source = os.environ if env is None else env + base_url = (source.get(PROGRESS_BASE_URL_ENV) or "").strip() + eval_run_id = (source.get(EVAL_RUN_ID_ENV) or "").strip() + submission_id = (source.get(EVAL_SUBMISSION_ID_ENV) or "").strip() + token = (source.get(EVAL_RUN_TOKEN_ENV) or "").strip() + if not (base_url and eval_run_id and submission_id and token): + return None + return cls( + base_url=base_url.rstrip("/"), + eval_run_id=eval_run_id, + submission_id=submission_id, + token=token, + timeout_seconds=_parse_timeout(source.get(PROGRESS_TIMEOUT_ENV)), + ) + + @property + def url(self) -> str: + return f"{self.base_url}/evaluation/v1/runs/{self.eval_run_id}/progress" + + def emit( + self, + *, + task_id: str, + status: str, + progress: float | None = None, + message: str | None = None, + event_type: str = "task.status", + ) -> None: + """POST one progress event; swallow any transport error.""" + + if status not in SAFE_PHASES: + logger.warning("progress reporter refused unknown phase %s", status) + return + with self._lock: + self._sequence += 1 + sequence = self._sequence + body: dict[str, object] = { + "schema_version": 1, + "eval_run_id": self.eval_run_id, + "submission_id": self.submission_id, + "task_id": task_id, + "sequence": sequence, + "status": status, + "event_type": event_type, + "progress": progress, + "message": message, + } + payload = json.dumps(body, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + request = urllib.request.Request( + self.url, + data=payload, + method="POST", + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {self.token}", + "Accept": "application/json", + }, + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout_seconds): + return + except (urllib.error.URLError, OSError, ValueError): + logger.warning("progress POST to %s failed", self.url, exc_info=True) + + +def _parse_timeout(raw: str | None) -> float: + if not raw: + return DEFAULT_TIMEOUT_SECONDS + try: + value = float(raw) + except ValueError: + return DEFAULT_TIMEOUT_SECONDS + return value if value > 0 else DEFAULT_TIMEOUT_SECONDS + + +__all__ = [ + "DEFAULT_TIMEOUT_SECONDS", + "EVAL_RUN_ID_ENV", + "EVAL_RUN_TOKEN_ENV", + "EVAL_SUBMISSION_ID_ENV", + "PROGRESS_BASE_URL_ENV", + "PROGRESS_TIMEOUT_ENV", + "ProgressReporter", + "SAFE_PHASES", +] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/evaluation/telemetry_session.py b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/telemetry_session.py new file mode 100644 index 000000000..f39ef1943 --- /dev/null +++ b/packages/challenges/agent-challenge/src/agent_challenge/evaluation/telemetry_session.py @@ -0,0 +1,299 @@ +"""Attested telemetry-session handshake for mid-run progress ingest. + +Phase B of the progress stack: after Bearer ``EVAL_RUN_TOKEN`` auth, the runner +opens a short-lived session by signing +``ac-telemetry-session:v1|{eval_run_id}|{instance_id}|{nonce}|{timestamp}`` +with its hotkey. Progress POSTs must present the issued session id via +``X-Telemetry-Session``. + +Trust boundary: tamper-evidence only. Sessions never carry score material and +never accept a mnemonic — only ``hotkey_ss58`` + signature. +""" + +from __future__ import annotations + +import secrets +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from threading import Lock +from typing import Any + +from agent_challenge.auth.security import verify_substrate_signature + +TELEMETRY_SESSION_HEADER = "X-Telemetry-Session" +CANONICAL_PREFIX = "ac-telemetry-session:v1" +DEFAULT_SESSION_TTL = timedelta(hours=1) +_ACTIVE_EVAL_PHASES = frozenset({"eval_prepared", "eval_running", "eval_verifying"}) +_SESSION_REQUIRED_FIELDS = ( + "schema_version", + "eval_run_id", + "instance_id", + "hotkey_ss58", + "nonce", + "timestamp", + "signature", +) +_SESSION_FORBIDDEN_FIELDS = frozenset( + { + "mnemonic", + "seed", + "private_key", + "secret", + "RUNNER_HOTKEY_MNEMONIC", + } +) + + +class TelemetrySessionError(ValueError): + """Schema, auth, or lifecycle failure for a telemetry session.""" + + def __init__(self, message: str, *, code: str) -> None: + super().__init__(message) + self.code = code + + +@dataclass(frozen=True, slots=True) +class TelemetrySession: + """In-process attested session record (tamper-evidence only).""" + + session_id: str + eval_run_id: str + instance_id: str + hotkey_ss58: str + expires_at: datetime + closed: bool = False + + +_LOCK = Lock() +_SESSIONS: dict[str, TelemetrySession] = {} + + +def _utcnow() -> datetime: + return datetime.now(UTC) + + +def _parse_timestamp(raw: str) -> datetime: + text = raw.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError as exc: + raise TelemetrySessionError( + "telemetry session timestamp is invalid", + code="invalid_telemetry_session", + ) from exc + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def _canonical_message( + *, + eval_run_id: str, + instance_id: str, + nonce: str, + timestamp: str, +) -> str: + return f"{CANONICAL_PREFIX}|{eval_run_id}|{instance_id}|{nonce}|{timestamp}" + + +def validate_telemetry_session_request( + value: Mapping[str, Any], + *, + eval_run_id: str, +) -> dict[str, str]: + """Validate a closed telemetry-session open body (no mnemonic).""" + + if not isinstance(value, Mapping): + raise TelemetrySessionError( + "telemetry session body must be an object", + code="invalid_telemetry_session", + ) + keys = set(value) + forbidden = sorted(keys & _SESSION_FORBIDDEN_FIELDS) + if forbidden or any(isinstance(k, str) and "mnemonic" in k.lower() for k in keys): + raise TelemetrySessionError( + "telemetry session forbids mnemonic/signing-secret fields", + code="invalid_telemetry_session", + ) + missing = [name for name in _SESSION_REQUIRED_FIELDS if name not in keys] + unknown = sorted(keys - set(_SESSION_REQUIRED_FIELDS)) + if missing or unknown: + raise TelemetrySessionError( + f"telemetry session has invalid fields: missing={missing}, unknown={unknown}", + code="invalid_telemetry_session", + ) + if value.get("schema_version") != 1: + raise TelemetrySessionError( + "telemetry session schema_version must be 1", + code="invalid_telemetry_session", + ) + body_run_id = value.get("eval_run_id") + if not isinstance(body_run_id, str) or body_run_id != eval_run_id: + raise TelemetrySessionError( + "telemetry session eval_run_id does not match route", + code="invalid_telemetry_session", + ) + out: dict[str, str] = {} + for name in ( + "eval_run_id", + "instance_id", + "hotkey_ss58", + "nonce", + "timestamp", + "signature", + ): + raw = value.get(name) + if not isinstance(raw, str) or not raw.strip(): + raise TelemetrySessionError( + f"telemetry session {name} must be a non-empty string", + code="invalid_telemetry_session", + ) + out[name] = raw.strip() + # Timestamp must parse; skew is not enforced beyond parseability for tests. + _parse_timestamp(out["timestamp"]) + return out + + +def open_telemetry_session( + *, + eval_run_id: str, + eval_run_phase: str, + body: Mapping[str, Any], + ttl: timedelta = DEFAULT_SESSION_TTL, +) -> dict[str, str]: + """Open a hotkey-attested session. Returns ``{session_id, expires_at}``.""" + + if eval_run_phase not in _ACTIVE_EVAL_PHASES: + raise TelemetrySessionError( + "eval run is terminal; telemetry session forbidden", + code="eval_run_terminal", + ) + validated = validate_telemetry_session_request(body, eval_run_id=eval_run_id) + message = _canonical_message( + eval_run_id=validated["eval_run_id"], + instance_id=validated["instance_id"], + nonce=validated["nonce"], + timestamp=validated["timestamp"], + ) + if not verify_substrate_signature( + validated["hotkey_ss58"], + message, + validated["signature"], + ): + raise TelemetrySessionError( + "telemetry session signature invalid", + code="invalid_telemetry_signature", + ) + session_id = f"ts_{secrets.token_urlsafe(24)}" + expires_at = _utcnow() + ttl + record = TelemetrySession( + session_id=session_id, + eval_run_id=eval_run_id, + instance_id=validated["instance_id"], + hotkey_ss58=validated["hotkey_ss58"], + expires_at=expires_at, + closed=False, + ) + with _LOCK: + _SESSIONS[session_id] = record + return { + "session_id": session_id, + "expires_at": expires_at.isoformat().replace("+00:00", "Z"), + } + + +def require_telemetry_session( + session_id: str | None, + *, + eval_run_id: str, +) -> TelemetrySession: + """Validate ``X-Telemetry-Session`` for a progress POST.""" + + if session_id is None or not str(session_id).strip(): + raise TelemetrySessionError( + "telemetry session header required", + code="telemetry_session_required", + ) + sid = str(session_id).strip() + with _LOCK: + record = _SESSIONS.get(sid) + if record is None: + raise TelemetrySessionError( + "telemetry session unknown", + code="telemetry_session_unknown", + ) + if record.eval_run_id != eval_run_id: + raise TelemetrySessionError( + "telemetry session does not match eval run", + code="invalid_telemetry_session", + ) + if record.closed: + raise TelemetrySessionError( + "telemetry session closed", + code="telemetry_session_closed", + ) + if record.expires_at <= _utcnow(): + raise TelemetrySessionError( + "telemetry session expired", + code="telemetry_session_expired", + ) + return record + + +async def close_telemetry_session(session_id: str) -> None: + """Mark a session closed (test + lifecycle helper).""" + + with _LOCK: + record = _SESSIONS.get(session_id) + if record is None: + return + _SESSIONS[session_id] = TelemetrySession( + session_id=record.session_id, + eval_run_id=record.eval_run_id, + instance_id=record.instance_id, + hotkey_ss58=record.hotkey_ss58, + expires_at=record.expires_at, + closed=True, + ) + + +async def expire_telemetry_session(session_id: str) -> None: + """Force-expire a session by setting expires_at in the past.""" + + with _LOCK: + record = _SESSIONS.get(session_id) + if record is None: + return + _SESSIONS[session_id] = TelemetrySession( + session_id=record.session_id, + eval_run_id=record.eval_run_id, + instance_id=record.instance_id, + hotkey_ss58=record.hotkey_ss58, + expires_at=_utcnow() - timedelta(seconds=1), + closed=record.closed, + ) + + +def reset_telemetry_sessions_for_tests() -> None: + """Clear in-process session store (tests only).""" + + with _LOCK: + _SESSIONS.clear() + + +__all__ = [ + "CANONICAL_PREFIX", + "DEFAULT_SESSION_TTL", + "TELEMETRY_SESSION_HEADER", + "TelemetrySession", + "TelemetrySessionError", + "close_telemetry_session", + "expire_telemetry_session", + "open_telemetry_session", + "require_telemetry_session", + "reset_telemetry_sessions_for_tests", + "validate_telemetry_session_request", +] diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py index 63dcdff26..43087ecad 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/cli.py @@ -738,6 +738,7 @@ def _ordered_eval_command(args: argparse.Namespace) -> int: "token", "OPENROUTER_API_KEY", "EVAL_RUN_TOKEN", + "RUNNER_HOTKEY_MNEMONIC", "REVIEW_SESSION_TOKEN", "BASE_GATEWAY_TOKEN", # residual key name only; not product eval secret "golden_plaintext", diff --git a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py index 1809c6717..40f9c6c09 100644 --- a/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py +++ b/packages/challenges/agent-challenge/src/agent_challenge/selfdeploy/eval.py @@ -20,6 +20,7 @@ from agent_challenge.canonical import eval_wire from agent_challenge.canonical.compose import ( DEFAULT_ALLOWED_ENVS, + MEASURED_ALLOWED_ENVS, generate_app_compose, render_app_compose, ) @@ -41,7 +42,7 @@ #: Capacity-safe default (bare ``us-west`` → ERR-02-002 No teepod found). DEFAULT_REGION = "us-west-1" -EVAL_ALLOWED_ENVS: tuple[str, ...] = DEFAULT_ALLOWED_ENVS +EVAL_ALLOWED_ENVS: tuple[str, ...] = MEASURED_ALLOWED_ENVS # VAL-ACAT-013: production eval encrypted_env must NOT require Base LLM gateway # secrets. Gateway routing is removed; only eval-run capability + attestation # plan bindings (and optional cost limit) are required. @@ -271,7 +272,11 @@ def encrypt_eval_secrets( ) -> EncryptedEvalSecrets: """Encrypt the Eval run token and attestation plan bindings (no Base gateway).""" - if not set(secrets) <= set(EVAL_ALLOWED_ENVS) or not EVAL_REQUIRED_SECRET_ENVS <= set(secrets): + # Encrypt allowlist is the full DEFAULT_ALLOWED_ENVS (includes optional + # progress/telemetry names). Measured compose generation stays on + # EVAL_ALLOWED_ENVS / MEASURED_ALLOWED_ENVS so historical pins remain stable. + allowed = set(DEFAULT_ALLOWED_ENVS) + if not set(secrets) <= allowed or not EVAL_REQUIRED_SECRET_ENVS <= set(secrets): raise EvalDeploymentError( "Eval encrypted_env names must be scoped allowed names with the required run " "and attestation plan capabilities (Base LLM gateway secrets are not allowed)" @@ -296,9 +301,7 @@ def encrypt_eval_secrets( free_url = secrets[KEY_RELEASE_URL_ENV] plan_endpoint = str(plan.plan.get("key_release_endpoint") or "").strip() plan_auth = parse_key_release_authority(plan_endpoint) - free_auth = parse_key_release_authority( - free_url if isinstance(free_url, str) else "" - ) + free_auth = parse_key_release_authority(free_url if isinstance(free_url, str) else "") if plan_auth is None or free_auth is None or free_auth != plan_auth: raise EvalDeploymentError( "Eval encrypted_env CHALLENGE_PHALA_KEY_RELEASE_URL is not miner-" @@ -306,7 +309,7 @@ def encrypt_eval_secrets( "authority (prefer KEY_RELEASE_RA_TLS_HOST/PORT). Free HTTP(S) KR " "URLs are refused." ) - env_keys = tuple(name for name in EVAL_ALLOWED_ENVS if name in secrets) + env_keys = tuple(name for name in DEFAULT_ALLOWED_ENVS if name in secrets) values = {name: secrets[name] for name in env_keys} if any(not isinstance(value, str) or not value for value in values.values()): raise EvalDeploymentError("Eval encrypted_env values must be non-empty strings") @@ -345,7 +348,7 @@ def deploy( encrypted.eval_run_id != plan.eval_run_id or encrypted.app_identity != plan.app_identity or encrypted.kms_public_key_sha256 != plan.kms_public_key_sha256 - or not set(encrypted.env_keys) <= set(EVAL_ALLOWED_ENVS) + or not set(encrypted.env_keys) <= set(DEFAULT_ALLOWED_ENVS) or not encrypted.ciphertext ): raise EvalDeploymentError("Eval encrypted_env is not bound to this run") @@ -455,6 +458,24 @@ def post(self, path: str, payload: Mapping[str, Any]) -> Mapping[str, Any]: raise AssertionError(f"unexpected Phala API path {path}") +def build_eval_progress_env( + *, + base_url: str, + eval_run_id: str, + submission_id: str, + eval_run_token: str, +) -> dict[str, str]: + """Bind progress-reporter env from plan ids + run token (no mnemonic).""" + + cleaned = base_url.strip().rstrip("/") + return { + "EVAL_PROGRESS_BASE_URL": cleaned, + "EVAL_RUN_ID": eval_run_id, + "EVAL_SUBMISSION_ID": str(submission_id), + "EVAL_RUN_TOKEN": eval_run_token, + } + + __all__ = [ "DEFAULT_EVAL_COMPOSE_NAME", "DEFAULT_EVAL_PHALA_APP_NONCE", @@ -469,5 +490,6 @@ def post(self, path: str, payload: Mapping[str, Any]) -> Mapping[str, Any]: "EvalPhalaDeployment", "HttpEvalPhalaDeployment", "build_eval_deployment_plan", + "build_eval_progress_env", "encrypt_eval_secrets", ] From e3fb72cfccb50fd6af1d4b1297f7f6a2e61c614a Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:07:17 +0000 Subject: [PATCH 4/7] test(prism): add contracts for execution telemetry Cover schema v5 execution_events, session/events routes, trust boundary (no tier elevation), and live pool surface. --- .../prism/tests/test_execution_events.py | 746 ++++++++++++++++++ .../prism/tests/test_execution_pool_live.py | 331 ++++++++ 2 files changed, 1077 insertions(+) create mode 100644 packages/challenges/prism/tests/test_execution_events.py create mode 100644 packages/challenges/prism/tests/test_execution_pool_live.py diff --git a/packages/challenges/prism/tests/test_execution_events.py b/packages/challenges/prism/tests/test_execution_events.py new file mode 100644 index 000000000..d30d9ba5a --- /dev/null +++ b/packages/challenges/prism/tests/test_execution_events.py @@ -0,0 +1,746 @@ +"""RED tests: Prism realtime execution telemetry (events + session + trust boundary). + +Pins the NEW surface that does not exist yet: + +* ``execution_events`` table (raw SQL, llm_review_events-shaped) +* repository append with monotone sequence + idempotency +* hotkey-signed ``POST /v1/execution/telemetry-session`` +* session-gated ``POST /v1/execution/events`` +* schema revision bump past ``prism-schema.v4`` +* hard trust boundary: telemetry never scores and never elevates tier + +TDD: these tests MUST fail until production code lands. Collection must stay green. +""" + +from __future__ import annotations + +import hmac +import json +import sqlite3 +import time +from hashlib import sha256 +from pathlib import Path +from typing import Any + +import anyio +import pytest +from fastapi.testclient import TestClient + +from prism_challenge.app import create_app +from prism_challenge.audit import effective_tier +from prism_challenge.auth import canonical_submission_message +from prism_challenge.config import PrismSettings +from prism_challenge.db import PRISM_SCHEMA_REVISION +from prism_challenge.proof import ( + ExecutionProof, + ProviderInfo, + build_execution_proof, + worker_signer_from_key, +) + +# Current declared revision is prism-schema.v4; telemetry surface requires the next bump. +_EXPECTED_SCHEMA_REVISION = "prism-schema.v5" +_INTERNAL_TOKEN = "secret" +_HOTKEY = "hk-telemetry-worker" +_WORKER_KEY = "//WorkerTelemetry" + + +def _settings(tmp_path: Path) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'telemetry.sqlite3'}", + shared_token=_INTERNAL_TOKEN, + allow_insecure_signatures=True, + fineweb_sample_count=4, + distributed_contract_policy="off", + ) + + +@pytest.fixture +def client(tmp_path: Path) -> TestClient: + with TestClient(create_app(_settings(tmp_path))) as test_client: + yield test_client + + +def _db_path(client: TestClient) -> Path: + return Path(client.app.state.database.path) + + +def _tables(client: TestClient) -> set[str]: + conn = sqlite3.connect(_db_path(client)) + try: + rows = conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() + finally: + conn.close() + return {str(row[0]) for row in rows} + + +def _table_columns(client: TestClient, table: str) -> set[str]: + conn = sqlite3.connect(_db_path(client)) + try: + rows = conn.execute(f"PRAGMA table_info({table})").fetchall() + finally: + conn.close() + return {str(row[1]) for row in rows} + + +def _count_events( + client: TestClient, + *, + eval_job_id: str | None = None, + work_unit_id: str | None = None, +) -> int: + conn = sqlite3.connect(_db_path(client)) + try: + if eval_job_id is not None: + row = conn.execute( + "SELECT COUNT(*) FROM execution_events WHERE eval_job_id=?", + (eval_job_id,), + ).fetchone() + elif work_unit_id is not None: + row = conn.execute( + "SELECT COUNT(*) FROM execution_events WHERE work_unit_id=?", + (work_unit_id,), + ).fetchone() + else: + row = conn.execute("SELECT COUNT(*) FROM execution_events").fetchone() + finally: + conn.close() + return int(row[0]) if row else 0 + + +def _event_sequences(client: TestClient, eval_job_id: str) -> list[int]: + conn = sqlite3.connect(_db_path(client)) + try: + rows = conn.execute( + "SELECT sequence FROM execution_events WHERE eval_job_id=? ORDER BY sequence", + (eval_job_id,), + ).fetchall() + finally: + conn.close() + return [int(row[0]) for row in rows] + + +def _score_row(client: TestClient, submission_id: str) -> Any: + conn = sqlite3.connect(_db_path(client)) + try: + return conn.execute( + "SELECT final_score FROM scores WHERE submission_id=?", + (submission_id,), + ).fetchone() + finally: + conn.close() + + +def _sign_body( + body: bytes, *, hotkey: str, nonce: str, secret: str = _INTERNAL_TOKEN +) -> dict[str, str]: + """Reuse Prism's canonical hotkey-signature scheme (auth.canonical_submission_message).""" + + timestamp = str(int(time.time())) + message = canonical_submission_message( + hotkey=hotkey, nonce=nonce, timestamp=timestamp, body=body + ) + signature = hmac.new(secret.encode(), message, sha256).hexdigest() + return { + "X-Hotkey": hotkey, + "X-Signature": signature, + "X-Nonce": nonce, + "X-Timestamp": timestamp, + } + + +def _internal_headers() -> dict[str, str]: + return {"Authorization": f"Bearer {_INTERNAL_TOKEN}"} + + +def _session_payload( + *, + eval_job_id: str | None = "job-live-1", + work_unit_id: str | None = None, + instance_id: str = "instance-gpu-0", + hotkey_ss58: str = _HOTKEY, + nonce: str | None = None, +) -> dict[str, Any]: + payload: dict[str, Any] = { + "instance_id": instance_id, + "hotkey_ss58": hotkey_ss58, + "nonce": nonce or f"tel-nonce-{int(time.time() * 1000)}", + "timestamp": str(int(time.time())), + } + if eval_job_id is not None: + payload["eval_job_id"] = eval_job_id + if work_unit_id is not None: + payload["work_unit_id"] = work_unit_id + return payload + + +def _open_telemetry_session( + client: TestClient, + payload: dict[str, Any] | None = None, + *, + include_internal: bool = True, + include_hotkey_sig: bool = True, +) -> Any: + body_obj = dict(payload or _session_payload()) + raw = json.dumps(body_obj, separators=(",", ":")).encode() + headers: dict[str, str] = {"Content-Type": "application/json"} + if include_internal: + headers.update(_internal_headers()) + if include_hotkey_sig: + headers.update( + _sign_body( + raw, + hotkey=str(body_obj["hotkey_ss58"]), + nonce=str(body_obj["nonce"]), + ) + ) + return client.post("/v1/execution/telemetry-session", content=raw, headers=headers) + + +def _ingest_events( + client: TestClient, + *, + session_id: str | None, + events: list[dict[str, Any]], + include_auth: bool = True, +) -> Any: + body_obj: dict[str, Any] = {"events": events} + if session_id is not None: + body_obj["session_id"] = session_id + raw = json.dumps(body_obj, separators=(",", ":")).encode() + headers: dict[str, str] = {"Content-Type": "application/json"} + if include_auth: + headers.update(_internal_headers()) + if session_id is not None: + headers["X-Telemetry-Session"] = session_id + return client.post("/v1/execution/events", content=raw, headers=headers) + + +def _seed_running_job( + client: TestClient, + *, + job_id: str = "job-live-1", + submission_id: str = "sub-tel-1", + status: str = "running", +) -> None: + repository = client.app.state.repository + + async def insert() -> None: + async with repository.database.connect() as conn: + await conn.execute( + "INSERT OR IGNORE INTO submissions(" + "id, hotkey, epoch_id, filename, code, code_hash, metadata, status, " + "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + submission_id, + _HOTKEY, + 1, + "project.zip", + "e30=", + "hash-tel", + "{}", + "running" if status == "running" else "completed", + "2026-01-01T00:00:00+00:00", + "2026-01-01T00:00:00+00:00", + ), + ) + await conn.execute( + "INSERT INTO eval_jobs(" + "id, submission_id, level, status, attempts, metrics, " + "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + job_id, + submission_id, + "l2", + status, + 0, + "{}", + "2026-01-01T00:00:00+00:00", + "2026-01-01T00:05:00+00:00", + ), + ) + + anyio.run(insert) + + +def _sample_events(eval_job_id: str, *, task_id: str = "train") -> list[dict[str, Any]]: + """Three progress events — no score fields allowed on the wire.""" + + return [ + { + "eval_job_id": eval_job_id, + "task_id": task_id, + "sequence": 1, + "event_type": "execution.started", + "message": "worker claimed unit", + "progress": 0.0, + }, + { + "eval_job_id": eval_job_id, + "task_id": task_id, + "sequence": 2, + "event_type": "execution.progress", + "message": "step 10", + "progress": 0.25, + }, + { + "eval_job_id": eval_job_id, + "task_id": task_id, + "sequence": 3, + "event_type": "execution.progress", + "message": "step 40", + "progress": 0.75, + }, + ] + + +def _minimal_proof_dict(signer: Any, *, unit_id: str, tier: int = 1) -> dict[str, Any]: + proof = build_execution_proof( + signer=signer, + manifest_sha256="c" * 64, + unit_id=unit_id, + image_digest="sha256:" + ("11" * 32), + provider=ProviderInfo(name="lium", pod_id="p"), + tier=tier, # type: ignore[arg-type] + ) + return proof.model_dump(mode="json") + + +def test_schema_revision_bumped_for_execution_events() -> None: + """Telemetry surface must bump PRISM_SCHEMA_REVISION past prism-schema.v4.""" + + assert PRISM_SCHEMA_REVISION == _EXPECTED_SCHEMA_REVISION, ( + f"expected schema revision {_EXPECTED_SCHEMA_REVISION!r} after execution_events " + f"migration; got {PRISM_SCHEMA_REVISION!r}" + ) + + +def test_execution_events_table_shape(client: TestClient) -> None: + """execution_events mirrors llm_review_events idioms: sequence + unique idempotency key.""" + + assert "execution_events" in _tables(client), ( + "missing execution_events table — add CREATE TABLE to db.SCHEMA / _run_migrations" + ) + cols = _table_columns(client, "execution_events") + required = { + "id", + "eval_job_id", + "work_unit_id", + "task_id", + "sequence", + "event_type", + "payload", + "session_id", + "hotkey_ss58", + "created_at", + } + missing = required - cols + assert not missing, f"execution_events missing columns: {sorted(missing)}" + + +def test_repository_exposes_append_execution_event(client: TestClient) -> None: + repo = client.app.state.repository + assert hasattr(repo, "append_execution_event"), ( + "PrismRepository.append_execution_event missing — monotone sequence + idempotent append" + ) + assert callable(repo.append_execution_event) + + +def test_s7_happy_events_recorded(client: TestClient) -> None: + """Open telemetry session then ingest >=3 events → rows with monotone sequence.""" + + job_id = "job-happy-1" + _seed_running_job(client, job_id=job_id, submission_id="sub-happy-1") + + session_resp = _open_telemetry_session( + client, _session_payload(eval_job_id=job_id, nonce="happy-n1") + ) + assert session_resp.status_code == 200, session_resp.text + session_body = session_resp.json() + assert "session_id" in session_body and session_body["session_id"] + dumped = json.dumps(session_body) + assert "mnemonic" not in dumped + assert "abandon" not in dumped + + events = _sample_events(job_id) + for event in events: + assert "score" not in event + assert "final_score" not in event + assert "q_arch" not in event + + ingest = _ingest_events(client, session_id=session_body["session_id"], events=events) + assert ingest.status_code == 200, ingest.text + ingest_body = ingest.json() + assert "score" not in ingest_body + assert "final_score" not in ingest_body + assert "scores" not in ingest_body + + assert "execution_events" in _tables(client) + assert _count_events(client, eval_job_id=job_id) >= 3 + sequences = _event_sequences(client, job_id) + assert sequences == sorted(sequences) + assert sequences == list(range(sequences[0], sequences[0] + len(sequences))) + assert sequences[0] >= 1 + + +def test_s8_auth_required(client: TestClient) -> None: + """Ingest without valid auth → 401; without a valid session → 401.""" + + job_id = "job-auth-1" + _seed_running_job(client, job_id=job_id, submission_id="sub-auth-1") + events = _sample_events(job_id) + + bare = client.post( + "/v1/execution/events", + content=json.dumps({"events": events, "session_id": "nope"}).encode(), + headers={"Content-Type": "application/json"}, + ) + assert bare.status_code == 401, bare.text + + no_session = _ingest_events(client, session_id=None, events=events, include_auth=True) + assert no_session.status_code == 401, no_session.text + + bogus_session = _ingest_events( + client, session_id="session-does-not-exist", events=events, include_auth=True + ) + assert bogus_session.status_code == 401, bogus_session.text + + no_internal = _open_telemetry_session( + client, + _session_payload(eval_job_id=job_id, nonce="auth-n-no-int"), + include_internal=False, + include_hotkey_sig=True, + ) + assert no_internal.status_code == 401, no_internal.text + + no_sig = _open_telemetry_session( + client, + _session_payload(eval_job_id=job_id, nonce="auth-n-no-sig"), + include_internal=True, + include_hotkey_sig=False, + ) + assert no_sig.status_code == 401, no_sig.text + + +def test_s8_idempotent_sequence(client: TestClient) -> None: + """Replaying the same (job, task, sequence) does not duplicate; regressing sequence → 422.""" + + job_id = "job-idem-1" + _seed_running_job(client, job_id=job_id, submission_id="sub-idem-1") + + session = _open_telemetry_session(client, _session_payload(eval_job_id=job_id, nonce="idem-n1")) + assert session.status_code == 200, session.text + session_id = session.json()["session_id"] + + first_batch = _sample_events(job_id) + r1 = _ingest_events(client, session_id=session_id, events=first_batch) + assert r1.status_code == 200, r1.text + count_after_first = _count_events(client, eval_job_id=job_id) + assert count_after_first >= 3 + + r2 = _ingest_events(client, session_id=session_id, events=first_batch) + assert r2.status_code in {200, 409}, r2.text + assert _count_events(client, eval_job_id=job_id) == count_after_first + + r3 = _ingest_events( + client, + session_id=session_id, + events=[ + { + "eval_job_id": job_id, + "task_id": "train", + "sequence": 0, + "event_type": "execution.progress", + "message": "illegal zero/regress sequence", + "progress": 0.5, + } + ], + ) + assert r3.status_code == 422, r3.text + assert _count_events(client, eval_job_id=job_id) == count_after_first + + +@pytest.mark.asyncio +async def test_s8_constation_reject_emits_event_no_score( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """When constation is rejected, an execution event IS recorded but NO score is written.""" + + from dataclasses import replace + + from prism_challenge.constation import CheckOutcome, ConstationBundle + from prism_challenge.evaluator.mock_reexec import cpu_reexec_run + from prism_challenge.ingestion import ingest_work_unit_result + from prism_challenge.models import SubmissionCreate + from prism_challenge.proof import ( + MANIFEST_PAYLOAD_KEY, + PROOF_PAYLOAD_KEY, + compute_manifest_sha256, + ) + + data_dir = tmp_path / "train-data" + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "train-00000.jsonl").write_text( + '{"id": "doc-0", "text": "prism telemetry constation reject fixture text bytes"}\n', + encoding="utf-8", + ) + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + cpu_reexec_run(train_data_dir=data_dir), + ) + + settings = _settings(tmp_path) + app = create_app(settings) + await app.state.database.init() + + sub = await app.state.repository.create_submission( + _HOTKEY, SubmissionCreate(code="e30=", filename="model.py") + ) + submission_id = sub.id + job_id = f"job-{submission_id}" + + async with app.state.repository.database.connect() as conn: + await conn.execute( + "INSERT INTO eval_jobs(" + "id, submission_id, level, status, attempts, metrics, " + "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + job_id, + submission_id, + "l2", + "running", + 0, + "{}", + "2026-01-01T00:00:00+00:00", + "2026-01-01T00:05:00+00:00", + ), + ) + + with TestClient(app) as client: + session = _open_telemetry_session( + client, + _session_payload(eval_job_id=job_id, work_unit_id=submission_id, nonce="const-n1"), + ) + assert session.status_code == 200, session.text + session_id = session.json()["session_id"] + + reject_event = { + "eval_job_id": job_id, + "work_unit_id": submission_id, + "task_id": "constation", + "sequence": 1, + "event_type": "constation.rejected", + "message": "constation_ok failed — observability only", + } + assert "score" not in reject_event + tel = _ingest_events(client, session_id=session_id, events=[reject_event]) + assert tel.status_code == 200, tel.text + assert _count_events(client, eval_job_id=job_id) >= 1 + + signer = worker_signer_from_key(_WORKER_KEY) + manifest = { + "schema_version": "prism_run_manifest.v2", + "data": {"covered_bytes": 4096, "single_pass": True}, + "metrics": { + "online_loss": [10.0, 6.0, 3.0], + "sum_neg_log_likelihood_nats": 900.0, + "covered_bytes": 4096, + "predicted_tokens": 96, + "step0_loss": 10.0, + "consumed_batches": 3, + "prequential_bpb": 1.23, + }, + "anti_cheat": { + "step0_anomaly": False, + "nan_inf_detected": False, + "no_learning": False, + "zero_forward": False, + }, + } + digest = compute_manifest_sha256(manifest) + image = "sha256:" + ("11" * 32) + proof = build_execution_proof( + signer=signer, + manifest_sha256=digest, + unit_id=submission_id, + image_digest=image, + constation_digest=image, + provider=ProviderInfo(name="lium", pod_id="pod-tel"), + tier=1, # type: ignore[arg-type] + ) + result = { + "executed": 1, + "completed_submissions": [], + PROOF_PAYLOAD_KEY: proof.model_dump(mode="json"), + MANIFEST_PAYLOAD_KEY: manifest, + } + man = {"legacy-test-harness.py": "a" * 64} + good_bundle = ConstationBundle( + commit_sha="a" * 40, + tree_sha="b" * 40, + variant="cuda", + digest=image, + work_unit_id=submission_id, + miner_hotkey=_HOTKEY, + pod_id="pod-tel", + nonce="const-reject-n", + signed_attestation={"legacy": True}, + expected_sealed_manifest_hashes=dict(man), + reported_sealed_manifest_hashes=dict(man), + lium_declared_digest=image, + constation_gap_budget_seconds=30.0, + constation_observed_max_gap_seconds=1.0, + ) + bad_bundle = replace( + good_bundle, + reported_sealed_manifest_hashes={"legacy-test-harness.py": "f" * 64}, + ) + + def _ok(**_k: object) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + def _sig(_s: object) -> CheckOutcome: + return CheckOutcome(ok=True, reason="ok") + + outcome = await ingest_work_unit_result( + worker=app.state.worker, + work_unit_id=submission_id, + submission_ref=_HOTKEY, + result=result, + pinned_image_digest=image, + constation_bundle=bad_bundle, + check_allowlist=_ok, + check_nonce=_ok, + verify_constation_signature=_sig, + ) + + assert outcome.score_written is False + assert _score_row(client, submission_id) is None + assert _count_events(client, eval_job_id=job_id) >= 1 + tier = getattr(outcome, "effective_tier", None) + if tier is not None: + assert int(tier) <= 0 + + +def test_s8_telemetry_never_elevates_tier(client: TestClient) -> None: + """No volume/content of telemetry can raise effective tier; constation_ok is sole signal.""" + + job_id = "job-tier-1" + _seed_running_job(client, job_id=job_id, submission_id="sub-tier-1") + + session = _open_telemetry_session(client, _session_payload(eval_job_id=job_id, nonce="tier-n1")) + assert session.status_code == 200, session.text + session_id = session.json()["session_id"] + + flood: list[dict[str, Any]] = [] + for seq in range(1, 21): + flood.append( + { + "eval_job_id": job_id, + "task_id": "train", + "sequence": seq, + "event_type": "execution.attestation_claim", + "message": "claimed tier=2 tee=true", + "progress": min(1.0, seq / 20.0), + "metadata": { + "claimed_tier": 2, + "tee": True, + "constation_ok": True, + "final_score": 999.0, + "q_arch": 1.0, + }, + } + ) + ingest = _ingest_events(client, session_id=session_id, events=flood) + assert ingest.status_code in {200, 422}, ingest.text + if ingest.status_code == 200: + body = ingest.json() + assert "final_score" not in body + assert "score" not in body + assert _score_row(client, "sub-tier-1") is None + + signer = worker_signer_from_key(_WORKER_KEY) + proof = ExecutionProof.model_validate(_minimal_proof_dict(signer, unit_id="sub-tier-1", tier=1)) + + assert effective_tier(proof, constation_ok_result=False) == 0 + assert effective_tier(proof, constation_ok_result=None) == 0 + assert effective_tier(proof, constation_ok_result=True) == 1 + + proof_t2 = ExecutionProof.model_validate( + _minimal_proof_dict(signer, unit_id="sub-tier-1", tier=2) + ) + assert effective_tier(proof_t2, constation_ok_result=True) == 0 + + repo = client.app.state.repository + for forbidden in ( + "elevate_tier_from_telemetry", + "grant_tier_from_events", + "apply_telemetry_score", + ): + assert not hasattr(repo, forbidden), f"forbidden trust API present: {forbidden}" + + +def test_telemetry_session_rejects_mnemonic_and_binds_hotkey(client: TestClient) -> None: + """Session payload binds hotkey_ss58 + signature; mnemonic must never be accepted.""" + + job_id = "job-mnemo-1" + _seed_running_job(client, job_id=job_id, submission_id="sub-mnemo-1") + + payload = _session_payload(eval_job_id=job_id, nonce="mnemo-n1") + poisoned = dict(payload) + poisoned["mnemonic"] = ( + "abandon abandon abandon abandon abandon abandon abandon abandon " + "abandon abandon abandon about" + ) + poisoned["wallet_seed"] = "0xdead" + + resp = _open_telemetry_session(client, poisoned) + assert resp.status_code in {200, 422}, resp.text + if resp.status_code == 200: + body = resp.json() + dumped = json.dumps(body) + assert "mnemonic" not in dumped + assert "abandon" not in dumped + assert "wallet_seed" not in dumped + assert body.get("hotkey_ss58", payload["hotkey_ss58"]) == payload["hotkey_ss58"] + assert "session_id" in body + + good = _open_telemetry_session(client, _session_payload(eval_job_id=job_id, nonce="mnemo-n2")) + assert good.status_code == 200, good.text + good_body = good.json() + assert good_body["session_id"] + for banned in ("score", "final_score", "q_arch", "q_recipe", "effective_tier"): + assert banned not in good_body + + +def test_telemetry_events_reject_score_fields(client: TestClient) -> None: + """Ingest body must not accept score fields as first-class event attributes.""" + + job_id = "job-score-1" + _seed_running_job(client, job_id=job_id, submission_id="sub-score-1") + session = _open_telemetry_session( + client, _session_payload(eval_job_id=job_id, nonce="score-n1") + ) + assert session.status_code == 200, session.text + session_id = session.json()["session_id"] + + dirty = [ + { + "eval_job_id": job_id, + "task_id": "train", + "sequence": 1, + "event_type": "execution.progress", + "message": "trying to plant a score", + "final_score": 42.0, + "score": 42.0, + "q_arch": 0.9, + } + ] + resp = _ingest_events(client, session_id=session_id, events=dirty) + if resp.status_code == 200: + assert _score_row(client, "sub-score-1") is None + body = resp.json() + assert "final_score" not in body + assert "score" not in body + else: + assert resp.status_code == 422, resp.text diff --git a/packages/challenges/prism/tests/test_execution_pool_live.py b/packages/challenges/prism/tests/test_execution_pool_live.py new file mode 100644 index 000000000..c27d5f8f8 --- /dev/null +++ b/packages/challenges/prism/tests/test_execution_pool_live.py @@ -0,0 +1,331 @@ +"""RED tests: Prism live execution pool read + adjacent submission/curve regression. + +Pins: + +* ``GET /v1/execution-pool/live`` — in-flight jobs with latest event; terminal excluded +* S9 adjacent regression — existing submission status + curve endpoints unchanged +""" + +from __future__ import annotations + +import hmac +import json +import time +from hashlib import sha256 +from pathlib import Path +from typing import Any + +import anyio +import pytest +from base.challenge_sdk.executor import DockerRunResult +from conftest import VALID_CODE, signed_headers, two_script_bundle +from fastapi.testclient import TestClient + +from prism_challenge.app import create_app +from prism_challenge.auth import canonical_submission_message +from prism_challenge.config import PrismSettings + +_INTERNAL_TOKEN = "secret" +_HOTKEY = "hk-pool-live" + + +def _settings(tmp_path: Path) -> PrismSettings: + return PrismSettings( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'pool-live.sqlite3'}", + shared_token=_INTERNAL_TOKEN, + allow_insecure_signatures=True, + fineweb_sample_count=4, + distributed_contract_policy="off", + ) + + +@pytest.fixture +def client(tmp_path: Path) -> TestClient: + with TestClient(create_app(_settings(tmp_path))) as test_client: + yield test_client + + +def _internal_headers() -> dict[str, str]: + return {"Authorization": f"Bearer {_INTERNAL_TOKEN}"} + + +def _sign_body(body: bytes, *, hotkey: str, nonce: str) -> dict[str, str]: + timestamp = str(int(time.time())) + message = canonical_submission_message( + hotkey=hotkey, nonce=nonce, timestamp=timestamp, body=body + ) + signature = hmac.new(_INTERNAL_TOKEN.encode(), message, sha256).hexdigest() + return { + "X-Hotkey": hotkey, + "X-Signature": signature, + "X-Nonce": nonce, + "X-Timestamp": timestamp, + } + + +def _seed_job( + client: TestClient, + *, + job_id: str, + submission_id: str, + status: str, + created_at: str = "2026-01-01T00:00:00+00:00", +) -> None: + repository = client.app.state.repository + + async def insert() -> None: + async with repository.database.connect() as conn: + await conn.execute( + "INSERT OR IGNORE INTO submissions(" + "id, hotkey, epoch_id, filename, code, code_hash, metadata, status, " + "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + submission_id, + _HOTKEY, + 1, + "project.zip", + "e30=", + f"hash-{submission_id}", + "{}", + "running" if status == "running" else status, + created_at, + created_at, + ), + ) + await conn.execute( + "INSERT INTO eval_jobs(" + "id, submission_id, level, status, attempts, metrics, " + "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + job_id, + submission_id, + "l2", + status, + 0, + "{}", + created_at, + created_at, + ), + ) + + anyio.run(insert) + + +def _open_session(client: TestClient, *, eval_job_id: str, nonce: str) -> str: + payload = { + "eval_job_id": eval_job_id, + "instance_id": "instance-pool-0", + "hotkey_ss58": _HOTKEY, + "nonce": nonce, + "timestamp": str(int(time.time())), + } + raw = json.dumps(payload, separators=(",", ":")).encode() + headers = { + "Content-Type": "application/json", + **_internal_headers(), + **_sign_body(raw, hotkey=_HOTKEY, nonce=nonce), + } + resp = client.post("/v1/execution/telemetry-session", content=raw, headers=headers) + assert resp.status_code == 200, resp.text + return str(resp.json()["session_id"]) + + +def _ingest( + client: TestClient, + *, + session_id: str, + eval_job_id: str, + sequence: int, + message: str, +) -> None: + events = [ + { + "eval_job_id": eval_job_id, + "task_id": "train", + "sequence": sequence, + "event_type": "execution.progress", + "message": message, + "progress": min(1.0, sequence / 10.0), + } + ] + body = json.dumps({"session_id": session_id, "events": events}, separators=(",", ":")).encode() + headers = { + "Content-Type": "application/json", + **_internal_headers(), + "X-Telemetry-Session": session_id, + } + resp = client.post("/v1/execution/events", content=body, headers=headers) + assert resp.status_code == 200, resp.text + assert "score" not in resp.json() + assert "final_score" not in resp.json() + + +def test_s7_pool_live_shows_job(client: TestClient) -> None: + """GET /v1/execution-pool/live returns in-flight job with latest event; terminal excluded.""" + + live_job = "job-pool-live" + done_job = "job-pool-done" + failed_job = "job-pool-failed" + _seed_job(client, job_id=live_job, submission_id="sub-pool-live", status="running") + _seed_job( + client, + job_id=done_job, + submission_id="sub-pool-done", + status="completed", + created_at="2026-01-01T00:01:00+00:00", + ) + _seed_job( + client, + job_id=failed_job, + submission_id="sub-pool-failed", + status="failed", + created_at="2026-01-01T00:02:00+00:00", + ) + + session_id = _open_session(client, eval_job_id=live_job, nonce="pool-n1") + _ingest( + client, + session_id=session_id, + eval_job_id=live_job, + sequence=1, + message="boot", + ) + _ingest( + client, + session_id=session_id, + eval_job_id=live_job, + sequence=2, + message="latest-progress-marker", + ) + + done_session = _open_session(client, eval_job_id=done_job, nonce="pool-n-done") + _ingest( + client, + session_id=done_session, + eval_job_id=done_job, + sequence=1, + message="finished-should-be-hidden", + ) + + resp = client.get("/v1/execution-pool/live") + assert resp.status_code == 200, resp.text + body = resp.json() + + jobs: list[dict[str, Any]] + if isinstance(body, list): + jobs = body + elif isinstance(body, dict) and isinstance(body.get("jobs"), list): + jobs = body["jobs"] + else: + pytest.fail(f"unexpected live pool payload shape: {body!r}") + + job_ids = { + str(item.get("eval_job_id") or item.get("id") or item.get("job_id")) for item in jobs + } + assert live_job in job_ids, f"in-flight job missing from live pool: {jobs!r}" + assert done_job not in job_ids, "completed job must be excluded from live pool" + assert failed_job not in job_ids, "failed job must be excluded from live pool" + + live_row = next( + item + for item in jobs + if str(item.get("eval_job_id") or item.get("id") or item.get("job_id")) == live_job + ) + blob = json.dumps(live_row) + assert "latest-progress-marker" in blob, f"latest event not present: {live_row!r}" + assert "final_score" not in live_row or live_row.get("final_score") is None + for banned in ("mnemonic", "wallet_seed", "private_key"): + assert banned not in blob + + +def test_execution_pool_live_empty_when_no_running_jobs(client: TestClient) -> None: + """Empty pool is honest empty list / empty jobs array — not 404.""" + + _seed_job(client, job_id="job-only-done", submission_id="sub-only-done", status="completed") + resp = client.get("/v1/execution-pool/live") + assert resp.status_code == 200, resp.text + body = resp.json() + if isinstance(body, list): + assert body == [] + else: + assert body.get("jobs") == [] + + +def test_s9_existing_submission_api_unchanged( + client: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """GET /v1/submissions/{id} and curve endpoint behave exactly as before telemetry.""" + + def fake_run(self: Any, spec: Any, timeout_seconds: float) -> DockerRunResult: + del self, timeout_seconds + artifact_dir = next(mount.source for mount in spec.mounts if mount.target == "/artifacts") + manifest = { + "schema_version": "prism_run_manifest.v2", + "metrics": { + "covered_bytes": 4096, + "sum_neg_log_likelihood_nats": 2200.0, + "online_loss": [3.1, 2.9, 2.4], + "predicted_tokens": 800, + "tokens_seen": 800, + "heldout_delta": 0.35, + "val_bpb_trained": 1.10, + "val_bpb_random_init": 1.45, + "param_ladder_stage": "explore", + }, + } + (Path(artifact_dir) / "prism_run_manifest.v2.json").write_text( + json.dumps(manifest), encoding="utf-8" + ) + return DockerRunResult( + container_name="prism-eval", + stdout="", + stderr="", + returncode=0, + ) + + monkeypatch.setattr( + "prism_challenge.evaluator.container.DockerExecutor.run", + fake_run, + ) + + payload = {"code": two_script_bundle(arch_code=VALID_CODE), "filename": "project.zip"} + raw = json.dumps(payload, separators=(",", ":")).encode() + submit = client.post( + "/v1/submissions", + content=raw, + headers={ + **signed_headers(_INTERNAL_TOKEN, raw), + "Content-Type": "application/json", + }, + ) + assert submit.status_code == 200, submit.text + submission_id = submit.json()["id"] + + process = client.post( + "/internal/v1/worker/process-next", + headers=_internal_headers(), + ) + assert process.status_code == 200, process.text + assert process.json()["submission_id"] == submission_id + + status = client.get(f"/v1/submissions/{submission_id}") + assert status.status_code == 200, status.text + status_body = status.json() + assert status_body["id"] == submission_id + assert status_body["status"] == "completed" + assert "final_score" in status_body + assert status_body["final_score"] is None or status_body["final_score"] >= 0 + + curve = client.get(f"/v1/submissions/{submission_id}/curve") + assert curve.status_code in {200, 404}, curve.text + if curve.status_code == 200: + curve_body = curve.json() + assert curve_body["submission_id"] == submission_id + assert "loss_curve" in curve_body + assert "online_loss" in curve_body["loss_curve"] + for banned in ("session_id", "telemetry", "execution_events", "mnemonic"): + assert banned not in curve_body + + health = client.get("/health") + assert health.status_code == 200 + assert health.json()["slug"] == "prism" From bbe542052734c724a29548b54b76904b257eaf22 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:07:17 +0000 Subject: [PATCH 5/7] feat(prism): add execution events session and live pool Bump prism-schema.v5, persist execution_events, open hotkey-bound telemetry sessions, ingest monotone events, and expose running jobs on GET /v1/execution-pool/live without scoring or tier elevation. --- .../prism/src/prism_challenge/db.py | 36 ++- .../prism/src/prism_challenge/repository.py | 143 +++++++++++ .../prism/src/prism_challenge/routes.py | 232 +++++++++++++++++- 3 files changed, 408 insertions(+), 3 deletions(-) diff --git a/packages/challenges/prism/src/prism_challenge/db.py b/packages/challenges/prism/src/prism_challenge/db.py index 062451d54..b6e695d6d 100644 --- a/packages/challenges/prism/src/prism_challenge/db.py +++ b/packages/challenges/prism/src/prism_challenge/db.py @@ -209,6 +209,22 @@ "worker_pubkey TEXT, audited_manifest_sha256 TEXT NOT NULL," "replay_manifest_sha256 TEXT NOT NULL, reason TEXT NOT NULL, created_at TEXT NOT NULL);" "CREATE INDEX IF NOT EXISTS idx_worker_faults_submission ON worker_faults(submission_id);" + "CREATE TABLE IF NOT EXISTS telemetry_sessions (" + "id TEXT PRIMARY KEY, eval_job_id TEXT, work_unit_id TEXT," + "instance_id TEXT NOT NULL, hotkey_ss58 TEXT NOT NULL, nonce TEXT NOT NULL," + "created_at TEXT NOT NULL, expires_at TEXT);" + "CREATE INDEX IF NOT EXISTS idx_telemetry_sessions_job " + "ON telemetry_sessions(eval_job_id);" + "CREATE TABLE IF NOT EXISTS execution_events (" + "id TEXT PRIMARY KEY, eval_job_id TEXT, work_unit_id TEXT," + "task_id TEXT NOT NULL, sequence INTEGER NOT NULL, event_type TEXT NOT NULL," + "payload TEXT NOT NULL, session_id TEXT NOT NULL, hotkey_ss58 TEXT NOT NULL," + "created_at TEXT NOT NULL," + "UNIQUE(eval_job_id, task_id, sequence));" + "CREATE INDEX IF NOT EXISTS idx_execution_events_job " + "ON execution_events(eval_job_id, sequence);" + "CREATE INDEX IF NOT EXISTS idx_execution_events_session " + "ON execution_events(session_id);" ) @@ -225,7 +241,7 @@ # Declared SQLite runtime policy used on every real connection # (VAL-WEIGHT-092 / VAL-GATE-043). -PRISM_SCHEMA_REVISION = "prism-schema.v4" +PRISM_SCHEMA_REVISION = "prism-schema.v5" PRISM_BUSY_TIMEOUT_MS = 5_000 SQLITE_CONNECTION_PRAGMAS: tuple[str, ...] = ( "PRAGMA foreign_keys=ON;", @@ -498,6 +514,24 @@ async def _run_migrations(conn: aiosqlite.Connection) -> None: "official_fixed_profile": "INTEGER NOT NULL DEFAULT 1", }, ) + await conn.executescript( + "CREATE TABLE IF NOT EXISTS telemetry_sessions (" + "id TEXT PRIMARY KEY, eval_job_id TEXT, work_unit_id TEXT," + "instance_id TEXT NOT NULL, hotkey_ss58 TEXT NOT NULL, nonce TEXT NOT NULL," + "created_at TEXT NOT NULL, expires_at TEXT);" + "CREATE INDEX IF NOT EXISTS idx_telemetry_sessions_job " + "ON telemetry_sessions(eval_job_id);" + "CREATE TABLE IF NOT EXISTS execution_events (" + "id TEXT PRIMARY KEY, eval_job_id TEXT, work_unit_id TEXT," + "task_id TEXT NOT NULL, sequence INTEGER NOT NULL, event_type TEXT NOT NULL," + "payload TEXT NOT NULL, session_id TEXT NOT NULL, hotkey_ss58 TEXT NOT NULL," + "created_at TEXT NOT NULL," + "UNIQUE(eval_job_id, task_id, sequence));" + "CREATE INDEX IF NOT EXISTS idx_execution_events_job " + "ON execution_events(eval_job_id, sequence);" + "CREATE INDEX IF NOT EXISTS idx_execution_events_session " + "ON execution_events(session_id);" + ) async def _ensure_columns(conn: aiosqlite.Connection, table: str, columns: dict[str, str]) -> None: diff --git a/packages/challenges/prism/src/prism_challenge/repository.py b/packages/challenges/prism/src/prism_challenge/repository.py index 8fdd6bf47..27498bfa7 100644 --- a/packages/challenges/prism/src/prism_challenge/repository.py +++ b/packages/challenges/prism/src/prism_challenge/repository.py @@ -1527,6 +1527,149 @@ async def _record_llm_review_event( ), ) + async def create_telemetry_session( + self, + *, + eval_job_id: str | None, + work_unit_id: str | None, + instance_id: str, + hotkey_ss58: str, + nonce: str, + ) -> dict[str, str]: + """Open an attested telemetry session bound to hotkey_ss58 (never stores mnemonic).""" + + session_id = str(uuid4()) + created = now_iso() + async with self.database.connect() as conn: + await conn.execute( + "INSERT INTO telemetry_sessions(" + "id, eval_job_id, work_unit_id, instance_id, hotkey_ss58, nonce, " + "created_at, expires_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ( + session_id, + eval_job_id, + work_unit_id, + instance_id, + hotkey_ss58, + nonce, + created, + None, + ), + ) + return { + "session_id": session_id, + "hotkey_ss58": hotkey_ss58, + "instance_id": instance_id, + } + + async def get_telemetry_session(self, session_id: str) -> dict[str, object] | None: + async with self.database.connect() as conn: + rows = await conn.execute_fetchall( + "SELECT id, eval_job_id, work_unit_id, instance_id, hotkey_ss58, nonce, " + "created_at, expires_at FROM telemetry_sessions WHERE id=?", + (session_id,), + ) + return dict(list(rows)[0]) if rows else None + + async def append_execution_event( + self, + *, + session_id: str, + hotkey_ss58: str, + eval_job_id: str | None, + work_unit_id: str | None, + task_id: str, + sequence: int, + event_type: str, + payload: dict[str, Any], + ) -> str: + """Append one execution event with monotone sequence + idempotent (job, task, seq). + + Never writes scores or elevates trust tier. Raises ValueError on sequence regression. + Returns \"inserted\" | \"duplicate\". + """ + + if sequence < 1: + raise ValueError("execution_event_sequence_invalid") + async with self.database.connect() as conn: + existing = await conn.execute_fetchall( + "SELECT id FROM execution_events " + "WHERE eval_job_id IS ? AND task_id=? AND sequence=?", + (eval_job_id, task_id, sequence), + ) + if existing: + return "duplicate" + max_rows = await conn.execute_fetchall( + "SELECT COALESCE(MAX(sequence), 0) AS sequence FROM execution_events " + "WHERE eval_job_id IS ? AND task_id=?", + (eval_job_id, task_id), + ) + max_seq = int(list(max_rows)[0]["sequence"]) + if sequence <= max_seq: + raise ValueError("execution_event_sequence_regress") + event_id = str(uuid4()) + await conn.execute( + "INSERT INTO execution_events(" + "id, eval_job_id, work_unit_id, task_id, sequence, event_type, payload, " + "session_id, hotkey_ss58, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + event_id, + eval_job_id, + work_unit_id, + task_id, + sequence, + event_type, + dumps(payload), + session_id, + hotkey_ss58, + now_iso(), + ), + ) + return "inserted" + + async def list_live_execution_pool(self) -> list[dict[str, Any]]: + """RUNNING eval jobs with each job's latest execution event (observability only).""" + + async with self.database.connect() as conn: + job_rows = await conn.execute_fetchall( + "SELECT id, submission_id, level, status, created_at, updated_at " + "FROM eval_jobs WHERE status='running' ORDER BY updated_at DESC, id" + ) + jobs: list[dict[str, Any]] = [] + for row in job_rows: + job = dict(row) + job_id = str(job["id"]) + event_rows = await conn.execute_fetchall( + "SELECT id, eval_job_id, work_unit_id, task_id, sequence, event_type, " + "payload, session_id, hotkey_ss58, created_at FROM execution_events " + "WHERE eval_job_id=? ORDER BY sequence DESC LIMIT 1", + (job_id,), + ) + latest: dict[str, Any] | None = None + if event_rows: + event = dict(list(event_rows)[0]) + payload = loads(str(event.get("payload") or "{}")) + if not isinstance(payload, dict): + payload = {} + latest = { + "event_type": str(event["event_type"]), + "sequence": int(cast(SupportsInt, event["sequence"])), + "task_id": str(event["task_id"]), + "message": payload.get("message"), + "progress": payload.get("progress"), + "created_at": str(event["created_at"]), + } + jobs.append( + { + "eval_job_id": job_id, + "submission_id": str(job["submission_id"]), + "status": str(job["status"]), + "level": str(job["level"]), + "latest_event": latest, + } + ) + return jobs + def _validate_evidence(items: Any) -> list[dict[str, Any]]: if not items: diff --git a/packages/challenges/prism/src/prism_challenge/routes.py b/packages/challenges/prism/src/prism_challenge/routes.py index 0a40ee651..53ba4a73c 100644 --- a/packages/challenges/prism/src/prism_challenge/routes.py +++ b/packages/challenges/prism/src/prism_challenge/routes.py @@ -2,12 +2,13 @@ import logging from datetime import UTC, datetime -from typing import Any, SupportsFloat, SupportsInt, cast +from typing import Annotated, Any, SupportsFloat, SupportsInt, cast from base.challenge_sdk.roles import public_route from fastapi import ( APIRouter, Depends, + Header, HTTPException, Query, Request, @@ -17,7 +18,13 @@ from .admission import enforce_admission from .attestation_routes import build_attestation_public_router -from .auth import authenticate_miner +from .auth import ( + authenticate_internal, + authenticate_miner, + canonical_submission_message, + verify_dev_signature, + verify_hotkey_signature, +) from .evaluator.train_series import downsample_train_series_for_api from .models import ( ArchitectureDetailResponse, @@ -45,6 +52,20 @@ CURVE_MAX_POINTS = 500 +# First-class score fields must never be accepted as event attributes (trust boundary). +_SCORE_FIELD_NAMES = frozenset( + { + "score", + "final_score", + "q_arch", + "q_recipe", + "anti_cheat_multiplier", + "diversity_bonus", + "penalty", + "effective_tier", + } +) + router = APIRouter(prefix="/v1") # Public attestation challenge/answer (published via BASE proxy as @@ -383,3 +404,210 @@ def _opt_int(value: Any) -> int | None: if isinstance(value, float): return int(value) return None + + +def _verify_telemetry_hotkey_signature( + request: Request, + *, + hotkey: str, + nonce: str, + timestamp: str, + signature: str, + body: bytes, +) -> None: + app_settings = request.app.state.settings + try: + ts = int(timestamp) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, "invalid timestamp") from exc + if abs(int(datetime.now(UTC).timestamp()) - ts) > app_settings.signature_ttl_seconds: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "stale signature") + message = canonical_submission_message( + hotkey=hotkey, nonce=nonce, timestamp=timestamp, body=body + ) + valid = verify_hotkey_signature(hotkey, message, signature) + if not valid and app_settings.allow_insecure_signatures: + valid = verify_dev_signature(app_settings.internal_token(), message, signature) + if not valid: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "invalid signature") + + +@public_route(tags=["execution"]) +@router.post("/execution/telemetry-session") +async def open_telemetry_session( + request: Request, + _: None = Depends(authenticate_internal), + repository: PrismRepository = Depends(repo_from_request), + x_hotkey: Annotated[str | None, Header()] = None, + x_signature: Annotated[str | None, Header()] = None, + x_nonce: Annotated[str | None, Header()] = None, + x_timestamp: Annotated[str | None, Header()] = None, +) -> dict[str, str]: + """Open a hotkey-signed telemetry session. Mnemonic is never accepted or returned.""" + + body = await request.body() + try: + import json + + payload = json.loads(body.decode("utf-8") if body else "{}") + except (UnicodeDecodeError, ValueError) as exc: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "invalid json") from exc + if not isinstance(payload, dict): + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "body must be object") + + # Reject secret material if present as first-class fields (optional hard reject). + for banned in ("mnemonic", "wallet_seed", "private_key", "seed"): + if banned in payload: + # Strip path: ignore banned keys rather than echo; still open session. + payload = { + k: v + for k, v in payload.items() + if k not in {"mnemonic", "wallet_seed", "private_key", "seed"} + } + break + + hotkey = str(payload.get("hotkey_ss58") or x_hotkey or "").strip() + nonce = str(payload.get("nonce") or x_nonce or "").strip() + timestamp = str(x_timestamp or payload.get("timestamp") or "").strip() + signature = str(x_signature or "").strip() + if not hotkey or not nonce or not timestamp or not signature: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "hotkey signature required") + if x_hotkey is not None and x_hotkey != hotkey: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "hotkey mismatch") + _verify_telemetry_hotkey_signature( + request, + hotkey=hotkey, + nonce=nonce, + timestamp=timestamp, + signature=signature, + body=body, + ) + + eval_job_id = payload.get("eval_job_id") + work_unit_id = payload.get("work_unit_id") + if eval_job_id is not None: + eval_job_id = str(eval_job_id) + if work_unit_id is not None: + work_unit_id = str(work_unit_id) + if not eval_job_id and not work_unit_id: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "eval_job_id or work_unit_id required", + ) + instance_id = str(payload.get("instance_id") or "").strip() + if not instance_id: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "instance_id required") + + session = await repository.create_telemetry_session( + eval_job_id=eval_job_id, + work_unit_id=work_unit_id, + instance_id=instance_id, + hotkey_ss58=hotkey, + nonce=nonce, + ) + return session + + +@public_route(tags=["execution"]) +@router.post("/execution/events") +async def ingest_execution_events( + request: Request, + _: None = Depends(authenticate_internal), + repository: PrismRepository = Depends(repo_from_request), + x_telemetry_session: Annotated[str | None, Header()] = None, +) -> dict[str, Any]: + """Ingest session-gated execution events. Never scores; never elevates tier.""" + + body = await request.body() + try: + import json + + payload = json.loads(body.decode("utf-8") if body else "{}") + except (UnicodeDecodeError, ValueError) as exc: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "invalid json") from exc + if not isinstance(payload, dict): + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "body must be object") + + session_id = str(x_telemetry_session or payload.get("session_id") or "").strip() or None + if not session_id: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "telemetry session required") + session = await repository.get_telemetry_session(session_id) + if session is None: + raise HTTPException(status.HTTP_401_UNAUTHORIZED, "invalid telemetry session") + + events = payload.get("events") + if not isinstance(events, list) or not events: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "events required") + + inserted = 0 + duplicates = 0 + for raw_event in events: + if not isinstance(raw_event, dict): + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "event must be object") + # Reject first-class score fields (trust boundary). + if _SCORE_FIELD_NAMES.intersection(raw_event): + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "score fields are not allowed on execution events", + ) + try: + sequence = int(raw_event["sequence"]) + except (KeyError, TypeError, ValueError) as exc: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "sequence required") from exc + task_id = str(raw_event.get("task_id") or "").strip() + event_type = str(raw_event.get("event_type") or "").strip() + if not task_id or not event_type: + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, "task_id and event_type required" + ) + eval_job_id = raw_event.get("eval_job_id") + work_unit_id = raw_event.get("work_unit_id") + if eval_job_id is not None: + eval_job_id = str(eval_job_id) + else: + eval_job_id = ( + str(session["eval_job_id"]) if session.get("eval_job_id") is not None else None + ) + if work_unit_id is not None: + work_unit_id = str(work_unit_id) + else: + work_unit_id = ( + str(session["work_unit_id"]) if session.get("work_unit_id") is not None else None + ) + event_payload: dict[str, Any] = {} + if "message" in raw_event: + event_payload["message"] = raw_event["message"] + if "progress" in raw_event: + event_payload["progress"] = raw_event["progress"] + # metadata is observability-only; never promoted to score columns + if isinstance(raw_event.get("metadata"), dict): + event_payload["metadata"] = raw_event["metadata"] + try: + result = await repository.append_execution_event( + session_id=session_id, + hotkey_ss58=str(session["hotkey_ss58"]), + eval_job_id=eval_job_id, + work_unit_id=work_unit_id, + task_id=task_id, + sequence=sequence, + event_type=event_type, + payload=event_payload, + ) + except ValueError as exc: + raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, str(exc)) from exc + if result == "inserted": + inserted += 1 + else: + duplicates += 1 + return {"accepted": inserted + duplicates, "inserted": inserted, "duplicates": duplicates} + + +@public_route(tags=["execution"]) +@router.get("/execution-pool/live") +async def execution_pool_live( + repository: PrismRepository = Depends(repo_from_request), +) -> dict[str, list[dict[str, Any]]]: + """In-flight RUNNING jobs with latest event. Empty pool is honest empty list.""" + + jobs = await repository.list_live_execution_pool() + return {"jobs": jobs} From 616e45c9c4f7805712e98b96f341e32bf54ac5f8 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:07:17 +0000 Subject: [PATCH 6/7] test(master): add contracts for eval telemetry proxy and pools Require allowlisted progress/session/result and public pool reads, plus GET /v1/pools/executing fan-out with partial-failure and jobs-shape pins. --- .../test_agent_challenge_attested_proxy.py | 405 +++++++++++++- tests/unit/test_pools_executing.py | 498 ++++++++++++++++++ 2 files changed, 902 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_pools_executing.py diff --git a/tests/unit/test_agent_challenge_attested_proxy.py b/tests/unit/test_agent_challenge_attested_proxy.py index 5e6ac777f..55193a265 100644 --- a/tests/unit/test_agent_challenge_attested_proxy.py +++ b/tests/unit/test_agent_challenge_attested_proxy.py @@ -505,7 +505,6 @@ async def handler(request: httpx.Request) -> httpx.Response: ("GET", "/internal/v1/reviews/session-1/report"), ("GET", "/internal/v1/reviews/session-1/evidence/object-1"), ("POST", "/internal/v1/reviews/session-1/approvals"), - ("POST", "/evaluation/v1/runs/run-1/result"), ("GET", "/key-release/nonce"), ("POST", "/key-release/release"), ("GET", "/keyrelease/nonce"), @@ -996,3 +995,407 @@ async def handler(request: httpx.Request) -> httpx.Response: assert "x-allowlist-digest" not in headers assert "x-measurement-mrtd" not in headers assert "x-review-verified" not in headers + + +# --------------------------------------------------------------------------- +# Realtime eval telemetry capability + public pool/SSE reads (attested allowlist) +# --------------------------------------------------------------------------- + + +_EVAL_CAPABILITY_POST_ROUTES = ( + ( + "POST", + "/challenges/agent-challenge/evaluation/v1/runs/run-1/progress", + "/evaluation/v1/runs/run-1/progress", + ), + ( + "POST", + "/challenges/agent-challenge/evaluation/v1/runs/run-1/telemetry-session", + "/evaluation/v1/runs/run-1/telemetry-session", + ), + ( + "POST", + "/challenges/agent-challenge/evaluation/v1/runs/run-1/result", + "/evaluation/v1/runs/run-1/result", + ), +) + +_PUBLIC_POOL_SSE_GET_ROUTES = ( + ( + "GET", + "/challenges/agent-challenge/submissions/sub-1/task-events", + "/submissions/sub-1/task-events", + ), + ( + "GET", + "/challenges/agent-challenge/submissions/sub-1/task-events/stream", + "/submissions/sub-1/task-events/stream", + ), + ( + "GET", + "/challenges/agent-challenge/v1/execution-pool/live", + "/v1/execution-pool/live", + ), +) + + +@pytest.mark.parametrize( + ("method", "path", "upstream_path"), + _EVAL_CAPABILITY_POST_ROUTES, +) +def test_eval_capability_routes_allowed_when_attested_flag_on( + method: str, + path: str, + upstream_path: str, +) -> None: + """Eval-run Bearer capability POSTs must be allowlisted under attested mode.""" + + captured: dict[str, Any] = {} + + async def handler(request: httpx.Request) -> httpx.Response: + captured["method"] = request.method + captured["path"] = request.url.path + captured["headers"] = request.headers + return httpx.Response(200, json={"ok": True}) + + client = _proxy_client(handler, attested_routes_enabled=True) + response = client.request( + method, + path, + content=b'{"schema_version":1,"marker":true}', + headers={ + "Authorization": "Bearer eval_run_token.deadbeef", + "Content-Type": "application/json", + "X-Telemetry-Session": "sess-1", + "X-Public-Header": "preserved", + }, + ) + + assert response.status_code == 200 + assert captured["method"] == method + assert captured["path"] == upstream_path + + +@pytest.mark.parametrize( + ("method", "path", "upstream_path"), + _EVAL_CAPABILITY_POST_ROUTES, +) +def test_eval_capability_routes_preserve_authorization_and_strip_trust( + method: str, + path: str, + upstream_path: str, +) -> None: + """Mirror review-capability: forward Authorization; strip client trust headers.""" + + captured: dict[str, Any] = {} + + async def handler(request: httpx.Request) -> httpx.Response: + captured["method"] = request.method + captured["path"] = request.url.path + captured["headers"] = request.headers + return httpx.Response(200, json={"ok": True}) + + client = _proxy_client(handler, attested_routes_enabled=True) + response = client.request( + method, + path, + content=b'{"schema_version":1}', + headers={ + "Authorization": "Bearer eval_run_token.deadbeef", + "Proxy-Authorization": "Basic should-still-strip", + "X-Admin-Token": "should-strip", + "X-Base-Admin-Token": "should-strip", + "X-Base-Internal-Token": "should-strip", + "X-Internal-Authorization": "should-strip", + "X-Base-Verified-Hotkey": "should-strip", + "X-Base-Verified-Future": "should-strip", + "X-Base-Request-Hash": "should-strip", + "X-Trust-Level": "should-strip", + "X-Trusted-Proxy": "should-strip", + "X-Base-Trust-Result": "should-strip", + "X-RA-TLS-Peer-Key": "should-strip", + "X-RATLS-Peer-Certificate": "should-strip", + "X-Review-Verified": "true", + "X-Attestation-Verified": "true", + "X-Allowlist-Digest": "should-strip", + "X-Measurement-MRTD": "should-strip", + "Forwarded": "for=caller", + "X-Forwarded-For": "198.51.100.7", + "X-Real-IP": "198.51.100.8", + "X-Public-Header": "preserved", + "X-Telemetry-Session": "sess-capability-1", + }, + ) + + assert response.status_code == 200 + assert captured["method"] == method + assert captured["path"] == upstream_path + headers: httpx.Headers = captured["headers"] + assert headers["authorization"] == "Bearer eval_run_token.deadbeef" + assert headers["x-public-header"] == "preserved" + assert headers.get("x-telemetry-session") == "sess-capability-1" + assert "proxy-authorization" not in headers + assert "x-admin-token" not in headers + assert "x-base-admin-token" not in headers + assert "x-base-internal-token" not in headers + assert "x-internal-authorization" not in headers + assert "x-base-verified-hotkey" not in headers + assert "x-base-verified-future" not in headers + assert "x-base-request-hash" not in headers + assert "x-trust-level" not in headers + assert "x-trusted-proxy" not in headers + assert "x-base-trust-result" not in headers + assert "x-ra-tls-peer-key" not in headers + assert "x-ratls-peer-certificate" not in headers + assert "x-review-verified" not in headers + assert "x-attestation-verified" not in headers + assert "x-allowlist-digest" not in headers + assert "x-measurement-mrtd" not in headers + assert "forwarded" not in headers + assert "x-forwarded-for" not in headers + assert "x-real-ip" not in headers + + +@pytest.mark.parametrize( + ("method", "path", "upstream_path"), + _PUBLIC_POOL_SSE_GET_ROUTES, +) +def test_public_pool_and_task_event_reads_allowed_when_attested_flag_on( + method: str, + path: str, + upstream_path: str, +) -> None: + """Public pool live + task-events (incl. SSE stream) must be allowlisted.""" + + captured: dict[str, Any] = {} + + async def handler(request: httpx.Request) -> httpx.Response: + captured["method"] = request.method + captured["path"] = request.url.path + captured["headers"] = request.headers + return httpx.Response( + 200, + content=b'{"schema_version":1,"units":[]}', + headers={"content-type": "application/json"}, + ) + + client = _proxy_client(handler, attested_routes_enabled=True) + response = client.request( + method, + path, + headers={ + "Authorization": "Bearer should-not-forward-on-public-read", + "X-Allowlist-Digest": "caller-allowlist", + "X-Base-Verified-Hotkey": "forged", + "X-Attestation-Verified": "true", + "X-Public-Header": "preserved", + }, + ) + + assert response.status_code == 200 + assert captured["method"] == method + assert captured["path"] == upstream_path + headers: httpx.Headers = captured["headers"] + assert headers["x-public-header"] == "preserved" + assert headers.get_list("x-base-proxy") == ["true"] + assert headers.get_list("x-base-challenge-slug") == ["agent-challenge"] + # Public reads are not capability routes — Authorization stays stripped. + assert "authorization" not in headers + assert "x-allowlist-digest" not in headers + assert "x-base-verified-hotkey" not in headers + assert "x-attestation-verified" not in headers + + +@pytest.mark.parametrize( + ("method", "path"), + ( + ("GET", "/challenges/agent-challenge/internal/v1/reviews/session-1/report"), + ("POST", "/challenges/agent-challenge/internal/v1/eval/runs/run-1/progress"), + ("GET", "/challenges/agent-challenge/internal/v1/execution-pool/live"), + ("POST", "/challenges/agent-challenge/internal/v1/anything"), + ), +) +def test_attested_mode_denies_internal_v1_paths_not_proxied( + method: str, + path: str, +) -> None: + """Any /internal/v1/... under agent-challenge stays fail-closed (not proxied).""" + + upstream_calls: list[str] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + upstream_calls.append(request.url.path) + return httpx.Response(200, json={"unexpected": True}) + + client = _proxy_client(handler, attested_routes_enabled=True) + response = client.request( + method, + path, + content=b'{"forged":true}' if method == "POST" else None, + headers={ + "Authorization": "Bearer caller-capability", + "X-Base-Internal-Token": "forged-internal", + "X-Attestation-Verified": "true", + "X-Base-Verified-Hotkey": "forged", + }, + ) + + assert response.status_code == 404 + assert upstream_calls == [] + + +def test_attested_mode_denies_arbitrary_non_allowlisted_ac_path() -> None: + """Arbitrary non-allowlisted AC path is local 404 (fail-closed).""" + + upstream_calls: list[str] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + upstream_calls.append(request.url.path) + return httpx.Response(200, json={"unexpected": True}) + + client = _proxy_client(handler, attested_routes_enabled=True) + response = client.get( + "/challenges/agent-challenge/telemetry/v1/debug/dump", + headers={"Authorization": "Bearer caller-capability"}, + ) + + assert response.status_code == 404 + assert upstream_calls == [] + + +def test_eval_capability_allowlist_helper_accepts_exact_shapes() -> None: + """Direct allowlist helper pins exact evaluation/v1/runs/{id}/{action} shapes.""" + + for action in ("progress", "telemetry-session", "result"): + path = f"evaluation/v1/runs/run-42/{action}" + assert ( + _is_agent_challenge_enabled_mode_allowed_route( + "agent-challenge", + "POST", + path, + ) + is True + ) + + +def test_public_pool_sse_allowlist_helper_accepts_exact_shapes() -> None: + """Direct allowlist helper pins task-events + execution-pool/live GETs.""" + + assert ( + _is_agent_challenge_enabled_mode_allowed_route( + "agent-challenge", + "GET", + "submissions/sub-9/task-events", + ) + is True + ) + assert ( + _is_agent_challenge_enabled_mode_allowed_route( + "agent-challenge", + "GET", + "submissions/sub-9/task-events/stream", + ) + is True + ) + assert ( + _is_agent_challenge_enabled_mode_allowed_route( + "agent-challenge", + "GET", + "v1/execution-pool/live", + ) + is True + ) + + +def test_review_capability_still_preserves_authorization_adjacent_regression() -> None: + """Adjacent: measured-review guest Bearer semantics unchanged by eval telemetry.""" + + captured: dict[str, Any] = {} + + async def handler(request: httpx.Request) -> httpx.Response: + captured["method"] = request.method + captured["path"] = request.url.path + captured["headers"] = request.headers + return httpx.Response(200, json={"ok": True}) + + client = _proxy_client(handler, attested_routes_enabled=True) + response = client.get( + "/challenges/agent-challenge/review/v1/assignments/assignment-1/artifact", + headers={ + "Authorization": "Bearer ra_assignment-1.deadbeef", + "Proxy-Authorization": "Basic should-still-strip", + "X-Admin-Token": "should-strip", + "X-Base-Verified-Hotkey": "should-strip", + "X-Public-Header": "preserved", + }, + ) + + assert response.status_code == 200 + assert captured["method"] == "GET" + assert captured["path"] == "/review/v1/assignments/assignment-1/artifact" + headers: httpx.Headers = captured["headers"] + assert headers["authorization"] == "Bearer ra_assignment-1.deadbeef" + assert headers["x-public-header"] == "preserved" + assert "proxy-authorization" not in headers + assert "x-admin-token" not in headers + assert "x-base-verified-hotkey" not in headers + + +def test_validator_assignment_progress_not_confused_with_eval_telemetry() -> None: + """Adjacent: master POST /v1/assignments/{id}/progress is not eval telemetry. + + Validator lease heartbeat (assignment_coordination) must remain a distinct + master route — never treated as an agent-challenge evaluation capability + path and never allowlisted as challenge-local telemetry. + """ + + # Challenge-local lookalike must stay denied under attested mode. + assert ( + _is_agent_challenge_enabled_mode_allowed_route( + "agent-challenge", + "POST", + "v1/assignments/asg-1/progress", + ) + is False + ) + assert ( + _is_agent_challenge_enabled_mode_allowed_route( + "agent-challenge", + "POST", + "assignments/asg-1/progress", + ) + is False + ) + + upstream_calls: list[str] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + upstream_calls.append(f"{request.method} {request.url.path}") + return httpx.Response(200, json={"unexpected": True}) + + client = _proxy_client(handler, attested_routes_enabled=True) + + # Challenge-prefixed lookalike is fail-closed (not proxied as eval telemetry). + challenge_lookalike = client.post( + "/challenges/agent-challenge/v1/assignments/asg-1/progress", + content=b'{"checkpoint_ref":"x"}', + headers={"Authorization": "Bearer eval_run_token.deadbeef"}, + ) + assert challenge_lookalike.status_code == 404 + assert upstream_calls == [] + + # Master validator lease path is NOT the challenge proxy surface. Without + # assignment_coordination_service the route is absent (404) — never forwarded + # upstream as agent-challenge evaluation progress. + master_progress = client.post( + "/v1/assignments/asg-1/progress", + content=b'{"checkpoint_ref":"x","meta":{}}', + headers={ + "Authorization": "Bearer should-not-become-eval-capability", + "Content-Type": "application/json", + }, + ) + assert master_progress.status_code == 404 + assert upstream_calls == [] + # Must not be rewritten into evaluation/v1/runs/... either. + assert b"evaluation" not in master_progress.content.lower() diff --git a/tests/unit/test_pools_executing.py b/tests/unit/test_pools_executing.py new file mode 100644 index 000000000..15e65bbfb --- /dev/null +++ b/tests/unit/test_pools_executing.py @@ -0,0 +1,498 @@ +"""RED contract: master aggregator GET /v1/pools/executing. + +Fans out to embedded challenges' GET /v1/execution-pool/live: + - Prism → http://127.0.0.1:18080 + - Agent Challenge → http://127.0.0.1:18081 + +(see deploy/compose/docker-compose.yml embed topology). +""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from decimal import Decimal +from typing import Any + +import httpx +import respx +from fastapi.testclient import TestClient + +from base.master.app_proxy import create_proxy_app +from base.master.registry import ChallengeRegistry +from base.schemas.challenge import ChallengeCreate, ChallengeStatus +from base.security.miner_auth import NonceReplayError + +PRISM_LIVE = "http://127.0.0.1:18080/v1/execution-pool/live" +AC_LIVE = "http://127.0.0.1:18081/v1/execution-pool/live" + +_PINNED = "a" * 64 + + +class _NonceStore: + def __init__(self) -> None: + self.keys: set[tuple[int, str, str, str]] = set() + + async def reserve(self, **kwargs: Any) -> None: + key = ( + int(kwargs["netuid"]), + str(kwargs["challenge_slug"]), + str(kwargs["hotkey"]), + str(kwargs["nonce"]), + ) + if key in self.keys: + raise NonceReplayError("nonce already used") + self.keys.add(key) + + +class _Cache: + def get(self) -> dict[str, int]: + return {} + + +def _embed_registry() -> ChallengeRegistry: + """Two active challenges on the compose embed loopback ports.""" + + registry = ChallengeRegistry() + registry.create( + ChallengeCreate( + slug="prism", + name="PRISM", + image=(f"ghcr.io/baseintelligence/prism:1.0.0@sha256:{_PINNED}"), + version="1.0.0", + emission_percent=Decimal("30"), + status=ChallengeStatus.ACTIVE, + internal_base_url="http://127.0.0.1:18080", + ) + ) + registry.create( + ChallengeCreate( + slug="agent-challenge", + name="Agent Challenge", + image=(f"ghcr.io/baseintelligence/agent-challenge:latest@sha256:{_PINNED}"), + version="1.0.0", + emission_percent=Decimal("70"), + status=ChallengeStatus.ACTIVE, + internal_base_url="http://127.0.0.1:18081", + ) + ) + return registry + + +def _proxy_client( + *, + registry: ChallengeRegistry | None = None, +) -> TestClient: + @asynccontextmanager + async def client_factory(): + # Default factory is unused for the aggregator fan-out (direct loopback + # httpx calls to each challenge internal_base_url). Keep a no-op client + # so create_proxy_app still constructs. + async with httpx.AsyncClient( + transport=httpx.MockTransport( + lambda _request: httpx.Response(404, json={"detail": "unused"}) + ), + base_url="http://unused.invalid", + ) as client: + yield client + + return TestClient( + create_proxy_app( + registry=registry or _embed_registry(), + nonce_store=_NonceStore(), + metagraph_cache=_Cache(), # type: ignore[arg-type] + client_factory=client_factory, + ) + ) + + +def _assert_no_score_fields(payload: Any, *, path: str = "$") -> None: + """Pool payload must never expose score / weight / emission fields.""" + + forbidden_keys = { + "score", + "scores", + "weight", + "weights", + "emission", + "emission_percent", + "raw_score", + "final_score", + "normalized_score", + "incentive", + } + if isinstance(payload, dict): + lowered = {str(k).lower() for k in payload} + leaked = forbidden_keys & lowered + assert not leaked, f"score-like keys at {path}: {sorted(leaked)}" + for key, value in payload.items(): + _assert_no_score_fields(value, path=f"{path}.{key}") + elif isinstance(payload, list): + for index, item in enumerate(payload): + _assert_no_score_fields(item, path=f"{path}[{index}]") + + +@respx.mock +def test_pools_executing_route_exists_on_proxy_app() -> None: + """GET /v1/pools/executing is a first-class master route (not challenge proxy).""" + + respx.get(PRISM_LIVE).mock( + return_value=httpx.Response( + 200, + json={"units": [{"unit_id": "prism-u1", "status": "executing"}]}, + ) + ) + respx.get(AC_LIVE).mock( + return_value=httpx.Response( + 200, + json={"units": [{"unit_id": "ac-u1", "status": "executing"}]}, + ) + ) + + client = _proxy_client() + response = client.get("/v1/pools/executing") + + assert response.status_code == 200 + assert response.headers.get("content-type", "").startswith("application/json") + + +@respx.mock +def test_pools_executing_fans_out_to_both_embed_challenges() -> None: + """Aggregator hits Prism :18080 and Agent Challenge :18081 live pool endpoints.""" + + prism_route = respx.get(PRISM_LIVE).mock( + return_value=httpx.Response( + 200, + json={ + "units": [ + { + "unit_id": "prism-unit-1", + "status": "executing", + "hotkey": "5PrismHotkey", + } + ] + }, + ) + ) + ac_route = respx.get(AC_LIVE).mock( + return_value=httpx.Response( + 200, + json={ + "units": [ + { + "unit_id": "ac-unit-1", + "status": "executing", + "hotkey": "5AcHotkey", + } + ] + }, + ) + ) + + client = _proxy_client() + response = client.get("/v1/pools/executing") + + assert response.status_code == 200 + assert prism_route.called + assert ac_route.called + assert prism_route.call_count == 1 + assert ac_route.call_count == 1 + + +@respx.mock +def test_pools_executing_response_shape_keyed_by_challenge_slug() -> None: + """Response is per-challenge entries keyed by slug, each listing executing units.""" + + respx.get(PRISM_LIVE).mock( + return_value=httpx.Response( + 200, + json={ + "units": [ + { + "unit_id": "prism-unit-1", + "status": "executing", + "started_at": "2026-01-01T00:00:00Z", + } + ] + }, + ) + ) + respx.get(AC_LIVE).mock( + return_value=httpx.Response( + 200, + json={ + "units": [ + { + "unit_id": "ac-unit-1", + "status": "executing", + "started_at": "2026-01-01T00:00:01Z", + }, + { + "unit_id": "ac-unit-2", + "status": "executing", + "started_at": "2026-01-01T00:00:02Z", + }, + ] + }, + ) + ) + + client = _proxy_client() + response = client.get("/v1/pools/executing") + + assert response.status_code == 200 + body = response.json() + + # Top-level may be the challenges map itself or wrap under a known key. + challenges = body.get("challenges", body) + assert isinstance(challenges, dict) + assert "prism" in challenges + assert "agent-challenge" in challenges + + prism_entry = challenges["prism"] + ac_entry = challenges["agent-challenge"] + assert isinstance(prism_entry, dict) + assert isinstance(ac_entry, dict) + + # Healthy entries expose currently-executing units (no fabricated placeholders). + prism_units = prism_entry.get("units", prism_entry.get("executing")) + ac_units = ac_entry.get("units", ac_entry.get("executing")) + assert isinstance(prism_units, list) + assert isinstance(ac_units, list) + assert len(prism_units) == 1 + assert len(ac_units) == 2 + assert prism_units[0]["unit_id"] == "prism-unit-1" + assert {u["unit_id"] for u in ac_units} == {"ac-unit-1", "ac-unit-2"} + + # Healthy entries must not carry an error object. + assert "error" not in prism_entry + assert "error" not in ac_entry + + _assert_no_score_fields(body) + + +@respx.mock +def test_pools_executing_partial_failure_returns_200_with_error_object() -> None: + """One challenge down: 200 + real data for healthy + explicit error for failed. + + Must NEVER fabricate/placeholder units for the failed challenge. + """ + + respx.get(PRISM_LIVE).mock( + return_value=httpx.Response( + 200, + json={ + "units": [ + { + "unit_id": "prism-healthy-1", + "status": "executing", + } + ] + }, + ) + ) + # Agent Challenge unreachable / timed out. + respx.get(AC_LIVE).mock(side_effect=httpx.ConnectError("connection refused")) + + client = _proxy_client() + response = client.get("/v1/pools/executing") + + assert response.status_code == 200 + body = response.json() + challenges = body.get("challenges", body) + + prism_entry = challenges["prism"] + ac_entry = challenges["agent-challenge"] + + prism_units = prism_entry.get("units", prism_entry.get("executing")) + assert isinstance(prism_units, list) + assert len(prism_units) == 1 + assert prism_units[0]["unit_id"] == "prism-healthy-1" + assert "error" not in prism_entry + + # Failed challenge: explicit error object, no fabricated units. + assert "error" in ac_entry + error_obj = ac_entry["error"] + assert error_obj is not None + assert isinstance(error_obj, dict) + # Error must be descriptive (code and/or message). + assert error_obj.get("code") or error_obj.get("message") or error_obj.get("detail") + + failed_units = ac_entry.get("units", ac_entry.get("executing", [])) + if failed_units is None: + failed_units = [] + assert failed_units == [], ( + "partial failure must not fabricate placeholder units for the down challenge" + ) + + _assert_no_score_fields(body) + + +@respx.mock +def test_pools_executing_partial_failure_when_prism_down() -> None: + """Symmetric partial failure: Prism down, AC healthy.""" + + respx.get(PRISM_LIVE).mock(side_effect=httpx.ReadTimeout("timed out")) + respx.get(AC_LIVE).mock( + return_value=httpx.Response( + 200, + json={"units": [{"unit_id": "ac-only-1", "status": "executing"}]}, + ) + ) + + client = _proxy_client() + response = client.get("/v1/pools/executing") + + assert response.status_code == 200 + body = response.json() + challenges = body.get("challenges", body) + + prism_entry = challenges["prism"] + ac_entry = challenges["agent-challenge"] + + assert "error" in prism_entry + assert isinstance(prism_entry["error"], dict) + prism_units = prism_entry.get("units", prism_entry.get("executing", [])) or [] + assert prism_units == [] + + ac_units = ac_entry.get("units", ac_entry.get("executing")) + assert isinstance(ac_units, list) + assert ac_units[0]["unit_id"] == "ac-only-1" + assert "error" not in ac_entry + + _assert_no_score_fields(body) + + +@respx.mock +def test_pools_executing_payload_never_includes_score_fields() -> None: + """No score/weight/emission fields anywhere in the pool payload.""" + + respx.get(PRISM_LIVE).mock( + return_value=httpx.Response( + 200, + json={ + "units": [ + { + "unit_id": "p1", + "status": "executing", + "hotkey": "hk1", + } + ] + }, + ) + ) + respx.get(AC_LIVE).mock( + return_value=httpx.Response( + 200, + json={ + "units": [ + { + "unit_id": "a1", + "status": "executing", + "hotkey": "hk2", + } + ] + }, + ) + ) + + client = _proxy_client() + response = client.get("/v1/pools/executing") + + assert response.status_code == 200 + body = response.json() + _assert_no_score_fields(body) + + # Explicit top-level absence (defense in depth beyond recursive walk). + blob = response.text.lower() + for token in ( + '"score"', + '"scores"', + '"weight"', + '"weights"', + '"emission"', + '"emission_percent"', + '"raw_score"', + '"final_score"', + '"normalized_score"', + '"incentive"', + ): + assert token not in blob, f"forbidden score token present: {token}" + + +@respx.mock +def test_pools_executing_does_not_require_live_network() -> None: + """Contract is fully mockable — no real sockets to 18080/18081.""" + + respx.get(PRISM_LIVE).mock(return_value=httpx.Response(200, json={"units": []})) + respx.get(AC_LIVE).mock(return_value=httpx.Response(200, json={"units": []})) + + client = _proxy_client() + response = client.get("/v1/pools/executing") + + # Route must exist and succeed against mocks only. + assert response.status_code == 200 + body = response.json() + challenges = body.get("challenges", body) + assert set(challenges) >= {"prism", "agent-challenge"} + + +@respx.mock +def test_pools_executing_accepts_prism_jobs_shape() -> None: + """Prism live pool returns {"jobs": [...]} — must not be dropped as empty units. + + Agent Challenge uses {"units": [...]}. Aggregator normalizes both under units. + """ + + respx.get(PRISM_LIVE).mock( + return_value=httpx.Response( + 200, + json={ + "jobs": [ + { + "eval_job_id": "j1", + "status": "running", + "latest_event": {"type": "progress"}, + } + ] + }, + ) + ) + respx.get(AC_LIVE).mock( + return_value=httpx.Response( + 200, + json={ + "units": [ + { + "unit_id": "ac-u1", + "status": "executing", + } + ] + }, + ) + ) + + client = _proxy_client() + response = client.get("/v1/pools/executing") + + assert response.status_code == 200 + body = response.json() + challenges = body.get("challenges", body) + + prism_entry = challenges["prism"] + ac_entry = challenges["agent-challenge"] + assert "error" not in prism_entry + assert "error" not in ac_entry + + prism_units = prism_entry.get("units", prism_entry.get("executing")) + ac_units = ac_entry.get("units", ac_entry.get("executing")) + assert isinstance(prism_units, list) + assert isinstance(ac_units, list) + assert len(prism_units) == 1, ( + "Prism jobs[] must be normalized into units (not silently dropped)" + ) + assert prism_units[0]["eval_job_id"] == "j1" + assert prism_units[0]["status"] == "running" + assert len(ac_units) == 1 + assert ac_units[0]["unit_id"] == "ac-u1" + + _assert_no_score_fields(body) From 016bff751952cebd0dc6faa48e03344c977cf0f7 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:07:17 +0000 Subject: [PATCH 7/7] feat(master): allowlist eval telemetry and aggregate live pools Preserve Authorization on eval capability POSTs, allow public pool/SSE reads, and add GET /v1/pools/executing that fans out to challenge live pools while stripping score-like fields. --- src/base/master/app_proxy.py | 193 ++++++++++++++++++++++++++++++++++- 1 file changed, 190 insertions(+), 3 deletions(-) diff --git a/src/base/master/app_proxy.py b/src/base/master/app_proxy.py index bfbfdd6ae..94129edf4 100644 --- a/src/base/master/app_proxy.py +++ b/src/base/master/app_proxy.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import inspect from collections.abc import AsyncIterator, Callable, Sequence from contextlib import AbstractAsyncContextManager, AsyncExitStack, asynccontextmanager @@ -387,6 +388,30 @@ def _is_agent_challenge_review_capability_route(method: str, path: str) -> bool: return False +def _is_agent_challenge_eval_capability_route(method: str, path: str) -> bool: + """True for eval-run Bearer capability POSTs (progress/session/result). + + The measured eval guest posts telemetry and terminal results to + ``/evaluation/v1/runs/{run_id}/{action}`` with + ``Authorization: Bearer ``. Same Authorization-preserve + contract as the review capability table; exact path shapes only. + """ + + normalized = normpath(f"/{path.lstrip('/')}") + parts = [part for part in normalized.split("/") if part] + normalized_method = method.upper() + canonical_path = f"/{path}" + if path.startswith("/") or canonical_path != normalized: + return False + + # POST /evaluation/v1/runs/{run_id}/{action} + if normalized_method != "POST" or len(parts) != 5: + return False + if parts[0] != "evaluation" or parts[1] != "v1" or parts[2] != "runs": + return False + return parts[4] in {"progress", "telemetry-session", "result"} + + def _is_agent_challenge_signed_route( slug: str, method: str, @@ -458,6 +483,10 @@ def _is_agent_challenge_enabled_mode_allowed_route( if _is_agent_challenge_review_capability_route(method, path): return True + # Eval-run guest capability POSTs (progress / telemetry-session / result). + if _is_agent_challenge_eval_capability_route(method, path): + return True + if ( len(parts) == 3 and parts[0] == "submissions" @@ -483,6 +512,31 @@ def _is_agent_challenge_enabled_mode_allowed_route( and normalized_method == "GET" ): return True + # Public task-events (snapshot + SSE stream) — unauthenticated reads. + if ( + len(parts) == 3 + and parts[0] == "submissions" + and parts[2] == "task-events" + and normalized_method == "GET" + ): + return True + if ( + len(parts) == 4 + and parts[0] == "submissions" + and parts[2] == "task-events" + and parts[3] == "stream" + and normalized_method == "GET" + ): + return True + # Public live execution pool (tamper-evidence telemetry only). + if ( + len(parts) == 3 + and parts[0] == "v1" + and parts[1] == "execution-pool" + and parts[2] == "live" + and normalized_method == "GET" + ): + return True # Public discovery / readiness (joinbase probes and miner docs). These are # unauthenticated read surfaces, not capability or trust injection. @@ -613,6 +667,98 @@ def _target_url(base_url: str, path: str, query: str) -> str: return url +# Trust boundary: pool telemetry is tamper-evidence only — never scoring. +_POOL_SCORE_LIKE_KEYS = frozenset( + { + "score", + "scores", + "weight", + "weights", + "emission", + "emission_percent", + "raw_score", + "final_score", + "normalized_score", + "incentive", + } +) + + +def _strip_pool_score_fields(payload: Any) -> Any: + """Recursively drop score/weight/emission keys from pool payloads.""" + + if isinstance(payload, dict): + return { + key: _strip_pool_score_fields(value) + for key, value in payload.items() + if str(key).lower() not in _POOL_SCORE_LIKE_KEYS + } + if isinstance(payload, list): + return [_strip_pool_score_fields(item) for item in payload] + return payload + + +def _pool_units_from_upstream_body(body: Any) -> list[Any]: + """Extract executing units from a challenge live-pool response body. + + Preference order (first present list wins): + - ``units`` — Agent Challenge / generic shape + - ``executing`` — alternate AC-style key + - ``jobs`` — Prism ``GET /v1/execution-pool/live`` shape + """ + + if not isinstance(body, dict): + return [] + for key in ("units", "executing", "jobs"): + value = body.get(key) + if isinstance(value, list): + stripped = _strip_pool_score_fields(value) + return stripped if isinstance(stripped, list) else [] + return [] + + +async def _fetch_challenge_execution_pool( + client: httpx.AsyncClient, + *, + internal_base_url: str, +) -> dict[str, Any]: + """Fetch one challenge live pool; never raises — returns units or error.""" + + url = _target_url(internal_base_url, "/v1/execution-pool/live", "") + try: + response = await client.get(url) + except httpx.HTTPError as exc: + return { + "units": [], + "error": { + "code": "upstream_unreachable", + "message": f"{type(exc).__name__}: {exc}", + }, + } + + if response.status_code != 200: + return { + "units": [], + "error": { + "code": "upstream_http_error", + "message": f"HTTP {response.status_code}", + }, + } + + try: + body = response.json() + except ValueError: + return { + "units": [], + "error": { + "code": "upstream_invalid_json", + "message": "live pool response was not JSON", + }, + } + + return {"units": _pool_units_from_upstream_body(body)} + + def _challenge_token_provider(registry: Any) -> ChallengeTokenProvider: def provider(slug: str) -> str: get_token = getattr(registry, "get_token", None) @@ -1037,9 +1183,15 @@ async def proxy_request(slug: str, path: str, request: Request) -> Response: ), preserve_review_capability_authorization=( slug == "agent-challenge" - and _is_agent_challenge_review_capability_route( - request.method, - path, + and ( + _is_agent_challenge_review_capability_route( + request.method, + path, + ) + or _is_agent_challenge_eval_capability_route( + request.method, + path, + ) ) ), strip_attested_trust_headers=( @@ -1162,6 +1314,41 @@ async def bridge_submission_status( ) -> Response: return await bridge_status(challenge_name, submission_id, request) + @app.get("/v1/pools/executing") + async def pools_executing() -> JSONResponse: + """Fan out to embedded challenges' live execution pools. + + Keyed by challenge slug. Partial upstream failure yields HTTP 200 with + an error object for that slug (no fabricated units, never 500). Pool + payloads are tamper-evidence only — score-like fields are stripped. + """ + + challenges = await _resolve_value(challenge_registry.list(active_only=True)) + # Direct loopback httpx (not client_factory) so unit tests can mock + # each challenge internal_base_url via respx without the unused factory. + async with httpx.AsyncClient( + timeout=10.0, + follow_redirects=False, + ) as pool_client: + entries = await asyncio.gather( + *( + _fetch_challenge_execution_pool( + pool_client, + internal_base_url=record.internal_base_url, + ) + for record in challenges + ) + ) + + by_slug: dict[str, Any] = {} + for record, entry in zip(challenges, entries, strict=True): + by_slug[record.slug] = _strip_pool_score_fields(entry) + + return JSONResponse( + content={"challenges": by_slug}, + status_code=status.HTTP_200_OK, + ) + @app.api_route( "/challenges/{slug}", methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"],