diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d4108d1..9dc35d57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ - `fp-cloud-cli`: typer 0.27.1 → 0.27.2, click 8.4.2 → 8.5.0, posthog 7.42.0 → 7.44.2 (#771) +- `osv-scanner.toml` ignores GHSA-8mgp-746c-j5xp (nltk 3.10.3, CVSS 8.3) until 2026-11-20. Surfaced 2026-09-03 against an already-approved PR — nothing on the branch introduced it and nothing on the branch can resolve it, which is the case the allow-list exists for. nltk is transitive via `llama-index-core` and appears only in `sdk/python/uv.lock`, the dev/test lockfile that pins every extra so CI can exercise the adapters; the SDK's own runtime dependency list is empty and the `llamaindex` extra is imported lazily, so it reaches no shipped code path. OSV reports "0 vulnerabilities can be fixed" and an empty FIXED VERSION, so the only alternative is dropping the extra and its adapter tests. Dated to match the chromadb entries so the list is revisited in one pass + - browserslist pinned to 4.28.8 in `overrides`, closing GHSA-73wf-gq98-2v4g and GHSA-c83g-rgw3-j3cx (both 7.5, both fixed in 4.28.7). They turned `main` red on its own scheduled Supply Chain run rather than on any PR's change — disclosed after this branch's first CI run, the same surface-late mechanism `osv-scanner.toml` documents for chromadb. browserslist is transitive-only (via `@babel/helper-compilation-targets`'s `^4.24.0`), so this is an override pin, not a dependency bump — and not `bun update browserslist`, which adds it to `dependencies` as a direct dep it is not and leaves 4.28.2 nested under `@babel/helper-compilation-targets`, keeping the gate red (#771) ## 1.0.3 — 2026-08-31 diff --git a/fp-cloud-cli/fp_cli/permissions.py b/fp-cloud-cli/fp_cli/permissions.py index 39560990..e475074c 100644 --- a/fp-cloud-cli/fp_cli/permissions.py +++ b/fp-cloud-cli/fp_cli/permissions.py @@ -26,6 +26,7 @@ "users:delete", "evaluations:read", "evaluations:trigger", + "evaluations:run", "dashboards:read", "dashboards:write", "dashboards:delete", diff --git a/osv-scanner.toml b/osv-scanner.toml index d55fa7e6..16392c99 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -82,3 +82,27 @@ reason = "chromadb 1.1.1 (published 2026-06-12, surfaced 2026-08-24). No fixed v id = "GHSA-xph7-9rjv-w5fr" ignoreUntil = 2026-11-20 reason = "chromadb 1.1.1 (published 2026-06-12, surfaced 2026-08-24). No fixed version. Transitive via crewai, dev/test lockfile only; see the shared justification above. Re-review by 2026-11-20." + +# --------------------------------------------------------------------------- +# nltk 3.10.3 — one advisory, no fixed version. +# +# Surfaced 2026-09-03 against an already-approved PR, which is the case this +# file exists for: nothing on the branch introduced it, and nothing on the +# branch can resolve it. +# +# nltk is transitive via llama-index-core and appears only in +# sdk/python/uv.lock, the dev/test lockfile that pins every extra so CI can +# exercise the adapters. The SDK's own runtime dependency list is empty — the +# `llamaindex` extra is opt-in and the integration module imports it lazily — +# so nltk reaches no shipped code path. OSV reports "0 vulnerabilities can be +# fixed" and an empty FIXED VERSION, so there is nothing to bump to; the only +# alternative is dropping the llamaindex extra and the adapter tests with it. +# +# Same re-review date as the chromadb entries above, so the whole list is +# revisited in one pass. +# --------------------------------------------------------------------------- + +[[IgnoredVulns]] +id = "GHSA-8mgp-746c-j5xp" +ignoreUntil = 2026-11-20 +reason = "nltk 3.10.3 (surfaced 2026-09-03, CVSS 8.3). No fixed version. Transitive via llama-index-core, dev/test lockfile only; the SDK ships zero runtime dependencies. Re-review by 2026-11-20." diff --git a/sdk/python/CHANGELOG.md b/sdk/python/CHANGELOG.md index 57aa24a9..e06eb6cc 100644 --- a/sdk/python/CHANGELOG.md +++ b/sdk/python/CHANGELOG.md @@ -16,6 +16,154 @@ moved the version here automatically; nothing has landed against `0.0.1b2` yet. Add entries as changes merge — this section becomes the GitHub Release body when it ships. +- Retire the old inbound evaluator boundary and add evaluator authoring plus the + outbound-only v2 worker runtime under the lazy `failproofai_sdk.evaluator` + namespace. +- Harden the managed-evaluator source sandbox against a class of escapes an + adversarial review found: `str.format`/`format_map` C-level field traversal, + generator/frame introspection (`gi_frame.f_globals`) that reached the eval + globals and could poison a process-shared namespace across evaluations, and + `type.mro()` type-object reach. Attribute access is now **default-deny** (an + allowlist of the transcript data surface plus pure string/collection methods, + so every current and future introspection attribute is rejected), each eval + runs with **fresh per-call globals**, and a result whose text embeds a runtime + object repr (`<... at 0x...>`, a heap-pointer/ASLR disclosure that falls out of + any bound method's repr) is rejected at the output boundary. `enumerate` and + bare generator expressions are no longer permitted — both were gratuitous + pointer-repr sources; use `range(len(...))` and list/set/dict comprehensions. +- Report **why** a server-authored definition was rejected. Every failure + collapsed to `evaluation raised `, so a hosted definition that can + never run reported only `evaluation raised UnsafeEvaluatorSource` — on every + session, forever, with nothing telling the author what was wrong. It matters + because the server accepts any source passing its size and key checks and does + not validate the sandbox's single-expression grammar, so a structurally + unrunnable definition is published successfully and then fails silently. + `UnsafeEvaluatorSource` now carries its detail (`evaluator_source must be one + expression`), bounded to `MAX_ERROR_MESSAGE_BYTES`. Deliberately narrower than + the generic handler, which still reports the type name only: this exception is + raised by our own validator before any customer source executes and describes + the source's shape, so it embeds no transcript content. +- Scrub the sandbox child's environment. `subprocess.Popen` inherited + `os.environ`, so the process executing untrusted server-authored source ran + with `FAILPROOFAI_EVALUATOR_TOKEN` in its environment — on the managed pod, + the cross-tenant credential. Defence in depth rather than a live escape (the + AST allowlist and empty `__builtins__` already stop a managed expression + reaching `os.environ`): a future gap there can no longer be escalated into + credential theft. Only what the interpreter needs is forwarded, `PYTHONPATH` + included. +- Contain a poison managed definition to its own run: source is now compiled + lazily inside the per-run executor, so a definition the sandbox rejects + dead-letters as one bounded `failed`/`eval_error` run instead of crashing the + assignment task and forcing it to be reclaimed until its attempt budget runs + out. +- Run managed (server-authored) evaluations in a **killable fork+exec'd + subprocess** with hard `RLIMIT_CPU` + `RLIMIT_AS` + a parent wall-clock kill — + so a compute/memory bomb in a hosted definition (`sum(range(10**20))`) can no + longer exhaust the worker (SEC-001). Cancelling an in-process thread does not + stop it; a fresh subprocess the kernel bounds and the parent terminates does. A + plain `os.fork()` would deadlock — the worker is multi-threaded (asyncio loop, + executor, writer) and forking one hangs the child on an inherited lock — so the + sandbox execs a fresh `python -m ..._sandbox_runner` that sets its own limits; + the transcript crosses in via `to_wire`, only the result crosses back. The + effective budget is **clamped to a hard ceiling** (`MAX_SANDBOX_TIMEOUT_SECONDS`, + 60s) so a large server-provided `timeout_seconds` cannot remove the bound. + Managed conditions, which previously ran with no timeout at all, are sandboxed + the same way. The result crossing back is **bounded on both sides** — the child + validates it (`result_items`, the 25-result limit) and refuses to serialize + anything over 1 MiB, and the parent reads at most that before killing the child + — so an oversized result (`metrics={str(x): 1 for x in range(100000)}`) cannot + OOM the worker either. The per-sandbox address space is capped (512 MiB) and the number of concurrent sandbox processes is bounded (a semaphore), so the AGGREGATE memory is bounded independent of the worker's `max_concurrency` — a fleet of concurrent runs can't OOM the host. Fails **closed** (`EvaluationSandboxUnavailable`) if the + sandbox cannot be spawned or the transcript cannot be serialized. Defense in depth at + compile time: reject `**` with a large/non-constant exponent and cap total AST + size. A managed condition the sandbox rejects now dead-letters as + `condition_error` instead of stranding the assignment. Only server-authored + source is isolated this way; customer evaluators still run in-process. +- Require `execution_mode` on the wire instead of coercing a falsy/missing value + to `local` — a malformed value silently ran a `python` definition down the + customer path (or vice-versa); it is now a hard protocol error. +- Count the sandbox-slot wait against the execution timeout (SEC-001). `_run_sandboxed` + acquired the `MAX_CONCURRENT_SANDBOXES` slot with an UNBOUNDED wait and only started + its wall-clock deadline afterward — so a run queued behind busy slots could, after the + runtime's `asyncio.wait_for` already reported it timed out (that cancels only the + awaiter, not the executor thread), still acquire a slot and launch a sandbox; 28 + threads could pile up behind 4 long sandboxes and starve the worker (conditions have + no runtime-level wait at all). One wall-clock deadline now covers BOTH the slot wait + and execution: the slot is acquired with the remaining budget, and on timeout the run + raises `EvaluationTimeout` **without spawning a child**. Regression test: more + concurrent compute bombs than slots all resolve within ~one budget, not N serialized + budgets. +- A managed (`python`) definition's applicability is now governed by the SERVER's + `condition_source`, never a colliding local condition (COR-001). `process_assignment` + keyed the local-definition lookup on `(eval_key, eval_version)` alone and selected + `local.condition` whenever a local definition with that key existed — so a managed + definition whose server condition was false could be forced to run anyway if the + worker had also registered a local definition under the same key whose condition was + true, executing server-managed source against the operator's intent. Condition + selection now branches on `execution_mode`, mirroring the evaluator branch: `LOCAL` + uses `local.condition`, `PYTHON` compiles and runs the server's `condition_source` + regardless of any key collision. Regression test: identical local+managed keys, local + condition true and managed false, asserts the definition is skipped and no managed run + is submitted. +- Recognize the server's `incomplete_plan` terminal error (API-001). The server rejects a + plan that fails to cover every snapshotted definition with `422 incomplete_plan`; that + code is now in the SDK's `ERROR_SPECS` mirror and the shared `contract.json` fixture + (byte-identical with the server's), so a worker no longer treats a valid server-defined + failure as an unrecognized error. The fixture-equality test covers it. +- Close a heap-address disclosure bypass in the managed-source sandbox + (adversarial-audit SEC). The output-boundary guard that rejects a `` + repr in a result field was anchored on the literal `<`, so an allow-listed + `str(payload.get).replace("<", "")` — or an f-string / `%`-format of a bare bound + method — kept the live heap address while stripping the match, leaking an + ASLR/memory-layout primitive of the sandbox process into a persisted result. The fix + moves the defense to compile time: a bound method (the only reachable value with a + pointer repr — the transcript and result types are all frozen, pointer-free + dataclasses) may now only be **called**, never referenced as a bare value, so no + reachable value can carry a pointer repr through `str()`, an f-string, or `%`. The + output-boundary scan is kept and broadened (no longer requires the leading `<`) as + defense in depth. Legitimate evaluations — which call methods and read data + attributes — are unaffected; regression tests cover the `.replace("<","")`, f-string, + and `%` bypasses and confirm called-method/data-attribute stringification still works. +- Switch the worker from long-polling to **normal (short) polling**, matching the + cadence of our other cloud surfaces. `claim` no longer sends `wait_seconds` and + the server returns immediately; when a claim comes back empty the worker sleeps + the server-advertised `poll_interval_seconds` (from the register response, + default 10 s) before polling again, instead of holding a request open for up to + 25 s. Removes the `claim_wait_seconds` config knob and the + `request_timeout_seconds > claim_wait_seconds` constraint; the poll cadence is + now tuned centrally by the server, not per worker. +- Bound the pre-plan condition phase by the assignment lease (hermes advisory). + Conditions were evaluated serially with no lease awareness before the plan was + created, and a managed condition could run its full sandbox budget — so a few + near-budget conditions could burn the whole lease before the plan request and + the server would fence the plan as `lease_lost`, reclaiming the assignment in a + loop instead of submitting a result. Each condition is now capped to the lease + time remaining before a plan-submission margin (using `lease_expires_at` when it + is in the future, else the negotiated lease duration), and once that budget is + gone the remaining conditions are skipped as `lease_exhausted` rather than run. + Local conditions, which previously had no timeout at all, are bounded the same + way. The complete fix — renewing the lease *during* the condition phase — needs + a server-side pre-plan heartbeat and is tracked separately. +- Make a timed-out **synchronous** evaluator observable and stop it starving the + worker (hermes advisory). A synchronous evaluator that overruns its timeout runs + in the executor thread and cannot be cancelled (CPython cannot interrupt a + running thread), so its thread was permanently lost; with the pool sized to the + concurrency limit, one such orphan on a single-slot worker silently stopped all + further local evaluation. The eval executor now carries headroom over the + semaphore so an orphaned thread does not immediately starve live capacity — the + semaphore stays the real concurrency bound — and each orphan increments + `sync_evaluations_orphaned` and logs a warning naming the evaluator, so a hung + one is findable. This is a finite cushion, not a cure for a permanently-blocked + evaluator; prefer `async def` evaluators (cooperatively cancellable) or managed + `python` evaluators (subprocess-isolated, hard-killed) for long or untrusted work. +- Let a managed source's list/set/dict comprehension read `session` on CPython + 3.10/3.11 (hermes COR-001). The sandbox eval put `session` in the eval *locals*, + but a comprehension runs in its own scope and resolves a free name like + `session` from *globals* — so on 3.10 (a supported version) an allowed source + such as `all([session.event_count > 0 for i in range(1)])` raised `NameError`. + `session` now goes in a fresh per-call globals mapping and both eval paths use + empty locals, which keeps isolation and works across 3.10–3.14. Regression test + runs on the whole version matrix. + ## 0.0.1b1 — 2026-08-24 The first release under this name. Everything below describes the package as it diff --git a/sdk/python/README.md b/sdk/python/README.md index f974587d..3cb8adaf 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -10,6 +10,14 @@ the platform. - **Dependencies:** none. Standard library only, so installing it constrains nothing else in your environment. +## Evaluator v2 status + +The legacy inbound `agenteye-evaluator` package has been retired; do not build new +evaluator services against its server-push HTTP contract. Evaluator v2 authoring +and its customer-hosted, outbound-only worker runtime live under the lazy +`failproofai_sdk.evaluator` namespace. Importing the top-level tracing SDK does not +import or start the evaluator runtime. + ## Installation ```bash diff --git a/sdk/python/examples/evaluator_worker.py b/sdk/python/examples/evaluator_worker.py new file mode 100644 index 00000000..912b61e5 --- /dev/null +++ b/sdk/python/examples/evaluator_worker.py @@ -0,0 +1,121 @@ +"""Customer evaluator with deterministic and optional async judge checks.""" + +from __future__ import annotations + +import asyncio +import ipaddress +import json +import os +from urllib.parse import urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from failproofai_sdk.evaluator import ( + ConditionResult, + EvalResult, + Evaluator, + Metric, + Score, +) + +app = Evaluator(name="customer-production", version="2026.08.1") + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + return None + + +@app.eval( + "tool_efficiency", + version="1.0.0", + labels=["tools", "deterministic"], + when=lambda session: ConditionResult( + session.count("tool_use") > 0, "no_tool_calls" + ), +) +def tool_efficiency(session): + calls = session.events_of_type("tool_use") + distinct = { + event.payload.get("tool_name") + for event in calls + if event.payload.get("tool_name") + } + value = len(distinct) / len(calls) + return EvalResult( + score=Score(value, passed=value >= 0.7), + metrics={ + "tool_call_count": Metric(len(calls), unit="events"), + "distinct_tool_count": Metric(len(distinct), unit="tools"), + }, + reasoning=f"{len(distinct)} distinct tools across {len(calls)} calls", + ) + + +def _judge_configured(session): + configured = bool(os.environ.get("EXAMPLE_JUDGE_URL")) + return ConditionResult(configured, "judge_not_configured") + + +def _last_content(session, event_type): + events = session.events_of_type(event_type) + if not events: + return None + payload = events[-1].payload + fields = { + "human_input": ("response",), + "model_response": ("content",), + "agent_end": ("summary",), + }.get(event_type, ("content", "summary", "response")) + return next((payload.get(field) for field in fields if payload.get(field)), None) + + +def _call_judge(question, answer): + url = os.environ["EXAMPLE_JUDGE_URL"] + parsed = urlsplit(url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("EXAMPLE_JUDGE_URL must be an absolute http(s) URL") + hostname = parsed.hostname + loopback = hostname == "localhost" + if hostname is not None and not loopback: + try: + loopback = ipaddress.ip_address(hostname).is_loopback + except ValueError: + loopback = False + if parsed.scheme != "https" and not loopback: + raise ValueError("EXAMPLE_JUDGE_URL must use https unless it targets loopback") + token = os.environ.get("EXAMPLE_JUDGE_TOKEN") + body = json.dumps({"question": question, "answer": answer}).encode("utf-8") + headers = {"Content-Type": "application/json", "Accept": "application/json"} + if token: + headers["Authorization"] = f"Bearer {token}" + request = Request(url, data=body, headers=headers, method="POST") + with build_opener(_RejectRedirects()).open(request, timeout=25) as response: # nosec B310 + result = json.loads(response.read(64 * 1024)) + return float(result["score"]), str( + result.get("reasoning") or "Judge returned no reasoning" + ) + + +@app.eval( + "answer_relevance", + version="judge-api-v1", + labels=["llm_judge", "relevance"], + when=_judge_configured, + timeout_seconds=30, +) +async def answer_relevance(session): + question = _last_content(session, "human_input") + answer = _last_content(session, "model_response") + if question is None or answer is None: + raise ValueError("answer relevance requires human input and model output") + value, reasoning = await asyncio.to_thread(_call_judge, question, answer) + value = min(max(value, 0.0), 1.0) + return EvalResult( + score=Score(value, passed=value >= 0.7), + reasoning=reasoning, + labels=("llm_judge", "relevance"), + ) + + +if __name__ == "__main__": + app.run_from_env() diff --git a/sdk/python/failproofai_sdk/evaluator/__init__.py b/sdk/python/failproofai_sdk/evaluator/__init__.py new file mode 100644 index 00000000..a94155ce --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/__init__.py @@ -0,0 +1,101 @@ +"""Authoring and worker primitives for FailproofAI Evaluator v2. + +This namespace is intentionally lazy relative to :mod:`failproofai_sdk`: users +who only emit telemetry do not import evaluator networking or runtime code. +""" + +from failproofai_sdk.evaluator.authoring import ( + Assertion, + ConditionResult, + EvalDefinition, + EvalResult, + Evaluator, + Metric, + Score, +) +from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient +from failproofai_sdk.evaluator.protocol import ( + Assignment, + AssignmentDefinition, + CatalogDefinition, + ClaimRequest, + ClaimResponse, + ErrorResponse, + EvalSelection, + ExecutionMode, + EvaluatorKind, + HeartbeatRequest, + HeartbeatResponse, + HeartbeatRun, + PlannedRun, + PlanRequest, + PlanResponse, + DefinitionsResponse, + ProtocolError, + RegisterRequest, + RegisterResponse, + RemoteError, + ResultItem, + ResultKind, + ResultRequest, + ResultResponse, + SessionTranscript, + SkippedEval, + TerminalRunStatus, + TranscriptEvent, + UnsupportedProtocolVersion, +) +from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime +from failproofai_sdk.evaluator.source import ( + UnsafeEvaluatorSource, + compile_condition, + compile_evaluator, + source_checksum, +) + +__all__ = [ + "Assertion", + "Assignment", + "AssignmentDefinition", + "CatalogDefinition", + "ClaimRequest", + "ClaimResponse", + "ConditionResult", + "ErrorResponse", + "EvalDefinition", + "EvalResult", + "EvalSelection", + "ExecutionMode", + "Evaluator", + "EvaluatorAPIError", + "EvaluatorClient", + "EvaluatorKind", + "HeartbeatRequest", + "HeartbeatResponse", + "HeartbeatRun", + "Metric", + "PlanRequest", + "PlanResponse", + "DefinitionsResponse", + "PlannedRun", + "ProtocolError", + "RegisterRequest", + "RegisterResponse", + "RemoteError", + "ResultItem", + "ResultKind", + "ResultRequest", + "ResultResponse", + "Score", + "SessionTranscript", + "SkippedEval", + "TerminalRunStatus", + "TranscriptEvent", + "UnsupportedProtocolVersion", + "WorkerConfig", + "WorkerRuntime", + "UnsafeEvaluatorSource", + "compile_condition", + "compile_evaluator", + "source_checksum", +] diff --git a/sdk/python/failproofai_sdk/evaluator/__main__.py b/sdk/python/failproofai_sdk/evaluator/__main__.py new file mode 100644 index 00000000..f7bff1c5 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/__main__.py @@ -0,0 +1,49 @@ +"""Run an evaluator declared as ``module:attribute``.""" + +from __future__ import annotations + +import argparse +import importlib +import os +from collections.abc import Sequence + +from failproofai_sdk.evaluator.authoring import Evaluator + + +def load_evaluator(spec: str) -> Evaluator: + module_name, separator, attribute = spec.partition(":") + if not module_name: + raise ValueError("evaluator module must not be empty") + if not separator: + attribute = "app" + if not attribute: + raise ValueError("evaluator attribute must not be empty") + module = importlib.import_module(module_name) + try: + evaluator = getattr(module, attribute) + except AttributeError as error: + raise ValueError(f"{spec!r} does not define {attribute!r}") from error + if not isinstance(evaluator, Evaluator): + raise TypeError( + f"{spec!r} resolved to {type(evaluator).__name__}, not Evaluator" + ) + return evaluator + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m failproofai_sdk.evaluator") + parser.add_argument( + "module", + nargs="?", + default=os.environ.get("FAILPROOFAI_EVALUATOR_MODULE"), + help="Python module and optional attribute (for example my_evals:app)", + ) + args = parser.parse_args(argv) + if not args.module: + parser.error("module is required (or set FAILPROOFAI_EVALUATOR_MODULE)") + load_evaluator(args.module).run_from_env() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py b/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py new file mode 100644 index 00000000..1b773e26 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py @@ -0,0 +1,63 @@ +"""Subprocess entry point for the managed-source sandbox. + +Invoked as ``python -m failproofai_sdk.evaluator._sandbox_runner `` by +``source._run_sandboxed``. The input file holds a pickled +``(kind, source, session_wire, cpu_seconds, mem_bytes, eval_key)`` tuple. This +process installs hard ``RLIMIT_CPU`` + ``RLIMIT_AS`` limits ON ITSELF, evaluates +the re-validated server-authored source against the reconstructed transcript, +validates + bounds the result, and writes a pickled ``("ok", result)`` / +``("err", type_name, message)`` outcome to stdout. + +This is a FRESH exec'd process — never a fork of the multi-threaded worker — so +there is no inherited-lock deadlock (forking a process that has an asyncio loop, +an executor pool and a writer daemon hangs the child). The parent enforces the +wall-clock bound and the output-size bound by reading only so far and killing +this process on timeout or overflow. +""" + +from __future__ import annotations + +import pickle +import sys + + +def _main() -> int: + with open(sys.argv[1], "rb") as handle: + kind, source, session_wire, cpu_seconds, mem_bytes, eval_key = pickle.loads( + handle.read() + ) + # Imported here, in the child, so the import cost is never on the worker's path. + from failproofai_sdk.evaluator.protocol import SessionTranscript + from failproofai_sdk.evaluator.source import ( + SANDBOX_MAX_RESULT_BYTES, + _install_limits, + _raw_eval, + ) + + try: + session = SessionTranscript.from_wire(session_wire) + # Compile + re-validate BEFORE the limits so validation cost is not charged + # against the eval's CPU budget; the limits bind the eval itself. + run = _raw_eval(source, kind) + _install_limits(cpu_seconds, mem_bytes) + result = run(session) + # Bound the result INSIDE the sandbox before it crosses back: result_items + # enforces the 25-result limit + field validation, so a huge result + # (`metrics={str(x):1 for x in range(100000)}`) raises here instead of + # being serialized and shipped to the parent. + if kind == "evaluator": + result.result_items(eval_key or "result") + payload = pickle.dumps(("ok", result)) + if len(payload) > SANDBOX_MAX_RESULT_BYTES: + payload = pickle.dumps( + ("err", "ResultTooLarge", "evaluation result exceeds the size limit") + ) + except BaseException as error: # noqa: BLE001 - relay type+msg, this process is the boundary + payload = pickle.dumps(("err", type(error).__name__, str(error)[:500])) + sys.stdout.buffer.write(payload) + sys.stdout.buffer.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/sdk/python/failproofai_sdk/evaluator/authoring.py b/sdk/python/failproofai_sdk/evaluator/authoring.py new file mode 100644 index 00000000..c33fdee1 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/authoring.py @@ -0,0 +1,404 @@ +"""Evaluator definition registry and typed author results.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import math +import re +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from failproofai_sdk.evaluator.protocol import ( + MAX_CATALOG_DEFINITIONS, + MAX_DESCRIPTION_BYTES, + MAX_DISPLAY_NAME_BYTES, + MAX_DISPLAY_VALUE_BYTES, + MAX_EVAL_KEY_BYTES, + MAX_LABEL_BYTES, + MAX_LABELS_PER_RESULT, + MAX_REASONING_BYTES, + MAX_RESULTS_PER_RUN, + MAX_SUMMARY_BYTES, + MAX_UNIT_BYTES, + MAX_VERSION_BYTES, + CatalogDefinition, + ResultItem, + ResultKind, + SessionTranscript, +) + +_KEY = re.compile(r"^[a-z][a-z0-9_]*$") +EvalFunction = Callable[[SessionTranscript], "EvalResult | Awaitable[EvalResult]"] +ConditionFunction = Callable[ + [SessionTranscript], "bool | ConditionResult | Awaitable[bool | ConditionResult]" +] +CancellationFunction = Callable[[SessionTranscript], "Any | Awaitable[Any]"] + + +def _bounded(value: str, *, field_name: str, maximum: int) -> str: + if not isinstance(value, str): + raise TypeError(f"{field_name} must be a string") + if not value: + raise ValueError(f"{field_name} must not be empty") + size = len(value.encode("utf-8")) + if size > maximum: + raise ValueError(f"{field_name} is {size} bytes; maximum is {maximum}") + # Reject C0 control characters and DEL, matching the server's `check_bounded` + # (server/src/evaluator/protocol.rs). Without this the SDK accepts a string — + # e.g. reasoning/summary quoting transcript text that contains an ANSI escape + # or NUL — that the server then rejects with a NON-RETRYABLE 422, so a + # successful evaluation is silently lost and its assignment dead-letters. + # TAB, LF and CR are kept because real multi-line reasoning uses them. + bad = next( + ( + ch + for ch in value + if (ord(ch) < 0x20 and ch not in "\t\n\r") or ord(ch) == 0x7F + ), + None, + ) + if bad is not None: + raise ValueError( + f"{field_name} must not contain control characters " + f"(found U+{ord(bad):04X})" + ) + return value + + +def _finite(value: float, field_name: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{field_name} must be a number") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{field_name} must be finite") + return result + + +def _labels(values: tuple[str, ...] | list[str]) -> tuple[str, ...]: + if len(values) > MAX_LABELS_PER_RESULT: + raise ValueError(f"at most {MAX_LABELS_PER_RESULT} labels are allowed") + normalized = [] + for label in values: + normalized.append(_bounded(label, field_name="label", maximum=MAX_LABEL_BYTES)) + if len(set(normalized)) != len(normalized): + raise ValueError("labels must be unique") + return tuple(sorted(normalized)) + + +@dataclass(frozen=True) +class Score: + value: float + passed: bool | None = None + unit: str = "ratio" + display_value: str | None = None + description: str | None = None + + def __post_init__(self) -> None: + value = _finite(self.value, "score value") + if not 0 <= value <= 1: + raise ValueError("score value must be between 0 and 1") + object.__setattr__(self, "value", value) + if self.passed is not None and not isinstance(self.passed, bool): + raise TypeError("score passed must be a boolean or None") + _validate_result_text(self.unit, self.display_value, self.description) + + +@dataclass(frozen=True) +class Metric: + value: float + unit: str = "" + display_value: str | None = None + description: str | None = None + + def __post_init__(self) -> None: + object.__setattr__(self, "value", _finite(self.value, "metric value")) + _validate_result_text(self.unit, self.display_value, self.description) + + +@dataclass(frozen=True) +class Assertion: + passed: bool + description: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.passed, bool): + raise TypeError("assertion passed must be a boolean") + if self.description is not None: + _bounded( + self.description, + field_name="description", + maximum=MAX_DESCRIPTION_BYTES, + ) + + +@dataclass(frozen=True) +class ConditionResult: + applicable: bool + reason_code: str = "condition_false" + + def __post_init__(self) -> None: + if not isinstance(self.applicable, bool): + raise TypeError("condition applicable must be a boolean") + _validate_key(self.reason_code, "condition reason code") + + +def _validate_result_text( + unit: str, display_value: str | None, description: str | None +) -> None: + if unit: + _bounded(unit, field_name="unit", maximum=MAX_UNIT_BYTES) + if display_value is not None: + _bounded( + display_value, + field_name="display value", + maximum=MAX_DISPLAY_VALUE_BYTES, + ) + if description is not None: + _bounded( + description, + field_name="description", + maximum=MAX_DESCRIPTION_BYTES, + ) + + +@dataclass(frozen=True) +class EvalResult: + score: Score | None = None + metrics: Mapping[str, Metric | float] = field(default_factory=dict) + assertions: Mapping[str, Assertion | bool] = field(default_factory=dict) + reasoning: str | None = None + summary: str | None = None + labels: tuple[str, ...] = () + + def __post_init__(self) -> None: + if self.reasoning is not None: + _bounded( + self.reasoning, + field_name="reasoning", + maximum=MAX_REASONING_BYTES, + ) + if self.summary is not None: + _bounded(self.summary, field_name="summary", maximum=MAX_SUMMARY_BYTES) + object.__setattr__(self, "labels", _labels(list(self.labels))) + + def result_items(self, eval_key: str) -> tuple[ResultItem, ...]: + items: list[ResultItem] = [] + if self.score is not None: + items.append( + ResultItem( + result_key=eval_key, + result_kind=ResultKind.SCORE, + numeric_value=self.score.value, + bool_value=self.score.passed, + unit=self.score.unit, + display_value=self.score.display_value, + description=self.score.description, + reasoning=self.reasoning, + labels=self.labels, + ) + ) + for key, raw_metric in sorted(self.metrics.items()): + _validate_key(key, "metric key") + metric = ( + raw_metric if isinstance(raw_metric, Metric) else Metric(raw_metric) + ) + items.append( + ResultItem( + result_key=key, + result_kind=ResultKind.METRIC, + numeric_value=metric.value, + unit=metric.unit, + display_value=metric.display_value, + description=metric.description, + # A metric-kind eval's primary result IS the metric whose + # key equals eval_key; attach the eval's reasoning there so + # it is not silently dropped for non-score evals. + reasoning=self.reasoning if key == eval_key else None, + labels=self.labels, + ) + ) + for key, raw_assertion in sorted(self.assertions.items()): + _validate_key(key, "assertion key") + assertion = ( + raw_assertion + if isinstance(raw_assertion, Assertion) + else Assertion(raw_assertion) + ) + items.append( + ResultItem( + result_key=key, + result_kind=ResultKind.ASSERTION, + bool_value=assertion.passed, + description=assertion.description, + # An assertion-kind eval's primary result is the assertion + # whose key equals eval_key; carry the eval's reasoning there + # so a non-score eval does not lose it. + reasoning=self.reasoning if key == eval_key else None, + labels=self.labels, + ) + ) + if not items: + raise ValueError("an EvalResult must contain a score, metric, or assertion") + if len(items) > MAX_RESULTS_PER_RUN: + raise ValueError( + f"an EvalResult may contain at most {MAX_RESULTS_PER_RUN} results" + ) + keys = [item.result_key for item in items] + if len(keys) != len(set(keys)): + raise ValueError("result keys must be unique within one evaluation run") + return tuple(items) + + +def _validate_key(value: str, field_name: str = "eval_key") -> str: + _bounded(value, field_name=field_name, maximum=MAX_EVAL_KEY_BYTES) + if not _KEY.fullmatch(value): + raise ValueError(f"{field_name} must match {_KEY.pattern}") + return value + + +@dataclass(frozen=True) +class EvalDefinition: + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] + function: EvalFunction + condition: ConditionFunction | None + on_cancel: CancellationFunction | None + timeout_seconds: float | None + + def catalog_definition(self) -> CatalogDefinition: + return CatalogDefinition( + eval_key=self.eval_key, + display_name=self.display_name, + eval_version=self.eval_version, + result_kind=self.result_kind, + labels=self.labels, + ) + + +class Evaluator: + """A process-local collection of explicitly versioned evaluations.""" + + def __init__(self, *, name: str, version: str) -> None: + self.name = _bounded(name, field_name="name", maximum=MAX_DISPLAY_NAME_BYTES) + self.version = _bounded( + version, field_name="version", maximum=MAX_VERSION_BYTES + ) + self._definitions: dict[str, EvalDefinition] = {} + + def eval( + self, + eval_key: str, + *, + version: str, + display_name: str | None = None, + result_kind: ResultKind | str = ResultKind.SCORE, + labels: tuple[str, ...] | list[str] = (), + when: ConditionFunction | None = None, + on_cancel: CancellationFunction | None = None, + timeout_seconds: float | None = None, + ) -> Callable[[EvalFunction], EvalFunction]: + key = _validate_key(eval_key) + eval_version = _bounded( + version, field_name="eval version", maximum=MAX_VERSION_BYTES + ) + display = _bounded( + display_name or eval_key.replace("_", " ").capitalize(), + field_name="display name", + maximum=MAX_DISPLAY_NAME_BYTES, + ) + kind = ResultKind(result_kind) + normalized_labels = _labels(list(labels)) + if timeout_seconds is not None: + timeout_seconds = _finite(timeout_seconds, "timeout_seconds") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be greater than zero") + + def register(function: EvalFunction) -> EvalFunction: + if key in self._definitions: + raise ValueError(f"duplicate eval key: {key}") + if len(self._definitions) >= MAX_CATALOG_DEFINITIONS: + raise ValueError( + f"an evaluator may define at most {MAX_CATALOG_DEFINITIONS} evaluations" + ) + if not callable(function): + raise TypeError("evaluation must be callable") + if when is not None and not callable(when): + raise TypeError("when must be callable") + if on_cancel is not None and not callable(on_cancel): + raise TypeError("on_cancel must be callable") + self._definitions[key] = EvalDefinition( + eval_key=key, + display_name=display, + eval_version=eval_version, + result_kind=kind, + labels=normalized_labels, + function=function, + condition=when, + on_cancel=on_cancel, + timeout_seconds=timeout_seconds, + ) + return function + + return register + + @property + def definitions(self) -> tuple[EvalDefinition, ...]: + return tuple(self._definitions[key] for key in sorted(self._definitions)) + + def catalog(self) -> tuple[CatalogDefinition, ...]: + return tuple(definition.catalog_definition() for definition in self.definitions) + + @property + def catalog_revision(self) -> str: + payload = [item.to_wire() for item in self.catalog()] + canonical = json.dumps( + payload, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return "sha256:" + hashlib.sha256(canonical).hexdigest() + + def definition(self, eval_key: str) -> EvalDefinition: + try: + return self._definitions[eval_key] + except KeyError as error: + raise KeyError(f"unknown eval key: {eval_key}") from error + + def run_from_env(self) -> None: + """Run this evaluator until the process receives a stop request.""" + import asyncio + import signal + + from failproofai_sdk.evaluator.runtime import WorkerConfig, WorkerRuntime + + async def run() -> None: + runtime = WorkerRuntime(self, WorkerConfig.from_env()) + loop = asyncio.get_running_loop() + for name in ("SIGINT", "SIGTERM"): + process_signal = getattr(signal, name, None) + if process_signal is None: + continue + try: + loop.add_signal_handler(process_signal, runtime.stop) + except (NotImplementedError, RuntimeError): + pass + await runtime.run_forever() + + asyncio.run(run()) + + @staticmethod + async def call( + function: EvalFunction | ConditionFunction, session: SessionTranscript + ) -> Any: + result = function(session) + if inspect.isawaitable(result): + return await result + return result diff --git a/sdk/python/failproofai_sdk/evaluator/client.py b/sdk/python/failproofai_sdk/evaluator/client.py new file mode 100644 index 00000000..fc3c28ce --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/client.py @@ -0,0 +1,299 @@ +"""Standard-library HTTP client for the Evaluator v2 worker protocol.""" + +from __future__ import annotations + +import ipaddress +import json +import random +import time +from collections.abc import Callable, Mapping +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from failproofai_sdk.evaluator.protocol import ( + CLAIM_PATH, + DEFINITIONS_PATH, + HEARTBEAT_PATH, + LEASE_GENERATION_HEADER, + MAX_TRANSCRIPT_BYTES, + PLAN_PATH, + REGISTER_PATH, + RESULT_PATH, + WORKER_ID_HEADER, + Assignment, + DefinitionsResponse, + ClaimRequest, + ClaimResponse, + ErrorResponse, + HeartbeatRequest, + HeartbeatResponse, + PlanRequest, + PlanResponse, + RegisterRequest, + RegisterResponse, + ResultRequest, + ResultResponse, + SessionTranscript, + WireModel, +) + +_DEFAULT_RESPONSE_LIMIT = 2 * 1024 * 1024 +_RETRYABLE_HTTP_STATUSES = frozenset({429, 502, 503, 504}) + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, request, file_pointer, code, message, headers, new_url): + return None + + +def _open_without_redirects(request: Request, *, timeout: float): + return build_opener(_RejectRedirects()).open(request, timeout=timeout) + + +class EvaluatorAPIError(RuntimeError): + def __init__( + self, + *, + status: int | None, + code: str, + message: str, + retryable: bool, + request_id: str | None = None, + ) -> None: + super().__init__(f"{code}: {message}") + self.status = status + self.code = code + self.retryable = retryable + self.request_id = request_id + + +class EvaluatorClient: + """Client for the public Evaluator v2 machine API. + + Hosted workers normally use the FailproofAI dashboard origin. Its ``/v1`` + passthrough forwards this worker's bearer credential to the private server. + """ + + def __init__( + self, + *, + base_url: str, + credential: str, + timeout_seconds: float = 30, + max_retries: int = 3, + allow_insecure_http: bool = False, + opener: Callable[..., Any] | None = None, + sleeper: Callable[[float], None] = time.sleep, + ) -> None: + parsed = urlsplit(base_url) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("base_url must be an absolute http(s) URL") + hostname = parsed.hostname + loopback = hostname == "localhost" + if hostname is not None and not loopback: + try: + loopback = ipaddress.ip_address(hostname).is_loopback + except ValueError: + loopback = False + if parsed.scheme != "https" and not loopback and not allow_insecure_http: + raise ValueError("base_url must use https unless it targets loopback") + if not credential or not credential.strip(): + raise ValueError("credential must not be empty") + if any( + ord(character) < 32 or ord(character) == 127 for character in credential + ): + raise ValueError("credential must not contain control characters") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be greater than zero") + if max_retries < 0: + raise ValueError("max_retries must not be negative") + self._base_url = base_url.rstrip("/") + "/" + self._origin = (parsed.scheme, parsed.netloc) + self._credential = credential + self._timeout_seconds = timeout_seconds + self._max_retries = max_retries + self._opener = opener or _open_without_redirects + self._sleeper = sleeper + + def register(self, request: RegisterRequest) -> RegisterResponse: + return RegisterResponse.from_wire( + self._json("POST", REGISTER_PATH, request, retry=True) + ) + + def claim(self, request: ClaimRequest) -> ClaimResponse: + # A lost claim response may already have leased work. Do not hide a + # second claim behind transport retry; the runtime recalculates capacity. + return ClaimResponse.from_wire( + self._json("POST", CLAIM_PATH, request, retry=False) + ) + + def transcript( + self, assignment: Assignment, *, worker_id: str + ) -> SessionTranscript: + headers = { + WORKER_ID_HEADER: worker_id, + LEASE_GENERATION_HEADER: str(assignment.lease_generation), + } + return SessionTranscript.from_wire( + self._json( + "GET", + assignment.transcript_url, + None, + retry=True, + headers=headers, + response_limit=MAX_TRANSCRIPT_BYTES, + ) + ) + + def definitions( + self, assignment: Assignment, *, worker_id: str + ) -> DefinitionsResponse: + headers = { + WORKER_ID_HEADER: worker_id, + LEASE_GENERATION_HEADER: str(assignment.lease_generation), + } + path = assignment.definitions_url or DEFINITIONS_PATH.format( + assignment_id=assignment.assignment_id + ) + return DefinitionsResponse.from_wire( + self._json("GET", path, None, retry=True, headers=headers) + ) + + def plan(self, assignment_id: str, request: PlanRequest) -> PlanResponse: + return PlanResponse.from_wire( + self._json( + "POST", + PLAN_PATH.format(assignment_id=assignment_id), + request, + retry=True, + ) + ) + + def heartbeat(self, request: HeartbeatRequest) -> HeartbeatResponse: + return HeartbeatResponse.from_wire( + self._json("POST", HEARTBEAT_PATH, request, retry=True) + ) + + def submit_result(self, run_id: str, request: ResultRequest) -> ResultResponse: + return ResultResponse.from_wire( + self._json( + "POST", + RESULT_PATH.format(evaluation_run_id=run_id), + request, + retry=True, + ) + ) + + def _url(self, path: str) -> str: + url = urljoin(self._base_url, path) + parsed = urlsplit(url) + if (parsed.scheme, parsed.netloc) != self._origin: + raise EvaluatorAPIError( + status=None, + code="invalid_transcript_url", + message="server supplied a URL outside the configured API origin", + retryable=False, + ) + return url + + def _json( + self, + method: str, + path: str, + body: WireModel | None, + *, + retry: bool, + headers: Mapping[str, str] | None = None, + response_limit: int = _DEFAULT_RESPONSE_LIMIT, + ) -> dict[str, Any]: + encoded = None + request_headers = { + "Accept": "application/json", + "Authorization": f"Bearer {self._credential}", + "User-Agent": "failproofai-sdk-evaluator/2", + } + if body is not None: + encoded = json.dumps( + body.to_wire(), + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if headers: + request_headers.update(headers) + + attempts = self._max_retries + 1 if retry else 1 + for attempt in range(attempts): + request = Request( + self._url(path), data=encoded, headers=request_headers, method=method + ) + try: + with self._opener(request, timeout=self._timeout_seconds) as response: + return self._decode( + response.read(response_limit + 1), response_limit + ) + except HTTPError as error: + api_error = self._http_error(error, response_limit) + if attempt + 1 == attempts or not api_error.retryable: + raise api_error from error + except (URLError, TimeoutError, OSError) as error: + if attempt + 1 == attempts: + raise EvaluatorAPIError( + status=None, + code="transport_error", + message=str(error), + retryable=True, + ) from error + # Jitter is scheduling noise, not a security decision. + self._sleeper(random.uniform(0, min(0.25 * (2**attempt), 2.0))) # nosec B311 + raise AssertionError("retry loop exhausted without returning or raising") + + @staticmethod + def _decode(raw: bytes, limit: int) -> dict[str, Any]: + if len(raw) > limit: + raise EvaluatorAPIError( + status=None, + code="response_too_large", + message=f"server response exceeds {limit} bytes", + retryable=False, + ) + try: + value = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise EvaluatorAPIError( + status=None, + code="invalid_response", + message="server response was not valid JSON", + retryable=False, + ) from error + if not isinstance(value, dict): + raise EvaluatorAPIError( + status=None, + code="invalid_response", + message="server response must be a JSON object", + retryable=False, + ) + return value + + @classmethod + def _http_error(cls, error: HTTPError, limit: int) -> EvaluatorAPIError: + raw = error.read(limit + 1) + try: + response = ErrorResponse.from_wire(cls._decode(raw, limit)) + except (ValueError, EvaluatorAPIError): + return EvaluatorAPIError( + status=error.code, + code="http_error", + message=f"server returned HTTP {error.code}", + retryable=error.code in _RETRYABLE_HTTP_STATUSES, + ) + return EvaluatorAPIError( + status=error.code, + code=response.error.code, + message=response.error.message, + retryable=response.error.retryable, + request_id=response.error.request_id, + ) diff --git a/sdk/python/failproofai_sdk/evaluator/protocol.py b/sdk/python/failproofai_sdk/evaluator/protocol.py new file mode 100644 index 00000000..70f46352 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/protocol.py @@ -0,0 +1,754 @@ +"""Dependency-free wire models for the outbound Evaluator v2 protocol.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from dataclasses import asdict, dataclass +from enum import Enum +from typing import Any, TypeVar + +PROTOCOL_VERSION = "2" +TRANSCRIPT_SCHEMA_VERSION = "2" +RESULT_SCHEMA_VERSION = "2" + +REGISTER_PATH = "/v1/evaluator/workers/register" +CLAIM_PATH = "/v1/evaluator/assignments/claim" +TRANSCRIPT_PATH = "/v1/evaluator/assignments/{assignment_id}/transcript" +DEFINITIONS_PATH = "/v1/evaluator/assignments/{assignment_id}/definitions" +PLAN_PATH = "/v1/evaluator/assignments/{assignment_id}/plan" +HEARTBEAT_PATH = "/v1/evaluator/runs/heartbeat" +RESULT_PATH = "/v1/evaluator/runs/{evaluation_run_id}/result" +WORKER_ID_HEADER = "X-FailproofAI-Worker-Id" +LEASE_GENERATION_HEADER = "X-FailproofAI-Lease-Generation" + +HEARTBEAT_INTERVAL_SECONDS = 30 +LEASE_DURATION_SECONDS = 120 +# Fallback poll cadence if the register response omits poll_interval_seconds. The +# worker prefers the server-advertised value; claims are normal short polls, never +# long-polls, so this only bounds idle latency, not connection lifetime. +DEFAULT_POLL_INTERVAL_SECONDS = 10 +MAX_ATTEMPTS = 5 + +MAX_CATALOG_DEFINITIONS = 100 +MAX_CLAIM_CAPACITY = 32 +MAX_TRANSCRIPT_BYTES = 25 * 1024 * 1024 +MAX_RESULTS_PER_RUN = 25 +MAX_EVAL_KEY_BYTES = 128 +MAX_DISPLAY_NAME_BYTES = 128 +MAX_VERSION_BYTES = 128 +MAX_WORKER_ID_BYTES = 128 +MAX_LABEL_BYTES = 64 +MAX_LABELS_PER_RESULT = 20 +MAX_SUMMARY_BYTES = 4 * 1024 +MAX_REASONING_BYTES = 16 * 1024 +MAX_UNIT_BYTES = 64 +MAX_DISPLAY_VALUE_BYTES = 256 +MAX_DESCRIPTION_BYTES = 1_000 +MAX_ERROR_CODE_BYTES = 64 +MAX_ERROR_MESSAGE_BYTES = 4 * 1024 + +ERROR_SPECS = { + "invalid_credentials": {"http_status": 401, "retryable": False}, + "instance_disabled": {"http_status": 403, "retryable": False}, + "insufficient_permissions": {"http_status": 403, "retryable": False}, + "assignment_not_found": {"http_status": 404, "retryable": False}, + "run_not_found": {"http_status": 404, "retryable": False}, + "catalog_mismatch": {"http_status": 409, "retryable": False}, + "lease_lost": {"http_status": 409, "retryable": False}, + "plan_conflict": {"http_status": 409, "retryable": False}, + "submission_conflict": {"http_status": 409, "retryable": False}, + "retry_budget_exhausted": {"http_status": 409, "retryable": False}, + "transcript_too_large": {"http_status": 413, "retryable": False}, + "invalid_request": {"http_status": 422, "retryable": False}, + "invalid_catalog": {"http_status": 422, "retryable": False}, + "incomplete_plan": {"http_status": 422, "retryable": False}, + "unsupported_protocol_version": {"http_status": 426, "retryable": False}, + "internal_error": {"http_status": 500, "retryable": True}, +} + + +class ProtocolError(ValueError): + """A local or remote evaluator protocol contract violation.""" + + +class UnsupportedProtocolVersion(ProtocolError): + def __init__(self, received: str) -> None: + super().__init__( + f"unsupported evaluator protocol version {received!r}; " + f"supported major version is {PROTOCOL_VERSION}" + ) + self.received = received + + +def validate_protocol_version(version: str) -> None: + if version != PROTOCOL_VERSION: + raise UnsupportedProtocolVersion(version) + + +class EvaluatorKind(str, Enum): + MANAGED = "managed" + CUSTOMER = "customer" + + +class ResultKind(str, Enum): + SCORE = "score" + METRIC = "metric" + ASSERTION = "assertion" + + +class ExecutionMode(str, Enum): + LOCAL = "local" + PYTHON = "python" + + +class TerminalRunStatus(str, Enum): + SUCCEEDED = "succeeded" + FAILED = "failed" + TIMED_OUT = "timed_out" + CANCELLED = "cancelled" + + +_EnumT = TypeVar("_EnumT", bound=Enum) + + +def _wire(value: Any) -> Any: + if isinstance(value, Enum): + return value.value + if hasattr(value, "__dataclass_fields__"): + return {key: _wire(item) for key, item in asdict(value).items()} + if isinstance(value, (list, tuple)): + return [_wire(item) for item in value] + if isinstance(value, dict): + return {key: _wire(item) for key, item in value.items()} + return value + + +class WireModel: + def to_wire(self) -> dict[str, Any]: + return _wire(self) + + +def _string(data: Mapping[str, Any], key: str) -> str: + value = data.get(key) + if not isinstance(value, str): + raise ProtocolError(f"{key} must be a string") + return value + + +def _integer(data: Mapping[str, Any], key: str) -> int: + value = data.get(key) + if isinstance(value, bool) or not isinstance(value, int): + raise ProtocolError(f"{key} must be an integer") + return value + + +def _list(data: Mapping[str, Any], key: str) -> list[Any]: + value = data.get(key) + if not isinstance(value, list): + raise ProtocolError(f"{key} must be an array") + return value + + +def _object(value: Any, field_name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ProtocolError(f"{field_name} must be an object") + return value + + +def _object_list(data: Mapping[str, Any], key: str) -> tuple[Mapping[str, Any], ...]: + return tuple( + _object(value, f"{key}[{index}]") + for index, value in enumerate(_list(data, key)) + ) + + +def _string_list(data: Mapping[str, Any], key: str) -> tuple[str, ...]: + values = _list(data, key) + for index, value in enumerate(values): + if not isinstance(value, str): + raise ProtocolError(f"{key}[{index}] must be a string") + return tuple(values) + + +def _enum(enum_type: type[_EnumT], data: Mapping[str, Any], key: str) -> _EnumT: + value = _string(data, key) + try: + return enum_type(value) + except ValueError as error: + allowed = ", ".join(repr(item.value) for item in enum_type) + raise ProtocolError(f"{key} must be one of {allowed}") from error + + +def _positive_integer(data: Mapping[str, Any], key: str) -> int: + value = _integer(data, key) + if value <= 0: + raise ProtocolError(f"{key} must be greater than zero") + return value + + +def _nonnegative_integer(data: Mapping[str, Any], key: str) -> int: + value = _integer(data, key) + if value < 0: + raise ProtocolError(f"{key} must not be negative") + return value + + +def _optional_string(data: Mapping[str, Any], key: str) -> str | None: + value = data.get(key) + if value is not None and not isinstance(value, str): + raise ProtocolError(f"{key} must be a string or null") + return value + + +@dataclass(frozen=True) +class CatalogDefinition(WireModel): + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] = () + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> CatalogDefinition: + return cls( + eval_key=_string(data, "eval_key"), + display_name=_string(data, "display_name"), + eval_version=_string(data, "eval_version"), + result_kind=_enum(ResultKind, data, "result_kind"), + labels=_string_list(data, "labels"), + ) + + +@dataclass(frozen=True) +class RegisterRequest(WireModel): + worker_id: str + sdk_version: str + catalog_revision: str + max_concurrency: int + definitions: tuple[CatalogDefinition, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RegisterRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + sdk_version=_string(data, "sdk_version"), + catalog_revision=_string(data, "catalog_revision"), + max_concurrency=_integer(data, "max_concurrency"), + definitions=tuple( + CatalogDefinition.from_wire(item) + for item in _object_list(data, "definitions") + ), + ) + + +@dataclass(frozen=True) +class RegisterResponse(WireModel): + evaluator_instance_id: str + evaluator_kind: EvaluatorKind + heartbeat_interval_seconds: int + lease_duration_seconds: int + poll_interval_seconds: int + claim_limit: int + disabled_definitions: tuple[str, ...] = () + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RegisterResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + evaluator_instance_id=_string(data, "evaluator_instance_id"), + evaluator_kind=_enum(EvaluatorKind, data, "evaluator_kind"), + heartbeat_interval_seconds=_integer(data, "heartbeat_interval_seconds"), + lease_duration_seconds=_integer(data, "lease_duration_seconds"), + poll_interval_seconds=_integer(data, "poll_interval_seconds"), + claim_limit=_integer(data, "claim_limit"), + disabled_definitions=_string_list(data, "disabled_definitions"), + ) + + +@dataclass(frozen=True) +class ClaimRequest(WireModel): + worker_id: str + catalog_revision: str + capacity: int + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ClaimRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + catalog_revision=_string(data, "catalog_revision"), + capacity=_integer(data, "capacity"), + ) + + +@dataclass(frozen=True) +class Assignment(WireModel): + assignment_id: str + lease_generation: int + lease_expires_at: str + session_id: str + session_revision_id: str + agent_id: str + environment: str + trigger_reason: str + event_count: int + transcript_url: str + definitions_url: str = "" + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> Assignment: + return cls( + assignment_id=_string(data, "assignment_id"), + lease_generation=_positive_integer(data, "lease_generation"), + lease_expires_at=_string(data, "lease_expires_at"), + session_id=_string(data, "session_id"), + session_revision_id=_string(data, "session_revision_id"), + agent_id=_string(data, "agent_id"), + environment=_string(data, "environment"), + trigger_reason=_string(data, "trigger_reason"), + event_count=_nonnegative_integer(data, "event_count"), + transcript_url=_string(data, "transcript_url"), + definitions_url=str(data.get("definitions_url") or ""), + ) + + +@dataclass(frozen=True) +class AssignmentDefinition(WireModel): + eval_key: str + display_name: str + eval_version: str + result_kind: ResultKind + labels: tuple[str, ...] = () + execution_mode: ExecutionMode = ExecutionMode.LOCAL + condition_source: str | None = None + source_checksum: str | None = None + timeout_seconds: float | None = None + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> AssignmentDefinition: + timeout = data.get("timeout_seconds") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ProtocolError("timeout_seconds must be a number or null") + timeout = float(timeout) + if not math.isfinite(timeout) or timeout <= 0: + raise ProtocolError("timeout_seconds must be finite and greater than zero") + return cls( + eval_key=_string(data, "eval_key"), + display_name=_string(data, "display_name"), + eval_version=_string(data, "eval_version"), + result_kind=_enum(ResultKind, data, "result_kind"), + labels=_string_list(data, "labels"), + # Require execution_mode explicitly. Coercing a falsy/missing value to + # 'local' silently ran a server-authored ('python') definition down the + # customer-local path (or vice-versa); a malformed wire value is a + # protocol error, not a default (F2). + execution_mode=_enum(ExecutionMode, data, "execution_mode"), + condition_source=_optional_string(data, "condition_source"), + source_checksum=_optional_string(data, "source_checksum"), + timeout_seconds=timeout, + ) + + +@dataclass(frozen=True) +class DefinitionsResponse(WireModel): + assignment_id: str + catalog_revision: str + definitions: tuple[AssignmentDefinition, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> DefinitionsResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + assignment_id=_string(data, "assignment_id"), + catalog_revision=_string(data, "catalog_revision"), + definitions=tuple( + AssignmentDefinition.from_wire(item) + for item in _object_list(data, "definitions") + ), + ) + + +@dataclass(frozen=True) +class ClaimResponse(WireModel): + assignments: tuple[Assignment, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ClaimResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + tuple( + Assignment.from_wire(item) for item in _object_list(data, "assignments") + ) + ) + + +@dataclass(frozen=True) +class TranscriptEvent(WireModel): + id: str + ts: str + event_type: str + payload: Mapping[str, Any] + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> TranscriptEvent: + payload = data.get("payload") + if not isinstance(payload, Mapping): + raise ProtocolError("payload must be an object") + return cls( + id=_string(data, "id"), + ts=_string(data, "ts"), + event_type=_string(data, "event_type"), + payload=dict(payload), + ) + + +@dataclass(frozen=True) +class SessionTranscript(WireModel): + assignment_id: str + session_id: str + session_revision_id: str + agent_id: str + environment: str + started_at: str + ended_at: str + event_count: int + events: tuple[TranscriptEvent, ...] + schema_version: str = TRANSCRIPT_SCHEMA_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> SessionTranscript: + version = _string(data, "schema_version") + if version != TRANSCRIPT_SCHEMA_VERSION: + raise ProtocolError(f"unsupported transcript schema version {version!r}") + events = tuple( + TranscriptEvent.from_wire(item) for item in _object_list(data, "events") + ) + event_count = _nonnegative_integer(data, "event_count") + if event_count != len(events): + raise ProtocolError( + f"event_count is {event_count}, but transcript contains {len(events)} events" + ) + return cls( + assignment_id=_string(data, "assignment_id"), + session_id=_string(data, "session_id"), + session_revision_id=_string(data, "session_revision_id"), + agent_id=_string(data, "agent_id"), + environment=_string(data, "environment"), + started_at=_string(data, "started_at"), + ended_at=_string(data, "ended_at"), + event_count=event_count, + events=events, + ) + + def events_of_type(self, event_type: str) -> tuple[TranscriptEvent, ...]: + return tuple(event for event in self.events if event.event_type == event_type) + + def count(self, event_type: str) -> int: + return sum(event.event_type == event_type for event in self.events) + + +@dataclass(frozen=True) +class EvalSelection(WireModel): + eval_key: str + eval_version: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> EvalSelection: + return cls(_string(data, "eval_key"), _string(data, "eval_version")) + + +@dataclass(frozen=True) +class SkippedEval(WireModel): + eval_key: str + eval_version: str + reason_code: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> SkippedEval: + return cls( + _string(data, "eval_key"), + _string(data, "eval_version"), + _string(data, "reason_code"), + ) + + +@dataclass(frozen=True) +class PlanRequest(WireModel): + worker_id: str + lease_generation: int + selected: tuple[EvalSelection, ...] = () + skipped: tuple[SkippedEval, ...] = () + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlanRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + selected=tuple( + EvalSelection.from_wire(item) for item in _object_list(data, "selected") + ), + skipped=tuple( + SkippedEval.from_wire(item) for item in _object_list(data, "skipped") + ), + ) + + +@dataclass(frozen=True) +class PlannedRun(WireModel): + evaluation_run_id: str + eval_key: str + eval_version: str + execution_mode: ExecutionMode = ExecutionMode.LOCAL + evaluator_source: str | None = None + source_checksum: str | None = None + timeout_seconds: float | None = None + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlannedRun: + timeout = data.get("timeout_seconds") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ProtocolError("timeout_seconds must be a number or null") + timeout = float(timeout) + if not math.isfinite(timeout) or timeout <= 0: + raise ProtocolError("timeout_seconds must be finite and greater than zero") + return cls( + evaluation_run_id=_string(data, "evaluation_run_id"), + eval_key=_string(data, "eval_key"), + eval_version=_string(data, "eval_version"), + # Require execution_mode explicitly. Coercing a falsy/missing value to + # 'local' silently ran a server-authored ('python') definition down the + # customer-local path (or vice-versa); a malformed wire value is a + # protocol error, not a default (F2). + execution_mode=_enum(ExecutionMode, data, "execution_mode"), + evaluator_source=_optional_string(data, "evaluator_source"), + source_checksum=_optional_string(data, "source_checksum"), + timeout_seconds=timeout, + ) + + +@dataclass(frozen=True) +class PlanResponse(WireModel): + assignment_id: str + assignment_status: str + runs: tuple[PlannedRun, ...] + idempotent_replay: bool = False + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> PlanResponse: + validate_protocol_version(_string(data, "protocol_version")) + replay = data.get("idempotent_replay", False) + if not isinstance(replay, bool): + raise ProtocolError("idempotent_replay must be a boolean") + return cls( + assignment_id=_string(data, "assignment_id"), + assignment_status=_string(data, "assignment_status"), + runs=tuple( + PlannedRun.from_wire(item) for item in _object_list(data, "runs") + ), + idempotent_replay=replay, + ) + + +@dataclass(frozen=True) +class HeartbeatRun(WireModel): + evaluation_run_id: str + state: str + progress: float | None = None + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatRun: + progress = data.get("progress") + if progress is not None: + if isinstance(progress, bool) or not isinstance(progress, (int, float)): + raise ProtocolError("progress must be a number or null") + progress = float(progress) + if not math.isfinite(progress) or not 0 <= progress <= 1: + raise ProtocolError("progress must be finite and between 0 and 1") + return cls( + evaluation_run_id=_string(data, "evaluation_run_id"), + state=_string(data, "state"), + progress=progress, + ) + + +@dataclass(frozen=True) +class HeartbeatRequest(WireModel): + worker_id: str + lease_generation: int + runs: tuple[HeartbeatRun, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatRequest: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + runs=tuple( + HeartbeatRun.from_wire(item) for item in _object_list(data, "runs") + ), + ) + + +@dataclass(frozen=True) +class HeartbeatResponse(WireModel): + lease_expires_at: str + accepted_run_ids: tuple[str, ...] + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> HeartbeatResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls( + lease_expires_at=_string(data, "lease_expires_at"), + accepted_run_ids=_string_list(data, "accepted_run_ids"), + ) + + +@dataclass(frozen=True) +class ResultItem(WireModel): + result_key: str + result_kind: ResultKind + numeric_value: float | None = None + bool_value: bool | None = None + text_value: str | None = None + unit: str = "" + display_value: str | None = None + description: str | None = None + reasoning: str | None = None + labels: tuple[str, ...] = () + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultItem: + numeric = data.get("numeric_value") + if numeric is not None: + if isinstance(numeric, bool) or not isinstance(numeric, (int, float)): + raise ProtocolError("numeric_value must be a number or null") + numeric = float(numeric) + if not math.isfinite(numeric): + raise ProtocolError("numeric_value must be finite") + boolean = data.get("bool_value") + if boolean is not None and not isinstance(boolean, bool): + raise ProtocolError("bool_value must be a boolean or null") + return cls( + result_key=_string(data, "result_key"), + result_kind=_enum(ResultKind, data, "result_kind"), + numeric_value=numeric, + bool_value=boolean, + text_value=_optional_string(data, "text_value"), + unit=_string(data, "unit"), + display_value=_optional_string(data, "display_value"), + description=_optional_string(data, "description"), + reasoning=_optional_string(data, "reasoning"), + labels=_string_list(data, "labels"), + ) + + +@dataclass(frozen=True) +class ResultRequest(WireModel): + submission_id: str + worker_id: str + lease_generation: int + status: TerminalRunStatus + started_at: str + finished_at: str + duration_ms: int + summary: str | None + results: tuple[ResultItem, ...] + error_code: str | None + error_message: str | None + protocol_version: str = PROTOCOL_VERSION + result_schema_version: str = RESULT_SCHEMA_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultRequest: + validate_protocol_version(_string(data, "protocol_version")) + result_schema_version = _string(data, "result_schema_version") + if result_schema_version != RESULT_SCHEMA_VERSION: + raise ProtocolError( + f"unsupported result schema version {result_schema_version!r}" + ) + return cls( + submission_id=_string(data, "submission_id"), + worker_id=_string(data, "worker_id"), + lease_generation=_positive_integer(data, "lease_generation"), + status=_enum(TerminalRunStatus, data, "status"), + started_at=_string(data, "started_at"), + finished_at=_string(data, "finished_at"), + duration_ms=_nonnegative_integer(data, "duration_ms"), + summary=_optional_string(data, "summary"), + results=tuple( + ResultItem.from_wire(item) for item in _object_list(data, "results") + ), + error_code=_optional_string(data, "error_code"), + error_message=_optional_string(data, "error_message"), + ) + + +@dataclass(frozen=True) +class ResultResponse(WireModel): + evaluation_run_id: str + submission_id: str + status: str + idempotent_replay: bool + result_count: int + result_checksum: str + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ResultResponse: + validate_protocol_version(_string(data, "protocol_version")) + replay = data.get("idempotent_replay") + if not isinstance(replay, bool): + raise ProtocolError("idempotent_replay must be a boolean") + return cls( + evaluation_run_id=_string(data, "evaluation_run_id"), + submission_id=_string(data, "submission_id"), + status=_string(data, "status"), + idempotent_replay=replay, + result_count=_nonnegative_integer(data, "result_count"), + result_checksum=_string(data, "result_checksum"), + ) + + +@dataclass(frozen=True) +class RemoteError(WireModel): + code: str + message: str + retryable: bool + request_id: str + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> RemoteError: + retryable = data.get("retryable") + if not isinstance(retryable, bool): + raise ProtocolError("retryable must be a boolean") + return cls( + code=_string(data, "code"), + message=_string(data, "message"), + retryable=retryable, + request_id=_string(data, "request_id"), + ) + + +@dataclass(frozen=True) +class ErrorResponse(WireModel): + error: RemoteError + protocol_version: str = PROTOCOL_VERSION + + @classmethod + def from_wire(cls, data: Mapping[str, Any]) -> ErrorResponse: + validate_protocol_version(_string(data, "protocol_version")) + return cls(RemoteError.from_wire(_object(data.get("error"), "error"))) diff --git a/sdk/python/failproofai_sdk/evaluator/runtime.py b/sdk/python/failproofai_sdk/evaluator/runtime.py new file mode 100644 index 00000000..50aa0008 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/runtime.py @@ -0,0 +1,936 @@ +"""Async worker state machine for Evaluator v2.""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import inspect +import logging +import os +import socket +import threading +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any +from uuid import uuid4 + +from failproofai_sdk import __version__ +from failproofai_sdk.evaluator.authoring import ( + ConditionResult, + EvalDefinition, + EvalResult, + Evaluator, +) +from failproofai_sdk.evaluator.client import EvaluatorAPIError, EvaluatorClient +from failproofai_sdk.evaluator.protocol import ( + DEFAULT_POLL_INTERVAL_SECONDS, + MAX_CLAIM_CAPACITY, + MAX_ERROR_MESSAGE_BYTES, + MAX_WORKER_ID_BYTES, + Assignment, + AssignmentDefinition, + ClaimRequest, + EvalSelection, + ExecutionMode, + HeartbeatRequest, + HeartbeatRun, + PlanRequest, + RegisterRequest, + ResultRequest, + SkippedEval, + TerminalRunStatus, +) +from failproofai_sdk.evaluator.source import ( + EvaluationTimeout, + UnsafeEvaluatorSource, + compile_condition, + compile_evaluator, + source_checksum, +) + +logger = logging.getLogger("failproofai_sdk.evaluator") + +# A synchronous evaluator (or condition) that overruns its timeout cannot be +# cancelled: the executor thread runs the customer function to completion no +# matter what `asyncio.wait_for` does, because CPython cannot interrupt a running +# thread. To stop one such orphaned thread from starving live capacity, the eval +# executor is sized with headroom OVER the concurrency limit — the semaphore, not +# the thread pool, stays the real bound on how many evaluations run at once. This +# is a finite cushion, not a cure: a permanently-blocked synchronous evaluator +# invoked once per session leaks one thread per session, and no fixed pool +# survives that. `sync_evaluations_orphaned` and a warning make the offending +# evaluator findable; prefer `async def` evaluators (cooperatively cancellable) or +# managed PYTHON evaluators (subprocess-isolated, hard-killed) for long or +# untrusted work. +_EVAL_EXECUTOR_ORPHAN_HEADROOM = 8 + +# Reserved out of the lease for the plan request (and network jitter) so the +# pre-plan condition phase always leaves time to submit the plan before the lease +# expires. See `WorkerRuntime._condition_phase_deadline`. +_CONDITION_PHASE_SAFETY_MARGIN_SECONDS = 5.0 + +# Fallback wall-clock bound for an evaluation whose definition declares no +# `timeout_seconds`. A LOCAL (customer-authored) eval is a plain coroutine/thread +# with no sandbox backstop, so without this an eval that hangs — a wedged +# `await`, an unbounded judge HTTP call — runs forever, permanently wedging its +# worker slot and holding the assignment lease. Matches the storage contract's +# 5-minute per-eval default. (Managed evals are additionally hard-capped inside +# the fork sandbox, so this is only their outer bound.) +DEFAULT_EVAL_TIMEOUT_SECONDS = 300.0 + + +def _utc_now() -> str: + return ( + datetime.now(timezone.utc) + .isoformat(timespec="microseconds") + .replace("+00:00", "Z") + ) + + +def _deferred_managed_eval(source: str, timeout_seconds: int | None, eval_key: str): + """Compile server-authored source lazily, at invocation time. + + Compilation can reject unsafe or malformed source (``UnsafeEvaluatorSource``). + Building the definition with this thunk instead of a pre-compiled function + routes that failure through the same per-run ``try/except`` that turns any + evaluation error into a bounded ``FAILED`` result — so a poison definition + dead-letters cleanly as one failed run instead of raising out of assignment + setup, crashing the task, and forcing the whole assignment to be reclaimed + and retried until its attempt budget is exhausted. + """ + + def evaluate(session: Any) -> Any: + return compile_evaluator( + source, timeout_seconds=timeout_seconds, eval_key=eval_key + )(session) + + return evaluate + + +def _positive_int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None: + return default + try: + value = int(raw) + except ValueError as error: + raise ValueError(f"{name} must be an integer") from error + if value <= 0: + raise ValueError(f"{name} must be greater than zero") + return value + + +def _boolean(name: str, default: bool = False) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + normalized = raw.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ValueError(f"{name} must be a boolean") + + +@dataclass(frozen=True) +class WorkerConfig: + server_url: str + credential: str + worker_id: str + max_concurrency: int = 1 + request_timeout_seconds: int = 30 + drain_timeout_seconds: int = 60 + allow_insecure_http: bool = False + + @classmethod + def from_env(cls) -> WorkerConfig: + server_url = os.environ.get("FAILPROOFAI_EVALUATOR_URL", "").strip() + credential = os.environ.get("FAILPROOFAI_EVALUATOR_TOKEN", "").strip() + if not server_url: + raise ValueError("FAILPROOFAI_EVALUATOR_URL is required") + if not credential: + raise ValueError("FAILPROOFAI_EVALUATOR_TOKEN is required") + worker_id = os.environ.get("FAILPROOFAI_EVALUATOR_WORKER_ID", "").strip() + if not worker_id: + worker_id = f"{socket.gethostname()}-{os.getpid()}" + if len(worker_id.encode("utf-8")) > MAX_WORKER_ID_BYTES: + raise ValueError( + f"FAILPROOFAI_EVALUATOR_WORKER_ID exceeds {MAX_WORKER_ID_BYTES} bytes" + ) + if any(ord(character) < 32 or ord(character) == 127 for character in worker_id): + raise ValueError( + "FAILPROOFAI_EVALUATOR_WORKER_ID must not contain control characters" + ) + config = cls( + server_url=server_url, + credential=credential, + worker_id=worker_id, + max_concurrency=_positive_int("FAILPROOFAI_EVALUATOR_CONCURRENCY", 1), + request_timeout_seconds=_positive_int( + "FAILPROOFAI_EVALUATOR_REQUEST_TIMEOUT_SECONDS", 30 + ), + drain_timeout_seconds=_positive_int( + "FAILPROOFAI_EVALUATOR_DRAIN_TIMEOUT_SECONDS", 60 + ), + allow_insecure_http=_boolean( + "FAILPROOFAI_EVALUATOR_ALLOW_INSECURE_HTTP" + ), + ) + if config.max_concurrency > MAX_CLAIM_CAPACITY: + raise ValueError( + f"FAILPROOFAI_EVALUATOR_CONCURRENCY exceeds {MAX_CLAIM_CAPACITY}" + ) + return config + + +class WorkerRuntime: + def __init__( + self, + evaluator: Evaluator, + config: WorkerConfig, + *, + client: EvaluatorClient | None = None, + ) -> None: + self.evaluator = evaluator + self.config = config + self.client = client or EvaluatorClient( + base_url=config.server_url, + credential=config.credential, + timeout_seconds=config.request_timeout_seconds, + allow_insecure_http=config.allow_insecure_http, + ) + self._stopping = asyncio.Event() + self._active: set[asyncio.Task[None]] = set() + self._heartbeat_interval = 30 + self._poll_interval = DEFAULT_POLL_INTERVAL_SECONDS + self._claim_limit = config.max_concurrency + self._lease_duration = 120 + self._disabled_definitions: set[str] = set() + self._eval_semaphore = asyncio.Semaphore(config.max_concurrency) + # Headroom over the semaphore so a timed-out-but-still-running synchronous + # evaluator (an unkillable orphaned thread) does not immediately starve + # live capacity — the semaphore remains the true concurrency bound. See + # `_EVAL_EXECUTOR_ORPHAN_HEADROOM`. + self._eval_executor = concurrent.futures.ThreadPoolExecutor( + max_workers=config.max_concurrency + _EVAL_EXECUTOR_ORPHAN_HEADROOM, + thread_name_prefix="failproof-eval", + ) + self._registered = False + self._last_server_contact: float | None = None + self._metric_lock = threading.Lock() + self._metrics: dict[str, int] = {} + + async def register(self) -> None: + try: + response = await self._call_client( + self.client.register, + RegisterRequest( + worker_id=self.config.worker_id, + sdk_version=__version__, + catalog_revision=self.evaluator.catalog_revision, + max_concurrency=self.config.max_concurrency, + definitions=self.evaluator.catalog(), + ), + ) + except Exception: + self._increment("registration_failure") + raise + self._heartbeat_interval = response.heartbeat_interval_seconds + self._poll_interval = response.poll_interval_seconds + self._lease_duration = response.lease_duration_seconds + self._claim_limit = min(self.config.max_concurrency, response.claim_limit) + if ( + self._heartbeat_interval <= 0 + or self._poll_interval <= 0 + or self._lease_duration <= self._heartbeat_interval + or self._claim_limit <= 0 + ): + self._increment("registration_failure") + raise RuntimeError( + "server returned invalid evaluator timing or claim limits" + ) + self._disabled_definitions = set(response.disabled_definitions) + self._registered = True + self._increment("registration_success") + + async def run_forever(self) -> None: + await self.register() + retry_delay = 1.0 + try: + while not self._stopping.is_set(): + self._reap_finished() + capacity = self._claim_limit - len(self._active) + if capacity <= 0: + await self._wait_for_progress() + continue + try: + response = await self._call_client( + self.client.claim, + ClaimRequest( + worker_id=self.config.worker_id, + catalog_revision=self.evaluator.catalog_revision, + capacity=capacity, + ), + ) + except EvaluatorAPIError as error: + self._increment("claim_failures") + logger.warning( + "evaluator claim failed", + extra={"code": error.code, "retryable": error.retryable}, + ) + if not error.retryable: + raise + delay = ( + float(self._lease_duration) + if error.status is None + else retry_delay + ) + await self._wait_or_stop(delay) + retry_delay = min(retry_delay * 2.0, 30.0) + continue + retry_delay = 1.0 + assignments = self._validated_assignments( + response.assignments, capacity + ) + for assignment in assignments: + task = asyncio.create_task(self.process_assignment(assignment)) + self._active.add(task) + self._increment("assignments_claimed", len(assignments)) + if not assignments: + # Normal short poll: the server returns immediately, so when + # nothing is queued we wait the advertised interval before + # polling again instead of hot-looping. When work IS returned + # we loop straight back to drain any backlog up to capacity. + await self._wait_or_stop(float(self._poll_interval)) + finally: + await self.drain() + self._eval_executor.shutdown(wait=False, cancel_futures=True) + + async def run_once(self) -> int: + """Claim once and finish the returned assignments; useful for jobs/tests.""" + response = await self._call_client( + self.client.claim, + ClaimRequest( + worker_id=self.config.worker_id, + catalog_revision=self.evaluator.catalog_revision, + capacity=self._claim_limit, + ), + ) + assignments = self._validated_assignments( + response.assignments, self._claim_limit + ) + self._increment("assignments_claimed", len(assignments)) + await asyncio.gather(*(self.process_assignment(item) for item in assignments)) + return len(assignments) + + def stop(self) -> None: + self._stopping.set() + + async def drain(self) -> None: + self._reap_finished() + if not self._active: + return + done, pending = await asyncio.wait( + self._active, timeout=self.config.drain_timeout_seconds + ) + for task in done: + self._consume_task(task) + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + self._active.clear() + + async def process_assignment(self, assignment: Assignment) -> None: + try: + session = await self._call_client( + self.client.transcript, + assignment, + worker_id=self.config.worker_id, + ) + except EvaluatorAPIError as error: + if error.code == "transcript_too_large": + # The session transcript exceeds the hard ceiling. No runs are + # planned yet, so there is nothing to submit a per-run result for, + # and the error is non-retryable — re-raising would only wedge the + # poll loop and burn the assignment's whole retry budget against a + # transcript that can never shrink. Log and return; the server + # terminalizes the assignment as `too_large`. + logger.warning( + "assignment %s transcript is too large to evaluate; skipping", + assignment.assignment_id, + ) + self._increment("transcripts_too_large") + return + raise + if session.session_revision_id != assignment.session_revision_id: + raise RuntimeError("transcript session revision does not match assignment") + + descriptors = await self._assignment_definitions(assignment) + # Every descriptor the assignment carries, keyed for reconstruction: on an + # idempotent replay the server re-serves the first attempt's run set, which + # may include a run this attempt's re-derived plan would have skipped. + descriptor_by_key = { + (item.eval_key, item.eval_version): item for item in descriptors + } + selected: list[tuple[AssignmentDefinition, EvalDefinition | None]] = [] + skipped: list[SkippedEval] = [] + local_definitions = { + (item.eval_key, item.eval_version): item + for item in self.evaluator.definitions + } + # The assignment lease is fixed at claim time and cannot be renewed until + # the plan is submitted (the server only extends a lease for a *planned* + # assignment with running runs). A slow condition phase can therefore burn + # the whole lease and get the plan fenced as lease_lost, so every + # condition is bounded by the lease it must leave time to plan within. + condition_deadline = self._condition_phase_deadline(assignment) + for descriptor in descriptors: + local = local_definitions.get( + (descriptor.eval_key, descriptor.eval_version) + ) + if descriptor.execution_mode is ExecutionMode.LOCAL and local is None: + raise RuntimeError("server requested a definition absent from this worker") + if descriptor.eval_key in self._disabled_definitions: + skipped.append(self._skipped_descriptor(descriptor, "disabled_by_server")) + self._increment("conditions_skipped") + continue + try: + # Whose condition decides applicability follows the execution mode, + # mirroring the evaluator branch below (`run.execution_mode`): a LOCAL + # definition's condition is client-authored (`local.condition`); a + # PYTHON (managed) definition's is server-authored and MUST govern even + # when the worker also registered the same key/version locally. Keying + # `local` on `(eval_key, eval_version)` alone means a managed def can + # collide with a local one; selecting `local.condition` there would let + # a matching local condition override the server's managed rule and run + # the managed evaluator against the operator's intent (COR-001). + # Compile INSIDE the try: a managed condition the sandbox rejects + # (unsafe/malformed source) must dead-letter as `condition_error`, + # not raise out of the plan loop and strand the whole assignment + # until its retry budget is exhausted. + managed_condition_source: str | None = None + if descriptor.execution_mode is ExecutionMode.LOCAL: + condition_function = local.condition if local is not None else None + elif descriptor.condition_source: + # Compiled below, once the lease budget is known, so the + # sandbox subprocess is bounded by whatever lease remains. + managed_condition_source = descriptor.condition_source + condition_function = None + else: + condition_function = None + if condition_function is None and managed_condition_source is None: + # No condition to run — applicable by default, no lease spent. + selected.append((descriptor, local)) + continue + budget = self._condition_budget( + condition_deadline, descriptor.timeout_seconds + ) + if budget <= 0.0: + # Not enough lease left to evaluate this condition and still + # submit the plan in time; skip it (and, as the loop proceeds, + # every later condition) rather than do work the server will + # fence as lease_lost and reclaim in a loop. + skipped.append( + self._skipped_descriptor(descriptor, "lease_exhausted") + ) + self._increment("conditions_skipped") + self._increment("conditions_lease_exhausted") + continue + if managed_condition_source is not None: + condition_function = compile_condition( + managed_condition_source, + timeout_seconds=budget, + ) + condition = await asyncio.wait_for( + self._invoke(condition_function, session), timeout=budget + ) + if isinstance(condition, ConditionResult): + applicable = condition.applicable + reason_code = condition.reason_code + elif isinstance(condition, bool): + applicable = condition + reason_code = "condition_false" + else: + raise TypeError("condition must return bool or ConditionResult") + except Exception as error: # noqa: BLE001 - isolates customer condition code + logger.warning( + "evaluator condition failed", + extra={ + "assignment_id": assignment.assignment_id, + "error_type": type(error).__name__, + }, + ) + skipped.append(self._skipped_descriptor(descriptor, "condition_error")) + self._increment("conditions_skipped") + continue + if applicable: + selected.append((descriptor, local)) + self._increment("conditions_selected") + else: + skipped.append(self._skipped_descriptor(descriptor, reason_code)) + self._increment("conditions_skipped") + + plan = await self._call_client( + self.client.plan, + assignment.assignment_id, + PlanRequest( + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + selected=tuple( + EvalSelection(item.eval_key, item.eval_version) + for item, _local in selected + ), + skipped=tuple(skipped), + ), + ) + if plan.assignment_id != assignment.assignment_id: + raise RuntimeError("server returned a plan for a different assignment") + # On an idempotent replay the server's status is authoritative: this + # attempt may have selected a different set than the first, so a mismatch + # against our own `selected` is expected, not an error. + if not plan.idempotent_replay: + expected_status = "planned" if selected else "skipped" + if plan.assignment_status != expected_status: + raise RuntimeError("server returned an inconsistent assignment status") + + definitions = { + (item.eval_key, item.eval_version): (item, local) + for item, local in selected + } + run_definitions: list[tuple[str, EvalDefinition]] = [] + run_ids: set[str] = set() + for run in plan.runs: + if run.evaluation_run_id in run_ids: + raise RuntimeError("server returned a duplicate evaluation run id") + run_ids.add(run.evaluation_run_id) + run_key = (run.eval_key, run.eval_version) + selected_definition = definitions.pop(run_key, None) + if selected_definition is None: + # On an idempotent replay the server's run set is AUTHORITATIVE — + # it re-serves the first attempt's runs even for a definition this + # attempt's condition phase would have skipped. Reconstruct the + # definition from the assignment's descriptors rather than raising + # and dead-lettering an assignment that could otherwise never + # converge (the divergence-abort bug). + if plan.idempotent_replay: + replay_descriptor = descriptor_by_key.get(run_key) + if replay_descriptor is not None: + selected_definition = ( + replay_descriptor, + local_definitions.get(run_key), + ) + if selected_definition is None: + raise RuntimeError("server returned an unrequested evaluation run") + descriptor, local = selected_definition + if run.execution_mode is not descriptor.execution_mode: + raise RuntimeError("server changed the evaluation execution mode") + if run.execution_mode is ExecutionMode.LOCAL: + if local is None: + raise RuntimeError("local evaluation definition is unavailable") + definition = local + else: + if not run.evaluator_source or not run.source_checksum: + raise RuntimeError("server omitted managed evaluation source") + expected = source_checksum( + descriptor.condition_source, run.evaluator_source + ) + if expected != run.source_checksum or ( + descriptor.source_checksum + and descriptor.source_checksum != run.source_checksum + ): + raise RuntimeError("managed evaluation source checksum mismatch") + definition = EvalDefinition( + eval_key=descriptor.eval_key, + display_name=descriptor.display_name, + eval_version=descriptor.eval_version, + result_kind=descriptor.result_kind, + labels=descriptor.labels, + function=_deferred_managed_eval( + run.evaluator_source, + run.timeout_seconds or descriptor.timeout_seconds, + descriptor.eval_key, + ), + condition=None, + on_cancel=None, + timeout_seconds=run.timeout_seconds or descriptor.timeout_seconds, + ) + run_definitions.append((run.evaluation_run_id, definition)) + if definitions and not plan.idempotent_replay: + raise RuntimeError("server omitted a selected evaluation run") + + tasks = { + run_id: asyncio.create_task( + self._execute_run(assignment, run_id, definition, session) + ) + for run_id, definition in run_definitions + } + heartbeat = asyncio.create_task(self._heartbeat(assignment, tasks)) + try: + outcomes = await asyncio.gather(*tasks.values(), return_exceptions=True) + for outcome in outcomes: + if isinstance(outcome, BaseException): + raise outcome + finally: + heartbeat.cancel() + await asyncio.gather(heartbeat, return_exceptions=True) + + async def _execute_run( + self, + assignment: Assignment, + run_id: str, + definition: EvalDefinition, + session: Any, + ) -> None: + async with self._eval_semaphore: + await self._execute_run_in_slot(assignment, run_id, definition, session) + + async def _execute_run_in_slot( + self, + assignment: Assignment, + run_id: str, + definition: EvalDefinition, + session: Any, + ) -> None: + started_at = _utc_now() + started = time.monotonic() + # A synchronous evaluator runs in the executor thread; if it overruns the + # wall-clock timeout below, the thread cannot be cancelled and is orphaned. + sync_function = not inspect.iscoroutinefunction(definition.function) + try: + invocation = self._invoke(definition.function, session) + # Always bound the evaluation. A definition with no declared + # timeout_seconds falls back to DEFAULT_EVAL_TIMEOUT_SECONDS rather + # than awaiting unbounded — an unbounded local eval that hangs would + # wedge its worker slot and hold the lease forever. + eval_timeout = ( + definition.timeout_seconds + if definition.timeout_seconds is not None + else DEFAULT_EVAL_TIMEOUT_SECONDS + ) + result = await asyncio.wait_for(invocation, timeout=eval_timeout) + if not isinstance(result, EvalResult): + raise TypeError("evaluation must return EvalResult") + items = result.result_items(definition.eval_key) + if not any( + item.result_key == definition.eval_key + and item.result_kind == definition.result_kind + for item in items + ): + raise ValueError( + "evaluation result does not contain its declared primary result" + ) + status = TerminalRunStatus.SUCCEEDED + summary = result.summary + error_code = None + error_message = None + except (asyncio.TimeoutError, EvaluationTimeout) as timeout_error: + # asyncio.TimeoutError: the awaiter hit the wall-clock. EvaluationTimeout: + # the forked managed sandbox was killed by its CPU/memory/time budget — + # the real, thread-uncancellable case. Both are a timed-out run. + await self._cancel_hook(definition, session) + if isinstance(timeout_error, asyncio.TimeoutError) and sync_function: + # The awaiter gave up while a SYNCHRONOUS evaluator was still + # running in the executor. CPython cannot interrupt that thread, + # so it is now orphaned — it runs to completion (or forever) + # holding a worker thread. Count it and name the evaluator so a + # hung one is findable; the executor's headroom keeps this one + # orphan from immediately starving live capacity. + self._increment("sync_evaluations_orphaned") + logger.warning( + "synchronous evaluation exceeded its timeout and cannot be " + "cancelled; its worker thread is orphaned until it returns", + extra={ + "assignment_id": assignment.assignment_id, + "eval_key": definition.eval_key, + }, + ) + items = () + status = TerminalRunStatus.TIMED_OUT + summary = None + error_code = "eval_timeout" + error_message = "evaluation exceeded its configured timeout" + except asyncio.CancelledError: + await self._cancel_hook(definition, session) + raise + except UnsafeEvaluatorSource as error: + # Surface the REASON for a rejected server-authored definition. + # + # This is deliberately narrower than the generic handler below. + # UnsafeEvaluatorSource is raised by our own validator before any + # customer source executes, and its message is SDK-authored text + # about the source's shape ("evaluator_source must be one + # expression", "contains disallowed syntax: Assign") — it embeds no + # transcript content, so it is safe to send back over the wire. + # + # Without this the author saw only "evaluation raised + # UnsafeEvaluatorSource" on every session, with no way to learn what + # was wrong: the server accepts any source that passes its size and + # key checks, so a definition that can never run is published + # successfully and then fails silently and permanently. + items = () + status = TerminalRunStatus.FAILED + summary = None + error_code = "eval_error" + detail = str(error).strip() + error_message = ( + f"evaluator source rejected: {detail}" + if detail + else "evaluator source rejected by the sandbox validator" + ) + encoded = error_message.encode("utf-8") + if len(encoded) > MAX_ERROR_MESSAGE_BYTES: + error_message = encoded[:MAX_ERROR_MESSAGE_BYTES].decode( + "utf-8", "ignore" + ) + except Exception as error: # noqa: BLE001 - converts customer eval failures + # Type name ONLY on the wire. A customer eval's exception text can + # quote the transcript it was reading, and this field is persisted + # and shown in the dashboard, so the message itself is not repeated + # there. But log the FULL traceback LOCALLY: this runs on the + # customer's own pod over their own data, and without it an author + # whose eval raises sees only "evaluation raised HTTPError" in the + # dashboard and nothing at all in their pod logs — no way to debug + # their own eval. + logger.warning( + "evaluation %r raised %s; reported to the server as a failed run", + definition.eval_key, + type(error).__name__, + exc_info=True, + ) + items = () + status = TerminalRunStatus.FAILED + summary = None + error_code = "eval_error" + error_message = f"evaluation raised {type(error).__name__}" + + request = ResultRequest( + submission_id=str(uuid4()), + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + status=status, + started_at=started_at, + finished_at=_utc_now(), + duration_ms=max(0, round((time.monotonic() - started) * 1_000)), + summary=summary, + results=items, + error_code=error_code, + error_message=error_message, + ) + await self._call_client(self.client.submit_result, run_id, request) + self._increment(f"runs_{status.value}") + + async def _cancel_hook(self, definition: EvalDefinition, session: Any) -> None: + if definition.on_cancel is None: + return + try: + await self._invoke(definition.on_cancel, session) + except Exception as error: # noqa: BLE001 - cancellation hooks are customer code + logger.warning( + "evaluator cancellation hook failed", + extra={"error_type": type(error).__name__}, + ) + + async def _heartbeat( + self, + assignment: Assignment, + tasks: dict[str, asyncio.Task[None]], + ) -> None: + # Beat IMMEDIATELY, before the first sleep. The pre-plan condition phase + # may have consumed most of the claim-time lease, and the server only + # renews a planned assignment's lease on heartbeat — so sleeping a full + # interval here can let the lease expire before the first renewal, after + # the runs have already started, cancelling every one of them. The first + # beat renews the lease the moment the runs are live. + first = True + while True: + if not first: + await asyncio.sleep(self._heartbeat_interval) + first = False + active = tuple( + HeartbeatRun(evaluation_run_id=run_id, state="running") + for run_id, task in tasks.items() + if not task.done() + ) + if not active: + return + try: + response = await self._call_client( + self.client.heartbeat, + HeartbeatRequest( + worker_id=self.config.worker_id, + lease_generation=assignment.lease_generation, + runs=active, + ), + ) + accepted = set(response.accepted_run_ids) + for run_id, task in tasks.items(): + if not task.done() and run_id not in accepted: + task.cancel() + except EvaluatorAPIError as error: + if error.code == "lease_lost": + self._increment("leases_lost") + for task in tasks.values(): + task.cancel() + return + logger.warning( + "evaluator heartbeat failed", + extra={ + "assignment_id": assignment.assignment_id, + "code": error.code, + }, + ) + self._increment("heartbeat_failures") + except Exception as error: # noqa: BLE001 - keep lease renewal alive + logger.warning( + "evaluator heartbeat error", + extra={ + "assignment_id": assignment.assignment_id, + "error_type": type(error).__name__, + }, + ) + self._increment("heartbeat_failures") + + def _condition_phase_deadline(self, assignment: Assignment) -> float: + """Monotonic-clock reading by which the pre-plan condition phase must end. + + The real `lease_expires_at` is used when it is in the future (production); + a past or unparseable value (clock skew, or a replayed transcript in a + test) falls back to the negotiated lease duration measured from now, so + the bound never fires spuriously on a stale deadline. + """ + remaining = float(self._lease_duration) + try: + expires = datetime.fromisoformat( + assignment.lease_expires_at.replace("Z", "+00:00") + ) + parsed = (expires - datetime.now(timezone.utc)).total_seconds() + if parsed > 0: + remaining = parsed + except (ValueError, AttributeError): + pass + return time.monotonic() + remaining + + def _condition_budget( + self, deadline: float, timeout_seconds: int | None + ) -> float: + """Seconds a single condition may run: the lease left before the + plan-submission margin, capped by the definition's own timeout.""" + remaining = ( + deadline - time.monotonic() - _CONDITION_PHASE_SAFETY_MARGIN_SECONDS + ) + if timeout_seconds is not None: + remaining = min(remaining, float(timeout_seconds)) + return remaining + + async def _invoke(self, function, session): + if inspect.iscoroutinefunction(function): + return await function(session) + loop = asyncio.get_running_loop() + result = await loop.run_in_executor(self._eval_executor, function, session) + if inspect.isawaitable(result): + return await result + return result + + @staticmethod + def _skipped(definition: EvalDefinition, reason: str) -> SkippedEval: + return SkippedEval(definition.eval_key, definition.eval_version, reason) + + @staticmethod + def _skipped_descriptor( + definition: AssignmentDefinition, reason: str + ) -> SkippedEval: + return SkippedEval(definition.eval_key, definition.eval_version, reason) + + async def _assignment_definitions( + self, assignment: Assignment + ) -> tuple[AssignmentDefinition, ...]: + if assignment.definitions_url: + response = await self._call_client( + self.client.definitions, + assignment, + worker_id=self.config.worker_id, + ) + if response.assignment_id != assignment.assignment_id: + raise RuntimeError("server returned definitions for another assignment") + return response.definitions + return tuple( + AssignmentDefinition( + eval_key=item.eval_key, + display_name=item.display_name, + eval_version=item.eval_version, + result_kind=item.result_kind, + labels=item.labels, + ) + for item in self.evaluator.definitions + ) + + def _reap_finished(self) -> None: + done = {task for task in self._active if task.done()} + self._active.difference_update(done) + for task in done: + self._consume_task(task) + + @staticmethod + def _validated_assignments( + assignments: tuple[Assignment, ...], capacity: int + ) -> tuple[Assignment, ...]: + if len(assignments) > capacity: + raise RuntimeError("server returned more assignments than requested") + assignment_ids = [item.assignment_id for item in assignments] + if len(assignment_ids) != len(set(assignment_ids)): + raise RuntimeError("server returned duplicate assignments") + return assignments + + @staticmethod + def _consume_task(task: asyncio.Task[None]) -> None: + try: + task.result() + except asyncio.CancelledError: + pass + except Exception: + logger.exception("evaluator assignment failed") + + async def _wait_for_progress(self) -> None: + if not self._active: + return + stop_task = asyncio.create_task(self._stopping.wait()) + try: + await asyncio.wait( + (*self._active, stop_task), return_when=asyncio.FIRST_COMPLETED + ) + finally: + if not stop_task.done(): + stop_task.cancel() + await asyncio.gather(stop_task, return_exceptions=True) + + async def _wait_or_stop(self, seconds: float) -> None: + try: + await asyncio.wait_for(self._stopping.wait(), timeout=seconds) + except asyncio.TimeoutError: + pass + + async def _call_client(self, function, *args, **kwargs): + result = await asyncio.to_thread(function, *args, **kwargs) + self._last_server_contact = time.monotonic() + return result + + def _increment(self, name: str, amount: int = 1) -> None: + with self._metric_lock: + self._metrics[name] = self._metrics.get(name, 0) + amount + + def metrics(self) -> dict[str, int]: + with self._metric_lock: + return dict(self._metrics) + + def is_ready(self) -> bool: + if ( + self._stopping.is_set() + or not self._registered + or self._last_server_contact is None + ): + return False + return time.monotonic() - self._last_server_contact <= max( + float(self._lease_duration), 60.0 + ) diff --git a/sdk/python/failproofai_sdk/evaluator/source.py b/sdk/python/failproofai_sdk/evaluator/source.py new file mode 100644 index 00000000..dbdba2c9 --- /dev/null +++ b/sdk/python/failproofai_sdk/evaluator/source.py @@ -0,0 +1,684 @@ +"""Restricted deterministic expression compiler for server-authored evaluations. + +The managed worker never executes a module, statements, imports, or ambient +builtins from tenant-authored source. Definitions are single Python expressions +evaluated with a small constructor/helper surface and the immutable transcript +bound as ``session``. +""" + +from __future__ import annotations + +import ast +import builtins +import hashlib +import os +import pickle +import re +import select +import subprocess +import sys +import tempfile +import threading +import time +from collections.abc import Callable +from typing import Any + +try: + import resource as _resource +except ImportError: # pragma: no cover - non-POSIX + _resource = None # type: ignore[assignment] + +from failproofai_sdk.evaluator.authoring import ( + Assertion, + ConditionResult, + EvalResult, + Metric, + Score, +) + +MAX_CONDITION_SOURCE_BYTES = 16 * 1024 +MAX_EVALUATOR_SOURCE_BYTES = 128 * 1024 + +# Static defense-in-depth bounds applied at COMPILE time (see `_compile`). They +# reject the obvious authoring bombs early; they are NOT the primary defense — +# a runtime-computed size (`range(len(session.events) ** 40)`) slips past any +# static check, which is exactly why the fork sandbox below is the real bound. +MAX_AST_NODES = 5_000 +MAX_POW_EXPONENT = 64 + +# Hard ceilings for ONE sandboxed evaluation, enforced by the kernel in a +# fork+exec'd subprocess (see `_run_sandboxed`). RLIMIT_CPU + the parent's +# wall-clock kill both bound compute bombs; RLIMIT_AS is the memory backstop for a +# giant-int / huge-allocation bomb. +DEFAULT_SANDBOX_TIMEOUT_SECONDS = 30 +# The effective budget is CLAMPED to this ceiling regardless of the (server-set) +# per-definition timeout, so a large `timeout_seconds` can never remove the +# execution bound (SEC-001). Wall-clock and CPU are both capped here. +MAX_SANDBOX_TIMEOUT_SECONDS = 60 +# Per-sandbox address-space cap. A managed eval works over a transcript (<=25 MiB) +# and returns a small result, so this is generous; it also rejects an allocation +# bomb (`[0] * 200000000` is ~1.6 GiB > this) before it returns a valid result. +SANDBOX_MEMORY_BYTES = 512 * 1024 * 1024 # 512 MiB +# ...but a per-process cap alone does not bound the HOST: a worker with +# max_concurrency=32 could run 32 sandboxes at once. Cap the number of concurrent +# sandbox processes so the AGGREGATE (MAX_CONCURRENT_SANDBOXES * SANDBOX_MEMORY_BYTES, +# ~2 GiB) is bounded independent of the worker's claim concurrency; extra evals +# queue on the semaphore rather than pile up memory. +MAX_CONCURRENT_SANDBOXES = 4 +_SANDBOX_SLOTS = threading.Semaphore(MAX_CONCURRENT_SANDBOXES) +# The result crossing back is bounded on BOTH sides: the child refuses to serialize +# a result larger than this, and the parent stops reading (and kills the child) +# past it — so a permitted expression that builds a huge result +# (`EvalResult(metrics={str(x): 1 for x in range(100000)})`) cannot OOM the worker +# even though the child's RLIMIT_AS lets it construct one. A valid result (<=25 +# items, bounded fields) is far under this. +SANDBOX_MAX_RESULT_BYTES = 1 * 1024 * 1024 # 1 MiB + +# The sandbox child is scrubbed of the worker's environment. +# +# `subprocess.Popen` inherits `os.environ` by default, which on a worker means +# FAILPROOFAI_EVALUATOR_TOKEN — and on the FailproofAI-managed pod that token is +# the CROSS-TENANT credential the whole fleet authenticates with. The AST and +# empty-builtins restrictions already stop a managed expression from reading +# `os.environ`, so this is defence in depth rather than a fix for a live escape: +# it means a future gap in those restrictions cannot be escalated into credential +# theft. Only the variables the interpreter itself needs are forwarded — notably +# PYTHONPATH, without which the child cannot import the sandbox runner at all. +_SANDBOX_ENV_PASSTHROUGH = ( + "PATH", + "PYTHONPATH", + "PYTHONHOME", + "PYTHONDONTWRITEBYTECODE", + "PYTHONUNBUFFERED", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TMPDIR", + "SYSTEMROOT", # Windows: CPython fails to start without it +) + + +def _sandbox_env() -> dict[str, str]: + return { + name: os.environ[name] + for name in _SANDBOX_ENV_PASSTHROUGH + if os.environ.get(name) + } + + +def _clamp_budget(timeout_seconds: float | None) -> float: + """The wall-clock/CPU budget for one evaluation: a positive value no larger + than MAX_SANDBOX_TIMEOUT_SECONDS. Server-provided timeouts cannot exceed it.""" + requested = float(timeout_seconds or DEFAULT_SANDBOX_TIMEOUT_SECONDS) + if requested <= 0: + requested = DEFAULT_SANDBOX_TIMEOUT_SECONDS + return min(requested, float(MAX_SANDBOX_TIMEOUT_SECONDS)) + + +class EvaluationTimeout(Exception): + """A sandboxed evaluation exceeded its CPU/memory/wall-clock budget. + + Distinct from an eval that *returned* an error: the computation was forcibly + terminated because it could not be allowed to keep running. + """ + + +class EvaluationSandboxUnavailable(Exception): + """The killable-process sandbox could not be established. + + Raised instead of running server-authored source unsandboxed — if the sandbox + subprocess cannot be started, or the transcript cannot be serialized into it, + there is no way to bound or terminate the evaluation, so we fail closed + (SEC-001). + """ + + +def _install_limits(cpu_seconds: float, mem_bytes: int) -> None: + """Install hard CPU + address-space limits on the CURRENT process. + + Called by the sandbox subprocess on itself, right before it evaluates. + """ + if _resource is None: # pragma: no cover - non-POSIX + return + cpu = max(1, int(cpu_seconds)) + _resource.setrlimit(_resource.RLIMIT_CPU, (cpu, cpu)) + _resource.setrlimit(_resource.RLIMIT_AS, (mem_bytes, mem_bytes)) + + +def _run_sandboxed( + kind: str, + source: str, + session: Any, + *, + wall_timeout: float, + cpu_seconds: float, + mem_bytes: int, + eval_key: str | None = None, +) -> Any: + """Evaluate server-authored ``source`` against ``session`` in a fork+exec'd + subprocess that CANNOT outlive its budget or flood this process. + + A FRESH ``python -m ..._sandbox_runner`` process — never a fork of this + multi-threaded worker (forking one deadlocks the child on a lock some other + thread holds) — reads its input from a temp file, installs hard RLIMIT_CPU + + RLIMIT_AS on itself, evaluates, and writes a bounded result to stdout. This + parent reads stdout up to ``SANDBOX_MAX_RESULT_BYTES`` and no further, killing + the child on timeout OR oversize — so neither compute (``sum(range(10**20))``) + nor an oversized result (``metrics={str(x):1 for x in range(100000)}``) can + exhaust the worker. + """ + # SEC-001: without the stdlib ``resource`` module (e.g. Windows) the sandbox + # child cannot install RLIMIT_CPU / RLIMIT_AS on itself (``_install_limits`` + # no-ops), so a permitted expression could allocate unbounded memory before + # the parent's wall-clock kill lands. Refuse BEFORE spawning any child rather + # than run server-authored source without the advertised limits. + if _resource is None: # pragma: no cover - non-POSIX + raise EvaluationSandboxUnavailable( + "kernel resource limits (RLIMIT_CPU/RLIMIT_AS) are unavailable on this " + "platform; managed evaluation cannot be bounded, refusing to run" + ) + try: + session_wire = session.to_wire() + except AttributeError as error: + raise EvaluationSandboxUnavailable( + "sandboxed evaluation requires a serializable transcript" + ) from error + payload = pickle.dumps( + (kind, source, session_wire, cpu_seconds, mem_bytes, eval_key) + ) + # Input via a temp file, not stdin: the transcript can be large (up to the + # transcript ceiling) and feeding a big stdin while bounding stdout invites a + # pipe deadlock. The child reads the file; we only read its stdout. + handle, path = tempfile.mkstemp(prefix="fpai-sandbox-", suffix=".pkl") + try: + with os.fdopen(handle, "wb") as tmp: + tmp.write(payload) + chunks: list[bytes] = [] + total = 0 + timed_out = False + too_large = False + # A slot is held for the whole subprocess lifetime so no more than + # MAX_CONCURRENT_SANDBOXES run at once — bounding aggregate memory across + # concurrent sandboxes. But acquiring it must COUNT AGAINST the wall-clock + # budget: the runtime runs this in a thread and `asyncio.wait_for` only + # cancels the awaiter, so a thread that blocked here UNBOUNDED past its + # deadline would still go on to launch a sandbox after its run was already + # reported timed out — 28 such threads could queue behind 4 long sandboxes + # and starve the worker (conditions have no runtime-level wait at all). One + # deadline therefore covers BOTH the slot wait and execution: we acquire the + # slot with the remaining budget and, on failure, time out WITHOUT spawning. + deadline = time.monotonic() + wall_timeout + acquire_timeout = deadline - time.monotonic() + if acquire_timeout <= 0 or not _SANDBOX_SLOTS.acquire(timeout=acquire_timeout): + raise EvaluationTimeout("evaluation timed out waiting for a sandbox slot") + try: + if deadline - time.monotonic() <= 0: + # Slot acquired exactly at the deadline: a child launched now could + # only be killed immediately, so do not spawn one at all. + raise EvaluationTimeout("evaluation timed out waiting for a sandbox slot") + try: + proc = subprocess.Popen( # noqa: S603 - fixed argv, no shell + [sys.executable, "-m", "failproofai_sdk.evaluator._sandbox_runner", path], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=_sandbox_env(), + ) + except OSError as error: + raise EvaluationSandboxUnavailable( + f"could not start the evaluation sandbox: {error}" + ) from error + out_fd = proc.stdout.fileno() + try: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + timed_out = True + break + ready, _, _ = select.select([out_fd], [], [], remaining) + if not ready: + timed_out = True + break + chunk = os.read(out_fd, 65536) + if not chunk: + break + total += len(chunk) + if total > SANDBOX_MAX_RESULT_BYTES: + too_large = True + break + chunks.append(chunk) + finally: + proc.stdout.close() + if proc.poll() is None: + proc.kill() + proc.wait() + finally: + _SANDBOX_SLOTS.release() + finally: + try: + os.unlink(path) + except OSError: + pass + + if timed_out: + raise EvaluationTimeout("evaluation exceeded its wall-clock budget") + if too_large: + raise EvaluationTimeout("evaluation result exceeded the size limit") + data = b"".join(chunks) + if not data: + # Killed by RLIMIT_CPU/RLIMIT_AS (or otherwise died) before it could write. + raise EvaluationTimeout("evaluation was terminated before producing a result") + outcome = pickle.loads(data) + if outcome[0] == "ok": + return outcome[1] + # Preserve the child's original exception SEMANTICS: an eval's + # NameError/TypeError/ZeroDivisionError/... and the sandbox's own + # UnsafeEvaluatorSource must read the same as they did in-process. Reconstruct + # any builtin exception by name; anything else collapses to a generic error — + # still caught as a failed run upstream. + _, name, message = outcome + if name == UnsafeEvaluatorSource.__name__: + raise UnsafeEvaluatorSource(message) + builtin = getattr(builtins, name, None) + if isinstance(builtin, type) and issubclass(builtin, BaseException): + raise builtin(message) + raise RuntimeError(f"{name}: {message}") + +_ALLOWED_NODES = ( + ast.Expression, + ast.BoolOp, + ast.BinOp, + ast.UnaryOp, + ast.IfExp, + ast.Dict, + ast.Set, + ast.List, + ast.Tuple, + ast.ListComp, + ast.SetComp, + ast.DictComp, + # `ast.GeneratorExp` is intentionally NOT allowed: a bare generator object's + # default repr is ``, which leaks a live host + # heap address (an ASLR/memory-layout disclosure) the moment it is coerced to + # a string into any result field. List/set/dict comprehensions render as their + # data (`[...]`, `{...}`) and cover the same ground — wrap a generator in `[]`. + ast.comprehension, + ast.Compare, + ast.Call, + ast.FormattedValue, + ast.JoinedStr, + ast.Constant, + ast.Name, + ast.Load, + ast.Store, + ast.Attribute, + ast.Subscript, + ast.Slice, + ast.keyword, + ast.And, + ast.Or, + ast.Add, + ast.Sub, + ast.Mult, + ast.Div, + ast.FloorDiv, + ast.Mod, + ast.Pow, + ast.USub, + ast.UAdd, + ast.Not, + ast.Eq, + ast.NotEq, + ast.Lt, + ast.LtE, + ast.Gt, + ast.GtE, + ast.In, + ast.NotIn, + ast.Is, + ast.IsNot, +) + +_SAFE_GLOBALS = { + "__builtins__": {}, + "Assertion": Assertion, + "ConditionResult": ConditionResult, + "EvalResult": EvalResult, + "Metric": Metric, + "Score": Score, + "abs": abs, + "all": all, + "any": any, + "bool": bool, + "dict": dict, + # `enumerate` is intentionally excluded: an enumerate object's default repr is + # ``, leaking a live host heap address into any + # result field. Index-aware iteration can use `range(len(...))` instead. + "float": float, + "int": int, + "len": len, + "list": list, + "max": max, + "min": min, + "range": range, + "round": round, + "set": set, + "sorted": sorted, + "str": str, + "sum": sum, + "tuple": tuple, +} + + +# Attribute access is DEFAULT-DENY. A denylist is unwinnable here: dunder access +# is only one door. `str.format`/`format_map` traverse a format string's fields +# at the C level; `(x for x in [1]).gi_frame.f_globals` reaches the eval globals +# through generator/frame introspection; `str.mro()[-1]` reaches the `object` +# type — and NONE of `format`, `gi_frame`, `f_globals`, `co_names`, `mro`, ... +# start with an underscore, so the dunder guard never sees them. Rather than +# chase each introspection family, we allow ONLY the attribute names a real +# session evaluation needs: the transcript/event data surface plus a fixed set +# of pure string/collection data methods. Anything else — every current and +# future introspection attribute — is rejected. `format`/`format_map` are simply +# absent from this set, so the C-level format escape is closed too. +# Data attributes on the transcript surface — safe to READ as a value: each is a +# field of a frozen dataclass (SessionTranscript / TranscriptEvent, whose reprs are +# field-based and pointer-free) or a JSON scalar/container from an event payload. +_DATA_ATTRS = frozenset( + { + # SessionTranscript + TranscriptEvent data surface (see protocol.py). + "events", + "event_count", + "event_type", + "payload", + "id", + "ts", + "agent_id", + "environment", + "session_id", + "session_revision_id", + "assignment_id", + "started_at", + "ended_at", + "schema_version", + } +) + +# Method attributes — pure data methods that must be CALLED, never referenced as a +# bare value. A bound method's repr is `<... at 0x...>`, a live heap address; a bare +# reference (`payload.get` uncalled) is only ever useful for smuggling that address +# into a result field via `str()`, an f-string, or `%`-formatting — none of which a +# real evaluation needs. `_compile` requires each of these names to appear at a call +# site, which closes every text-coercion leak at its source: no reachable value can +# then carry a pointer repr, so the output-boundary scan is only defense in depth. +_METHOD_ATTRS = frozenset( + { + # SessionTranscript methods. + "events_of_type", + "count", + # dict data methods. + "get", + "keys", + "values", + "items", + # str / bytes pure data methods. + "lower", + "upper", + "strip", + "lstrip", + "rstrip", + "split", + "rsplit", + "splitlines", + "startswith", + "endswith", + "replace", + "find", + "rfind", + "index", + "join", + "title", + "capitalize", + "casefold", + "swapcase", + "isdigit", + "isalpha", + "isalnum", + "isspace", + "isnumeric", + "isdecimal", + "islower", + "isupper", + "istitle", + "zfill", + "ljust", + "rjust", + "center", + "partition", + "rpartition", + "removeprefix", + "removesuffix", + "encode", + "decode", + "hex", + # set data methods. + "union", + "intersection", + "difference", + "symmetric_difference", + "issubset", + "issuperset", + "isdisjoint", + } +) + +# The walk rejects any attribute outside this union, and additionally requires every +# name in `_METHOD_ATTRS` to appear only as the function of a call. +_ALLOWED_ATTRS = _DATA_ATTRS | _METHOD_ATTRS + + +def _fresh_globals() -> dict[str, Any]: + """A throwaway globals mapping for one eval call. + + Every evaluation gets its own copy — with a fresh empty ``__builtins__`` — + so that even if a future reach exposes the eval's globals (e.g. through a + frame object), a mutation cannot persist into another evaluation and poison + a shared, process-wide namespace. + """ + return {**_SAFE_GLOBALS, "__builtins__": {}} + + +class UnsafeEvaluatorSource(ValueError): + """Raised before any disallowed server-authored source can execute.""" + + +def source_checksum(condition_source: str | None, evaluator_source: str) -> str: + payload = (condition_source or "").encode("utf-8") + b"\0" + evaluator_source.encode( + "utf-8" + ) + return "sha256:" + hashlib.sha256(payload).hexdigest() + + +def _compile(source: str, *, field_name: str, maximum: int) -> Any: + if not isinstance(source, str) or not source.strip(): + raise UnsafeEvaluatorSource(f"{field_name} must not be empty") + if len(source.encode("utf-8")) > maximum: + raise UnsafeEvaluatorSource(f"{field_name} exceeds {maximum} bytes") + try: + tree = ast.parse(source, mode="eval") + except SyntaxError as error: + raise UnsafeEvaluatorSource(f"{field_name} must be one expression") from error + # An Attribute that is the function of a Call is a method invocation; any other + # Attribute naming a method (`_METHOD_ATTRS`) is a bare bound-method reference, + # whose only use is leaking the method's `<... at 0xADDR>` repr into a result. + called_method_nodes = { + node.func + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + node_count = 0 + for node in ast.walk(tree): + node_count += 1 + if node_count > MAX_AST_NODES: + raise UnsafeEvaluatorSource( + f"{field_name} is too large ({MAX_AST_NODES}-node ceiling)" + ) + if not isinstance(node, _ALLOWED_NODES): + raise UnsafeEvaluatorSource( + f"{field_name} contains disallowed syntax: {type(node).__name__}" + ) + # Defense in depth: a literal `10 ** 20` (or worse, `2 ** (10**8)`) builds a + # giant int — a memory bomb — at compile-time-visible size. Require Pow's + # exponent to be a small non-negative integer constant. Runtime-sized bombs + # still exist and are caught by the fork sandbox, not here. + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Pow): + exponent = node.right + if not ( + isinstance(exponent, ast.Constant) + and isinstance(exponent.value, int) + and not isinstance(exponent.value, bool) + and 0 <= exponent.value <= MAX_POW_EXPONENT + ): + raise UnsafeEvaluatorSource( + f"{field_name} exponent must be an integer constant " + f"in 0..{MAX_POW_EXPONENT}" + ) + if isinstance(node, ast.Attribute): + if node.attr.startswith("_"): + raise UnsafeEvaluatorSource( + f"{field_name} may not access private or dunder attributes" + ) + if node.attr not in _ALLOWED_ATTRS: + raise UnsafeEvaluatorSource( + f"{field_name} may not access attribute '{node.attr}'" + ) + if node.attr in _METHOD_ATTRS and node not in called_method_nodes: + raise UnsafeEvaluatorSource( + f"{field_name} may reference method '{node.attr}' only to call it; " + "a bare bound method leaks a heap address when stringified" + ) + if isinstance(node, ast.Name) and node.id.startswith("_"): + raise UnsafeEvaluatorSource(f"{field_name} may not access private names") + return compile(tree, f"<{field_name}>", "eval", dont_inherit=True, optimize=2) + + +# CPython's default object repr — `<... at 0x7f...>` — embeds a live heap address +# (an ASLR/memory-layout disclosure). The PRIMARY defense is at compile time: a bound +# method (the only reachable object with such a repr — the result and transcript types +# are all frozen, pointer-free dataclasses) can no longer be referenced as a value +# (`_METHOD_ATTRS` must be called), so no reachable value carries a pointer repr to +# begin with. This output-boundary scan is DEFENSE IN DEPTH. It matches the "... at +# 0xADDR" tail every default object repr shares — which survives even a reshaped +# wrapper such as `str(x).replace("<","")`, since stripping the leading `<` leaves +# the " at 0x..." tail intact. A bare `0x`+hex run is deliberately NOT matched: it +# false-rejects legitimate result text (a hex colour like `0xFFFFFF`, a git-style +# digest, or an address the agent itself logged and the eval quotes), marking a +# correct evaluation as failed for embedding an ordinary hex literal. +_OBJECT_REPR = re.compile(r" at 0x[0-9a-fA-F]+") + + +def _forbid_object_reprs(field_name: str, value: Any) -> Any: + if _OBJECT_REPR.search(repr(value)): + raise UnsafeEvaluatorSource( + f"{field_name} result may not embed a runtime object repr" + ) + return value + + +def _raw_eval(source: str, kind: str) -> Callable[[Any], Any]: + """Compile server-authored source and return a function that evaluates it and + validates the result. + + Runs INSIDE the sandbox subprocess (see `_sandbox_runner`) — there is no + isolation here. `compile_condition`/`compile_evaluator` have already validated + the AST in the parent; this recompiles as defense in depth so a subprocess + can never eval source the parent has not vetted. + """ + if kind == "condition": + code = _compile( + source, field_name="condition_source", maximum=MAX_CONDITION_SOURCE_BYTES + ) + + def run(session: Any) -> Any: + # `session` goes in the (fresh, per-call) GLOBALS, not locals: on + # CPython 3.10 a list/set/dict comprehension resolves a free name like + # `session` from globals, so an allowed source such as + # `all([session.event_count > 0 for _ in range(1)])` raises NameError + # if `session` is only a local. Globals stay fresh per call for + # isolation (see `_fresh_globals`); locals are empty. + value = eval(code, {**_fresh_globals(), "session": session}, {}) # noqa: S307 + if not isinstance(value, (bool, ConditionResult)): + raise TypeError("condition_source must return bool or ConditionResult") + return _forbid_object_reprs("condition_source", value) + + else: + code = _compile( + source, field_name="evaluator_source", maximum=MAX_EVALUATOR_SOURCE_BYTES + ) + + def run(session: Any) -> Any: + # `session` goes in the (fresh, per-call) GLOBALS, not locals: on + # CPython 3.10 a list/set/dict comprehension resolves a free name like + # `session` from globals, so an allowed source such as + # `all([session.event_count > 0 for _ in range(1)])` raises NameError + # if `session` is only a local. Globals stay fresh per call for + # isolation (see `_fresh_globals`); locals are empty. + value = eval(code, {**_fresh_globals(), "session": session}, {}) # noqa: S307 + if not isinstance(value, EvalResult): + raise TypeError("evaluator_source must return EvalResult") + return _forbid_object_reprs("evaluator_source", value) + + return run + + +def compile_condition( + source: str, + *, + timeout_seconds: float | None = DEFAULT_SANDBOX_TIMEOUT_SECONDS, +) -> Callable[[Any], bool | ConditionResult]: + # Validate the AST in THIS (parent) process so unsafe/malformed source is + # rejected up front, before any subprocess is spawned. + _compile(source, field_name="condition_source", maximum=MAX_CONDITION_SOURCE_BYTES) + budget = _clamp_budget(timeout_seconds) + + def condition(session: Any) -> bool | ConditionResult: + # Managed conditions are sandboxed like evaluators — `sum(range(10**10)) > 0` + # in a condition would otherwise block the worker with NO timeout at all. + return _run_sandboxed( + "condition", + source, + session, + wall_timeout=budget, + cpu_seconds=budget, + mem_bytes=SANDBOX_MEMORY_BYTES, + ) + + return condition + + +def compile_evaluator( + source: str, + *, + timeout_seconds: float | None = DEFAULT_SANDBOX_TIMEOUT_SECONDS, + eval_key: str | None = None, +) -> Callable[[Any], EvalResult]: + _compile(source, field_name="evaluator_source", maximum=MAX_EVALUATOR_SOURCE_BYTES) + budget = _clamp_budget(timeout_seconds) + + def evaluate(session: Any) -> EvalResult: + # The kernel-enforced boundary: this runs in a fork+exec'd subprocess with + # hard CPU/memory/wall-clock limits and a bounded result, killed if it + # exceeds them, so a server-authored compute or result bomb cannot exhaust + # the worker (SEC-001). `eval_key` lets the child validate the result's + # 25-item limit before it crosses back. + return _run_sandboxed( + "evaluator", + source, + session, + wall_timeout=budget, + cpu_seconds=budget, + mem_bytes=SANDBOX_MEMORY_BYTES, + eval_key=eval_key, + ) + + return evaluate diff --git a/sdk/python/tests/fixtures/evaluator_v2/README.md b/sdk/python/tests/fixtures/evaluator_v2/README.md new file mode 100644 index 00000000..e57c93bd --- /dev/null +++ b/sdk/python/tests/fixtures/evaluator_v2/README.md @@ -0,0 +1,28 @@ +# Evaluator v2 contract fixtures + +`contract.json` is the Checkpoint 0 wire contract shared by the Rust server and +the zero-dependency Python SDK. The matching copy lives at +`server/tests/fixtures/evaluator_v2/contract.json` in the `agenteye` repository. +Change both copies together. + +Contract rules: + +- The only accepted protocol major is the exact string `"2"`. Unsupported + majors return `426 unsupported_protocol_version`. +- Unknown JSON fields are ignored so either side may add optional fields within + major version 2. Removing, renaming, or changing the meaning of a field needs + a new major version. +- Worker payloads never carry authoritative tenant or evaluator-instance + identity. The server derives those from the credential and leased record. +- `lease_generation` is the fencing token. `409 lease_lost` is terminal for the + affected local execution; the SDK must stop heartbeating or submitting it. +- `submission_id` is an idempotency key. Replaying identical content succeeds; + reusing it for different content returns `409 submission_conflict`. +- Transcript overflow is terminal in v2 (`413 transcript_too_large`); the server + never silently truncates the evaluated input. +- Only errors marked `retryable` may be retried automatically. HTTP method alone + is not enough to decide whether a protocol operation is safe to replay. + +The timing and payload limits in the fixture are normative defaults. A register +response may lower the worker's effective concurrency, heartbeat interval, or +lease duration, but may not raise a client-side payload bound. diff --git a/sdk/python/tests/fixtures/evaluator_v2/contract.json b/sdk/python/tests/fixtures/evaluator_v2/contract.json new file mode 100644 index 00000000..c467f930 --- /dev/null +++ b/sdk/python/tests/fixtures/evaluator_v2/contract.json @@ -0,0 +1,252 @@ +{ + "fixture_revision": "evaluator-v2-2026-08-28.3", + "protocol": { + "supported_major_versions": ["2"], + "transcript_schema_version": "2", + "result_schema_version": "2" + }, + "http": { + "register": "/v1/evaluator/workers/register", + "claim": "/v1/evaluator/assignments/claim", + "transcript": "/v1/evaluator/assignments/{assignment_id}/transcript", + "definitions": "/v1/evaluator/assignments/{assignment_id}/definitions", + "plan": "/v1/evaluator/assignments/{assignment_id}/plan", + "heartbeat": "/v1/evaluator/runs/heartbeat", + "result": "/v1/evaluator/runs/{evaluation_run_id}/result", + "worker_id_header": "X-FailproofAI-Worker-Id", + "lease_generation_header": "X-FailproofAI-Lease-Generation" + }, + "timing": { + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "poll_interval_seconds": 10, + "max_attempts": 5 + }, + "limits": { + "max_catalog_definitions": 100, + "max_claim_capacity": 32, + "max_transcript_bytes": 26214400, + "max_results_per_run": 25, + "max_eval_key_bytes": 128, + "max_display_name_bytes": 128, + "max_version_bytes": 128, + "max_worker_id_bytes": 128, + "max_label_bytes": 64, + "max_labels_per_result": 20, + "max_summary_bytes": 4096, + "max_reasoning_bytes": 16384, + "max_unit_bytes": 64, + "max_display_value_bytes": 256, + "max_description_bytes": 1000, + "max_error_code_bytes": 64, + "max_error_message_bytes": 4096 + }, + "errors": { + "invalid_credentials": {"http_status": 401, "retryable": false}, + "instance_disabled": {"http_status": 403, "retryable": false}, + "insufficient_permissions": {"http_status": 403, "retryable": false}, + "assignment_not_found": {"http_status": 404, "retryable": false}, + "run_not_found": {"http_status": 404, "retryable": false}, + "catalog_mismatch": {"http_status": 409, "retryable": false}, + "lease_lost": {"http_status": 409, "retryable": false}, + "plan_conflict": {"http_status": 409, "retryable": false}, + "submission_conflict": {"http_status": 409, "retryable": false}, + "retry_budget_exhausted": {"http_status": 409, "retryable": false}, + "transcript_too_large": {"http_status": 413, "retryable": false}, + "invalid_request": {"http_status": 422, "retryable": false}, + "invalid_catalog": {"http_status": 422, "retryable": false}, + "incomplete_plan": {"http_status": 422, "retryable": false}, + "unsupported_protocol_version": {"http_status": 426, "retryable": false}, + "internal_error": {"http_status": 500, "retryable": true} + }, + "samples": { + "register_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "sdk_version": "0.0.1b2", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "max_concurrency": 4, + "definitions": [ + { + "eval_key": "tool_efficiency", + "display_name": "Tool efficiency", + "eval_version": "1.2.0", + "result_kind": "score", + "labels": ["tools", "deterministic"] + } + ] + }, + "register_response": { + "protocol_version": "2", + "evaluator_instance_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23101", + "evaluator_kind": "customer", + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "poll_interval_seconds": 10, + "claim_limit": 4, + "disabled_definitions": [] + }, + "claim_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "capacity": 2 + }, + "claim_response": { + "protocol_version": "2", + "assignments": [ + { + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "lease_generation": 3, + "lease_expires_at": "2026-08-28T12:02:00.000000Z", + "session_id": "session-42", + "session_revision_id": "evt-agent-end-42", + "agent_id": "support-agent", + "environment": "production", + "trigger_reason": "agent_end", + "event_count": 42, + "transcript_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/transcript", + "definitions_url": "/v1/evaluator/assignments/018f47a8-7c1d-7e21-a22a-79f7a4d23102/definitions" + } + ] + }, + "definitions_response": { + "protocol_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "catalog_revision": "sha256:b4e2c077aa9f91b5de1f3184ebf96811bce07eec00c7f417896ab269c67c88fb", + "definitions": [ + { + "eval_key": "tool_efficiency", + "display_name": "Tool efficiency", + "eval_version": "1.2.0", + "result_kind": "score", + "labels": ["tools", "deterministic"], + "execution_mode": "python", + "condition_source": null, + "source_checksum": "sha256:da6cf174ea9199dd8412af4abebd40bd27dea482f738cb8c28523076472501ea", + "timeout_seconds": 30.0 + } + ] + }, + "transcript_response": { + "schema_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "session_id": "session-42", + "session_revision_id": "evt-agent-end-42", + "agent_id": "support-agent", + "environment": "production", + "started_at": "2026-08-28T11:58:00.000000Z", + "ended_at": "2026-08-28T12:00:00.000000Z", + "event_count": 2, + "events": [ + { + "id": "evt-tool-1", + "ts": "2026-08-28T11:59:00.000000Z", + "event_type": "tool_use", + "payload": {"tool_name": "search"} + }, + { + "id": "evt-end-1", + "ts": "2026-08-28T12:00:00.000000Z", + "event_type": "agent_end", + "payload": {"summary": "Done"} + } + ] + }, + "plan_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "selected": [ + {"eval_key": "tool_efficiency", "eval_version": "1.2.0"} + ], + "skipped": [ + { + "eval_key": "answer_groundedness", + "eval_version": "2.1.0", + "reason_code": "no_retrieval_events" + } + ] + }, + "plan_response": { + "protocol_version": "2", + "assignment_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23102", + "assignment_status": "planned", + "idempotent_replay": false, + "runs": [ + { + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "eval_key": "tool_efficiency", + "eval_version": "1.2.0", + "execution_mode": "python", + "evaluator_source": "EvalResult(score=Score(1.0))", + "source_checksum": "sha256:da6cf174ea9199dd8412af4abebd40bd27dea482f738cb8c28523076472501ea", + "timeout_seconds": 30.0 + } + ] + }, + "heartbeat_request": { + "protocol_version": "2", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "runs": [ + { + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "state": "running", + "progress": 0.5 + } + ] + }, + "heartbeat_response": { + "protocol_version": "2", + "lease_expires_at": "2026-08-28T12:02:30.000000Z", + "accepted_run_ids": ["018f47a8-7c1d-7e21-a22a-79f7a4d23103"] + }, + "result_request": { + "protocol_version": "2", + "result_schema_version": "2", + "submission_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23104", + "worker_id": "pod-7f8c9", + "lease_generation": 3, + "status": "succeeded", + "started_at": "2026-08-28T12:00:10.000000Z", + "finished_at": "2026-08-28T12:00:10.812000Z", + "duration_ms": 812, + "summary": "Used a compact tool set without retries.", + "results": [ + { + "result_key": "tool_efficiency", + "result_kind": "score", + "numeric_value": 0.92, + "bool_value": true, + "text_value": null, + "unit": "ratio", + "display_value": "92%", + "description": "Distinct tools divided by total tool calls", + "reasoning": "3 distinct tools across 3 calls", + "labels": ["tools", "deterministic"] + } + ], + "error_code": null, + "error_message": null + }, + "result_response": { + "protocol_version": "2", + "evaluation_run_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23103", + "submission_id": "018f47a8-7c1d-7e21-a22a-79f7a4d23104", + "status": "committed", + "idempotent_replay": false, + "result_count": 1, + "result_checksum": "sha256:8cbd34f2d95d" + }, + "error_response": { + "protocol_version": "2", + "error": { + "code": "lease_lost", + "message": "The assignment lease is no longer owned by this worker.", + "retryable": false, + "request_id": "req-018f47a8" + } + } + } +} diff --git a/sdk/python/tests/test_evaluator_authoring.py b/sdk/python/tests/test_evaluator_authoring.py new file mode 100644 index 00000000..578b1220 --- /dev/null +++ b/sdk/python/tests/test_evaluator_authoring.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import asyncio +import math + +import pytest + +from failproofai_sdk.evaluator import ( + Assertion, + EvalResult, + Evaluator, + Metric, + ResultKind, + Score, +) + + +def test_catalog_is_stable_across_registration_order(): + first = Evaluator(name="acme", version="2026.08.1") + second = Evaluator(name="acme", version="2026.08.1") + + @first.eval("zeta_check", version="1", labels=["z", "a"]) + def first_zeta(session): + return EvalResult(score=Score(1)) + + @first.eval("alpha_check", version="1") + def first_alpha(session): + return EvalResult(score=Score(1)) + + @second.eval("alpha_check", version="1") + def second_alpha(session): + return EvalResult(score=Score(1)) + + @second.eval("zeta_check", version="1", labels=["a", "z"]) + def second_zeta(session): + return EvalResult(score=Score(1)) + + assert first.catalog_revision == second.catalog_revision + assert [item.eval_key for item in first.catalog()] == ["alpha_check", "zeta_check"] + + +def test_duplicate_eval_keys_are_rejected_even_when_versions_differ(): + evaluator = Evaluator(name="acme", version="1") + + @evaluator.eval("quality", version="1") + def quality_v1(session): + return EvalResult(score=Score(1)) + + with pytest.raises(ValueError, match="duplicate eval key"): + + @evaluator.eval("quality", version="2") + def quality_v2(session): + return EvalResult(score=Score(1)) + + +@pytest.mark.parametrize("value", [-0.01, 1.01, math.nan, math.inf]) +def test_scores_are_finite_ratios(value): + with pytest.raises(ValueError): + Score(value) + + +def test_result_presentation_fields_are_bounded_before_networking(): + with pytest.raises(ValueError, match="unit is 65 bytes"): + Metric(1, unit="u" * 65) + with pytest.raises(ValueError, match="display value is 257 bytes"): + Score(1, display_value="x" * 257) + with pytest.raises(ValueError, match="description is 1001 bytes"): + Assertion(True, description="x" * 1001) + + +def test_eval_result_expands_to_typed_long_form_rows(): + result = EvalResult( + score=Score(0.75, passed=True, unit="ratio"), + metrics={"call_count": Metric(4, unit="calls")}, + assertions={"had_output": Assertion(True)}, + reasoning="Three useful calls out of four.", + labels=("tools",), + ) + + items = result.result_items("tool_efficiency") + assert [item.result_kind for item in items] == [ + ResultKind.SCORE, + ResultKind.METRIC, + ResultKind.ASSERTION, + ] + assert items[0].reasoning == "Three useful calls out of four." + assert items[1].numeric_value == 4 + assert items[2].bool_value is True + + +def test_empty_eval_result_is_rejected_when_serialized(): + with pytest.raises(ValueError, match="must contain"): + EvalResult().result_items("quality") + + +def test_result_keys_must_be_unique_across_kinds(): + result = EvalResult(score=Score(1), metrics={"quality": 1}) + with pytest.raises(ValueError, match="result keys must be unique"): + result.result_items("quality") + + +def test_sync_and_async_functions_share_one_call_path(): + async def async_eval(session): + return EvalResult(score=Score(1)) + + def sync_eval(session): + return EvalResult(score=Score(0.5)) + + async def exercise(): + sync_result = await Evaluator.call(sync_eval, None) + async_result = await Evaluator.call(async_eval, None) + return sync_result, async_result + + sync_result, async_result = asyncio.run(exercise()) + assert sync_result.score.value == 0.5 + assert async_result.score.value == 1 + + +def test_keys_are_machine_safe_and_versions_are_explicit(): + evaluator = Evaluator(name="acme", version="1") + with pytest.raises(ValueError, match="must match"): + evaluator.eval("Not Safe", version="1") + with pytest.raises(ValueError, match="must not be empty"): + evaluator.eval("safe", version="") diff --git a/sdk/python/tests/test_evaluator_client.py b/sdk/python/tests/test_evaluator_client.py new file mode 100644 index 00000000..d74c60dc --- /dev/null +++ b/sdk/python/tests/test_evaluator_client.py @@ -0,0 +1,253 @@ +from __future__ import annotations + +import io +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.error import HTTPError, URLError + +import pytest + +from failproofai_sdk.evaluator import ( + Assignment, + ClaimRequest, + EvaluatorAPIError, + EvaluatorClient, + ResultRequest, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _samples(): + return json.loads(FIXTURE.read_text(encoding="utf-8"))["samples"] + + +class Response: + def __init__(self, body): + self.body = json.dumps(body).encode() + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def read(self, amount): + return self.body[:amount] + + +class RawResponse(Response): + def __init__(self, body: bytes): + self.body = body + + +def test_claim_sends_bearer_auth_and_does_not_retry(): + calls = [] + + def opener(request, timeout): + calls.append((request, timeout)) + raise URLError("offline") + + client = EvaluatorClient( + base_url="https://cloud.example/api/", + credential="secret", + opener=opener, + sleeper=lambda _: None, + ) + with pytest.raises(EvaluatorAPIError, match="transport_error"): + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + + assert len(calls) == 1 + request, timeout = calls[0] + assert request.full_url == "https://cloud.example/v1/evaluator/assignments/claim" + assert request.get_header("Authorization") == "Bearer secret" + assert timeout == 30 + + +def test_idempotent_result_submission_retries_transport_failure(): + samples = _samples() + calls = 0 + + def opener(request, timeout): + nonlocal calls + calls += 1 + if calls == 1: + raise URLError("reset") + return Response(samples["result_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=opener, + sleeper=lambda _: None, + ) + response = client.submit_result( + samples["result_response"]["evaluation_run_id"], + ResultRequest.from_wire(samples["result_request"]), + ) + assert response.status == "committed" + assert calls == 2 + + +def test_transcript_url_cannot_exfiltrate_the_worker_credential(): + sample = _samples()["claim_response"]["assignments"][0] + assignment = Assignment.from_wire( + {**sample, "transcript_url": "https://evil.test/read"} + ) + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=lambda *_args, **_kwargs: pytest.fail("network must not be reached"), + ) + with pytest.raises(EvaluatorAPIError, match="outside the configured API origin"): + client.transcript(assignment, worker_id="worker") + + +def test_machine_error_envelope_controls_retryability(): + body = json.dumps(_samples()["error_response"]).encode() + + def opener(request, timeout): + raise HTTPError(request.full_url, 409, "Conflict", {}, io.BytesIO(body)) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + assert caught.value.code == "lease_lost" + assert caught.value.status == 409 + assert caught.value.retryable is False + assert caught.value.request_id == "req-018f47a8" + + +@pytest.mark.parametrize( + ("body", "code"), + [ + (b"not-json", "invalid_response"), + (b"[]", "invalid_response"), + ( + b"{" + b'"padding":"' + b"x" * (2 * 1024 * 1024) + b'"}', + "response_too_large", + ), + ], +) +def test_malformed_or_oversized_server_responses_fail_closed(body, code): + client = EvaluatorClient( + base_url="https://cloud.example/", + credential="secret", + opener=lambda request, timeout: RawResponse(body), + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 20)) + assert caught.value.code == code + assert caught.value.retryable is False + + +def test_transcript_identity_is_sent_as_fencing_headers(): + samples = _samples() + assignment = Assignment.from_wire(samples["claim_response"]["assignments"][0]) + captured = None + + def opener(request, timeout): + nonlocal captured + captured = request + return Response(samples["transcript_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + client.transcript(assignment, worker_id="worker-7") + assert captured.get_header("X-failproofai-worker-id") == "worker-7" + assert captured.get_header("X-failproofai-lease-generation") == "3" + + +def test_definitions_use_the_server_supplied_path_and_fencing_headers(): + samples = _samples() + assignment = Assignment.from_wire(samples["claim_response"]["assignments"][0]) + captured = None + + def opener(request, timeout): + nonlocal captured + captured = request + return Response(samples["definitions_response"]) + + client = EvaluatorClient( + base_url="https://cloud.example/", credential="secret", opener=opener + ) + response = client.definitions(assignment, worker_id="worker-7") + + assert response.assignment_id == assignment.assignment_id + assert response.definitions[0].execution_mode.value == "python" + assert captured.full_url.endswith(assignment.definitions_url) + assert captured.get_header("X-failproofai-worker-id") == "worker-7" + assert captured.get_header("X-failproofai-lease-generation") == "3" + + +def test_constructor_rejects_unsafe_or_incomplete_configuration(): + with pytest.raises(ValueError, match="absolute"): + EvaluatorClient(base_url="localhost:8080", credential="secret") + with pytest.raises(ValueError, match="credential"): + EvaluatorClient(base_url="https://cloud.example", credential="") + with pytest.raises(ValueError, match="control characters"): + EvaluatorClient(base_url="https://cloud.example", credential="secret\nleak") + + +def test_private_cluster_http_requires_an_explicit_opt_in(): + with pytest.raises(ValueError, match="must use https"): + EvaluatorClient(base_url="http://server:8080", credential="secret") + EvaluatorClient( + base_url="http://server:8080", + credential="secret", + allow_insecure_http=True, + ) + + +def test_protocol_redirect_does_not_forward_the_bearer_credential(): + exfiltration_attempts = [] + + class Sink(BaseHTTPRequestHandler): + def do_POST(self): + exfiltration_attempts.append(self.headers.get("Authorization")) + self.send_response(200) + self.end_headers() + + def log_message(self, format, *args): + return + + sink = ThreadingHTTPServer(("127.0.0.1", 0), Sink) + sink_thread = threading.Thread(target=sink.serve_forever, daemon=True) + sink_thread.start() + + location = f"http://127.0.0.1:{sink.server_address[1]}/steal" + + class Redirector(BaseHTTPRequestHandler): + def do_POST(self): + self.send_response(307) + self.send_header("Location", location) + self.end_headers() + + def log_message(self, format, *args): + return + + redirector = ThreadingHTTPServer(("127.0.0.1", 0), Redirector) + redirector_thread = threading.Thread(target=redirector.serve_forever, daemon=True) + redirector_thread.start() + try: + client = EvaluatorClient( + base_url=f"http://127.0.0.1:{redirector.server_address[1]}", + credential="must-not-leak", + max_retries=0, + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.claim(ClaimRequest("worker", "sha256:x", 1, 0)) + assert caught.value.status == 307 + assert exfiltration_attempts == [] + finally: + redirector.shutdown() + redirector.server_close() + redirector_thread.join(timeout=5) + sink.shutdown() + sink.server_close() + sink_thread.join(timeout=5) diff --git a/sdk/python/tests/test_evaluator_example.py b/sdk/python/tests/test_evaluator_example.py new file mode 100644 index 00000000..e85a545e --- /dev/null +++ b/sdk/python/tests/test_evaluator_example.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ConditionResult + + +def _example_module(): + path = Path(__file__).parents[1] / "examples" / "evaluator_worker.py" + spec = importlib.util.spec_from_file_location("evaluator_worker_example", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def test_example_registers_deterministic_and_async_evals(monkeypatch): + monkeypatch.delenv("EXAMPLE_JUDGE_URL", raising=False) + module = _example_module() + definitions = {item.eval_key: item for item in module.app.definitions} + assert set(definitions) == {"answer_relevance", "tool_efficiency"} + assert definitions["answer_relevance"].eval_version == "judge-api-v1" + + skipped = definitions["answer_relevance"].condition(None) + assert skipped == ConditionResult(False, "judge_not_configured") + + +def test_example_rejects_non_http_judge_urls(monkeypatch): + module = _example_module() + monkeypatch.setenv("EXAMPLE_JUDGE_URL", "file:///etc/passwd") + with pytest.raises(ValueError, match="absolute http"): + module._call_judge("question", "answer") diff --git a/sdk/python/tests/test_evaluator_http_e2e.py b/sdk/python/tests/test_evaluator_http_e2e.py new file mode 100644 index 00000000..42591bb1 --- /dev/null +++ b/sdk/python/tests/test_evaluator_http_e2e.py @@ -0,0 +1,636 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import socket +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import urlsplit +from uuid import UUID, uuid5 + +import pytest + +from failproofai_sdk.evaluator import ( + Assignment, + ClaimRequest, + EvalResult, + EvalSelection, + Evaluator, + EvaluatorAPIError, + EvaluatorClient, + HeartbeatRequest, + HeartbeatRun, + PlanRequest, + ResultItem, + ResultKind, + ResultRequest, + Score, + TerminalRunStatus, + WorkerConfig, + WorkerRuntime, +) + +_NAMESPACE = UUID("4d592d9c-aed4-4f07-9b2d-e14963399df6") + + +class ProtocolState: + def __init__(self) -> None: + self.lock = threading.Lock() + self.base_url = "" + self.instances = { + "customer-a-token": ("instance-customer-a", "customer", "org-a"), + "customer-b-token": ("instance-customer-b", "customer", "org-b"), + "managed-token": ("instance-managed", "managed", None), + } + self.registrations: dict[str, dict] = {} + self.assignments: dict[str, dict] = {} + self.runs: dict[str, dict] = {} + self.result_attempts = 0 + self.result_commits = 0 + self.last_result_body: dict | None = None + self.drop_first_result_response = False + + def add_assignment(self, name: str, *, token: str, org: str) -> str: + assignment_id = str(uuid5(_NAMESPACE, name)) + self.assignments[assignment_id] = { + "token": token, + "org": org, + "status": "available", + "worker_id": None, + "lease_generation": 0, + "expired": False, + "session_id": f"session-{name}", + "session_revision_id": f"revision-{name}", + } + return assignment_id + + def expire(self, assignment_id: str) -> None: + with self.lock: + self.assignments[assignment_id]["expired"] = True + + +class ProtocolServer: + def __init__(self, state: ProtocolState) -> None: + self.state = state + handler = _handler_for(state) + self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + self.server.daemon_threads = True + state.base_url = f"http://127.0.0.1:{self.server.server_address[1]}" + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + + def __enter__(self) -> ProtocolServer: # noqa: PYI034 - Python 3.10 lacks Self + self.thread.start() + return self + + def __exit__(self, *_args) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + + +def _handler_for(state: ProtocolState): + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: + token = self._token() + if token not in state.instances: + self._error(401, "invalid_credentials", False) + return + body = self._body() + path = urlsplit(self.path).path + if path == "/v1/evaluator/workers/register": + state.registrations[token] = body + instance_id, kind, _org = state.instances[token] + self._json( + 200, + { + "protocol_version": "2", + "evaluator_instance_id": instance_id, + "evaluator_kind": kind, + "heartbeat_interval_seconds": 30, + "lease_duration_seconds": 120, + "poll_interval_seconds": 10, + "claim_limit": body["max_concurrency"], + "disabled_definitions": [], + }, + ) + return + if path == "/v1/evaluator/assignments/claim": + self._claim(token, body) + return + if path.endswith("/plan"): + self._plan(token, path.split("/")[-2], body) + return + if path == "/v1/evaluator/runs/heartbeat": + self._heartbeat(token, body) + return + if path.endswith("/result"): + self._result(token, path.split("/")[-2], body) + return + self._error(404, "assignment_not_found", False) + + def do_GET(self) -> None: + token = self._token() + if token not in state.instances: + self._error(401, "invalid_credentials", False) + return + path = urlsplit(self.path).path + if path.endswith("/transcript"): + self._transcript(token, path.split("/")[-2]) + return + self._error(404, "assignment_not_found", False) + + def _claim(self, token: str, body: dict) -> None: + claimed = [] + with state.lock: + for assignment_id, item in state.assignments.items(): + if len(claimed) >= body["capacity"]: + break + if item["token"] != token: + continue + if item["status"] in {"leased", "planned"} and not item["expired"]: + continue + if item["status"] not in {"available", "leased", "planned"}: + continue + item["status"] = "leased" + item["expired"] = False + item["worker_id"] = body["worker_id"] + item["lease_generation"] += 1 + claimed.append(self._assignment_wire(assignment_id, item)) + self._json(200, {"protocol_version": "2", "assignments": claimed}) + + def _transcript(self, token: str, assignment_id: str) -> None: + item = self._leased_assignment( + token, + assignment_id, + self.headers.get("X-FailproofAI-Worker-Id"), + self.headers.get("X-FailproofAI-Lease-Generation"), + ) + if item is None: + return + self._json( + 200, + { + "schema_version": "2", + "assignment_id": assignment_id, + "session_id": item["session_id"], + "session_revision_id": item["session_revision_id"], + "agent_id": "agent-e2e", + "environment": "test", + "started_at": "2026-08-28T12:00:00.000000Z", + "ended_at": "2026-08-28T12:00:01.000000Z", + "event_count": 1, + "events": [ + { + "id": "event-1", + "ts": "2026-08-28T12:00:00.500000Z", + "event_type": "model_response", + "payload": {"content": "done"}, + } + ], + }, + ) + + def _plan(self, token: str, assignment_id: str, body: dict) -> None: + item = self._leased_assignment( + token, assignment_id, body["worker_id"], body["lease_generation"] + ) + if item is None: + return + runs = [] + with state.lock: + for selected in body["selected"]: + run_id = str( + uuid5( + _NAMESPACE, + f"{assignment_id}:{selected['eval_key']}:{selected['eval_version']}", + ) + ) + run = state.runs.setdefault( + run_id, + { + "token": token, + "assignment_id": assignment_id, + "worker_id": body["worker_id"], + "lease_generation": body["lease_generation"], + "submission_id": None, + "checksum": None, + }, + ) + if run["submission_id"] is None: + run["worker_id"] = body["worker_id"] + run["lease_generation"] = body["lease_generation"] + runs.append( + { + "evaluation_run_id": run_id, + "execution_mode": "local", + **selected, + } + ) + item["status"] = "planned" if runs else "skipped" + self._json( + 200, + { + "protocol_version": "2", + "assignment_id": assignment_id, + "assignment_status": item["status"], + "runs": runs, + }, + ) + + def _heartbeat(self, token: str, body: dict) -> None: + accepted = [] + with state.lock: + for requested in body["runs"]: + run = state.runs.get(requested["evaluation_run_id"]) + assignment = ( + state.assignments.get(run["assignment_id"]) + if run is not None + else None + ) + if ( + run is not None + and assignment is not None + and run["token"] == token + and run["worker_id"] == body["worker_id"] + and run["lease_generation"] == body["lease_generation"] + and assignment["worker_id"] == body["worker_id"] + and assignment["lease_generation"] == body["lease_generation"] + and not assignment["expired"] + ): + accepted.append(requested["evaluation_run_id"]) + if not accepted: + self._error(409, "lease_lost", False) + return + self._json( + 200, + { + "protocol_version": "2", + "lease_expires_at": "2026-08-28T12:02:30.000000Z", + "accepted_run_ids": accepted, + }, + ) + + def _result(self, token: str, run_id: str, body: dict) -> None: + with state.lock: + run = state.runs.get(run_id) + if run is None or run["token"] != token: + self._error(404, "run_not_found", False) + return + if ( + run["worker_id"] != body["worker_id"] + or run["lease_generation"] != body["lease_generation"] + ): + self._error(409, "lease_lost", False) + return + checksum = hashlib.sha256( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + if run["submission_id"] is not None: + if ( + run["submission_id"] != body["submission_id"] + or run["checksum"] != checksum + ): + self._error(409, "submission_conflict", False) + return + replay = True + else: + run["submission_id"] = body["submission_id"] + run["checksum"] = checksum + state.result_commits += 1 + state.last_result_body = body + replay = False + state.result_attempts += 1 + drop = state.drop_first_result_response and state.result_attempts == 1 + if drop: + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + return + self._json( + 200, + { + "protocol_version": "2", + "evaluation_run_id": run_id, + "submission_id": body["submission_id"], + "status": "committed", + "idempotent_replay": replay, + "result_count": len(body["results"]), + "result_checksum": checksum, + }, + ) + + def _leased_assignment( + self, + token: str, + assignment_id: str, + worker_id: str | None, + generation: str | int | None, + ) -> dict | None: + with state.lock: + item = state.assignments.get(assignment_id) + if item is None or item["token"] != token: + self._error(404, "assignment_not_found", False) + return None + try: + generation = int(generation) if generation is not None else None + except ValueError: + generation = None + if ( + item["status"] != "leased" + or item["expired"] + or item["worker_id"] != worker_id + or item["lease_generation"] != generation + ): + self._error(409, "lease_lost", False) + return None + return item + + def _assignment_wire(self, assignment_id: str, item: dict) -> dict: + return { + "assignment_id": assignment_id, + "lease_generation": item["lease_generation"], + "lease_expires_at": "2026-08-28T12:02:00.000000Z", + "session_id": item["session_id"], + "session_revision_id": item["session_revision_id"], + "agent_id": "agent-e2e", + "environment": "test", + "trigger_reason": "agent_end", + "event_count": 1, + "transcript_url": ( + f"{state.base_url}/v1/evaluator/assignments/" + f"{assignment_id}/transcript" + ), + } + + def _token(self) -> str | None: + value = self.headers.get("Authorization", "") + return ( + value.removeprefix("Bearer ") if value.startswith("Bearer ") else None + ) + + def _body(self) -> dict: + size = int(self.headers.get("Content-Length", "0")) + return json.loads(self.rfile.read(size)) + + def _error(self, status: int, code: str, retryable: bool) -> None: + self._json( + status, + { + "protocol_version": "2", + "error": { + "code": code, + "message": code.replace("_", " "), + "retryable": retryable, + "request_id": "request-e2e", + }, + }, + ) + + def _json(self, status: int, value: dict) -> None: + body = json.dumps(value, separators=(",", ":")).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args) -> None: + return + + return Handler + + +def _client(state: ProtocolState, token: str) -> EvaluatorClient: + return EvaluatorClient( + base_url=state.base_url, + credential=token, + max_retries=2, + sleeper=lambda _seconds: None, + ) + + +def _claim(client: EvaluatorClient, worker_id: str): + return client.claim( + ClaimRequest( + worker_id=worker_id, + catalog_revision="sha256:" + "a" * 64, + capacity=1, + ) + ) + + +def test_real_http_worker_survives_lost_result_response_without_duplicate_commit(): + state = ProtocolState() + state.add_assignment("runtime", token="customer-a-token", org="org-a") + state.drop_first_result_response = True + evaluator = Evaluator(name="e2e", version="1") + + @evaluator.eval("completion_present", version="1") + def completion_present(session): + return EvalResult(score=Score(float(bool(session.events)))) + + with ProtocolServer(state): + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url=state.base_url, + credential="customer-a-token", + worker_id="worker-a", + ), + client=_client(state, "customer-a-token"), + ) + + async def run(): + await runtime.register() + return await runtime.run_once() + + assert asyncio.run(run()) == 1 + + run_id = next(iter(state.runs)) + committed = ResultRequest.from_wire(state.last_result_body) + replay = runtime.client.submit_result(run_id, committed) + assert replay.idempotent_replay is True + + with pytest.raises(EvaluatorAPIError) as caught: + runtime.client.submit_result( + run_id, + replace(committed, summary="different content"), + ) + assert caught.value.code == "submission_conflict" + + assert state.result_attempts == 3 + assert state.result_commits == 1 + assert len(state.runs) == 1 + assert next(iter(state.runs.values()))["submission_id"] is not None + assert runtime.metrics()["runs_succeeded"] == 1 + + +def test_two_workers_racing_receive_one_unique_lease(): + state = ProtocolState() + assignment_id = state.add_assignment("race", token="customer-a-token", org="org-a") + with ProtocolServer(state): + first = _client(state, "customer-a-token") + second = _client(state, "customer-a-token") + with ThreadPoolExecutor(max_workers=2) as executor: + responses = list( + executor.map( + lambda pair: _claim(*pair), + [(first, "worker-a"), (second, "worker-b")], + ) + ) + + claimed = [item for response in responses for item in response.assignments] + assert [item.assignment_id for item in claimed] == [assignment_id] + assert state.assignments[assignment_id]["lease_generation"] == 1 + + +def test_expired_lease_is_reclaimed_and_stale_worker_is_fenced(): + state = ProtocolState() + assignment_id = state.add_assignment( + "reclaim", token="customer-a-token", org="org-a" + ) + with ProtocolServer(state): + client = _client(state, "customer-a-token") + first = _claim(client, "worker-a").assignments[0] + plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + state.expire(assignment_id) + second = _claim(client, "worker-b").assignments[0] + + assert second.lease_generation == first.lease_generation + 1 + with pytest.raises(EvaluatorAPIError) as caught: + client.transcript(first, worker_id="worker-a") + assert caught.value.code == "lease_lost" + assert caught.value.retryable is False + + with pytest.raises(EvaluatorAPIError) as caught: + client.heartbeat( + HeartbeatRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + runs=(HeartbeatRun(plan.runs[0].evaluation_run_id, "running"),), + ) + ) + assert caught.value.code == "lease_lost" + + +def test_replacement_worker_finishes_after_forced_worker_loss(): + state = ProtocolState() + assignment_id = state.add_assignment( + "forced-worker-loss", token="customer-a-token", org="org-a" + ) + result_sample = ResultRequest( + submission_id=str(uuid5(_NAMESPACE, "forced-worker-loss-result")), + worker_id="worker-a", + lease_generation=1, + status=TerminalRunStatus.SUCCEEDED, + started_at="2026-08-28T12:00:10.000000Z", + finished_at="2026-08-28T12:00:10.100000Z", + duration_ms=100, + summary="Replacement worker completed the evaluation.", + results=( + ResultItem( + result_key="quality", + result_kind=ResultKind.SCORE, + numeric_value=1.0, + ), + ), + error_code=None, + error_message=None, + ) + + with ProtocolServer(state): + client = _client(state, "customer-a-token") + first = _claim(client, "worker-a").assignments[0] + first_plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-a", + lease_generation=first.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + + # Worker A disappears after planning. Its lease expires and worker B + # reclaims the same logical assignment and deterministic run. + state.expire(assignment_id) + second = _claim(client, "worker-b").assignments[0] + second_plan = client.plan( + assignment_id, + PlanRequest( + worker_id="worker-b", + lease_generation=second.lease_generation, + selected=(EvalSelection("quality", "1"),), + ), + ) + assert ( + second_plan.runs[0].evaluation_run_id + == first_plan.runs[0].evaluation_run_id + ) + + stale_result = replace( + result_sample, + worker_id="worker-a", + lease_generation=first.lease_generation, + ) + with pytest.raises(EvaluatorAPIError) as caught: + client.submit_result(first_plan.runs[0].evaluation_run_id, stale_result) + assert caught.value.code == "lease_lost" + + replacement_result = replace( + result_sample, + worker_id="worker-b", + lease_generation=second.lease_generation, + ) + committed = client.submit_result( + second_plan.runs[0].evaluation_run_id, replacement_result + ) + assert committed.status == "committed" + assert committed.idempotent_replay is False + + assert state.result_attempts == 1 + assert state.result_commits == 1 + + +def test_customer_tenants_are_isolated_and_managed_worker_coexists(): + state = ProtocolState() + customer_id = state.add_assignment( + "customer", token="customer-a-token", org="org-a" + ) + managed_id = state.add_assignment("managed", token="managed-token", org="org-b") + with ProtocolServer(state): + customer_a = _client(state, "customer-a-token") + customer_b = _client(state, "customer-b-token") + managed = _client(state, "managed-token") + + customer_assignment = _claim(customer_a, "worker-a").assignments[0] + assert customer_assignment.assignment_id == customer_id + assert _claim(customer_b, "worker-b").assignments == () + assert ( + _claim(managed, "worker-managed").assignments[0].assignment_id == managed_id + ) + + stolen = Assignment( + assignment_id=customer_assignment.assignment_id, + lease_generation=customer_assignment.lease_generation, + lease_expires_at=customer_assignment.lease_expires_at, + session_id=customer_assignment.session_id, + session_revision_id=customer_assignment.session_revision_id, + agent_id=customer_assignment.agent_id, + environment=customer_assignment.environment, + trigger_reason=customer_assignment.trigger_reason, + event_count=customer_assignment.event_count, + transcript_url=customer_assignment.transcript_url, + ) + with pytest.raises(EvaluatorAPIError) as caught: + customer_b.transcript(stolen, worker_id="worker-a") + assert caught.value.status == 404 + assert caught.value.code == "assignment_not_found" diff --git a/sdk/python/tests/test_evaluator_main.py b/sdk/python/tests/test_evaluator_main.py new file mode 100644 index 00000000..c846abdc --- /dev/null +++ b/sdk/python/tests/test_evaluator_main.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import sys + +import pytest + +from failproofai_sdk.evaluator import Evaluator +from failproofai_sdk.evaluator.__main__ import load_evaluator + + +def test_module_loader_defaults_to_app(tmp_path, monkeypatch): + (tmp_path / "my_evals.py").write_text( + "from failproofai_sdk.evaluator import Evaluator\n" + "app = Evaluator(name='example', version='1')\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + try: + loaded = load_evaluator("my_evals") + finally: + sys.modules.pop("my_evals", None) + assert isinstance(loaded, Evaluator) + assert loaded.name == "example" + + +def test_module_loader_supports_an_explicit_attribute(tmp_path, monkeypatch): + (tmp_path / "custom_evals.py").write_text( + "from failproofai_sdk.evaluator import Evaluator\n" + "worker = Evaluator(name='custom', version='1')\n", + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(tmp_path)) + try: + loaded = load_evaluator("custom_evals:worker") + finally: + sys.modules.pop("custom_evals", None) + assert loaded.name == "custom" + + +def test_module_loader_rejects_the_wrong_object_type(tmp_path, monkeypatch): + (tmp_path / "not_evals.py").write_text("app = object()\n", encoding="utf-8") + monkeypatch.syspath_prepend(str(tmp_path)) + try: + with pytest.raises(TypeError, match="not Evaluator"): + load_evaluator("not_evals") + finally: + sys.modules.pop("not_evals", None) diff --git a/sdk/python/tests/test_evaluator_protocol.py b/sdk/python/tests/test_evaluator_protocol.py new file mode 100644 index 00000000..e23b2474 --- /dev/null +++ b/sdk/python/tests/test_evaluator_protocol.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ( + ClaimRequest, + ClaimResponse, + DefinitionsResponse, + ErrorResponse, + HeartbeatRequest, + HeartbeatResponse, + PlanRequest, + PlanResponse, + ProtocolError, + RegisterRequest, + RegisterResponse, + ResultRequest, + ResultResponse, + SessionTranscript, + UnsupportedProtocolVersion, + protocol, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _contract(): + return json.loads(FIXTURE.read_text(encoding="utf-8")) + + +@pytest.mark.parametrize( + ("sample", "model"), + [ + ("register_request", RegisterRequest), + ("register_response", RegisterResponse), + ("claim_request", ClaimRequest), + ("claim_response", ClaimResponse), + ("definitions_response", DefinitionsResponse), + ("transcript_response", SessionTranscript), + ("plan_request", PlanRequest), + ("plan_response", PlanResponse), + ("heartbeat_request", HeartbeatRequest), + ("heartbeat_response", HeartbeatResponse), + ("result_request", ResultRequest), + ("result_response", ResultResponse), + ("error_response", ErrorResponse), + ], +) +def test_golden_messages_round_trip(sample, model): + wire = _contract()["samples"][sample] + assert model.from_wire(wire).to_wire() == wire + + +def test_unknown_additive_fields_are_tolerated(): + wire = dict(_contract()["samples"]["claim_request"]) + wire["future_optional_field"] = True + assert ClaimRequest.from_wire(wire).capacity == 2 + + +def test_unsupported_major_version_fails_loudly(): + wire = dict(_contract()["samples"]["claim_request"]) + wire["protocol_version"] = "3" + with pytest.raises( + UnsupportedProtocolVersion, match="supported major version is 2" + ): + ClaimRequest.from_wire(wire) + + +def test_transcript_event_count_is_an_integrity_check(): + wire = dict(_contract()["samples"]["transcript_response"]) + wire["event_count"] = 99 + with pytest.raises(ProtocolError, match="transcript contains 2 events"): + SessionTranscript.from_wire(wire) + + +@pytest.mark.parametrize( + ("sample", "model", "path", "value", "message"), + [ + ( + "claim_response", + ClaimResponse, + ("assignments", 0), + 42, + r"assignments\[0\] must be an object", + ), + ( + "register_response", + RegisterResponse, + ("disabled_definitions",), + [42], + r"disabled_definitions\[0\] must be a string", + ), + ( + "heartbeat_response", + HeartbeatResponse, + ("accepted_run_ids", 0), + None, + r"accepted_run_ids\[0\] must be a string", + ), + ( + "plan_response", + PlanResponse, + ("runs", 0), + "not-an-object", + r"runs\[0\] must be an object", + ), + ( + "result_request", + ResultRequest, + ("results", 0, "labels", 0), + 7, + r"labels\[0\] must be a string", + ), + ( + "register_request", + RegisterRequest, + ("definitions", 0, "result_kind"), + "unknown", + "result_kind must be one of", + ), + ], +) +def test_nested_wire_values_fail_with_protocol_errors( + sample, model, path, value, message +): + wire = copy.deepcopy(_contract()["samples"][sample]) + target = wire + for part in path[:-1]: + target = target[part] + target[path[-1]] = value + with pytest.raises(ProtocolError, match=message): + model.from_wire(wire) + + +@pytest.mark.parametrize( + ("sample", "model", "field", "value", "message"), + [ + ( + "claim_response", + ClaimResponse, + "lease_generation", + 0, + "lease_generation must be greater than zero", + ), + ( + "claim_response", + ClaimResponse, + "event_count", + -1, + "event_count must not be negative", + ), + ( + "result_response", + ResultResponse, + "result_count", + -1, + "result_count must not be negative", + ), + ], +) +def test_server_response_counters_and_generations_are_bounded( + sample, model, field, value, message +): + wire = copy.deepcopy(_contract()["samples"][sample]) + if sample == "claim_response": + wire["assignments"][0][field] = value + else: + wire[field] = value + with pytest.raises(ProtocolError, match=message): + model.from_wire(wire) + + +def test_fixture_constants_match_the_sdk_contract(): + contract = _contract() + assert contract["protocol"] == { + "supported_major_versions": [protocol.PROTOCOL_VERSION], + "transcript_schema_version": protocol.TRANSCRIPT_SCHEMA_VERSION, + "result_schema_version": protocol.RESULT_SCHEMA_VERSION, + } + assert contract["http"] == { + "register": protocol.REGISTER_PATH, + "claim": protocol.CLAIM_PATH, + "transcript": protocol.TRANSCRIPT_PATH, + "definitions": protocol.DEFINITIONS_PATH, + "plan": protocol.PLAN_PATH, + "heartbeat": protocol.HEARTBEAT_PATH, + "result": protocol.RESULT_PATH, + "worker_id_header": protocol.WORKER_ID_HEADER, + "lease_generation_header": protocol.LEASE_GENERATION_HEADER, + } + assert contract["timing"] == { + "heartbeat_interval_seconds": protocol.HEARTBEAT_INTERVAL_SECONDS, + "lease_duration_seconds": protocol.LEASE_DURATION_SECONDS, + "poll_interval_seconds": protocol.DEFAULT_POLL_INTERVAL_SECONDS, + "max_attempts": protocol.MAX_ATTEMPTS, + } + assert contract["limits"] == { + "max_catalog_definitions": protocol.MAX_CATALOG_DEFINITIONS, + "max_claim_capacity": protocol.MAX_CLAIM_CAPACITY, + "max_transcript_bytes": protocol.MAX_TRANSCRIPT_BYTES, + "max_results_per_run": protocol.MAX_RESULTS_PER_RUN, + "max_eval_key_bytes": protocol.MAX_EVAL_KEY_BYTES, + "max_display_name_bytes": protocol.MAX_DISPLAY_NAME_BYTES, + "max_version_bytes": protocol.MAX_VERSION_BYTES, + "max_worker_id_bytes": protocol.MAX_WORKER_ID_BYTES, + "max_label_bytes": protocol.MAX_LABEL_BYTES, + "max_labels_per_result": protocol.MAX_LABELS_PER_RESULT, + "max_summary_bytes": protocol.MAX_SUMMARY_BYTES, + "max_reasoning_bytes": protocol.MAX_REASONING_BYTES, + "max_unit_bytes": protocol.MAX_UNIT_BYTES, + "max_display_value_bytes": protocol.MAX_DISPLAY_VALUE_BYTES, + "max_description_bytes": protocol.MAX_DESCRIPTION_BYTES, + "max_error_code_bytes": protocol.MAX_ERROR_CODE_BYTES, + "max_error_message_bytes": protocol.MAX_ERROR_MESSAGE_BYTES, + } + assert contract["errors"] == protocol.ERROR_SPECS + + +def test_session_helpers_use_the_protocol_event_vocabulary(): + session = SessionTranscript.from_wire(_contract()["samples"]["transcript_response"]) + assert session.count("tool_use") == 1 + assert session.events_of_type("agent_end")[0].payload["summary"] == "Done" + + +def test_falsy_or_missing_execution_mode_is_rejected_not_defaulted(): + # F2: a falsy/absent execution_mode was silently coerced to 'local', which + # could run a server-authored ('python') definition down the customer path. + # It must now be a hard protocol error, not a default. + samples = _contract()["samples"] + for bad in ("", None): + defs = json.loads(json.dumps(samples["definitions_response"])) + defs["definitions"][0]["execution_mode"] = bad + with pytest.raises(ProtocolError, match="execution_mode"): + DefinitionsResponse.from_wire(defs) + plan = json.loads(json.dumps(samples["plan_response"])) + plan["runs"][0]["execution_mode"] = bad + with pytest.raises(ProtocolError, match="execution_mode"): + PlanResponse.from_wire(plan) + absent = json.loads(json.dumps(samples["definitions_response"])) + del absent["definitions"][0]["execution_mode"] + with pytest.raises(ProtocolError, match="execution_mode"): + DefinitionsResponse.from_wire(absent) diff --git a/sdk/python/tests/test_evaluator_review_fixes.py b/sdk/python/tests/test_evaluator_review_fixes.py new file mode 100644 index 00000000..d3f70047 --- /dev/null +++ b/sdk/python/tests/test_evaluator_review_fixes.py @@ -0,0 +1,164 @@ +"""Regression tests for Evaluator v2 PR-review fixes. + +Covers: control-character rejection in bounded result text, reasoning carried on +non-score primary results, and the object-repr guard no longer false-rejecting +ordinary hex literals. +""" +from __future__ import annotations + +import pytest + +from failproofai_sdk.evaluator.authoring import EvalResult, Metric, Score +from failproofai_sdk.evaluator.source import _forbid_object_reprs, UnsafeEvaluatorSource + + +def test_bounded_rejects_c0_control_characters(): + # NUL / ESC in reasoning would be accepted by the SDK but rejected by the + # server with a non-retryable 422, silently losing a successful eval. + with pytest.raises(ValueError): + EvalResult(score=Score(1.0, passed=True), reasoning="bad\x00value") + with pytest.raises(ValueError): + EvalResult(score=Score(1.0, passed=True), summary="esc\x1b[31m") + + +def test_bounded_keeps_tab_newline_cr(): + r = EvalResult(score=Score(1.0, passed=True), reasoning="line one\nline\ttwo\r") + items = r.result_items("q") + assert items[0].reasoning == "line one\nline\ttwo\r" + + +def test_reasoning_carried_on_metric_primary(): + # A metric-kind eval's primary result is the metric whose key == eval_key. + r = EvalResult(metrics={"latency": Metric(12.0)}, reasoning="slow tail") + items = {i.result_key: i for i in r.result_items("latency")} + assert items["latency"].reasoning == "slow tail" + + +def test_reasoning_not_smeared_onto_secondary_metrics(): + r = EvalResult( + score=Score(1.0, passed=True), + metrics={"aux": Metric(3.0)}, + reasoning="about the score", + ) + items = {i.result_key: i for i in r.result_items("q")} + assert items["q"].reasoning == "about the score" + assert items["aux"].reasoning is None + + +def test_object_repr_guard_allows_plain_hex_literals(): + # A hex colour / digest in result text must not be mistaken for a pointer repr. + assert _forbid_object_reprs("summary", "background 0xFFFFFF, sha 0xdeadbeef1234") == ( + "background 0xFFFFFF, sha 0xdeadbeef1234" + ) + + +def test_object_repr_guard_still_rejects_pointer_reprs(): + with pytest.raises(UnsafeEvaluatorSource): + _forbid_object_reprs("summary", "") + # ...even with the leading '<' stripped (the reshape bypass). + with pytest.raises(UnsafeEvaluatorSource): + _forbid_object_reprs("summary", "foo.Bar object at 0x7f9c1a2b3c4d>") + + +# ---- runtime fixes (Section 1/2 of the follow-up review) -------------------- +import asyncio +from dataclasses import replace + +from failproofai_sdk.evaluator import ( + ClaimResponse, EvaluatorAPIError, PlannedRun, PlanResponse, SessionTranscript, + WorkerConfig, WorkerRuntime, +) +from failproofai_sdk.evaluator import RegisterResponse, HeartbeatResponse +import json as _json +from pathlib import Path as _Path + + +def _samples_fx(): + return _json.loads( + (_Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json").read_text() + )["samples"] + + +class _FakeClient: + def __init__(self): + s = _samples_fx() + self.assignment = replace( + ClaimResponse.from_wire(s["claim_response"]).assignments[0], + definitions_url="", + ) + self.session = SessionTranscript.from_wire(s["transcript_response"]) + self.plans, self.submissions, self.heartbeats = [], [], [] + + def transcript(self, assignment, *, worker_id): + return self.session + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned" if request.selected else "skipped", + runs=tuple( + PlannedRun(f"run-{i.eval_key}", i.eval_key, i.eval_version) + for i in request.selected + ), + ) + + def submit_result(self, run_id, request): + self.submissions.append((run_id, request)) + + def heartbeat(self, request): + self.heartbeats.append(request) + return HeartbeatResponse( + lease_expires_at="2026-08-28T12:02:30.000000Z", + accepted_run_ids=tuple(r.evaluation_run_id for r in request.runs), + ) + + +def _rt(evaluator, client): + return WorkerRuntime( + evaluator, + WorkerConfig(server_url="https://cloud.example", credential="x", + worker_id="worker-test", max_concurrency=2), + client=client, + ) + + +def test_transcript_too_large_skips_assignment_without_crashing(): + from failproofai_sdk.evaluator import Evaluator, EvalResult, Score + ev = Evaluator(name="t", version="1") + ev.eval("quality", version="1")(lambda s: EvalResult(score=Score(1.0, passed=True))) + + class TooLarge(_FakeClient): + def transcript(self, assignment, *, worker_id): + raise EvaluatorAPIError( + status=413, code="transcript_too_large", + message="transcript exceeds 26214400 bytes", retryable=False, + ) + + c = TooLarge() + # returns cleanly — no plan, no submission, no exception + asyncio.run(_rt(ev, c).process_assignment(c.assignment)) + assert c.plans == [] and c.submissions == [] + + +def test_idempotent_replay_runs_a_definition_this_attempt_skipped(): + from failproofai_sdk.evaluator import Evaluator, EvalResult, Score, ConditionResult + ev = Evaluator(name="t", version="1") + + @ev.eval("quality", version="1", when=lambda s: ConditionResult(False, "nope")) + def quality(session): + return EvalResult(score=Score(1.0, passed=True)) + + class ReplaySkipped(_FakeClient): + def plan(self, assignment_id, request): + # This attempt selected nothing, but the server replays the first + # attempt's run for the now-skipped eval. + return PlanResponse( + assignment_id=assignment_id, assignment_status="planned", + runs=(PlannedRun("run-quality", "quality", "1"),), + idempotent_replay=True, + ) + + c = ReplaySkipped() + asyncio.run(_rt(ev, c).process_assignment(c.assignment)) + assert [rid for rid, _ in c.submissions] == ["run-quality"] diff --git a/sdk/python/tests/test_evaluator_runtime.py b/sdk/python/tests/test_evaluator_runtime.py new file mode 100644 index 00000000..dff10869 --- /dev/null +++ b/sdk/python/tests/test_evaluator_runtime.py @@ -0,0 +1,1103 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import time +from dataclasses import replace +from pathlib import Path + +import pytest + +from failproofai_sdk.evaluator import ( + AssignmentDefinition, + ClaimResponse, + ConditionResult, + DefinitionsResponse, + EvalResult, + Evaluator, + EvaluatorAPIError, + ExecutionMode, + HeartbeatResponse, + PlannedRun, + PlanResponse, + RegisterResponse, + ResultKind, + Score, + SessionTranscript, + WorkerConfig, + WorkerRuntime, + source_checksum, +) + +FIXTURE = Path(__file__).parent / "fixtures" / "evaluator_v2" / "contract.json" + + +def _samples(): + return json.loads(FIXTURE.read_text(encoding="utf-8"))["samples"] + + +class FakeClient: + def __init__(self): + samples = _samples() + self.assignment = ClaimResponse.from_wire( + samples["claim_response"] + ).assignments[0] + self.assignment = replace(self.assignment, definitions_url="") + self.session = SessionTranscript.from_wire(samples["transcript_response"]) + self.register_requests = [] + self.claim_requests = [] + self.plans = [] + self.submissions = [] + self.heartbeats = [] + + def register(self, request): + self.register_requests.append(request) + return RegisterResponse.from_wire(_samples()["register_response"]) + + def claim(self, request): + self.claim_requests.append(request) + return ClaimResponse(assignments=(self.assignment,)) + + def transcript(self, assignment, *, worker_id): + assert assignment == self.assignment + assert worker_id == "worker-test" + return self.session + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned" if request.selected else "skipped", + runs=tuple( + PlannedRun(f"run-{item.eval_key}", item.eval_key, item.eval_version) + for item in request.selected + ), + ) + + def submit_result(self, run_id, request): + self.submissions.append((run_id, request)) + + def heartbeat(self, request): + self.heartbeats.append(request) + return HeartbeatResponse( + lease_expires_at="2026-08-28T12:02:30.000000Z", + accepted_run_ids=tuple(item.evaluation_run_id for item in request.runs), + ) + + +def _runtime(evaluator, client): + return WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + max_concurrency=2, + ), + client=client, + ) + + +def test_managed_definition_is_fetched_verified_and_executed(): + source = "EvalResult(score=Score(0.75, passed=True), summary='hosted')" + + class HostedClient(FakeClient): + def __init__(self): + super().__init__() + self.assignment = replace( + self.assignment, + definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions", + ) + + def definitions(self, assignment, *, worker_id): + assert assignment == self.assignment + assert worker_id == "worker-test" + return DefinitionsResponse( + assignment_id=assignment.assignment_id, + catalog_revision="sha256:hosted", + definitions=( + AssignmentDefinition( + eval_key="hosted_quality", + display_name="Hosted quality", + eval_version="1", + result_kind=ResultKind.SCORE, + execution_mode=ExecutionMode.PYTHON, + source_checksum=source_checksum(None, source), + ), + ), + ) + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun( + "run-hosted", + "hosted_quality", + "1", + execution_mode=ExecutionMode.PYTHON, + evaluator_source=source, + source_checksum=source_checksum(None, source), + timeout_seconds=1, + ), + ), + ) + + client = HostedClient() + asyncio.run( + _runtime(Evaluator(name="managed", version="1"), client).process_assignment( + client.assignment + ) + ) + + assert len(client.submissions) == 1 + run_id, result = client.submissions[0] + assert run_id == "run-hosted" + assert result.status.value == "succeeded" + assert result.summary == "hosted" + assert result.results[0].numeric_value == 0.75 + + +def test_managed_definition_that_fails_to_compile_dead_letters_as_one_failed_run(): + # Unsafe/malformed server-authored source is rejected by the sandbox at + # compile time. That rejection must surface as a single bounded FAILED run, + # NOT as an exception out of assignment setup that crashes the task and + # forces the whole assignment to be reclaimed and retried. + unsafe = ( + 'EvalResult(score=Score(1.0), ' + 'reasoning="{0.__class__}".format(session))' + ) + + class HostedClient(FakeClient): + def __init__(self): + super().__init__() + self.assignment = replace( + self.assignment, + definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions", + ) + + def definitions(self, assignment, *, worker_id): + return DefinitionsResponse( + assignment_id=assignment.assignment_id, + catalog_revision="sha256:hosted", + definitions=( + AssignmentDefinition( + eval_key="hosted_quality", + display_name="Hosted quality", + eval_version="1", + result_kind=ResultKind.SCORE, + execution_mode=ExecutionMode.PYTHON, + source_checksum=source_checksum(None, unsafe), + ), + ), + ) + + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun( + "run-hosted", + "hosted_quality", + "1", + execution_mode=ExecutionMode.PYTHON, + evaluator_source=unsafe, + source_checksum=source_checksum(None, unsafe), + timeout_seconds=1, + ), + ), + ) + + client = HostedClient() + # Must NOT raise — the poison definition is contained to its own run. + asyncio.run( + _runtime(Evaluator(name="managed", version="1"), client).process_assignment( + client.assignment + ) + ) + + assert len(client.submissions) == 1 + run_id, result = client.submissions[0] + assert run_id == "run-hosted" + assert result.status.value == "failed" + assert result.error_code == "eval_error" + # Nothing derived from the rejected source may be reported. + assert result.results == () + assert result.summary is None + + +def test_managed_condition_governs_even_when_a_local_key_collides(): + # COR-001: `local` is keyed on (eval_key, eval_version) alone, so a managed + # (PYTHON) definition can collide with a local one the worker also registered. + # The server's managed condition must decide applicability — NOT the matching + # local condition. Here the local condition returns True and the managed + # `condition_source` is "False": the definition must be recorded as skipped + # (condition_false) and the managed evaluator source must never run. + source = "EvalResult(score=Score(1.0), summary='should never run')" + + evaluator = Evaluator(name="managed", version="1") + + @evaluator.eval("hosted_quality", version="1", when=lambda session: True) + def hosted_quality(session): # a colliding LOCAL definition, condition True + return EvalResult(score=Score(1.0, passed=True), summary="local") + + class HostedClient(FakeClient): + def __init__(self): + super().__init__() + self.assignment = replace( + self.assignment, + definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions", + ) + + def definitions(self, assignment, *, worker_id): + return DefinitionsResponse( + assignment_id=assignment.assignment_id, + catalog_revision="sha256:hosted", + definitions=( + AssignmentDefinition( + eval_key="hosted_quality", + display_name="Hosted quality", + eval_version="1", + result_kind=ResultKind.SCORE, + execution_mode=ExecutionMode.PYTHON, + condition_source="False", + source_checksum=source_checksum("False", source), + timeout_seconds=1, + ), + ), + ) + + client = HostedClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + # The server's managed condition (False) wins over the local one (True): + # recorded as skipped, nothing selected, and no managed run submitted. + assert client.plans[0].selected == () + assert {(item.eval_key, item.reason_code) for item in client.plans[0].skipped} == { + ("hosted_quality", "condition_false"), + } + assert client.submissions == [] + + +def test_two_assignments_share_the_bounded_sync_eval_pool_and_keep_heartbeating(): + evaluator = Evaluator(name="parallel", version="1") + lock = threading.Lock() + active = 0 + peak = 0 + + def measured(_session): + nonlocal active, peak + with lock: + active += 1 + peak = max(peak, active) + time.sleep(0.04) + with lock: + active -= 1 + return EvalResult(score=Score(1)) + + for index in range(5): + evaluator.eval( + f"eval_{index}", + version="1", + when=lambda session, index=index: ( + index < 3 if session.session_id == "session-a" else index >= 3 + ), + )(measured) + + class ParallelClient(FakeClient): + def transcript(self, assignment, *, worker_id): + assert worker_id == "worker-test" + return replace( + self.session, + assignment_id=assignment.assignment_id, + session_id=assignment.session_id, + session_revision_id=assignment.session_revision_id, + ) + + client = ParallelClient() + first = replace( + client.assignment, + assignment_id="assignment-a", + session_id="session-a", + session_revision_id="revision-a", + ) + second = replace( + client.assignment, + assignment_id="assignment-b", + session_id="session-b", + session_revision_id="revision-b", + ) + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + + async def exercise(): + await asyncio.gather( + runtime.process_assignment(first), runtime.process_assignment(second) + ) + + asyncio.run(exercise()) + + assert peak == 2 + assert len(client.submissions) == 5 + assert client.heartbeats + + +def test_condition_failures_are_isolated_and_plan_is_declared_first(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1", when=lambda session: True) + def selected(session): + return EvalResult(score=Score(1)) + + @evaluator.eval("not_applicable", version="1", when=lambda session: False) + def not_applicable(session): + return EvalResult(score=Score(1)) + + def broken_condition(session): + raise RuntimeError("condition exploded") + + @evaluator.eval("broken_condition", version="1", when=broken_condition) + def never_runs(session): + raise AssertionError("must not run") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + assert len(client.plans) == 1 + assert [item.eval_key for item in client.plans[0].selected] == ["selected"] + assert {(item.eval_key, item.reason_code) for item in client.plans[0].skipped} == { + ("not_applicable", "condition_false"), + ("broken_condition", "condition_error"), + } + assert [run_id for run_id, _ in client.submissions] == ["run-selected"] + + +def test_condition_can_supply_a_stable_skip_reason(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval( + "retrieval_only", + version="1", + when=lambda session: ConditionResult(False, "no_retrieval_events"), + ) + def retrieval_only(session): + raise AssertionError("must not run") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.plans[0].skipped[0].reason_code == "no_retrieval_events" + + +def test_one_eval_failure_does_not_block_another_result(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("fails", version="1") + def fails(session): + raise RuntimeError("secret details should be bounded") + + @evaluator.eval("succeeds", version="1") + async def succeeds(session): + await asyncio.sleep(0) + return EvalResult(score=Score(0.8), summary="good") + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + + by_run = {run_id: request for run_id, request in client.submissions} + assert by_run["run-fails"].status.value == "failed" + assert by_run["run-fails"].error_code == "eval_error" + assert by_run["run-fails"].results == () + assert by_run["run-succeeds"].status.value == "succeeded" + assert by_run["run-succeeds"].results[0].result_kind == ResultKind.SCORE + + +def test_timeout_is_submitted_as_a_terminal_run(): + evaluator = Evaluator(name="test", version="1") + cancelled = [] + + @evaluator.eval( + "slow", + version="1", + timeout_seconds=0.01, + on_cancel=lambda session: cancelled.append(session.session_revision_id), + ) + async def slow(session): + await asyncio.sleep(1) + return EvalResult(score=Score(1)) + + client = FakeClient() + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + request = client.submissions[0][1] + assert request.status.value == "timed_out" + assert request.error_code == "eval_timeout" + assert cancelled == [client.assignment.session_revision_id] + + +def test_lost_lease_cancels_local_execution(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("slow", version="1") + async def slow(session): + await asyncio.sleep(1) + return EvalResult(score=Score(1)) + + class LeaseLostClient(FakeClient): + def heartbeat(self, request): + raise EvaluatorAPIError( + status=409, + code="lease_lost", + message="gone", + retryable=False, + ) + + client = LeaseLostClient() + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + with pytest.raises(asyncio.CancelledError): + asyncio.run(runtime.process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_partial_heartbeat_acceptance_cancels_only_the_fenced_run(): + evaluator = Evaluator(name="test", version="1") + + class PartialHeartbeatClient(FakeClient): + def heartbeat(self, request): + self.heartbeats.append(request) + return HeartbeatResponse( + lease_expires_at="2026-08-28T12:02:30.000000Z", + accepted_run_ids=(request.runs[0].evaluation_run_id,), + ) + + client = PartialHeartbeatClient() + runtime = _runtime(evaluator, client) + runtime._heartbeat_interval = 0.01 + + async def exercise(): + first = asyncio.create_task(asyncio.sleep(60)) + second = asyncio.create_task(asyncio.sleep(60)) + heartbeat = asyncio.create_task( + runtime._heartbeat( + client.assignment, {"run-first": first, "run-second": second} + ) + ) + while not client.heartbeats: + await asyncio.sleep(0.001) + for _ in range(100): + if second.done(): + break + await asyncio.sleep(0.001) + assert first.done() is False + assert second.cancelled() is True + heartbeat.cancel() + first.cancel() + await asyncio.gather(first, second, heartbeat, return_exceptions=True) + + asyncio.run(exercise()) + + +def test_transcript_revision_must_match_the_claimed_assignment(): + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + client.session = SessionTranscript.from_wire( + { + **_samples()["transcript_response"], + "session_revision_id": "different-revision", + } + ) + + with pytest.raises(RuntimeError, match="revision does not match"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.plans == [] + + +def test_server_cannot_add_a_run_when_every_eval_was_skipped(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("skipped", version="1", when=lambda session: False) + def skipped(session): + raise AssertionError("must not run") + + class UnexpectedRunClient(FakeClient): + def plan(self, assignment_id, request): + self.plans.append(request) + return PlanResponse( + assignment_id=assignment_id, + assignment_status="skipped", + runs=(PlannedRun("run-injected", "skipped", "1"),), + ) + + client = UnexpectedRunClient() + with pytest.raises(RuntimeError, match="unrequested evaluation run"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_server_plan_must_match_assignment_and_include_each_new_selected_eval(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1") + def selected(session): + return EvalResult(score=Score(1)) + + class WrongAssignmentClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id="another-assignment", + assignment_status="planned", + runs=(PlannedRun("run-selected", "selected", "1"),), + ) + + wrong_assignment = WrongAssignmentClient() + with pytest.raises(RuntimeError, match="different assignment"): + asyncio.run( + _runtime(evaluator, wrong_assignment).process_assignment( + wrong_assignment.assignment + ) + ) + + class WrongStatusClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="skipped", + runs=(PlannedRun("run-selected", "selected", "1"),), + ) + + wrong_status = WrongStatusClient() + with pytest.raises(RuntimeError, match="inconsistent assignment status"): + asyncio.run( + _runtime(evaluator, wrong_status).process_assignment( + wrong_status.assignment + ) + ) + + class OmittedRunClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=(), + ) + + omitted = OmittedRunClient() + with pytest.raises(RuntimeError, match="omitted a selected evaluation run"): + asyncio.run(_runtime(evaluator, omitted).process_assignment(omitted.assignment)) + + class ReplayedPlanClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=(), + idempotent_replay=True, + ) + + replayed = ReplayedPlanClient() + asyncio.run(_runtime(evaluator, replayed).process_assignment(replayed.assignment)) + assert replayed.submissions == [] + + +def test_server_plan_rejects_duplicate_run_ids(): + evaluator = Evaluator(name="test", version="1") + evaluator.eval("first", version="1")(lambda session: EvalResult(score=Score(1))) + evaluator.eval("second", version="1")(lambda session: EvalResult(score=Score(1))) + + class DuplicateRunClient(FakeClient): + def plan(self, assignment_id, request): + return PlanResponse( + assignment_id=assignment_id, + assignment_status="planned", + runs=( + PlannedRun("same-run", "first", "1"), + PlannedRun("same-run", "second", "1"), + ), + ) + + client = DuplicateRunClient() + with pytest.raises(RuntimeError, match="duplicate evaluation run id"): + asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment)) + assert client.submissions == [] + + +def test_register_advertises_the_deterministic_catalog(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("quality", version="7") + def quality(session): + return EvalResult(score=Score(1)) + + client = FakeClient() + runtime = _runtime(evaluator, client) + asyncio.run(runtime.register()) + request = client.register_requests[0] + assert request.catalog_revision == evaluator.catalog_revision + assert request.definitions[0].eval_version == "7" + assert runtime._heartbeat_interval == 30 + + +def test_runtime_readiness_tracks_registration_contact_and_shutdown(monkeypatch): + evaluator = Evaluator(name="test", version="1") + runtime = _runtime(evaluator, FakeClient()) + + assert runtime.is_ready() is False + assert runtime.metrics() == {} + + asyncio.run(runtime.register()) + assert runtime.is_ready() is True + assert runtime.metrics() == {"registration_success": 1} + + last_contact = runtime._last_server_contact + assert last_contact is not None + monkeypatch.setattr(time, "monotonic", lambda: last_contact + 121) + assert runtime.is_ready() is False + + monkeypatch.setattr(time, "monotonic", lambda: last_contact) + runtime.stop() + assert runtime.is_ready() is False + + +def test_runtime_metrics_count_claims_conditions_and_outcomes(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("selected", version="1", when=lambda session: True) + def selected(session): + return EvalResult(score=Score(1)) + + @evaluator.eval("skipped", version="1", when=lambda session: False) + def skipped(session): + raise AssertionError("must not run") + + runtime = _runtime(evaluator, FakeClient()) + + async def exercise(): + await runtime.register() + return await runtime.run_once() + + assert asyncio.run(exercise()) == 1 + assert runtime.metrics() == { + "assignments_claimed": 1, + "conditions_selected": 1, + "conditions_skipped": 1, + "registration_success": 1, + "runs_succeeded": 1, + } + + +def test_runtime_metrics_count_registration_failure(): + evaluator = Evaluator(name="test", version="1") + + class BrokenClient(FakeClient): + def register(self, request): + raise EvaluatorAPIError( + status=503, + code="unavailable", + message="try later", + retryable=True, + ) + + runtime = _runtime(evaluator, BrokenClient()) + with pytest.raises(EvaluatorAPIError): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False + assert runtime.metrics() == {"registration_failure": 1} + + +def test_invalid_registration_response_does_not_make_runtime_ready(): + evaluator = Evaluator(name="test", version="1") + + class InvalidTimingClient(FakeClient): + def register(self, request): + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=120, + lease_duration_seconds=120, + poll_interval_seconds=10, + claim_limit=1, + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + runtime = _runtime(evaluator, InvalidTimingClient()) + with pytest.raises(RuntimeError, match="invalid evaluator timing"): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False + assert runtime.metrics() == {"registration_failure": 1} + + +def test_lost_claim_response_waits_out_the_lease_before_claiming_again(): + evaluator = Evaluator(name="test", version="1") + + class LostResponseClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + raise EvaluatorAPIError( + status=None, + code="transport_error", + message="response lost", + retryable=True, + ) + + runtime = _runtime(evaluator, LostResponseClient()) + waits = [] + + async def stop_after_wait(seconds): + waits.append(seconds) + runtime.stop() + + runtime._wait_or_stop = stop_after_wait + asyncio.run(runtime.run_forever()) + + assert waits == [120.0] + assert len(runtime.client.claim_requests) == 1 + assert runtime.metrics() == { + "claim_failures": 1, + "registration_success": 1, + } + + +def test_idle_claim_waits_the_advertised_poll_interval_before_polling_again(): + # Normal short polling: an empty claim returns immediately (no long-poll), so + # the worker sleeps the server-advertised poll_interval_seconds — 10 in the + # fixture register response — instead of hot-looping. The claim request also no + # longer carries a wait_seconds field. + evaluator = Evaluator(name="test", version="1") + + class IdleClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + return ClaimResponse(assignments=()) + + runtime = _runtime(evaluator, IdleClient()) + waits = [] + + async def stop_after_wait(seconds): + waits.append(seconds) + runtime.stop() + + runtime._wait_or_stop = stop_after_wait + asyncio.run(runtime.run_forever()) + + assert waits == [10.0] + assert len(runtime.client.claim_requests) == 1 + assert not hasattr(runtime.client.claim_requests[0], "wait_seconds") + + +def test_nonretryable_claim_failure_stops_the_worker(): + evaluator = Evaluator(name="test", version="1") + + class RejectedClaimClient(FakeClient): + def claim(self, request): + self.claim_requests.append(request) + raise EvaluatorAPIError( + status=409, + code="catalog_mismatch", + message="register again with the current catalog", + retryable=False, + ) + + runtime = _runtime(evaluator, RejectedClaimClient()) + with pytest.raises(EvaluatorAPIError, match="catalog_mismatch"): + asyncio.run(runtime.run_forever()) + assert len(runtime.client.claim_requests) == 1 + assert runtime.metrics() == { + "claim_failures": 1, + "registration_success": 1, + } + + +@pytest.mark.parametrize( + ("assignments", "message"), + [ + (lambda item: (item, item), "duplicate assignments"), + ( + lambda item: tuple( + replace(item, assignment_id=f"assignment-{index}") for index in range(3) + ), + "more assignments than requested", + ), + ], +) +def test_claim_response_cannot_exceed_capacity_or_repeat_work(assignments, message): + evaluator = Evaluator(name="test", version="1") + + class InvalidClaimClient(FakeClient): + def claim(self, request): + return ClaimResponse(assignments=assignments(self.assignment)) + + runtime = _runtime(evaluator, InvalidClaimClient()) + with pytest.raises(RuntimeError, match=message): + asyncio.run(runtime.run_once()) + assert runtime.metrics() == {} + + +def test_register_applies_server_claim_limit_and_disabled_definitions(): + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("disabled", version="1") + def disabled(session): + raise AssertionError("disabled eval must not run") + + class RestrictedClient(FakeClient): + def register(self, request): + self.register_requests.append(request) + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=10, + lease_duration_seconds=120, + poll_interval_seconds=10, + claim_limit=1, + disabled_definitions=("disabled",), + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + client = RestrictedClient() + runtime = _runtime(evaluator, client) + + async def exercise(): + await runtime.register() + await runtime.run_once() + + asyncio.run(exercise()) + assert runtime._claim_limit == 1 + assert client.claim_requests[0].capacity == 1 + assert client.plans[0].selected == () + assert client.plans[0].skipped[0].reason_code == "disabled_by_server" + + +def test_worker_config_requires_dedicated_credentials(monkeypatch): + monkeypatch.delenv("FAILPROOFAI_EVALUATOR_URL", raising=False) + monkeypatch.delenv("FAILPROOFAI_EVALUATOR_TOKEN", raising=False) + with pytest.raises(ValueError, match="URL is required"): + WorkerConfig.from_env() + + +def test_register_rejects_non_positive_poll_interval(): + # The worker adopts the server-advertised poll_interval_seconds (normal short + # polling — there is no long-poll wait). A non-positive interval would make the + # claim loop hot-spin, so registration must refuse it. + evaluator = Evaluator(name="test", version="1") + + class ZeroPollClient(FakeClient): + def register(self, request): + return RegisterResponse( + evaluator_instance_id="instance", + evaluator_kind=self._kind(), + heartbeat_interval_seconds=30, + lease_duration_seconds=120, + poll_interval_seconds=0, + claim_limit=1, + ) + + @staticmethod + def _kind(): + from failproofai_sdk.evaluator import EvaluatorKind + + return EvaluatorKind.CUSTOMER + + runtime = _runtime(evaluator, ZeroPollClient()) + with pytest.raises(RuntimeError, match="invalid evaluator timing"): + asyncio.run(runtime.register()) + assert runtime.is_ready() is False + + +def test_worker_config_rejects_header_control_characters(monkeypatch): + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_URL", "https://cloud.example") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_TOKEN", "secret") + monkeypatch.setenv("FAILPROOFAI_EVALUATOR_WORKER_ID", "worker\nforged") + with pytest.raises(ValueError, match="control characters"): + WorkerConfig.from_env() + + +def test_graceful_drain_cancels_work_after_the_configured_deadline(): + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + drain_timeout_seconds=1, + ), + client=client, + ) + cancelled = False + + async def exercise(): + nonlocal cancelled + + async def active_work(): + nonlocal cancelled + try: + await asyncio.sleep(60) + except asyncio.CancelledError: + cancelled = True + raise + + task = asyncio.create_task(active_work()) + runtime._active.add(task) + await asyncio.sleep(0) + runtime.config = WorkerConfig( + server_url=runtime.config.server_url, + credential=runtime.config.credential, + worker_id=runtime.config.worker_id, + drain_timeout_seconds=0, + ) + await runtime.drain() + + asyncio.run(exercise()) + assert cancelled is True + assert runtime._active == set() + + +def test_stop_interrupts_capacity_wait_and_enters_drain(): + runtime = _runtime(Evaluator(name="test", version="1"), FakeClient()) + + async def exercise(): + blocker = asyncio.Event() + work = asyncio.create_task(blocker.wait()) + runtime._active.add(work) + await asyncio.sleep(0) + + runtime.stop() + await asyncio.wait_for(runtime._wait_for_progress(), timeout=0.1) + + assert work.done() is False + work.cancel() + await asyncio.gather(work, return_exceptions=True) + + asyncio.run(exercise()) + + +def test_eval_execution_respects_process_concurrency(): + evaluator = Evaluator(name="test", version="1") + active = 0 + peak = 0 + + async def measured(session): + nonlocal active, peak + active += 1 + peak = max(peak, active) + await asyncio.sleep(0.01) + active -= 1 + return EvalResult(score=Score(1)) + + evaluator.eval("first", version="1")(measured) + evaluator.eval("second", version="1")(measured) + client = FakeClient() + runtime = WorkerRuntime( + evaluator, + WorkerConfig( + server_url="https://cloud.example", + credential="secret", + worker_id="worker-test", + max_concurrency=1, + ), + client=client, + ) + asyncio.run(runtime.process_assignment(client.assignment)) + assert peak == 1 + DefinitionsResponse, + ExecutionMode, + + +def test_synchronous_evaluation_timeout_is_counted_as_orphaned(): + # A synchronous evaluator that overruns its timeout cannot be cancelled: the + # runtime submits a terminal timed_out result and records the orphaned thread + # so a hung evaluator is findable. The executor is sized with headroom over + # the concurrency limit so this orphan does not starve live capacity. + evaluator = Evaluator(name="test", version="1") + + @evaluator.eval("slow", version="1", timeout_seconds=0.05) + def slow(session): + time.sleep(0.5) + return EvalResult(score=Score(1)) + + client = FakeClient() + runtime = _runtime(evaluator, client) + try: + asyncio.run(runtime.process_assignment(client.assignment)) + request = client.submissions[0][1] + assert request.status.value == "timed_out" + assert request.error_code == "eval_timeout" + assert runtime.metrics().get("sync_evaluations_orphaned") == 1 + assert runtime._eval_executor._max_workers > runtime.config.max_concurrency + finally: + runtime._eval_executor.shutdown(wait=True) + + +def test_conditions_are_skipped_when_the_lease_is_exhausted(): + # With no lease time left before the plan must be submitted, the worker skips + # the condition (without running it) instead of burning the lease and getting + # the plan fenced as lease_lost. + evaluator = Evaluator(name="test", version="1") + ran = [] + + def gate(session): + ran.append(True) + return True + + @evaluator.eval("slow", version="1", when=gate) + def slow(session): + return EvalResult(score=Score(1)) + + client = FakeClient() + runtime = _runtime(evaluator, client) + # Force an already-exhausted condition-phase deadline. + runtime._condition_phase_deadline = lambda assignment: time.monotonic() + asyncio.run(runtime.process_assignment(client.assignment)) + + assert ran == [], "the condition must not run once the lease is exhausted" + assert client.plans, "a plan must still be submitted" + plan_request = client.plans[-1] + assert not plan_request.selected + reasons = {(s.eval_key, s.reason_code) for s in plan_request.skipped} + assert ("slow", "lease_exhausted") in reasons + assert runtime.metrics().get("conditions_lease_exhausted") == 1 + assert client.submissions == [] + + +def test_condition_phase_deadline_and_budget_are_lease_bounded(): + from datetime import datetime, timedelta, timezone + + evaluator = Evaluator(name="test", version="1") + client = FakeClient() + runtime = _runtime(evaluator, client) + runtime._lease_duration = 120 + + # A stale (past) lease_expires_at falls back to the negotiated lease duration, + # so the bound never fires spuriously under clock skew or a replayed fixture. + stale = replace(client.assignment, lease_expires_at="2000-01-01T00:00:00.000000Z") + fallback = runtime._condition_phase_deadline(stale) - time.monotonic() + assert 110 <= fallback <= 125 + + # A future lease is honored. + future_ts = (datetime.now(timezone.utc) + timedelta(seconds=300)).strftime( + "%Y-%m-%dT%H:%M:%S.%f" + ) + "Z" + future = replace(client.assignment, lease_expires_at=future_ts) + ahead = runtime._condition_phase_deadline(future) - time.monotonic() + assert 250 <= ahead <= 305 + + # Budget is capped by both the remaining lease and the per-definition timeout. + deadline = time.monotonic() + 100 + assert runtime._condition_budget(deadline, None) == pytest.approx(95, abs=2) + assert runtime._condition_budget(deadline, 10) == pytest.approx(10, abs=0.05) + assert runtime._condition_budget(time.monotonic(), None) < 0 diff --git a/sdk/python/tests/test_evaluator_source.py b/sdk/python/tests/test_evaluator_source.py new file mode 100644 index 00000000..fc65bcd5 --- /dev/null +++ b/sdk/python/tests/test_evaluator_source.py @@ -0,0 +1,430 @@ +from __future__ import annotations + +import pytest + +from failproofai_sdk.evaluator import EvalResult, Score +from failproofai_sdk.evaluator.protocol import SessionTranscript, TranscriptEvent +from failproofai_sdk.evaluator.source import ( + MAX_AST_NODES, + MAX_EVALUATOR_SOURCE_BYTES, + MAX_POW_EXPONENT, + MAX_SANDBOX_TIMEOUT_SECONDS, + EvaluationSandboxUnavailable, + EvaluationTimeout, + UnsafeEvaluatorSource, + _clamp_budget, + compile_condition, + compile_evaluator, + source_checksum, +) + + +def Session(event_count: int = 3) -> SessionTranscript: + """A real, serializable transcript — managed evals now run in a subprocess and + the transcript crosses the boundary via `to_wire`, so a dummy object won't do. + Each event carries a dict payload (so `events[0].payload.get` is reachable).""" + events = tuple( + TranscriptEvent( + id=f"e{i}", + ts="2026-08-28T12:00:00.000000Z", + event_type="tool_use", + payload={"k": "v", "tool_name": "search"}, + ) + for i in range(event_count) + ) + return SessionTranscript( + assignment_id="a", + session_id="s", + session_revision_id="r", + agent_id="agent", + environment="test", + started_at="2026-08-28T12:00:00.000000Z", + ended_at="2026-08-28T12:00:01.000000Z", + event_count=event_count, + events=events, + ) + + +def test_restricted_expressions_can_evaluate_conditions_and_results(): + assert compile_condition("session.event_count > 0")(Session()) is True + result = compile_evaluator("EvalResult(score=Score(0.75, passed=True))")( + Session() + ) + assert isinstance(result, EvalResult) + assert result.score == Score(0.75, passed=True) + + +@pytest.mark.parametrize( + "source", + [ + "__import__('os').system('id')", + "session.__class__", + "(lambda: 1)()", + "[x for x in ().__class__.__base__.__subclasses__()]", + ], +) +def test_restricted_expressions_reject_escape_primitives(source): + with pytest.raises(UnsafeEvaluatorSource): + compile_evaluator(source) + + +@pytest.mark.parametrize( + "source", + [ + # `str.format` / `str.format_map` traverse a format string's field spec + # at the C level, reaching attributes the AST dunder guard never sees. + # These reached real `__builtins__` before the denylist landed. + '"{0.__class__.__init__.__globals__[__builtins__][__import__]}".format(session)', + '"{0.__class__}".format(session)', + 'str.format("{0.__class__}", session)', + '"{a.__class__}".format_map({"a": session})', + # A reasoning string is where a leak would surface — block it there too. + 'EvalResult(score=Score(0.5), reasoning="{0.__class__}".format(session))', + ], +) +def test_restricted_expressions_reject_format_string_traversal(source): + with pytest.raises(UnsafeEvaluatorSource): + compile_evaluator(source) + + +@pytest.mark.parametrize( + "source", + [ + # Generator/frame/code introspection reaches the eval globals and, via + # dict.update on them, could poison a shared namespace. None of these + # attribute names start with "_", so only the default-deny allowlist + # stops them. + "EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_frame.f_globals))", + "EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_code))", + 'EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_frame.f_globals.update({"P": 1})))', + # `mro` is a public method on the type metaclass; it reaches `object`. + "EvalResult(score=Score(0.5), reasoning=str(str.mro()[-1]))", + "EvalResult(score=Score(0.5), reasoning=str(int.mro()))", + # A live function's identity would leak a heap pointer (ASLR defeat). + "EvalResult(score=Score(0.5), reasoning=str(EvalResult.result_items))", + ], +) +def test_restricted_expressions_reject_introspection_attributes(source): + with pytest.raises(UnsafeEvaluatorSource, match="attribute"): + compile_evaluator(source) + + +def test_no_reachable_construct_leaks_a_heap_pointer_repr(): + # An object's default repr (`<... object at 0x...>`) leaks a live host heap + # address (ASLR/memory-layout disclosure) if coerced into a result field. + # Generator expressions are rejected at compile; `enumerate` is not bound, so + # it raises NameError at run time and becomes a bounded failed run instead of + # a disclosure. Either way, a pointer must never reach a result string. + import re + + from failproofai_sdk.evaluator.source import _SAFE_GLOBALS + + with pytest.raises(UnsafeEvaluatorSource, match="GeneratorExp"): + compile_evaluator("EvalResult(score=Score(1.0), reasoning=str((x for x in [1])))") + + evaluate = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(enumerate([1])))" + ) + with pytest.raises(NameError): + evaluate(Session()) + + # No value bound into the evaluation namespace reprs to a heap pointer. + pointer = re.compile(r"0x[0-9a-fA-F]+") + leaky = { + name: repr(value) + for name, value in _SAFE_GLOBALS.items() + if name != "__builtins__" and pointer.search(repr(value)) + } + assert leaky == {} + + +@pytest.mark.parametrize( + "inner", + [ + # A bound method's repr is `<... at 0xADDR>` — a live heap pointer. These + # methods stay allowlisted (real evals CALL them), but referencing one as a + # bare VALUE only serves to stringify that repr, so it is now rejected at + # COMPILE — the disclosure is closed at its source, not at the output. + "session.events[0].payload.get", + "''.join", + "'x'.encode", + "'a,b'.split", + ], +) +def test_bare_bound_method_reference_is_rejected_at_compile(inner): + for field in ( + f'EvalResult(score=Score(1.0), reasoning=str({inner}))', + f'EvalResult(score=Score(1.0), summary=str({inner}))', + f'EvalResult(score=Score(1.0, display_value=str({inner})))', + f'EvalResult(score=Score(1.0), labels=(str({inner}),))', + ): + with pytest.raises(UnsafeEvaluatorSource, match="only to call it"): + compile_evaluator(field) + + +def test_heap_pointer_output_guard_bypasses_are_closed_at_compile(): + # An adversarial-review finding: the output-boundary regex was anchored on `<`, + # so a managed source could keep the address while reshaping the wrapper text — + # str(...).replace("<",""), an f-string, or %-formatting all coerce a bound + # method at a point the old scan missed. Each needs a BARE bound-method + # reference, which the compile-time call-site rule now rejects outright. + bypasses = [ + # str(...).replace("<","") strips the old regex's `<` anchor. + 'EvalResult(score=Score(1.0), reasoning=str(dict().get).replace("<", ""))', + # f-strings coerce at the C level, past the `str` global. + 'EvalResult(score=Score(1.0), reasoning=f"{session.events[0].payload.get}")', + # %-formatting coerces at the C level too. + 'EvalResult(score=Score(1.0), reasoning="%s" % session.events[0].payload.get)', + ] + for src in bypasses: + with pytest.raises(UnsafeEvaluatorSource, match="only to call it"): + compile_evaluator(src) + + +def test_called_methods_and_data_attributes_still_stringify(): + # The call-site rule blocks only BARE method references. Calling methods and + # reading data attributes (both pointer-free) must still work — including the + # f-string and %-formatting paths — so legitimate evaluations are unaffected. + reasoning_call = compile_evaluator( + 'EvalResult(score=Score(1.0), ' + 'reasoning=str(session.events[0].payload.get("tool_name")))' + )(Session()) + assert reasoning_call.reasoning == "search" + + fstring = compile_evaluator( + 'EvalResult(score=Score(1.0), reasoning=f"n={session.event_count}")' + )(Session()) + assert fstring.reasoning == "n=3" + + percent = compile_evaluator( + 'EvalResult(score=Score(1.0), reasoning="pct=%d" % (session.event_count * 10))' + )(Session()) + assert percent.reasoning == "pct=30" + + +def test_each_evaluation_gets_isolated_globals_so_it_cannot_poison_the_next(): + # Even setting aside the allowlist, one evaluation must not be able to leave + # state behind for the next. Compiling and running twice must not share a + # mutable namespace. + from failproofai_sdk.evaluator.source import _fresh_globals + + first = _fresh_globals() + second = _fresh_globals() + assert first is not second + assert first["__builtins__"] is not second["__builtins__"] + first["__poison__"] = "leaked" + assert "__poison__" not in second + + +def test_format_denylist_does_not_block_legitimate_string_methods(): + # The fix is a targeted denylist of `format`/`format_map`, not a ban on all + # string methods — ordinary evaluations must still compile and run. + evaluate = compile_evaluator( + 'EvalResult(score=Score(0.9, passed=True), ' + 'reasoning="tools=" + str(session.event_count).upper())' + ) + result = evaluate(Session()) + assert result.reasoning == "tools=3" + + +def test_restricted_expressions_reject_statements_and_oversized_source(): + with pytest.raises(UnsafeEvaluatorSource, match="one expression"): + compile_evaluator("import os") + with pytest.raises(UnsafeEvaluatorSource, match="exceeds"): + compile_evaluator("x" * (MAX_EVALUATOR_SOURCE_BYTES + 1)) + + +def test_result_and_condition_types_are_checked_at_runtime(): + with pytest.raises(TypeError, match="EvalResult"): + compile_evaluator("True")(Session()) + with pytest.raises(TypeError, match="bool or ConditionResult"): + compile_condition("1")(Session()) + + +def test_source_checksum_covers_condition_and_evaluator_together(): + base = source_checksum(None, "EvalResult()") + assert base == source_checksum(None, "EvalResult()") + assert base != source_checksum("True", "EvalResult()") + assert base != source_checksum(None, "EvalResult(summary='changed')") + + +# --- SEC-001: managed source cannot exhaust the worker (killable-fork sandbox) --- + + +def test_compute_bomb_is_killed_within_its_budget(): + # `sum(range(10**9))` would burn CPU for ~20s in-process, uncancellable — a + # CPU-bound loop (not a big allocation) so the wall-clock/CPU budget is what + # stops it, deterministically, rather than the memory ceiling. The forked + # sandbox kills it at its budget. + import time as _time + + evaluate = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(sum(range(10**9))))", + timeout_seconds=1, + ) + started = _time.monotonic() + with pytest.raises(EvaluationTimeout): + evaluate(Session()) + assert _time.monotonic() - started < 5 # bounded by the ~1s budget, not ~20s + + +def test_condition_compute_bomb_is_also_bounded(): + # `10**9`, matching the evaluator bomb above, NOT `10**8`. + # + # The property under test is "a CPU bomb in a condition is stopped by the + # sandbox budget", and the bomb has to be big enough that it cannot finish + # inside that budget on ANY machine the suite runs on. At 10**8 it was only + # ~1.35 CPU-seconds against a 1-second budget — a 1.35x margin — so on a fast + # runner the sum simply completed and nothing timed out. It failed exactly + # that way on CI under Python 3.14, which is faster here than 3.13 (1.35s vs + # 1.44s measured), while passing locally: a machine-speed coin flip, not a + # real signal about the sandbox. + # + # 10**9 restores the ~13x margin the evaluator twin already had. It costs no + # extra wall-clock: the sandbox kills the child at its budget either way, so + # a bigger bomb only widens the gap between "killed" and "could have + # finished". Do not shrink it back. + condition = compile_condition("sum(range(10**9)) > 0", timeout_seconds=1) + with pytest.raises(EvaluationTimeout): + condition(Session()) + + +def test_literal_pow_exponent_bomb_is_rejected_at_compile(): + with pytest.raises(UnsafeEvaluatorSource, match="exponent"): + compile_evaluator(f"EvalResult(score=Score(10 ** {MAX_POW_EXPONENT + 1}))") + # A small constant exponent stays allowed. + result = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(2 ** 3))" + )(Session()) + assert result.reasoning == "8" + + +def test_oversized_expression_is_rejected_at_compile(): + huge = "[" + ",".join("1" for _ in range(MAX_AST_NODES)) + "]" + with pytest.raises(UnsafeEvaluatorSource, match="too large"): + compile_evaluator(f"EvalResult(score=Score(len({huge}) / len({huge})))") + + +def test_normal_managed_eval_survives_the_fork_boundary(): + # A session-dependent result must round-trip out of the forked child intact. + result = compile_evaluator( + "EvalResult(score=Score(1.0 if session.event_count > 0 else 0.0), " + "reasoning=str(session.event_count))" + )(Session()) + assert isinstance(result, EvalResult) + assert result.score.value == 1.0 + assert result.reasoning == "3" + + +def test_sandbox_fails_closed_without_a_serializable_transcript(): + # The transcript crosses into the subprocess via `to_wire`. A session that + # can't be serialized cannot be sandboxed, so refuse rather than run unbounded. + class NotATranscript: + event_count = 1 + + with pytest.raises(EvaluationSandboxUnavailable): + compile_evaluator("EvalResult(score=Score(1.0))")(NotATranscript()) + + +def test_server_timeout_cannot_exceed_the_hard_ceiling(): + # SEC-001: a large server-provided timeout must not remove the execution bound. + assert _clamp_budget(10**9) == float(MAX_SANDBOX_TIMEOUT_SECONDS) + assert _clamp_budget(0) == 30.0 + assert _clamp_budget(None) == 30.0 + assert _clamp_budget(5) == 5.0 + + +def test_oversized_result_is_rejected_before_it_crosses_back(): + # A result with far more than the 25-item limit must be rejected INSIDE the + # sandbox (via result_items), so a huge result can never be serialized and + # shipped back to OOM the worker (SEC-001). + src = ( + "EvalResult(score=Score(1.0), " + "metrics={'m' + str(i): float(i) for i in range(200)})" + ) + with pytest.raises(ValueError, match="at most"): + compile_evaluator(src, eval_key="q")(Session()) + + +def test_sandbox_slot_wait_counts_against_the_timeout(monkeypatch): + # SEC-001: acquiring a concurrency slot must count against the wall-clock budget. + # `asyncio.wait_for` only cancels the awaiter, so a run that blocked UNBOUNDED on + # a busy slot would still launch a sandbox after its caller was reported timed + # out — 28 threads could queue behind 4 long sandboxes and starve the worker. + # With one slot and three 1s compute bombs, all three must resolve within ~one + # budget (the holder is killed at ~1s; the two queued behind it exhaust their + # budget waiting and time out WITHOUT ever spawning a child), not three serialized + # budgets (~3s). + import threading + import time as _time + + from failproofai_sdk.evaluator import source as _source + + monkeypatch.setattr(_source, "_SANDBOX_SLOTS", threading.Semaphore(1)) + bomb = compile_evaluator( + "EvalResult(score=Score(1.0), reasoning=str(sum(range(10**9))))", + timeout_seconds=1, + ) + session = Session() + errors: list[str] = [] + + def run(): + try: + bomb(session) + except Exception as error: # noqa: BLE001 + errors.append(type(error).__name__) + + threads = [threading.Thread(target=run) for _ in range(3)] + started = _time.monotonic() + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=15) + elapsed = _time.monotonic() - started + + assert errors == ["EvaluationTimeout"] * 3, errors + # Bounded by ~one budget, NOT three serialized ones — proving the queued runs + # timed out on slot acquisition instead of each waiting then running in turn. + assert elapsed < 2.5, f"queued sandboxes were not bounded by the timeout: {elapsed:.2f}s" + + +def test_allocation_bomb_is_bounded_by_the_per_sandbox_memory_limit(): + # A ~1.6 GiB allocation exceeds the per-sandbox RLIMIT_AS and is killed, so it + # cannot exhaust the worker even wrapped in an otherwise-valid result. With the + # concurrent-sandbox cap this also bounds the aggregate across concurrent runs. + src = "EvalResult(score=Score(1.0 if len([0] * 200000000) >= 0 else 0.0))" + with pytest.raises((EvaluationTimeout, MemoryError)): + compile_evaluator(src, timeout_seconds=5)(Session()) + + +def test_sandbox_fails_closed_when_kernel_resource_limits_are_unavailable(monkeypatch): + # SEC-001: on a platform without the stdlib ``resource`` module (e.g. Windows), + # the sandbox child cannot install RLIMIT_CPU / RLIMIT_AS on itself, so managed + # source must be refused BEFORE any child is spawned rather than run unbounded. + import failproofai_sdk.evaluator.source as source + + monkeypatch.setattr(source, "_resource", None) + + def _no_spawn(*args, **kwargs): + raise AssertionError("a sandbox child must not be started when limits are unavailable") + + monkeypatch.setattr(source.subprocess, "Popen", _no_spawn) + + with pytest.raises(EvaluationSandboxUnavailable): + source.compile_evaluator("EvalResult(score=Score(1.0))")(Session()) + + +def test_comprehension_body_can_read_session(): + # A list/set/dict comprehension resolves a free name like `session` from + # GLOBALS. When `session` was only in eval locals, such a source raised + # NameError on CPython 3.10 (a supported version). Both sandbox paths must + # now evaluate a session-dependent comprehension (COR-001). + assert ( + compile_condition("len([session.event_count for i in range(1)]) > 0")(Session()) + is True + ) + result = compile_evaluator( + "EvalResult(score=Score(1.0), " + "reasoning=str([session.event_count for i in range(1)]))" + )(Session(event_count=3)) + assert result.reasoning == "[3]" diff --git a/sdk/python/tests/test_zero_dependencies.py b/sdk/python/tests/test_zero_dependencies.py index a27fd75f..95be7b72 100644 --- a/sdk/python/tests/test_zero_dependencies.py +++ b/sdk/python/tests/test_zero_dependencies.py @@ -300,6 +300,23 @@ def test_importing_the_package_loads_no_framework(): ) +def test_importing_the_package_does_not_load_the_evaluator_runtime(): + """Telemetry-only users do not pay for the separate worker surface.""" + import json + import subprocess + + probe = ( + "import json, sys; import failproofai_sdk; " + "print(json.dumps(sorted(m for m in sys.modules " + "if m.startswith('failproofai_sdk.evaluator'))))" + ) + result = subprocess.run( + [sys.executable, "-c", probe], capture_output=True, text=True, cwd=str(ROOT), timeout=60 + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout.strip()) == [] + + def test_the_adapter_registry_holds_strings_not_modules(): """`_REGISTRY` maps a name to a dotted path; importing it here would defeat it.""" from failproofai_sdk.integrations import _REGISTRY