From 46edc08a2ec19fd235698721b2a44b5fba484f98 Mon Sep 17 00:00:00 2001 From: song Date: Sun, 1 Nov 2026 10:00:11 +0800 Subject: [PATCH 1/2] fix(benchmark): preserve nonfinite results across local JSON boundaries Signed-off-by: song --- benchmark/README.md | 12 ++ benchmark/edgebench/README.md | 11 ++ benchmark/edgebench/checkpoint_judge.py | 10 +- benchmark/edgebench/offline_scoring.py | 9 +- benchmark/runtime/checkpoint.py | 5 +- benchmark/runtime/sforge_checkpoint.py | 5 +- benchmark/runtime/sforge_json.py | 119 ++++++++++++++++++ benchmark/tests/test_checkpoint_judge.py | 105 ++++++++++++++++ benchmark/tests/test_edgebench_online.py | 112 +++++++++++++++++ benchmark/tests/test_sforge_json.py | 102 +++++++++++++++ .../test_research_checkpoint_capture.py | 19 ++- 11 files changed, 498 insertions(+), 11 deletions(-) create mode 100644 benchmark/runtime/sforge_json.py create mode 100644 benchmark/tests/test_sforge_json.py diff --git a/benchmark/README.md b/benchmark/README.md index bb61cc6b29..25c6fe4740 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -508,6 +508,18 @@ The local `sforge_checkpoint_opportunity_v0` artifact records these facts; it grants no restored admission. Disabling capture means omitting both the judge flag and worker declaration option on a new run. +Finite opportunity responses and saved participants keep that existing JSON +shape. Released history/results containing non-finite numbers use the explicit +[`sforge_nonfinite_json: 1` envelope](edgebench/README.md#feedback-protocol-new-run-default-native) +over the same logical artifact. The client decodes it before checking the +incumbent; checkpoint capture saves it through the same lossless codec. +Only released result fields may contain encoded numbers: opportunity clocks, +round counters and authority fields retain their original strict representation. +The archive binding covers the canonical encoded state, including numeric kinds, +so repeated NaN reads retain a binding and changed released data invalidates it. +Use `benchmark.runtime.sforge_json.loads` for saved participant readback. This +transport neither changes native selection nor qualifies continued execution. + For an isolated single-Goal trial, `"loopx_state": true` adds the existing state-backup owner's inventory: registered active state, project research state, runtime authority/effect/quota journals and registry. The declared worker and diff --git a/benchmark/edgebench/README.md b/benchmark/edgebench/README.md index b094e70260..d84d855c41 100644 --- a/benchmark/edgebench/README.md +++ b/benchmark/edgebench/README.md @@ -511,6 +511,17 @@ covers online submissions only; use the complete offline result for post-run qualification. Neither report by itself certifies integrity or score countability. No offline result is routed to the worker. +Offline score files, the combined result and the command's summary retain native +non-finite scores and nested metrics. Finite documents keep their existing JSON. +Other documents use the local `sforge_nonfinite_json: 1` transport envelope: +`value` contains JSON data with each non-finite number replaced by `null`, and +`nonfinite_values` records its path and `positive_infinity`, `negative_infinity` +or `nan` kind. Use `benchmark.runtime.sforge_json.loads` to read either form. +Restart decodes saved scores before calling the unchanged native selector; +it does not turn these results into zero, missing feedback or grading errors. +Literal marker-shaped metrics remain data, including an escaped root marker. +Malformed paths, duplicate keys and unmarked non-finite constants are rejected. + The first completed valid finite score establishes a silent baseline. The task's native selection policy is the sole improvement criterion among valid scored snapshots. Only a strictly better native rank updates `/opt/edgebench-feedback/latest.json`; ties, diff --git a/benchmark/edgebench/checkpoint_judge.py b/benchmark/edgebench/checkpoint_judge.py index 77117b7e80..e91a2b8c8b 100644 --- a/benchmark/edgebench/checkpoint_judge.py +++ b/benchmark/edgebench/checkpoint_judge.py @@ -15,7 +15,8 @@ import uuid from dataclasses import replace from pathlib import Path -from benchmark.runtime.sforge_checkpoint import ARCHIVE_ROUTE, ROUTE, SCHEMA +from benchmark.runtime import sforge_json +from benchmark.runtime.sforge_checkpoint import ARCHIVE_ROUTE, ROUTE, SCHEMA, RELEASED_FIELDS def _read_archive(root, run, task, round_id): @@ -142,8 +143,10 @@ def snapshot_locked(token): binding = {key: value for key, value in packet.items() if key not in {"observed_at_epoch", "remaining_cooldown_seconds"}} packet["capture_binding_sha256"] = hashlib.sha256( - json.dumps(binding, sort_keys=True, separators=(",", ":")).encode()).hexdigest() - return json.loads(json.dumps(packet)), raw # No live references or tokens leave the lock. + sforge_json.dumps(binding, fields=RELEASED_FIELDS, + sort_keys=True, separators=(",", ":")).encode()).hexdigest() + # The wire object contains no non-JSON constants or live references. + return json.loads(sforge_json.dumps(packet, fields=RELEASED_FIELDS)), raw def capture(request): with state._admission_lock, state._tokens_lock, state._history_lock: @@ -152,6 +155,7 @@ def capture(request): def archive(request): with state._admission_lock, state._tokens_lock, state._history_lock: packet, raw = snapshot_locked(request.token) + packet = sforge_json.decode_document(packet, fields=RELEASED_FIELDS) incumbent = packet["native_incumbent"] if packet["capture_binding_sha256"] != request.capture_binding_sha256 or incumbent is None: raise HTTPException(409, "Native checkpoint basis moved or has no selected incumbent") diff --git a/benchmark/edgebench/offline_scoring.py b/benchmark/edgebench/offline_scoring.py index 45440e7b74..72de9ae538 100644 --- a/benchmark/edgebench/offline_scoring.py +++ b/benchmark/edgebench/offline_scoring.py @@ -18,6 +18,7 @@ from sforge.harness.run_evaluation import judge_submission from sforge.harness.selection import select_best from sforge.harness.task_spec import make_task_spec +from benchmark.runtime import sforge_json from .online_judge import cohort_lock, resource_preflight @@ -51,7 +52,7 @@ def score_captures(trial, task, config, backend): directory.mkdir(exist_ok=True) result_path = directory / "score.json" if result_path.is_file(): - entry = json.loads(result_path.read_text()) + entry = sforge_json.loads(result_path.read_text()) if (entry["source_sha256"] != digest or entry["task_sha256"] != receipt["task_sha256"] or entry.get("evaluator_sha256") != evaluator_digest): raise ValueError("Offline score provenance differs from the trial") @@ -67,7 +68,7 @@ def score_captures(trial, task, config, backend): except Exception as error: entry.update(status="error", error_kind=type(error).__name__) pending = result_path.with_suffix(".tmp") - pending.write_text(json.dumps(entry, allow_nan=False)) + pending.write_text(sforge_json.dumps(entry)) pending.replace(result_path) entries.append(entry) result = dict(run_id=receipt["run_id"], task=task.task_id, @@ -76,7 +77,7 @@ def score_captures(trial, task, config, backend): **select_best(entries, task.judge.score_direction, task.judge.selection), entries=entries) path = destination / "result.json" pending = path.with_suffix(".tmp") - pending.write_text(json.dumps(result, allow_nan=False)) + pending.write_text(sforge_json.dumps(result)) pending.replace(path) return result @@ -105,7 +106,7 @@ def main(): result = score_captures(trial, task, config, create_backend_from_config(config)) except BlockingIOError: parser.error("Online cohort still owns resources; defer offline scoring") - print(json.dumps({k: v for k, v in result.items() if k != "entries"})) + print(sforge_json.dumps({k: v for k, v in result.items() if k != "entries"})) return 0 if result["offline_scoring_complete"] else 1 diff --git a/benchmark/runtime/checkpoint.py b/benchmark/runtime/checkpoint.py index 0628d04dea..0f5d37c5ea 100644 --- a/benchmark/runtime/checkpoint.py +++ b/benchmark/runtime/checkpoint.py @@ -340,12 +340,13 @@ def capture_after_wake(env: dict[str, str], receipt: dict[str, Any], *, components, research = research_participants(env, components, metadata, destination, evidence, deadline) opportunity = None if request.get("evaluation_opportunity") and not request.get("controller_handoff"): - from .sforge_checkpoint import capture_opportunity + from . import sforge_json + from .sforge_checkpoint import capture_opportunity, RELEASED_FIELDS evidence = Path(env["LOOPX_WAKE_LOG_DIR"]) / receipt["turn_id"] / "checkpoint-opportunity.json" incumbent = evidence.with_name("checkpoint-incumbent.tar.gz") opportunity = capture_opportunity(env, timeout=max(.1, deadline - time.time()), incumbent_path=incumbent) - evidence.write_text(json.dumps(opportunity), encoding="utf-8") + evidence.write_text(sforge_json.dumps(opportunity, fields=RELEASED_FIELDS), encoding="utf-8") evidence.chmod(0o600) opportunity_id = _participant_id(components, "evaluation_opportunity") components = [*components, {"id": opportunity_id, "source_path": str(evidence)}] diff --git a/benchmark/runtime/sforge_checkpoint.py b/benchmark/runtime/sforge_checkpoint.py index 2f19c7da96..885614b437 100644 --- a/benchmark/runtime/sforge_checkpoint.py +++ b/benchmark/runtime/sforge_checkpoint.py @@ -8,9 +8,12 @@ from urllib.request import Request, urlopen from pathlib import Path +from . import sforge_json + SCHEMA = "sforge_checkpoint_opportunity_v0" # Local provider artifact, not a LoopX authority contract. ROUTE = "/api/v1/checkpoint/opportunity" ARCHIVE_ROUTE = "/api/v1/checkpoint/incumbent" +RELEASED_FIELDS = ("released_history", "released_results") def capture_opportunity(env, *, timeout, incumbent_path: Path): @@ -23,7 +26,7 @@ def capture_opportunity(env, *, timeout, incumbent_path: Path): data=json.dumps({"token": env["SFORGE_TOKEN"]}).encode(), headers={"Content-Type": "application/json"}, method="POST") with urlopen(request, timeout=max(.1, stop - time.time())) as response: - packet = json.load(response) + packet = sforge_json.loads(response.read(), fields=RELEASED_FIELDS) if (packet.get("schema_version") != SCHEMA or packet.get("execution_authority_granted") is not False or packet.get("outstanding_evaluations") != 0): raise ValueError("native opportunity checkpoint readback is unavailable") diff --git a/benchmark/runtime/sforge_json.py b/benchmark/runtime/sforge_json.py new file mode 100644 index 0000000000..1bc0349cae --- /dev/null +++ b/benchmark/runtime/sforge_json.py @@ -0,0 +1,119 @@ +"""Lossless JSON transport for local SForge result artifacts. + +Finite legacy documents keep their ordinary JSON representation. Nonfinite +numbers use an explicit envelope and paths into its value, never inline tags +that could be confused with a task's literal metric. This is transport only; +native grading and selection still receive the original numeric values. +""" +from __future__ import annotations + +import json +import math + +_MARKER = "sforge_nonfinite_json" +_KINDS = {"positive_infinity": "inf", "negative_infinity": "-inf", "nan": "nan"} + + +def encode_document(document, *, fields=None): + """Return a JSON-safe document; optional fields restrict numeric encoding.""" + if not isinstance(document, dict): + raise ValueError("SForge result transport requires an object") + values = [] + + def visit(value, path): + if isinstance(value, float) and not math.isfinite(value): + if fields is not None and path[0] not in fields: + raise ValueError("Nonfinite number outside released result fields") + kind = "nan" if math.isnan(value) else ( + "positive_infinity" if value > 0 else "negative_infinity") + values.append({"path": path, "kind": kind}) + return None + if isinstance(value, dict): + pairs = {} + for key, item in value.items(): + # Follow JSON's existing conversion of scalar object keys. + if not isinstance(key, str): + key = next(iter(json.loads(json.dumps({key: None}, allow_nan=False)))) + if key in pairs: + raise ValueError("SForge result keys collide in JSON") + pairs[key] = item + # Stable path order also keeps capture bindings independent of dict order. + return {key: visit(item, [*path, key]) for key, item in sorted(pairs.items())} + if isinstance(value, (list, tuple)): + return [visit(item, [*path, index]) for index, item in enumerate(value)] + return value + + value = visit(document, []) + if not values and _MARKER not in document: + return document + # Escape a literal root marker as data, even when it contains only finite + # numbers. Decoding unwraps exactly once; nested marker-shaped values stay data. + return {_MARKER: 1, "value": value, "nonfinite_values": values} + + +def decode_document(document, *, fields=None): + """Decode one envelope from a strictly parsed JSON document.""" + if not isinstance(document, dict): + raise ValueError("SForge result transport requires an object") + if _MARKER not in document: + return document + if (set(document) != {_MARKER, "value", "nonfinite_values"} + or type(document[_MARKER]) is not int or document[_MARKER] != 1 + or not isinstance(document["value"], dict)): + raise ValueError("Invalid SForge nonfinite transport envelope") + value, entries = document["value"], document["nonfinite_values"] + if not isinstance(entries, list) or (not entries and _MARKER not in value): + raise ValueError("SForge nonfinite transport requires value paths") + + def lookup(container, key): + if isinstance(container, dict) and isinstance(key, str) and key in container: + return container[key] + if isinstance(container, list) and type(key) is int and 0 <= key < len(container): + return container[key] + raise ValueError("SForge nonfinite value path is unavailable") + + seen = set() + for entry in entries: + if not isinstance(entry, dict) or set(entry) != {"path", "kind"}: + raise ValueError("Invalid SForge nonfinite value") + path, kind = entry["path"], entry["kind"] + if (not isinstance(path, list) or not path + or not all(type(key) is str or (type(key) is int and key >= 0) for key in path) + or not isinstance(kind, str) or kind not in _KINDS + or (fields is not None and path[0] not in fields)): + raise ValueError("Invalid SForge nonfinite value path or kind") + identity = tuple(path) + if identity in seen: + raise ValueError("Duplicate SForge nonfinite value path") + seen.add(identity) + container = value + for key in path[:-1]: + container = lookup(container, key) + if lookup(container, path[-1]) is not None: + raise ValueError("SForge nonfinite value must replace null") + container[path[-1]] = float(_KINDS[kind]) + return value + + +def dumps(document, *, fields=None, **options): + return json.dumps(encode_document(document, fields=fields), allow_nan=False, **options) + + +def loads(encoded, *, fields=None): + def finite_number(text): + value = float(text) + if not math.isfinite(value): + raise ValueError("Unmarked nonfinite SForge result number") + return value + + def unique_object(pairs): + value = {} + for key, item in pairs: + if key in value: + raise ValueError("Duplicate SForge result object key") + value[key] = item + return value + + document = json.loads(encoded, parse_float=finite_number, parse_constant=finite_number, + object_pairs_hook=unique_object) + return decode_document(document, fields=fields) diff --git a/benchmark/tests/test_checkpoint_judge.py b/benchmark/tests/test_checkpoint_judge.py index 0c6708ccae..0920f980a4 100644 --- a/benchmark/tests/test_checkpoint_judge.py +++ b/benchmark/tests/test_checkpoint_judge.py @@ -1,6 +1,8 @@ """Native owner semantics; real grading/transport is qualified by the Docker smoke.""" import json import hashlib +import math +import time from types import SimpleNamespace import pytest @@ -168,3 +170,106 @@ def read(request, *, timeout): with pytest.raises(FileExistsError): sforge_checkpoint.capture_opportunity(env, timeout=10, incumbent_path=target) assert target.read_bytes() == b'prior participant' + + +@pytest.mark.parametrize("score", [-2.5, float("inf"), float("-inf"), float("nan")]) +def test_nonfinite_opportunity_crosses_real_http_client_and_retains_binding(native_config, score): + from urllib.error import HTTPError + from urllib.request import Request, urlopen + from sforge.harness.grading import EvalReport + from benchmark.edgebench.checkpoint_judge import create_app + from benchmark.runtime import sforge_checkpoint, sforge_json + from benchmark.tests.test_checkpoint_controller import server + + app = create_app(native_config, checkpoint_capture=True) + state = app.state.judge + with server(app) as url: + token = state.register_session("fixture", "source", max_agent_submissions=3, submission_cooldown=3600) + state.consume_round(token) + literal = {"sforge_nonfinite_json": 1, "value": {"score": None}, + "nonfinite_values": [{"path": ["score"], "kind": "nan"}]} + report = EvalReport(task_id="fixture", submission_id="accepted", pass_rate=1, + score=score, metrics={"nested": [score, None], "literal": literal}).to_dict() + state._record_submission("source", "accepted", "fixture", "agent-1", report, None) + state.submissions["accepted"] = {"run_id": "source", "task_id": "fixture", "round": "agent-1", + "status": "completed", "report": report, "error": None} + archive = native_config.log_dir / "runs/source/fixture/submissions/agent-1/submission.tar.gz" + archive.parent.mkdir(parents=True) + original = b"opaque accepted synthetic candidate" + archive.write_bytes(original) + before = state.resolve_token(token) + history_before = json.dumps(state.get_run_history("source", "fixture"), sort_keys=True) + + def request(route, payload): + return urlopen(Request(url + route, data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, method="POST"), timeout=10) + + with request(sforge_checkpoint.ROUTE, {"token": token}) as response: + raw = response.read() + # The actual FastAPI response is standards-compliant JSON; the client, + # rather than a helper-only round trip, restores its numeric semantics. + def unmarked(value): + raise AssertionError("Non-JSON numeric constant in HTTP response") + wire = json.loads(raw, parse_constant=unmarked) + assert ("sforge_nonfinite_json" in wire) is not math.isfinite(score) + env = {"SFORGE_JUDGE_URL": url, "SFORGE_TOKEN": token, + "LOOPX_PHASE_DEADLINE_EPOCH": str(time.time() + 60)} + point = sforge_checkpoint.capture_opportunity(env, timeout=10, + incumbent_path=native_config.log_dir / "incumbent.tar.gz") + assert point["schema_version"] == sforge_checkpoint.SCHEMA + measured = point["released_results"]["accepted"]["report"]["metrics"]["nested"][0] + assert math.isnan(measured) if math.isnan(score) else measured == score + selected_score = point["released_history"][0]["score"] + assert math.isnan(selected_score) if math.isnan(score) else selected_score == score + assert point["released_results"]["accepted"]["report"]["metrics"]["literal"] == literal + assert point["native_incumbent"]["round"] == "agent-1" + assert (native_config.log_dir / "incumbent.tar.gz").read_bytes() == original + assert point["next_agent_round"] == 2 and point["max_agent_submissions"] == 3 + assert point["submission_cooldown_seconds"] == 3600 + assert 0 < point["remaining_cooldown_seconds"] <= 3600 + assert point["execution_authority_granted"] is False and point["continuation_qualified"] is False + assert token not in raw.decode() and "admin_secret" not in raw.decode() + + # Repeated NaN reads and JSON-equivalent object ordering retain the same + # binding. A changed released metric must invalidate the archive read. + report["metrics"] = dict(reversed(list(report["metrics"].items()))) + with request(sforge_checkpoint.ROUTE, {"token": token}) as response: + repeated = sforge_json.loads(response.read(), fields=sforge_checkpoint.RELEASED_FIELDS) + assert repeated["capture_binding_sha256"] == point["capture_binding_sha256"] + assert state.resolve_token(token) == before + assert json.dumps(state.get_run_history("source", "fixture"), sort_keys=True) == history_before + report["metrics"]["nested"][0] = 42.0 + with pytest.raises(HTTPError) as moved: + request(sforge_checkpoint.ARCHIVE_ROUTE, {"token": token, + "capture_binding_sha256": point["capture_binding_sha256"]}) + assert moved.value.code == 409 + assert archive.read_bytes() == original and state.resolve_token(token) == before + + +def test_malformed_http_nonfinite_paths_cannot_download_an_incumbent(tmp_path): + from fastapi import FastAPI + from benchmark.runtime import sforge_checkpoint, sforge_json + from benchmark.tests.test_checkpoint_controller import server + + app = FastAPI() + downloads = [] + packet = sforge_json.encode_document({"schema_version": sforge_checkpoint.SCHEMA, + "execution_authority_granted": False, "outstanding_evaluations": 0, + "capture_binding_sha256": "synthetic-binding", "released_results": {"score": float("inf")}, + "native_incumbent": {"archive_bytes": 1, "archive_sha256": "a" * 64}}) + packet["nonfinite_values"].append(packet["nonfinite_values"][0]) + + @app.post(sforge_checkpoint.ROUTE) + def malformed(): + return packet + + @app.post(sforge_checkpoint.ARCHIVE_ROUTE) + def archive(): + downloads.append(True) + + with server(app) as url: + target = tmp_path / "incumbent.tar.gz" + with pytest.raises(ValueError, match="Duplicate"): + sforge_checkpoint.capture_opportunity({"SFORGE_JUDGE_URL": url, "SFORGE_TOKEN": "synthetic", + "LOOPX_PHASE_DEADLINE_EPOCH": str(time.time() + 60)}, timeout=10, incumbent_path=target) + assert not target.exists() and not downloads diff --git a/benchmark/tests/test_edgebench_online.py b/benchmark/tests/test_edgebench_online.py index 4a82701fb8..21c09d09e6 100644 --- a/benchmark/tests/test_edgebench_online.py +++ b/benchmark/tests/test_edgebench_online.py @@ -2,6 +2,7 @@ import hashlib import json import logging +import math from types import SimpleNamespace import pytest @@ -222,6 +223,117 @@ def grade(**kwargs): offline_scoring.score_captures(tmp_path, task, None, None) +@pytest.mark.parametrize("score,max_round,min_round", [ + (-2.5, "capture-3", "capture-2"), (float("inf"), "capture-2", "capture-1"), + (float("-inf"), "capture-3", "capture-2"), (float("nan"), "capture-3", "capture-1"), +]) +@pytest.mark.parametrize("direction", ["maximize", "minimize"]) +@pytest.mark.parametrize("policy", ["score_first", "pass_rate_first", "valid_then_score"]) +def test_offline_nonfinite_reports_preserve_native_selection_on_restart( + tmp_path, monkeypatch, score, max_round, min_round, direction, policy): + from sforge.harness.grading import EvalReport + from benchmark.edgebench import offline_scoring + from benchmark.runtime import sforge_json + + queue = sampler(tmp_path) + for archive in (b"first", b"nonfinite", b"final"): + queue.capture(archive, final=archive == b"final") + receipt = tmp_path / "runtime-receipt.json" + receipt.write_text(json.dumps(dict(status="terminal", feedback="best-only", task="fixture", + run_id="run", task_sha256="source-pin"))) + native_before = receipt.read_bytes(), (queue.directory / "index.json").read_bytes() + task = SimpleNamespace(task_id="fixture", judge=SimpleNamespace(score_direction=direction, selection=policy)) + calls = [] + literal = {"sforge_nonfinite_json": 1, "value": {"score": None}, + "nonfinite_values": [{"path": ["score"], "kind": "nan"}]} + + def grade(**kwargs): + calls.append(kwargs["archive"]) + return EvalReport(task_id="fixture", submission_id=kwargs["submission_id"], pass_rate=1, + score={b"first": 1.0, b"nonfinite": score, b"final": 2.0}[kwargs["archive"]], + metrics={"measured": score, "literal": literal, "missing": None}) + + monkeypatch.setattr(offline_scoring, "judge_submission", grade) + for _ in range(2): + result = offline_scoring.score_captures(tmp_path, task, None, None) + assert result["offline_scoring_complete"] and result["failed_captures"] == 0 + assert result["best_round"] == (max_round if direction == "maximize" else min_round) + for entry in result["entries"]: + measured = entry["metrics"]["measured"] + assert math.isnan(measured) if math.isnan(score) else measured == score + assert entry["metrics"]["literal"] == literal and entry["metrics"]["missing"] is None + restored = sforge_json.loads((tmp_path / "offline-scoring/result.json").read_text()) + assert restored["best_round"] == result["best_round"] + if math.isfinite(score): + assert (tmp_path / "offline-scoring/result.json").read_text() == json.dumps(result, allow_nan=False) + assert calls == [b"first", b"nonfinite", b"final"] # Cached scores are decoded before native ranking. + assert (receipt.read_bytes(), (queue.directory / "index.json").read_bytes()) == native_before + + +def test_offline_nan_incumbent_and_malformed_cached_transport_are_not_regraded(tmp_path, monkeypatch): + from sforge.harness.grading import EvalReport + from benchmark.edgebench import offline_scoring + + queue = sampler(tmp_path) + queue.capture(b"first") + queue.capture(b"final", final=True) + (tmp_path / "runtime-receipt.json").write_text(json.dumps(dict( + status="terminal", feedback="best-only", task="fixture", run_id="run", task_sha256="source-pin"))) + task = SimpleNamespace(task_id="fixture", judge=SimpleNamespace(score_direction="maximize", selection="score_first")) + calls = [] + + def grade(**kwargs): + calls.append(kwargs["archive"]) + return EvalReport(task_id="fixture", submission_id=kwargs["submission_id"], pass_rate=1, + score=float("nan") if kwargs["archive"] == b"first" else 10.0) + + monkeypatch.setattr(offline_scoring, "judge_submission", grade) + for _ in range(2): + result = offline_scoring.score_captures(tmp_path, task, None, None) + # Preserve native first-entry comparison semantics, including NaN. + assert result["best_round"] == "capture-1" and math.isnan(result["best_score"]) + prior = (tmp_path / "offline-scoring/result.json").read_bytes() + path = tmp_path / "offline-scoring/capture-1/score.json" + wire = json.loads(path.read_text()) + wire["nonfinite_values"].append(wire["nonfinite_values"][0]) + path.write_text(json.dumps(wire)) + with pytest.raises(ValueError, match="Duplicate"): + offline_scoring.score_captures(tmp_path, task, None, None) + assert calls == [b"first", b"final"] + assert (tmp_path / "offline-scoring/result.json").read_bytes() == prior + + +def test_offline_cli_summary_uses_the_same_lossless_transport(tmp_path, monkeypatch, capsys): + from contextlib import nullcontext + import sys + import docker + from sforge.harness.config import SForgeConfig + from sforge.harness.grading import EvalReport + from benchmark.edgebench import offline_scoring + from benchmark.runtime import sforge_json + + sampler(tmp_path).capture(b"final", final=True) + task_file = tmp_path / "fixture.json" + task_file.write_text("{}") + (tmp_path / "runtime-receipt.json").write_text(json.dumps(dict(status="terminal", feedback="best-only", + task="fixture", run_id="run", task_sha256=hashlib.sha256(task_file.read_bytes()).hexdigest()))) + task = SimpleNamespace(task_id="fixture", judge=SimpleNamespace(score_direction="maximize", selection="score_first")) + monkeypatch.setattr(sys, "argv", ["offline_scoring", "--trial", str(tmp_path), "--tasks-dir", str(tmp_path)]) + monkeypatch.setattr(offline_scoring, "load_benchmark", lambda path: None) + monkeypatch.setattr(offline_scoring, "make_task_spec", lambda *args: task) + monkeypatch.setattr(offline_scoring, "load_config", lambda: SForgeConfig()) + monkeypatch.setattr(offline_scoring, "cohort_lock", nullcontext) + monkeypatch.setattr(docker, "from_env", lambda: SimpleNamespace(close=lambda: None)) + monkeypatch.setattr(offline_scoring, "resource_preflight", lambda *args, **kwargs: None) + monkeypatch.setattr(offline_scoring, "create_backend_from_config", lambda config: None) + monkeypatch.setattr(offline_scoring, "judge_submission", lambda **kwargs: EvalReport( + task_id="fixture", submission_id=kwargs["submission_id"], score=float("inf"), pass_rate=1)) + assert offline_scoring.main() == 0 + summary = sforge_json.loads(capsys.readouterr().out) + assert summary["best_score"] == float("inf") and summary["best_round"] == "capture-1" + assert summary["offline_scoring_complete"] and "entries" not in summary + + def test_offline_needs_terminal_solver_final_capture_and_exclusive_capacity(tmp_path, monkeypatch): from benchmark.edgebench.offline_scoring import score_captures from benchmark.edgebench.online_judge import cohort_lock diff --git a/benchmark/tests/test_sforge_json.py b/benchmark/tests/test_sforge_json.py new file mode 100644 index 0000000000..c4be658e3c --- /dev/null +++ b/benchmark/tests/test_sforge_json.py @@ -0,0 +1,102 @@ +"""Native result numbers survive JSON without interpreting literal task data.""" +import copy +import json +import math + +import pytest + +from benchmark.runtime import sforge_json + + +def test_finite_legacy_representation_and_literal_tags_are_unchanged(): + value = {"score": -2.5, "zero": 0.0, "missing": None, "metrics": { + "nonfinite_values": [{"path": ["score"], "kind": "nan"}], + "sforge_nonfinite_json": 1, "value": {"score": None}, + "text": ["NaN", "Infinity", "-Infinity"], "negative_zero": -0.0}} + assert sforge_json.dumps(value) == json.dumps(value, allow_nan=False) + assert sforge_json.loads(sforge_json.dumps(value)) == value + assert math.copysign(1, sforge_json.loads(sforge_json.dumps(value))["metrics"]["negative_zero"]) == -1 + + +def test_nonfinite_values_and_literal_envelopes_are_lossless_without_source_mutation(): + literal = {"sforge_nonfinite_json": 1, "value": {"score": None}, + "nonfinite_values": [{"path": ["score"], "kind": "nan"}]} + value = {"score": float("inf"), "metrics": {"nested": [float("-inf"), float("nan"), None], + "literal": literal, 7: float("inf")}} + restored = sforge_json.loads(sforge_json.dumps(value)) + assert restored["score"] == float("inf") + assert restored["metrics"]["nested"][0] == float("-inf") + assert math.isnan(restored["metrics"]["nested"][1]) + assert restored["metrics"]["nested"][2] is None + assert restored["metrics"]["7"] == float("inf") + assert restored["metrics"]["literal"] == literal + assert sforge_json.loads(sforge_json.dumps(literal)) == literal + assert sforge_json.loads(sforge_json.dumps({"sforge_nonfinite_json": "literal"})) == { + "sforge_nonfinite_json": "literal"} + assert value["score"] == float("inf") and 7 in value["metrics"] + assert math.isnan(value["metrics"]["nested"][1]) + + +def test_nonfinite_binding_is_independent_of_object_key_order(): + first = {"released_results": {"b": float("inf"), "a": float("nan")}, "next_agent_round": 3} + second = {"next_agent_round": 3, "released_results": {"a": float("nan"), "b": float("inf")}} + assert sforge_json.dumps(first, sort_keys=True) == sforge_json.dumps(second, sort_keys=True) + + +@pytest.mark.parametrize("fault", [ + "version", "boolean_version", "extra_envelope_field", "missing_values", "empty_values", + "wrong_value", "entry_field", "unknown_kind", "boolean_index", "negative_index", + "missing_path", "empty_path", "non_null", "duplicate", "outside_results", +]) +def test_malformed_nonfinite_transport_is_rejected(fault): + packet = sforge_json.encode_document({"released_results": {"nested": [float("inf")]}, + "next_agent_round": None}) + entry = packet["nonfinite_values"][0] + if fault == "version": + packet["sforge_nonfinite_json"] = 2 + elif fault == "boolean_version": + packet["sforge_nonfinite_json"] = True + elif fault == "extra_envelope_field": + packet["unknown"] = 1 + elif fault == "missing_values": + del packet["nonfinite_values"] + elif fault == "empty_values": + packet["nonfinite_values"] = [] + elif fault == "wrong_value": + packet["value"] = [] + elif fault == "entry_field": + entry["unknown"] = 1 + elif fault == "unknown_kind": + entry["kind"] = "infinity" + elif fault == "boolean_index": + entry["path"][-1] = False + elif fault == "negative_index": + entry["path"][-1] = -1 + elif fault == "missing_path": + entry["path"] = ["released_results", "missing"] + elif fault == "empty_path": + entry["path"] = [] + elif fault == "non_null": + packet["value"]["released_results"]["nested"][0] = 0 + elif fault == "duplicate": + packet["nonfinite_values"].append(copy.deepcopy(entry)) + else: + entry["path"] = ["next_agent_round"] + with pytest.raises(ValueError): + sforge_json.loads(json.dumps(packet), fields=("released_results",)) + + +@pytest.mark.parametrize("encoded", [ + '{"score":NaN}', '{"score":Infinity}', '{"score":-Infinity}', '{"score":1e999}', + '{"score":1,"score":2}', '{"metrics":{"same":1,"same":2}}', '[]', +]) +def test_unmarked_numbers_duplicate_keys_and_nonobjects_are_rejected(encoded): + with pytest.raises(ValueError): + sforge_json.loads(encoded) + + +def test_encoding_cannot_hide_invalid_opportunity_or_colliding_json_keys(): + with pytest.raises(ValueError, match="outside released"): + sforge_json.dumps({"next_agent_round": float("inf")}, fields=("released_results",)) + with pytest.raises(ValueError, match="collide"): + sforge_json.dumps({"metrics": {1: "first", "1": float("inf")}}) diff --git a/tests/control_plane/test_research_checkpoint_capture.py b/tests/control_plane/test_research_checkpoint_capture.py index fe004dcaee..f7a7b5d3ac 100644 --- a/tests/control_plane/test_research_checkpoint_capture.py +++ b/tests/control_plane/test_research_checkpoint_capture.py @@ -134,6 +134,7 @@ def denied_read(path): @pytest.mark.parametrize("opportunity,corrupt,collisions", [ (False, False, False), (True, False, False), (True, True, False), (True, False, True), (True, True, True), ("empty", False, True), + ("nonfinite", False, False), ]) def test_real_cli_settlement_capture_verification_and_inert_materialization( tmp_path, monkeypatch, opportunity, corrupt, collisions): @@ -155,6 +156,9 @@ def run(*args): "round": "agent-1", "archive_sha256": hashlib.sha256(incumbent_bytes).hexdigest(), "archive_bytes": len(incumbent_bytes)}} if opportunity == "empty": opportunity_packet["native_incumbent"] = None + if opportunity == "nonfinite": + opportunity_packet["released_results"] = {"synthetic": {"report": { + "score": float("inf"), "metrics": {"nested": [float("nan"), float("-inf")]}}}} def export(env, *, timeout, incumbent_path): assert 0 < timeout <= 30 observed.append(True) @@ -214,7 +218,19 @@ def altered_probe(method, params, **kwargs): assert goal_state.read_bytes() == retained_goal if opportunity: mapping = json.loads((target / "checkpoint.json").read_text())["point_metadata"]["evaluation_opportunity"] - assert json.loads((target / "components" / mapping["component_id"]).read_text()) == opportunity_packet + retained_opportunity = (target / "components" / mapping["component_id"]).read_text() + if opportunity == "nonfinite": + import math + from benchmark.runtime import sforge_json + restored_opportunity = sforge_json.loads(retained_opportunity) + report = restored_opportunity["released_results"]["synthetic"]["report"] + assert report["score"] == float("inf") + assert math.isnan(report["metrics"]["nested"][0]) + assert report["metrics"]["nested"][1] == float("-inf") + assert restored_opportunity["native_incumbent"] == opportunity_packet["native_incumbent"] + assert json.loads(retained_opportunity)["sforge_nonfinite_json"] == 1 + else: + assert json.loads(retained_opportunity) == opportunity_packet if opportunity == "empty": assert mapping["incumbent_component_id"] is None assert not (wake / cli.TURN_ID / "checkpoint-incumbent.tar.gz").exists() @@ -229,6 +245,7 @@ def altered_probe(method, params, **kwargs): assert restored["activation_performed"] is False assert (tmp_path / "inert/components/code").read_text() == "retained_candidate = 7\n" if opportunity: + assert (tmp_path / "inert/components" / mapping["component_id"]).read_text() == retained_opportunity if opportunity != "empty": assert (tmp_path / "inert/components" / mapping["incumbent_component_id"]).read_bytes() == incumbent_bytes for member in caller_components: From 1493936d080d5d5299a70d93f23bf11f1a5db65f Mon Sep 17 00:00:00 2001 From: song Date: Sun, 1 Nov 2026 10:15:01 +0800 Subject: [PATCH 2/2] test(benchmark): guard optional FastAPI HTTP coverage Signed-off-by: song --- benchmark/tests/test_checkpoint_judge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/tests/test_checkpoint_judge.py b/benchmark/tests/test_checkpoint_judge.py index 0920f980a4..3025d6f1ca 100644 --- a/benchmark/tests/test_checkpoint_judge.py +++ b/benchmark/tests/test_checkpoint_judge.py @@ -247,6 +247,7 @@ def unmarked(value): def test_malformed_http_nonfinite_paths_cannot_download_an_incumbent(tmp_path): + pytest.importorskip("fastapi") from fastapi import FastAPI from benchmark.runtime import sforge_checkpoint, sforge_json from benchmark.tests.test_checkpoint_controller import server