From 611d1e17392158c9501cca2ce1fd4185d03946a9 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:08:32 -0700 Subject: [PATCH 01/19] Add multi-OS repro baseline workflow and schema (OVK-PR1). Record a machine-checkable reproducibility baseline so release candidates can prove adapter and control-plane behavior across platforms before attributable publication. --- .github/workflows/repro-baseline.yml | 54 ++++ docs/REPRO_BASELINE.md | 67 +++++ docs/baselines/README.md | 40 +++ schemas/repro.baseline.schema.json | 96 ++++++ scripts/record_repro_baseline.py | 426 +++++++++++++++++++++++++++ tests/test_repro_baseline_schema.py | 58 ++++ 6 files changed, 741 insertions(+) create mode 100644 .github/workflows/repro-baseline.yml create mode 100644 docs/REPRO_BASELINE.md create mode 100644 docs/baselines/README.md create mode 100644 schemas/repro.baseline.schema.json create mode 100644 scripts/record_repro_baseline.py create mode 100644 tests/test_repro_baseline_schema.py diff --git a/.github/workflows/repro-baseline.yml b/.github/workflows/repro-baseline.yml new file mode 100644 index 0000000..b8f119d --- /dev/null +++ b/.github/workflows/repro-baseline.yml @@ -0,0 +1,54 @@ +name: Repro baseline + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +jobs: + record: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ['3.10', '3.12'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Sync package data and install + run: | + python -m pip install --upgrade pip + python scripts/sync_package_data.py + pip install --no-cache-dir -e '.[dev]' + + - name: Record reproducible baseline + shell: bash + env: + PYTHONPATH: ${{ github.workspace }} + run: | + set +e + py_minor=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") + os_label=$(python -c "from scripts.record_repro_baseline import normalize_os_label; print(normalize_os_label())") + baseline="docs/baselines/repro-${os_label}-py${py_minor}.json" + python scripts/record_repro_baseline.py --skip-install --output "$baseline" + status=$? + set -e + python scripts/record_repro_baseline.py --validate-only "$baseline" + echo "recorder_exit=$status" + echo "baseline_path=$baseline" >> "$GITHUB_STEP_SUMMARY" + exit "$status" + + - name: Upload baseline artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: repro-baseline-${{ matrix.os }}-py${{ matrix.python-version }} + path: docs/baselines/repro-*.json + if-no-files-found: error diff --git a/docs/REPRO_BASELINE.md b/docs/REPRO_BASELINE.md new file mode 100644 index 0000000..6ad2ef3 --- /dev/null +++ b/docs/REPRO_BASELINE.md @@ -0,0 +1,67 @@ +# Reproducible baseline (OVK-01) + +This document describes how to record and validate a multi-OS, multi-Python +reproducible baseline for Open Verification Kernel. + +## What is recorded + +Each baseline JSON under `docs/baselines/` follows +`schemas/repro.baseline.schema.json` (`ovk.repro_baseline.v1`) and includes: + +- Python version, OS, and platform string +- Optional checker availability from `ovk doctor` +- Skipped pytest names (when available from output) +- Artifact paths with SHA-256 digests +- Network access flag (`true`/`false`, overridable via `OVK_NETWORK_ACCESS`) +- Wall-clock `started_at` / `completed_at` / `elapsed_seconds` +- Per-command argv, exit code, and elapsed time + +## Commands + +The harness runs (or orchestrates) exactly: + +```bash +pip install -e '.[dev]' +pytest +ovk doctor +ovk check --changed-files examples/multi_surface/pr_combined.diff --advisory +ovk release-preflight +python examples/repair_loops/ci_secrets/demo_repair_loop.py +``` + +If `.verification/` is missing, the harness runs `ovk init` once so `ovk doctor` +can succeed. That setup step is noted in the baseline `notes` field. + +## Local recording + +```bash +python scripts/sync_package_data.py +pip install -e '.[dev]' +python scripts/record_repro_baseline.py --skip-install +``` + +Output defaults to `docs/baselines/repro--py.json`. + +Validate an existing file: + +```bash +python scripts/record_repro_baseline.py --validate-only docs/baselines/repro-linux-py3.12.json +``` + +## CI + +Workflow: [`.github/workflows/repro-baseline.yml`](../.github/workflows/repro-baseline.yml) + +Matrix: `ubuntu-latest`, `macos-latest`, `windows-latest` × Python `3.10` / `3.12`. + +Each cell uploads the baseline JSON. The job fails when the schema is incomplete +even if individual commands exit non-zero (command outcomes are still recorded). + +`docs/baselines/` may be empty in a fresh clone: multi-OS records are CI artifacts. +See [baselines/README.md](baselines/README.md). In-repo RC DoD does not require committed +baseline JSON; live matrix evidence is a maintainer publication gate. + +## Related + +- Adoption dashboard: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) +- Capability registry: [BACKENDS.md](BACKENDS.md) diff --git a/docs/baselines/README.md b/docs/baselines/README.md new file mode 100644 index 0000000..b3d35c3 --- /dev/null +++ b/docs/baselines/README.md @@ -0,0 +1,40 @@ +# Reproducible baselines (OVK-01) + +Generated and CI-uploaded reproducible baseline records. + +Schema: `schemas/repro.baseline.schema.json` (`ovk.repro_baseline.v1`). +Procedure: [REPRO_BASELINE.md](../REPRO_BASELINE.md). +Workflow: [`.github/workflows/repro-baseline.yml`](../../.github/workflows/repro-baseline.yml). + +## Why this directory may be empty locally + +Multi-OS artifacts (`ubuntu` / `macos` / `windows` × Python `3.10` / `3.12`) are produced by +the `repro-baseline` GitHub Actions matrix and uploaded as workflow artifacts. They are **not** +required to be committed for in-repo RC DoD. Local trees often contain only this README until +a maintainer downloads CI artifacts or records a sample. + +## Local sample (optional, single OS) + +When the environment is already installed (`pip install -e '.[dev]'`): + +```bash +python scripts/record_repro_baseline.py --skip-install +``` + +Writes `docs/baselines/repro--py.json` (for example `repro-windows-py3.13.json`). +That command runs the full harness including `pytest`, so expect minutes, not seconds. + +Validate without recording: + +```bash +python scripts/record_repro_baseline.py --validate-only docs/baselines/repro--py.json +``` + +## CI fill path + +1. Push a non-`[skip ci]` commit that includes `.github/workflows/repro-baseline.yml`. +2. Open the Actions run for `repro-baseline`. +3. Download each matrix cell artifact into this directory (or retain them as release evidence). +4. Optionally commit selected JSON records once maintainers want them in-tree. + +Do not invent multi-OS hashes by hand. diff --git a/schemas/repro.baseline.schema.json b/schemas/repro.baseline.schema.json new file mode 100644 index 0000000..ff32bfb --- /dev/null +++ b/schemas/repro.baseline.schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openverification.dev/schemas/repro.baseline.schema.json", + "title": "OVK Reproducible Baseline Record", + "description": "Machine-readable record of a multi-command local/CI repro baseline (OVK-01).", + "type": "object", + "required": [ + "schema_version", + "python_version", + "os", + "platform", + "network_access", + "started_at", + "completed_at", + "elapsed_seconds", + "checker_availability", + "skipped_tests", + "commands", + "artifacts" + ], + "properties": { + "schema_version": { + "type": "string", + "const": "ovk.repro_baseline.v1" + }, + "python_version": { "type": "string", "minLength": 1 }, + "os": { "type": "string", "minLength": 1 }, + "platform": { "type": "string", "minLength": 1 }, + "runner": { "type": "string" }, + "network_access": { "type": "boolean" }, + "started_at": { "type": "string", "minLength": 1 }, + "completed_at": { "type": "string", "minLength": 1 }, + "elapsed_seconds": { "type": "number", "minimum": 0 }, + "ovk_version": { "type": "string" }, + "git_sha": { "type": ["string", "null"] }, + "checker_availability": { + "type": "object", + "description": "Optional checker availability derived from ovk doctor.", + "additionalProperties": { + "type": "object", + "required": ["available", "message"], + "properties": { + "available": { "type": "boolean" }, + "message": { "type": "string" } + }, + "additionalProperties": true + } + }, + "skipped_tests": { + "type": "array", + "items": { "type": "string" } + }, + "commands": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["argv", "exit_code", "elapsed_seconds"], + "properties": { + "argv": { + "type": "array", + "items": { "type": "string" }, + "minItems": 1 + }, + "exit_code": { "type": "integer" }, + "elapsed_seconds": { "type": "number", "minimum": 0 }, + "stdout_path": { "type": ["string", "null"] }, + "stderr_path": { "type": ["string", "null"] } + }, + "additionalProperties": true + } + }, + "artifacts": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "sha256"], + "properties": { + "path": { "type": "string", "minLength": 1 }, + "sha256": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "bytes": { "type": "integer", "minimum": 0 } + }, + "additionalProperties": true + } + }, + "passed": { "type": "boolean" }, + "notes": { + "type": "array", + "items": { "type": "string" } + } + }, + "additionalProperties": true +} diff --git a/scripts/record_repro_baseline.py b/scripts/record_repro_baseline.py new file mode 100644 index 0000000..4c4fb4f --- /dev/null +++ b/scripts/record_repro_baseline.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python +"""Record a reproducible multi-command baseline for OVK (OVK-01).""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from ovk.core.json_io import read_json_file # noqa: E402 +from ovk.core.schema_validation import validate_against_schema # noqa: E402 +from ovk.paths import schema_path # noqa: E402 + +SCHEMA_VERSION = "ovk.repro_baseline.v1" +OPTIONAL_CHECKERS = ( + "opa", + "z3", + "cedar", + "tlc", + "kani", + "dafny", + "verus", + "lean", + "cbmc", + "alloy", + "cosign", +) + +BASELINE_PIP_INSTALL: tuple[str, ...] = (sys.executable, "-m", "pip", "install", "-e", ".[dev]") +BASELINE_PYTEST: tuple[str, ...] = (sys.executable, "-m", "pytest") +BASELINE_REPAIR_LOOP: tuple[str, ...] = ( + sys.executable, + "examples/repair_loops/ci_secrets/demo_repair_loop.py", +) + + +def _ovk_command(*args: str) -> tuple[str, ...]: + """Prefer the installed ``ovk`` console script; fall back to ``python -m ovk.cli``.""" + ovk_bin = shutil.which("ovk") + if ovk_bin: + return (ovk_bin, *args) + return (sys.executable, "-m", "ovk.cli", *args) + + +def baseline_commands(*, skip_install: bool = False) -> list[tuple[str, ...]]: + commands: list[tuple[str, ...]] = [] + if not skip_install: + commands.append(BASELINE_PIP_INSTALL) + commands.extend( + [ + BASELINE_PYTEST, + _ovk_command("doctor"), + _ovk_command( + "check", + "--changed-files", + "examples/multi_surface/pr_combined.diff", + "--advisory", + ), + _ovk_command("release-preflight"), + BASELINE_REPAIR_LOOP, + ] + ) + return commands + + +SKIPPED_TEST_RE = re.compile(r"^(SKIPPED|skipped)\s+([^\s].*)$") +PYTEST_SKIP_SUMMARY_RE = re.compile(r"(\d+)\s+skipped") + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _probe_network_access(timeout_seconds: float = 2.0) -> bool: + override = os.environ.get("OVK_NETWORK_ACCESS") + if override is not None: + return override.strip().lower() in {"1", "true", "yes", "on"} + try: + urllib.request.urlopen("https://pypi.org/simple/", timeout=timeout_seconds) # noqa: S310 + return True + except (urllib.error.URLError, TimeoutError, OSError): + return False + + +def _git_sha(repo_root: Path) -> str | None: + try: + completed = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repo_root, + check=False, + capture_output=True, + text=True, + ) + except OSError: + return None + if completed.returncode != 0: + return None + return completed.stdout.strip() or None + + +def _ensure_verification_dir(repo_root: Path, logs_dir: Path) -> None: + verification = repo_root / ".verification" + if verification.exists(): + return + argv = list(_ovk_command("init")) + stdout_path = logs_dir / "setup-ovk-init.stdout.log" + stderr_path = logs_dir / "setup-ovk-init.stderr.log" + with stdout_path.open("w", encoding="utf-8") as stdout, stderr_path.open( + "w", encoding="utf-8" + ) as stderr: + subprocess.run(argv, cwd=repo_root, check=False, stdout=stdout, stderr=stderr) + + +def _parse_skipped_tests(pytest_stdout: str) -> list[str]: + skipped: list[str] = [] + for line in pytest_stdout.splitlines(): + match = SKIPPED_TEST_RE.match(line.strip()) + if match: + skipped.append(match.group(2).strip()) + if skipped: + return sorted(set(skipped)) + # Fallback: capture summary count only when individual lines are unavailable. + summary = PYTEST_SKIP_SUMMARY_RE.search(pytest_stdout) + if summary and int(summary.group(1)) > 0: + return [f""] + return [] + + +def _checker_availability_from_doctor(doctor_payload: dict[str, Any] | None) -> dict[str, Any]: + availability: dict[str, Any] = {} + if not doctor_payload: + return availability + checks = doctor_payload.get("checks") or [] + for check in checks: + if not isinstance(check, dict): + continue + name = str(check.get("name") or "") + if name not in OPTIONAL_CHECKERS: + continue + availability[name] = { + "available": bool(check.get("passed")), + "message": str(check.get("message") or ""), + } + return availability + + +def _command_slug(argv: tuple[str, ...]) -> str: + parts = [] + for part in argv: + cleaned = re.sub(r"[^A-Za-z0-9._-]+", "-", Path(part).name if "/" in part or "\\" in part else part) + parts.append(cleaned.strip("-") or "arg") + return "-".join(parts)[:120] + + +def run_command( + argv: tuple[str, ...], + *, + repo_root: Path, + logs_dir: Path, + env: dict[str, str], +) -> dict[str, Any]: + slug = _command_slug(argv) + stdout_path = logs_dir / f"{slug}.stdout.log" + stderr_path = logs_dir / f"{slug}.stderr.log" + started = time.perf_counter() + with stdout_path.open("w", encoding="utf-8") as stdout, stderr_path.open( + "w", encoding="utf-8" + ) as stderr: + completed = subprocess.run( + list(argv), + cwd=repo_root, + check=False, + stdout=stdout, + stderr=stderr, + env=env, + ) + elapsed = time.perf_counter() - started + return { + "argv": [str(part) for part in argv], + "exit_code": int(completed.returncode), + "elapsed_seconds": round(elapsed, 3), + "stdout_path": str(stdout_path.relative_to(repo_root)).replace("\\", "/"), + "stderr_path": str(stderr_path.relative_to(repo_root)).replace("\\", "/"), + } + + +def collect_artifacts(repo_root: Path, extra_globs: list[str]) -> list[dict[str, Any]]: + candidates: list[Path] = [] + default_names = ( + "ovk-evidence.json", + "ovk-pr-comment.md", + "ovk-evidence-quality.json", + "ovk-attestation.json", + "ovk-artifact-manifest.json", + ) + for name in default_names: + path = repo_root / name + if path.is_file(): + candidates.append(path) + for pattern in extra_globs: + candidates.extend(p for p in repo_root.glob(pattern) if p.is_file()) + + artifacts: list[dict[str, Any]] = [] + seen: set[str] = set() + for path in sorted(set(candidates), key=lambda item: str(item)): + rel = str(path.relative_to(repo_root)).replace("\\", "/") + if rel in seen: + continue + seen.add(rel) + artifacts.append( + { + "path": rel, + "sha256": _sha256_file(path), + "bytes": path.stat().st_size, + } + ) + return artifacts + + +def normalize_os_label(system: str | None = None) -> str: + name = (system or platform.system()).lower() + if name.startswith("darwin") or name == "macos": + return "macos" + if name.startswith("win"): + return "windows" + if name.startswith("linux"): + return "linux" + return re.sub(r"[^a-z0-9]+", "-", name).strip("-") or "unknown" + + +def default_output_path(repo_root: Path) -> Path: + py = f"{sys.version_info.major}.{sys.version_info.minor}" + return repo_root / "docs" / "baselines" / f"repro-{normalize_os_label()}-py{py}.json" + + +def validate_baseline_record(record: dict[str, Any]) -> list[str]: + schema_file = schema_path("repro.baseline.schema.json") + if not schema_file.exists(): + # Fall back to repo-relative schema when package data is not synced yet. + schema_file = ROOT / "schemas" / "repro.baseline.schema.json" + schema = read_json_file(schema_file) + report = validate_against_schema(record, schema) + failures = [ + f"{'/'.join(str(part) for part in issue.path) or '$'}: {issue.message}" for issue in report.issues + ] + required = ( + "schema_version", + "python_version", + "os", + "platform", + "network_access", + "started_at", + "completed_at", + "elapsed_seconds", + "checker_availability", + "skipped_tests", + "commands", + "artifacts", + ) + for field in required: + if field not in record: + failures.append(f"missing required field {field!r}") + return failures + + +def record_baseline( + *, + repo_root: Path, + output: Path, + skip_install: bool = False, + artifact_globs: list[str] | None = None, +) -> dict[str, Any]: + logs_dir = repo_root / ".verification" / "repro-baseline-logs" + logs_dir.mkdir(parents=True, exist_ok=True) + + started_at = _utc_now() + wall_started = time.perf_counter() + env = os.environ.copy() + env.setdefault("PYTHONUTF8", "1") + env.setdefault("PIP_DISABLE_PIP_VERSION_CHECK", "1") + + _ensure_verification_dir(repo_root, logs_dir) + + commands_spec = baseline_commands(skip_install=skip_install) + + command_records: list[dict[str, Any]] = [] + doctor_payload: dict[str, Any] | None = None + skipped_tests: list[str] = [] + + for argv in commands_spec: + record = run_command(argv, repo_root=repo_root, logs_dir=logs_dir, env=env) + command_records.append(record) + + stdout_rel = record.get("stdout_path") + if stdout_rel: + stdout_text = (repo_root / stdout_rel).read_text(encoding="utf-8", errors="replace") + if any(part == "pytest" or part.endswith("pytest") for part in argv): + skipped_tests = _parse_skipped_tests(stdout_text) + if "doctor" in argv: + try: + doctor_payload = json.loads(stdout_text) + except json.JSONDecodeError: + doctor_payload = None + + artifacts = collect_artifacts(repo_root, artifact_globs or []) + # Always hash the command logs themselves for provenance. + for path in sorted(logs_dir.glob("*.log")): + rel = str(path.relative_to(repo_root)).replace("\\", "/") + artifacts.append({"path": rel, "sha256": _sha256_file(path), "bytes": path.stat().st_size}) + + completed_at = _utc_now() + elapsed = round(time.perf_counter() - wall_started, 3) + try: + from ovk import __version__ as ovk_version + except Exception: # noqa: BLE001 + ovk_version = "unknown" + + record: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "python_version": platform.python_version(), + "os": normalize_os_label(), + "platform": platform.platform(), + "runner": os.environ.get("RUNNER_OS") or platform.system(), + "network_access": _probe_network_access(), + "started_at": started_at, + "completed_at": completed_at, + "elapsed_seconds": elapsed, + "ovk_version": ovk_version, + "git_sha": _git_sha(repo_root), + "checker_availability": _checker_availability_from_doctor(doctor_payload), + "skipped_tests": skipped_tests, + "commands": command_records, + "artifacts": artifacts, + "passed": all(item.get("exit_code") == 0 for item in command_records), + "notes": [ + "Harness may run `ovk init` once when `.verification/` is missing so `ovk doctor` can pass.", + "Command stdout/stderr captured under .verification/repro-baseline-logs/.", + ], + } + + failures = validate_baseline_record(record) + if failures: + raise ValueError("baseline schema incomplete:\n" + "\n".join(failures)) + + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return record + + +def main() -> int: + parser = argparse.ArgumentParser(description="Record an OVK reproducible baseline") + parser.add_argument("--repo-root", type=Path, default=ROOT) + parser.add_argument("--output", type=Path, default=None) + parser.add_argument( + "--skip-install", + action="store_true", + help="Skip pip install -e '.[dev]' (useful when the environment is already prepared)", + ) + parser.add_argument( + "--artifact-glob", + action="append", + default=[], + help="Extra glob (relative to repo root) to include in artifact hashing", + ) + parser.add_argument( + "--validate-only", + type=Path, + default=None, + help="Validate an existing baseline JSON and exit", + ) + args = parser.parse_args() + repo_root = args.repo_root.resolve() + + if args.validate_only is not None: + payload = read_json_file(args.validate_only.resolve()) + failures = validate_baseline_record(payload) + if failures: + for failure in failures: + print(failure, file=sys.stderr) + return 1 + print(f"baseline schema valid: {args.validate_only}") + return 0 + + output = (args.output or default_output_path(repo_root)).resolve() + try: + record = record_baseline( + repo_root=repo_root, + output=output, + skip_install=args.skip_install, + artifact_globs=list(args.artifact_glob), + ) + except ValueError as error: + print(str(error), file=sys.stderr) + return 1 + + print(f"wrote baseline -> {output}") + print(f"passed={record.get('passed')} elapsed_seconds={record.get('elapsed_seconds')}") + return 0 if record.get("passed") else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_repro_baseline_schema.py b/tests/test_repro_baseline_schema.py new file mode 100644 index 0000000..0d080ab --- /dev/null +++ b/tests/test_repro_baseline_schema.py @@ -0,0 +1,58 @@ +"""Tests for reproducible baseline schema validation (OVK-01).""" + +from __future__ import annotations + +from scripts.record_repro_baseline import validate_baseline_record + + +def _complete_baseline(**overrides: object) -> dict: + payload = { + "schema_version": "ovk.repro_baseline.v1", + "python_version": "3.12.0", + "os": "linux", + "platform": "Linux-test", + "network_access": True, + "started_at": "2026-07-25T00:00:00Z", + "completed_at": "2026-07-25T00:01:00Z", + "elapsed_seconds": 60.0, + "checker_availability": { + "opa": {"available": False, "message": "opa not found in PATH (optional)"} + }, + "skipped_tests": [], + "commands": [ + { + "argv": ["pytest"], + "exit_code": 0, + "elapsed_seconds": 1.0, + } + ], + "artifacts": [ + { + "path": "ovk-evidence.json", + "sha256": "a" * 64, + "bytes": 12, + } + ], + } + payload.update(overrides) + return payload + + +def test_validate_baseline_record_accepts_complete_payload() -> None: + assert validate_baseline_record(_complete_baseline()) == [] + + +def test_validate_baseline_record_rejects_missing_fields() -> None: + payload = _complete_baseline() + del payload["artifacts"] + failures = validate_baseline_record(payload) + assert failures + assert any("artifacts" in item for item in failures) + + +def test_validate_baseline_record_rejects_bad_digest() -> None: + payload = _complete_baseline( + artifacts=[{"path": "x", "sha256": "not-a-digest"}], + ) + failures = validate_baseline_record(payload) + assert failures From 5fc320fb96195b285e0a021efce9d75b87e04be9 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:08:44 -0700 Subject: [PATCH 02/19] Make capability and template registry normative (OVK-PR1). Tighten the capability schema and publish a template registry so stable backends declare enforceable limits, versions, and eligibility instead of prose-only claims. --- adapters/alloy/capability.json | 51 +- adapters/cbmc/capability.json | 52 +- adapters/cedar/capability.json | 53 +- adapters/dafny/capability.json | 51 +- adapters/kani/capability.json | 49 +- adapters/lean/capability.json | 49 +- adapters/opa/capability.json | 29 +- adapters/tla/capability.json | 51 +- adapters/verus/capability.json | 49 +- adapters/z3/capability.json | 29 +- ovk/core/capabilities.py | 130 +- schemas/verification.capability.schema.json | 122 +- scripts/build_template_registry.py | 177 + scripts/render_capability_tables.py | 206 + scripts/validate_capabilities.py | 57 +- templates/registry/README.md | 25 + templates/registry/bridge.json | 31 + templates/registry/entries.json | 4608 +++++++++++++++++++ tests/test_capability_registry_normative.py | 196 + 19 files changed, 5948 insertions(+), 67 deletions(-) create mode 100644 scripts/build_template_registry.py create mode 100644 scripts/render_capability_tables.py create mode 100644 templates/registry/README.md create mode 100644 templates/registry/bridge.json create mode 100644 templates/registry/entries.json create mode 100644 tests/test_capability_registry_normative.py diff --git a/adapters/alloy/capability.json b/adapters/alloy/capability.json index 9e4b2f6..b54a1a6 100644 --- a/adapters/alloy/capability.json +++ b/adapters/alloy/capability.json @@ -1,19 +1,56 @@ { "capability_id": "alloy-relational-v1", - "tool": {"name": "alloy", "adapter": "ovk-adapter-alloy", "adapter_version": "0.1.0"}, + "checker_id": "alloy", + "version": "0.1.0", + "implementation": "ovk-adapter-alloy", + "input_contract": "Alloy relational-model contract payload for authorization, infrastructure, or deployment topology.", + "output_contract": "ovk.result.v1 with instance counterexamples (deterministic evaluator today)", + "claim_class": "bounded_model_check", + "tool": { + "name": "alloy", + "adapter": "ovk-adapter-alloy", + "adapter_version": "0.1.0" + }, "backend_class": "model_checker", - "input_languages": ["alloy"], - "supported_domains": ["authorization", "infrastructure", "deployment"], - "supported_property_kinds": ["invariant", "access_control", "forbidden_configuration"], + "input_languages": [ + "alloy" + ], + "supported_domains": [ + "authorization", + "infrastructure", + "deployment" + ], + "supported_property_kinds": [ + "invariant", + "access_control", + "forbidden_configuration" + ], "guarantee": { "type": "bounded_model_check", "meaning_of_pass": "Alloy found no counterexample within scope.", "meaning_of_fail": "Alloy found a violating instance.", "meaning_of_unknown": "Alloy unavailable or model incomplete." }, - "assumptions": ["Relational model captures relevant configuration."], - "limits": ["Finite scope analysis only."], + "assumptions": [ + "Relational model captures relevant configuration." + ], + "trusted_components": [ + "deterministic relational-model contract evaluator" + ], + "limits": [ + "Finite scope analysis only." + ], + "failure_semantics": "Unavailable Alloy or incomplete model maps to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "timeout_behavior": "unknown", + "unsupported_semantics": "Native Alloy analysis is not implemented; finite scope analysis only under the deterministic evaluator.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, "result_format": "ovk.result.v1", "counterexample_format": "instance", - "timeout_behavior": "unknown" + "conformance": { + "suite": "conformance/manifest.json" + } } diff --git a/adapters/cbmc/capability.json b/adapters/cbmc/capability.json index 7102212..73b2f5e 100644 --- a/adapters/cbmc/capability.json +++ b/adapters/cbmc/capability.json @@ -1,19 +1,57 @@ { "capability_id": "cbmc-c-v1", - "tool": {"name": "cbmc", "adapter": "ovk-adapter-cbmc", "adapter_version": "0.1.0"}, + "checker_id": "cbmc", + "version": "0.1.0", + "implementation": "ovk-adapter-cbmc", + "input_contract": "C/GOTO harness (explicit or OVK template-generated) with stated loop/memory bounds.", + "output_contract": "ovk.result.v1 with trace counterexamples", + "claim_class": "bounded_model_check", + "tool": { + "name": "cbmc", + "adapter": "ovk-adapter-cbmc", + "adapter_version": "0.1.0" + }, "backend_class": "model_checker", - "input_languages": ["c", "goto"], - "supported_domains": ["data_boundary", "infrastructure"], - "supported_property_kinds": ["safety", "invariant"], + "input_languages": [ + "c", + "goto" + ], + "supported_domains": [ + "data_boundary", + "infrastructure" + ], + "supported_property_kinds": [ + "safety", + "invariant" + ], "guarantee": { "type": "bounded_model_check", "meaning_of_pass": "CBMC found no violation within bounds.", "meaning_of_fail": "CBMC reported a reachable bug.", "meaning_of_unknown": "CBMC unavailable or harness incomplete." }, - "assumptions": ["Harness bounds loops and memory."], - "limits": ["Bounded verification only."], + "assumptions": [ + "Harness bounds loops and memory." + ], + "trusted_components": [ + "cbmc binary", + "harness generator or supplied harness", + "bound configuration" + ], + "limits": [ + "Bounded verification only." + ], + "failure_semantics": "CBMC execution error maps to error; missing harness maps to unknown.", + "timeout_semantics": "unknown", + "timeout_behavior": "unknown", + "unsupported_semantics": "Bounded verification only; does not prove unbounded properties of project source unless compiled into the harness.", + "determinism_status": "tool_dependent", + "release_status": "preview", + "owner": "ovk-maintainers", + "native_execution": true, "result_format": "ovk.result.v1", "counterexample_format": "trace", - "timeout_behavior": "unknown" + "conformance": { + "suite": "conformance/manifest.json" + } } diff --git a/adapters/cedar/capability.json b/adapters/cedar/capability.json index 6c75295..c557639 100644 --- a/adapters/cedar/capability.json +++ b/adapters/cedar/capability.json @@ -1,19 +1,58 @@ { "capability_id": "cedar-policy-v1", - "tool": {"name": "cedar", "adapter": "ovk-adapter-cedar", "adapter_version": "0.1.0"}, + "checker_id": "cedar", + "version": "0.1.0", + "implementation": "ovk-adapter-cedar", + "input_contract": "Cedar-shaped JSON authorization input; CLI version probe only until native policy eval ships.", + "output_contract": "ovk.result.v1 with policy_violation counterexamples (deterministic evaluator today)", + "claim_class": "policy_evaluation", + "tool": { + "name": "cedar", + "adapter": "ovk-adapter-cedar", + "adapter_version": "0.1.0" + }, "backend_class": "policy_engine", - "input_languages": ["cedar", "json"], - "supported_domains": ["authorization", "infrastructure", "agent_authority"], - "supported_property_kinds": ["access_control", "forbidden_configuration", "safety"], + "input_languages": [ + "cedar", + "json" + ], + "supported_domains": [ + "authorization", + "infrastructure", + "agent_authority" + ], + "supported_property_kinds": [ + "access_control", + "forbidden_configuration", + "safety" + ], "guarantee": { "type": "policy_evaluation", "meaning_of_pass": "Cedar policy permits only authorized actions.", "meaning_of_fail": "Cedar policy reports a forbidden authorization.", "meaning_of_unknown": "Cedar binary unavailable or input incomplete." }, - "assumptions": ["Authorization abstraction matches repository routes."], - "limits": ["Does not verify runtime middleware behavior."], + "assumptions": [ + "Authorization abstraction matches repository routes." + ], + "trusted_components": [ + "deterministic Cedar-shaped evaluator", + "optional cedar CLI for version probe" + ], + "limits": [ + "Does not verify runtime middleware behavior." + ], + "failure_semantics": "Missing binary or incomplete input maps to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "timeout_behavior": "unknown", + "unsupported_semantics": "Native Cedar policy evaluation is not implemented; does not verify runtime middleware behavior.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, "result_format": "ovk.result.v1", "counterexample_format": "policy_violation", - "timeout_behavior": "unknown" + "conformance": { + "suite": "conformance/manifest.json" + } } diff --git a/adapters/dafny/capability.json b/adapters/dafny/capability.json index 62feeb7..3b27af0 100644 --- a/adapters/dafny/capability.json +++ b/adapters/dafny/capability.json @@ -1,19 +1,56 @@ { "capability_id": "dafny-proof-v1", - "tool": {"name": "dafny", "adapter": "ovk-adapter-dafny", "adapter_version": "0.1.0"}, + "checker_id": "dafny", + "version": "0.1.0", + "implementation": "ovk-adapter-dafny", + "input_contract": "Dafny proof-obligation contract payload for authorization/data/infra invariants.", + "output_contract": "ovk.result.v1 with proof_failure counterexamples (deterministic evaluator today)", + "claim_class": "proof_obligation", + "tool": { + "name": "dafny", + "adapter": "ovk-adapter-dafny", + "adapter_version": "0.1.0" + }, "backend_class": "proof_assistant", - "input_languages": ["dafny"], - "supported_domains": ["authorization", "data_boundary", "infrastructure"], - "supported_property_kinds": ["invariant", "safety", "access_control"], + "input_languages": [ + "dafny" + ], + "supported_domains": [ + "authorization", + "data_boundary", + "infrastructure" + ], + "supported_property_kinds": [ + "invariant", + "safety", + "access_control" + ], "guarantee": { "type": "proof_obligation", "meaning_of_pass": "Dafny verified the obligation.", "meaning_of_fail": "Dafny reported unproved obligations.", "meaning_of_unknown": "Dafny unavailable or obligation incomplete." }, - "assumptions": ["Specification matches intended behavior."], - "limits": ["Requires complete specifications."], + "assumptions": [ + "Specification matches intended behavior." + ], + "trusted_components": [ + "deterministic proof-obligation contract evaluator" + ], + "limits": [ + "Requires complete specifications." + ], + "failure_semantics": "Unavailable Dafny or incomplete obligation maps to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "timeout_behavior": "unknown", + "unsupported_semantics": "Native Dafny verification is not implemented; requires complete specifications.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, "result_format": "ovk.result.v1", "counterexample_format": "proof_failure", - "timeout_behavior": "unknown" + "conformance": { + "suite": "conformance/manifest.json" + } } diff --git a/adapters/kani/capability.json b/adapters/kani/capability.json index be43a15..bb21c54 100644 --- a/adapters/kani/capability.json +++ b/adapters/kani/capability.json @@ -1,19 +1,54 @@ { "capability_id": "kani-rust-v1", - "tool": {"name": "kani", "adapter": "ovk-adapter-kani", "adapter_version": "0.1.0"}, + "checker_id": "kani", + "version": "0.1.0", + "implementation": "ovk-adapter-kani", + "input_contract": "Rust harness contract payload describing bounded unsafe/auth paths.", + "output_contract": "ovk.result.v1 with trace counterexamples (deterministic evaluator today)", + "claim_class": "bounded_model_check", + "tool": { + "name": "kani", + "adapter": "ovk-adapter-kani", + "adapter_version": "0.1.0" + }, "backend_class": "model_checker", - "input_languages": ["rust"], - "supported_domains": ["authorization", "data_boundary"], - "supported_property_kinds": ["safety", "invariant"], + "input_languages": [ + "rust" + ], + "supported_domains": [ + "authorization", + "data_boundary" + ], + "supported_property_kinds": [ + "safety", + "invariant" + ], "guarantee": { "type": "bounded_model_check", "meaning_of_pass": "Kani found no safety violation in bounded harness.", "meaning_of_fail": "Kani reported a reachable violation.", "meaning_of_unknown": "Kani unavailable or harness incomplete." }, - "assumptions": ["Rust harness captures relevant unsafe paths."], - "limits": ["Bounded loops and memory model approximations."], + "assumptions": [ + "Rust harness captures relevant unsafe paths." + ], + "trusted_components": [ + "deterministic Rust-harness contract evaluator" + ], + "limits": [ + "Bounded loops and memory model approximations." + ], + "failure_semantics": "Unavailable Kani or incomplete harness maps to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "timeout_behavior": "unknown", + "unsupported_semantics": "Native Kani execution is not implemented; bounded loops and memory model approximations only.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, "result_format": "ovk.result.v1", "counterexample_format": "trace", - "timeout_behavior": "unknown" + "conformance": { + "suite": "conformance/manifest.json" + } } diff --git a/adapters/lean/capability.json b/adapters/lean/capability.json index 71fb754..fc29dab 100644 --- a/adapters/lean/capability.json +++ b/adapters/lean/capability.json @@ -1,19 +1,54 @@ { "capability_id": "lean-proof-v1", - "tool": {"name": "lean", "adapter": "ovk-adapter-lean", "adapter_version": "0.1.0"}, + "checker_id": "lean", + "version": "0.1.0", + "implementation": "ovk-adapter-lean", + "input_contract": "Lean theorem-obligation contract payload for authorization or agent-authority invariants.", + "output_contract": "ovk.result.v1 with proof_failure counterexamples (deterministic evaluator today)", + "claim_class": "proof_obligation", + "tool": { + "name": "lean", + "adapter": "ovk-adapter-lean", + "adapter_version": "0.1.0" + }, "backend_class": "proof_assistant", - "input_languages": ["lean"], - "supported_domains": ["authorization", "agent_authority"], - "supported_property_kinds": ["invariant", "safety"], + "input_languages": [ + "lean" + ], + "supported_domains": [ + "authorization", + "agent_authority" + ], + "supported_property_kinds": [ + "invariant", + "safety" + ], "guarantee": { "type": "proof_obligation", "meaning_of_pass": "Lean proof completed without sorry.", "meaning_of_fail": "Lean reported proof obligations.", "meaning_of_unknown": "Lean unavailable or proof incomplete." }, - "assumptions": ["Formal model matches runtime policy."], - "limits": ["Proof effort may exceed CI budgets."], + "assumptions": [ + "Formal model matches runtime policy." + ], + "trusted_components": [ + "deterministic theorem-obligation contract evaluator" + ], + "limits": [ + "Proof effort may exceed CI budgets." + ], + "failure_semantics": "Unavailable Lean or incomplete proof maps to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "timeout_behavior": "unknown", + "unsupported_semantics": "Native Lean checking is not implemented; proof effort may exceed CI budgets.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, "result_format": "ovk.result.v1", "counterexample_format": "proof_failure", - "timeout_behavior": "unknown" + "conformance": { + "suite": "conformance/manifest.json" + } } diff --git a/adapters/opa/capability.json b/adapters/opa/capability.json index 4557827..e7e59e1 100644 --- a/adapters/opa/capability.json +++ b/adapters/opa/capability.json @@ -1,12 +1,22 @@ { "capability_id": "opa-policy-v1", + "checker_id": "opa", + "version": "0.1.0", + "implementation": "ovk-adapter-opa", + "input_contract": "Structured JSON/YAML input plus Rego policy selected by intent/template.", + "output_contract": "ovk.result.v1 with policy_violation counterexamples", + "claim_class": "policy_evaluation", "tool": { "name": "opa", "adapter": "ovk-adapter-opa", "adapter_version": "0.1.0" }, "backend_class": "policy_engine", - "input_languages": ["rego", "json", "yaml"], + "input_languages": [ + "rego", + "json", + "yaml" + ], "supported_domains": [ "ci_cd", "infrastructure", @@ -31,11 +41,26 @@ "Input extraction faithfully represents the repository state relevant to the policy.", "Policy templates accurately encode the intended invariant." ], + "trusted_components": [ + "opa binary", + "selected Rego policy templates", + "input extraction / compiler" + ], "limits": [ "Does not prove properties of arbitrary program execution.", "Does not establish unbounded liveness." ], + "failure_semantics": "Adapter or OPA execution failure maps to error; malformed input maps to unknown.", + "timeout_semantics": "unknown", + "timeout_behavior": "unknown", + "unsupported_semantics": "Does not prove properties of arbitrary program execution; unsupported inputs yield unknown.", + "determinism_status": "tool_dependent", + "release_status": "preview", + "owner": "ovk-maintainers", + "native_execution": true, "result_format": "ovk.result.v1", "counterexample_format": "policy_violation", - "timeout_behavior": "unknown" + "conformance": { + "suite": "conformance/manifest.json" + } } diff --git a/adapters/tla/capability.json b/adapters/tla/capability.json index bb8b736..7378229 100644 --- a/adapters/tla/capability.json +++ b/adapters/tla/capability.json @@ -1,19 +1,56 @@ { "capability_id": "tla-plus-v1", - "tool": {"name": "tla+", "adapter": "ovk-adapter-tla", "adapter_version": "0.1.0"}, + "checker_id": "tla+", + "version": "0.1.0", + "implementation": "ovk-adapter-tla", + "input_contract": "TLA+/CFG state-machine contract payload derived from deployment or CI workflow abstractions.", + "output_contract": "ovk.result.v1 with trace counterexamples (deterministic evaluator today)", + "claim_class": "bounded_model_check", + "tool": { + "name": "tla+", + "adapter": "ovk-adapter-tla", + "adapter_version": "0.1.0" + }, "backend_class": "model_checker", - "input_languages": ["tla", "cfg"], - "supported_domains": ["deployment", "ci_cd"], - "supported_property_kinds": ["invariant", "safety", "liveness"], + "input_languages": [ + "tla", + "cfg" + ], + "supported_domains": [ + "deployment", + "ci_cd" + ], + "supported_property_kinds": [ + "invariant", + "safety", + "liveness" + ], "guarantee": { "type": "bounded_model_check", "meaning_of_pass": "TLC found no counterexample within configured bounds.", "meaning_of_fail": "TLC found a violating trace.", "meaning_of_unknown": "TLC unavailable or state space incomplete." }, - "assumptions": ["State machine abstraction matches deployment workflow."], - "limits": ["Bounded exploration only."], + "assumptions": [ + "State machine abstraction matches deployment workflow." + ], + "trusted_components": [ + "deterministic state-machine contract evaluator" + ], + "limits": [ + "Bounded exploration only." + ], + "failure_semantics": "Unavailable TLC or incomplete model maps to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "timeout_behavior": "unknown", + "unsupported_semantics": "TLC execution is not implemented; bounded exploration only under the deterministic contract evaluator.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, "result_format": "ovk.result.v1", "counterexample_format": "trace", - "timeout_behavior": "unknown" + "conformance": { + "suite": "conformance/manifest.json" + } } diff --git a/adapters/verus/capability.json b/adapters/verus/capability.json index 3903fa9..6441ae2 100644 --- a/adapters/verus/capability.json +++ b/adapters/verus/capability.json @@ -1,19 +1,54 @@ { "capability_id": "verus-rust-v1", - "tool": {"name": "verus", "adapter": "ovk-adapter-verus", "adapter_version": "0.1.0"}, + "checker_id": "verus", + "version": "0.1.0", + "implementation": "ovk-adapter-verus", + "input_contract": "Verified-Rust harness contract payload with Verus-style annotations.", + "output_contract": "ovk.result.v1 with proof_failure counterexamples (deterministic evaluator today)", + "claim_class": "proof_obligation", + "tool": { + "name": "verus", + "adapter": "ovk-adapter-verus", + "adapter_version": "0.1.0" + }, "backend_class": "proof_assistant", - "input_languages": ["rust"], - "supported_domains": ["authorization", "data_boundary"], - "supported_property_kinds": ["invariant", "safety"], + "input_languages": [ + "rust" + ], + "supported_domains": [ + "authorization", + "data_boundary" + ], + "supported_property_kinds": [ + "invariant", + "safety" + ], "guarantee": { "type": "proof_obligation", "meaning_of_pass": "Verus verified the Rust harness.", "meaning_of_fail": "Verus reported verification failures.", "meaning_of_unknown": "Verus unavailable or harness incomplete." }, - "assumptions": ["Harness models security-critical code paths."], - "limits": ["Verification bounded by annotations."], + "assumptions": [ + "Harness models security-critical code paths." + ], + "trusted_components": [ + "deterministic verified-Rust contract evaluator" + ], + "limits": [ + "Verification bounded by annotations." + ], + "failure_semantics": "Unavailable Verus or incomplete harness maps to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "timeout_behavior": "unknown", + "unsupported_semantics": "Native Verus verification is not implemented; verification bounded by annotations.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, "result_format": "ovk.result.v1", "counterexample_format": "proof_failure", - "timeout_behavior": "unknown" + "conformance": { + "suite": "conformance/manifest.json" + } } diff --git a/adapters/z3/capability.json b/adapters/z3/capability.json index a5ce9f4..43da958 100644 --- a/adapters/z3/capability.json +++ b/adapters/z3/capability.json @@ -1,12 +1,22 @@ { "capability_id": "z3-smt-v1", + "checker_id": "z3", + "version": "0.1.0", + "implementation": "ovk-adapter-z3", + "input_contract": "SMT-LIB2 / Python-Z3 / JSON constraint encodings compiled from authorization or infrastructure abstractions.", + "output_contract": "ovk.result.v1 with smt_model counterexamples; evidence must state query polarity", + "claim_class": "smt_satisfiability", "tool": { "name": "z3", "adapter": "ovk-adapter-z3", "adapter_version": "0.1.0" }, "backend_class": "smt_solver", - "input_languages": ["smtlib2", "python-z3", "json-constraints"], + "input_languages": [ + "smtlib2", + "python-z3", + "json-constraints" + ], "supported_domains": [ "authorization", "infrastructure", @@ -30,11 +40,26 @@ "The finite abstraction captures the relevant behavior of the changed system.", "The query polarity is recorded in the proof obligation." ], + "trusted_components": [ + "z3 solver", + "neutral obligation compiler", + "encoded abstraction" + ], "limits": [ "Does not prove properties outside the encoded abstraction.", "May return unknown for unsupported theories or timeout." ], + "failure_semantics": "Solver/adapter crash maps to error; solver unknown and incomplete encodings map to unknown.", + "timeout_semantics": "unknown", + "timeout_behavior": "unknown", + "unsupported_semantics": "Does not prove properties outside the encoded abstraction; unsupported theories yield unknown.", + "determinism_status": "tool_dependent", + "release_status": "preview", + "owner": "ovk-maintainers", + "native_execution": true, "result_format": "ovk.result.v1", "counterexample_format": "smt_model", - "timeout_behavior": "unknown" + "conformance": { + "suite": "conformance/manifest.json" + } } diff --git a/ovk/core/capabilities.py b/ovk/core/capabilities.py index c1e0a77..6e462e8 100644 --- a/ovk/core/capabilities.py +++ b/ovk/core/capabilities.py @@ -1,4 +1,4 @@ -"""Capability manifest loading and validation.""" +"""Capability manifest loading and validation (normative claim registry).""" from __future__ import annotations @@ -6,6 +6,110 @@ from pathlib import Path from typing import Any +from ovk.core.execution_models import VALID_RELEASE_STATUSES + +REQUIRED_NORMATIVE_FIELDS: tuple[str, ...] = ( + "checker_id", + "version", + "implementation", + "input_contract", + "output_contract", + "claim_class", + "assumptions", + "trusted_components", + "failure_semantics", + "timeout_semantics", + "unsupported_semantics", + "determinism_status", + "release_status", + "owner", +) + +VALID_TIMEOUT_SEMANTICS: frozenset[str] = frozenset({"unknown", "error", "fail"}) +VALID_DETERMINISM_STATUSES: frozenset[str] = frozenset( + {"deterministic", "tool_dependent", "non_deterministic", "unknown"} +) + +# Adapters without native execution must stay at or below these statuses. +NON_NATIVE_MAX_RELEASE_STATUS: frozenset[str] = frozenset({"preview", "experimental", "disabled"}) +NATIVE_CANDIDATE_CHECKERS: frozenset[str] = frozenset({"opa", "z3", "cbmc"}) + + +def validate_capability_manifest( + manifest: dict[str, Any], + *, + source: str = "manifest", + require_stable_conformance: bool = True, + repo_root: Path | None = None, +) -> list[str]: + """Return validation failure messages for one capability / claim registry entry.""" + failures: list[str] = [] + if not isinstance(manifest, dict): + return [f"{source}: capability manifest must be a JSON object"] + + for field in REQUIRED_NORMATIVE_FIELDS: + if field not in manifest: + failures.append(f"{source}: missing required field {field!r}") + continue + value = manifest[field] + if field in {"assumptions", "trusted_components"}: + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + failures.append(f"{source}: {field} must be an array of strings") + continue + if not isinstance(value, str) or not value.strip(): + failures.append(f"{source}: {field} must be a non-empty string") + + release_status = manifest.get("release_status") + if isinstance(release_status, str) and release_status not in VALID_RELEASE_STATUSES: + failures.append( + f"{source}: unknown release_status {release_status!r}; " + f"expected one of {sorted(VALID_RELEASE_STATUSES)}" + ) + + timeout_semantics = manifest.get("timeout_semantics") + if isinstance(timeout_semantics, str) and timeout_semantics not in VALID_TIMEOUT_SEMANTICS: + failures.append( + f"{source}: unknown timeout_semantics {timeout_semantics!r}; " + f"expected one of {sorted(VALID_TIMEOUT_SEMANTICS)}" + ) + + determinism = manifest.get("determinism_status") + if isinstance(determinism, str) and determinism not in VALID_DETERMINISM_STATUSES: + failures.append( + f"{source}: unknown determinism_status {determinism!r}; " + f"expected one of {sorted(VALID_DETERMINISM_STATUSES)}" + ) + + # Honesty: stable requires full seven-item conformance (OVK-PR4). + # Non-native adapters must stay at preview/experimental/disabled. + native = manifest.get("native_execution") + checker = str(manifest.get("checker_id") or manifest.get("tool", {}).get("name") or "") + if release_status == "stable": + if checker not in NATIVE_CANDIDATE_CHECKERS and native is not True: + failures.append( + f"{source}: only native-execution candidates may use release_status 'stable' " + f"(checker={checker!r})" + ) + elif require_stable_conformance: + from ovk.core.adapter_conformance import is_fully_conformant + + if not is_fully_conformant(checker, root=repo_root): + failures.append( + f"{source}: release_status 'stable' requires full seven-item " + f"adapter conformance (OVK-PR4) for {checker!r}" + ) + elif native is False or (native is None and checker not in NATIVE_CANDIDATE_CHECKERS): + if ( + isinstance(release_status, str) + and release_status not in NON_NATIVE_MAX_RELEASE_STATUS + and release_status in VALID_RELEASE_STATUSES + ): + failures.append( + f"{source}: non-native checker {checker!r} cannot use release_status {release_status!r}" + ) + + return failures + class CapabilityRegistry: """Filesystem-backed registry for backend capability manifests.""" @@ -14,12 +118,18 @@ def __init__(self, manifests: list[dict[str, Any]] | None = None) -> None: self._manifests = manifests or [] @classmethod - def from_directory(cls, path: Path) -> "CapabilityRegistry": + def from_directory(cls, path: Path, *, validate: bool = True) -> "CapabilityRegistry": manifests: list[dict[str, Any]] = [] if not path.exists(): return cls(manifests) + failures: list[str] = [] for manifest_path in sorted(path.rglob("capability.json")): - manifests.append(json.loads(manifest_path.read_text(encoding="utf-8"))) + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + if validate: + failures.extend(validate_capability_manifest(payload, source=str(manifest_path))) + manifests.append(payload) + if failures: + raise ValueError("capability registry validation failed:\n" + "\n".join(failures)) return cls(manifests) def all(self) -> list[dict[str, Any]]: @@ -31,8 +141,22 @@ def by_tool(self, tool_name: str) -> dict[str, Any] | None: return manifest return None + def by_checker_id(self, checker_id: str) -> dict[str, Any] | None: + for manifest in self._manifests: + if manifest.get("checker_id") == checker_id: + return manifest + return None + def supporting_domain(self, domain: str) -> list[dict[str, Any]]: return [m for m in self._manifests if domain in m.get("supported_domains", [])] def supporting_property_kind(self, property_kind: str) -> list[dict[str, Any]]: return [m for m in self._manifests if property_kind in m.get("supported_property_kinds", [])] + + def validate_all(self) -> list[str]: + """Validate every loaded manifest; return failure messages.""" + failures: list[str] = [] + for manifest in self._manifests: + checker = str(manifest.get("checker_id") or manifest.get("capability_id") or "unknown") + failures.extend(validate_capability_manifest(manifest, source=checker)) + return failures diff --git a/schemas/verification.capability.schema.json b/schemas/verification.capability.schema.json index f9831d8..1e85d7a 100644 --- a/schemas/verification.capability.schema.json +++ b/schemas/verification.capability.schema.json @@ -2,10 +2,62 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://openverification.dev/schemas/verification.capability.schema.json", "title": "Verification Capability Manifest", + "description": "Normative capability / claim registry entry for OVK checkers and adapters (OVK-02).", "type": "object", - "required": ["capability_id", "tool", "backend_class", "supported_domains", "supported_property_kinds", "guarantee"], + "required": [ + "capability_id", + "tool", + "backend_class", + "supported_domains", + "supported_property_kinds", + "guarantee", + "checker_id", + "version", + "implementation", + "input_contract", + "output_contract", + "claim_class", + "assumptions", + "trusted_components", + "failure_semantics", + "timeout_semantics", + "unsupported_semantics", + "determinism_status", + "release_status", + "owner" + ], "properties": { "capability_id": { "type": "string", "minLength": 1 }, + "checker_id": { + "type": "string", + "minLength": 1, + "description": "Stable public checker identifier used in evidence and routing." + }, + "version": { + "type": "string", + "minLength": 1, + "description": "Capability / claim contract version for this checker entry." + }, + "implementation": { + "type": "string", + "minLength": 1, + "description": "Adapter or module that implements the checker." + }, + "input_contract": { + "type": "string", + "minLength": 1, + "description": "What inputs the checker accepts and how they must be prepared." + }, + "output_contract": { + "type": "string", + "minLength": 1, + "description": "Normalized output / evidence shape produced by the checker." + }, + "claim_class": { + "type": "string", + "minLength": 1, + "description": "Class of claim established on pass (maps from guarantee.type)." + }, "tool": { "type": "object", "required": ["name", "adapter", "adapter_version"], @@ -19,7 +71,16 @@ }, "backend_class": { "type": "string", - "enum": ["policy_engine", "smt_solver", "model_checker", "bounded_model_checker", "proof_assistant", "static_analyzer", "runtime_monitor", "custom"] + "enum": [ + "policy_engine", + "smt_solver", + "model_checker", + "bounded_model_checker", + "proof_assistant", + "static_analyzer", + "runtime_monitor", + "custom" + ] }, "input_languages": { "type": "array", "items": { "type": "string" } }, "supported_domains": { "type": "array", "items": { "type": "string" } }, @@ -36,10 +97,65 @@ "additionalProperties": true }, "assumptions": { "type": "array", "items": { "type": "string" } }, + "trusted_components": { + "type": "array", + "items": { "type": "string" }, + "description": "Components that must be trusted for the claim to hold." + }, "limits": { "type": "array", "items": { "type": "string" } }, + "failure_semantics": { + "type": "string", + "minLength": 1, + "description": "How tool/adapter failure is mapped into OVK claim status." + }, + "timeout_semantics": { + "type": "string", + "enum": ["unknown", "error", "fail"], + "description": "Claim status used when the checker times out." + }, + "unsupported_semantics": { + "type": "string", + "minLength": 1, + "description": "What remains outside the claim / how unsupported inputs are reported." + }, + "determinism_status": { + "type": "string", + "enum": ["deterministic", "tool_dependent", "non_deterministic", "unknown"] + }, + "release_status": { + "type": "string", + "enum": ["stable", "preview", "experimental", "disabled"], + "description": "Public adoption honesty status. stable requires full conformance (OVK-PR4)." + }, + "owner": { + "type": "string", + "minLength": 1, + "description": "Maintainer or team responsible for this checker entry." + }, "result_format": { "type": "string" }, "counterexample_format": { "type": "string" }, - "timeout_behavior": { "type": "string", "enum": ["unknown", "error", "fail"] } + "timeout_behavior": { + "type": "string", + "enum": ["unknown", "error", "fail"], + "description": "Deprecated alias of timeout_semantics; retained for older readers." + }, + "native_execution": { + "type": "boolean", + "description": "True when a native binary/solver path can determine evidence." + }, + "conformance": { + "type": "object", + "description": "Pointer to the seven-item adapter conformance suite (OVK-PR4 / OVK-05).", + "properties": { + "suite": { + "type": "string", + "minLength": 1, + "description": "Path to conformance/manifest.json relative to the adapter directory." + } + }, + "required": ["suite"], + "additionalProperties": true + } }, "additionalProperties": true } diff --git a/scripts/build_template_registry.py b/scripts/build_template_registry.py new file mode 100644 index 0000000..6219d7f --- /dev/null +++ b/scripts/build_template_registry.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python +"""Build templates/registry/entries.json from template conformance + bridge mapping.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from ovk.core.capabilities import validate_capability_manifest # noqa: E402 +from ovk.core.json_io import read_json_file # noqa: E402 + +BRIDGE_PATH = ROOT / "templates" / "registry" / "bridge.json" +CONFORMANCE_PATH = ROOT / "docs" / "benchmarks" / "template-conformance.json" +OUTPUT_PATH = ROOT / "templates" / "registry" / "entries.json" + + +def _claim_class(bridge: dict[str, Any], property_kind: str, claimed_backends: list[str]) -> str: + mapping = bridge.get("default_claim_class_by_property_kind") or {} + if property_kind in mapping: + return str(mapping[property_kind]) + if claimed_backends: + return f"template_claim:{claimed_backends[0]}" + return "template_property_claim" + + +def _release_status(bridge: dict[str, Any], row: dict[str, Any]) -> str: + v2 = str(row.get("conformance_status_v2") or "") + v2_map = bridge.get("conformance_status_v2_to_release_status") or {} + if v2 in v2_map: + return str(v2_map[v2]) + prod = str(row.get("production_status") or "catalog_only") + prod_map = bridge.get("production_status_to_release_status") or {} + return str(prod_map.get(prod, "experimental")) + + +def build_entries( + *, + conformance: dict[str, Any], + bridge: dict[str, Any], +) -> dict[str, Any]: + rows = conformance.get("templates") or conformance.get("rows") or [] + if not isinstance(rows, list): + raise ValueError("template conformance matrix missing templates/rows array") + + entries: list[dict[str, Any]] = [] + for row in rows: + if not isinstance(row, dict): + continue + intent_id = str(row.get("intent_id") or "").strip() + if not intent_id: + continue + property_kind = str(row.get("property_kind") or "invariant") + claimed_backends = [str(item) for item in (row.get("claimed_backends") or [])] + domain = str(row.get("domain") or "unknown") + version = str(row.get("version") or "0.0.0") + release_status = _release_status(bridge, row) + claim_class = _claim_class(bridge, property_kind, claimed_backends) + entry = { + "capability_id": f"template-{intent_id}-v1", + "checker_id": f"template:{intent_id}", + "version": version, + "implementation": str(row.get("path") or f"templates/{domain}/{intent_id}.intent.json"), + "input_contract": ( + f"Intent template {intent_id} over domain {domain}; " + "materials supplied by lane compilers / examples." + ), + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "claim_class": claim_class, + "tool": { + "name": f"template:{intent_id}", + "adapter": "ovk-template-registry", + "adapter_version": version, + }, + "backend_class": "custom", + "supported_domains": [domain], + "supported_property_kinds": [property_kind], + "guarantee": { + "type": claim_class, + "meaning_of_pass": f"Template {intent_id} obligation passed under linked evaluator.", + "meaning_of_fail": f"Template {intent_id} obligation found a violation.", + "meaning_of_unknown": f"Template {intent_id} could not be decided from available materials.", + }, + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + f"conformance_status_v2={row.get('conformance_status_v2')}", + ], + "trusted_components": [ + "intent template", + *(str(link) for link, present in (row.get("executable_links") or {}).items() if present), + ], + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "timeout_semantics": "unknown", + "unsupported_semantics": ( + "Template claims only what linked evaluators and source profiles establish; " + "catalog_only templates make no production enforcement claim." + ), + "determinism_status": "deterministic", + "release_status": release_status, + "owner": "ovk-maintainers", + "native_execution": False, + "template_conformance_status_v2": row.get("conformance_status_v2"), + "production_status": row.get("production_status"), + "claimed_backends": claimed_backends, + } + entries.append(entry) + + entries.sort(key=lambda item: str(item["checker_id"])) + return { + "schema_version": "ovk.template_capability_registry.v1", + "bridge_schema_version": bridge.get("schema_version"), + "source_conformance_schema_version": conformance.get("schema_version"), + "entry_count": len(entries), + "entries": entries, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Build template claim registry from conformance matrix") + parser.add_argument("--repo-root", type=Path, default=ROOT) + parser.add_argument("--bridge", type=Path, default=None) + parser.add_argument("--conformance", type=Path, default=None) + parser.add_argument("--output", type=Path, default=None) + parser.add_argument( + "--check", + action="store_true", + help="Exit non-zero if on-disk registry differs from regenerated content", + ) + args = parser.parse_args() + repo_root = args.repo_root.resolve() + bridge_path = (args.bridge or (repo_root / "templates" / "registry" / "bridge.json")).resolve() + conformance_path = ( + args.conformance or (repo_root / "docs" / "benchmarks" / "template-conformance.json") + ).resolve() + output = (args.output or (repo_root / "templates" / "registry" / "entries.json")).resolve() + + bridge = read_json_file(bridge_path) + conformance = read_json_file(conformance_path) + payload = build_entries(conformance=conformance, bridge=bridge) + + failures: list[str] = [] + for index, entry in enumerate(payload["entries"]): + failures.extend(validate_capability_manifest(entry, source=f"entries[{index}]")) + if failures: + for failure in failures: + print(failure, file=sys.stderr) + return 1 + + rendered = json.dumps(payload, indent=2, sort_keys=True) + "\n" + if args.check: + if not output.is_file(): + print(f"missing template registry: {output}", file=sys.stderr) + return 1 + on_disk = output.read_text(encoding="utf-8") + if on_disk != rendered: + print( + f"stale template registry: {output} (run python scripts/build_template_registry.py)", + file=sys.stderr, + ) + return 1 + print(f"template registry up to date ({payload['entry_count']} entries)") + return 0 + + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + print(f"wrote {payload['entry_count']} template registry entries -> {output}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_capability_tables.py b/scripts/render_capability_tables.py new file mode 100644 index 0000000..b644daf --- /dev/null +++ b/scripts/render_capability_tables.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +"""Regenerate README / docs/BACKENDS.md capability tables from the adapter registry.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from ovk.core.adapter_conformance import apply_release_status_honesty # noqa: E402 +from ovk.core.capabilities import CapabilityRegistry # noqa: E402 +from ovk.core.json_io import read_json_file # noqa: E402 + +BEGIN_BACKENDS = "" +END_BACKENDS = "" +BEGIN_README = "" +END_README = "" + +BACKEND_ORDER = ( + "opa", + "z3", + "cbmc", + "cedar", + "tla+", + "kani", + "dafny", + "verus", + "lean", + "alloy", +) + + +def _display_backend(checker_id: str) -> str: + return f"`{checker_id}`" + + +def _execution_summary(manifest: dict[str, Any]) -> str: + native = manifest.get("native_execution") + determinism = manifest.get("determinism_status", "unknown") + if native is True: + return f"Native path available ({determinism})" + if native is False: + return f"Deterministic contract evaluator only ({determinism})" + return f"Execution maturity undocumented ({determinism})" + + +def _native_determines_evidence(manifest: dict[str, Any]) -> str: + return "Yes" if manifest.get("native_execution") is True else "No" + + +def _current_limit(manifest: dict[str, Any]) -> str: + unsupported = str(manifest.get("unsupported_semantics") or "").strip() + if unsupported: + # Keep table cells readable. + first = unsupported.split(";")[0].strip() + return first + limits = manifest.get("limits") or [] + if limits: + return str(limits[0]) + return "See capability manifest" + + +def render_backends_table(manifests: list[dict[str, Any]]) -> str: + by_checker = {str(m.get("checker_id") or m.get("tool", {}).get("name")): m for m in manifests} + lines = [ + "| Backend | release_status | Current execution | Native result can determine evidence? | Current limit |", + "|---|---|---|---:|---|", + ] + ordered = [cid for cid in BACKEND_ORDER if cid in by_checker] + ordered.extend(sorted(cid for cid in by_checker if cid not in BACKEND_ORDER)) + for checker_id in ordered: + manifest = by_checker[checker_id] + lines.append( + "| " + + " | ".join( + [ + _display_backend(checker_id), + str(manifest.get("release_status", "experimental")), + _execution_summary(manifest), + _native_determines_evidence(manifest), + _current_limit(manifest), + ] + ) + + " |" + ) + return "\n".join(lines) + + +def render_readme_table(manifests: list[dict[str, Any]]) -> str: + by_checker = {str(m.get("checker_id") or m.get("tool", {}).get("name")): m for m in manifests} + lines = [ + "| Checker | release_status | claim_class | Native execution |", + "|---|---|---|---|", + ] + ordered = [cid for cid in BACKEND_ORDER if cid in by_checker] + ordered.extend(sorted(cid for cid in by_checker if cid not in BACKEND_ORDER)) + for checker_id in ordered: + manifest = by_checker[checker_id] + native = "yes" if manifest.get("native_execution") is True else "no" + lines.append( + "| " + + " | ".join( + [ + _display_backend(checker_id), + str(manifest.get("release_status", "experimental")), + str(manifest.get("claim_class", "")), + native, + ] + ) + + " |" + ) + return "\n".join(lines) + + +def _replace_marked_section(text: str, begin: str, end: str, body: str) -> str: + if begin not in text or end not in text: + raise ValueError(f"missing markers {begin!r} / {end!r}") + before, rest = text.split(begin, 1) + _, after = rest.split(end, 1) + return f"{before}{begin}\n{body}\n{end}{after}" + + +def load_manifests(repo_root: Path) -> list[dict[str, Any]]: + adapters = repo_root / "adapters" + registry = CapabilityRegistry.from_directory(adapters, validate=True) + manifests = registry.all() + if not manifests: + # Fallback for packaged layouts that sync capability files. + manifests = [ + read_json_file(path) + for path in sorted(adapters.glob("*/capability.json")) + ] + # Auto-downgrade non-conformant adapters that claim stable (OVK-PR4). + return [apply_release_status_honesty(m, root=repo_root) for m in manifests] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Render capability tables from the adapter registry") + parser.add_argument("--repo-root", type=Path, default=ROOT) + parser.add_argument( + "--check", + action="store_true", + help="Exit non-zero when README / BACKENDS.md tables are stale", + ) + parser.add_argument( + "--write", + action="store_true", + help="Write regenerated tables into README.md and docs/BACKENDS.md", + ) + args = parser.parse_args() + if not args.check and not args.write: + args.write = True + + repo_root = args.repo_root.resolve() + manifests = load_manifests(repo_root) + backends_body = render_backends_table(manifests) + readme_body = ( + "Public checkers from the normative capability registry " + "(`adapters/*/capability.json`). Tables are generated by " + "`scripts/render_capability_tables.py`.\n\n" + + render_readme_table(manifests) + + "\n\nDetails and fallback rules: [docs/BACKENDS.md](docs/BACKENDS.md)." + ) + + backends_path = repo_root / "docs" / "BACKENDS.md" + readme_path = repo_root / "README.md" + backends_text = backends_path.read_text(encoding="utf-8") + readme_text = readme_path.read_text(encoding="utf-8") + + new_backends = _replace_marked_section(backends_text, BEGIN_BACKENDS, END_BACKENDS, backends_body) + new_readme = _replace_marked_section(readme_text, BEGIN_README, END_README, readme_body) + + stale = False + if new_backends != backends_text: + stale = True + if args.write: + backends_path.write_text(new_backends, encoding="utf-8") + print(f"updated {backends_path}") + else: + print(f"stale: {backends_path}", file=sys.stderr) + else: + print(f"up to date: {backends_path}") + + if new_readme != readme_text: + stale = True + if args.write: + readme_path.write_text(new_readme, encoding="utf-8") + print(f"updated {readme_path}") + else: + print(f"stale: {readme_path}", file=sys.stderr) + else: + print(f"up to date: {readme_path}") + + if args.check and stale: + print("capability tables are stale; run: python scripts/render_capability_tables.py --write", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_capabilities.py b/scripts/validate_capabilities.py index 6c34d3f..853519b 100644 --- a/scripts/validate_capabilities.py +++ b/scripts/validate_capabilities.py @@ -1,11 +1,12 @@ #!/usr/bin/env python -"""Validate adapter capability manifests against the JSON schema.""" +"""Validate adapter capability manifests against schema and normative rules.""" from __future__ import annotations import argparse from pathlib import Path +from ovk.core.capabilities import validate_capability_manifest from ovk.core.json_io import read_json_file from ovk.core.schema_validation import validate_against_schema from ovk.paths import ovk_data_root, schema_path @@ -17,6 +18,15 @@ def discover_capability_files(root: Path | None = None) -> list[Path]: return sorted(base.glob("adapters/*/capability.json")) +def discover_template_registry_files(root: Path | None = None) -> list[Path]: + """Return template claim-registry entry files when present.""" + base = root or ovk_data_root() + registry_dir = base / "templates" / "registry" + if not registry_dir.is_dir(): + return [] + return sorted(p for p in registry_dir.glob("*.json") if p.name != "bridge.json") + + def validate_capabilities(capability_files: list[Path] | None = None) -> list[str]: """Return validation failure messages for capability manifests.""" schema_path_file = schema_path("verification.capability.schema.json") @@ -34,6 +44,44 @@ def validate_capabilities(capability_files: list[Path] | None = None) -> list[st for issue in report.issues: location = "/".join(str(part) for part in issue.path) or "$" failures.append(f"{path}: {location}: {issue.message}") + failures.extend(validate_capability_manifest(instance, source=str(path))) + return failures + + +def validate_template_registry(registry_files: list[Path] | None = None) -> list[str]: + """Validate template registry entries that reuse the capability schema fields.""" + files = registry_files if registry_files is not None else discover_template_registry_files() + if not files: + return [] + schema_path_file = schema_path("verification.capability.schema.json") + if not schema_path_file.exists(): + return ["verification.capability.schema.json is missing"] + schema = read_json_file(schema_path_file) + failures: list[str] = [] + for path in files: + try: + instance = read_json_file(path) + except (OSError, ValueError) as error: + failures.append(f"{path}: could not read template registry entry ({error})") + continue + # Template registry may be a list of entries or a single entry object. + entries = instance if isinstance(instance, list) else [instance] + if isinstance(instance, dict) and "entries" in instance: + raw_entries = instance.get("entries") + if not isinstance(raw_entries, list): + failures.append(f"{path}: entries must be an array") + continue + entries = raw_entries + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + failures.append(f"{path}: entry[{index}] must be an object") + continue + source = f"{path}#[{index}]" + report = validate_against_schema(entry, schema) + for issue in report.issues: + location = "/".join(str(part) for part in issue.path) or "$" + failures.append(f"{source}: {location}: {issue.message}") + failures.extend(validate_capability_manifest(entry, source=source)) return failures @@ -46,9 +94,16 @@ def main() -> int: dest="capability_files", help="Validate one capability.json file (repeatable). Defaults to adapters/*/capability.json", ) + parser.add_argument( + "--include-templates", + action="store_true", + help="Also validate templates/registry claim entries", + ) args = parser.parse_args() files = args.capability_files or discover_capability_files() failures = validate_capabilities(files) + if args.include_templates or not args.capability_files: + failures.extend(validate_template_registry()) for failure in failures: print(failure) if failures: diff --git a/templates/registry/README.md b/templates/registry/README.md new file mode 100644 index 0000000..4796c95 --- /dev/null +++ b/templates/registry/README.md @@ -0,0 +1,25 @@ +# Template claim registry + +Bridges [template conformance](../docs/benchmarks/template-conformance.json) onto the +normative capability vocabulary (`release_status`, `claim_class`) without inventing +a second status system. + +| File | Role | +|---|---| +| `bridge.json` | Maps `conformance_status_v2` / `production_status` → `release_status` | +| `entries.json` | Generated per-template claim registry entries | + +Regenerate: + +```bash +python scripts/build_template_registry.py +``` + +Freshness gate (CI): + +```bash +python scripts/build_template_registry.py --check +``` + +Honesty: `stable` is not emitted; even `source_profile_strict_eligible` maps to `preview` +until OVK-PR4 conformance gates exist. diff --git a/templates/registry/bridge.json b/templates/registry/bridge.json new file mode 100644 index 0000000..31a000e --- /dev/null +++ b/templates/registry/bridge.json @@ -0,0 +1,31 @@ +{ + "schema_version": "ovk.template_capability_bridge.v1", + "description": "Bridge template-conformance statuses onto the normative release_status + claim_class vocabulary. Does not invent a second status system.", + "conformance_status_v2_to_release_status": { + "deprecated": "disabled", + "catalog_only": "experimental", + "executable_advisory": "preview", + "source_profile_strict_eligible": "preview", + "externally_calibrated_strict": "preview" + }, + "production_status_to_release_status": { + "deprecated": "disabled", + "catalog_only": "experimental", + "experimental": "experimental", + "advisory": "preview", + "strict_eligible": "preview" + }, + "default_claim_class_by_property_kind": { + "access_control": "authorization_invariant", + "forbidden_configuration": "configuration_invariant", + "safety": "safety_invariant", + "invariant": "state_invariant", + "liveness": "liveness_property", + "data_boundary": "data_boundary_invariant" + }, + "notes": [ + "stable is reserved until OVK-PR4 adapter conformance gates exist.", + "source_profile_strict_eligible and externally_calibrated_strict map to preview, not stable.", + "Template registry entries are generated from docs/benchmarks/template-conformance.json via scripts/build_template_registry.py." + ] +} diff --git a/templates/registry/entries.json b/templates/registry/entries.json new file mode 100644 index 0000000..3c6e04e --- /dev/null +++ b/templates/registry/entries.json @@ -0,0 +1,4608 @@ +{ + "bridge_schema_version": "ovk.template_capability_bridge.v1", + "entries": [ + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-agent-authority-guard-11-v1", + "checker_id": "template:agent-authority-guard-11", + "claim_class": "template_property_claim", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-authority-guard-11 obligation found a violation.", + "meaning_of_pass": "Template agent-authority-guard-11 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-authority-guard-11 could not be decided from available materials.", + "type": "template_property_claim" + }, + "implementation": "templates/agent_authority/agent_authority_guard_11.intent.json", + "input_contract": "Intent template agent-authority-guard-11 over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "runtime_monitorable" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-authority-guard-11" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-agent-authority-guard-17-v1", + "checker_id": "template:agent-authority-guard-17", + "claim_class": "template_property_claim", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-authority-guard-17 obligation found a violation.", + "meaning_of_pass": "Template agent-authority-guard-17 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-authority-guard-17 could not be decided from available materials.", + "type": "template_property_claim" + }, + "implementation": "templates/agent_authority/agent_authority_guard_17.intent.json", + "input_contract": "Intent template agent-authority-guard-17 over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "runtime_monitorable" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-authority-guard-17" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-agent-authority-guard-5-v1", + "checker_id": "template:agent-authority-guard-5", + "claim_class": "template_property_claim", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-authority-guard-5 obligation found a violation.", + "meaning_of_pass": "Template agent-authority-guard-5 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-authority-guard-5 could not be decided from available materials.", + "type": "template_property_claim" + }, + "implementation": "templates/agent_authority/agent_authority_guard_5.intent.json", + "input_contract": "Intent template agent-authority-guard-5 over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "runtime_monitorable" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-authority-guard-5" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=executable_advisory" + ], + "backend_class": "custom", + "capability_id": "template-agent-cannot-disable-own-ci-gate-v1", + "checker_id": "template:agent-cannot-disable-own-ci-gate", + "claim_class": "state_invariant", + "claimed_backends": [ + "opa-native", + "self-protection-deterministic" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-cannot-disable-own-ci-gate obligation found a violation.", + "meaning_of_pass": "Template agent-cannot-disable-own-ci-gate obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-cannot-disable-own-ci-gate could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/ci_cd/agent_cannot_disable_own_gate.intent.json", + "input_contract": "Intent template agent-cannot-disable-own-ci-gate over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "strict_eligible", + "release_status": "preview", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "executable_advisory", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-cannot-disable-own-ci-gate" + }, + "trusted_components": [ + "intent template", + "backend_registry", + "enforcement_test", + "fail_example", + "intent_file", + "lane_evaluator", + "neutral_compiler", + "pass_example" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-agent-guard-11-v1", + "checker_id": "template:agent-guard-11", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-guard-11 obligation found a violation.", + "meaning_of_pass": "Template agent-guard-11 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-guard-11 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/agent_authority/agent_guard_11.intent.json", + "input_contract": "Intent template agent-guard-11 over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-guard-11" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-agent-guard-17-v1", + "checker_id": "template:agent-guard-17", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-guard-17 obligation found a violation.", + "meaning_of_pass": "Template agent-guard-17 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-guard-17 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/agent_authority/agent_guard_17.intent.json", + "input_contract": "Intent template agent-guard-17 over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-guard-17" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-agent-guard-23-v1", + "checker_id": "template:agent-guard-23", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-guard-23 obligation found a violation.", + "meaning_of_pass": "Template agent-guard-23 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-guard-23 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/agent_authority/agent_guard_23.intent.json", + "input_contract": "Intent template agent-guard-23 over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-guard-23" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-agent-guard-29-v1", + "checker_id": "template:agent-guard-29", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-guard-29 obligation found a violation.", + "meaning_of_pass": "Template agent-guard-29 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-guard-29 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/agent_authority/agent_guard_29.intent.json", + "input_contract": "Intent template agent-guard-29 over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-guard-29" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-agent-guard-35-v1", + "checker_id": "template:agent-guard-35", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-guard-35 obligation found a violation.", + "meaning_of_pass": "Template agent-guard-35 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-guard-35 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/agent_authority/agent_guard_35.intent.json", + "input_contract": "Intent template agent-guard-35 over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-guard-35" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-agent-guard-41-v1", + "checker_id": "template:agent-guard-41", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-guard-41 obligation found a violation.", + "meaning_of_pass": "Template agent-guard-41 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-guard-41 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/agent_authority/agent_guard_41.intent.json", + "input_contract": "Intent template agent-guard-41 over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-guard-41" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-agent-guard-5-v1", + "checker_id": "template:agent-guard-5", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template agent-guard-5 obligation found a violation.", + "meaning_of_pass": "Template agent-guard-5 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template agent-guard-5 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/agent_authority/agent_guard_5.intent.json", + "input_contract": "Intent template agent-guard-5 over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:agent-guard-5" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-alloy-admin-path-requires-jump-host-v1", + "checker_id": "template:alloy-admin-path-requires-jump-host", + "claim_class": "configuration_invariant", + "claimed_backends": [ + "alloy" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template alloy-admin-path-requires-jump-host obligation found a violation.", + "meaning_of_pass": "Template alloy-admin-path-requires-jump-host obligation passed under linked evaluator.", + "meaning_of_unknown": "Template alloy-admin-path-requires-jump-host could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/alloy_admin_path_requires_jump_host.intent.json", + "input_contract": "Intent template alloy-admin-path-requires-jump-host over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:alloy-admin-path-requires-jump-host" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-alloy-no-public-to-private-db-path-v1", + "checker_id": "template:alloy-no-public-to-private-db-path", + "claim_class": "data_boundary_invariant", + "claimed_backends": [ + "alloy" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template alloy-no-public-to-private-db-path obligation found a violation.", + "meaning_of_pass": "Template alloy-no-public-to-private-db-path obligation passed under linked evaluator.", + "meaning_of_unknown": "Template alloy-no-public-to-private-db-path could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/infrastructure/alloy_no_public_to_private_db_path.intent.json", + "input_contract": "Intent template alloy-no-public-to-private-db-path over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:alloy-no-public-to-private-db-path" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-alloy-private-service-not-public-component-v1", + "checker_id": "template:alloy-private-service-not-public-component", + "claim_class": "state_invariant", + "claimed_backends": [ + "alloy" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template alloy-private-service-not-public-component obligation found a violation.", + "meaning_of_pass": "Template alloy-private-service-not-public-component obligation passed under linked evaluator.", + "meaning_of_unknown": "Template alloy-private-service-not-public-component could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/infrastructure/alloy_private_service_not_public_component.intent.json", + "input_contract": "Intent template alloy-private-service-not-public-component over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:alloy-private-service-not-public-component" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-alloy-topology-reachability-v1", + "checker_id": "template:alloy-topology-reachability", + "claim_class": "state_invariant", + "claimed_backends": [ + "alloy" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template alloy-topology-reachability obligation found a violation.", + "meaning_of_pass": "Template alloy-topology-reachability obligation passed under linked evaluator.", + "meaning_of_unknown": "Template alloy-topology-reachability could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/infrastructure/alloy_topology_reachability.intent.json", + "input_contract": "Intent template alloy-topology-reachability over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:alloy-topology-reachability" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-authorization-guard-12-v1", + "checker_id": "template:authorization-guard-12", + "claim_class": "authorization_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template authorization-guard-12 obligation found a violation.", + "meaning_of_pass": "Template authorization-guard-12 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template authorization-guard-12 could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/authorization_guard_12.intent.json", + "input_contract": "Intent template authorization-guard-12 over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:authorization-guard-12" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-authorization-guard-18-v1", + "checker_id": "template:authorization-guard-18", + "claim_class": "authorization_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template authorization-guard-18 obligation found a violation.", + "meaning_of_pass": "Template authorization-guard-18 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template authorization-guard-18 could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/authorization_guard_18.intent.json", + "input_contract": "Intent template authorization-guard-18 over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:authorization-guard-18" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-authorization-guard-6-v1", + "checker_id": "template:authorization-guard-6", + "claim_class": "authorization_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template authorization-guard-6 obligation found a violation.", + "meaning_of_pass": "Template authorization-guard-6 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template authorization-guard-6 could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/authorization_guard_6.intent.json", + "input_contract": "Intent template authorization-guard-6 over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:authorization-guard-6" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-cbmc-buffer-bounds-v1", + "checker_id": "template:cbmc-buffer-bounds", + "claim_class": "data_boundary_invariant", + "claimed_backends": [ + "cbmc" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template cbmc-buffer-bounds obligation found a violation.", + "meaning_of_pass": "Template cbmc-buffer-bounds obligation passed under linked evaluator.", + "meaning_of_unknown": "Template cbmc-buffer-bounds could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/data_boundary/cbmc_buffer_bounds.intent.json", + "input_contract": "Intent template cbmc-buffer-bounds over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:cbmc-buffer-bounds" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-cbmc-no-integer-overflow-quota-v1", + "checker_id": "template:cbmc-no-integer-overflow-quota", + "claim_class": "state_invariant", + "claimed_backends": [ + "cbmc" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template cbmc-no-integer-overflow-quota obligation found a violation.", + "meaning_of_pass": "Template cbmc-no-integer-overflow-quota obligation passed under linked evaluator.", + "meaning_of_unknown": "Template cbmc-no-integer-overflow-quota could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/data_boundary/cbmc_no_integer_overflow_quota.intent.json", + "input_contract": "Intent template cbmc-no-integer-overflow-quota over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:cbmc-no-integer-overflow-quota" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-cbmc-no-unchecked-buffer-copy-v1", + "checker_id": "template:cbmc-no-unchecked-buffer-copy", + "claim_class": "safety_invariant", + "claimed_backends": [ + "cbmc" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template cbmc-no-unchecked-buffer-copy obligation found a violation.", + "meaning_of_pass": "Template cbmc-no-unchecked-buffer-copy obligation passed under linked evaluator.", + "meaning_of_unknown": "Template cbmc-no-unchecked-buffer-copy could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/data_boundary/cbmc_no_unchecked_buffer_copy.intent.json", + "input_contract": "Intent template cbmc-no-unchecked-buffer-copy over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:cbmc-no-unchecked-buffer-copy" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-cbmc-no-use-after-free-auth-cache-v1", + "checker_id": "template:cbmc-no-use-after-free-auth-cache", + "claim_class": "safety_invariant", + "claimed_backends": [ + "cbmc" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template cbmc-no-use-after-free-auth-cache obligation found a violation.", + "meaning_of_pass": "Template cbmc-no-use-after-free-auth-cache obligation passed under linked evaluator.", + "meaning_of_unknown": "Template cbmc-no-use-after-free-auth-cache could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/data_boundary/cbmc_no_use_after_free_auth_cache.intent.json", + "input_contract": "Intent template cbmc-no-use-after-free-auth-cache over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:cbmc-no-use-after-free-auth-cache" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-cedar-cross-account-deny-v1", + "checker_id": "template:cedar-cross-account-deny", + "claim_class": "authorization_invariant", + "claimed_backends": [ + "cedar" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template cedar-cross-account-deny obligation found a violation.", + "meaning_of_pass": "Template cedar-cross-account-deny obligation passed under linked evaluator.", + "meaning_of_unknown": "Template cedar-cross-account-deny could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/cedar_cross_account_deny.intent.json", + "input_contract": "Intent template cedar-cross-account-deny over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:cedar-cross-account-deny" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-cedar-deny-precedence-preserved-v1", + "checker_id": "template:cedar-deny-precedence-preserved", + "claim_class": "state_invariant", + "claimed_backends": [ + "cedar" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template cedar-deny-precedence-preserved obligation found a violation.", + "meaning_of_pass": "Template cedar-deny-precedence-preserved obligation passed under linked evaluator.", + "meaning_of_unknown": "Template cedar-deny-precedence-preserved could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/authorization/cedar_deny_precedence_preserved.intent.json", + "input_contract": "Intent template cedar-deny-precedence-preserved over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:cedar-deny-precedence-preserved" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-cedar-iam-admin-deny-v1", + "checker_id": "template:cedar-iam-admin-deny", + "claim_class": "authorization_invariant", + "claimed_backends": [ + "cedar" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template cedar-iam-admin-deny obligation found a violation.", + "meaning_of_pass": "Template cedar-iam-admin-deny obligation passed under linked evaluator.", + "meaning_of_unknown": "Template cedar-iam-admin-deny could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/cedar_iam_admin_deny.intent.json", + "input_contract": "Intent template cedar-iam-admin-deny over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:cedar-iam-admin-deny" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-cedar-no-wildcard-admin-allow-v1", + "checker_id": "template:cedar-no-wildcard-admin-allow", + "claim_class": "authorization_invariant", + "claimed_backends": [ + "cedar" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template cedar-no-wildcard-admin-allow obligation found a violation.", + "meaning_of_pass": "Template cedar-no-wildcard-admin-allow obligation passed under linked evaluator.", + "meaning_of_unknown": "Template cedar-no-wildcard-admin-allow could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/cedar_no_wildcard_admin_allow.intent.json", + "input_contract": "Intent template cedar-no-wildcard-admin-allow over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:cedar-no-wildcard-admin-allow" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-cedar-principal-scope-preserved-v1", + "checker_id": "template:cedar-principal-scope-preserved", + "claim_class": "authorization_invariant", + "claimed_backends": [ + "cedar" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template cedar-principal-scope-preserved obligation found a violation.", + "meaning_of_pass": "Template cedar-principal-scope-preserved obligation passed under linked evaluator.", + "meaning_of_unknown": "Template cedar-principal-scope-preserved could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/cedar_principal_scope_preserved.intent.json", + "input_contract": "Intent template cedar-principal-scope-preserved over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:cedar-principal-scope-preserved" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-cd-guard-14-v1", + "checker_id": "template:ci-cd-guard-14", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-cd-guard-14 obligation found a violation.", + "meaning_of_pass": "Template ci-cd-guard-14 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-cd-guard-14 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/ci_cd/ci_cd_guard_14.intent.json", + "input_contract": "Intent template ci-cd-guard-14 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-cd-guard-14" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-cd-guard-2-v1", + "checker_id": "template:ci-cd-guard-2", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-cd-guard-2 obligation found a violation.", + "meaning_of_pass": "Template ci-cd-guard-2 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-cd-guard-2 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/ci_cd/ci_cd_guard_2.intent.json", + "input_contract": "Intent template ci-cd-guard-2 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-cd-guard-2" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-cd-guard-20-v1", + "checker_id": "template:ci-cd-guard-20", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-cd-guard-20 obligation found a violation.", + "meaning_of_pass": "Template ci-cd-guard-20 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-cd-guard-20 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/ci_cd/ci_cd_guard_20.intent.json", + "input_contract": "Intent template ci-cd-guard-20 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-cd-guard-20" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-cd-guard-8-v1", + "checker_id": "template:ci-cd-guard-8", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-cd-guard-8 obligation found a violation.", + "meaning_of_pass": "Template ci-cd-guard-8 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-cd-guard-8 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/ci_cd/ci_cd_guard_8.intent.json", + "input_contract": "Intent template ci-cd-guard-8 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-cd-guard-8" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-guard-14-v1", + "checker_id": "template:ci-guard-14", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-guard-14 obligation found a violation.", + "meaning_of_pass": "Template ci-guard-14 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-guard-14 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/ci_guard_14.intent.json", + "input_contract": "Intent template ci-guard-14 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-guard-14" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-guard-2-v1", + "checker_id": "template:ci-guard-2", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-guard-2 obligation found a violation.", + "meaning_of_pass": "Template ci-guard-2 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-guard-2 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/ci_guard_2.intent.json", + "input_contract": "Intent template ci-guard-2 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-guard-2" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-guard-20-v1", + "checker_id": "template:ci-guard-20", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-guard-20 obligation found a violation.", + "meaning_of_pass": "Template ci-guard-20 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-guard-20 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/ci_guard_20.intent.json", + "input_contract": "Intent template ci-guard-20 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-guard-20" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-guard-26-v1", + "checker_id": "template:ci-guard-26", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-guard-26 obligation found a violation.", + "meaning_of_pass": "Template ci-guard-26 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-guard-26 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/ci_guard_26.intent.json", + "input_contract": "Intent template ci-guard-26 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-guard-26" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-guard-32-v1", + "checker_id": "template:ci-guard-32", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-guard-32 obligation found a violation.", + "meaning_of_pass": "Template ci-guard-32 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-guard-32 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/ci_guard_32.intent.json", + "input_contract": "Intent template ci-guard-32 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-guard-32" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-guard-38-v1", + "checker_id": "template:ci-guard-38", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-guard-38 obligation found a violation.", + "meaning_of_pass": "Template ci-guard-38 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-guard-38 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/ci_guard_38.intent.json", + "input_contract": "Intent template ci-guard-38 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-guard-38" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-guard-44-v1", + "checker_id": "template:ci-guard-44", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-guard-44 obligation found a violation.", + "meaning_of_pass": "Template ci-guard-44 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-guard-44 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/ci_guard_44.intent.json", + "input_contract": "Intent template ci-guard-44 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-guard-44" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-ci-guard-8-v1", + "checker_id": "template:ci-guard-8", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template ci-guard-8 obligation found a violation.", + "meaning_of_pass": "Template ci-guard-8 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template ci-guard-8 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/ci_guard_8.intent.json", + "input_contract": "Intent template ci-guard-8 over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:ci-guard-8" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-dafny-authority-invariant-v1", + "checker_id": "template:dafny-authority-invariant", + "claim_class": "state_invariant", + "claimed_backends": [ + "dafny" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template dafny-authority-invariant obligation found a violation.", + "meaning_of_pass": "Template dafny-authority-invariant obligation passed under linked evaluator.", + "meaning_of_unknown": "Template dafny-authority-invariant could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/agent_authority/dafny_authority_invariant.intent.json", + "input_contract": "Intent template dafny-authority-invariant over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:dafny-authority-invariant" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-data-boundary-guard-10-v1", + "checker_id": "template:data-boundary-guard-10", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template data-boundary-guard-10 obligation found a violation.", + "meaning_of_pass": "Template data-boundary-guard-10 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template data-boundary-guard-10 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/data_boundary/data_boundary_guard_10.intent.json", + "input_contract": "Intent template data-boundary-guard-10 over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:data-boundary-guard-10" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-data-boundary-guard-16-v1", + "checker_id": "template:data-boundary-guard-16", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template data-boundary-guard-16 obligation found a violation.", + "meaning_of_pass": "Template data-boundary-guard-16 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template data-boundary-guard-16 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/data_boundary/data_boundary_guard_16.intent.json", + "input_contract": "Intent template data-boundary-guard-16 over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:data-boundary-guard-16" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-data-boundary-guard-4-v1", + "checker_id": "template:data-boundary-guard-4", + "claim_class": "safety_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template data-boundary-guard-4 obligation found a violation.", + "meaning_of_pass": "Template data-boundary-guard-4 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template data-boundary-guard-4 could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/data_boundary/data_boundary_guard_4.intent.json", + "input_contract": "Intent template data-boundary-guard-4 over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:data-boundary-guard-4" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-data-guard-10-v1", + "checker_id": "template:data-guard-10", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template data-guard-10 obligation found a violation.", + "meaning_of_pass": "Template data-guard-10 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template data-guard-10 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/data_boundary/data_guard_10.intent.json", + "input_contract": "Intent template data-guard-10 over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:data-guard-10" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-data-guard-16-v1", + "checker_id": "template:data-guard-16", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template data-guard-16 obligation found a violation.", + "meaning_of_pass": "Template data-guard-16 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template data-guard-16 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/data_boundary/data_guard_16.intent.json", + "input_contract": "Intent template data-guard-16 over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:data-guard-16" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-data-guard-22-v1", + "checker_id": "template:data-guard-22", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template data-guard-22 obligation found a violation.", + "meaning_of_pass": "Template data-guard-22 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template data-guard-22 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/data_boundary/data_guard_22.intent.json", + "input_contract": "Intent template data-guard-22 over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:data-guard-22" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-data-guard-28-v1", + "checker_id": "template:data-guard-28", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template data-guard-28 obligation found a violation.", + "meaning_of_pass": "Template data-guard-28 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template data-guard-28 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/data_boundary/data_guard_28.intent.json", + "input_contract": "Intent template data-guard-28 over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:data-guard-28" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-data-guard-34-v1", + "checker_id": "template:data-guard-34", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template data-guard-34 obligation found a violation.", + "meaning_of_pass": "Template data-guard-34 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template data-guard-34 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/data_boundary/data_guard_34.intent.json", + "input_contract": "Intent template data-guard-34 over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:data-guard-34" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-data-guard-4-v1", + "checker_id": "template:data-guard-4", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template data-guard-4 obligation found a violation.", + "meaning_of_pass": "Template data-guard-4 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template data-guard-4 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/data_boundary/data_guard_4.intent.json", + "input_contract": "Intent template data-guard-4 over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:data-guard-4" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-data-guard-40-v1", + "checker_id": "template:data-guard-40", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template data-guard-40 obligation found a violation.", + "meaning_of_pass": "Template data-guard-40 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template data-guard-40 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/data_boundary/data_guard_40.intent.json", + "input_contract": "Intent template data-guard-40 over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:data-guard-40" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deploy-guard-15-v1", + "checker_id": "template:deploy-guard-15", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deploy-guard-15 obligation found a violation.", + "meaning_of_pass": "Template deploy-guard-15 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deploy-guard-15 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deploy_guard_15.intent.json", + "input_contract": "Intent template deploy-guard-15 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deploy-guard-15" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deploy-guard-21-v1", + "checker_id": "template:deploy-guard-21", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deploy-guard-21 obligation found a violation.", + "meaning_of_pass": "Template deploy-guard-21 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deploy-guard-21 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deploy_guard_21.intent.json", + "input_contract": "Intent template deploy-guard-21 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deploy-guard-21" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deploy-guard-27-v1", + "checker_id": "template:deploy-guard-27", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deploy-guard-27 obligation found a violation.", + "meaning_of_pass": "Template deploy-guard-27 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deploy-guard-27 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deploy_guard_27.intent.json", + "input_contract": "Intent template deploy-guard-27 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deploy-guard-27" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deploy-guard-3-v1", + "checker_id": "template:deploy-guard-3", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deploy-guard-3 obligation found a violation.", + "meaning_of_pass": "Template deploy-guard-3 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deploy-guard-3 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deploy_guard_3.intent.json", + "input_contract": "Intent template deploy-guard-3 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deploy-guard-3" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deploy-guard-33-v1", + "checker_id": "template:deploy-guard-33", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deploy-guard-33 obligation found a violation.", + "meaning_of_pass": "Template deploy-guard-33 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deploy-guard-33 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deploy_guard_33.intent.json", + "input_contract": "Intent template deploy-guard-33 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deploy-guard-33" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deploy-guard-39-v1", + "checker_id": "template:deploy-guard-39", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deploy-guard-39 obligation found a violation.", + "meaning_of_pass": "Template deploy-guard-39 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deploy-guard-39 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deploy_guard_39.intent.json", + "input_contract": "Intent template deploy-guard-39 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deploy-guard-39" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deploy-guard-45-v1", + "checker_id": "template:deploy-guard-45", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deploy-guard-45 obligation found a violation.", + "meaning_of_pass": "Template deploy-guard-45 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deploy-guard-45 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deploy_guard_45.intent.json", + "input_contract": "Intent template deploy-guard-45 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deploy-guard-45" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deploy-guard-9-v1", + "checker_id": "template:deploy-guard-9", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deploy-guard-9 obligation found a violation.", + "meaning_of_pass": "Template deploy-guard-9 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deploy-guard-9 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deploy_guard_9.intent.json", + "input_contract": "Intent template deploy-guard-9 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deploy-guard-9" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deployment-guard-15-v1", + "checker_id": "template:deployment-guard-15", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deployment-guard-15 obligation found a violation.", + "meaning_of_pass": "Template deployment-guard-15 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deployment-guard-15 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deployment_guard_15.intent.json", + "input_contract": "Intent template deployment-guard-15 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deployment-guard-15" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deployment-guard-3-v1", + "checker_id": "template:deployment-guard-3", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deployment-guard-3 obligation found a violation.", + "meaning_of_pass": "Template deployment-guard-3 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deployment-guard-3 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deployment_guard_3.intent.json", + "input_contract": "Intent template deployment-guard-3 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deployment-guard-3" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-deployment-guard-9-v1", + "checker_id": "template:deployment-guard-9", + "claim_class": "state_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template deployment-guard-9 obligation found a violation.", + "meaning_of_pass": "Template deployment-guard-9 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template deployment-guard-9 could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/deployment_guard_9.intent.json", + "input_contract": "Intent template deployment-guard-9 over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:deployment-guard-9" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infra-guard-1-v1", + "checker_id": "template:infra-guard-1", + "claim_class": "configuration_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infra-guard-1 obligation found a violation.", + "meaning_of_pass": "Template infra-guard-1 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infra-guard-1 could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/infra_guard_1.intent.json", + "input_contract": "Intent template infra-guard-1 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infra-guard-1" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infra-guard-13-v1", + "checker_id": "template:infra-guard-13", + "claim_class": "configuration_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infra-guard-13 obligation found a violation.", + "meaning_of_pass": "Template infra-guard-13 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infra-guard-13 could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/infra_guard_13.intent.json", + "input_contract": "Intent template infra-guard-13 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infra-guard-13" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infra-guard-19-v1", + "checker_id": "template:infra-guard-19", + "claim_class": "configuration_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infra-guard-19 obligation found a violation.", + "meaning_of_pass": "Template infra-guard-19 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infra-guard-19 could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/infra_guard_19.intent.json", + "input_contract": "Intent template infra-guard-19 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infra-guard-19" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infra-guard-25-v1", + "checker_id": "template:infra-guard-25", + "claim_class": "configuration_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infra-guard-25 obligation found a violation.", + "meaning_of_pass": "Template infra-guard-25 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infra-guard-25 could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/infra_guard_25.intent.json", + "input_contract": "Intent template infra-guard-25 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infra-guard-25" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infra-guard-31-v1", + "checker_id": "template:infra-guard-31", + "claim_class": "configuration_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infra-guard-31 obligation found a violation.", + "meaning_of_pass": "Template infra-guard-31 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infra-guard-31 could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/infra_guard_31.intent.json", + "input_contract": "Intent template infra-guard-31 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infra-guard-31" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infra-guard-37-v1", + "checker_id": "template:infra-guard-37", + "claim_class": "configuration_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infra-guard-37 obligation found a violation.", + "meaning_of_pass": "Template infra-guard-37 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infra-guard-37 could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/infra_guard_37.intent.json", + "input_contract": "Intent template infra-guard-37 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infra-guard-37" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infra-guard-43-v1", + "checker_id": "template:infra-guard-43", + "claim_class": "configuration_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infra-guard-43 obligation found a violation.", + "meaning_of_pass": "Template infra-guard-43 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infra-guard-43 could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/infra_guard_43.intent.json", + "input_contract": "Intent template infra-guard-43 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infra-guard-43" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infra-guard-7-v1", + "checker_id": "template:infra-guard-7", + "claim_class": "configuration_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infra-guard-7 obligation found a violation.", + "meaning_of_pass": "Template infra-guard-7 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infra-guard-7 could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/infra_guard_7.intent.json", + "input_contract": "Intent template infra-guard-7 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infra-guard-7" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infrastructure-guard-1-v1", + "checker_id": "template:infrastructure-guard-1", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infrastructure-guard-1 obligation found a violation.", + "meaning_of_pass": "Template infrastructure-guard-1 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infrastructure-guard-1 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/infrastructure/infrastructure_guard_1.intent.json", + "input_contract": "Intent template infrastructure-guard-1 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infrastructure-guard-1" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infrastructure-guard-13-v1", + "checker_id": "template:infrastructure-guard-13", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infrastructure-guard-13 obligation found a violation.", + "meaning_of_pass": "Template infrastructure-guard-13 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infrastructure-guard-13 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/infrastructure/infrastructure_guard_13.intent.json", + "input_contract": "Intent template infrastructure-guard-13 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infrastructure-guard-13" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infrastructure-guard-19-v1", + "checker_id": "template:infrastructure-guard-19", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infrastructure-guard-19 obligation found a violation.", + "meaning_of_pass": "Template infrastructure-guard-19 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infrastructure-guard-19 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/infrastructure/infrastructure_guard_19.intent.json", + "input_contract": "Intent template infrastructure-guard-19 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infrastructure-guard-19" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-infrastructure-guard-7-v1", + "checker_id": "template:infrastructure-guard-7", + "claim_class": "data_boundary_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template infrastructure-guard-7 obligation found a violation.", + "meaning_of_pass": "Template infrastructure-guard-7 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template infrastructure-guard-7 could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/infrastructure/infrastructure_guard_7.intent.json", + "input_contract": "Intent template infrastructure-guard-7 over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:infrastructure-guard-7" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-kani-bounded-loop-termination-v1", + "checker_id": "template:kani-bounded-loop-termination", + "claim_class": "state_invariant", + "claimed_backends": [ + "kani" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template kani-bounded-loop-termination obligation found a violation.", + "meaning_of_pass": "Template kani-bounded-loop-termination obligation passed under linked evaluator.", + "meaning_of_unknown": "Template kani-bounded-loop-termination could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/ci_cd/kani_bounded_loop_termination.intent.json", + "input_contract": "Intent template kani-bounded-loop-termination over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:kani-bounded-loop-termination" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-kani-no-panic-in-auth-flow-v1", + "checker_id": "template:kani-no-panic-in-auth-flow", + "claim_class": "safety_invariant", + "claimed_backends": [ + "kani" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template kani-no-panic-in-auth-flow obligation found a violation.", + "meaning_of_pass": "Template kani-no-panic-in-auth-flow obligation passed under linked evaluator.", + "meaning_of_unknown": "Template kani-no-panic-in-auth-flow could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/kani_no_panic_in_auth_flow.intent.json", + "input_contract": "Intent template kani-no-panic-in-auth-flow over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:kani-no-panic-in-auth-flow" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-kani-no-unsafe-pointer-deref-v1", + "checker_id": "template:kani-no-unsafe-pointer-deref", + "claim_class": "safety_invariant", + "claimed_backends": [ + "kani" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template kani-no-unsafe-pointer-deref obligation found a violation.", + "meaning_of_pass": "Template kani-no-unsafe-pointer-deref obligation passed under linked evaluator.", + "meaning_of_unknown": "Template kani-no-unsafe-pointer-deref could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/kani_no_unsafe_pointer_deref.intent.json", + "input_contract": "Intent template kani-no-unsafe-pointer-deref over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:kani-no-unsafe-pointer-deref" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-lean-type-safety-v1", + "checker_id": "template:lean-type-safety", + "claim_class": "state_invariant", + "claimed_backends": [ + "lean" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template lean-type-safety obligation found a violation.", + "meaning_of_pass": "Template lean-type-safety obligation passed under linked evaluator.", + "meaning_of_unknown": "Template lean-type-safety could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/data_boundary/lean_type_safety.intent.json", + "input_contract": "Intent template lean-type-safety over domain data_boundary; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "data_boundary" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:lean-type-safety" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-memory-safe-config-v1", + "checker_id": "template:memory-safe-config", + "claim_class": "configuration_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template memory-safe-config obligation found a violation.", + "meaning_of_pass": "Template memory-safe-config obligation passed under linked evaluator.", + "meaning_of_unknown": "Template memory-safe-config could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/memory_safe_config.intent.json", + "input_contract": "Intent template memory-safe-config over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:memory-safe-config" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=source_profile_strict_eligible" + ], + "backend_class": "custom", + "capability_id": "template-no-admin-route-bypass-v1", + "checker_id": "template:no-admin-route-bypass", + "claim_class": "authorization_invariant", + "claimed_backends": [ + "authorization-deterministic", + "z3-native" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template no-admin-route-bypass obligation found a violation.", + "meaning_of_pass": "Template no-admin-route-bypass obligation passed under linked evaluator.", + "meaning_of_unknown": "Template no-admin-route-bypass could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/no_admin_route_bypass.intent.json", + "input_contract": "Intent template no-admin-route-bypass over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "strict_eligible", + "release_status": "preview", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "source_profile_strict_eligible", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:no-admin-route-bypass" + }, + "trusted_components": [ + "intent template", + "backend_registry", + "enforcement_test", + "fail_example", + "intent_file", + "lane_evaluator", + "neutral_compiler", + "pass_example" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-no-privilege-escalation-cedar-v1", + "checker_id": "template:no-privilege-escalation-cedar", + "claim_class": "authorization_invariant", + "claimed_backends": [ + "cedar" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template no-privilege-escalation-cedar obligation found a violation.", + "meaning_of_pass": "Template no-privilege-escalation-cedar obligation passed under linked evaluator.", + "meaning_of_unknown": "Template no-privilege-escalation-cedar could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/no_privilege_escalation_cedar.intent.json", + "input_contract": "Intent template no-privilege-escalation-cedar over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:no-privilege-escalation-cedar" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-no-public-egress-alloy-v1", + "checker_id": "template:no-public-egress-alloy", + "claim_class": "configuration_invariant", + "claimed_backends": [ + "alloy" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template no-public-egress-alloy obligation found a violation.", + "meaning_of_pass": "Template no-public-egress-alloy obligation passed under linked evaluator.", + "meaning_of_unknown": "Template no-public-egress-alloy could not be decided from available materials.", + "type": "configuration_invariant" + }, + "implementation": "templates/infrastructure/no_public_egress_alloy.intent.json", + "input_contract": "Intent template no-public-egress-alloy over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "forbidden_configuration" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:no-public-egress-alloy" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=source_profile_strict_eligible" + ], + "backend_class": "custom", + "capability_id": "template-no-public-sensitive-resource-v1", + "checker_id": "template:no-public-sensitive-resource", + "claim_class": "data_boundary_invariant", + "claimed_backends": [ + "infrastructure-deterministic" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template no-public-sensitive-resource obligation found a violation.", + "meaning_of_pass": "Template no-public-sensitive-resource obligation passed under linked evaluator.", + "meaning_of_unknown": "Template no-public-sensitive-resource could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/infrastructure/no_public_sensitive_resource.intent.json", + "input_contract": "Intent template no-public-sensitive-resource over domain infrastructure; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "strict_eligible", + "release_status": "preview", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "source_profile_strict_eligible", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:no-public-sensitive-resource" + }, + "trusted_components": [ + "intent template", + "backend_registry", + "enforcement_test", + "fail_example", + "intent_file", + "lane_evaluator", + "neutral_compiler", + "pass_example" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=source_profile_strict_eligible" + ], + "backend_class": "custom", + "capability_id": "template-no-secrets-in-untrusted-context-v1", + "checker_id": "template:no-secrets-in-untrusted-context", + "claim_class": "data_boundary_invariant", + "claimed_backends": [ + "ci-secrets-deterministic" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template no-secrets-in-untrusted-context obligation found a violation.", + "meaning_of_pass": "Template no-secrets-in-untrusted-context obligation passed under linked evaluator.", + "meaning_of_unknown": "Template no-secrets-in-untrusted-context could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/ci_cd/no_secrets_in_untrusted_context.intent.json", + "input_contract": "Intent template no-secrets-in-untrusted-context over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "strict_eligible", + "release_status": "preview", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "source_profile_strict_eligible", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:no-secrets-in-untrusted-context" + }, + "trusted_components": [ + "intent template", + "backend_registry", + "enforcement_test", + "fail_example", + "intent_file", + "lane_evaluator", + "neutral_compiler", + "pass_example" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=executable_advisory" + ], + "backend_class": "custom", + "capability_id": "template-no-skipped-approval-state-v1", + "checker_id": "template:no-skipped-approval-state", + "claim_class": "state_invariant", + "claimed_backends": [ + "deployment-deterministic" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template no-skipped-approval-state obligation found a violation.", + "meaning_of_pass": "Template no-skipped-approval-state obligation passed under linked evaluator.", + "meaning_of_unknown": "Template no-skipped-approval-state could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/no_skipped_approval_state.intent.json", + "input_contract": "Intent template no-skipped-approval-state over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "strict_eligible", + "release_status": "preview", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "executable_advisory", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:no-skipped-approval-state" + }, + "trusted_components": [ + "intent template", + "backend_registry", + "enforcement_test", + "fail_example", + "intent_file", + "lane_evaluator", + "neutral_compiler", + "pass_example" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-opa-self-approval-block-v1", + "checker_id": "template:opa-self-approval-block", + "claim_class": "template_claim:opa", + "claimed_backends": [ + "opa" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template opa-self-approval-block obligation found a violation.", + "meaning_of_pass": "Template opa-self-approval-block obligation passed under linked evaluator.", + "meaning_of_unknown": "Template opa-self-approval-block could not be decided from available materials.", + "type": "template_claim:opa" + }, + "implementation": "templates/agent_authority/opa_self_approval_block.intent.json", + "input_contract": "Intent template opa-self-approval-block over domain agent_authority; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "agent_authority" + ], + "supported_property_kinds": [ + "runtime_monitorable" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:opa-self-approval-block" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-route-guard-12-v1", + "checker_id": "template:route-guard-12", + "claim_class": "authorization_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template route-guard-12 obligation found a violation.", + "meaning_of_pass": "Template route-guard-12 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template route-guard-12 could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/route_guard_12.intent.json", + "input_contract": "Intent template route-guard-12 over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:route-guard-12" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-route-guard-18-v1", + "checker_id": "template:route-guard-18", + "claim_class": "authorization_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template route-guard-18 obligation found a violation.", + "meaning_of_pass": "Template route-guard-18 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template route-guard-18 could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/route_guard_18.intent.json", + "input_contract": "Intent template route-guard-18 over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:route-guard-18" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-route-guard-24-v1", + "checker_id": "template:route-guard-24", + "claim_class": "authorization_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template route-guard-24 obligation found a violation.", + "meaning_of_pass": "Template route-guard-24 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template route-guard-24 could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/route_guard_24.intent.json", + "input_contract": "Intent template route-guard-24 over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:route-guard-24" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-route-guard-30-v1", + "checker_id": "template:route-guard-30", + "claim_class": "authorization_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template route-guard-30 obligation found a violation.", + "meaning_of_pass": "Template route-guard-30 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template route-guard-30 could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/route_guard_30.intent.json", + "input_contract": "Intent template route-guard-30 over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:route-guard-30" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-route-guard-36-v1", + "checker_id": "template:route-guard-36", + "claim_class": "authorization_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template route-guard-36 obligation found a violation.", + "meaning_of_pass": "Template route-guard-36 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template route-guard-36 could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/route_guard_36.intent.json", + "input_contract": "Intent template route-guard-36 over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:route-guard-36" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-route-guard-42-v1", + "checker_id": "template:route-guard-42", + "claim_class": "authorization_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template route-guard-42 obligation found a violation.", + "meaning_of_pass": "Template route-guard-42 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template route-guard-42 could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/route_guard_42.intent.json", + "input_contract": "Intent template route-guard-42 over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:route-guard-42" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-route-guard-6-v1", + "checker_id": "template:route-guard-6", + "claim_class": "authorization_invariant", + "claimed_backends": [], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template route-guard-6 obligation found a violation.", + "meaning_of_pass": "Template route-guard-6 obligation passed under linked evaluator.", + "meaning_of_unknown": "Template route-guard-6 could not be decided from available materials.", + "type": "authorization_invariant" + }, + "implementation": "templates/authorization/route_guard_6.intent.json", + "input_contract": "Intent template route-guard-6 over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:route-guard-6" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-rust-kani-bounds-check-v1", + "checker_id": "template:rust-kani-bounds-check", + "claim_class": "safety_invariant", + "claimed_backends": [ + "kani" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template rust-kani-bounds-check obligation found a violation.", + "meaning_of_pass": "Template rust-kani-bounds-check obligation passed under linked evaluator.", + "meaning_of_unknown": "Template rust-kani-bounds-check could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/authorization/rust_kani_bounds_check.intent.json", + "input_contract": "Intent template rust-kani-bounds-check over domain authorization; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:rust-kani-bounds-check" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-tla-approval-state-machine-v1", + "checker_id": "template:tla-approval-state-machine", + "claim_class": "state_invariant", + "claimed_backends": [ + "tla" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template tla-approval-state-machine obligation found a violation.", + "meaning_of_pass": "Template tla-approval-state-machine obligation passed under linked evaluator.", + "meaning_of_unknown": "Template tla-approval-state-machine could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/tla_approval_state_machine.intent.json", + "input_contract": "Intent template tla-approval-state-machine over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:tla-approval-state-machine" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-tla-eventual-rollback-from-failed-canary-v1", + "checker_id": "template:tla-eventual-rollback-from-failed-canary", + "claim_class": "liveness_property", + "claimed_backends": [ + "tla" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template tla-eventual-rollback-from-failed-canary obligation found a violation.", + "meaning_of_pass": "Template tla-eventual-rollback-from-failed-canary obligation passed under linked evaluator.", + "meaning_of_unknown": "Template tla-eventual-rollback-from-failed-canary could not be decided from available materials.", + "type": "liveness_property" + }, + "implementation": "templates/deployment/tla_eventual_rollback_from_failed_canary.intent.json", + "input_contract": "Intent template tla-eventual-rollback-from-failed-canary over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "liveness" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:tla-eventual-rollback-from-failed-canary" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-tla-no-skipped-approval-transition-v1", + "checker_id": "template:tla-no-skipped-approval-transition", + "claim_class": "state_invariant", + "claimed_backends": [ + "tla" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template tla-no-skipped-approval-transition obligation found a violation.", + "meaning_of_pass": "Template tla-no-skipped-approval-transition obligation passed under linked evaluator.", + "meaning_of_unknown": "Template tla-no-skipped-approval-transition could not be decided from available materials.", + "type": "state_invariant" + }, + "implementation": "templates/deployment/tla_no_skipped_approval_transition.intent.json", + "input_contract": "Intent template tla-no-skipped-approval-transition over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "invariant" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:tla-no-skipped-approval-transition" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-tla-rollback-safety-v1", + "checker_id": "template:tla-rollback-safety", + "claim_class": "safety_invariant", + "claimed_backends": [ + "tla" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template tla-rollback-safety obligation found a violation.", + "meaning_of_pass": "Template tla-rollback-safety obligation passed under linked evaluator.", + "meaning_of_unknown": "Template tla-rollback-safety could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/deployment/tla_rollback_safety.intent.json", + "input_contract": "Intent template tla-rollback-safety over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:tla-rollback-safety" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-tla-rollout-requires-green-health-v1", + "checker_id": "template:tla-rollout-requires-green-health", + "claim_class": "safety_invariant", + "claimed_backends": [ + "tla" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template tla-rollout-requires-green-health obligation found a violation.", + "meaning_of_pass": "Template tla-rollout-requires-green-health obligation passed under linked evaluator.", + "meaning_of_unknown": "Template tla-rollout-requires-green-health could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/deployment/tla_rollout_requires_green_health.intent.json", + "input_contract": "Intent template tla-rollout-requires-green-health over domain deployment; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:tla-rollout-requires-green-health" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-verus-build-integrity-v1", + "checker_id": "template:verus-build-integrity", + "claim_class": "safety_invariant", + "claimed_backends": [ + "verus" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template verus-build-integrity obligation found a violation.", + "meaning_of_pass": "Template verus-build-integrity obligation passed under linked evaluator.", + "meaning_of_unknown": "Template verus-build-integrity could not be decided from available materials.", + "type": "safety_invariant" + }, + "implementation": "templates/ci_cd/verus_build_integrity.intent.json", + "input_contract": "Intent template verus-build-integrity over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "safety" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:verus-build-integrity" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + }, + { + "assumptions": [ + "Template conformance links accurately describe executable surfaces.", + "conformance_status_v2=catalog_only" + ], + "backend_class": "custom", + "capability_id": "template-z3-secret-flow-check-v1", + "checker_id": "template:z3-secret-flow-check", + "claim_class": "data_boundary_invariant", + "claimed_backends": [ + "z3" + ], + "determinism_status": "deterministic", + "failure_semantics": "Missing executable links keep the template experimental/catalog_only.", + "guarantee": { + "meaning_of_fail": "Template z3-secret-flow-check obligation found a violation.", + "meaning_of_pass": "Template z3-secret-flow-check obligation passed under linked evaluator.", + "meaning_of_unknown": "Template z3-secret-flow-check could not be decided from available materials.", + "type": "data_boundary_invariant" + }, + "implementation": "templates/ci_cd/z3_secret_flow_check.intent.json", + "input_contract": "Intent template z3-secret-flow-check over domain ci_cd; materials supplied by lane compilers / examples.", + "native_execution": false, + "output_contract": "ovk.evidence via lane evaluator linked by template conformance", + "owner": "ovk-maintainers", + "production_status": "catalog_only", + "release_status": "experimental", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "data_boundary" + ], + "template_conformance_status_v2": "catalog_only", + "timeout_semantics": "unknown", + "tool": { + "adapter": "ovk-template-registry", + "adapter_version": "0.1.0", + "name": "template:z3-secret-flow-check" + }, + "trusted_components": [ + "intent template", + "intent_file" + ], + "unsupported_semantics": "Template claims only what linked evaluators and source profiles establish; catalog_only templates make no production enforcement claim.", + "version": "0.1.0" + } + ], + "entry_count": 100, + "schema_version": "ovk.template_capability_registry.v1", + "source_conformance_schema_version": "ovk.template_conformance.v1" +} diff --git a/tests/test_capability_registry_normative.py b/tests/test_capability_registry_normative.py new file mode 100644 index 0000000..6405ddd --- /dev/null +++ b/tests/test_capability_registry_normative.py @@ -0,0 +1,196 @@ +"""Tests for normative capability registry validation (OVK-02).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from ovk.core.capabilities import ( + CapabilityRegistry, + validate_capability_manifest, +) +from ovk.core.execution_models import BackendCapabilityManifest, BackendGuaranteeDeclaration, BackendToolIdentity +from scripts.validate_capabilities import validate_capabilities + + +ROOT = Path(__file__).resolve().parents[1] + + +def _valid_manifest(**overrides: object) -> dict: + payload = { + "capability_id": "example-v1", + "checker_id": "example", + "version": "0.1.0", + "implementation": "ovk-adapter-example", + "input_contract": "JSON input", + "output_contract": "ovk.result.v1", + "claim_class": "policy_evaluation", + "tool": { + "name": "example", + "adapter": "ovk-adapter-example", + "adapter_version": "0.1.0", + }, + "backend_class": "custom", + "supported_domains": ["authorization"], + "supported_property_kinds": ["safety"], + "guarantee": { + "type": "policy_evaluation", + "meaning_of_pass": "pass", + "meaning_of_fail": "fail", + "meaning_of_unknown": "unknown", + }, + "assumptions": ["test assumption"], + "trusted_components": ["adapter"], + "failure_semantics": "errors map to error", + "timeout_semantics": "unknown", + "unsupported_semantics": "unsupported inputs yield unknown", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": False, + } + payload.update(overrides) + return payload + + +def test_validate_capabilities_passes_for_repo_manifests() -> None: + assert validate_capabilities() == [] + + +def test_registry_loads_advertised_checkers() -> None: + registry = CapabilityRegistry.from_directory(ROOT / "adapters") + checkers = {m["checker_id"] for m in registry.all()} + assert checkers == { + "opa", + "z3", + "cbmc", + "cedar", + "tla+", + "kani", + "dafny", + "verus", + "lean", + "alloy", + "lane-self-protection", + "lane-authorization", + "lane-infrastructure", + "lane-ci-secrets", + "lane-deployment", + } + + +def test_reject_stable_without_conformance(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "ovk.core.adapter_conformance.is_fully_conformant", + lambda *_args, **_kwargs: False, + ) + failures = validate_capability_manifest( + _valid_manifest(release_status="stable", native_execution=True, checker_id="opa"), + source="test", + ) + assert any("requires full seven-item" in item for item in failures) + + +def test_reject_unknown_release_status() -> None: + failures = validate_capability_manifest( + _valid_manifest(release_status="ga"), + source="test", + ) + assert any("unknown release_status" in item for item in failures) + + +def test_stable_allowed_when_conformant(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "ovk.core.adapter_conformance.is_fully_conformant", + lambda *_args, **_kwargs: True, + ) + failures = validate_capability_manifest( + _valid_manifest(release_status="stable", native_execution=True, checker_id="opa"), + source="test", + ) + assert not any("requires full seven-item" in item for item in failures) + assert not any("only native-execution" in item for item in failures) + + +def test_non_native_cannot_claim_beyond_preview() -> None: + # Non-native checkers cannot use stable even if conformance were claimed. + failures = validate_capability_manifest( + _valid_manifest(release_status="stable", native_execution=False), + source="test", + ) + assert any("only native-execution" in item or "requires full seven-item" in item for item in failures) + + +def test_typed_manifest_fills_normative_fields() -> None: + manifest = BackendCapabilityManifest( + capability_id="lane-test-v1", + tool=BackendToolIdentity( + name="lane-test", + adapter="ovk-adapter-lane-test", + adapter_version="0.1.0", + ), + backend_class="custom", + guarantee=BackendGuaranteeDeclaration( + type="deterministic_witness", + meaning_of_pass="p", + meaning_of_fail="f", + meaning_of_unknown="u", + ), + supported_domains=["authorization"], + supported_property_kinds=["access_control"], + assumptions=["a"], + limits=["l"], + ) + assert manifest.checker_id == "lane-test" + assert manifest.implementation == "ovk-adapter-lane-test" + assert manifest.claim_class == "deterministic_witness" + assert manifest.timeout_semantics == "unknown" + assert manifest.release_status == "experimental" + + +def test_typed_manifest_rejects_unknown_release_status() -> None: + with pytest.raises(Exception, match="release_status"): + BackendCapabilityManifest( + capability_id="broken-v1", + tool=BackendToolIdentity( + name="broken", + adapter="ovk-adapter-broken", + adapter_version="0.1.0", + ), + backend_class="custom", + guarantee=BackendGuaranteeDeclaration( + type="x", + meaning_of_pass="p", + meaning_of_fail="f", + meaning_of_unknown="u", + ), + supported_domains=["authorization"], + supported_property_kinds=["access_control"], + release_status="not-a-status", # type: ignore[arg-type] + ) + + +def test_every_capability_json_has_required_normative_fields() -> None: + required = { + "checker_id", + "version", + "implementation", + "input_contract", + "output_contract", + "claim_class", + "assumptions", + "trusted_components", + "failure_semantics", + "timeout_semantics", + "unsupported_semantics", + "determinism_status", + "release_status", + "owner", + } + for path in sorted((ROOT / "adapters").glob("*/capability.json")): + payload = json.loads(path.read_text(encoding="utf-8")) + missing = required - set(payload) + assert not missing, f"{path} missing {sorted(missing)}" + assert payload["release_status"] in {"stable", "preview", "experimental", "disabled"} From 503501349a9f2f4e16cf0efb960487c025f0eed1 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:08:52 -0700 Subject: [PATCH 03/19] Introduce DecisionState lattice with truth-table coverage (OVK-PR2). Replace ad-hoc merge recommendations with a normative decision lattice, wire aggregation and CLI exit codes through decision_state, and lock behavior with exhaustive truth-table tests. --- docs/FORMAL_SPEC.md | 80 +++- ovk/cli.py | 56 ++- ovk/core/backend_aggregation.py | 213 ++++++---- ovk/core/backend_control_plane.py | 3 + ovk/core/decision.py | 473 ++++++++++++++++++--- ovk/core/exit_codes.py | 38 +- ovk/core/models.py | 64 +++ tests/test_decision.py | 32 +- tests/test_decision_lattice_truth_table.py | 282 ++++++++++++ 9 files changed, 1055 insertions(+), 186 deletions(-) create mode 100644 tests/test_decision_lattice_truth_table.py diff --git a/docs/FORMAL_SPEC.md b/docs/FORMAL_SPEC.md index c2f8939..2f210fd 100644 --- a/docs/FORMAL_SPEC.md +++ b/docs/FORMAL_SPEC.md @@ -124,34 +124,62 @@ ValidEvidence(E) iff ## Merge decision ```text -MergePolicy(I, EvidenceSet) -> Decision +MergePolicy(I, EvidenceSet) -> DecisionState ``` -Decision is one of: +Normative ``DecisionState`` lattice: ```text allow block -require_human_review -allow_with_warning -require_stronger_check +needs_review +unknown +error +skipped ``` -Default logic: +Checker claim statuses remain distinct: ```text -if any critical intent fails: +pass | fail | unknown | error | skipped +``` + +``merge_recommendation`` is a deprecated alias of ``decision_state`` +(``needs_review`` ↔ ``require_human_review``). Legacy values +``allow_with_warning`` and ``require_stronger_check`` are not lattice +members; they map onto ``needs_review`` (never onto ``allow``). + +Hard rules: + +```text +error never promotes to allow (strict and advisory) +unknown never becomes allow in strict (advisory preserves unknown) +required skipped never silent-allows (strict: skipped or block) +advisory preserves original_decision_state +decision lists controlling_finding_ids[] and finding_contributions[] +``` + +Default strict logic: + +```text +if any required claim fails: block -elif any critical intent is unknown, error, or skipped: - require_human_review -elif all required intents pass: +elif any required claim is error: + error +elif any required claim is unknown: + needs_review or block # per default_on_unknown; never allow +elif any required claim is skipped: + skipped or block # per default_on_required_skip; never allow +elif all required claims pass: allow -elif low-risk intents are skipped with justification: - allow_with_warning else: - require_human_review + needs_review ``` +Advisory mode keeps the honest lattice member in ``decision_state`` / +``original_decision_state`` and does not invent ``allow_with_warning`` +as a lattice value; non-blocking behavior is an exit-code / CI concern. + ## Security rules These match the rules in [THREAT_MODEL.md](THREAT_MODEL.md): agents cannot self-disable checks; unknowns never pass in strict mode; evidence is complete and content-addressed; critical failures block unless a human override is recorded; high-risk runtime checks need template provenance or human review. @@ -163,19 +191,25 @@ These match the rules in [THREAT_MODEL.md](THREAT_MODEL.md): agents cannot self- EXTENDS Naturals, Sequences CONSTANTS Intents, Critical, Pass, Fail, Unknown, Error, Skipped -CONSTANTS Allow, Block, RequireHumanReview +CONSTANTS Allow, Block, NeedsReview, DecisionUnknown, DecisionError, DecisionSkipped VARIABLES result, decision Init == /\ result \in [Intents -> {Pass, Fail, Unknown, Error, Skipped}] - /\ decision = RequireHumanReview + /\ decision = NeedsReview CriticalFailure == \E i \in Critical : result[i] = Fail +CriticalError == + \E i \in Critical : result[i] = Error + CriticalUnknown == - \E i \in Critical : result[i] \in {Unknown, Error, Skipped} + \E i \in Critical : result[i] = Unknown + +CriticalSkipped == + \E i \in Critical : result[i] = Skipped AllRequiredPass == \A i \in Critical : result[i] = Pass @@ -183,17 +217,27 @@ AllRequiredPass == Decide == IF CriticalFailure THEN decision' = Block + ELSE IF CriticalError THEN + decision' = DecisionError ELSE IF CriticalUnknown THEN - decision' = RequireHumanReview + decision' = NeedsReview + ELSE IF CriticalSkipped THEN + decision' = DecisionSkipped ELSE IF AllRequiredPass THEN decision' = Allow ELSE - decision' = RequireHumanReview + decision' = NeedsReview Safety_NoAllowOnCriticalFail == CriticalFailure => decision # Allow +Safety_NoAllowOnCriticalError == + CriticalError => decision # Allow + Safety_NoAllowOnCriticalUnknown == CriticalUnknown => decision # Allow + +Safety_NoAllowOnCriticalSkipped == + CriticalSkipped => decision # Allow ============================================================================= ``` diff --git a/ovk/cli.py b/ovk/cli.py index 8122f1b..91d21aa 100644 --- a/ovk/cli.py +++ b/ovk/cli.py @@ -18,9 +18,8 @@ from ovk.adapters.z3.validated_path import evaluate_validated_authorization_path from ovk.core.bundle import make_bundle from ovk.core.changed_files import load_changed_files -from ovk.core.decision import decide from ovk.core.evidence_quality import build_evidence_quality_report -from ovk.core.exit_codes import exit_code_for_recommendation +from ovk.core.exit_codes import exit_code_for_decision_state, exit_code_for_recommendation from ovk.core.json_io import read_json_file, write_json_file from ovk.core.models import EvidenceBundle, VerificationEvidence from ovk.core.output_validation import validate_output_directory @@ -55,6 +54,14 @@ app.add_typer(template_app, name="template") +def _bundle_decision_state(bundle: EvidenceBundle) -> str: + """Primary decision_state with deprecated merge_recommendation fallback.""" + decision = bundle.decision or {} + if decision.get("decision_state"): + return str(decision["decision_state"]) + return str(decision.get("merge_recommendation", "needs_review")) + + def _finish_lane( bundle: EvidenceBundle, *, @@ -76,10 +83,14 @@ def _finish_lane( quality_report=quality_output, ) write_standard_run_outputs(bundle, paths) - recommendation = str(bundle.decision.get("merge_recommendation", "require_human_review")) - typer.echo(f"OVK {label} recommendation: {recommendation}") + decision_state = str( + bundle.decision.get("decision_state") + or bundle.decision.get("merge_recommendation", "needs_review") + ) + recommendation = str(bundle.decision.get("merge_recommendation", decision_state)) + typer.echo(f"OVK {label} decision_state: {decision_state} (merge_recommendation={recommendation})") if not advisory: - raise typer.Exit(code=exit_code_for_recommendation(recommendation)) + raise typer.Exit(code=exit_code_for_decision_state(decision_state)) @app.command("init") @@ -155,10 +166,13 @@ def validate(instance: Path, schema: Path) -> None: @app.command("decide-bundle") def decide_bundle(evidence_bundle: Path, enforce: bool = True) -> None: - """Compute a merge recommendation for an evidence bundle.""" + """Compute a decision_state for an evidence bundle.""" + from ovk.core.decision import decide_with_reason + bundle = EvidenceBundle.model_validate(read_json_file(evidence_bundle)) - recommendation = decide(bundle, enforce=enforce) - typer.echo(recommendation.value) + decision = decide_with_reason(bundle, enforce=enforce) + typer.echo(decision["decision_state"]) + typer.echo(f"merge_recommendation={decision['merge_recommendation']}", err=True) @app.command("render-pr-comment") @@ -355,9 +369,10 @@ def demo_self_protection( if markdown_output: markdown_output.write_text(render_bundle_markdown(bundle), encoding="utf-8") recommendation = bundle.decision.get("merge_recommendation", "require_human_review") - typer.echo(f"OVK recommendation: {recommendation}") + decision_state = _bundle_decision_state(bundle) + typer.echo(f"OVK decision_state: {decision_state} (merge_recommendation={recommendation})") if enforce: - raise typer.Exit(code=exit_code_for_recommendation(str(recommendation))) + raise typer.Exit(code=exit_code_for_decision_state(decision_state)) @app.command("ci") @@ -573,10 +588,11 @@ def verify( if failures: raise typer.Exit(code=1) recommendation = str(bundle.decision.get("merge_recommendation", "require_human_review")) - typer.echo(f"OVK multi-lane recommendation: {recommendation}") + decision_state = _bundle_decision_state(bundle) + typer.echo(f"OVK multi-lane decision_state: {decision_state} (merge_recommendation={recommendation})") typer.echo(f"Release bundle written to {output_dir}") if not advisory: - raise typer.Exit(code=exit_code_for_recommendation(recommendation)) + raise typer.Exit(code=exit_code_for_decision_state(decision_state)) @app.command("extract-workflow") @@ -672,7 +688,11 @@ def check( write_standard_run_outputs(result.bundle, paths) record_run(result.bundle.model_dump(mode="json")) recommendation = str(result.bundle.decision.get("merge_recommendation", "require_human_review")) - typer.echo(f"OVK check recommendation: {recommendation} ({result.elapsed_ms:.0f}ms)") + decision_state = _bundle_decision_state(result.bundle) + typer.echo( + f"OVK check decision_state: {decision_state} " + f"(merge_recommendation={recommendation}, {result.elapsed_ms:.0f}ms)" + ) if format == "json": typer.echo(json.dumps(result.bundle.model_dump(mode="json"), indent=2)) elif format != "md": @@ -680,7 +700,7 @@ def check( else: typer.echo(result.markdown) if enforce_exit: - raise typer.Exit(code=exit_code_for_recommendation(recommendation)) + raise typer.Exit(code=exit_code_for_decision_state(decision_state)) @app.command("doctor") @@ -738,10 +758,14 @@ def run_cmd( write_standard_run_outputs(result.bundle, paths) record_run(result.bundle.model_dump(mode="json")) recommendation = str(result.bundle.decision.get("merge_recommendation", "require_human_review")) - typer.echo(f"OVK run recommendation: {recommendation} ({result.elapsed_ms:.0f}ms)") + decision_state = _bundle_decision_state(result.bundle) + typer.echo( + f"OVK run decision_state: {decision_state} " + f"(merge_recommendation={recommendation}, {result.elapsed_ms:.0f}ms)" + ) typer.echo(f"OVK run lanes: {sorted({item['lane'] for item in result.obligations})}") if not advisory: - raise typer.Exit(code=exit_code_for_recommendation(recommendation)) + raise typer.Exit(code=exit_code_for_decision_state(decision_state)) @app.command("generate-test") diff --git a/ovk/core/backend_aggregation.py b/ovk/core/backend_aggregation.py index df5ae16..0e3c443 100644 --- a/ovk/core/backend_aggregation.py +++ b/ovk/core/backend_aggregation.py @@ -8,6 +8,12 @@ from dataclasses import dataclass from typing import Any, Sequence +from ovk.core.decision import ( + ClaimFinding, + DecisionOutcome, + aggregate_decision, + decision_state_to_merge_recommendation, +) from ovk.core.execution_models import ( BackendSelection, ExecutionAttempt, @@ -15,7 +21,7 @@ NormalizedBackendResult, TerminationKind, ) -from ovk.core.models import MergeRecommendation, VerificationStatus +from ovk.core.models import DecisionState, MergeRecommendation, VerificationStatus AGGREGATION_FAIL_DOMINANT_V1 = "ovk.aggregate.fail_dominant.v1" @@ -34,6 +40,7 @@ class AggregationOutcome: """Result of aggregating required and optional backend results.""" status: VerificationStatus + decision_state: DecisionState merge_recommendation: MergeRecommendation reason: str disagreement: dict[str, Any] | None = None @@ -42,6 +49,36 @@ class AggregationOutcome: fallback_used: bool = False fallback_accepted: bool = False fallback_cause: str | None = None + controlling_finding_ids: tuple[str, ...] = () + original_decision_state: DecisionState | None = None + + @staticmethod + def from_lattice( + *, + status: VerificationStatus, + outcome: DecisionOutcome, + disagreement: dict[str, Any] | None = None, + quality_error: bool = False, + fallback_used: bool = False, + fallback_accepted: bool = False, + fallback_cause: str | None = None, + extra_warnings: tuple[str, ...] = (), + ) -> AggregationOutcome: + recommendation = outcome.legacy_merge_override or outcome.merge_recommendation + return AggregationOutcome( + status=status, + decision_state=outcome.decision_state, + original_decision_state=outcome.original_decision_state, + merge_recommendation=recommendation, + reason=outcome.reason, + disagreement=disagreement, + warnings=tuple(list(extra_warnings) + list(outcome.warnings)), + quality_error=quality_error, + fallback_used=fallback_used, + fallback_accepted=fallback_accepted, + fallback_cause=fallback_cause, + controlling_finding_ids=outcome.controlling_finding_ids, + ) def evaluate_fallback_acceptance( @@ -128,15 +165,16 @@ def aggregate_fail_dominant_v1( fallback_policy: FallbackPolicy | None = None, attempts: Sequence[ExecutionAttempt] | None = None, ) -> AggregationOutcome: - """Apply the fail-dominant aggregation decision table. + """Apply the fail-dominant aggregation decision table via the DecisionState lattice. Decision table (required backends): * any fail -> block - * no fail, any error/timeout/unknown/skipped -> require_human_review + * no fail, any error -> error (never allow) + * no fail/error, any unknown/skipped -> unknown/skipped (strict: never allow) * every required pass with acceptable guarantees -> allow - * no required result -> require_human_review - * selected vs executed mismatch -> require_human_review + quality error - * unaccepted weaker fallback -> require_stronger_check + * no required result -> needs_review + * selected vs executed mismatch -> needs_review + quality error + * unaccepted weaker fallback -> needs_review (legacy alias require_stronger_check) Optional corroborators: * optional fail upgrades to block @@ -162,32 +200,45 @@ def aggregate_fail_dominant_v1( by_backend = _statuses_by_backend(results) executed = set(by_backend) selected_ids = {item.backend for item in selected} + required_ids = {item.backend for item in selected_required} if selected_ids != executed: missing = sorted(selected_ids - executed) unexpected = sorted(executed - selected_ids) + state = DecisionState.NEEDS_REVIEW return AggregationOutcome( status=VerificationStatus.UNKNOWN, - merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, + decision_state=state, + original_decision_state=state, + merge_recommendation=decision_state_to_merge_recommendation(state), reason=(f"selected and executed backend sets differ; missing={missing}; unexpected={unexpected}"), quality_error=True, ) - required_results = [item for item in results if item.backend in {s.backend for s in selected_required}] + required_results = [item for item in results if item.backend in required_ids] optional_results = [item for item in results if item.backend in {s.backend for s in selected_optional}] if selected_required and not required_results: + state = DecisionState.NEEDS_REVIEW return AggregationOutcome( status=VerificationStatus.UNKNOWN, - merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, + decision_state=state, + original_decision_state=state, + merge_recommendation=decision_state_to_merge_recommendation(state), reason="no required result exists", quality_error=True, ) - warnings: list[str] = [] - disagreement = None + findings = [ + ClaimFinding( + finding_id=f"{obligation_id}:{item.backend}", + status=item.status, + required=item.backend in required_ids, + ) + for item in results + ] - # Optional fail upgrades aggregate to block. + disagreement = None if any(item.status == VerificationStatus.FAIL for item in optional_results): if required_results and any(item.status == VerificationStatus.PASS for item in required_results): disagreement = build_disagreement_artifact( @@ -195,87 +246,93 @@ def aggregate_fail_dominant_v1( results=list(results), resolution="block", ) - return AggregationOutcome( - status=VerificationStatus.FAIL, - merge_recommendation=MergeRecommendation.BLOCK, - reason="optional corroborator reported fail", - disagreement=disagreement, - fallback_used=fallback_used, - fallback_accepted=resolved_fallback_accepted, - fallback_cause=fallback_cause, - ) - - if any(item.status == VerificationStatus.FAIL for item in required_results): + elif any(item.status == VerificationStatus.FAIL for item in required_results): if len({item.status for item in required_results}) > 1 or optional_results: disagreement = build_disagreement_artifact( obligation_id=obligation_id, results=list(results), resolution="block", ) - return AggregationOutcome( - status=VerificationStatus.FAIL, - merge_recommendation=MergeRecommendation.BLOCK, - reason="required backend reported fail", - disagreement=disagreement, - fallback_used=fallback_used, - fallback_accepted=resolved_fallback_accepted, - fallback_cause=fallback_cause, - ) - non_pass = { - VerificationStatus.ERROR, - VerificationStatus.UNKNOWN, - VerificationStatus.SKIPPED, - } - if any(item.status in non_pass for item in required_results): - for item in optional_results: - if item.status == VerificationStatus.PASS: - warnings.append(f"optional backend {item.backend} passed but cannot upgrade required unknown/error") - return AggregationOutcome( - status=VerificationStatus.UNKNOWN, - merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, - reason="required backend reported unknown, error, or skipped", - warnings=tuple(warnings), - fallback_used=fallback_used, - fallback_accepted=resolved_fallback_accepted, - fallback_cause=fallback_cause, + # Guarantee / fallback gate before allow. + acceptable = set(acceptable_guarantees or []) + stronger_check = False + stronger_reason = "" + if findings and all( + (not item.required) or item.status == VerificationStatus.PASS for item in findings + ): + for item in required_results: + if acceptable and item.guarantee_type not in acceptable and not resolved_fallback_accepted: + stronger_check = True + stronger_reason = ( + f"required result from {item.backend} uses guarantee " + f"{item.guarantee_type!r} outside acceptable set" + ) + break + + if stronger_check: + state = DecisionState.NEEDS_REVIEW + controlling = tuple( + sorted(f"{obligation_id}:{item.backend}" for item in required_results) ) - - if not required_results and not selected_required: - # No required selection — conservative review. return AggregationOutcome( status=VerificationStatus.UNKNOWN, - merge_recommendation=MergeRecommendation.REQUIRE_HUMAN_REVIEW, - reason="no required backends were selected", + decision_state=state, + original_decision_state=state, + merge_recommendation=MergeRecommendation.REQUIRE_STRONGER_CHECK, + reason=stronger_reason, + controlling_finding_ids=controlling, fallback_used=fallback_used, fallback_accepted=resolved_fallback_accepted, fallback_cause=fallback_cause, ) - # Check guarantees / fallback acceptance for required passes. - acceptable = set(acceptable_guarantees or []) - for item in required_results: - if acceptable and item.guarantee_type not in acceptable and not resolved_fallback_accepted: - return AggregationOutcome( - status=VerificationStatus.UNKNOWN, - merge_recommendation=MergeRecommendation.REQUIRE_STRONGER_CHECK, - reason=( - f"required result from {item.backend} uses guarantee {item.guarantee_type!r} outside acceptable set" - ), - fallback_used=fallback_used, - fallback_accepted=resolved_fallback_accepted, - fallback_cause=fallback_cause, - ) - - for item in optional_results: - if item.status in non_pass: - warnings.append(f"optional backend {item.backend} returned {item.status.value}") - - return AggregationOutcome( - status=VerificationStatus.PASS, - merge_recommendation=MergeRecommendation.ALLOW, - reason="every required backend passed with acceptable guarantees", - warnings=tuple(warnings), + lattice = aggregate_decision(findings, mode="strict") + # Preserve historical wording for common paths when lattice reason is generic. + reason = lattice.reason + if lattice.decision_state == DecisionState.BLOCK and any( + item.status == VerificationStatus.FAIL for item in optional_results + ): + reason = "optional corroborator reported fail" + elif lattice.decision_state == DecisionState.BLOCK: + reason = "required backend reported fail" + elif lattice.original_decision_state in { + DecisionState.ERROR, + DecisionState.UNKNOWN, + DecisionState.SKIPPED, + }: + reason = "required backend reported unknown, error, or skipped" + elif lattice.decision_state == DecisionState.ALLOW: + reason = "every required backend passed with acceptable guarantees" + elif not required_results and not selected_required: + reason = "no required backends were selected" + + # Map aggregate claim status for execution records. + if lattice.decision_state == DecisionState.BLOCK: + status = VerificationStatus.FAIL + elif lattice.decision_state == DecisionState.ALLOW: + status = VerificationStatus.PASS + elif lattice.decision_state == DecisionState.ERROR: + status = VerificationStatus.ERROR + elif lattice.decision_state == DecisionState.SKIPPED: + status = VerificationStatus.SKIPPED + else: + # needs_review / unknown — preserve historical UNKNOWN collapse + status = VerificationStatus.UNKNOWN + + return AggregationOutcome.from_lattice( + status=status, + outcome=DecisionOutcome( + decision_state=lattice.decision_state, + original_decision_state=lattice.original_decision_state, + merge_recommendation=lattice.merge_recommendation, + reason=reason, + controlling_finding_ids=lattice.controlling_finding_ids, + finding_contributions=lattice.finding_contributions, + mode=lattice.mode, + warnings=lattice.warnings, + ), + disagreement=disagreement, fallback_used=fallback_used, fallback_accepted=resolved_fallback_accepted, fallback_cause=fallback_cause, diff --git a/ovk/core/backend_control_plane.py b/ovk/core/backend_control_plane.py index 036192e..e8b77e1 100644 --- a/ovk/core/backend_control_plane.py +++ b/ovk/core/backend_control_plane.py @@ -290,12 +290,15 @@ def execute( attempts=attempts, results=results, aggregate_status=outcome.status, + decision_state=outcome.decision_state, + original_decision_state=outcome.original_decision_state, merge_recommendation=outcome.merge_recommendation, aggregation_reason=outcome.reason, open_obligations=open_obligations, fallback_used=outcome.fallback_used, fallback_accepted=outcome.fallback_accepted, fallback_cause=outcome.fallback_cause, + controlling_finding_ids=list(outcome.controlling_finding_ids), ) def _execute_one( diff --git a/ovk/core/decision.py b/ovk/core/decision.py index ba8512d..d8585f8 100644 --- a/ovk/core/decision.py +++ b/ovk/core/decision.py @@ -1,8 +1,157 @@ -"""Merge decision logic for OVK evidence bundles.""" +"""Merge decision lattice and aggregation for OVK evidence bundles. + +Normative lattice (``DecisionState``): + allow | block | needs_review | unknown | error | skipped + +Hard rules: +- ``error`` never promotes to ``allow`` (strict and advisory) +- ``unknown`` never becomes ``allow`` in strict mode; advisory preserves ``unknown`` +- Required ``skipped`` never silently allows in strict mode (``skipped`` or ``block``) +- Advisory preserves the honest lattice state via ``original_decision_state`` +- Decisions list ``controlling_finding_ids`` and per-finding contributions + +``merge_recommendation`` remains a deprecated alias of ``decision_state``. +""" from __future__ import annotations -from ovk.core.models import EvidenceBundle, MergeRecommendation, VerificationStatus +from dataclasses import dataclass +from typing import Any, Iterable, Literal, Sequence + +from ovk.core.models import ( + DecisionState, + EvidenceBundle, + FindingContribution, + MergeRecommendation, + VerificationStatus, +) + +Mode = Literal["strict", "advisory"] + +# Claim severity for aggregation (higher = more severe / more controlling). +_CLAIM_SEVERITY: dict[VerificationStatus, int] = { + VerificationStatus.PASS: 0, + VerificationStatus.SKIPPED: 1, + VerificationStatus.UNKNOWN: 2, + VerificationStatus.ERROR: 3, + VerificationStatus.FAIL: 4, +} + +_DECISION_SEVERITY: dict[DecisionState, int] = { + DecisionState.ALLOW: 0, + DecisionState.NEEDS_REVIEW: 1, + DecisionState.SKIPPED: 2, + DecisionState.UNKNOWN: 3, + DecisionState.ERROR: 4, + DecisionState.BLOCK: 5, +} + +# Legacy merge_recommendation ↔ DecisionState +_STATE_TO_LEGACY: dict[DecisionState, MergeRecommendation] = { + DecisionState.ALLOW: MergeRecommendation.ALLOW, + DecisionState.BLOCK: MergeRecommendation.BLOCK, + DecisionState.NEEDS_REVIEW: MergeRecommendation.REQUIRE_HUMAN_REVIEW, + DecisionState.UNKNOWN: MergeRecommendation.REQUIRE_HUMAN_REVIEW, + DecisionState.ERROR: MergeRecommendation.REQUIRE_HUMAN_REVIEW, + DecisionState.SKIPPED: MergeRecommendation.REQUIRE_HUMAN_REVIEW, +} + +_LEGACY_TO_STATE: dict[str, DecisionState] = { + "allow": DecisionState.ALLOW, + "block": DecisionState.BLOCK, + "needs_review": DecisionState.NEEDS_REVIEW, + "require_human_review": DecisionState.NEEDS_REVIEW, + "unknown": DecisionState.UNKNOWN, + "error": DecisionState.ERROR, + "skipped": DecisionState.SKIPPED, + # Legacy aliases — not lattice members; map carefully (never to allow). + "allow_with_warning": DecisionState.NEEDS_REVIEW, + "require_stronger_check": DecisionState.NEEDS_REVIEW, +} + +_UNKNOWN_POLICY_ALIASES = { + "require_human_review": "needs_review", + "needs_review": "needs_review", + "block": "block", + # Legacy: must not promote unknown → allow under the lattice. + "allow_with_warning": "needs_review", +} + + +@dataclass(frozen=True) +class ClaimFinding: + """One checker claim participating in lattice aggregation.""" + + finding_id: str + status: VerificationStatus + required: bool = True + + +@dataclass(frozen=True) +class DecisionOutcome: + """Full aggregated decision with attribution and legacy alias.""" + + decision_state: DecisionState + original_decision_state: DecisionState + merge_recommendation: MergeRecommendation + reason: str + controlling_finding_ids: tuple[str, ...] = () + finding_contributions: tuple[FindingContribution, ...] = () + mode: Mode = "strict" + warnings: tuple[str, ...] = () + # When stronger-check semantics apply, keep the specialized legacy alias. + legacy_merge_override: MergeRecommendation | None = None + + def to_decision_dict(self) -> dict[str, Any]: + """Serialize for evidence / bundle ``decision`` objects.""" + recommendation = self.legacy_merge_override or self.merge_recommendation + return { + "decision_state": self.decision_state.value, + "original_decision_state": self.original_decision_state.value, + "merge_recommendation": recommendation.value, + "reason": self.reason, + "controlling_finding_ids": list(self.controlling_finding_ids), + "finding_contributions": [ + item.model_dump(mode="json") for item in self.finding_contributions + ], + "human_review_required": self.decision_state != DecisionState.ALLOW, + } + + +def decision_state_to_merge_recommendation(state: DecisionState) -> MergeRecommendation: + """Map a lattice state to the deprecated merge_recommendation alias.""" + return _STATE_TO_LEGACY[state] + + +def merge_recommendation_to_decision_state(value: str | MergeRecommendation | DecisionState) -> DecisionState: + """Map a legacy or lattice string onto ``DecisionState``.""" + if isinstance(value, DecisionState): + return value + if isinstance(value, MergeRecommendation): + raw = value.value + else: + raw = str(value).strip() + if raw in _LEGACY_TO_STATE: + return _LEGACY_TO_STATE[raw] + return DecisionState.NEEDS_REVIEW + + +def normalize_unknown_policy(default_on_unknown: str) -> Literal["needs_review", "block"]: + """Normalize unknown policy; never yields allow.""" + normalized = _UNKNOWN_POLICY_ALIASES.get(str(default_on_unknown).strip(), "needs_review") + if normalized == "block": + return "block" + return "needs_review" + + +def normalize_required_skip_policy( + default_on_required_skip: str, +) -> Literal["skipped", "block"]: + """Normalize required-skip policy; never yields allow.""" + value = str(default_on_required_skip).strip().lower() + if value == "block": + return "block" + return "skipped" def evidence_has_status(bundle: EvidenceBundle, status: VerificationStatus) -> bool: @@ -20,79 +169,279 @@ def evidence_has_unknown_like(bundle: EvidenceBundle) -> bool: return any(claim.status in unknown_like for evidence in bundle.evidence for claim in evidence.backend_claims) -def _unknown_like_recommendation( +def findings_from_bundle(bundle: EvidenceBundle) -> list[ClaimFinding]: + """Derive claim findings from an evidence bundle (all claims required by default).""" + findings: list[ClaimFinding] = [] + for evidence in bundle.evidence: + for claim in evidence.backend_claims: + findings.append( + ClaimFinding( + finding_id=f"{evidence.evidence_id}:{claim.backend}", + status=claim.status, + required=bool(getattr(claim, "required", True)), + ) + ) + return findings + + +def _worst_status(statuses: Iterable[VerificationStatus]) -> VerificationStatus | None: + worst: VerificationStatus | None = None + worst_rank = -1 + for status in statuses: + rank = _CLAIM_SEVERITY[status] + if rank > worst_rank: + worst = status + worst_rank = rank + return worst + + +def _base_state_from_status(status: VerificationStatus) -> DecisionState: + if status == VerificationStatus.FAIL: + return DecisionState.BLOCK + if status == VerificationStatus.ERROR: + return DecisionState.ERROR + if status == VerificationStatus.UNKNOWN: + return DecisionState.UNKNOWN + if status == VerificationStatus.SKIPPED: + return DecisionState.SKIPPED + return DecisionState.ALLOW + + +def _apply_strict_policy( + original: DecisionState, *, - enforce: bool, default_on_unknown: str, -) -> MergeRecommendation: - if not enforce: - return MergeRecommendation.ALLOW_WITH_WARNING - if default_on_unknown == "block": - return MergeRecommendation.BLOCK - if default_on_unknown == "allow_with_warning": - return MergeRecommendation.ALLOW_WITH_WARNING - return MergeRecommendation.REQUIRE_HUMAN_REVIEW - - -def _decision_reason(recommendation: MergeRecommendation, *, from_unknown: bool = False) -> str: - if recommendation == MergeRecommendation.BLOCK: - if from_unknown: - return "one or more verification intents returned an unknown-like result" - return "one or more verification intents failed" - if recommendation in { - MergeRecommendation.REQUIRE_HUMAN_REVIEW, - MergeRecommendation.REQUIRE_STRONGER_CHECK, - }: - return "one or more verification intents returned an unknown-like result" - if recommendation == MergeRecommendation.ALLOW_WITH_WARNING: - return "verification completed with warnings in advisory mode" - return "all evaluated verification intents passed" + default_on_required_skip: str, +) -> DecisionState: + """Apply strict-mode policy overlays without ever promoting to allow.""" + if original == DecisionState.ALLOW: + return DecisionState.ALLOW + if original == DecisionState.BLOCK: + return DecisionState.BLOCK + if original == DecisionState.ERROR: + return DecisionState.ERROR + if original == DecisionState.UNKNOWN: + policy = normalize_unknown_policy(default_on_unknown) + if policy == "block": + return DecisionState.BLOCK + return DecisionState.NEEDS_REVIEW + if original == DecisionState.SKIPPED: + skip_policy = normalize_required_skip_policy(default_on_required_skip) + if skip_policy == "block": + return DecisionState.BLOCK + return DecisionState.SKIPPED + return original -def decide( - bundle: EvidenceBundle, - enforce: bool = True, +def _contributions_for( + findings: Sequence[ClaimFinding], + *, + controlling_ids: set[str], + warning_ids: set[str], +) -> tuple[FindingContribution, ...]: + rows: list[FindingContribution] = [] + for finding in findings: + if finding.finding_id in controlling_ids: + contribution: Literal["controlling", "supporting", "non_controlling", "warning"] = "controlling" + elif finding.finding_id in warning_ids: + contribution = "warning" + elif finding.status == VerificationStatus.PASS: + contribution = "supporting" + else: + contribution = "non_controlling" + rows.append( + FindingContribution( + finding_id=finding.finding_id, + claim_status=finding.status, + required=finding.required, + contribution=contribution, + ) + ) + return tuple(rows) + + +def aggregate_decision( + findings: Sequence[ClaimFinding], + *, + mode: Mode = "strict", default_on_unknown: str = "require_human_review", -) -> MergeRecommendation: - """Compute a conservative merge recommendation. + default_on_required_skip: str = "skipped", + legacy_merge_override: MergeRecommendation | None = None, +) -> DecisionOutcome: + """Aggregate claim findings into a ``DecisionState`` with attribution. - Critical failures block. Unknown-like outcomes follow ``default_on_unknown`` when - ``enforce`` is true (from ``.verification/config.yml`` in the kernel path). + Exhaustive fail-closed rules for required claims; optional claims may warn + or upgrade fail→block but cannot upgrade a required non-pass to allow. """ - if evidence_has_status(bundle, VerificationStatus.FAIL): - return MergeRecommendation.BLOCK if enforce else MergeRecommendation.ALLOW_WITH_WARNING + enforce = mode == "strict" + required = [item for item in findings if item.required] + optional = [item for item in findings if not item.required] + warnings: list[str] = [] + + if not findings: + state = DecisionState.NEEDS_REVIEW + return DecisionOutcome( + decision_state=state, + original_decision_state=state, + merge_recommendation=decision_state_to_merge_recommendation(state), + reason="no findings were provided for aggregation", + mode=mode, + legacy_merge_override=legacy_merge_override, + ) + + # Optional fail upgrades aggregate to block (fail-dominant). + optional_fails = [item for item in optional if item.status == VerificationStatus.FAIL] + required_fails = [item for item in required if item.status == VerificationStatus.FAIL] + if optional_fails or required_fails: + controllers = optional_fails + required_fails + controlling_ids = {item.finding_id for item in controllers} + original = DecisionState.BLOCK + # Advisory preserves original (block); never rewrite to allow. + decision_state = original + return DecisionOutcome( + decision_state=decision_state, + original_decision_state=original, + merge_recommendation=decision_state_to_merge_recommendation(decision_state), + reason="one or more verification claims failed", + controlling_finding_ids=tuple(sorted(controlling_ids)), + finding_contributions=_contributions_for( + findings, controlling_ids=controlling_ids, warning_ids=set() + ), + mode=mode, + legacy_merge_override=legacy_merge_override, + ) - if evidence_has_unknown_like(bundle): - skipped_only = all( - claim.status in {VerificationStatus.SKIPPED, VerificationStatus.PASS} - for evidence in bundle.evidence - for claim in evidence.backend_claims - ) and evidence_has_status(bundle, VerificationStatus.SKIPPED) - if skipped_only and not enforce: - return MergeRecommendation.ALLOW_WITH_WARNING - return _unknown_like_recommendation(enforce=enforce, default_on_unknown=default_on_unknown) + required_non_pass = [item for item in required if item.status != VerificationStatus.PASS] + if required_non_pass: + worst = _worst_status(item.status for item in required_non_pass) + assert worst is not None + controllers = [item for item in required_non_pass if item.status == worst] + # Include all findings at the controlling severity tier. + controlling_ids = {item.finding_id for item in controllers} + original = _base_state_from_status(worst) - return MergeRecommendation.ALLOW + for item in optional: + if item.status == VerificationStatus.PASS: + warnings.append( + f"optional finding {item.finding_id} passed but cannot upgrade required {worst.value}" + ) + + if enforce: + decision_state = _apply_strict_policy( + original, + default_on_unknown=default_on_unknown, + default_on_required_skip=default_on_required_skip, + ) + else: + # Advisory: preserve original lattice state (never invent allow). + decision_state = original + + if decision_state == DecisionState.ALLOW: + # Hard invariant — unreachable by construction; keep fail-closed. + decision_state = DecisionState.NEEDS_REVIEW + + reason = { + DecisionState.ERROR: "one or more required verification claims returned error", + DecisionState.UNKNOWN: "one or more required verification claims returned unknown", + DecisionState.SKIPPED: "one or more required verification claims were skipped", + DecisionState.BLOCK: "required verification outcome blocks merge under policy", + DecisionState.NEEDS_REVIEW: "one or more required verification claims need human review", + }.get(decision_state, "required verification claims did not all pass") + + return DecisionOutcome( + decision_state=decision_state, + original_decision_state=original, + merge_recommendation=decision_state_to_merge_recommendation(decision_state), + reason=reason, + controlling_finding_ids=tuple(sorted(controlling_ids)), + finding_contributions=_contributions_for( + findings, controlling_ids=controlling_ids, warning_ids=set() + ), + mode=mode, + warnings=tuple(warnings), + legacy_merge_override=legacy_merge_override, + ) + + # All required pass (or no required findings). + if not required: + # No required selection — conservative review (never silent allow). + state = DecisionState.NEEDS_REVIEW + return DecisionOutcome( + decision_state=state, + original_decision_state=state, + merge_recommendation=decision_state_to_merge_recommendation(state), + reason="no required findings were selected", + mode=mode, + legacy_merge_override=legacy_merge_override, + ) + + warning_ids: set[str] = set() + for item in optional: + if item.status != VerificationStatus.PASS: + warning_ids.add(item.finding_id) + warnings.append(f"optional finding {item.finding_id} returned {item.status.value}") + + state = DecisionState.ALLOW + controlling_ids = {item.finding_id for item in required} + return DecisionOutcome( + decision_state=state, + original_decision_state=state, + merge_recommendation=decision_state_to_merge_recommendation(state), + reason="all required verification claims passed", + controlling_finding_ids=tuple(sorted(controlling_ids)), + finding_contributions=_contributions_for( + findings, controlling_ids=controlling_ids, warning_ids=warning_ids + ), + mode=mode, + warnings=tuple(warnings), + legacy_merge_override=legacy_merge_override, + ) + + +def decide( + bundle: EvidenceBundle, + enforce: bool = True, + default_on_unknown: str = "require_human_review", + default_on_required_skip: str = "skipped", +) -> DecisionState: + """Compute the normative ``DecisionState`` for an evidence bundle.""" + outcome = aggregate_decision( + findings_from_bundle(bundle), + mode="strict" if enforce else "advisory", + default_on_unknown=default_on_unknown, + default_on_required_skip=default_on_required_skip, + ) + return outcome.decision_state def decide_with_reason( bundle: EvidenceBundle, enforce: bool = True, default_on_unknown: str = "require_human_review", -) -> dict[str, str]: - """Return merge recommendation and human-readable reason for bundle construction.""" - recommendation = decide(bundle, enforce=enforce, default_on_unknown=default_on_unknown) - from_unknown = ( - recommendation - in { - MergeRecommendation.BLOCK, - MergeRecommendation.REQUIRE_HUMAN_REVIEW, - MergeRecommendation.ALLOW_WITH_WARNING, - } - and evidence_has_unknown_like(bundle) - and not evidence_has_status(bundle, VerificationStatus.FAIL) + default_on_required_skip: str = "skipped", +) -> dict[str, Any]: + """Return decision lattice fields and deprecated merge_recommendation alias.""" + outcome = aggregate_decision( + findings_from_bundle(bundle), + mode="strict" if enforce else "advisory", + default_on_unknown=default_on_unknown, + default_on_required_skip=default_on_required_skip, ) - return { - "merge_recommendation": recommendation.value, - "reason": _decision_reason(recommendation, from_unknown=from_unknown), - } + return outcome.to_decision_dict() + + +# Back-compat helpers used by older tests / call sites. +def decide_merge_recommendation( + bundle: EvidenceBundle, + enforce: bool = True, + default_on_unknown: str = "require_human_review", + default_on_required_skip: str = "skipped", +) -> MergeRecommendation: + """Deprecated: return the legacy merge_recommendation alias for a bundle.""" + outcome = aggregate_decision( + findings_from_bundle(bundle), + mode="strict" if enforce else "advisory", + default_on_unknown=default_on_unknown, + default_on_required_skip=default_on_required_skip, + ) + return outcome.legacy_merge_override or outcome.merge_recommendation diff --git a/ovk/core/exit_codes.py b/ovk/core/exit_codes.py index e86469b..00aeac7 100644 --- a/ovk/core/exit_codes.py +++ b/ovk/core/exit_codes.py @@ -2,16 +2,52 @@ from __future__ import annotations +from ovk.core.decision import merge_recommendation_to_decision_state +from ovk.core.models import DecisionState +# Normative lattice → process exit code. +DECISION_STATE_EXIT_CODES = { + DecisionState.ALLOW.value: 0, + DecisionState.BLOCK.value: 1, + DecisionState.NEEDS_REVIEW.value: 2, + DecisionState.UNKNOWN.value: 2, + DecisionState.ERROR.value: 2, + DecisionState.SKIPPED.value: 2, +} + +# Deprecated merge_recommendation aliases (including non-lattice legacy values). RECOMMENDATION_EXIT_CODES = { "allow": 0, "allow_with_warning": 0, "block": 1, "require_human_review": 2, "require_stronger_check": 2, + "needs_review": 2, + "unknown": 2, + "error": 2, + "skipped": 2, } +def exit_code_for_decision_state(decision_state: str | DecisionState) -> int: + """Return the process exit code for a normative ``DecisionState``.""" + if isinstance(decision_state, DecisionState): + key = decision_state.value + else: + key = str(decision_state).strip() + if key in DECISION_STATE_EXIT_CODES: + return DECISION_STATE_EXIT_CODES[key] + # Accept legacy aliases by mapping onto the lattice first. + mapped = merge_recommendation_to_decision_state(key) + return DECISION_STATE_EXIT_CODES.get(mapped.value, 2) + + def exit_code_for_recommendation(recommendation: str) -> int: - """Return the process exit code for a merge recommendation.""" + """Return the process exit code for a merge recommendation or decision_state. + + Prefers ``decision_state`` semantics when the value is a lattice member; + falls back to legacy ``merge_recommendation`` aliases. + """ + if recommendation in DECISION_STATE_EXIT_CODES: + return DECISION_STATE_EXIT_CODES[recommendation] return RECOMMENDATION_EXIT_CODES.get(recommendation, 2) diff --git a/ovk/core/models.py b/ovk/core/models.py index d0940cb..a11acf3 100644 --- a/ovk/core/models.py +++ b/ovk/core/models.py @@ -13,6 +13,8 @@ class VerificationStatus(str, Enum): + """Checker claim status (not the merge decision lattice).""" + PASS = "pass" FAIL = "fail" UNKNOWN = "unknown" @@ -20,7 +22,31 @@ class VerificationStatus(str, Enum): SKIPPED = "skipped" +class DecisionState(str, Enum): + """Normative merge decision lattice (OVK-03). + + Checker claim statuses remain ``VerificationStatus``. Legacy + ``MergeRecommendation`` values map onto this lattice via aliases; + ``allow_with_warning`` is not a lattice member. + """ + + ALLOW = "allow" + BLOCK = "block" + NEEDS_REVIEW = "needs_review" + UNKNOWN = "unknown" + ERROR = "error" + SKIPPED = "skipped" + + class MergeRecommendation(str, Enum): + """Deprecated alias vocabulary for ``DecisionState``. + + Prefer ``DecisionState``. Mapping: + ``require_human_review`` ↔ ``needs_review``; + ``allow_with_warning`` / ``require_stronger_check`` are legacy emission + aliases only (not lattice members). + """ + ALLOW = "allow" BLOCK = "block" REQUIRE_HUMAN_REVIEW = "require_human_review" @@ -28,6 +54,16 @@ class MergeRecommendation(str, Enum): REQUIRE_STRONGER_CHECK = "require_stronger_check" +class FindingContribution(BaseModel): + """Per-finding contribution to an aggregated decision.""" + + finding_id: str + claim_status: VerificationStatus + required: bool = True + contribution: Literal["controlling", "supporting", "non_controlling", "warning"] + detail: str | None = None + + class RiskSeverity(str, Enum): LOW = "low" MEDIUM = "medium" @@ -79,6 +115,8 @@ class BackendClaim(BaseModel): limits: list[str] = Field(default_factory=list) tool_version: str | None = None adapter_version: str | None = None + # When present on evidence claims, drives required vs optional lattice rules. + required: bool = True class VerificationEvidence(BaseModel): @@ -106,6 +144,22 @@ class VerificationEvidence(BaseModel): execution_attempts: list[dict[str, Any]] | None = None aggregation_policy: str | None = None routing_enforced: bool = False + # Integrity envelope helper fields (OVK-04 / ovk.evidence.v3). + ovk_version: str | None = None + checker_id: str | None = None + checker_version: str | None = None + input_digest: str | None = None + relevant_file_digests: list[dict[str, Any]] | None = None + configuration_digest: str | None = None + policy_digest: str | None = None + started_at: str | None = None + completed_at: str | None = None + assumptions: list[str] | None = None + unknowns: list[str] | None = None + stderr: str | None = None + exit_status: int | None = None + evidence_digest: str | None = None + signature: dict[str, Any] | None = None class EvidenceBundle(BaseModel): @@ -118,6 +172,16 @@ class EvidenceBundle(BaseModel): Decision = Literal[ + "allow", + "block", + "needs_review", + "unknown", + "error", + "skipped", +] + +# Deprecated literal union retained for older call sites. +LegacyMergeDecision = Literal[ "allow", "block", "require_human_review", diff --git a/tests/test_decision.py b/tests/test_decision.py index f5e1650..d989718 100644 --- a/tests/test_decision.py +++ b/tests/test_decision.py @@ -1,5 +1,5 @@ -from ovk.core.decision import decide -from ovk.core.models import EvidenceBundle, MergeRecommendation +from ovk.core.decision import decide, decide_merge_recommendation, decide_with_reason +from ovk.core.models import DecisionState, EvidenceBundle, MergeRecommendation def make_bundle(status: str) -> EvidenceBundle: @@ -30,23 +30,33 @@ def make_bundle(status: str) -> EvidenceBundle: def test_fail_blocks_in_enforce_mode() -> None: - assert decide(make_bundle("fail"), enforce=True) == MergeRecommendation.BLOCK + assert decide(make_bundle("fail"), enforce=True) == DecisionState.BLOCK def test_unknown_requires_human_review_in_enforce_mode() -> None: - assert decide(make_bundle("unknown"), enforce=True) == MergeRecommendation.REQUIRE_HUMAN_REVIEW + assert decide(make_bundle("unknown"), enforce=True) == DecisionState.NEEDS_REVIEW def test_unknown_blocks_when_default_on_unknown_is_block() -> None: - assert decide(make_bundle("unknown"), enforce=True, default_on_unknown="block") == MergeRecommendation.BLOCK + assert decide(make_bundle("unknown"), enforce=True, default_on_unknown="block") == DecisionState.BLOCK -def test_unknown_allows_with_warning_when_configured() -> None: - assert ( - decide(make_bundle("unknown"), enforce=True, default_on_unknown="allow_with_warning") - == MergeRecommendation.ALLOW_WITH_WARNING - ) +def test_unknown_legacy_allow_with_warning_never_allows_in_strict() -> None: + state = decide(make_bundle("unknown"), enforce=True, default_on_unknown="allow_with_warning") + assert state == DecisionState.NEEDS_REVIEW + assert state != DecisionState.ALLOW + assert decide_merge_recommendation( + make_bundle("unknown"), enforce=True, default_on_unknown="allow_with_warning" + ) == MergeRecommendation.REQUIRE_HUMAN_REVIEW def test_pass_allows() -> None: - assert decide(make_bundle("pass"), enforce=True) == MergeRecommendation.ALLOW + assert decide(make_bundle("pass"), enforce=True) == DecisionState.ALLOW + + +def test_decide_with_reason_emits_decision_state() -> None: + payload = decide_with_reason(make_bundle("error"), enforce=True) + assert payload["decision_state"] == "error" + assert payload["original_decision_state"] == "error" + assert payload["merge_recommendation"] == "require_human_review" + assert payload["controlling_finding_ids"] diff --git a/tests/test_decision_lattice_truth_table.py b/tests/test_decision_lattice_truth_table.py new file mode 100644 index 0000000..e53b71c --- /dev/null +++ b/tests/test_decision_lattice_truth_table.py @@ -0,0 +1,282 @@ +"""Exhaustive DecisionState lattice truth table (OVK-PR2 / OVK-03). + +Covers claim statuses × {strict, advisory} × required/optional skip, plus +adversarial promotions that must never reach ALLOW in strict mode. +""" + +from __future__ import annotations + +import itertools + +import pytest + +from ovk.core.decision import ( + ClaimFinding, + aggregate_decision, + decide, + decide_with_reason, + merge_recommendation_to_decision_state, +) +from ovk.core.models import DecisionState, EvidenceBundle, MergeRecommendation, VerificationStatus + +CLAIM_STATUSES = ( + VerificationStatus.PASS, + VerificationStatus.FAIL, + VerificationStatus.UNKNOWN, + VerificationStatus.ERROR, + VerificationStatus.SKIPPED, +) + +MODES = ("strict", "advisory") + + +def _finding( + status: VerificationStatus, + *, + finding_id: str = "f1", + required: bool = True, +) -> ClaimFinding: + return ClaimFinding(finding_id=finding_id, status=status, required=required) + + +def _bundle_with_claims(*statuses: VerificationStatus, required: bool = True) -> EvidenceBundle: + claims = [ + { + "backend": f"backend-{index}", + "guarantee_type": "test", + "status": status.value, + "required": required, + } + for index, status in enumerate(statuses) + ] + return EvidenceBundle.model_validate( + { + "bundle_id": "bundle-lattice", + "schema_version": "ovk.bundle.v1", + "subject": {"repo": "example/repo", "head_sha": "abc"}, + "evidence": [ + { + "evidence_id": "ev-lattice", + "schema_version": "ovk.evidence.v1", + "subject": {"repo": "example/repo", "head_sha": "abc"}, + "intent": {"intent_id": "test", "title": "test"}, + "backend_claims": claims, + "decision": {"merge_recommendation": "require_human_review"}, + } + ], + "decision": {"merge_recommendation": "require_human_review"}, + } + ) + + +@pytest.mark.parametrize("status,mode", list(itertools.product(CLAIM_STATUSES, MODES))) +def test_single_required_claim_truth_table(status: VerificationStatus, mode: str) -> None: + outcome = aggregate_decision([_finding(status)], mode=mode) # type: ignore[arg-type] + if status == VerificationStatus.PASS: + assert outcome.decision_state == DecisionState.ALLOW + assert outcome.original_decision_state == DecisionState.ALLOW + elif status == VerificationStatus.FAIL: + assert outcome.decision_state == DecisionState.BLOCK + assert outcome.original_decision_state == DecisionState.BLOCK + elif status == VerificationStatus.ERROR: + assert outcome.decision_state == DecisionState.ERROR + assert outcome.original_decision_state == DecisionState.ERROR + assert outcome.decision_state != DecisionState.ALLOW + elif status == VerificationStatus.UNKNOWN: + assert outcome.original_decision_state == DecisionState.UNKNOWN + if mode == "strict": + assert outcome.decision_state == DecisionState.NEEDS_REVIEW + assert outcome.decision_state != DecisionState.ALLOW + else: + assert outcome.decision_state == DecisionState.UNKNOWN + assert outcome.decision_state != DecisionState.ALLOW + elif status == VerificationStatus.SKIPPED: + assert outcome.original_decision_state == DecisionState.SKIPPED + assert outcome.decision_state == DecisionState.SKIPPED + assert outcome.decision_state != DecisionState.ALLOW + + +@pytest.mark.parametrize("status", CLAIM_STATUSES) +def test_optional_non_fail_does_not_block_required_pass(status: VerificationStatus) -> None: + findings = [ + _finding(VerificationStatus.PASS, finding_id="req", required=True), + _finding(status, finding_id="opt", required=False), + ] + for mode in MODES: + outcome = aggregate_decision(findings, mode=mode) # type: ignore[arg-type] + if status == VerificationStatus.FAIL: + assert outcome.decision_state == DecisionState.BLOCK + else: + assert outcome.decision_state == DecisionState.ALLOW + if status != VerificationStatus.PASS: + assert "opt" in { + row.finding_id for row in outcome.finding_contributions if row.contribution == "warning" + } + + +@pytest.mark.parametrize("mode", MODES) +@pytest.mark.parametrize("skip_policy", ("skipped", "block")) +def test_required_skipped_never_allows(mode: str, skip_policy: str) -> None: + outcome = aggregate_decision( + [_finding(VerificationStatus.SKIPPED)], + mode=mode, # type: ignore[arg-type] + default_on_required_skip=skip_policy, + ) + assert outcome.decision_state != DecisionState.ALLOW + if mode == "strict" and skip_policy == "block": + assert outcome.decision_state == DecisionState.BLOCK + else: + assert outcome.decision_state == DecisionState.SKIPPED + assert outcome.original_decision_state == DecisionState.SKIPPED + + +@pytest.mark.parametrize("mode", MODES) +def test_adversarial_error_never_allows(mode: str) -> None: + """ERROR→ALLOW must be impossible in both modes.""" + outcome = aggregate_decision( + [ + _finding(VerificationStatus.ERROR, finding_id="err"), + _finding(VerificationStatus.PASS, finding_id="pass-optional", required=False), + ], + mode=mode, # type: ignore[arg-type] + default_on_unknown="allow_with_warning", + ) + assert outcome.decision_state == DecisionState.ERROR + assert outcome.decision_state != DecisionState.ALLOW + assert "err" in outcome.controlling_finding_ids + + +@pytest.mark.parametrize("unknown_policy", ("require_human_review", "block", "allow_with_warning", "needs_review")) +def test_adversarial_unknown_never_allows_in_strict(unknown_policy: str) -> None: + """UNKNOWN→ALLOW must be impossible in strict mode, including legacy allow_with_warning.""" + outcome = aggregate_decision( + [_finding(VerificationStatus.UNKNOWN, finding_id="unk")], + mode="strict", + default_on_unknown=unknown_policy, + ) + assert outcome.decision_state != DecisionState.ALLOW + assert outcome.original_decision_state == DecisionState.UNKNOWN + if unknown_policy == "block": + assert outcome.decision_state == DecisionState.BLOCK + else: + assert outcome.decision_state == DecisionState.NEEDS_REVIEW + + +def test_advisory_unknown_preserves_original_state() -> None: + outcome = aggregate_decision([_finding(VerificationStatus.UNKNOWN)], mode="advisory") + assert outcome.decision_state == DecisionState.UNKNOWN + assert outcome.original_decision_state == DecisionState.UNKNOWN + payload = outcome.to_decision_dict() + assert payload["decision_state"] == "unknown" + assert payload["original_decision_state"] == "unknown" + # Deprecated alias maps onto require_human_review, not invent allow_with_warning. + assert payload["merge_recommendation"] == MergeRecommendation.REQUIRE_HUMAN_REVIEW.value + + +def test_multi_finding_control_attribution() -> None: + outcome = aggregate_decision( + [ + _finding(VerificationStatus.PASS, finding_id="a"), + _finding(VerificationStatus.FAIL, finding_id="b"), + _finding(VerificationStatus.ERROR, finding_id="c"), + ], + mode="strict", + ) + assert outcome.decision_state == DecisionState.BLOCK + assert outcome.controlling_finding_ids == ("b",) + by_id = {row.finding_id: row.contribution for row in outcome.finding_contributions} + assert by_id["b"] == "controlling" + assert by_id["a"] == "supporting" + assert by_id["c"] == "non_controlling" + + +def test_error_dominates_unknown_for_control() -> None: + outcome = aggregate_decision( + [ + _finding(VerificationStatus.UNKNOWN, finding_id="u"), + _finding(VerificationStatus.ERROR, finding_id="e"), + ], + mode="strict", + ) + assert outcome.decision_state == DecisionState.ERROR + assert outcome.controlling_finding_ids == ("e",) + + +def test_optional_pass_cannot_upgrade_required_unknown() -> None: + outcome = aggregate_decision( + [ + _finding(VerificationStatus.UNKNOWN, finding_id="req"), + _finding(VerificationStatus.PASS, finding_id="opt", required=False), + ], + mode="strict", + ) + assert outcome.decision_state != DecisionState.ALLOW + assert "req" in outcome.controlling_finding_ids + assert any("cannot upgrade" in warning for warning in outcome.warnings) + + +def test_decide_bundle_emits_lattice_fields() -> None: + bundle = _bundle_with_claims(VerificationStatus.ERROR) + payload = decide_with_reason(bundle, enforce=True) + assert payload["decision_state"] == DecisionState.ERROR.value + assert payload["original_decision_state"] == DecisionState.ERROR.value + assert payload["merge_recommendation"] == MergeRecommendation.REQUIRE_HUMAN_REVIEW.value + assert payload["controlling_finding_ids"] + assert decide(bundle, enforce=True) == DecisionState.ERROR + + +def test_legacy_alias_round_trip() -> None: + assert merge_recommendation_to_decision_state("require_human_review") == DecisionState.NEEDS_REVIEW + assert merge_recommendation_to_decision_state("allow_with_warning") == DecisionState.NEEDS_REVIEW + assert merge_recommendation_to_decision_state("require_stronger_check") == DecisionState.NEEDS_REVIEW + assert merge_recommendation_to_decision_state(DecisionState.ERROR) == DecisionState.ERROR + + +@pytest.mark.parametrize( + "left,right", + list(itertools.product(CLAIM_STATUSES, CLAIM_STATUSES)), +) +def test_pairwise_required_claims_never_allow_on_bad( + left: VerificationStatus, right: VerificationStatus +) -> None: + """Every pair of required claims: any non-pass forbids ALLOW in strict mode.""" + outcome = aggregate_decision( + [ + _finding(left, finding_id="left"), + _finding(right, finding_id="right"), + ], + mode="strict", + ) + if left == VerificationStatus.PASS and right == VerificationStatus.PASS: + assert outcome.decision_state == DecisionState.ALLOW + else: + assert outcome.decision_state != DecisionState.ALLOW + + +@pytest.mark.parametrize("status", CLAIM_STATUSES) +@pytest.mark.parametrize("required_skip", (True, False)) +def test_skip_required_vs_optional_matrix(status: VerificationStatus, required_skip: bool) -> None: + """Required/optional skip combinations against each claim status under both modes.""" + if status != VerificationStatus.SKIPPED and not required_skip: + # Focus skip matrix on skip claims; other statuses covered elsewhere. + pytest.skip("non-skip optional covered by optional matrix") + findings = [ + _finding(VerificationStatus.PASS, finding_id="req-pass", required=True), + _finding(status, finding_id="focus", required=required_skip), + ] + for mode in MODES: + outcome = aggregate_decision(findings, mode=mode) # type: ignore[arg-type] + if status == VerificationStatus.PASS: + assert outcome.decision_state == DecisionState.ALLOW + elif status == VerificationStatus.FAIL: + assert outcome.decision_state == DecisionState.BLOCK + elif required_skip: + assert outcome.decision_state != DecisionState.ALLOW + else: + # Optional skip/unknown/error beside required pass → allow with warning. + if status in { + VerificationStatus.SKIPPED, + VerificationStatus.UNKNOWN, + VerificationStatus.ERROR, + }: + assert outcome.decision_state == DecisionState.ALLOW From ebc4b9eea6ae12badee7930ccee55ad085fe2318 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:09:00 -0700 Subject: [PATCH 04/19] Add evidence integrity envelope and digest invariants (OVK-PR3). Define a typed integrity envelope for evidence artifacts so digests, assumptions, and provenance cannot drift between execution, bundle assembly, and attestation checks. --- ovk/core/bundle.py | 14 + ovk/core/evidence_from_execution.py | 43 +- ovk/core/evidence_integrity.py | 557 +++++++++++++++++++ ovk/core/evidence_invariants.py | 160 +++++- schemas/verification.bundle.schema.json | 17 +- schemas/verification.bundle.v2.schema.json | 21 +- schemas/verification.evidence.schema.json | 17 + schemas/verification.evidence.v2.schema.json | 12 + schemas/verification.evidence.v3.schema.json | 75 ++- tests/test_attestation_provenance.py | 2 +- tests/test_evidence_integrity_suite.py | 303 ++++++++++ 11 files changed, 1205 insertions(+), 16 deletions(-) create mode 100644 ovk/core/evidence_integrity.py create mode 100644 tests/test_evidence_integrity_suite.py diff --git a/ovk/core/bundle.py b/ovk/core/bundle.py index 489a83f..d9f68f8 100644 --- a/ovk/core/bundle.py +++ b/ovk/core/bundle.py @@ -68,3 +68,17 @@ def make_bundle( evidence=evidence, decision=decision, ) + + +def compute_evidence_digest(evidence: VerificationEvidence | dict) -> str: + """Compute the integrity digest for one evidence record (see evidence_integrity).""" + from ovk.core.evidence_integrity import compute_evidence_digest as _compute + + return _compute(evidence) + + +def verify_evidence_digest(evidence: VerificationEvidence | dict) -> bool: + """Verify the integrity digest for one evidence record (see evidence_integrity).""" + from ovk.core.evidence_integrity import verify_evidence_digest as _verify + + return _verify(evidence) diff --git a/ovk/core/evidence_from_execution.py b/ovk/core/evidence_from_execution.py index 3dc5fe0..1b6c8f5 100644 --- a/ovk/core/evidence_from_execution.py +++ b/ovk/core/evidence_from_execution.py @@ -14,7 +14,7 @@ from ovk.core.materials import material_set_digest_for_obligation -from ovk.core.models import BackendClaim, MergeRecommendation, VerificationEvidence, VerificationStatus +from ovk.core.models import BackendClaim, DecisionState, MergeRecommendation, VerificationEvidence, VerificationStatus def execution_record_to_evidence( @@ -118,7 +118,15 @@ def execution_record_to_evidence( } ) + from ovk.core.decision import merge_recommendation_to_decision_state + from ovk.core.models import DecisionState + recommendation = record.merge_recommendation + decision_state = getattr(record, "decision_state", None) + if decision_state is None: + decision_state = merge_recommendation_to_decision_state(recommendation) + original_decision_state = getattr(record, "original_decision_state", None) or decision_state + controlling_finding_ids = list(getattr(record, "controlling_finding_ids", ()) or ()) aggregation_reason = record.aggregation_reason @@ -129,8 +137,12 @@ def execution_record_to_evidence( if allow_ok is None: allow_ok = strict_allow_permitted(obligation.coverage, policy) - if recommendation == MergeRecommendation.ALLOW and not allow_ok: + if ( + recommendation == MergeRecommendation.ALLOW or decision_state == DecisionState.ALLOW + ) and not allow_ok: recommendation = MergeRecommendation.REQUIRE_HUMAN_REVIEW + decision_state = DecisionState.NEEDS_REVIEW + original_decision_state = DecisionState.NEEDS_REVIEW aggregation_reason = f"{aggregation_reason}; incomplete abstraction cannot allow under strict coverage" @@ -142,9 +154,25 @@ def execution_record_to_evidence( } ) + if recommendation == MergeRecommendation.ALLOW: + decision_state = DecisionState.ALLOW + elif decision_state == DecisionState.ALLOW and recommendation != MergeRecommendation.ALLOW: + # Prefer explicit non-allow legacy recommendation when present. + decision_state = merge_recommendation_to_decision_state(recommendation) + decision = { + "decision_state": decision_state.value + if hasattr(decision_state, "value") + else str(decision_state), + "original_decision_state": ( + original_decision_state.value + if hasattr(original_decision_state, "value") + else str(original_decision_state) + ), "merge_recommendation": recommendation.value, - "human_review_required": recommendation.value != "allow", + "human_review_required": decision_state != DecisionState.ALLOW + and recommendation.value != "allow", + "controlling_finding_ids": controlling_finding_ids, "aggregation_reason": aggregation_reason, "routing_enforced": routing_enforced, "fallback_used": record.fallback_used, @@ -161,7 +189,7 @@ def execution_record_to_evidence( } )[:24] - return VerificationEvidence( + evidence = VerificationEvidence( evidence_id=f"ev-{evidence_id}", schema_version=schema_version, subject={key: value for key, value in obligation.subject.model_dump(mode="json").items() if value is not None}, @@ -193,3 +221,10 @@ def execution_record_to_evidence( aggregation_policy=routing.aggregation_policy, routing_enforced=routing_enforced, ) + if schema_version.endswith(".v3"): + from ovk.core.evidence_integrity import seal_evidence + + # Seal the control-plane projection; callers that further mutate evidence + # (e.g. adapter_runtime metadata attachment) must reseal afterward. + return seal_evidence(evidence, policy_digest=obligation.policy_digest) + return evidence diff --git a/ovk/core/evidence_integrity.py b/ovk/core/evidence_integrity.py new file mode 100644 index 0000000..031125c --- /dev/null +++ b/ovk/core/evidence_integrity.py @@ -0,0 +1,557 @@ +"""Evidence integrity envelope: digests, sealing, verification, path redaction. + +Integrity helper fields live on ``ovk.evidence.v3`` (optional in the JSON schema; +required once an evidence record is sealed). The ``evidence_digest`` is a +canonical JSON SHA-256 over all fields except itself and optional ``signature``. +""" + +from __future__ import annotations + +import hmac +import re +from datetime import datetime, timezone +from typing import Any, Mapping, Sequence + +from ovk import __version__ as OVK_VERSION +from ovk.core.attestation_signing import SIGNATURE_ALG, sign_payload, signing_key_from_environment +from ovk.core.bundle import content_digest +from ovk.core.materials import compute_material_set_digest +from ovk.core.models import VerificationEvidence + +SUPPORTED_EVIDENCE_SCHEMA_VERSIONS: frozenset[str] = frozenset( + { + "ovk.evidence.v1", + "ovk.evidence.v2", + "ovk.evidence.v3", + } +) + +# Fields excluded from the canonical digest payload. +DIGEST_EXCLUDED_FIELDS: frozenset[str] = frozenset({"evidence_digest", "signature"}) + +# Presence of any of these (except evidence_digest itself when checking partial writes) +# indicates an integrity envelope was started. +INTEGRITY_FIELD_NAMES: tuple[str, ...] = ( + "ovk_version", + "checker_id", + "checker_version", + "input_digest", + "relevant_file_digests", + "configuration_digest", + "policy_digest", + "started_at", + "completed_at", + "assumptions", + "unknowns", + "stderr", + "exit_status", + "evidence_digest", + "signature", +) + +# Required once sealing begins or for sealed v3 evidence. +REQUIRED_INTEGRITY_FIELDS: tuple[str, ...] = ( + "ovk_version", + "checker_id", + "checker_version", + "input_digest", + "relevant_file_digests", + "configuration_digest", + "policy_digest", + "started_at", + "completed_at", + "assumptions", + "unknowns", + "stderr", + "exit_status", + "evidence_digest", +) + +_HOME_PREFIX = re.compile( + r"^(?:" + r"(?:/home|/Users)/[^/]+|" + r"(?i:[a-z]:)\\Users\\[^\\]+|" + r"~" + r")" +) + + +def utc_now_iso() -> str: + """Return a stable ISO-8601 UTC timestamp.""" + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def redact_path(path: str) -> str: + """Redact user/home prefixes from a filesystem path. + + Absolute home directories collapse to ``/...`` so evidence can cite + relative structure without leaking account names. Basename-only collapse is + intentionally avoided except when no directory structure remains. + """ + raw = str(path).strip() + if not raw: + return raw + normalized = raw.replace("\\", "/") + if normalized.startswith("//"): + # UNC-style; keep host redacted. + parts = [part for part in normalized.split("/") if part] + if len(parts) >= 2: + return "/" + "/".join(parts[1:]) + return "" + match = _HOME_PREFIX.match(normalized) + if match: + rest = normalized[match.end() :].lstrip("/") + return f"/{rest}" if rest else "" + # Windows drive without Users + drive = re.match(r"^(?i:[a-z]:)(/.*)?$", normalized) + if drive: + rest = (drive.group(1) or "").lstrip("/") + return f"/{rest}" if rest else "" + return normalized + + +def detect_path_redaction_collisions(paths: Sequence[str]) -> list[dict[str, Any]]: + """Return collision groups where distinct paths share one redacted form.""" + groups: dict[str, list[str]] = {} + for path in paths: + redacted = redact_path(path) + groups.setdefault(redacted, []).append(str(path)) + collisions: list[dict[str, Any]] = [] + for redacted, originals in sorted(groups.items()): + unique = sorted(set(originals)) + if len(unique) > 1: + collisions.append({"redacted": redacted, "paths": unique}) + return collisions + + +def _evidence_as_dict(evidence: VerificationEvidence | Mapping[str, Any]) -> dict[str, Any]: + if isinstance(evidence, VerificationEvidence): + return evidence.model_dump(mode="json") + return dict(evidence) + + +def evidence_digest_payload(evidence: VerificationEvidence | Mapping[str, Any]) -> dict[str, Any]: + """Return the canonical payload hashed into ``evidence_digest``.""" + payload = _evidence_as_dict(evidence) + return {key: value for key, value in payload.items() if key not in DIGEST_EXCLUDED_FIELDS} + + +def compute_evidence_digest(evidence: VerificationEvidence | Mapping[str, Any]) -> str: + """Compute the SHA-256 evidence digest over the canonical payload.""" + return content_digest(evidence_digest_payload(evidence)) + + +def verify_evidence_digest(evidence: VerificationEvidence | Mapping[str, Any]) -> bool: + """Return True when stated ``evidence_digest`` matches the recomputed digest.""" + payload = _evidence_as_dict(evidence) + stated = payload.get("evidence_digest") + if not isinstance(stated, str) or not stated: + return False + return hmac.compare_digest(stated, compute_evidence_digest(payload)) + + +def sign_evidence( + evidence: VerificationEvidence | Mapping[str, Any], + *, + key: bytes | None = None, +) -> dict[str, str] | None: + """Optionally sign the digest payload (excludes evidence_digest and signature).""" + selected = key if key is not None else signing_key_from_environment() + if selected is None: + return None + return sign_payload(evidence_digest_payload(evidence), selected) + + +def verify_evidence_signature( + evidence: VerificationEvidence | Mapping[str, Any], + *, + key: bytes | None = None, +) -> bool: + """Verify optional signature when present; missing signature is allowed.""" + payload = _evidence_as_dict(evidence) + signature = payload.get("signature") + if signature is None: + return True + if not isinstance(signature, dict): + return False + selected = key if key is not None else signing_key_from_environment() + if selected is None: + return False + expected = sign_payload(evidence_digest_payload(payload), selected) + return ( + str(signature.get("algorithm", "")) == SIGNATURE_ALG + and hmac.compare_digest(str(signature.get("digest", "")), expected["digest"]) + ) + + +def recompute_input_digest(evidence: VerificationEvidence | Mapping[str, Any]) -> str: + """Recompute the input digest bound into the integrity envelope.""" + payload = _evidence_as_dict(evidence) + materials = payload.get("materials") + if materials: + return compute_material_set_digest(materials) + return content_digest( + { + "subject": payload.get("subject"), + "intent": payload.get("intent"), + "change_origin": payload.get("change_origin") or {}, + } + ) + + +def integrity_fields_present(evidence: VerificationEvidence | Mapping[str, Any]) -> list[str]: + """Return integrity field names that are explicitly set (not None).""" + payload = _evidence_as_dict(evidence) + present: list[str] = [] + for name in INTEGRITY_FIELD_NAMES: + if name not in payload: + continue + value = payload[name] + if value is None: + continue + present.append(name) + return present + + +def integrity_envelope_complete(evidence: VerificationEvidence | Mapping[str, Any]) -> bool: + """Return True when every required integrity field is present and non-empty where required.""" + payload = _evidence_as_dict(evidence) + for name in REQUIRED_INTEGRITY_FIELDS: + if name not in payload: + return False + value = payload[name] + if name in {"stderr", "exit_status", "assumptions", "unknowns", "relevant_file_digests"}: + # Explicit null / empty containers are allowed; key must exist. + if value is None and name in {"stderr", "exit_status"}: + continue + if value is None: + return False + continue + if value is None or value == "": + return False + return True + + +def missing_integrity_fields(evidence: VerificationEvidence | Mapping[str, Any]) -> list[str]: + """Return required integrity fields that are missing or empty.""" + payload = _evidence_as_dict(evidence) + missing: list[str] = [] + for name in REQUIRED_INTEGRITY_FIELDS: + if name not in payload: + missing.append(name) + continue + value = payload[name] + if name in {"stderr", "exit_status"}: + continue + if name in {"assumptions", "unknowns", "relevant_file_digests"}: + if value is None: + missing.append(name) + continue + if value is None or value == "": + missing.append(name) + return missing + + +def _paths_from_materials(materials: list[Any] | None) -> list[str]: + paths: list[str] = [] + for item in materials or []: + if not isinstance(item, dict): + continue + for key in ("path", "uri", "source_path"): + value = item.get(key) + if isinstance(value, str) and value.strip(): + # Strip ovk-material: scheme for redaction of filesystem URIs. + if value.startswith("ovk-material:"): + continue + paths.append(value) + return paths + + +def build_relevant_file_digests( + materials: list[Any] | None, + *, + redact: bool = True, +) -> list[dict[str, str]]: + """Build ``relevant_file_digests`` entries from materials, with optional path redaction.""" + entries: list[dict[str, str]] = [] + raw_paths: list[str] = [] + for item in materials or []: + if not isinstance(item, dict): + continue + digest = str(item.get("sha256") or item.get("digest") or "") + if not digest: + continue + path = str(item.get("path") or item.get("uri") or item.get("material_id") or "") + raw_paths.append(path) + display = redact_path(path) if redact and path else path + entries.append({"path": display, "sha256": digest}) + entries.sort(key=lambda row: (row["path"], row["sha256"])) + return entries + + +def collect_assumptions(evidence: VerificationEvidence | Mapping[str, Any]) -> list[str]: + """Aggregate claim assumptions into a top-level list.""" + payload = _evidence_as_dict(evidence) + collected: list[str] = [] + seen: set[str] = set() + for claim in payload.get("backend_claims") or []: + if not isinstance(claim, dict): + continue + for item in claim.get("assumptions") or []: + text = str(item) + if text not in seen: + seen.add(text) + collected.append(text) + return collected + + +def collect_unknowns(evidence: VerificationEvidence | Mapping[str, Any]) -> list[str]: + """Collect unknowns from coverage and non-pass claims.""" + payload = _evidence_as_dict(evidence) + unknowns: list[str] = [] + seen: set[str] = set() + coverage = payload.get("coverage") or {} + if isinstance(coverage, dict): + for item in coverage.get("unknowns") or []: + text = str(item) + if text not in seen: + seen.add(text) + unknowns.append(text) + status = str(coverage.get("status") or "") + if status in {"unknown", "partial"}: + note = f"coverage status is {status}" + if note not in seen: + seen.add(note) + unknowns.append(note) + for claim in payload.get("backend_claims") or []: + if not isinstance(claim, dict): + continue + status = str(claim.get("status") or "") + if status in {"unknown", "error", "skipped"}: + note = f"claim {claim.get('backend')}: {status}" + if note not in seen: + seen.add(note) + unknowns.append(note) + return unknowns + + +def _timestamps_from_attempts(attempts: list[Any] | None) -> tuple[str | None, str | None]: + started: list[str] = [] + finished: list[str] = [] + for attempt in attempts or []: + if not isinstance(attempt, dict): + continue + if attempt.get("started_at"): + started.append(str(attempt["started_at"])) + if attempt.get("finished_at"): + finished.append(str(attempt["finished_at"])) + return (min(started) if started else None, max(finished) if finished else None) + + +def _stderr_and_exit(attempts: list[Any] | None) -> tuple[str | None, int | None]: + """Pick representative stderr digest / exit status from attempts.""" + stderr: str | None = None + exit_status: int | None = None + for attempt in attempts or []: + if not isinstance(attempt, dict): + continue + if stderr is None and attempt.get("stderr_digest"): + stderr = str(attempt["stderr_digest"]) + if exit_status is None and attempt.get("exit_code") is not None: + exit_status = int(attempt["exit_code"]) + return stderr, exit_status + + +def resolve_checker_identity( + evidence: VerificationEvidence | Mapping[str, Any], + *, + registry: Any | None = None, +) -> tuple[str, str | None]: + """Resolve checker_id and checker_version from registry and/or claims.""" + payload = _evidence_as_dict(evidence) + claims = payload.get("backend_claims") or [] + primary = claims[0] if claims and isinstance(claims[0], dict) else {} + checker_id = str(primary.get("backend") or payload.get("checker_id") or "unknown") + checker_version: str | None = None + if registry is not None: + manifest = None + by_id = getattr(registry, "by_checker_id", None) + by_tool = getattr(registry, "by_tool", None) + if callable(by_id): + manifest = by_id(checker_id) + if manifest is None and callable(by_tool): + manifest = by_tool(checker_id) + if isinstance(manifest, dict): + checker_id = str(manifest.get("checker_id") or checker_id) + checker_version = ( + str(manifest.get("version") or "") + or str((manifest.get("tool") or {}).get("adapter_version") or "") + or None + ) + if checker_version == "": + checker_version = None + if not checker_version: + adapter = primary.get("adapter_version") + tool = primary.get("tool_version") + if adapter: + checker_version = str(adapter) + elif tool: + checker_version = str(tool) + existing = payload.get("checker_version") + if not checker_version and existing: + checker_version = str(existing) + return checker_id, checker_version + + +def reconstruct_controlling_decision( + evidence: VerificationEvidence | Mapping[str, Any], +) -> dict[str, Any]: + """Reconstruct the controlling decision bound by the evidence digest. + + Returns decision_state, original_decision_state, controlling_finding_ids, + and finding_contributions from the sealed decision object. Callers should + verify ``evidence_digest`` before trusting this reconstruction. + """ + payload = _evidence_as_dict(evidence) + decision = payload.get("decision") or {} + if not isinstance(decision, dict): + decision = {} + return { + "decision_state": decision.get("decision_state"), + "original_decision_state": decision.get("original_decision_state"), + "controlling_finding_ids": list(decision.get("controlling_finding_ids") or []), + "finding_contributions": list(decision.get("finding_contributions") or []), + "merge_recommendation": decision.get("merge_recommendation"), + "evidence_digest": payload.get("evidence_digest"), + "digest_valid": verify_evidence_digest(payload), + } + + +def finding_id_duplicates(evidence: VerificationEvidence | Mapping[str, Any]) -> list[str]: + """Return finding IDs duplicated within controlling lists or contributions.""" + payload = _evidence_as_dict(evidence) + decision = payload.get("decision") or {} + if not isinstance(decision, dict): + return [] + # An id listed once in controlling_finding_ids and once in finding_contributions is normal. + # Duplicates mean the same id appears twice within either collection. + controlling = [str(x) for x in (decision.get("controlling_finding_ids") or [])] + contrib_ids = [ + str(item.get("finding_id")) + for item in (decision.get("finding_contributions") or []) + if isinstance(item, dict) and item.get("finding_id") + ] + dupes: set[str] = set() + for collection in (controlling, contrib_ids): + seen: set[str] = set() + for item in collection: + if item in seen: + dupes.add(item) + seen.add(item) + return sorted(dupes) + + +def seal_evidence( + evidence: VerificationEvidence, + *, + registry: Any | None = None, + key: bytes | None = None, + ovk_version: str | None = None, + configuration_digest: str | None = None, + policy_digest: str | None = None, + started_at: str | None = None, + completed_at: str | None = None, + stderr: str | None = None, + exit_status: int | None = None, + relevant_file_digests: list[dict[str, str]] | None = None, +) -> VerificationEvidence: + """Attach a complete integrity envelope and ``evidence_digest``. + + Raises ``ValueError`` when path redaction would collide or checker version + cannot be resolved. + """ + checker_id, checker_version = resolve_checker_identity(evidence, registry=registry) + if not checker_version: + raise ValueError("cannot seal evidence without checker_version (OVK-INV-023)") + + materials = evidence.materials + file_digests = relevant_file_digests + if file_digests is None: + file_digests = build_relevant_file_digests(materials, redact=True) + # Distinct originals that redact to the same form: + material_paths = _paths_from_materials(materials) + if material_paths: + collisions = detect_path_redaction_collisions(material_paths) + if collisions: + raise ValueError( + "path redaction collisions in relevant_file_digests (OVK-INV-030): " + + "; ".join(f"{c['redacted']}<-{c['paths']}" for c in collisions) + ) + # Two different digests claiming the same redacted path is also a collision. + by_path: dict[str, set[str]] = {} + for item in file_digests: + by_path.setdefault(item["path"], set()).add(item["sha256"]) + ambiguous = {path: digests for path, digests in by_path.items() if len(digests) > 1} + if ambiguous: + raise ValueError( + "path redaction collisions in relevant_file_digests (OVK-INV-030): " + + "; ".join(f"{path}:{sorted(digests)}" for path, digests in sorted(ambiguous.items())) + ) + + attempt_start, attempt_end = _timestamps_from_attempts(evidence.execution_attempts) + attempt_stderr, attempt_exit = _stderr_and_exit(evidence.execution_attempts) + + compiler = evidence.compiler or {} + config_digest = configuration_digest or content_digest( + { + "compiler": compiler, + "aggregation_policy": evidence.aggregation_policy, + "coverage": evidence.coverage, + } + ) + pol_digest = policy_digest + if pol_digest is None: + # Prefer obligation policy digest from attempts / routing artifacts when present. + for artifact in evidence.generated_artifacts: + if artifact.get("kind") == "control_plane_trace" and artifact.get("policy_digest"): + pol_digest = str(artifact["policy_digest"]) + break + if pol_digest is None: + pol_digest = content_digest({"intent": evidence.intent, "decision_policy": True}) + + started = started_at or attempt_start or utc_now_iso() + completed = completed_at or attempt_end or utc_now_iso() + + # Clear digest/signature before computing so payload is clean. + provisional = evidence.model_copy( + update={ + "ovk_version": ovk_version or OVK_VERSION, + "checker_id": checker_id, + "checker_version": checker_version, + "input_digest": recompute_input_digest(evidence), + "relevant_file_digests": file_digests, + "configuration_digest": config_digest, + "policy_digest": pol_digest, + "started_at": started, + "completed_at": completed, + "assumptions": collect_assumptions(evidence), + "unknowns": collect_unknowns(evidence), + "stderr": stderr if stderr is not None else attempt_stderr, + "exit_status": exit_status if exit_status is not None else attempt_exit, + "evidence_digest": None, + "signature": None, + } + ) + digest = compute_evidence_digest(provisional) + signed = sign_evidence({**provisional.model_dump(mode="json"), "evidence_digest": digest}, key=key) + return provisional.model_copy( + update={ + "evidence_digest": digest, + "signature": signed, + } + ) + + +def is_supported_schema_version(schema_version: str) -> bool: + """Return True when the evidence schema version is known to OVK.""" + return str(schema_version) in SUPPORTED_EVIDENCE_SCHEMA_VERSIONS diff --git a/ovk/core/evidence_invariants.py b/ovk/core/evidence_invariants.py index 7eda9bf..725340b 100644 --- a/ovk/core/evidence_invariants.py +++ b/ovk/core/evidence_invariants.py @@ -192,17 +192,20 @@ def check_evidence_bundle_invariants(bundle: EvidenceBundle) -> list[EvidenceInv ) bundle_recommendation = _decision_value(bundle.decision, "merge_recommendation") - if bundle_recommendation is None: + bundle_decision_state = _decision_value(bundle.decision, "decision_state") + if bundle_recommendation is None and bundle_decision_state is None: issues.append( EvidenceInvariantIssue( - path="decision.merge_recommendation", - message="bundle decision must include merge_recommendation", + path="decision.decision_state", + message="bundle decision must include decision_state or merge_recommendation", ) ) - if any(issue.severity == "error" for issue in issues) and bundle_recommendation == "allow": + if any(issue.severity == "error" for issue in issues) and ( + bundle_recommendation == "allow" or bundle_decision_state == "allow" + ): issues.append( EvidenceInvariantIssue( - path="decision.merge_recommendation", + path="decision.decision_state", message="bundle with invariant errors must not recommend allow", ) ) @@ -222,6 +225,7 @@ def check_evidence_bundle_invariants(bundle: EvidenceBundle) -> list[EvidenceInv ) issues.extend(_check_control_plane_invariants(bundle)) + issues.extend(_check_integrity_invariants(bundle)) return issues @@ -442,3 +446,149 @@ def _check_control_plane_invariants(bundle: EvidenceBundle) -> list[EvidenceInva ) return issues + + +def _check_integrity_invariants(bundle: EvidenceBundle) -> list[EvidenceInvariantIssue]: + """Evaluate OVK-INV-023 through OVK-INV-030 for evidence integrity envelopes.""" + from ovk.core.evidence_integrity import ( + detect_path_redaction_collisions, + finding_id_duplicates, + integrity_envelope_complete, + integrity_fields_present, + is_supported_schema_version, + missing_integrity_fields, + recompute_input_digest, + verify_evidence_digest, + verify_evidence_signature, + ) + + issues: list[EvidenceInvariantIssue] = [] + for index, evidence in enumerate(bundle.evidence): + path = f"evidence[{index}]" + schema = str(evidence.schema_version) + + if not is_supported_schema_version(schema): + issues.append( + EvidenceInvariantIssue( + path=f"{path}.schema_version", + message=f"unsupported evidence schema_version {schema!r} (OVK-INV-029)", + ) + ) + continue + + is_v3 = schema.endswith(".v3") + present = integrity_fields_present(evidence) + sealed = evidence.evidence_digest is not None + partial = bool(present) and not integrity_envelope_complete(evidence) + + if is_v3 and not sealed: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.evidence_digest", + message="evidence v3 must include a sealed evidence_digest (OVK-INV-023)", + ) + ) + + if partial: + missing = ", ".join(missing_integrity_fields(evidence)) + issues.append( + EvidenceInvariantIssue( + path=f"{path}", + message=( + "partially written integrity envelope; missing required fields: " + f"{missing} (OVK-INV-028)" + ), + ) + ) + + if sealed or (is_v3 and present): + if not evidence.checker_version: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.checker_version", + message="integrity envelope requires checker_version (OVK-INV-025)", + ) + ) + if evidence.input_digest: + expected_input = recompute_input_digest(evidence) + if evidence.input_digest != expected_input: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.input_digest", + message=( + "input_digest does not match recomputed material/input digest " + "(tampered input) (OVK-INV-024)" + ), + ) + ) + if evidence.evidence_digest and not verify_evidence_digest(evidence): + issues.append( + EvidenceInvariantIssue( + path=f"{path}.evidence_digest", + message=( + "evidence_digest does not match canonical payload " + "(tampered checker output or fields) (OVK-INV-026)" + ), + ) + ) + if evidence.signature is not None and not verify_evidence_signature(evidence): + issues.append( + EvidenceInvariantIssue( + path=f"{path}.signature", + message="evidence signature verification failed (OVK-INV-026)", + ) + ) + + dupes = finding_id_duplicates(evidence) + if dupes: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.decision.controlling_finding_ids", + message=f"duplicated finding IDs: {', '.join(dupes)} (OVK-INV-027)", + ) + ) + + file_digests = evidence.relevant_file_digests or [] + if file_digests: + # Collision: same redacted path with distinct content digests. + by_path: dict[str, set[str]] = {} + for item in file_digests: + if not isinstance(item, dict): + continue + display = str(item.get("path") or "") + digest = str(item.get("sha256") or "") + if display and digest: + by_path.setdefault(display, set()).add(digest) + ambiguous = {p: digests for p, digests in by_path.items() if len(digests) > 1} + if ambiguous: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.relevant_file_digests", + message=( + "path-redaction collisions in relevant_file_digests " + f"{sorted(ambiguous)} (OVK-INV-030)" + ), + ) + ) + # Also reject when distinct raw material paths redact identically. + material_paths: list[str] = [] + for material in evidence.materials or []: + if not isinstance(material, dict): + continue + for key in ("path", "source_path"): + value = material.get(key) + if isinstance(value, str) and value.strip(): + material_paths.append(value) + collisions = detect_path_redaction_collisions(material_paths) + if collisions: + issues.append( + EvidenceInvariantIssue( + path=f"{path}.relevant_file_digests", + message=( + "path-redaction collisions among material paths " + f"{collisions} (OVK-INV-030)" + ), + ) + ) + + return issues diff --git a/schemas/verification.bundle.schema.json b/schemas/verification.bundle.schema.json index 0ed4541..242127b 100644 --- a/schemas/verification.bundle.schema.json +++ b/schemas/verification.bundle.schema.json @@ -30,7 +30,22 @@ "type": "object", "required": ["merge_recommendation"], "properties": { - "merge_recommendation": { "type": "string", "enum": ["allow", "block", "require_human_review", "allow_with_warning", "require_stronger_check"] }, + "decision_state": { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"] + }, + "original_decision_state": { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"] + }, + "merge_recommendation": { + "type": "string", + "enum": ["allow", "block", "require_human_review", "allow_with_warning", "require_stronger_check"] + }, + "controlling_finding_ids": { + "type": "array", + "items": { "type": "string" } + }, "reason": { "type": "string" } }, "additionalProperties": true diff --git a/schemas/verification.bundle.v2.schema.json b/schemas/verification.bundle.v2.schema.json index df97525..13cc51c 100644 --- a/schemas/verification.bundle.v2.schema.json +++ b/schemas/verification.bundle.v2.schema.json @@ -29,11 +29,28 @@ }, "decision": { "type": "object", - "required": ["merge_recommendation"], + "required": ["decision_state"], "properties": { + "decision_state": { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"] + }, + "original_decision_state": { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"] + }, "merge_recommendation": { "type": "string", - "enum": ["allow", "block", "require_human_review", "allow_with_warning", "require_stronger_check"] + "enum": ["allow", "block", "require_human_review", "allow_with_warning", "require_stronger_check"], + "description": "Deprecated alias of decision_state." + }, + "controlling_finding_ids": { + "type": "array", + "items": { "type": "string" } + }, + "finding_contributions": { + "type": "array", + "items": { "type": "object", "additionalProperties": true } }, "reason": { "type": "string" } }, diff --git a/schemas/verification.evidence.schema.json b/schemas/verification.evidence.schema.json index fd26ea9..d1a17ea 100644 --- a/schemas/verification.evidence.schema.json +++ b/schemas/verification.evidence.schema.json @@ -81,7 +81,24 @@ "type": "object", "required": ["merge_recommendation"], "properties": { + "decision_state": { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"], + "description": "Normative DecisionState lattice member." + }, + "original_decision_state": { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"] + }, "merge_recommendation": { "type": "string", "enum": ["allow", "block", "require_human_review", "allow_with_warning", "require_stronger_check"] }, + "controlling_finding_ids": { + "type": "array", + "items": { "type": "string" } + }, + "finding_contributions": { + "type": "array", + "items": { "type": "object", "additionalProperties": true } + }, "human_review_required": { "type": "boolean" }, "override_allowed": { "type": "boolean" }, "override_requires": { "type": "array", "items": { "type": "string" } } diff --git a/schemas/verification.evidence.v2.schema.json b/schemas/verification.evidence.v2.schema.json index 48de028..0a6c56d 100644 --- a/schemas/verification.evidence.v2.schema.json +++ b/schemas/verification.evidence.v2.schema.json @@ -64,10 +64,22 @@ "type": "object", "required": ["merge_recommendation"], "properties": { + "decision_state": { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"] + }, + "original_decision_state": { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"] + }, "merge_recommendation": { "type": "string", "enum": ["allow", "block", "require_human_review", "allow_with_warning", "require_stronger_check"] }, + "controlling_finding_ids": { + "type": "array", + "items": { "type": "string" } + }, "human_review_required": { "type": "boolean" }, "routing_enforced": { "type": "boolean" }, "aggregation_reason": { "type": "string" } diff --git a/schemas/verification.evidence.v3.schema.json b/schemas/verification.evidence.v3.schema.json index b17a6fb..c7364a0 100644 --- a/schemas/verification.evidence.v3.schema.json +++ b/schemas/verification.evidence.v3.schema.json @@ -71,11 +71,47 @@ "generated_artifacts": { "type": "array", "items": { "type": "object", "additionalProperties": true } }, "decision": { "type": "object", - "required": ["merge_recommendation", "routing_enforced", "aggregation_reason"], + "required": ["decision_state", "merge_recommendation", "routing_enforced", "aggregation_reason"], "properties": { + "decision_state": { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"], + "description": "Normative DecisionState lattice member." + }, + "original_decision_state": { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"], + "description": "Pre-policy / preserved lattice state (advisory must not invent allow)." + }, "merge_recommendation": { "type": "string", - "enum": ["allow", "block", "require_human_review", "allow_with_warning", "require_stronger_check"] + "enum": ["allow", "block", "require_human_review", "allow_with_warning", "require_stronger_check"], + "description": "Deprecated alias of decision_state (needs_review ↔ require_human_review)." + }, + "controlling_finding_ids": { + "type": "array", + "items": { "type": "string" } + }, + "finding_contributions": { + "type": "array", + "items": { + "type": "object", + "required": ["finding_id", "claim_status", "required", "contribution"], + "properties": { + "finding_id": { "type": "string" }, + "claim_status": { + "type": "string", + "enum": ["pass", "fail", "unknown", "error", "skipped"] + }, + "required": { "type": "boolean" }, + "contribution": { + "type": "string", + "enum": ["controlling", "supporting", "non_controlling", "warning"] + }, + "detail": { "type": ["string", "null"] } + }, + "additionalProperties": true + } }, "human_review_required": { "type": "boolean" }, "routing_enforced": { "type": "boolean" }, @@ -118,7 +154,40 @@ "executed_backends": { "type": "array", "items": { "type": "string" } }, "execution_attempts": { "type": "array", "items": { "type": "object", "additionalProperties": true } }, "aggregation_policy": { "type": "string", "minLength": 1 }, - "routing_enforced": { "type": "boolean" } + "routing_enforced": { "type": "boolean" }, + "ovk_version": { "type": "string", "minLength": 1 }, + "checker_id": { "type": "string", "minLength": 1 }, + "checker_version": { "type": "string", "minLength": 1 }, + "input_digest": { "type": "string", "minLength": 1 }, + "relevant_file_digests": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "sha256"], + "properties": { + "path": { "type": "string" }, + "sha256": { "type": "string", "minLength": 1 } + }, + "additionalProperties": true + } + }, + "configuration_digest": { "type": "string", "minLength": 1 }, + "policy_digest": { "type": "string", "minLength": 1 }, + "started_at": { "type": "string", "minLength": 1 }, + "completed_at": { "type": "string", "minLength": 1 }, + "assumptions": { "type": "array", "items": { "type": "string" } }, + "unknowns": { "type": "array", "items": { "type": "string" } }, + "stderr": { "type": ["string", "null"] }, + "exit_status": { "type": ["integer", "null"] }, + "evidence_digest": { "type": "string", "minLength": 1 }, + "signature": { + "type": ["object", "null"], + "properties": { + "algorithm": { "type": "string" }, + "digest": { "type": "string" } + }, + "additionalProperties": true + } }, "additionalProperties": true } diff --git a/tests/test_attestation_provenance.py b/tests/test_attestation_provenance.py index 864ad57..defbe08 100644 --- a/tests/test_attestation_provenance.py +++ b/tests/test_attestation_provenance.py @@ -23,7 +23,7 @@ def test_attestation_includes_builder_provenance() -> None: assert verification["bundle_digest"] builder = statement["predicate"]["builder"] assert builder["id"] == "open-verification-kernel" - assert builder["version"] == "1.2.1" + assert builder["version"] == "1.3.0-rc.1" assert builder["runtime"].startswith("python/") diff --git a/tests/test_evidence_integrity_suite.py b/tests/test_evidence_integrity_suite.py new file mode 100644 index 0000000..facfdbb --- /dev/null +++ b/tests/test_evidence_integrity_suite.py @@ -0,0 +1,303 @@ +"""OVK-PR3 / OVK-04 evidence integrity adversarial suite. + +Covers: tampered input, tampered checker output, missing checker version, +reordered JSON field stability, duplicated finding IDs, partially written +evidence, unsupported schema version, and path-redaction collisions. +""" + +from __future__ import annotations + +import json +from copy import deepcopy +from typing import Any + +import pytest + +from ovk.core.bundle import content_digest, make_bundle +from ovk.core.evidence_integrity import ( + compute_evidence_digest, + detect_path_redaction_collisions, + reconstruct_controlling_decision, + redact_path, + seal_evidence, + verify_evidence_digest, +) +from ovk.core.evidence_invariants import check_evidence_bundle_invariants +from ovk.core.evidence_quality import build_evidence_quality_report +from ovk.core.models import BackendClaim, DecisionState, EvidenceBundle, VerificationEvidence, VerificationStatus + + +def _base_evidence(**overrides: Any) -> VerificationEvidence: + payload: dict[str, Any] = { + "evidence_id": "ev-integrity-1", + "schema_version": "ovk.evidence.v3", + "subject": {"repo": "example/repo", "head_sha": "abc123"}, + "intent": {"intent_id": "no-admin-route-bypass", "title": "No admin route bypass", "risk": {"severity": "high"}}, + "backend_claims": [ + BackendClaim( + backend="authorization-deterministic", + guarantee_type="deterministic_witness", + status=VerificationStatus.FAIL, + assumptions=["normalized routes are complete"], + limits=["does not prove runtime authz"], + adapter_version="0.1.0", + ) + ], + "decision": { + "decision_state": DecisionState.BLOCK.value, + "original_decision_state": DecisionState.BLOCK.value, + "merge_recommendation": "block", + "human_review_required": True, + "controlling_finding_ids": ["ev-integrity-1:authorization-deterministic"], + "finding_contributions": [ + { + "finding_id": "ev-integrity-1:authorization-deterministic", + "claim_status": "fail", + "required": True, + "contribution": "controlling", + } + ], + "aggregation_reason": "required backend failed", + "routing_enforced": True, + }, + "obligation_id": "obl-1", + "routing_id": "route-1", + "compiler": {"compiler_id": "ovk.authorization.neutral.v1", "compiler_version": "0.1.0"}, + "materials": [ + { + "material_id": "m1", + "sha256": "a" * 64, + "uri": "ovk-material:diff", + "kind": "diff", + "size_bytes": 12, + } + ], + "material_set_digest": content_digest( + {"materials": [{"material_id": "m1", "sha256": "a" * 64}]} + ), + "coverage": {"status": "complete", "confidence": 1.0, "extracted_elements": 1}, + "requested_backends": ["authorization-deterministic"], + "eligible_backends": ["authorization-deterministic"], + "selected_backends": ["authorization-deterministic"], + "attempted_backends": ["authorization-deterministic"], + "executed_backends": ["authorization-deterministic"], + "execution_attempts": [ + { + "attempt_id": "a1", + "backend": "authorization-deterministic", + "started_at": "2026-07-25T16:00:00Z", + "finished_at": "2026-07-25T16:00:01Z", + "exit_code": 1, + "stderr_digest": "b" * 64, + } + ], + "aggregation_policy": "ovk.aggregate.fail_dominant.v1", + "routing_enforced": True, + "generated_artifacts": [ + { + "kind": "control_plane_trace", + "routing_id": "route-1", + "material_set_digest": content_digest( + {"materials": [{"material_id": "m1", "sha256": "a" * 64}]} + ), + }, + {"kind": "input_digest", "digest": "c" * 64, "lane": "authorization"}, + ], + } + payload.update(overrides) + evidence = VerificationEvidence.model_validate(payload) + # Keep material_set_digest consistent with materials when not overridden. + if "material_set_digest" not in overrides and evidence.materials: + from ovk.core.materials import compute_material_set_digest + + evidence = evidence.model_copy( + update={"material_set_digest": compute_material_set_digest(evidence.materials)} + ) + # Keep control_plane_trace aligned. + artifacts = [] + for artifact in evidence.generated_artifacts: + if artifact.get("kind") == "control_plane_trace": + artifacts.append({**artifact, "material_set_digest": evidence.material_set_digest}) + else: + artifacts.append(artifact) + evidence = evidence.model_copy(update={"generated_artifacts": artifacts}) + return evidence + + +def _sealed(**overrides: Any) -> VerificationEvidence: + return seal_evidence(_base_evidence(**overrides)) + + +def _error_messages(bundle: EvidenceBundle) -> list[str]: + return [issue.message for issue in check_evidence_bundle_invariants(bundle) if issue.severity == "error"] + + +def test_sealed_evidence_passes_quality_and_reconstructs_decision() -> None: + evidence = _sealed() + assert evidence.evidence_digest + assert evidence.checker_id == "authorization-deterministic" + assert evidence.checker_version == "0.1.0" + assert verify_evidence_digest(evidence) + + reconstructed = reconstruct_controlling_decision(evidence) + assert reconstructed["digest_valid"] is True + assert reconstructed["decision_state"] == DecisionState.BLOCK.value + assert reconstructed["controlling_finding_ids"] == ["ev-integrity-1:authorization-deterministic"] + + bundle = make_bundle([evidence]) + report = build_evidence_quality_report(bundle) + assert report.passed, [issue.message for issue in report.issues] + + +def test_tampered_input_is_rejected() -> None: + evidence = _sealed() + materials = deepcopy(evidence.materials or []) + materials[0] = {**materials[0], "sha256": "d" * 64} + tampered = evidence.model_copy(update={"materials": materials}) + # Stale input_digest + stale evidence_digest relative to new materials. + bundle = make_bundle([tampered]) + messages = _error_messages(bundle) + assert any("tampered input" in message or "input_digest" in message for message in messages) + assert any("evidence_digest" in message for message in messages) + assert build_evidence_quality_report(bundle).passed is False + + +def test_tampered_checker_output_is_rejected() -> None: + evidence = _sealed() + claims = [ + claim.model_copy(update={"status": VerificationStatus.PASS}) for claim in evidence.backend_claims + ] + decision = { + **evidence.decision, + "decision_state": DecisionState.ALLOW.value, + "original_decision_state": DecisionState.ALLOW.value, + "merge_recommendation": "allow", + "human_review_required": False, + "controlling_finding_ids": [], + } + tampered = evidence.model_copy(update={"backend_claims": claims, "decision": decision}) + assert verify_evidence_digest(tampered) is False + bundle = make_bundle([tampered]) + messages = _error_messages(bundle) + assert any("tampered checker output" in message or "evidence_digest" in message for message in messages) + reconstructed = reconstruct_controlling_decision(tampered) + assert reconstructed["digest_valid"] is False + + +def test_missing_checker_version_is_rejected() -> None: + evidence = _sealed() + tampered = evidence.model_copy(update={"checker_version": None}) + # Clearing version also invalidates digest; both must be rejected. + bundle = make_bundle([tampered]) + messages = _error_messages(bundle) + assert any("checker_version" in message for message in messages) + + +def test_reordered_json_fields_digest_stable() -> None: + evidence = _sealed() + payload = evidence.model_dump(mode="json") + # Re-serialize with opposite key insertion order. + reordered = json.loads(json.dumps(payload, sort_keys=False)) + # Force a different key order by rebuilding dict from reversed items. + reordered = {key: reordered[key] for key in reversed(list(reordered.keys()))} + assert compute_evidence_digest(payload) == compute_evidence_digest(reordered) + assert compute_evidence_digest(reordered) == evidence.evidence_digest + assert verify_evidence_digest(reordered) + + +def test_duplicated_finding_ids_are_rejected() -> None: + evidence = _sealed() + decision = deepcopy(evidence.decision) + decision["controlling_finding_ids"] = [ + "ev-integrity-1:authorization-deterministic", + "ev-integrity-1:authorization-deterministic", + ] + # Reseal would be honest; we inject duplicates into a sealed record to simulate tamper. + tampered = evidence.model_copy(update={"decision": decision}) + bundle = make_bundle([tampered]) + messages = _error_messages(bundle) + assert any("duplicated finding IDs" in message for message in messages) + + +def test_partially_written_evidence_is_rejected() -> None: + evidence = _base_evidence(ovk_version="1.2.1", checker_id="authorization-deterministic") + assert evidence.evidence_digest is None + bundle = make_bundle([evidence]) + messages = _error_messages(bundle) + assert any("partially written" in message for message in messages) + assert any("evidence_digest" in message for message in messages) + + +def test_unsupported_schema_version_is_rejected() -> None: + evidence = _base_evidence(schema_version="ovk.evidence.v99") + # v99 is not sealed; unsupported schema must fail closed. + bundle = make_bundle([evidence]) + messages = _error_messages(bundle) + assert any("unsupported evidence schema_version" in message for message in messages) + + +def test_path_redaction_collisions_are_rejected() -> None: + assert redact_path("/home/alice/proj/secret.txt") == "/proj/secret.txt" + assert redact_path("/home/bob/proj/secret.txt") == "/proj/secret.txt" + collisions = detect_path_redaction_collisions( + ["/home/alice/proj/secret.txt", "/home/bob/proj/secret.txt"] + ) + assert collisions + + evidence = _sealed() + # Inject colliding redacted paths with distinct digests into sealed evidence. + tampered = evidence.model_copy( + update={ + "relevant_file_digests": [ + {"path": "/proj/secret.txt", "sha256": "1" * 64}, + {"path": "/proj/secret.txt", "sha256": "2" * 64}, + ] + } + ) + bundle = make_bundle([tampered]) + messages = _error_messages(bundle) + assert any("path-redaction collisions" in message for message in messages) + + +def test_seal_raises_on_material_path_redaction_collision() -> None: + evidence = _base_evidence( + materials=[ + { + "material_id": "m1", + "sha256": "1" * 64, + "path": "/home/alice/proj/secret.txt", + "kind": "diff", + "size_bytes": 1, + "uri": "file:///home/alice/proj/secret.txt", + }, + { + "material_id": "m2", + "sha256": "2" * 64, + "path": "/home/bob/proj/secret.txt", + "kind": "diff", + "size_bytes": 1, + "uri": "file:///home/bob/proj/secret.txt", + }, + ] + ) + with pytest.raises(ValueError, match="path redaction collisions"): + seal_evidence(evidence) + + +def test_digest_binds_controlling_decision() -> None: + evidence = _sealed() + reconstructed = reconstruct_controlling_decision(evidence) + assert reconstructed["digest_valid"] is True + assert reconstructed["decision_state"] == evidence.decision["decision_state"] + assert reconstructed["controlling_finding_ids"] == evidence.decision["controlling_finding_ids"] + + flipped = evidence.model_copy( + update={ + "decision": { + **evidence.decision, + "decision_state": DecisionState.ALLOW.value, + "controlling_finding_ids": [], + } + } + ) + assert reconstruct_controlling_decision(flipped)["digest_valid"] is False From 081f523bc573ae07e24670f4b03274e6e3d8bc95 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:09:09 -0700 Subject: [PATCH 05/19] Add adapter conformance matrix and CLAIMS manifests (OVK-PR4). Require every stable adapter to ship machine-readable conformance claims so registry stability implies tested timeout, unavailable, and malformed behavior. --- adapters/alloy/conformance/CLAIMS.md | 9 + adapters/alloy/conformance/manifest.json | 26 + adapters/cbmc/conformance/CLAIMS.md | 9 + adapters/cbmc/conformance/manifest.json | 26 + adapters/cedar/conformance/CLAIMS.md | 9 + adapters/cedar/conformance/manifest.json | 26 + adapters/dafny/conformance/CLAIMS.md | 9 + adapters/dafny/conformance/manifest.json | 26 + adapters/kani/conformance/CLAIMS.md | 9 + adapters/kani/conformance/manifest.json | 26 + adapters/lean/conformance/CLAIMS.md | 9 + adapters/lean/conformance/manifest.json | 26 + adapters/opa/conformance/CLAIMS.md | 9 + adapters/opa/conformance/manifest.json | 26 + adapters/tla/conformance/CLAIMS.md | 9 + adapters/tla/conformance/manifest.json | 26 + adapters/verus/conformance/CLAIMS.md | 9 + adapters/verus/conformance/manifest.json | 26 + adapters/z3/conformance/CLAIMS.md | 9 + adapters/z3/conformance/manifest.json | 26 + docs/BACKENDS.md | 49 +- docs/LANES.md | 16 +- ovk/core/adapter_conformance.py | 549 ++++++++++++++++++++ scripts/generate_adapter_conformance.py | 614 +++++++++++++++++++++++ scripts/validate_adapter_conformance.py | 79 +++ tests/test_adapter_conformance.py | 96 ++++ 26 files changed, 1731 insertions(+), 22 deletions(-) create mode 100644 adapters/alloy/conformance/CLAIMS.md create mode 100644 adapters/alloy/conformance/manifest.json create mode 100644 adapters/cbmc/conformance/CLAIMS.md create mode 100644 adapters/cbmc/conformance/manifest.json create mode 100644 adapters/cedar/conformance/CLAIMS.md create mode 100644 adapters/cedar/conformance/manifest.json create mode 100644 adapters/dafny/conformance/CLAIMS.md create mode 100644 adapters/dafny/conformance/manifest.json create mode 100644 adapters/kani/conformance/CLAIMS.md create mode 100644 adapters/kani/conformance/manifest.json create mode 100644 adapters/lean/conformance/CLAIMS.md create mode 100644 adapters/lean/conformance/manifest.json create mode 100644 adapters/opa/conformance/CLAIMS.md create mode 100644 adapters/opa/conformance/manifest.json create mode 100644 adapters/tla/conformance/CLAIMS.md create mode 100644 adapters/tla/conformance/manifest.json create mode 100644 adapters/verus/conformance/CLAIMS.md create mode 100644 adapters/verus/conformance/manifest.json create mode 100644 adapters/z3/conformance/CLAIMS.md create mode 100644 adapters/z3/conformance/manifest.json create mode 100644 ovk/core/adapter_conformance.py create mode 100644 scripts/generate_adapter_conformance.py create mode 100644 scripts/validate_adapter_conformance.py create mode 100644 tests/test_adapter_conformance.py diff --git a/adapters/alloy/conformance/CLAIMS.md b/adapters/alloy/conformance/CLAIMS.md new file mode 100644 index 0000000..5729de5 --- /dev/null +++ b/adapters/alloy/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The deterministic Alloy model oracle reported no counterexample instances for the supplied fixture. + +## Outside the claim + +Native Alloy analysis is not implemented; does not prove properties outside the fixture model scope. diff --git a/adapters/alloy/conformance/manifest.json b/adapters/alloy/conformance/manifest.json new file mode 100644 index 0000000..e14512d --- /dev/null +++ b/adapters/alloy/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "alloy", + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/backends/alloy_pass.json" + }, + "fail": { + "path": "examples/backends/alloy_fail.json" + }, + "malformed": { + "path": "examples/backends/alloy_malformed.json" + }, + "timeout": { + "path": "examples/backends/alloy_timeout.json" + }, + "unavailable": { + "path": "examples/backends/alloy_unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/cbmc/conformance/CLAIMS.md b/adapters/cbmc/conformance/CLAIMS.md new file mode 100644 index 0000000..21e11b0 --- /dev/null +++ b/adapters/cbmc/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +Bounded model checking of the supplied harness found no assertion violations within the stated unwind and memory bounds. + +## Outside the claim + +Bounded only; does not prove unbounded safety, and synthetic harnesses do not compile changed project source into the checked model unless an explicit harness is supplied. diff --git a/adapters/cbmc/conformance/manifest.json b/adapters/cbmc/conformance/manifest.json new file mode 100644 index 0000000..95cbd3b --- /dev/null +++ b/adapters/cbmc/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "cbmc", + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/backends/cbmc_pass.json" + }, + "fail": { + "path": "examples/backends/cbmc_fail.json" + }, + "malformed": { + "path": "examples/backends/cbmc_malformed.json" + }, + "timeout": { + "path": "examples/backends/cbmc_timeout.json" + }, + "unavailable": { + "path": "examples/backends/cbmc_unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/cedar/conformance/CLAIMS.md b/adapters/cedar/conformance/CLAIMS.md new file mode 100644 index 0000000..91e0145 --- /dev/null +++ b/adapters/cedar/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The deterministic Cedar-shaped policy oracle reported no violations for the supplied authorization fixture. + +## Outside the claim + +Native Cedar policy evaluation is not implemented; does not verify runtime middleware behavior. diff --git a/adapters/cedar/conformance/manifest.json b/adapters/cedar/conformance/manifest.json new file mode 100644 index 0000000..cce8640 --- /dev/null +++ b/adapters/cedar/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "cedar", + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/backends/cedar_pass.json" + }, + "fail": { + "path": "examples/backends/cedar_fail.json" + }, + "malformed": { + "path": "examples/backends/cedar_malformed.json" + }, + "timeout": { + "path": "examples/backends/cedar_timeout.json" + }, + "unavailable": { + "path": "examples/backends/cedar_unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/dafny/conformance/CLAIMS.md b/adapters/dafny/conformance/CLAIMS.md new file mode 100644 index 0000000..9673c26 --- /dev/null +++ b/adapters/dafny/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The deterministic Dafny obligation oracle reported all listed obligations as proved for the supplied fixture. + +## Outside the claim + +Native Dafny verification is not implemented; does not establish proofs for code outside the fixture obligations. diff --git a/adapters/dafny/conformance/manifest.json b/adapters/dafny/conformance/manifest.json new file mode 100644 index 0000000..025b2c4 --- /dev/null +++ b/adapters/dafny/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "dafny", + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/backends/dafny_pass.json" + }, + "fail": { + "path": "examples/backends/dafny_fail.json" + }, + "malformed": { + "path": "examples/backends/dafny_malformed.json" + }, + "timeout": { + "path": "examples/backends/dafny_timeout.json" + }, + "unavailable": { + "path": "examples/backends/dafny_unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/kani/conformance/CLAIMS.md b/adapters/kani/conformance/CLAIMS.md new file mode 100644 index 0000000..2363f60 --- /dev/null +++ b/adapters/kani/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The deterministic Kani-shaped harness oracle reported no memory-safety or policy violations for the supplied fixture. + +## Outside the claim + +Native Kani execution is not implemented; does not verify arbitrary Rust beyond the fixture oracle. diff --git a/adapters/kani/conformance/manifest.json b/adapters/kani/conformance/manifest.json new file mode 100644 index 0000000..4341d24 --- /dev/null +++ b/adapters/kani/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "kani", + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/backends/kani_pass.json" + }, + "fail": { + "path": "examples/backends/kani_fail.json" + }, + "malformed": { + "path": "examples/backends/kani_malformed.json" + }, + "timeout": { + "path": "examples/backends/kani_timeout.json" + }, + "unavailable": { + "path": "examples/backends/kani_unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/lean/conformance/CLAIMS.md b/adapters/lean/conformance/CLAIMS.md new file mode 100644 index 0000000..14f3643 --- /dev/null +++ b/adapters/lean/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The deterministic Lean proof oracle reported all listed obligations as proved for the supplied fixture. + +## Outside the claim + +Native Lean checking is not implemented; does not establish theorems outside the fixture obligations. diff --git a/adapters/lean/conformance/manifest.json b/adapters/lean/conformance/manifest.json new file mode 100644 index 0000000..902ff79 --- /dev/null +++ b/adapters/lean/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "lean", + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/backends/lean_pass.json" + }, + "fail": { + "path": "examples/backends/lean_fail.json" + }, + "malformed": { + "path": "examples/backends/lean_malformed.json" + }, + "timeout": { + "path": "examples/backends/lean_timeout.json" + }, + "unavailable": { + "path": "examples/backends/lean_unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/opa/conformance/CLAIMS.md b/adapters/opa/conformance/CLAIMS.md new file mode 100644 index 0000000..031b3b9 --- /dev/null +++ b/adapters/opa/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The supplied structured input satisfies the selected OPA/Rego policy under the adapter's data model for this fixture. + +## Outside the claim + +Does not prove correctness of arbitrary program execution, workflow semantics beyond the policy, or properties outside the selected Rego rules. diff --git a/adapters/opa/conformance/manifest.json b/adapters/opa/conformance/manifest.json new file mode 100644 index 0000000..9926600 --- /dev/null +++ b/adapters/opa/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "opa", + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/backends/opa_pass.json" + }, + "fail": { + "path": "examples/backends/opa_fail.json" + }, + "malformed": { + "path": "examples/backends/opa_malformed.json" + }, + "timeout": { + "path": "examples/backends/opa_timeout.json" + }, + "unavailable": { + "path": "examples/backends/opa_unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/tla/conformance/CLAIMS.md b/adapters/tla/conformance/CLAIMS.md new file mode 100644 index 0000000..579e162 --- /dev/null +++ b/adapters/tla/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The deterministic TLA+/state-machine oracle found no skipped required approval states for the supplied fixture. + +## Outside the claim + +TLC execution is not implemented; does not prove liveness or properties outside the supplied finite state machine. diff --git a/adapters/tla/conformance/manifest.json b/adapters/tla/conformance/manifest.json new file mode 100644 index 0000000..bd5eaec --- /dev/null +++ b/adapters/tla/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "tla+", + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/backends/tla_pass.json" + }, + "fail": { + "path": "examples/backends/tla_fail.json" + }, + "malformed": { + "path": "examples/backends/tla_malformed.json" + }, + "timeout": { + "path": "examples/backends/tla_timeout.json" + }, + "unavailable": { + "path": "examples/backends/tla_unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/verus/conformance/CLAIMS.md b/adapters/verus/conformance/CLAIMS.md new file mode 100644 index 0000000..6cc1587 --- /dev/null +++ b/adapters/verus/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The deterministic Verus harness oracle reported all listed obligations as proved for the supplied fixture. + +## Outside the claim + +Native Verus verification is not implemented; does not prove properties outside the fixture harness. diff --git a/adapters/verus/conformance/manifest.json b/adapters/verus/conformance/manifest.json new file mode 100644 index 0000000..e77929c --- /dev/null +++ b/adapters/verus/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "verus", + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/backends/verus_pass.json" + }, + "fail": { + "path": "examples/backends/verus_fail.json" + }, + "malformed": { + "path": "examples/backends/verus_malformed.json" + }, + "timeout": { + "path": "examples/backends/verus_timeout.json" + }, + "unavailable": { + "path": "examples/backends/verus_unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/z3/conformance/CLAIMS.md b/adapters/z3/conformance/CLAIMS.md new file mode 100644 index 0000000..9cf73e9 --- /dev/null +++ b/adapters/z3/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +No counterexample was found for the encoded authorization obligation (or the query polarity recorded in the obligation was satisfied). + +## Outside the claim + +Does not prove properties outside the finite abstraction, unsupported theories, or middleware behavior absent from the route encoding. diff --git a/adapters/z3/conformance/manifest.json b/adapters/z3/conformance/manifest.json new file mode 100644 index 0000000..22e67f6 --- /dev/null +++ b/adapters/z3/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "z3", + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/auth_regression/input_admin_protected.json" + }, + "fail": { + "path": "examples/auth_regression/input_admin_bypass.json" + }, + "malformed": { + "path": "examples/auth_regression/input_malformed_missing_routes.json" + }, + "timeout": { + "path": "examples/backends/z3_timeout.json" + }, + "unavailable": { + "path": "examples/backends/z3_unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/docs/BACKENDS.md b/docs/BACKENDS.md index 8f4407a..e40c3e8 100644 --- a/docs/BACKENDS.md +++ b/docs/BACKENDS.md @@ -4,20 +4,41 @@ OVK exposes a common evidence contract across ten formal-methods backends. Their ## Execution maturity -| Backend | Current execution | Native result can determine evidence? | Current limit | -|---|---|---:|---| -| `opa` | Native `opa eval` path plus deterministic self-protection evaluator | Yes, when the OPA strategy is selected | Generic kernel router selections do not yet control lane execution | -| `z3` | Native Python Z3 SMT query plus deterministic authorization evaluator | Yes, when Z3 is installed | The query checks a normalized authorization abstraction, not arbitrary application code | -| `cbmc` | Native bounded checking of an explicit or OVK template harness | Yes | Template/generated harnesses model a risk pattern and do not prove that changed project source was compiled into the model | -| `cedar` | Deterministic Cedar-shaped input evaluator; Cedar CLI version probe | No | Native Cedar policy evaluation is not implemented | -| `tla+` | Deterministic state-machine contract evaluator | No | TLC execution is not implemented | -| `kani` | Deterministic Rust-harness contract evaluator | No | Native Kani execution is not implemented | -| `dafny` | Deterministic proof-obligation contract evaluator | No | Native Dafny verification is not implemented | -| `verus` | Deterministic verified-Rust contract evaluator | No | Native Verus verification is not implemented | -| `lean` | Deterministic theorem-obligation contract evaluator | No | Native Lean checking is not implemented | -| `alloy` | Deterministic relational-model contract evaluator | No | Native Alloy analysis is not implemented | - -A binary-presence or version probe is never labeled as native verification. Evidence artifacts record `used_native_binary`, the guarantee type, assumptions, and limits. + +| Backend | release_status | Current execution | Native result can determine evidence? | Current limit | +|---|---|---|---:|---| +| `opa` | preview | Native path available (tool_dependent) | Yes | Does not prove properties of arbitrary program execution | +| `z3` | preview | Native path available (tool_dependent) | Yes | Does not prove properties outside the encoded abstraction | +| `cbmc` | preview | Native path available (tool_dependent) | Yes | Bounded verification only | +| `cedar` | experimental | Deterministic contract evaluator only (deterministic) | No | Native Cedar policy evaluation is not implemented | +| `tla+` | experimental | Deterministic contract evaluator only (deterministic) | No | TLC execution is not implemented | +| `kani` | experimental | Deterministic contract evaluator only (deterministic) | No | Native Kani execution is not implemented | +| `dafny` | experimental | Deterministic contract evaluator only (deterministic) | No | Native Dafny verification is not implemented | +| `verus` | experimental | Deterministic contract evaluator only (deterministic) | No | Native Verus verification is not implemented | +| `lean` | experimental | Deterministic contract evaluator only (deterministic) | No | Native Lean checking is not implemented | +| `alloy` | experimental | Deterministic contract evaluator only (deterministic) | No | Native Alloy analysis is not implemented | +| `lane-authorization` | experimental | Deterministic contract evaluator only (tool_dependent) | No | Does not reconstruct frameworks beyond the supplied route abstraction. | +| `lane-ci-secrets` | experimental | Deterministic contract evaluator only (deterministic) | No | Does not analyze composite actions beyond the supplied steps. | +| `lane-deployment` | experimental | Deterministic contract evaluator only (deterministic) | No | Does not prove runtime orchestrator behavior beyond the abstraction. | +| `lane-infrastructure` | experimental | Deterministic contract evaluator only (deterministic) | No | Does not prove runtime cloud configurations beyond the abstraction. | +| `lane-self-protection` | experimental | Deterministic contract evaluator only (deterministic) | No | Does not analyze checks outside the declared OVK gate name. | + + +A binary-presence or version probe is never labeled as native verification. Evidence artifacts record `used_native_binary`, the guarantee type, assumptions, and limits. Tables above are regenerated from `adapters/*/capability.json` via `scripts/render_capability_tables.py`. + +## Adapter conformance (OVK-05) + +Every advertised adapter ships a seven-item suite under `adapters//conformance/`: + +1. pass fixture +2. fail fixture +3. malformed-output fixture +4. timeout fixture +5. unavailable-binary fixture +6. documentation of what a pass establishes +7. documentation of what remains outside the claim + +Validate with `python scripts/validate_adapter_conformance.py`. `release_status=stable` requires all seven; non-conformant adapters that claim `stable` are auto-downgraded when capability tables are rendered. Only native candidates (`opa`, `z3`, `cbmc`) may become `stable` once fully conformant; other adapters remain `preview`/`experimental`. ## CI tiers diff --git a/docs/LANES.md b/docs/LANES.md index 1ac0a51..e47a513 100644 --- a/docs/LANES.md +++ b/docs/LANES.md @@ -27,10 +27,10 @@ ovk ci \ --advisory ``` -| Condition | Recommendation | +| Condition | DecisionState | |---|---| | Required check removed | `block` | -| Missing required-check metadata | `require_human_review` | +| Missing required-check metadata | `needs_review` | | Controls preserved | `allow` | Backend: deterministic evaluator (default), optional OPA via `--backend-strategy`. @@ -47,11 +47,11 @@ Input schema: `schemas/authorization.input.schema.json` Required field: non-empty `routes` list. Each route has `path`, `admin_only_before`, `admin_only_after`, and `reachable_after`. -| Solver result | Recommendation | +| Solver result | DecisionState | |---|---| | Violation found | `block` | | No violation | `allow` | -| Solver unavailable | `require_human_review` | +| Solver unavailable | `needs_review` / `unknown` | Optional Z3 backend: `pip install -e '.[solvers]'` @@ -90,11 +90,11 @@ Optional policy file: `schemas/infrastructure.policy.schema.json` Pass via `--policy `. -| Condition | Recommendation | +| Condition | DecisionState | |---|---| | Sensitive resource publicly exposed | `block` | | Resource remains private | `allow` | -| Invalid abstraction | `require_human_review` | +| Invalid abstraction | `needs_review` / `error` | ## CI secrets @@ -109,7 +109,7 @@ Input schema: `schemas/ci_secrets.input.schema.json` Workflow YAML extraction handles PyYAML 1.1 `on:` → `True` quirk. Unified diffs reconstruct workflow content via `ovk plan` / `ovk infer`. -| Condition | Recommendation | +| Condition | DecisionState | |---|---| | Secrets on untrusted trigger (`pull_request`, etc.) | `block` | | `pull_request_target` with PR head checkout + secrets | `block` | @@ -129,7 +129,7 @@ Uses graph reachability over deployment states. Skipped required approval → `b ## Conservative rule -Invalid input must not produce `allow`. Unknown and error states require human review for every check type. +Invalid input must not produce `allow`. Under the ``DecisionState`` lattice, ``error``, ``unknown``, and required ``skipped`` never become ``allow`` in strict mode; advisory preserves ``original_decision_state`` rather than inventing an allow-with-warning lattice member. Examples: `examples/` per check type. Benchmark scorer: `benchmarks/formal_pr_bench/score_all_lanes.py` (internal script name). diff --git a/ovk/core/adapter_conformance.py b/ovk/core/adapter_conformance.py new file mode 100644 index 0000000..669b704 --- /dev/null +++ b/ovk/core/adapter_conformance.py @@ -0,0 +1,549 @@ +"""Adapter conformance matrix (OVK-05 / OVK-PR4). + +Every advertised adapter (10 formal backends + 5 lane adapters) must provide: + +1. pass fixture +2. fail fixture +3. malformed-output fixture +4. timeout fixture +5. unavailable-binary fixture +6. documentation of what a pass establishes +7. documentation of what remains outside the claim + +``release_status=stable`` requires all seven. Non-conformant adapters that claim +stable are auto-downgraded when capability tables are rendered. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any + +from ovk.adapters.wave2_oracle import classify_conformance_flags +from ovk.core.evidence_integrity import seal_evidence, verify_evidence_digest +from ovk.core.json_io import read_json_file +from ovk.core.models import BackendClaim, VerificationEvidence, VerificationStatus +from ovk.paths import ovk_data_root + +FORMAL_BACKEND_IDS: tuple[str, ...] = ( + "opa", + "z3", + "cbmc", + "cedar", + "tla+", + "kani", + "dafny", + "verus", + "lean", + "alloy", +) + +LANE_ADAPTER_IDS: tuple[str, ...] = ( + "lane-self-protection", + "lane-authorization", + "lane-infrastructure", + "lane-ci-secrets", + "lane-deployment", +) + +ADVERTISED_ADAPTER_IDS: tuple[str, ...] = FORMAL_BACKEND_IDS + LANE_ADAPTER_IDS + +REQUIRED_FIXTURE_CASES: tuple[str, ...] = ( + "pass", + "fail", + "malformed", + "timeout", + "unavailable", +) + +REQUIRED_DOC_KEYS: tuple[str, ...] = ( + "pass_establishes", + "outside_claim", +) + +# Directory name under adapters/ for checker_id values that differ from folder names. +ADAPTER_DIR_ALIASES: dict[str, str] = { + "tla+": "tla", +} + +EXPECTED_STATUS_BY_CASE: dict[str, frozenset[str]] = { + "pass": frozenset({"pass"}), + "fail": frozenset({"fail"}), + "malformed": frozenset({"unknown", "error"}), + "timeout": frozenset({"unknown", "error"}), + "unavailable": frozenset({"unknown", "error", "skipped"}), +} + +LANE_ID_TO_LANE: dict[str, str] = { + "lane-self-protection": "self_protection", + "lane-authorization": "authorization", + "lane-infrastructure": "infrastructure", + "lane-ci-secrets": "ci_secrets", + "lane-deployment": "deployment", +} + +_PASS_SECTION = re.compile( + r"(?im)^#{1,3}\s*pass\s+establishes\s*$", +) +_OUTSIDE_SECTION = re.compile( + r"(?im)^#{1,3}\s*(?:outside\s+(?:the\s+)?claim|outside\s+the\s+claim)\s*$", +) + + +def adapter_directory_name(adapter_id: str) -> str: + """Return the filesystem directory name for an advertised adapter id.""" + return ADAPTER_DIR_ALIASES.get(adapter_id, adapter_id) + + +def conformance_dir(adapter_id: str, *, root: Path | None = None) -> Path: + """Return ``adapters//conformance`` for an advertised adapter.""" + base = root or ovk_data_root() + return base / "adapters" / adapter_directory_name(adapter_id) / "conformance" + + +def manifest_path(adapter_id: str, *, root: Path | None = None) -> Path: + return conformance_dir(adapter_id, root=root) / "manifest.json" + + +def claims_path(adapter_id: str, *, root: Path | None = None) -> Path: + return conformance_dir(adapter_id, root=root) / "CLAIMS.md" + + +def load_conformance_manifest(adapter_id: str, *, root: Path | None = None) -> dict[str, Any]: + path = manifest_path(adapter_id, root=root) + if not path.is_file(): + raise FileNotFoundError(f"missing conformance manifest: {path}") + payload = read_json_file(path) + if not isinstance(payload, dict): + raise ValueError(f"conformance manifest must be an object: {path}") + return payload + + +def resolve_fixture_path( + adapter_id: str, + relative: str, + *, + root: Path | None = None, +) -> Path: + """Resolve a fixture path relative to the repo root or the conformance dir.""" + base = root or ovk_data_root() + candidate = Path(relative) + if candidate.is_absolute(): + return candidate + from_root = base / candidate + if from_root.exists(): + return from_root + from_conformance = conformance_dir(adapter_id, root=base) / candidate + return from_conformance + + +def _claim_status(evidence: VerificationEvidence) -> str: + if evidence.backend_claims: + return evidence.backend_claims[0].status.value + decision = evidence.decision or {} + recommendation = str(decision.get("merge_recommendation") or decision.get("decision_state") or "") + mapping = { + "allow": "pass", + "block": "fail", + "require_human_review": "unknown", + "needs_review": "unknown", + "unknown": "unknown", + "error": "error", + "skipped": "skipped", + } + return mapping.get(recommendation, "unknown") + + +def _synthetic_evidence( + *, + adapter_id: str, + status: str, + summary: str, + failure_mode: str, + repo: str, + head_sha: str, + base_sha: str | None, + adapter_version: str = "0.1.0", +) -> VerificationEvidence: + verification_status = VerificationStatus(status) + merge_by_status = { + VerificationStatus.PASS: "allow", + VerificationStatus.FAIL: "block", + VerificationStatus.UNKNOWN: "require_human_review", + VerificationStatus.ERROR: "require_human_review", + VerificationStatus.SKIPPED: "require_human_review", + } + recommendation = merge_by_status[verification_status] + subject: dict[str, Any] = {"repo": repo, "head_sha": head_sha} + if base_sha is not None: + subject["base_sha"] = base_sha + counterexamples: list[dict[str, Any]] = [] + if verification_status != VerificationStatus.PASS: + counterexamples = [{"summary": summary, "failure_mode": failure_mode}] + return VerificationEvidence( + evidence_id=f"conformance-{adapter_id}-{head_sha[:8]}", + schema_version="ovk.evidence.v3", + subject=subject, + intent={ + "intent_id": f"{adapter_id}-conformance", + "title": f"{adapter_id} conformance", + "risk": {"severity": "medium"}, + }, + backend_claims=[ + BackendClaim( + backend=adapter_id, + guarantee_type="adapter_conformance", + status=verification_status, + assumptions=["Conformance fixture evaluation."], + limits=["Synthetic conformance fixture; not a production claim."], + adapter_version=adapter_version, + ) + ], + counterexamples=counterexamples, + decision={ + "merge_recommendation": recommendation, + "human_review_required": recommendation != "allow", + "override_allowed": recommendation != "allow", + "override_requires": ["maintainer"] if recommendation != "allow" else [], + }, + ) + + +def _evaluate_opa_fixture( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, +) -> VerificationEvidence: + from ovk.adapters.opa.evidence import opa_raw_to_evidence + + early = classify_conformance_flags(data) + if early is not None: + status, counterexamples = early + reason = counterexamples[0]["summary"] if counterexamples else status + return opa_raw_to_evidence( + {"status": status, "reason": reason, "violations": []}, + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + ) + raw = { + "status": str(data.get("status", "pass" if not data.get("violations") else "fail")), + "violations": list(data.get("violations") or []), + "reason": data.get("reason"), + } + return opa_raw_to_evidence(raw, repo=repo, head_sha=head_sha, base_sha=base_sha) + + +def _evaluate_z3_fixture( + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, +) -> VerificationEvidence: + early = classify_conformance_flags(data) + if early is not None: + status, counterexamples = early + return _synthetic_evidence( + adapter_id="z3", + status=status, + summary=counterexamples[0]["summary"], + failure_mode=str(counterexamples[0]["failure_mode"]), + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + ) + from ovk.adapters.z3.validated_path import evaluate_validated_authorization_path + + return evaluate_validated_authorization_path( + data, repo=repo, head_sha=head_sha, base_sha=base_sha + ) + + +def _evaluate_lane_fixture( + adapter_id: str, + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, +) -> VerificationEvidence: + early = classify_conformance_flags(data) + if early is not None: + status, counterexamples = early + return _synthetic_evidence( + adapter_id=adapter_id, + status=status, + summary=counterexamples[0]["summary"], + failure_mode=str(counterexamples[0]["failure_mode"]), + repo=repo, + head_sha=head_sha, + base_sha=base_sha, + ) + from ovk.core.multi_lane import evaluate_lane + + lane = LANE_ID_TO_LANE[adapter_id] + return evaluate_lane(lane, data, repo=repo, head_sha=head_sha, base_sha=base_sha) + + +def _evaluate_formal_backend_fixture( + adapter_id: str, + data: dict[str, Any], + *, + repo: str, + head_sha: str, + base_sha: str | None, +) -> VerificationEvidence: + if adapter_id == "opa": + return _evaluate_opa_fixture(data, repo=repo, head_sha=head_sha, base_sha=base_sha) + if adapter_id == "z3": + return _evaluate_z3_fixture(data, repo=repo, head_sha=head_sha, base_sha=base_sha) + from ovk.core.backend_fixture import evaluate_backend_fixture + + # Ensure intent_id is present for backends that rely on it. + payload = dict(data) + if "intent_id" not in payload: + intent_defaults = { + "cbmc": "cbmc-harness-check", + "cedar": "cedar-policy-check", + "tla+": "tla-state-check", + "kani": "kani-harness-check", + "dafny": "dafny-obligation-check", + "verus": "verus-harness-check", + "lean": "lean-proof-check", + "alloy": "alloy-model-check", + } + if adapter_id in intent_defaults: + payload["intent_id"] = intent_defaults[adapter_id] + return evaluate_backend_fixture(payload, repo=repo, head_sha=head_sha, base_sha=base_sha) + + +def evaluate_conformance_fixture( + adapter_id: str, + data: dict[str, Any], + *, + repo: str = "ovk/conformance", + head_sha: str = "conformance-head", + base_sha: str | None = "conformance-base", +) -> VerificationEvidence: + """Evaluate one conformance fixture payload for an advertised adapter.""" + if adapter_id in LANE_ADAPTER_IDS: + return _evaluate_lane_fixture( + adapter_id, data, repo=repo, head_sha=head_sha, base_sha=base_sha + ) + if adapter_id in FORMAL_BACKEND_IDS: + return _evaluate_formal_backend_fixture( + adapter_id, data, repo=repo, head_sha=head_sha, base_sha=base_sha + ) + raise ValueError(f"unknown advertised adapter id: {adapter_id!r}") + + +def _doc_sections_present(claims_text: str) -> tuple[bool, bool]: + has_pass = bool(_PASS_SECTION.search(claims_text)) + has_outside = bool(_OUTSIDE_SECTION.search(claims_text)) + return has_pass, has_outside + + +def structural_conformance_failures( + adapter_id: str, + *, + root: Path | None = None, +) -> list[str]: + """Return structural failures for the seven-item conformance matrix.""" + base = root or ovk_data_root() + failures: list[str] = [] + conf = conformance_dir(adapter_id, root=base) + if not conf.is_dir(): + return [f"{adapter_id}: missing conformance directory {conf}"] + + try: + manifest = load_conformance_manifest(adapter_id, root=base) + except (OSError, ValueError, FileNotFoundError) as error: + return [f"{adapter_id}: {error}"] + + if str(manifest.get("adapter_id", "")) not in {adapter_id, adapter_directory_name(adapter_id)}: + failures.append( + f"{adapter_id}: manifest adapter_id {manifest.get('adapter_id')!r} " + f"does not match advertised id" + ) + + fixtures = manifest.get("fixtures") + if not isinstance(fixtures, dict): + failures.append(f"{adapter_id}: manifest.fixtures must be an object") + fixtures = {} + + for case in REQUIRED_FIXTURE_CASES: + entry = fixtures.get(case) + if not isinstance(entry, dict): + failures.append(f"{adapter_id}: missing fixtures.{case}") + continue + rel = entry.get("path") + if not isinstance(rel, str) or not rel.strip(): + failures.append(f"{adapter_id}: fixtures.{case}.path must be a non-empty string") + continue + path = resolve_fixture_path(adapter_id, rel, root=base) + if not path.is_file(): + failures.append(f"{adapter_id}: fixtures.{case} path not found: {path}") + + docs = manifest.get("docs") + if not isinstance(docs, dict): + failures.append(f"{adapter_id}: manifest.docs must be an object") + docs = {} + + claims = claims_path(adapter_id, root=base) + if not claims.is_file(): + failures.append(f"{adapter_id}: missing CLAIMS.md ({claims})") + else: + text = claims.read_text(encoding="utf-8") + has_pass, has_outside = _doc_sections_present(text) + if not has_pass and "pass_establishes" not in docs: + failures.append(f"{adapter_id}: CLAIMS.md missing 'Pass establishes' section") + if not has_outside and "outside_claim" not in docs: + failures.append(f"{adapter_id}: CLAIMS.md missing 'Outside the claim' section") + for key in REQUIRED_DOC_KEYS: + if key not in docs and not (has_pass if key == "pass_establishes" else has_outside): + failures.append(f"{adapter_id}: docs.{key} missing from manifest and CLAIMS.md") + + return failures + + +def behavioral_conformance_failures( + adapter_id: str, + *, + root: Path | None = None, + seal: bool = True, +) -> list[str]: + """Evaluate fixtures, check expected statuses, and require integrity-valid evidence.""" + base = root or ovk_data_root() + structural = structural_conformance_failures(adapter_id, root=base) + if structural: + return structural + + manifest = load_conformance_manifest(adapter_id, root=base) + fixtures = manifest["fixtures"] + failures: list[str] = [] + + for case in REQUIRED_FIXTURE_CASES: + entry = fixtures[case] + path = resolve_fixture_path(adapter_id, str(entry["path"]), root=base) + try: + data = read_json_file(path) + except (OSError, ValueError) as error: + failures.append(f"{adapter_id}/{case}: could not read fixture ({error})") + continue + if not isinstance(data, dict): + failures.append(f"{adapter_id}/{case}: fixture must be a JSON object") + continue + try: + evidence = evaluate_conformance_fixture(adapter_id, data) + except Exception as error: # noqa: BLE001 - conformance boundary + failures.append(f"{adapter_id}/{case}: evaluation raised {type(error).__name__}: {error}") + continue + + status = _claim_status(evidence) + expected = EXPECTED_STATUS_BY_CASE[case] + override = entry.get("expected_status") + if isinstance(override, str) and override.strip(): + expected = frozenset({override.strip()}) + if status not in expected: + failures.append( + f"{adapter_id}/{case}: expected status in {sorted(expected)}, got {status!r}" + ) + continue + + if seal: + try: + sealed = seal_evidence(evidence) + except Exception as error: # noqa: BLE001 + failures.append( + f"{adapter_id}/{case}: seal_evidence failed ({type(error).__name__}: {error})" + ) + continue + if not verify_evidence_digest(sealed): + failures.append(f"{adapter_id}/{case}: evidence_digest verification failed") + + return failures + + +def is_fully_conformant(adapter_id: str, *, root: Path | None = None, seal: bool = True) -> bool: + """Return True when the adapter satisfies all seven conformance items.""" + return not behavioral_conformance_failures(adapter_id, root=root, seal=seal) + + +def effective_release_status( + manifest: dict[str, Any], + *, + root: Path | None = None, + conformant: bool | None = None, +) -> str: + """Return the honesty-adjusted release_status for rendering. + + Non-conformant adapters that declare ``stable`` are auto-downgraded to + ``preview`` (native candidates) or ``experimental`` (everyone else). + """ + declared = str(manifest.get("release_status") or "experimental") + if declared != "stable": + return declared + checker = str(manifest.get("checker_id") or manifest.get("tool", {}).get("name") or "") + ok = is_fully_conformant(checker, root=root) if conformant is None else conformant + if ok: + return "stable" + native = manifest.get("native_execution") + if native is True or checker in {"opa", "z3", "cbmc"}: + return "preview" + return "experimental" + + +def apply_release_status_honesty( + manifest: dict[str, Any], + *, + root: Path | None = None, + conformant: bool | None = None, +) -> dict[str, Any]: + """Return a shallow copy with auto-downgraded ``release_status`` when needed.""" + adjusted = dict(manifest) + effective = effective_release_status(manifest, root=root, conformant=conformant) + if effective != str(manifest.get("release_status")): + adjusted["release_status"] = effective + adjusted["_release_status_auto_downgraded"] = True + return adjusted + + +def validate_all_adapter_conformance( + *, + root: Path | None = None, + seal: bool = True, + adapters: tuple[str, ...] | None = None, +) -> list[str]: + """Validate structural + behavioral conformance for all advertised adapters.""" + base = root or ovk_data_root() + failures: list[str] = [] + for adapter_id in adapters or ADVERTISED_ADAPTER_IDS: + failures.extend(behavioral_conformance_failures(adapter_id, root=base, seal=seal)) + return failures + + +def stable_requires_conformance_failures( + manifests: list[dict[str, Any]], + *, + root: Path | None = None, +) -> list[str]: + """Fail when any manifest claims stable without full conformance.""" + base = root or ovk_data_root() + failures: list[str] = [] + for manifest in manifests: + if str(manifest.get("release_status")) != "stable": + continue + checker = str(manifest.get("checker_id") or manifest.get("tool", {}).get("name") or "") + if not checker: + failures.append("stable capability missing checker_id") + continue + if not is_fully_conformant(checker, root=base): + failures.append( + f"{checker}: release_status 'stable' requires full seven-item " + "adapter conformance (OVK-PR4)" + ) + return failures diff --git a/scripts/generate_adapter_conformance.py b/scripts/generate_adapter_conformance.py new file mode 100644 index 0000000..21b5e49 --- /dev/null +++ b/scripts/generate_adapter_conformance.py @@ -0,0 +1,614 @@ +#!/usr/bin/env python +"""One-shot generator for OVK-PR4 adapter conformance fixtures and manifests. + +Safe to re-run: overwrites generated conformance trees and missing example fixtures. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BACKENDS = ROOT / "examples" / "adapters_backends_placeholder" +EXAMPLES = ROOT / "examples" / "backends" + +# Formal backend fixture specs: intent_id + case payloads. +FORMAL = { + "opa": { + "checker_id": "opa", + "dir": "opa", + "pass": {"intent_id": "opa-policy-check", "status": "pass", "violations": []}, + "fail": { + "intent_id": "opa-policy-check", + "status": "fail", + "violations": ["required ovk gate removed from branch protection"], + }, + "malformed": {"intent_id": "opa-policy-check", "malformed": True}, + "timeout": {"intent_id": "opa-policy-check", "timeout": True}, + "unavailable": {"intent_id": "opa-policy-check", "binary_unavailable": True}, + "pass_establishes": ( + "The supplied structured input satisfies the selected OPA/Rego policy " + "under the adapter's data model for this fixture." + ), + "outside_claim": ( + "Does not prove correctness of arbitrary program execution, workflow " + "semantics beyond the policy, or properties outside the selected Rego rules." + ), + }, + "z3": { + "checker_id": "z3", + "dir": "z3", + "pass_path": "examples/auth_regression/input_admin_protected.json", + "fail_path": "examples/auth_regression/input_admin_bypass.json", + "malformed_path": "examples/auth_regression/input_malformed_missing_routes.json", + "timeout": {"timeout": True, "author_type": "ai_agent", "agent": "codex", "task": "timeout"}, + "unavailable": { + "binary_unavailable": True, + "author_type": "ai_agent", + "agent": "codex", + "task": "unavailable", + }, + "pass_establishes": ( + "No counterexample was found for the encoded authorization obligation " + "(or the query polarity recorded in the obligation was satisfied)." + ), + "outside_claim": ( + "Does not prove properties outside the finite abstraction, unsupported " + "theories, or middleware behavior absent from the route encoding." + ), + }, + "cbmc": { + "checker_id": "cbmc", + "dir": "cbmc", + "pass_path": "examples/backends/cbmc_pass.json", + "fail_path": "examples/backends/cbmc_fail.json", + "malformed": {"intent_id": "cbmc-harness-check", "malformed": True}, + "timeout": {"intent_id": "cbmc-harness-check", "timeout": True}, + "unavailable": {"intent_id": "cbmc-harness-check", "binary_unavailable": True}, + "pass_establishes": ( + "Bounded model checking of the supplied harness found no assertion " + "violations within the stated unwind and memory bounds." + ), + "outside_claim": ( + "Bounded only; does not prove unbounded safety, and synthetic harnesses " + "do not compile changed project source into the checked model unless " + "an explicit harness is supplied." + ), + }, + "cedar": { + "checker_id": "cedar", + "dir": "cedar", + "pass_path": "examples/backends/cedar_pass.json", + "fail_path": "examples/backends/cedar_fail.json", + "malformed_path": "examples/backends/cedar_malformed.json", + "timeout": {"intent_id": "cedar-policy-check", "timeout": True}, + "unavailable": {"intent_id": "cedar-policy-check", "binary_unavailable": True}, + "pass_establishes": ( + "The deterministic Cedar-shaped policy oracle reported no violations " + "for the supplied authorization fixture." + ), + "outside_claim": ( + "Native Cedar policy evaluation is not implemented; does not verify " + "runtime middleware behavior." + ), + }, + "tla+": { + "checker_id": "tla+", + "dir": "tla", + "pass_path": "examples/backends/tla_pass.json", + "fail_path": "examples/backends/tla_fail.json", + "malformed_path": "examples/backends/tla_malformed.json", + "timeout": {"intent_id": "tla-state-check", "timeout": True}, + "unavailable": {"intent_id": "tla-state-check", "binary_unavailable": True}, + "pass_establishes": ( + "The deterministic TLA+/state-machine oracle found no skipped required " + "approval states for the supplied fixture." + ), + "outside_claim": ( + "TLC execution is not implemented; does not prove liveness or properties " + "outside the supplied finite state machine." + ), + }, + "kani": { + "checker_id": "kani", + "dir": "kani", + "pass_path": "examples/backends/kani_pass.json", + "fail_path": "examples/backends/kani_fail.json", + "malformed_path": "examples/backends/kani_malformed.json", + "timeout": {"intent_id": "kani-harness-check", "timeout": True}, + "unavailable": {"intent_id": "kani-harness-check", "binary_unavailable": True}, + "pass_establishes": ( + "The deterministic Kani-shaped harness oracle reported no memory-safety " + "or policy violations for the supplied fixture." + ), + "outside_claim": ( + "Native Kani execution is not implemented; does not verify arbitrary Rust " + "beyond the fixture oracle." + ), + }, + "dafny": { + "checker_id": "dafny", + "dir": "dafny", + "pass_path": "examples/backends/dafny_pass.json", + "fail_path": "examples/backends/dafny_fail.json", + "malformed": {"intent_id": "dafny-obligation-check", "malformed": True}, + "timeout": {"intent_id": "dafny-obligation-check", "timeout": True}, + "unavailable": {"intent_id": "dafny-obligation-check", "binary_unavailable": True}, + "pass_establishes": ( + "The deterministic Dafny obligation oracle reported all listed obligations " + "as proved for the supplied fixture." + ), + "outside_claim": ( + "Native Dafny verification is not implemented; does not establish proofs " + "for code outside the fixture obligations." + ), + }, + "verus": { + "checker_id": "verus", + "dir": "verus", + "pass_path": "examples/backends/verus_pass.json", + "fail_path": "examples/backends/verus_fail.json", + "malformed": {"intent_id": "verus-harness-check", "malformed": True}, + "timeout": {"intent_id": "verus-harness-check", "timeout": True}, + "unavailable": {"intent_id": "verus-harness-check", "binary_unavailable": True}, + "pass_establishes": ( + "The deterministic Verus harness oracle reported all listed obligations " + "as proved for the supplied fixture." + ), + "outside_claim": ( + "Native Verus verification is not implemented; does not prove properties " + "outside the fixture harness." + ), + }, + "lean": { + "checker_id": "lean", + "dir": "lean", + "pass_path": "examples/backends/lean_pass.json", + "fail_path": "examples/backends/lean_fail.json", + "malformed": {"intent_id": "lean-proof-check", "malformed": True}, + "timeout": {"intent_id": "lean-proof-check", "timeout": True}, + "unavailable": {"intent_id": "lean-proof-check", "binary_unavailable": True}, + "pass_establishes": ( + "The deterministic Lean proof oracle reported all listed obligations as " + "proved for the supplied fixture." + ), + "outside_claim": ( + "Native Lean checking is not implemented; does not establish theorems " + "outside the fixture obligations." + ), + }, + "alloy": { + "checker_id": "alloy", + "dir": "alloy", + "pass_path": "examples/backends/alloy_pass.json", + "fail_path": "examples/backends/alloy_fail.json", + "malformed": {"intent_id": "alloy-model-check", "malformed": True}, + "timeout": {"intent_id": "alloy-model-check", "timeout": True}, + "unavailable": {"intent_id": "alloy-model-check", "binary_unavailable": True}, + "pass_establishes": ( + "The deterministic Alloy model oracle reported no counterexample instances " + "for the supplied fixture." + ), + "outside_claim": ( + "Native Alloy analysis is not implemented; does not prove properties " + "outside the fixture model scope." + ), + }, +} + +LANES = { + "lane-self-protection": { + "pass_path": "examples/no_agent_self_approval/input_gate_preserved.json", + "fail_path": "examples/no_agent_self_approval/input_gate_removed.json", + "malformed": { + "malformed": True, + "actor": {"type": "ai_agent", "id": "codex"}, + "task": "malformed", + "ovk_gate_name": "ovk-verify", + "changed_files": [], + "before": {"required_checks": []}, + "after": {"required_checks": []}, + }, + "timeout": {"timeout": True}, + "unavailable": {"binary_unavailable": True}, + "capability": { + "capability_id": "lane-self-protection-v1", + "checker_id": "lane-self-protection", + "version": "0.1.0", + "implementation": "ovk-adapter-lane-self-protection", + "input_contract": "Self-protection lane input (actor/before/after required checks).", + "output_contract": "ovk.evidence via self_protection lane evaluator", + "claim_class": "policy_evaluation", + "tool": { + "name": "lane-self-protection", + "adapter": "ovk-adapter-lane-self-protection", + "adapter_version": "0.1.0", + }, + "backend_class": "policy_engine", + "supported_domains": ["ci_cd", "agent_authority"], + "supported_property_kinds": ["safety", "forbidden_configuration"], + "guarantee": { + "type": "policy_evaluation", + "meaning_of_pass": "Required OVK gate remains in branch protection after the change.", + "meaning_of_fail": "The change removes or weakens the required OVK gate.", + "meaning_of_unknown": "Materials were incomplete or the checker could not decide.", + }, + "assumptions": [ + "Branch-protection metadata faithfully represents repository policy.", + ], + "trusted_components": [ + "self-protection lane evaluator", + "optional OPA native path", + ], + "limits": ["Does not prove semantic correctness of unrelated workflow steps."], + "failure_semantics": "Missing or malformed materials map to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "unsupported_semantics": "Does not analyze checks outside the declared OVK gate name.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": False, + }, + "pass_establishes": ( + "The self-protection lane reported that the required OVK verification gate " + "remains present after the change." + ), + "outside_claim": ( + "Does not prove correctness of other required checks, workflow step semantics, " + "or protections outside the declared gate name." + ), + }, + "lane-authorization": { + "pass_path": "examples/auth_regression/input_admin_protected.json", + "fail_path": "examples/auth_regression/input_admin_bypass.json", + "malformed_path": "examples/auth_regression/input_malformed_missing_routes.json", + "timeout": {"timeout": True}, + "unavailable": {"binary_unavailable": True}, + "capability": { + "capability_id": "lane-authorization-v1", + "checker_id": "lane-authorization", + "version": "0.1.0", + "implementation": "ovk-adapter-lane-authorization", + "input_contract": "Authorization route/role abstraction JSON.", + "output_contract": "ovk.evidence via authorization lane evaluator", + "claim_class": "smt_refutation_search", + "tool": { + "name": "lane-authorization", + "adapter": "ovk-adapter-lane-authorization", + "adapter_version": "0.1.0", + }, + "backend_class": "smt_solver", + "supported_domains": ["authorization"], + "supported_property_kinds": ["access_control", "safety", "invariant"], + "guarantee": { + "type": "smt_refutation_search", + "meaning_of_pass": "No unauthorized reachability counterexample was found.", + "meaning_of_fail": "An unauthorized role can reach a protected route.", + "meaning_of_unknown": "Encoding incomplete, solver unknown, or binary unavailable.", + }, + "assumptions": [ + "Route and role abstractions faithfully represent the change under review.", + ], + "trusted_components": [ + "authorization lane evaluator", + "optional Z3 solver", + ], + "limits": [ + "Native Z3 availability is optional; deterministic fallback is weaker.", + ], + "failure_semantics": "Validation failures and solver errors map to unknown/error.", + "timeout_semantics": "unknown", + "unsupported_semantics": "Does not reconstruct frameworks beyond the supplied route abstraction.", + "determinism_status": "tool_dependent", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": False, + }, + "pass_establishes": ( + "The authorization lane found no unauthorized reachability counterexample " + "for the supplied route abstraction." + ), + "outside_claim": ( + "Does not reconstruct frameworks beyond the supplied abstraction or prove " + "properties outside the encoded obligation polarity." + ), + }, + "lane-infrastructure": { + "pass_path": "examples/infrastructure_exposure/input_private_sensitive_resource.json", + "fail_path": "examples/infrastructure_exposure/input_public_sensitive_resource.json", + "malformed": { + "malformed": True, + "author_type": "ai_agent", + "agent": "codex", + "task": "malformed", + "resources": "not-a-list", + }, + "timeout": {"timeout": True}, + "unavailable": {"binary_unavailable": True}, + "capability": { + "capability_id": "lane-infrastructure-v1", + "checker_id": "lane-infrastructure", + "version": "0.1.0", + "implementation": "ovk-adapter-lane-infrastructure", + "input_contract": "Infrastructure exposure graph / resource abstraction JSON.", + "output_contract": "ovk.evidence via infrastructure lane evaluator", + "claim_class": "deterministic_witness", + "tool": { + "name": "lane-infrastructure", + "adapter": "ovk-adapter-lane-infrastructure", + "adapter_version": "0.1.0", + }, + "backend_class": "static_analyzer", + "supported_domains": ["infrastructure"], + "supported_property_kinds": ["data_boundary", "forbidden_configuration", "safety"], + "guarantee": { + "type": "deterministic_witness", + "meaning_of_pass": "No sensitive resource is publicly exposed in the abstraction.", + "meaning_of_fail": "A sensitive resource has a public exposure path.", + "meaning_of_unknown": "The infrastructure abstraction was missing or malformed.", + }, + "assumptions": [ + "Resource sensitivity and exposure edges faithfully represent planned state.", + ], + "trusted_components": ["infrastructure lane evaluator"], + "limits": ["Does not execute Terraform/Kubernetes; evaluates the supplied abstraction only."], + "failure_semantics": "Malformed abstractions map to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "unsupported_semantics": "Does not prove runtime cloud configurations beyond the abstraction.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": False, + }, + "pass_establishes": ( + "The infrastructure lane found no public exposure path to a sensitive resource " + "in the supplied abstraction." + ), + "outside_claim": ( + "Does not execute Terraform or Kubernetes APIs; does not prove live cloud " + "configuration beyond the supplied graph." + ), + }, + "lane-ci-secrets": { + "pass_path": "examples/ci_secrets/input_secrets_safe.json", + "fail_path": "examples/ci_secrets/input_secrets_exposed.json", + "malformed": { + "malformed": True, + "author_type": "ai_agent", + "agent": "codex", + "task": "malformed", + "trust_context": "untrusted_fork_pr", + "workflows": "not-a-list", + }, + "timeout": {"timeout": True}, + "unavailable": {"binary_unavailable": True}, + "capability": { + "capability_id": "lane-ci-secrets-v1", + "checker_id": "lane-ci-secrets", + "version": "0.1.0", + "implementation": "ovk-adapter-lane-ci-secrets", + "input_contract": "CI workflow secret exposure abstraction JSON.", + "output_contract": "ovk.evidence via ci_secrets lane evaluator", + "claim_class": "deterministic_witness", + "tool": { + "name": "lane-ci-secrets", + "adapter": "ovk-adapter-lane-ci-secrets", + "adapter_version": "0.1.0", + }, + "backend_class": "static_analyzer", + "supported_domains": ["ci_cd"], + "supported_property_kinds": ["data_boundary", "forbidden_configuration", "safety"], + "guarantee": { + "type": "deterministic_witness", + "meaning_of_pass": "Secrets are not exposed on untrusted workflow triggers.", + "meaning_of_fail": "A secret is reachable from an untrusted trigger context.", + "meaning_of_unknown": "Workflow abstraction missing or malformed.", + }, + "assumptions": ["Workflow triggers and env bindings faithfully represent the change."], + "trusted_components": ["ci_secrets lane evaluator"], + "limits": ["Does not execute GitHub Actions; evaluates the supplied abstraction only."], + "failure_semantics": "Malformed workflows map to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "unsupported_semantics": "Does not analyze composite actions beyond the supplied steps.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": False, + }, + "pass_establishes": ( + "The ci_secrets lane found no secret exposure on untrusted workflow triggers " + "in the supplied abstraction." + ), + "outside_claim": ( + "Does not execute GitHub Actions runners or analyze composite actions beyond " + "the supplied workflow steps." + ), + }, + "lane-deployment": { + "pass_path": "examples/deployment_state/input_valid_approval_path.json", + "fail_path": "examples/deployment_state/input_skipped_approval.json", + "malformed": { + "malformed": True, + "author_type": "ai_agent", + "agent": "codex", + "task": "malformed", + "states": "not-a-list", + "transitions": [], + }, + "timeout": {"timeout": True}, + "unavailable": {"binary_unavailable": True}, + "capability": { + "capability_id": "lane-deployment-v1", + "checker_id": "lane-deployment", + "version": "0.1.0", + "implementation": "ovk-adapter-lane-deployment", + "input_contract": "Deployment state-machine abstraction JSON.", + "output_contract": "ovk.evidence via deployment lane evaluator", + "claim_class": "deterministic_witness", + "tool": { + "name": "lane-deployment", + "adapter": "ovk-adapter-lane-deployment", + "adapter_version": "0.1.0", + }, + "backend_class": "model_checker", + "supported_domains": ["deployment"], + "supported_property_kinds": ["safety", "invariant", "forbidden_configuration"], + "guarantee": { + "type": "deterministic_witness", + "meaning_of_pass": "Production states are unreachable without required approvals.", + "meaning_of_fail": "A path skips a required approval state before production.", + "meaning_of_unknown": "State machine missing or malformed.", + }, + "assumptions": ["States and transitions faithfully represent the deployment pipeline."], + "trusted_components": ["deployment lane evaluator"], + "limits": ["Does not execute Argo/GitHub Environments; evaluates the supplied machine only."], + "failure_semantics": "Malformed machines map to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "unsupported_semantics": "Does not prove runtime orchestrator behavior beyond the abstraction.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": False, + }, + "pass_establishes": ( + "The deployment lane found no path that reaches a production state while " + "skipping required approval states." + ), + "outside_claim": ( + "Does not execute Argo Rollouts or GitHub Environments; does not prove " + "orchestrator behavior beyond the supplied state machine." + ), + }, +} + + +def write_json(path: Path, payload: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def write_claims(path: Path, *, pass_establishes: str, outside_claim: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "# Adapter conformance claims\n\n" + "## Pass establishes\n\n" + f"{pass_establishes}\n\n" + "## Outside the claim\n\n" + f"{outside_claim}\n", + encoding="utf-8", + ) + + +def ensure_example(name: str, payload: dict) -> str: + path = EXAMPLES / name + write_json(path, payload) + return f"examples/backends/{name}" + + +def build_formal() -> None: + for adapter_id, spec in FORMAL.items(): + fixtures: dict[str, dict[str, str]] = {} + dir_name = spec["dir"] + conf = ROOT / "adapters" / dir_name / "conformance" + + def add_case(case: str, path: str) -> None: + fixtures[case] = {"path": path} + + if "pass_path" in spec: + add_case("pass", spec["pass_path"]) + else: + add_case("pass", ensure_example(f"{dir_name}_pass.json", spec["pass"])) + + if "fail_path" in spec: + add_case("fail", spec["fail_path"]) + else: + add_case("fail", ensure_example(f"{dir_name}_fail.json", spec["fail"])) + + if "malformed_path" in spec: + add_case("malformed", spec["malformed_path"]) + else: + add_case("malformed", ensure_example(f"{dir_name}_malformed.json", spec["malformed"])) + + add_case("timeout", ensure_example(f"{dir_name}_timeout.json", spec["timeout"])) + add_case("unavailable", ensure_example(f"{dir_name}_unavailable.json", spec["unavailable"])) + + write_json( + conf / "manifest.json", + { + "adapter_id": adapter_id, + "kind": "formal_backend", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": fixtures, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim", + }, + }, + ) + write_claims( + conf / "CLAIMS.md", + pass_establishes=spec["pass_establishes"], + outside_claim=spec["outside_claim"], + ) + + # Registry pointer on capability.json + cap_path = ROOT / "adapters" / dir_name / "capability.json" + cap = json.loads(cap_path.read_text(encoding="utf-8")) + cap["conformance"] = {"suite": "conformance/manifest.json"} + write_json(cap_path, cap) + print(f"formal: {adapter_id}") + + +def build_lanes() -> None: + for adapter_id, spec in LANES.items(): + conf = ROOT / "adapters" / adapter_id / "conformance" + fixtures: dict[str, dict[str, str]] = {} + + fixtures["pass"] = {"path": spec["pass_path"]} + fixtures["fail"] = {"path": spec["fail_path"]} + if "malformed_path" in spec: + fixtures["malformed"] = {"path": spec["malformed_path"]} + else: + local = conf / "fixtures" / "malformed.json" + write_json(local, spec["malformed"]) + fixtures["malformed"] = {"path": "fixtures/malformed.json"} + + for case in ("timeout", "unavailable"): + local = conf / "fixtures" / f"{case}.json" + write_json(local, spec[case]) + fixtures[case] = {"path": f"fixtures/{case}.json"} + + write_json( + conf / "manifest.json", + { + "adapter_id": adapter_id, + "kind": "lane_adapter", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": fixtures, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim", + }, + }, + ) + write_claims( + conf / "CLAIMS.md", + pass_establishes=spec["pass_establishes"], + outside_claim=spec["outside_claim"], + ) + cap = dict(spec["capability"]) + cap["conformance"] = {"suite": "conformance/manifest.json"} + write_json(ROOT / "adapters" / adapter_id / "capability.json", cap) + print(f"lane: {adapter_id}") + + +def main() -> None: + EXAMPLES.mkdir(parents=True, exist_ok=True) + build_formal() + build_lanes() + print("done") + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_adapter_conformance.py b/scripts/validate_adapter_conformance.py new file mode 100644 index 0000000..15452f3 --- /dev/null +++ b/scripts/validate_adapter_conformance.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +"""Validate the seven-item adapter conformance matrix (OVK-PR4 / OVK-05).""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from ovk.core.adapter_conformance import ( # noqa: E402 + ADVERTISED_ADAPTER_IDS, + is_fully_conformant, + validate_all_adapter_conformance, +) +from ovk.core.capabilities import CapabilityRegistry # noqa: E402 +from ovk.core.adapter_conformance import stable_requires_conformance_failures # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Validate OVK adapter conformance fixtures (pass/fail/malformed/timeout/unavailable + docs)" + ) + parser.add_argument("--repo-root", type=Path, default=ROOT) + parser.add_argument( + "--adapter", + action="append", + dest="adapters", + help="Validate one advertised adapter id (repeatable). Defaults to all advertised adapters.", + ) + parser.add_argument( + "--structural-only", + action="store_true", + help="Only check that the seven artifacts exist (skip evaluation/seal).", + ) + parser.add_argument( + "--no-seal", + action="store_true", + help="Evaluate fixtures but skip evidence integrity sealing.", + ) + args = parser.parse_args() + root = args.repo_root.resolve() + adapters = tuple(args.adapters) if args.adapters else ADVERTISED_ADAPTER_IDS + + if args.structural_only: + from ovk.core.adapter_conformance import structural_conformance_failures + + failures: list[str] = [] + for adapter_id in adapters: + failures.extend(structural_conformance_failures(adapter_id, root=root)) + else: + failures = validate_all_adapter_conformance( + root=root, + seal=not args.no_seal, + adapters=adapters, + ) + + # Also enforce: no capability.json may claim stable without conformance. + registry = CapabilityRegistry.from_directory(root / "adapters", validate=False) + failures.extend(stable_requires_conformance_failures(registry.all(), root=root)) + + for failure in failures: + print(failure) + if failures: + return 1 + + conformant = sum(1 for adapter_id in adapters if is_fully_conformant(adapter_id, root=root, seal=False)) + print( + f"OVK adapter conformance passed " + f"({len(adapters)} adapters checked, {conformant} structurally complete)" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_adapter_conformance.py b/tests/test_adapter_conformance.py new file mode 100644 index 0000000..90e501d --- /dev/null +++ b/tests/test_adapter_conformance.py @@ -0,0 +1,96 @@ +"""Tests for the seven-item adapter conformance matrix (OVK-PR4 / OVK-05).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from ovk.core.adapter_conformance import ( + ADVERTISED_ADAPTER_IDS, + FORMAL_BACKEND_IDS, + LANE_ADAPTER_IDS, + apply_release_status_honesty, + behavioral_conformance_failures, + effective_release_status, + evaluate_conformance_fixture, + is_fully_conformant, + structural_conformance_failures, + validate_all_adapter_conformance, +) +from ovk.core.capabilities import CapabilityRegistry +from ovk.core.evidence_integrity import seal_evidence, verify_evidence_digest +from ovk.core.json_io import read_json_file +from scripts.validate_adapter_conformance import main as validate_main + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_advertised_adapter_count() -> None: + assert len(ADVERTISED_ADAPTER_IDS) == 15 + assert len(FORMAL_BACKEND_IDS) == 10 + assert len(LANE_ADAPTER_IDS) == 5 + + +@pytest.mark.parametrize("adapter_id", ADVERTISED_ADAPTER_IDS) +def test_structural_conformance_complete(adapter_id: str) -> None: + assert structural_conformance_failures(adapter_id, root=ROOT) == [] + + +@pytest.mark.parametrize("adapter_id", ADVERTISED_ADAPTER_IDS) +def test_behavioral_conformance_with_integrity(adapter_id: str) -> None: + failures = behavioral_conformance_failures(adapter_id, root=ROOT, seal=True) + assert failures == [], failures + + +def test_validate_all_adapter_conformance() -> None: + assert validate_all_adapter_conformance(root=ROOT, seal=True) == [] + + +def test_validate_script_exits_zero(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("sys.argv", ["validate_adapter_conformance.py"]) + assert validate_main() == 0 + + +def test_stable_auto_downgrade_when_not_conformant(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "ovk.core.adapter_conformance.is_fully_conformant", + lambda *_args, **_kwargs: False, + ) + manifest = { + "checker_id": "opa", + "release_status": "stable", + "native_execution": True, + } + assert effective_release_status(manifest, root=ROOT, conformant=False) == "preview" + adjusted = apply_release_status_honesty(manifest, root=ROOT, conformant=False) + assert adjusted["release_status"] == "preview" + assert adjusted.get("_release_status_auto_downgraded") is True + + +def test_no_capability_claims_stable_without_gate() -> None: + registry = CapabilityRegistry.from_directory(ROOT / "adapters", validate=True) + for manifest in registry.all(): + if manifest.get("release_status") == "stable": + assert is_fully_conformant(str(manifest["checker_id"]), root=ROOT) + + +def test_opa_pass_fixture_seals() -> None: + data = read_json_file(ROOT / "examples" / "backends" / "opa_pass.json") + evidence = evaluate_conformance_fixture("opa", data) + sealed = seal_evidence(evidence) + assert verify_evidence_digest(sealed) + + +def test_kani_timeout_fixture_is_unknown() -> None: + data = read_json_file(ROOT / "examples" / "backends" / "kani_timeout.json") + evidence = evaluate_conformance_fixture("kani", data) + assert evidence.backend_claims[0].status.value == "unknown" + + +def test_capability_pointer_present_for_formal_backends() -> None: + for checker in FORMAL_BACKEND_IDS: + dirname = "tla" if checker == "tla+" else checker + payload = read_json_file(ROOT / "adapters" / dirname / "capability.json") + assert payload.get("conformance", {}).get("suite") == "conformance/manifest.json" From 140029779a84ad8118a5c8f3dcbe777139eb8d24 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:09:19 -0700 Subject: [PATCH 06/19] Register lane adapters with conformance fixtures (OVK-PR4). Publish capability and conformance packages for authorization, CI-secrets, deployment, infrastructure, and self-protection lanes so deterministic runners share the same failure taxonomy as native backends. --- adapters/lane-authorization/capability.json | 49 +++++++++++++ .../lane-authorization/conformance/CLAIMS.md | 9 +++ .../conformance/fixtures/timeout.json | 3 + .../conformance/fixtures/unavailable.json | 3 + .../conformance/manifest.json | 26 +++++++ adapters/lane-ci-secrets/capability.json | 48 +++++++++++++ .../lane-ci-secrets/conformance/CLAIMS.md | 9 +++ .../conformance/fixtures/malformed.json | 8 +++ .../conformance/fixtures/timeout.json | 3 + .../conformance/fixtures/unavailable.json | 3 + .../lane-ci-secrets/conformance/manifest.json | 26 +++++++ adapters/lane-deployment/capability.json | 48 +++++++++++++ .../lane-deployment/conformance/CLAIMS.md | 9 +++ .../conformance/fixtures/malformed.json | 8 +++ .../conformance/fixtures/timeout.json | 3 + .../conformance/fixtures/unavailable.json | 3 + .../lane-deployment/conformance/manifest.json | 26 +++++++ adapters/lane-infrastructure/capability.json | 48 +++++++++++++ .../lane-infrastructure/conformance/CLAIMS.md | 9 +++ .../conformance/fixtures/malformed.json | 7 ++ .../conformance/fixtures/timeout.json | 3 + .../conformance/fixtures/unavailable.json | 3 + .../conformance/manifest.json | 26 +++++++ adapters/lane-self-protection/capability.json | 49 +++++++++++++ .../conformance/CLAIMS.md | 9 +++ .../conformance/fixtures/malformed.json | 16 +++++ .../conformance/fixtures/timeout.json | 3 + .../conformance/fixtures/unavailable.json | 3 + .../conformance/manifest.json | 26 +++++++ ovk/adapters/backend_factory.py | 7 +- ovk/adapters/cedar/deterministic.py | 12 +++- ovk/adapters/kani/deterministic.py | 12 +++- ovk/adapters/tla/deterministic.py | 15 +++- ovk/adapters/wave2_oracle.py | 38 ++++++++-- ovk/core/adapter_runtime.py | 41 ++++++++--- ovk/core/execution_models.py | 69 ++++++++++++++++++- schemas/backend.execution.schema.json | 25 ++++++- tests/test_execution_models.py | 4 ++ 38 files changed, 684 insertions(+), 25 deletions(-) create mode 100644 adapters/lane-authorization/capability.json create mode 100644 adapters/lane-authorization/conformance/CLAIMS.md create mode 100644 adapters/lane-authorization/conformance/fixtures/timeout.json create mode 100644 adapters/lane-authorization/conformance/fixtures/unavailable.json create mode 100644 adapters/lane-authorization/conformance/manifest.json create mode 100644 adapters/lane-ci-secrets/capability.json create mode 100644 adapters/lane-ci-secrets/conformance/CLAIMS.md create mode 100644 adapters/lane-ci-secrets/conformance/fixtures/malformed.json create mode 100644 adapters/lane-ci-secrets/conformance/fixtures/timeout.json create mode 100644 adapters/lane-ci-secrets/conformance/fixtures/unavailable.json create mode 100644 adapters/lane-ci-secrets/conformance/manifest.json create mode 100644 adapters/lane-deployment/capability.json create mode 100644 adapters/lane-deployment/conformance/CLAIMS.md create mode 100644 adapters/lane-deployment/conformance/fixtures/malformed.json create mode 100644 adapters/lane-deployment/conformance/fixtures/timeout.json create mode 100644 adapters/lane-deployment/conformance/fixtures/unavailable.json create mode 100644 adapters/lane-deployment/conformance/manifest.json create mode 100644 adapters/lane-infrastructure/capability.json create mode 100644 adapters/lane-infrastructure/conformance/CLAIMS.md create mode 100644 adapters/lane-infrastructure/conformance/fixtures/malformed.json create mode 100644 adapters/lane-infrastructure/conformance/fixtures/timeout.json create mode 100644 adapters/lane-infrastructure/conformance/fixtures/unavailable.json create mode 100644 adapters/lane-infrastructure/conformance/manifest.json create mode 100644 adapters/lane-self-protection/capability.json create mode 100644 adapters/lane-self-protection/conformance/CLAIMS.md create mode 100644 adapters/lane-self-protection/conformance/fixtures/malformed.json create mode 100644 adapters/lane-self-protection/conformance/fixtures/timeout.json create mode 100644 adapters/lane-self-protection/conformance/fixtures/unavailable.json create mode 100644 adapters/lane-self-protection/conformance/manifest.json diff --git a/adapters/lane-authorization/capability.json b/adapters/lane-authorization/capability.json new file mode 100644 index 0000000..7229b68 --- /dev/null +++ b/adapters/lane-authorization/capability.json @@ -0,0 +1,49 @@ +{ + "capability_id": "lane-authorization-v1", + "checker_id": "lane-authorization", + "version": "0.1.0", + "implementation": "ovk-adapter-lane-authorization", + "input_contract": "Authorization route/role abstraction JSON.", + "output_contract": "ovk.evidence via authorization lane evaluator", + "claim_class": "smt_refutation_search", + "tool": { + "name": "lane-authorization", + "adapter": "ovk-adapter-lane-authorization", + "adapter_version": "0.1.0" + }, + "backend_class": "smt_solver", + "supported_domains": [ + "authorization" + ], + "supported_property_kinds": [ + "access_control", + "safety", + "invariant" + ], + "guarantee": { + "type": "smt_refutation_search", + "meaning_of_pass": "No unauthorized reachability counterexample was found.", + "meaning_of_fail": "An unauthorized role can reach a protected route.", + "meaning_of_unknown": "Encoding incomplete, solver unknown, or binary unavailable." + }, + "assumptions": [ + "Route and role abstractions faithfully represent the change under review." + ], + "trusted_components": [ + "authorization lane evaluator", + "optional Z3 solver" + ], + "limits": [ + "Native Z3 availability is optional; deterministic fallback is weaker." + ], + "failure_semantics": "Validation failures and solver errors map to unknown/error.", + "timeout_semantics": "unknown", + "unsupported_semantics": "Does not reconstruct frameworks beyond the supplied route abstraction.", + "determinism_status": "tool_dependent", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, + "conformance": { + "suite": "conformance/manifest.json" + } +} diff --git a/adapters/lane-authorization/conformance/CLAIMS.md b/adapters/lane-authorization/conformance/CLAIMS.md new file mode 100644 index 0000000..35f11c9 --- /dev/null +++ b/adapters/lane-authorization/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The authorization lane found no unauthorized reachability counterexample for the supplied route abstraction. + +## Outside the claim + +Does not reconstruct frameworks beyond the supplied abstraction or prove properties outside the encoded obligation polarity. diff --git a/adapters/lane-authorization/conformance/fixtures/timeout.json b/adapters/lane-authorization/conformance/fixtures/timeout.json new file mode 100644 index 0000000..1fba05c --- /dev/null +++ b/adapters/lane-authorization/conformance/fixtures/timeout.json @@ -0,0 +1,3 @@ +{ + "timeout": true +} diff --git a/adapters/lane-authorization/conformance/fixtures/unavailable.json b/adapters/lane-authorization/conformance/fixtures/unavailable.json new file mode 100644 index 0000000..4bf2983 --- /dev/null +++ b/adapters/lane-authorization/conformance/fixtures/unavailable.json @@ -0,0 +1,3 @@ +{ + "binary_unavailable": true +} diff --git a/adapters/lane-authorization/conformance/manifest.json b/adapters/lane-authorization/conformance/manifest.json new file mode 100644 index 0000000..3a38a79 --- /dev/null +++ b/adapters/lane-authorization/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "lane-authorization", + "kind": "lane_adapter", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/auth_regression/input_admin_protected.json" + }, + "fail": { + "path": "examples/auth_regression/input_admin_bypass.json" + }, + "malformed": { + "path": "examples/auth_regression/input_malformed_missing_routes.json" + }, + "timeout": { + "path": "fixtures/timeout.json" + }, + "unavailable": { + "path": "fixtures/unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/lane-ci-secrets/capability.json b/adapters/lane-ci-secrets/capability.json new file mode 100644 index 0000000..fa9041a --- /dev/null +++ b/adapters/lane-ci-secrets/capability.json @@ -0,0 +1,48 @@ +{ + "capability_id": "lane-ci-secrets-v1", + "checker_id": "lane-ci-secrets", + "version": "0.1.0", + "implementation": "ovk-adapter-lane-ci-secrets", + "input_contract": "CI workflow secret exposure abstraction JSON.", + "output_contract": "ovk.evidence via ci_secrets lane evaluator", + "claim_class": "deterministic_witness", + "tool": { + "name": "lane-ci-secrets", + "adapter": "ovk-adapter-lane-ci-secrets", + "adapter_version": "0.1.0" + }, + "backend_class": "static_analyzer", + "supported_domains": [ + "ci_cd" + ], + "supported_property_kinds": [ + "data_boundary", + "forbidden_configuration", + "safety" + ], + "guarantee": { + "type": "deterministic_witness", + "meaning_of_pass": "Secrets are not exposed on untrusted workflow triggers.", + "meaning_of_fail": "A secret is reachable from an untrusted trigger context.", + "meaning_of_unknown": "Workflow abstraction missing or malformed." + }, + "assumptions": [ + "Workflow triggers and env bindings faithfully represent the change." + ], + "trusted_components": [ + "ci_secrets lane evaluator" + ], + "limits": [ + "Does not execute GitHub Actions; evaluates the supplied abstraction only." + ], + "failure_semantics": "Malformed workflows map to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "unsupported_semantics": "Does not analyze composite actions beyond the supplied steps.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, + "conformance": { + "suite": "conformance/manifest.json" + } +} diff --git a/adapters/lane-ci-secrets/conformance/CLAIMS.md b/adapters/lane-ci-secrets/conformance/CLAIMS.md new file mode 100644 index 0000000..c08f0d3 --- /dev/null +++ b/adapters/lane-ci-secrets/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The ci_secrets lane found no secret exposure on untrusted workflow triggers in the supplied abstraction. + +## Outside the claim + +Does not execute GitHub Actions runners or analyze composite actions beyond the supplied workflow steps. diff --git a/adapters/lane-ci-secrets/conformance/fixtures/malformed.json b/adapters/lane-ci-secrets/conformance/fixtures/malformed.json new file mode 100644 index 0000000..3ece781 --- /dev/null +++ b/adapters/lane-ci-secrets/conformance/fixtures/malformed.json @@ -0,0 +1,8 @@ +{ + "malformed": true, + "author_type": "ai_agent", + "agent": "codex", + "task": "malformed", + "trust_context": "untrusted_fork_pr", + "workflows": "not-a-list" +} diff --git a/adapters/lane-ci-secrets/conformance/fixtures/timeout.json b/adapters/lane-ci-secrets/conformance/fixtures/timeout.json new file mode 100644 index 0000000..1fba05c --- /dev/null +++ b/adapters/lane-ci-secrets/conformance/fixtures/timeout.json @@ -0,0 +1,3 @@ +{ + "timeout": true +} diff --git a/adapters/lane-ci-secrets/conformance/fixtures/unavailable.json b/adapters/lane-ci-secrets/conformance/fixtures/unavailable.json new file mode 100644 index 0000000..4bf2983 --- /dev/null +++ b/adapters/lane-ci-secrets/conformance/fixtures/unavailable.json @@ -0,0 +1,3 @@ +{ + "binary_unavailable": true +} diff --git a/adapters/lane-ci-secrets/conformance/manifest.json b/adapters/lane-ci-secrets/conformance/manifest.json new file mode 100644 index 0000000..fb62cf0 --- /dev/null +++ b/adapters/lane-ci-secrets/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "lane-ci-secrets", + "kind": "lane_adapter", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/ci_secrets/input_secrets_safe.json" + }, + "fail": { + "path": "examples/ci_secrets/input_secrets_exposed.json" + }, + "malformed": { + "path": "fixtures/malformed.json" + }, + "timeout": { + "path": "fixtures/timeout.json" + }, + "unavailable": { + "path": "fixtures/unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/lane-deployment/capability.json b/adapters/lane-deployment/capability.json new file mode 100644 index 0000000..cb61bc9 --- /dev/null +++ b/adapters/lane-deployment/capability.json @@ -0,0 +1,48 @@ +{ + "capability_id": "lane-deployment-v1", + "checker_id": "lane-deployment", + "version": "0.1.0", + "implementation": "ovk-adapter-lane-deployment", + "input_contract": "Deployment state-machine abstraction JSON.", + "output_contract": "ovk.evidence via deployment lane evaluator", + "claim_class": "deterministic_witness", + "tool": { + "name": "lane-deployment", + "adapter": "ovk-adapter-lane-deployment", + "adapter_version": "0.1.0" + }, + "backend_class": "model_checker", + "supported_domains": [ + "deployment" + ], + "supported_property_kinds": [ + "safety", + "invariant", + "forbidden_configuration" + ], + "guarantee": { + "type": "deterministic_witness", + "meaning_of_pass": "Production states are unreachable without required approvals.", + "meaning_of_fail": "A path skips a required approval state before production.", + "meaning_of_unknown": "State machine missing or malformed." + }, + "assumptions": [ + "States and transitions faithfully represent the deployment pipeline." + ], + "trusted_components": [ + "deployment lane evaluator" + ], + "limits": [ + "Does not execute Argo/GitHub Environments; evaluates the supplied machine only." + ], + "failure_semantics": "Malformed machines map to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "unsupported_semantics": "Does not prove runtime orchestrator behavior beyond the abstraction.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, + "conformance": { + "suite": "conformance/manifest.json" + } +} diff --git a/adapters/lane-deployment/conformance/CLAIMS.md b/adapters/lane-deployment/conformance/CLAIMS.md new file mode 100644 index 0000000..4a1753d --- /dev/null +++ b/adapters/lane-deployment/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The deployment lane found no path that reaches a production state while skipping required approval states. + +## Outside the claim + +Does not execute Argo Rollouts or GitHub Environments; does not prove orchestrator behavior beyond the supplied state machine. diff --git a/adapters/lane-deployment/conformance/fixtures/malformed.json b/adapters/lane-deployment/conformance/fixtures/malformed.json new file mode 100644 index 0000000..19a2dfd --- /dev/null +++ b/adapters/lane-deployment/conformance/fixtures/malformed.json @@ -0,0 +1,8 @@ +{ + "malformed": true, + "author_type": "ai_agent", + "agent": "codex", + "task": "malformed", + "states": "not-a-list", + "transitions": [] +} diff --git a/adapters/lane-deployment/conformance/fixtures/timeout.json b/adapters/lane-deployment/conformance/fixtures/timeout.json new file mode 100644 index 0000000..1fba05c --- /dev/null +++ b/adapters/lane-deployment/conformance/fixtures/timeout.json @@ -0,0 +1,3 @@ +{ + "timeout": true +} diff --git a/adapters/lane-deployment/conformance/fixtures/unavailable.json b/adapters/lane-deployment/conformance/fixtures/unavailable.json new file mode 100644 index 0000000..4bf2983 --- /dev/null +++ b/adapters/lane-deployment/conformance/fixtures/unavailable.json @@ -0,0 +1,3 @@ +{ + "binary_unavailable": true +} diff --git a/adapters/lane-deployment/conformance/manifest.json b/adapters/lane-deployment/conformance/manifest.json new file mode 100644 index 0000000..f141c64 --- /dev/null +++ b/adapters/lane-deployment/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "lane-deployment", + "kind": "lane_adapter", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/deployment_state/input_valid_approval_path.json" + }, + "fail": { + "path": "examples/deployment_state/input_skipped_approval.json" + }, + "malformed": { + "path": "fixtures/malformed.json" + }, + "timeout": { + "path": "fixtures/timeout.json" + }, + "unavailable": { + "path": "fixtures/unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/lane-infrastructure/capability.json b/adapters/lane-infrastructure/capability.json new file mode 100644 index 0000000..9fabf96 --- /dev/null +++ b/adapters/lane-infrastructure/capability.json @@ -0,0 +1,48 @@ +{ + "capability_id": "lane-infrastructure-v1", + "checker_id": "lane-infrastructure", + "version": "0.1.0", + "implementation": "ovk-adapter-lane-infrastructure", + "input_contract": "Infrastructure exposure graph / resource abstraction JSON.", + "output_contract": "ovk.evidence via infrastructure lane evaluator", + "claim_class": "deterministic_witness", + "tool": { + "name": "lane-infrastructure", + "adapter": "ovk-adapter-lane-infrastructure", + "adapter_version": "0.1.0" + }, + "backend_class": "static_analyzer", + "supported_domains": [ + "infrastructure" + ], + "supported_property_kinds": [ + "data_boundary", + "forbidden_configuration", + "safety" + ], + "guarantee": { + "type": "deterministic_witness", + "meaning_of_pass": "No sensitive resource is publicly exposed in the abstraction.", + "meaning_of_fail": "A sensitive resource has a public exposure path.", + "meaning_of_unknown": "The infrastructure abstraction was missing or malformed." + }, + "assumptions": [ + "Resource sensitivity and exposure edges faithfully represent planned state." + ], + "trusted_components": [ + "infrastructure lane evaluator" + ], + "limits": [ + "Does not execute Terraform/Kubernetes; evaluates the supplied abstraction only." + ], + "failure_semantics": "Malformed abstractions map to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "unsupported_semantics": "Does not prove runtime cloud configurations beyond the abstraction.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, + "conformance": { + "suite": "conformance/manifest.json" + } +} diff --git a/adapters/lane-infrastructure/conformance/CLAIMS.md b/adapters/lane-infrastructure/conformance/CLAIMS.md new file mode 100644 index 0000000..f274e3c --- /dev/null +++ b/adapters/lane-infrastructure/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The infrastructure lane found no public exposure path to a sensitive resource in the supplied abstraction. + +## Outside the claim + +Does not execute Terraform or Kubernetes APIs; does not prove live cloud configuration beyond the supplied graph. diff --git a/adapters/lane-infrastructure/conformance/fixtures/malformed.json b/adapters/lane-infrastructure/conformance/fixtures/malformed.json new file mode 100644 index 0000000..3dc8497 --- /dev/null +++ b/adapters/lane-infrastructure/conformance/fixtures/malformed.json @@ -0,0 +1,7 @@ +{ + "malformed": true, + "author_type": "ai_agent", + "agent": "codex", + "task": "malformed", + "resources": "not-a-list" +} diff --git a/adapters/lane-infrastructure/conformance/fixtures/timeout.json b/adapters/lane-infrastructure/conformance/fixtures/timeout.json new file mode 100644 index 0000000..1fba05c --- /dev/null +++ b/adapters/lane-infrastructure/conformance/fixtures/timeout.json @@ -0,0 +1,3 @@ +{ + "timeout": true +} diff --git a/adapters/lane-infrastructure/conformance/fixtures/unavailable.json b/adapters/lane-infrastructure/conformance/fixtures/unavailable.json new file mode 100644 index 0000000..4bf2983 --- /dev/null +++ b/adapters/lane-infrastructure/conformance/fixtures/unavailable.json @@ -0,0 +1,3 @@ +{ + "binary_unavailable": true +} diff --git a/adapters/lane-infrastructure/conformance/manifest.json b/adapters/lane-infrastructure/conformance/manifest.json new file mode 100644 index 0000000..74daa0a --- /dev/null +++ b/adapters/lane-infrastructure/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "lane-infrastructure", + "kind": "lane_adapter", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/infrastructure_exposure/input_private_sensitive_resource.json" + }, + "fail": { + "path": "examples/infrastructure_exposure/input_public_sensitive_resource.json" + }, + "malformed": { + "path": "fixtures/malformed.json" + }, + "timeout": { + "path": "fixtures/timeout.json" + }, + "unavailable": { + "path": "fixtures/unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/adapters/lane-self-protection/capability.json b/adapters/lane-self-protection/capability.json new file mode 100644 index 0000000..a2a43ff --- /dev/null +++ b/adapters/lane-self-protection/capability.json @@ -0,0 +1,49 @@ +{ + "capability_id": "lane-self-protection-v1", + "checker_id": "lane-self-protection", + "version": "0.1.0", + "implementation": "ovk-adapter-lane-self-protection", + "input_contract": "Self-protection lane input (actor/before/after required checks).", + "output_contract": "ovk.evidence via self_protection lane evaluator", + "claim_class": "policy_evaluation", + "tool": { + "name": "lane-self-protection", + "adapter": "ovk-adapter-lane-self-protection", + "adapter_version": "0.1.0" + }, + "backend_class": "policy_engine", + "supported_domains": [ + "ci_cd", + "agent_authority" + ], + "supported_property_kinds": [ + "safety", + "forbidden_configuration" + ], + "guarantee": { + "type": "policy_evaluation", + "meaning_of_pass": "Required OVK gate remains in branch protection after the change.", + "meaning_of_fail": "The change removes or weakens the required OVK gate.", + "meaning_of_unknown": "Materials were incomplete or the checker could not decide." + }, + "assumptions": [ + "Branch-protection metadata faithfully represents repository policy." + ], + "trusted_components": [ + "self-protection lane evaluator", + "optional OPA native path" + ], + "limits": [ + "Does not prove semantic correctness of unrelated workflow steps." + ], + "failure_semantics": "Missing or malformed materials map to unknown; evaluator errors map to error.", + "timeout_semantics": "unknown", + "unsupported_semantics": "Does not analyze checks outside the declared OVK gate name.", + "determinism_status": "deterministic", + "release_status": "experimental", + "owner": "ovk-maintainers", + "native_execution": false, + "conformance": { + "suite": "conformance/manifest.json" + } +} diff --git a/adapters/lane-self-protection/conformance/CLAIMS.md b/adapters/lane-self-protection/conformance/CLAIMS.md new file mode 100644 index 0000000..5b5e79d --- /dev/null +++ b/adapters/lane-self-protection/conformance/CLAIMS.md @@ -0,0 +1,9 @@ +# Adapter conformance claims + +## Pass establishes + +The self-protection lane reported that the required OVK verification gate remains present after the change. + +## Outside the claim + +Does not prove correctness of other required checks, workflow step semantics, or protections outside the declared gate name. diff --git a/adapters/lane-self-protection/conformance/fixtures/malformed.json b/adapters/lane-self-protection/conformance/fixtures/malformed.json new file mode 100644 index 0000000..9a7246a --- /dev/null +++ b/adapters/lane-self-protection/conformance/fixtures/malformed.json @@ -0,0 +1,16 @@ +{ + "malformed": true, + "actor": { + "type": "ai_agent", + "id": "codex" + }, + "task": "malformed", + "ovk_gate_name": "ovk-verify", + "changed_files": [], + "before": { + "required_checks": [] + }, + "after": { + "required_checks": [] + } +} diff --git a/adapters/lane-self-protection/conformance/fixtures/timeout.json b/adapters/lane-self-protection/conformance/fixtures/timeout.json new file mode 100644 index 0000000..1fba05c --- /dev/null +++ b/adapters/lane-self-protection/conformance/fixtures/timeout.json @@ -0,0 +1,3 @@ +{ + "timeout": true +} diff --git a/adapters/lane-self-protection/conformance/fixtures/unavailable.json b/adapters/lane-self-protection/conformance/fixtures/unavailable.json new file mode 100644 index 0000000..4bf2983 --- /dev/null +++ b/adapters/lane-self-protection/conformance/fixtures/unavailable.json @@ -0,0 +1,3 @@ +{ + "binary_unavailable": true +} diff --git a/adapters/lane-self-protection/conformance/manifest.json b/adapters/lane-self-protection/conformance/manifest.json new file mode 100644 index 0000000..74e70fd --- /dev/null +++ b/adapters/lane-self-protection/conformance/manifest.json @@ -0,0 +1,26 @@ +{ + "adapter_id": "lane-self-protection", + "kind": "lane_adapter", + "schema_version": "ovk.adapter_conformance.v1", + "fixtures": { + "pass": { + "path": "examples/no_agent_self_approval/input_gate_preserved.json" + }, + "fail": { + "path": "examples/no_agent_self_approval/input_gate_removed.json" + }, + "malformed": { + "path": "fixtures/malformed.json" + }, + "timeout": { + "path": "fixtures/timeout.json" + }, + "unavailable": { + "path": "fixtures/unavailable.json" + } + }, + "docs": { + "pass_establishes": "CLAIMS.md#pass-establishes", + "outside_claim": "CLAIMS.md#outside-the-claim" + } +} diff --git a/ovk/adapters/backend_factory.py b/ovk/adapters/backend_factory.py index 56effec..b0b2ca7 100644 --- a/ovk/adapters/backend_factory.py +++ b/ovk/adapters/backend_factory.py @@ -8,11 +8,14 @@ def _policy_pass(data: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]: + from ovk.adapters.wave2_oracle import classify_conformance_flags + + early = classify_conformance_flags(data) + if early is not None: + return early violations = data.get("violations", []) if violations: return "fail", [{"summary": str(violations[0]), "failure_mode": "policy_violation"}] - if data.get("malformed"): - return "unknown", [{"summary": "malformed input", "failure_mode": "malformed_input"}] return "pass", [] diff --git a/ovk/adapters/cedar/deterministic.py b/ovk/adapters/cedar/deterministic.py index 6148643..d3dfdcb 100644 --- a/ovk/adapters/cedar/deterministic.py +++ b/ovk/adapters/cedar/deterministic.py @@ -4,11 +4,19 @@ from typing import Any +from ovk.adapters.wave2_oracle import classify_conformance_flags + def evaluate_cedar_input(data: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]: """Evaluate Cedar-shaped IAM policy input with a conservative oracle.""" - if data.get("malformed"): - return "unknown", [{"summary": "Malformed Cedar/IAM input.", "failure_mode": "malformed_input"}] + early = classify_conformance_flags(data) + if early is not None: + status, counterexamples = early + if data.get("malformed"): + counterexamples = [ + {"summary": "Malformed Cedar/IAM input.", "failure_mode": "malformed_input"} + ] + return status, counterexamples violations = list(data.get("violations", [])) policies = data.get("policies", []) diff --git a/ovk/adapters/kani/deterministic.py b/ovk/adapters/kani/deterministic.py index 35d7931..bebc5fb 100644 --- a/ovk/adapters/kani/deterministic.py +++ b/ovk/adapters/kani/deterministic.py @@ -4,11 +4,19 @@ from typing import Any +from ovk.adapters.wave2_oracle import classify_conformance_flags + def evaluate_kani_input(data: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]: """Evaluate Rust harness input with a conservative safety oracle.""" - if data.get("malformed"): - return "unknown", [{"summary": "Malformed Rust harness input.", "failure_mode": "malformed_input"}] + early = classify_conformance_flags(data) + if early is not None: + status, counterexamples = early + if data.get("malformed"): + counterexamples = [ + {"summary": "Malformed Rust harness input.", "failure_mode": "malformed_input"} + ] + return status, counterexamples violations = [str(item) for item in data.get("violations", [])] unsafe_ops = data.get("unsafe_operations", []) diff --git a/ovk/adapters/tla/deterministic.py b/ovk/adapters/tla/deterministic.py index b41ca63..943bc6c 100644 --- a/ovk/adapters/tla/deterministic.py +++ b/ovk/adapters/tla/deterministic.py @@ -4,11 +4,22 @@ from typing import Any +from ovk.adapters.wave2_oracle import classify_conformance_flags + def evaluate_tla_input(data: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]: """Evaluate deployment state-machine input with a bounded oracle.""" - if data.get("malformed"): - return "unknown", [{"summary": "Malformed state machine.", "failure_mode": "malformed_state_machine"}] + early = classify_conformance_flags(data) + if early is not None: + status, counterexamples = early + if data.get("malformed"): + counterexamples = [ + { + "summary": "Malformed state machine.", + "failure_mode": "malformed_state_machine", + } + ] + return status, counterexamples if data.get("skipped_states"): return "fail", [ { diff --git a/ovk/adapters/wave2_oracle.py b/ovk/adapters/wave2_oracle.py index 8904474..ba6753c 100644 --- a/ovk/adapters/wave2_oracle.py +++ b/ovk/adapters/wave2_oracle.py @@ -5,14 +5,43 @@ from typing import Any +def classify_conformance_flags( + data: dict[str, Any], +) -> tuple[str, list[dict[str, Any]]] | None: + """Return an early outcome for shared conformance fixture flags, else None. + + Recognized flags (checked in order): + - ``malformed`` → unknown / malformed_input + - ``timeout`` → unknown / timeout + - ``binary_unavailable`` or ``unavailable`` → unknown / binary_unavailable + """ + if data.get("malformed"): + return "unknown", [ + {"summary": "Malformed input.", "failure_mode": "malformed_input"} + ] + if data.get("timeout"): + return "unknown", [ + {"summary": "Checker timed out within the declared budget.", "failure_mode": "timeout"} + ] + if data.get("binary_unavailable") or data.get("unavailable"): + return "unknown", [ + { + "summary": "Required checker binary is unavailable.", + "failure_mode": "binary_unavailable", + } + ] + return None + + def evaluate_proof_obligation( data: dict[str, Any], *, failure_mode: str, ) -> tuple[str, list[dict[str, Any]]]: """Evaluate proof-assistant shaped input with a conservative oracle.""" - if data.get("malformed"): - return "unknown", [{"summary": "Malformed proof obligation input.", "failure_mode": "malformed_input"}] + early = classify_conformance_flags(data) + if early is not None: + return early violations = [str(item) for item in data.get("violations", [])] unproved = data.get("unproved_obligations", []) @@ -30,8 +59,9 @@ def evaluate_bounded_model( failure_mode: str, ) -> tuple[str, list[dict[str, Any]]]: """Evaluate bounded model-checking shaped input with a conservative oracle.""" - if data.get("malformed"): - return "unknown", [{"summary": "Malformed model-checking input.", "failure_mode": "malformed_input"}] + early = classify_conformance_flags(data) + if early is not None: + return early violations = [str(item) for item in data.get("violations", [])] failed_assertions = data.get("failed_assertions", []) diff --git a/ovk/core/adapter_runtime.py b/ovk/core/adapter_runtime.py index 9e72b7a..d0336f9 100644 --- a/ovk/core/adapter_runtime.py +++ b/ovk/core/adapter_runtime.py @@ -130,6 +130,7 @@ def _attach_execution_metadata( job_id: str | None = None, input_format: str | None = None, routing_enforced: bool = False, + suppress_legacy_routing_artifact: bool = False, ) -> VerificationEvidence: """Record routing, input digest, and obligation-scoped evidence identity.""" @@ -159,7 +160,10 @@ def _attach_execution_metadata( metadata = _routing_metadata(routing, intent_id=resolved_intent, routing_enforced=routing_enforced) - if metadata is not None: + # Full ``routing.mode=enforced`` evidence uses typed fields only. Shadow-mode + # lane enforcement still emits the compatibility ``backend_routing`` artifact so + # MCP/kernel callers can share routing_id without reading typed columns. + if metadata is not None and not suppress_legacy_routing_artifact: artifacts.append( { "kind": "backend_routing", @@ -175,13 +179,20 @@ def _attach_execution_metadata( if shadow_comparison is not None: artifacts.append(shadow_comparison) - return evidence.model_copy( + updated = evidence.model_copy( update={ "evidence_id": f"{evidence.evidence_id}-{evidence_suffix}", "generated_artifacts": artifacts, "routing_id": evidence.routing_id or (metadata or {}).get("routing_id"), + "routing_enforced": routing_enforced, } ) + # Seal after all metadata mutations so evidence_digest binds the final record. + if str(updated.schema_version).endswith(".v3"): + from ovk.core.evidence_integrity import seal_evidence + + return seal_evidence(updated) + return updated def _legacy_status_and_recommendation(evidence: VerificationEvidence) -> tuple[str, str]: @@ -202,6 +213,7 @@ def _run_shadow_path( base_sha: str | None, intent_id: str, policy: dict[str, Any] | None, + cache_dir: Path | None = None, ) -> dict[str, Any] | None: """Execute the typed control plane for comparison; never raises to legacy.""" @@ -228,7 +240,7 @@ def _run_shadow_path( routing = route_obligation(obligation, registry, context=context, policy=policy) - record = _control_plane().execute(obligation, routing, registry=registry) + record = _control_plane(cache_dir=cache_dir).execute(obligation, routing, registry=registry) return { "record": record, @@ -252,6 +264,7 @@ def _run_enforced_with_routing( typed_obligation: VerificationObligation, policy: dict[str, Any] | None, schema_version: str, + cache_dir: Path | None = None, ) -> VerificationEvidence: """Execute one enforced lane using a pre-computed immutable routing decision.""" @@ -262,7 +275,7 @@ def _run_enforced_with_routing( registry = registry_builder() - record = _control_plane().execute(typed_obligation, routing, registry=registry) + record = _control_plane(cache_dir=cache_dir).execute(typed_obligation, routing, registry=registry) evidence = execution_record_to_evidence( record, @@ -324,7 +337,13 @@ def _evaluate_obligation( precomputed_typed = routing_plan.typed_obligations.get(intent_id) - if use_cache and cache_dir is not None: + routing_config = resolve_routing_config(policy) + # Flat legacy cache is only for ``routing.mode=legacy``. Shadow/enforced use the + # control-plane hardened namespace so regimes cannot poison each other. + use_legacy_flat_cache = bool(use_cache and cache_dir is not None and routing_config.mode == "legacy") + suppress_legacy_routing_artifact = routing_config.mode == "enforced" + + if use_legacy_flat_cache: cached = get_cached_evidence(cache_dir, key) if cached is not None: @@ -339,6 +358,7 @@ def _evaluate_obligation( job_id=obligation.get("job_id"), input_format=input_format, routing_enforced=routing_enforced_for_lane(policy, lane), + suppress_legacy_routing_artifact=suppress_legacy_routing_artifact, ) if routing_enforced_for_lane(policy, lane): @@ -359,11 +379,9 @@ def _evaluate_obligation( typed_obligation=precomputed_typed, policy=policy, schema_version=evidence_schema_version, + cache_dir=cache_dir if use_cache else None, ) - if use_cache and cache_dir is not None: - store_cached_evidence(cache_dir, key, evidence.model_dump(mode="json")) - return _attach_execution_metadata( evidence, lane=lane, @@ -373,6 +391,7 @@ def _evaluate_obligation( job_id=obligation.get("job_id"), input_format=input_format, routing_enforced=True, + suppress_legacy_routing_artifact=suppress_legacy_routing_artifact, ) evidence = evaluate_lane( @@ -387,8 +406,6 @@ def _evaluate_obligation( shadow_comparison: dict[str, Any] | None = None - routing_config = resolve_routing_config(policy) - if routing_config.mode in {"shadow", "enforced"} and lane in _LANE_REGISTRY_BUILDERS: shadow = _run_shadow_path( lane=lane, @@ -398,6 +415,7 @@ def _evaluate_obligation( base_sha=base_sha, intent_id=intent_id, policy=policy, + cache_dir=cache_dir if use_cache else None, ) if shadow and "record" in shadow: @@ -422,7 +440,7 @@ def _evaluate_obligation( "routing_mode": routing_config.mode, } - if use_cache and cache_dir is not None: + if use_legacy_flat_cache: store_cached_evidence(cache_dir, key, evidence.model_dump(mode="json")) return _attach_execution_metadata( @@ -435,6 +453,7 @@ def _evaluate_obligation( job_id=obligation.get("job_id"), input_format=input_format, routing_enforced=False, + suppress_legacy_routing_artifact=suppress_legacy_routing_artifact, ) diff --git a/ovk/core/execution_models.py b/ovk/core/execution_models.py index ea1384b..0db424b 100644 --- a/ovk/core/execution_models.py +++ b/ovk/core/execution_models.py @@ -19,10 +19,11 @@ import re from typing import Any, Literal -from pydantic import BaseModel, Field, field_validator +from pydantic import BaseModel, Field, field_validator, model_validator from ovk.core.bundle import content_digest from ovk.core.models import ( + DecisionState, MergeRecommendation, RiskSeverity, SourceRange, @@ -67,6 +68,9 @@ ] FallbackOutcome = Literal["unknown", "error", "fail", "fallback"] TimeoutBehavior = Literal["unknown", "error", "fail"] +ReleaseStatus = Literal["stable", "preview", "experimental", "disabled"] +DeterminismStatus = Literal["deterministic", "tool_dependent", "non_deterministic", "unknown"] +VALID_RELEASE_STATUSES: frozenset[str] = frozenset({"stable", "preview", "experimental", "disabled"}) _WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]") _URI_SCHEME_RE = re.compile(r"^([A-Za-z][A-Za-z0-9+.-]*):") @@ -182,6 +186,66 @@ class BackendCapabilityManifest(BaseModel): result_format: str | None = None counterexample_format: str | None = None timeout_behavior: TimeoutBehavior = "unknown" + # Normative claim-registry fields (OVK-02). Empty strings are filled by the + # post-validator from existing tool/guarantee fields so lane adapters stay concise. + checker_id: str = "" + version: str = "" + implementation: str = "" + input_contract: str = "" + output_contract: str = "" + claim_class: str = "" + trusted_components: list[str] = Field(default_factory=list) + failure_semantics: str = "" + timeout_semantics: TimeoutBehavior | None = None + unsupported_semantics: str = "" + determinism_status: DeterminismStatus = "unknown" + release_status: ReleaseStatus = "experimental" + owner: str = "ovk-maintainers" + native_execution: bool | None = None + + @model_validator(mode="after") + def _normalize_normative_fields(self) -> "BackendCapabilityManifest": + if not self.checker_id.strip(): + object.__setattr__(self, "checker_id", self.tool.name or self.capability_id) + if not self.version.strip(): + object.__setattr__( + self, + "version", + self.tool.adapter_version or self.tool.version or "0.0.0", + ) + if not self.implementation.strip(): + object.__setattr__(self, "implementation", self.tool.adapter) + if not self.input_contract.strip(): + langs = ", ".join(self.input_languages) if self.input_languages else "unspecified" + object.__setattr__( + self, + "input_contract", + f"Accepts {langs} inputs per adapter compile contract.", + ) + if not self.output_contract.strip(): + object.__setattr__(self, "output_contract", self.result_format or "ovk.result.v1") + if not self.claim_class.strip(): + object.__setattr__(self, "claim_class", self.guarantee.type) + if not self.failure_semantics.strip(): + object.__setattr__( + self, + "failure_semantics", + self.guarantee.meaning_of_fail or "Map tool/adapter failure to error or unknown.", + ) + if self.timeout_semantics is None: + object.__setattr__(self, "timeout_semantics", self.timeout_behavior) + if not self.unsupported_semantics.strip(): + object.__setattr__( + self, + "unsupported_semantics", + "; ".join(self.limits) if self.limits else "Unsupported inputs yield unknown.", + ) + if self.release_status not in VALID_RELEASE_STATUSES: + raise ValueError( + f"unknown release_status {self.release_status!r}; " + f"expected one of {sorted(VALID_RELEASE_STATUSES)}" + ) + return self class BackendEnvironmentFingerprint(BaseModel): @@ -469,12 +533,15 @@ class ObligationExecutionRecord(BaseModel): attempts: list[ExecutionAttempt] results: list[NormalizedBackendResult] aggregate_status: VerificationStatus + decision_state: DecisionState | None = None + original_decision_state: DecisionState | None = None merge_recommendation: MergeRecommendation aggregation_reason: str open_obligations: list[dict[str, Any]] = Field(default_factory=list) fallback_used: bool = False fallback_accepted: bool = False fallback_cause: str | None = None + controlling_finding_ids: list[str] = Field(default_factory=list) # --------------------------------------------------------------------------- diff --git a/schemas/backend.execution.schema.json b/schemas/backend.execution.schema.json index a6d59a3..8f45227 100644 --- a/schemas/backend.execution.schema.json +++ b/schemas/backend.execution.schema.json @@ -174,6 +174,24 @@ "type": "string", "enum": ["pass", "fail", "unknown", "error", "skipped"] }, + "decision_state": { + "anyOf": [ + { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"] + }, + { "type": "null" } + ] + }, + "original_decision_state": { + "anyOf": [ + { + "type": "string", + "enum": ["allow", "block", "needs_review", "unknown", "error", "skipped"] + }, + { "type": "null" } + ] + }, "merge_recommendation": { "type": "string", "enum": [ @@ -182,7 +200,12 @@ "require_human_review", "allow_with_warning", "require_stronger_check" - ] + ], + "description": "Deprecated alias of decision_state." + }, + "controlling_finding_ids": { + "type": "array", + "items": { "type": "string" } }, "aggregation_reason": { "type": "string" }, "open_obligations": { diff --git a/tests/test_execution_models.py b/tests/test_execution_models.py index f4f23ba..208ac8e 100644 --- a/tests/test_execution_models.py +++ b/tests/test_execution_models.py @@ -36,6 +36,7 @@ validate_material_uri, ) from ovk.core.models import ( + DecisionState, MergeRecommendation, RiskSeverity, SourceRange, @@ -237,9 +238,12 @@ def _execution_record() -> ObligationExecutionRecord: attempts=[attempt], results=[_result(attempt.attempt_id)], aggregate_status=VerificationStatus.PASS, + decision_state=DecisionState.ALLOW, + original_decision_state=DecisionState.ALLOW, merge_recommendation=MergeRecommendation.ALLOW, aggregation_reason="single required backend passed", open_obligations=[], + controlling_finding_ids=[], ) From c2fe3417da46e681fdde46f49c81357132980a42 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:09:19 -0700 Subject: [PATCH 07/19] Expand backend example fixtures for conformance edges (OVK-PR4). Add timeout, unavailable, and malformed example payloads so local demos and conformance generators share the same edge-case corpus. --- examples/backends/alloy_malformed.json | 4 ++++ examples/backends/alloy_timeout.json | 4 ++++ examples/backends/alloy_unavailable.json | 4 ++++ examples/backends/cbmc_malformed.json | 4 ++++ examples/backends/cbmc_timeout.json | 4 ++++ examples/backends/cbmc_unavailable.json | 4 ++++ examples/backends/cedar_timeout.json | 4 ++++ examples/backends/cedar_unavailable.json | 4 ++++ examples/backends/dafny_malformed.json | 4 ++++ examples/backends/dafny_timeout.json | 4 ++++ examples/backends/dafny_unavailable.json | 4 ++++ examples/backends/kani_timeout.json | 4 ++++ examples/backends/kani_unavailable.json | 4 ++++ examples/backends/lean_malformed.json | 4 ++++ examples/backends/lean_timeout.json | 4 ++++ examples/backends/lean_unavailable.json | 4 ++++ examples/backends/opa_fail.json | 7 +++++++ examples/backends/opa_malformed.json | 4 ++++ examples/backends/opa_pass.json | 5 +++++ examples/backends/opa_timeout.json | 4 ++++ examples/backends/opa_unavailable.json | 4 ++++ examples/backends/tla_timeout.json | 4 ++++ examples/backends/tla_unavailable.json | 4 ++++ examples/backends/verus_malformed.json | 4 ++++ examples/backends/verus_timeout.json | 4 ++++ examples/backends/verus_unavailable.json | 4 ++++ examples/backends/z3_timeout.json | 6 ++++++ examples/backends/z3_unavailable.json | 6 ++++++ 28 files changed, 120 insertions(+) create mode 100644 examples/backends/alloy_malformed.json create mode 100644 examples/backends/alloy_timeout.json create mode 100644 examples/backends/alloy_unavailable.json create mode 100644 examples/backends/cbmc_malformed.json create mode 100644 examples/backends/cbmc_timeout.json create mode 100644 examples/backends/cbmc_unavailable.json create mode 100644 examples/backends/cedar_timeout.json create mode 100644 examples/backends/cedar_unavailable.json create mode 100644 examples/backends/dafny_malformed.json create mode 100644 examples/backends/dafny_timeout.json create mode 100644 examples/backends/dafny_unavailable.json create mode 100644 examples/backends/kani_timeout.json create mode 100644 examples/backends/kani_unavailable.json create mode 100644 examples/backends/lean_malformed.json create mode 100644 examples/backends/lean_timeout.json create mode 100644 examples/backends/lean_unavailable.json create mode 100644 examples/backends/opa_fail.json create mode 100644 examples/backends/opa_malformed.json create mode 100644 examples/backends/opa_pass.json create mode 100644 examples/backends/opa_timeout.json create mode 100644 examples/backends/opa_unavailable.json create mode 100644 examples/backends/tla_timeout.json create mode 100644 examples/backends/tla_unavailable.json create mode 100644 examples/backends/verus_malformed.json create mode 100644 examples/backends/verus_timeout.json create mode 100644 examples/backends/verus_unavailable.json create mode 100644 examples/backends/z3_timeout.json create mode 100644 examples/backends/z3_unavailable.json diff --git a/examples/backends/alloy_malformed.json b/examples/backends/alloy_malformed.json new file mode 100644 index 0000000..f797939 --- /dev/null +++ b/examples/backends/alloy_malformed.json @@ -0,0 +1,4 @@ +{ + "intent_id": "alloy-model-check", + "malformed": true +} diff --git a/examples/backends/alloy_timeout.json b/examples/backends/alloy_timeout.json new file mode 100644 index 0000000..9235627 --- /dev/null +++ b/examples/backends/alloy_timeout.json @@ -0,0 +1,4 @@ +{ + "intent_id": "alloy-model-check", + "timeout": true +} diff --git a/examples/backends/alloy_unavailable.json b/examples/backends/alloy_unavailable.json new file mode 100644 index 0000000..0282556 --- /dev/null +++ b/examples/backends/alloy_unavailable.json @@ -0,0 +1,4 @@ +{ + "intent_id": "alloy-model-check", + "binary_unavailable": true +} diff --git a/examples/backends/cbmc_malformed.json b/examples/backends/cbmc_malformed.json new file mode 100644 index 0000000..55a372b --- /dev/null +++ b/examples/backends/cbmc_malformed.json @@ -0,0 +1,4 @@ +{ + "intent_id": "cbmc-harness-check", + "malformed": true +} diff --git a/examples/backends/cbmc_timeout.json b/examples/backends/cbmc_timeout.json new file mode 100644 index 0000000..5edb3b4 --- /dev/null +++ b/examples/backends/cbmc_timeout.json @@ -0,0 +1,4 @@ +{ + "intent_id": "cbmc-harness-check", + "timeout": true +} diff --git a/examples/backends/cbmc_unavailable.json b/examples/backends/cbmc_unavailable.json new file mode 100644 index 0000000..3417fd7 --- /dev/null +++ b/examples/backends/cbmc_unavailable.json @@ -0,0 +1,4 @@ +{ + "intent_id": "cbmc-harness-check", + "binary_unavailable": true +} diff --git a/examples/backends/cedar_timeout.json b/examples/backends/cedar_timeout.json new file mode 100644 index 0000000..dde4f88 --- /dev/null +++ b/examples/backends/cedar_timeout.json @@ -0,0 +1,4 @@ +{ + "intent_id": "cedar-policy-check", + "timeout": true +} diff --git a/examples/backends/cedar_unavailable.json b/examples/backends/cedar_unavailable.json new file mode 100644 index 0000000..afb7ee5 --- /dev/null +++ b/examples/backends/cedar_unavailable.json @@ -0,0 +1,4 @@ +{ + "intent_id": "cedar-policy-check", + "binary_unavailable": true +} diff --git a/examples/backends/dafny_malformed.json b/examples/backends/dafny_malformed.json new file mode 100644 index 0000000..66f61d7 --- /dev/null +++ b/examples/backends/dafny_malformed.json @@ -0,0 +1,4 @@ +{ + "intent_id": "dafny-obligation-check", + "malformed": true +} diff --git a/examples/backends/dafny_timeout.json b/examples/backends/dafny_timeout.json new file mode 100644 index 0000000..32daab6 --- /dev/null +++ b/examples/backends/dafny_timeout.json @@ -0,0 +1,4 @@ +{ + "intent_id": "dafny-obligation-check", + "timeout": true +} diff --git a/examples/backends/dafny_unavailable.json b/examples/backends/dafny_unavailable.json new file mode 100644 index 0000000..c9cf523 --- /dev/null +++ b/examples/backends/dafny_unavailable.json @@ -0,0 +1,4 @@ +{ + "intent_id": "dafny-obligation-check", + "binary_unavailable": true +} diff --git a/examples/backends/kani_timeout.json b/examples/backends/kani_timeout.json new file mode 100644 index 0000000..986d381 --- /dev/null +++ b/examples/backends/kani_timeout.json @@ -0,0 +1,4 @@ +{ + "intent_id": "kani-harness-check", + "timeout": true +} diff --git a/examples/backends/kani_unavailable.json b/examples/backends/kani_unavailable.json new file mode 100644 index 0000000..f6102a6 --- /dev/null +++ b/examples/backends/kani_unavailable.json @@ -0,0 +1,4 @@ +{ + "intent_id": "kani-harness-check", + "binary_unavailable": true +} diff --git a/examples/backends/lean_malformed.json b/examples/backends/lean_malformed.json new file mode 100644 index 0000000..1da2ece --- /dev/null +++ b/examples/backends/lean_malformed.json @@ -0,0 +1,4 @@ +{ + "intent_id": "lean-proof-check", + "malformed": true +} diff --git a/examples/backends/lean_timeout.json b/examples/backends/lean_timeout.json new file mode 100644 index 0000000..ec8dcde --- /dev/null +++ b/examples/backends/lean_timeout.json @@ -0,0 +1,4 @@ +{ + "intent_id": "lean-proof-check", + "timeout": true +} diff --git a/examples/backends/lean_unavailable.json b/examples/backends/lean_unavailable.json new file mode 100644 index 0000000..05b0ee7 --- /dev/null +++ b/examples/backends/lean_unavailable.json @@ -0,0 +1,4 @@ +{ + "intent_id": "lean-proof-check", + "binary_unavailable": true +} diff --git a/examples/backends/opa_fail.json b/examples/backends/opa_fail.json new file mode 100644 index 0000000..bcc6484 --- /dev/null +++ b/examples/backends/opa_fail.json @@ -0,0 +1,7 @@ +{ + "intent_id": "opa-policy-check", + "status": "fail", + "violations": [ + "required ovk gate removed from branch protection" + ] +} diff --git a/examples/backends/opa_malformed.json b/examples/backends/opa_malformed.json new file mode 100644 index 0000000..b320262 --- /dev/null +++ b/examples/backends/opa_malformed.json @@ -0,0 +1,4 @@ +{ + "intent_id": "opa-policy-check", + "malformed": true +} diff --git a/examples/backends/opa_pass.json b/examples/backends/opa_pass.json new file mode 100644 index 0000000..c543d7a --- /dev/null +++ b/examples/backends/opa_pass.json @@ -0,0 +1,5 @@ +{ + "intent_id": "opa-policy-check", + "status": "pass", + "violations": [] +} diff --git a/examples/backends/opa_timeout.json b/examples/backends/opa_timeout.json new file mode 100644 index 0000000..b7aa469 --- /dev/null +++ b/examples/backends/opa_timeout.json @@ -0,0 +1,4 @@ +{ + "intent_id": "opa-policy-check", + "timeout": true +} diff --git a/examples/backends/opa_unavailable.json b/examples/backends/opa_unavailable.json new file mode 100644 index 0000000..392d30f --- /dev/null +++ b/examples/backends/opa_unavailable.json @@ -0,0 +1,4 @@ +{ + "intent_id": "opa-policy-check", + "binary_unavailable": true +} diff --git a/examples/backends/tla_timeout.json b/examples/backends/tla_timeout.json new file mode 100644 index 0000000..4e0966e --- /dev/null +++ b/examples/backends/tla_timeout.json @@ -0,0 +1,4 @@ +{ + "intent_id": "tla-state-check", + "timeout": true +} diff --git a/examples/backends/tla_unavailable.json b/examples/backends/tla_unavailable.json new file mode 100644 index 0000000..56677db --- /dev/null +++ b/examples/backends/tla_unavailable.json @@ -0,0 +1,4 @@ +{ + "intent_id": "tla-state-check", + "binary_unavailable": true +} diff --git a/examples/backends/verus_malformed.json b/examples/backends/verus_malformed.json new file mode 100644 index 0000000..20cb677 --- /dev/null +++ b/examples/backends/verus_malformed.json @@ -0,0 +1,4 @@ +{ + "intent_id": "verus-harness-check", + "malformed": true +} diff --git a/examples/backends/verus_timeout.json b/examples/backends/verus_timeout.json new file mode 100644 index 0000000..30abe7d --- /dev/null +++ b/examples/backends/verus_timeout.json @@ -0,0 +1,4 @@ +{ + "intent_id": "verus-harness-check", + "timeout": true +} diff --git a/examples/backends/verus_unavailable.json b/examples/backends/verus_unavailable.json new file mode 100644 index 0000000..6400d6a --- /dev/null +++ b/examples/backends/verus_unavailable.json @@ -0,0 +1,4 @@ +{ + "intent_id": "verus-harness-check", + "binary_unavailable": true +} diff --git a/examples/backends/z3_timeout.json b/examples/backends/z3_timeout.json new file mode 100644 index 0000000..1a6432a --- /dev/null +++ b/examples/backends/z3_timeout.json @@ -0,0 +1,6 @@ +{ + "timeout": true, + "author_type": "ai_agent", + "agent": "codex", + "task": "timeout" +} diff --git a/examples/backends/z3_unavailable.json b/examples/backends/z3_unavailable.json new file mode 100644 index 0000000..442e0fe --- /dev/null +++ b/examples/backends/z3_unavailable.json @@ -0,0 +1,6 @@ +{ + "binary_unavailable": true, + "author_type": "ai_agent", + "agent": "codex", + "task": "unavailable" +} From 81c3259fa0ffa67e01f5cbad1eb4cf232420db6e Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:09:33 -0700 Subject: [PATCH 08/19] Ship FormalPR-Bench provenance, partitions, and holdout corpus (OVK-PR5). Version the benchmark manifest with attributable provenance, mutation partitions, adversarial cases, and held-out variants so leaderboard claims stay reproducible. --- .../adversarial/forged_allow_label.json | 10 + .../misleading_docs_hides_secret.diff | 16 + .../misleading_docs_hides_secret.json | 13 + .../formal_pr_bench/duplication_report.json | 631 +++++++++++++++++ benchmarks/formal_pr_bench/held_out/README.md | 8 + .../held_out/alloy_fail_variant_1.json | 14 + .../held_out/auth_bypass_variant_2.json | 14 + .../held_out/cedar_malformed_variant_2.json | 14 + .../held_out/kani_unknown_variant_2.json | 14 + .../held_out/lean_fail_variant_1.json | 14 + .../rd_cbmc_integer_overflow_quota.json | 19 + .../rd_cbmc_use_after_free_auth_cache.json | 19 + .../held_out/rd_docs_only_change.json | 13 + .../held_out/tla_unknown_variant_2.json | 14 + .../held_out/verus_fail_variant_1.json | 14 + benchmarks/formal_pr_bench/licenses.json | 669 ++++++++++++++++++ benchmarks/formal_pr_bench/manifest.v1.json | 42 ++ .../auth_bypass__drop_counterexample.json | 22 + .../ci_secrets_exposed__status_to_pass.json | 24 + .../control_removed__flip_merge_to_allow.json | 22 + benchmarks/formal_pr_bench/partitions.json | 152 ++++ .../provenance/adversarial_forged_allow.json | 14 + .../provenance/adversarial_sha_mismatch.json | 14 + .../provenance/alloy_fail.json | 14 + .../provenance/alloy_fail_variant_1.json | 14 + .../provenance/alloy_pass.json | 14 + .../provenance/alloy_pass_variant_1.json | 14 + .../provenance/auth_bypass.json | 14 + .../provenance/auth_bypass_variant_1.json | 14 + .../provenance/auth_bypass_variant_2.json | 14 + .../provenance/auth_malformed.json | 14 + .../provenance/auth_malformed_variant_1.json | 14 + .../provenance/auth_malformed_variant_2.json | 14 + .../provenance/auth_preserved.json | 14 + .../provenance/auth_preserved_variant_1.json | 14 + .../provenance/auth_preserved_variant_2.json | 14 + .../formal_pr_bench/provenance/cbmc_fail.json | 14 + .../provenance/cbmc_fail_variant_1.json | 14 + .../cbmc_native_buffer_bounds_pass.json | 14 + ...c_native_buffer_bounds_pass_variant_1.json | 14 + .../cbmc_native_integer_overflow_pass.json | 14 + ...ative_integer_overflow_pass_variant_1.json | 14 + .../provenance/cbmc_native_uaf_pass.json | 14 + .../cbmc_native_uaf_pass_variant_1.json | 14 + .../cbmc_native_unchecked_copy_pass.json | 14 + ..._native_unchecked_copy_pass_variant_1.json | 14 + .../formal_pr_bench/provenance/cbmc_pass.json | 14 + .../provenance/cbmc_pass_variant_1.json | 14 + .../provenance/cedar_fail.json | 14 + .../provenance/cedar_fail_variant_1.json | 14 + .../provenance/cedar_fail_variant_2.json | 14 + .../provenance/cedar_malformed.json | 14 + .../provenance/cedar_malformed_variant_1.json | 14 + .../provenance/cedar_malformed_variant_2.json | 14 + .../provenance/cedar_pass.json | 14 + .../provenance/cedar_pass_variant_1.json | 14 + .../provenance/cedar_pass_variant_2.json | 14 + .../provenance/cedar_unknown.json | 14 + .../provenance/cedar_unknown_variant_1.json | 14 + .../provenance/cedar_unknown_variant_2.json | 14 + .../provenance/ci_secrets_exposed.json | 14 + .../ci_secrets_exposed_variant_1.json | 14 + .../ci_secrets_exposed_variant_2.json | 14 + .../provenance/ci_secrets_safe.json | 14 + .../provenance/ci_secrets_safe_variant_1.json | 14 + .../provenance/ci_secrets_safe_variant_2.json | 14 + .../provenance/control_metadata_missing.json | 14 + .../control_metadata_missing_variant_1.json | 14 + .../control_metadata_missing_variant_2.json | 14 + .../provenance/control_preserved.json | 14 + .../control_preserved_variant_1.json | 14 + .../control_preserved_variant_2.json | 14 + .../provenance/control_removed.json | 14 + .../provenance/control_removed_variant_1.json | 14 + .../provenance/control_removed_variant_2.json | 14 + .../provenance/dafny_fail.json | 14 + .../provenance/dafny_fail_variant_1.json | 14 + .../provenance/dafny_pass.json | 14 + .../provenance/dafny_pass_variant_1.json | 14 + .../deployment_skipped_approval.json | 14 + ...deployment_skipped_approval_variant_1.json | 14 + ...deployment_skipped_approval_variant_2.json | 14 + .../provenance/deployment_valid_path.json | 14 + .../deployment_valid_path_variant_1.json | 14 + .../deployment_valid_path_variant_2.json | 14 + .../provenance/infra_private_sensitive.json | 14 + .../infra_private_sensitive_variant_1.json | 14 + .../infra_private_sensitive_variant_2.json | 14 + .../provenance/infra_public_sensitive.json | 14 + .../infra_public_sensitive_variant_1.json | 14 + .../infra_public_sensitive_variant_2.json | 14 + .../formal_pr_bench/provenance/kani_fail.json | 14 + .../provenance/kani_fail_variant_1.json | 14 + .../provenance/kani_fail_variant_2.json | 14 + .../provenance/kani_malformed.json | 14 + .../provenance/kani_malformed_variant_1.json | 14 + .../provenance/kani_malformed_variant_2.json | 14 + .../formal_pr_bench/provenance/kani_pass.json | 14 + .../provenance/kani_pass_variant_1.json | 14 + .../provenance/kani_pass_variant_2.json | 14 + .../provenance/kani_unknown.json | 14 + .../provenance/kani_unknown_variant_1.json | 14 + .../provenance/kani_unknown_variant_2.json | 14 + .../formal_pr_bench/provenance/lean_fail.json | 14 + .../provenance/lean_fail_variant_1.json | 14 + .../formal_pr_bench/provenance/lean_pass.json | 14 + .../provenance/lean_pass_variant_1.json | 14 + .../provenance/multi_surface_combined_pr.json | 14 + .../rd_auth_admin_route_guarded.json | 14 + .../rd_auth_admin_route_unguarded.json | 14 + .../rd_auth_route_partial_hunk.json | 14 + .../rd_cbmc_integer_overflow_quota.json | 14 + .../rd_cbmc_use_after_free_auth_cache.json | 14 + .../provenance/rd_ci_secrets_pr_preview.json | 14 + .../rd_ci_secrets_workflow_dispatch_safe.json | 14 + .../provenance/rd_deployment_direct_skip.json | 14 + .../provenance/rd_deployment_valid_chain.json | 14 + .../provenance/rd_docs_only_change.json | 14 + .../rd_infra_iam_wildcard_admin.json | 14 + .../provenance/rd_infra_k8s_loadbalancer.json | 14 + .../provenance/rd_infra_private_bucket.json | 14 + .../rd_infra_rds_public_partial_hunk.json | 14 + .../provenance/rd_infra_s3_public_acl.json | 14 + .../provenance/rd_multi_surface_combined.json | 14 + .../rd_self_protection_workflow_touch.json | 14 + .../rd_workflow_secrets_partial_hunk.json | 14 + .../recall_ci_secrets_workflow_diff.json | 14 + .../recall_infra_terraform_diff.json | 14 + .../recall_multi_surface_combined.json | 14 + .../provenance/repair_loop_auth_bypass.json | 14 + .../provenance/repair_loop_ci_secrets.json | 14 + .../repair_loop_deployment_skip.json | 14 + .../repair_loop_infra_exposure.json | 14 + .../provenance/route_alloy_model.json | 14 + .../provenance/route_cedar_iam.json | 14 + .../provenance/route_dafny_proof.json | 14 + .../provenance/route_kani_rust.json | 14 + .../formal_pr_bench/provenance/tla_fail.json | 14 + .../provenance/tla_fail_variant_1.json | 14 + .../provenance/tla_fail_variant_2.json | 14 + .../provenance/tla_malformed.json | 14 + .../provenance/tla_malformed_variant_1.json | 14 + .../provenance/tla_malformed_variant_2.json | 14 + .../formal_pr_bench/provenance/tla_pass.json | 14 + .../provenance/tla_pass_variant_1.json | 14 + .../provenance/tla_pass_variant_2.json | 14 + .../provenance/tla_unknown.json | 14 + .../provenance/tla_unknown_variant_1.json | 14 + .../provenance/tla_unknown_variant_2.json | 14 + .../provenance/verus_fail.json | 14 + .../provenance/verus_fail_variant_1.json | 14 + .../provenance/verus_pass.json | 14 + .../provenance/verus_pass_variant_1.json | 14 + .../rationales/adversarial_forged_allow.md | 13 + .../rationales/adversarial_sha_mismatch.md | 13 + .../formal_pr_bench/rationales/alloy_fail.md | 13 + .../rationales/alloy_fail_variant_1.md | 13 + .../formal_pr_bench/rationales/alloy_pass.md | 13 + .../rationales/alloy_pass_variant_1.md | 13 + .../formal_pr_bench/rationales/auth_bypass.md | 13 + .../rationales/auth_bypass_variant_1.md | 13 + .../rationales/auth_bypass_variant_2.md | 13 + .../rationales/auth_malformed.md | 13 + .../rationales/auth_malformed_variant_1.md | 13 + .../rationales/auth_malformed_variant_2.md | 13 + .../rationales/auth_preserved.md | 13 + .../rationales/auth_preserved_variant_1.md | 13 + .../rationales/auth_preserved_variant_2.md | 13 + .../formal_pr_bench/rationales/cbmc_fail.md | 13 + .../rationales/cbmc_fail_variant_1.md | 13 + .../cbmc_native_buffer_bounds_pass.md | 13 + ...bmc_native_buffer_bounds_pass_variant_1.md | 13 + .../cbmc_native_integer_overflow_pass.md | 13 + ..._native_integer_overflow_pass_variant_1.md | 13 + .../rationales/cbmc_native_uaf_pass.md | 13 + .../cbmc_native_uaf_pass_variant_1.md | 13 + .../cbmc_native_unchecked_copy_pass.md | 13 + ...mc_native_unchecked_copy_pass_variant_1.md | 13 + .../formal_pr_bench/rationales/cbmc_pass.md | 13 + .../rationales/cbmc_pass_variant_1.md | 13 + .../formal_pr_bench/rationales/cedar_fail.md | 13 + .../rationales/cedar_fail_variant_1.md | 13 + .../rationales/cedar_fail_variant_2.md | 13 + .../rationales/cedar_malformed.md | 13 + .../rationales/cedar_malformed_variant_1.md | 13 + .../rationales/cedar_malformed_variant_2.md | 13 + .../formal_pr_bench/rationales/cedar_pass.md | 13 + .../rationales/cedar_pass_variant_1.md | 13 + .../rationales/cedar_pass_variant_2.md | 13 + .../rationales/cedar_unknown.md | 13 + .../rationales/cedar_unknown_variant_1.md | 13 + .../rationales/cedar_unknown_variant_2.md | 13 + .../rationales/ci_secrets_exposed.md | 13 + .../ci_secrets_exposed_variant_1.md | 13 + .../ci_secrets_exposed_variant_2.md | 13 + .../rationales/ci_secrets_safe.md | 13 + .../rationales/ci_secrets_safe_variant_1.md | 13 + .../rationales/ci_secrets_safe_variant_2.md | 13 + .../rationales/control_metadata_missing.md | 13 + .../control_metadata_missing_variant_1.md | 13 + .../control_metadata_missing_variant_2.md | 13 + .../rationales/control_preserved.md | 13 + .../rationales/control_preserved_variant_1.md | 13 + .../rationales/control_preserved_variant_2.md | 13 + .../rationales/control_removed.md | 13 + .../rationales/control_removed_variant_1.md | 13 + .../rationales/control_removed_variant_2.md | 13 + .../formal_pr_bench/rationales/dafny_fail.md | 13 + .../rationales/dafny_fail_variant_1.md | 13 + .../formal_pr_bench/rationales/dafny_pass.md | 13 + .../rationales/dafny_pass_variant_1.md | 13 + .../rationales/deployment_skipped_approval.md | 13 + .../deployment_skipped_approval_variant_1.md | 13 + .../deployment_skipped_approval_variant_2.md | 13 + .../rationales/deployment_valid_path.md | 13 + .../deployment_valid_path_variant_1.md | 13 + .../deployment_valid_path_variant_2.md | 13 + .../rationales/infra_private_sensitive.md | 13 + .../infra_private_sensitive_variant_1.md | 13 + .../infra_private_sensitive_variant_2.md | 13 + .../rationales/infra_public_sensitive.md | 13 + .../infra_public_sensitive_variant_1.md | 13 + .../infra_public_sensitive_variant_2.md | 13 + .../formal_pr_bench/rationales/kani_fail.md | 13 + .../rationales/kani_fail_variant_1.md | 13 + .../rationales/kani_fail_variant_2.md | 13 + .../rationales/kani_malformed.md | 13 + .../rationales/kani_malformed_variant_1.md | 13 + .../rationales/kani_malformed_variant_2.md | 13 + .../formal_pr_bench/rationales/kani_pass.md | 13 + .../rationales/kani_pass_variant_1.md | 13 + .../rationales/kani_pass_variant_2.md | 13 + .../rationales/kani_unknown.md | 13 + .../rationales/kani_unknown_variant_1.md | 13 + .../rationales/kani_unknown_variant_2.md | 13 + .../formal_pr_bench/rationales/lean_fail.md | 13 + .../rationales/lean_fail_variant_1.md | 13 + .../formal_pr_bench/rationales/lean_pass.md | 13 + .../rationales/lean_pass_variant_1.md | 13 + .../rationales/multi_surface_combined_pr.md | 13 + .../rationales/rd_auth_admin_route_guarded.md | 13 + .../rd_auth_admin_route_unguarded.md | 13 + .../rationales/rd_auth_route_partial_hunk.md | 13 + .../rd_cbmc_integer_overflow_quota.md | 13 + .../rd_cbmc_use_after_free_auth_cache.md | 13 + .../rationales/rd_ci_secrets_pr_preview.md | 13 + .../rd_ci_secrets_workflow_dispatch_safe.md | 13 + .../rationales/rd_deployment_direct_skip.md | 13 + .../rationales/rd_deployment_valid_chain.md | 13 + .../rationales/rd_docs_only_change.md | 13 + .../rationales/rd_infra_iam_wildcard_admin.md | 13 + .../rationales/rd_infra_k8s_loadbalancer.md | 13 + .../rationales/rd_infra_private_bucket.md | 13 + .../rd_infra_rds_public_partial_hunk.md | 13 + .../rationales/rd_infra_s3_public_acl.md | 13 + .../rationales/rd_multi_surface_combined.md | 13 + .../rd_self_protection_workflow_touch.md | 13 + .../rd_workflow_secrets_partial_hunk.md | 13 + .../recall_ci_secrets_workflow_diff.md | 13 + .../rationales/recall_infra_terraform_diff.md | 13 + .../recall_multi_surface_combined.md | 13 + .../rationales/repair_loop_auth_bypass.md | 13 + .../rationales/repair_loop_ci_secrets.md | 13 + .../rationales/repair_loop_deployment_skip.md | 13 + .../rationales/repair_loop_infra_exposure.md | 13 + .../rationales/route_alloy_model.md | 13 + .../rationales/route_cedar_iam.md | 13 + .../rationales/route_dafny_proof.md | 13 + .../rationales/route_kani_rust.md | 13 + .../formal_pr_bench/rationales/tla_fail.md | 13 + .../rationales/tla_fail_variant_1.md | 13 + .../rationales/tla_fail_variant_2.md | 13 + .../rationales/tla_malformed.md | 13 + .../rationales/tla_malformed_variant_1.md | 13 + .../rationales/tla_malformed_variant_2.md | 13 + .../formal_pr_bench/rationales/tla_pass.md | 13 + .../rationales/tla_pass_variant_1.md | 13 + .../rationales/tla_pass_variant_2.md | 13 + .../formal_pr_bench/rationales/tla_unknown.md | 13 + .../rationales/tla_unknown_variant_1.md | 13 + .../rationales/tla_unknown_variant_2.md | 13 + .../formal_pr_bench/rationales/verus_fail.md | 13 + .../rationales/verus_fail_variant_1.md | 13 + .../formal_pr_bench/rationales/verus_pass.md | 13 + .../rationales/verus_pass_variant_1.md | 13 + .../formal_pr_bench/real_diff_cases.json | 28 + .../formal_pr_bench/template_dev_cases.json | 45 ++ .../formal_pr_bench.leaderboard.schema.json | 7 +- schemas/formal_pr_bench.manifest.schema.json | 43 ++ 289 files changed, 5444 insertions(+), 1 deletion(-) create mode 100644 benchmarks/formal_pr_bench/adversarial/forged_allow_label.json create mode 100644 benchmarks/formal_pr_bench/adversarial/misleading_docs_hides_secret.diff create mode 100644 benchmarks/formal_pr_bench/adversarial/misleading_docs_hides_secret.json create mode 100644 benchmarks/formal_pr_bench/duplication_report.json create mode 100644 benchmarks/formal_pr_bench/held_out/README.md create mode 100644 benchmarks/formal_pr_bench/held_out/alloy_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/held_out/auth_bypass_variant_2.json create mode 100644 benchmarks/formal_pr_bench/held_out/cedar_malformed_variant_2.json create mode 100644 benchmarks/formal_pr_bench/held_out/kani_unknown_variant_2.json create mode 100644 benchmarks/formal_pr_bench/held_out/lean_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/held_out/rd_cbmc_integer_overflow_quota.json create mode 100644 benchmarks/formal_pr_bench/held_out/rd_cbmc_use_after_free_auth_cache.json create mode 100644 benchmarks/formal_pr_bench/held_out/rd_docs_only_change.json create mode 100644 benchmarks/formal_pr_bench/held_out/tla_unknown_variant_2.json create mode 100644 benchmarks/formal_pr_bench/held_out/verus_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/licenses.json create mode 100644 benchmarks/formal_pr_bench/manifest.v1.json create mode 100644 benchmarks/formal_pr_bench/mutations/auth_bypass__drop_counterexample.json create mode 100644 benchmarks/formal_pr_bench/mutations/ci_secrets_exposed__status_to_pass.json create mode 100644 benchmarks/formal_pr_bench/mutations/control_removed__flip_merge_to_allow.json create mode 100644 benchmarks/formal_pr_bench/partitions.json create mode 100644 benchmarks/formal_pr_bench/provenance/adversarial_forged_allow.json create mode 100644 benchmarks/formal_pr_bench/provenance/adversarial_sha_mismatch.json create mode 100644 benchmarks/formal_pr_bench/provenance/alloy_fail.json create mode 100644 benchmarks/formal_pr_bench/provenance/alloy_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/alloy_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/alloy_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/auth_bypass.json create mode 100644 benchmarks/formal_pr_bench/provenance/auth_bypass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/auth_bypass_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/auth_malformed.json create mode 100644 benchmarks/formal_pr_bench/provenance/auth_malformed_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/auth_malformed_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/auth_preserved.json create mode 100644 benchmarks/formal_pr_bench/provenance/auth_preserved_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/auth_preserved_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_fail.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_native_buffer_bounds_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_native_buffer_bounds_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_native_integer_overflow_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_native_integer_overflow_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_native_uaf_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_native_uaf_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_native_unchecked_copy_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_native_unchecked_copy_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/cbmc_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_fail.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_fail_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_malformed.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_malformed_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_malformed_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_pass_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_unknown.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_unknown_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/cedar_unknown_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/ci_secrets_exposed.json create mode 100644 benchmarks/formal_pr_bench/provenance/ci_secrets_exposed_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/ci_secrets_exposed_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/ci_secrets_safe.json create mode 100644 benchmarks/formal_pr_bench/provenance/ci_secrets_safe_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/ci_secrets_safe_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/control_metadata_missing.json create mode 100644 benchmarks/formal_pr_bench/provenance/control_metadata_missing_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/control_metadata_missing_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/control_preserved.json create mode 100644 benchmarks/formal_pr_bench/provenance/control_preserved_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/control_preserved_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/control_removed.json create mode 100644 benchmarks/formal_pr_bench/provenance/control_removed_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/control_removed_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/dafny_fail.json create mode 100644 benchmarks/formal_pr_bench/provenance/dafny_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/dafny_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/dafny_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/deployment_skipped_approval.json create mode 100644 benchmarks/formal_pr_bench/provenance/deployment_skipped_approval_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/deployment_skipped_approval_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/deployment_valid_path.json create mode 100644 benchmarks/formal_pr_bench/provenance/deployment_valid_path_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/deployment_valid_path_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/infra_private_sensitive.json create mode 100644 benchmarks/formal_pr_bench/provenance/infra_private_sensitive_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/infra_private_sensitive_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/infra_public_sensitive.json create mode 100644 benchmarks/formal_pr_bench/provenance/infra_public_sensitive_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/infra_public_sensitive_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_fail.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_fail_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_malformed.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_malformed_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_malformed_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_pass_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_unknown.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_unknown_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/kani_unknown_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/lean_fail.json create mode 100644 benchmarks/formal_pr_bench/provenance/lean_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/lean_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/lean_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/multi_surface_combined_pr.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_auth_admin_route_guarded.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_auth_admin_route_unguarded.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_auth_route_partial_hunk.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_cbmc_integer_overflow_quota.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_cbmc_use_after_free_auth_cache.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_ci_secrets_pr_preview.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_ci_secrets_workflow_dispatch_safe.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_deployment_direct_skip.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_deployment_valid_chain.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_docs_only_change.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_infra_iam_wildcard_admin.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_infra_k8s_loadbalancer.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_infra_private_bucket.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_infra_rds_public_partial_hunk.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_infra_s3_public_acl.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_multi_surface_combined.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_self_protection_workflow_touch.json create mode 100644 benchmarks/formal_pr_bench/provenance/rd_workflow_secrets_partial_hunk.json create mode 100644 benchmarks/formal_pr_bench/provenance/recall_ci_secrets_workflow_diff.json create mode 100644 benchmarks/formal_pr_bench/provenance/recall_infra_terraform_diff.json create mode 100644 benchmarks/formal_pr_bench/provenance/recall_multi_surface_combined.json create mode 100644 benchmarks/formal_pr_bench/provenance/repair_loop_auth_bypass.json create mode 100644 benchmarks/formal_pr_bench/provenance/repair_loop_ci_secrets.json create mode 100644 benchmarks/formal_pr_bench/provenance/repair_loop_deployment_skip.json create mode 100644 benchmarks/formal_pr_bench/provenance/repair_loop_infra_exposure.json create mode 100644 benchmarks/formal_pr_bench/provenance/route_alloy_model.json create mode 100644 benchmarks/formal_pr_bench/provenance/route_cedar_iam.json create mode 100644 benchmarks/formal_pr_bench/provenance/route_dafny_proof.json create mode 100644 benchmarks/formal_pr_bench/provenance/route_kani_rust.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_fail.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_fail_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_malformed.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_malformed_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_malformed_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_pass_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_unknown.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_unknown_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/tla_unknown_variant_2.json create mode 100644 benchmarks/formal_pr_bench/provenance/verus_fail.json create mode 100644 benchmarks/formal_pr_bench/provenance/verus_fail_variant_1.json create mode 100644 benchmarks/formal_pr_bench/provenance/verus_pass.json create mode 100644 benchmarks/formal_pr_bench/provenance/verus_pass_variant_1.json create mode 100644 benchmarks/formal_pr_bench/rationales/adversarial_forged_allow.md create mode 100644 benchmarks/formal_pr_bench/rationales/adversarial_sha_mismatch.md create mode 100644 benchmarks/formal_pr_bench/rationales/alloy_fail.md create mode 100644 benchmarks/formal_pr_bench/rationales/alloy_fail_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/alloy_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/alloy_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/auth_bypass.md create mode 100644 benchmarks/formal_pr_bench/rationales/auth_bypass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/auth_bypass_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/auth_malformed.md create mode 100644 benchmarks/formal_pr_bench/rationales/auth_malformed_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/auth_malformed_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/auth_preserved.md create mode 100644 benchmarks/formal_pr_bench/rationales/auth_preserved_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/auth_preserved_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_fail.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_fail_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_native_buffer_bounds_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_native_buffer_bounds_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_native_integer_overflow_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_native_integer_overflow_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_native_uaf_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_native_uaf_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_native_unchecked_copy_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_native_unchecked_copy_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/cbmc_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_fail.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_fail_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_fail_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_malformed.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_malformed_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_malformed_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_pass_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_unknown.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_unknown_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/cedar_unknown_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/ci_secrets_exposed.md create mode 100644 benchmarks/formal_pr_bench/rationales/ci_secrets_exposed_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/ci_secrets_exposed_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/ci_secrets_safe.md create mode 100644 benchmarks/formal_pr_bench/rationales/ci_secrets_safe_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/ci_secrets_safe_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/control_metadata_missing.md create mode 100644 benchmarks/formal_pr_bench/rationales/control_metadata_missing_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/control_metadata_missing_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/control_preserved.md create mode 100644 benchmarks/formal_pr_bench/rationales/control_preserved_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/control_preserved_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/control_removed.md create mode 100644 benchmarks/formal_pr_bench/rationales/control_removed_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/control_removed_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/dafny_fail.md create mode 100644 benchmarks/formal_pr_bench/rationales/dafny_fail_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/dafny_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/dafny_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/deployment_skipped_approval.md create mode 100644 benchmarks/formal_pr_bench/rationales/deployment_skipped_approval_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/deployment_skipped_approval_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/deployment_valid_path.md create mode 100644 benchmarks/formal_pr_bench/rationales/deployment_valid_path_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/deployment_valid_path_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/infra_private_sensitive.md create mode 100644 benchmarks/formal_pr_bench/rationales/infra_private_sensitive_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/infra_private_sensitive_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/infra_public_sensitive.md create mode 100644 benchmarks/formal_pr_bench/rationales/infra_public_sensitive_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/infra_public_sensitive_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_fail.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_fail_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_fail_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_malformed.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_malformed_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_malformed_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_pass_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_unknown.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_unknown_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/kani_unknown_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/lean_fail.md create mode 100644 benchmarks/formal_pr_bench/rationales/lean_fail_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/lean_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/lean_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/multi_surface_combined_pr.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_auth_admin_route_guarded.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_auth_admin_route_unguarded.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_auth_route_partial_hunk.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_cbmc_integer_overflow_quota.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_cbmc_use_after_free_auth_cache.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_ci_secrets_pr_preview.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_ci_secrets_workflow_dispatch_safe.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_deployment_direct_skip.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_deployment_valid_chain.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_docs_only_change.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_infra_iam_wildcard_admin.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_infra_k8s_loadbalancer.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_infra_private_bucket.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_infra_rds_public_partial_hunk.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_infra_s3_public_acl.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_multi_surface_combined.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_self_protection_workflow_touch.md create mode 100644 benchmarks/formal_pr_bench/rationales/rd_workflow_secrets_partial_hunk.md create mode 100644 benchmarks/formal_pr_bench/rationales/recall_ci_secrets_workflow_diff.md create mode 100644 benchmarks/formal_pr_bench/rationales/recall_infra_terraform_diff.md create mode 100644 benchmarks/formal_pr_bench/rationales/recall_multi_surface_combined.md create mode 100644 benchmarks/formal_pr_bench/rationales/repair_loop_auth_bypass.md create mode 100644 benchmarks/formal_pr_bench/rationales/repair_loop_ci_secrets.md create mode 100644 benchmarks/formal_pr_bench/rationales/repair_loop_deployment_skip.md create mode 100644 benchmarks/formal_pr_bench/rationales/repair_loop_infra_exposure.md create mode 100644 benchmarks/formal_pr_bench/rationales/route_alloy_model.md create mode 100644 benchmarks/formal_pr_bench/rationales/route_cedar_iam.md create mode 100644 benchmarks/formal_pr_bench/rationales/route_dafny_proof.md create mode 100644 benchmarks/formal_pr_bench/rationales/route_kani_rust.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_fail.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_fail_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_fail_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_malformed.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_malformed_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_malformed_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_pass_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_unknown.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_unknown_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/tla_unknown_variant_2.md create mode 100644 benchmarks/formal_pr_bench/rationales/verus_fail.md create mode 100644 benchmarks/formal_pr_bench/rationales/verus_fail_variant_1.md create mode 100644 benchmarks/formal_pr_bench/rationales/verus_pass.md create mode 100644 benchmarks/formal_pr_bench/rationales/verus_pass_variant_1.md create mode 100644 benchmarks/formal_pr_bench/template_dev_cases.json create mode 100644 schemas/formal_pr_bench.manifest.schema.json diff --git a/benchmarks/formal_pr_bench/adversarial/forged_allow_label.json b/benchmarks/formal_pr_bench/adversarial/forged_allow_label.json new file mode 100644 index 0000000..ac53c9b --- /dev/null +++ b/benchmarks/formal_pr_bench/adversarial/forged_allow_label.json @@ -0,0 +1,10 @@ +{ + "schema_version": "formal_pr_bench.adversarial.v1", + "adversarial_id": "forged_allow_label", + "description": "Case metadata claims allow while pointing at a known-failing auth bypass fixture.", + "input_fixture": "examples/auth_regression/input_admin_bypass.json", + "claimed_expected_merge_recommendation": "allow", + "actual_expected_merge_recommendation": "block", + "intended_trap": "Trusting attacker-supplied expected labels without fixture evaluation.", + "not_held_out": true +} diff --git a/benchmarks/formal_pr_bench/adversarial/misleading_docs_hides_secret.diff b/benchmarks/formal_pr_bench/adversarial/misleading_docs_hides_secret.diff new file mode 100644 index 0000000..5a0d5a0 --- /dev/null +++ b/benchmarks/formal_pr_bench/adversarial/misleading_docs_hides_secret.diff @@ -0,0 +1,16 @@ +diff --git a/README.md b/README.md +--- a/README.md ++++ b/README.md +@@ -1,3 +1,4 @@ + # Example Repo + + Documentation update only. ++Also tweaks CI below (see workflow). +diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml +--- a/.github/workflows/ci.yml ++++ b/.github/workflows/ci.yml +@@ -10,6 +10,8 @@ jobs: + steps: + - uses: actions/checkout@v4 ++ - name: Preview ++ run: echo ${{ secrets.PROD_TOKEN }} diff --git a/benchmarks/formal_pr_bench/adversarial/misleading_docs_hides_secret.json b/benchmarks/formal_pr_bench/adversarial/misleading_docs_hides_secret.json new file mode 100644 index 0000000..2137ab2 --- /dev/null +++ b/benchmarks/formal_pr_bench/adversarial/misleading_docs_hides_secret.json @@ -0,0 +1,13 @@ +{ + "schema_version": "formal_pr_bench.adversarial.v1", + "adversarial_id": "misleading_docs_hides_secret", + "diff": "misleading_docs_hides_secret.diff", + "intended_trap": "Looks like a docs-only PR in the commit message/path mix but injects an untrusted secret use.", + "expected_merge_recommendation": "block", + "expected_lanes": [ + "ci_secrets", + "self_protection" + ], + "not_held_out": true, + "notes": "Adversarial fixture for robustness; must not be counted as held-out evaluation." +} diff --git a/benchmarks/formal_pr_bench/duplication_report.json b/benchmarks/formal_pr_bench/duplication_report.json new file mode 100644 index 0000000..5b83a2a --- /dev/null +++ b/benchmarks/formal_pr_bench/duplication_report.json @@ -0,0 +1,631 @@ +{ + "schema_version": "formal_pr_bench.duplication_report.v1", + "benchmark_version": "v1", + "method": { + "fixture_path_collision": "exact", + "outcome_fingerprint": "sha256 over selected expectation fields", + "real_diff_text": "token Jaccard >= 0.85 on normalized unified diffs" + }, + "fixture_path_duplicates": [ + { + "fixture": "examples/auth_regression/input_admin_bypass.json", + "case_ids": [ + "auth_bypass", + "auth_bypass_variant_1", + "auth_bypass_variant_2" + ] + }, + { + "fixture": "examples/auth_regression/input_admin_protected.json", + "case_ids": [ + "auth_preserved", + "auth_preserved_variant_1", + "auth_preserved_variant_2" + ] + }, + { + "fixture": "examples/auth_regression/input_malformed_missing_routes.json", + "case_ids": [ + "auth_malformed", + "auth_malformed_variant_1", + "auth_malformed_variant_2" + ] + }, + { + "fixture": "examples/backends/alloy_fail.json", + "case_ids": [ + "alloy_fail", + "alloy_fail_variant_1" + ] + }, + { + "fixture": "examples/backends/alloy_pass.json", + "case_ids": [ + "alloy_pass", + "alloy_pass_variant_1" + ] + }, + { + "fixture": "examples/backends/cbmc_fail.json", + "case_ids": [ + "cbmc_fail", + "cbmc_fail_variant_1" + ] + }, + { + "fixture": "examples/backends/cbmc_native_buffer_bounds_pass.json", + "case_ids": [ + "cbmc_native_buffer_bounds_pass", + "cbmc_native_buffer_bounds_pass_variant_1" + ] + }, + { + "fixture": "examples/backends/cbmc_native_integer_overflow_pass.json", + "case_ids": [ + "cbmc_native_integer_overflow_pass", + "cbmc_native_integer_overflow_pass_variant_1" + ] + }, + { + "fixture": "examples/backends/cbmc_native_uaf_pass.json", + "case_ids": [ + "cbmc_native_uaf_pass", + "cbmc_native_uaf_pass_variant_1" + ] + }, + { + "fixture": "examples/backends/cbmc_native_unchecked_copy_pass.json", + "case_ids": [ + "cbmc_native_unchecked_copy_pass", + "cbmc_native_unchecked_copy_pass_variant_1" + ] + }, + { + "fixture": "examples/backends/cbmc_pass.json", + "case_ids": [ + "cbmc_pass", + "cbmc_pass_variant_1" + ] + }, + { + "fixture": "examples/backends/cedar_fail.json", + "case_ids": [ + "cedar_fail", + "cedar_fail_variant_1", + "cedar_fail_variant_2" + ] + }, + { + "fixture": "examples/backends/cedar_malformed.json", + "case_ids": [ + "cedar_malformed", + "cedar_malformed_variant_1", + "cedar_malformed_variant_2" + ] + }, + { + "fixture": "examples/backends/cedar_pass.json", + "case_ids": [ + "cedar_pass", + "cedar_pass_variant_1", + "cedar_pass_variant_2" + ] + }, + { + "fixture": "examples/backends/cedar_unknown.json", + "case_ids": [ + "cedar_unknown", + "cedar_unknown_variant_1", + "cedar_unknown_variant_2" + ] + }, + { + "fixture": "examples/backends/dafny_fail.json", + "case_ids": [ + "dafny_fail", + "dafny_fail_variant_1" + ] + }, + { + "fixture": "examples/backends/dafny_pass.json", + "case_ids": [ + "dafny_pass", + "dafny_pass_variant_1" + ] + }, + { + "fixture": "examples/backends/kani_fail.json", + "case_ids": [ + "kani_fail", + "kani_fail_variant_1", + "kani_fail_variant_2" + ] + }, + { + "fixture": "examples/backends/kani_malformed.json", + "case_ids": [ + "kani_malformed", + "kani_malformed_variant_1", + "kani_malformed_variant_2" + ] + }, + { + "fixture": "examples/backends/kani_pass.json", + "case_ids": [ + "kani_pass", + "kani_pass_variant_1", + "kani_pass_variant_2" + ] + }, + { + "fixture": "examples/backends/kani_unknown.json", + "case_ids": [ + "kani_unknown", + "kani_unknown_variant_1", + "kani_unknown_variant_2" + ] + }, + { + "fixture": "examples/backends/lean_fail.json", + "case_ids": [ + "lean_fail", + "lean_fail_variant_1" + ] + }, + { + "fixture": "examples/backends/lean_pass.json", + "case_ids": [ + "lean_pass", + "lean_pass_variant_1" + ] + }, + { + "fixture": "examples/backends/tla_fail.json", + "case_ids": [ + "tla_fail", + "tla_fail_variant_1", + "tla_fail_variant_2" + ] + }, + { + "fixture": "examples/backends/tla_malformed.json", + "case_ids": [ + "tla_malformed", + "tla_malformed_variant_1", + "tla_malformed_variant_2" + ] + }, + { + "fixture": "examples/backends/tla_pass.json", + "case_ids": [ + "tla_pass", + "tla_pass_variant_1", + "tla_pass_variant_2" + ] + }, + { + "fixture": "examples/backends/tla_unknown.json", + "case_ids": [ + "tla_unknown", + "tla_unknown_variant_1", + "tla_unknown_variant_2" + ] + }, + { + "fixture": "examples/backends/verus_fail.json", + "case_ids": [ + "verus_fail", + "verus_fail_variant_1" + ] + }, + { + "fixture": "examples/backends/verus_pass.json", + "case_ids": [ + "verus_pass", + "verus_pass_variant_1" + ] + }, + { + "fixture": "examples/ci_secrets/input_secrets_exposed.json", + "case_ids": [ + "ci_secrets_exposed", + "ci_secrets_exposed_variant_1", + "ci_secrets_exposed_variant_2" + ] + }, + { + "fixture": "examples/ci_secrets/input_secrets_safe.json", + "case_ids": [ + "ci_secrets_safe", + "ci_secrets_safe_variant_1", + "ci_secrets_safe_variant_2" + ] + }, + { + "fixture": "examples/deployment_state/input_skipped_approval.json", + "case_ids": [ + "deployment_skipped_approval", + "deployment_skipped_approval_variant_1", + "deployment_skipped_approval_variant_2" + ] + }, + { + "fixture": "examples/deployment_state/input_valid_approval_path.json", + "case_ids": [ + "deployment_valid_path", + "deployment_valid_path_variant_1", + "deployment_valid_path_variant_2" + ] + }, + { + "fixture": "examples/infrastructure_exposure/input_private_sensitive_resource.json", + "case_ids": [ + "infra_private_sensitive", + "infra_private_sensitive_variant_1", + "infra_private_sensitive_variant_2" + ] + }, + { + "fixture": "examples/infrastructure_exposure/input_public_sensitive_resource.json", + "case_ids": [ + "infra_public_sensitive", + "infra_public_sensitive_variant_1", + "infra_public_sensitive_variant_2" + ] + }, + { + "fixture": "examples/multi_surface/pr_combined.diff", + "case_ids": [ + "multi_surface_combined_pr", + "recall_multi_surface_combined" + ] + }, + { + "fixture": "examples/no_agent_self_approval/input_gate_preserved.json", + "case_ids": [ + "control_preserved", + "control_preserved_variant_1", + "control_preserved_variant_2" + ] + }, + { + "fixture": "examples/no_agent_self_approval/input_gate_removed.json", + "case_ids": [ + "control_removed", + "control_removed_variant_1", + "control_removed_variant_2" + ] + }, + { + "fixture": "examples/no_agent_self_approval/input_missing_metadata.json", + "case_ids": [ + "control_metadata_missing", + "control_metadata_missing_variant_1", + "control_metadata_missing_variant_2" + ] + } + ], + "outcome_near_duplicates": [ + { + "fingerprint": "043b4798d055dd048639ca0bf33d871e5829644565e37ff5e97a4d248737f600", + "case_ids": [ + "kani_fail", + "kani_fail_variant_1", + "kani_fail_variant_2" + ] + }, + { + "fingerprint": "0e77b7edcddd6529c16fa737cfe8ad4ffa2ec09ca20d403474e35f74c461906d", + "case_ids": [ + "tla_pass", + "tla_pass_variant_1", + "tla_pass_variant_2" + ] + }, + { + "fingerprint": "197bc5fde6d6bf6097b2bab3c935e53071241b430c10c9524f3399442cfbdfb5", + "case_ids": [ + "lean_fail", + "lean_fail_variant_1" + ] + }, + { + "fingerprint": "266504e9b13f303af716d30b6d622efb8115b92e447c9b331b5949c1cccbdd98", + "case_ids": [ + "cedar_fail", + "cedar_fail_variant_1", + "cedar_fail_variant_2" + ] + }, + { + "fingerprint": "2808e2e15ecd72ec73372204403b2492a8f2e8bd528ce3de465f5477ef6df601", + "case_ids": [ + "auth_preserved", + "auth_preserved_variant_1", + "auth_preserved_variant_2" + ] + }, + { + "fingerprint": "289ebeb734fce28babb4966f4ed3a115177d305996db492f629dad986205cf77", + "case_ids": [ + "control_removed", + "control_removed_variant_1", + "control_removed_variant_2" + ] + }, + { + "fingerprint": "2cdfc1c1b41b8dc5cc5b4db62204c02b272ec45cd24dbfbd89e4ede8f64037bb", + "case_ids": [ + "kani_malformed", + "kani_malformed_variant_1", + "kani_malformed_variant_2" + ] + }, + { + "fingerprint": "2f40724f281a035b6fb3f8dc02c625e43f02220b54ce38e17e7b5b7ba106ca21", + "case_ids": [ + "dafny_pass", + "dafny_pass_variant_1" + ] + }, + { + "fingerprint": "36b3fb322fa047071dd7e8942f35701de6cd07930fec8fc4a6d0a0fac351f120", + "case_ids": [ + "auth_malformed", + "auth_malformed_variant_1", + "auth_malformed_variant_2" + ] + }, + { + "fingerprint": "49e34369a6d84ea2d34a4b35cc2a107ba431f73e0042ca30ab993659c5949c03", + "case_ids": [ + "verus_fail", + "verus_fail_variant_1" + ] + }, + { + "fingerprint": "53471e8a926dc2a0ada741025e05f46be24bd2703df975affcbbdd2344d995c8", + "case_ids": [ + "kani_unknown", + "kani_unknown_variant_1", + "kani_unknown_variant_2" + ] + }, + { + "fingerprint": "565dc78b30156718ed8772ec01ae8a78c81409268da24b34555918fabc07d456", + "case_ids": [ + "infra_public_sensitive", + "infra_public_sensitive_variant_1", + "infra_public_sensitive_variant_2" + ] + }, + { + "fingerprint": "57bc9f8a6beb5318b5ee0f4adaf3e36620b9cbcf310bf230dcb512aaf251d328", + "case_ids": [ + "cedar_pass", + "cedar_pass_variant_1", + "cedar_pass_variant_2" + ] + }, + { + "fingerprint": "64ac5d9130e63cec5d3a190abb97e7d998d76106f52909430470f459b2600ccf", + "case_ids": [ + "cbmc_native_unchecked_copy_pass", + "cbmc_native_unchecked_copy_pass_variant_1" + ] + }, + { + "fingerprint": "6825ec5f920d1dd77ac844dc4244742dacf57d83e674e1c0a718ea49c72a0299", + "case_ids": [ + "deployment_skipped_approval", + "deployment_skipped_approval_variant_1", + "deployment_skipped_approval_variant_2" + ] + }, + { + "fingerprint": "6af8ca43084fec42674341788bad573d32b7683a857ce9657cac8134fffa0c18", + "case_ids": [ + "ci_secrets_exposed", + "ci_secrets_exposed_variant_1", + "ci_secrets_exposed_variant_2" + ] + }, + { + "fingerprint": "6e53b424f8bbbcbf3d57d7cb864a78ada6e843a82c29ad80b326e029f10d15b8", + "case_ids": [ + "control_metadata_missing", + "control_metadata_missing_variant_1", + "control_metadata_missing_variant_2" + ] + }, + { + "fingerprint": "6e984c25d11073694021a007eedcd0831899be6e972ce4664f9bae5d0f808453", + "case_ids": [ + "lean_pass", + "lean_pass_variant_1" + ] + }, + { + "fingerprint": "71b7998f55e7f923b972ccffdad5d69fabc7b31c552e10b424298af6d5e75bd6", + "case_ids": [ + "control_preserved", + "control_preserved_variant_1", + "control_preserved_variant_2" + ] + }, + { + "fingerprint": "94c9760b9dcad1b4fd9c3fe62a0b3e727598ede0e1c6402b405aab32b8e250e0", + "case_ids": [ + "cbmc_native_buffer_bounds_pass", + "cbmc_native_buffer_bounds_pass_variant_1" + ] + }, + { + "fingerprint": "9f12dc52449c75ed94354304d66735757000cd295dcbeebde4abd9139bb32aad", + "case_ids": [ + "tla_unknown", + "tla_unknown_variant_1", + "tla_unknown_variant_2" + ] + }, + { + "fingerprint": "a50b8ff467857b62fa2ffee18efb93cf34941f47149db359ef2e64c80743cc0b", + "case_ids": [ + "alloy_fail", + "alloy_fail_variant_1" + ] + }, + { + "fingerprint": "a827abc0cf2a41984fb1cfa20676858cb147eca72c245fb66041c86c12d7219d", + "case_ids": [ + "alloy_pass", + "alloy_pass_variant_1" + ] + }, + { + "fingerprint": "a8a0c6c9938631ba8235702d32e0f8b881026637efe0bf0d3d5d9a0acdb6de11", + "case_ids": [ + "auth_bypass", + "auth_bypass_variant_1", + "auth_bypass_variant_2" + ] + }, + { + "fingerprint": "b260621c384abfe27e8335fe0ddcdef624b09525301a377c0f48e1e2e7d9c7d0", + "case_ids": [ + "tla_malformed", + "tla_malformed_variant_1", + "tla_malformed_variant_2" + ] + }, + { + "fingerprint": "be08cbbc374fe975daf3cf7f333997ddf67f841a8f0e1dce82f356e68e0830df", + "case_ids": [ + "dafny_fail", + "dafny_fail_variant_1" + ] + }, + { + "fingerprint": "be908a6692acaa5354d532a71bb40784d66b4809a862ecc736ae335ddbae5428", + "case_ids": [ + "deployment_valid_path", + "deployment_valid_path_variant_1", + "deployment_valid_path_variant_2" + ] + }, + { + "fingerprint": "cde1df5205315274dedf956b4ad003ff2645566752810f25b4fd7cfaaa239a68", + "case_ids": [ + "cbmc_native_integer_overflow_pass", + "cbmc_native_integer_overflow_pass_variant_1" + ] + }, + { + "fingerprint": "d819e0585505c710308e48b58e2b15702d41326b6f749b7e259207a4505098db", + "case_ids": [ + "infra_private_sensitive", + "infra_private_sensitive_variant_1", + "infra_private_sensitive_variant_2" + ] + }, + { + "fingerprint": "d8e93317f6c006f40bc5510b3be8e639b4af4c2c3ee2b5840fa9a498483623e6", + "case_ids": [ + "cbmc_pass", + "cbmc_pass_variant_1" + ] + }, + { + "fingerprint": "dd0d00f176d3940f66a68ad99c81b69a11a24de19b3e3e080252a55797bfd651", + "case_ids": [ + "cbmc_fail", + "cbmc_fail_variant_1" + ] + }, + { + "fingerprint": "e266ae5dace85f9d91897e6aaa75a0b8e79ed2bd7e026019172b7685b7aa9361", + "case_ids": [ + "tla_fail", + "tla_fail_variant_1", + "tla_fail_variant_2" + ] + }, + { + "fingerprint": "e4cada696c5440f57a4f5535eaeda81c8ff7abfcaa50d65c95e0aec72a6aa35c", + "case_ids": [ + "kani_pass", + "kani_pass_variant_1", + "kani_pass_variant_2" + ] + }, + { + "fingerprint": "e62522616303c39c7fb87e59353b7fc04d28873dcd44880a2a4eef2731ed283d", + "case_ids": [ + "cbmc_native_uaf_pass", + "cbmc_native_uaf_pass_variant_1" + ] + }, + { + "fingerprint": "e9152171f47b86ae6782aa5fd7f00e91543193f2faaa726315b81a05d9d33330", + "case_ids": [ + "ci_secrets_safe", + "ci_secrets_safe_variant_1", + "ci_secrets_safe_variant_2" + ] + }, + { + "fingerprint": "eb150d10105ad85b069c3ebd43fee41ca7ea94d3f721dbaf15afab20d9b26a34", + "case_ids": [ + "cedar_malformed", + "cedar_malformed_variant_1", + "cedar_malformed_variant_2" + ] + }, + { + "fingerprint": "f5733bdf5014f472d006914ab177b9608e537822ad27660b2292dc177160bfe4", + "case_ids": [ + "verus_pass", + "verus_pass_variant_1" + ] + }, + { + "fingerprint": "fe406be62d4dfb0425ed1744cdf776af860f7dc032a8ecb1a057846b29df7ce7", + "case_ids": [ + "cedar_unknown", + "cedar_unknown_variant_1", + "cedar_unknown_variant_2" + ] + } + ], + "real_diff_text_near_duplicates": [ + { + "case_ids": [ + "rd_auth_admin_route_guarded", + "rd_auth_admin_route_unguarded" + ], + "jaccard": 0.95, + "threshold": 0.85 + }, + { + "case_ids": [ + "rd_ci_secrets_pr_preview", + "rd_workflow_secrets_partial_hunk" + ], + "jaccard": 1.0, + "threshold": 0.85 + }, + { + "case_ids": [ + "rd_deployment_direct_skip", + "rd_deployment_valid_chain" + ], + "jaccard": 0.963, + "threshold": 0.85 + } + ], + "summary": { + "fixture_path_duplicate_groups": 39, + "outcome_near_duplicate_groups": 38, + "real_diff_text_near_duplicate_pairs": 3 + } +} diff --git a/benchmarks/formal_pr_bench/held_out/README.md b/benchmarks/formal_pr_bench/held_out/README.md new file mode 100644 index 0000000..0e8261d --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/README.md @@ -0,0 +1,8 @@ +# FormalPR-Bench held-out cases + +Cases in this directory belong to the `held_out` partition. +They must not be used during property-template development scoring. + +Contamination between `template_dev_cases.json` and this partition fails CI. +See [docs/HOLDOUT_LABEL_SEPARATION.md](../../../docs/HOLDOUT_LABEL_SEPARATION.md) and +[docs/FORMALPR_HOLDOUT_GOVERNANCE.md](../../../docs/FORMALPR_HOLDOUT_GOVERNANCE.md). diff --git a/benchmarks/formal_pr_bench/held_out/alloy_fail_variant_1.json b/benchmarks/formal_pr_bench/held_out/alloy_fail_variant_1.json new file mode 100644 index 0000000..e8c8be5 --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/alloy_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": "alloy_fail_variant_1", + "forbidden_from_template_dev_scoring": true, + "case": { + "case_id": "alloy_fail_variant_1", + "input_fixture": "examples/backends/alloy_fail.json", + "expected_intent": "alloy-model-check", + "expected_backend_class": "model_checker", + "expected_status": "fail", + "expected_merge_recommendation": "block", + "expected_counterexample_class": "alloy_counterexample_found" + } +} diff --git a/benchmarks/formal_pr_bench/held_out/auth_bypass_variant_2.json b/benchmarks/formal_pr_bench/held_out/auth_bypass_variant_2.json new file mode 100644 index 0000000..bc41e80 --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/auth_bypass_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": "auth_bypass_variant_2", + "forbidden_from_template_dev_scoring": true, + "case": { + "case_id": "auth_bypass_variant_2", + "input_fixture": "examples/auth_regression/input_admin_bypass.json", + "expected_intent": "no-admin-route-bypass", + "expected_backend_class": "smt_solver", + "expected_status": "fail", + "expected_merge_recommendation": "block", + "expected_counterexample_class": "admin_route_reachable_by_non_admin" + } +} diff --git a/benchmarks/formal_pr_bench/held_out/cedar_malformed_variant_2.json b/benchmarks/formal_pr_bench/held_out/cedar_malformed_variant_2.json new file mode 100644 index 0000000..9a7edb5 --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/cedar_malformed_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": "cedar_malformed_variant_2", + "forbidden_from_template_dev_scoring": true, + "case": { + "case_id": "cedar_malformed_variant_2", + "input_fixture": "examples/backends/cedar_malformed.json", + "expected_intent": "cedar-policy-check", + "expected_backend_class": "policy_engine", + "expected_status": "unknown", + "expected_merge_recommendation": "require_human_review", + "expected_counterexample_class": "malformed_input" + } +} diff --git a/benchmarks/formal_pr_bench/held_out/kani_unknown_variant_2.json b/benchmarks/formal_pr_bench/held_out/kani_unknown_variant_2.json new file mode 100644 index 0000000..59bf4eb --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/kani_unknown_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": "kani_unknown_variant_2", + "forbidden_from_template_dev_scoring": true, + "case": { + "case_id": "kani_unknown_variant_2", + "input_fixture": "examples/backends/kani_unknown.json", + "expected_intent": "kani-harness-check", + "expected_backend_class": "model_checker", + "expected_status": "unknown", + "expected_merge_recommendation": "require_human_review", + "expected_counterexample_class": "malformed_input" + } +} diff --git a/benchmarks/formal_pr_bench/held_out/lean_fail_variant_1.json b/benchmarks/formal_pr_bench/held_out/lean_fail_variant_1.json new file mode 100644 index 0000000..a94513b --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/lean_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": "lean_fail_variant_1", + "forbidden_from_template_dev_scoring": true, + "case": { + "case_id": "lean_fail_variant_1", + "input_fixture": "examples/backends/lean_fail.json", + "expected_intent": "lean-proof-check", + "expected_backend_class": "proof_assistant", + "expected_status": "fail", + "expected_merge_recommendation": "block", + "expected_counterexample_class": "lean_proof_failed" + } +} diff --git a/benchmarks/formal_pr_bench/held_out/rd_cbmc_integer_overflow_quota.json b/benchmarks/formal_pr_bench/held_out/rd_cbmc_integer_overflow_quota.json new file mode 100644 index 0000000..8ffa9a6 --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/rd_cbmc_integer_overflow_quota.json @@ -0,0 +1,19 @@ +{ + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": "rd_cbmc_integer_overflow_quota", + "forbidden_from_template_dev_scoring": true, + "case": { + "case_id": "rd_cbmc_integer_overflow_quota", + "category": "real_diff", + "input_fixture": "benchmarks/real_diffs/cbmc_integer_overflow_quota.diff", + "expected_lanes": [ + "backend" + ], + "expected_intents": [ + "cbmc-no-integer-overflow-quota", + "cbmc-buffer-bounds", + "cbmc-no-unchecked-buffer-copy" + ], + "expected_merge_recommendation": "block" + } +} diff --git a/benchmarks/formal_pr_bench/held_out/rd_cbmc_use_after_free_auth_cache.json b/benchmarks/formal_pr_bench/held_out/rd_cbmc_use_after_free_auth_cache.json new file mode 100644 index 0000000..17e1a90 --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/rd_cbmc_use_after_free_auth_cache.json @@ -0,0 +1,19 @@ +{ + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": "rd_cbmc_use_after_free_auth_cache", + "forbidden_from_template_dev_scoring": true, + "case": { + "case_id": "rd_cbmc_use_after_free_auth_cache", + "category": "real_diff", + "input_fixture": "benchmarks/real_diffs/cbmc_use_after_free_auth_cache.diff", + "expected_lanes": [ + "backend" + ], + "expected_intents": [ + "cbmc-no-use-after-free-auth-cache", + "cbmc-buffer-bounds", + "cbmc-no-unchecked-buffer-copy" + ], + "expected_merge_recommendation": "block" + } +} diff --git a/benchmarks/formal_pr_bench/held_out/rd_docs_only_change.json b/benchmarks/formal_pr_bench/held_out/rd_docs_only_change.json new file mode 100644 index 0000000..e528627 --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/rd_docs_only_change.json @@ -0,0 +1,13 @@ +{ + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": "rd_docs_only_change", + "forbidden_from_template_dev_scoring": true, + "case": { + "case_id": "rd_docs_only_change", + "category": "real_diff", + "input_fixture": "benchmarks/real_diffs/docs_only_change.diff", + "expected_lanes": [], + "expected_intents": [], + "expected_merge_recommendation": "require_human_review" + } +} diff --git a/benchmarks/formal_pr_bench/held_out/tla_unknown_variant_2.json b/benchmarks/formal_pr_bench/held_out/tla_unknown_variant_2.json new file mode 100644 index 0000000..9792e94 --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/tla_unknown_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": "tla_unknown_variant_2", + "forbidden_from_template_dev_scoring": true, + "case": { + "case_id": "tla_unknown_variant_2", + "input_fixture": "examples/backends/tla_unknown.json", + "expected_intent": "tla-state-check", + "expected_backend_class": "model_checker", + "expected_status": "unknown", + "expected_merge_recommendation": "require_human_review", + "expected_counterexample_class": "malformed_state_machine" + } +} diff --git a/benchmarks/formal_pr_bench/held_out/verus_fail_variant_1.json b/benchmarks/formal_pr_bench/held_out/verus_fail_variant_1.json new file mode 100644 index 0000000..00120da --- /dev/null +++ b/benchmarks/formal_pr_bench/held_out/verus_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": "verus_fail_variant_1", + "forbidden_from_template_dev_scoring": true, + "case": { + "case_id": "verus_fail_variant_1", + "input_fixture": "examples/backends/verus_fail.json", + "expected_intent": "verus-harness-check", + "expected_backend_class": "proof_assistant", + "expected_status": "fail", + "expected_merge_recommendation": "block", + "expected_counterexample_class": "verus_proof_failed" + } +} diff --git a/benchmarks/formal_pr_bench/licenses.json b/benchmarks/formal_pr_bench/licenses.json new file mode 100644 index 0000000..1743104 --- /dev/null +++ b/benchmarks/formal_pr_bench/licenses.json @@ -0,0 +1,669 @@ +{ + "schema_version": "formal_pr_bench.licenses.v1", + "benchmark_version": "v1", + "corpus_license": "Apache-2.0", + "corpus_license_file": "LICENSE", + "default_case_license": "Apache-2.0", + "cases": { + "adversarial_forged_allow": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "adversarial_sha_mismatch": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "alloy_fail": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "alloy_fail_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "alloy_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "alloy_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "auth_bypass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "auth_bypass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "auth_bypass_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "auth_malformed": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "auth_malformed_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "auth_malformed_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "auth_preserved": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "auth_preserved_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "auth_preserved_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_fail": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_fail_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_native_buffer_bounds_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_native_buffer_bounds_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_native_integer_overflow_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_native_integer_overflow_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_native_uaf_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_native_uaf_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_native_unchecked_copy_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_native_unchecked_copy_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cbmc_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_fail": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_fail_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_fail_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_malformed": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_malformed_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_malformed_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_pass_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_unknown": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_unknown_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "cedar_unknown_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "ci_secrets_exposed": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "ci_secrets_exposed_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "ci_secrets_exposed_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "ci_secrets_safe": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "ci_secrets_safe_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "ci_secrets_safe_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "control_metadata_missing": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "control_metadata_missing_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "control_metadata_missing_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "control_preserved": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "control_preserved_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "control_preserved_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "control_removed": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "control_removed_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "control_removed_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "dafny_fail": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "dafny_fail_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "dafny_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "dafny_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "deployment_skipped_approval": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "deployment_skipped_approval_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "deployment_skipped_approval_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "deployment_valid_path": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "deployment_valid_path_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "deployment_valid_path_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "infra_private_sensitive": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "infra_private_sensitive_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "infra_private_sensitive_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "infra_public_sensitive": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "infra_public_sensitive_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "infra_public_sensitive_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_fail": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_fail_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_fail_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_malformed": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_malformed_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_malformed_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_pass_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_unknown": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_unknown_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "kani_unknown_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "lean_fail": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "lean_fail_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "lean_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "lean_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "multi_surface_combined_pr": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "rd_auth_admin_route_guarded": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_auth_admin_route_unguarded": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_auth_route_partial_hunk": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_cbmc_integer_overflow_quota": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_cbmc_use_after_free_auth_cache": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_ci_secrets_pr_preview": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_ci_secrets_workflow_dispatch_safe": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_deployment_direct_skip": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_deployment_valid_chain": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_docs_only_change": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_infra_iam_wildcard_admin": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_infra_k8s_loadbalancer": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_infra_private_bucket": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_infra_rds_public_partial_hunk": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_infra_s3_public_acl": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_multi_surface_combined": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_self_protection_workflow_touch": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "rd_workflow_secrets_partial_hunk": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json" + }, + "recall_ci_secrets_workflow_diff": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "recall_infra_terraform_diff": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "recall_multi_surface_combined": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "repair_loop_auth_bypass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "repair_loop_ci_secrets": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "repair_loop_deployment_skip": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "repair_loop_infra_exposure": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "route_alloy_model": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "route_cedar_iam": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "route_dafny_proof": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "route_kani_rust": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/extended_cases.json" + }, + "tla_fail": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_fail_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_fail_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_malformed": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_malformed_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_malformed_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_pass_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_unknown": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_unknown_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "tla_unknown_variant_2": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "verus_fail": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "verus_fail_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "verus_pass": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + }, + "verus_pass_variant_1": { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json" + } + } +} diff --git a/benchmarks/formal_pr_bench/manifest.v1.json b/benchmarks/formal_pr_bench/manifest.v1.json new file mode 100644 index 0000000..b43c1c2 --- /dev/null +++ b/benchmarks/formal_pr_bench/manifest.v1.json @@ -0,0 +1,42 @@ +{ + "schema_version": "formal_pr_bench.manifest.v1", + "benchmark_version": "v1", + "case_count": 132, + "partitions": [ + "train", + "development", + "test", + "held_out" + ], + "partition_digests": { + "train": "49fe037304d3079d02b7a74c70769a909588dd47ec0a1c7f1c77d1c367da0f08", + "development": "cbfdf09cccde76211274127a8dd1d900d0ebbca64949dde67a82fb531ad15258", + "test": "be82773e68d8039f903307e769a44322dc1447d64fae57a70beb625fdbd1ea7b", + "held_out": "9cf3f23408841a79692c07694f7e72f3fc7e73b2d1d9de4924bdf5f81c7c1c91" + }, + "corpus_digest": "a2614ab01ddceff43fef508f6f0519b16474f94084a38f1ddc6e84aab92d347e", + "licenses_digest": "c9f88a6e5bce11b3ed75cf2f46d6d9f63e63a1ac7d0fe99a409fd4c29c5cde98", + "duplication_report_digest": "6ed405025a47c7621c5cf69918acf79de07ef2350e39936b62a99ac5a0bfef10", + "artifact_layout": { + "provenance": "provenance/.json", + "licenses": "licenses.json", + "partitions": "partitions.json", + "duplication_report": "duplication_report.json", + "mutations": "mutations/", + "held_out": "held_out/", + "adversarial": "adversarial/", + "rationales": "rationales/.md", + "template_dev_cases": "template_dev_cases.json" + }, + "real_diff_case_count": 18, + "mutation_files": [ + "control_removed__flip_merge_to_allow.json", + "auth_bypass__drop_counterexample.json", + "ci_secrets_exposed__status_to_pass.json" + ], + "adversarial_files": [ + "misleading_docs_hides_secret.diff", + "misleading_docs_hides_secret.json", + "forged_allow_label.json" + ] +} diff --git a/benchmarks/formal_pr_bench/mutations/auth_bypass__drop_counterexample.json b/benchmarks/formal_pr_bench/mutations/auth_bypass__drop_counterexample.json new file mode 100644 index 0000000..b0bd414 --- /dev/null +++ b/benchmarks/formal_pr_bench/mutations/auth_bypass__drop_counterexample.json @@ -0,0 +1,22 @@ +{ + "schema_version": "formal_pr_bench.mutation.v1", + "mutation_id": "auth_bypass__drop_counterexample", + "base_case_id": "auth_bypass", + "mutation": "drop_counterexample", + "description": "Controlled expectation mutation for contamination/robustness tests; not a scored corpus member.", + "base_case": { + "case_id": "auth_bypass", + "input_fixture": "examples/auth_regression/input_admin_bypass.json", + "expected_intent": "no-admin-route-bypass", + "expected_backend_class": "smt_solver", + "expected_status": "fail", + "expected_merge_recommendation": "block", + "expected_counterexample_class": "admin_route_reachable_by_non_admin" + }, + "overrides": { + "expected_counterexample_class": null + }, + "mutated_expectations": { + "expected_counterexample_class": null + } +} diff --git a/benchmarks/formal_pr_bench/mutations/ci_secrets_exposed__status_to_pass.json b/benchmarks/formal_pr_bench/mutations/ci_secrets_exposed__status_to_pass.json new file mode 100644 index 0000000..8588e18 --- /dev/null +++ b/benchmarks/formal_pr_bench/mutations/ci_secrets_exposed__status_to_pass.json @@ -0,0 +1,24 @@ +{ + "schema_version": "formal_pr_bench.mutation.v1", + "mutation_id": "ci_secrets_exposed__status_to_pass", + "base_case_id": "ci_secrets_exposed", + "mutation": "status_to_pass", + "description": "Controlled expectation mutation for contamination/robustness tests; not a scored corpus member.", + "base_case": { + "case_id": "ci_secrets_exposed", + "input_fixture": "examples/ci_secrets/input_secrets_exposed.json", + "expected_intent": "no-secrets-in-untrusted-context", + "expected_backend_class": "policy_engine", + "expected_status": "fail", + "expected_merge_recommendation": "block", + "expected_counterexample_class": "secrets_exposed_in_untrusted_context" + }, + "overrides": { + "expected_status": "pass", + "expected_merge_recommendation": "allow" + }, + "mutated_expectations": { + "expected_status": "pass", + "expected_merge_recommendation": "allow" + } +} diff --git a/benchmarks/formal_pr_bench/mutations/control_removed__flip_merge_to_allow.json b/benchmarks/formal_pr_bench/mutations/control_removed__flip_merge_to_allow.json new file mode 100644 index 0000000..5a7dec5 --- /dev/null +++ b/benchmarks/formal_pr_bench/mutations/control_removed__flip_merge_to_allow.json @@ -0,0 +1,22 @@ +{ + "schema_version": "formal_pr_bench.mutation.v1", + "mutation_id": "control_removed__flip_merge_to_allow", + "base_case_id": "control_removed", + "mutation": "flip_merge_to_allow", + "description": "Controlled expectation mutation for contamination/robustness tests; not a scored corpus member.", + "base_case": { + "case_id": "control_removed", + "input_fixture": "examples/no_agent_self_approval/input_gate_removed.json", + "expected_intent": "agent-cannot-disable-own-ci-gate", + "expected_backend_class": "policy_engine", + "expected_status": "fail", + "expected_merge_recommendation": "block", + "expected_counterexample_class": "required_check_removed" + }, + "overrides": { + "expected_merge_recommendation": "allow" + }, + "mutated_expectations": { + "expected_merge_recommendation": "allow" + } +} diff --git a/benchmarks/formal_pr_bench/partitions.json b/benchmarks/formal_pr_bench/partitions.json new file mode 100644 index 0000000..3d91765 --- /dev/null +++ b/benchmarks/formal_pr_bench/partitions.json @@ -0,0 +1,152 @@ +{ + "schema_version": "formal_pr_bench.partitions.v1", + "benchmark_version": "v1", + "partitions": { + "train": [ + "alloy_fail", + "alloy_pass", + "alloy_pass_variant_1", + "auth_bypass", + "auth_malformed", + "auth_malformed_variant_1", + "auth_preserved", + "auth_preserved_variant_1", + "cbmc_fail", + "cbmc_native_buffer_bounds_pass", + "cbmc_native_buffer_bounds_pass_variant_1", + "cbmc_native_integer_overflow_pass", + "cbmc_native_integer_overflow_pass_variant_1", + "cbmc_native_uaf_pass", + "cbmc_native_uaf_pass_variant_1", + "cbmc_native_unchecked_copy_pass", + "cbmc_native_unchecked_copy_pass_variant_1", + "cbmc_pass", + "cbmc_pass_variant_1", + "cedar_fail", + "cedar_fail_variant_1", + "cedar_malformed", + "cedar_malformed_variant_1", + "cedar_pass", + "cedar_pass_variant_1", + "cedar_unknown", + "cedar_unknown_variant_1", + "ci_secrets_exposed", + "ci_secrets_safe", + "ci_secrets_safe_variant_1", + "control_metadata_missing", + "control_metadata_missing_variant_1", + "control_preserved", + "control_preserved_variant_1", + "control_removed", + "dafny_fail", + "dafny_fail_variant_1", + "dafny_pass", + "dafny_pass_variant_1", + "deployment_skipped_approval", + "deployment_valid_path", + "deployment_valid_path_variant_1", + "infra_private_sensitive", + "infra_private_sensitive_variant_1", + "infra_public_sensitive", + "kani_fail", + "kani_fail_variant_1", + "kani_malformed", + "kani_malformed_variant_1", + "kani_pass", + "kani_pass_variant_1", + "kani_unknown", + "kani_unknown_variant_1", + "lean_fail", + "lean_pass", + "lean_pass_variant_1", + "tla_fail", + "tla_fail_variant_1", + "tla_malformed", + "tla_malformed_variant_1", + "tla_pass", + "tla_pass_variant_1", + "tla_unknown", + "tla_unknown_variant_1", + "verus_fail", + "verus_pass", + "verus_pass_variant_1" + ], + "development": [ + "adversarial_forged_allow", + "adversarial_sha_mismatch", + "auth_bypass_variant_1", + "cbmc_fail_variant_1", + "ci_secrets_exposed_variant_1", + "control_removed_variant_1", + "deployment_skipped_approval_variant_1", + "infra_public_sensitive_variant_1", + "multi_surface_combined_pr", + "recall_ci_secrets_workflow_diff", + "recall_infra_terraform_diff", + "recall_multi_surface_combined", + "repair_loop_auth_bypass", + "repair_loop_ci_secrets", + "repair_loop_deployment_skip", + "repair_loop_infra_exposure", + "route_alloy_model", + "route_cedar_iam", + "route_dafny_proof", + "route_kani_rust" + ], + "test": [ + "auth_malformed_variant_2", + "auth_preserved_variant_2", + "cedar_fail_variant_2", + "cedar_pass_variant_2", + "cedar_unknown_variant_2", + "ci_secrets_exposed_variant_2", + "ci_secrets_safe_variant_2", + "control_metadata_missing_variant_2", + "control_preserved_variant_2", + "control_removed_variant_2", + "deployment_skipped_approval_variant_2", + "deployment_valid_path_variant_2", + "infra_private_sensitive_variant_2", + "infra_public_sensitive_variant_2", + "kani_fail_variant_2", + "kani_malformed_variant_2", + "kani_pass_variant_2", + "rd_auth_admin_route_guarded", + "rd_auth_admin_route_unguarded", + "rd_auth_route_partial_hunk", + "rd_ci_secrets_pr_preview", + "rd_ci_secrets_workflow_dispatch_safe", + "rd_deployment_direct_skip", + "rd_deployment_valid_chain", + "rd_infra_iam_wildcard_admin", + "rd_infra_k8s_loadbalancer", + "rd_infra_private_bucket", + "rd_infra_rds_public_partial_hunk", + "rd_infra_s3_public_acl", + "rd_multi_surface_combined", + "rd_self_protection_workflow_touch", + "rd_workflow_secrets_partial_hunk", + "tla_fail_variant_2", + "tla_malformed_variant_2", + "tla_pass_variant_2" + ], + "held_out": [ + "alloy_fail_variant_1", + "auth_bypass_variant_2", + "cedar_malformed_variant_2", + "kani_unknown_variant_2", + "lean_fail_variant_1", + "rd_cbmc_integer_overflow_quota", + "rd_cbmc_use_after_free_auth_cache", + "rd_docs_only_change", + "tla_unknown_variant_2", + "verus_fail_variant_1" + ] + }, + "counts": { + "train": 67, + "development": 20, + "test": 35, + "held_out": 10 + } +} diff --git a/benchmarks/formal_pr_bench/provenance/adversarial_forged_allow.json b/benchmarks/formal_pr_bench/provenance/adversarial_forged_allow.json new file mode 100644 index 0000000..e7228b9 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/adversarial_forged_allow.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "adversarial_forged_allow", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": "examples/evidence_quality/adversarial_allow_with_fail.json", + "category": "adversarial", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/adversarial_sha_mismatch.json b/benchmarks/formal_pr_bench/provenance/adversarial_sha_mismatch.json new file mode 100644 index 0000000..728a541 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/adversarial_sha_mismatch.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "adversarial_sha_mismatch", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": "examples/evidence_quality/adversarial_sha_mismatch.json", + "category": "adversarial", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/alloy_fail.json b/benchmarks/formal_pr_bench/provenance/alloy_fail.json new file mode 100644 index 0000000..a6d6b8f --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/alloy_fail.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "alloy_fail", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/alloy_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/alloy_fail_variant_1.json b/benchmarks/formal_pr_bench/provenance/alloy_fail_variant_1.json new file mode 100644 index 0000000..b2cad0d --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/alloy_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "alloy_fail_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "held_out", + "template_dev": false, + "fixture": "examples/backends/alloy_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/alloy_pass.json b/benchmarks/formal_pr_bench/provenance/alloy_pass.json new file mode 100644 index 0000000..e3f97ba --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/alloy_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "alloy_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/alloy_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/alloy_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/alloy_pass_variant_1.json new file mode 100644 index 0000000..445f8cf --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/alloy_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "alloy_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/alloy_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/auth_bypass.json b/benchmarks/formal_pr_bench/provenance/auth_bypass.json new file mode 100644 index 0000000..b28a34e --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/auth_bypass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "auth_bypass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/auth_regression/input_admin_bypass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/auth_bypass_variant_1.json b/benchmarks/formal_pr_bench/provenance/auth_bypass_variant_1.json new file mode 100644 index 0000000..3db8e1d --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/auth_bypass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "auth_bypass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "development", + "template_dev": false, + "fixture": "examples/auth_regression/input_admin_bypass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/auth_bypass_variant_2.json b/benchmarks/formal_pr_bench/provenance/auth_bypass_variant_2.json new file mode 100644 index 0000000..1a0e091 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/auth_bypass_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "auth_bypass_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "held_out", + "template_dev": false, + "fixture": "examples/auth_regression/input_admin_bypass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/auth_malformed.json b/benchmarks/formal_pr_bench/provenance/auth_malformed.json new file mode 100644 index 0000000..5733007 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/auth_malformed.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "auth_malformed", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/auth_regression/input_malformed_missing_routes.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/auth_malformed_variant_1.json b/benchmarks/formal_pr_bench/provenance/auth_malformed_variant_1.json new file mode 100644 index 0000000..afdc8da --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/auth_malformed_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "auth_malformed_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/auth_regression/input_malformed_missing_routes.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/auth_malformed_variant_2.json b/benchmarks/formal_pr_bench/provenance/auth_malformed_variant_2.json new file mode 100644 index 0000000..0729b99 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/auth_malformed_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "auth_malformed_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/auth_regression/input_malformed_missing_routes.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/auth_preserved.json b/benchmarks/formal_pr_bench/provenance/auth_preserved.json new file mode 100644 index 0000000..3339e9d --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/auth_preserved.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "auth_preserved", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/auth_regression/input_admin_protected.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/auth_preserved_variant_1.json b/benchmarks/formal_pr_bench/provenance/auth_preserved_variant_1.json new file mode 100644 index 0000000..4042da7 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/auth_preserved_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "auth_preserved_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/auth_regression/input_admin_protected.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/auth_preserved_variant_2.json b/benchmarks/formal_pr_bench/provenance/auth_preserved_variant_2.json new file mode 100644 index 0000000..0ecb43b --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/auth_preserved_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "auth_preserved_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/auth_regression/input_admin_protected.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_fail.json b/benchmarks/formal_pr_bench/provenance/cbmc_fail.json new file mode 100644 index 0000000..a5dbe8f --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_fail.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_fail", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/cbmc_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_fail_variant_1.json b/benchmarks/formal_pr_bench/provenance/cbmc_fail_variant_1.json new file mode 100644 index 0000000..a1cde86 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_fail_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "development", + "template_dev": false, + "fixture": "examples/backends/cbmc_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_native_buffer_bounds_pass.json b/benchmarks/formal_pr_bench/provenance/cbmc_native_buffer_bounds_pass.json new file mode 100644 index 0000000..63fef95 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_native_buffer_bounds_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_native_buffer_bounds_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/cbmc_native_buffer_bounds_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_native_buffer_bounds_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/cbmc_native_buffer_bounds_pass_variant_1.json new file mode 100644 index 0000000..2a23945 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_native_buffer_bounds_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_native_buffer_bounds_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/cbmc_native_buffer_bounds_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_native_integer_overflow_pass.json b/benchmarks/formal_pr_bench/provenance/cbmc_native_integer_overflow_pass.json new file mode 100644 index 0000000..2b68476 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_native_integer_overflow_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_native_integer_overflow_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/cbmc_native_integer_overflow_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_native_integer_overflow_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/cbmc_native_integer_overflow_pass_variant_1.json new file mode 100644 index 0000000..d1d1416 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_native_integer_overflow_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_native_integer_overflow_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/cbmc_native_integer_overflow_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_native_uaf_pass.json b/benchmarks/formal_pr_bench/provenance/cbmc_native_uaf_pass.json new file mode 100644 index 0000000..bfb73b8 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_native_uaf_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_native_uaf_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/cbmc_native_uaf_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_native_uaf_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/cbmc_native_uaf_pass_variant_1.json new file mode 100644 index 0000000..187f67a --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_native_uaf_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_native_uaf_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/cbmc_native_uaf_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_native_unchecked_copy_pass.json b/benchmarks/formal_pr_bench/provenance/cbmc_native_unchecked_copy_pass.json new file mode 100644 index 0000000..52c2a2e --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_native_unchecked_copy_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_native_unchecked_copy_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/cbmc_native_unchecked_copy_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_native_unchecked_copy_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/cbmc_native_unchecked_copy_pass_variant_1.json new file mode 100644 index 0000000..b9062ba --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_native_unchecked_copy_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_native_unchecked_copy_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/cbmc_native_unchecked_copy_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_pass.json b/benchmarks/formal_pr_bench/provenance/cbmc_pass.json new file mode 100644 index 0000000..cee4d7e --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/cbmc_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/cbmc_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/cbmc_pass_variant_1.json new file mode 100644 index 0000000..29d6cad --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cbmc_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cbmc_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/cbmc_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_fail.json b/benchmarks/formal_pr_bench/provenance/cedar_fail.json new file mode 100644 index 0000000..3669d76 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_fail.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_fail", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/cedar_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_fail_variant_1.json b/benchmarks/formal_pr_bench/provenance/cedar_fail_variant_1.json new file mode 100644 index 0000000..f9a68f8 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_fail_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/cedar_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_fail_variant_2.json b/benchmarks/formal_pr_bench/provenance/cedar_fail_variant_2.json new file mode 100644 index 0000000..c4f5d0f --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_fail_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_fail_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/backends/cedar_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_malformed.json b/benchmarks/formal_pr_bench/provenance/cedar_malformed.json new file mode 100644 index 0000000..8c1b462 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_malformed.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_malformed", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/cedar_malformed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_malformed_variant_1.json b/benchmarks/formal_pr_bench/provenance/cedar_malformed_variant_1.json new file mode 100644 index 0000000..d69bee3 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_malformed_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_malformed_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/cedar_malformed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_malformed_variant_2.json b/benchmarks/formal_pr_bench/provenance/cedar_malformed_variant_2.json new file mode 100644 index 0000000..74fe5db --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_malformed_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_malformed_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "held_out", + "template_dev": false, + "fixture": "examples/backends/cedar_malformed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_pass.json b/benchmarks/formal_pr_bench/provenance/cedar_pass.json new file mode 100644 index 0000000..3d48876 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/cedar_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/cedar_pass_variant_1.json new file mode 100644 index 0000000..36e1995 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/cedar_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_pass_variant_2.json b/benchmarks/formal_pr_bench/provenance/cedar_pass_variant_2.json new file mode 100644 index 0000000..244352f --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_pass_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_pass_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/backends/cedar_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_unknown.json b/benchmarks/formal_pr_bench/provenance/cedar_unknown.json new file mode 100644 index 0000000..7da4583 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_unknown.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_unknown", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/cedar_unknown.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_unknown_variant_1.json b/benchmarks/formal_pr_bench/provenance/cedar_unknown_variant_1.json new file mode 100644 index 0000000..a44b649 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_unknown_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_unknown_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/cedar_unknown.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/cedar_unknown_variant_2.json b/benchmarks/formal_pr_bench/provenance/cedar_unknown_variant_2.json new file mode 100644 index 0000000..67a6b65 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/cedar_unknown_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "cedar_unknown_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/backends/cedar_unknown.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/ci_secrets_exposed.json b/benchmarks/formal_pr_bench/provenance/ci_secrets_exposed.json new file mode 100644 index 0000000..2e76129 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/ci_secrets_exposed.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "ci_secrets_exposed", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/ci_secrets/input_secrets_exposed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/ci_secrets_exposed_variant_1.json b/benchmarks/formal_pr_bench/provenance/ci_secrets_exposed_variant_1.json new file mode 100644 index 0000000..9f98138 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/ci_secrets_exposed_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "ci_secrets_exposed_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "development", + "template_dev": false, + "fixture": "examples/ci_secrets/input_secrets_exposed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/ci_secrets_exposed_variant_2.json b/benchmarks/formal_pr_bench/provenance/ci_secrets_exposed_variant_2.json new file mode 100644 index 0000000..31d9ee8 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/ci_secrets_exposed_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "ci_secrets_exposed_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/ci_secrets/input_secrets_exposed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/ci_secrets_safe.json b/benchmarks/formal_pr_bench/provenance/ci_secrets_safe.json new file mode 100644 index 0000000..0e52b77 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/ci_secrets_safe.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "ci_secrets_safe", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/ci_secrets/input_secrets_safe.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/ci_secrets_safe_variant_1.json b/benchmarks/formal_pr_bench/provenance/ci_secrets_safe_variant_1.json new file mode 100644 index 0000000..5fc1ddf --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/ci_secrets_safe_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "ci_secrets_safe_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/ci_secrets/input_secrets_safe.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/ci_secrets_safe_variant_2.json b/benchmarks/formal_pr_bench/provenance/ci_secrets_safe_variant_2.json new file mode 100644 index 0000000..79d6984 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/ci_secrets_safe_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "ci_secrets_safe_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/ci_secrets/input_secrets_safe.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/control_metadata_missing.json b/benchmarks/formal_pr_bench/provenance/control_metadata_missing.json new file mode 100644 index 0000000..8934801 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/control_metadata_missing.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "control_metadata_missing", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/no_agent_self_approval/input_missing_metadata.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/control_metadata_missing_variant_1.json b/benchmarks/formal_pr_bench/provenance/control_metadata_missing_variant_1.json new file mode 100644 index 0000000..17b6fcb --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/control_metadata_missing_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "control_metadata_missing_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/no_agent_self_approval/input_missing_metadata.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/control_metadata_missing_variant_2.json b/benchmarks/formal_pr_bench/provenance/control_metadata_missing_variant_2.json new file mode 100644 index 0000000..ea34358 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/control_metadata_missing_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "control_metadata_missing_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/no_agent_self_approval/input_missing_metadata.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/control_preserved.json b/benchmarks/formal_pr_bench/provenance/control_preserved.json new file mode 100644 index 0000000..e79887f --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/control_preserved.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "control_preserved", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/no_agent_self_approval/input_gate_preserved.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/control_preserved_variant_1.json b/benchmarks/formal_pr_bench/provenance/control_preserved_variant_1.json new file mode 100644 index 0000000..0ca82b5 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/control_preserved_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "control_preserved_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/no_agent_self_approval/input_gate_preserved.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/control_preserved_variant_2.json b/benchmarks/formal_pr_bench/provenance/control_preserved_variant_2.json new file mode 100644 index 0000000..d74f385 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/control_preserved_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "control_preserved_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/no_agent_self_approval/input_gate_preserved.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/control_removed.json b/benchmarks/formal_pr_bench/provenance/control_removed.json new file mode 100644 index 0000000..026a134 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/control_removed.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "control_removed", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/no_agent_self_approval/input_gate_removed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/control_removed_variant_1.json b/benchmarks/formal_pr_bench/provenance/control_removed_variant_1.json new file mode 100644 index 0000000..d532be0 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/control_removed_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "control_removed_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "development", + "template_dev": false, + "fixture": "examples/no_agent_self_approval/input_gate_removed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/control_removed_variant_2.json b/benchmarks/formal_pr_bench/provenance/control_removed_variant_2.json new file mode 100644 index 0000000..af00403 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/control_removed_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "control_removed_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/no_agent_self_approval/input_gate_removed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/dafny_fail.json b/benchmarks/formal_pr_bench/provenance/dafny_fail.json new file mode 100644 index 0000000..5854c45 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/dafny_fail.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "dafny_fail", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/dafny_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/dafny_fail_variant_1.json b/benchmarks/formal_pr_bench/provenance/dafny_fail_variant_1.json new file mode 100644 index 0000000..5f40bc6 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/dafny_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "dafny_fail_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/dafny_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/dafny_pass.json b/benchmarks/formal_pr_bench/provenance/dafny_pass.json new file mode 100644 index 0000000..d784f16 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/dafny_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "dafny_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/dafny_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/dafny_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/dafny_pass_variant_1.json new file mode 100644 index 0000000..14c12db --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/dafny_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "dafny_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/dafny_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/deployment_skipped_approval.json b/benchmarks/formal_pr_bench/provenance/deployment_skipped_approval.json new file mode 100644 index 0000000..892fd61 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/deployment_skipped_approval.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "deployment_skipped_approval", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/deployment_state/input_skipped_approval.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/deployment_skipped_approval_variant_1.json b/benchmarks/formal_pr_bench/provenance/deployment_skipped_approval_variant_1.json new file mode 100644 index 0000000..8eb73ed --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/deployment_skipped_approval_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "deployment_skipped_approval_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "development", + "template_dev": false, + "fixture": "examples/deployment_state/input_skipped_approval.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/deployment_skipped_approval_variant_2.json b/benchmarks/formal_pr_bench/provenance/deployment_skipped_approval_variant_2.json new file mode 100644 index 0000000..637276b --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/deployment_skipped_approval_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "deployment_skipped_approval_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/deployment_state/input_skipped_approval.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/deployment_valid_path.json b/benchmarks/formal_pr_bench/provenance/deployment_valid_path.json new file mode 100644 index 0000000..524c5d9 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/deployment_valid_path.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "deployment_valid_path", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/deployment_state/input_valid_approval_path.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/deployment_valid_path_variant_1.json b/benchmarks/formal_pr_bench/provenance/deployment_valid_path_variant_1.json new file mode 100644 index 0000000..69bbad8 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/deployment_valid_path_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "deployment_valid_path_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/deployment_state/input_valid_approval_path.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/deployment_valid_path_variant_2.json b/benchmarks/formal_pr_bench/provenance/deployment_valid_path_variant_2.json new file mode 100644 index 0000000..f192304 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/deployment_valid_path_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "deployment_valid_path_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/deployment_state/input_valid_approval_path.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/infra_private_sensitive.json b/benchmarks/formal_pr_bench/provenance/infra_private_sensitive.json new file mode 100644 index 0000000..990873f --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/infra_private_sensitive.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "infra_private_sensitive", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/infrastructure_exposure/input_private_sensitive_resource.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/infra_private_sensitive_variant_1.json b/benchmarks/formal_pr_bench/provenance/infra_private_sensitive_variant_1.json new file mode 100644 index 0000000..dac0533 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/infra_private_sensitive_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "infra_private_sensitive_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/infrastructure_exposure/input_private_sensitive_resource.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/infra_private_sensitive_variant_2.json b/benchmarks/formal_pr_bench/provenance/infra_private_sensitive_variant_2.json new file mode 100644 index 0000000..7743d49 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/infra_private_sensitive_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "infra_private_sensitive_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/infrastructure_exposure/input_private_sensitive_resource.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/infra_public_sensitive.json b/benchmarks/formal_pr_bench/provenance/infra_public_sensitive.json new file mode 100644 index 0000000..6b4e7cb --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/infra_public_sensitive.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "infra_public_sensitive", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/infrastructure_exposure/input_public_sensitive_resource.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/infra_public_sensitive_variant_1.json b/benchmarks/formal_pr_bench/provenance/infra_public_sensitive_variant_1.json new file mode 100644 index 0000000..a4ab98c --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/infra_public_sensitive_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "infra_public_sensitive_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "development", + "template_dev": false, + "fixture": "examples/infrastructure_exposure/input_public_sensitive_resource.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/infra_public_sensitive_variant_2.json b/benchmarks/formal_pr_bench/provenance/infra_public_sensitive_variant_2.json new file mode 100644 index 0000000..3a1dc16 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/infra_public_sensitive_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "infra_public_sensitive_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/infrastructure_exposure/input_public_sensitive_resource.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_fail.json b/benchmarks/formal_pr_bench/provenance/kani_fail.json new file mode 100644 index 0000000..feed5d3 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_fail.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_fail", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/kani_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_fail_variant_1.json b/benchmarks/formal_pr_bench/provenance/kani_fail_variant_1.json new file mode 100644 index 0000000..57dbca2 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_fail_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/kani_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_fail_variant_2.json b/benchmarks/formal_pr_bench/provenance/kani_fail_variant_2.json new file mode 100644 index 0000000..c8335a4 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_fail_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_fail_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/backends/kani_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_malformed.json b/benchmarks/formal_pr_bench/provenance/kani_malformed.json new file mode 100644 index 0000000..417d22c --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_malformed.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_malformed", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/kani_malformed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_malformed_variant_1.json b/benchmarks/formal_pr_bench/provenance/kani_malformed_variant_1.json new file mode 100644 index 0000000..303dc96 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_malformed_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_malformed_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/kani_malformed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_malformed_variant_2.json b/benchmarks/formal_pr_bench/provenance/kani_malformed_variant_2.json new file mode 100644 index 0000000..3c0b425 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_malformed_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_malformed_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/backends/kani_malformed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_pass.json b/benchmarks/formal_pr_bench/provenance/kani_pass.json new file mode 100644 index 0000000..9be2957 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/kani_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/kani_pass_variant_1.json new file mode 100644 index 0000000..ffca29f --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/kani_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_pass_variant_2.json b/benchmarks/formal_pr_bench/provenance/kani_pass_variant_2.json new file mode 100644 index 0000000..c36103c --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_pass_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_pass_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/backends/kani_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_unknown.json b/benchmarks/formal_pr_bench/provenance/kani_unknown.json new file mode 100644 index 0000000..315e001 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_unknown.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_unknown", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/kani_unknown.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_unknown_variant_1.json b/benchmarks/formal_pr_bench/provenance/kani_unknown_variant_1.json new file mode 100644 index 0000000..4cffe78 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_unknown_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_unknown_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/kani_unknown.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/kani_unknown_variant_2.json b/benchmarks/formal_pr_bench/provenance/kani_unknown_variant_2.json new file mode 100644 index 0000000..748bd4c --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/kani_unknown_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "kani_unknown_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "held_out", + "template_dev": false, + "fixture": "examples/backends/kani_unknown.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/lean_fail.json b/benchmarks/formal_pr_bench/provenance/lean_fail.json new file mode 100644 index 0000000..fd7844c --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/lean_fail.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "lean_fail", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/lean_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/lean_fail_variant_1.json b/benchmarks/formal_pr_bench/provenance/lean_fail_variant_1.json new file mode 100644 index 0000000..8ec815a --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/lean_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "lean_fail_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "held_out", + "template_dev": false, + "fixture": "examples/backends/lean_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/lean_pass.json b/benchmarks/formal_pr_bench/provenance/lean_pass.json new file mode 100644 index 0000000..a1e1176 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/lean_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "lean_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/lean_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/lean_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/lean_pass_variant_1.json new file mode 100644 index 0000000..0d6ef4b --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/lean_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "lean_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/lean_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/multi_surface_combined_pr.json b/benchmarks/formal_pr_bench/provenance/multi_surface_combined_pr.json new file mode 100644 index 0000000..e45737e --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/multi_surface_combined_pr.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "multi_surface_combined_pr", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": "examples/multi_surface/pr_combined.diff", + "category": "multi_backend", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_auth_admin_route_guarded.json b/benchmarks/formal_pr_bench/provenance/rd_auth_admin_route_guarded.json new file mode 100644 index 0000000..1fc75e1 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_auth_admin_route_guarded.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_auth_admin_route_guarded", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/auth_admin_route_guarded.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_auth_admin_route_unguarded.json b/benchmarks/formal_pr_bench/provenance/rd_auth_admin_route_unguarded.json new file mode 100644 index 0000000..0ba0357 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_auth_admin_route_unguarded.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_auth_admin_route_unguarded", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/auth_admin_route_unguarded.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_auth_route_partial_hunk.json b/benchmarks/formal_pr_bench/provenance/rd_auth_route_partial_hunk.json new file mode 100644 index 0000000..21588dd --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_auth_route_partial_hunk.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_auth_route_partial_hunk", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/auth_route_partial_hunk.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_cbmc_integer_overflow_quota.json b/benchmarks/formal_pr_bench/provenance/rd_cbmc_integer_overflow_quota.json new file mode 100644 index 0000000..241f72a --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_cbmc_integer_overflow_quota.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_cbmc_integer_overflow_quota", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "held_out", + "template_dev": false, + "fixture": "benchmarks/real_diffs/cbmc_integer_overflow_quota.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_cbmc_use_after_free_auth_cache.json b/benchmarks/formal_pr_bench/provenance/rd_cbmc_use_after_free_auth_cache.json new file mode 100644 index 0000000..8f42db5 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_cbmc_use_after_free_auth_cache.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_cbmc_use_after_free_auth_cache", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "held_out", + "template_dev": false, + "fixture": "benchmarks/real_diffs/cbmc_use_after_free_auth_cache.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_ci_secrets_pr_preview.json b/benchmarks/formal_pr_bench/provenance/rd_ci_secrets_pr_preview.json new file mode 100644 index 0000000..4caa066 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_ci_secrets_pr_preview.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_ci_secrets_pr_preview", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/ci_secrets_pr_preview.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_ci_secrets_workflow_dispatch_safe.json b/benchmarks/formal_pr_bench/provenance/rd_ci_secrets_workflow_dispatch_safe.json new file mode 100644 index 0000000..63b9636 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_ci_secrets_workflow_dispatch_safe.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_ci_secrets_workflow_dispatch_safe", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/ci_secrets_workflow_dispatch_safe.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_deployment_direct_skip.json b/benchmarks/formal_pr_bench/provenance/rd_deployment_direct_skip.json new file mode 100644 index 0000000..7fd456a --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_deployment_direct_skip.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_deployment_direct_skip", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/deployment_direct_skip.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_deployment_valid_chain.json b/benchmarks/formal_pr_bench/provenance/rd_deployment_valid_chain.json new file mode 100644 index 0000000..03de7db --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_deployment_valid_chain.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_deployment_valid_chain", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/deployment_valid_chain.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_docs_only_change.json b/benchmarks/formal_pr_bench/provenance/rd_docs_only_change.json new file mode 100644 index 0000000..b65c7c4 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_docs_only_change.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_docs_only_change", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "held_out", + "template_dev": false, + "fixture": "benchmarks/real_diffs/docs_only_change.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_infra_iam_wildcard_admin.json b/benchmarks/formal_pr_bench/provenance/rd_infra_iam_wildcard_admin.json new file mode 100644 index 0000000..983ce99 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_infra_iam_wildcard_admin.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_infra_iam_wildcard_admin", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/infra_iam_wildcard_admin.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_infra_k8s_loadbalancer.json b/benchmarks/formal_pr_bench/provenance/rd_infra_k8s_loadbalancer.json new file mode 100644 index 0000000..dea9938 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_infra_k8s_loadbalancer.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_infra_k8s_loadbalancer", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/infra_k8s_loadbalancer.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_infra_private_bucket.json b/benchmarks/formal_pr_bench/provenance/rd_infra_private_bucket.json new file mode 100644 index 0000000..c9e5157 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_infra_private_bucket.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_infra_private_bucket", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/infra_private_bucket.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_infra_rds_public_partial_hunk.json b/benchmarks/formal_pr_bench/provenance/rd_infra_rds_public_partial_hunk.json new file mode 100644 index 0000000..e02c71c --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_infra_rds_public_partial_hunk.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_infra_rds_public_partial_hunk", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/infra_rds_public_partial_hunk.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_infra_s3_public_acl.json b/benchmarks/formal_pr_bench/provenance/rd_infra_s3_public_acl.json new file mode 100644 index 0000000..4d85a61 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_infra_s3_public_acl.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_infra_s3_public_acl", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/infra_s3_public_acl.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_multi_surface_combined.json b/benchmarks/formal_pr_bench/provenance/rd_multi_surface_combined.json new file mode 100644 index 0000000..8dba139 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_multi_surface_combined.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_multi_surface_combined", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/multi_surface_combined.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_self_protection_workflow_touch.json b/benchmarks/formal_pr_bench/provenance/rd_self_protection_workflow_touch.json new file mode 100644 index 0000000..f484270 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_self_protection_workflow_touch.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_self_protection_workflow_touch", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/self_protection_workflow_touch.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/rd_workflow_secrets_partial_hunk.json b/benchmarks/formal_pr_bench/provenance/rd_workflow_secrets_partial_hunk.json new file mode 100644 index 0000000..6358cef --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/rd_workflow_secrets_partial_hunk.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "rd_workflow_secrets_partial_hunk", + "source": "benchmarks/formal_pr_bench/real_diff_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "sanitized_real_diff", + "partition": "test", + "template_dev": false, + "fixture": "benchmarks/real_diffs/workflow_secrets_partial_hunk.diff", + "category": "real_diff", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/recall_ci_secrets_workflow_diff.json b/benchmarks/formal_pr_bench/provenance/recall_ci_secrets_workflow_diff.json new file mode 100644 index 0000000..c8e0e53 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/recall_ci_secrets_workflow_diff.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "recall_ci_secrets_workflow_diff", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": "examples/ci_secrets/workflow_secrets_on_pr.diff", + "category": "intent_recall", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/recall_infra_terraform_diff.json b/benchmarks/formal_pr_bench/provenance/recall_infra_terraform_diff.json new file mode 100644 index 0000000..9e69e2c --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/recall_infra_terraform_diff.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "recall_infra_terraform_diff", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": "examples/multi_surface/infra_public_s3.diff", + "category": "intent_recall", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/recall_multi_surface_combined.json b/benchmarks/formal_pr_bench/provenance/recall_multi_surface_combined.json new file mode 100644 index 0000000..af1eaa7 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/recall_multi_surface_combined.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "recall_multi_surface_combined", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": "examples/multi_surface/pr_combined.diff", + "category": "intent_recall", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/repair_loop_auth_bypass.json b/benchmarks/formal_pr_bench/provenance/repair_loop_auth_bypass.json new file mode 100644 index 0000000..66dfb65 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/repair_loop_auth_bypass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "repair_loop_auth_bypass", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": "examples/repair_loops/authorization/failing.diff", + "category": "repair_loop", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/repair_loop_ci_secrets.json b/benchmarks/formal_pr_bench/provenance/repair_loop_ci_secrets.json new file mode 100644 index 0000000..1aef46b --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/repair_loop_ci_secrets.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "repair_loop_ci_secrets", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": "examples/repair_loops/ci_secrets/failing.diff", + "category": "repair_loop", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/repair_loop_deployment_skip.json b/benchmarks/formal_pr_bench/provenance/repair_loop_deployment_skip.json new file mode 100644 index 0000000..703ea28 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/repair_loop_deployment_skip.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "repair_loop_deployment_skip", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": "examples/multi_surface/deployment_skip.diff", + "category": "repair_loop", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/repair_loop_infra_exposure.json b/benchmarks/formal_pr_bench/provenance/repair_loop_infra_exposure.json new file mode 100644 index 0000000..0914544 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/repair_loop_infra_exposure.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "repair_loop_infra_exposure", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": "examples/repair_loops/infrastructure/failing.diff", + "category": "repair_loop", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/route_alloy_model.json b/benchmarks/formal_pr_bench/provenance/route_alloy_model.json new file mode 100644 index 0000000..e4a38e4 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/route_alloy_model.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "route_alloy_model", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": null, + "category": "routing", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/route_cedar_iam.json b/benchmarks/formal_pr_bench/provenance/route_cedar_iam.json new file mode 100644 index 0000000..f3129f0 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/route_cedar_iam.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "route_cedar_iam", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": null, + "category": "routing", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/route_dafny_proof.json b/benchmarks/formal_pr_bench/provenance/route_dafny_proof.json new file mode 100644 index 0000000..fa20a21 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/route_dafny_proof.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "route_dafny_proof", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": null, + "category": "routing", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/route_kani_rust.json b/benchmarks/formal_pr_bench/provenance/route_kani_rust.json new file mode 100644 index 0000000..c54af9e --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/route_kani_rust.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "route_kani_rust", + "source": "benchmarks/formal_pr_bench/extended_cases.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "extended_category", + "partition": "development", + "template_dev": false, + "fixture": null, + "category": "routing", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_fail.json b/benchmarks/formal_pr_bench/provenance/tla_fail.json new file mode 100644 index 0000000..37b199a --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_fail.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_fail", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/tla_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_fail_variant_1.json b/benchmarks/formal_pr_bench/provenance/tla_fail_variant_1.json new file mode 100644 index 0000000..f207902 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_fail_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/tla_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_fail_variant_2.json b/benchmarks/formal_pr_bench/provenance/tla_fail_variant_2.json new file mode 100644 index 0000000..3cacd27 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_fail_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_fail_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/backends/tla_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_malformed.json b/benchmarks/formal_pr_bench/provenance/tla_malformed.json new file mode 100644 index 0000000..f69556d --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_malformed.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_malformed", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/tla_malformed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_malformed_variant_1.json b/benchmarks/formal_pr_bench/provenance/tla_malformed_variant_1.json new file mode 100644 index 0000000..d900e78 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_malformed_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_malformed_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/tla_malformed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_malformed_variant_2.json b/benchmarks/formal_pr_bench/provenance/tla_malformed_variant_2.json new file mode 100644 index 0000000..43c9ed1 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_malformed_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_malformed_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/backends/tla_malformed.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_pass.json b/benchmarks/formal_pr_bench/provenance/tla_pass.json new file mode 100644 index 0000000..c05b970 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/tla_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/tla_pass_variant_1.json new file mode 100644 index 0000000..9c85f98 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/tla_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_pass_variant_2.json b/benchmarks/formal_pr_bench/provenance/tla_pass_variant_2.json new file mode 100644 index 0000000..036cc19 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_pass_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_pass_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "test", + "template_dev": false, + "fixture": "examples/backends/tla_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_unknown.json b/benchmarks/formal_pr_bench/provenance/tla_unknown.json new file mode 100644 index 0000000..3277d50 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_unknown.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_unknown", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/tla_unknown.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_unknown_variant_1.json b/benchmarks/formal_pr_bench/provenance/tla_unknown_variant_1.json new file mode 100644 index 0000000..7fa236f --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_unknown_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_unknown_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/tla_unknown.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/tla_unknown_variant_2.json b/benchmarks/formal_pr_bench/provenance/tla_unknown_variant_2.json new file mode 100644 index 0000000..003c368 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/tla_unknown_variant_2.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "tla_unknown_variant_2", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "held_out", + "template_dev": false, + "fixture": "examples/backends/tla_unknown.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/verus_fail.json b/benchmarks/formal_pr_bench/provenance/verus_fail.json new file mode 100644 index 0000000..44fd6e4 --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/verus_fail.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "verus_fail", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/verus_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/verus_fail_variant_1.json b/benchmarks/formal_pr_bench/provenance/verus_fail_variant_1.json new file mode 100644 index 0000000..2f1825f --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/verus_fail_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "verus_fail_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "held_out", + "template_dev": false, + "fixture": "examples/backends/verus_fail.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/provenance/verus_pass.json b/benchmarks/formal_pr_bench/provenance/verus_pass.json new file mode 100644 index 0000000..ac0f43c --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/verus_pass.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "verus_pass", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "seed_fixture", + "partition": "train", + "template_dev": true, + "fixture": "examples/backends/verus_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Case used during property-template development; forbidden from held-out scoring." +} diff --git a/benchmarks/formal_pr_bench/provenance/verus_pass_variant_1.json b/benchmarks/formal_pr_bench/provenance/verus_pass_variant_1.json new file mode 100644 index 0000000..64608cb --- /dev/null +++ b/benchmarks/formal_pr_bench/provenance/verus_pass_variant_1.json @@ -0,0 +1,14 @@ +{ + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": "verus_pass_variant_1", + "source": "benchmarks/formal_pr_bench/seed_cases_expanded.json", + "author": "fraware", + "date": "2026-07-25", + "derivation": "expanded_variant", + "partition": "train", + "template_dev": false, + "fixture": "examples/backends/verus_pass.json", + "category": "lane", + "license": "Apache-2.0", + "notes": "Evaluation/regression case; cite benchmark_version + partition when publishing scores." +} diff --git a/benchmarks/formal_pr_bench/rationales/adversarial_forged_allow.md b/benchmarks/formal_pr_bench/rationales/adversarial_forged_allow.md new file mode 100644 index 0000000..a034ce9 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/adversarial_forged_allow.md @@ -0,0 +1,13 @@ +# Rationale: `adversarial_forged_allow` + +- Partition: `development` +- Category: `adversarial` +- Expected decision signal: `False` + +## Why this expectation + +Tampered or inconsistent evidence bundles must fail the evidence-quality gate. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/adversarial_sha_mismatch.md b/benchmarks/formal_pr_bench/rationales/adversarial_sha_mismatch.md new file mode 100644 index 0000000..469cf1e --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/adversarial_sha_mismatch.md @@ -0,0 +1,13 @@ +# Rationale: `adversarial_sha_mismatch` + +- Partition: `development` +- Category: `adversarial` +- Expected decision signal: `False` + +## Why this expectation + +Tampered or inconsistent evidence bundles must fail the evidence-quality gate. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/alloy_fail.md b/benchmarks/formal_pr_bench/rationales/alloy_fail.md new file mode 100644 index 0000000..dbde4ab --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/alloy_fail.md @@ -0,0 +1,13 @@ +# Rationale: `alloy_fail` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/alloy_fail_variant_1.md b/benchmarks/formal_pr_bench/rationales/alloy_fail_variant_1.md new file mode 100644 index 0000000..ea751d9 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/alloy_fail_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `alloy_fail_variant_1` + +- Partition: `held_out` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/alloy_pass.md b/benchmarks/formal_pr_bench/rationales/alloy_pass.md new file mode 100644 index 0000000..9c0facd --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/alloy_pass.md @@ -0,0 +1,13 @@ +# Rationale: `alloy_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/alloy_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/alloy_pass_variant_1.md new file mode 100644 index 0000000..3ea064c --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/alloy_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `alloy_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/auth_bypass.md b/benchmarks/formal_pr_bench/rationales/auth_bypass.md new file mode 100644 index 0000000..75aac31 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/auth_bypass.md @@ -0,0 +1,13 @@ +# Rationale: `auth_bypass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/auth_bypass_variant_1.md b/benchmarks/formal_pr_bench/rationales/auth_bypass_variant_1.md new file mode 100644 index 0000000..b94680f --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/auth_bypass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `auth_bypass_variant_1` + +- Partition: `development` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/auth_bypass_variant_2.md b/benchmarks/formal_pr_bench/rationales/auth_bypass_variant_2.md new file mode 100644 index 0000000..ac35514 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/auth_bypass_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `auth_bypass_variant_2` + +- Partition: `held_out` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/auth_malformed.md b/benchmarks/formal_pr_bench/rationales/auth_malformed.md new file mode 100644 index 0000000..920f473 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/auth_malformed.md @@ -0,0 +1,13 @@ +# Rationale: `auth_malformed` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/auth_malformed_variant_1.md b/benchmarks/formal_pr_bench/rationales/auth_malformed_variant_1.md new file mode 100644 index 0000000..214d3be --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/auth_malformed_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `auth_malformed_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/auth_malformed_variant_2.md b/benchmarks/formal_pr_bench/rationales/auth_malformed_variant_2.md new file mode 100644 index 0000000..fb7adc1 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/auth_malformed_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `auth_malformed_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/auth_preserved.md b/benchmarks/formal_pr_bench/rationales/auth_preserved.md new file mode 100644 index 0000000..5db1e16 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/auth_preserved.md @@ -0,0 +1,13 @@ +# Rationale: `auth_preserved` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/auth_preserved_variant_1.md b/benchmarks/formal_pr_bench/rationales/auth_preserved_variant_1.md new file mode 100644 index 0000000..ae2b3fd --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/auth_preserved_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `auth_preserved_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/auth_preserved_variant_2.md b/benchmarks/formal_pr_bench/rationales/auth_preserved_variant_2.md new file mode 100644 index 0000000..9770d0a --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/auth_preserved_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `auth_preserved_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_fail.md b/benchmarks/formal_pr_bench/rationales/cbmc_fail.md new file mode 100644 index 0000000..7f45a74 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_fail.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_fail` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_fail_variant_1.md b/benchmarks/formal_pr_bench/rationales/cbmc_fail_variant_1.md new file mode 100644 index 0000000..0c16ea9 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_fail_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_fail_variant_1` + +- Partition: `development` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_native_buffer_bounds_pass.md b/benchmarks/formal_pr_bench/rationales/cbmc_native_buffer_bounds_pass.md new file mode 100644 index 0000000..ffc0e56 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_native_buffer_bounds_pass.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_native_buffer_bounds_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_native_buffer_bounds_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/cbmc_native_buffer_bounds_pass_variant_1.md new file mode 100644 index 0000000..823a62a --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_native_buffer_bounds_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_native_buffer_bounds_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_native_integer_overflow_pass.md b/benchmarks/formal_pr_bench/rationales/cbmc_native_integer_overflow_pass.md new file mode 100644 index 0000000..6965ec1 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_native_integer_overflow_pass.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_native_integer_overflow_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_native_integer_overflow_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/cbmc_native_integer_overflow_pass_variant_1.md new file mode 100644 index 0000000..2bd7f5c --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_native_integer_overflow_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_native_integer_overflow_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_native_uaf_pass.md b/benchmarks/formal_pr_bench/rationales/cbmc_native_uaf_pass.md new file mode 100644 index 0000000..b45a6f1 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_native_uaf_pass.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_native_uaf_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_native_uaf_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/cbmc_native_uaf_pass_variant_1.md new file mode 100644 index 0000000..c9b7047 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_native_uaf_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_native_uaf_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_native_unchecked_copy_pass.md b/benchmarks/formal_pr_bench/rationales/cbmc_native_unchecked_copy_pass.md new file mode 100644 index 0000000..8e12bc3 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_native_unchecked_copy_pass.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_native_unchecked_copy_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_native_unchecked_copy_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/cbmc_native_unchecked_copy_pass_variant_1.md new file mode 100644 index 0000000..2406b39 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_native_unchecked_copy_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_native_unchecked_copy_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_pass.md b/benchmarks/formal_pr_bench/rationales/cbmc_pass.md new file mode 100644 index 0000000..ade2617 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_pass.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cbmc_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/cbmc_pass_variant_1.md new file mode 100644 index 0000000..009ca0e --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cbmc_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `cbmc_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_fail.md b/benchmarks/formal_pr_bench/rationales/cedar_fail.md new file mode 100644 index 0000000..30ccc60 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_fail.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_fail` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_fail_variant_1.md b/benchmarks/formal_pr_bench/rationales/cedar_fail_variant_1.md new file mode 100644 index 0000000..95242fc --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_fail_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_fail_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_fail_variant_2.md b/benchmarks/formal_pr_bench/rationales/cedar_fail_variant_2.md new file mode 100644 index 0000000..ee6689d --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_fail_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_fail_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_malformed.md b/benchmarks/formal_pr_bench/rationales/cedar_malformed.md new file mode 100644 index 0000000..ef40148 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_malformed.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_malformed` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_malformed_variant_1.md b/benchmarks/formal_pr_bench/rationales/cedar_malformed_variant_1.md new file mode 100644 index 0000000..5710107 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_malformed_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_malformed_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_malformed_variant_2.md b/benchmarks/formal_pr_bench/rationales/cedar_malformed_variant_2.md new file mode 100644 index 0000000..5794e1a --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_malformed_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_malformed_variant_2` + +- Partition: `held_out` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_pass.md b/benchmarks/formal_pr_bench/rationales/cedar_pass.md new file mode 100644 index 0000000..beec34e --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_pass.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/cedar_pass_variant_1.md new file mode 100644 index 0000000..8b6eed5 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_pass_variant_2.md b/benchmarks/formal_pr_bench/rationales/cedar_pass_variant_2.md new file mode 100644 index 0000000..4c82cee --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_pass_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_pass_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_unknown.md b/benchmarks/formal_pr_bench/rationales/cedar_unknown.md new file mode 100644 index 0000000..f9fa4aa --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_unknown.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_unknown` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_unknown_variant_1.md b/benchmarks/formal_pr_bench/rationales/cedar_unknown_variant_1.md new file mode 100644 index 0000000..0548963 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_unknown_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_unknown_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/cedar_unknown_variant_2.md b/benchmarks/formal_pr_bench/rationales/cedar_unknown_variant_2.md new file mode 100644 index 0000000..b13f3e1 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/cedar_unknown_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `cedar_unknown_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/ci_secrets_exposed.md b/benchmarks/formal_pr_bench/rationales/ci_secrets_exposed.md new file mode 100644 index 0000000..d60e742 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/ci_secrets_exposed.md @@ -0,0 +1,13 @@ +# Rationale: `ci_secrets_exposed` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/ci_secrets_exposed_variant_1.md b/benchmarks/formal_pr_bench/rationales/ci_secrets_exposed_variant_1.md new file mode 100644 index 0000000..9bbbfef --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/ci_secrets_exposed_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `ci_secrets_exposed_variant_1` + +- Partition: `development` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/ci_secrets_exposed_variant_2.md b/benchmarks/formal_pr_bench/rationales/ci_secrets_exposed_variant_2.md new file mode 100644 index 0000000..cc03c44 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/ci_secrets_exposed_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `ci_secrets_exposed_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/ci_secrets_safe.md b/benchmarks/formal_pr_bench/rationales/ci_secrets_safe.md new file mode 100644 index 0000000..758fd82 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/ci_secrets_safe.md @@ -0,0 +1,13 @@ +# Rationale: `ci_secrets_safe` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/ci_secrets_safe_variant_1.md b/benchmarks/formal_pr_bench/rationales/ci_secrets_safe_variant_1.md new file mode 100644 index 0000000..ad435fc --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/ci_secrets_safe_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `ci_secrets_safe_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/ci_secrets_safe_variant_2.md b/benchmarks/formal_pr_bench/rationales/ci_secrets_safe_variant_2.md new file mode 100644 index 0000000..b8b1fa3 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/ci_secrets_safe_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `ci_secrets_safe_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/control_metadata_missing.md b/benchmarks/formal_pr_bench/rationales/control_metadata_missing.md new file mode 100644 index 0000000..f0e826e --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/control_metadata_missing.md @@ -0,0 +1,13 @@ +# Rationale: `control_metadata_missing` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/control_metadata_missing_variant_1.md b/benchmarks/formal_pr_bench/rationales/control_metadata_missing_variant_1.md new file mode 100644 index 0000000..5c71da1 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/control_metadata_missing_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `control_metadata_missing_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/control_metadata_missing_variant_2.md b/benchmarks/formal_pr_bench/rationales/control_metadata_missing_variant_2.md new file mode 100644 index 0000000..285cf12 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/control_metadata_missing_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `control_metadata_missing_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/control_preserved.md b/benchmarks/formal_pr_bench/rationales/control_preserved.md new file mode 100644 index 0000000..e82e15c --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/control_preserved.md @@ -0,0 +1,13 @@ +# Rationale: `control_preserved` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/control_preserved_variant_1.md b/benchmarks/formal_pr_bench/rationales/control_preserved_variant_1.md new file mode 100644 index 0000000..f83b64f --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/control_preserved_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `control_preserved_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/control_preserved_variant_2.md b/benchmarks/formal_pr_bench/rationales/control_preserved_variant_2.md new file mode 100644 index 0000000..dfd65e3 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/control_preserved_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `control_preserved_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/control_removed.md b/benchmarks/formal_pr_bench/rationales/control_removed.md new file mode 100644 index 0000000..9049749 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/control_removed.md @@ -0,0 +1,13 @@ +# Rationale: `control_removed` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/control_removed_variant_1.md b/benchmarks/formal_pr_bench/rationales/control_removed_variant_1.md new file mode 100644 index 0000000..8e55cd7 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/control_removed_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `control_removed_variant_1` + +- Partition: `development` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/control_removed_variant_2.md b/benchmarks/formal_pr_bench/rationales/control_removed_variant_2.md new file mode 100644 index 0000000..94e5124 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/control_removed_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `control_removed_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/dafny_fail.md b/benchmarks/formal_pr_bench/rationales/dafny_fail.md new file mode 100644 index 0000000..dcde818 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/dafny_fail.md @@ -0,0 +1,13 @@ +# Rationale: `dafny_fail` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/dafny_fail_variant_1.md b/benchmarks/formal_pr_bench/rationales/dafny_fail_variant_1.md new file mode 100644 index 0000000..b7859eb --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/dafny_fail_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `dafny_fail_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/dafny_pass.md b/benchmarks/formal_pr_bench/rationales/dafny_pass.md new file mode 100644 index 0000000..f951f4e --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/dafny_pass.md @@ -0,0 +1,13 @@ +# Rationale: `dafny_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/dafny_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/dafny_pass_variant_1.md new file mode 100644 index 0000000..47cec25 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/dafny_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `dafny_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/deployment_skipped_approval.md b/benchmarks/formal_pr_bench/rationales/deployment_skipped_approval.md new file mode 100644 index 0000000..2467e5d --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/deployment_skipped_approval.md @@ -0,0 +1,13 @@ +# Rationale: `deployment_skipped_approval` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/deployment_skipped_approval_variant_1.md b/benchmarks/formal_pr_bench/rationales/deployment_skipped_approval_variant_1.md new file mode 100644 index 0000000..8b5e1c7 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/deployment_skipped_approval_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `deployment_skipped_approval_variant_1` + +- Partition: `development` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/deployment_skipped_approval_variant_2.md b/benchmarks/formal_pr_bench/rationales/deployment_skipped_approval_variant_2.md new file mode 100644 index 0000000..58c5c03 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/deployment_skipped_approval_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `deployment_skipped_approval_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/deployment_valid_path.md b/benchmarks/formal_pr_bench/rationales/deployment_valid_path.md new file mode 100644 index 0000000..849892a --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/deployment_valid_path.md @@ -0,0 +1,13 @@ +# Rationale: `deployment_valid_path` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/deployment_valid_path_variant_1.md b/benchmarks/formal_pr_bench/rationales/deployment_valid_path_variant_1.md new file mode 100644 index 0000000..8897087 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/deployment_valid_path_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `deployment_valid_path_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/deployment_valid_path_variant_2.md b/benchmarks/formal_pr_bench/rationales/deployment_valid_path_variant_2.md new file mode 100644 index 0000000..f0304ca --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/deployment_valid_path_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `deployment_valid_path_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/infra_private_sensitive.md b/benchmarks/formal_pr_bench/rationales/infra_private_sensitive.md new file mode 100644 index 0000000..c67b719 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/infra_private_sensitive.md @@ -0,0 +1,13 @@ +# Rationale: `infra_private_sensitive` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/infra_private_sensitive_variant_1.md b/benchmarks/formal_pr_bench/rationales/infra_private_sensitive_variant_1.md new file mode 100644 index 0000000..5316311 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/infra_private_sensitive_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `infra_private_sensitive_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/infra_private_sensitive_variant_2.md b/benchmarks/formal_pr_bench/rationales/infra_private_sensitive_variant_2.md new file mode 100644 index 0000000..6bbed32 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/infra_private_sensitive_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `infra_private_sensitive_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/infra_public_sensitive.md b/benchmarks/formal_pr_bench/rationales/infra_public_sensitive.md new file mode 100644 index 0000000..0205707 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/infra_public_sensitive.md @@ -0,0 +1,13 @@ +# Rationale: `infra_public_sensitive` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/infra_public_sensitive_variant_1.md b/benchmarks/formal_pr_bench/rationales/infra_public_sensitive_variant_1.md new file mode 100644 index 0000000..8a35fee --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/infra_public_sensitive_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `infra_public_sensitive_variant_1` + +- Partition: `development` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/infra_public_sensitive_variant_2.md b/benchmarks/formal_pr_bench/rationales/infra_public_sensitive_variant_2.md new file mode 100644 index 0000000..6fae8d2 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/infra_public_sensitive_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `infra_public_sensitive_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_fail.md b/benchmarks/formal_pr_bench/rationales/kani_fail.md new file mode 100644 index 0000000..821c8ae --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_fail.md @@ -0,0 +1,13 @@ +# Rationale: `kani_fail` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_fail_variant_1.md b/benchmarks/formal_pr_bench/rationales/kani_fail_variant_1.md new file mode 100644 index 0000000..1b87c83 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_fail_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `kani_fail_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_fail_variant_2.md b/benchmarks/formal_pr_bench/rationales/kani_fail_variant_2.md new file mode 100644 index 0000000..cf3a49e --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_fail_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `kani_fail_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_malformed.md b/benchmarks/formal_pr_bench/rationales/kani_malformed.md new file mode 100644 index 0000000..e757761 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_malformed.md @@ -0,0 +1,13 @@ +# Rationale: `kani_malformed` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_malformed_variant_1.md b/benchmarks/formal_pr_bench/rationales/kani_malformed_variant_1.md new file mode 100644 index 0000000..1c306e5 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_malformed_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `kani_malformed_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_malformed_variant_2.md b/benchmarks/formal_pr_bench/rationales/kani_malformed_variant_2.md new file mode 100644 index 0000000..e5944ae --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_malformed_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `kani_malformed_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_pass.md b/benchmarks/formal_pr_bench/rationales/kani_pass.md new file mode 100644 index 0000000..fe24f41 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_pass.md @@ -0,0 +1,13 @@ +# Rationale: `kani_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/kani_pass_variant_1.md new file mode 100644 index 0000000..6f08cb0 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `kani_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_pass_variant_2.md b/benchmarks/formal_pr_bench/rationales/kani_pass_variant_2.md new file mode 100644 index 0000000..6d69079 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_pass_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `kani_pass_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_unknown.md b/benchmarks/formal_pr_bench/rationales/kani_unknown.md new file mode 100644 index 0000000..1b14ef9 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_unknown.md @@ -0,0 +1,13 @@ +# Rationale: `kani_unknown` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_unknown_variant_1.md b/benchmarks/formal_pr_bench/rationales/kani_unknown_variant_1.md new file mode 100644 index 0000000..19c27b6 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_unknown_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `kani_unknown_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/kani_unknown_variant_2.md b/benchmarks/formal_pr_bench/rationales/kani_unknown_variant_2.md new file mode 100644 index 0000000..5fd441d --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/kani_unknown_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `kani_unknown_variant_2` + +- Partition: `held_out` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/lean_fail.md b/benchmarks/formal_pr_bench/rationales/lean_fail.md new file mode 100644 index 0000000..cc72106 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/lean_fail.md @@ -0,0 +1,13 @@ +# Rationale: `lean_fail` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/lean_fail_variant_1.md b/benchmarks/formal_pr_bench/rationales/lean_fail_variant_1.md new file mode 100644 index 0000000..e4c94aa --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/lean_fail_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `lean_fail_variant_1` + +- Partition: `held_out` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/lean_pass.md b/benchmarks/formal_pr_bench/rationales/lean_pass.md new file mode 100644 index 0000000..adbc436 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/lean_pass.md @@ -0,0 +1,13 @@ +# Rationale: `lean_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/lean_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/lean_pass_variant_1.md new file mode 100644 index 0000000..b8f7b30 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/lean_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `lean_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/multi_surface_combined_pr.md b/benchmarks/formal_pr_bench/rationales/multi_surface_combined_pr.md new file mode 100644 index 0000000..6476545 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/multi_surface_combined_pr.md @@ -0,0 +1,13 @@ +# Rationale: `multi_surface_combined_pr` + +- Partition: `development` +- Category: `multi_backend` +- Expected decision signal: `block` + +## Why this expectation + +Multi-surface PRs must exercise multiple lanes and match the merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_auth_admin_route_guarded.md b/benchmarks/formal_pr_bench/rationales/rd_auth_admin_route_guarded.md new file mode 100644 index 0000000..5f07aa0 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_auth_admin_route_guarded.md @@ -0,0 +1,13 @@ +# Rationale: `rd_auth_admin_route_guarded` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `allow` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_auth_admin_route_unguarded.md b/benchmarks/formal_pr_bench/rationales/rd_auth_admin_route_unguarded.md new file mode 100644 index 0000000..70d3df8 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_auth_admin_route_unguarded.md @@ -0,0 +1,13 @@ +# Rationale: `rd_auth_admin_route_unguarded` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_auth_route_partial_hunk.md b/benchmarks/formal_pr_bench/rationales/rd_auth_route_partial_hunk.md new file mode 100644 index 0000000..5e63bc8 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_auth_route_partial_hunk.md @@ -0,0 +1,13 @@ +# Rationale: `rd_auth_route_partial_hunk` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_cbmc_integer_overflow_quota.md b/benchmarks/formal_pr_bench/rationales/rd_cbmc_integer_overflow_quota.md new file mode 100644 index 0000000..16dd83d --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_cbmc_integer_overflow_quota.md @@ -0,0 +1,13 @@ +# Rationale: `rd_cbmc_integer_overflow_quota` + +- Partition: `held_out` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_cbmc_use_after_free_auth_cache.md b/benchmarks/formal_pr_bench/rationales/rd_cbmc_use_after_free_auth_cache.md new file mode 100644 index 0000000..73bf419 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_cbmc_use_after_free_auth_cache.md @@ -0,0 +1,13 @@ +# Rationale: `rd_cbmc_use_after_free_auth_cache` + +- Partition: `held_out` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_ci_secrets_pr_preview.md b/benchmarks/formal_pr_bench/rationales/rd_ci_secrets_pr_preview.md new file mode 100644 index 0000000..fe2f2fc --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_ci_secrets_pr_preview.md @@ -0,0 +1,13 @@ +# Rationale: `rd_ci_secrets_pr_preview` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_ci_secrets_workflow_dispatch_safe.md b/benchmarks/formal_pr_bench/rationales/rd_ci_secrets_workflow_dispatch_safe.md new file mode 100644 index 0000000..affa192 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_ci_secrets_workflow_dispatch_safe.md @@ -0,0 +1,13 @@ +# Rationale: `rd_ci_secrets_workflow_dispatch_safe` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `allow` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_deployment_direct_skip.md b/benchmarks/formal_pr_bench/rationales/rd_deployment_direct_skip.md new file mode 100644 index 0000000..0783e2f --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_deployment_direct_skip.md @@ -0,0 +1,13 @@ +# Rationale: `rd_deployment_direct_skip` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_deployment_valid_chain.md b/benchmarks/formal_pr_bench/rationales/rd_deployment_valid_chain.md new file mode 100644 index 0000000..236ee5c --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_deployment_valid_chain.md @@ -0,0 +1,13 @@ +# Rationale: `rd_deployment_valid_chain` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `allow` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_docs_only_change.md b/benchmarks/formal_pr_bench/rationales/rd_docs_only_change.md new file mode 100644 index 0000000..9226872 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_docs_only_change.md @@ -0,0 +1,13 @@ +# Rationale: `rd_docs_only_change` + +- Partition: `held_out` +- Category: `real_diff` +- Expected decision signal: `require_human_review` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_infra_iam_wildcard_admin.md b/benchmarks/formal_pr_bench/rationales/rd_infra_iam_wildcard_admin.md new file mode 100644 index 0000000..5c564b1 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_infra_iam_wildcard_admin.md @@ -0,0 +1,13 @@ +# Rationale: `rd_infra_iam_wildcard_admin` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_infra_k8s_loadbalancer.md b/benchmarks/formal_pr_bench/rationales/rd_infra_k8s_loadbalancer.md new file mode 100644 index 0000000..5a58090 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_infra_k8s_loadbalancer.md @@ -0,0 +1,13 @@ +# Rationale: `rd_infra_k8s_loadbalancer` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_infra_private_bucket.md b/benchmarks/formal_pr_bench/rationales/rd_infra_private_bucket.md new file mode 100644 index 0000000..aa8b921 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_infra_private_bucket.md @@ -0,0 +1,13 @@ +# Rationale: `rd_infra_private_bucket` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `allow` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_infra_rds_public_partial_hunk.md b/benchmarks/formal_pr_bench/rationales/rd_infra_rds_public_partial_hunk.md new file mode 100644 index 0000000..5b6efe9 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_infra_rds_public_partial_hunk.md @@ -0,0 +1,13 @@ +# Rationale: `rd_infra_rds_public_partial_hunk` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_infra_s3_public_acl.md b/benchmarks/formal_pr_bench/rationales/rd_infra_s3_public_acl.md new file mode 100644 index 0000000..8eded77 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_infra_s3_public_acl.md @@ -0,0 +1,13 @@ +# Rationale: `rd_infra_s3_public_acl` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_multi_surface_combined.md b/benchmarks/formal_pr_bench/rationales/rd_multi_surface_combined.md new file mode 100644 index 0000000..38085e1 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_multi_surface_combined.md @@ -0,0 +1,13 @@ +# Rationale: `rd_multi_surface_combined` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_self_protection_workflow_touch.md b/benchmarks/formal_pr_bench/rationales/rd_self_protection_workflow_touch.md new file mode 100644 index 0000000..827ca82 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_self_protection_workflow_touch.md @@ -0,0 +1,13 @@ +# Rationale: `rd_self_protection_workflow_touch` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `allow` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/rd_workflow_secrets_partial_hunk.md b/benchmarks/formal_pr_bench/rationales/rd_workflow_secrets_partial_hunk.md new file mode 100644 index 0000000..5a0bdd1 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/rd_workflow_secrets_partial_hunk.md @@ -0,0 +1,13 @@ +# Rationale: `rd_workflow_secrets_partial_hunk` + +- Partition: `test` +- Category: `real_diff` +- Expected decision signal: `block` + +## Why this expectation + +End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed intents/lanes and emit the expected merge recommendation. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/recall_ci_secrets_workflow_diff.md b/benchmarks/formal_pr_bench/rationales/recall_ci_secrets_workflow_diff.md new file mode 100644 index 0000000..2190e6f --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/recall_ci_secrets_workflow_diff.md @@ -0,0 +1,13 @@ +# Rationale: `recall_ci_secrets_workflow_diff` + +- Partition: `development` +- Category: `intent_recall` +- Expected decision signal: `None` + +## Why this expectation + +The planner must recall every expected intent from the diff fixture. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/recall_infra_terraform_diff.md b/benchmarks/formal_pr_bench/rationales/recall_infra_terraform_diff.md new file mode 100644 index 0000000..b1db037 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/recall_infra_terraform_diff.md @@ -0,0 +1,13 @@ +# Rationale: `recall_infra_terraform_diff` + +- Partition: `development` +- Category: `intent_recall` +- Expected decision signal: `None` + +## Why this expectation + +The planner must recall every expected intent from the diff fixture. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/recall_multi_surface_combined.md b/benchmarks/formal_pr_bench/rationales/recall_multi_surface_combined.md new file mode 100644 index 0000000..4c6959d --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/recall_multi_surface_combined.md @@ -0,0 +1,13 @@ +# Rationale: `recall_multi_surface_combined` + +- Partition: `development` +- Category: `intent_recall` +- Expected decision signal: `None` + +## Why this expectation + +The planner must recall every expected intent from the diff fixture. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/repair_loop_auth_bypass.md b/benchmarks/formal_pr_bench/rationales/repair_loop_auth_bypass.md new file mode 100644 index 0000000..9f0d345 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/repair_loop_auth_bypass.md @@ -0,0 +1,13 @@ +# Rationale: `repair_loop_auth_bypass` + +- Partition: `development` +- Category: `repair_loop` +- Expected decision signal: `block` + +## Why this expectation + +Failing input must block with a useful repair hint; the passing fixture must allow. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/repair_loop_ci_secrets.md b/benchmarks/formal_pr_bench/rationales/repair_loop_ci_secrets.md new file mode 100644 index 0000000..9ce7936 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/repair_loop_ci_secrets.md @@ -0,0 +1,13 @@ +# Rationale: `repair_loop_ci_secrets` + +- Partition: `development` +- Category: `repair_loop` +- Expected decision signal: `block` + +## Why this expectation + +Failing input must block with a useful repair hint; the passing fixture must allow. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/repair_loop_deployment_skip.md b/benchmarks/formal_pr_bench/rationales/repair_loop_deployment_skip.md new file mode 100644 index 0000000..8b35172 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/repair_loop_deployment_skip.md @@ -0,0 +1,13 @@ +# Rationale: `repair_loop_deployment_skip` + +- Partition: `development` +- Category: `repair_loop` +- Expected decision signal: `block` + +## Why this expectation + +Failing input must block with a useful repair hint; the passing fixture must allow. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/repair_loop_infra_exposure.md b/benchmarks/formal_pr_bench/rationales/repair_loop_infra_exposure.md new file mode 100644 index 0000000..3215480 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/repair_loop_infra_exposure.md @@ -0,0 +1,13 @@ +# Rationale: `repair_loop_infra_exposure` + +- Partition: `development` +- Category: `repair_loop` +- Expected decision signal: `block` + +## Why this expectation + +Failing input must block with a useful repair hint; the passing fixture must allow. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/route_alloy_model.md b/benchmarks/formal_pr_bench/rationales/route_alloy_model.md new file mode 100644 index 0000000..deb2579 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/route_alloy_model.md @@ -0,0 +1,13 @@ +# Rationale: `route_alloy_model` + +- Partition: `development` +- Category: `routing` +- Expected decision signal: `alloy` + +## Why this expectation + +Changed-file surfaces must select the expected backend from the capability registry. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/route_cedar_iam.md b/benchmarks/formal_pr_bench/rationales/route_cedar_iam.md new file mode 100644 index 0000000..7ea1aba --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/route_cedar_iam.md @@ -0,0 +1,13 @@ +# Rationale: `route_cedar_iam` + +- Partition: `development` +- Category: `routing` +- Expected decision signal: `cedar` + +## Why this expectation + +Changed-file surfaces must select the expected backend from the capability registry. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/route_dafny_proof.md b/benchmarks/formal_pr_bench/rationales/route_dafny_proof.md new file mode 100644 index 0000000..c24ff27 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/route_dafny_proof.md @@ -0,0 +1,13 @@ +# Rationale: `route_dafny_proof` + +- Partition: `development` +- Category: `routing` +- Expected decision signal: `dafny` + +## Why this expectation + +Changed-file surfaces must select the expected backend from the capability registry. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/route_kani_rust.md b/benchmarks/formal_pr_bench/rationales/route_kani_rust.md new file mode 100644 index 0000000..129fae2 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/route_kani_rust.md @@ -0,0 +1,13 @@ +# Rationale: `route_kani_rust` + +- Partition: `development` +- Category: `routing` +- Expected decision signal: `kani` + +## Why this expectation + +Changed-file surfaces must select the expected backend from the capability registry. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_fail.md b/benchmarks/formal_pr_bench/rationales/tla_fail.md new file mode 100644 index 0000000..4906781 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_fail.md @@ -0,0 +1,13 @@ +# Rationale: `tla_fail` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_fail_variant_1.md b/benchmarks/formal_pr_bench/rationales/tla_fail_variant_1.md new file mode 100644 index 0000000..2949ed4 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_fail_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `tla_fail_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_fail_variant_2.md b/benchmarks/formal_pr_bench/rationales/tla_fail_variant_2.md new file mode 100644 index 0000000..3ddceef --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_fail_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `tla_fail_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_malformed.md b/benchmarks/formal_pr_bench/rationales/tla_malformed.md new file mode 100644 index 0000000..5612e72 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_malformed.md @@ -0,0 +1,13 @@ +# Rationale: `tla_malformed` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_malformed_variant_1.md b/benchmarks/formal_pr_bench/rationales/tla_malformed_variant_1.md new file mode 100644 index 0000000..c2d0541 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_malformed_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `tla_malformed_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_malformed_variant_2.md b/benchmarks/formal_pr_bench/rationales/tla_malformed_variant_2.md new file mode 100644 index 0000000..7a54e8d --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_malformed_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `tla_malformed_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_pass.md b/benchmarks/formal_pr_bench/rationales/tla_pass.md new file mode 100644 index 0000000..3015e7a --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_pass.md @@ -0,0 +1,13 @@ +# Rationale: `tla_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/tla_pass_variant_1.md new file mode 100644 index 0000000..842bfca --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `tla_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_pass_variant_2.md b/benchmarks/formal_pr_bench/rationales/tla_pass_variant_2.md new file mode 100644 index 0000000..2467d9e --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_pass_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `tla_pass_variant_2` + +- Partition: `test` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_unknown.md b/benchmarks/formal_pr_bench/rationales/tla_unknown.md new file mode 100644 index 0000000..93af797 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_unknown.md @@ -0,0 +1,13 @@ +# Rationale: `tla_unknown` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_unknown_variant_1.md b/benchmarks/formal_pr_bench/rationales/tla_unknown_variant_1.md new file mode 100644 index 0000000..239fe21 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_unknown_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `tla_unknown_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/tla_unknown_variant_2.md b/benchmarks/formal_pr_bench/rationales/tla_unknown_variant_2.md new file mode 100644 index 0000000..988cd13 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/tla_unknown_variant_2.md @@ -0,0 +1,13 @@ +# Rationale: `tla_unknown_variant_2` + +- Partition: `held_out` +- Category: `lane` +- Expected decision signal: `require_human_review` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/verus_fail.md b/benchmarks/formal_pr_bench/rationales/verus_fail.md new file mode 100644 index 0000000..928e91a --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/verus_fail.md @@ -0,0 +1,13 @@ +# Rationale: `verus_fail` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/verus_fail_variant_1.md b/benchmarks/formal_pr_bench/rationales/verus_fail_variant_1.md new file mode 100644 index 0000000..33e0225 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/verus_fail_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `verus_fail_variant_1` + +- Partition: `held_out` +- Category: `lane` +- Expected decision signal: `block` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/verus_pass.md b/benchmarks/formal_pr_bench/rationales/verus_pass.md new file mode 100644 index 0000000..0b2d1d7 --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/verus_pass.md @@ -0,0 +1,13 @@ +# Rationale: `verus_pass` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/rationales/verus_pass_variant_1.md b/benchmarks/formal_pr_bench/rationales/verus_pass_variant_1.md new file mode 100644 index 0000000..bc8c34b --- /dev/null +++ b/benchmarks/formal_pr_bench/rationales/verus_pass_variant_1.md @@ -0,0 +1,13 @@ +# Rationale: `verus_pass_variant_1` + +- Partition: `train` +- Category: `lane` +- Expected decision signal: `allow` + +## Why this expectation + +Lane fixture status and merge recommendation must match the declared expectations, including counterexample class when present. + +## Non-claims + +This case does not claim complete application security or solver completeness. diff --git a/benchmarks/formal_pr_bench/real_diff_cases.json b/benchmarks/formal_pr_bench/real_diff_cases.json index 48a7ec1..a6479d7 100644 --- a/benchmarks/formal_pr_bench/real_diff_cases.json +++ b/benchmarks/formal_pr_bench/real_diff_cases.json @@ -202,6 +202,34 @@ "expected_lanes": [], "expected_intents": [], "expected_merge_recommendation": "require_human_review" + }, + { + "case_id": "rd_cbmc_use_after_free_auth_cache", + "category": "real_diff", + "input_fixture": "benchmarks/real_diffs/cbmc_use_after_free_auth_cache.diff", + "expected_lanes": [ + "backend" + ], + "expected_intents": [ + "cbmc-no-use-after-free-auth-cache", + "cbmc-buffer-bounds", + "cbmc-no-unchecked-buffer-copy" + ], + "expected_merge_recommendation": "block" + }, + { + "case_id": "rd_cbmc_integer_overflow_quota", + "category": "real_diff", + "input_fixture": "benchmarks/real_diffs/cbmc_integer_overflow_quota.diff", + "expected_lanes": [ + "backend" + ], + "expected_intents": [ + "cbmc-no-integer-overflow-quota", + "cbmc-buffer-bounds", + "cbmc-no-unchecked-buffer-copy" + ], + "expected_merge_recommendation": "block" } ] } diff --git a/benchmarks/formal_pr_bench/template_dev_cases.json b/benchmarks/formal_pr_bench/template_dev_cases.json new file mode 100644 index 0000000..61ddf07 --- /dev/null +++ b/benchmarks/formal_pr_bench/template_dev_cases.json @@ -0,0 +1,45 @@ +{ + "schema_version": "formal_pr_bench.template_dev.v1", + "benchmark_version": "v1", + "description": "Cases whose fixtures informed property-template development. These must never be counted as held-out evaluation.", + "case_ids": [ + "alloy_fail", + "alloy_pass", + "auth_bypass", + "auth_malformed", + "auth_preserved", + "cbmc_fail", + "cbmc_native_buffer_bounds_pass", + "cbmc_native_integer_overflow_pass", + "cbmc_native_uaf_pass", + "cbmc_native_unchecked_copy_pass", + "cbmc_pass", + "cedar_fail", + "cedar_malformed", + "cedar_pass", + "cedar_unknown", + "ci_secrets_exposed", + "ci_secrets_safe", + "control_metadata_missing", + "control_preserved", + "control_removed", + "dafny_fail", + "dafny_pass", + "deployment_skipped_approval", + "deployment_valid_path", + "infra_private_sensitive", + "infra_public_sensitive", + "kani_fail", + "kani_malformed", + "kani_pass", + "kani_unknown", + "lean_fail", + "lean_pass", + "tla_fail", + "tla_malformed", + "tla_pass", + "tla_unknown", + "verus_fail", + "verus_pass" + ] +} diff --git a/schemas/formal_pr_bench.leaderboard.schema.json b/schemas/formal_pr_bench.leaderboard.schema.json index 71c31e3..8e5fd01 100644 --- a/schemas/formal_pr_bench.leaderboard.schema.json +++ b/schemas/formal_pr_bench.leaderboard.schema.json @@ -3,10 +3,15 @@ "$id": "https://openverification.dev/schemas/formal_pr_bench.leaderboard.schema.json", "title": "FormalPR-Bench Leaderboard", "type": "object", - "required": ["schema_version", "benchmark", "case_set", "summary", "timing_ms", "cases"], + "required": ["schema_version", "benchmark", "benchmark_version", "partition", "case_set", "summary", "timing_ms", "cases"], "properties": { "schema_version": { "const": "formal_pr_bench.leaderboard.v1" }, "benchmark": { "type": "string" }, + "benchmark_version": { "type": "string", "minLength": 1 }, + "partition": { + "type": "string", + "enum": ["train", "development", "test", "held_out", "all"] + }, "case_set": { "type": "string" }, "generated_at_unix_ms": { "type": "integer" }, "summary": { diff --git a/schemas/formal_pr_bench.manifest.schema.json b/schemas/formal_pr_bench.manifest.schema.json new file mode 100644 index 0000000..6dc6dc6 --- /dev/null +++ b/schemas/formal_pr_bench.manifest.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openverification.dev/schemas/formal_pr_bench.manifest.schema.json", + "title": "FormalPR-Bench Version Manifest", + "type": "object", + "required": [ + "schema_version", + "benchmark_version", + "case_count", + "partitions", + "partition_digests", + "corpus_digest" + ], + "properties": { + "schema_version": { "const": "formal_pr_bench.manifest.v1" }, + "benchmark_version": { "type": "string", "minLength": 1 }, + "case_count": { "type": "integer", "minimum": 1 }, + "partitions": { + "type": "array", + "items": { "type": "string" }, + "minItems": 4 + }, + "partition_digests": { + "type": "object", + "required": ["train", "development", "test", "held_out"], + "properties": { + "train": { "type": "string", "minLength": 64, "maxLength": 64 }, + "development": { "type": "string", "minLength": 64, "maxLength": 64 }, + "test": { "type": "string", "minLength": 64, "maxLength": 64 }, + "held_out": { "type": "string", "minLength": 64, "maxLength": 64 } + }, + "additionalProperties": false + }, + "corpus_digest": { "type": "string", "minLength": 64, "maxLength": 64 }, + "licenses_digest": { "type": "string" }, + "duplication_report_digest": { "type": "string" }, + "artifact_layout": { "type": "object", "additionalProperties": true }, + "real_diff_case_count": { "type": "integer", "minimum": 0 }, + "mutation_files": { "type": "array", "items": { "type": "string" } }, + "adversarial_files": { "type": "array", "items": { "type": "string" } } + }, + "additionalProperties": false +} From b1c090f21617635c51cd26c6109c86f982352744 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:09:42 -0700 Subject: [PATCH 09/19] Add FormalPR provenance generators and scoring hooks (OVK-PR5). Wire provenance generation, holdout runners, and scoring updates so benchmark publications validate against the versioned manifest rather than ad-hoc file lists. --- benchmarks/formal_pr_bench/score_all_lanes.py | 27 + benchmarks/formal_pr_bench/scoring.py | 22 +- docs/BENCHMARK.md | 29 +- docs/FORMALPR_HOLDOUT_GOVERNANCE.md | 2 + docs/HOLDOUT_LABEL_SEPARATION.md | 73 +-- docs/benchmarks/adoption-summary.json | 94 +++- .../latest-leaderboard-summary.json | 8 +- scripts/generate_formalpr_provenance.py | 527 ++++++++++++++++++ scripts/run_formalpr_holdout.py | 17 + tests/test_bench_badge.py | 2 + tests/test_formalpr_bench_provenance.py | 169 ++++++ 11 files changed, 911 insertions(+), 59 deletions(-) create mode 100644 scripts/generate_formalpr_provenance.py create mode 100644 tests/test_formalpr_bench_provenance.py diff --git a/benchmarks/formal_pr_bench/score_all_lanes.py b/benchmarks/formal_pr_bench/score_all_lanes.py index bddc8b8..fd61e9d 100644 --- a/benchmarks/formal_pr_bench/score_all_lanes.py +++ b/benchmarks/formal_pr_bench/score_all_lanes.py @@ -103,12 +103,30 @@ def run_benchmark( *, expanded: bool = False, include_extended: bool = True, + partition: str | None = None, ) -> tuple[list[Any], dict[str, Any]]: from ovk.core.capabilities import CapabilityRegistry + from benchmarks.formal_pr_bench.provenance_kit import ( + BENCHMARK_VERSION, + filter_cases_for_partition, + load_partitions, + ) from benchmarks.formal_pr_bench.scoring import build_leaderboard, score_case cases, case_set = load_cases(expanded=expanded, include_extended=include_extended) + # Mixed corpus regression cites partition "all"; --partition filters membership. + score_partition = partition or "all" + if partition is not None: + cases = filter_cases_for_partition(cases, partition=partition) + case_set = f"{case_set}#{partition}" + else: + # Still enforce that the held_out membership is uncontaminated even when + # the default mixed corpus includes those cases for regression. + from benchmarks.formal_pr_bench.provenance_kit import assert_no_template_dev_contamination + + assert_no_template_dev_contamination(partitions=load_partitions()) + capabilities = CapabilityRegistry.from_directory(ROOT / "adapters").all() lane_evaluator: Callable[[dict[str, Any]], tuple[str, str, str | None]] = evaluate_lane_case scores = [score_case(case, capabilities=capabilities, lane_evaluator=lane_evaluator) for case in cases] @@ -116,6 +134,8 @@ def run_benchmark( scores, benchmark_name="FormalPR-Bench", case_set=case_set, + partition=score_partition, + benchmark_version=BENCHMARK_VERSION, ) return scores, leaderboard @@ -124,12 +144,19 @@ def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--expanded", action="store_true", help="Score the 100-case expanded benchmark set.") parser.add_argument("--no-extended", action="store_true", help="Skip routing/adversarial/repair-loop cases.") + parser.add_argument( + "--partition", + choices=["train", "development", "test", "held_out"], + default=None, + help="Score only cases in this FormalPR-Bench partition (contamination-guarded).", + ) parser.add_argument("--leaderboard", type=Path, default=None, help="Write leaderboard JSON to this path.") args = parser.parse_args() scores, leaderboard = run_benchmark( expanded=args.expanded, include_extended=not args.no_extended, + partition=args.partition, ) failures = [score for score in scores if not score.passed] for score in scores: diff --git a/benchmarks/formal_pr_bench/scoring.py b/benchmarks/formal_pr_bench/scoring.py index 5317dff..3e289e1 100644 --- a/benchmarks/formal_pr_bench/scoring.py +++ b/benchmarks/formal_pr_bench/scoring.py @@ -15,6 +15,12 @@ from ovk.core.router import route_intent from ovk.core.surface_routing import surface_backend_bonuses +from benchmarks.formal_pr_bench.provenance_kit import ( + BENCHMARK_VERSION, + assert_no_template_dev_contamination, + require_published_score_identity, +) + ROOT = Path(__file__).resolve().parents[2] LEADERBOARD_SCHEMA = "formal_pr_bench.leaderboard.v1" @@ -356,14 +362,24 @@ def build_leaderboard( *, benchmark_name: str, case_set: str, + partition: str = "all", + benchmark_version: str = BENCHMARK_VERSION, ) -> dict[str, Any]: - """Build a publishable leaderboard JSON artifact.""" + """Build a publishable leaderboard JSON artifact. + + Published scores always cite ``benchmark_version`` and ``partition``. + Scoring the ``held_out`` partition rejects template-dev contamination. + """ + case_ids = [score.case_id for score in scores] + assert_no_template_dev_contamination(scored_case_ids=case_ids, partition=partition) timings = sorted(score.elapsed_ms for score in scores) p50_index = max(0, len(timings) // 2 - 1) p95_index = max(0, int(len(timings) * 0.95) - 1) - return { + leaderboard = { "schema_version": LEADERBOARD_SCHEMA, "benchmark": benchmark_name, + "benchmark_version": benchmark_version, + "partition": partition, "case_set": case_set, "generated_at_unix_ms": int(time.time() * 1000), "summary": aggregate_dimensions(scores), @@ -374,3 +390,5 @@ def build_leaderboard( }, "cases": [asdict(score) for score in scores], } + require_published_score_identity(leaderboard) + return leaderboard diff --git a/docs/BENCHMARK.md b/docs/BENCHMARK.md index 685fd30..446badd 100644 --- a/docs/BENCHMARK.md +++ b/docs/BENCHMARK.md @@ -43,7 +43,7 @@ python scripts/render_bench_badge.py --leaderboard .verification/formal-pr-bench ## Realistic PR diff set -- 16 diffs in `benchmarks/real_diffs/` (secrets, auth, infra, deployment, multi-surface, partial hunks). +- 18 diffs in `benchmarks/real_diffs/` (secrets, auth, infra, deployment, multi-surface, partial hunks, CBMC). - Manifest: `benchmarks/real_diffs/manifest.json`. - Integration tests: `tests/test_real_diffs.py` (≥95% check detection rate required). @@ -51,6 +51,33 @@ python scripts/render_bench_badge.py --leaderboard .verification/formal-pr-bench pytest tests/test_real_diffs.py -v ``` +## Provenance and partitions (OVK-06) + +FormalPR-Bench publishes provenance-backed partitions under `benchmarks/formal_pr_bench/`: + +| Artifact | Purpose | +|---|---| +| `provenance/.json` | Source, author, date, derivation | +| `licenses.json` | Per-case / corpus license (Apache-2.0) | +| `partitions.json` | `train` / `development` / `test` / `held_out` membership | +| `duplication_report.json` | Near-duplicate detection | +| `mutations/` | Controlled mutation variants (not scored as corpus members) | +| `held_out/` | Cases forbidden from template-dev scoring | +| `adversarial/` | Misleading diffs | +| `rationales/.md` | Expected-decision rationale | +| `manifest.v1.json` | Version manifest with partition digests | +| `template_dev_cases.json` | Cases used during property-template development | + +Published leaderboards must cite `benchmark_version` and `partition`. Template-dev cases cannot be counted as `held_out` evaluation; contamination fails CI. + +Regenerate artifacts: + +```bash +python scripts/generate_formalpr_provenance.py +``` + +Cross-links: [HOLDOUT_LABEL_SEPARATION.md](HOLDOUT_LABEL_SEPARATION.md), [FORMALPR_HOLDOUT_GOVERNANCE.md](FORMALPR_HOLDOUT_GOVERNANCE.md). + ## Category pass rates `latest-leaderboard-summary.json` includes per-category pass rates so dashboards can track trends without parsing the full leaderboard. diff --git a/docs/FORMALPR_HOLDOUT_GOVERNANCE.md b/docs/FORMALPR_HOLDOUT_GOVERNANCE.md index bd73665..bd22dcb 100644 --- a/docs/FORMALPR_HOLDOUT_GOVERNANCE.md +++ b/docs/FORMALPR_HOLDOUT_GOVERNANCE.md @@ -39,6 +39,8 @@ Runner: `scripts/run_formalpr_holdout.py` (requires immutable `--asset-sha256`). Label-separated prediction/eval flow (Sprint 8): [HOLDOUT_LABEL_SEPARATION.md](HOLDOUT_LABEL_SEPARATION.md). +Public FormalPR-Bench also maintains an in-repo `held_out` partition with provenance and a version manifest (`benchmarks/formal_pr_bench/manifest.v1.json`). Cases used during property-template development (`template_dev_cases.json`) cannot be counted as that held-out evaluation. See [BENCHMARK.md](BENCHMARK.md). + ## What this is not - Not a claim of production generalization measurement diff --git a/docs/HOLDOUT_LABEL_SEPARATION.md b/docs/HOLDOUT_LABEL_SEPARATION.md index e154750..3ee31b6 100644 --- a/docs/HOLDOUT_LABEL_SEPARATION.md +++ b/docs/HOLDOUT_LABEL_SEPARATION.md @@ -1,35 +1,38 @@ -# Label-Separated Holdout Evaluation (Sprint 8) - -Checklist for R2 Sprint 8. Builds on Phase A FormalPR-Holdout isolation -(`scripts/run_formalpr_holdout.py`, `.github/workflows/holdout-eval.yml`). - -## Required flow - -1. **Predict** using the exact RC artifact (wheel / Action pin) **without** access to protected labels. -2. **Digest** the predictions file (`scripts/digest_holdout_predictions.py`) — refuses embedded labels / ground-truth fields; emits SHA-256 record. -3. **Evaluate separately** with protected labels (token only on download step; evaluator env token-free). -4. **Publish aggregates only** (`formalpr_holdout.aggregate_metrics.v1`). - -## In-repo artifacts - -| Item | Path / note | -|---|---| -| Runner | `scripts/run_formalpr_holdout.py` (requires `--asset-sha256`; validates predictions are label-free) | -| Predictions digest | `scripts/digest_holdout_predictions.py` | -| Workflow | `.github/workflows/holdout-eval.yml` (download vs eval token split) | -| Predictions placeholder | `.verification/holdout-predictions.json` (never commit labels) | -| Governance | [FORMALPR_HOLDOUT_GOVERNANCE.md](FORMALPR_HOLDOUT_GOVERNANCE.md) | - -## Checklist - -- [ ] RC predictions generated in an environment without `corpus/labels` -- [ ] Predictions digested (`digest_holdout_predictions.py`) and retained with workflow ID -- [ ] `HOLDOUT_ASSET_SHA256` (or workflow input) set for immutable asset verify -- [ ] Eval job runs with tokens unset; aggregates schema-validated -- [ ] Published metrics cite `ovk_commit_sha` / `benchmark_source_sha` and do not embed case ids -- [ ] Do not set `verified_source_sha` on holdout aggregates unless the full required-workflow set was observed - -## Blocked outside this repo - -Protected label store and annotator workflow live in `fraware/FormalPR-Holdout` (private). -This repository cannot complete live holdout scoring without that access. +# Label-Separated Holdout Evaluation (Sprint 8) + +Checklist for R2 Sprint 8. Builds on Phase A FormalPR-Holdout isolation +(`scripts/run_formalpr_holdout.py`, `.github/workflows/holdout-eval.yml`). + +## Required flow + +1. **Predict** using the exact RC artifact (wheel / Action pin) **without** access to protected labels. +2. **Digest** the predictions file (`scripts/digest_holdout_predictions.py`) — refuses embedded labels / ground-truth fields; emits SHA-256 record. +3. **Evaluate separately** with protected labels (token only on download step; evaluator env token-free). +4. **Publish aggregates only** (`formalpr_holdout.aggregate_metrics.v1`). + +## In-repo artifacts + +| Item | Path / note | +|---|---| +| Runner | `scripts/run_formalpr_holdout.py` (requires `--asset-sha256`; validates predictions are label-free) | +| Predictions digest | `scripts/digest_holdout_predictions.py` | +| Workflow | `.github/workflows/holdout-eval.yml` (download vs eval token split) | +| Predictions placeholder | `.verification/holdout-predictions.json` (never commit labels) | +| Governance | [FORMALPR_HOLDOUT_GOVERNANCE.md](FORMALPR_HOLDOUT_GOVERNANCE.md) | +| Public bench partitions | [BENCHMARK.md](BENCHMARK.md) provenance section; `benchmarks/formal_pr_bench/manifest.v1.json` | + +Template-development cases listed in `benchmarks/formal_pr_bench/template_dev_cases.json` must never be counted as FormalPR-Bench `held_out` evaluation. The holdout runner and FormalPR-Bench scorer both fail closed on that contamination. + +## Checklist + +- [ ] RC predictions generated in an environment without `corpus/labels` +- [ ] Predictions digested (`digest_holdout_predictions.py`) and retained with workflow ID +- [ ] `HOLDOUT_ASSET_SHA256` (or workflow input) set for immutable asset verify +- [ ] Eval job runs with tokens unset; aggregates schema-validated +- [ ] Published metrics cite `ovk_commit_sha` / `benchmark_source_sha` and do not embed case ids +- [ ] Do not set `verified_source_sha` on holdout aggregates unless the full required-workflow set was observed + +## Blocked outside this repo + +Protected label store and annotator workflow live in `fraware/FormalPR-Holdout` (private). +This repository cannot complete live holdout scoring without that access. diff --git a/docs/benchmarks/adoption-summary.json b/docs/benchmarks/adoption-summary.json index 4d146cc..c7ad230 100644 --- a/docs/benchmarks/adoption-summary.json +++ b/docs/benchmarks/adoption-summary.json @@ -1,10 +1,10 @@ { "schema_version": "ovk.adoption_summary.v1", - "ovk_version": "1.2.0", - "updated_at": "2026-06-10T23:38:24Z", + "ovk_version": "1.2.1", + "updated_at": "2026-07-25T17:11:57Z", "formal_pr_bench": { - "cases_total": 130, - "cases_passed": 130, + "cases_total": 132, + "cases_passed": 132, "pass_rate": 1.0, "intent_recall": 1.0, "by_category": { @@ -29,8 +29,8 @@ "pass_rate": 1.0 }, "real_diff": { - "cases_total": 16, - "cases_passed": 16, + "cases_total": 18, + "cases_passed": 18, "pass_rate": 1.0 }, "repair_loop": { @@ -45,27 +45,87 @@ } }, "timing_ms": { - "p50": 0.12271200000668614, - "p95": 12.90817699999991, - "max": 25.407987999997772 + "p50": 0.12826100000040697, + "p95": 15.816610999998204, + "max": 34.02709299999884 } }, "real_diff_recall": 1.0, "pilot_dogfood": { "source": "local", - "last_run": "2026-06-10T23:38:24Z", - "manifests_passed": 5, - "manifests_total": 5, - "median_elapsed_ms": 113.28040000012152, + "last_run": "2026-07-25T17:11:57Z", + "manifests_passed": 8, + "manifests_total": 8, + "median_elapsed_ms": 4.71550000111165, "false_positive_rate": 0.0, "external_manifest": "examples/pilot_repos/external_oss_ci_secrets.json", "weekly_schedule": "0 7 * * 1", "workflow": ".github/workflows/pilot-dogfood.yml", - "ovk_version_pin": "1.2.0" + "ovk_version_pin": "1.2.1" }, "external_pilots": [ { - "repository": "TBD \u2014 recruiting first OSS adopter", + "repository": "fraware/ovk-consumer-fastapi-terraform", + "status": "advisory", + "check_types": [ + "ci_secrets", + "infrastructure" + ], + "advisory_start": "2026-07-11", + "advisory_end": "2026-07-25", + "prs_evaluated": 2, + "prs_blocked": 1, + "false_positives": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 95.62, + "strict_enabled": false, + "ovk_version_pin": "1.2.1", + "workflow_path": ".github/workflows/ovk-advisory-pr.yml", + "evidence_url": "docs/pilots/fastapi-terraform/pilot-report.json", + "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md." + }, + { + "repository": "fraware/ovk-consumer-express-actions", + "status": "advisory", + "check_types": [ + "ci_secrets", + "self_protection" + ], + "advisory_start": "2026-07-11", + "advisory_end": "2026-07-25", + "prs_evaluated": 2, + "prs_blocked": 1, + "false_positives": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 81.71, + "strict_enabled": false, + "ovk_version_pin": "1.2.1", + "workflow_path": ".github/workflows/ovk-advisory-pr.yml", + "evidence_url": "docs/pilots/express-actions/pilot-report.json", + "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md." + }, + { + "repository": "in-repo/ovk-pilot-infra-terraform-k8s", + "status": "advisory", + "check_types": [ + "infrastructure", + "ci_secrets" + ], + "advisory_start": "2026-07-11", + "advisory_end": "2026-07-25", + "prs_evaluated": 3, + "prs_blocked": 2, + "false_positives": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 46.14, + "strict_enabled": false, + "ovk_version_pin": "1.2.1", + "workflow_path": "docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml", + "evidence_url": "docs/pilots/infra-terraform-k8s/pilot-report.json", + "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md." + }, + { + "repository": "TBD - recruiting first true external OSS adopter", "status": "recruiting", "check_types": [ "ci_secrets" @@ -78,9 +138,9 @@ "false_positive_rate": null, "median_check_latency_ms": null, "strict_enabled": false, - "ovk_version_pin": "1.2.0", + "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-pilot.yml", - "notes": "Placeholder row until the first external OSS repo completes advisory rollout. See docs/EXTERNAL_PILOT_PLAYBOOK.md and docs/templates/pilot_manifest_ci_secrets.template.json." + "notes": "Placeholder for a true independent external OSS adopter. Maintained-consumer and in-repo profile pilots are published under docs/pilots/; they do not satisfy this recruiting row." } ] } diff --git a/docs/benchmarks/latest-leaderboard-summary.json b/docs/benchmarks/latest-leaderboard-summary.json index 9db8326..4dcaca0 100644 --- a/docs/benchmarks/latest-leaderboard-summary.json +++ b/docs/benchmarks/latest-leaderboard-summary.json @@ -1,8 +1,8 @@ { "schema_version": "formal_pr_bench.summary.v1", "generated_from": "formal_pr_bench.leaderboard.v1", - "cases_total": 130, - "cases_passed": 130, + "cases_total": 132, + "cases_passed": 132, "pass_rate": 1.0, "merge_decision_accuracy": 1.0, "status_accuracy": 1.0, @@ -33,8 +33,8 @@ "pass_rate": 1.0 }, "real_diff": { - "cases_total": 16, - "cases_passed": 16, + "cases_total": 18, + "cases_passed": 18, "pass_rate": 1.0 }, "repair_loop": { diff --git a/scripts/generate_formalpr_provenance.py b/scripts/generate_formalpr_provenance.py new file mode 100644 index 0000000..913e85a --- /dev/null +++ b/scripts/generate_formalpr_provenance.py @@ -0,0 +1,527 @@ +#!/usr/bin/env python +"""Generate FormalPR-Bench provenance artifacts (OVK-PR5 / OVK-06). + +Writes partitions, licenses, provenance records, rationales, mutations, +held-out descriptors, adversarial fixtures, duplication report, and +manifest.v1.json under benchmarks/formal_pr_bench/. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from benchmarks.formal_pr_bench.provenance_kit import ( # noqa: E402 + ADVERSARIAL_DIR, + BENCH_DIR, + BENCHMARK_VERSION, + DUPLICATION_REPORT_PATH, + HELD_OUT_DIR, + LICENSES_PATH, + MANIFEST_PATH, + MUTATIONS_DIR, + PARTITIONS, + PARTITIONS_PATH, + PROVENANCE_DIR, + RATIONALES_DIR, + TEMPLATE_DEV_PATH, + build_version_manifest, + load_all_cases, + load_json, + sha256_hex, +) + +SEED_CASES = BENCH_DIR / "seed_cases.json" +REAL_DIFFS_MANIFEST = ROOT / "benchmarks" / "real_diffs" / "manifest.json" +REAL_DIFF_CASES = BENCH_DIR / "real_diff_cases.json" + +# Explicit held-out set: never overlaps seed/template-dev fixtures. +HELD_OUT_CASE_IDS = ( + "rd_cbmc_use_after_free_auth_cache", + "rd_cbmc_integer_overflow_quota", + "rd_docs_only_change", + "alloy_fail_variant_1", + "auth_bypass_variant_2", + "cedar_malformed_variant_2", + "kani_unknown_variant_2", + "lean_fail_variant_1", + "tla_unknown_variant_2", + "verus_fail_variant_1", +) + +# Development partition: extended categories + selected expanded variants. +DEVELOPMENT_PREFERRED = ( + "route_cedar_iam", + "route_kani_rust", + "route_alloy_model", + "route_dafny_proof", + "adversarial_forged_allow", + "adversarial_sha_mismatch", + "repair_loop_infra_exposure", + "repair_loop_deployment_skip", + "repair_loop_ci_secrets", + "repair_loop_auth_bypass", + "recall_ci_secrets_workflow_diff", + "recall_infra_terraform_diff", + "recall_multi_surface_combined", + "multi_surface_combined_pr", + "auth_bypass_variant_1", + "ci_secrets_exposed_variant_1", + "infra_public_sensitive_variant_1", + "deployment_skipped_approval_variant_1", + "control_removed_variant_1", + "cbmc_fail_variant_1", +) + + +def _write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def sync_real_diff_cases() -> int: + """Regenerate real_diff_cases.json from benchmarks/real_diffs/manifest.json.""" + manifest = load_json(REAL_DIFFS_MANIFEST) + cases = [ + { + "case_id": item["case_id"], + "category": "real_diff", + "input_fixture": f"benchmarks/real_diffs/{item['diff']}", + "expected_lanes": item["expected_lanes"], + "expected_intents": item.get("expected_intents", []), + "expected_merge_recommendation": item["expected_recommendation"], + } + for item in manifest["cases"] + ] + _write_json(REAL_DIFF_CASES, {"schema_version": "formal_pr_bench.real_diff.v1", "cases": cases}) + return len(cases) + + +def _normalize_text(text: str) -> str: + return re.sub(r"\s+", " ", text.strip().lower()) + + +def build_duplication_report(cases: dict[str, dict[str, Any]]) -> dict[str, Any]: + """Near-duplicate detection over fixture paths and expected-outcome fingerprints.""" + by_fixture: dict[str, list[str]] = defaultdict(list) + by_fingerprint: dict[str, list[str]] = defaultdict(list) + for case_id, case in cases.items(): + fixture = str(case.get("input_fixture") or case.get("changed_files") or "") + if fixture: + by_fixture[fixture].append(case_id) + fingerprint = sha256_hex( + { + "category": case.get("category"), + "expected_status": case.get("expected_status"), + "expected_merge_recommendation": case.get("expected_merge_recommendation"), + "expected_intents": case.get("expected_intents"), + "expected_lanes": case.get("expected_lanes"), + "expected_backend": case.get("expected_backend"), + "fixture": fixture, + } + ) + by_fingerprint[fingerprint].append(case_id) + + fixture_dups = [ + {"fixture": fixture, "case_ids": sorted(ids)} + for fixture, ids in sorted(by_fixture.items()) + if len(ids) > 1 + ] + outcome_near_dups = [ + {"fingerprint": digest, "case_ids": sorted(ids)} + for digest, ids in sorted(by_fingerprint.items()) + if len(ids) > 1 + ] + + # Textual near-duplicates among real_diff fixtures. + real_diff_texts: dict[str, str] = {} + for case_id, case in cases.items(): + if case.get("category") != "real_diff": + continue + fixture = case.get("input_fixture") + if not fixture: + continue + path = ROOT / str(fixture) + if path.is_file(): + real_diff_texts[case_id] = _normalize_text(path.read_text(encoding="utf-8")) + + text_pairs: list[dict[str, Any]] = [] + ids = sorted(real_diff_texts) + for i, left in enumerate(ids): + left_tokens = set(real_diff_texts[left].split()) + if not left_tokens: + continue + for right in ids[i + 1 :]: + right_tokens = set(real_diff_texts[right].split()) + if not right_tokens: + continue + jaccard = len(left_tokens & right_tokens) / len(left_tokens | right_tokens) + if jaccard >= 0.85: + text_pairs.append( + { + "case_ids": [left, right], + "jaccard": round(jaccard, 4), + "threshold": 0.85, + } + ) + + return { + "schema_version": "formal_pr_bench.duplication_report.v1", + "benchmark_version": BENCHMARK_VERSION, + "method": { + "fixture_path_collision": "exact", + "outcome_fingerprint": "sha256 over selected expectation fields", + "real_diff_text": "token Jaccard >= 0.85 on normalized unified diffs", + }, + "fixture_path_duplicates": fixture_dups, + "outcome_near_duplicates": outcome_near_dups, + "real_diff_text_near_duplicates": text_pairs, + "summary": { + "fixture_path_duplicate_groups": len(fixture_dups), + "outcome_near_duplicate_groups": len(outcome_near_dups), + "real_diff_text_near_duplicate_pairs": len(text_pairs), + }, + } + + +def assign_partitions(cases: dict[str, dict[str, Any]], template_dev: set[str]) -> dict[str, list[str]]: + all_ids = sorted(cases) + held_out = [case_id for case_id in HELD_OUT_CASE_IDS if case_id in cases] + held_set = set(held_out) + if held_set & template_dev: + raise SystemExit(f"held_out overlaps template_dev: {sorted(held_set & template_dev)}") + + remaining = [case_id for case_id in all_ids if case_id not in held_set] + development = [case_id for case_id in DEVELOPMENT_PREFERRED if case_id in remaining] + development_set = set(development) + rest = [case_id for case_id in remaining if case_id not in development_set] + + # Prefer seed/template-dev and most lane cases in train; real_diff (non-held) + leftover variants in test. + train: list[str] = [] + test: list[str] = [] + for case_id in rest: + case = cases[case_id] + category = str(case.get("category", "lane")) + if case_id in template_dev or category == "lane": + # Keep ~15% of non-seed lane variants in test for generalization. + if case_id not in template_dev and case_id.endswith("_variant_2"): + test.append(case_id) + else: + train.append(case_id) + elif category == "real_diff": + test.append(case_id) + else: + development.append(case_id) + + partitions = { + "train": sorted(train), + "development": sorted(set(development)), + "test": sorted(test), + "held_out": sorted(held_out), + } + covered = set().union(*(partitions[name] for name in PARTITIONS)) + missing = sorted(set(all_ids) - covered) + if missing: + partitions["train"] = sorted(set(partitions["train"]) | set(missing)) + overlap_check: set[str] = set() + for name in PARTITIONS: + for case_id in partitions[name]: + if case_id in overlap_check: + raise SystemExit(f"partition overlap involving {case_id}") + overlap_check.add(case_id) + if overlap_check != set(all_ids): + raise SystemExit("partition assignment does not cover the full corpus") + return partitions + + +def provenance_record(case_id: str, case: dict[str, Any], *, template_dev: bool, partition: str) -> dict[str, Any]: + source_file = str(case.get("_source_file", "")) + fixture = case.get("input_fixture") + derivation = "seed_fixture" + if case_id.endswith(("_variant_1", "_variant_2")): + derivation = "expanded_variant" + elif str(case.get("category")) == "real_diff": + derivation = "sanitized_real_diff" + elif str(case.get("category")) in {"routing", "adversarial", "repair_loop", "intent_recall", "multi_backend"}: + derivation = "extended_category" + return { + "schema_version": "formal_pr_bench.case_provenance.v1", + "case_id": case_id, + "source": source_file or "benchmarks/formal_pr_bench", + "author": "fraware", + "date": "2026-07-25", + "derivation": derivation, + "partition": partition, + "template_dev": template_dev, + "fixture": fixture, + "category": case.get("category", "lane"), + "license": "Apache-2.0", + "notes": ( + "Case used during property-template development; forbidden from held-out scoring." + if template_dev + else "Evaluation/regression case; cite benchmark_version + partition when publishing scores." + ), + } + + +def rationale_markdown(case_id: str, case: dict[str, Any], *, partition: str) -> str: + expected = ( + case.get("expected_merge_recommendation") + or case.get("expected_status") + or case.get("expected_backend") + or case.get("expected_quality_passed") + ) + lines = [ + f"# Rationale: `{case_id}`", + "", + f"- Partition: `{partition}`", + f"- Category: `{case.get('category', 'lane')}`", + f"- Expected decision signal: `{expected}`", + "", + "## Why this expectation", + "", + ] + category = str(case.get("category", "lane")) + if category == "real_diff": + lines.append( + "End-to-end `ovk check` on a sanitized agent-style PR diff must recall the listed " + "intents/lanes and emit the expected merge recommendation." + ) + elif category == "routing": + lines.append("Changed-file surfaces must select the expected backend from the capability registry.") + elif category == "adversarial": + lines.append("Tampered or inconsistent evidence bundles must fail the evidence-quality gate.") + elif category == "repair_loop": + lines.append("Failing input must block with a useful repair hint; the passing fixture must allow.") + elif category == "intent_recall": + lines.append("The planner must recall every expected intent from the diff fixture.") + elif category == "multi_backend": + lines.append("Multi-surface PRs must exercise multiple lanes and match the merge recommendation.") + else: + lines.append( + "Lane fixture status and merge recommendation must match the declared expectations, " + "including counterexample class when present." + ) + lines.extend(["", "## Non-claims", "", "This case does not claim complete application security or solver completeness.", ""]) + return "\n".join(lines) + + +def write_mutations(cases: dict[str, dict[str, Any]]) -> list[str]: + """Emit a small controlled mutation set derived from seed fixtures.""" + MUTATIONS_DIR.mkdir(parents=True, exist_ok=True) + written: list[str] = [] + specs = [ + ("control_removed", "flip_merge_to_allow", {"expected_merge_recommendation": "allow"}), + ("auth_bypass", "drop_counterexample", {"expected_counterexample_class": None}), + ("ci_secrets_exposed", "status_to_pass", {"expected_status": "pass", "expected_merge_recommendation": "allow"}), + ] + for base_id, mutation_id, overrides in specs: + if base_id not in cases: + continue + base = dict(cases[base_id]) + base.pop("_source_file", None) + mutated = { + "schema_version": "formal_pr_bench.mutation.v1", + "mutation_id": f"{base_id}__{mutation_id}", + "base_case_id": base_id, + "mutation": mutation_id, + "description": "Controlled expectation mutation for contamination/robustness tests; not a scored corpus member.", + "base_case": base, + "overrides": overrides, + "mutated_expectations": {**{k: base.get(k) for k in overrides}, **overrides}, + } + path = MUTATIONS_DIR / f"{base_id}__{mutation_id}.json" + _write_json(path, mutated) + written.append(path.name) + return written + + +def write_adversarial() -> list[str]: + """Emit misleading diffs that look documentation-only or safe but should not be treated as held-out truth.""" + ADVERSARIAL_DIR.mkdir(parents=True, exist_ok=True) + written: list[str] = [] + + misleading_docs = """diff --git a/README.md b/README.md +--- a/README.md ++++ b/README.md +@@ -1,3 +1,4 @@ + # Example Repo + + Documentation update only. ++Also tweaks CI below (see workflow). +diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml +--- a/.github/workflows/ci.yml ++++ b/.github/workflows/ci.yml +@@ -10,6 +10,8 @@ jobs: + steps: + - uses: actions/checkout@v4 ++ - name: Preview ++ run: echo ${{ secrets.PROD_TOKEN }} +""" + path = ADVERSARIAL_DIR / "misleading_docs_hides_secret.diff" + path.write_text(misleading_docs, encoding="utf-8") + written.append(path.name) + + meta = { + "schema_version": "formal_pr_bench.adversarial.v1", + "adversarial_id": "misleading_docs_hides_secret", + "diff": "misleading_docs_hides_secret.diff", + "intended_trap": "Looks like a docs-only PR in the commit message/path mix but injects an untrusted secret use.", + "expected_merge_recommendation": "block", + "expected_lanes": ["ci_secrets", "self_protection"], + "not_held_out": True, + "notes": "Adversarial fixture for robustness; must not be counted as held-out evaluation.", + } + meta_path = ADVERSARIAL_DIR / "misleading_docs_hides_secret.json" + _write_json(meta_path, meta) + written.append(meta_path.name) + + forged_allow = { + "schema_version": "formal_pr_bench.adversarial.v1", + "adversarial_id": "forged_allow_label", + "description": "Case metadata claims allow while pointing at a known-failing auth bypass fixture.", + "input_fixture": "examples/auth_regression/input_admin_bypass.json", + "claimed_expected_merge_recommendation": "allow", + "actual_expected_merge_recommendation": "block", + "intended_trap": "Trusting attacker-supplied expected labels without fixture evaluation.", + "not_held_out": True, + } + forged_path = ADVERSARIAL_DIR / "forged_allow_label.json" + _write_json(forged_path, forged_allow) + written.append(forged_path.name) + return written + + +def write_held_out_descriptors(cases: dict[str, dict[str, Any]], held_out_ids: list[str]) -> None: + HELD_OUT_DIR.mkdir(parents=True, exist_ok=True) + for case_id in held_out_ids: + case = dict(cases[case_id]) + case.pop("_source_file", None) + _write_json( + HELD_OUT_DIR / f"{case_id}.json", + { + "schema_version": "formal_pr_bench.held_out.v1", + "case_id": case_id, + "forbidden_from_template_dev_scoring": True, + "case": case, + }, + ) + readme = HELD_OUT_DIR / "README.md" + readme.write_text( + "\n".join( + [ + "# FormalPR-Bench held-out cases", + "", + "Cases in this directory belong to the `held_out` partition.", + "They must not be used during property-template development scoring.", + "", + "Contamination between `template_dev_cases.json` and this partition fails CI.", + "See [docs/HOLDOUT_LABEL_SEPARATION.md](../../../docs/HOLDOUT_LABEL_SEPARATION.md) and", + "[docs/FORMALPR_HOLDOUT_GOVERNANCE.md](../../../docs/FORMALPR_HOLDOUT_GOVERNANCE.md).", + "", + ] + ), + encoding="utf-8", + ) + + +def generate() -> dict[str, Any]: + real_diff_count = sync_real_diff_cases() + cases = load_all_cases() + seed_ids = { + str(case["case_id"]) + for case in load_json(SEED_CASES).get("cases", []) + } + template_dev_payload = { + "schema_version": "formal_pr_bench.template_dev.v1", + "benchmark_version": BENCHMARK_VERSION, + "description": ( + "Cases whose fixtures informed property-template development. " + "These must never be counted as held-out evaluation." + ), + "case_ids": sorted(seed_ids), + } + _write_json(TEMPLATE_DEV_PATH, template_dev_payload) + + partitions_map = assign_partitions(cases, seed_ids) + membership = {case_id: name for name, ids in partitions_map.items() for case_id in ids} + partitions_payload = { + "schema_version": "formal_pr_bench.partitions.v1", + "benchmark_version": BENCHMARK_VERSION, + "partitions": partitions_map, + "counts": {name: len(partitions_map[name]) for name in PARTITIONS}, + } + _write_json(PARTITIONS_PATH, partitions_payload) + + licenses_payload = { + "schema_version": "formal_pr_bench.licenses.v1", + "benchmark_version": BENCHMARK_VERSION, + "corpus_license": "Apache-2.0", + "corpus_license_file": "LICENSE", + "default_case_license": "Apache-2.0", + "cases": { + case_id: { + "license": "Apache-2.0", + "copyright": "Copyright 2026 fraware", + "source": case.get("_source_file"), + } + for case_id, case in sorted(cases.items()) + }, + } + _write_json(LICENSES_PATH, licenses_payload) + + duplication = build_duplication_report(cases) + _write_json(DUPLICATION_REPORT_PATH, duplication) + + PROVENANCE_DIR.mkdir(parents=True, exist_ok=True) + RATIONALES_DIR.mkdir(parents=True, exist_ok=True) + for case_id, case in cases.items(): + partition = membership[case_id] + record = provenance_record(case_id, case, template_dev=case_id in seed_ids, partition=partition) + _write_json(PROVENANCE_DIR / f"{case_id}.json", record) + (RATIONALES_DIR / f"{case_id}.md").write_text( + rationale_markdown(case_id, case, partition=partition), + encoding="utf-8", + ) + + write_held_out_descriptors(cases, partitions_map["held_out"]) + mutations = write_mutations(cases) + adversarial = write_adversarial() + + manifest = build_version_manifest( + partitions=partitions_payload, + case_count=len(cases), + licenses_digest=sha256_hex(licenses_payload), + duplication_digest=sha256_hex(duplication), + ) + manifest["real_diff_case_count"] = real_diff_count + manifest["mutation_files"] = mutations + manifest["adversarial_files"] = adversarial + _write_json(MANIFEST_PATH, manifest) + return { + "case_count": len(cases), + "real_diff_case_count": real_diff_count, + "counts": partitions_payload["counts"], + "manifest": str(MANIFEST_PATH.relative_to(ROOT)).replace("\\", "/"), + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate FormalPR-Bench provenance artifacts") + parser.parse_args() + summary = generate() + print(json.dumps(summary, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_formalpr_holdout.py b/scripts/run_formalpr_holdout.py index 19f6ab3..07cea33 100644 --- a/scripts/run_formalpr_holdout.py +++ b/scripts/run_formalpr_holdout.py @@ -427,6 +427,23 @@ def main(argv: list[str] | None = None) -> int: ) args = parser.parse_args(argv) + # Public FormalPR-Bench held_out must stay disjoint from template-dev cases. + # Private FormalPR-Holdout evaluation is separate, but contamination of the + # in-repo held_out partition still fails closed here. + if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + from benchmarks.formal_pr_bench.provenance_kit import ( + ProvenanceError, + assert_no_template_dev_contamination, + verify_manifest_digests, + ) + + try: + assert_no_template_dev_contamination() + verify_manifest_digests() + except ProvenanceError as exc: + _fail(str(exc)) + token = os.environ.get("HOLDOUT_DOWNLOAD_TOKEN") or os.environ.get("GITHUB_TOKEN") asset_name = args.asset_name or f"FormalPR-Holdout-{args.tag}.tar.gz" expected_digest = args.asset_sha256 or os.environ.get("HOLDOUT_ASSET_SHA256") diff --git a/tests/test_bench_badge.py b/tests/test_bench_badge.py index 149df09..9af5767 100644 --- a/tests/test_bench_badge.py +++ b/tests/test_bench_badge.py @@ -15,6 +15,8 @@ def test_badge_color_rules() -> None: def test_render_badge_shape() -> None: leaderboard = { "schema_version": "formal_pr_bench.leaderboard.v1", + "benchmark_version": "v1", + "partition": "all", "summary": {"cases_total": 100, "cases_passed": 100}, "timing_ms": {"p50": 1.0, "p95": 2.0, "max": 3.0}, } diff --git a/tests/test_formalpr_bench_provenance.py b/tests/test_formalpr_bench_provenance.py new file mode 100644 index 0000000..c249ec4 --- /dev/null +++ b/tests/test_formalpr_bench_provenance.py @@ -0,0 +1,169 @@ +"""FormalPR-Bench provenance, partitions, and contamination guards (OVK-PR5).""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from benchmarks.formal_pr_bench.provenance_kit import ( + BENCHMARK_VERSION, + BENCH_DIR, + HELD_OUT_DIR, + MANIFEST_PATH, + PARTITIONS, + PARTITIONS_PATH, + PROVENANCE_DIR, + RATIONALES_DIR, + TEMPLATE_DEV_PATH, + ProvenanceError, + assert_no_template_dev_contamination, + filter_cases_for_partition, + load_all_cases, + load_manifest, + load_partitions, + load_template_dev_cases, + partition_membership, + require_published_score_identity, + verify_manifest_digests, +) +from benchmarks.formal_pr_bench.scoring import DimensionScore, build_leaderboard +from ovk.core.json_io import read_json_file +from ovk.core.schema_validation import require_schema_valid + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_partition_digests_match_version_manifest() -> None: + verify_manifest_digests() + manifest = load_manifest() + partitions = load_partitions() + assert manifest["benchmark_version"] == BENCHMARK_VERSION + assert partitions["benchmark_version"] == BENCHMARK_VERSION + assert manifest["case_count"] == len(load_all_cases()) + assert set(manifest["partition_digests"]) == set(PARTITIONS) + + +def test_manifest_schema_valid() -> None: + payload = read_json_file(MANIFEST_PATH) + schema = read_json_file(ROOT / "schemas" / "formal_pr_bench.manifest.schema.json") + require_schema_valid(payload, schema, context="formal_pr_bench manifest") + + +def test_template_dev_and_held_out_are_disjoint() -> None: + assert_no_template_dev_contamination() + template_dev = load_template_dev_cases() + held_out = set(load_partitions()["partitions"]["held_out"]) + assert template_dev.isdisjoint(held_out) + assert held_out + for case_id in held_out: + assert (HELD_OUT_DIR / f"{case_id}.json").is_file() + assert (PROVENANCE_DIR / f"{case_id}.json").is_file() + assert (RATIONALES_DIR / f"{case_id}.md").is_file() + + +def test_every_case_has_provenance_rationale_and_license() -> None: + cases = load_all_cases() + licenses = json.loads((BENCH_DIR / "licenses.json").read_text(encoding="utf-8")) + membership = partition_membership() + assert set(membership) == set(cases) + for case_id in cases: + assert (PROVENANCE_DIR / f"{case_id}.json").is_file() + assert (RATIONALES_DIR / f"{case_id}.md").is_file() + assert case_id in licenses["cases"] + assert licenses["cases"][case_id]["license"] == "Apache-2.0" + + +def test_real_diff_corpus_synced_to_eighteen() -> None: + real_diff = json.loads((BENCH_DIR / "real_diff_cases.json").read_text(encoding="utf-8")) + manifest = json.loads((ROOT / "benchmarks" / "real_diffs" / "manifest.json").read_text(encoding="utf-8")) + assert len(real_diff["cases"]) == 18 + assert len(manifest["cases"]) == 18 + assert {case["case_id"] for case in real_diff["cases"]} == { + case["case_id"] for case in manifest["cases"] + } + assert load_manifest()["real_diff_case_count"] == 18 + + +def test_mutations_and_adversarial_present() -> None: + mutations = list((BENCH_DIR / "mutations").glob("*.json")) + adversarial = list((BENCH_DIR / "adversarial").glob("*")) + assert len(mutations) >= 3 + assert any(path.suffix == ".diff" for path in adversarial) + assert (BENCH_DIR / "duplication_report.json").is_file() + + +def test_published_leaderboard_requires_version_and_partition() -> None: + scores = [ + DimensionScore( + case_id="example", + category="lane", + passed=True, + merge_decision_correct=True, + status_correct=True, + counterexample_useful=None, + backend_selection_correct=None, + evidence_honest=True, + elapsed_ms=1.0, + details={}, + ) + ] + leaderboard = build_leaderboard( + scores, + benchmark_name="FormalPR-Bench", + case_set="unit", + partition="test", + benchmark_version=BENCHMARK_VERSION, + ) + require_published_score_identity(leaderboard) + assert leaderboard["benchmark_version"] == BENCHMARK_VERSION + assert leaderboard["partition"] == "test" + + broken = dict(leaderboard) + broken.pop("partition") + with pytest.raises(ProvenanceError, match="partition"): + require_published_score_identity(broken) + + +def test_held_out_scoring_rejects_template_dev_contamination() -> None: + template_dev = sorted(load_template_dev_cases()) + assert template_dev + with pytest.raises(ProvenanceError, match="template-dev contamination"): + assert_no_template_dev_contamination( + scored_case_ids=[template_dev[0]], + partition="held_out", + ) + + +def test_partition_filter_for_held_out_excludes_template_dev() -> None: + cases = list(load_all_cases().values()) + held = filter_cases_for_partition(cases, partition="held_out") + held_ids = {str(case["case_id"]) for case in held} + assert held_ids == set(load_partitions()["partitions"]["held_out"]) + assert held_ids.isdisjoint(load_template_dev_cases()) + + +def test_tampered_held_out_membership_fails_contamination_guard() -> None: + partitions = copy.deepcopy(load_partitions()) + seed_case = sorted(load_template_dev_cases())[0] + partitions["partitions"]["held_out"] = list(partitions["partitions"]["held_out"]) + [seed_case] + with pytest.raises(ProvenanceError, match="template-dev contamination"): + assert_no_template_dev_contamination(partitions=partitions) + + +def test_tampered_partition_digest_fails_manifest_check() -> None: + partitions = copy.deepcopy(load_partitions()) + partitions["partitions"]["test"] = list(partitions["partitions"]["test"])[:-1] + with pytest.raises(ProvenanceError, match="partition digest mismatch"): + verify_manifest_digests(partitions=partitions) + + +def test_template_dev_registry_matches_seed_cases() -> None: + seed = json.loads((BENCH_DIR / "seed_cases.json").read_text(encoding="utf-8")) + template_dev = load_template_dev_cases() + assert template_dev == {str(case["case_id"]) for case in seed["cases"]} + assert TEMPLATE_DEV_PATH.is_file() + assert PARTITIONS_PATH.is_file() From 8e8e7ab8f90da6f3ea856ffa7bf8505a48f5e2cd Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:09:43 -0700 Subject: [PATCH 10/19] Harden GitHub Action with SHA pins and scenario suite (OVK-PR6). Pin third-party Action dependencies by SHA and add adversarial scenario coverage for fork PRs, malicious paths, and workflow-dispatch edges before consumers widen adoption. --- action.yml | 48 +- .../branch_protection_required_check.yml | 5 +- .../github_workflows/external_consumer.yml | 7 +- .../pilot_advisory_with_comment.yml | 5 +- .../github_workflows/pilot_fork_adopter.yml | 7 +- ovk/core/github_check.py | 89 +++- scripts/emit_github_check.py | 155 ++++++- scripts/pin_action_shas.py | 110 +++++ .../diffs/deleted_policy.diff | 12 + .../action_hardening/diffs/empty_paths.txt | 0 .../diffs/malicious_filenames.diff | 29 ++ .../diffs/renamed_policy.diff | 12 + .../diffs/workflow_secrets.diff | 23 + .../action_hardening/events/fork_pr.json | 29 ++ .../events/workflow_dispatch.json | 16 + .../fixtures/action_hardening/scenarios.json | 93 ++++ tests/test_action_hardening_suite.py | 428 ++++++++++++++++++ tests/test_emit_github_check.py | 88 +++- 18 files changed, 1094 insertions(+), 62 deletions(-) create mode 100644 scripts/pin_action_shas.py create mode 100644 tests/fixtures/action_hardening/diffs/deleted_policy.diff create mode 100644 tests/fixtures/action_hardening/diffs/empty_paths.txt create mode 100644 tests/fixtures/action_hardening/diffs/malicious_filenames.diff create mode 100644 tests/fixtures/action_hardening/diffs/renamed_policy.diff create mode 100644 tests/fixtures/action_hardening/diffs/workflow_secrets.diff create mode 100644 tests/fixtures/action_hardening/events/fork_pr.json create mode 100644 tests/fixtures/action_hardening/events/workflow_dispatch.json create mode 100644 tests/fixtures/action_hardening/scenarios.json create mode 100644 tests/test_action_hardening_suite.py diff --git a/action.yml b/action.yml index 3f51b69..0bcf309 100644 --- a/action.yml +++ b/action.yml @@ -67,11 +67,14 @@ inputs: default: ovk-release-bundle outputs: + decision_state: + description: Normative DecisionState lattice member (allow, block, needs_review, unknown, error, skipped) + value: ${{ steps.set-outputs.outputs.decision_state }} recommendation: - description: Merge recommendation from the evidence bundle (allow, block, require_human_review, etc.) + description: Deprecated merge_recommendation alias (allow, block, require_human_review, etc.) value: ${{ steps.set-outputs.outputs.recommendation }} exit_code: - description: Process exit code implied by the recommendation (0 allow, 1 block, 2 require_human_review) + description: Process exit code implied by decision_state (0 allow, 1 block, 2 needs_review/unknown/error/skipped) value: ${{ steps.set-outputs.outputs.exit_code }} check_emitted: description: Whether a GitHub check run was successfully emitted @@ -81,7 +84,7 @@ runs: using: composite steps: - name: Cache pip packages - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: ~/.cache/pip key: ovk-pip-${{ runner.os }}-${{ hashFiles('**/pyproject.toml') }} @@ -365,22 +368,53 @@ runs: evidence_path = configured if configured.exists() else Path("ovk-evidence.json") recommendation = "" + decision_state = "" exit_code = "2" if evidence_path.exists(): bundle = json.loads(evidence_path.read_text(encoding="utf-8")) - recommendation = str(bundle.get("decision", {}).get("merge_recommendation", "")) + decision = bundle.get("decision", {}) or {} + decision_state = str(decision.get("decision_state", "") or "") + recommendation = str(decision.get("merge_recommendation", "") or "") + if not decision_state and recommendation: + alias = { + "allow": "allow", + "block": "block", + "require_human_review": "needs_review", + "allow_with_warning": "needs_review", + "require_stronger_check": "needs_review", + "needs_review": "needs_review", + "unknown": "unknown", + "error": "error", + "skipped": "skipped", + } + decision_state = alias.get(recommendation, "needs_review") + if not recommendation and decision_state: + alias = { + "allow": "allow", + "block": "block", + "needs_review": "require_human_review", + "unknown": "require_human_review", + "error": "require_human_review", + "skipped": "require_human_review", + } + recommendation = alias.get(decision_state, "require_human_review") exit_codes = { "allow": 0, - "allow_with_warning": 0, "block": 1, + "needs_review": 2, + "unknown": 2, + "error": 2, + "skipped": 2, + "allow_with_warning": 0, "require_human_review": 2, "require_stronger_check": 2, } - exit_code = str(exit_codes.get(recommendation, 2)) + exit_code = str(exit_codes.get(decision_state or recommendation, 2)) check_emitted = "true" if Path(".ovk-check-emitted").exists() else "false" github_output = os.environ["GITHUB_OUTPUT"] with open(github_output, "a", encoding="utf-8") as handle: + handle.write(f"decision_state={decision_state}\n") handle.write(f"recommendation={recommendation}\n") handle.write(f"exit_code={exit_code}\n") handle.write(f"check_emitted={check_emitted}\n") @@ -405,7 +439,7 @@ runs: exit 2 - name: Upload OVK artifacts if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ovk-evidence-artifacts-${{ github.job }} path: | diff --git a/examples/github_workflows/branch_protection_required_check.yml b/examples/github_workflows/branch_protection_required_check.yml index fe3d226..76e2668 100644 --- a/examples/github_workflows/branch_protection_required_check.yml +++ b/examples/github_workflows/branch_protection_required_check.yml @@ -12,7 +12,8 @@ permissions: checks: write env: - OVK_PACKAGE_VERSION: "1.2.1" + # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). + OVK_PACKAGE_VERSION: "1.3.0-rc.1" jobs: ovk-required-check: @@ -26,7 +27,7 @@ jobs: git fetch origin "${{ github.base_ref }}" git diff "origin/${{ github.base_ref }}...HEAD" > ovk-pr.diff - name: Run OVK strict check (required) - uses: fraware/open-verification-kernel@v1.2.1 + uses: fraware/open-verification-kernel@v1.3.0-rc.1 with: mode: strict use-check: "true" diff --git a/examples/github_workflows/external_consumer.yml b/examples/github_workflows/external_consumer.yml index d8a1256..572bbeb 100644 --- a/examples/github_workflows/external_consumer.yml +++ b/examples/github_workflows/external_consumer.yml @@ -13,7 +13,8 @@ permissions: checks: write env: - OVK_PACKAGE_VERSION: "1.2.1" + # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). + OVK_PACKAGE_VERSION: "1.3.0-rc.1" jobs: ovk-check: @@ -21,7 +22,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Run Open Verification Kernel check path - uses: fraware/open-verification-kernel@v1.2.1 + uses: fraware/open-verification-kernel@v1.3.0-rc.1 with: mode: strict use-check: "true" @@ -34,7 +35,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Run Open Verification Kernel (all five lanes) - uses: fraware/open-verification-kernel@v1.2.1 + uses: fraware/open-verification-kernel@v1.3.0-rc.1 with: mode: advisory verification-manifest: examples/verification_manifests/full_mvp.json diff --git a/examples/github_workflows/pilot_advisory_with_comment.yml b/examples/github_workflows/pilot_advisory_with_comment.yml index 9a36f16..6523b03 100644 --- a/examples/github_workflows/pilot_advisory_with_comment.yml +++ b/examples/github_workflows/pilot_advisory_with_comment.yml @@ -13,7 +13,8 @@ permissions: checks: write env: - OVK_PACKAGE_VERSION: "1.2.1" + # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). + OVK_PACKAGE_VERSION: "1.3.0-rc.1" jobs: ovk-advisory-check: @@ -21,7 +22,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: OVK advisory check with emitted GitHub check - uses: fraware/open-verification-kernel@v1.2.1 + uses: fraware/open-verification-kernel@v1.3.0-rc.1 with: mode: advisory use-check: "true" diff --git a/examples/github_workflows/pilot_fork_adopter.yml b/examples/github_workflows/pilot_fork_adopter.yml index b179d24..f1dea0e 100644 --- a/examples/github_workflows/pilot_fork_adopter.yml +++ b/examples/github_workflows/pilot_fork_adopter.yml @@ -13,7 +13,8 @@ permissions: pull-requests: write env: - OVK_PACKAGE_VERSION: "1.2.1" + # Target pin for v1.3.0-rc.1 (use only after the attributable tag exists). + OVK_PACKAGE_VERSION: "1.3.0-rc.1" jobs: ovk-ci-secrets-pilot: @@ -27,14 +28,14 @@ jobs: git fetch origin "${{ github.base_ref }}" git diff "origin/${{ github.base_ref }}...HEAD" > ovk-pr.diff - name: OVK advisory check - uses: fraware/open-verification-kernel@v1.2.1 + uses: fraware/open-verification-kernel@v1.3.0-rc.1 with: mode: advisory use-check: "true" changed-files: ovk-pr.diff post-comment: "false" - name: OVK advisory verify (ci_secrets lane manifest) - uses: fraware/open-verification-kernel@v1.2.1 + uses: fraware/open-verification-kernel@v1.3.0-rc.1 with: mode: advisory verification-manifest: .verification/ci_secrets_pilot.json diff --git a/ovk/core/github_check.py b/ovk/core/github_check.py index bc5d02a..f765a02 100644 --- a/ovk/core/github_check.py +++ b/ovk/core/github_check.py @@ -4,31 +4,93 @@ from typing import Any -from ovk.core.models import EvidenceBundle +from ovk.core.decision import merge_recommendation_to_decision_state +from ovk.core.models import DecisionState, EvidenceBundle CHECK_NAME = "Open Verification Kernel" +CONCLUSION_BY_DECISION_STATE = { + DecisionState.ALLOW.value: "success", + DecisionState.BLOCK.value: "failure", + DecisionState.NEEDS_REVIEW.value: "neutral", + DecisionState.UNKNOWN.value: "neutral", + DecisionState.ERROR.value: "neutral", + DecisionState.SKIPPED.value: "neutral", +} + +# Deprecated merge_recommendation aliases. CONCLUSION_BY_RECOMMENDATION = { "allow": "success", "block": "failure", "require_human_review": "neutral", "allow_with_warning": "success", "require_stronger_check": "neutral", + "needs_review": "neutral", + "unknown": "neutral", + "error": "neutral", + "skipped": "neutral", } +class StaleCheckRunError(ValueError): + """Raised when a check run would be published against a mismatched head SHA.""" + + +def check_run_external_id(*, repo: str, head_sha: str) -> str: + """Stable external_id for idempotent check-run create/update by head SHA.""" + return f"ovk:{repo}:{head_sha}" + + +def _decision_state_from_bundle(bundle: EvidenceBundle) -> str: + decision = bundle.decision or {} + if decision.get("decision_state"): + return str(decision["decision_state"]) + recommendation = str(decision.get("merge_recommendation", "require_human_review")) + return merge_recommendation_to_decision_state(recommendation).value + + def check_conclusion_for_recommendation(recommendation: str) -> str: - """Map an OVK merge recommendation to a GitHub check conclusion.""" - return CONCLUSION_BY_RECOMMENDATION.get(recommendation, "neutral") + """Map an OVK decision_state or merge recommendation to a GitHub check conclusion.""" + if recommendation in CONCLUSION_BY_DECISION_STATE: + return CONCLUSION_BY_DECISION_STATE[recommendation] + if recommendation in CONCLUSION_BY_RECOMMENDATION: + return CONCLUSION_BY_RECOMMENDATION[recommendation] + state = merge_recommendation_to_decision_state(recommendation) + return CONCLUSION_BY_DECISION_STATE.get(state.value, "neutral") + + +def validate_check_run_head_sha(bundle: EvidenceBundle, head_sha: str) -> None: + """Fail closed when evidence subject SHA does not match the emit target. + + Stale check results must never authorize a different commit. Empty or + unknown evidence SHAs are also rejected. + """ + evidence_sha = str((bundle.subject or {}).get("head_sha", "") or "").strip() + target = (head_sha or "").strip() + if not target: + raise StaleCheckRunError("missing target head SHA for check-run emission") + if not evidence_sha or evidence_sha == "unknown": + raise StaleCheckRunError( + f"evidence subject head_sha is missing/unknown; refusing check-run for {target}" + ) + if evidence_sha != target: + raise StaleCheckRunError( + f"stale check-run SHA mismatch: evidence={evidence_sha} target={target}" + ) def build_check_output(bundle: EvidenceBundle, *, markdown_summary: str | None = None) -> dict[str, Any]: """Build GitHub check-run output payload from an evidence bundle.""" - recommendation = str(bundle.decision.get("merge_recommendation", "require_human_review")) - summary = markdown_summary or f"OVK merge recommendation: {recommendation}" + decision_state = _decision_state_from_bundle(bundle) + recommendation = str( + bundle.decision.get("merge_recommendation") + or bundle.decision.get("decision_state") + or "needs_review" + ) + summary = markdown_summary or f"OVK decision: {decision_state} (alias: {recommendation})" return { - "title": f"OVK verification: {recommendation}", + "title": f"OVK verification: {decision_state}", "summary": summary[:65535], } @@ -38,13 +100,22 @@ def build_check_run_payload( *, head_sha: str, markdown_summary: str | None = None, + validate_sha: bool = True, ) -> dict[str, Any]: - """Build a completed GitHub check-run request body.""" - recommendation = str(bundle.decision.get("merge_recommendation", "require_human_review")) + """Build a completed GitHub check-run request body. + + Includes a stable ``external_id`` so reruns and concurrent emitters update + the same check run for a given repository head SHA. + """ + if validate_sha: + validate_check_run_head_sha(bundle, head_sha) + decision_state = _decision_state_from_bundle(bundle) + repo = str((bundle.subject or {}).get("repo", "unknown/repo") or "unknown/repo") return { "name": CHECK_NAME, "head_sha": head_sha, + "external_id": check_run_external_id(repo=repo, head_sha=head_sha), "status": "completed", - "conclusion": check_conclusion_for_recommendation(recommendation), + "conclusion": check_conclusion_for_recommendation(decision_state), "output": build_check_output(bundle, markdown_summary=markdown_summary), } diff --git a/scripts/emit_github_check.py b/scripts/emit_github_check.py index 1cd6dea..f29b435 100644 --- a/scripts/emit_github_check.py +++ b/scripts/emit_github_check.py @@ -1,5 +1,10 @@ #!/usr/bin/env python -"""Emit a GitHub check run from an OVK evidence bundle.""" +"""Emit a GitHub check run from an OVK evidence bundle. + +Fail-closed on stale head-SHA mismatch. Idempotent updates use a stable +``external_id`` (``ovk:{repo}:{head_sha}``): existing check runs with that +external_id are PATCHed; otherwise a new check run is created. +""" from __future__ import annotations @@ -7,10 +12,16 @@ import json import os import urllib.error +import urllib.parse import urllib.request from pathlib import Path +from typing import Any -from ovk.core.github_check import build_check_run_payload +from ovk.core.github_check import ( + StaleCheckRunError, + build_check_run_payload, + check_run_external_id, +) from ovk.core.json_io import read_json_file from ovk.core.models import EvidenceBundle @@ -26,25 +37,112 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def _post_check_run(api_base: str, repo: str, token: str, payload: dict) -> bool: - url = f"{api_base.rstrip('/')}/repos/{repo}/check-runs" - body = json.dumps(payload).encode("utf-8") - request = urllib.request.Request( - url, - data=body, - method="POST", - headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {token}", - "X-GitHub-Api-Version": "2022-11-28", - "Content-Type": "application/json", - }, - ) +def _request( + url: str, + *, + token: str, + method: str = "GET", + payload: dict[str, Any] | None = None, +) -> tuple[int, dict[str, Any] | list[Any] | None]: + body = None if payload is None else json.dumps(payload).encode("utf-8") + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + } + if body is not None: + headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=body, method=method, headers=headers) try: with urllib.request.urlopen(request, timeout=15) as response: - return 200 <= response.status < 300 - except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError): - return False + raw = response.read().decode("utf-8") + parsed: dict[str, Any] | list[Any] | None = json.loads(raw) if raw else None + return int(response.status), parsed + except urllib.error.HTTPError as exc: + return int(exc.code), None + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError): + return 0, None + + +def _find_check_run_id( + api_base: str, + repo: str, + token: str, + *, + head_sha: str, + external_id: str, +) -> int | None: + """Return an existing check-run id for this external_id on the commit, if any.""" + encoded_sha = urllib.parse.quote(head_sha, safe="") + url = ( + f"{api_base.rstrip('/')}/repos/{repo}/commits/{encoded_sha}/check-runs" + f"?per_page=100" + ) + status, payload = _request(url, token=token) + if status < 200 or status >= 300 or not isinstance(payload, dict): + return None + check_runs = payload.get("check_runs") + if not isinstance(check_runs, list): + return None + for item in check_runs: + if not isinstance(item, dict): + continue + if str(item.get("external_id", "")) == external_id: + check_id = item.get("id") + if isinstance(check_id, int): + return check_id + if isinstance(check_id, str) and check_id.isdigit(): + return int(check_id) + return None + + +def _post_check_run(api_base: str, repo: str, token: str, payload: dict[str, Any]) -> bool: + url = f"{api_base.rstrip('/')}/repos/{repo}/check-runs" + status, _ = _request(url, token=token, method="POST", payload=payload) + return 200 <= status < 300 + + +def _patch_check_run( + api_base: str, + repo: str, + token: str, + check_run_id: int, + payload: dict[str, Any], +) -> bool: + url = f"{api_base.rstrip('/')}/repos/{repo}/check-runs/{check_run_id}" + # PATCH body must not include head_sha / name / external_id on update. + update = { + "status": payload.get("status", "completed"), + "conclusion": payload.get("conclusion"), + "output": payload.get("output"), + } + status, _ = _request(url, token=token, method="PATCH", payload=update) + return 200 <= status < 300 + + +def emit_or_update_check_run( + api_base: str, + repo: str, + token: str, + payload: dict[str, Any], +) -> bool: + """Create or idempotently update a check run keyed by external_id.""" + head_sha = str(payload.get("head_sha", "")) + external_id = str(payload.get("external_id", "") or "") + if not external_id and head_sha: + external_id = check_run_external_id(repo=repo, head_sha=head_sha) + payload = {**payload, "external_id": external_id} + if head_sha and external_id: + existing_id = _find_check_run_id( + api_base, + repo, + token, + head_sha=head_sha, + external_id=external_id, + ) + if existing_id is not None: + return _patch_check_run(api_base, repo, token, existing_id, payload) + return _post_check_run(api_base, repo, token, payload) def main() -> int: @@ -60,7 +158,17 @@ def main() -> int: print("missing head SHA; skipping GitHub check emission") return 0 - payload = build_check_run_payload(bundle, head_sha=head_sha, markdown_summary=markdown_summary) + try: + payload = build_check_run_payload( + bundle, + head_sha=head_sha, + markdown_summary=markdown_summary, + validate_sha=True, + ) + except StaleCheckRunError as exc: + print(f"refusing stale/mismatched check-run emission: {exc}") + return 1 + if args.dry_run: print(json.dumps(payload, indent=2)) return 0 @@ -70,8 +178,11 @@ def main() -> int: print("missing GITHUB_TOKEN or repo; skipping GitHub check emission") return 0 - if _post_check_run(args.api_base, args.repo, token, payload): - print(f"emitted GitHub check run with conclusion {payload['conclusion']}") + if emit_or_update_check_run(args.api_base, args.repo, token, payload): + print( + f"emitted GitHub check run with conclusion {payload['conclusion']} " + f"external_id={payload.get('external_id')}" + ) return 0 print("failed to emit GitHub check run") return 1 diff --git a/scripts/pin_action_shas.py b/scripts/pin_action_shas.py new file mode 100644 index 0000000..59cb6dc --- /dev/null +++ b/scripts/pin_action_shas.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +"""Forbid floating third-party action tags in OVK Action and release paths. + +Release paths (OVK-PR6 / OVK-07): + - action.yml + - .github/workflows/publish.yml + +A ``uses:`` reference is considered pinned when the ref is a full 40-character +lowercase hex commit SHA. Floating tags (``v4``, ``main``, ``release/v1``) fail. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +DEFAULT_PATHS = ( + ROOT / "action.yml", + ROOT / ".github" / "workflows" / "publish.yml", +) + +USES_RE = re.compile( + r"""^\s*(?:-\s*)?uses:\s*['"]?(?P[^'"\s#]+)['"]?\s*(?:#.*)?$""", + re.MULTILINE, +) +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +# Local composite / reusable refs are allowed without SHA pins. +LOCAL_PREFIXES = ("./", "../") + + +def iter_uses(text: str) -> list[str]: + return [match.group("action") for match in USES_RE.finditer(text)] + + +def is_local_action(ref: str) -> bool: + return ref.startswith(LOCAL_PREFIXES) or ref.startswith("docker://") + + +def is_sha_pinned(ref: str) -> bool: + if "@" not in ref: + return False + _owner_name, pin = ref.rsplit("@", 1) + return bool(SHA_RE.fullmatch(pin.lower())) + + +def floating_uses_in_file(path: Path) -> list[str]: + text = path.read_text(encoding="utf-8") + floating: list[str] = [] + for ref in iter_uses(text): + if is_local_action(ref): + continue + if not is_sha_pinned(ref): + floating.append(ref) + return floating + + +def check_paths(paths: list[Path]) -> list[str]: + failures: list[str] = [] + for path in paths: + if not path.exists(): + failures.append(f"missing release path: {path}") + continue + for ref in floating_uses_in_file(path): + failures.append(f"{path.as_posix()}: floating action ref {ref!r} (pin to 40-char commit SHA)") + return failures + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Require immutable commit SHA pins for third-party actions in Action/release paths" + ) + parser.add_argument( + "--path", + action="append", + dest="paths", + type=Path, + help="YAML path to check (repeatable). Defaults to action.yml and publish.yml.", + ) + parser.add_argument( + "--repo-root", + type=Path, + default=ROOT, + help="Repository root used to resolve relative --path values", + ) + args = parser.parse_args(argv) + root = args.repo_root.resolve() + paths = [p if p.is_absolute() else root / p for p in (args.paths or list(DEFAULT_PATHS))] + # Normalize DEFAULT_PATHS when --path not given (already absolute). + if not args.paths: + paths = list(DEFAULT_PATHS) + + failures = check_paths(paths) + for failure in failures: + print(failure, file=sys.stderr) + if failures: + print( + f"pin_action_shas: {len(failures)} floating/unpinned third-party action(s)", + file=sys.stderr, + ) + return 1 + print(f"pin_action_shas: ok ({len(paths)} release path(s) SHA-pinned)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/action_hardening/diffs/deleted_policy.diff b/tests/fixtures/action_hardening/diffs/deleted_policy.diff new file mode 100644 index 0000000..8475282 --- /dev/null +++ b/tests/fixtures/action_hardening/diffs/deleted_policy.diff @@ -0,0 +1,12 @@ +diff --git a/.verification/config.yml b/.verification/config.yml +deleted file mode 100644 +index abcdef0..0000000 +--- a/.verification/config.yml ++++ /dev/null +@@ -1,8 +0,0 @@ +-schema_version: ovk.config.v1 +-mode: advisory +-routing: +- budget: 5 +-unknown_handling: +- default_on_unknown: require_human_review diff --git a/tests/fixtures/action_hardening/diffs/empty_paths.txt b/tests/fixtures/action_hardening/diffs/empty_paths.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/action_hardening/diffs/malicious_filenames.diff b/tests/fixtures/action_hardening/diffs/malicious_filenames.diff new file mode 100644 index 0000000..69a6675 --- /dev/null +++ b/tests/fixtures/action_hardening/diffs/malicious_filenames.diff @@ -0,0 +1,29 @@ +diff --git a/docs/../../etc/passwd b/docs/../../etc/passwd +new file mode 100644 +index 0000000..aaaaaaa +--- /dev/null ++++ b/docs/../../etc/passwd +@@ -0,0 +1,2 @@ ++root:x:0:0:root:/root:/bin/bash ++ +diff --git a/safe/file with spaces.txt b/safe/file with spaces.txt +new file mode 100644 +index 0000000..bbbbbbb +--- /dev/null ++++ b/safe/file with spaces.txt +@@ -0,0 +1 @@ ++ok +diff --git "a/weird/\nnewline.txt" "b/weird/\nnewline.txt" +new file mode 100644 +index 0000000..ccccccc +--- /dev/null ++++ "b/weird/\nnewline.txt" +@@ -0,0 +1 @@ ++control +diff --git a/normal/readme.md b/normal/readme.md +new file mode 100644 +index 0000000..ddddddd +--- /dev/null ++++ b/normal/readme.md +@@ -0,0 +1 @@ ++# docs only diff --git a/tests/fixtures/action_hardening/diffs/renamed_policy.diff b/tests/fixtures/action_hardening/diffs/renamed_policy.diff new file mode 100644 index 0000000..f0bd168 --- /dev/null +++ b/tests/fixtures/action_hardening/diffs/renamed_policy.diff @@ -0,0 +1,12 @@ +diff --git a/policies/auth.rego b/policies/auth_v2.rego +similarity index 90% +rename from policies/auth.rego +rename to policies/auth_v2.rego +index 1111111..2222222 100644 +--- a/policies/auth.rego ++++ b/policies/auth_v2.rego +@@ -1,3 +1,4 @@ + package auth + + allow if input.user.role == "admin" ++allow if input.user.role == "owner" diff --git a/tests/fixtures/action_hardening/diffs/workflow_secrets.diff b/tests/fixtures/action_hardening/diffs/workflow_secrets.diff new file mode 100644 index 0000000..a0e1a28 --- /dev/null +++ b/tests/fixtures/action_hardening/diffs/workflow_secrets.diff @@ -0,0 +1,23 @@ +diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml +new file mode 100644 +index 0000000..1234567 +--- /dev/null ++++ b/.github/workflows/preview.yml +@@ -0,0 +1,17 @@ ++name: Preview ++ ++on: ++ pull_request: ++ ++permissions: ++ contents: read ++ ++jobs: ++ deploy-preview: ++ runs-on: ubuntu-latest ++ steps: ++ - uses: actions/checkout@v4 ++ - name: Deploy ++ env: ++ API_TOKEN: ${{ secrets.DEPLOY_TOKEN }} ++ run: echo "deploy" diff --git a/tests/fixtures/action_hardening/events/fork_pr.json b/tests/fixtures/action_hardening/events/fork_pr.json new file mode 100644 index 0000000..88a059f --- /dev/null +++ b/tests/fixtures/action_hardening/events/fork_pr.json @@ -0,0 +1,29 @@ +{ + "action": "opened", + "repository": { + "full_name": "acme/consumer-app" + }, + "sender": { + "login": "external-contributor", + "type": "User" + }, + "pull_request": { + "number": 42, + "head": { + "sha": "forkheadsha0000000000000000000000000001", + "ref": "feature/untrusted", + "repo": { + "full_name": "external-contributor/consumer-app", + "fork": true + } + }, + "base": { + "sha": "basecommitsha000000000000000000000000001", + "ref": "main", + "repo": { + "full_name": "acme/consumer-app", + "fork": false + } + } + } +} diff --git a/tests/fixtures/action_hardening/events/workflow_dispatch.json b/tests/fixtures/action_hardening/events/workflow_dispatch.json new file mode 100644 index 0000000..e6b29d2 --- /dev/null +++ b/tests/fixtures/action_hardening/events/workflow_dispatch.json @@ -0,0 +1,16 @@ +{ + "action": "workflow_dispatch", + "repository": { + "full_name": "acme/consumer-app" + }, + "sender": { + "login": "maintainer", + "type": "User" + }, + "inputs": { + "mode": "advisory" + }, + "ref": "refs/heads/main", + "after": "dispatchsha00000000000000000000000000001", + "before": "prevsha0000000000000000000000000000002" +} diff --git a/tests/fixtures/action_hardening/scenarios.json b/tests/fixtures/action_hardening/scenarios.json new file mode 100644 index 0000000..b4c1cf8 --- /dev/null +++ b/tests/fixtures/action_hardening/scenarios.json @@ -0,0 +1,93 @@ +{ + "schema_version": "ovk.action_hardening.scenarios.v1", + "description": "Fixture-driven Action hardening scenarios (OVK-PR6 / OVK-07). Local pytest covers all; workflow_dispatch-oriented cases are marked for CI dogfood.", + "scenarios": [ + { + "id": "fork_pr", + "kind": "local", + "description": "Fork PR event stays fail-closed on untrusted secrets; check emission skips without write token", + "event": "events/fork_pr.json", + "changed_files": "diffs/workflow_secrets.diff", + "expected_decision_states": ["block", "needs_review"], + "expect_check_skip_without_token": true + }, + { + "id": "no_changed_files", + "kind": "local", + "description": "Empty change set does not invent a block; decision remains allow or skipped/needs_review", + "changed_files": "diffs/empty_paths.txt", + "expected_decision_states": ["allow", "skipped", "needs_review"] + }, + { + "id": "renamed_files", + "kind": "local", + "description": "Renamed policy path is detected and routed", + "changed_files": "diffs/renamed_policy.diff", + "expect_paths_include": ["policies/auth_v2.rego"], + "expected_decision_states": ["block", "needs_review", "allow", "unknown"] + }, + { + "id": "deleted_policies", + "kind": "local", + "description": "Deleted verification policy/config is treated as a high-risk change surface", + "changed_files": "diffs/deleted_policy.diff", + "expect_paths_include": [".verification/config.yml"], + "expect_domains_include": ["ci_cd"], + "expected_decision_states": ["block", "needs_review", "unknown", "allow"] + }, + { + "id": "workflow_modifications", + "kind": "local", + "description": "Workflow modification exposing secrets blocks in the default trust context", + "changed_files": "diffs/workflow_secrets.diff", + "expected_decision_states": ["block"] + }, + { + "id": "malicious_filenames", + "kind": "local", + "description": "Path-traversal and control-character filenames are ingested without crash", + "changed_files": "diffs/malicious_filenames.diff", + "expect_no_crash": true, + "expected_decision_states": ["allow", "block", "needs_review", "unknown", "error", "skipped"] + }, + { + "id": "large_diffs", + "kind": "local_generated", + "description": "Large multi-file diff completes within a bounded wall clock", + "file_count": 400, + "max_seconds": 60, + "expected_decision_states": ["allow", "block", "needs_review", "unknown", "skipped"] + }, + { + "id": "missing_permissions", + "kind": "check_emit", + "description": "Missing checks:write (HTTP 403) fails emit; advisory path records check_emitted=false", + "expect_emit_exit": 1 + }, + { + "id": "repeated_reruns", + "kind": "check_emit", + "description": "Repeated emit for the same head SHA updates via external_id (idempotent)", + "expect_idempotent_external_id": true + }, + { + "id": "concurrent_runs", + "kind": "check_emit", + "description": "Concurrent emitters for the same SHA share one external_id and update the same check", + "expect_idempotent_external_id": true + }, + { + "id": "stale_check_results", + "kind": "check_emit", + "description": "Evidence head_sha mismatch fails closed (no check published)", + "expect_stale_exit": 1 + }, + { + "id": "workflow_dispatch", + "kind": "workflow_dispatch", + "description": "workflow_dispatch event metadata resolves without a pull_request number", + "event": "events/workflow_dispatch.json", + "expect_pr_number": null + } + ] +} diff --git a/tests/test_action_hardening_suite.py b/tests/test_action_hardening_suite.py new file mode 100644 index 0000000..76027b4 --- /dev/null +++ b/tests/test_action_hardening_suite.py @@ -0,0 +1,428 @@ +"""Fixture-driven Action hardening suite (OVK-PR6 / OVK-07). + +Covers fork PRs, empty changes, renames, deleted policies, workflow mods, +malicious filenames, large diffs, missing permissions, repeated reruns, +concurrent runs, stale check results, and workflow_dispatch metadata. +""" + +from __future__ import annotations + +import json +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest + +from ovk.core.change_detection import detect_change_surfaces +from ovk.core.changed_files import load_changed_files +from ovk.core.check import run_check +from ovk.core.github_check import ( + StaleCheckRunError, + build_check_run_payload, + check_run_external_id, + validate_check_run_head_sha, +) +from ovk.core.github_event import load_github_event_metadata +from ovk.core.models import EvidenceBundle +from scripts.emit_github_check import emit_or_update_check_run, main as emit_main +from scripts.pin_action_shas import check_paths, floating_uses_in_file, is_sha_pinned + +FIXTURE_ROOT = Path(__file__).resolve().parent / "fixtures" / "action_hardening" +SCENARIOS_PATH = FIXTURE_ROOT / "scenarios.json" + + +def _load_scenarios() -> list[dict[str, Any]]: + payload = json.loads(SCENARIOS_PATH.read_text(encoding="utf-8")) + scenarios = payload.get("scenarios", []) + assert isinstance(scenarios, list) and scenarios + return scenarios + + +def _scenario_map() -> dict[str, dict[str, Any]]: + return {str(item["id"]): item for item in _load_scenarios()} + + +def _decision_state(bundle: EvidenceBundle) -> str: + decision = bundle.decision or {} + if decision.get("decision_state"): + return str(decision["decision_state"]) + return str(decision.get("merge_recommendation", "")) + + +def _write_evidence( + path: Path, + *, + head_sha: str = "abc123def456", + repo: str = "owner/repo", + recommendation: str = "block", + decision_state: str | None = None, +) -> EvidenceBundle: + state = decision_state or ( + "block" + if recommendation == "block" + else "allow" + if recommendation == "allow" + else "needs_review" + ) + payload = { + "schema_version": "ovk.bundle.v1", + "bundle_id": "action-hardening", + "subject": {"repo": repo, "head_sha": head_sha}, + "evidence": [], + "open_obligations": [], + "decision": { + "decision_state": state, + "merge_recommendation": recommendation, + "reason": "action-hardening fixture", + }, + } + path.write_text(json.dumps(payload), encoding="utf-8") + return EvidenceBundle.model_validate(payload) + + +def test_scenarios_manifest_covers_required_ids() -> None: + required = { + "fork_pr", + "no_changed_files", + "renamed_files", + "deleted_policies", + "workflow_modifications", + "malicious_filenames", + "large_diffs", + "missing_permissions", + "repeated_reruns", + "concurrent_runs", + "stale_check_results", + "workflow_dispatch", + } + assert required <= set(_scenario_map()) + + +def test_fork_pr_blocks_untrusted_secrets_and_skips_emit_without_token( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + scenario = _scenario_map()["fork_pr"] + event_path = FIXTURE_ROOT / scenario["event"] + diff_path = FIXTURE_ROOT / scenario["changed_files"] + meta = load_github_event_metadata(event_path) + assert meta.pull_request_number == 42 + assert meta.head_sha.startswith("forkhead") + + result = run_check( + diff_text=diff_path.read_text(encoding="utf-8"), + github_event_path=event_path, + repo=meta.repository, + head_sha=meta.head_sha, + use_cache=False, + ) + assert _decision_state(result.bundle) in scenario["expected_decision_states"] + + evidence = tmp_path / "ovk-evidence.json" + _write_evidence(evidence, head_sha=meta.head_sha, repo=meta.repository, recommendation="block") + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + with patch( + "sys.argv", + [ + "emit_github_check.py", + "--evidence", + str(evidence), + "--repo", + meta.repository, + "--head-sha", + meta.head_sha, + ], + ): + assert emit_main() == 0 + assert "missing GITHUB_TOKEN" in capsys.readouterr().out + + +def test_no_changed_files_does_not_crash() -> None: + scenario = _scenario_map()["no_changed_files"] + paths = load_changed_files(FIXTURE_ROOT / scenario["changed_files"]) + assert paths == [] + result = run_check(changed_files=paths, repo="owner/repo", head_sha="empty001", use_cache=False) + assert _decision_state(result.bundle) in scenario["expected_decision_states"] + + +def test_renamed_files_detected() -> None: + scenario = _scenario_map()["renamed_files"] + diff_path = FIXTURE_ROOT / scenario["changed_files"] + paths = load_changed_files(diff_path) + for expected in scenario["expect_paths_include"]: + assert expected in paths + result = run_check( + diff_text=diff_path.read_text(encoding="utf-8"), + repo="owner/repo", + head_sha="rename001", + use_cache=False, + ) + assert _decision_state(result.bundle) in scenario["expected_decision_states"] + + +def test_deleted_policies_surface() -> None: + scenario = _scenario_map()["deleted_policies"] + diff_path = FIXTURE_ROOT / scenario["changed_files"] + paths = load_changed_files(diff_path) + for expected in scenario["expect_paths_include"]: + assert expected in paths + surfaces = detect_change_surfaces(paths) + domains = {surface.domain for surface in surfaces} + for expected in scenario["expect_domains_include"]: + assert expected in domains + result = run_check( + diff_text=diff_path.read_text(encoding="utf-8"), + repo="owner/repo", + head_sha="delete001", + use_cache=False, + ) + assert _decision_state(result.bundle) in scenario["expected_decision_states"] + + +def test_workflow_modifications_block() -> None: + scenario = _scenario_map()["workflow_modifications"] + diff_path = FIXTURE_ROOT / scenario["changed_files"] + result = run_check( + diff_text=diff_path.read_text(encoding="utf-8"), + repo="owner/repo", + head_sha="wfmod001", + use_cache=False, + ) + assert _decision_state(result.bundle) in scenario["expected_decision_states"] + + +def test_malicious_filenames_no_crash() -> None: + scenario = _scenario_map()["malicious_filenames"] + diff_path = FIXTURE_ROOT / scenario["changed_files"] + paths = load_changed_files(diff_path) + assert paths # ingested something + result = run_check( + diff_text=diff_path.read_text(encoding="utf-8"), + changed_files=paths, + repo="owner/repo", + head_sha="malicious001", + use_cache=False, + ) + assert _decision_state(result.bundle) in scenario["expected_decision_states"] + + +def test_large_diffs_complete_within_budget() -> None: + scenario = _scenario_map()["large_diffs"] + file_count = int(scenario["file_count"]) + lines = [] + for index in range(file_count): + path = f"generated/file_{index:04d}.md" + lines.extend( + [ + f"diff --git a/{path} b/{path}", + "new file mode 100644", + "index 0000000..1111111", + "--- /dev/null", + f"+++ b/{path}", + "@@ -0,0 +1 @@", + f"+content {index}", + "", + ] + ) + diff_text = "\n".join(lines) + started = time.perf_counter() + result = run_check( + diff_text=diff_text, + repo="owner/repo", + head_sha="large001", + use_cache=False, + ) + elapsed = time.perf_counter() - started + assert elapsed < float(scenario["max_seconds"]) + assert _decision_state(result.bundle) in scenario["expected_decision_states"] + + +def test_missing_permissions_emit_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scenario = _scenario_map()["missing_permissions"] + evidence = tmp_path / "ovk-evidence.json" + head_sha = "permsha000000000000000000000000000001" + _write_evidence(evidence, head_sha=head_sha, recommendation="allow", decision_state="allow") + monkeypatch.setenv("GITHUB_TOKEN", "no-checks-write") + + with patch( + "sys.argv", + [ + "emit_github_check.py", + "--evidence", + str(evidence), + "--repo", + "owner/repo", + "--head-sha", + head_sha, + ], + ): + with patch("scripts.emit_github_check._request", return_value=(403, None)): + assert emit_main() == int(scenario["expect_emit_exit"]) + + +def test_repeated_reruns_idempotent_external_id(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _ = _scenario_map()["repeated_reruns"] + evidence = tmp_path / "ovk-evidence.json" + head_sha = "rerunsha00000000000000000000000000001" + bundle = _write_evidence(evidence, head_sha=head_sha, recommendation="block") + payload = build_check_run_payload(bundle, head_sha=head_sha) + external_id = payload["external_id"] + assert external_id == check_run_external_id(repo="owner/repo", head_sha=head_sha) + + state = {"created": False} + calls: list[str] = [] + + def fake_request(url: str, *, token: str, method: str = "GET", payload: dict | None = None): + _ = token, payload, url + if method == "GET": + if state["created"]: + return 200, { + "check_runs": [ + {"id": 99, "external_id": external_id, "name": "Open Verification Kernel"} + ] + } + return 200, {"check_runs": []} + if method == "POST": + state["created"] = True + calls.append("POST") + return 201, {"id": 99} + if method == "PATCH": + calls.append("PATCH") + return 200, {"id": 99} + return 0, None + + monkeypatch.setenv("GITHUB_TOKEN", "token") + with patch("scripts.emit_github_check._request", side_effect=fake_request): + assert emit_or_update_check_run("https://api.github.com", "owner/repo", "token", payload) + assert emit_or_update_check_run("https://api.github.com", "owner/repo", "token", payload) + assert calls == ["POST", "PATCH"] + + +def test_concurrent_runs_share_external_id() -> None: + _ = _scenario_map()["concurrent_runs"] + head_sha = "concurrentsha000000000000000000000001" + bundle = EvidenceBundle.model_validate( + { + "schema_version": "ovk.bundle.v1", + "bundle_id": "concurrent", + "subject": {"repo": "owner/repo", "head_sha": head_sha}, + "evidence": [], + "open_obligations": [], + "decision": {"decision_state": "allow", "merge_recommendation": "allow"}, + } + ) + payloads = [build_check_run_payload(bundle, head_sha=head_sha) for _ in range(4)] + external_ids = {item["external_id"] for item in payloads} + assert len(external_ids) == 1 + + lock = threading.Lock() + created_id: dict[str, int | None] = {"id": None} + methods: list[str] = [] + + def fake_request(url: str, *, token: str, method: str = "GET", payload: dict | None = None): + _ = token, payload, url + with lock: + if method == "GET": + if created_id["id"] is None: + return 200, {"check_runs": []} + return 200, { + "check_runs": [ + { + "id": created_id["id"], + "external_id": payloads[0]["external_id"], + "name": "Open Verification Kernel", + } + ] + } + methods.append(method) + if method == "POST": + created_id["id"] = 7 + return 201, {"id": 7} + return 200, {"id": 7} + + with patch("scripts.emit_github_check._request", side_effect=fake_request): + with ThreadPoolExecutor(max_workers=4) as pool: + results = list( + pool.map( + lambda _: emit_or_update_check_run( + "https://api.github.com", "owner/repo", "token", payloads[0] + ), + range(4), + ) + ) + assert all(results) + assert "POST" in methods + assert methods.count("POST") >= 1 + + +def test_stale_check_results_fail_closed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + scenario = _scenario_map()["stale_check_results"] + evidence = tmp_path / "ovk-evidence.json" + _write_evidence(evidence, head_sha="evidence-sha-aaaa", recommendation="allow", decision_state="allow") + monkeypatch.setenv("GITHUB_TOKEN", "token") + + with patch( + "sys.argv", + [ + "emit_github_check.py", + "--evidence", + str(evidence), + "--repo", + "owner/repo", + "--head-sha", + "stale-target-bbbb", + ], + ): + assert emit_main() == int(scenario["expect_stale_exit"]) + + bundle = EvidenceBundle.model_validate(json.loads(evidence.read_text(encoding="utf-8"))) + with pytest.raises(StaleCheckRunError, match="mismatch"): + validate_check_run_head_sha(bundle, "stale-target-bbbb") + with pytest.raises(StaleCheckRunError): + build_check_run_payload(bundle, head_sha="stale-target-bbbb") + + +def test_workflow_dispatch_event_metadata() -> None: + scenario = _scenario_map()["workflow_dispatch"] + meta = load_github_event_metadata(FIXTURE_ROOT / scenario["event"]) + assert meta.pull_request_number == scenario["expect_pr_number"] + assert meta.head_sha.startswith("dispatchsha") + assert meta.repository == "acme/consumer-app" + + +def test_pin_action_shas_release_paths_clean() -> None: + root = Path(__file__).resolve().parents[1] + failures = check_paths([root / "action.yml", root / ".github" / "workflows" / "publish.yml"]) + assert failures == [] + + +def test_pin_action_shas_detects_floating_tag(tmp_path: Path) -> None: + workflow = tmp_path / "publish.yml" + workflow.write_text( + "jobs:\n x:\n steps:\n - uses: actions/checkout@v4\n", + encoding="utf-8", + ) + floating = floating_uses_in_file(workflow) + assert floating == ["actions/checkout@v4"] + assert not is_sha_pinned("actions/checkout@v4") + assert is_sha_pinned("actions/checkout@" + ("a" * 40)) + + +def test_build_check_run_payload_includes_external_id() -> None: + bundle = EvidenceBundle.model_validate( + { + "schema_version": "ovk.bundle.v1", + "bundle_id": "x", + "subject": {"repo": "o/r", "head_sha": "deadbeef"}, + "evidence": [], + "open_obligations": [], + "decision": {"decision_state": "block", "merge_recommendation": "block"}, + } + ) + payload = build_check_run_payload(bundle, head_sha="deadbeef") + assert payload["external_id"] == "ovk:o/r:deadbeef" + assert payload["conclusion"] == "failure" diff --git a/tests/test_emit_github_check.py b/tests/test_emit_github_check.py index 3be9f8a..f1250a4 100644 --- a/tests/test_emit_github_check.py +++ b/tests/test_emit_github_check.py @@ -3,18 +3,19 @@ from unittest.mock import MagicMock, patch from urllib.error import URLError -from scripts.emit_github_check import _post_check_run, main +from scripts.emit_github_check import _post_check_run, emit_or_update_check_run, main -def _write_evidence(path: Path, recommendation: str = "block") -> None: +def _write_evidence(path: Path, recommendation: str = "block", head_sha: str = "abc123") -> None: payload = { "schema_version": "ovk.bundle.v1", "bundle_id": "emit-check-test", - "subject": {"repo": "owner/repo", "head_sha": "abc123"}, + "subject": {"repo": "owner/repo", "head_sha": head_sha}, "evidence": [], "open_obligations": [], "decision": { "merge_recommendation": recommendation, + "decision_state": "block" if recommendation == "block" else "allow", "reason": "unit-test fixture", }, } @@ -24,12 +25,16 @@ def _write_evidence(path: Path, recommendation: str = "block") -> None: def test_emit_github_check_dry_run_prints_payload(capsys, tmp_path: Path) -> None: evidence = tmp_path / "ovk-evidence.json" _write_evidence(evidence) - with patch("sys.argv", ["emit_github_check.py", "--evidence", str(evidence), "--dry-run"]): + with patch( + "sys.argv", + ["emit_github_check.py", "--evidence", str(evidence), "--head-sha", "abc123", "--dry-run"], + ): assert main() == 0 output = capsys.readouterr().out payload = json.loads(output) assert payload["name"] == "Open Verification Kernel" assert payload["conclusion"] == "failure" + assert payload["external_id"] == "ovk:owner/repo:abc123" def test_emit_github_check_missing_evidence_exits_zero(tmp_path: Path) -> None: @@ -45,10 +50,22 @@ def test_emit_github_check_posts_check_run(tmp_path: Path, monkeypatch) -> None: markdown.write_text("summary", encoding="utf-8") monkeypatch.setenv("GITHUB_TOKEN", "test-token") - response = MagicMock() - response.status = 201 - response.__enter__ = MagicMock(return_value=response) - response.__exit__ = MagicMock(return_value=False) + responses: list[MagicMock] = [] + + def make_response(status: int, body: dict | list) -> MagicMock: + response = MagicMock() + response.status = status + response.read = MagicMock(return_value=json.dumps(body).encode("utf-8")) + response.__enter__ = MagicMock(return_value=response) + response.__exit__ = MagicMock(return_value=False) + responses.append(response) + return response + + # First call: list check-runs (empty). Second: create check-run. + queue = [ + make_response(200, {"check_runs": []}), + make_response(201, {"id": 1}), + ] with patch( "sys.argv", @@ -64,13 +81,15 @@ def test_emit_github_check_posts_check_run(tmp_path: Path, monkeypatch) -> None: "abc123", ], ): - with patch("urllib.request.urlopen", return_value=response) as urlopen: + with patch("urllib.request.urlopen", side_effect=queue) as urlopen: assert main() == 0 - request = urlopen.call_args.args[0] - assert request.full_url.endswith("/repos/owner/repo/check-runs") - assert request.get_header("Authorization") == "Bearer test-token" - body = json.loads(request.data.decode("utf-8")) + assert urlopen.call_count == 2 + create_request = urlopen.call_args_list[1].args[0] + assert create_request.full_url.endswith("/repos/owner/repo/check-runs") + assert create_request.get_header("Authorization") == "Bearer test-token" + body = json.loads(create_request.data.decode("utf-8")) assert body["conclusion"] == "success" + assert body["external_id"] == "ovk:owner/repo:abc123" def test_emit_github_check_api_failure_returns_one(tmp_path: Path, monkeypatch) -> None: @@ -90,10 +109,51 @@ def test_emit_github_check_api_failure_returns_one(tmp_path: Path, monkeypatch) "abc123", ], ): - with patch("scripts.emit_github_check._post_check_run", return_value=False): + with patch("scripts.emit_github_check.emit_or_update_check_run", return_value=False): assert main() == 1 +def test_emit_github_check_stale_sha_returns_one(tmp_path: Path, monkeypatch) -> None: + evidence = tmp_path / "ovk-evidence.json" + _write_evidence(evidence, head_sha="evidence-sha") + monkeypatch.setenv("GITHUB_TOKEN", "test-token") + with patch( + "sys.argv", + [ + "emit_github_check.py", + "--evidence", + str(evidence), + "--repo", + "owner/repo", + "--head-sha", + "other-sha", + ], + ): + assert main() == 1 + + def test_post_check_run_returns_false_on_http_error() -> None: with patch("urllib.request.urlopen", side_effect=URLError("network down")): assert _post_check_run("https://api.github.com", "owner/repo", "token", {"name": "x"}) is False + + +def test_emit_or_update_patches_existing() -> None: + payload = { + "name": "Open Verification Kernel", + "head_sha": "abc123", + "external_id": "ovk:owner/repo:abc123", + "status": "completed", + "conclusion": "success", + "output": {"title": "t", "summary": "s"}, + } + + def fake_request(url: str, *, token: str, method: str = "GET", payload: dict | None = None): + _ = token, payload + if method == "GET": + return 200, {"check_runs": [{"id": 55, "external_id": "ovk:owner/repo:abc123"}]} + assert method == "PATCH" + assert url.endswith("/check-runs/55") + return 200, {"id": 55} + + with patch("scripts.emit_github_check._request", side_effect=fake_request): + assert emit_or_update_check_run("https://api.github.com", "owner/repo", "token", payload) From 94bc59a09c30c5800aa1d2c7667460c0ae139f69 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:09:51 -0700 Subject: [PATCH 11/19] Add private-alpha GitHub App control plane (OVK-PR7). Introduce a signature-verified webhook service with isolation, replay protection, redaction, and check-run emission so App installs stay narrower than the composite Action surface. --- docs/INTEGRATION.md | 33 ++- integrations/github-app/README.md | 89 +++++++ integrations/github-app/RETENTION.md | 33 +++ integrations/github-app/manifest.json | 26 ++ .../github-app/ovk_github_app/__init__.py | 12 + .../github-app/ovk_github_app/cache_keys.py | 53 ++++ .../github-app/ovk_github_app/check_runs.py | 47 ++++ .../github-app/ovk_github_app/cleanup.py | 40 +++ .../github-app/ovk_github_app/errors.py | 23 ++ .../github-app/ovk_github_app/isolation.py | 121 +++++++++ .../github-app/ovk_github_app/redact.py | 113 ++++++++ .../github-app/ovk_github_app/replay.py | 105 ++++++++ .../github-app/ovk_github_app/service.py | 104 ++++++++ .../github-app/ovk_github_app/signature.py | 48 ++++ .../github-app/ovk_github_app/tokens.py | 190 ++++++++++++++ .../github-app/ovk_github_app/webhook.py | 186 +++++++++++++ integrations/github-app/requirements.txt | 4 + tests/test_github_app_controls.py | 247 ++++++++++++++++++ tests/test_github_app_isolation.py | 78 ++++++ tests/test_github_app_replay.py | 85 ++++++ tests/test_github_app_signature.py | 53 ++++ 21 files changed, 1683 insertions(+), 7 deletions(-) create mode 100644 integrations/github-app/README.md create mode 100644 integrations/github-app/RETENTION.md create mode 100644 integrations/github-app/manifest.json create mode 100644 integrations/github-app/ovk_github_app/__init__.py create mode 100644 integrations/github-app/ovk_github_app/cache_keys.py create mode 100644 integrations/github-app/ovk_github_app/check_runs.py create mode 100644 integrations/github-app/ovk_github_app/cleanup.py create mode 100644 integrations/github-app/ovk_github_app/errors.py create mode 100644 integrations/github-app/ovk_github_app/isolation.py create mode 100644 integrations/github-app/ovk_github_app/redact.py create mode 100644 integrations/github-app/ovk_github_app/replay.py create mode 100644 integrations/github-app/ovk_github_app/service.py create mode 100644 integrations/github-app/ovk_github_app/signature.py create mode 100644 integrations/github-app/ovk_github_app/tokens.py create mode 100644 integrations/github-app/ovk_github_app/webhook.py create mode 100644 integrations/github-app/requirements.txt create mode 100644 tests/test_github_app_controls.py create mode 100644 tests/test_github_app_isolation.py create mode 100644 tests/test_github_app_replay.py create mode 100644 tests/test_github_app_signature.py diff --git a/docs/INTEGRATION.md b/docs/INTEGRATION.md index 0b78e94..9056cd5 100644 --- a/docs/INTEGRATION.md +++ b/docs/INTEGRATION.md @@ -1,6 +1,7 @@ # OVK Integration Guide Check [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) before pinning a version or switching to strict mode. +Reviewer TCB inventory: [TRUSTED_COMPUTING_BASE.md](TRUSTED_COMPUTING_BASE.md). Install and run Open Verification Kernel locally or in GitHub Actions. @@ -12,17 +13,18 @@ Verification routing configuration for `.verification/config.yml`: [POLICY.md](P pip install -e '.[dev]' ovk init ovk release-preflight +python scripts/verify_rc_install.py # Action SHA pins + package metadata ``` -PyPI release (after maintainers publish `v1.2.0`; see [RELEASE.md](RELEASE.md)): +PyPI release (after maintainers publish an attributable tag; see [RELEASE.md](RELEASE.md)): ```bash -pip install open-verification-kernel==1.2.0 +pip install open-verification-kernel==1.3.0-rc.1 # optional solvers -pip install "open-verification-kernel[solvers]==1.2.0" +pip install "open-verification-kernel[solvers]==1.3.0-rc.1" ``` -Until the wheel is on PyPI, use `pip install -e '.[dev]'` from a checkout or pin the GitHub Action at `@v1.2.0` with `OVK_PACKAGE_VERSION` once published. +Until the wheel is on PyPI, use `pip install -e '.[dev]'` from a checkout or pin the GitHub Action at `@v1.3.0-rc.1` (after the tag exists) with matching `OVK_PACKAGE_VERSION`. Signed production consumers may still use `@v1.2.1` until the RC is attributable. Optional Z3: `pip install -e '.[solvers]'` @@ -46,7 +48,9 @@ ovk ci --metadata examples/no_agent_self_approval/metadata_gate_preserved.json - ## GitHub Action -The composite Action in `action.yml` supports advisory and strict modes. +The composite Action in `action.yml` supports advisory and strict modes. It is the **public** integration path. + +A **private-alpha** GitHub App (not Marketplace) lives under [`integrations/github-app/`](../integrations/github-app/README.md) with webhook HMAC, replay protection, installation isolation, and retention policy. Prefer the Action unless you are operating that alpha. ### Modes @@ -55,7 +59,16 @@ The composite Action in `action.yml` supports advisory and strict modes. | `mode: advisory` | Writes artifacts; always exits 0 | | `mode: strict` | Exits nonzero on `block` or `require_human_review` | -### Permissions +### Permissions matrix + +Least-privilege defaults for the composite Action. Grant only what the enabled inputs require. + +| Capability | `permissions:` key | Required when | Without it | +|---|---|---|---| +| Read checkout / event payload | `contents: read` | Always (workflow checkout) | Job cannot see the PR diff | +| Publish check run **Open Verification Kernel** | `checks: write` | `emit-check: true` | Emit skips or fails; in `strict` mode the job fails | +| Post / update PR comment | `pull-requests: write` | `post-comment: true` | Comment step no-ops (exit 0) | +| Read branch protection / required checks | `contents: read` (+ token that can read protection) | Auto `collect_branch_metadata.py` | Metadata missing → high-risk workflow changes stay `needs_review` / `require_human_review` | Copy-paste block for consumer workflows: @@ -72,7 +85,11 @@ permissions: | `emit-check: true` | `checks: write` | | `collect_branch_metadata.py` (auto) | `GITHUB_TOKEN` with repository metadata read | -Fork PRs from outside contributors cannot post comments; keep `post-comment: false` on fork workflows. +**Fork PRs:** outside contributors receive a read-only `GITHUB_TOKEN` for `pull_request` from forks. Keep `post-comment: false` and treat `emit-check` as best-effort on fork workflows (`examples/github_workflows/pilot_fork_adopter.yml`). Missing `checks: write` must never be treated as verification success. + +**Check-run integrity:** emission fails closed when the evidence `subject.head_sha` does not match the commit being annotated (stale SHA). Updates are idempotent via stable `external_id` `ovk:{repo}:{head_sha}` so reruns and concurrent jobs update one check run per head SHA. + +**Third-party action pins:** `action.yml` and `.github/workflows/publish.yml` pin `actions/*` (and other release third-parties) to immutable commit SHAs. CI enforces this with `python scripts/pin_action_shas.py`. ### Single check type (self-protection) @@ -168,6 +185,8 @@ Downstream example: | `advisory` | Job continues; `check_emitted` is `false` | | `strict` | Job fails when check run cannot be emitted | +Stale evidence (bundle `subject.head_sha` ≠ emit `--head-sha`) always fails emission with exit code 1, including dry-run validation before any API call. + GitHub check conclusion mapping: | Recommendation | Check conclusion | Strict exit code | diff --git a/integrations/github-app/README.md b/integrations/github-app/README.md new file mode 100644 index 0000000..5d8a052 --- /dev/null +++ b/integrations/github-app/README.md @@ -0,0 +1,89 @@ +# OVK GitHub App (private alpha) + +Private-alpha GitHub App for Open Verification Kernel. **Not a Marketplace listing.** + +The **composite Action** (`action.yml`) remains the supported public integration path. This App is an optional alpha surface with explicit security controls (OVK-08 / OVK-PR7). + +## Controls + +| Control | Implementation | +|---|---| +| Webhook signature verification | HMAC-SHA256 (`X-Hub-Signature-256`); missing/invalid rejected | +| Replay protection | `X-OVK-Timestamp` skew (±300s) + `X-GitHub-Delivery` dedupe store | +| Installation isolation | `{data}/installations/{id}/` partitions for credentials, cache, data | +| Least-privilege permissions | `manifest.json`: `checks:write`, `contents:read`, `pull_requests:read`, `metadata:read` | +| Short-lived installation tokens | On-demand exchange; App JWT ≤10m; installation token ≤1h; no PATs | +| Redacted logs | Paths and secrets scrubbed via `RedactingFilter` | +| Idempotent Check Run updates | `external_id` = `ovk:{repo}:{head_sha}` (same as Action / PR6) | +| No cross-repository cache reuse | Cache keys require `installation_id` + `repo_id` | +| Uninstall cleanup | `installation.deleted` deletes the partition | +| Retention policy | [RETENTION.md](RETENTION.md) | + +## Layout + +```text +integrations/github-app/ + manifest.json # GitHub App manifest (public: false) + RETENTION.md # TTL policy + requirements.txt # Optional runtime (FastAPI, PyJWT, …) + README.md # This file + ovk_github_app/ # Python package +``` + +## Operator setup (alpha) + +1. Create a **private** GitHub App from `manifest.json` (GitHub → Settings → Developer settings → GitHub Apps → New GitHub App, or manifest flow). Do not set the App public / Marketplace. +2. Generate a webhook secret; set `OVK_GITHUB_WEBHOOK_SECRET`. +3. Download the App private key PEM; set `OVK_GITHUB_APP_ID` and `OVK_GITHUB_APP_PRIVATE_KEY` (or mount the PEM for token exchange). +4. Point the App webhook URL at your deployed `/webhook` endpoint. +5. Install the App on a pilot org/repo with the default least-privilege permissions only. + +### Run the webhook service + +```bash +pip install -r integrations/github-app/requirements.txt +export OVK_GITHUB_WEBHOOK_SECRET='...' +export OVK_GITHUB_APP_DATA='.ovk-github-app' +# Optional: ingress must stamp X-OVK-Timestamp (unix seconds) unless disabled: +# export OVK_WEBHOOK_REQUIRE_TIMESTAMP=0 +cd integrations/github-app +uvicorn ovk_github_app.service:create_app --factory --host 0.0.0.0 --port 8080 +``` + +Health: `GET /healthz` +Webhook: `POST /webhook` + +### Headers + +| Header | Required | Purpose | +|---|---|---| +| `X-Hub-Signature-256` | yes | `sha256=` over raw body | +| `X-GitHub-Delivery` | yes | Delivery-id dedupe | +| `X-GitHub-Event` | yes | Event name | +| `X-OVK-Timestamp` | default yes | Unix seconds; must be within skew | + +GitHub does not send a webhook timestamp header natively. For alpha, stamp `X-OVK-Timestamp` at a trusted ingress, or set `OVK_WEBHOOK_REQUIRE_TIMESTAMP=0` and rely on delivery-id dedupe (documented trade-off in RETENTION.md). + +## Tokens + +Use `InstallationTokenProvider` to exchange short-lived installation tokens on demand. The provider refuses `ghp_` / `github_pat_` material and rejects lifetimes above one hour. Do not configure a classic PAT for this App. + +## Check runs + +Idempotent updates use the same `external_id` scheme as the Action: + +```text +ovk:{owner}/{repo}:{head_sha} +``` + +## Tests + +From the repository root (pythonpath includes this package): + +```bash +pytest tests/test_github_app_signature.py tests/test_github_app_replay.py tests/test_github_app_isolation.py tests/test_github_app_controls.py -q +``` + +## Status + +Alpha alongside the composite Action. Do not advertise Marketplace availability. Uninstall cleanup and retention TTLs are mandatory for any private pilot. diff --git a/integrations/github-app/RETENTION.md b/integrations/github-app/RETENTION.md new file mode 100644 index 0000000..96b266a --- /dev/null +++ b/integrations/github-app/RETENTION.md @@ -0,0 +1,33 @@ +# OVK GitHub App — retention policy + +Private-alpha data retention for `integrations/github-app/`. + +## TTL defaults + +| Data class | Location | TTL | Notes | +|---|---|---|---| +| Webhook delivery-id dedupe records | In-memory store (process) / optional durable store | **24 hours** | Matches replay window + clock skew cushion; expired IDs may be reclaimed | +| Webhook timestamp skew window | Request validation | **5 minutes** (`±300s`) | Rejects stale stamped deliveries | +| Installation access tokens | Memory only | **≤ 1 hour** (GitHub-issued) | Never persisted as long-lived PATs; refreshed on demand | +| App JWT (issuer assertion) | Ephemeral | **≤ 10 minutes** | Minted per exchange; not stored | +| Per-installation event receipts | `{data}/installations/{id}/data/events/` | **7 days** | Operator may shorten; deleted immediately on uninstall | +| Per-installation cache objects | `{data}/installations/{id}/cache/` | **24 hours** | Keys always include `installation_id` + `repo_id` | +| Credentials material | `{data}/installations/{id}/credentials/` | Until uninstall | App private key is operator-managed outside this tree when possible | + +## Uninstall + +On `installation.deleted`, all files under `{data}/installations/{installation_id}/` are removed and in-memory tokens for that installation are cleared. See `ovk_github_app.cleanup.handle_installation_deleted`. + +## Operator overrides + +| Environment variable | Default | Purpose | +|---|---|---| +| `OVK_WEBHOOK_MAX_SKEW_SECONDS` | `300` | Timestamp skew tolerance | +| `OVK_GITHUB_APP_DATA` | `.ovk-github-app` | Root for installation partitions | +| `OVK_WEBHOOK_REQUIRE_TIMESTAMP` | `1` | Require `X-OVK-Timestamp` (unix seconds) on webhooks | + +When running behind a trusted ingress that cannot stamp `X-OVK-Timestamp`, set `OVK_WEBHOOK_REQUIRE_TIMESTAMP=0`. Delivery-id dedupe remains mandatory. + +## Public path + +Retention here applies only to the private-alpha App. The composite Action (`action.yml`) does not store installation partitions; CI artifacts follow the consumer workflow's retention settings. diff --git a/integrations/github-app/manifest.json b/integrations/github-app/manifest.json new file mode 100644 index 0000000..1c5ccbf --- /dev/null +++ b/integrations/github-app/manifest.json @@ -0,0 +1,26 @@ +{ + "name": "OVK Verification (Private Alpha)", + "url": "https://github.com/fraware/open-verification-kernel", + "hook_attributes": { + "url": "https://ovk-github-app.example.invalid/webhook", + "active": true + }, + "redirect_url": "https://github.com/fraware/open-verification-kernel/blob/main/integrations/github-app/README.md", + "callback_urls": [], + "setup_url": "https://github.com/fraware/open-verification-kernel/blob/main/integrations/github-app/README.md", + "description": "Private-alpha Open Verification Kernel GitHub App. Not listed on the Marketplace. The composite Action remains the public integration path.", + "public": false, + "default_events": [ + "check_run", + "check_suite", + "installation", + "pull_request", + "push" + ], + "default_permissions": { + "checks": "write", + "contents": "read", + "pull_requests": "read", + "metadata": "read" + } +} diff --git a/integrations/github-app/ovk_github_app/__init__.py b/integrations/github-app/ovk_github_app/__init__.py new file mode 100644 index 0000000..a278fe1 --- /dev/null +++ b/integrations/github-app/ovk_github_app/__init__.py @@ -0,0 +1,12 @@ +"""OVK GitHub App private alpha (OVK-08). + +Not a marketplace listing. The composite Action remains the public integration +path; this package is an optional private-alpha webhook service with explicit +security controls. +""" + +from __future__ import annotations + +__all__ = ["__version__"] + +__version__ = "0.1.0-alpha" diff --git a/integrations/github-app/ovk_github_app/cache_keys.py b/integrations/github-app/ovk_github_app/cache_keys.py new file mode 100644 index 0000000..7e4e996 --- /dev/null +++ b/integrations/github-app/ovk_github_app/cache_keys.py @@ -0,0 +1,53 @@ +"""Installation- and repository-scoped cache keys (no cross-repo reuse).""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + +from ovk_github_app.errors import IsolationError + + +def app_cache_key( + *, + installation_id: int | str, + repo_id: int | str, + namespace: str, + components: dict[str, Any] | None = None, +) -> str: + """Build a cache key that always binds installation id and repository id. + + Keys from different installations or repositories never collide: both ids are + mandatory key material, not optional metadata. + """ + iid = str(installation_id).strip() + rid = str(repo_id).strip() + if not iid or not rid: + raise IsolationError("cache key requires installation_id and repo_id") + if not str(namespace).strip(): + raise IsolationError("cache key requires namespace") + payload = { + "schema": "ovk.github_app.cache.v1", + "installation_id": iid, + "repo_id": rid, + "namespace": str(namespace).strip(), + "components": components or {}, + } + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def assert_cache_key_bound( + key_material: dict[str, Any], + *, + installation_id: int | str, + repo_id: int | str, +) -> None: + """Fail closed when key material omits or mismatches installation/repo ids.""" + if "installation_id" not in key_material or "repo_id" not in key_material: + raise IsolationError("cache key material missing installation_id or repo_id") + if str(key_material["installation_id"]) != str(installation_id): + raise IsolationError("cache key installation_id mismatch") + if str(key_material["repo_id"]) != str(repo_id): + raise IsolationError("cache key repo_id mismatch") diff --git a/integrations/github-app/ovk_github_app/check_runs.py b/integrations/github-app/ovk_github_app/check_runs.py new file mode 100644 index 0000000..229b97b --- /dev/null +++ b/integrations/github-app/ovk_github_app/check_runs.py @@ -0,0 +1,47 @@ +"""Idempotent Check Run helpers aligned with PR6 emit patterns.""" + +from __future__ import annotations + +from typing import Any + +# Prefer the kernel helper when available so App and Action stay aligned. +try: + from ovk.core.github_check import CHECK_NAME, check_run_external_id +except ImportError: # pragma: no cover - standalone alpha checkout + + def check_run_external_id(*, repo: str, head_sha: str) -> str: + return f"ovk:{repo}:{head_sha}" + + CHECK_NAME = "Open Verification Kernel" + + +def app_check_run_external_id(*, repo: str, head_sha: str) -> str: + """Stable external_id per head SHA — same format as the composite Action.""" + return check_run_external_id(repo=repo, head_sha=head_sha) + + +def build_check_run_update_payload( + *, + repo: str, + head_sha: str, + conclusion: str, + title: str, + summary: str, + status: str = "completed", +) -> dict[str, Any]: + """Build a check-run create/update body with idempotent ``external_id``.""" + if not head_sha or not str(head_sha).strip(): + raise ValueError("head_sha is required for check-run updates") + if not repo or not str(repo).strip(): + raise ValueError("repo is required for check-run updates") + return { + "name": CHECK_NAME, + "head_sha": str(head_sha).strip(), + "external_id": app_check_run_external_id(repo=str(repo).strip(), head_sha=str(head_sha).strip()), + "status": status, + "conclusion": conclusion, + "output": { + "title": title[:255], + "summary": summary[:65535], + }, + } diff --git a/integrations/github-app/ovk_github_app/cleanup.py b/integrations/github-app/ovk_github_app/cleanup.py new file mode 100644 index 0000000..a7a5231 --- /dev/null +++ b/integrations/github-app/ovk_github_app/cleanup.py @@ -0,0 +1,40 @@ +"""Uninstall cleanup for installation-scoped data.""" + +from __future__ import annotations + +import logging +from typing import Any + +from ovk_github_app.isolation import InstallationStore +from ovk_github_app.redact import redact_message +from ovk_github_app.tokens import InstallationTokenProvider + +logger = logging.getLogger(__name__) + + +def handle_installation_deleted( + payload: dict[str, Any], + *, + store: InstallationStore, + token_provider: InstallationTokenProvider | None = None, +) -> dict[str, Any]: + """Delete installation-scoped data when GitHub sends ``installation.deleted``. + + Clears filesystem partitions and any in-memory installation tokens. + """ + installation = payload.get("installation") if isinstance(payload.get("installation"), dict) else {} + raw_id = installation.get("id") + if raw_id is None: + raise ValueError("installation.deleted payload missing installation.id") + installation_id = int(raw_id) + deleted = store.delete_installation(installation_id) + if token_provider is not None: + token_provider.clear_cache(installation_id) + logger.info( + redact_message(f"installation cleanup complete id={installation_id} deleted={deleted}") + ) + return { + "installation_id": installation_id, + "deleted": deleted, + "action": "installation.deleted", + } diff --git a/integrations/github-app/ovk_github_app/errors.py b/integrations/github-app/ovk_github_app/errors.py new file mode 100644 index 0000000..e407f80 --- /dev/null +++ b/integrations/github-app/ovk_github_app/errors.py @@ -0,0 +1,23 @@ +"""Typed failures for the GitHub App alpha controls.""" + +from __future__ import annotations + + +class GitHubAppError(Exception): + """Base error for the private-alpha GitHub App service.""" + + +class SignatureError(GitHubAppError): + """Webhook HMAC signature missing or invalid.""" + + +class ReplayError(GitHubAppError): + """Webhook rejected by timestamp skew or delivery-id dedupe.""" + + +class IsolationError(GitHubAppError): + """Cross-installation or cross-repository boundary violation.""" + + +class TokenError(GitHubAppError): + """Installation token exchange or lifetime policy failure.""" diff --git a/integrations/github-app/ovk_github_app/isolation.py b/integrations/github-app/ovk_github_app/isolation.py new file mode 100644 index 0000000..88243c8 --- /dev/null +++ b/integrations/github-app/ovk_github_app/isolation.py @@ -0,0 +1,121 @@ +"""Per-installation credentials and data partitions.""" + +from __future__ import annotations + +import json +import re +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from ovk_github_app.errors import IsolationError + +_SAFE_INSTALLATION = re.compile(r"^[1-9][0-9]*$") + + +def _require_installation_id(installation_id: int | str) -> str: + text = str(installation_id).strip() + if not _SAFE_INSTALLATION.fullmatch(text): + raise IsolationError(f"invalid installation id: {installation_id!r}") + return text + + +@dataclass(frozen=True) +class InstallationPartition: + """Filesystem layout for one GitHub App installation.""" + + installation_id: str + root: Path + + @property + def credentials_dir(self) -> Path: + return self.root / "credentials" + + @property + def cache_dir(self) -> Path: + return self.root / "cache" + + @property + def data_dir(self) -> Path: + return self.root / "data" + + def ensure(self) -> None: + for path in (self.root, self.credentials_dir, self.cache_dir, self.data_dir): + path.mkdir(parents=True, exist_ok=True) + + +class InstallationStore: + """Isolate credentials and cached data by installation id. + + Paths are always ``{root}/installations/{installation_id}/...``. Callers must + never construct sibling paths from untrusted repo names. + """ + + def __init__(self, root: Path) -> None: + self.root = Path(root) + self.installations_root = self.root / "installations" + self.installations_root.mkdir(parents=True, exist_ok=True) + + def partition(self, installation_id: int | str) -> InstallationPartition: + iid = _require_installation_id(installation_id) + part = InstallationPartition( + installation_id=iid, + root=self.installations_root / iid, + ) + part.ensure() + return part + + def assert_path_in_partition(self, installation_id: int | str, path: Path) -> Path: + """Resolve ``path`` and reject escape outside the installation partition.""" + part = self.partition(installation_id) + resolved = Path(path).resolve() + root = part.root.resolve() + try: + resolved.relative_to(root) + except ValueError as exc: + raise IsolationError( + f"path {resolved} escapes installation {part.installation_id} partition" + ) from exc + return resolved + + def write_json(self, installation_id: int | str, relative: str, payload: dict[str, Any]) -> Path: + part = self.partition(installation_id) + target = (part.data_dir / relative).resolve() + self.assert_path_in_partition(installation_id, target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n", encoding="utf-8") + return target + + def read_json(self, installation_id: int | str, relative: str) -> dict[str, Any] | None: + part = self.partition(installation_id) + target = (part.data_dir / relative).resolve() + self.assert_path_in_partition(installation_id, target) + if not target.is_file(): + return None + return json.loads(target.read_text(encoding="utf-8")) + + def delete_installation(self, installation_id: int | str) -> bool: + """Remove all installation-scoped data. Returns True if anything was deleted.""" + iid = _require_installation_id(installation_id) + target = (self.installations_root / iid).resolve() + root = self.installations_root.resolve() + try: + target.relative_to(root) + except ValueError as exc: + raise IsolationError(f"refusing to delete outside installations root: {target}") from exc + if target == root: + raise IsolationError("refusing to delete installations root") + if not target.exists(): + return False + shutil.rmtree(target) + return True + + def list_installations(self) -> list[str]: + if not self.installations_root.is_dir(): + return [] + return sorted( + path.name + for path in self.installations_root.iterdir() + if path.is_dir() and _SAFE_INSTALLATION.fullmatch(path.name) + ) diff --git a/integrations/github-app/ovk_github_app/redact.py b/integrations/github-app/ovk_github_app/redact.py new file mode 100644 index 0000000..7ca2228 --- /dev/null +++ b/integrations/github-app/ovk_github_app/redact.py @@ -0,0 +1,113 @@ +"""Redacted logging helpers for the GitHub App alpha.""" + +from __future__ import annotations + +import logging +import re +from typing import Any + +# Home / user path prefixes (aligned with ovk.core.evidence_integrity.redact_path intent). +_HOME_PREFIX = re.compile( + r"^(?:" + r"(?i:[a-z]:)/Users/[^/]+|" + r"/Users/[^/]+|" + r"/home/[^/]+" + r")" +) +_SECRET_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"(?i)(authorization:\s*bearer\s+)(\S+)"), + re.compile(r"(?i)(x-hub-signature-256:\s*)(\S+)"), + re.compile(r"(ghp_[A-Za-z0-9_]{20,})"), + re.compile(r"(github_pat_[A-Za-z0-9_]{20,})"), + re.compile(r"(ghs_[A-Za-z0-9_]{20,})"), + re.compile(r"(-----BEGIN[^-]+PRIVATE KEY-----)(.*?)(-----END[^-]+PRIVATE KEY-----)", re.DOTALL), + re.compile(r"(?i)(webhook[_-]?secret|client[_-]?secret|private[_-]?key)\s*[:=]\s*(\S+)"), +) + + +def redact_path(path: str) -> str: + """Scrub account home prefixes from filesystem paths.""" + raw = str(path).strip() + if not raw: + return raw + normalized = raw.replace("\\", "/") + match = _HOME_PREFIX.match(normalized) + if match: + rest = normalized[match.end() :].lstrip("/") + return f"/{rest}" if rest else "" + drive = re.match(r"^(?i:[a-z]:)(/.*)?$", normalized) + if drive: + rest = (drive.group(1) or "").lstrip("/") + return f"/{rest}" if rest else "" + return normalized + + +def redact_secrets(text: str) -> str: + """Scrub bearer tokens, PATs, and PEM private key material from log text.""" + out = str(text) + for pattern in _SECRET_PATTERNS: + if pattern.groups >= 3 and "PRIVATE KEY" in pattern.pattern: + out = pattern.sub(r"\1\3", out) + elif pattern.groups >= 2 and ( + "authorization" in pattern.pattern.lower() + or "signature" in pattern.pattern.lower() + or "secret" in pattern.pattern.lower() + ): + out = pattern.sub(r"\1", out) + else: + out = pattern.sub("", out) + return out + + +def redact_message(message: str) -> str: + """Redact a free-form log message (paths embedded in prose + secrets).""" + scrubbed = redact_secrets(message) + scrubbed = re.sub( + r"(?P

(?:/Users/|/home/|(?i:[a-z]:)/Users/)[^\s\"']+)", + lambda m: redact_path(m.group("p")), + scrubbed, + ) + scrubbed = re.sub( + r"(?P

(?i:[a-z]:)\\[^\s\"']+|(?i:[a-z]:)/[^\s\"']+)", + lambda m: redact_path(m.group("p")), + scrubbed, + ) + return scrubbed + + +def redact_text(text: str) -> str: + """Apply path and secret scrubbing suitable for operator logs.""" + return redact_message(text) + + +class RedactingFilter(logging.Filter): + """Logging filter that scrubs paths and secrets from record messages.""" + + def filter(self, record: logging.LogRecord) -> bool: + try: + msg = record.getMessage() + except Exception: # noqa: BLE001 + return True + redacted = redact_message(msg) + record.msg = redacted + record.args = () + return True + + +def attach_redacting_filter(logger: logging.Logger | None = None) -> logging.Filter: + """Attach :class:`RedactingFilter` to ``logger`` (default root).""" + target = logger or logging.getLogger() + filt = RedactingFilter() + target.addFilter(filt) + return filt + + +def safe_log_extra(data: dict[str, Any]) -> dict[str, Any]: + """Return a shallow copy with string values redacted.""" + out: dict[str, Any] = {} + for key, value in data.items(): + if isinstance(value, str): + out[key] = redact_message(value) + else: + out[key] = value + return out diff --git a/integrations/github-app/ovk_github_app/replay.py b/integrations/github-app/ovk_github_app/replay.py new file mode 100644 index 0000000..6b2efa2 --- /dev/null +++ b/integrations/github-app/ovk_github_app/replay.py @@ -0,0 +1,105 @@ +"""Replay protection: timestamp skew + delivery-id dedupe.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from typing import Protocol + +from ovk_github_app.errors import ReplayError + +DEFAULT_MAX_SKEW_SECONDS = 300 +DELIVERY_HEADER = "X-GitHub-Delivery" +TIMESTAMP_HEADER = "X-OVK-Timestamp" + + +class DeliveryDedupeStore(Protocol): + """Persistent or in-memory claim store for GitHub delivery IDs.""" + + def try_claim(self, delivery_id: str, *, seen_at: int) -> bool: + """Return True if this delivery_id is newly claimed; False if duplicate.""" + + def has(self, delivery_id: str) -> bool: + """Return True when delivery_id was previously claimed.""" + + +@dataclass +class MemoryDeliveryDedupeStore: + """Process-local delivery-id store with optional TTL eviction.""" + + ttl_seconds: int = 86_400 + _seen: dict[str, int] = field(default_factory=dict) + + def try_claim(self, delivery_id: str, *, seen_at: int) -> bool: + self._evict(now=seen_at) + if delivery_id in self._seen: + return False + self._seen[delivery_id] = seen_at + return True + + def has(self, delivery_id: str) -> bool: + return delivery_id in self._seen + + def _evict(self, *, now: int) -> None: + if self.ttl_seconds <= 0: + return + expired = [key for key, ts in self._seen.items() if now - ts > self.ttl_seconds] + for key in expired: + del self._seen[key] + + +def parse_webhook_timestamp(raw: str | int | None) -> int: + """Parse a unix-epoch timestamp from header or payload field.""" + if raw is None or raw == "": + raise ReplayError("missing webhook timestamp") + try: + value = int(raw) + except (TypeError, ValueError) as exc: + raise ReplayError("invalid webhook timestamp") from exc + return value + + +def assert_timestamp_fresh( + timestamp: int, + *, + now: int | None = None, + max_skew_seconds: int = DEFAULT_MAX_SKEW_SECONDS, +) -> None: + """Reject timestamps outside the allowed clock skew window.""" + if max_skew_seconds < 0: + raise ReplayError("max_skew_seconds must be non-negative") + current = int(time.time()) if now is None else int(now) + if abs(current - int(timestamp)) > int(max_skew_seconds): + raise ReplayError( + f"timestamp skew exceeded: ts={timestamp} now={current} max_skew={max_skew_seconds}" + ) + + +def assert_new_delivery( + delivery_id: str | None, + *, + store: DeliveryDedupeStore, + seen_at: int | None = None, +) -> None: + """Claim a GitHub delivery id; reject duplicates and missing ids.""" + if delivery_id is None or not str(delivery_id).strip(): + raise ReplayError("missing X-GitHub-Delivery header") + claimed_at = int(time.time()) if seen_at is None else int(seen_at) + if not store.try_claim(str(delivery_id).strip(), seen_at=claimed_at): + raise ReplayError(f"duplicate delivery id: {delivery_id}") + + +def protect_against_replay( + *, + delivery_id: str | None, + timestamp: int | str | None, + store: DeliveryDedupeStore, + now: int | None = None, + max_skew_seconds: int = DEFAULT_MAX_SKEW_SECONDS, +) -> int: + """Apply timestamp skew then delivery-id dedupe; return normalized timestamp.""" + current = int(time.time()) if now is None else int(now) + ts = parse_webhook_timestamp(timestamp) + assert_timestamp_fresh(ts, now=current, max_skew_seconds=max_skew_seconds) + assert_new_delivery(delivery_id, store=store, seen_at=current) + return ts diff --git a/integrations/github-app/ovk_github_app/service.py b/integrations/github-app/ovk_github_app/service.py new file mode 100644 index 0000000..cb75b6a --- /dev/null +++ b/integrations/github-app/ovk_github_app/service.py @@ -0,0 +1,104 @@ +"""Minimal FastAPI (or Starlette) webhook service for the private alpha. + +Runtime extras (not part of the core ``ovk`` package):: + + pip install -r integrations/github-app/requirements.txt + +Core security modules and unit tests do not require FastAPI. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from ovk_github_app.isolation import InstallationStore +from ovk_github_app.replay import MemoryDeliveryDedupeStore +from ovk_github_app.webhook import WebhookProcessor + + +def _env(name: str, default: str | None = None) -> str | None: + value = os.environ.get(name) + if value is None or value == "": + return default + return value + + +def build_processor() -> WebhookProcessor: + secret = _env("OVK_GITHUB_WEBHOOK_SECRET", "") + if not secret: + raise RuntimeError("OVK_GITHUB_WEBHOOK_SECRET is required") + data_root = Path(_env("OVK_GITHUB_APP_DATA", ".ovk-github-app") or ".ovk-github-app") + require_ts = (_env("OVK_WEBHOOK_REQUIRE_TIMESTAMP", "1") or "1") not in {"0", "false", "False"} + max_skew = int(_env("OVK_WEBHOOK_MAX_SKEW_SECONDS", "300") or "300") + return WebhookProcessor( + webhook_secret=secret, + store=InstallationStore(data_root), + dedupe=MemoryDeliveryDedupeStore(ttl_seconds=86_400), + max_skew_seconds=max_skew, + require_timestamp_header=require_ts, + ) + + +def create_app(): + """Create the ASGI app. Imports FastAPI lazily so unit tests stay dep-light.""" + try: + from fastapi import FastAPI, Header, Request, Response + except ImportError as exc: # pragma: no cover + raise RuntimeError( + "fastapi is required to run the GitHub App service; " + "install integrations/github-app/requirements.txt" + ) from exc + + processor = build_processor() + app = FastAPI( + title="OVK GitHub App (private alpha)", + version="0.1.0-alpha", + docs_url=None, + redoc_url=None, + ) + + @app.get("/healthz") + def healthz() -> dict[str, str]: + return {"status": "ok", "surface": "github-app-alpha"} + + @app.post("/webhook") + async def webhook( + request: Request, + x_hub_signature_256: str | None = Header(default=None), + x_github_delivery: str | None = Header(default=None), + x_github_event: str | None = Header(default=None), + x_ovk_timestamp: str | None = Header(default=None), + ) -> Response: + body = await request.body() + headers = { + "X-Hub-Signature-256": x_hub_signature_256 or "", + "X-GitHub-Delivery": x_github_delivery or "", + "X-GitHub-Event": x_github_event or "", + "X-OVK-Timestamp": x_ovk_timestamp or "", + } + # Drop empty optional timestamp so processor can apply require_timestamp policy. + if not headers["X-OVK-Timestamp"]: + del headers["X-OVK-Timestamp"] + if not headers["X-Hub-Signature-256"]: + del headers["X-Hub-Signature-256"] + if not headers["X-GitHub-Delivery"]: + del headers["X-GitHub-Delivery"] + result = processor.process(headers=headers, body=body) + return Response( + content=__import__("json").dumps(result.body), + status_code=result.status_code, + media_type="application/json", + ) + + return app + + +app = None + +try: + if _env("OVK_GITHUB_WEBHOOK_SECRET"): + app = create_app() +except Exception: + # Import-time app construction is best-effort; operators call create_app(). + app = None diff --git a/integrations/github-app/ovk_github_app/signature.py b/integrations/github-app/ovk_github_app/signature.py new file mode 100644 index 0000000..1b12c0c --- /dev/null +++ b/integrations/github-app/ovk_github_app/signature.py @@ -0,0 +1,48 @@ +"""GitHub webhook HMAC-SHA256 signature verification.""" + +from __future__ import annotations + +import hashlib +import hmac +import secrets + +from ovk_github_app.errors import SignatureError + +SIGNATURE_HEADER = "X-Hub-Signature-256" +SIGNATURE_PREFIX = "sha256=" + + +def compute_signature(*, secret: str | bytes, body: bytes) -> str: + """Return the ``sha256=`` digest for ``body`` under ``secret``.""" + key = secret.encode("utf-8") if isinstance(secret, str) else secret + digest = hmac.new(key, body, hashlib.sha256).hexdigest() + return f"{SIGNATURE_PREFIX}{digest}" + + +def verify_signature( + *, + secret: str | bytes, + body: bytes, + signature_header: str | None, +) -> None: + """Verify ``X-Hub-Signature-256``; reject missing or invalid signatures. + + Uses constant-time comparison. Empty secrets are rejected so misconfigured + deployments fail closed rather than accepting unsigned traffic. + """ + if not secret: + raise SignatureError("webhook secret is not configured") + if signature_header is None or not str(signature_header).strip(): + raise SignatureError("missing X-Hub-Signature-256 header") + + expected = compute_signature(secret=secret, body=body) + provided = str(signature_header).strip() + if not provided.startswith(SIGNATURE_PREFIX): + raise SignatureError("invalid webhook signature") + if not hmac.compare_digest(expected, provided): + raise SignatureError("invalid webhook signature") + + +def new_webhook_secret(*, nbytes: int = 32) -> str: + """Generate a high-entropy webhook secret for private-alpha installs.""" + return secrets.token_hex(nbytes) diff --git a/integrations/github-app/ovk_github_app/tokens.py b/integrations/github-app/ovk_github_app/tokens.py new file mode 100644 index 0000000..9c0a5d1 --- /dev/null +++ b/integrations/github-app/ovk_github_app/tokens.py @@ -0,0 +1,190 @@ +"""Short-lived GitHub App installation tokens (no long-lived PATs).""" + +from __future__ import annotations + +import json +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any, Callable + +from ovk_github_app.errors import TokenError + +# GitHub allows App JWTs for at most 10 minutes. +DEFAULT_APP_JWT_LIFETIME_SECONDS = 600 +# Installation tokens from GitHub expire in ~1 hour; we treat anything longer as policy failure. +MAX_INSTALLATION_TOKEN_LIFETIME_SECONDS = 3600 + + +@dataclass(frozen=True) +class InstallationToken: + """Ephemeral installation access token.""" + + installation_id: int + token: str + expires_at: int + permissions: dict[str, str] + + def remaining_seconds(self, *, now: int | None = None) -> int: + current = int(time.time()) if now is None else int(now) + return max(0, int(self.expires_at) - current) + + def is_expired(self, *, now: int | None = None, skew_seconds: int = 30) -> bool: + return self.remaining_seconds(now=now) <= skew_seconds + + +HttpPoster = Callable[[str, dict[str, str], bytes | None], tuple[int, dict[str, Any]]] + + +def _default_http_json( + url: str, + headers: dict[str, str], + body: bytes | None, +) -> tuple[int, dict[str, Any]]: + request = urllib.request.Request(url, data=body, headers=headers, method="POST" if body is not None else "GET") + try: + with urllib.request.urlopen(request, timeout=30) as response: # noqa: S310 — GitHub API only + raw = response.read().decode("utf-8") + payload = json.loads(raw) if raw else {} + return int(response.status), payload if isinstance(payload, dict) else {} + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8", errors="replace") + try: + payload = json.loads(raw) if raw else {} + except json.JSONDecodeError: + payload = {"message": raw} + if not isinstance(payload, dict): + payload = {"message": raw} + return int(exc.code), payload + + +def build_app_jwt( + *, + app_id: str | int, + private_key_pem: str, + now: int | None = None, + lifetime_seconds: int = DEFAULT_APP_JWT_LIFETIME_SECONDS, +) -> str: + """Create a short-lived RS256 JWT for GitHub App authentication. + + Requires PyJWT + cryptography at runtime. Unit tests inject a signer or mock + the exchange HTTP layer instead of minting real JWTs. + """ + if lifetime_seconds <= 0 or lifetime_seconds > DEFAULT_APP_JWT_LIFETIME_SECONDS: + raise TokenError( + f"app JWT lifetime must be in 1..{DEFAULT_APP_JWT_LIFETIME_SECONDS} seconds" + ) + try: + import jwt + except ImportError as exc: # pragma: no cover - optional runtime dep + raise TokenError("PyJWT is required to mint GitHub App JWTs") from exc + + current = int(time.time()) if now is None else int(now) + payload = { + "iat": current - 60, # GitHub recommends clock skew cushion + "exp": current + int(lifetime_seconds), + "iss": str(app_id), + } + try: + return jwt.encode(payload, private_key_pem, algorithm="RS256") + except Exception as exc: # noqa: BLE001 — surface as TokenError + raise TokenError(f"failed to mint app JWT: {exc}") from exc + + +@dataclass +class InstallationTokenProvider: + """Exchange installation tokens on demand; never persist long-lived PATs. + + Tokens are cached in memory only until ``expires_at`` (minus skew). There is + no API to store a classic ``ghp_`` personal access token. + """ + + app_id: str | int + private_key_pem: str + api_base: str = "https://api.github.com" + http_post: HttpPoster = _default_http_json + jwt_builder: Callable[..., str] | None = None + _cache: dict[int, InstallationToken] | None = None + + def __post_init__(self) -> None: + if self._cache is None: + self._cache = {} + + def get_token( + self, + installation_id: int, + *, + now: int | None = None, + force_refresh: bool = False, + ) -> InstallationToken: + current = int(time.time()) if now is None else int(now) + cached = (self._cache or {}).get(int(installation_id)) + if cached is not None and not force_refresh and not cached.is_expired(now=current): + return cached + token = self._exchange(installation_id, now=current) + assert self._cache is not None + self._cache[int(installation_id)] = token + return token + + def clear_cache(self, installation_id: int | None = None) -> None: + assert self._cache is not None + if installation_id is None: + self._cache.clear() + else: + self._cache.pop(int(installation_id), None) + + def _exchange(self, installation_id: int, *, now: int) -> InstallationToken: + builder = self.jwt_builder or build_app_jwt + app_jwt = builder( + app_id=self.app_id, + private_key_pem=self.private_key_pem, + now=now, + ) + url = f"{self.api_base.rstrip('/')}/app/installations/{int(installation_id)}/access_tokens" + headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {app_jwt}", + "X-GitHub-Api-Version": "2022-11-28", + "Content-Type": "application/json", + "User-Agent": "ovk-github-app-alpha", + } + status, payload = self.http_post(url, headers, b"{}") + if status not in (200, 201): + raise TokenError(f"installation token exchange failed: HTTP {status}") + token = str(payload.get("token", "") or "") + if not token: + raise TokenError("installation token exchange returned empty token") + if token.startswith("ghp_") or token.startswith("github_pat_"): + raise TokenError("refusing long-lived personal access token material") + expires_at = _parse_expires_at(payload.get("expires_at")) + lifetime = expires_at - now + if lifetime <= 0 or lifetime > MAX_INSTALLATION_TOKEN_LIFETIME_SECONDS: + raise TokenError( + f"installation token lifetime out of policy: {lifetime}s " + f"(max {MAX_INSTALLATION_TOKEN_LIFETIME_SECONDS}s)" + ) + permissions = payload.get("permissions") if isinstance(payload.get("permissions"), dict) else {} + return InstallationToken( + installation_id=int(installation_id), + token=token, + expires_at=expires_at, + permissions={str(k): str(v) for k, v in permissions.items()}, + ) + + +def _parse_expires_at(raw: Any) -> int: + if raw is None: + raise TokenError("installation token missing expires_at") + if isinstance(raw, (int, float)): + return int(raw) + text = str(raw).strip() + # GitHub returns ISO-8601 UTC, e.g. 2026-07-25T17:00:00Z + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + from datetime import datetime + + return int(datetime.fromisoformat(text).timestamp()) + except ValueError as exc: + raise TokenError(f"invalid expires_at: {raw!r}") from exc diff --git a/integrations/github-app/ovk_github_app/webhook.py b/integrations/github-app/ovk_github_app/webhook.py new file mode 100644 index 0000000..5ddbbb9 --- /dev/null +++ b/integrations/github-app/ovk_github_app/webhook.py @@ -0,0 +1,186 @@ +"""Webhook request processing with required security controls.""" + +from __future__ import annotations + +import json +import logging +import time +from dataclasses import dataclass, field +from typing import Any + +from ovk_github_app.cache_keys import app_cache_key +from ovk_github_app.check_runs import app_check_run_external_id, build_check_run_update_payload +from ovk_github_app.cleanup import handle_installation_deleted +from ovk_github_app.errors import GitHubAppError, ReplayError, SignatureError +from ovk_github_app.isolation import InstallationStore +from ovk_github_app.redact import RedactingFilter, redact_message +from ovk_github_app.replay import ( + DEFAULT_MAX_SKEW_SECONDS, + DELIVERY_HEADER, + TIMESTAMP_HEADER, + DeliveryDedupeStore, + MemoryDeliveryDedupeStore, + protect_against_replay, +) +from ovk_github_app.signature import SIGNATURE_HEADER, verify_signature +from ovk_github_app.tokens import InstallationTokenProvider + +logger = logging.getLogger("ovk_github_app.webhook") +logger.addFilter(RedactingFilter()) + + +@dataclass +class WebhookResult: + status_code: int + body: dict[str, Any] + + +@dataclass +class WebhookProcessor: + """Verify, dedupe, isolate, and dispatch GitHub App webhook events.""" + + webhook_secret: str + store: InstallationStore + dedupe: DeliveryDedupeStore = field(default_factory=MemoryDeliveryDedupeStore) + token_provider: InstallationTokenProvider | None = None + max_skew_seconds: int = DEFAULT_MAX_SKEW_SECONDS + require_timestamp_header: bool = True + + def process( + self, + *, + headers: dict[str, str], + body: bytes, + now: int | None = None, + ) -> WebhookResult: + current = int(time.time()) if now is None else int(now) + normalized = {_normalize_header(k): v for k, v in headers.items()} + try: + verify_signature( + secret=self.webhook_secret, + body=body, + signature_header=normalized.get(SIGNATURE_HEADER.lower()), + ) + timestamp_raw = normalized.get(TIMESTAMP_HEADER.lower()) + if timestamp_raw is None and not self.require_timestamp_header: + timestamp_raw = str(current) + protect_against_replay( + delivery_id=normalized.get(DELIVERY_HEADER.lower()), + timestamp=timestamp_raw, + store=self.dedupe, + now=current, + max_skew_seconds=self.max_skew_seconds, + ) + event = normalized.get("x-github-event", "") + payload = json.loads(body.decode("utf-8") or "{}") + if not isinstance(payload, dict): + raise GitHubAppError("webhook payload must be a JSON object") + return self._dispatch(event=event, payload=payload, now=current) + except SignatureError as exc: + logger.warning(redact_message(f"webhook signature rejected: {exc}")) + return WebhookResult(401, {"error": "invalid_signature", "detail": str(exc)}) + except ReplayError as exc: + logger.warning(redact_message(f"webhook replay rejected: {exc}")) + return WebhookResult(409, {"error": "replay_rejected", "detail": str(exc)}) + except (GitHubAppError, ValueError, json.JSONDecodeError) as exc: + logger.warning(redact_message(f"webhook processing failed: {exc}")) + return WebhookResult(400, {"error": "bad_request", "detail": str(exc)}) + + def _dispatch(self, *, event: str, payload: dict[str, Any], now: int) -> WebhookResult: + action = str(payload.get("action", "") or "") + installation = payload.get("installation") if isinstance(payload.get("installation"), dict) else {} + installation_id = installation.get("id") + repository = payload.get("repository") if isinstance(payload.get("repository"), dict) else {} + repo_id = repository.get("id") + full_name = str(repository.get("full_name") or "") + + if event == "installation" and action == "deleted": + result = handle_installation_deleted( + payload, + store=self.store, + token_provider=self.token_provider, + ) + return WebhookResult(200, {"ok": True, "handled": "installation.deleted", **result}) + + if installation_id is not None: + # Touch the installation partition so credentials/data stay isolated. + self.store.partition(int(installation_id)) + + if event in {"check_suite", "pull_request", "push"} and installation_id and repo_id: + cache_key = app_cache_key( + installation_id=int(installation_id), + repo_id=int(repo_id), + namespace=event, + components={"action": action, "delivery_bound": True}, + ) + head_sha = _extract_head_sha(event, payload) + external_id = ( + app_check_run_external_id(repo=full_name or "unknown/repo", head_sha=head_sha) + if head_sha + else None + ) + check_payload = None + if head_sha and full_name: + check_payload = build_check_run_update_payload( + repo=full_name, + head_sha=head_sha, + conclusion="neutral", + title="OVK GitHub App alpha", + summary="Private alpha acknowledged event; composite Action remains the public path.", + status="in_progress", + ) + self.store.write_json( + int(installation_id), + f"events/{event}-{now}.json", + { + "event": event, + "action": action, + "repo_id": int(repo_id), + "cache_key": cache_key, + "external_id": external_id, + "check_run": check_payload, + }, + ) + return WebhookResult( + 200, + { + "ok": True, + "handled": event, + "installation_id": int(installation_id), + "repo_id": int(repo_id), + "cache_key": cache_key, + "external_id": external_id, + }, + ) + + if installation_id is not None: + return WebhookResult( + 200, + { + "ok": True, + "handled": event or "unknown", + "installation_id": int(installation_id), + "action": action, + }, + ) + return WebhookResult(200, {"ok": True, "handled": event or "unknown", "action": action}) + + +def _normalize_header(name: str) -> str: + return str(name).strip().lower() + + +def _extract_head_sha(event: str, payload: dict[str, Any]) -> str | None: + if event == "push": + sha = payload.get("after") + return str(sha) if sha else None + if event == "pull_request": + pr = payload.get("pull_request") if isinstance(payload.get("pull_request"), dict) else {} + head = pr.get("head") if isinstance(pr.get("head"), dict) else {} + sha = head.get("sha") + return str(sha) if sha else None + if event == "check_suite": + suite = payload.get("check_suite") if isinstance(payload.get("check_suite"), dict) else {} + sha = suite.get("head_sha") + return str(sha) if sha else None + return None diff --git a/integrations/github-app/requirements.txt b/integrations/github-app/requirements.txt new file mode 100644 index 0000000..1880d5c --- /dev/null +++ b/integrations/github-app/requirements.txt @@ -0,0 +1,4 @@ +fastapi>=0.115.0 +uvicorn>=0.30.0 +PyJWT>=2.8.0 +cryptography>=42.0.0 diff --git a/tests/test_github_app_controls.py b/tests/test_github_app_controls.py new file mode 100644 index 0000000..598a65f --- /dev/null +++ b/tests/test_github_app_controls.py @@ -0,0 +1,247 @@ +"""Remaining GitHub App alpha controls: tokens, cache, checks, redact, webhook.""" + +from __future__ import annotations + +import json +import logging +import time +from pathlib import Path + +import pytest + +from ovk.core.github_check import check_run_external_id +from ovk_github_app.cache_keys import app_cache_key, assert_cache_key_bound +from ovk_github_app.check_runs import app_check_run_external_id, build_check_run_update_payload +from ovk_github_app.errors import IsolationError, TokenError +from ovk_github_app.isolation import InstallationStore +from ovk_github_app.redact import RedactingFilter, redact_message, redact_path, redact_secrets +from ovk_github_app.replay import MemoryDeliveryDedupeStore +from ovk_github_app.signature import compute_signature +from ovk_github_app.tokens import InstallationTokenProvider +from ovk_github_app.webhook import WebhookProcessor + + +def test_check_run_external_id_aligns_with_pr6() -> None: + repo = "acme/widgets" + sha = "deadbeefcafebabe" + assert app_check_run_external_id(repo=repo, head_sha=sha) == check_run_external_id( + repo=repo, head_sha=sha + ) + assert app_check_run_external_id(repo=repo, head_sha=sha) == f"ovk:{repo}:{sha}" + + +def test_check_run_payload_is_idempotent_per_head_sha() -> None: + a = build_check_run_update_payload( + repo="o/r", + head_sha="abc", + conclusion="success", + title="t", + summary="s", + ) + b = build_check_run_update_payload( + repo="o/r", + head_sha="abc", + conclusion="failure", + title="t2", + summary="s2", + ) + assert a["external_id"] == b["external_id"] == "ovk:o/r:abc" + + +def test_cache_key_includes_installation_and_repo() -> None: + k1 = app_cache_key(installation_id=1, repo_id=10, namespace="pull_request", components={"a": 1}) + k2 = app_cache_key(installation_id=2, repo_id=10, namespace="pull_request", components={"a": 1}) + k3 = app_cache_key(installation_id=1, repo_id=11, namespace="pull_request", components={"a": 1}) + assert k1 != k2 + assert k1 != k3 + assert k2 != k3 + + +def test_cache_key_rejects_missing_binding() -> None: + with pytest.raises(IsolationError): + app_cache_key(installation_id="", repo_id=1, namespace="x") + with pytest.raises(IsolationError): + assert_cache_key_bound({"repo_id": 1}, installation_id=1, repo_id=1) + + +def test_token_provider_exchanges_on_demand_and_rejects_pat() -> None: + now = int(time.time()) + calls: list[str] = [] + + def fake_jwt(**kwargs): # noqa: ANN003 + return "app-jwt" + + def fake_http(url: str, headers: dict[str, str], body: bytes | None): + calls.append(url) + assert headers["Authorization"] == "Bearer app-jwt" + assert "ghp_" not in (body or b"").decode() + return ( + 201, + { + "token": "ghs_installation_short", + "expires_at": now + 3600, + "permissions": {"checks": "write", "contents": "read"}, + }, + ) + + provider = InstallationTokenProvider( + app_id=42, + private_key_pem="-----BEGIN PRIVATE KEY-----\nTEST\n-----END PRIVATE KEY-----", + jwt_builder=fake_jwt, + http_post=fake_http, + ) + token = provider.get_token(7, now=now) + assert token.token.startswith("ghs_") + assert token.expires_at == now + 3600 + assert len(calls) == 1 + # Cached within lifetime — no second exchange. + again = provider.get_token(7, now=now + 10) + assert again.token == token.token + assert len(calls) == 1 + + def pat_http(url: str, headers: dict[str, str], body: bytes | None): + return (201, {"token": "ghp_long_lived_pat_value_xxxxxxxxxxxx", "expires_at": now + 100}) + + bad = InstallationTokenProvider( + app_id=42, + private_key_pem="x", + jwt_builder=fake_jwt, + http_post=pat_http, + ) + with pytest.raises(TokenError, match="personal access token"): + bad.get_token(1, now=now) + + +def test_token_provider_rejects_oversized_lifetime() -> None: + now = int(time.time()) + + def fake_http(url: str, headers: dict[str, str], body: bytes | None): + return (201, {"token": "ghs_x", "expires_at": now + 7200}) + + provider = InstallationTokenProvider( + app_id=1, + private_key_pem="x", + jwt_builder=lambda **_: "jwt", + http_post=fake_http, + ) + with pytest.raises(TokenError, match="lifetime"): + provider.get_token(1, now=now) + + +def test_redact_paths_and_secrets() -> None: + assert "" in redact_path("/Users/mateo/secret/repo/file.py") + assert "mateo" not in redact_path("/Users/mateo/secret/repo/file.py") + assert "" in redact_secrets("token=ghp_abcdefghijklmnopqrstuvwxyz012345") + msg = redact_message( + "auth Authorization: Bearer ghs_abcdefghijklmnopqrstuvwxyz path=/Users/mateo/proj/a.py" + ) + assert "ghs_" not in msg + assert "mateo" not in msg + assert "" in msg or "" in msg + + +def test_redacting_filter_on_logger(caplog: pytest.LogCaptureFixture) -> None: + log = logging.getLogger("ovk_github_app.test_redact") + log.addFilter(RedactingFilter()) + log.setLevel(logging.INFO) + with caplog.at_level(logging.INFO, logger="ovk_github_app.test_redact"): + log.info("key ghp_abcdefghijklmnopqrstuvwxyz012345 at /Users/mateo/x") + text = " ".join(r.message for r in caplog.records) + assert "ghp_" not in text + assert "mateo" not in text + + +def test_webhook_processor_happy_path_and_idempotent_external_id(tmp_path: Path) -> None: + secret = "whsec" + store = InstallationStore(tmp_path) + processor = WebhookProcessor( + webhook_secret=secret, + store=store, + dedupe=MemoryDeliveryDedupeStore(), + require_timestamp_header=True, + max_skew_seconds=300, + ) + now = int(time.time()) + payload = { + "action": "opened", + "installation": {"id": 55}, + "repository": {"id": 9001, "full_name": "acme/widgets"}, + "pull_request": {"head": {"sha": "abc123"}}, + } + body = json.dumps(payload).encode("utf-8") + headers = { + "X-Hub-Signature-256": compute_signature(secret=secret, body=body), + "X-GitHub-Delivery": "del-1", + "X-GitHub-Event": "pull_request", + "X-OVK-Timestamp": str(now), + } + result = processor.process(headers=headers, body=body, now=now) + assert result.status_code == 200 + assert result.body["ok"] is True + assert result.body["external_id"] == "ovk:acme/widgets:abc123" + assert result.body["installation_id"] == 55 + assert result.body["repo_id"] == 9001 + # Replay same delivery → 409 + replay = processor.process(headers=headers, body=body, now=now) + assert replay.status_code == 409 + + +def test_webhook_rejects_missing_signature(tmp_path: Path) -> None: + processor = WebhookProcessor( + webhook_secret="s", + store=InstallationStore(tmp_path), + dedupe=MemoryDeliveryDedupeStore(), + ) + now = int(time.time()) + result = processor.process( + headers={ + "X-GitHub-Delivery": "d1", + "X-GitHub-Event": "ping", + "X-OVK-Timestamp": str(now), + }, + body=b"{}", + now=now, + ) + assert result.status_code == 401 + + +def test_webhook_installation_deleted(tmp_path: Path) -> None: + secret = "s" + store = InstallationStore(tmp_path) + store.write_json(77, "x.json", {"v": 1}) + processor = WebhookProcessor( + webhook_secret=secret, + store=store, + dedupe=MemoryDeliveryDedupeStore(), + ) + now = int(time.time()) + payload = {"action": "deleted", "installation": {"id": 77}} + body = json.dumps(payload).encode("utf-8") + result = processor.process( + headers={ + "X-Hub-Signature-256": compute_signature(secret=secret, body=body), + "X-GitHub-Delivery": "del-uninstall", + "X-GitHub-Event": "installation", + "X-OVK-Timestamp": str(now), + }, + body=body, + now=now, + ) + assert result.status_code == 200 + assert result.body["handled"] == "installation.deleted" + assert not (tmp_path / "installations" / "77").exists() + + +def test_manifest_is_private_with_least_privilege() -> None: + manifest_path = ( + Path(__file__).resolve().parents[1] / "integrations" / "github-app" / "manifest.json" + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert manifest["public"] is False + perms = manifest["default_permissions"] + assert perms["checks"] == "write" + assert perms["contents"] == "read" + assert perms["pull_requests"] == "read" + # No broad admin / workflows / members permissions. + for forbidden in ("administration", "members", "workflows", "actions"): + assert forbidden not in perms diff --git a/tests/test_github_app_isolation.py b/tests/test_github_app_isolation.py new file mode 100644 index 0000000..8e9a669 --- /dev/null +++ b/tests/test_github_app_isolation.py @@ -0,0 +1,78 @@ +"""Installation isolation and uninstall cleanup tests (OVK-PR7).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from ovk_github_app.cleanup import handle_installation_deleted +from ovk_github_app.errors import IsolationError +from ovk_github_app.isolation import InstallationStore +from ovk_github_app.tokens import InstallationToken, InstallationTokenProvider + + +def test_partitions_are_installation_scoped(tmp_path: Path) -> None: + store = InstallationStore(tmp_path) + a = store.partition(11) + b = store.partition(22) + assert a.root != b.root + assert a.root.parent == b.root.parent == store.installations_root + store.write_json(11, "note.json", {"who": "a"}) + store.write_json(22, "note.json", {"who": "b"}) + assert store.read_json(11, "note.json") == {"who": "a"} + assert store.read_json(22, "note.json") == {"who": "b"} + + +def test_path_escape_rejected(tmp_path: Path) -> None: + store = InstallationStore(tmp_path) + part = store.partition(7) + escape = (part.root / ".." / "22" / "data" / "x.json").resolve() + with pytest.raises(IsolationError, match="escapes"): + store.assert_path_in_partition(7, escape) + + +def test_invalid_installation_id_rejected(tmp_path: Path) -> None: + store = InstallationStore(tmp_path) + with pytest.raises(IsolationError, match="invalid"): + store.partition("../evil") + with pytest.raises(IsolationError, match="invalid"): + store.partition("0") + + +def test_installation_deleted_removes_partition(tmp_path: Path) -> None: + store = InstallationStore(tmp_path) + store.write_json(99, "keep.json", {"x": 1}) + assert (tmp_path / "installations" / "99").is_dir() + + tokens = InstallationTokenProvider( + app_id=1, + private_key_pem="unused", + jwt_builder=lambda **_: "jwt", + http_post=lambda *_a, **_k: (201, {"token": "ghs_test", "expires_at": 9_999_999_999}), + ) + # Seed cache without network by injecting directly. + assert tokens._cache is not None + tokens._cache[99] = InstallationToken( + installation_id=99, + token="ghs_cached", + expires_at=9_999_999_999, + permissions={}, + ) + + result = handle_installation_deleted( + {"action": "deleted", "installation": {"id": 99}}, + store=store, + token_provider=tokens, + ) + assert result["deleted"] is True + assert not (tmp_path / "installations" / "99").exists() + assert 99 not in (tokens._cache or {}) + # Sibling installation untouched. + store.write_json(100, "ok.json", {"ok": True}) + handle_installation_deleted( + {"action": "deleted", "installation": {"id": 99}}, + store=store, + token_provider=tokens, + ) + assert store.read_json(100, "ok.json") == {"ok": True} diff --git a/tests/test_github_app_replay.py b/tests/test_github_app_replay.py new file mode 100644 index 0000000..601eaa1 --- /dev/null +++ b/tests/test_github_app_replay.py @@ -0,0 +1,85 @@ +"""Replay protection tests: timestamp skew + delivery-id dedupe (OVK-PR7).""" + +from __future__ import annotations + +import pytest + +from ovk_github_app.errors import ReplayError +from ovk_github_app.replay import ( + MemoryDeliveryDedupeStore, + assert_new_delivery, + assert_timestamp_fresh, + protect_against_replay, +) + + +def test_timestamp_within_skew_accepted() -> None: + now = 1_700_000_000 + assert_timestamp_fresh(now - 10, now=now, max_skew_seconds=300) + assert_timestamp_fresh(now + 10, now=now, max_skew_seconds=300) + + +def test_timestamp_outside_skew_rejected() -> None: + now = 1_700_000_000 + with pytest.raises(ReplayError, match="skew"): + assert_timestamp_fresh(now - 301, now=now, max_skew_seconds=300) + with pytest.raises(ReplayError, match="skew"): + assert_timestamp_fresh(now + 500, now=now, max_skew_seconds=300) + + +def test_delivery_id_dedupe_rejects_replay() -> None: + store = MemoryDeliveryDedupeStore() + assert_new_delivery("delivery-1", store=store, seen_at=100) + with pytest.raises(ReplayError, match="duplicate"): + assert_new_delivery("delivery-1", store=store, seen_at=101) + + +def test_delivery_id_missing_rejected() -> None: + store = MemoryDeliveryDedupeStore() + with pytest.raises(ReplayError, match="missing"): + assert_new_delivery(None, store=store, seen_at=1) + with pytest.raises(ReplayError, match="missing"): + assert_new_delivery(" ", store=store, seen_at=1) + + +def test_protect_against_replay_combines_guards() -> None: + store = MemoryDeliveryDedupeStore() + now = 1_700_000_100 + protect_against_replay( + delivery_id="abc-def", + timestamp=now - 5, + store=store, + now=now, + max_skew_seconds=300, + ) + with pytest.raises(ReplayError, match="duplicate"): + protect_against_replay( + delivery_id="abc-def", + timestamp=now - 5, + store=store, + now=now, + max_skew_seconds=300, + ) + + +def test_protect_against_replay_stale_timestamp_before_dedupe() -> None: + store = MemoryDeliveryDedupeStore() + now = 1_700_000_100 + with pytest.raises(ReplayError, match="skew"): + protect_against_replay( + delivery_id="never-claimed", + timestamp=now - 10_000, + store=store, + now=now, + max_skew_seconds=300, + ) + assert not store.has("never-claimed") + + +def test_dedupe_ttl_eviction_allows_reclaim() -> None: + store = MemoryDeliveryDedupeStore(ttl_seconds=60) + store.try_claim("old", seen_at=100) + assert store.has("old") + # Advance past TTL; claim triggers eviction. + assert store.try_claim("new", seen_at=200) + assert not store.has("old") diff --git a/tests/test_github_app_signature.py b/tests/test_github_app_signature.py new file mode 100644 index 0000000..ed1f03f --- /dev/null +++ b/tests/test_github_app_signature.py @@ -0,0 +1,53 @@ +"""Webhook HMAC signature verification tests (OVK-PR7).""" + +from __future__ import annotations + +import pytest + +from ovk_github_app.errors import SignatureError +from ovk_github_app.signature import compute_signature, verify_signature + + +def test_verify_signature_accepts_valid_hmac() -> None: + secret = "test-webhook-secret" + body = b'{"action":"opened"}' + header = compute_signature(secret=secret, body=body) + verify_signature(secret=secret, body=body, signature_header=header) + + +def test_verify_signature_rejects_missing() -> None: + with pytest.raises(SignatureError, match="missing"): + verify_signature(secret="s", body=b"{}", signature_header=None) + with pytest.raises(SignatureError, match="missing"): + verify_signature(secret="s", body=b"{}", signature_header=" ") + + +def test_verify_signature_rejects_invalid() -> None: + secret = "test-webhook-secret" + body = b'{"ok":true}' + with pytest.raises(SignatureError, match="invalid"): + verify_signature( + secret=secret, + body=body, + signature_header="sha256=" + ("ab" * 32), + ) + + +def test_verify_signature_rejects_empty_secret() -> None: + with pytest.raises(SignatureError, match="not configured"): + verify_signature(secret="", body=b"{}", signature_header="sha256=abc") + + +def test_verify_signature_rejects_wrong_secret() -> None: + body = b'{"x":1}' + header = compute_signature(secret="correct", body=body) + with pytest.raises(SignatureError, match="invalid"): + verify_signature(secret="wrong", body=body, signature_header=header) + + +def test_verify_signature_rejects_prefixless_digest() -> None: + secret = "s" + body = b"{}" + digest = compute_signature(secret=secret, body=body).removeprefix("sha256=") + with pytest.raises(SignatureError, match="invalid"): + verify_signature(secret=secret, body=body, signature_header=digest) From fad3b4e091fba4339d2630481c2c6a2bf2843fa8 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:09:51 -0700 Subject: [PATCH 12/19] Publish three advisory external pilot reports (OVK-PR8). Land express-actions, fastapi-terraform, and infra-terraform-k8s pilot reports with check-case evidence so adoption guidance is backed by attributable advisory runs. --- docs/EXTERNAL_PILOT_PLAYBOOK.md | 2 + docs/PILOT_CASE_STUDIES.md | 13 ++- docs/benchmarks/external-pilots-registry.json | 72 +++++++++++++- docs/pilots/README.md | 49 ++++++++++ docs/pilots/express-actions/REPORT.md | 82 ++++++++++++++++ .../express-actions/check-case-results.json | 24 +++++ .../external_pilot_report.json | 50 ++++++++++ docs/pilots/express-actions/pilot-report.json | 28 ++++++ docs/pilots/fastapi-terraform/REPORT.md | 85 +++++++++++++++++ .../fastapi-terraform/check-case-results.json | 24 +++++ .../external_pilot_report.json | 50 ++++++++++ .../fastapi-terraform/pilot-report.json | 28 ++++++ docs/pilots/infra-terraform-k8s/REPORT.md | 71 ++++++++++++++ .../check-case-results.json | 31 +++++++ .../external_pilot_report.json | 57 ++++++++++++ .../infra-terraform-k8s/pilot-report.json | 18 ++++ .../infra-terraform-k8s/profile/README.md | 33 +++++++ .../profile/ovk-pilot.workflow.yml | 55 +++++++++++ .../pilot_repos/express_actions_consumer.json | 17 ++++ .../fastapi_terraform_consumer.json | 17 ++++ examples/pilot_repos/infra_terraform_k8s.json | 17 ++++ tests/test_published_pilot_reports.py | 93 +++++++++++++++++++ tests/test_render_pilot_metrics.py | 2 +- 23 files changed, 908 insertions(+), 10 deletions(-) create mode 100644 docs/pilots/README.md create mode 100644 docs/pilots/express-actions/REPORT.md create mode 100644 docs/pilots/express-actions/check-case-results.json create mode 100644 docs/pilots/express-actions/external_pilot_report.json create mode 100644 docs/pilots/express-actions/pilot-report.json create mode 100644 docs/pilots/fastapi-terraform/REPORT.md create mode 100644 docs/pilots/fastapi-terraform/check-case-results.json create mode 100644 docs/pilots/fastapi-terraform/external_pilot_report.json create mode 100644 docs/pilots/fastapi-terraform/pilot-report.json create mode 100644 docs/pilots/infra-terraform-k8s/REPORT.md create mode 100644 docs/pilots/infra-terraform-k8s/check-case-results.json create mode 100644 docs/pilots/infra-terraform-k8s/external_pilot_report.json create mode 100644 docs/pilots/infra-terraform-k8s/pilot-report.json create mode 100644 docs/pilots/infra-terraform-k8s/profile/README.md create mode 100644 docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml create mode 100644 examples/pilot_repos/express_actions_consumer.json create mode 100644 examples/pilot_repos/fastapi_terraform_consumer.json create mode 100644 examples/pilot_repos/infra_terraform_k8s.json create mode 100644 tests/test_published_pilot_reports.py diff --git a/docs/EXTERNAL_PILOT_PLAYBOOK.md b/docs/EXTERNAL_PILOT_PLAYBOOK.md index 94890e2..7135ebe 100644 --- a/docs/EXTERNAL_PILOT_PLAYBOOK.md +++ b/docs/EXTERNAL_PILOT_PLAYBOOK.md @@ -122,6 +122,8 @@ See `examples/github_workflows/pilot_fork_adopter.yml` and the full example in S Weekly in-repo pilot workflow: `.github/workflows/pilot-dogfood.yml` with `scripts/collect_pilot_metrics.py`. +Published advisory reports (maintained consumers + infra profile): [pilots/README.md](pilots/README.md). + ## Support artifacts - Repair loop walkthrough: [AGENT_REPAIR_LOOP.md](AGENT_REPAIR_LOOP.md) diff --git a/docs/PILOT_CASE_STUDIES.md b/docs/PILOT_CASE_STUDIES.md index 4b6cdc2..7e46b64 100644 --- a/docs/PILOT_CASE_STUDIES.md +++ b/docs/PILOT_CASE_STUDIES.md @@ -114,15 +114,20 @@ Playbook: [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) Manifest template: [templates/pilot_manifest_ci_secrets.template.json](templates/pilot_manifest_ci_secrets.template.json) +Published advisory reports (OVK-PR8): [pilots/README.md](pilots/README.md) + ### Active and recruiting Source of truth: [external-pilots-registry.json](benchmarks/external-pilots-registry.json) (merged into [adoption-summary.json](benchmarks/adoption-summary.json) by `scripts/render_pilot_metrics.py`). -| Repository | Status | Check type | Advisory period | False positive rate | Strict enabled | -|-------|--------|------|-----------------|---------------------|----------------| -| TBD — recruiting first OSS adopter (see registry) | recruiting | ci_secrets | — | — | no | +| Repository | Status | Kind | Check types | Advisory period | False positive rate | Strict enabled | +|-------|--------|------|-------------|-----------------|---------------------|----------------| +| [fraware/ovk-consumer-fastapi-terraform](pilots/fastapi-terraform/REPORT.md) | advisory | Maintained consumer | ci_secrets, infrastructure | 2026-07-11 – 2026-07-25 | 0.0 (fixtures) | no | +| [fraware/ovk-consumer-express-actions](pilots/express-actions/REPORT.md) | advisory | Maintained consumer | ci_secrets, self_protection | 2026-07-11 – 2026-07-25 | 0.0 (fixtures) | no | +| [in-repo/ovk-pilot-infra-terraform-k8s](pilots/infra-terraform-k8s/REPORT.md) | advisory | In-repo maintained profile | infrastructure, ci_secrets | 2026-07-11 – 2026-07-25 | 0.0 (fixtures) | no | +| TBD - recruiting first true external OSS adopter | recruiting | True external OSS (open) | ci_secrets | — | — | no | -When an external repo completes advisory rollout, maintainer ingests artifacts, updates the registry, re-renders the adoption summary, and replaces the recruiting row with measured metrics. Target: under 5% false positives before enabling strict mode on protected branches. +Maintained-consumer and in-repo profile rows are **fixture/dogfood** measurements with `strict_mode_recommendation: remain_advisory`. They do not replace a true independent external OSS adopter. When such a repo completes advisory rollout, ingest artifacts, update the registry, re-render the adoption summary, and keep the kind label explicit. Target: under 5% false positives before enabling strict mode on protected branches. ### Reporting template diff --git a/docs/benchmarks/external-pilots-registry.json b/docs/benchmarks/external-pilots-registry.json index f9e4df2..01a6beb 100644 --- a/docs/benchmarks/external-pilots-registry.json +++ b/docs/benchmarks/external-pilots-registry.json @@ -1,11 +1,73 @@ { "schema_version": "ovk.external_pilots_registry.v1", - "updated_at": "2026-06-10T00:00:00Z", + "updated_at": "2026-07-25T17:08:12Z", "external_pilots": [ { - "repository": "TBD — recruiting first OSS adopter", + "repository": "fraware/ovk-consumer-fastapi-terraform", + "status": "advisory", + "check_types": [ + "ci_secrets", + "infrastructure" + ], + "advisory_start": "2026-07-11", + "advisory_end": "2026-07-25", + "prs_evaluated": 2, + "prs_blocked": 1, + "false_positives": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 95.62, + "strict_enabled": false, + "ovk_version_pin": "1.2.1", + "workflow_path": ".github/workflows/ovk-advisory-pr.yml", + "evidence_url": "docs/pilots/fastapi-terraform/pilot-report.json", + "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md." + }, + { + "repository": "fraware/ovk-consumer-express-actions", + "status": "advisory", + "check_types": [ + "ci_secrets", + "self_protection" + ], + "advisory_start": "2026-07-11", + "advisory_end": "2026-07-25", + "prs_evaluated": 2, + "prs_blocked": 1, + "false_positives": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 81.71, + "strict_enabled": false, + "ovk_version_pin": "1.2.1", + "workflow_path": ".github/workflows/ovk-advisory-pr.yml", + "evidence_url": "docs/pilots/express-actions/pilot-report.json", + "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md." + }, + { + "repository": "in-repo/ovk-pilot-infra-terraform-k8s", + "status": "advisory", + "check_types": [ + "infrastructure", + "ci_secrets" + ], + "advisory_start": "2026-07-11", + "advisory_end": "2026-07-25", + "prs_evaluated": 3, + "prs_blocked": 2, + "false_positives": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 46.14, + "strict_enabled": false, + "ovk_version_pin": "1.2.1", + "workflow_path": "docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml", + "evidence_url": "docs/pilots/infra-terraform-k8s/pilot-report.json", + "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md." + }, + { + "repository": "TBD - recruiting first true external OSS adopter", "status": "recruiting", - "check_types": ["ci_secrets"], + "check_types": [ + "ci_secrets" + ], "advisory_start": null, "advisory_end": null, "prs_evaluated": null, @@ -14,9 +76,9 @@ "false_positive_rate": null, "median_check_latency_ms": null, "strict_enabled": false, - "ovk_version_pin": "1.2.0", + "ovk_version_pin": "1.2.1", "workflow_path": ".github/workflows/ovk-pilot.yml", - "notes": "Placeholder row until the first external OSS repo completes advisory rollout. See docs/EXTERNAL_PILOT_PLAYBOOK.md and docs/templates/pilot_manifest_ci_secrets.template.json." + "notes": "Placeholder for a true independent external OSS adopter. Maintained-consumer and in-repo profile pilots are published under docs/pilots/; they do not satisfy this recruiting row." } ] } diff --git a/docs/pilots/README.md b/docs/pilots/README.md new file mode 100644 index 0000000..2e6ce11 --- /dev/null +++ b/docs/pilots/README.md @@ -0,0 +1,49 @@ +# OVK External Pilot Reports (OVK-PR8 / OVK-09) + +Published advisory pilot evidence for the adoption-surface program. These reports are +**maintained-consumer / in-repo profile** measurements, not claims of independent +external OSS production readiness. + +| Pilot | Repository | Stack | Kind | Workflow reproduction | Report | +|---|---|---|---|---|---| +| Python | [fraware/ovk-consumer-fastapi-terraform](https://github.com/fraware/ovk-consumer-fastapi-terraform) | FastAPI + Terraform | Maintained consumer | Complete | [fastapi-terraform/REPORT.md](fastapi-terraform/REPORT.md) | +| JS/TS | [fraware/ovk-consumer-express-actions](https://github.com/fraware/ovk-consumer-express-actions) | Express + Actions | Maintained consumer | Complete | [express-actions/REPORT.md](express-actions/REPORT.md) | +| Infrastructure | `in-repo/ovk-pilot-infra-terraform-k8s` | Terraform / K8s fixtures | In-repo maintained profile (no live remote) | Advisory metrics published | [infra-terraform-k8s/REPORT.md](infra-terraform-k8s/REPORT.md) | + +## Artifacts per pilot + +| File | Schema / role | +|---|---| +| `pilot-report.json` | [`schemas/pilot.report.schema.json`](../../schemas/pilot.report.schema.json) (`ovk.pilot_report.v1`) | +| `external_pilot_report.json` | Playbook self-report; ingested into [`docs/benchmarks/external-pilots-registry.json`](../benchmarks/external-pilots-registry.json) | +| `REPORT.md` | Human-readable profile, metrics, and strict-mode recommendation | +| `check-case-results.json` | Fixture-level `ovk check` outcomes used for FP/FN/unknown counts | + +## Measurement honesty + +- Metrics are from **fixture and dogfood runs** (consumer scenario diffs + in-repo manifests). +- Rows are labeled **maintained consumer** or **in-repo maintained profile**, not true independent external OSS. +- Consumer ledgers keep `production_gate_met: false` until 30 human-adjudicated PRs exist ([CONSUMER_VALIDATION_CHECKLIST.md](../CONSUMER_VALIDATION_CHECKLIST.md)). +- Strict mode is **not** recommended from these pilots alone. + +## Reproduce + +```bash +# Manifest pilot program (includes consumer-aligned manifests under examples/pilot_repos/) +ovk pilot --output .verification/pilot-program-report.json + +# Re-ingest published self-reports into the registry +python scripts/ingest_external_pilot_metrics.py \ + --repo fraware/ovk-consumer-fastapi-terraform \ + --report docs/pilots/fastapi-terraform/external_pilot_report.json +python scripts/ingest_external_pilot_metrics.py \ + --repo fraware/ovk-consumer-express-actions \ + --report docs/pilots/express-actions/external_pilot_report.json +python scripts/ingest_external_pilot_metrics.py \ + --repo in-repo/ovk-pilot-infra-terraform-k8s \ + --report docs/pilots/infra-terraform-k8s/external_pilot_report.json + +python scripts/render_pilot_metrics.py --registry docs/benchmarks/external-pilots-registry.json +``` + +Playbook: [EXTERNAL_PILOT_PLAYBOOK.md](../EXTERNAL_PILOT_PLAYBOOK.md). Case-study index: [PILOT_CASE_STUDIES.md](../PILOT_CASE_STUDIES.md). diff --git a/docs/pilots/express-actions/REPORT.md b/docs/pilots/express-actions/REPORT.md new file mode 100644 index 0000000..094a29d --- /dev/null +++ b/docs/pilots/express-actions/REPORT.md @@ -0,0 +1,82 @@ +# Pilot report — JS/TS / Express + GitHub Actions + +**Status:** Advisory metrics published (complete playbook workflow reproduction on fixtures). +**Consumer kind:** Maintained consumer (not true independent external OSS). +**Repository:** [fraware/ovk-consumer-express-actions](https://github.com/fraware/ovk-consumer-express-actions) +**Measurement basis:** Fixture and dogfood runs (2026-07-11 – 2026-07-25). +**OVK pin:** `1.2.1` / Action `@v1.2.1` + +Machine-readable companions: [`pilot-report.json`](pilot-report.json) (`ovk.pilot_report.v1`), [`external_pilot_report.json`](external_pilot_report.json). + +## Repository profile + +| Field | Value | +|---|---| +| Stack | TypeScript Express service with GitHub Actions workflows | +| Role | Independent maintained consumer gate (program section 23) | +| Advisory workflow | `.github/workflows/ovk-advisory-pr.yml` | +| Pilot ledger | Consumer `pilot/ledger.json` (automated_scenario rows only) | +| Production gate | `production_gate_met: false` | + +## Checks selected + +| Check type | Why | +|---|---| +| `ci_secrets` | Primary playbook starter for workflow-heavy repos | +| `self_protection` | Agent CI-gate integrity for Actions-centric consumers | + +Manifests exercised: `examples/pilot_repos/express_actions_consumer.json`, `examples/pilot_repos/external_oss_ci_secrets.json`. + +## Workflow reproduction (complete) + +Documented playbook path reproduced locally against the consumer clone: + +1. Advisory `ovk check` on `fixtures/diffs/advisory_passing.diff` → `allow` +2. Advisory `ovk check` on `fixtures/diffs/advisory_failing.diff` → `block` +3. Advisory pilot manifest verify for CI secrets + self-protection safe fixtures → `allow` +4. Metrics packaged into `pilot-report.json` + `external_pilot_report.json` for registry ingest + +## False positives + +| Metric | Measured | +|---|---| +| False positives | **0** | +| False positive rate | **0.0** (0/2 fixture cases) | + +## False negatives + +| Metric | Measured | +|---|---| +| False negatives | **0** | + +Unsafe secrets-on-PR fixture blocked as expected. + +## Unknowns + +| Source | Count / note | +|---|---| +| Fixture check cases in this report | **0** unknowns | +| Broader consumer scenario matrix | Timeout/unavailable backend scenarios remain honest non-pass outcomes in the ledger | + +Live production unknown rate: **unmeasured**. + +## Human review burden + +| Signal | Observation | +|---|---| +| Fixture advisory runs | Low — decisions matched fixtures | +| Consumer ledger | Automated scenarios only | +| Live PR review load | Not measured | + +## Configuration changes + +- Pin Action to `fraware/open-verification-kernel@v1.2.1`. +- Advisory mode + `default_on_unknown: require_human_review`. +- `.verification/ci_secrets_pilot.json` in the consumer; in-repo mirror `examples/pilot_repos/express_actions_consumer.json`. +- `post-comment` / `emit-check` exercised on the advisory PR workflow path. + +## Strict-mode recommendation + +**Remain advisory (`remain_advisory`).** Fixture metrics meet the playbook FP target numerically, but without a live human-adjudicated advisory window this report does not authorize strict branch protection. + +Median `ovk check` latency on fixture diffs: ~82 ms. diff --git a/docs/pilots/express-actions/check-case-results.json b/docs/pilots/express-actions/check-case-results.json new file mode 100644 index 0000000..7ba78d9 --- /dev/null +++ b/docs/pilots/express-actions/check-case-results.json @@ -0,0 +1,24 @@ +{ + "cases": [ + { + "case": "advisory_passing", + "expected": "allow", + "merge_recommendation": "allow", + "elapsed_ms": 80.652, + "intents": [ + "agent-cannot-disable-own-ci-gate", + "no-secrets-in-untrusted-context" + ] + }, + { + "case": "advisory_failing", + "expected": "block", + "merge_recommendation": "block", + "elapsed_ms": 82.773, + "intents": [ + "agent-cannot-disable-own-ci-gate", + "no-secrets-in-untrusted-context" + ] + } + ] +} diff --git a/docs/pilots/express-actions/external_pilot_report.json b/docs/pilots/express-actions/external_pilot_report.json new file mode 100644 index 0000000..ad0d63b --- /dev/null +++ b/docs/pilots/express-actions/external_pilot_report.json @@ -0,0 +1,50 @@ +{ + "schema_version": "ovk.external_pilot_report.v1", + "repository": "fraware/ovk-consumer-express-actions", + "repository_url": "https://github.com/fraware/ovk-consumer-express-actions", + "consumer_kind": "maintained_consumer", + "stack": "TypeScript Express + GitHub Actions", + "check_types": [ + "ci_secrets", + "self_protection" + ], + "advisory_start": "2026-07-11", + "advisory_end": "2026-07-25", + "prs_evaluated": 2, + "prs_blocked": 1, + "false_positives": 0, + "false_negatives": 0, + "unknowns": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 81.71, + "strict_enabled": false, + "strict_mode_recommendation": "remain_advisory", + "ovk_version_pin": "1.2.1", + "workflow_path": ".github/workflows/ovk-advisory-pr.yml", + "evidence_url": "docs/pilots/express-actions/pilot-report.json", + "measurement_basis": "fixture_and_dogfood", + "complete_workflow_reproduction": true, + "notes": "Maintained consumer (fraware/ovk-consumer-express-actions), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/express-actions/REPORT.md.", + "check_case_results": [ + { + "case": "advisory_passing", + "expected": "allow", + "merge_recommendation": "allow", + "elapsed_ms": 80.65, + "intents": [ + "agent-cannot-disable-own-ci-gate", + "no-secrets-in-untrusted-context" + ] + }, + { + "case": "advisory_failing", + "expected": "block", + "merge_recommendation": "block", + "elapsed_ms": 82.77, + "intents": [ + "agent-cannot-disable-own-ci-gate", + "no-secrets-in-untrusted-context" + ] + } + ] +} diff --git a/docs/pilots/express-actions/pilot-report.json b/docs/pilots/express-actions/pilot-report.json new file mode 100644 index 0000000..1192618 --- /dev/null +++ b/docs/pilots/express-actions/pilot-report.json @@ -0,0 +1,28 @@ +{ + "schema_version": "ovk.pilot_report.v1", + "pilot_dir": "docs/pilots/express-actions", + "manifests_total": 2, + "manifests_passed": 2, + "results": [ + { + "manifest": "examples/pilot_repos/express_actions_consumer.json", + "name": "pilot-express-actions-consumer", + "description": "Maintained JS/TS consumer profile: CI secrets + self-protection (Express + GitHub Actions).", + "lane_count": 2, + "evidence_count": 2, + "merge_recommendation": "allow", + "elapsed_ms": 6.101499999203952, + "passed": true + }, + { + "manifest": "examples/pilot_repos/external_oss_ci_secrets.json", + "name": "external-oss-ci-secrets-pilot", + "description": "External OSS pilot manifest: CI secrets lane only. Copy to .verification/ci_secrets_pilot.json in your fork.", + "lane_count": 1, + "evidence_count": 1, + "merge_recommendation": "allow", + "elapsed_ms": 4.04049999997369, + "passed": true + } + ] +} diff --git a/docs/pilots/fastapi-terraform/REPORT.md b/docs/pilots/fastapi-terraform/REPORT.md new file mode 100644 index 0000000..53c6776 --- /dev/null +++ b/docs/pilots/fastapi-terraform/REPORT.md @@ -0,0 +1,85 @@ +# Pilot report — Python / FastAPI + Terraform + +**Status:** Advisory metrics published (complete playbook workflow reproduction on fixtures). +**Consumer kind:** Maintained consumer (not true independent external OSS). +**Repository:** [fraware/ovk-consumer-fastapi-terraform](https://github.com/fraware/ovk-consumer-fastapi-terraform) +**Measurement basis:** Fixture and dogfood runs (2026-07-11 – 2026-07-25). +**OVK pin:** `1.2.1` / Action `@v1.2.1` + +Machine-readable companions: [`pilot-report.json`](pilot-report.json) (`ovk.pilot_report.v1`), [`external_pilot_report.json`](external_pilot_report.json). + +## Repository profile + +| Field | Value | +|---|---| +| Stack | FastAPI web app with Terraform infrastructure | +| Role | Independent maintained consumer gate (program section 23) | +| Advisory workflow | `.github/workflows/ovk-advisory-pr.yml` | +| Pilot ledger | Consumer `pilot/ledger.json` (automated_scenario rows only) | +| Production gate | `production_gate_met: false` (no 30 human adjudications yet) | + +## Checks selected + +| Check type | Why | +|---|---| +| `ci_secrets` | Primary playbook starter for agent-authored workflow PRs | +| `infrastructure` | Aligns with Terraform surface in the consumer | + +Manifests exercised: `examples/pilot_repos/fastapi_terraform_consumer.json`, `examples/pilot_repos/ci_secrets_only.json`. + +## Workflow reproduction (complete) + +Documented playbook path reproduced locally against the consumer clone: + +1. Advisory `ovk check` on `fixtures/diffs/advisory_passing.diff` → `allow` +2. Advisory `ovk check` on `fixtures/diffs/advisory_failing.diff` → `block` (job remains non-blocking in advisory) +3. Advisory `ovk verify` / pilot manifest run for CI secrets + infrastructure safe fixtures → `allow` +4. Metrics packaged into `pilot-report.json` + `external_pilot_report.json` for registry ingest + +## False positives + +| Metric | Measured | +|---|---| +| False positives | **0** | +| False positive rate | **0.0** (0/2 fixture cases) | + +Passing workflow fixture was allowed; no incorrect blocks on the known-good case. + +## False negatives + +| Metric | Measured | +|---|---| +| False negatives | **0** | + +Known-bad secrets-on-`pull_request` fixture was blocked as expected (100% block rate on that unsafe fixture). + +## Unknowns + +| Source | Count / note | +|---|---| +| Fixture check cases in this report | **0** unknowns | +| Broader consumer scenario matrix | Native backend timeout path records `unknown` honestly (ledger `auto-native_backend_timeout`); not counted as FP/FN | + +No live PR adjudications yet — unknown appropriateness for production traffic remains **unmeasured**. + +## Human review burden + +| Signal | Observation | +|---|---| +| Fixture advisory runs | Low — allow/block decisions matched fixtures without manual override | +| Consumer ledger | All rows `human_adjudication: automated_scenario` | +| Live PR review load | Not measured (no 14-day live adopter window in this publication) | +| Gate to reduce review | Accumulate human adjudications per [CONSUMER_VALIDATION_CHECKLIST.md](../../CONSUMER_VALIDATION_CHECKLIST.md) | + +## Configuration changes + +- Pin Action to `fraware/open-verification-kernel@v1.2.1` (immutable tag / audited SHA only). +- Advisory mode via `.verification/config.yml` (`mode: advisory`, `default_on_unknown: require_human_review`). +- Pilot manifest under `.verification/ci_secrets_pilot.json` (consumer) mirrored by in-repo `examples/pilot_repos/fastapi_terraform_consumer.json`. +- Artifact upload of evidence / comment outputs for ingest. + +## Strict-mode recommendation + +**Remain advisory (`remain_advisory`).** Fixture FP rate is under 5%, but this publication is maintained-consumer dogfood without human-adjudicated live PRs. Do not enable strict required checks on protected branches from this evidence alone. + +Median `ovk check` latency on fixture diffs: ~96 ms. diff --git a/docs/pilots/fastapi-terraform/check-case-results.json b/docs/pilots/fastapi-terraform/check-case-results.json new file mode 100644 index 0000000..a9dbc62 --- /dev/null +++ b/docs/pilots/fastapi-terraform/check-case-results.json @@ -0,0 +1,24 @@ +{ + "cases": [ + { + "case": "advisory_passing", + "expected": "allow", + "merge_recommendation": "allow", + "elapsed_ms": 100.651, + "intents": [ + "agent-cannot-disable-own-ci-gate", + "no-secrets-in-untrusted-context" + ] + }, + { + "case": "advisory_failing", + "expected": "block", + "merge_recommendation": "block", + "elapsed_ms": 90.58, + "intents": [ + "agent-cannot-disable-own-ci-gate", + "no-secrets-in-untrusted-context" + ] + } + ] +} diff --git a/docs/pilots/fastapi-terraform/external_pilot_report.json b/docs/pilots/fastapi-terraform/external_pilot_report.json new file mode 100644 index 0000000..7de9a54 --- /dev/null +++ b/docs/pilots/fastapi-terraform/external_pilot_report.json @@ -0,0 +1,50 @@ +{ + "schema_version": "ovk.external_pilot_report.v1", + "repository": "fraware/ovk-consumer-fastapi-terraform", + "repository_url": "https://github.com/fraware/ovk-consumer-fastapi-terraform", + "consumer_kind": "maintained_consumer", + "stack": "Python FastAPI + Terraform", + "check_types": [ + "ci_secrets", + "infrastructure" + ], + "advisory_start": "2026-07-11", + "advisory_end": "2026-07-25", + "prs_evaluated": 2, + "prs_blocked": 1, + "false_positives": 0, + "false_negatives": 0, + "unknowns": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 95.62, + "strict_enabled": false, + "strict_mode_recommendation": "remain_advisory", + "ovk_version_pin": "1.2.1", + "workflow_path": ".github/workflows/ovk-advisory-pr.yml", + "evidence_url": "docs/pilots/fastapi-terraform/pilot-report.json", + "measurement_basis": "fixture_and_dogfood", + "complete_workflow_reproduction": true, + "notes": "Maintained consumer (fraware/ovk-consumer-fastapi-terraform), not true independent external OSS. Complete advisory workflow reproduction on fixture diffs + manifests. Measured-from-fixture/dogfood. Strict remain_advisory. See docs/pilots/fastapi-terraform/REPORT.md.", + "check_case_results": [ + { + "case": "advisory_passing", + "expected": "allow", + "merge_recommendation": "allow", + "elapsed_ms": 100.65, + "intents": [ + "agent-cannot-disable-own-ci-gate", + "no-secrets-in-untrusted-context" + ] + }, + { + "case": "advisory_failing", + "expected": "block", + "merge_recommendation": "block", + "elapsed_ms": 90.58, + "intents": [ + "agent-cannot-disable-own-ci-gate", + "no-secrets-in-untrusted-context" + ] + } + ] +} diff --git a/docs/pilots/fastapi-terraform/pilot-report.json b/docs/pilots/fastapi-terraform/pilot-report.json new file mode 100644 index 0000000..4b1048f --- /dev/null +++ b/docs/pilots/fastapi-terraform/pilot-report.json @@ -0,0 +1,28 @@ +{ + "schema_version": "ovk.pilot_report.v1", + "pilot_dir": "docs/pilots/fastapi-terraform", + "manifests_total": 2, + "manifests_passed": 2, + "results": [ + { + "manifest": "examples/pilot_repos/fastapi_terraform_consumer.json", + "name": "pilot-fastapi-terraform-consumer", + "description": "Maintained Python consumer profile: CI secrets + infrastructure (FastAPI + Terraform).", + "lane_count": 2, + "evidence_count": 2, + "merge_recommendation": "allow", + "elapsed_ms": 19.658799999888288, + "passed": true + }, + { + "manifest": "examples/pilot_repos/ci_secrets_only.json", + "name": "pilot-ci-secrets-only", + "description": "Pilot manifest: CI secrets lane for workflow-heavy repositories.", + "lane_count": 1, + "evidence_count": 1, + "merge_recommendation": "allow", + "elapsed_ms": 3.492200001346646, + "passed": true + } + ] +} diff --git a/docs/pilots/infra-terraform-k8s/REPORT.md b/docs/pilots/infra-terraform-k8s/REPORT.md new file mode 100644 index 0000000..d1135f0 --- /dev/null +++ b/docs/pilots/infra-terraform-k8s/REPORT.md @@ -0,0 +1,71 @@ +# Pilot report — Infrastructure (Terraform / Kubernetes profile) + +**Status:** Advisory metrics published (scaffold; no live remote consumer). +**Consumer kind:** In-repo maintained profile — **not** a true independent external OSS repository. +**Repository id:** `in-repo/ovk-pilot-infra-terraform-k8s` +**Profile:** [profile/README.md](profile/README.md) +**Measurement basis:** In-repo infrastructure fixtures and repair-loop diffs (2026-07-11 – 2026-07-25). +**OVK pin:** working-tree package version (`1.2.1`) + +Machine-readable companions: [`pilot-report.json`](pilot-report.json) (`ovk.pilot_report.v1`), [`external_pilot_report.json`](external_pilot_report.json). + +## Repository profile + +| Field | Value | +|---|---| +| Stack | Terraform-heavy infrastructure exposure + CI secrets (K8s-oriented adopter profile) | +| Live remote | **None** — designated in-repo pilot profile until an infra consumer repo exists | +| Manifest | `examples/pilot_repos/infra_terraform_k8s.json` | +| Workflow scaffold | [profile/ovk-pilot.workflow.yml](profile/ovk-pilot.workflow.yml) | + +This pilot satisfies OVK-PR8’s third-pilot requirement at the **advisory metrics publication** bar. It does **not** claim a complete remote playbook reproduction. + +## Checks selected + +| Check type | Why | +|---|---| +| `infrastructure` | Primary signal for public/sensitive resource exposure | +| `ci_secrets` | Shared starter check from the external pilot playbook | + +## False positives + +| Metric | Measured | +|---|---| +| False positives | **0** | +| False positive rate | **0.0** (0/3 fixture cases) | + +Private / repaired infrastructure fixtures allowed when expected. + +## False negatives + +| Metric | Measured | +|---|---| +| False negatives | **0** | + +Public-sensitive and failing repair-loop fixtures blocked as expected. + +## Unknowns + +| Source | Count / note | +|---|---| +| Fixture check cases in this report | **0** unknowns | +| Live infra adopter traffic | **N/A** — no remote consumer | + +## Human review burden + +| Signal | Observation | +|---|---| +| Fixture runs | Low for the curated infra diffs | +| Expected for a future live infra consumer | Higher until exposure-graph coverage and policy digests stabilize; keep advisory | + +## Configuration changes (profile scaffold) + +- Copy [profile/ovk-pilot.workflow.yml](profile/ovk-pilot.workflow.yml) to `.github/workflows/ovk-pilot.yml` in a future infra consumer. +- Install manifest from `examples/pilot_repos/infra_terraform_k8s.json` (or consumer-local copy under `.verification/`). +- Start advisory-only; ingest artifacts via `scripts/ingest_external_pilot_metrics.py`. + +## Strict-mode recommendation + +**Remain advisory (`remain_advisory`).** Publish-only scaffold with fixture metrics. Promote to strict only after a live infra consumer completes the playbook window with FP under 5% and human adjudication. + +Median `ovk check` latency on fixture diffs: ~46 ms. diff --git a/docs/pilots/infra-terraform-k8s/check-case-results.json b/docs/pilots/infra-terraform-k8s/check-case-results.json new file mode 100644 index 0000000..a01044f --- /dev/null +++ b/docs/pilots/infra-terraform-k8s/check-case-results.json @@ -0,0 +1,31 @@ +{ + "cases": [ + { + "case": "infra_passing", + "expected": "allow", + "merge_recommendation": "allow", + "elapsed_ms": 46.139, + "intents": [ + "no-public-sensitive-resource" + ] + }, + { + "case": "infra_failing", + "expected": "block", + "merge_recommendation": "block", + "elapsed_ms": 44.99, + "intents": [ + "no-public-sensitive-resource" + ] + }, + { + "case": "infra_public_s3", + "expected": "block", + "merge_recommendation": "block", + "elapsed_ms": 46.143, + "intents": [ + "no-public-sensitive-resource" + ] + } + ] +} diff --git a/docs/pilots/infra-terraform-k8s/external_pilot_report.json b/docs/pilots/infra-terraform-k8s/external_pilot_report.json new file mode 100644 index 0000000..47df93b --- /dev/null +++ b/docs/pilots/infra-terraform-k8s/external_pilot_report.json @@ -0,0 +1,57 @@ +{ + "schema_version": "ovk.external_pilot_report.v1", + "repository": "in-repo/ovk-pilot-infra-terraform-k8s", + "repository_url": "https://github.com/fraware/open-verification-kernel/tree/main/docs/pilots/infra-terraform-k8s", + "consumer_kind": "in_repo_maintained_profile", + "stack": "Terraform + Kubernetes-oriented infrastructure fixtures", + "check_types": [ + "infrastructure", + "ci_secrets" + ], + "advisory_start": "2026-07-11", + "advisory_end": "2026-07-25", + "prs_evaluated": 3, + "prs_blocked": 2, + "false_positives": 0, + "false_negatives": 0, + "unknowns": 0, + "false_positive_rate": 0.0, + "median_check_latency_ms": 46.14, + "strict_enabled": false, + "strict_mode_recommendation": "remain_advisory", + "ovk_version_pin": "1.2.1", + "workflow_path": "docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml", + "evidence_url": "docs/pilots/infra-terraform-k8s/pilot-report.json", + "measurement_basis": "fixture_and_dogfood", + "complete_workflow_reproduction": false, + "notes": "In-repo maintained infrastructure pilot profile (no live remote). Advisory metrics published from Terraform/K8s-oriented fixtures. Not true external OSS. Strict remain_advisory. See docs/pilots/infra-terraform-k8s/REPORT.md.", + "check_case_results": [ + { + "case": "infra_passing", + "expected": "allow", + "merge_recommendation": "allow", + "elapsed_ms": 46.14, + "intents": [ + "no-public-sensitive-resource" + ] + }, + { + "case": "infra_failing", + "expected": "block", + "merge_recommendation": "block", + "elapsed_ms": 44.99, + "intents": [ + "no-public-sensitive-resource" + ] + }, + { + "case": "infra_public_s3", + "expected": "block", + "merge_recommendation": "block", + "elapsed_ms": 46.14, + "intents": [ + "no-public-sensitive-resource" + ] + } + ] +} diff --git a/docs/pilots/infra-terraform-k8s/pilot-report.json b/docs/pilots/infra-terraform-k8s/pilot-report.json new file mode 100644 index 0000000..768aec6 --- /dev/null +++ b/docs/pilots/infra-terraform-k8s/pilot-report.json @@ -0,0 +1,18 @@ +{ + "schema_version": "ovk.pilot_report.v1", + "pilot_dir": "docs/pilots/infra-terraform-k8s", + "manifests_total": 1, + "manifests_passed": 1, + "results": [ + { + "manifest": "examples/pilot_repos/infra_terraform_k8s.json", + "name": "pilot-infra-terraform-k8s", + "description": "Infrastructure-heavy pilot: infrastructure exposure + CI secrets (Terraform/K8s consumer profile).", + "lane_count": 2, + "evidence_count": 2, + "merge_recommendation": "allow", + "elapsed_ms": 4.408300002978649, + "passed": true + } + ] +} diff --git a/docs/pilots/infra-terraform-k8s/profile/README.md b/docs/pilots/infra-terraform-k8s/profile/README.md new file mode 100644 index 0000000..dc7e5fd --- /dev/null +++ b/docs/pilots/infra-terraform-k8s/profile/README.md @@ -0,0 +1,33 @@ +# In-repo infrastructure pilot profile + +Designated Terraform / Kubernetes-oriented adopter profile used when no live +`ovk-consumer-*-infra` remote exists. + +| Field | Value | +|---|---| +| Kind | Maintained in-repo profile (not true external OSS) | +| Registry id | `in-repo/ovk-pilot-infra-terraform-k8s` | +| Manifest | [`examples/pilot_repos/infra_terraform_k8s.json`](../../../examples/pilot_repos/infra_terraform_k8s.json) | +| Workflow scaffold | [`ovk-pilot.workflow.yml`](ovk-pilot.workflow.yml) | +| Report | [`../REPORT.md`](../REPORT.md) | + +## Intent + +Provide a reproducible advisory path for infrastructure-heavy stacks: + +1. Run infrastructure exposure + CI secrets manifests in advisory mode. +2. Exercise known-bad public resource diffs (`examples/multi_surface/infra_public_s3.diff`, repair-loop failing diff). +3. Publish metrics under `docs/pilots/infra-terraform-k8s/` and ingest into the external pilots registry. + +## Local reproduction + +```bash +ovk verify --manifest examples/pilot_repos/infra_terraform_k8s.json --advisory +ovk check --changed-files examples/repair_loops/infrastructure/passing.diff --advisory +ovk check --changed-files examples/repair_loops/infrastructure/failing.diff --advisory +ovk check --changed-files examples/multi_surface/infra_public_s3.diff --advisory +``` + +When a live infra consumer remote is created, replace the registry id, keep the +`maintained_consumer` vs `true_external_oss` label explicit, and re-run the +[EXTERNAL_PILOT_PLAYBOOK.md](../../../EXTERNAL_PILOT_PLAYBOOK.md) window before strict mode. diff --git a/docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml b/docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml new file mode 100644 index 0000000..49df758 --- /dev/null +++ b/docs/pilots/infra-terraform-k8s/profile/ovk-pilot.workflow.yml @@ -0,0 +1,55 @@ +name: OVK Infra Pilot (Advisory) + +# Scaffold for a future Terraform/K8s-heavy consumer repository. +# Copy to .github/workflows/ovk-pilot.yml and point verification-manifest at a +# local copy of examples/pilot_repos/infra_terraform_k8s.json. +# See docs/EXTERNAL_PILOT_PLAYBOOK.md and docs/pilots/infra-terraform-k8s/REPORT.md. + +on: + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + +env: + OVK_PACKAGE_VERSION: "1.2.1" + +jobs: + ovk-infra-advisory: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Build PR diff for OVK + run: | + git fetch origin "${{ github.base_ref }}" + git diff "origin/${{ github.base_ref }}...HEAD" > ovk-pr.diff + - name: OVK advisory check + uses: fraware/open-verification-kernel@v1.2.1 + with: + mode: advisory + use-check: "true" + changed-files: ovk-pr.diff + post-comment: "false" + - name: OVK advisory verify (infra + ci_secrets) + uses: fraware/open-verification-kernel@v1.2.1 + with: + mode: advisory + verification-manifest: .verification/infra_pilot.json + bundle-output-dir: ovk-pilot-bundle + post-comment: "false" + - name: Upload pilot artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: ovk-pilot-artifacts + path: | + ovk-evidence.json + ovk-pilot-bundle/** + external_pilot_report.json + retention-days: 30 + if-no-files-found: ignore diff --git a/examples/pilot_repos/express_actions_consumer.json b/examples/pilot_repos/express_actions_consumer.json new file mode 100644 index 0000000..a8831e8 --- /dev/null +++ b/examples/pilot_repos/express_actions_consumer.json @@ -0,0 +1,17 @@ +{ + "schema_version": "ovk.verification_manifest.v1", + "name": "pilot-express-actions-consumer", + "description": "Maintained JS/TS consumer profile: CI secrets + self-protection (Express + GitHub Actions).", + "lanes": [ + { + "lane": "ci_secrets", + "input": "../ci_secrets/input_secrets_safe.json", + "input_format": "infra" + }, + { + "lane": "self_protection", + "input": "../no_agent_self_approval/input_gate_preserved.json", + "input_format": "infra" + } + ] +} diff --git a/examples/pilot_repos/fastapi_terraform_consumer.json b/examples/pilot_repos/fastapi_terraform_consumer.json new file mode 100644 index 0000000..5ee82cc --- /dev/null +++ b/examples/pilot_repos/fastapi_terraform_consumer.json @@ -0,0 +1,17 @@ +{ + "schema_version": "ovk.verification_manifest.v1", + "name": "pilot-fastapi-terraform-consumer", + "description": "Maintained Python consumer profile: CI secrets + infrastructure (FastAPI + Terraform).", + "lanes": [ + { + "lane": "ci_secrets", + "input": "../ci_secrets/input_secrets_safe.json", + "input_format": "infra" + }, + { + "lane": "infrastructure", + "input": "../infrastructure_exposure/input_private_sensitive_resource.json", + "input_format": "infra" + } + ] +} diff --git a/examples/pilot_repos/infra_terraform_k8s.json b/examples/pilot_repos/infra_terraform_k8s.json new file mode 100644 index 0000000..d4f2abe --- /dev/null +++ b/examples/pilot_repos/infra_terraform_k8s.json @@ -0,0 +1,17 @@ +{ + "schema_version": "ovk.verification_manifest.v1", + "name": "pilot-infra-terraform-k8s", + "description": "Infrastructure-heavy pilot: infrastructure exposure + CI secrets (Terraform/K8s consumer profile).", + "lanes": [ + { + "lane": "infrastructure", + "input": "../infrastructure_exposure/input_private_sensitive_resource.json", + "input_format": "infra" + }, + { + "lane": "ci_secrets", + "input": "../ci_secrets/input_secrets_safe.json", + "input_format": "infra" + } + ] +} diff --git a/tests/test_published_pilot_reports.py b/tests/test_published_pilot_reports.py new file mode 100644 index 0000000..d31c55b --- /dev/null +++ b/tests/test_published_pilot_reports.py @@ -0,0 +1,93 @@ +"""Validate published OVK-PR8 pilot reports under docs/pilots/.""" + +from __future__ import annotations + +from pathlib import Path + +from ovk.core.json_io import read_json_file +from ovk.core.schema_validation import require_schema_valid + +ROOT = Path(__file__).resolve().parents[1] +PILOTS_DIR = ROOT / "docs" / "pilots" +PILOT_REPORT_SCHEMA = ROOT / "schemas" / "pilot.report.schema.json" +REGISTRY = ROOT / "docs" / "benchmarks" / "external-pilots-registry.json" + +REQUIRED_SLUGS = ( + "fastapi-terraform", + "express-actions", + "infra-terraform-k8s", +) + +REQUIRED_REPORT_SECTIONS = ( + "Repository profile", + "Checks selected", + "False positives", + "False negatives", + "Unknowns", + "Human review burden", + "Configuration changes", + "Strict-mode recommendation", +) + + +def test_three_pilot_reports_published() -> None: + for slug in REQUIRED_SLUGS: + report_md = PILOTS_DIR / slug / "REPORT.md" + pilot_report = PILOTS_DIR / slug / "pilot-report.json" + external_report = PILOTS_DIR / slug / "external_pilot_report.json" + assert report_md.is_file(), f"missing {report_md}" + assert pilot_report.is_file(), f"missing {pilot_report}" + assert external_report.is_file(), f"missing {external_report}" + + +def test_published_pilot_reports_match_schema() -> None: + schema = read_json_file(PILOT_REPORT_SCHEMA) + for slug in REQUIRED_SLUGS: + payload = read_json_file(PILOTS_DIR / slug / "pilot-report.json") + require_schema_valid(payload, schema, context=f"docs/pilots/{slug}/pilot-report.json") + assert payload["manifests_total"] >= 1 + assert payload["manifests_passed"] == payload["manifests_total"] + + +def test_published_report_markdown_covers_required_sections() -> None: + for slug in REQUIRED_SLUGS: + text = (PILOTS_DIR / slug / "REPORT.md").read_text(encoding="utf-8") + for section in REQUIRED_REPORT_SECTIONS: + assert section in text, f"{slug} REPORT.md missing section: {section}" + + +def test_external_reports_mark_maintained_vs_oss_and_remain_advisory() -> None: + fastapi = read_json_file(PILOTS_DIR / "fastapi-terraform" / "external_pilot_report.json") + express = read_json_file(PILOTS_DIR / "express-actions" / "external_pilot_report.json") + infra = read_json_file(PILOTS_DIR / "infra-terraform-k8s" / "external_pilot_report.json") + + assert fastapi["consumer_kind"] == "maintained_consumer" + assert express["consumer_kind"] == "maintained_consumer" + assert infra["consumer_kind"] == "in_repo_maintained_profile" + assert fastapi["complete_workflow_reproduction"] is True + assert express["complete_workflow_reproduction"] is True + assert infra["complete_workflow_reproduction"] is False + + for payload in (fastapi, express, infra): + assert payload["strict_enabled"] is False + assert payload["strict_mode_recommendation"] == "remain_advisory" + assert payload["measurement_basis"] == "fixture_and_dogfood" + assert payload["false_positive_rate"] == 0.0 + + +def test_registry_includes_published_pilots() -> None: + registry = read_json_file(REGISTRY) + repos = {str(item["repository"]) for item in registry["external_pilots"]} + assert "fraware/ovk-consumer-fastapi-terraform" in repos + assert "fraware/ovk-consumer-express-actions" in repos + assert "in-repo/ovk-pilot-infra-terraform-k8s" in repos + assert any(item.get("status") == "recruiting" for item in registry["external_pilots"]) + + +def test_infra_profile_scaffold_exists() -> None: + profile = PILOTS_DIR / "infra-terraform-k8s" / "profile" + assert (profile / "README.md").is_file() + assert (profile / "ovk-pilot.workflow.yml").is_file() + assert (ROOT / "examples" / "pilot_repos" / "infra_terraform_k8s.json").is_file() + assert (ROOT / "examples" / "pilot_repos" / "fastapi_terraform_consumer.json").is_file() + assert (ROOT / "examples" / "pilot_repos" / "express_actions_consumer.json").is_file() diff --git a/tests/test_render_pilot_metrics.py b/tests/test_render_pilot_metrics.py index 124b87c..7b48d3a 100644 --- a/tests/test_render_pilot_metrics.py +++ b/tests/test_render_pilot_metrics.py @@ -21,7 +21,7 @@ def test_render_adoption_summary_shape() -> None: summary = render_adoption_summary(metrics) validate_summary(summary) assert summary["real_diff_recall"] == 1.0 - assert summary["formal_pr_bench"]["cases_total"] == 130 + assert summary["formal_pr_bench"]["cases_total"] == 132 assert summary["formal_pr_bench"]["pass_rate"] == 1.0 assert summary["updated_at"] is not None assert summary["pilot_dogfood"]["ovk_version_pin"] == metrics["ovk_version"] From ff1089295b138c96125f8cb2658be16de4d3f39b Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:10:00 -0700 Subject: [PATCH 13/19] Bump package metadata to v1.3.0-rc.1 (OVK-PR9). Advance release metadata and preflight report shape so install pins and consumer checklists target the in-repo release candidate explicitly. --- docs/RELEASE_NOTES_v1.3.0-rc.1.md | 49 ++++++++++++++++++++ ovk/core/release_metadata.py | 2 +- ovk/core/release_preflight_report.py | 27 +++++++++++ pyproject.toml | 4 +- tests/test_release_metadata.py | 4 +- tests/test_release_preflight_report_shape.py | 3 ++ 6 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 docs/RELEASE_NOTES_v1.3.0-rc.1.md diff --git a/docs/RELEASE_NOTES_v1.3.0-rc.1.md b/docs/RELEASE_NOTES_v1.3.0-rc.1.md new file mode 100644 index 0000000..65b3bce --- /dev/null +++ b/docs/RELEASE_NOTES_v1.3.0-rc.1.md @@ -0,0 +1,49 @@ +# OVK v1.3.0-rc.1 + +Release-candidate notes for the adoption-surface cut. **Do not treat this document as attributable publication evidence** until a signed immutable tag exists and [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md) live gates are filled. + +## Highlights + +- Normative capability registry with honest `release_status` and generated backend tables +- DecisionState lattice (`allow` / `block` / `needs_review` / `unknown` / `error` / `skipped`) with strict fail-closed aggregation +- Evidence integrity envelope (digests, timestamps, controlling findings, optional signature) +- Seven-item adapter conformance; `stable` requires full suite +- FormalPR-Bench provenance, partitions, mutations/held-out guards, version manifest +- Composite Action hardening + SHA-pinned third-party actions in release paths +- Private GitHub App alpha (`integrations/github-app/`) +- Three advisory pilot reports under `docs/pilots/` +- Reviewer TCB inventory: [TRUSTED_COMPUTING_BASE.md](TRUSTED_COMPUTING_BASE.md) + +## Install (after the tag exists) + +```bash +pip install open-verification-kernel==1.3.0-rc.1 +``` + +Composite Action (immutable pin): + +```yaml +env: + OVK_PACKAGE_VERSION: "1.3.0-rc.1" +steps: + - uses: fraware/open-verification-kernel@v1.3.0-rc.1 +``` + +Until the tag is published, local/dev installs remain `pip install -e '.[dev]'` from this tree; the Action falls back to checkout install when PyPI lacks the RC. + +## Local RC preflight + +```bash +python scripts/verify_rc_dod.py +python scripts/verify_rc_install.py +ovk release-preflight +``` + +## Known limits for this RC + +- Live non-`[skip ci]` workflow IDs and Sigstore identity for **this** version are not yet recorded +- Independent consumers still pin signed `v1.2.1` until remotes are bumped +- Default product path remains advisory / shadow until attributable strict-mode calibration +- Package classifier remains Beta + +See [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) and [ROADMAP.md](ROADMAP.md). diff --git a/ovk/core/release_metadata.py b/ovk/core/release_metadata.py index 99746fe..9d63051 100644 --- a/ovk/core/release_metadata.py +++ b/ovk/core/release_metadata.py @@ -5,7 +5,7 @@ from typing import Any -OVK_VERSION = "1.2.1" +OVK_VERSION = "1.3.0-rc.1" OVK_RELEASE_CANDIDATE = OVK_VERSION diff --git a/ovk/core/release_preflight_report.py b/ovk/core/release_preflight_report.py index 2546e1d..6cac546 100644 --- a/ovk/core/release_preflight_report.py +++ b/ovk/core/release_preflight_report.py @@ -237,6 +237,30 @@ def _check_adapter_capabilities() -> list[str]: return validate_capabilities() +def _check_rc_dod() -> list[str]: + """Verify in-repo OVK-PR9 definition-of-done items.""" + ensure_repo_on_path() + from scripts.verify_rc_dod import verify_rc_dod + + return verify_rc_dod() + + +def _check_rc_install() -> list[str]: + """Verify pip metadata + composite Action SHA-pin install surface.""" + ensure_repo_on_path() + from scripts.verify_rc_install import verify_rc_install + + return verify_rc_install(wheel=False) + + +def _check_tcb_doc_fresh() -> list[str]: + """Ensure TRUSTED_COMPUTING_BASE.md matches registry + Action surfaces.""" + ensure_repo_on_path() + from scripts.render_tcb_doc import tcb_doc_stale + + return tcb_doc_stale() + + def build_release_preflight_report() -> PreflightReport: """Run release preflight checks and return a structured report.""" ensure_repo_on_path() @@ -264,6 +288,9 @@ def build_release_preflight_report() -> PreflightReport: check_from_failures("pilot_program", _check_pilot_program()), check_from_failures("release_layout_schema_coverage", _check_release_layout_schema_coverage()), check_from_failures("adapter_capabilities", _check_adapter_capabilities()), + check_from_failures("rc_dod", _check_rc_dod()), + check_from_failures("rc_install", _check_rc_install()), + check_from_failures("tcb_doc", _check_tcb_doc_fresh()), ), optional_checks=(check_from_failures("pilot_metrics_dry_run", _check_pilot_metrics_dry_run()),), ) diff --git a/pyproject.toml b/pyproject.toml index 2389899..f2d2d7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "open-verification-kernel" -version = "1.2.1" +version = "1.3.0-rc.1" description = "Solver-agnostic verification kernel for AI-agent engineering workflows" readme = "README.md" requires-python = ">=3.10" @@ -72,7 +72,7 @@ line-length = 120 [tool.pytest.ini_options] testpaths = ["tests"] -pythonpath = ["."] +pythonpath = [".", "integrations/github-app"] markers = [ "native_backend(name): native backend integration probe by backend name", ] diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py index df49d89..16b2719 100644 --- a/tests/test_release_metadata.py +++ b/tests/test_release_metadata.py @@ -3,8 +3,8 @@ def test_release_metadata_contains_release_candidate() -> None: metadata = release_metadata() - assert metadata["version"] == "1.2.1" - assert metadata["release_candidate"] == "1.2.1" + assert metadata["version"] == "1.3.0-rc.1" + assert metadata["release_candidate"] == "1.3.0-rc.1" def test_release_metadata_lists_core_commands() -> None: diff --git a/tests/test_release_preflight_report_shape.py b/tests/test_release_preflight_report_shape.py index 0a69c82..ed9043f 100644 --- a/tests/test_release_preflight_report_shape.py +++ b/tests/test_release_preflight_report_shape.py @@ -20,6 +20,9 @@ def test_release_preflight_report_is_serializable() -> None: "pilot_program", "release_layout_schema_coverage", "adapter_capabilities", + "rc_dod", + "rc_install", + "tcb_doc", } optional_names = {check["name"] for check in payload.get("optional_checks", [])} assert optional_names == {"pilot_metrics_dry_run"} From 84313264ac47ec55a516176ce2016d0e7cf7d392 Mon Sep 17 00:00:00 2001 From: fraware Date: Sat, 25 Jul 2026 11:10:01 -0700 Subject: [PATCH 14/19] Add TCB doc and RC install/DoD verification gates (OVK-PR9). Document the trusted computing base and add machine-checkable RC definition-of-done plus install verification so attributable publication cannot proceed on incomplete gates. --- .github/workflows/publish.yml | 28 +- docs/ATTRIBUTABLE_PUBLICATION.md | 241 +++++++++++--- docs/CONSUMER_VALIDATION_CHECKLIST.md | 196 +++++------ docs/CURRENT_RELEASE_STATUS.md | 265 ++++++++------- docs/RELEASE.md | 36 +- docs/TRUSTED_COMPUTING_BASE.md | 136 ++++++++ .../consumer_validation.workflow.yml | 4 +- scripts/render_tcb_doc.py | 307 ++++++++++++++++++ scripts/verify_rc_dod.py | 212 ++++++++++++ scripts/verify_rc_install.py | 182 +++++++++++ tests/test_rc_pr9.py | 39 +++ 11 files changed, 1342 insertions(+), 304 deletions(-) create mode 100644 docs/TRUSTED_COMPUTING_BASE.md create mode 100644 scripts/render_tcb_doc.py create mode 100644 scripts/verify_rc_dod.py create mode 100644 scripts/verify_rc_install.py create mode 100644 tests/test_rc_pr9.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f2acefd..3270d70 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,8 +21,8 @@ jobs: if: github.event_name == 'release' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install and verify release gates @@ -58,7 +58,7 @@ jobs: /tmp/ovk-release-wheel/bin/ovk doctor /tmp/ovk-release-wheel/bin/python -c 'from ovk import mcp_server; from ovk.core.templates_cli import list_templates; assert len(list_templates()) >= 100; assert len(mcp_server.list_capabilities()["capabilities"]) >= 10' - name: Upload built distributions - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ovk-dist path: dist/* @@ -69,8 +69,8 @@ jobs: if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Build package for Sigstore dry-run @@ -83,7 +83,7 @@ jobs: python -m build twine check dist/* - name: Upload built distributions - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ovk-dist path: dist/* @@ -102,17 +102,17 @@ jobs: contents: write id-token: write steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Download built distributions - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ovk-dist path: dist - name: Install cosign - uses: sigstore/cosign-installer@v3.8.1 + uses: sigstore/cosign-installer@d7d6bc7722e3daa8354c50bcb52f4837da5e9b6a # v3.8.1 - name: Resolve certificate identity id: identity run: | @@ -162,7 +162,7 @@ jobs: --git-ref "${GITHUB_REF}" fi - name: Retain cosign bundles - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ovk-sigstore-bundles path: | @@ -192,13 +192,13 @@ jobs: contents: read id-token: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Download built distributions - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ovk-dist path: dist - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1 (release/v1) with: password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/docs/ATTRIBUTABLE_PUBLICATION.md b/docs/ATTRIBUTABLE_PUBLICATION.md index 83affe7..adc860d 100644 --- a/docs/ATTRIBUTABLE_PUBLICATION.md +++ b/docs/ATTRIBUTABLE_PUBLICATION.md @@ -1,51 +1,190 @@ -# Attributable Publication Checklist (Sprint 10) - -Gate for publishing **`v1.3.0-rc.1`** and later promoting to **`v1.3.0`**. -Authority: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) 18-condition gate. - -## Terminology - -| Field | Use | -|---|---| -| `benchmark_source_sha` | FormalPR-Bench / badge measurement identity | -| `verified_source_sha` | Complete observed required-workflow set only | - -Never label a `[skip ci]` badge commit as verified. Never re-attribute `v1.2.1` -Sigstore / consumer evidence to typed-control-plane commits. - -## Collect workflow evidence (when Actions are available) - -```bash -python scripts/collect_workflow_evidence.py \ - --sha \ - --output .verification/workflow-evidence-.json -``` - -The collector records run IDs/URLs under `benchmark_source_sha` and leaves -`verified_source_sha` unset until maintainers confirm the full required set. - -## Pre-tag checklist (`v1.3.0-rc.1`) - -- [ ] P0 trust PRs 1–9 landed on the tag source -- [ ] Non-`[skip ci]` CI, native Tier 1, wheel smoke, Action dogfood, release preflight green -- [ ] Expanded FormalPR-Bench recorded with `benchmark_source_sha` -- [ ] Template conformance v2 matrix regenerated from semantic statuses -- [ ] Both consumers dispatched on immutable rc.1 pin (or audited commit); evidence downloaded and verified -- [ ] Label-separated holdout aggregates retained (predictions digest + eval workflow IDs) -- [ ] Release artifacts signed; workflow IDs and digests recorded in [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) - -## Promote to `v1.3.0` - -Only after: - -- [ ] All 18 completion-gate conditions hold -- [ ] P0 closure (PRs 1–9) on the exact tag source -- [ ] Consumer validation on the exact pin -- [ ] Attributable holdout aggregates (predictions digest + eval) -- [ ] Human pilot ledgers remain separate from automated fixtures -- [ ] No re-attribution of `v1.2.1` Sigstore evidence to typed-control-plane commits - -## Blocked without external access - -Live GitHub Actions run URLs, consumer repo pin PRs, and private holdout evaluation require -maintainer credentials outside this working tree. +# Attributable Publication Checklist (Sprint 10 / OVK-PR9) + +Gate for publishing **`v1.3.0-rc.1`** and later promoting to **`v1.3.0`**. +Authority: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) 18-condition gate. +Status dashboard: [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md). TCB: [TRUSTED_COMPUTING_BASE.md](TRUSTED_COMPUTING_BASE.md). +Release procedure: [RELEASE.md](RELEASE.md). Consumer pins: [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md). + +## Terminology + +| Field | Use | +|---|---| +| `benchmark_source_sha` | FormalPR-Bench / badge measurement identity | +| `verified_source_sha` | Complete observed required-workflow set only | + +Never label a `[skip ci]` badge commit as verified. Never re-attribute `v1.2.1` +Sigstore / consumer evidence to typed-control-plane commits. + +## In-repo readiness (OVK-PR9) — complete without live secrets + +Run these locally before asking maintainers for a tag: + +```bash +python scripts/check_release_metadata.py +python scripts/render_capability_tables.py --check +python scripts/render_tcb_doc.py --check +python scripts/validate_capabilities.py +python scripts/validate_adapter_conformance.py +python scripts/verify_rc_dod.py +python scripts/verify_rc_install.py # Action SHA pins + package metadata +python scripts/verify_rc_install.py --wheel # outside-checkout wheel import +ovk release-preflight +``` + +| Item | Status | +|---|---| +| Package / `__version__` / metadata = `1.3.0-rc.1` | Done in working tree | +| Registry covers every public checker; `stable ⊆` conformant | Done (DoD script) | +| Strict fail-closed lattice + evidence integrity suites present | Done (PR2+PR3) | +| Evidence reconstructs controlling decision APIs | Done | +| Bench version manifest + partition digests | Done (PR5) | +| ≥2 pilot reports under `docs/pilots/` | Done (PR8; three published) | +| TCB doc generated from registry + Action/App surfaces | Done | +| Installable via pip wheel path **and** composite Action (SHA-pinned deps) | In-repo verified; PyPI/tag still live | + +## Exact maintainer publication sequence (requires push + secrets) + +Replace `` with a **non-`[skip ci]`** commit that carries this tree. +Do **not** tag from a badge-only commit. + +### 1. Land source and observe required workflows + +```bash +# After push to origin (this workspace does not push): +git push origin HEAD:main # or open/merge a PR — maintainer only + +# Confirm the commit message does NOT contain [skip ci] +git log -1 --format=%B +``` + +Required workflow names (collector): `CI`, `Native Tier 1`, `Release`, `Bench` +(plus Action dogfood / wheel smoke as recorded in [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md)). + +```bash +python scripts/collect_workflow_evidence.py \ + --sha \ + --output .verification/workflow-evidence-.json + +# Optional direct inspection: +gh run list --repo fraware/open-verification-kernel --commit --limit 30 \ + --json databaseId,workflowName,status,conclusion,url,headSha +``` + +Only after the complete required set is green on ``, set +`verified_source_sha=` in [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) +and paste run URLs / IDs. Until then cite `benchmark_source_sha` only. + +### 2. Signed immutable tag + GitHub Release + +Tag binding: Publish requires `github.event.release.tag_name` (without leading `v`) +to equal `ovk.__version__` exactly — tag **`v1.3.0-rc.1`**, package **`1.3.0-rc.1`**. + +```bash +git fetch origin +git checkout +git tag -s v1.3.0-rc.1 +git push origin v1.3.0-rc.1 + +gh release create v1.3.0-rc.1 \ + --verify-tag \ + --title "OVK v1.3.0-rc.1" \ + --notes-file docs/RELEASE_NOTES_v1.3.0-rc.1.md +``` + +Do not move historical tags (`v1.2.1`, …). + +### 3. Sigstore / cosign (identity-bound) + +Protected Publish workflow (`.github/workflows/publish.yml`) keyless-signs distributions +in the `sigstore` environment. Production verification identity for this RC: + +```text +https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.3.0-rc.1 +``` + +OIDC issuer: + +```text +https://token.actions.githubusercontent.com +``` + +```bash +# Watch the Publish run attached to the Release: +gh run list --repo fraware/open-verification-kernel --workflow Publish.yml --limit 5 + +# Consumer-side verify (after downloading wheel + *.cosign.bundle.json from the Release): +export OVK_COSIGN_IDENTITY='https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.3.0-rc.1' +export OVK_COSIGN_ISSUER='https://token.actions.githubusercontent.com' +cosign verify-blob \ + --bundle path/to/artifact.cosign.bundle.json \ + --certificate-identity "$OVK_COSIGN_IDENTITY" \ + --certificate-oidc-issuer "$OVK_COSIGN_ISSUER" \ + path/to/open_verification_kernel-1.3.0rc1-*.whl +``` + +Optional dry-run (no PyPI; **not** a production pin — bound to branch ref, not the tag): + +```bash +gh workflow run Publish.yml --ref main -f dry_run=true +gh run watch +``` + +### 4. Consumer pin bumps (separate remotes; do not push from this workspace alone) + +In-repo templates already target `v1.3.0-rc.1` +([templates/consumer_validation.workflow.yml](templates/consumer_validation.workflow.yml), +[examples/github_workflows/](../examples/github_workflows/)). + +After the tag exists, in each consumer: + +```bash +# Example for fastapi consumer (repeat for express): +gh api repos/fraware/ovk-consumer-fastapi-terraform/contents/.github/workflows/ \ + --jq '.[].name' # locate validation workflow + +# Bump uses: fraware/open-verification-kernel@v1.3.0-rc.1 +# and OVK_PACKAGE_VERSION: "1.3.0-rc.1" via PR, then: +gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-fastapi-terraform +gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-express-actions + +gh run list --repo fraware/ovk-consumer-fastapi-terraform --limit 5 +gh run download --repo fraware/ovk-consumer-fastapi-terraform \ + -n -D ./consumer-evidence/fastapi/ +``` + +Full checklist: [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md). + +### 5. Record status + +Update [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) with: + +- `verified_source_sha` +- CI / Native Tier 1 / Action dogfood / Publish workflow IDs + URLs +- Sigstore identity string for `v1.3.0-rc.1` +- Consumer pin SHAs / run URLs + +## Pre-tag checklist (`v1.3.0-rc.1`) — remaining maintainer actions + +- [x] Adoption-surface PRs 1–9 landed in the working tree (in-repo) +- [ ] Non-`[skip ci]` CI, native Tier 1, wheel smoke, Action dogfood, release preflight green on the tag source +- [ ] Expanded FormalPR-Bench recorded with `benchmark_source_sha` +- [ ] Template conformance v2 matrix regenerated from semantic statuses (as needed on release SHA) +- [ ] Both consumers dispatched on immutable rc.1 pin (or audited commit); evidence downloaded and verified +- [ ] Label-separated holdout aggregates retained when promoting beyond RC (predictions digest + eval workflow IDs) +- [ ] Release artifacts signed; workflow IDs and digests recorded in [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) + +## Promote to `v1.3.0` + +Only after: + +- [ ] All 18 completion-gate conditions hold +- [ ] P0 closure (R2 PRs 1–9) on the exact tag source +- [ ] Consumer validation on the exact pin +- [ ] Attributable holdout aggregates (predictions digest + eval) +- [ ] Human pilot ledgers remain separate from automated fixtures +- [ ] No re-attribution of `v1.2.1` Sigstore evidence to typed-control-plane commits + +## Blocked without external access + +Live GitHub Actions run URLs, consumer repo pin PRs, protected Publish/Sigstore environments, +and private holdout evaluation require maintainer credentials outside this working tree. diff --git a/docs/CONSUMER_VALIDATION_CHECKLIST.md b/docs/CONSUMER_VALIDATION_CHECKLIST.md index da4b213..cf180ae 100644 --- a/docs/CONSUMER_VALIDATION_CHECKLIST.md +++ b/docs/CONSUMER_VALIDATION_CHECKLIST.md @@ -1,98 +1,98 @@ -# Consumer Validation Checklist - -Scaffolding and live pointers for independent consumer repositories validating OVK. -Completing this checklist for one repo does **not** satisfy the multi-repo production -exit criterion (30 human-adjudicated PRs per independent consumer). - -## Live independent consumers - -| Repository | Stack | Current pin | Target pin (Sprint 9) | -|---|---|---|---| -| [fraware/ovk-consumer-fastapi-terraform](https://github.com/fraware/ovk-consumer-fastapi-terraform) | FastAPI + Terraform | `v1.2.1` | immutable `v1.3.0-rc.1` (or audited commit) | -| [fraware/ovk-consumer-express-actions](https://github.com/fraware/ovk-consumer-express-actions) | Express + GitHub Actions | `v1.2.1` | immutable `v1.3.0-rc.1` (or audited commit) | - -`v1.2.1` validates the **pre-control-plane** signed release only. Typed control-plane -commits must not inherit that consumer evidence. Both consumers use -`scripts/assert_ovk_pin.py` to fail on pin drift. - -## Immutable pin requirements - -Consumers must pin an **immutable** OVK commit SHA or release tag. - -```yaml -env: - OVK_PACKAGE_VERSION: "1.3.0rc1" # after rc.1 cut; until then keep 1.2.1 - OVK_ACTION_REF: "v1.3.0-rc.1" -steps: - - uses: fraware/open-verification-kernel@v1.3.0-rc.1 -``` - -In-repo template: [templates/consumer_validation.workflow.yml](templates/consumer_validation.workflow.yml). - -Forbidden: - -- `uses: ./` -- `uses: fraware/open-verification-kernel@main` -- floating refs without a tag or full commit SHA - -## Maintainer steps after `v1.3.0-rc.1` exists (do not push from this workspace alone) - -For each consumer repository: - -1. Open a pin PR that bumps Action `uses:` and `OVK_PACKAGE_VERSION` to the immutable rc.1 tag (or full SHA). -2. Merge the pin PR (or push to a validation branch) so workflows can see the new pin. -3. Dispatch validation: - ```bash - gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-fastapi-terraform - gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-express-actions - ``` - (Use the exact workflow name as defined in each consumer.) -4. Await conclusions; download evidence artifacts: - ```bash - gh run download --repo -n -D ./consumer-evidence// - ``` -5. Verify bundles with the OVK release verifier + cosign as applicable for the pin. -6. Exercise a true cross-fork PR path (`docs/FORK_PR.md` in each consumer). -7. Update the pilot ledger: keep `automated_scenario` rows distinct from human adjudications. - -Local clone prep (optional, no push): - -```bash -git clone https://github.com/fraware/ovk-consumer-fastapi-terraform.git -git clone https://github.com/fraware/ovk-consumer-express-actions.git -# Edit workflow pins locally; do not git push until maintainers cut rc.1. -``` - -## Sprint 9 checklist (per consumer) — prepare in this repo; land in consumer repos - -In-repo preparation (this repository): - -- [x] Document rc.1 target pins and provenance correction (this checklist + R2 status) -- [x] Keep human pilot ledgers separate from automated fixtures (see consumer `pilot/ledger.json` policy) -- [x] Template workflow targets `v1.3.0-rc.1` (copy only after tag exists) -- [ ] Cut attributable `v1.3.0-rc.1` tag on verified source (Sprint 10) - -In consumer repositories (requires write access — **blocked from this workspace alone**): - -- [ ] Bump Action pin from `v1.2.1` → `v1.3.0-rc.1` (or audited full SHA) -- [ ] Bump `OVK_PACKAGE_VERSION` / wheel install scripts to match -- [ ] Dispatch validation workflows; await conclusions -- [ ] Download evidence bundles; verify with release verifier + cosign as applicable -- [ ] Exercise true cross-fork PR path (`docs/FORK_PR.md`) -- [ ] Update ledger: automated scenarios remain distinct from human adjudications - -## Checklist (per consumer) — ongoing - -- [x] Workflow copies from `docs/templates/consumer_validation.workflow.yml` (or equivalent) with an immutable pin. -- [x] Automated scenario matrix covers program section 23.1 intents (see consumer README). -- [x] Adjudication rows recorded in a pilot ledger conforming to `schemas/pilot.ledger.schema.json`. -- [ ] Human adjudications reach 30 PRs (entries must not remain `automated_scenario` / `pending` only). -- [ ] True cross-fork PR exercised and ledger-adjudicated (see consumer `docs/FORK_PR.md`). -- [ ] Prefer PyPI once published; until then Release wheel + cosign verify-blob at the **current** pin. - -## What this does not claim - -- Declaring two independent consumer repos with 30 adjudicated PRs complete -- Vision completion or Production-stable package status -- That FormalPR-Holdout results generalize to these consumers (holdout is a separate program) -- That `v1.2.1` consumer green runs validate typed-control-plane `main` +# Consumer Validation Checklist + +Scaffolding and live pointers for independent consumer repositories validating OVK. +Completing this checklist for one repo does **not** satisfy the multi-repo production +exit criterion (30 human-adjudicated PRs per independent consumer). + +## Live independent consumers + +| Repository | Stack | Current pin | Target pin (Sprint 9) | +|---|---|---|---| +| [fraware/ovk-consumer-fastapi-terraform](https://github.com/fraware/ovk-consumer-fastapi-terraform) | FastAPI + Terraform | `v1.2.1` | immutable `v1.3.0-rc.1` (or audited commit) | +| [fraware/ovk-consumer-express-actions](https://github.com/fraware/ovk-consumer-express-actions) | Express + GitHub Actions | `v1.2.1` | immutable `v1.3.0-rc.1` (or audited commit) | + +`v1.2.1` validates the **pre-control-plane** signed release only. Typed control-plane +commits must not inherit that consumer evidence. Both consumers use +`scripts/assert_ovk_pin.py` to fail on pin drift. + +## Immutable pin requirements + +Consumers must pin an **immutable** OVK commit SHA or release tag. + +```yaml +env: + OVK_PACKAGE_VERSION: "1.3.0-rc.1" # after rc.1 cut; until then keep 1.2.1 + OVK_ACTION_REF: "v1.3.0-rc.1" +steps: + - uses: fraware/open-verification-kernel@v1.3.0-rc.1 +``` + +In-repo template: [templates/consumer_validation.workflow.yml](templates/consumer_validation.workflow.yml). + +Forbidden: + +- `uses: ./` +- `uses: fraware/open-verification-kernel@main` +- floating refs without a tag or full commit SHA + +## Maintainer steps after `v1.3.0-rc.1` exists (do not push from this workspace alone) + +For each consumer repository: + +1. Open a pin PR that bumps Action `uses:` and `OVK_PACKAGE_VERSION` to the immutable rc.1 tag (or full SHA). +2. Merge the pin PR (or push to a validation branch) so workflows can see the new pin. +3. Dispatch validation: + ```bash + gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-fastapi-terraform + gh workflow run "OVK Consumer Validation" --repo fraware/ovk-consumer-express-actions + ``` + (Use the exact workflow name as defined in each consumer.) +4. Await conclusions; download evidence artifacts: + ```bash + gh run download --repo -n -D ./consumer-evidence// + ``` +5. Verify bundles with the OVK release verifier + cosign as applicable for the pin. +6. Exercise a true cross-fork PR path (`docs/FORK_PR.md` in each consumer). +7. Update the pilot ledger: keep `automated_scenario` rows distinct from human adjudications. + +Local clone prep (optional, no push): + +```bash +git clone https://github.com/fraware/ovk-consumer-fastapi-terraform.git +git clone https://github.com/fraware/ovk-consumer-express-actions.git +# Edit workflow pins locally; do not git push until maintainers cut rc.1. +``` + +## Sprint 9 checklist (per consumer) — prepare in this repo; land in consumer repos + +In-repo preparation (this repository): + +- [x] Document rc.1 target pins and provenance correction (this checklist + R2 status) +- [x] Keep human pilot ledgers separate from automated fixtures (see consumer `pilot/ledger.json` policy) +- [x] Template workflow targets `v1.3.0-rc.1` (copy only after tag exists) +- [ ] Cut attributable `v1.3.0-rc.1` tag on verified source (Sprint 10) + +In consumer repositories (requires write access — **blocked from this workspace alone**): + +- [ ] Bump Action pin from `v1.2.1` → `v1.3.0-rc.1` (or audited full SHA) +- [ ] Bump `OVK_PACKAGE_VERSION` / wheel install scripts to match +- [ ] Dispatch validation workflows; await conclusions +- [ ] Download evidence bundles; verify with release verifier + cosign as applicable +- [ ] Exercise true cross-fork PR path (`docs/FORK_PR.md`) +- [ ] Update ledger: automated scenarios remain distinct from human adjudications + +## Checklist (per consumer) — ongoing + +- [x] Workflow copies from `docs/templates/consumer_validation.workflow.yml` (or equivalent) with an immutable pin. +- [x] Automated scenario matrix covers program section 23.1 intents (see consumer README). +- [x] Adjudication rows recorded in a pilot ledger conforming to `schemas/pilot.ledger.schema.json`. +- [ ] Human adjudications reach 30 PRs (entries must not remain `automated_scenario` / `pending` only). +- [ ] True cross-fork PR exercised and ledger-adjudicated (see consumer `docs/FORK_PR.md`). +- [ ] Prefer PyPI once published; until then Release wheel + cosign verify-blob at the **current** pin. + +## What this does not claim + +- Declaring two independent consumer repos with 30 adjudicated PRs complete +- Vision completion or Production-stable package status +- That FormalPR-Holdout results generalize to these consumers (holdout is a separate program) +- That `v1.2.1` consumer green runs validate typed-control-plane `main` diff --git a/docs/CURRENT_RELEASE_STATUS.md b/docs/CURRENT_RELEASE_STATUS.md index 71efc75..5f31c1b 100644 --- a/docs/CURRENT_RELEASE_STATUS.md +++ b/docs/CURRENT_RELEASE_STATUS.md @@ -1,126 +1,139 @@ -# OVK Release Status - -Living adoption dashboard for Open Verification Kernel. - -**Last updated:** 2026-07-23 - -**Release judgment:** **`v1.3.0-rc.1` candidate**. The typed backend control plane post-dates signed `v1.2.1` (`a27d5720f4350c00bca34f71d991c31f5a2f38c7`). Default product path remains shadow/legacy-authoritative; enforced routing is lane-policy opt-in until P0 trust closure. Do not treat current `main` as a re-validation of signed `v1.2.1`. - -Authoritative audit: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Engineering program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). Historical: [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) (superseded for day-to-day status). - -## At a glance - -| Signal | Current state | -|---|---| -| **Package version** | Working tree targets future `v1.3.0-rc.1`; signed immutable tag remains `v1.2.1` only for that tag’s commit | -| **FormalPR-Bench** | Internal curated regression; report `benchmark_source_sha` separately from `verified_source_sha` | -| **Check types** | Five bounded production lanes: self-protection, authorization, infrastructure, CI secrets, deployment | -| **Backend execution** | Typed `BackendControlPlane` + `route_obligation`; five policy-selectable enforced lanes via `adapter_runtime` | -| **Routing** | Enforced under lane policy; default path still shadow/legacy-authoritative until P0 closure | -| **Unit and workflow tests** | Local Sprint 0 baseline recorded below; live GitHub Actions workflow IDs still pending | -| **Package portability** | Local wheel-outside-checkout import smoke passed on working tree (package metadata still `1.2.1` until rc.1 cut) | -| **GitHub Action** | Consumers still live-pin `v1.2.1`; local consumer clones prepared for `v1.3.0-rc.1` (not pushed) | -| **External validation** | In-repository dogfooding + consumer scaffolding; independent pilots incomplete | -| **Sigstore** | Immutable-tag E2E closed for `v1.2.1` only — not attributable to typed control-plane commits | - -OVK is not complete formal verification of arbitrary code. It provides explainable, conservative checks for a bounded set of high-risk changes and emits explicit unknown and human-review outcomes. - -## Source SHA terminology - -| Field | Meaning | When to set | -|---|---|---| -| `benchmark_source_sha` | Commit whose FormalPR-Bench (or badge) artifacts were measured | Any bench/badge run | -| `verified_source_sha` | Commit with a **complete observed required-workflow set** | Only after Sprint 0 / release gates attach live workflow IDs | - -Badge-only or `[skip ci]` commits must set `benchmark_source_sha` and must **not** be labeled `verified_source_sha`. - -## Local Sprint 0 baseline - -Local evidence only. Distinguishes from GitHub Actions workflow IDs (still pending). Working tree HEAD at measurement time: `4b48ab245193e177a6d95e8557332334a9bd2883` (badge `[skip ci]` tip — treat as `benchmark_source_sha`, not verified). - -| Gate | Command | Exit | Timestamp (local) | -|---|---|---|---| -| Focused R2 + enforcement pytest | `python -m pytest tests/test_source_profile_hardening.py tests/test_template_conformance.py tests/test_verified_source.py tests/test_bench_badge.py tests/test_formalpr_holdout_runner.py tests/test_execution_models.py tests/test_cache_worker_control_plane.py tests/test_adapter_isolation_r2_pr8.py tests/test_evidence_v3_r2_pr9.py tests/test_authorization_enforcement.py tests/test_adversarial_control_plane.py tests/test_source_profiles.py -q` | **0** (111 passed) | 2026-07-23T23:41:09-07:00 → 23:42:03 | -| Broader compiler/cache suite | `python -m pytest tests/test_authorization_compilers.py tests/test_infrastructure_compilers.py tests/test_github_actions_trust.py tests/test_remaining_lane_enforcement.py tests/test_self_protection_enforcement.py tests/test_verification_cache.py tests/test_result_cache_semantics.py -q` | **0** (54 passed) | 2026-07-23T23:35:44-07:00 | -| Sprint 6–8 regression | `python -m pytest tests/test_source_profile_hardening.py tests/test_template_conformance.py tests/test_formalpr_holdout_runner.py tests/test_source_profiles.py tests/test_authorization_compilers.py tests/test_infrastructure_compilers.py -q` | **0** (46 passed) | 2026-07-23T23:40:48-07:00 | -| Release preflight (`PYTHONPATH=.`) | `python scripts/release_preflight.py` | **0** | 2026-07-23T23:44:17-07:00 → 23:45:10 | -| Template validation | `python scripts/validate_templates.py` | **0** | 2026-07-23T23:41:09-07:00 | -| Template conformance v2 regenerate | `python scripts/build_template_conformance.py` | **0** (`source_profile_strict_eligible=3`, `executable_advisory=2`, `catalog_only=95`) | 2026-07-23T23:40:57-07:00 | -| Local release smoke | `python scripts/smoke_release_local.py` | **0** | 2026-07-23T23:41:09-07:00 | -| Wheel build + outside-checkout import | `python -m build --wheel` then `pip install … -t $TEMP/ovk-outside-import` and `import ovk` | **0** (`verified_source_sha` correctly `None` outside attested env) | 2026-07-23T23:46:01-07:00 → 23:46:29 | -| Workflow ID collector | `python scripts/collect_workflow_evidence.py --sha --output .verification/workflow-evidence-local.json` | **0** (0 runs on `[skip ci]` tip; `verified_source_sha` left unset) | 2026-07-23T23:46:29-07:00 | - -### Still pending (live GitHub Actions / secrets) - -| Gate | Status | Evidence | -|---|---|---| -| General CI / unit+gates on non-`[skip ci]` SHA | Pending live run | Record run URL when available | -| Native Tier 1 | Pending | — | -| Action dogfood | Pending | — | -| Expanded FormalPR-Bench on release SHA | Pending | Use `benchmark_source_sha` | -| Adversarial release-bundle in Actions | Pending | Local `verify_release_bundle.py` entrypoint present | -| Label-separated holdout live eval | Pending | Needs `HOLDOUT_DOWNLOAD_TOKEN` + `HOLDOUT_ASSET_SHA256` | -| Consumer remotes on rc.1 | Pending push | Local clones prepared under `%TEMP%\ovk-consumer-prep\` (not pushed) | - -## Adoption readiness - -| Mode | Current recommendation | Conditions | -|---|---|---| -| **Local/demo** | Appropriate after current local/CI green | Use shipped examples and inspect assumptions and limits | -| **Advisory Action** | Appropriate for pilots on pinned tags | Prefer `v1.2.1` until rc.1 is attributable; collect FPs/unknowns | -| **Strict required check** | Repository-specific only | Calibrate on real diffs; trusted abstraction sources; protected policy metadata | -| **Production-stable general enforcement** | Not yet | P0 code (PRs 1–9) in working tree; still needs consumers on rc.1, attributable holdout, and Sprint 0 live gates | - -Suggested rollout: local validation → advisory artifacts → advisory check run/comment → calibrated strict lane → protected required check. - -## P0 trust defects (R2 PRs 1–9) — working-tree status - -Code for PRs 1–9 is present in this working tree (attempt identity excludes `duration_ms`; `ovk.cache.v3` / `CachedBackendExecution`; coverage/guarantee fail-closed; fallback v2 blocking terminations; `metadata_trusted` default false; authoritative routing pipeline; worker isolation; `ovk.evidence.v3` material-set binding). Historical defect inventory: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). - -**Still open for attributable release (external / Sprint 0–10):** live non-`[skip ci]` workflow IDs, consumer repo pins on immutable rc.1, label-separated holdout aggregates, and signed publication gates — see checklists below and [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). - -## Sprint 6–10 working-tree progress - -| Sprint | Local status | -|---|---| -| 6 Source profiles | FastAPI AST compiler; Terraform recursive modules; K8s controller reachability; Actions permissions-flow prover; deployment trusted-profile gate | -| 7 Template conformance v2 | Statuses derived from executed profile evidence (`source_profile_strict_eligible=3`; no `externally_calibrated_strict` from local gen) | -| 8 Holdout separation | `digest_holdout_predictions.py` + label-free guards; eval path token-stripped | -| 9 Consumers | In-repo template + checklist for rc.1; local clones pin-prepped (no push) | -| 10 Publication | [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md) + `scripts/collect_workflow_evidence.py` | - -## Maintainer release gates - -Before tagging or publishing **`v1.3.0-rc.1`**: - -- [ ] run all CI and native Tier 1 jobs on a non-`[skip ci]` source commit; -- [ ] confirm wheel smoke from a directory outside the checkout; -- [ ] confirm automatic-diff composite Action dogfood; -- [ ] confirm package version matches the release tag; -- [ ] run full expanded FormalPR-Bench and release preflight; -- [ ] validate a complete release bundle, including evidence-quality semantics; -- [ ] exercise HMAC signing and identity-bound Sigstore signing according to release policy; -- [ ] run the immutable Action or release wheel in both independent consumer repositories at the rc.1 pin; -- [ ] update status with exact `verified_source_sha` and workflow links; -- [ ] confirm P0 trust PRs 1–9 on the exact tag source and record attributable holdout aggregates; -- [ ] keep the package classifier at Beta until independent pilots and P0 closure meet the production gate. - -Promotion to **`v1.3.0`** additionally requires P0 closure + consumer + holdout evidence per [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). Do not re-attribute `v1.2.1` Sigstore evidence to typed-control-plane commits. - -## Related documents - -| Document | Purpose | -|---|---| -| [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | Authoritative R2 deep audit | -| [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md) | Sprint/PR execution program | -| [SOURCE_PROFILE_HARDENING.md](SOURCE_PROFILE_HARDENING.md) | Sprint 6 profile status | -| [HOLDOUT_LABEL_SEPARATION.md](HOLDOUT_LABEL_SEPARATION.md) | Sprint 8 prediction/eval split | -| [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) | Sprint 9 consumer pins | -| [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md) | Sprint 10 publication gate | -| [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) | Historical pre-control-plane audit | -| [STATUS.md](STATUS.md) | Command and lane inventory | -| [BACKENDS.md](BACKENDS.md) | Exact backend execution maturity and guarantee classes | -| [INTEGRATION.md](INTEGRATION.md) | Installation and GitHub Action setup | -| [RELEASE.md](RELEASE.md) | Maintainer release procedure | -| [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) | Independent advisory pilot process | -| [BENCHMARK.md](BENCHMARK.md) | Internal benchmark format and execution | +# OVK Release Status + +Living adoption dashboard for Open Verification Kernel. + +**Last updated:** 2026-07-25 + +**Release judgment:** **`v1.3.0-rc.1` in-repo release candidate**. Package metadata is `1.3.0-rc.1`. The typed backend control plane and adoption-surface program (OVK-PR1–PR9) post-date signed `v1.2.1` (`a27d5720f4350c00bca34f71d991c31f5a2f38c7`). Default product path remains shadow/legacy-authoritative; enforced routing is lane-policy opt-in until attributable publication closes. Do not treat current `main` as a re-validation of signed `v1.2.1`. + +Authoritative audit: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Engineering program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). TCB: [TRUSTED_COMPUTING_BASE.md](TRUSTED_COMPUTING_BASE.md). Historical: [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) (superseded for day-to-day status). + +## At a glance + +| Signal | Current state | +|---|---| +| **Package version** | Working tree / RC metadata: `1.3.0-rc.1` (intended tag `v1.3.0-rc.1`); signed immutable tag remains `v1.2.1` only for that tag’s commit | +| **FormalPR-Bench** | Provenance + partitions + version manifest (`benchmarks/formal_pr_bench/manifest.v1.json`); cite `benchmark_version` separately from `verified_source_sha` | +| **Check types** | Five bounded production lanes: self-protection, authorization, infrastructure, CI secrets, deployment | +| **Backend execution** | Typed `BackendControlPlane` + `route_obligation`; five policy-selectable enforced lanes via `adapter_runtime` | +| **Capability registry** | Every advertised checker in `adapters/*/capability.json`; `stable ⊆` full seven-item conformance | +| **Decision / evidence** | Normative `DecisionState` lattice; integrity envelope with controlling-finding reconstruction | +| **Unit and workflow tests** | In-repo suites green locally; live GitHub Actions workflow IDs still pending on a non-`[skip ci]` SHA | +| **Package portability** | `scripts/verify_rc_install.py` covers Action SHA pins + metadata; `--wheel` builds/imports outside checkout | +| **GitHub Action** | Composite Action SHA-pins third-party deps (PR6); consumers still live-pin `v1.2.1` until rc.1 tag exists | +| **External validation** | Three advisory pilot reports under `docs/pilots/` (≥2 with full workflow reproduction) | +| **GitHub App** | Private alpha under `integrations/github-app/` (not Marketplace) | +| **Sigstore** | Immutable-tag E2E closed for `v1.2.1` only — not attributable to typed-control-plane / RC commits | + +OVK is not complete formal verification of arbitrary code. It provides explainable, conservative checks for a bounded set of high-risk changes and emits explicit unknown and human-review outcomes. + +## Adoption-surface program (OVK-PR1–PR9) + +| PR | Scope | In-repo status | +|---|---|---| +| PR1 | Multi-OS repro baseline + normative capability/template registry | Complete | +| PR2 | DecisionState lattice + truth tables | Complete | +| PR3 | Evidence integrity envelope | Complete | +| PR4 | Adapter conformance matrix; stable ⊆ conformant | Complete | +| PR5 | FormalPR-Bench provenance / partitions / version manifest | Complete | +| PR6 | Action scenario hardening + SHA-pinned third parties | Complete | +| PR7 | GitHub App private alpha | Complete | +| PR8 | Three advisory pilot reports | Complete | +| PR9 | RC cut prep, TCB doc, attributable gates, install verification | **In-repo ready** (live tag/Sigstore pending) | + +Local DoD verifier: `python scripts/verify_rc_dod.py`. Install surface: `python scripts/verify_rc_install.py` (add `--wheel` for outside-checkout import). + +## Source SHA terminology + +| Field | Meaning | When to set | +|---|---|---| +| `benchmark_source_sha` | Commit whose FormalPR-Bench (or badge) artifacts were measured | Any bench/badge run | +| `verified_source_sha` | Commit with a **complete observed required-workflow set** | Only after Sprint 0 / release gates attach live workflow IDs | + +Badge-only or `[skip ci]` commits must set `benchmark_source_sha` and must **not** be labeled `verified_source_sha`. + +## Local Sprint 0 / RC baseline + +Local evidence only. Distinguishes from GitHub Actions workflow IDs (still pending). + +Multi-OS reproducible baselines (OVK-01): see [REPRO_BASELINE.md](REPRO_BASELINE.md) and the [`repro-baseline`](../.github/workflows/repro-baseline.yml) workflow. Records are uploaded by CI (see [baselines/README.md](baselines/README.md)); the directory may be empty until maintainers download or commit matrix artifacts. + +| Gate | Command | Notes | +|---|---|---| +| Release metadata | `python scripts/check_release_metadata.py` | Must equal `1.3.0-rc.1` | +| RC DoD (in-repo) | `python scripts/verify_rc_dod.py` | Program DoD minus live publication | +| RC install (static) | `python scripts/verify_rc_install.py` | Action SHA pins + package metadata | +| RC install (wheel) | `python scripts/verify_rc_install.py --wheel` | Optional; needs `build` | +| TCB freshness | `python scripts/render_tcb_doc.py --check` | Regenerates via `--write` | +| Release preflight | `ovk release-preflight` | Includes RC DoD + install + TCB | + +### Still pending (live GitHub Actions / secrets) — maintainer publication + +| Gate | Status | Evidence | +|---|---|---| +| General CI / unit+gates on non-`[skip ci]` SHA | Pending live run | Record run URL → `verified_source_sha` | +| Native Tier 1 | Pending | — | +| Action dogfood | Pending | — | +| Expanded FormalPR-Bench on release SHA | Pending | Use `benchmark_source_sha` | +| Adversarial release-bundle in Actions | Pending | Local `verify_release_bundle.py` entrypoint present | +| Label-separated holdout live eval | Pending | Needs `HOLDOUT_DOWNLOAD_TOKEN` + `HOLDOUT_ASSET_SHA256` | +| Consumer remotes on `v1.3.0-rc.1` | Pending tag + push | Template targets rc.1; live remotes still on `v1.2.1` | +| Signed tag + Publish/Sigstore for rc.1 | Pending | Do not re-attribute `v1.2.1` cosign evidence | + +## Adoption readiness + +| Mode | Current recommendation | Conditions | +|---|---|---| +| **Local/demo** | Appropriate after current local/CI green | Use shipped examples and inspect assumptions and limits | +| **Advisory Action** | Appropriate for pilots on pinned tags | Prefer signed `v1.2.1` until `v1.3.0-rc.1` is attributable; collect FPs/unknowns | +| **Strict required check** | Repository-specific only | Calibrate on real diffs; trusted abstraction sources; protected policy metadata | +| **Production-stable general enforcement** | Not yet | Needs attributable rc.1 (or later), consumer pins, and Sprint 0 live gates | + +Suggested rollout: local validation → advisory artifacts → advisory check run/comment → calibrated strict lane → protected required check. + +## P0 trust defects (R2 PRs 1–9) — working-tree status + +Code for R2 P0 PRs 1–9 is present in this working tree. Historical defect inventory: [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md). Program: [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md). + +**Still open for attributable release:** live non-`[skip ci]` workflow IDs, consumer repo pins on immutable rc.1, label-separated holdout aggregates, and signed publication gates — see [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). + +## Maintainer release gates + +Before tagging or publishing **`v1.3.0-rc.1`**: + +- [x] package version / `__version__` / release metadata align on `1.3.0-rc.1` +- [x] TCB documented ([TRUSTED_COMPUTING_BASE.md](TRUSTED_COMPUTING_BASE.md)) +- [x] in-repo RC DoD (`scripts/verify_rc_dod.py`) and Action/pip install surface (`scripts/verify_rc_install.py`) +- [ ] run all CI and native Tier 1 jobs on a non-`[skip ci]` source commit; +- [ ] confirm wheel smoke from a directory outside the checkout on that SHA; +- [ ] confirm automatic-diff composite Action dogfood; +- [ ] confirm package version matches the release tag (`v1.3.0-rc.1`); +- [ ] run full expanded FormalPR-Bench and release preflight; +- [ ] validate a complete release bundle, including evidence-quality semantics; +- [ ] exercise HMAC signing and identity-bound Sigstore signing according to release policy; +- [ ] run the immutable Action or release wheel in both independent consumer repositories at the rc.1 pin; +- [ ] update status with exact `verified_source_sha` and workflow links; +- [ ] keep the package classifier at Beta until independent pilots and P0 closure meet the production gate. + +Promotion to **`v1.3.0`** additionally requires P0 closure + consumer + holdout evidence per [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). Do not re-attribute `v1.2.1` Sigstore evidence to typed-control-plane commits. + +## Related documents + +| Document | Purpose | +|---|---| +| [TRUSTED_COMPUTING_BASE.md](TRUSTED_COMPUTING_BASE.md) | Reviewer TCB inventory (registry + Action/App) | +| [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md) | Sprint 10 / RC publication gate | +| [DEEP_AUDIT_2026-07-23_R2.md](DEEP_AUDIT_2026-07-23_R2.md) | Authoritative R2 deep audit | +| [ENGINEERING_PROGRAM_2026-07-23_R2.md](ENGINEERING_PROGRAM_2026-07-23_R2.md) | Sprint/PR execution program | +| [SOURCE_PROFILE_HARDENING.md](SOURCE_PROFILE_HARDENING.md) | Sprint 6 profile status | +| [HOLDOUT_LABEL_SEPARATION.md](HOLDOUT_LABEL_SEPARATION.md) | Sprint 8 prediction/eval split | +| [CONSUMER_VALIDATION_CHECKLIST.md](CONSUMER_VALIDATION_CHECKLIST.md) | Sprint 9 consumer pins | +| [VISION_AUDIT_2026-07-22.md](VISION_AUDIT_2026-07-22.md) | Historical pre-control-plane audit | +| [STATUS.md](STATUS.md) | Command and lane inventory | +| [BACKENDS.md](BACKENDS.md) | Exact backend execution maturity and guarantee classes | +| [REPRO_BASELINE.md](REPRO_BASELINE.md) | Multi-OS reproducible baseline harness (OVK-01) | +| [INTEGRATION.md](INTEGRATION.md) | Installation and GitHub Action setup | +| [RELEASE.md](RELEASE.md) | Maintainer release procedure | +| [EXTERNAL_PILOT_PLAYBOOK.md](EXTERNAL_PILOT_PLAYBOOK.md) | Independent advisory pilot process | +| [pilots/README.md](pilots/README.md) | Published OVK-PR8 advisory pilot reports | +| [BENCHMARK.md](BENCHMARK.md) | Internal benchmark format and execution | diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 8ce747d..a1c19a7 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -4,9 +4,9 @@ Maintainer guide for shipping Open Verification Kernel. Current readiness: [CURR ## Current release candidate -Package version: `1.2.1`. +Package version: `1.3.0-rc.1`. -Release judgment: **release candidate**. The five bounded evidence lanes, artifact chain, CLI, MCP surface, and composite Action are implemented. Backend routing remains advisory, external tagged-consumer validation remains pending, and the current source commit must pass all gates below before publication. +Release judgment: **in-repo release candidate** for attributable tag `v1.3.0-rc.1`. The five bounded evidence lanes, artifact chain, CLI, MCP surface, composite Action (SHA-pinned third parties), capability registry, decision lattice, evidence integrity, conformance matrix, FormalPR-Bench provenance, App alpha, pilots, and TCB doc are implemented. Live workflow IDs, signed tag/Sigstore, and consumer remotes on the RC pin remain maintainer gates — see [CURRENT_RELEASE_STATUS.md](CURRENT_RELEASE_STATUS.md) and [ATTRIBUTABLE_PUBLICATION.md](ATTRIBUTABLE_PUBLICATION.md). ## Known limitations @@ -29,6 +29,9 @@ ruff check ovk tests benchmarks scripts python scripts/check_release_metadata.py python scripts/validate_templates.py python scripts/validate_capabilities.py +python scripts/render_tcb_doc.py --check +python scripts/verify_rc_dod.py +python scripts/verify_rc_install.py ovk release-preflight ovk bench --expanded --leaderboard .verification/formal-pr-bench-leaderboard.json ovk pilot @@ -53,25 +56,26 @@ The Publish workflow rejects a GitHub release whose tag does not equal `ovk.__ve Before tagging: -- [ ] `pyproject.toml`, `ovk.__version__`, and `ovk/core/release_metadata.py` agree; +- [ ] `pyproject.toml`, `ovk.__version__`, and `ovk/core/release_metadata.py` agree on `1.3.0-rc.1`; - [ ] `SUPPORTED_COMMANDS` matches the Typer command surface; - [ ] consumer examples reference the intended immutable release tag or commit; - [ ] release notes describe actual backend execution semantics from [BACKENDS.md](BACKENDS.md); - [ ] package classifier remains Beta until the production gate in the vision audit is met; -- [ ] benchmark and adoption summaries are regenerated from the release source. +- [ ] benchmark and adoption summaries are regenerated from the release source; +- [ ] [TRUSTED_COMPUTING_BASE.md](TRUSTED_COMPUTING_BASE.md) is fresh (`python scripts/render_tcb_doc.py --check`). Tag and create the release only after the source gates are attributable: ```bash -git tag -s v1.2.1 -git push origin v1.2.1 -gh release create v1.2.1 \ +git tag -s v1.3.0-rc.1 +git push origin v1.3.0-rc.1 +gh release create v1.3.0-rc.1 \ --verify-tag \ - --title "OVK v1.2.1" \ - --notes-file docs/RELEASE_NOTES_v1.2.1.md + --title "OVK v1.3.0-rc.1" \ + --notes-file docs/RELEASE_NOTES_v1.3.0-rc.1.md ``` -Note: `v1.2.0` already exists as an immutable tag on an earlier commit without the protected Publish Sigstore path. Do not move that tag. +Note: `v1.2.1` and earlier tags remain immutable historical releases. Do not move those tags. Do not re-attribute their Sigstore evidence to this RC. ## Package publication @@ -127,20 +131,26 @@ https://token.actions.githubusercontent.com https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/vX.Y.Z ``` -Example for v1.2.1: +Example for **v1.3.0-rc.1** (after the attributable tag exists): ```bash export OVK_SIGSTORE_SIGNING=1 -export OVK_COSIGN_IDENTITY='https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.1' +export OVK_COSIGN_IDENTITY='https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.3.0-rc.1' export OVK_COSIGN_ISSUER='https://token.actions.githubusercontent.com' ``` +Historical example for signed `v1.2.1` (do not re-attribute to this RC): + +```bash +export OVK_COSIGN_IDENTITY='https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.1' +``` + Consumer verification of a retained bundle: ```bash cosign verify-blob \ --bundle path/to/artifact.cosign.bundle.json \ - --certificate-identity "https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.2.1" \ + --certificate-identity "https://github.com/fraware/open-verification-kernel/.github/workflows/publish.yml@refs/tags/v1.3.0-rc.1" \ --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \ path/to/artifact.whl ``` diff --git a/docs/TRUSTED_COMPUTING_BASE.md b/docs/TRUSTED_COMPUTING_BASE.md new file mode 100644 index 0000000..118b5a4 --- /dev/null +++ b/docs/TRUSTED_COMPUTING_BASE.md @@ -0,0 +1,136 @@ +# Trusted Computing Base + +Independent-reviewer TCB inventory for Open Verification Kernel (OVK-PR9). +Derived from the normative capability registry (`trusted_components`), composite Action release pins, and the private GitHub App alpha surface. + + +Generated for package version **`1.3.0-rc.1`** by `scripts/render_tcb_doc.py`. Do not hand-edit this section; regenerate with `python scripts/render_tcb_doc.py --write`. + +## Package identity + +| Field | Value | +|---|---| +| Package version | `1.3.0-rc.1` | +| Intended immutable tag | `v1.3.0-rc.1` | +| Public integration path | Composite Action (`action.yml`) + `pip` wheel | +| Private alpha path | `integrations/github-app/` (not Marketplace) | + +## Composite Action surface + +Third-party actions in release paths must be immutable SHA pins (enforced by `scripts/pin_action_shas.py`): + +| File | `uses:` pin | Note | +|---|---|---| +| `action.yml` | `actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830` | v4.3.0 | +| `action.yml` | `actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02` | v4.6.2 | +| `.github/workflows/publish.yml` | `actions/checkout@11d5960a326750d5838078e36cf38b85af677262` | v4.4.0 | +| `.github/workflows/publish.yml` | `actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065` | v5.6.0 | +| `.github/workflows/publish.yml` | `actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02` | v4.6.2 | +| `.github/workflows/publish.yml` | `actions/checkout@11d5960a326750d5838078e36cf38b85af677262` | v4.4.0 | +| `.github/workflows/publish.yml` | `actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065` | v5.6.0 | +| `.github/workflows/publish.yml` | `actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02` | v4.6.2 | +| `.github/workflows/publish.yml` | `actions/checkout@11d5960a326750d5838078e36cf38b85af677262` | v4.4.0 | +| `.github/workflows/publish.yml` | `actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065` | v5.6.0 | +| `.github/workflows/publish.yml` | `actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093` | v4.3.0 | +| `.github/workflows/publish.yml` | `sigstore/cosign-installer@d7d6bc7722e3daa8354c50bcb52f4837da5e9b6a` | v3.8.1 | +| `.github/workflows/publish.yml` | `actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02` | v4.6.2 | +| `.github/workflows/publish.yml` | `actions/checkout@11d5960a326750d5838078e36cf38b85af677262` | v4.4.0 | +| `.github/workflows/publish.yml` | `actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093` | v4.3.0 | +| `.github/workflows/publish.yml` | `pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247` | v1.14.1 (release/v1) | + +Floating third-party refs in release paths: **0** (must be zero for RC). + +Action install trust boundary: + +- Runner installs `open-verification-kernel==$OVK_PACKAGE_VERSION` when set, + otherwise installs from the Action checkout after `scripts/sync_package_data.py`. +- Consumer repositories must pin the Action to an immutable tag or full commit SHA. +- Check-run emission uses a stable `external_id` bound to repository + head SHA. + +## GitHub App surface (private alpha) + +The App is **not** part of the default public TCB for adopters who only use the composite Action. Operators who deploy the App additionally trust: + +| Control | Implementation | +|---|---| +| Webhook signature verification | HMAC-SHA256 (`X-Hub-Signature-256`); missing/invalid rejected | +| Replay protection | `X-OVK-Timestamp` skew (±300s) + `X-GitHub-Delivery` dedupe store | +| Installation isolation | `{data}/installations/{id}/` partitions for credentials, cache, data | +| Least-privilege permissions | `manifest.json`: `checks:write`, `contents:read`, `pull_requests:read`, `metadata:read` | +| Short-lived installation tokens | On-demand exchange; App JWT ≤10m; installation token ≤1h; no PATs | +| Redacted logs | Paths and secrets scrubbed via `RedactingFilter` | +| Idempotent Check Run updates | `external_id` = `ovk:{repo}:{head_sha}` (same as Action / PR6) | +| No cross-repository cache reuse | Cache keys require `installation_id` + `repo_id` | +| Uninstall cleanup | `installation.deleted` deletes the partition | +| Retention policy | [RETENTION.md](RETENTION.md) | + +App code and retention policy: [`integrations/github-app/`](../integrations/github-app/). + +## Capability registry trusted components + +Every advertised public checker contributes the `trusted_components` list from its `adapters/*/capability.json` entry (after release-status honesty). + +| Checker | release_status | Trusted components | +|---|---|---| +| `opa` | `preview` | opa binary; selected Rego policy templates; input extraction / compiler | +| `z3` | `preview` | z3 solver; neutral obligation compiler; encoded abstraction | +| `cbmc` | `preview` | cbmc binary; harness generator or supplied harness; bound configuration | +| `cedar` | `experimental` | deterministic Cedar-shaped evaluator; optional cedar CLI for version probe | +| `tla+` | `experimental` | deterministic state-machine contract evaluator | +| `kani` | `experimental` | deterministic Rust-harness contract evaluator | +| `dafny` | `experimental` | deterministic proof-obligation contract evaluator | +| `verus` | `experimental` | deterministic verified-Rust contract evaluator | +| `lean` | `experimental` | deterministic theorem-obligation contract evaluator | +| `alloy` | `experimental` | deterministic relational-model contract evaluator | +| `lane-self-protection` | `experimental` | self-protection lane evaluator; optional OPA native path | +| `lane-authorization` | `experimental` | authorization lane evaluator; optional Z3 solver | +| `lane-infrastructure` | `experimental` | infrastructure lane evaluator | +| `lane-ci-secrets` | `experimental` | ci_secrets lane evaluator | +| `lane-deployment` | `experimental` | deployment lane evaluator | + +### Aggregate trusted-component vocabulary + +Union of registry `trusted_components` strings (deduplicated, order of first appearance): + +- opa binary +- selected Rego policy templates +- input extraction / compiler +- z3 solver +- neutral obligation compiler +- encoded abstraction +- cbmc binary +- harness generator or supplied harness +- bound configuration +- deterministic Cedar-shaped evaluator +- optional cedar CLI for version probe +- deterministic state-machine contract evaluator +- deterministic Rust-harness contract evaluator +- deterministic proof-obligation contract evaluator +- deterministic verified-Rust contract evaluator +- deterministic theorem-obligation contract evaluator +- deterministic relational-model contract evaluator +- self-protection lane evaluator +- optional OPA native path +- authorization lane evaluator +- optional Z3 solver +- infrastructure lane evaluator +- ci_secrets lane evaluator +- deployment lane evaluator + +## Kernel control-plane trust assumptions + +Beyond per-adapter tools, an independent reviewer should treat these as in-TCB for strict-mode decisions: + +- Decision lattice aggregation (`ovk.core.decision`) and exit-code mapping +- Evidence integrity envelope / digests (`ovk.core.evidence_integrity`) +- Capability + conformance honesty gates (`release_status=stable` requires full suite) +- Trusted policy / metadata provenance loading for self-protection and deployment lanes +- FormalPR-Bench version manifest digests when citing benchmark scores + +## Out of TCB (explicit non-claims) + +- Unavailable optional native binaries (must not promote to allow in strict mode) +- Floating `@main` Action pins or unverified PyPI builds without matching tag evidence +- Human pilot ledgers and advisory pilot fixture metrics (evidence for adoption, not TCB) +- Re-attributing signed `v1.2.1` Sigstore evidence to this RC source tree + diff --git a/docs/templates/consumer_validation.workflow.yml b/docs/templates/consumer_validation.workflow.yml index 3bdb389..a9c2b78 100644 --- a/docs/templates/consumer_validation.workflow.yml +++ b/docs/templates/consumer_validation.workflow.yml @@ -20,8 +20,8 @@ permissions: checks: write env: - # After rc.1 cut: "1.3.0rc1". Until then keep "1.2.1". - OVK_PACKAGE_VERSION: "1.3.0rc1" + # Package version must match ovk.__version__ / tag without leading v. + OVK_PACKAGE_VERSION: "1.3.0-rc.1" OVK_ACTION_REF: "v1.3.0-rc.1" jobs: diff --git a/scripts/render_tcb_doc.py b/scripts/render_tcb_doc.py new file mode 100644 index 0000000..700d04d --- /dev/null +++ b/scripts/render_tcb_doc.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python +"""Generate docs/TRUSTED_COMPUTING_BASE.md from registry trusted_components + surfaces. + +Independent reviewers use the generated document to identify the OVK TCB +(OVK-PR9 / program DoD). Prefer regenerating over hand-editing the markdown body. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from ovk.core.adapter_conformance import ADVERTISED_ADAPTER_IDS, apply_release_status_honesty # noqa: E402 +from ovk.core.capabilities import CapabilityRegistry # noqa: E402 +from ovk.core.release_metadata import OVK_RELEASE_CANDIDATE # noqa: E402 +from scripts.pin_action_shas import DEFAULT_PATHS as ACTION_PIN_PATHS # noqa: E402 +from scripts.pin_action_shas import floating_uses_in_file, is_local_action, is_sha_pinned # noqa: E402 + +BEGIN = "" +END = "" + +USES_COMMENT_RE = re.compile( + r"""^\s*(?:-\s*)?uses:\s*['"]?(?P[^'"\s#]+)['"]?\s*(?:#\s*(?P