From 1bb4f04825484a4944025b55b8f27393a548cc57 Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 2 Sep 2026 16:42:27 +0000 Subject: [PATCH 1/8] feat(verify): add conditional metamorphic relations and a parameter-addressing contract Three no-golden-answer relations -- input permutation invariance, duplicate resistance, and positive-buffer monotonicity -- each declare the precondition that makes them valid, execute the project's own canonical entrypoint in an isolated variant workspace, and report not_testable with the reason when the precondition does not hold on the data. Unknown relations, non-growing variants, and duplicate resistance declared for a count or sum are declaration failures rather than skipped checks. runtime.implementation.parameters (openmapstack-parameters/v1) addresses one knob of the pipeline through an argument or environment binding; verify fails when the declared canonical value drifts from the bound step. The fixture generator gains the parameter plus three pipeline-logic break modes that its copied pipeline.py reproduces on rerun; cases 015 and 923-925 prove the healthy, deliberate-defect, and isolation paths. Refs #13 (B4). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01TMMy6A73zi7yGjK3wHBhju --- README.md | 10 + docs/verify-applicability.md | 7 +- evals/COVERAGE.md | 8 +- .../015-metamorphic-relations/expected.yaml | 29 + .../923-order-dependent-output/expected.yaml | 27 + .../expected.yaml | 27 + .../expected.yaml | 27 + evals/fixtures/reference_pipeline/gen.py | 96 ++- evals/run.py | 8 + openmapstack/checks/metamorphic.py | 74 ++ openmapstack/checks/project.py | 26 + openmapstack/cli.py | 6 + openmapstack/metamorphic.py | 661 ++++++++++++++++++ openmapstack/parameters.py | 190 +++++ openmapstack/rerun.py | 92 ++- openmapstack/schemas/project-v1.schema.json | 97 +++ openmapstack/verify.py | 36 +- references/project-spec.md | 82 ++- templates/project.yaml | 19 + tests/test_metamorphic.py | 452 ++++++++++++ 20 files changed, 1930 insertions(+), 44 deletions(-) create mode 100644 evals/cases/015-metamorphic-relations/expected.yaml create mode 100644 evals/cases/923-order-dependent-output/expected.yaml create mode 100644 evals/cases/924-inverted-distance-predicate/expected.yaml create mode 100644 evals/cases/925-duplicate-sensitive-candidates/expected.yaml create mode 100644 openmapstack/checks/metamorphic.py create mode 100644 openmapstack/metamorphic.py create mode 100644 openmapstack/parameters.py create mode 100644 tests/test_metamorphic.py diff --git a/README.md b/README.md index a083a00..5ef25a8 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,7 @@ PyQGIS is available. ```bash openmapstack verify path/to/project.yaml openmapstack verify path/to/project.yaml --rerun # + rebuild from source and compare +openmapstack verify path/to/project.yaml --metamorphic # + run declared no-oracle relations openmapstack verify path/to/project.yaml --json --output validation/verify-report.json openmapstack verify path/to/project.yaml --strict # warnings and not-testable also return 1 ``` @@ -217,6 +218,15 @@ inputs, or a retained local evidence file invalidates the attestation and returns it to warning status. See [the project contract](references/project-spec.md#26-validation). +Where no golden answer exists at all, `validation.metamorphic[]` declares +relations that must hold under a controlled perturbation: shuffle a source and +the result must not change, duplicate every feature and a keyed set must not +change, widen an inclusion buffer and no candidate may disappear. Each relation +states the precondition that makes it valid, is executed by +`verify --metamorphic` in an isolated copy against the project's own pipeline, +and reports `not_testable` with the reason when the precondition does not hold +on the actual data. See [the project contract](references/project-spec.md#26-validation). + `validate` checks manifest structure, source retrieval/version/licensing data, CRS declarations, processing graph resolution, override provenance and files, output existence, validation-report parity/status propagation, override diff --git a/docs/verify-applicability.md b/docs/verify-applicability.md index 227cc52..c27fb5f 100644 --- a/docs/verify-applicability.md +++ b/docs/verify-applicability.md @@ -32,6 +32,7 @@ checker where applicable. | `project.one_canonical_pipeline` | always | one executable entrypoint is declared | missing declaration is a failure | 005 | | `project.assumptions_have_rationale` | always | assumptions are explicit and reasoned | none | 001, 070–073 | | `project.status_agrees_with_validation_report` | always | project status does not launder report state | missing report is `not_testable` | 004 | +| `project.parameters_match_steps` | `runtime.implementation.parameters` is declared | parameters are well-formed and agree with the processing steps they are bound to | none; malformed or drifting declarations fail | direct parameter-contract tests; case 015 | | `project.declared_files_exist` | at least one output path is declared | all declared outputs exist | none; missing output fails | 001, 909 | | `provenance.every_source_has_provider_and_access` | always | sources identify provider, method, and retrieval time | none | broad contract suite | | `provenance.every_source_pinned` | always | sources carry a non-`latest` version identity | current checker does not yet recognise warehouse snapshot classes | 906 | @@ -44,6 +45,8 @@ checker where applicable. | `validation.warning_or_failed_propagates_to_status` | always | report aggregate reflects non-passing checks | missing report is `not_testable` | 004, 071, 072 | | `validation.run_record_matches` | always | report, manifest, inventories, hashes, and files agree | missing report is `not_testable`; missing or false records fail | 001; mutations 901–921 | | `expectation.` | once per `validation.expectations[]` entry | an independently attested known answer agrees with the produced artifact | unverified, incomplete, changed, or input-stale attestations warn without executing; allowlisted checks need DuckDB Spatial | direct attestation, staleness, path-safety, and checker-failure tests | +| `metamorphic.declarations_valid` | `validation.metamorphic` is declared | every relation parses, names an implemented relation, and addresses declared outputs | none; structural only, nothing executes | direct declaration tests; cases 015, 923–925 | +| `metamorphic.` | `--metamorphic` and the relation is declared | the declared invariant holds under the relation's controlled perturbation | executes the canonical entrypoint in an isolated copy; unmet data preconditions, unsupported source/output formats, DuckDB absent for Parquet, timeouts, and oversize sources are `not_testable`; a crashing variant or one that mutates the project's inputs fails | 015 (holds); 923 `permutation_changed_output`, 924 `monotonicity_violated`, 925 `duplicates_changed_output` | | `geodata.crs_not_used_for_metrics` | always | manifest does not declare geographic CRS for metric work | none | 001, 007, 902, 914 | | `geodata.geometry_all_valid` | once per declared readable geodata output | every geometry in the artifact is valid | DuckDB Spatial; unsupported formats and unreadable artifacts are `not_testable`, missing files fail separately | 001, 011, 918 | | `geodata.dataset_crs_is` | once per readable geodata output with declared EPSG | artifact CRS agrees with the manifest | DuckDB Spatial; absent EPSG/addressing and unreadable metadata are `not_testable` | 001, 007, 914, 919 | @@ -80,7 +83,9 @@ address them without guessing: the `verify` plan; Playwright being installed does not imply they ran; - validation evidence recomputation requires machine-readable declarations mapping report fields to artifacts and metrics; -- clean-rerun checks run only when the user supplies `--rerun`. +- clean-rerun checks run only when the user supplies `--rerun`, and declared + metamorphic relations execute only with `--metamorphic`; both rerun the + pipeline, which the static plan must never do implicitly. These omissions are reachability gaps, not implicit passes. They should enter the plan only with a versioned addressing contract and their own mutation diff --git a/evals/COVERAGE.md b/evals/COVERAGE.md index 97bf2ed..0013b85 100644 --- a/evals/COVERAGE.md +++ b/evals/COVERAGE.md @@ -100,13 +100,15 @@ Legend: ✅ covered · ⚠️ partially covered · ❌ not covered (tracked belo |---|---| | Reprojection invariance (4326-stored input must recover the surveyed 3301 line) | 007 (round-trip verified to sub-mm; golden distances/areas baked) | | Duplicate-input resistance | 008 (duplicated poi-z must not inflate the join) | -| Input-order permutation | ⚠️ join ordered by pair_id; an explicit permuted-input run is not yet a separate case | -| Monotonic buffer behaviour | ❌ not yet exercised | +| Input-order permutation | 015 `parcel-order` (source shuffled, outputs must be equal); mutation 923 `permutation_changed_output` | +| Monotonic buffer behaviour | 015 `road-distance-monotonic` (threshold tripled through the declared parameter, baseline keys must survive); mutation 924 `monotonicity_violated` | +| Duplicate-input resistance (declared) | 015 `parcel-duplicates` (every parcel appended once more, outputs must be equal); mutation 925 `duplicates_changed_output` | +| Invalid-precondition refusal | unit tests: count/sum semantics, missing tie-break, non-growing variant, unsupported format, source already duplicated, oversize source | ## Known gaps (tracked) 1. **PostGIS / warehouse canary** — needs a live service; candidate design is a scheduled-container case in the visual/benchmark workflow. 2. **True MultiPolygon inputs** and **nearest-neighbour tie-breaking**. -3. **Explicit input-permutation and monotonic-buffer metamorphic cases**. +3. **CRS round-trip, subset-additivity, and area-scale metamorphic relations** (the framework rejects them as undeclared rather than guessing). 4. **Raster analysis** (the suite is vector-only so far). diff --git a/evals/cases/015-metamorphic-relations/expected.yaml b/evals/cases/015-metamorphic-relations/expected.yaml new file mode 100644 index 0000000..5d78fa1 --- /dev/null +++ b/evals/cases/015-metamorphic-relations/expected.yaml @@ -0,0 +1,29 @@ +id: 015-metamorphic-relations +case_type: positive +modes: [fixture] +score_types: + fixture: contract_ci +project_dir: project +hard_gate: true +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +# No golden answer is consulted here. Each relation reruns the project's own +# pipeline on a perturbed copy of its inputs (or with its declared parameter +# turned) and tests the invariant the manifest declares -- and declares the +# precondition that makes the invariant valid for this analysis. +assertions: + - assert: project.conforms_to_schema + - assert: project.parameters_match_steps + - assert: metamorphic.declarations_valid + - assert: metamorphic.relation_holds + args: { id: parcel-order } + - assert: metamorphic.relation_holds + args: { id: parcel-duplicates } + - assert: metamorphic.relation_holds + args: { id: road-distance-monotonic } + - assert: validation.run_record_matches diff --git a/evals/cases/923-order-dependent-output/expected.yaml b/evals/cases/923-order-dependent-output/expected.yaml new file mode 100644 index 0000000..261edbc --- /dev/null +++ b/evals/cases/923-order-dependent-output/expected.yaml @@ -0,0 +1,27 @@ +id: 923-order-dependent-output +case_type: mutation +modes: [fixture] +score_types: { fixture: mutation_tests } +project_dir: project +hard_gate: true +mutation: + control_generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir} --break=order_dependent" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +assertions: + - assert: project.conforms_to_schema + - assert: validation.run_record_matches + - assert: metamorphic.declarations_valid + - assert: metamorphic.relation_holds + args: { id: parcel-order } + expect: failed + expect_code: permutation_changed_output + - assert: metamorphic.relation_holds + args: { id: parcel-duplicates } + - assert: metamorphic.relation_holds + args: { id: road-distance-monotonic } diff --git a/evals/cases/924-inverted-distance-predicate/expected.yaml b/evals/cases/924-inverted-distance-predicate/expected.yaml new file mode 100644 index 0000000..22e8ebf --- /dev/null +++ b/evals/cases/924-inverted-distance-predicate/expected.yaml @@ -0,0 +1,27 @@ +id: 924-inverted-distance-predicate +case_type: mutation +modes: [fixture] +score_types: { fixture: mutation_tests } +project_dir: project +hard_gate: true +mutation: + control_generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir} --break=distance_inverted" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +assertions: + - assert: project.conforms_to_schema + - assert: validation.run_record_matches + - assert: metamorphic.declarations_valid + - assert: metamorphic.relation_holds + args: { id: parcel-order } + - assert: metamorphic.relation_holds + args: { id: parcel-duplicates } + - assert: metamorphic.relation_holds + args: { id: road-distance-monotonic } + expect: failed + expect_code: monotonicity_violated diff --git a/evals/cases/925-duplicate-sensitive-candidates/expected.yaml b/evals/cases/925-duplicate-sensitive-candidates/expected.yaml new file mode 100644 index 0000000..a51d5d6 --- /dev/null +++ b/evals/cases/925-duplicate-sensitive-candidates/expected.yaml @@ -0,0 +1,27 @@ +id: 925-duplicate-sensitive-candidates +case_type: mutation +modes: [fixture] +score_types: { fixture: mutation_tests } +project_dir: project +hard_gate: true +mutation: + control_generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir} --break=duplicate_sensitive" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +assertions: + - assert: project.conforms_to_schema + - assert: validation.run_record_matches + - assert: metamorphic.declarations_valid + - assert: metamorphic.relation_holds + args: { id: parcel-order } + - assert: metamorphic.relation_holds + args: { id: parcel-duplicates } + expect: failed + expect_code: duplicates_changed_output + - assert: metamorphic.relation_holds + args: { id: road-distance-monotonic } diff --git a/evals/fixtures/reference_pipeline/gen.py b/evals/fixtures/reference_pipeline/gen.py index f8d2d37..61169af 100755 --- a/evals/fixtures/reference_pipeline/gen.py +++ b/evals/fixtures/reference_pipeline/gen.py @@ -26,6 +26,13 @@ qgis_incomplete_crs QGIS layers carry authid-only invalid CRS blocks incomplete_pagination roads source reports numberMatched > returned mutated_source pipeline rewrites its own copied "immutable" source file + order_dependent candidate rows carry their input file position (order-dependent output) + distance_inverted road-distance predicate inverted (>= instead of <=) + duplicate_sensitive duplicated source parcels produce duplicated candidates + +--road-distance-m overrides the canonical 2000 m threshold. It is the +project's one declared runtime parameter (runtime.implementation.parameters), +which is what lets a metamorphic relation vary it without editing the code. """ from __future__ import annotations @@ -99,6 +106,15 @@ def _inventory(root: Path, paths: list[Path]) -> list[dict[str, str]]: ] +ROAD_DISTANCE_CANONICAL_M = 2000 + +# Break modes that live in the pipeline's logic rather than in the generated +# bookkeeping. A generated project's copied pipeline.py reproduces these on +# rerun -- as a genuinely defective pipeline would -- so the metamorphic +# relations, which rerun the pipeline on perturbed input, can observe them. +PIPELINE_LOGIC_BREAK_MODES = {"order_dependent", "distance_inverted", "duplicate_sensitive"} + + def build( output_dir: Path, apply_override: bool, @@ -106,6 +122,7 @@ def build( with_scenario_road: bool = False, uncertain_completeness: bool = False, source_dir: Path | None = None, + road_distance_m: float = ROAD_DISTANCE_CANONICAL_M, ) -> None: output_dir.mkdir(parents=True, exist_ok=True) (output_dir / "data" / "source").mkdir(parents=True, exist_ok=True) @@ -146,9 +163,12 @@ def build( analysis_crs = "EPSG:4326" if break_mode == "wrong_crs" else "EPSG:3301" con.execute(f"CREATE TABLE parcels_raw AS SELECT * FROM ST_Read('{parcels_path}')") + # ``input_rank`` is the parcel's position in the source file. A correct + # pipeline never lets it reach the output; the two order/duplicate break + # modes below do, which is exactly what the metamorphic relations catch. con.execute( "CREATE TABLE large_parcels AS SELECT cadastral_id, land_use, municipality, geom, " - "ST_Area(geom) AS area_m2 FROM parcels_raw " + "ST_Area(geom) AS area_m2, row_number() OVER () AS input_rank FROM parcels_raw " "WHERE ST_Area(geom) >= 8000 AND land_use IN ('ARIMAA','MAATULUNDUSMAA','TOOTMISMAA')" ) con.execute(f"CREATE TABLE official_roads AS SELECT road_id, road_class, name, geom FROM ST_Read('{roads_path}')") @@ -188,12 +208,24 @@ def build( con.execute("CREATE TABLE pois_effective AS SELECT * FROM pois_raw") override_status = "applied" if apply_override else "not_testable" + distance_operator = ">=" if break_mode == "distance_inverted" else "<=" + if break_mode == "order_dependent": + # The file position leaks into the result. MIN() keeps the pipeline + # duplicate-resistant, so only the permutation relation can see it. + candidate_columns = "lp.* EXCLUDE (input_rank), MIN(lp.input_rank) AS input_rank" + group_extra = "" + elif break_mode == "duplicate_sensitive": + candidate_columns = "lp.* EXCLUDE (input_rank)" + group_extra = ", lp.input_rank" + else: + candidate_columns = "lp.* EXCLUDE (input_rank)" + group_extra = "" con.execute( "CREATE TABLE candidate_parcels AS " - "SELECT lp.*, MIN(ST_Distance(lp.geom, r.geom)) AS dist_main_road_m " + f"SELECT {candidate_columns}, MIN(ST_Distance(lp.geom, r.geom)) AS dist_main_road_m " "FROM large_parcels lp, roads r " - "GROUP BY lp.cadastral_id, lp.land_use, lp.municipality, lp.geom, lp.area_m2 " - "HAVING MIN(ST_Distance(lp.geom, r.geom)) <= 2000 " + f"GROUP BY lp.cadastral_id, lp.land_use, lp.municipality, lp.geom, lp.area_m2{group_extra} " + f"HAVING MIN(ST_Distance(lp.geom, r.geom)) {distance_operator} {float(road_distance_m)} " "ORDER BY lp.cadastral_id" ) @@ -255,7 +287,7 @@ def build( {"id": "apply_poi_override", "operation": "apply_override", "input": "pois_raw", "override": "OVERRIDE-001", "output": "pois_effective"}, {"id": "road_distance", "operation": "distance_filter", "input": "large_parcels", "target": "road_network", - "max_distance_m": 2000, "crs": analysis_crs, "output": "candidate_parcels"}, + "max_distance_m": ROAD_DISTANCE_CANONICAL_M, "crs": analysis_crs, "output": "candidate_parcels"}, ] if break_mode == "dangling_graph": steps.append({ @@ -378,6 +410,25 @@ def build( "domain_checks": [ {"name": "parcel_area_range", "expression": "area_m2 > 0 AND area_m2 < 1000000"}, ], + # No-golden-answer relations the candidate set must satisfy. Each + # declares the precondition that makes it valid here: candidates + # are a keyed *set* (not a count), and the road threshold is an + # inclusion predicate, so widening it can only add parcels. + "metamorphic": [ + {"id": "parcel-order", "relation": "input_permutation_invariance", + "source": {"path": "data/source/parcels.geojson"}, + "outputs": ["candidate_parcels", "candidate_parcels_geojson"], "key": "cadastral_id", + "preconditions": {"tie_break": "candidates are keyed by cadastral_id and ordered by it; " + "no selection depends on input order"}}, + {"id": "parcel-duplicates", "relation": "duplicate_resistance", + "source": {"path": "data/source/parcels.geojson"}, + "outputs": ["candidate_parcels", "candidate_parcels_geojson"], "key": "cadastral_id", + "preconditions": {"dedup_key": "cadastral_id", "measure": "set"}}, + {"id": "road-distance-monotonic", "relation": "positive_buffer_monotonicity", + "parameter": "road_distance_m", "variant": {"multiply": 3}, + "outputs": ["candidate_parcels"], "key": "cadastral_id", + "preconditions": {"predicate": "within_distance", "expected": "superset"}}, + ], }, "presentation": { "intent": "analytical_workspace", @@ -436,7 +487,13 @@ def build( # the dashboard loads it locally, so a clean rerun must carry it or # the rebuilt project would silently lose its map library. "runtime": {"implementation": {"preferred_engine": "duckdb-spatial", "pipeline": "pipeline.py", - "dependencies": ["README.md", "vendor/maplibre-gl"]}, + "dependencies": ["README.md", "vendor/maplibre-gl"], + "parameters": [{ + "id": "road_distance_m", "type": "number", + "canonical": ROAD_DISTANCE_CANONICAL_M, + "binding": {"argument": "--road-distance-m"}, + "step": "road_distance", "field": "max_distance_m", + }]}, "environment": {"python": "3.12", "duckdb": duckdb.__version__}}, } @@ -1049,13 +1106,33 @@ def main() -> int: if isinstance(project, dict) else {} ) + # The one declared runtime parameter. The canonical run passes nothing + # and reads the manifest; a variant run binds --road-distance-m. + road_distance_m = float(ROAD_DISTANCE_CANONICAL_M) + for parameter in ((project.get("runtime") or {}).get("implementation") or {}).get("parameters") or []: + if isinstance(parameter, dict) and parameter.get("id") == "road_distance_m": + road_distance_m = float(parameter.get("canonical", road_distance_m)) + argv = sys.argv[1:] + if "--road-distance-m" in argv: + road_distance_m = float(argv[argv.index("--road-distance-m") + 1]) + break_mode = next( + ( + warning.get("issue") + for warning in (project.get("warnings") or [] if isinstance(project, dict) else []) + if isinstance(warning, dict) + and warning.get("id") == "EVAL-BREAK" + and warning.get("issue") in PIPELINE_LOGIC_BREAK_MODES + ), + None, + ) build( output_dir, apply_override=any(item.get("id") == "OVERRIDE-001" for item in overrides), - break_mode=None, + break_mode=break_mode, with_scenario_road=any(item.get("id") == "OVERRIDE-002" for item in overrides), uncertain_completeness="completeness" not in source_selection, source_dir=output_dir / "data" / "source", + road_distance_m=road_distance_m, ) print(f"rebuilt {output_dir} from local project inputs") return 0 @@ -1067,12 +1144,15 @@ def main() -> int: parser.add_argument("--scenario-road", action="store_true", help="add OVERRIDE-002 planned connector road") parser.add_argument("--uncertain-completeness", action="store_true", help="drop POI completeness counts and add a completeness warning") + parser.add_argument("--road-distance-m", type=float, default=float(ROAD_DISTANCE_CANONICAL_M), + help="road-distance threshold in metres (declared runtime parameter)") args = parser.parse_args() if args.output_dir.exists(): shutil.rmtree(args.output_dir) build(args.output_dir, apply_override=not args.no_override, break_mode=args.break_mode, - with_scenario_road=args.scenario_road, uncertain_completeness=args.uncertain_completeness) + with_scenario_road=args.scenario_road, uncertain_completeness=args.uncertain_completeness, + road_distance_m=args.road_distance_m) print(f"wrote {args.output_dir}") return 0 diff --git a/evals/run.py b/evals/run.py index 9205431..c4bf840 100755 --- a/evals/run.py +++ b/evals/run.py @@ -81,6 +81,10 @@ def _load_eval_schema(name: str) -> dict[str, Any]: "qgis": "presentation_contract", "presentation": "presentation_contract", "rerun": "rerun_success", + # Kept apart from gis_correctness on purpose: a relation that holds is + # evidence that the analysis is self-consistent under perturbation, not + # that its answer is right. The two must not share a denominator. + "metamorphic": "metamorphic_evidence", } @@ -815,6 +819,10 @@ def _evaluate_assertions( if args.get("hashes_before") == "$SOURCE_HASHES": args["hashes_before"] = source_hashes_before args["require_complete_tree"] = True + if assert_name == "metamorphic.relation_holds" and "forbidden_fragments" not in args: + # A variant run is a rerun: it must not reach back into the + # reference generator either. + args["forbidden_fragments"] = list(eval_forbidden_rerun_fragments()) declared_expect = entry.get("expect", "passed") mutation_role = "target" if declared_expect != "passed" else "guard" diff --git a/openmapstack/checks/metamorphic.py b/openmapstack/checks/metamorphic.py new file mode 100644 index 0000000..49e194a --- /dev/null +++ b/openmapstack/checks/metamorphic.py @@ -0,0 +1,74 @@ +"""Metamorphic-relation assertions. + +Thin check-library entry points over ``openmapstack.metamorphic`` so an eval +case (``assert: metamorphic.relation_holds``) and ``openmapstack verify +--metamorphic`` grade the same thing. See that module for the relations, +their preconditions, and the result vocabulary. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from . import AssertionResult, failed, load_project_yaml, not_testable, passed, project_root + + +def declarations_valid(workspace: Path, project_dir: str = ".") -> AssertionResult: + """Every ``validation.metamorphic[]`` entry parses, names a known relation, + and addresses declared outputs. Structural only: nothing is executed.""" + from ..metamorphic import DeclarationError, declared_relations, parse_declaration + + proj = load_project_yaml(workspace, project_dir) + if proj is None: + return failed("project.yaml missing", code="manifest_missing") + try: + raw_declarations = declared_relations(proj) + except DeclarationError as exc: + return failed(str(exc), code="metamorphic_declaration_invalid") + if not raw_declarations: + return not_testable("no metamorphic relations are declared", code="metamorphic_undeclared") + errors: list[str] = [] + seen: set[str] = set() + outputs = proj.get("outputs") if isinstance(proj.get("outputs"), dict) else {} + for raw in raw_declarations: + try: + declaration = parse_declaration(raw) + except DeclarationError as exc: + errors.append(str(exc)) + continue + if declaration.id in seen: + errors.append(f"{declaration.id}: duplicate relation id") + seen.add(declaration.id) + missing = [key for key in declaration.outputs if key not in outputs] + if missing: + errors.append(f"{declaration.id}: outputs {missing} are not declared outputs") + if errors: + return failed("; ".join(errors), code="metamorphic_declaration_invalid") + return passed(f"{len(raw_declarations)} metamorphic relation(s) are well-formed") + + +def relation_holds( + workspace: Path, + id: str, + project_dir: str = ".", + forbidden_fragments: list[str] | None = None, +) -> AssertionResult: + """Execute the declared relation ``id`` in an isolated variant workspace.""" + from ..metamorphic import DeclarationError, declared_relations, run_relation + + root = project_root(workspace, project_dir) + proj = load_project_yaml(workspace, project_dir) + if proj is None: + return failed("project.yaml missing", code="manifest_missing") + try: + raw_declarations = declared_relations(proj) + except DeclarationError as exc: + return failed(str(exc), code="metamorphic_declaration_invalid") + matches = [raw for raw in raw_declarations if isinstance(raw, dict) and raw.get("id") == id] + if not matches: + return failed(f"no metamorphic relation with id {id!r} is declared", code="metamorphic_relation_undeclared") + result, evidence = run_relation(root, proj, matches[0], forbidden_fragments=tuple(forbidden_fragments or ())) + data: dict[str, Any] = dict(result.data) + data["evidence"] = evidence + return AssertionResult(result.status, result.detail, data) diff --git a/openmapstack/checks/project.py b/openmapstack/checks/project.py index d13b669..26a34ed 100644 --- a/openmapstack/checks/project.py +++ b/openmapstack/checks/project.py @@ -259,3 +259,29 @@ def assumptions_have_rationale(workspace: Path, project_dir: str = ".") -> Asser code="assumption_missing_rationale", ) return passed(f"all {len(assumptions)} assumptions have statement + rationale") + + +def parameters_match_steps(workspace: Path, project_dir: str = ".") -> AssertionResult: + """``runtime.implementation.parameters`` is well-formed and each parameter + bound to a processing step agrees with that step's declared value. + + A manifest that advertises one threshold under ``parameters`` while the + step declares another is drift of the same kind as a presentation + control that disagrees with the pipeline: the whole view becomes a + confident lie. See ``openmapstack.parameters``. + """ + from ..parameters import ParameterError, declared_parameters + + proj = load_project_yaml(workspace, project_dir) + if proj is None: + return failed("project.yaml missing", code="manifest_missing") + try: + parameters = declared_parameters(proj) + except ParameterError as exc: + return failed(str(exc), code="parameters_invalid") + if not parameters: + return not_testable("no runtime parameters are declared", code="parameters_undeclared") + bound = sum(1 for parameter in parameters if parameter.step) + return passed( + f"{len(parameters)} runtime parameter(s) declared; {bound} bound to a processing step agree with it" + ) diff --git a/openmapstack/cli.py b/openmapstack/cli.py index a8e6b14..f2bfdf7 100644 --- a/openmapstack/cli.py +++ b/openmapstack/cli.py @@ -72,6 +72,11 @@ def build_parser() -> argparse.ArgumentParser: default=1800.0, help="seconds allowed for the clean rerun's canonical entrypoint (default: 1800)", ) + verify_parser.add_argument( + "--metamorphic", + action="store_true", + help="also execute every declared validation.metamorphic relation in an isolated variant workspace", + ) verify_parser.add_argument("--strict", action="store_true", help="return failure when warnings or not-testable checks exist") verify_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") verify_parser.add_argument("--output", type=Path, help="also write the JSON result to this path") @@ -119,6 +124,7 @@ def _cmd_verify(args: argparse.Namespace) -> int: args.project, rerun=args.rerun, rerun_timeout_s=args.rerun_timeout, + metamorphic=args.metamorphic, ) except ProjectError as exc: print(f"openmapstack: {exc}", file=sys.stderr) diff --git a/openmapstack/metamorphic.py b/openmapstack/metamorphic.py new file mode 100644 index 0000000..ed1c0d8 --- /dev/null +++ b/openmapstack/metamorphic.py @@ -0,0 +1,661 @@ +"""Conditional metamorphic relations for projects with no golden answer. + +A metamorphic relation asks: if I perturb an input or a parameter in a +controlled way, does the output change the way the analysis semantics say it +must? It needs no frozen answer, which is what makes it usable on a user's +own data -- and it is *conditional*: each relation holds only under declared +preconditions, and a relation that cannot be addressed safely reports +``not_testable`` with the reason, never a pass. + +Relations are declared in ``validation.metamorphic[]``: + +.. code-block:: yaml + + validation: + metamorphic: + - id: parcel-order + relation: input_permutation_invariance + source: {path: data/source/parcels.geojson} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: + tie_break: "candidates are keyed by cadastral_id; no order-dependent selection" + - id: parcel-duplicates + relation: duplicate_resistance + source: {path: data/source/parcels.geojson} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: {dedup_key: cadastral_id, measure: set} + - id: road-distance-monotonic + relation: positive_buffer_monotonicity + parameter: road_distance_m + variant: {multiply: 1.5} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: {predicate: within_distance, expected: superset} + +Every relation runs the canonical entrypoint in an isolated copy prepared +exactly like a clean rerun (``openmapstack.rerun.prepare_clean_workspace``), +perturbs only that copy, compares the copy's outputs with the project's +produced outputs, and removes the copy. The project's own ``data/source`` and +``data/overrides`` are hashed before and after; a variant that reaches back +and mutates them fails outright. + +Implemented relations and their machine-checked preconditions: + +``input_permutation_invariance`` + Source features are shuffled (deterministically, from ``seed``). Every + listed output must be semantically equal. Precondition: a declared + ``tie_break`` rule and a ``key`` that is unique in the baseline output. +``duplicate_resistance`` + Every source feature is appended once more. Outputs must be equal. + Precondition: ``dedup_key`` is unique in the source, and ``measure`` is + ``set``. This relation is invalid for counts and sums, which legitimately + change when rows are duplicated; declaring another measure is rejected. +``positive_buffer_monotonicity`` + A declared numeric parameter is increased through its binding. Every + baseline output key must survive in the variant (``superset``). + Precondition: the parameter is declared, positive, and the variant is a + strict increase; the ``predicate`` names an inclusion predicate. + +Candidate relations that are *not* implemented here (CRS round trip, subset +additivity, area-scale consistency) must not be declared; the verifier +rejects unknown relation names rather than silently skipping them. +""" + +from __future__ import annotations + +import hashlib +import json +import random +import re +import shutil +import tempfile +import time +from collections.abc import Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from .checks import AssertionResult, failed, not_testable, passed +from .checks.rerun import _semantic_snapshot +from .checks.spatial import connect_spatial +from .parameters import ParameterError, declared_parameters +from .project import get_in, project_path +from .rerun import execute_canonical, prepare_clean_workspace + +METAMORPHIC_SCHEMA = "openmapstack-metamorphic/v1" +RELATIONS = ( + "input_permutation_invariance", + "duplicate_resistance", + "positive_buffer_monotonicity", +) +INCLUSION_PREDICATES = ("within_distance", "intersects_buffer", "within_buffer") +DEFAULT_TIMEOUT_S = 600.0 +DEFAULT_MAX_SOURCE_BYTES = 64 * 1024 * 1024 +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_DECLARATION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_TRANSFORMABLE_SUFFIXES = {".geojson", ".json", ".parquet"} +_READABLE_OUTPUT_SUFFIXES = {".geojson", ".json", ".parquet"} + + +class DeclarationError(ValueError): + """A metamorphic declaration is structurally invalid.""" + + +@dataclass +class Declaration: + id: str + relation: str + outputs: list[str] + key: str + source_path: str | None = None + parameter: str | None = None + variant: dict[str, Any] = field(default_factory=dict) + preconditions: dict[str, Any] = field(default_factory=dict) + tolerance: dict[str, Any] = field(default_factory=dict) + timeout_s: float = DEFAULT_TIMEOUT_S + max_source_bytes: int = DEFAULT_MAX_SOURCE_BYTES + seed: int = 7 + + +def parse_declaration(raw: object) -> Declaration: + """Validate one ``validation.metamorphic[]`` entry structurally.""" + if not isinstance(raw, dict): + raise DeclarationError("declaration must be a mapping") + errors: list[str] = [] + declaration_id = raw.get("id") + if not isinstance(declaration_id, str) or _DECLARATION_ID.fullmatch(declaration_id) is None: + raise DeclarationError("declaration id is required") + relation = raw.get("relation") + if relation not in RELATIONS: + raise DeclarationError(f"{declaration_id}: relation must be one of {list(RELATIONS)}, got {relation!r}") + allowed = { + "id", "relation", "outputs", "key", "source", "parameter", "variant", + "preconditions", "tolerance", "limits", "seed", "description", + } + unknown = set(raw) - allowed + if unknown: + errors.append(f"unknown keys {sorted(unknown)}") + outputs = raw.get("outputs") + if not isinstance(outputs, list) or not outputs or any(not isinstance(item, str) or not item for item in outputs): + errors.append("outputs must be a non-empty list of output keys") + outputs = [] + key = raw.get("key") + if not isinstance(key, str) or _IDENTIFIER.fullmatch(key) is None: + errors.append("key must be a simple field identifier") + key = "" + preconditions = raw.get("preconditions") or {} + if not isinstance(preconditions, dict): + errors.append("preconditions must be a mapping") + preconditions = {} + tolerance = raw.get("tolerance") or {} + if not isinstance(tolerance, dict): + errors.append("tolerance must be a mapping") + tolerance = {} + limits = raw.get("limits") or {} + if not isinstance(limits, dict): + errors.append("limits must be a mapping") + limits = {} + timeout_s = limits.get("timeout_s", DEFAULT_TIMEOUT_S) + max_source_bytes = limits.get("max_source_bytes", DEFAULT_MAX_SOURCE_BYTES) + if isinstance(timeout_s, bool) or not isinstance(timeout_s, (int, float)) or timeout_s <= 0: + errors.append("limits.timeout_s must be a positive number") + timeout_s = DEFAULT_TIMEOUT_S + if isinstance(max_source_bytes, bool) or not isinstance(max_source_bytes, int) or max_source_bytes <= 0: + errors.append("limits.max_source_bytes must be a positive integer") + max_source_bytes = DEFAULT_MAX_SOURCE_BYTES + seed = raw.get("seed", 7) + if isinstance(seed, bool) or not isinstance(seed, int): + errors.append("seed must be an integer") + seed = 7 + + source_path: str | None = None + parameter: str | None = None + variant: dict[str, Any] = {} + if relation in {"input_permutation_invariance", "duplicate_resistance"}: + source = raw.get("source") + source_path = source.get("path") if isinstance(source, dict) else None + if not isinstance(source_path, str) or not source_path.strip(): + errors.append("source.path is required for this relation") + source_path = None + elif not (source_path.startswith("data/source/") or source_path.startswith("data/overrides/")): + errors.append("source.path must name a file under data/source/ or data/overrides/") + if "parameter" in raw or "variant" in raw: + errors.append("parameter/variant do not apply to this relation") + if relation == "input_permutation_invariance": + tie_break = preconditions.get("tie_break") + if not isinstance(tie_break, str) or not tie_break.strip(): + errors.append("preconditions.tie_break must state the deterministic ordering rule") + if relation == "duplicate_resistance": + dedup_key = preconditions.get("dedup_key") + if not isinstance(dedup_key, str) or _IDENTIFIER.fullmatch(dedup_key) is None: + errors.append("preconditions.dedup_key must be a simple field identifier") + measure = preconditions.get("measure", "set") + if measure != "set": + errors.append( + f"duplicate_resistance is valid only for set semantics; measure {measure!r} " + "(counts and sums change legitimately when rows are duplicated)" + ) + if relation == "positive_buffer_monotonicity": + parameter = raw.get("parameter") + if not isinstance(parameter, str) or _IDENTIFIER.fullmatch(parameter) is None: + errors.append("parameter must name a declared runtime parameter") + parameter = None + variant = raw.get("variant") or {} + if not isinstance(variant, dict) or len(variant) != 1 or not ({"multiply", "add"} & set(variant)): + errors.append("variant must declare exactly one of multiply/add") + variant = {} + else: + operation, amount = next(iter(variant.items())) + if isinstance(amount, bool) or not isinstance(amount, (int, float)): + errors.append(f"variant.{operation} must be a number") + variant = {} + elif operation == "multiply" and amount <= 1: + errors.append("variant.multiply must be > 1 so the buffer strictly grows") + variant = {} + elif operation == "add" and amount <= 0: + errors.append("variant.add must be > 0 so the buffer strictly grows") + variant = {} + predicate = preconditions.get("predicate") + if predicate not in INCLUSION_PREDICATES: + errors.append(f"preconditions.predicate must be one of {list(INCLUSION_PREDICATES)}") + if preconditions.get("expected", "superset") != "superset": + errors.append("positive_buffer_monotonicity establishes only expected: superset") + if "source" in raw: + errors.append("source does not apply to a parameter relation") + if errors: + raise DeclarationError(f"{declaration_id}: " + "; ".join(errors)) + return Declaration( + id=declaration_id, + relation=relation, + outputs=list(outputs), + key=key, + source_path=source_path, + parameter=parameter, + variant=dict(variant), + preconditions=dict(preconditions), + tolerance=dict(tolerance), + timeout_s=float(timeout_s), + max_source_bytes=int(max_source_bytes), + seed=int(seed), + ) + + +def declared_relations(manifest: dict[str, Any]) -> list[object]: + raw = get_in(manifest, "validation", "metamorphic") + if raw is None: + return [] + if not isinstance(raw, list): + raise DeclarationError("validation.metamorphic must be a list") + return list(raw) + + +# --- data access ------------------------------------------------------------ + + +def _read_features(path: Path) -> tuple[dict[str, Any], list[Any]]: + document = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(document, dict) or document.get("type") != "FeatureCollection": + raise ValueError(f"{path.name} is not a GeoJSON FeatureCollection") + features = document.get("features") + if not isinstance(features, list): + raise ValueError(f"{path.name} has no features list") + return document, features + + +def _write_features(path: Path, document: dict[str, Any], features: list[Any]) -> None: + replaced = dict(document) + replaced["features"] = features + path.write_text(json.dumps(replaced, ensure_ascii=False), encoding="utf-8") + + +def _duckdb_or_none(): + return connect_spatial() + + +def _key_values(path: Path, key: str) -> list[Any]: + """Every value of ``key`` in an output artifact, in file order.""" + suffix = path.suffix.lower() + if suffix in {".geojson", ".json"}: + _, features = _read_features(path) + values = [] + for feature in features: + properties = feature.get("properties") if isinstance(feature, dict) else None + if not isinstance(properties, dict) or key not in properties: + raise KeyError(key) + values.append(properties[key]) + return values + if suffix == ".parquet": + connection = _duckdb_or_none() + if connection is None: + raise RuntimeError("duckdb_unavailable") + try: + escaped = path.as_posix().replace("'", "''") + columns = {row[0] for row in connection.execute(f"DESCRIBE SELECT * FROM read_parquet('{escaped}')").fetchall()} + if key not in columns: + raise KeyError(key) + rows = connection.execute(f'SELECT "{key}" FROM read_parquet(\'{escaped}\')').fetchall() + finally: + connection.close() + return [row[0] for row in rows] + raise ValueError("unsupported_format") + + +def _source_key_values(path: Path, key: str) -> list[Any]: + return _key_values(path, key) + + +def _hash_tree(root: Path) -> dict[str, str]: + hashes: dict[str, str] = {} + for directory in ("data/source", "data/overrides"): + base = root / directory + if not base.is_dir(): + continue + for item in sorted(base.rglob("*")): + if item.is_file(): + hashes[item.relative_to(root).as_posix()] = hashlib.sha256(item.read_bytes()).hexdigest() + return hashes + + +# --- transformations ---------------------------------------------------------- + + +def _permute_source(path: Path, seed: int) -> dict[str, Any]: + suffix = path.suffix.lower() + if suffix in {".geojson", ".json"}: + document, features = _read_features(path) + order = list(range(len(features))) + random.Random(seed).shuffle(order) + if len(features) > 1 and order == list(range(len(features))): + order.reverse() + _write_features(path, document, [features[index] for index in order]) + return {"transformation": "permute_features", "features": len(features), "seed": seed} + if suffix == ".parquet": + connection = _duckdb_or_none() + if connection is None: + raise RuntimeError("duckdb_unavailable") + try: + escaped = path.as_posix().replace("'", "''") + count = connection.execute(f"SELECT COUNT(*) FROM read_parquet('{escaped}')").fetchone()[0] + connection.execute(f"SELECT setseed({(seed % 1000) / 1000.0})") + temp = path.with_suffix(".permuted.tmp.parquet") + connection.execute( + f"COPY (SELECT * FROM read_parquet('{escaped}') ORDER BY random()) " + f"TO '{temp.as_posix().replace(chr(39), chr(39) * 2)}' (FORMAT PARQUET)" + ) + finally: + connection.close() + temp.replace(path) + return {"transformation": "permute_rows", "rows": count, "seed": seed} + raise ValueError("unsupported_format") + + +def _duplicate_source(path: Path) -> dict[str, Any]: + suffix = path.suffix.lower() + if suffix in {".geojson", ".json"}: + document, features = _read_features(path) + _write_features(path, document, features + [json.loads(json.dumps(item)) for item in features]) + return {"transformation": "duplicate_features", "features": len(features), "duplicated": len(features)} + if suffix == ".parquet": + connection = _duckdb_or_none() + if connection is None: + raise RuntimeError("duckdb_unavailable") + try: + escaped = path.as_posix().replace("'", "''") + count = connection.execute(f"SELECT COUNT(*) FROM read_parquet('{escaped}')").fetchone()[0] + temp = path.with_suffix(".duplicated.tmp.parquet") + connection.execute( + f"COPY (SELECT * FROM read_parquet('{escaped}') UNION ALL SELECT * FROM read_parquet('{escaped}')) " + f"TO '{temp.as_posix().replace(chr(39), chr(39) * 2)}' (FORMAT PARQUET)" + ) + finally: + connection.close() + temp.replace(path) + return {"transformation": "duplicate_rows", "rows": count, "duplicated": count} + raise ValueError("unsupported_format") + + +# --- execution ---------------------------------------------------------------- + + +def _declared_output_files(manifest: dict[str, Any], keys: Sequence[str]) -> dict[str, str]: + outputs = manifest.get("outputs") if isinstance(manifest.get("outputs"), dict) else {} + resolved: dict[str, str] = {} + for key in keys: + spec = outputs.get(key) + path = spec.get("path") if isinstance(spec, dict) else None + if not isinstance(path, str) or not path.strip(): + raise DeclarationError(f"outputs[{key!r}] is not a declared output with a path") + resolved[key] = path + return resolved + + +def run_relation( + project_root: Path, + manifest: dict[str, Any], + raw_declaration: object, + *, + forbidden_fragments: Sequence[str] = (), +) -> tuple[AssertionResult, dict[str, Any]]: + """Execute one declared relation and return (result, evidence). + + The result vocabulary: + + - ``failed``: the relation was executed and does not hold, or the + declaration is invalid, or the variant reached back into the project's + immutable inputs; + - ``not_testable``: a precondition does not hold on this data or the + environment cannot run the relation (unsupported format, DuckDB absent, + timeout, oversize source); + - ``passed``: the variant ran and the declared relation holds. + """ + evidence: dict[str, Any] = {"schema": METAMORPHIC_SCHEMA} + try: + declaration = parse_declaration(raw_declaration) + except DeclarationError as exc: + evidence["class"] = "invalid" + return failed(str(exc), code="metamorphic_declaration_invalid"), evidence + evidence.update({"id": declaration.id, "relation": declaration.relation}) + root = project_root.resolve() + + try: + output_files = _declared_output_files(manifest, declaration.outputs) + except DeclarationError as exc: + evidence["class"] = "invalid" + return failed(f"{declaration.id}: {exc}", code="metamorphic_declaration_invalid"), evidence + for output_key, relative in output_files.items(): + target = project_path(root, relative) + if target is None: + evidence["class"] = "invalid" + return failed(f"{declaration.id}: output {output_key!r} path is unsafe", code="metamorphic_declaration_invalid"), evidence + if not target.is_file(): + return not_testable( + f"{declaration.id}: baseline output {relative} does not exist; run the pipeline first", + code="baseline_missing", + ), evidence + if target.suffix.lower() not in _READABLE_OUTPUT_SUFFIXES: + return not_testable( + f"{declaration.id}: output {relative} is {target.suffix or 'extensionless'}, which the relation cannot compare", + code="unsupported_format", + ), evidence + + # Baseline keys: every listed output must expose a unique key. + baseline_keys: dict[str, list[Any]] = {} + for output_key, relative in output_files.items(): + try: + values = _key_values(root / relative, declaration.key) + except KeyError: + return not_testable( + f"{declaration.id}: output {relative} has no field {declaration.key!r}", + code="precondition_unmet", + ), evidence + except RuntimeError as exc: + return not_testable(f"{declaration.id}: {exc}", code="duckdb_unavailable"), evidence + except (ValueError, OSError, json.JSONDecodeError) as exc: + return not_testable(f"{declaration.id}: cannot read {relative}: {exc}", code="unsupported_format"), evidence + if len(set(map(_hashable, values))) != len(values): + return not_testable( + f"{declaration.id}: key {declaration.key!r} is not unique in {relative}, so set semantics cannot be asserted", + code="precondition_unmet", + ), evidence + baseline_keys[output_key] = values + + # Relation-specific preconditions and the variant plan. + extra_argv: list[str] = [] + extra_env: dict[str, str] = {} + source_relative: str | None = None + if declaration.relation in {"input_permutation_invariance", "duplicate_resistance"}: + assert declaration.source_path is not None + source_relative = declaration.source_path + source_target = project_path(root, source_relative) + if source_target is None or not source_target.is_file(): + return not_testable(f"{declaration.id}: source {source_relative} does not exist", code="precondition_unmet"), evidence + if source_target.suffix.lower() not in _TRANSFORMABLE_SUFFIXES: + return not_testable( + f"{declaration.id}: source {source_relative} is {source_target.suffix or 'extensionless'}, which cannot be transformed", + code="unsupported_format", + ), evidence + size = source_target.stat().st_size + if size > declaration.max_source_bytes: + return not_testable( + f"{declaration.id}: source {source_relative} is {size} bytes, above limits.max_source_bytes={declaration.max_source_bytes}", + code="resource_limit", + ), evidence + if declaration.relation == "duplicate_resistance": + dedup_key = declaration.preconditions["dedup_key"] + try: + source_values = _source_key_values(source_target, dedup_key) + except KeyError: + return not_testable( + f"{declaration.id}: source {source_relative} has no field {dedup_key!r}", + code="precondition_unmet", + ), evidence + except RuntimeError as exc: + return not_testable(f"{declaration.id}: {exc}", code="duckdb_unavailable"), evidence + except (ValueError, OSError, json.JSONDecodeError) as exc: + return not_testable(f"{declaration.id}: cannot read {source_relative}: {exc}", code="unsupported_format"), evidence + if len(set(map(_hashable, source_values))) != len(source_values): + return not_testable( + f"{declaration.id}: source {source_relative} already has duplicate {dedup_key!r} values; " + "the analysis cannot be deduplicating on that key", + code="precondition_unmet", + ), evidence + evidence["dedup_key"] = dedup_key + else: + assert declaration.parameter is not None + try: + parameters = {parameter.id: parameter for parameter in declared_parameters(manifest)} + except ParameterError as exc: + return failed(f"{declaration.id}: {exc}", code="parameters_invalid"), evidence + parameter = parameters.get(declaration.parameter) + if parameter is None: + return failed( + f"{declaration.id}: parameter {declaration.parameter!r} is not declared under runtime.implementation.parameters", + code="metamorphic_declaration_invalid", + ), evidence + if parameter.type not in {"integer", "number"}: + return not_testable( + f"{declaration.id}: parameter {parameter.id!r} is {parameter.type}, not numeric", + code="precondition_unmet", + ), evidence + if parameter.canonical <= 0: + return not_testable( + f"{declaration.id}: parameter {parameter.id!r} canonical value {parameter.canonical!r} is not positive", + code="precondition_unmet", + ), evidence + operation, amount = next(iter(declaration.variant.items())) + variant_value = parameter.canonical * amount if operation == "multiply" else parameter.canonical + amount + if parameter.type == "integer": + variant_value = int(round(variant_value)) + if variant_value <= parameter.canonical: + return not_testable( + f"{declaration.id}: variant does not strictly increase the integer parameter", + code="precondition_unmet", + ), evidence + extra_argv, extra_env = parameter.bind(variant_value) + evidence["parameter"] = {"id": parameter.id, "canonical": parameter.canonical, "variant": variant_value} + + # Run the variant in an isolated copy; the project itself is read-only. + original_hashes = _hash_tree(root) + variant_root = Path(tempfile.mkdtemp(prefix=f"openmapstack-metamorphic-{declaration.id}-")) + started = time.monotonic() + try: + try: + command, preserved, _ = prepare_clean_workspace(root, variant_root, forbidden_fragments=forbidden_fragments) + except ValueError as exc: + return failed(f"{declaration.id}: cannot prepare variant workspace: {exc}", code="variant_preparation_failed"), evidence + evidence["preserved_paths"] = sorted(preserved) + if source_relative is not None: + variant_source = variant_root / source_relative + try: + if declaration.relation == "input_permutation_invariance": + evidence["variant"] = _permute_source(variant_source, declaration.seed) + else: + evidence["variant"] = _duplicate_source(variant_source) + except RuntimeError as exc: + return not_testable(f"{declaration.id}: {exc}", code="duckdb_unavailable"), evidence + except (ValueError, OSError, json.JSONDecodeError) as exc: + return not_testable(f"{declaration.id}: cannot transform {source_relative}: {exc}", code="unsupported_format"), evidence + else: + evidence["variant"] = {"transformation": "parameter", "argv": extra_argv, "environment": sorted(extra_env)} + + execution, removed = execute_canonical( + command, variant_root, declaration.timeout_s, extra_argv=extra_argv, extra_env=extra_env + ) + evidence["command"] = [*command, *extra_argv] + evidence["removed_environment_keys"] = removed + evidence["duration_s"] = time.monotonic() - started + if execution.get("timed_out"): + return not_testable( + f"{declaration.id}: variant run exceeded limits.timeout_s={declaration.timeout_s}", + code="variant_timeout", + ), evidence + if execution.get("returncode") != 0: + evidence["stderr_tail"] = (execution.get("stderr") or "")[-2000:] + return failed( + f"{declaration.id}: variant run exited with status {execution.get('returncode')}", + code="variant_execution_failed", + ), evidence + + if _hash_tree(root) != original_hashes: + return failed( + f"{declaration.id}: the variant run mutated the project's declared-immutable inputs", + code="original_source_mutated", + ), evidence + + # Compare. + differences: list[str] = [] + for output_key, relative in output_files.items(): + baseline_file = root / relative + variant_file = variant_root / relative + if not variant_file.is_file(): + return failed( + f"{declaration.id}: variant run did not produce output {relative}", + code="variant_output_missing", + ), evidence + if declaration.relation == "positive_buffer_monotonicity": + try: + variant_keys = set(map(_hashable, _key_values(variant_file, declaration.key))) + except RuntimeError as exc: + return not_testable(f"{declaration.id}: {exc}", code="duckdb_unavailable"), evidence + except (KeyError, ValueError, OSError, json.JSONDecodeError) as exc: + return failed( + f"{declaration.id}: variant output {relative} lost field {declaration.key!r}: {exc}", + code="monotonicity_violated", + ), evidence + lost = sorted(str(value) for value in set(map(_hashable, baseline_keys[output_key])) - variant_keys) + evidence.setdefault("counts", {})[output_key] = { + "baseline": len(baseline_keys[output_key]), + "variant": len(variant_keys), + } + if lost: + differences.append(f"{relative} lost {len(lost)} baseline feature(s) when the buffer grew: {lost[:10]}") + else: + ignored = set(declaration.tolerance.get("ignored_fields") or []) + try: + equal = _snapshot(baseline_file, ignored, declaration.tolerance) == _snapshot( + variant_file, ignored, declaration.tolerance + ) + except RuntimeError as exc: + return not_testable(f"{declaration.id}: {exc}", code="duckdb_unavailable"), evidence + except Exception as exc: # noqa: BLE001 - normalization is best effort + return not_testable(f"{declaration.id}: cannot normalize {relative}: {exc}", code="normalize_error"), evidence + if not equal: + differences.append(f"{relative} changed") + if differences: + code = { + "input_permutation_invariance": "permutation_changed_output", + "duplicate_resistance": "duplicates_changed_output", + "positive_buffer_monotonicity": "monotonicity_violated", + }[declaration.relation] + return failed(f"{declaration.id}: {'; '.join(differences)}", code=code), evidence + return passed( + f"{declaration.id}: {declaration.relation} holds across {len(output_files)} output(s)" + ), evidence + finally: + shutil.rmtree(variant_root, ignore_errors=True) + + +def _snapshot(path: Path, ignored: set[str], tolerance: dict[str, Any]) -> Any: + snapshot = _semantic_snapshot(path, ignored) + digits = tolerance.get("round_numbers") + if isinstance(digits, int) and not isinstance(digits, bool): + snapshot = _round(snapshot, digits) + return snapshot + + +def _round(value: Any, digits: int) -> Any: + if isinstance(value, float): + return round(value, digits) + if isinstance(value, list): + return [_round(item, digits) for item in value] + if isinstance(value, dict): + return {key: _round(item, digits) for key, item in value.items()} + return value + + +def _hashable(value: Any) -> Any: + if isinstance(value, (list, dict)): + return json.dumps(value, sort_keys=True, default=str) + return value diff --git a/openmapstack/parameters.py b/openmapstack/parameters.py new file mode 100644 index 0000000..5cfb715 --- /dev/null +++ b/openmapstack/parameters.py @@ -0,0 +1,190 @@ +"""Versioned parameter addressing for a project's canonical entrypoint. + +A metamorphic relation that varies a threshold, and a benchmark that wants to +run the same pipeline at a different setting, both need one thing the +manifest did not previously give them: a way to *address* a parameter of the +pipeline without editing the pipeline. ``runtime.implementation.parameters`` +is that contract. + +.. code-block:: yaml + + runtime: + implementation: + pipeline: pipeline.py + parameters: + - id: road_distance_m + type: number + canonical: 2000 + binding: {argument: "--road-distance-m"} # or {environment: OMS_ROAD_DISTANCE_M} + step: road_distance # optional: the processing step that + field: max_distance_m # consumes it, so drift is checkable + +Rules: + +- ``id`` is a simple identifier, unique within the manifest; +- ``type`` is ``integer``, ``number``, or ``string`` and ``canonical`` has that + type (booleans are not numbers); +- exactly one binding: ``argument`` (a ``--long-flag``, passed as + ``--flag value``) or ``environment`` (an ``UPPER_SNAKE`` variable); +- ``step``/``field`` are optional but come together; when present the named + processing step must exist and its field must equal ``canonical``, so a + manifest cannot advertise one threshold while the step declares another. + +The canonical run passes nothing: a pipeline must produce the accepted result +with no arguments and no variables set. Bindings exist so a *variant* run can +say "same pipeline, this one knob turned". +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +from .project import get_in + +PARAMETERS_SCHEMA = "openmapstack-parameters/v1" +PARAMETER_TYPES = ("integer", "number", "string") + +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_ARGUMENT = re.compile(r"^--[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +_ENVIRONMENT = re.compile(r"^[A-Z][A-Z0-9_]*$") + + +class ParameterError(ValueError): + """The parameters block is malformed or drifts from the processing steps.""" + + +@dataclass(frozen=True) +class Parameter: + id: str + type: str + canonical: Any + argument: str | None = None + environment: str | None = None + step: str | None = None + field: str | None = None + + def bind(self, value: Any) -> tuple[list[str], dict[str, str]]: + """Return the argv suffix and environment additions for ``value``.""" + rendered = _render(value, self.type) + if self.argument is not None: + return [self.argument, rendered], {} + assert self.environment is not None + return [], {self.environment: rendered} + + +def _render(value: Any, type_name: str) -> str: + if type_name == "integer": + return str(int(value)) + if type_name == "number": + number = float(value) + return str(int(number)) if number.is_integer() else repr(number) + return str(value) + + +def value_has_type(value: Any, type_name: str) -> bool: + if isinstance(value, bool): + return False + if type_name == "integer": + return isinstance(value, int) + if type_name == "number": + return isinstance(value, (int, float)) + return isinstance(value, str) + + +def declared_parameters(manifest: dict[str, Any]) -> list[Parameter]: + """Parse and validate ``runtime.implementation.parameters``. + + Returns an empty list when nothing is declared. Raises ``ParameterError`` + describing every problem found, so a caller can report one failure that + names all of them. + """ + raw = get_in(manifest, "runtime", "implementation", "parameters") + if raw is None: + return [] + if not isinstance(raw, list): + raise ParameterError("runtime.implementation.parameters must be a list") + steps = get_in(manifest, "processing", "steps", default=[]) or [] + steps_by_id = { + str(step.get("id")): step for step in steps if isinstance(step, dict) and step.get("id") is not None + } + errors: list[str] = [] + seen: set[str] = set() + parameters: list[Parameter] = [] + for index, entry in enumerate(raw): + where = f"parameters[{index}]" + if not isinstance(entry, dict): + errors.append(f"{where} must be a mapping") + continue + unknown = set(entry) - {"id", "type", "canonical", "binding", "step", "field", "description"} + if unknown: + errors.append(f"{where} has unknown keys {sorted(unknown)}") + parameter_id = entry.get("id") + if not isinstance(parameter_id, str) or _IDENTIFIER.fullmatch(parameter_id) is None: + errors.append(f"{where}.id must be a simple identifier") + continue + where = f"parameters[{parameter_id}]" + if parameter_id in seen: + errors.append(f"{where} is declared more than once") + seen.add(parameter_id) + type_name = entry.get("type") + if type_name not in PARAMETER_TYPES: + errors.append(f"{where}.type must be one of {list(PARAMETER_TYPES)}") + continue + if "canonical" not in entry or not value_has_type(entry["canonical"], type_name): + errors.append(f"{where}.canonical must be a {type_name}") + continue + binding = entry.get("binding") + argument = environment = None + if not isinstance(binding, dict) or len(binding) != 1: + errors.append(f"{where}.binding must declare exactly one of argument/environment") + elif "argument" in binding: + argument = binding["argument"] + if not isinstance(argument, str) or _ARGUMENT.fullmatch(argument) is None: + errors.append(f"{where}.binding.argument must be a --long-flag") + argument = None + elif "environment" in binding: + environment = binding["environment"] + if not isinstance(environment, str) or _ENVIRONMENT.fullmatch(environment) is None: + errors.append(f"{where}.binding.environment must be an UPPER_SNAKE variable name") + environment = None + else: + errors.append(f"{where}.binding must declare exactly one of argument/environment") + step = entry.get("step") + field = entry.get("field") + if (step is None) != (field is None): + errors.append(f"{where} must declare step and field together") + elif step is not None: + if not isinstance(step, str) or step not in steps_by_id: + errors.append(f"{where}.step {step!r} is not a processing step") + elif not isinstance(field, str) or field not in steps_by_id[step]: + errors.append(f"{where}.field {field!r} is not declared on step {step!r}") + elif steps_by_id[step][field] != entry["canonical"]: + errors.append( + f"{where}.canonical {entry['canonical']!r} != processing step " + f"{step!r}.{field} = {steps_by_id[step][field]!r}" + ) + if argument is None and environment is None: + continue + parameters.append( + Parameter( + id=parameter_id, + type=type_name, + canonical=entry["canonical"], + argument=argument, + environment=environment, + step=step if isinstance(step, str) else None, + field=field if isinstance(field, str) else None, + ) + ) + if errors: + raise ParameterError("; ".join(errors)) + return parameters + + +def find_parameter(manifest: dict[str, Any], parameter_id: str) -> Parameter | None: + for parameter in declared_parameters(manifest): + if parameter.id == parameter_id: + return parameter + return None diff --git a/openmapstack/rerun.py b/openmapstack/rerun.py index 9b19175..a0b9cb6 100644 --- a/openmapstack/rerun.py +++ b/openmapstack/rerun.py @@ -235,6 +235,69 @@ def _clean_rerun_environment() -> tuple[dict[str, str], list[str]]: def _write_clean_rerun_evidence(rerun_root: Path, evidence: dict[str, Any]) -> None: (rerun_root / CLEAN_RERUN_EVIDENCE).write_text(json.dumps(evidence, indent=2, default=str), encoding="utf-8") +def prepare_clean_workspace( + project_root: Path, + rerun_root: Path, + *, + forbidden_fragments: Sequence[str] = (), +) -> tuple[list[str], set[str], dict[str, Any]]: + """Copy only the clean-rerun inputs into ``rerun_root``. + + Returns the resolved canonical command, the set of preserved + project-relative paths, and the loaded manifest. Raises ``ValueError`` + for a manifest that cannot be rerun safely. Shared by the clean rerun + and by metamorphic variant runs, which need the same isolation but then + perturb one input or parameter before executing. + """ + manifest_path = project_root / "project.yaml" + if not manifest_path.is_file(): + raise ValueError("project.yaml is missing") + try: + project = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, yaml.YAMLError) as exc: + raise ValueError(f"project.yaml cannot be loaded: {exc}") from exc + if not isinstance(project, dict): + raise ValueError("project.yaml must contain a mapping") + + preserved: set[str] = set() + command, declared_paths = canonical_rerun_command( + project_root, project, forbidden_fragments=forbidden_fragments + ) + _copy_clean_rerun_path(project_root, rerun_root, "project.yaml", "project manifest", preserved) + for conventional_path in ("data/source", "data/overrides"): + if (project_root / conventional_path).exists(): + _copy_clean_rerun_path( + project_root, + rerun_root, + conventional_path, + f"clean-rerun input {conventional_path}", + preserved, + ) + for path, field_name in declared_paths: + _copy_clean_rerun_path(project_root, rerun_root, path, field_name, preserved) + return command, preserved, project + + +def execute_canonical( + command: list[str], + rerun_root: Path, + timeout_s: int | float, + *, + extra_argv: Sequence[str] = (), + extra_env: dict[str, str] | None = None, +) -> tuple[dict[str, Any], list[str]]: + """Run the canonical entrypoint in a sanitized environment. + + Returns the execution record and the list of environment keys removed. + ``extra_argv``/``extra_env`` carry parameter bindings for variant runs. + """ + env, removed_environment = _clean_rerun_environment() + if extra_env: + env.update(extra_env) + execution = _execute_argv([*command, *extra_argv], rerun_root, timeout_s, env) + return execution, removed_environment + + def perform_clean_rerun( project_root: Path, rerun_root: Path, @@ -259,38 +322,15 @@ def perform_clean_rerun( } preserved: set[str] = set() try: - manifest_path = project_root / "project.yaml" - if not manifest_path.is_file(): - raise ValueError("project.yaml is missing") - try: - project = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, yaml.YAMLError) as exc: - raise ValueError(f"project.yaml cannot be loaded: {exc}") from exc - if not isinstance(project, dict): - raise ValueError("project.yaml must contain a mapping") - - command, declared_paths = canonical_rerun_command( - project_root, project, forbidden_fragments=forbidden_fragments + command, preserved, _project = prepare_clean_workspace( + project_root, rerun_root, forbidden_fragments=forbidden_fragments ) - _copy_clean_rerun_path(project_root, rerun_root, "project.yaml", "project manifest", preserved) - for conventional_path in ("data/source", "data/overrides"): - if (project_root / conventional_path).exists(): - _copy_clean_rerun_path( - project_root, - rerun_root, - conventional_path, - f"clean-rerun input {conventional_path}", - preserved, - ) - for path, field_name in declared_paths: - _copy_clean_rerun_path(project_root, rerun_root, path, field_name, preserved) evidence["preserved_paths"] = sorted(preserved) source_hashes_before = _hash_immutable_inputs(rerun_root, preserved) evidence["command"] = command - env, removed_environment = _clean_rerun_environment() + execution, removed_environment = execute_canonical(command, rerun_root, timeout_s) evidence["removed_environment_keys"] = removed_environment - execution = _execute_argv(command, rerun_root, timeout_s, env) evidence["execution"] = execution if execution.get("timed_out"): evidence["stage"] = "canonical_execution" diff --git a/openmapstack/schemas/project-v1.schema.json b/openmapstack/schemas/project-v1.schema.json index 7379e90..b01525d 100644 --- a/openmapstack/schemas/project-v1.schema.json +++ b/openmapstack/schemas/project-v1.schema.json @@ -24,6 +24,93 @@ "jsonScalar": { "type": ["string", "number", "boolean", "null"] }, + "identifier": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "runtimeParameter": { + "type": "object", + "required": ["id", "type", "canonical", "binding"], + "additionalProperties": false, + "properties": { + "id": {"$ref": "#/$defs/identifier"}, + "type": {"enum": ["integer", "number", "string"]}, + "canonical": {"type": ["integer", "number", "string"]}, + "binding": { + "type": "object", + "minProperties": 1, + "maxProperties": 1, + "additionalProperties": false, + "properties": { + "argument": {"type": "string", "pattern": "^--[a-z][a-z0-9]*(?:-[a-z0-9]+)*$"}, + "environment": {"type": "string", "pattern": "^[A-Z][A-Z0-9_]*$"} + } + }, + "step": {"type": "string", "minLength": 1}, + "field": {"$ref": "#/$defs/identifier"}, + "description": {"type": "string"} + }, + "dependentRequired": {"step": ["field"], "field": ["step"]} + }, + "metamorphicRelation": { + "type": "object", + "required": ["id", "relation", "outputs", "key"], + "additionalProperties": false, + "properties": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"}, + "relation": { + "enum": ["input_permutation_invariance", "duplicate_resistance", "positive_buffer_monotonicity"] + }, + "outputs": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}}, + "key": {"$ref": "#/$defs/identifier"}, + "source": { + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": {"path": {"$ref": "#/$defs/safeProjectPath"}} + }, + "parameter": {"$ref": "#/$defs/identifier"}, + "variant": { + "type": "object", + "minProperties": 1, + "maxProperties": 1, + "additionalProperties": false, + "properties": { + "multiply": {"type": "number", "exclusiveMinimum": 1}, + "add": {"type": "number", "exclusiveMinimum": 0} + } + }, + "preconditions": {"type": "object"}, + "tolerance": { + "type": "object", + "additionalProperties": false, + "properties": { + "ignored_fields": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "round_numbers": {"type": "integer", "minimum": 0} + } + }, + "limits": { + "type": "object", + "additionalProperties": false, + "properties": { + "timeout_s": {"type": "number", "exclusiveMinimum": 0}, + "max_source_bytes": {"type": "integer", "minimum": 1} + } + }, + "seed": {"type": "integer"}, + "description": {"type": "string"} + }, + "allOf": [ + { + "if": {"properties": {"relation": {"const": "positive_buffer_monotonicity"}}}, + "then": {"required": ["parameter", "variant", "preconditions"], "not": {"required": ["source"]}}, + "else": { + "required": ["source"], + "not": {"anyOf": [{"required": ["parameter"]}, {"required": ["variant"]}]} + } + } + ] + }, "expectationAttestation": { "type": "object", "required": ["status"], @@ -204,6 +291,10 @@ "expectations": { "type": "array", "items": {"$ref": "#/$defs/expectation"} + }, + "metamorphic": { + "type": "array", + "items": {"$ref": "#/$defs/metamorphicRelation"} } } }, @@ -237,6 +328,12 @@ "properties": { "implementation": { "type": "object", + "properties": { + "parameters": { + "type": "array", + "items": {"$ref": "#/$defs/runtimeParameter"} + } + }, "anyOf": [ {"required": ["pipeline"]}, {"required": ["command"]} diff --git a/openmapstack/verify.py b/openmapstack/verify.py index b03c2e3..f8626a8 100644 --- a/openmapstack/verify.py +++ b/openmapstack/verify.py @@ -29,6 +29,7 @@ from .checks import AssertionResult, not_testable from .checks import geodata as geodata_checks +from .checks import metamorphic as metamorphic_checks from .checks import overrides as overrides_checks from .checks import presentation as presentation_checks from .checks import project as project_checks @@ -37,7 +38,7 @@ from .checks import rerun as rerun_checks from .checks import validation as validation_checks from .expectations import evaluate_expectation -from .project import load_project +from .project import get_in, load_project from .rerun import perform_clean_rerun SCHEMA = "openmapstack-verify-result/v1" @@ -66,8 +67,9 @@ def to_dict(self) -> dict[str, Any]: } if self.args: payload["args"] = self.args - if self.evidence: - payload["evidence"] = self.evidence + evidence = self.evidence or (self.result.data or {}).get("evidence") + if evidence: + payload["evidence"] = evidence code = (self.result.data or {}).get("code") if code: payload["code"] = code @@ -189,9 +191,15 @@ def verify_project( *, rerun: bool = False, rerun_timeout_s: float = 1800, + metamorphic: bool = False, forbidden_fragments: Sequence[str] = (), ) -> VerifyResult: - """Run every applicable no-golden-answer check the environment supports.""" + """Run every applicable no-golden-answer check the environment supports. + + ``rerun`` and ``metamorphic`` both execute the project's canonical + entrypoint in isolated copies, so they are opt-in: the static plan must + stay cheap enough to run on every save. + """ project_file, manifest = load_project(project) root = project_file.parent result = VerifyResult(project_file=project_file) @@ -212,6 +220,8 @@ def verify_project( declared = [path for _, path, _ in _declared_output_paths(manifest)] if declared: _run(runs, "project.declared_files_exist", project_checks.declared_files_exist, root, files=declared) + if get_in(manifest, "runtime", "implementation", "parameters") is not None: + _run(runs, "project.parameters_match_steps", project_checks.parameters_match_steps, root) # -- provenance: sources are attributed, pinned, and licensed for name, fn in ( @@ -315,6 +325,24 @@ def verify_project( ): _run(runs, f"qgis.{name}", fn, root) + # -- metamorphic relations: declared invariants under controlled perturbation + relations = get_in(manifest, "validation", "metamorphic") + if relations is not None: + _run(runs, "metamorphic.declarations_valid", metamorphic_checks.declarations_valid, root) + if metamorphic and isinstance(relations, list): + for raw in relations: + relation_id = raw.get("id") if isinstance(raw, dict) else None + if not isinstance(relation_id, str) or not relation_id: + continue + _run( + runs, + f"metamorphic.{relation_id}", + metamorphic_checks.relation_holds, + root, + id=relation_id, + forbidden_fragments=list(forbidden_fragments), + ) + # -- reproducibility _run(runs, "rerun.no_chat_dependency", rerun_checks.no_chat_dependency, root) if rerun: diff --git a/references/project-spec.md b/references/project-spec.md index ec368c7..4a3cb30 100644 --- a/references/project-spec.md +++ b/references/project-spec.md @@ -328,6 +328,68 @@ evidence file makes the attestation stale and produces a warning. Attestation records reviewer evidence; it is not relabelled as an independently reproducible oracle in reports. +#### Metamorphic relations — invariants without a golden answer + +`validation.metamorphic[]` declares relations that must hold when an input or +parameter is perturbed in a controlled way. They need no frozen answer, so +they transfer to data nobody has an oracle for — and they are **conditional**: +each relation is valid only under preconditions the declaration must state, +and `openmapstack verify --metamorphic` reports `not_testable` with the reason +when a precondition does not hold on the actual data rather than guessing. + +```yaml +validation: + metamorphic: + - id: parcel-order + relation: input_permutation_invariance + source: {path: data/source/parcels.geojson} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: + tie_break: "candidates are keyed by cadastral_id; no selection depends on input order" + - id: parcel-duplicates + relation: duplicate_resistance + source: {path: data/source/parcels.geojson} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: {dedup_key: cadastral_id, measure: set} + - id: road-distance-monotonic + relation: positive_buffer_monotonicity + parameter: road_distance_m # declared under runtime.implementation.parameters + variant: {multiply: 3} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: {predicate: within_distance, expected: superset} + limits: {timeout_s: 600, max_source_bytes: 67108864} +``` + +| Relation | Transformation | Expected | Valid only when | +|---|---|---|---| +| `input_permutation_invariance` | source features shuffled (deterministic `seed`) | outputs semantically equal | a deterministic `tie_break` rule is declared and `key` is unique in the output | +| `duplicate_resistance` | every source feature appended once more | outputs equal | the analysis deduplicates on `dedup_key` (unique in the source) and the output is a keyed *set*; counts and sums are rejected | +| `positive_buffer_monotonicity` | a declared numeric parameter increased (`multiply` > 1 or `add` > 0) | every baseline key survives (`superset`) | the parameter drives an inclusion `predicate` (`within_distance`, `intersects_buffer`, `within_buffer`) | + +Each relation reruns the canonical entrypoint in an isolated copy prepared like +a clean rerun, perturbs only that copy, compares against the produced outputs, +and deletes the copy. The project's own `data/source/` and `data/overrides/` +are hashed before and after; a variant that mutates them fails. Unknown +relation names, `source` paths outside the immutable trees, non-growing +variants, and `duplicate_resistance` declared for a count or sum are +declaration failures, not skipped checks. + +**Counterexamples — do not enable a relation mechanically:** + +- a nearest-neighbour join with ties has no order invariance until the tie + rule is fixed in the pipeline; declare `tie_break` only once it is; +- a facility *count* per parcel is not duplicate-resistant — duplicating a + facility legitimately changes the count; only a keyed set of facilities is; +- an *exclusion* buffer (parcels farther than N m) is monotonic the other way; + `positive_buffer_monotonicity` only establishes `superset` for inclusion + predicates and refuses any other `predicate`; +- a threshold that also changes a classification (`tier1` below 1 km, `tier2` + below 2 km) keeps the key set but changes attributes; the relation only + compares keys, so declare it knowing that. + ### 2.7 Presentation semantics ```yaml @@ -535,6 +597,13 @@ runtime: dependencies: # project-local files needed in a clean run - requirements.lock - config/analysis.toml + parameters: # optional; how a variant run turns one knob + - id: road_distance_m + type: number # integer | number | string + canonical: 2000 + binding: {argument: "--road-distance-m"} # or {environment: OMS_ROAD_DISTANCE_M} + step: road_distance # optional pair: the step that consumes it + field: max_distance_m # ... whose value must equal `canonical` environment: python: "3.13" duckdb: "1.2.x" @@ -566,6 +635,14 @@ warnings: # known, unresolved data-quality limits consequential decisions. ``` +`runtime.implementation.parameters` is the versioned parameter-addressing +contract (`openmapstack-parameters/v1`). The canonical run passes nothing and +must produce the accepted result; a binding exists so a metamorphic relation or +a benchmark can run *the same pipeline with one knob turned* without editing +it. When `step`/`field` are given, `openmapstack verify` fails +`project.parameters_match_steps` if the step's declared value drifts from +`canonical` — the same honesty rule as `presentation.controls`. + **Warnings** give the explicit confidence/incompleteness handling. The rendered UX surfaces them (don't imply autoconfirmed geodata is current/complete). **Runs** capture what changed between executions and let a new engineer `rerun` tomorrow. The corresponding `runs/.json` record MUST contain `inputs` and `outputs` @@ -858,8 +935,9 @@ openmapstack inspect project.yaml - `verify` derives an applicable check plan from the manifest and inspects the produced artifacts without requiring a repository-owned golden answer. It reports partial execution as warning, evaluates only allowlisted and current - attestations, and optionally performs the independent clean rerun with - `--rerun`; `--json` includes applicability coverage and evidence class. + attestations, optionally performs the independent clean rerun with + `--rerun`, and optionally executes every declared metamorphic relation with + `--metamorphic`; `--json` includes applicability coverage and evidence class. - `run` first performs preflight validation, invokes exactly the pipeline or shell-free command in `runtime.implementation`, and then performs the full artifact validation. Python pipelines use the interpreter that installed the diff --git a/templates/project.yaml b/templates/project.yaml index 87096ee..47eec26 100644 --- a/templates/project.yaml +++ b/templates/project.yaml @@ -142,6 +142,16 @@ validation: # attestation: # status: unverified # reason: "Awaiting comparison with the authoritative register" + # Optional no-golden-answer invariants, executed by `openmapstack verify + # --metamorphic`. Declare a relation only when its precondition genuinely + # holds for this analysis (project-spec.md section 2.6 lists counterexamples). + metamorphic: [] + # - id: candidate-order + # relation: input_permutation_invariance + # source: {path: data/source/example.gpkg} + # outputs: [example_output] + # key: id + # preconditions: {tie_break: "keyed set; no order-dependent selection"} presentation: intent: analytical_workspace # analytical_workspace | discovery | report @@ -242,6 +252,15 @@ runtime: # dependencies: # - requirements.lock # - config/analysis.toml + # Declared knobs a variant run may turn (openmapstack-parameters/v1). + # The canonical run passes nothing; `step`/`field` let verify catch drift. + # parameters: + # - id: buffer_m + # type: number + # canonical: 2000 + # binding: {argument: "--buffer-m"} + # step: select_candidates + # field: max_distance_m environment: python: "3.13" duckdb: TODO diff --git a/tests/test_metamorphic.py b/tests/test_metamorphic.py new file mode 100644 index 0000000..e72bd7e --- /dev/null +++ b/tests/test_metamorphic.py @@ -0,0 +1,452 @@ +"""Metamorphic relations: positive, deliberate-defect, and invalid-precondition paths. + +Each relation is exercised three ways, on the same discipline as the mutation +cases: it must hold on a healthy pipeline, fail on a pipeline with exactly the +defect it exists to catch, and refuse (``not_testable`` or a declaration +failure) when its preconditions do not hold -- because a relation asserted +where it is not valid is a false failure, and a relation skipped silently is +a false pass. +""" + +from __future__ import annotations + +import io +import json +import os +import tempfile +import textwrap +import unittest +from contextlib import redirect_stdout +from copy import deepcopy +from pathlib import Path + +import yaml + +from openmapstack.checks import metamorphic as metamorphic_checks +from openmapstack.checks.project import parameters_match_steps +from openmapstack.cli import main +from openmapstack.metamorphic import DeclarationError, parse_declaration, run_relation +from openmapstack.parameters import ParameterError, declared_parameters +from openmapstack.schema import project_schema_errors +from openmapstack.verify import verify_project +from tests.evals.helpers import make_workspace, minimal_project, write_project + +# A deliberately boring pipeline: select points with x <= max_x, keyed by +# pid, deduplicated, written in pid order. ``MODE`` injects one defect. +PIPELINE = textwrap.dedent( + ''' + import json, os, sys, time + from pathlib import Path + + ROOT = Path(__file__).resolve().parent + MODE = {mode!r} + max_x = float(os.environ.get("OMS_MAX_X", "10")) + argv = sys.argv[1:] + if "--max-x" in argv: + max_x = float(argv[argv.index("--max-x") + 1]) + if MODE == "crash": + raise SystemExit(3) + if MODE == "slow": + time.sleep(5) + if MODE == "reach_back": + target = Path((ROOT / "data/source/origin.txt").read_text().strip()) + with target.open("a") as fh: + fh.write("touched\\n") + doc = json.loads((ROOT / "data/source/points.geojson").read_text()) + selected = {{}} + for index, feature in enumerate(doc["features"]): + x = feature["geometry"]["coordinates"][0] + keep = x >= max_x if MODE == "inverted" else x <= max_x + if not keep: + continue + pid = feature["properties"]["pid"] + props = dict(feature["properties"]) + if MODE == "order_dependent": + props["rank"] = index + out = {{"type": "Feature", "properties": props, "geometry": feature["geometry"]}} + if MODE == "duplicate_sensitive": + selected[(pid, index)] = out + else: + selected.setdefault(pid, out) + features = [selected[key] for key in sorted(selected, key=lambda k: str(k))] + (ROOT / "data/derived").mkdir(parents=True, exist_ok=True) + (ROOT / "data/derived/selected.geojson").write_text( + json.dumps({{"type": "FeatureCollection", "features": features}}) + ) + ''' +) + +POINTS = { + "type": "FeatureCollection", + "features": [ + {"type": "Feature", "properties": {"pid": f"p{i}"}, "geometry": {"type": "Point", "coordinates": [float(i), 0.0]}} + for i in range(1, 16) + ], +} + + +def _permutation(**extra): + declaration = { + "id": "point-order", + "relation": "input_permutation_invariance", + "source": {"path": "data/source/points.geojson"}, + "outputs": ["selected"], + "key": "pid", + "preconditions": {"tie_break": "selection is keyed by pid; output sorted by pid"}, + } + declaration.update(extra) + return declaration + + +def _duplicates(**extra): + declaration = { + "id": "point-duplicates", + "relation": "duplicate_resistance", + "source": {"path": "data/source/points.geojson"}, + "outputs": ["selected"], + "key": "pid", + "preconditions": {"dedup_key": "pid", "measure": "set"}, + } + declaration.update(extra) + return declaration + + +def _monotonic(**extra): + declaration = { + "id": "max-x-monotonic", + "relation": "positive_buffer_monotonicity", + "parameter": "max_x", + "variant": {"multiply": 1.5}, + "outputs": ["selected"], + "key": "pid", + "preconditions": {"predicate": "within_distance", "expected": "superset"}, + } + declaration.update(extra) + return declaration + + +def _parameter(**extra): + parameter = { + "id": "max_x", + "type": "number", + "canonical": 10, + "binding": {"argument": "--max-x"}, + "step": "select", + "field": "max_x", + } + parameter.update(extra) + return parameter + + +class _ProjectMixin: + def build(self, *, mode: str = "healthy", relations=None, parameters=None, run: bool = True): + workspace = make_workspace() + project = minimal_project() + project["processing"]["steps"] = [ + {"id": "load", "operation": "read", "source": "test_source", "output": "points"}, + {"id": "select", "operation": "distance_filter", "input": "points", "max_x": 10, "crs": "EPSG:3301", "output": "selected"}, + ] + project["outputs"] = {"selected": {"path": "data/derived/selected.geojson", "format": "GeoJSON", "generated_by": "select"}} + project["runtime"]["implementation"]["parameters"] = parameters if parameters is not None else [_parameter()] + project["validation"]["metamorphic"] = relations if relations is not None else [] + write_project(workspace, project) + (workspace / "pipeline.py").write_text(PIPELINE.format(mode=mode), encoding="utf-8") + (workspace / "data/source").mkdir(parents=True) + (workspace / "data/source/points.geojson").write_text(json.dumps(POINTS), encoding="utf-8") + if run: + import subprocess, sys + + subprocess.run([sys.executable, "pipeline.py"], cwd=workspace, check=True) + return workspace, project + + def relation(self, workspace, project, declaration): + return run_relation(workspace, project, declaration) + + +class PermutationInvarianceTests(_ProjectMixin, unittest.TestCase): + def test_holds_on_a_keyed_pipeline(self) -> None: + workspace, project = self.build() + result, evidence = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "passed", result.detail) + self.assertEqual(evidence["variant"]["transformation"], "permute_features") + + def test_detects_an_order_dependent_pipeline(self) -> None: + workspace, project = self.build(mode="order_dependent") + result, _ = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "permutation_changed_output") + + def test_requires_a_declared_tie_break_rule(self) -> None: + workspace, project = self.build() + result, evidence = self.relation(workspace, project, _permutation(preconditions={})) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "metamorphic_declaration_invalid") + self.assertEqual(evidence["class"], "invalid") + self.assertIn("tie_break", result.detail) + + def test_non_unique_output_key_is_not_testable(self) -> None: + # Set semantics cannot be asserted over a key that repeats. + workspace, project = self.build(mode="duplicate_sensitive") + doc = json.loads((workspace / "data/source/points.geojson").read_text()) + doc["features"].append(deepcopy(doc["features"][0])) + (workspace / "data/source/points.geojson").write_text(json.dumps(doc)) + import subprocess, sys + + subprocess.run([sys.executable, "pipeline.py"], cwd=workspace, check=True) + result, _ = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "precondition_unmet") + + +class DuplicateResistanceTests(_ProjectMixin, unittest.TestCase): + def test_holds_on_a_deduplicating_pipeline(self) -> None: + workspace, project = self.build() + result, evidence = self.relation(workspace, project, _duplicates()) + self.assertEqual(result.status, "passed", result.detail) + self.assertEqual(evidence["variant"]["duplicated"], len(POINTS["features"])) + + def test_detects_a_duplicate_sensitive_pipeline(self) -> None: + workspace, project = self.build(mode="duplicate_sensitive") + result, _ = self.relation(workspace, project, _duplicates()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "duplicates_changed_output") + + def test_rejects_count_and_sum_semantics(self) -> None: + # Counts legitimately double when rows are duplicated; declaring the + # relation for them is invalid use, not a relation that happens to fail. + workspace, project = self.build() + result, _ = self.relation( + workspace, project, _duplicates(preconditions={"dedup_key": "pid", "measure": "count"}) + ) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "metamorphic_declaration_invalid") + self.assertIn("set semantics", result.detail) + + def test_source_that_already_has_duplicates_is_not_testable(self) -> None: + workspace, project = self.build(run=False) + doc = json.loads((workspace / "data/source/points.geojson").read_text()) + doc["features"].append(deepcopy(doc["features"][0])) + (workspace / "data/source/points.geojson").write_text(json.dumps(doc)) + import subprocess, sys + + subprocess.run([sys.executable, "pipeline.py"], cwd=workspace, check=True) + result, _ = self.relation(workspace, project, _duplicates()) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "precondition_unmet") + self.assertIn("already has duplicate", result.detail) + + +class BufferMonotonicityTests(_ProjectMixin, unittest.TestCase): + def test_holds_when_a_larger_threshold_keeps_every_baseline_feature(self) -> None: + workspace, project = self.build() + result, evidence = self.relation(workspace, project, _monotonic()) + self.assertEqual(result.status, "passed", result.detail) + self.assertEqual(evidence["parameter"], {"id": "max_x", "canonical": 10, "variant": 15.0}) + self.assertEqual(evidence["command"][-2:], ["--max-x", "15"]) + self.assertEqual(evidence["counts"]["selected"], {"baseline": 10, "variant": 15}) + + def test_detects_an_inverted_predicate(self) -> None: + workspace, project = self.build(mode="inverted") + result, _ = self.relation(workspace, project, _monotonic()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "monotonicity_violated") + self.assertIn("lost", result.detail) + + def test_environment_binding_is_honoured(self) -> None: + workspace, project = self.build(parameters=[_parameter(binding={"environment": "OMS_MAX_X"})]) + result, evidence = self.relation(workspace, project, _monotonic()) + self.assertEqual(result.status, "passed", result.detail) + self.assertEqual(evidence["variant"]["environment"], ["OMS_MAX_X"]) + + def test_undeclared_parameter_is_a_declaration_failure(self) -> None: + workspace, project = self.build(parameters=[]) + result, _ = self.relation(workspace, project, _monotonic()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "metamorphic_declaration_invalid") + + def test_non_numeric_parameter_is_not_testable(self) -> None: + workspace, project = self.build( + parameters=[{"id": "max_x", "type": "string", "canonical": "ten", "binding": {"argument": "--max-x"}}] + ) + result, _ = self.relation(workspace, project, _monotonic()) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "precondition_unmet") + + def test_variant_must_strictly_grow(self) -> None: + with self.assertRaises(DeclarationError): + parse_declaration(_monotonic(variant={"multiply": 1})) + with self.assertRaises(DeclarationError): + parse_declaration(_monotonic(variant={"add": -5})) + with self.assertRaises(DeclarationError): + parse_declaration(_monotonic(preconditions={"predicate": "outside_distance"})) + + +class RelationSafetyTests(_ProjectMixin, unittest.TestCase): + def test_unknown_relation_is_rejected_not_skipped(self) -> None: + with self.assertRaises(DeclarationError): + parse_declaration(_permutation(relation="crs_round_trip_stability")) + + def test_unsupported_source_format_is_not_testable(self) -> None: + workspace, project = self.build() + (workspace / "data/source/points.gpkg").write_bytes(b"not really") + result, _ = self.relation(workspace, project, _permutation(source={"path": "data/source/points.gpkg"})) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "unsupported_format") + + def test_source_outside_immutable_trees_is_rejected(self) -> None: + with self.assertRaises(DeclarationError): + parse_declaration(_permutation(source={"path": "data/derived/selected.geojson"})) + + def test_oversize_source_is_a_resource_limit(self) -> None: + workspace, project = self.build() + result, _ = self.relation(workspace, project, _permutation(limits={"max_source_bytes": 10})) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "resource_limit") + + def test_missing_baseline_output_is_not_testable(self) -> None: + workspace, project = self.build(run=False) + result, _ = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "baseline_missing") + + def test_crashing_variant_is_a_failure(self) -> None: + workspace, project = self.build() + (workspace / "pipeline.py").write_text(PIPELINE.format(mode="crash"), encoding="utf-8") + result, evidence = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "variant_execution_failed") + self.assertIn("stderr_tail", evidence) + + def test_timeout_is_not_testable_and_the_variant_is_removed(self) -> None: + workspace, project = self.build() + (workspace / "pipeline.py").write_text(PIPELINE.format(mode="slow"), encoding="utf-8") + before = {p.name for p in Path(tempfile.gettempdir()).iterdir() if p.name.startswith("openmapstack-metamorphic-")} + result, _ = self.relation(workspace, project, _permutation(limits={"timeout_s": 0.5})) + after = {p.name for p in Path(tempfile.gettempdir()).iterdir() if p.name.startswith("openmapstack-metamorphic-")} + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "variant_timeout") + self.assertEqual(after - before, set()) + + def test_variant_that_mutates_the_original_source_fails(self) -> None: + workspace, project = self.build() + origin = workspace / "data/source/origin.txt" + origin.write_text(str(workspace / "data/source/points.geojson")) + (workspace / "pipeline.py").write_text(PIPELINE.format(mode="reach_back"), encoding="utf-8") + result, _ = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "original_source_mutated") + + def test_variant_workspace_is_removed_after_success(self) -> None: + workspace, project = self.build() + before = {p.name for p in Path(tempfile.gettempdir()).iterdir() if p.name.startswith("openmapstack-metamorphic-")} + self.relation(workspace, project, _permutation()) + after = {p.name for p in Path(tempfile.gettempdir()).iterdir() if p.name.startswith("openmapstack-metamorphic-")} + self.assertEqual(after - before, set()) + # And the original workspace still has exactly what it started with. + self.assertEqual(json.loads((workspace / "data/source/points.geojson").read_text()), POINTS) + + +class ParameterContractTests(_ProjectMixin, unittest.TestCase): + def test_parameters_parse_and_bind(self) -> None: + manifest = {"runtime": {"implementation": {"parameters": [_parameter()]}}, + "processing": {"steps": [{"id": "select", "max_x": 10}]}} + [parameter] = declared_parameters(manifest) + self.assertEqual(parameter.bind(15.0), (["--max-x", "15"], {})) + self.assertEqual(parameter.bind(12.5), (["--max-x", "12.5"], {})) + + def test_drift_between_canonical_and_step_is_rejected(self) -> None: + manifest = {"runtime": {"implementation": {"parameters": [_parameter(canonical=2000)]}}, + "processing": {"steps": [{"id": "select", "max_x": 10}]}} + with self.assertRaises(ParameterError) as caught: + declared_parameters(manifest) + self.assertIn("!= processing step", str(caught.exception)) + + def test_malformed_bindings_and_types_are_rejected(self) -> None: + for bad in ( + _parameter(binding={}), + _parameter(binding={"argument": "max-x"}), + _parameter(binding={"environment": "lower"}), + _parameter(binding={"argument": "--max-x", "environment": "OMS_MAX_X"}), + _parameter(type="number", canonical=True), + _parameter(type="integer", canonical=1.5), + {k: v for k, v in _parameter().items() if k != "field"}, + _parameter(step="missing", field="max_x"), + _parameter(id="not an identifier"), + ): + manifest = {"runtime": {"implementation": {"parameters": [bad]}}, + "processing": {"steps": [{"id": "select", "max_x": 10}]}} + with self.assertRaises(ParameterError, msg=bad): + declared_parameters(manifest) + + def test_project_check_reports_drift_and_absence(self) -> None: + workspace, _ = self.build(run=False) + self.assertEqual(parameters_match_steps(workspace).status, "passed") + workspace, _ = self.build(parameters=[_parameter(canonical=99)], run=False) + result = parameters_match_steps(workspace) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "parameters_invalid") + workspace, _ = self.build(parameters=[], run=False) + self.assertEqual(parameters_match_steps(workspace).status, "not_testable") + + def test_schema_accepts_the_contract_and_rejects_unknown_relations(self) -> None: + _, project = self.build(relations=[_permutation(), _duplicates(), _monotonic()], run=False) + project["runs"] = {"latest": {"id": "run-1", "started_at": "x", "completed_at": "x", "status": "passed", + "inputs_hash": "sha256:" + "0" * 64, "outputs_hash": "sha256:" + "0" * 64, + "validation_report": {"path": "validation/latest-report.json"}}} + self.assertEqual(project_schema_errors(project), []) + broken = deepcopy(project) + broken["validation"]["metamorphic"][0]["relation"] = "subset_additivity" + self.assertTrue(project_schema_errors(broken)) + broken = deepcopy(project) + broken["runtime"]["implementation"]["parameters"][0]["binding"] = {"argument": "bad flag"} + self.assertTrue(project_schema_errors(broken)) + + +class VerifyIntegrationTests(_ProjectMixin, unittest.TestCase): + def test_declarations_are_checked_statically_and_executed_only_on_request(self) -> None: + workspace, _ = self.build(relations=[_permutation(), _monotonic()]) + static = verify_project(workspace / "project.yaml") + names = [run.name for run in static.checks] + self.assertIn("metamorphic.declarations_valid", names) + self.assertIn("project.parameters_match_steps", names) + self.assertNotIn("metamorphic.point-order", names) + + executed = verify_project(workspace / "project.yaml", metamorphic=True) + by_name = {run.name: run for run in executed.checks} + self.assertEqual(by_name["metamorphic.point-order"].result.status, "passed") + self.assertEqual(by_name["metamorphic.max-x-monotonic"].result.status, "passed") + payload = executed.to_dict() + entry = next(item for item in payload["checks"] if item["check"] == "metamorphic.point-order") + self.assertEqual(entry["evidence"]["relation"], "input_permutation_invariance") + + def test_invalid_declaration_fails_the_static_plan(self) -> None: + workspace, _ = self.build(relations=[_duplicates(preconditions={"dedup_key": "pid", "measure": "count"})]) + result = verify_project(workspace / "project.yaml") + run = next(r for r in result.checks if r.name == "metamorphic.declarations_valid") + self.assertEqual(run.result.status, "failed") + self.assertEqual(result.status, "failed") + + def test_cli_flag_runs_relations(self) -> None: + workspace, _ = self.build(mode="inverted", relations=[_monotonic()]) + out = io.StringIO() + with redirect_stdout(out): + code = main(["verify", str(workspace / "project.yaml"), "--metamorphic"]) + self.assertEqual(code, 1) + self.assertIn("FAIL metamorphic.max-x-monotonic", out.getvalue()) + self.assertIn("lost 5 baseline feature(s)", out.getvalue()) + + def test_check_library_entry_matches_declared_id(self) -> None: + workspace, _ = self.build(relations=[_permutation()]) + self.assertEqual(metamorphic_checks.relation_holds(workspace, id="point-order").status, "passed") + missing = metamorphic_checks.relation_holds(workspace, id="nope") + self.assertEqual(missing.status, "failed") + self.assertEqual(missing.data["code"], "metamorphic_relation_undeclared") + self.assertEqual(metamorphic_checks.declarations_valid(workspace).status, "passed") + (workspace / "project.yaml").write_text( + yaml.safe_dump(minimal_project(), sort_keys=False), encoding="utf-8" + ) + self.assertEqual(metamorphic_checks.declarations_valid(workspace).status, "not_testable") + + +if __name__ == "__main__": + unittest.main() From 8a0379989baf8f293f7b65280e495e098165b197 Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 2 Sep 2026 16:50:51 +0000 Subject: [PATCH 2/8] feat(sources): typed pin classes and credential hygiene for warehouse sources A source whose bytes come from a query must say which of two pin classes froze them: a hash-matched local snapshot under data/source/, or a backend snapshot with an identifier, capture time, and retention limit. A snapshot that cannot deliver its bytes again -- missing, edited, expired, or last verified inaccessible -- is reported as not_reproducible rather than pinned because a string is present. A mutable alias such as 'latest' in version.identifier stays unpinned whatever else is declared. Secrets never enter project.yaml: access.connection must be a reference (env:, service:, keyring:, or a file outside the project) and every source block is scanned for password fragments, credentialed URLs, and well-known key shapes. Findings name the path and pattern, never the secret. The fixture generator pins all three sources as local snapshots and gains two break modes; cases 926 and 927 prove the failure paths. Refs #13 (B5a). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01TMMy6A73zi7yGjK3wHBhju --- docs/verify-applicability.md | 3 +- evals/COVERAGE.md | 3 + .../expected.yaml | 25 ++ .../927-inline-credentials/expected.yaml | 24 ++ evals/fixtures/reference_pipeline/gen.py | 34 ++- openmapstack/checks/provenance.py | 66 ++++- openmapstack/schemas/project-v1.schema.json | 62 ++++- openmapstack/sources.py | 250 ++++++++++++++++++ openmapstack/validation.py | 19 ++ openmapstack/verify.py | 1 + references/project-spec.md | 55 ++++ templates/project.yaml | 8 + tests/test_sources.py | 221 ++++++++++++++++ 13 files changed, 755 insertions(+), 16 deletions(-) create mode 100644 evals/cases/926-expired-backend-snapshot/expected.yaml create mode 100644 evals/cases/927-inline-credentials/expected.yaml create mode 100644 openmapstack/sources.py create mode 100644 tests/test_sources.py diff --git a/docs/verify-applicability.md b/docs/verify-applicability.md index c27fb5f..31376dc 100644 --- a/docs/verify-applicability.md +++ b/docs/verify-applicability.md @@ -35,7 +35,8 @@ checker where applicable. | `project.parameters_match_steps` | `runtime.implementation.parameters` is declared | parameters are well-formed and agree with the processing steps they are bound to | none; malformed or drifting declarations fail | direct parameter-contract tests; case 015 | | `project.declared_files_exist` | at least one output path is declared | all declared outputs exist | none; missing output fails | 001, 909 | | `provenance.every_source_has_provider_and_access` | always | sources identify provider, method, and retrieval time | none | broad contract suite | -| `provenance.every_source_pinned` | always | sources carry a non-`latest` version identity | current checker does not yet recognise warehouse snapshot classes | 906 | +| `provenance.every_source_pinned` | always | every source is pinned by version identity, a hash-matched local snapshot, or an unexpired, accessible backend snapshot | none; a mutable alias is `source_unpinned`, a snapshot that cannot deliver its bytes again is `not_reproducible`, a malformed pin is `pin_invalid` | 906, 926; direct pin-class tests | +| `provenance.no_inline_credentials` | always | no source embeds a secret and `access.connection` is a reference | none | 927; direct credential-hygiene tests | | `provenance.license_present_where_required` | always | every source has declared licence metadata | none | broad contract suite | | `provenance.rationale_present` | always | source selection rationale is recorded | none | broad contract suite | | `overrides.every_override_has_provenance` | always | declared overrides carry provenance | none; zero overrides is valid | 002, 003, 012, 013, 920, 921 | diff --git a/evals/COVERAGE.md b/evals/COVERAGE.md index 0013b85..1f3ad99 100644 --- a/evals/COVERAGE.md +++ b/evals/COVERAGE.md @@ -49,6 +49,9 @@ Legend: ✅ covered · ⚠️ partially covered · ❌ not covered (tracked belo | Risk | Positive | Mutation | |---|---|---| | Pinned versions | every case | 906 `unpinned-source` | +| Pin classes (hash-matched local snapshot) | every case (all three fixture sources carry `pin: local_snapshot`) | 911 `mutated-source` (byte identity) | +| Expired / inaccessible backend snapshot is `not_reproducible` | unit tests | 926 `expired-backend-snapshot` | +| No credentials in `project.yaml`; connections by reference | every case | 927 `inline-credentials` | | Completeness reporting | 001 (mini-Tartu) | 907 `incomplete-pagination` | | Source immutability (byte identity) | every case (auto-inserted check) | 911 `mutated-source` | | Provider/access metadata | every case | — | diff --git a/evals/cases/926-expired-backend-snapshot/expected.yaml b/evals/cases/926-expired-backend-snapshot/expected.yaml new file mode 100644 index 0000000..f01a425 --- /dev/null +++ b/evals/cases/926-expired-backend-snapshot/expected.yaml @@ -0,0 +1,25 @@ +id: 926-expired-backend-snapshot +case_type: mutation +modes: [fixture] +score_types: { fixture: mutation_tests } +project_dir: project +hard_gate: true +mutation: + control_generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir} --break=expired_snapshot" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +# A backend snapshot id plus a timestamp is not a pin once the backend may +# have dropped the snapshot. The source must read as not reproducible, not +# as pinned because a string is present. +assertions: + - assert: project.conforms_to_schema + - assert: validation.run_record_matches + - assert: provenance.no_inline_credentials + - assert: provenance.every_source_pinned + expect: failed + expect_code: not_reproducible diff --git a/evals/cases/927-inline-credentials/expected.yaml b/evals/cases/927-inline-credentials/expected.yaml new file mode 100644 index 0000000..0213d86 --- /dev/null +++ b/evals/cases/927-inline-credentials/expected.yaml @@ -0,0 +1,24 @@ +id: 927-inline-credentials +case_type: mutation +modes: [fixture] +score_types: { fixture: mutation_tests } +project_dir: project +hard_gate: true +mutation: + control_generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir} --break=inline_credentials" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +# Secrets never enter project.yaml. A credentialed DSN under access.connection +# is rejected even when everything else about the source is in order. +assertions: + - assert: project.conforms_to_schema + - assert: validation.run_record_matches + - assert: provenance.every_source_pinned + - assert: provenance.no_inline_credentials + expect: failed + expect_code: inline_credentials diff --git a/evals/fixtures/reference_pipeline/gen.py b/evals/fixtures/reference_pipeline/gen.py index 61169af..c9241b9 100755 --- a/evals/fixtures/reference_pipeline/gen.py +++ b/evals/fixtures/reference_pipeline/gen.py @@ -26,6 +26,8 @@ qgis_incomplete_crs QGIS layers carry authid-only invalid CRS blocks incomplete_pagination roads source reports numberMatched > returned mutated_source pipeline rewrites its own copied "immutable" source file + expired_snapshot pois source pinned to a backend snapshot whose retention has lapsed + inline_credentials roads source embeds a credentialed connection string order_dependent candidate rows carry their input file position (order-dependent output) distance_inverted road-distance predicate inverted (>= instead of <=) duplicate_sensitive duplicated source parcels produce duplicated candidates @@ -340,6 +342,33 @@ def build( source_identifier = "latest" if break_mode == "unpinned_source" else "mini-tartu-fixture-v1" + # Every copied source is a user-approved local snapshot: its pin is the + # real content hash of the bytes in data/source/, so a swapped or edited + # file is as visible as a swapped version tag. + def local_pin(name: str) -> dict: + return { + "class": "local_snapshot", + "path": f"data/source/{name}", + "sha256": _sha256_bytes((output_dir / "data" / "source" / name).read_bytes()), + "captured_at": "2026-08-25T08:00:00Z", + } + + pois_pin = local_pin("pois.geojson") + if break_mode == "expired_snapshot": + pois_pin = { + "class": "backend_snapshot", + "identifier": "pg_export_snapshot:00000003-000001A8-1", + "captured_at": "2026-08-25T08:00:00Z", + "retention_until": "2026-08-26T08:00:00Z", + } + roads_access = {"method": "local", "retrieved_at": "2026-08-25T08:00:00Z"} + if break_mode == "inline_credentials": + roads_access = { + "method": "postgis", + "retrieved_at": "2026-08-25T08:00:00Z", + "connection": "postgresql://gis:hunter2@db.example.invalid:5432/gis", + } + project = { "schema": "openmapstack-project/v1", "project": { @@ -363,6 +392,7 @@ def build( "dataset": "mini-tartu parcels", "source_url": "file://evals/fixtures/mini-tartu/parcels.geojson", "access": {"method": "local", "retrieved_at": "2026-08-25T08:00:00Z"}, "version": {"published_at": "2026-08-25", "identifier": source_identifier}, + "pin": local_pin("parcels.geojson"), "selection": { "filter": "area_m2 >= 8000", "semantic_predicates": [{ @@ -377,8 +407,9 @@ def build( "roads": { "role": "authoritative_input", "provider": "eval-fixture", "dataset": "mini-tartu roads", "source_url": "file://evals/fixtures/mini-tartu/roads.geojson", - "access": {"method": "local", "retrieved_at": "2026-08-25T08:00:00Z"}, + "access": roads_access, "version": {"published_at": "2026-08-25", "identifier": "mini-tartu-fixture-v1"}, + "pin": local_pin("roads.geojson"), "selection": ({"completeness": {"matched": 5, "returned": 1, "page_size": 1, "pages": 1}} if break_mode == "incomplete_pagination" else {"completeness": {"matched": 1, "returned": 1}}), @@ -390,6 +421,7 @@ def build( "dataset": "mini-tartu pois", "source_url": "file://evals/fixtures/mini-tartu/pois.geojson", "access": {"method": "local", "retrieved_at": "2026-08-25T08:00:00Z"}, "version": {"published_at": "2026-08-25", "identifier": "mini-tartu-fixture-v1"}, + "pin": pois_pin, "selection": ({} if uncertain_completeness else {"completeness": {"matched": 2, "returned": 2}}), "license": {"name": "eval fixture, public domain", "url": "https://example.invalid/license"}, "rationale": "Small deterministic fixture for CI evals.", diff --git a/openmapstack/checks/provenance.py b/openmapstack/checks/provenance.py index 3e12d3b..c59eb5f 100644 --- a/openmapstack/checks/provenance.py +++ b/openmapstack/checks/provenance.py @@ -7,7 +7,7 @@ from pathlib import Path -from . import AssertionResult, failed, get_in, load_project_yaml, passed, warning +from . import AssertionResult, failed, get_in, load_project_yaml, passed, project_root, warning def every_source_has_provider_and_access(workspace: Path, project_dir: str = ".") -> AssertionResult: @@ -31,25 +31,65 @@ def every_source_has_provider_and_access(workspace: Path, project_dir: str = "." def every_source_pinned(workspace: Path, project_dir: str = ".") -> AssertionResult: - """Pinning to 'latest' is not reproducible — version.identifier/published_at - must be present and not equal to the literal string 'latest'.""" + """Every source is reproducibly pinned. + + A source without a ``pin`` block must carry a ``version.identifier`` or + ``published_at`` that is not a mutable alias such as ``latest``. A source + with a pin block is held to its pin class (``openmapstack.sources``): a + local snapshot must exist and match its hash, and a backend snapshot must + be identified, unexpired, and not known to be inaccessible. A pin that + cannot deliver the bytes again is ``not_reproducible`` -- a timestamp + string alone does not make a warehouse table pinned. + """ + from ..sources import source_pin_summary + + proj = load_project_yaml(workspace, project_dir) + if proj is None: + return failed("project.yaml missing", code="manifest_missing") + sources = proj.get("sources") or {} + if not sources: + return failed("no sources declared", code="no_sources") + assessments = source_pin_summary(project_root(workspace, project_dir), sources) + by_status: dict[str, list[str]] = {} + for key, assessment in assessments.items(): + by_status.setdefault(assessment.status, []).append(f"{key}: {assessment.reason}") + if by_status.get("invalid"): + return failed(f"sources with malformed pins: {by_status['invalid']}", code="pin_invalid") + if by_status.get("not_reproducible"): + return failed( + f"sources whose pinned snapshot cannot be obtained again: {by_status['not_reproducible']}", + code="not_reproducible", + causes={key: assessment.details.get("cause") for key, assessment in assessments.items() if assessment.status == "not_reproducible"}, + ) + if by_status.get("unpinned"): + return failed(f"sources not pinned to a version/identifier: {by_status['unpinned']}", code="source_unpinned") + classes = sorted({assessment.pin_class for assessment in assessments.values()}) + return passed(f"all {len(sources)} sources are pinned ({', '.join(classes)})", pin_classes=classes) + + +def no_inline_credentials(workspace: Path, project_dir: str = ".") -> AssertionResult: + """No source embeds a secret, and warehouse connections are by reference.""" + from ..sources import connection_reference_error, find_inline_credentials + proj = load_project_yaml(workspace, project_dir) if proj is None: return failed("project.yaml missing", code="manifest_missing") sources = proj.get("sources") or {} if not sources: return failed("no sources declared", code="no_sources") - unpinned: list[str] = [] + root = project_root(workspace, project_dir) + problems: list[str] = [] for key, src in sources.items(): - identifier = get_in(src, "version.identifier") - published_at = get_in(src, "version.published_at") - if not identifier and not published_at: - unpinned.append(key) - elif str(identifier).strip().lower() == "latest": - unpinned.append(key) - if unpinned: - return failed(f"sources not pinned to a version/identifier: {unpinned}", code="source_unpinned") - return passed(f"all {len(sources)} sources are pinned") + if not isinstance(src, dict): + continue + for finding in find_inline_credentials(src, f"sources.{key}"): + problems.append(f"{finding['path']} ({finding['pattern']})") + error = connection_reference_error(root, get_in(src, "access.connection")) + if error: + problems.append(f"sources.{key}.access.connection: {error}") + if problems: + return failed(f"credentials or connection strings embedded in the manifest: {problems}", code="inline_credentials") + return passed(f"no inline credentials in {len(sources)} sources; connections are by reference") def license_present_where_required( diff --git a/openmapstack/schemas/project-v1.schema.json b/openmapstack/schemas/project-v1.schema.json index b01525d..9853486 100644 --- a/openmapstack/schemas/project-v1.schema.json +++ b/openmapstack/schemas/project-v1.schema.json @@ -111,6 +111,52 @@ } ] }, + "sourcePin": { + "type": "object", + "required": ["class", "captured_at"], + "properties": { + "class": {"enum": ["local_snapshot", "backend_snapshot"]}, + "captured_at": {"type": "string", "minLength": 1}, + "path": {"$ref": "#/$defs/safeProjectPath"}, + "sha256": {"$ref": "#/$defs/digest"}, + "identifier": {"type": "string", "minLength": 1}, + "retention_until": {"type": "string", "minLength": 1}, + "verification": { + "type": "object", + "required": ["at", "status"], + "properties": { + "at": {"type": "string", "minLength": 1}, + "status": {"enum": ["accessible", "inaccessible"]} + } + }, + "note": {"type": "string"} + }, + "allOf": [ + { + "if": {"properties": {"class": {"const": "local_snapshot"}}}, + "then": {"required": ["path", "sha256"]}, + "else": {"required": ["identifier", "retention_until"]} + } + ] + }, + "connectionReference": { + "description": "A credential reference (env:, file:, service:, keyring:). The shape is open here so that an embedded DSN is rejected by the semantic credential check with a useful message rather than by a regex.", + "type": ["string", "object"] + }, + "warehouseSource": { + "type": "object", + "required": ["backend"], + "properties": { + "backend": {"type": "string", "minLength": 1}, + "account": {"type": "string"}, + "database": {"type": "string"}, + "schema": {"type": "string"}, + "table": {"type": "string"}, + "query_sha256": {"$ref": "#/$defs/digest"}, + "schema_sha256": {"$ref": "#/$defs/digest"}, + "snapshot_capability": {"type": "string"} + } + }, "expectationAttestation": { "type": "object", "required": ["status"], @@ -270,7 +316,21 @@ "assumptions": {"type": "array"} } }, - "sources": {"type": "object", "minProperties": 1}, + "sources": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "properties": { + "pin": {"$ref": "#/$defs/sourcePin"}, + "warehouse": {"$ref": "#/$defs/warehouseSource"}, + "access": { + "type": "object", + "properties": {"connection": {"$ref": "#/$defs/connectionReference"}} + } + } + } + }, "overrides": {"type": "array"}, "processing": { "type": "object", diff --git a/openmapstack/sources.py b/openmapstack/sources.py new file mode 100644 index 0000000..4a3ebc3 --- /dev/null +++ b/openmapstack/sources.py @@ -0,0 +1,250 @@ +"""Source pin classes and credential hygiene for ``sources.*``. + +A source is reproducible only if the bytes it contributed can be obtained +again. ``version.identifier`` answers that for a published release or a +STAC item; it does not for a warehouse table, whose "version" is a moving +target unless something freezes it. This module recognises two pin classes +that do freeze it, and reports honestly when neither holds. + +.. code-block:: yaml + + sources: + parcels: + access: + method: postgis + connection: {ref: "env:PARCELS_DSN"} # credentials by reference only + warehouse: + backend: postgis # duckdb | postgis (pilot) + account: geo-prod # host/project identity, no secrets + database: gis + schema: cadastre + table: parcels + query_sha256: "sha256:..." # digest of the exact SELECT + schema_sha256: "sha256:..." # digest of the discovered columns + pin: + class: local_snapshot # (1) user-approved local copy + path: data/source/parcels.parquet + sha256: "sha256:..." + captured_at: "2026-08-30T10:00:00Z" + # -- or -- + pin: + class: backend_snapshot # (2) backend time travel / snapshot + identifier: "pg_export_snapshot:00000003-000001A8-1" + captured_at: "2026-08-30T10:00:00Z" + retention_until: "2026-12-31T00:00:00Z" # when the backend may drop it + verification: {at: "2026-08-30T10:05:00Z", status: accessible} + +``assess_pin`` returns one of: + +- ``pinned``: the pin class validates (local bytes match, or the backend + snapshot is identified, unexpired, and not known to be inaccessible); +- ``not_reproducible``: a pin is declared but cannot deliver the bytes again + (missing or changed snapshot file, expired or inaccessible backend snapshot); +- ``unpinned``: no pin and no usable version identity, or a mutable alias + such as ``latest``; +- ``invalid``: the pin block is malformed. + +A source without a ``pin`` block keeps the original rule: a non-``latest`` +``version.identifier`` or ``published_at`` counts as pinned. That rule is +adequate for immutable published releases and is all a file download needs. + +Secrets never belong in ``project.yaml``. ``find_inline_credentials`` walks +a source block for password fragments, credentialed URLs, and well-known key +shapes, and ``connection_reference_error`` requires ``access.connection`` to +be a reference (``env:``, ``file:``, ``service:``, ``keyring:``) rather than +a DSN. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import date, datetime, timezone +from pathlib import Path +from typing import Any + +from .integrity import normalize_digest, sha256_file +from .project import get_in, project_path + +PIN_CLASSES = ("local_snapshot", "backend_snapshot") +MUTABLE_ALIASES = {"latest", "current", "head", "now", "master", "main", "live", "today"} +CONNECTION_REFERENCE_SCHEMES = ("env", "file", "service", "keyring") + +_CREDENTIAL_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("password fragment", re.compile(r"(?i)(?:^|[\s;&?,{\"'])(?:password|passwd|pwd|secret|api[_-]?key|access[_-]?key|token)\s*[=:]\s*\S")), + ("credentialed URL", re.compile(r"(?i)\b[a-z][a-z0-9+.-]*://[^/\s:@]+:[^/\s@]+@")), + ("AWS access key id", re.compile(r"\bAKIA[0-9A-Z]{16}\b")), + ("bearer token", re.compile(r"(?i)\bbearer\s+[a-z0-9._~+/-]{16,}=*")), + ("private key block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")), +) + + +@dataclass +class PinAssessment: + status: str # pinned | not_reproducible | unpinned | invalid + pin_class: str # local_snapshot | backend_snapshot | version_identity | none + reason: str + details: dict[str, Any] = field(default_factory=dict) + + +def _parse_timestamp(value: object) -> datetime | None: + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=timezone.utc) + if isinstance(value, date): + return datetime(value.year, value.month, value.day, tzinfo=timezone.utc) + if not isinstance(value, str) or not value.strip(): + return None + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def _is_mutable_alias(value: object) -> bool: + return isinstance(value, str) and value.strip().lower() in MUTABLE_ALIASES + + +def assess_pin(root: Path, source: dict[str, Any], *, now: datetime | None = None) -> PinAssessment: + """Classify how (and whether) one source is pinned. Never raises.""" + now = now or datetime.now(timezone.utc) + pin = source.get("pin") + identifier = get_in(source, "version", "identifier") + published_at = get_in(source, "version", "published_at") + # A mutable alias is a lie about identity whatever else is declared: a + # snapshot of "latest" still cannot say *which* latest it froze. + if _is_mutable_alias(identifier) or (identifier in (None, "") and _is_mutable_alias(published_at)): + return PinAssessment("unpinned", "version_identity", f"version.identifier {identifier!r} is a mutable alias") + if pin is None: + if identifier in (None, "") and published_at in (None, ""): + return PinAssessment("unpinned", "none", "no pin block and no version.identifier/published_at") + return PinAssessment("pinned", "version_identity", "version identity is recorded and is not a mutable alias") + + if not isinstance(pin, dict): + return PinAssessment("invalid", "none", "pin must be a mapping") + pin_class = pin.get("class") + if pin_class not in PIN_CLASSES: + return PinAssessment("invalid", "none", f"pin.class must be one of {list(PIN_CLASSES)}, got {pin_class!r}") + + if pin_class == "local_snapshot": + relative = pin.get("path") + target = project_path(root, relative) + if target is None: + return PinAssessment("invalid", pin_class, "pin.path must be a safe project-relative path") + if not str(relative).replace("\\", "/").startswith("data/source/"): + return PinAssessment("invalid", pin_class, "a local snapshot must live under data/source/") + expected = normalize_digest(pin.get("sha256")) + if expected is None: + return PinAssessment("invalid", pin_class, "pin.sha256 must be a sha256 digest of the snapshot") + if _parse_timestamp(pin.get("captured_at")) is None: + return PinAssessment("invalid", pin_class, "pin.captured_at must be an ISO-8601 timestamp") + if not target.is_file(): + return PinAssessment( + "not_reproducible", pin_class, f"snapshot file is missing: {relative}", {"cause": "snapshot_missing"} + ) + actual = sha256_file(target) + if actual != expected: + return PinAssessment( + "not_reproducible", + pin_class, + f"snapshot {relative} does not match pin.sha256", + {"cause": "snapshot_hash_mismatch", "expected": expected, "actual": actual}, + ) + return PinAssessment("pinned", pin_class, f"local snapshot {relative} matches its content hash") + + identifier = pin.get("identifier") + if not isinstance(identifier, str) or not identifier.strip(): + return PinAssessment("invalid", pin_class, "pin.identifier must name the backend snapshot") + if _is_mutable_alias(identifier): + return PinAssessment("unpinned", pin_class, f"pin.identifier {identifier!r} is a mutable alias, not a snapshot") + if _parse_timestamp(pin.get("captured_at")) is None: + return PinAssessment("invalid", pin_class, "pin.captured_at must be an ISO-8601 timestamp") + retention = _parse_timestamp(pin.get("retention_until")) + if retention is None: + return PinAssessment( + "invalid", pin_class, "pin.retention_until must record when the backend may drop the snapshot" + ) + if retention <= now: + return PinAssessment( + "not_reproducible", + pin_class, + f"backend snapshot {identifier!r} retention expired at {pin.get('retention_until')}", + {"cause": "snapshot_expired", "retention_until": str(pin.get("retention_until"))}, + ) + verification = pin.get("verification") + if isinstance(verification, dict) and verification.get("status") == "inaccessible": + return PinAssessment( + "not_reproducible", + pin_class, + f"backend snapshot {identifier!r} was last verified inaccessible", + {"cause": "snapshot_inaccessible", "verified_at": verification.get("at")}, + ) + return PinAssessment("pinned", pin_class, f"backend snapshot {identifier!r} is identified and retained until {pin.get('retention_until')}") + + +def _walk_strings(value: Any, path: str): + if isinstance(value, str): + yield path, value + elif isinstance(value, dict): + for key, item in value.items(): + yield from _walk_strings(item, f"{path}.{key}" if path else str(key)) + elif isinstance(value, list): + for index, item in enumerate(value): + yield from _walk_strings(item, f"{path}[{index}]") + + +def find_inline_credentials(value: Any, path: str = "") -> list[dict[str, str]]: + """Return every string in ``value`` that looks like an embedded secret. + + Reports the manifest path and the pattern name, never the matched text, + so the finding itself cannot leak what it found. + """ + findings: list[dict[str, str]] = [] + for where, text in _walk_strings(value, path): + for name, pattern in _CREDENTIAL_PATTERNS: + if pattern.search(text): + findings.append({"path": where, "pattern": name}) + break + return findings + + +def connection_reference_error(root: Path, connection: object) -> str | None: + """``access.connection`` must be a reference, never a connection string.""" + if connection is None: + return None + reference = connection.get("ref") if isinstance(connection, dict) else connection + if not isinstance(reference, str) or not reference.strip(): + return "access.connection must be a string reference or {ref: ...}" + scheme, separator, remainder = reference.partition(":") + if not separator or scheme not in CONNECTION_REFERENCE_SCHEMES or not remainder.strip(): + return ( + "access.connection must reference credentials indirectly " + f"({', '.join(f'{item}:' for item in CONNECTION_REFERENCE_SCHEMES)}), not embed a connection string" + ) + if scheme == "file": + target = project_path(root, remainder.strip()) + if target is not None: + return "access.connection file references must point outside the project directory" + return None + + +def source_pin_summary(root: Path, sources: dict[str, Any], *, now: datetime | None = None) -> dict[str, PinAssessment]: + return { + str(key): assess_pin(root, source, now=now) if isinstance(source, dict) else PinAssessment("invalid", "none", "source must be a mapping") + for key, source in sources.items() + } + + +def redact(text: str) -> str: + """Mask credential-like fragments in free text before it is recorded.""" + redacted = re.sub(r"(?i)\b([a-z][a-z0-9+.-]*://[^/\s:@]+):[^/\s@]+@", r"\1:***@", text) + redacted = re.sub( + r"(?i)((?:password|passwd|pwd|secret|api[_-]?key|access[_-]?key|token)\s*[=:]\s*)\S+", + r"\1***", + redacted, + ) + redacted = re.sub(r"\bAKIA[0-9A-Z]{16}\b", "AKIA****************", redacted) + return redacted diff --git a/openmapstack/validation.py b/openmapstack/validation.py index 51862f5..3b5cfda 100644 --- a/openmapstack/validation.py +++ b/openmapstack/validation.py @@ -20,6 +20,7 @@ ) from .project import ProjectError, get_in, load_json, load_project, project_path, step_outputs from .schema import project_schema_errors +from .sources import assess_pin, connection_reference_error, find_inline_credentials SCHEMA = "openmapstack-project/v1" CHECK_STATUSES = {"passed", "failed", "warning", "not_testable"} @@ -254,6 +255,24 @@ def _sources(self) -> None: else: self.add("source.provenance", "passed", "source URL, retrieval, version, selection, and rationale are pinned", path=base) + pin = assess_pin(self.root, source) + if pin.status == "pinned": + self.add("source.pin", "passed", pin.reason, path=f"{base}.pin", pin_class=pin.pin_class) + elif pin.status == "not_reproducible": + self.add("source.pin", "failed", f"not reproducible: {pin.reason}", path=f"{base}.pin", pin_class=pin.pin_class, **pin.details) + elif pin.status == "invalid": + self.add("source.pin", "failed", pin.reason, path=f"{base}.pin") + elif pin.pin_class != "none": + self.add("source.pin", "failed", pin.reason, path=f"{base}.pin") + credential_findings = [f"{item['path']} ({item['pattern']})" for item in find_inline_credentials(source, base)] + connection_error = connection_reference_error(self.root, get_in(source, "access", "connection")) + if connection_error: + credential_findings.append(f"{base}.access.connection: {connection_error}") + if credential_findings: + self.add("source.credentials", "failed", f"secrets must never enter project.yaml: {credential_findings}", path=base) + elif get_in(source, "access", "connection") is not None or source.get("warehouse") is not None: + self.add("source.credentials", "passed", "warehouse connection is referenced, not embedded", path=f"{base}.access.connection") + license_block = source.get("license") if not isinstance(license_block, dict) or not _present(license_block.get("name")) or not _present(license_block.get("url")): self.add("source.license", "failed", "license.name and license.url are required", path=f"{base}.license") diff --git a/openmapstack/verify.py b/openmapstack/verify.py index f8626a8..5d8ecbb 100644 --- a/openmapstack/verify.py +++ b/openmapstack/verify.py @@ -227,6 +227,7 @@ def verify_project( for name, fn in ( ("every_source_has_provider_and_access", provenance_checks.every_source_has_provider_and_access), ("every_source_pinned", provenance_checks.every_source_pinned), + ("no_inline_credentials", provenance_checks.no_inline_credentials), ("license_present_where_required", provenance_checks.license_present_where_required), ("rationale_present", provenance_checks.rationale_present), ): diff --git a/references/project-spec.md b/references/project-spec.md index 4a3cb30..f39dc79 100644 --- a/references/project-spec.md +++ b/references/project-spec.md @@ -155,6 +155,61 @@ sources: - Describe the data you received in `schema` — its CRS, the key, the field roles the analysis depends on, and the columns. Earlier drafts used a bare `expected_fields` list; `schema.columns` supersedes it. - `access.retrieved_at` and `access.downloaded_at` are interchangeable to the validator, which needs one of the two. Record both when they differ (a cached extract retrieved later than it was published). +#### Pin classes — warehouse and mutable sources + +`version.identifier` pins a published release, a STAC item, or a dated +extract. It does not pin a warehouse table: "the parcels table on +2026-08-30" names a moving target unless something froze it. A source whose +bytes come from a query declares **which of two pin classes** froze them: + +```yaml +sources: + parcels: + access: + method: postgis + connection: {ref: "env:PARCELS_DSN"} # a reference, never a DSN + warehouse: + backend: postgis # duckdb | postgis (pilot backends) + account: geo-prod # host / project identity, no secrets + database: gis + schema: cadastre + table: parcels + query_sha256: "sha256:..." # digest of the exact SELECT used + schema_sha256: "sha256:..." # digest of the discovered column list + pin: + class: local_snapshot # (1) user-approved local copy + path: data/source/parcels.parquet + sha256: "sha256:..." # real content hash of that file + captured_at: "2026-08-30T10:00:00Z" + # -- or -- + pin: + class: backend_snapshot # (2) backend time travel / snapshot id + identifier: "pg_export_snapshot:00000003-000001A8-1" + captured_at: "2026-08-30T10:00:00Z" + retention_until: "2026-12-31T00:00:00Z" # when the backend may drop it + verification: {at: "2026-08-30T10:05:00Z", status: accessible} +``` + +| Pin | Reproducible when | Reported as | +|---|---|---| +| `local_snapshot` | the file exists under `data/source/` and matches `sha256` | `pinned`; a missing or edited file is `not_reproducible` | +| `backend_snapshot` | `identifier` names a real snapshot, `retention_until` is in the future, and the last `verification` did not find it inaccessible | `pinned`; expired or inaccessible is `not_reproducible` | +| none | `version.identifier` / `published_at` is present and not a mutable alias (`latest`, `current`, `head`, …) | `pinned` by version identity | + +`provenance.every_source_pinned` and `validate`'s `source.pin` check apply +this table. A backend snapshot id plus a timestamp is **not** a pin once the +retention has lapsed; the honest result is `not_reproducible`, never `pinned` +because a string is present. A mutable alias in `version.identifier` is +unpinned whatever else is declared. + +**Secrets never enter `project.yaml`.** `access.connection` is a reference — +`env:NAME`, `service:NAME` (a `pg_service.conf` entry), `keyring:NAME`, or +`file:/absolute/path/outside/the/project` — and the manifest is scanned for +password fragments, credentialed URLs, and well-known key shapes +(`source.credentials`, `provenance.no_inline_credentials`). Connector +discovery is read-only, and materialising warehouse data locally requires an +explicit approval; see `references/user-data-sources.md`. + ### 2.3 Overrides — analyst knowledge as data **Everything that is not a fact of an external dataset belongs in `overrides`.** Sources are immutable: diff --git a/templates/project.yaml b/templates/project.yaml index 47eec26..66890a0 100644 --- a/templates/project.yaml +++ b/templates/project.yaml @@ -57,6 +57,14 @@ sources: version: published_at: 2026-08-24 identifier: TODO # release tag, STAC item id, layer/table id + # A warehouse/query source also declares how its bytes were frozen + # (project-spec.md section 2.2, pin classes). Credentials stay out of + # this file: access.connection is a reference such as env:NAME. + # pin: + # class: local_snapshot # | backend_snapshot (identifier + retention_until) + # path: data/source/TODO.parquet + # sha256: "sha256:TODO" + # captured_at: 2026-08-25T00:00:00Z selection: bbox: [west, south, east, north] filter: TODO diff --git a/tests/test_sources.py b/tests/test_sources.py new file mode 100644 index 0000000..1632755 --- /dev/null +++ b/tests/test_sources.py @@ -0,0 +1,221 @@ +"""Source pin classes and credential hygiene (issue #13, B5a).""" + +from __future__ import annotations + +import hashlib +import unittest +from copy import deepcopy +from datetime import datetime, timezone + +from openmapstack.checks.provenance import every_source_pinned, no_inline_credentials +from openmapstack.schema import project_schema_errors +from openmapstack.sources import ( + assess_pin, + connection_reference_error, + find_inline_credentials, + redact, +) +from openmapstack.validation import validate_project +from tests.evals.helpers import make_workspace, minimal_project, write_project +from tests.test_cli import valid_manifest + +NOW = datetime(2026, 9, 1, tzinfo=timezone.utc) + + +def _snapshot(workspace, name="parcels.parquet", content=b"parquet bytes"): + target = workspace / "data/source" / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + return { + "class": "local_snapshot", + "path": f"data/source/{name}", + "sha256": "sha256:" + hashlib.sha256(content).hexdigest(), + "captured_at": "2026-08-30T10:00:00Z", + } + + +def _backend(**extra): + pin = { + "class": "backend_snapshot", + "identifier": "pg_export_snapshot:00000003-000001A8-1", + "captured_at": "2026-08-30T10:00:00Z", + "retention_until": "2026-12-31T00:00:00Z", + } + pin.update(extra) + return pin + + +def _source(**extra): + source = deepcopy(minimal_project()["sources"]["test_source"]) + source.update(extra) + return source + + +class PinAssessmentTests(unittest.TestCase): + def test_version_identity_without_pin_block_is_still_accepted(self) -> None: + assessment = assess_pin(make_workspace(), _source(), now=NOW) + self.assertEqual((assessment.status, assessment.pin_class), ("pinned", "version_identity")) + + def test_mutable_alias_is_unpinned_even_with_a_valid_snapshot(self) -> None: + workspace = make_workspace() + source = _source(version={"identifier": "latest"}, pin=_snapshot(workspace)) + self.assertEqual(assess_pin(workspace, source, now=NOW).status, "unpinned") + + def test_local_snapshot_pins_when_bytes_match(self) -> None: + workspace = make_workspace() + assessment = assess_pin(workspace, _source(pin=_snapshot(workspace)), now=NOW) + self.assertEqual((assessment.status, assessment.pin_class), ("pinned", "local_snapshot")) + + def test_local_snapshot_missing_or_changed_is_not_reproducible(self) -> None: + workspace = make_workspace() + pin = _snapshot(workspace) + (workspace / pin["path"]).write_bytes(b"edited") + changed = assess_pin(workspace, _source(pin=pin), now=NOW) + self.assertEqual((changed.status, changed.details["cause"]), ("not_reproducible", "snapshot_hash_mismatch")) + (workspace / pin["path"]).unlink() + missing = assess_pin(workspace, _source(pin=pin), now=NOW) + self.assertEqual((missing.status, missing.details["cause"]), ("not_reproducible", "snapshot_missing")) + + def test_local_snapshot_must_live_under_data_source(self) -> None: + workspace = make_workspace() + pin = _snapshot(workspace) + pin["path"] = "data/derived/parcels.parquet" + self.assertEqual(assess_pin(workspace, _source(pin=pin), now=NOW).status, "invalid") + pin["path"] = "../outside.parquet" + self.assertEqual(assess_pin(workspace, _source(pin=pin), now=NOW).status, "invalid") + + def test_backend_snapshot_pins_until_retention_lapses(self) -> None: + workspace = make_workspace() + self.assertEqual(assess_pin(workspace, _source(pin=_backend()), now=NOW).status, "pinned") + expired = assess_pin(workspace, _source(pin=_backend()), now=datetime(2027, 1, 1, tzinfo=timezone.utc)) + self.assertEqual((expired.status, expired.details["cause"]), ("not_reproducible", "snapshot_expired")) + + def test_backend_snapshot_verified_inaccessible_is_not_reproducible(self) -> None: + pin = _backend(verification={"at": "2026-08-31T00:00:00Z", "status": "inaccessible"}) + assessment = assess_pin(make_workspace(), _source(pin=pin), now=NOW) + self.assertEqual((assessment.status, assessment.details["cause"]), ("not_reproducible", "snapshot_inaccessible")) + + def test_backend_snapshot_needs_identifier_and_retention(self) -> None: + workspace = make_workspace() + for broken in ( + _backend(identifier="latest"), + {k: v for k, v in _backend().items() if k != "retention_until"}, + _backend(retention_until="whenever"), + _backend(captured_at=None), + {"class": "time_travel", "captured_at": "2026-08-30T10:00:00Z"}, + "not a mapping", + ): + assessment = assess_pin(workspace, _source(pin=broken), now=NOW) + self.assertIn(assessment.status, {"invalid", "unpinned"}, broken) + + def test_provenance_check_reports_each_class_with_its_own_code(self) -> None: + workspace = make_workspace() + project = minimal_project() + project["sources"]["test_source"]["pin"] = _snapshot(workspace) + write_project(workspace, project) + self.assertEqual(every_source_pinned(workspace).status, "passed") + + project["sources"]["test_source"]["pin"] = _backend(retention_until="2020-01-01T00:00:00Z") + write_project(workspace, project) + result = every_source_pinned(workspace) + self.assertEqual((result.status, result.data["code"]), ("failed", "not_reproducible")) + self.assertEqual(result.data["causes"], {"test_source": "snapshot_expired"}) + + project["sources"]["test_source"]["pin"] = {"class": "backend_snapshot"} + write_project(workspace, project) + self.assertEqual(every_source_pinned(workspace).data["code"], "pin_invalid") + + del project["sources"]["test_source"]["pin"] + project["sources"]["test_source"]["version"] = {"identifier": "latest"} + write_project(workspace, project) + self.assertEqual(every_source_pinned(workspace).data["code"], "source_unpinned") + + +class CredentialHygieneTests(unittest.TestCase): + def test_embedded_secrets_are_found_without_being_echoed(self) -> None: + source = _source( + access={"method": "postgis", "retrieved_at": "x", "connection": "postgresql://gis:hunter2@db/gis"}, + notes="password=opensesame", + ) + findings = find_inline_credentials(source, "sources.s") + paths = {item["path"] for item in findings} + self.assertIn("sources.s.access.connection", paths) + self.assertIn("sources.s.notes", paths) + self.assertNotIn("hunter2", str(findings)) + self.assertNotIn("opensesame", str(findings)) + + def test_ordinary_urls_and_prose_are_not_flagged(self) -> None: + source = _source( + source_url="https://geoportaal.maaruum.ee/eng/spatial-data/cadastral-data-p310.html", + rationale="The token field in the source schema is a land-use token, not a credential.", + ) + self.assertEqual(find_inline_credentials(source, "sources.s"), []) + + def test_connection_must_be_a_reference(self) -> None: + workspace = make_workspace() + self.assertIsNone(connection_reference_error(workspace, "env:PARCELS_DSN")) + self.assertIsNone(connection_reference_error(workspace, {"ref": "service:geo-prod"})) + self.assertIsNone(connection_reference_error(workspace, None)) + self.assertIsNotNone(connection_reference_error(workspace, "host=db dbname=gis")) + self.assertIsNotNone(connection_reference_error(workspace, "postgresql://db/gis")) + self.assertIsNotNone(connection_reference_error(workspace, {"ref": "env:"})) + # A secrets file inside the project directory would be committed. + self.assertIsNotNone(connection_reference_error(workspace, "file:secrets/dsn.txt")) + self.assertIsNone(connection_reference_error(workspace, "file:/etc/openmapstack/dsn")) + + def test_provenance_check_and_validate_reject_inline_credentials(self) -> None: + workspace = make_workspace() + project = minimal_project() + project["sources"]["test_source"]["access"]["connection"] = "postgresql://gis:hunter2@db/gis" + write_project(workspace, project) + result = no_inline_credentials(workspace) + self.assertEqual((result.status, result.data["code"]), ("failed", "inline_credentials")) + + manifest = valid_manifest() + key = next(iter(manifest["sources"])) + manifest["sources"][key]["access"]["connection"] = {"ref": "env:GIS_DSN"} + write_project(workspace, manifest) + (workspace / "pipeline.py").write_text("print('ok')\n", encoding="utf-8") + checks = {c.id: c for c in validate_project(workspace / "project.yaml", artifacts=False).checks} + self.assertEqual(checks["source.credentials"].status, "passed") + manifest["sources"][key]["access"]["connection"] = "postgresql://gis:hunter2@db/gis" + write_project(workspace, manifest) + checks = {c.id: c for c in validate_project(workspace / "project.yaml", artifacts=False).checks} + self.assertEqual(checks["source.credentials"].status, "failed") + self.assertNotIn("hunter2", checks["source.credentials"].message) + + def test_redaction_masks_secrets_in_recorded_text(self) -> None: + text = "postgresql://gis:hunter2@db/gis password=abc token: xyz AKIAABCDEFGHIJKLMNOP" + masked = redact(text) + for secret in ("hunter2", "abc", "xyz", "AKIAABCDEFGHIJKLMNOP"): + self.assertNotIn(secret, masked) + self.assertIn("postgresql://gis:***@db/gis", masked) + + +class SchemaTests(unittest.TestCase): + def test_pin_and_warehouse_blocks_validate(self) -> None: + manifest = valid_manifest() + key = next(iter(manifest["sources"])) + manifest["sources"][key]["pin"] = _backend() + manifest["sources"][key]["warehouse"] = {"backend": "postgis", "database": "gis", "schema": "cadastre", "table": "parcels"} + manifest["sources"][key]["access"]["connection"] = {"ref": "env:GIS_DSN"} + self.assertEqual(project_schema_errors(manifest), []) + manifest["sources"][key]["pin"] = {"class": "local_snapshot", "captured_at": "x"} + self.assertTrue(project_schema_errors(manifest)) + manifest["sources"][key]["pin"] = {"class": "elsewhere", "captured_at": "x"} + self.assertTrue(project_schema_errors(manifest)) + + def test_validate_reports_pin_class_per_source(self) -> None: + workspace = make_workspace() + manifest = valid_manifest() + key = next(iter(manifest["sources"])) + manifest["sources"][key]["pin"] = _snapshot(workspace) + write_project(workspace, manifest) + (workspace / "pipeline.py").write_text("print('ok')\n", encoding="utf-8") + checks = [c for c in validate_project(workspace / "project.yaml", artifacts=False).checks if c.id == "source.pin"] + self.assertEqual([c.status for c in checks], ["passed"]) + self.assertEqual(checks[0].details["pin_class"], "local_snapshot") + + +if __name__ == "__main__": + unittest.main() From 8ffd8b27c9b0e31005eb3123853248fbcf51ea22 Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 2 Sep 2026 16:59:03 +0000 Subject: [PATCH 3/8] feat(cli): read-only warehouse connectors with approval-gated snapshots (pilot) openmapstack source discover/snapshot implements the connector pilot for the reference pair, DuckDB local files and PostGIS. Credentials are resolved from a reference (env:, service:, file:) and never recorded; sessions are read-only with a statement timeout; only a single SELECT is accepted; a snapshot is a dry run until --approve, is bounded by row and byte limits, lands only under data/source/, never overwrites, and returns the local_snapshot pin plus query and schema digests for the manifest. The DuckDB connector confines file access to its root and exposes files as views. PostGIS has no durable time travel, so its pin is the local snapshot; pg_current_snapshot() is recorded as non-durable retrieval metadata. Unverified backends are refused rather than approximated. references/user-data-sources.md documents the four rules and the manual path for other warehouses; a scheduled workflow runs the live PostGIS round trip against a service container. Refs #13 (B5b). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01TMMy6A73zi7yGjK3wHBhju --- .github/workflows/eval-warehouse.yml | 74 +++++ README.md | 15 + SKILL.md | 1 + docs/maintainers/architecture.md | 12 + openmapstack/cli.py | 109 ++++++++ openmapstack/connectors/__init__.py | 356 ++++++++++++++++++++++++ openmapstack/connectors/duckdb_local.py | 240 ++++++++++++++++ openmapstack/connectors/postgis.py | 247 ++++++++++++++++ pyproject.toml | 5 +- references/user-data-sources.md | 157 +++++++++++ tests/test_connectors.py | 339 ++++++++++++++++++++++ 11 files changed, 1554 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/eval-warehouse.yml create mode 100644 openmapstack/connectors/__init__.py create mode 100644 openmapstack/connectors/duckdb_local.py create mode 100644 openmapstack/connectors/postgis.py create mode 100644 references/user-data-sources.md create mode 100644 tests/test_connectors.py diff --git a/.github/workflows/eval-warehouse.yml b/.github/workflows/eval-warehouse.yml new file mode 100644 index 0000000..552fdeb --- /dev/null +++ b/.github/workflows/eval-warehouse.yml @@ -0,0 +1,74 @@ +name: OpenMapStack warehouse connector pilot + +# The PostGIS connector is exercised end to end against a real PostGIS +# service: read-only discovery, dry-run plan, approved GeoParquet snapshot, +# and a pin the contract accepts. Scheduled/manual, and on changes to the +# connector code, so an unrelated PR is never blocked by a service container. +on: + workflow_dispatch: {} + schedule: + - cron: "37 5 * * 1" + pull_request: + paths: + - "openmapstack/connectors/**" + - "openmapstack/sources.py" + - "tests/test_connectors.py" + - ".github/workflows/eval-warehouse.yml" + +jobs: + postgis-connector: + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgis: + image: postgis/postgis:16-3.4 + env: + POSTGRES_PASSWORD: ci-only-password + POSTGRES_DB: gis + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d gis" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install the CLI with geodata and PostGIS support + run: pip install ".[geo,postgis]" + + - name: Prepare controlled DuckDB Spatial directory + run: | + echo "OPENMAPSTACK_SPATIAL_EXTENSION_DIR=${RUNNER_TEMP}/openmapstack-duckdb-extensions" >> "$GITHUB_ENV" + OPENMAPSTACK_SPATIAL_EXTENSION_DIR="${RUNNER_TEMP}/openmapstack-duckdb-extensions" python evals/prepare_spatial.py + + - name: Seed a parcels table and a read-only role + env: + PGPASSWORD: ci-only-password + run: | + sudo apt-get update -q && sudo apt-get install -y -q postgresql-client + psql -h 127.0.0.1 -U postgres -d gis -v ON_ERROR_STOP=1 <<'SQL' + CREATE EXTENSION IF NOT EXISTS postgis; + CREATE TABLE public.parcels (cadastral_id text primary key, land_use text, geom geometry(Polygon, 3301)); + INSERT INTO public.parcels VALUES + ('P1','ARIMAA', ST_GeomFromText('POLYGON((660100 6466500,660200 6466500,660200 6466600,660100 6466600,660100 6466500))',3301)), + ('P2','ARIMAA', ST_GeomFromText('POLYGON((660300 6466500,660400 6466500,660400 6466600,660300 6466600,660300 6466500))',3301)), + ('P3','TOOTMISMAA', ST_GeomFromText('POLYGON((660500 6466500,660600 6466500,660600 6466600,660500 6466600,660500 6466500))',3301)); + CREATE ROLE reader LOGIN PASSWORD 'ci-only-reader'; + GRANT CONNECT ON DATABASE gis TO reader; + GRANT USAGE ON SCHEMA public TO reader; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO reader; + ANALYZE public.parcels; + SQL + + - name: Run the connector suite against the live service + env: + # The credential reaches the connector only through this reference; + # the tests assert it never appears in any recorded output. + OPENMAPSTACK_TEST_POSTGIS_DSN: postgresql://reader:ci-only-reader@127.0.0.1:5432/gis + run: python -m unittest -v tests.test_connectors diff --git a/README.md b/README.md index 5ef25a8..1bdfbd8 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ It is open-first and cloud-native by default, built on shoulders of the awesome - [SKILL.md](SKILL.md) — the skill entry point: triggers, global defaults, format and compute decision matrices, anti-patterns, and a quick triage guide. - [references/data-sources.md](references/data-sources.md) - lists OSM, Overture, Sentinel/Landsat, regional portals, STAC catalogs and others. - [references/services-and-scale.md](references/services-and-scale.md) - depending on case use local installs or hosted/SaaS services for global-scale basemaps, elevation, routing, geocoding, place search, and postcodes. +- [references/user-data-sources.md](references/user-data-sources.md) - the user's own warehouse data: credentials by reference, read-only discovery, approval-gated snapshots, and the pin classes that make a warehouse table reproducible. - [references/formats-and-crs.md](references/formats-and-crs.md) - how to choose formats, conversions, projections, EPSG codes. - [references/processing.md](references/processing.md) - when and how to use GDAL/OGR, GeoPandas, xarray, DuckDB, PostGIS, PDAL and other open geo processing tools. - [references/analytics.md](references/analytics.md) — do vector/raster analytics, terrain, hydrology, network, point clouds, geocoding etc. @@ -151,6 +152,11 @@ openmapstack run path/to/project.yaml # Review sources, versions, overrides, ordered steps, outputs, and latest run. openmapstack inspect path/to/project.yaml + +# Read-only discovery of a warehouse source, then an approval-gated snapshot. +openmapstack source discover path/to/project.yaml --source parcels +openmapstack source snapshot path/to/project.yaml --source parcels \ + --query "SELECT id, geom FROM cadastre.parcels" --destination data/source/parcels.parquet --approve ``` Useful automation options: @@ -227,6 +233,15 @@ states the precondition that makes it valid, is executed by and reports `not_testable` with the reason when the precondition does not hold on the actual data. See [the project contract](references/project-spec.md#26-validation). +`openmapstack source` is the connector pilot for the user's own data +(DuckDB local files and PostGIS). Credentials are referenced, never stored; +discovery is read-only with a statement timeout; a snapshot is a dry run +until `--approve`, is limited by rows and bytes, lands only under +`data/source/`, and hands back the `pin` block that makes the source +reproducible. A warehouse table with only a timestamp is not pinned; an +expired backend snapshot is reported as `not_reproducible`. See +[user data sources](references/user-data-sources.md). + `validate` checks manifest structure, source retrieval/version/licensing data, CRS declarations, processing graph resolution, override provenance and files, output existence, validation-report parity/status propagation, override diff --git a/SKILL.md b/SKILL.md index 1fb2a1e..b7f599b 100644 --- a/SKILL.md +++ b/SKILL.md @@ -60,6 +60,7 @@ Hard rules for every material analysis — each is expanded in `references/proje | If the task involves... | Read | |---|---| | Finding or sourcing data (OSM, Overture, Sentinel, Landsat, building footprints, regional portals, STAC catalogs, MCP-based discovery) | `references/data-sources.md` | +| Reading the user's own warehouse or database (PostGIS, DuckDB, GeoParquet directories): credentials by reference, read-only discovery, approved snapshots, pin classes | `references/user-data-sources.md` | | Choosing local processing vs online/hosted/SaaS services for global or continental scale; basemaps, elevation, routing, geocoding, place search, postcode lookup APIs | `references/services-and-scale.md` | | Choosing a format, converting between formats, or any CRS / projection / EPSG question | `references/formats-and-crs.md` | | Compiling a reproducible GIS project artifact (`project.yaml`, pipeline, overrides, validation, presentation) | `references/project-spec.md` + `templates/` | diff --git a/docs/maintainers/architecture.md b/docs/maintainers/architecture.md index 34b0611..c169ec9 100644 --- a/docs/maintainers/architecture.md +++ b/docs/maintainers/architecture.md @@ -110,6 +110,18 @@ This environment cleanup is deliberately **not** a general sandbox or allowlist The eval harness additionally forbids reaching back into eval reference generators. That restriction is supplied by the eval caller; the shipped package intentionally does not know that `evals/` exists. +## Connectors are a trust boundary, like attestations + +`openmapstack/connectors/` reads user warehouse data on the user's behalf and is held to four rules that must survive any refactor: credentials are resolved from a reference and never recorded; sessions are read-only with a statement timeout; only a single `SELECT` reaches the backend; and nothing is materialised under `data/source/` without an explicit approval flag and within declared row/byte limits. Every message the package emits passes through `openmapstack.sources.redact`. + +The DuckDB local connector confines file access to its root (`allowed_directories` + `enable_external_access = false`) and exposes files as views so queries never spell paths. PostGIS has no durable time travel, so its pin is always a local snapshot; the transaction snapshot id is retrieval metadata, not a pin. Unverified backends are refused (`backend_unsupported`) rather than approximated. + +Pin classes live in `openmapstack/sources.py` and are shared by `validate` (`source.pin`, `source.credentials`) and `verify` (`provenance.every_source_pinned`, `provenance.no_inline_credentials`). Do not add a third interpretation. + +## Metamorphic relations execute the project's own pipeline + +`openmapstack/metamorphic.py` reuses the clean-rerun workspace preparation (`openmapstack/rerun.py`), perturbs only the copy, and compares against the produced outputs. A relation is valid only under its declared preconditions; unmet data preconditions are `not_testable`, invalid declarations fail, and unknown relation names are rejected rather than skipped. `runtime.implementation.parameters` (`openmapstack/parameters.py`) is the only sanctioned way to vary a pipeline setting from outside. Keep `metamorphic_evidence` a separate eval dimension from `gis_correctness`: a relation that holds is self-consistency, not a correct answer. + ## Integrity and path safety Project-relative paths are resolved through `openmapstack/project.py` helpers and must remain under the project root. Code that adds new file addressing should reuse the same safety model rather than joining unchecked user paths. diff --git a/openmapstack/cli.py b/openmapstack/cli.py index f2bfdf7..2ec1bbb 100644 --- a/openmapstack/cli.py +++ b/openmapstack/cli.py @@ -8,6 +8,9 @@ import subprocess import sys from collections.abc import Sequence + +import yaml + from pathlib import Path from typing import Any @@ -83,6 +86,35 @@ def build_parser() -> argparse.ArgumentParser: verify_parser.add_argument("--verbose", action="store_true", help="show passed checks in text output") verify_parser.set_defaults(handler=_cmd_verify) + source_parser = subparsers.add_parser( + "source", + help="read-only discovery and approval-gated snapshots of a warehouse source", + ) + source_commands = source_parser.add_subparsers(dest="source_command", required=True) + discover_parser = source_commands.add_parser("discover", help="list tables/files, geometry columns, SRIDs, and row estimates") + discover_parser.add_argument("project", help="project.yaml or its directory") + discover_parser.add_argument("--source", required=True, help="key under sources.* declaring warehouse.backend") + discover_parser.add_argument("--timeout", type=float, default=60.0, help="statement timeout in seconds (default: 60)") + discover_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + discover_parser.set_defaults(handler=_cmd_source_discover) + snapshot_parser = source_commands.add_parser( + "snapshot", + help="plan a query snapshot under data/source/; materialise only with --approve", + ) + snapshot_parser.add_argument("project", help="project.yaml or its directory") + snapshot_parser.add_argument("--source", required=True, help="key under sources.* declaring warehouse.backend") + query_group = snapshot_parser.add_mutually_exclusive_group(required=True) + query_group.add_argument("--query", help="one read-only SELECT statement") + query_group.add_argument("--query-file", type=Path, help="file containing one read-only SELECT statement") + snapshot_parser.add_argument("--destination", required=True, help="project-relative .parquet path under data/source/") + snapshot_parser.add_argument("--approve", action="store_true", help="actually write the snapshot (default is a dry run)") + snapshot_parser.add_argument("--write-manifest", action="store_true", help="after a materialised snapshot, record the pin and warehouse metadata in project.yaml (rewrites the file; YAML comments are not preserved)") + snapshot_parser.add_argument("--timeout", type=float, default=60.0, help="statement timeout in seconds (default: 60)") + snapshot_parser.add_argument("--max-rows", type=int, default=100_000, help="refuse queries returning more rows (default: 100000)") + snapshot_parser.add_argument("--max-bytes", type=int, default=256 * 1024 * 1024, help="refuse snapshots larger than this (default: 256 MiB)") + snapshot_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + snapshot_parser.set_defaults(handler=_cmd_source_snapshot) + inspect_parser = subparsers.add_parser("inspect", help="summarize a project for audit and review") inspect_parser.add_argument("project", nargs="?", default="project.yaml", help="project.yaml or its directory") inspect_parser.add_argument("--json", action="store_true", help="emit the full summary as JSON") @@ -262,6 +294,83 @@ def _cmd_run(args: argparse.Namespace) -> int: return 0 if validation.ok(strict=args.strict) else 1 +def _cmd_source_discover(args: argparse.Namespace) -> int: + from .connectors import ConnectorError, ConnectorLimits, discover_source + + try: + project_file, project = load_project(args.project) + discovery = discover_source( + project, args.source, project_root=project_file.parent, limits=ConnectorLimits(timeout_s=args.timeout) + ) + except (ProjectError, ConnectorError) as exc: + return _source_error(exc, args.json) + payload = discovery.to_dict() + if args.json: + print(_json(payload)) + return 0 + print(f"{discovery.backend} source {args.source!r}: {len(discovery.tables)} table(s)/file(s); read-only session: {discovery.read_only}") + for table in discovery.tables: + location = f"{table.schema}.{table.name}" if table.schema else table.name + srid = f"EPSG:{table.srid}" if table.srid else "srid unknown" + rows = f"~{table.row_estimate} rows" if table.row_estimate is not None else "rows unknown" + print(f" {location} [{table.kind}] geometry={table.geometry_column or '-'} {srid} {rows}") + for note in discovery.notes: + print(f" NOTE {note}") + return 0 + + +def _cmd_source_snapshot(args: argparse.Namespace) -> int: + from .connectors import ConnectorError, ConnectorLimits, apply_snapshot_to_manifest, snapshot_source + + query = args.query + if args.query_file is not None: + try: + query = args.query_file.read_text(encoding="utf-8") + except OSError as exc: + return _source_error(exc, args.json) + try: + project_file, project = load_project(args.project) + record = snapshot_source( + project, + args.source, + query, + args.destination, + project_root=project_file.parent, + approve=args.approve, + limits=ConnectorLimits(timeout_s=args.timeout, max_rows=args.max_rows, max_bytes=args.max_bytes), + ) + if args.write_manifest and record.get("materialized"): + updated = apply_snapshot_to_manifest(project, args.source, record) + project_file.write_text(yaml.safe_dump(updated, sort_keys=False, allow_unicode=True), encoding="utf-8") + record["manifest_written"] = str(project_file) + except (ProjectError, ConnectorError) as exc: + return _source_error(exc, args.json) + if args.json: + print(_json(record)) + return 0 + plan = record["plan"] + print(f"{record['backend']} source {args.source!r}: query {plan['query_sha256']} returns {plan['row_count']} row(s), {len(plan['columns'])} column(s)") + if not record["materialized"]: + print(f"DRY RUN: nothing written to {args.destination}; re-run with --approve to materialise") + return 0 + print(f"Wrote {args.destination} ({record['rows']} rows, {record['bytes']} bytes, {record['sha256']})") + if record.get("manifest_written"): + print(f"Updated {record['manifest_written']}") + else: + print("Add to project.yaml under sources.%s:" % args.source) + print(yaml.safe_dump({"pin": record["pin"], "warehouse": record["warehouse"]}, sort_keys=False).rstrip()) + return 0 + + +def _source_error(exc: Exception, as_json: bool) -> int: + code = getattr(exc, "code", type(exc).__name__) + if as_json: + print(_json({"schema": "openmapstack-source-error/v1", "status": "failed", "code": code, "error": str(exc)})) + else: + print(f"openmapstack source: {exc} [{code}]", file=sys.stderr) + return 2 + + def _cmd_inspect(args: argparse.Namespace) -> int: try: project_file, project = load_project(args.project) diff --git a/openmapstack/connectors/__init__.py b/openmapstack/connectors/__init__.py new file mode 100644 index 0000000..f2aae99 --- /dev/null +++ b/openmapstack/connectors/__init__.py @@ -0,0 +1,356 @@ +"""Safe, read-only connectors for user warehouse data (pilot). + +The connector surface is deliberately small and defensive: + +- **credentials by reference** -- ``access.connection`` names where the + credential lives (``env:NAME``, ``service:NAME``, ``file:/abs/path``); + the resolved value is used to open a session and is never recorded; +- **read-only discovery** -- a connector lists tables/files, geometry + columns, SRIDs, and row estimates through a session that cannot write; +- **explicit approval** -- ``snapshot_source`` never materialises data into + ``data/source/`` unless the caller passes ``approve=True``; without it the + call is a dry run that reports the schema and row count it *would* copy; +- **limits** -- a statement timeout, a row cap, and a byte cap apply to + every query; a query that exceeds any of them is refused, and a + half-written file is removed; +- **only SELECT** -- one statement, no DML/DDL/COPY/ATTACH keywords, no + statement separators; +- **redaction** -- every message a connector emits passes through + ``openmapstack.sources.redact``. + +Two backends are implemented as the reference pair: ``duckdb`` for local +files (GeoParquet, GeoJSON, GeoPackage, FlatGeobuf, ``.duckdb`` databases) +and ``postgis`` for PostgreSQL/PostGIS. Other warehouses are documented only +where their behaviour has been verified; they are not silently accepted. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ..integrity import sha256_file +from ..project import get_in, project_path +from ..sources import CONNECTION_REFERENCE_SCHEMES, redact + +BACKENDS = ("duckdb", "postgis") + +_FORBIDDEN_KEYWORDS = re.compile( + r"\b(insert|update|delete|drop|alter|create|copy|grant|revoke|truncate|call|execute|" + r"attach|detach|install|load|pragma|set|reset|vacuum|merge|into|import|export|checkpoint|" + r"begin|commit|rollback|do|listen|notify|refresh|lock|security_invoker|pg_read_file|" + r"read_text|read_blob|glob)\b", + re.IGNORECASE, +) + + +class ConnectorError(Exception): + """A connector refused or failed an operation. Messages are redacted.""" + + def __init__(self, message: str, *, code: str) -> None: + super().__init__(redact(message)) + self.code = code + + +class ConnectorUnavailable(ConnectorError): + """The backend driver is not installed in this environment.""" + + def __init__(self, message: str) -> None: + super().__init__(message, code="driver_unavailable") + + +@dataclass(frozen=True) +class ConnectorLimits: + timeout_s: float = 60.0 + max_rows: int = 100_000 + max_bytes: int = 256 * 1024 * 1024 + + def validate(self) -> None: + if self.timeout_s <= 0 or self.max_rows <= 0 or self.max_bytes <= 0: + raise ConnectorError("limits must all be positive", code="limits_invalid") + + +@dataclass +class TableInfo: + schema: str | None + name: str + geometry_column: str | None + srid: int | None + geometry_type: str | None + row_estimate: int | None + kind: str = "table" # table | view | file + + def to_dict(self) -> dict[str, Any]: + return { + "schema": self.schema, + "name": self.name, + "kind": self.kind, + "geometry_column": self.geometry_column, + "srid": self.srid, + "geometry_type": self.geometry_type, + "row_estimate": self.row_estimate, + } + + +@dataclass +class Discovery: + backend: str + identity: dict[str, Any] + tables: list[TableInfo] = field(default_factory=list) + read_only: bool = True + notes: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": "openmapstack-source-discovery/v1", + "backend": self.backend, + "identity": self.identity, + "read_only": self.read_only, + "tables": [table.to_dict() for table in self.tables], + "notes": self.notes, + } + + +@dataclass +class QueryPlan: + """What a snapshot would copy, established without materialising it.""" + + columns: list[dict[str, str]] + row_count: int + query_sha256: str + schema_sha256: str + backend_snapshot: dict[str, Any] | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "columns": self.columns, + "row_count": self.row_count, + "query_sha256": self.query_sha256, + "schema_sha256": self.schema_sha256, + "backend_snapshot": self.backend_snapshot, + } + + +def require_read_only_select(query: str) -> str: + """Accept exactly one SELECT/WITH statement with no side-effect keywords.""" + if not isinstance(query, str) or not query.strip(): + raise ConnectorError("query must be a non-empty SELECT statement", code="query_rejected") + text = query.strip().rstrip(";").strip() + stripped = re.sub(r"'(?:[^']|'')*'", "''", text) # ignore text inside string literals + if ";" in stripped: + raise ConnectorError("query must be a single statement", code="query_rejected") + if not re.match(r"(?is)^(select|with)\b", text): + raise ConnectorError("only SELECT (or WITH ... SELECT) queries are allowed", code="query_rejected") + match = _FORBIDDEN_KEYWORDS.search(stripped) + if match: + raise ConnectorError(f"query contains a forbidden keyword: {match.group(0).upper()}", code="query_rejected") + return text + + +def query_digest(query: str) -> str: + canonical = " ".join(query.strip().rstrip(";").split()) + return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def schema_digest(columns: list[dict[str, str]]) -> str: + canonical = json.dumps(columns, sort_keys=True, separators=(",", ":")) + return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def resolve_connection_reference(reference: object, *, environ: dict[str, str] | None = None) -> tuple[str, str]: + """Resolve ``access.connection`` to ``(scheme, secret)``. + + The secret is returned for opening a session only; callers must never + write it to a manifest, a log, or an evidence file. + """ + environ = os.environ if environ is None else environ + value = reference.get("ref") if isinstance(reference, dict) else reference + if not isinstance(value, str) or ":" not in value: + raise ConnectorError("access.connection must be a reference such as env:NAME", code="connection_reference_invalid") + scheme, _, remainder = value.partition(":") + remainder = remainder.strip() + if scheme not in CONNECTION_REFERENCE_SCHEMES or not remainder: + raise ConnectorError(f"unsupported connection reference scheme {scheme!r}", code="connection_reference_invalid") + if scheme == "env": + secret = environ.get(remainder) + if not secret: + raise ConnectorError(f"environment variable {remainder} is not set", code="connection_unresolved") + return scheme, secret + if scheme == "file": + path = Path(remainder).expanduser() + if not path.is_absolute(): + raise ConnectorError("file: connection references must be absolute paths outside the project", code="connection_reference_invalid") + try: + secret = path.read_text(encoding="utf-8").strip() + except OSError as exc: + raise ConnectorError(f"cannot read connection file: {type(exc).__name__}", code="connection_unresolved") from exc + if not secret: + raise ConnectorError("connection file is empty", code="connection_unresolved") + return scheme, secret + if scheme == "service": + return scheme, f"service={remainder}" + raise ConnectorError("keyring: references are not supported by the pilot connectors", code="connection_reference_invalid") + + +def load_connector(backend: str, connection: str, *, project_root: Path): + if backend == "duckdb": + from .duckdb_local import DuckDBLocalConnector + + return DuckDBLocalConnector(connection, project_root=project_root) + if backend == "postgis": + from .postgis import PostGISConnector + + return PostGISConnector(connection) + raise ConnectorError( + f"backend {backend!r} is not a verified connector; supported: {list(BACKENDS)}", + code="backend_unsupported", + ) + + +def _source_block(manifest: dict[str, Any], source_key: str) -> dict[str, Any]: + source = get_in(manifest, "sources", source_key) + if not isinstance(source, dict): + raise ConnectorError(f"source {source_key!r} is not declared", code="source_undeclared") + return source + + +def connector_for_source( + manifest: dict[str, Any], + source_key: str, + *, + project_root: Path, + environ: dict[str, str] | None = None, +): + source = _source_block(manifest, source_key) + backend = get_in(source, "warehouse", "backend") + if not isinstance(backend, str): + raise ConnectorError(f"source {source_key!r} declares no warehouse.backend", code="backend_undeclared") + reference = get_in(source, "access", "connection") + if backend == "duckdb" and reference is None: + # Local files need no credential: the "connection" is the project's + # own data/source directory. + return load_connector(backend, "", project_root=project_root) + _, secret = resolve_connection_reference(reference, environ=environ) + return load_connector(backend, secret, project_root=project_root) + + +def discover_source( + manifest: dict[str, Any], + source_key: str, + *, + project_root: Path, + limits: ConnectorLimits | None = None, + environ: dict[str, str] | None = None, +) -> Discovery: + limits = limits or ConnectorLimits() + limits.validate() + connector = connector_for_source(manifest, source_key, project_root=project_root, environ=environ) + return connector.discover(limits) + + +def snapshot_source( + manifest: dict[str, Any], + source_key: str, + query: str, + destination: str, + *, + project_root: Path, + approve: bool = False, + limits: ConnectorLimits | None = None, + environ: dict[str, str] | None = None, + now: datetime | None = None, +) -> dict[str, Any]: + """Plan, and with ``approve`` materialise, a query snapshot under data/source/. + + Returns a record with the plan, and when materialised the pin block and + warehouse metadata to place in the manifest. Nothing is written without + approval; a file that breaches ``limits`` is removed before returning. + """ + limits = limits or ConnectorLimits() + limits.validate() + query = require_read_only_select(query) + target = project_path(project_root, destination) + if target is None or not destination.replace("\\", "/").startswith("data/source/"): + raise ConnectorError("snapshot destination must be a project-relative path under data/source/", code="destination_invalid") + if target.suffix.lower() != ".parquet": + raise ConnectorError("snapshots are materialised as GeoParquet; use a .parquet destination", code="destination_invalid") + source = _source_block(manifest, source_key) + connector = connector_for_source(manifest, source_key, project_root=project_root, environ=environ) + plan = connector.plan(query, limits) + if plan.row_count > limits.max_rows: + raise ConnectorError( + f"query would return {plan.row_count} rows, above max_rows={limits.max_rows}", + code="row_limit_exceeded", + ) + record: dict[str, Any] = { + "schema": "openmapstack-source-snapshot/v1", + "source": source_key, + "backend": connector.backend, + "destination": destination, + "approved": bool(approve), + "materialized": False, + "plan": plan.to_dict(), + "limits": {"timeout_s": limits.timeout_s, "max_rows": limits.max_rows, "max_bytes": limits.max_bytes}, + } + if not approve: + record["note"] = "dry run: nothing was written; pass approve=True (--approve) to materialise" + return record + + if target.exists(): + raise ConnectorError( + f"{destination} already exists; sources are immutable, choose a new snapshot name", + code="destination_exists", + ) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".partial") + try: + rows = connector.materialize(query, temporary, limits) + size = temporary.stat().st_size + if size > limits.max_bytes: + raise ConnectorError(f"snapshot is {size} bytes, above max_bytes={limits.max_bytes}", code="byte_limit_exceeded") + temporary.replace(target) + finally: + if temporary.exists(): + temporary.unlink() + captured_at = (now or datetime.now(timezone.utc)).isoformat().replace("+00:00", "Z") + digest = sha256_file(target) + warehouse = dict(source.get("warehouse") or {}) + warehouse.update({"query_sha256": plan.query_sha256, "schema_sha256": plan.schema_sha256}) + warehouse.update(connector.identity_for_manifest()) + record.update( + { + "materialized": True, + "rows": rows, + "bytes": target.stat().st_size, + "sha256": digest, + "pin": {"class": "local_snapshot", "path": destination, "sha256": digest, "captured_at": captured_at}, + "warehouse": warehouse, + "access": {"retrieved_at": captured_at, "downloaded_at": captured_at}, + } + ) + if plan.backend_snapshot: + record["backend_snapshot"] = plan.backend_snapshot + return record + + +def apply_snapshot_to_manifest(manifest: dict[str, Any], source_key: str, record: dict[str, Any]) -> dict[str, Any]: + """Return a copy of ``manifest`` with the snapshot's pin and metadata applied.""" + import copy + + updated = copy.deepcopy(manifest) + source = _source_block(updated, source_key) + if not record.get("materialized"): + raise ConnectorError("only a materialised snapshot can be written to the manifest", code="not_materialized") + source["pin"] = record["pin"] + source["warehouse"] = record["warehouse"] + access = source.setdefault("access", {}) + access.update(record["access"]) + file_block = access.setdefault("file", {}) + file_block.update({"name": Path(record["destination"]).name, "format": "GeoParquet", "row_count": record["rows"], "size_bytes": record["bytes"]}) + return updated diff --git a/openmapstack/connectors/duckdb_local.py b/openmapstack/connectors/duckdb_local.py new file mode 100644 index 0000000..7b7e9f6 --- /dev/null +++ b/openmapstack/connectors/duckdb_local.py @@ -0,0 +1,240 @@ +"""DuckDB connector over local files (the local half of the reference pair). + +The "connection" is a directory of geodata files, or a ``.duckdb`` database +opened read-only. Discovery uses DuckDB Spatial's metadata readers and never +loads whole files. Queries run in a fresh connection whose file access is +confined to that directory when the installed DuckDB supports +``allowed_directories``; otherwise the confinement gap is recorded in the +discovery notes rather than hidden. +""" + +from __future__ import annotations + +import threading +from pathlib import Path +from typing import Any + +from ..checks.spatial import connect_spatial +from . import ( + ConnectorError, + ConnectorLimits, + ConnectorUnavailable, + Discovery, + QueryPlan, + TableInfo, + query_digest, + schema_digest, +) + +GEO_SUFFIXES = {".parquet": "GeoParquet", ".geojson": "GeoJSON", ".json": "GeoJSON", ".gpkg": "GeoPackage", ".fgb": "FlatGeobuf", ".shp": "Shapefile"} +_MAX_DISCOVERED_FILES = 500 + + +def _escape(value: str) -> str: + return value.replace("'", "''") + + +class _Timeout: + """Interrupt a DuckDB connection after ``seconds``; DuckDB has no statement timeout.""" + + def __init__(self, connection: Any, seconds: float) -> None: + self._connection = connection + self._timer = threading.Timer(seconds, self._interrupt) + self.fired = False + + def _interrupt(self) -> None: + self.fired = True + try: + self._connection.interrupt() + except Exception: # noqa: BLE001 - best effort + pass + + def __enter__(self) -> "_Timeout": + self._timer.start() + return self + + def __exit__(self, *_: object) -> None: + self._timer.cancel() + + +class DuckDBLocalConnector: + backend = "duckdb" + + def __init__(self, connection: str, *, project_root: Path) -> None: + root = Path(connection).expanduser() if connection else project_root / "data" / "source" + if not root.is_absolute(): + root = (project_root / root) + self.root = root.resolve() + self.database: Path | None = None + if self.root.is_file() and self.root.suffix.lower() == ".duckdb": + self.database = self.root + self.root = self.root.parent + if not self.root.is_dir(): + raise ConnectorError(f"duckdb connector root is not a directory: {self.root.name}", code="connection_unresolved") + self.notes: list[str] = [] + + # -- sessions --------------------------------------------------------------- + + def _connect(self, *, allow: tuple[Path, ...] = ()): + connection = connect_spatial() + if connection is None: + raise ConnectorUnavailable("duckdb with the Spatial extension is required (pip install 'openmapstack[geo]')") + allowed = ", ".join(f"'{_escape(str(path))}'" for path in (self.root, *allow)) + try: + # Both statements must be literal SETs after the database has + # started; once external access is off it cannot be re-enabled + # for this connection, which is the point. + connection.execute(f"SET allowed_directories = [{allowed}]") + connection.execute("SET enable_external_access = false") + except Exception: # noqa: BLE001 - older DuckDB + note = "installed DuckDB does not support allowed_directories; file access is not confined to the source root" + if note not in self.notes: + self.notes.append(note) + if self.database is not None: + connection.execute(f"ATTACH '{_escape(str(self.database))}' AS warehouse (READ_ONLY)") + connection.execute("USE warehouse") + self._register_file_views(connection) + return connection + + def _candidate_files(self) -> list[Path]: + candidates = sorted(path for path in self.root.rglob("*") if path.is_file() and path.suffix.lower() in GEO_SUFFIXES) + if len(candidates) > _MAX_DISCOVERED_FILES: + note = f"only the first {_MAX_DISCOVERED_FILES} of {len(candidates)} geodata files were registered" + if note not in self.notes: + self.notes.append(note) + candidates = candidates[:_MAX_DISCOVERED_FILES] + return candidates + + def _register_file_views(self, connection) -> None: + """Expose every geodata file under the root as a view named by its + root-relative path, so a query says ``FROM "parcels.geojson"`` and + never spells a filesystem path of its own.""" + for path in self._candidate_files(): + relative = path.relative_to(self.root).as_posix() + escaped = _escape(path.as_posix()) + reader = f"read_parquet('{escaped}')" if path.suffix.lower() == ".parquet" else f"ST_Read('{escaped}')" + try: + connection.execute(f'CREATE VIEW "{relative.replace(chr(34), chr(34) * 2)}" AS SELECT * FROM {reader}') + except Exception as exc: # noqa: BLE001 - describe what can be described + note = f"{relative}: not readable ({type(exc).__name__})" + if note not in self.notes: + self.notes.append(note) + + def identity_for_manifest(self) -> dict[str, Any]: + identity: dict[str, Any] = {"backend": "duckdb"} + if self.database is not None: + identity["database"] = self.database.name + return identity + + # -- discovery -------------------------------------------------------------- + + def discover(self, limits: ConnectorLimits) -> Discovery: + connection = self._connect() + tables: list[TableInfo] = [] + try: + with _Timeout(connection, limits.timeout_s): + if self.database is not None: + tables.extend(self._discover_database(connection)) + tables.extend(self._discover_files(connection)) + finally: + connection.close() + identity = {"root": self.root.name, "database": self.database.name if self.database else None} + return Discovery("duckdb", identity, tables, read_only=True, notes=list(self.notes)) + + def _discover_database(self, connection) -> list[TableInfo]: + rows = connection.execute( + "SELECT table_schema, table_name, table_type FROM information_schema.tables " + "WHERE table_catalog = 'warehouse' ORDER BY 1, 2" + ).fetchall() + found: list[TableInfo] = [] + for schema, name, table_type in rows: + columns = connection.execute( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_catalog = 'warehouse' AND table_schema = ? AND table_name = ?", + [schema, name], + ).fetchall() + geometry = next((column for column, type_name in columns if str(type_name).upper() == "GEOMETRY"), None) + estimate = connection.execute(f'SELECT COUNT(*) FROM "{schema}"."{name}"').fetchone()[0] + found.append(TableInfo(schema, name, geometry, None, None, int(estimate), kind="view" if "VIEW" in str(table_type).upper() else "table")) + return found + + def _discover_files(self, connection) -> list[TableInfo]: + found: list[TableInfo] = [] + for path in self._candidate_files(): + relative = path.relative_to(self.root).as_posix() + try: + found.append(self._describe_file(connection, path, relative)) + except Exception as exc: # noqa: BLE001 - describe what can be described + note = f"{relative}: not readable ({type(exc).__name__})" + if note not in self.notes: + self.notes.append(note) + return found + + def _describe_file(self, connection, path: Path, relative: str) -> TableInfo: + escaped = _escape(path.as_posix()) + if path.suffix.lower() == ".parquet": + columns = connection.execute(f"DESCRIBE SELECT * FROM read_parquet('{escaped}')").fetchall() + geometry = next((str(name) for name, type_name, *_ in columns if str(type_name).upper() == "GEOMETRY"), None) + estimate = connection.execute(f"SELECT COUNT(*) FROM read_parquet('{escaped}')").fetchone()[0] + srid = None + if geometry: + crs_row = connection.execute(f'SELECT ST_CRS("{geometry}") FROM read_parquet(\'{escaped}\') LIMIT 1').fetchone() + srid = _srid_from(crs_row[0] if crs_row else None) + return TableInfo(None, relative, geometry, srid, None, int(estimate), kind="file") + meta = connection.execute(f"SELECT layers FROM ST_Read_Meta('{escaped}')").fetchone() + layer = (meta[0] or [None])[0] if meta else None + geometry = srid = geometry_type = None + estimate = None + if isinstance(layer, dict): + fields = layer.get("geometry_fields") or [] + if fields: + geometry = fields[0].get("name") or "geom" + geometry_type = fields[0].get("type") + srid = _srid_from(((fields[0].get("crs") or {}).get("auth_code"))) + estimate = layer.get("feature_count") + return TableInfo(None, relative, geometry, srid, geometry_type, int(estimate) if estimate is not None else None, kind="file") + + # -- queries ---------------------------------------------------------------- + + def plan(self, query: str, limits: ConnectorLimits) -> QueryPlan: + connection = self._connect() + try: + with _Timeout(connection, limits.timeout_s) as timeout: + try: + described = connection.execute(f"DESCRIBE SELECT * FROM ({query}) AS q").fetchall() + count = connection.execute(f"SELECT COUNT(*) FROM ({query}) AS q").fetchone()[0] + except Exception as exc: # noqa: BLE001 + if timeout.fired: + raise ConnectorError(f"query exceeded timeout_s={limits.timeout_s}", code="timeout") from exc + raise ConnectorError(f"query failed: {type(exc).__name__}: {exc}", code="query_failed") from exc + finally: + connection.close() + columns = [{"name": str(name), "type": str(type_name)} for name, type_name, *_ in described] + return QueryPlan(columns, int(count), query_digest(query), schema_digest(columns)) + + def materialize(self, query: str, destination: Path, limits: ConnectorLimits) -> int: + connection = self._connect(allow=(destination.resolve().parent,)) + try: + with _Timeout(connection, limits.timeout_s) as timeout: + try: + connection.execute( + f"COPY (SELECT * FROM ({query}) AS q LIMIT {int(limits.max_rows)}) " + f"TO '{_escape(destination.as_posix())}' (FORMAT PARQUET)" + ) + rows = connection.execute(f"SELECT COUNT(*) FROM read_parquet('{_escape(destination.as_posix())}')").fetchone()[0] + except Exception as exc: # noqa: BLE001 + if timeout.fired: + raise ConnectorError(f"query exceeded timeout_s={limits.timeout_s}", code="timeout") from exc + raise ConnectorError(f"snapshot failed: {type(exc).__name__}: {exc}", code="query_failed") from exc + finally: + connection.close() + return int(rows) + + +def _srid_from(value: object) -> int | None: + if value is None: + return None + text = str(value).strip().upper() + if text.startswith("EPSG:"): + text = text[5:] + return int(text) if text.isdigit() else None diff --git a/openmapstack/connectors/postgis.py b/openmapstack/connectors/postgis.py new file mode 100644 index 0000000..1e3afa7 --- /dev/null +++ b/openmapstack/connectors/postgis.py @@ -0,0 +1,247 @@ +"""PostGIS connector (the warehouse half of the reference pair). + +Every session is opened with ``default_transaction_read_only = on`` and a +``statement_timeout``, discovery reads ``geometry_columns`` and planner +estimates, and a snapshot is materialised as GeoParquet through DuckDB from +rows fetched with geometry as WKB. + +PostgreSQL has no durable time travel: ``pg_export_snapshot()`` lives only as +long as its transaction. The pin for a PostGIS source is therefore always a +``local_snapshot``; the transaction snapshot identity and the schema digest +are recorded beside it as *retrieval* metadata, not as a pin. + +The DB-API driver (``psycopg`` 3, or ``psycopg2``) is imported lazily and +its absence is reported as ``driver_unavailable``. A ``connect`` callable +may be injected for tests. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from . import ( + ConnectorError, + ConnectorLimits, + ConnectorUnavailable, + Discovery, + QueryPlan, + TableInfo, + query_digest, + schema_digest, +) + +_GEOMETRY_TYPES = {"geometry", "geography"} + + +def _default_connect() -> Callable[[str], Any]: + try: + import psycopg # type: ignore[import-not-found] + + return lambda dsn: psycopg.connect(dsn) + except ImportError: + pass + try: + import psycopg2 # type: ignore[import-not-found] + + return lambda dsn: psycopg2.connect(dsn) + except ImportError as exc: + raise ConnectorUnavailable("a PostgreSQL driver (psycopg or psycopg2) is required for the postgis connector") from exc + + +class PostGISConnector: + backend = "postgis" + + def __init__(self, dsn: str, *, connect: Callable[[str], Any] | None = None) -> None: + self._dsn = dsn + self._connect = connect + self._identity: dict[str, Any] = {} + + # -- sessions --------------------------------------------------------------- + + def _session(self, limits: ConnectorLimits): + connect = self._connect or _default_connect() + try: + connection = connect(self._dsn) + except ConnectorError: + raise + except Exception as exc: # noqa: BLE001 - driver errors may carry the DSN + raise ConnectorError(f"cannot connect to PostGIS: {type(exc).__name__}", code="connection_failed") from exc + cursor = connection.cursor() + cursor.execute("SET default_transaction_read_only = on") + cursor.execute("SET transaction_read_only = on") + # SET cannot take a bound parameter; the value is an int we computed. + cursor.execute(f"SET statement_timeout = {int(limits.timeout_s * 1000)}") + return connection, cursor + + def identity_for_manifest(self) -> dict[str, Any]: + identity = {"backend": "postgis"} + for key in ("database", "server_version"): + if self._identity.get(key): + identity[key] = self._identity[key] + return identity + + # -- discovery -------------------------------------------------------------- + + def discover(self, limits: ConnectorLimits) -> Discovery: + connection, cursor = self._session(limits) + try: + cursor.execute("SELECT current_database(), current_user, version()") + database, user, version = cursor.fetchone() + self._identity = {"database": database, "user": user, "server_version": str(version).split(",")[0]} + cursor.execute( + "SELECT g.f_table_schema, g.f_table_name, g.f_geometry_column, g.srid, g.type, " + "c.reltuples::bigint, c.relkind " + "FROM geometry_columns g " + "LEFT JOIN pg_namespace n ON n.nspname = g.f_table_schema " + "LEFT JOIN pg_class c ON c.relname = g.f_table_name AND c.relnamespace = n.oid " + "ORDER BY 1, 2, 3" + ) + tables = [ + TableInfo( + str(schema), str(name), str(column), int(srid) if srid is not None else None, + str(geometry_type) if geometry_type else None, + int(estimate) if estimate is not None and estimate >= 0 else None, + kind="view" if relkind in ("v", "m") else "table", + ) + for schema, name, column, srid, geometry_type, estimate, relkind in cursor.fetchall() + ] + cursor.execute("SHOW default_transaction_read_only") + read_only = str(cursor.fetchone()[0]).lower() in {"on", "true", "1"} + except ConnectorError: + raise + except Exception as exc: # noqa: BLE001 + raise ConnectorError(f"discovery failed: {type(exc).__name__}", code="discovery_failed") from exc + finally: + _close(connection) + notes = ["row_estimate comes from the planner (pg_class.reltuples); -1 means never analysed"] + return Discovery("postgis", dict(self._identity), tables, read_only=read_only, notes=notes) + + # -- queries ---------------------------------------------------------------- + + def _columns(self, cursor, query: str) -> list[dict[str, str]]: + cursor.execute(f"SELECT * FROM ({query}) AS q LIMIT 0") + description = cursor.description or [] + oids = sorted({int(column[1]) for column in description if column[1] is not None}) + names: dict[int, str] = {} + if oids: + cursor.execute("SELECT oid, typname FROM pg_type WHERE oid = ANY(%s)", (oids,)) + names = {int(oid): str(typname) for oid, typname in cursor.fetchall()} + return [{"name": str(column[0]), "type": names.get(int(column[1]), str(column[1]))} for column in description] + + def plan(self, query: str, limits: ConnectorLimits) -> QueryPlan: + connection, cursor = self._session(limits) + try: + columns = self._columns(cursor, query) + cursor.execute(f"SELECT count(*) FROM ({query}) AS q") + count = int(cursor.fetchone()[0]) + backend_snapshot: dict[str, Any] | None = None + try: + cursor.execute("SELECT pg_current_snapshot()::text") + backend_snapshot = { + "kind": "pg_current_snapshot", + "value": str(cursor.fetchone()[0]), + "durable": False, + "note": "PostgreSQL keeps no durable time travel; the pin is the local snapshot", + } + except Exception: # noqa: BLE001 - PostgreSQL < 13 + try: + connection.rollback() + except Exception: # noqa: BLE001 + pass + except ConnectorError: + raise + except Exception as exc: # noqa: BLE001 + raise ConnectorError(f"query failed: {type(exc).__name__}", code="query_failed") from exc + finally: + _close(connection) + return QueryPlan(columns, count, query_digest(query), schema_digest(columns), backend_snapshot) + + def materialize(self, query: str, destination: Path, limits: ConnectorLimits) -> int: + from ..checks.spatial import connect_spatial + + duck = connect_spatial() + if duck is None: + raise ConnectorUnavailable("materialising GeoParquet requires duckdb with Spatial (pip install 'openmapstack[geo]')") + connection, cursor = self._session(limits) + try: + columns = self._columns(cursor, query) + geometry_columns = [column["name"] for column in columns if column["type"] in _GEOMETRY_TYPES] + selected = ", ".join( + f'ST_AsBinary("{column["name"]}") AS "{column["name"]}"' if column["name"] in geometry_columns else f'"{column["name"]}"' + for column in columns + ) + cursor.execute(f"SELECT {selected} FROM ({query}) AS q LIMIT %s", (int(limits.max_rows),)) + rows = cursor.fetchall() + srids: dict[str, int | None] = {} + for name in geometry_columns: + cursor.execute(f'SELECT ST_SRID("{name}") FROM ({query}) AS q WHERE "{name}" IS NOT NULL LIMIT 1') + row = cursor.fetchone() + srids[name] = int(row[0]) if row and row[0] else None + except ConnectorError: + raise + except Exception as exc: # noqa: BLE001 + raise ConnectorError(f"snapshot failed: {type(exc).__name__}", code="query_failed") from exc + finally: + _close(connection) + try: + _write_parquet(duck, destination, columns, geometry_columns, srids, rows) + finally: + duck.close() + return len(rows) + + +def _write_parquet(duck, destination: Path, columns, geometry_columns, srids, rows) -> None: + duck_types = { + "int2": "SMALLINT", "int4": "INTEGER", "int8": "BIGINT", "float4": "FLOAT", "float8": "DOUBLE", + "numeric": "DOUBLE", "bool": "BOOLEAN", "date": "DATE", "timestamp": "TIMESTAMP", "timestamptz": "TIMESTAMPTZ", + "json": "JSON", "jsonb": "JSON", "uuid": "UUID", + } + definitions = [] + for column in columns: + if column["name"] in geometry_columns: + definitions.append(f'"{column["name"]}" BLOB') + else: + definitions.append(f'"{column["name"]}" {duck_types.get(column["type"], "VARCHAR")}') + duck.execute(f"CREATE TABLE staging ({', '.join(definitions)})") + placeholders = ", ".join("?" for _ in columns) + if rows: + duck.executemany(f"INSERT INTO staging VALUES ({placeholders})", [tuple(_plain(value) for value in row) for row in rows]) + selected = [] + for column in columns: + name = column["name"] + if name in geometry_columns: + srid = srids.get(name) + geometry_expression = f'ST_GeomFromWKB("{name}")' + if srid: + try: + duck.execute(f"SELECT ST_SetSRID(ST_GeomFromWKB(NULL::BLOB), {int(srid)})") + geometry_expression = f'ST_SetSRID(ST_GeomFromWKB("{name}"), {int(srid)})' + except Exception: # noqa: BLE001 - older Spatial without CRS support + pass + selected.append(f'{geometry_expression} AS "{name}"') + else: + selected.append(f'"{name}"') + duck.execute(f"COPY (SELECT {', '.join(selected)} FROM staging) TO '{destination.as_posix().replace(chr(39), chr(39) * 2)}' (FORMAT PARQUET)") + + +def _plain(value: Any) -> Any: + if isinstance(value, memoryview): + return bytes(value) + if isinstance(value, (dict, list)): + import json + + return json.dumps(value) + return value + + +def _close(connection) -> None: + try: + connection.rollback() + except Exception: # noqa: BLE001 + pass + try: + connection.close() + except Exception: # noqa: BLE001 + pass diff --git a/pyproject.toml b/pyproject.toml index 8038f8d..286e904 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,10 @@ geo = ["duckdb>=1.2"] # Dashboard checks in a real browser. PyQGIS has no PyPI distribution, so # the QGIS checks depend on a system install and degrade to not_testable. visual = ["playwright>=1.40"] -all = ["openmapstack[geo,visual]"] +# `openmapstack source discover/snapshot` against PostGIS. The DuckDB +# local-file connector needs only [geo]. +postgis = ["psycopg[binary]>=3.1"] +all = ["openmapstack[geo,visual,postgis]"] [project.scripts] openmapstack = "openmapstack.cli:main" diff --git a/references/user-data-sources.md b/references/user-data-sources.md new file mode 100644 index 0000000..4d36e87 --- /dev/null +++ b/references/user-data-sources.md @@ -0,0 +1,157 @@ +# User data sources — warehouses, credentials, snapshots, clean reruns + +Read this when an analysis must read the user's own tables: a PostGIS +database, a DuckDB file, or a directory of GeoParquet/GeoPackage files that +is not a public download. It covers what `references/data-sources.md` does +not: data that has an owner, a credential, and no published version. + +Treat warehouse access as a **connector and security problem**, not only a +documentation problem. The rules below are enforced by `openmapstack validate` +and `openmapstack verify`; the CLI implements them for the two pilot +backends, and the rest of this file says what to do by hand elsewhere. + +## The four rules + +1. **Credentials by reference.** `project.yaml` never contains a password, + a token, a key, or a DSN with a password in it. `access.connection` is a + reference: `env:NAME`, `service:NAME` (an entry in `pg_service.conf`), + `keyring:NAME`, or `file:/absolute/path/outside/the/project`. Both + `validate` (`source.credentials`) and `verify` + (`provenance.no_inline_credentials`) scan every source for embedded + secrets and fail the project when they find one. Their findings name the + manifest path and the pattern, never the secret. +2. **Discovery is read-only.** A connector session opens with + `default_transaction_read_only = on` and a statement timeout, lists + tables, geometry columns, SRIDs, and row estimates, and only ever runs + a single `SELECT`. DML, DDL, `COPY`, `ATTACH`, `INSTALL`, `SET`, and + file-reading table functions are rejected before anything reaches the + server. +3. **Materialising data locally needs explicit approval.** + `openmapstack source snapshot` is a dry run by default: it reports the + schema and row count the query would copy. Only `--approve` writes the + snapshot, and it writes under `data/source/` only, never over an + existing file, and never beyond the row and byte limits. +4. **A warehouse table is pinned only by a pin class.** A timestamp string + is not a pin. Either the bytes are frozen locally (`local_snapshot`, + hash-matched) or the backend can serve that exact state again + (`backend_snapshot` with an identifier and a retention limit). An + expired or inaccessible backend snapshot is reported as + `not_reproducible`. See `project-spec.md` section 2.2. + +## Declaring a warehouse source + +```yaml +sources: + parcels: + role: authoritative_input + provider: City GIS department + dataset: cadastral parcels (warehouse copy) + source_url: postgresql://geo-prod.internal/gis # identity only, no credentials + access: + method: postgis + connection: {ref: "env:PARCELS_DSN"} + retrieved_at: "2026-08-30T10:00:00Z" + warehouse: + backend: postgis # duckdb | postgis are the verified pilot backends + account: geo-prod + database: gis + schema: cadastre + table: parcels + query_sha256: "sha256:..." # written by `source snapshot` + schema_sha256: "sha256:..." + pin: + class: local_snapshot + path: data/source/parcels.parquet + sha256: "sha256:..." + captured_at: "2026-08-30T10:00:00Z" + version: + identifier: "cadastre.parcels @ pg_current_snapshot 1001:1001: on 2026-08-30" + published_at: 2026-08-30 + selection: + filter: "municipality = 'Tartu linn'" + license: {name: "Internal — see data owner", url: https://intranet.example/gis-data-policy} + schema: {crs: EPSG:3301, key: cadastral_id, columns: [cadastral_id, land_use, geom]} + rationale: The warehouse copy is the department's authoritative parcel layer. +``` + +## The CLI path (DuckDB local files and PostGIS) + +```bash +# 1. Read-only discovery: what is there, which column is geometry, which SRID. +openmapstack source discover project.yaml --source parcels +openmapstack source discover project.yaml --source parcels --json + +# 2. Dry run: schema, row count, and digests of the query you intend to freeze. +openmapstack source snapshot project.yaml --source parcels \ + --query "SELECT cadastral_id, land_use, geom FROM cadastre.parcels WHERE municipality = 'Tartu linn'" \ + --destination data/source/parcels.parquet + +# 3. Materialise, after the user has approved the row count and the bytes. +openmapstack source snapshot project.yaml --source parcels --query-file queries/parcels.sql \ + --destination data/source/parcels.parquet --approve --max-rows 200000 --timeout 120 + +# 4. Record the pin (printed as YAML; --write-manifest rewrites project.yaml +# and drops YAML comments, so most projects paste the block instead). +``` + +The snapshot is GeoParquet with geometry typed and, where the installed +DuckDB Spatial supports it, carrying the source SRID. The command returns the +`pin` block, the `warehouse` digests, and `access.retrieved_at` to place in +the manifest. Once placed, `openmapstack validate` reports `source.pin` as +`passed` with `pin_class: local_snapshot`. + +### DuckDB local files + +With `warehouse.backend: duckdb` and no `access.connection`, the connector +root is the project's own `data/source/`. Every geodata file under it is +exposed as a view named by its relative path, so a query reads +`FROM "parcels.geojson"` and never spells a filesystem path. File access is +confined to that root (`allowed_directories` + `enable_external_access = +false`); a query that reaches outside it fails. A `.duckdb` database is +attached read-only when the connection names one. + +### PostGIS + +`access.connection` resolves to a DSN through the reference. The session is +read-only with `statement_timeout`; discovery reads `geometry_columns` and +planner estimates; the snapshot fetches geometry as WKB and writes GeoParquet +through DuckDB (`openmapstack[geo]`), with the driver from +`openmapstack[postgis]`. + +PostgreSQL has **no durable time travel**: `pg_export_snapshot()` lives only +as long as its transaction. The pin for a PostGIS source is therefore the +local snapshot; the connector records `pg_current_snapshot()` and the schema +digest beside it as retrieval metadata (`durable: false`), never as a pin. +Declaring `pin.class: backend_snapshot` for PostGIS is honest only when an +external mechanism (a logical replica frozen for the project, a `pg_dump` +retained under a stated policy) provides the retention you record. + +## Other backends + +Only DuckDB and PostGIS are verified. `warehouse.backend` may name another +system (`bigquery`, `snowflake`, `motherduck`, `databricks`, `redshift`, +`athena`, `iceberg`, `delta`), but the CLI refuses to connect to it +(`backend_unsupported`) rather than guessing its semantics. For those: + +- pull the data with the vendor's tooling into `data/source/` and pin it as + a `local_snapshot`, or +- use the backend's own snapshot/time-travel identity (Snowflake `AT + (STATEMENT => ...)`, BigQuery `FOR SYSTEM_TIME AS OF`, Iceberg/Delta + snapshot ids) as a `backend_snapshot`, **recording the retention limit + the vendor actually guarantees** (Snowflake Time Travel defaults to one + day; BigQuery keeps seven), and expect `verify` to report + `not_reproducible` once that passes. + +Never approximate a pin by pasting the current date. The point of the pin +contract is that a reviewer can tell the difference. + +## Clean rerun with warehouse sources + +A clean rerun copies only `data/source/`, `data/overrides/`, the manifest, +and declared dependencies, and executes the pipeline with session/provider +environment variables removed. A pipeline that reads the warehouse live at +run time therefore fails the rerun unless the credential reference resolves +in the rerun environment — and if it does, the rerun proves only that the +warehouse still answers, not that it answers the same thing. Read from the +local snapshot in the pipeline; keep the query that produced it under +`warehouse.query_sha256` so the snapshot can be refreshed deliberately. diff --git a/tests/test_connectors.py b/tests/test_connectors.py new file mode 100644 index 0000000..19ff869 --- /dev/null +++ b/tests/test_connectors.py @@ -0,0 +1,339 @@ +"""Connector pilot: read-only discovery, approval-gated snapshots, limits, redaction. + +The DuckDB local-file tests run wherever ``openmapstack[geo]`` is installed. +The PostGIS session/discovery/plan contract is tested against a fake DB-API +driver so it needs no server; the end-to-end snapshot path runs only when +``OPENMAPSTACK_TEST_POSTGIS_DSN`` names a reachable PostGIS database. +""" + +from __future__ import annotations + +import io +import json +import os +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +import yaml + +from openmapstack.checks.spatial import connect_spatial +from openmapstack.cli import main +from openmapstack.connectors import ( + ConnectorError, + ConnectorLimits, + ConnectorUnavailable, + apply_snapshot_to_manifest, + discover_source, + require_read_only_select, + resolve_connection_reference, + snapshot_source, +) +from openmapstack.connectors.postgis import PostGISConnector +from openmapstack.sources import assess_pin +from openmapstack.validation import validate_project +from tests.evals.helpers import make_workspace, minimal_project, write_project + +REPO_ROOT = Path(__file__).resolve().parents[1] +FIXTURES = REPO_ROOT / "evals" / "fixtures" / "mini-tartu" +DUCKDB_AVAILABLE = connect_spatial() is not None +LIVE_DSN = os.environ.get("OPENMAPSTACK_TEST_POSTGIS_DSN", "") + + +def _duckdb_project(): + workspace = make_workspace() + project = minimal_project() + project["sources"]["test_source"]["warehouse"] = {"backend": "duckdb"} + write_project(workspace, project) + (workspace / "data/source").mkdir(parents=True) + for name in ("parcels.geojson", "roads.geojson"): + (workspace / "data/source" / name).write_bytes((FIXTURES / name).read_bytes()) + return workspace, project + + +class QueryPolicyTests(unittest.TestCase): + def test_only_a_single_select_is_accepted(self) -> None: + self.assertTrue(require_read_only_select(" SELECT 1; ").startswith("SELECT")) + self.assertTrue(require_read_only_select("WITH t AS (SELECT 1) SELECT * FROM t")) + for bad in ( + "DROP TABLE parcels", + "SELECT 1; SELECT 2", + "SELECT * FROM read_text('/etc/passwd')", + "COPY (SELECT 1) TO '/tmp/x'", + "SELECT * INTO backup FROM parcels", + "SET enable_external_access = true", + "INSTALL httpfs", + "", + ): + with self.assertRaises(ConnectorError, msg=bad) as caught: + require_read_only_select(bad) + self.assertEqual(caught.exception.code, "query_rejected") + + def test_keywords_inside_string_literals_do_not_trip_the_policy(self) -> None: + self.assertTrue(require_read_only_select("SELECT 'drop' AS word, 'set;' AS other")) + + def test_connection_references_resolve_without_recording_secrets(self) -> None: + scheme, secret = resolve_connection_reference("env:OMS_TEST_DSN", environ={"OMS_TEST_DSN": "postgresql://u:pw@h/db"}) + self.assertEqual((scheme, secret), ("env", "postgresql://u:pw@h/db")) + self.assertEqual(resolve_connection_reference({"ref": "service:geo"}, environ={}), ("service", "service=geo")) + with self.assertRaises(ConnectorError) as caught: + resolve_connection_reference("env:OMS_TEST_DSN", environ={}) + self.assertEqual(caught.exception.code, "connection_unresolved") + with self.assertRaises(ConnectorError): + resolve_connection_reference("postgresql://u:pw@h/db", environ={}) + with self.assertRaises(ConnectorError): + resolve_connection_reference("file:relative/path", environ={}) + + def test_connector_errors_are_redacted(self) -> None: + error = ConnectorError("cannot open postgresql://gis:hunter2@db/gis", code="connection_failed") + self.assertNotIn("hunter2", str(error)) + self.assertIn("gis:***@db", str(error)) + + +@unittest.skipUnless(DUCKDB_AVAILABLE, "DuckDB Spatial is not available") +class DuckDBLocalConnectorTests(unittest.TestCase): + def test_discovery_describes_files_read_only(self) -> None: + workspace, project = _duckdb_project() + discovery = discover_source(project, "test_source", project_root=workspace) + self.assertEqual(discovery.backend, "duckdb") + self.assertTrue(discovery.read_only) + by_name = {table.name: table for table in discovery.tables} + self.assertEqual(by_name["parcels.geojson"].geometry_column, "geom") + self.assertEqual(by_name["parcels.geojson"].srid, 3301) + self.assertEqual(by_name["parcels.geojson"].row_estimate, 5) + self.assertEqual(discovery.notes, []) + + def test_snapshot_is_a_dry_run_unless_approved(self) -> None: + workspace, project = _duckdb_project() + query = 'SELECT cadastral_id, geom FROM "parcels.geojson" WHERE land_use = \'ARIMAA\'' + record = snapshot_source(project, "test_source", query, "data/source/arimaa.parquet", project_root=workspace) + self.assertFalse(record["materialized"]) + self.assertEqual(record["plan"]["row_count"], 2) + self.assertFalse((workspace / "data/source/arimaa.parquet").exists()) + + record = snapshot_source(project, "test_source", query, "data/source/arimaa.parquet", project_root=workspace, approve=True) + self.assertTrue(record["materialized"]) + self.assertEqual(record["rows"], 2) + self.assertEqual(record["pin"]["class"], "local_snapshot") + self.assertEqual(record["warehouse"]["query_sha256"], record["plan"]["query_sha256"]) + # The pin it hands back is one the pin contract accepts as pinned. + updated = apply_snapshot_to_manifest(project, "test_source", record) + self.assertEqual(assess_pin(workspace, updated["sources"]["test_source"]).status, "pinned") + write_project(workspace, updated) + (workspace / "pipeline.py").write_text("print('ok')\n", encoding="utf-8") + checks = {c.id: c.status for c in validate_project(workspace / "project.yaml", artifacts=False).checks if c.id in {"source.pin", "source.credentials"}} + self.assertEqual(checks, {"source.pin": "passed", "source.credentials": "passed"}) + + def test_snapshots_never_overwrite_an_existing_source(self) -> None: + workspace, project = _duckdb_project() + with self.assertRaises(ConnectorError) as caught: + snapshot_source(project, "test_source", 'SELECT * FROM "parcels.geojson"', "data/source/parcels.geojson", project_root=workspace, approve=True) + self.assertEqual(caught.exception.code, "destination_invalid") + (workspace / "data/source/taken.parquet").write_bytes(b"x") + with self.assertRaises(ConnectorError) as caught: + snapshot_source(project, "test_source", 'SELECT * FROM "parcels.geojson"', "data/source/taken.parquet", project_root=workspace, approve=True) + self.assertEqual(caught.exception.code, "destination_exists") + with self.assertRaises(ConnectorError) as caught: + snapshot_source(project, "test_source", 'SELECT * FROM "parcels.geojson"', "data/derived/out.parquet", project_root=workspace, approve=True) + self.assertEqual(caught.exception.code, "destination_invalid") + + def test_row_and_byte_limits_refuse_and_leave_no_partial_file(self) -> None: + workspace, project = _duckdb_project() + with self.assertRaises(ConnectorError) as caught: + snapshot_source(project, "test_source", 'SELECT * FROM "parcels.geojson"', "data/source/big.parquet", project_root=workspace, limits=ConnectorLimits(max_rows=2)) + self.assertEqual(caught.exception.code, "row_limit_exceeded") + with self.assertRaises(ConnectorError) as caught: + snapshot_source(project, "test_source", 'SELECT * FROM "parcels.geojson"', "data/source/tiny.parquet", project_root=workspace, approve=True, limits=ConnectorLimits(max_bytes=50)) + self.assertEqual(caught.exception.code, "byte_limit_exceeded") + self.assertEqual(sorted(p.name for p in (workspace / "data/source").iterdir()), ["parcels.geojson", "roads.geojson"]) + + def test_file_access_is_confined_to_the_source_root(self) -> None: + workspace, project = _duckdb_project() + outside = workspace / "outside.parquet" + (workspace / "data/derived").mkdir() + for target in (outside, workspace / "data/derived/secret.parquet"): + with self.assertRaises(ConnectorError, msg=target) as caught: + snapshot_source(project, "test_source", f"SELECT * FROM read_parquet('{target.as_posix()}')", "data/source/leak.parquet", project_root=workspace) + self.assertEqual(caught.exception.code, "query_failed") + + def test_cli_discover_and_snapshot(self) -> None: + workspace, _ = _duckdb_project() + out = io.StringIO() + with redirect_stdout(out): + self.assertEqual(main(["source", "discover", str(workspace), "--source", "test_source", "--json"]), 0) + self.assertEqual(json.loads(out.getvalue())["schema"], "openmapstack-source-discovery/v1") + out = io.StringIO() + with redirect_stdout(out): + code = main(["source", "snapshot", str(workspace), "--source", "test_source", "--query", 'SELECT * FROM "roads.geojson"', "--destination", "data/source/roads.parquet"]) + self.assertEqual(code, 0) + self.assertIn("DRY RUN", out.getvalue()) + out = io.StringIO() + with redirect_stdout(out): + code = main(["source", "snapshot", str(workspace), "--source", "test_source", "--query", 'SELECT * FROM "roads.geojson"', "--destination", "data/source/roads.parquet", "--approve", "--write-manifest", "--json"]) + self.assertEqual(code, 0) + record = json.loads(out.getvalue()) + self.assertTrue(record["materialized"]) + manifest = yaml.safe_load((workspace / "project.yaml").read_text(encoding="utf-8")) + self.assertEqual(manifest["sources"]["test_source"]["pin"]["path"], "data/source/roads.parquet") + err = io.StringIO() + with redirect_stderr(err), redirect_stdout(io.StringIO()): + self.assertEqual(main(["source", "snapshot", str(workspace), "--source", "test_source", "--query", "DROP TABLE x", "--destination", "data/source/x.parquet"]), 2) + self.assertIn("query_rejected", err.getvalue()) + + def test_unsupported_backend_is_refused_not_guessed(self) -> None: + workspace, project = _duckdb_project() + project["sources"]["test_source"]["warehouse"] = {"backend": "bigquery"} + project["sources"]["test_source"]["access"]["connection"] = "env:BQ" + with self.assertRaises(ConnectorError) as caught: + discover_source(project, "test_source", project_root=workspace, environ={"BQ": "x"}) + self.assertEqual(caught.exception.code, "backend_unsupported") + + +class _FakeCursor: + """Answers the exact statements the PostGIS connector issues.""" + + def __init__(self, log: list[str], *, snapshot_supported: bool = True) -> None: + self.log = log + self.description = None + self._rows: list[tuple] = [] + self._snapshot_supported = snapshot_supported + + def execute(self, statement: str, params=None) -> None: + self.log.append(statement if params is None else f"{statement} -- {params}") + text = statement.strip().lower() + self.description = None + if text.startswith("select current_database()"): + self._rows = [("gis", "reader", "PostgreSQL 16.3 (Debian), compiled by gcc")] + elif "from geometry_columns" in text: + self._rows = [("cadastre", "parcels", "geom", 3301, "MULTIPOLYGON", 79056, "r")] + elif text.startswith("show default_transaction_read_only"): + self._rows = [("on",)] + elif "limit 0" in text: + self.description = [("cadastral_id", 25, None, None, None, None, None), ("geom", 17000, None, None, None, None, None)] + self._rows = [] + elif text.startswith("select oid, typname"): + self._rows = [(25, "text"), (17000, "geometry")] + elif text.startswith("select count(*)"): + self._rows = [(3,)] + elif "pg_current_snapshot" in text: + if not self._snapshot_supported: + raise RuntimeError("function pg_current_snapshot() does not exist") + self._rows = [("1001:1001:",)] + else: + self._rows = [] + + def fetchone(self): + return self._rows[0] if self._rows else None + + def fetchall(self): + return list(self._rows) + + +class _FakeConnection: + def __init__(self, log: list[str], **kwargs) -> None: + self.log = log + self.kwargs = kwargs + self.closed = False + + def cursor(self): + return _FakeCursor(self.log, **self.kwargs) + + def rollback(self) -> None: + self.log.append("ROLLBACK") + + def close(self) -> None: + self.closed = True + + +class PostGISContractTests(unittest.TestCase): + def test_sessions_are_read_only_with_a_statement_timeout(self) -> None: + log: list[str] = [] + connector = PostGISConnector("postgresql://reader:pw@db/gis", connect=lambda dsn: _FakeConnection(log)) + discovery = connector.discover(ConnectorLimits(timeout_s=12.5)) + self.assertEqual(log[0], "SET default_transaction_read_only = on") + self.assertIn("SET statement_timeout = 12500", log) + self.assertTrue(discovery.read_only) + [table] = discovery.tables + self.assertEqual((table.schema, table.name, table.geometry_column, table.srid, table.row_estimate), ("cadastre", "parcels", "geom", 3301, 79056)) + self.assertEqual(discovery.identity["database"], "gis") + self.assertNotIn("pw", json.dumps(discovery.to_dict())) + + def test_plan_records_schema_digest_and_non_durable_backend_snapshot(self) -> None: + log: list[str] = [] + connector = PostGISConnector("dsn", connect=lambda dsn: _FakeConnection(log)) + plan = connector.plan("SELECT cadastral_id, geom FROM cadastre.parcels", ConnectorLimits()) + self.assertEqual(plan.row_count, 3) + self.assertEqual([c["type"] for c in plan.columns], ["text", "geometry"]) + self.assertEqual(plan.backend_snapshot["kind"], "pg_current_snapshot") + self.assertFalse(plan.backend_snapshot["durable"]) + self.assertTrue(plan.schema_sha256.startswith("sha256:")) + + def test_plan_survives_servers_without_pg_current_snapshot(self) -> None: + log: list[str] = [] + connector = PostGISConnector("dsn", connect=lambda dsn: _FakeConnection(log, snapshot_supported=False)) + plan = connector.plan("SELECT 1", ConnectorLimits()) + self.assertIsNone(plan.backend_snapshot) + + def test_missing_driver_is_reported_not_hidden(self) -> None: + import openmapstack.connectors.postgis as module + + original = module._default_connect + + def unavailable(): + raise ConnectorUnavailable("no driver") + + module._default_connect = unavailable + try: + with self.assertRaises(ConnectorUnavailable): + PostGISConnector("dsn").discover(ConnectorLimits()) + finally: + module._default_connect = original + + def test_driver_errors_never_leak_the_dsn(self) -> None: + def failing(dsn: str): + raise RuntimeError(f"could not connect to {dsn}") + + connector = PostGISConnector("postgresql://gis:hunter2@db/gis", connect=failing) + with self.assertRaises(ConnectorError) as caught: + connector.discover(ConnectorLimits()) + self.assertEqual(caught.exception.code, "connection_failed") + self.assertNotIn("hunter2", str(caught.exception)) + + +@unittest.skipUnless(LIVE_DSN and DUCKDB_AVAILABLE, "OPENMAPSTACK_TEST_POSTGIS_DSN is not set") +class PostGISLiveTests(unittest.TestCase): + """End-to-end against a real PostGIS: discovery, dry run, materialised + GeoParquet snapshot, and a pin the contract accepts.""" + + def test_snapshot_round_trip(self) -> None: + workspace = make_workspace() + project = minimal_project() + project["sources"]["test_source"]["warehouse"] = {"backend": "postgis", "schema": "public", "table": "parcels"} + project["sources"]["test_source"]["access"]["connection"] = "env:OPENMAPSTACK_TEST_POSTGIS_DSN" + write_project(workspace, project) + discovery = discover_source(project, "test_source", project_root=workspace) + self.assertTrue(discovery.read_only) + self.assertTrue(any(table.name == "parcels" for table in discovery.tables), discovery.to_dict()) + query = "SELECT cadastral_id, land_use, geom FROM public.parcels ORDER BY cadastral_id" + dry = snapshot_source(project, "test_source", query, "data/source/parcels.parquet", project_root=workspace) + self.assertFalse(dry["materialized"]) + record = snapshot_source(project, "test_source", query, "data/source/parcels.parquet", project_root=workspace, approve=True) + self.assertTrue(record["materialized"]) + self.assertEqual(record["rows"], dry["plan"]["row_count"]) + self.assertNotIn("oms-test-pw", json.dumps(record)) + updated = apply_snapshot_to_manifest(project, "test_source", record) + self.assertEqual(assess_pin(workspace, updated["sources"]["test_source"]).status, "pinned") + connection = connect_spatial() + try: + rows = connection.execute(f"SELECT cadastral_id, ST_GeometryType(geom) FROM read_parquet('{(workspace / 'data/source/parcels.parquet').as_posix()}') ORDER BY 1").fetchall() + finally: + connection.close() + self.assertEqual([row[0] for row in rows], ["P1", "P2", "P3"]) + self.assertTrue(all("POLYGON" in str(row[1]).upper() for row in rows), rows) + # A write attempt through the same reference must be refused by the policy. + with self.assertRaises(ConnectorError): + snapshot_source(project, "test_source", "DELETE FROM public.parcels", "data/source/x.parquet", project_root=workspace, approve=True) + + +if __name__ == "__main__": + unittest.main() From 8b83bbef363a4230ae927f80abfaa3e7d62569c0 Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 2 Sep 2026 17:03:42 +0000 Subject: [PATCH 4/8] feat(api): versioned check API and result schemas for external harnesses openmapstack-check-api/v1 is the surface OpenMapBench consumes without vendoring a check: api_info/negotiate for compatibility, list_checks for the catalogue with parameters, oracle_free flags and reporting dimensions, run_check returning an openmapstack-check-result/v1 record, and the packaged verify-result schema. A check that raises is not_testable with check_error, never a pass or a graded failure. Reporting dimensions are owned by openmapstack.api and imported by the eval runner; visual checks now report under their own visual_judgement bucket. CLI: openmapstack checks, check NAME WORKSPACE --arg k=v, api-info [--require-api --min-version --require-check]. The consumer fixture test proves the contract from the CLI and JSON schemas alone. Refs #13 (C1). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01TMMy6A73zi7yGjK3wHBhju --- README.md | 1 + docs/maintainers/architecture.md | 1 + .../0004-narrow-benchmark-interface.md | 37 ++ docs/openmapbench-interop.md | 97 ++++++ evals/run.py | 17 +- openmapstack/api.py | 319 ++++++++++++++++++ openmapstack/cli.py | 104 ++++++ .../schemas/check-result-v1.schema.json | 30 ++ .../schemas/verify-result-v1.schema.json | 50 +++ tests/test_check_api.py | 194 +++++++++++ 10 files changed, 836 insertions(+), 14 deletions(-) create mode 100644 docs/maintainers/decisions/0004-narrow-benchmark-interface.md create mode 100644 docs/openmapbench-interop.md create mode 100644 openmapstack/api.py create mode 100644 openmapstack/schemas/check-result-v1.schema.json create mode 100644 openmapstack/schemas/verify-result-v1.schema.json create mode 100644 tests/test_check_api.py diff --git a/README.md b/README.md index 1bdfbd8..d3c4e9f 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ It is open-first and cloud-native by default, built on shoulders of the awesome - [examples/tartu-development/](examples/tartu-development/) — a fully-worked reproducible project matching the acceptance scenario: source provenance + timestamps, explicit assumptions, two verified project overrides (a scenario attribute change with prior-value verification, and hypothetical scenario geometry), deterministic pipeline, machine-readable validation, and semantic presentation. - [evals/](evals/) — the eval suite grading whether an agent reaches the right analytical answer, respects the GIS-method guardrails, and reruns reproducibly, with the `openmapstack-project/v1` contract as the substrate that makes those independently checkable: `python evals/run.py --mode fixture` runs deterministic, no-LLM checks against real generated artifacts (analytical correctness against known geospatial truth, metric CRS, source immutability, schema, overrides, validation integrity, presentation contract, and clean reruns), plus adversarial cases and a pluggable live-agent benchmark (Claude Code, Codex, and any OpenAI-compatible API such as OpenRouter — URL and model via `OPENAI_COMPATIBLE_*` env, API key as a secret). - [`openmapstack/`](openmapstack/) — the installable `openmapstack validate/run/inspect` CLI for auditing and executing `openmapstack-project/v1` projects, plus [`openmapstack/checks/`](openmapstack/checks/): the reusable, semantic check library. All but five of its checks are oracle-free, so the same functions that grade the eval suite also grade a user's own project on data this repository has never seen. +- [docs/openmapbench-interop.md](docs/openmapbench-interop.md) — the narrow, versioned contract a benchmark harness such as OpenMapBench consumes: `openmapstack checks` / `check` / `api-info` (`openmapstack-check-api/v1`), the packaged result schemas, skill snapshots, arm provenance, and exported task bundles. - [`.claude-plugin/`](.claude-plugin/) — Claude Code plugin and marketplace manifests, so the repository can also be installed with `/plugin install`. Validated in CI by [`.github/workflows/plugin.yml`](.github/workflows/plugin.yml). My local Estonia-specific guidance (Maa- ja Ruumiamet, ETAK, EPSG:3301 / L-EST97) is included for convenience. But all the global sources are incuded for world-wide coverage. diff --git a/docs/maintainers/architecture.md b/docs/maintainers/architecture.md index c169ec9..f6ca4be 100644 --- a/docs/maintainers/architecture.md +++ b/docs/maintainers/architecture.md @@ -36,6 +36,7 @@ The distinction matters: `SKILL.md` is the product being developed and evaluated - Project contract: `references/project-spec.md`; machine validation is also constrained by `openmapstack/schemas/project-v1.schema.json` and `openmapstack/validation.py`. - Automatic `verify` plan and applicability: `docs/verify-applicability.md` plus `openmapstack/verify.py`. - Eval semantics: `evals/README.md`, `evals/schemas/`, `evals/run.py`, case definitions, and tests. +- External check consumption: `docs/openmapbench-interop.md` plus `openmapstack/api.py` and the packaged result schemas; reporting dimensions are owned by `openmapstack.api.DIMENSIONS`. - Roadmap/current work: GitHub issues. Do not mirror their checklists here. When these disagree, resolve the inconsistency at the owning layer rather than adding another interpretation here. diff --git a/docs/maintainers/decisions/0004-narrow-benchmark-interface.md b/docs/maintainers/decisions/0004-narrow-benchmark-interface.md new file mode 100644 index 0000000..1ae5f77 --- /dev/null +++ b/docs/maintainers/decisions/0004-narrow-benchmark-interface.md @@ -0,0 +1,37 @@ +# 0004 — A narrow versioned check API instead of an exported benchmark harness + +- Status: Accepted +- Date: 2026-09-02 +- Related: issue #13 (C1–C3); `openmapstack/api.py`; `openmapstack/snapshot.py`; `docs/openmapbench-interop.md`; `evals/run.py` + +## Context + +OpenMapBench needs to grade produced projects with the same checks this repository uses, record which skill snapshot and model configuration produced each result, and compare a plain arm with a skill arm. Two easy paths were available: let OpenMapBench vendor `openmapstack/checks/`, or export `evals/run.py` as the benchmark harness. Both couple the user-facing package to model-provider orchestration and let the two check libraries drift. + +## Decision + +Publish a small, versioned interface and nothing more: + +- `openmapstack-check-api/v1` (`api_info`, `negotiate`, `list_checks`, `run_check`, `verify --json`) with packaged JSON schemas for results; +- `openmapstack-skill-snapshot/v1` for the controlled copy of the shipped skill; +- `openmapstack-benchmark-arm/v1` as the provenance tuple a published arm must record; +- `openmapstack-benchmark-task/v1` bundles exported from the eval cases. + +Reporting dimensions are owned by `openmapstack.api.DIMENSIONS` and imported by the eval runner. Agent adapters, leaderboard policy, and run isolation remain in OpenMapBench. + +## Consequences + +- OpenMapBench upgrades by pinning a check API major and a minimum package version; `negotiate()` refuses to grade with an unknown API rather than grading wrongly. +- New checks are additive; renaming or removing one is a major bump with a migration note. +- Setup failures (`check_error`, adapter failure, missing CLI) stay outside scored denominators on both sides. +- Paired `plain`/`oms` runs in this repository exist to prove task parity and provenance recording; the public comparison is OpenMapBench's. + +## Alternatives considered + +### Vendor the check library into OpenMapBench + +Rejected: two copies of the same predicate drift, and a benchmark grading with a stale copy silently rewards a project the shipped verifier would fail. + +### Export `evals/run.py` as the benchmark harness + +Rejected: the runner's adapters, credential handling, and artifact layout are provider-specific compatibility surfaces. Making them the generic architecture would leak vendor terminology into shared interfaces and tie package releases to harness releases. diff --git a/docs/openmapbench-interop.md b/docs/openmapbench-interop.md new file mode 100644 index 0000000..d9af4fd --- /dev/null +++ b/docs/openmapbench-interop.md @@ -0,0 +1,97 @@ +# OpenMapBench interoperability contract + +OpenMapBench owns benchmark orchestration, run isolation, provider adapters, +leaderboard policy, and task governance. OpenMapStack owns the project +contract and the checks that grade a produced project. This document is the +narrow surface between them. Neither side copies the other's implementation: +OpenMapBench consumes a released `openmapstack` package through the API +below; OpenMapStack does not export a second benchmark harness. + +Owning code: `openmapstack/api.py`, `openmapstack/schemas/`, +`openmapstack/snapshot.py`, `evals/schemas/benchmark-arm-v1.schema.json`. +The consumer fixture that proves the contract without vendoring a check is +`tests/test_check_api.py::ConsumerFixtureTests`. + +## 1. Versioned check API — `openmapstack-check-api/v1` + +| Surface | Purpose | +|---|---| +| `openmapstack api-info --json` / `openmapstack.api.api_info()` | package version, check API version, project schema, result schemas, status vocabulary, dimensions | +| `openmapstack api-info --require-api … --min-version … --require-check …` / `negotiate()` | compatibility answer with every unmet requirement listed; exit 1 when incompatible | +| `openmapstack checks --json` / `list_checks()` | the catalogue: name, module, dimension, `oracle_free`, parameters with required/default | +| `openmapstack check NAME WORKSPACE --arg k=v --json` / `run_check()` | one check, one `openmapstack-check-result/v1` record | +| `openmapstack verify PROJECT --json` | the whole applicable plan, `openmapstack-verify-result/v1` | + +Additive changes (a new check, a new optional parameter, a new result +field) keep the major. Renaming or removing a check, changing a parameter's +meaning, or touching the four-state vocabulary bumps it. A consumer pins the +major and the minimum package version it was tested against. + +## 2. Result semantics a consumer may rely on + +- `status` ∈ `passed | failed | warning | not_testable`; a check that could + not establish its predicate is never `passed`. +- `code` is a stable machine identifier whenever `status` is not `passed`. + Grade on `status` and `code`; never on `detail` text. +- `dimension` names the reporting bucket (`gis_correctness`, + `reproducibility_compliance`, `provenance`, `override_handling`, + `validation_integrity`, `presentation_contract`, `rerun_success`, + `metamorphic_evidence`, `visual_judgement`). Buckets have separate + denominators and are never collapsed into one score. Deterministic + analytical correctness, metamorphic evidence, differential diagnostics, + and visual judgement stay apart. +- `oracle_free: false` marks the five known-answer checks. On arbitrary + data they are reachable only through attested `validation.expectations`; + a benchmark with a frozen expert reference may call them directly. +- A check that raises is `not_testable` with `code: check_error`. A + harness keeps such trials **outside the scored denominator** and reports + them prominently as setup failures, exactly as `evals/run.py` does + (`status: setup_failed`, exit 2). + +## 3. Reproducible arms — `openmapstack-benchmark-arm/v1` + +A published benchmark result identifies the *whole arm*, not a skill hash. +`evals/schemas/benchmark-arm-v1.schema.json` is the record OpenMapBench +must store per arm: + +| Field | Meaning | +|---|---| +| `arm` | `plain` (no skill) or `oms` (skill snapshot injected) | +| `skill` | mode, snapshot content hash, repository commit, dirty flag | +| `task_set` | case ids and a content hash over their prompts, expectations, and declared fixtures | +| `checker` | `openmapstack` package version and check API version | +| `harness` | harness repository commit and dirty flag | +| `runtime` | Python version, platform, DuckDB version, container image if any | +| `tool_surface` | adapter name and the exact agent CLI/API version it drove | +| `model` | provider, exact model id, provider revision/alias resolution when known | +| `sampling` | seed, temperature, reasoning configuration as the adapter reports them (nulls are allowed but must be present) | +| `price_catalog_date` | the date of the price list used for cost estimates | + +`openmapstack skill-snapshot --out DIR --json` produces the controlled +copy of `SKILL.md`, `references/`, and `templates/` with a per-file +inventory and content hash (`openmapstack-skill-snapshot/v1`); `--inspect` +re-verifies one. Symlinks and paths escaping the snapshot root are rejected. + +## 4. Task ownership and paired arms + +`evals/run.py --export-tasks DIR` writes the vendor-neutral task bundles +(`openmapstack-benchmark-task/v1`: prompt, declared fixtures, assertions, +hard gates, task hash) that OpenMapBench imports. Cases 070–073 (the +behavioural, prompt-style tasks) are exported as canonical OpenMapBench +tasks; this repository keeps them only as a scheduled smoke subset that +protects the integration and does not publish a competing benchmark. + +`evals/run.py --mode live --arms paired` runs `plain` and `oms` over the +same cases, trials, and seeds and reports them side by side: task parity, +per-arm success rate with a Wilson interval, per-arm median cost, tokens, +and duration, and trajectory diagnostics (event counts). It never emits a +single "success per dollar" number; quality and cost are reported as a +trade-off, and headline correctness is artifact-first. + +## 5. Evidence classes + +OpenMapBench distinguishes four evidence classes rather than one ground +truth: an authoritative answer, a frozen expert reference, metamorphic +evidence, and a differential diagnostic. Only the first two license a +known-answer check. VLM/visual review has its own denominator and judge +provenance and never turns an unverified analytical result into a pass. diff --git a/evals/run.py b/evals/run.py index c4bf840..9058961 100755 --- a/evals/run.py +++ b/evals/run.py @@ -72,20 +72,9 @@ def _load_eval_schema(name: str) -> dict[str, Any]: return json.loads((EVALS_DIR / "schemas" / name).read_text(encoding="utf-8")) -DIMENSIONS = { - "project": "reproducibility_compliance", - "overrides": "override_handling", - "provenance": "provenance", - "geodata": "gis_correctness", - "validation": "validation_integrity", - "qgis": "presentation_contract", - "presentation": "presentation_contract", - "rerun": "rerun_success", - # Kept apart from gis_correctness on purpose: a relation that holds is - # evidence that the analysis is self-consistent under perturbation, not - # that its answer is right. The two must not share a denominator. - "metamorphic": "metamorphic_evidence", -} +# Reporting buckets are owned by the shipped API so OpenMapBench and this +# runner cannot drift apart (tests assert the import). +from openmapstack.api import DIMENSIONS # noqa: E402 @dataclass diff --git a/openmapstack/api.py b/openmapstack/api.py new file mode 100644 index 0000000..886f479 --- /dev/null +++ b/openmapstack/api.py @@ -0,0 +1,319 @@ +"""The versioned check API that external harnesses consume. + +OpenMapBench (and any other benchmark or CI system) grades produced projects +with this package's checks. It must be able to do so without vendoring the +check implementations and without depending on module layout: the surface +it may rely on is exactly + +- ``CHECK_API_VERSION`` and ``api_info()`` for negotiation; +- ``list_checks()`` for the catalogue of check names and their parameters; +- ``run_check()`` for one check, returning a record that validates against + ``openmapstack-check-result/v1``; +- ``openmapstack verify --json``, returning ``openmapstack-verify-result/v1``; +- the JSON schemas packaged under ``openmapstack/schemas/``. + +Everything else in ``openmapstack.checks`` is implementation. + +Versioning: ``CHECK_API_VERSION`` follows ``/v``. A new check, +a new optional parameter, or a new result field is additive and does not +change the major. Renaming or removing a check, changing a parameter's +meaning, or changing the four-state status vocabulary does. A consumer +pins the major and the minimum package version it was tested against, and +``negotiate()`` answers whether the installed package satisfies both. + +Result semantics that a consumer may rely on: + +- ``status`` is one of ``passed | failed | warning | not_testable`` and a + check that could not establish its predicate is never ``passed``; +- ``code`` is a stable machine-readable identifier when the status is not + ``passed``; consumers grade on ``status`` and, for mutation-style + expectations, ``code`` -- never on ``detail`` text; +- ``dimension`` is the reporting bucket the check belongs to; buckets are + reported separately and must not be collapsed into one score; +- ``oracle_free`` is ``false`` for the checks that need a known answer. + Those transfer to arbitrary data only through attested expectations. + +Setup failures (a check that raises) are reported as ``not_testable`` with +``code: check_error`` by ``run_check`` so a broken environment cannot +produce either a pass or a graded failure; benchmark harnesses keep them +out of scored denominators. +""" + +from __future__ import annotations + +import importlib +import inspect +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from . import __version__ +from .checks import STATUSES, AssertionResult, not_testable +from .schema import validation_errors + +CHECK_API_VERSION = "openmapstack-check-api/v1" +CHECK_RESULT_SCHEMA = "openmapstack-check-result/v1" +API_INFO_SCHEMA = "openmapstack-api-info/v1" +VERIFY_RESULT_SCHEMA = "openmapstack-verify-result/v1" +PROJECT_SCHEMA = "openmapstack-project/v1" + +CHECK_MODULES = ( + "project", + "provenance", + "overrides", + "validation", + "geodata", + "presentation", + "qgis", + "visual", + "rerun", + "metamorphic", +) + +# Reporting buckets. Shared with evals/run.py, which asserts equality in +# tests so the two cannot drift apart. +DIMENSIONS = { + "project": "reproducibility_compliance", + "overrides": "override_handling", + "provenance": "provenance", + "geodata": "gis_correctness", + "validation": "validation_integrity", + "qgis": "presentation_contract", + "presentation": "presentation_contract", + "visual": "visual_judgement", + "rerun": "rerun_success", + "metamorphic": "metamorphic_evidence", +} + +# The checks that need a known answer. They are reachable on user data +# only through validation.expectations[] attestations. +KNOWN_ANSWER_CHECKS = frozenset( + { + "geodata.row_count", + "geodata.feature_present", + "geodata.feature_absent", + "geodata.feature_field_equals", + "geodata.field_range", + } +) + +_SCHEMA_DIR = Path(__file__).resolve().parent / "schemas" +_VERSION = re.compile(r"^(\d+)\.(\d+)\.(\d+)") + + +class CheckAPIError(ValueError): + """The consumer asked for something the API does not provide.""" + + +@dataclass(frozen=True) +class CheckParameter: + name: str + required: bool + default: Any = None + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = {"name": self.name, "required": self.required} + if not self.required: + payload["default"] = self.default + return payload + + +@dataclass(frozen=True) +class CheckDescriptor: + name: str + module: str + dimension: str + oracle_free: bool + summary: str + parameters: tuple[CheckParameter, ...] = field(default_factory=tuple) + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "module": self.module, + "dimension": self.dimension, + "oracle_free": self.oracle_free, + "summary": self.summary, + "parameters": [parameter.to_dict() for parameter in self.parameters], + } + + +def _load_schema(name: str) -> dict[str, Any]: + return json.loads((_SCHEMA_DIR / name).read_text(encoding="utf-8")) + + +def _describe(module_name: str, function_name: str, function: Any) -> CheckDescriptor: + signature = inspect.signature(function) + parameters: list[CheckParameter] = [] + for index, parameter in enumerate(signature.parameters.values()): + if index == 0: # workspace + continue + if parameter.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + continue + required = parameter.default is inspect.Parameter.empty + default = None if required else parameter.default + if isinstance(default, tuple): + default = list(default) + parameters.append(CheckParameter(parameter.name, required, default)) + doc = inspect.getdoc(function) or "" + summary = doc.strip().splitlines()[0].strip() if doc.strip() else "" + name = f"{module_name}.{function_name}" + return CheckDescriptor( + name=name, + module=module_name, + dimension=DIMENSIONS.get(module_name, "other"), + oracle_free=name not in KNOWN_ANSWER_CHECKS, + summary=summary, + parameters=tuple(parameters), + ) + + +def _is_check(function: Any) -> bool: + if not inspect.isfunction(function) or function.__name__.startswith("_"): + return False + try: + parameters = list(inspect.signature(function).parameters.values()) + except (TypeError, ValueError): + return False + return bool(parameters) and parameters[0].name == "workspace" + + +def list_checks() -> list[CheckDescriptor]: + """Every public check, discovered from the shipped modules.""" + descriptors: list[CheckDescriptor] = [] + for module_name in CHECK_MODULES: + module = importlib.import_module(f"openmapstack.checks.{module_name}") + for function_name, function in sorted(vars(module).items()): + if getattr(function, "__module__", None) != module.__name__: + continue + if _is_check(function): + descriptors.append(_describe(module_name, function_name, function)) + return descriptors + + +def describe_check(name: str) -> CheckDescriptor: + module_name, _, function_name = name.partition(".") + function = _resolve(name) + if getattr(function, "__module__", None) != f"openmapstack.checks.{module_name}": + raise CheckAPIError(f"unknown check {name!r}; see list_checks()") + return _describe(module_name, function_name, function) + + +def _resolve(name: str) -> Any: + module_name, _, function_name = name.partition(".") + if module_name not in CHECK_MODULES or not function_name: + raise CheckAPIError(f"unknown check {name!r}; see list_checks()") + module = importlib.import_module(f"openmapstack.checks.{module_name}") + function = getattr(module, function_name, None) + if function is None or not _is_check(function): + raise CheckAPIError(f"unknown check {name!r}; see list_checks()") + return function + + +def run_check(name: str, workspace: str | Path, args: dict[str, Any] | None = None) -> dict[str, Any]: + """Execute one check and return an ``openmapstack-check-result/v1`` record. + + Unknown check names and malformed arguments raise ``CheckAPIError`` + (a consumer configuration error). A check that raises while running is + reported as ``not_testable`` with ``code: check_error`` -- never as a + pass, and never as a graded failure. + """ + descriptor = describe_check(name) + function = _resolve(name) + args = dict(args or {}) + declared = {parameter.name for parameter in descriptor.parameters} + unknown = sorted(set(args) - declared) + missing = sorted(parameter.name for parameter in descriptor.parameters if parameter.required and parameter.name not in args) + if unknown or missing: + raise CheckAPIError(f"{name}: unknown args {unknown}, missing required args {missing}") + try: + result = function(Path(workspace), **args) + except Exception as exc: # noqa: BLE001 - a check must never take a harness down + result = not_testable(f"{type(exc).__name__}: {exc}", code="check_error") + if not isinstance(result, AssertionResult) or result.status not in STATUSES: + result = not_testable("check returned a malformed result", code="check_error") + data = {key: value for key, value in result.data.items() if key != "code"} + record: dict[str, Any] = { + "schema": CHECK_RESULT_SCHEMA, + "api_version": CHECK_API_VERSION, + "package_version": __version__, + "check": name, + "dimension": descriptor.dimension, + "oracle_free": descriptor.oracle_free, + "args": args, + "status": result.status, + "code": result.data.get("code"), + "detail": result.detail, + "data": json.loads(json.dumps(data, default=str)), + } + errors = validation_errors(record, _load_schema("check-result-v1.schema.json")) + if errors: # pragma: no cover - the record is built here; a failure is a bug + raise CheckAPIError(f"internal: check result does not validate: {errors}") + return record + + +def api_info() -> dict[str, Any]: + """What this installation offers, for a consumer to negotiate against.""" + checks = list_checks() + return { + "schema": API_INFO_SCHEMA, + "package": "openmapstack", + "package_version": __version__, + "check_api_version": CHECK_API_VERSION, + "project_schema": PROJECT_SCHEMA, + "result_schemas": { + "check": CHECK_RESULT_SCHEMA, + "verify": VERIFY_RESULT_SCHEMA, + }, + "statuses": list(STATUSES), + "dimensions": sorted(set(DIMENSIONS.values())), + "checks": len(checks), + "oracle_free_checks": sum(descriptor.oracle_free for descriptor in checks), + "known_answer_checks": sorted(KNOWN_ANSWER_CHECKS), + } + + +def _parse_version(text: str) -> tuple[int, int, int]: + match = _VERSION.match(text or "") + if match is None: + raise CheckAPIError(f"not a semantic version: {text!r}") + return tuple(int(part) for part in match.groups()) # type: ignore[return-value] + + +def negotiate( + *, + required_api: str = CHECK_API_VERSION, + min_package_version: str | None = None, + required_checks: list[str] | None = None, +) -> dict[str, Any]: + """Answer whether this installation satisfies a consumer's requirements. + + A consumer states the API major it was built for, the oldest package + version it was tested against, and the checks it needs. The answer + lists every unmet requirement so a harness can report *why* it is + refusing to grade rather than grading with a checker it does not + understand. + """ + problems: list[str] = [] + if required_api != CHECK_API_VERSION: + problems.append(f"check API {required_api!r} is not provided; this package offers {CHECK_API_VERSION!r}") + if min_package_version is not None and _parse_version(__version__) < _parse_version(min_package_version): + problems.append(f"package version {__version__} is older than the required {min_package_version}") + available = {descriptor.name for descriptor in list_checks()} + missing = sorted(name for name in (required_checks or []) if name not in available) + if missing: + problems.append(f"checks not provided: {missing}") + return { + "schema": "openmapstack-api-negotiation/v1", + "compatible": not problems, + "package_version": __version__, + "check_api_version": CHECK_API_VERSION, + "problems": problems, + } + + +def validate_verify_result(payload: dict[str, Any]) -> list[str]: + """Schema-validate an ``openmapstack verify --json`` document.""" + return validation_errors(payload, _load_schema("verify-result-v1.schema.json")) diff --git a/openmapstack/cli.py b/openmapstack/cli.py index 2ec1bbb..411d669 100644 --- a/openmapstack/cli.py +++ b/openmapstack/cli.py @@ -115,6 +115,30 @@ def build_parser() -> argparse.ArgumentParser: snapshot_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") snapshot_parser.set_defaults(handler=_cmd_source_snapshot) + checks_parser = subparsers.add_parser("checks", help="list the versioned check catalogue (openmapstack-check-api/v1)") + checks_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + checks_parser.set_defaults(handler=_cmd_checks) + + check_parser = subparsers.add_parser("check", help="run one named check against a project workspace") + check_parser.add_argument("name", help="check name, e.g. geodata.geometry_all_valid") + check_parser.add_argument("workspace", nargs="?", default=".", help="project directory (default: .)") + check_parser.add_argument( + "--arg", + action="append", + default=[], + metavar="KEY=VALUE", + help="check argument; VALUE is parsed as JSON when it is valid JSON, else used as a string", + ) + check_parser.add_argument("--json", action="store_true", help="emit the openmapstack-check-result/v1 record") + check_parser.set_defaults(handler=_cmd_check) + + api_parser = subparsers.add_parser("api-info", help="report API, schema, and package versions for consumers") + api_parser.add_argument("--require-api", help="check API the consumer was built for") + api_parser.add_argument("--min-version", help="oldest package version the consumer was tested against") + api_parser.add_argument("--require-check", action="append", default=[], help="a check the consumer needs; repeatable") + api_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + api_parser.set_defaults(handler=_cmd_api_info) + inspect_parser = subparsers.add_parser("inspect", help="summarize a project for audit and review") inspect_parser.add_argument("project", nargs="?", default="project.yaml", help="project.yaml or its directory") inspect_parser.add_argument("--json", action="store_true", help="emit the full summary as JSON") @@ -294,6 +318,86 @@ def _cmd_run(args: argparse.Namespace) -> int: return 0 if validation.ok(strict=args.strict) else 1 +def _cmd_checks(args: argparse.Namespace) -> int: + from .api import CHECK_API_VERSION, list_checks + + descriptors = list_checks() + if args.json: + print(_json({"schema": "openmapstack-check-catalogue/v1", "api_version": CHECK_API_VERSION, + "package_version": __version__, "checks": [d.to_dict() for d in descriptors]})) + return 0 + print(f"{CHECK_API_VERSION} ({len(descriptors)} checks, openmapstack {__version__})") + for descriptor in descriptors: + required = [p.name for p in descriptor.parameters if p.required] + optional = [p.name for p in descriptor.parameters if not p.required] + oracle = "" if descriptor.oracle_free else " [known-answer]" + print(f" {descriptor.name:48s} {descriptor.dimension}{oracle}") + if required or optional: + print(f" args: required={required} optional={optional}") + return 0 + + +def _parse_check_args(pairs: Sequence[str]) -> dict[str, Any]: + parsed: dict[str, Any] = {} + for pair in pairs: + key, separator, value = pair.partition("=") + if not separator or not key.strip(): + raise ValueError(f"--arg expects KEY=VALUE, got {pair!r}") + try: + parsed[key.strip()] = json.loads(value) + except json.JSONDecodeError: + parsed[key.strip()] = value + return parsed + + +def _cmd_check(args: argparse.Namespace) -> int: + from .api import CheckAPIError, run_check + + try: + record = run_check(args.name, args.workspace, _parse_check_args(args.arg)) + except (CheckAPIError, ValueError) as exc: + if args.json: + print(_json({"schema": "openmapstack-check-result/v1", "status": "not_testable", "code": "consumer_error", "error": str(exc)})) + else: + print(f"openmapstack check: {exc}", file=sys.stderr) + return 2 + if args.json: + print(_json(record)) + else: + mark = STATUS_MARKS.get(record["status"], record["status"].upper()) + code = f" [{record['code']}]" if record.get("code") else "" + print(f"{mark} {record['check']}{code}: {record['detail']}") + return 0 if record["status"] != "failed" else 1 + + +def _cmd_api_info(args: argparse.Namespace) -> int: + from .api import CHECK_API_VERSION, CheckAPIError, api_info, negotiate + + info = api_info() + negotiation = None + if args.require_api or args.min_version or args.require_check: + try: + negotiation = negotiate( + required_api=args.require_api or CHECK_API_VERSION, + min_package_version=args.min_version, + required_checks=list(args.require_check), + ) + except CheckAPIError as exc: + print(f"openmapstack api-info: {exc}", file=sys.stderr) + return 2 + info["negotiation"] = negotiation + if args.json: + print(_json(info)) + else: + print(f"openmapstack {info['package_version']}: {info['check_api_version']}, {info['checks']} checks " + f"({info['oracle_free_checks']} oracle-free), project schema {info['project_schema']}") + if negotiation is not None: + print("compatible" if negotiation["compatible"] else "INCOMPATIBLE: " + "; ".join(negotiation["problems"])) + if negotiation is not None and not negotiation["compatible"]: + return 1 + return 0 + + def _cmd_source_discover(args: argparse.Namespace) -> int: from .connectors import ConnectorError, ConnectorLimits, discover_source diff --git a/openmapstack/schemas/check-result-v1.schema.json b/openmapstack/schemas/check-result-v1.schema.json new file mode 100644 index 0000000..84aedee --- /dev/null +++ b/openmapstack/schemas/check-result-v1.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openmapstack/schemas/check-result-v1.schema.json", + "title": "One executed OpenMapStack check (openmapstack-check-result/v1)", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", "api_version", "package_version", "check", "dimension", + "oracle_free", "args", "status", "code", "detail", "data" + ], + "properties": { + "schema": {"const": "openmapstack-check-result/v1"}, + "api_version": {"const": "openmapstack-check-api/v1"}, + "package_version": {"type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+"}, + "check": {"type": "string", "pattern": "^[a-z_]+[.][a-z_]+$"}, + "dimension": {"type": "string", "minLength": 1}, + "oracle_free": {"type": "boolean"}, + "args": {"type": "object"}, + "status": {"enum": ["passed", "failed", "warning", "not_testable"]}, + "code": {"type": ["string", "null"]}, + "detail": {"type": "string"}, + "data": {"type": "object"} + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "passed"}}}, + "then": {"properties": {"code": {"type": "null"}}} + } + ] +} diff --git a/openmapstack/schemas/verify-result-v1.schema.json b/openmapstack/schemas/verify-result-v1.schema.json new file mode 100644 index 0000000..7191ca6 --- /dev/null +++ b/openmapstack/schemas/verify-result-v1.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openmapstack/schemas/verify-result-v1.schema.json", + "title": "openmapstack verify --json (openmapstack-verify-result/v1)", + "type": "object", + "required": ["schema", "project_file", "status", "counts", "coverage", "checks"], + "properties": { + "schema": {"const": "openmapstack-verify-result/v1"}, + "project_file": {"type": "string", "minLength": 1}, + "status": {"enum": ["passed", "failed", "warning", "not_testable"]}, + "counts": { + "type": "object", + "required": ["passed", "warning", "not_testable", "failed"], + "additionalProperties": false, + "properties": { + "passed": {"type": "integer", "minimum": 0}, + "warning": {"type": "integer", "minimum": 0}, + "not_testable": {"type": "integer", "minimum": 0}, + "failed": {"type": "integer", "minimum": 0} + } + }, + "coverage": { + "type": "object", + "required": ["applicable", "executed", "not_testable", "execution_rate"], + "additionalProperties": false, + "properties": { + "applicable": {"type": "integer", "minimum": 0}, + "executed": {"type": "integer", "minimum": 0}, + "not_testable": {"type": "integer", "minimum": 0}, + "execution_rate": {"type": ["number", "null"], "minimum": 0, "maximum": 1} + } + }, + "checks": { + "type": "array", + "items": { + "type": "object", + "required": ["check", "status", "message"], + "properties": { + "check": {"type": "string", "minLength": 1}, + "status": {"enum": ["passed", "failed", "warning", "not_testable"]}, + "message": {"type": "string"}, + "args": {"type": "object"}, + "evidence": {"type": "object"}, + "code": {"type": "string", "minLength": 1} + } + } + }, + "clean_rerun": {"type": "object"} + } +} diff --git a/tests/test_check_api.py b/tests/test_check_api.py new file mode 100644 index 0000000..74790d1 --- /dev/null +++ b/tests/test_check_api.py @@ -0,0 +1,194 @@ +"""The versioned check API an external harness (OpenMapBench) consumes. + +``ConsumerFixtureTests`` plays the role of that harness: it uses only the +public API, the CLI, and the packaged JSON schemas -- never +``openmapstack.checks`` directly -- and proves it can list, negotiate, run, +and validate a result without vendoring a single check. +""" + +from __future__ import annotations + +import importlib.util +import io +import json +import subprocess +import sys +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +from openmapstack import __version__, api +from openmapstack.cli import main +from openmapstack.schema import validation_errors +from openmapstack.verify import verify_project +from tests.evals.helpers import make_workspace, minimal_project, write_project + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCHEMAS = REPO_ROOT / "openmapstack" / "schemas" + + +def _schema(name: str) -> dict: + return json.loads((SCHEMAS / name).read_text(encoding="utf-8")) + + +class CatalogueTests(unittest.TestCase): + def test_every_public_check_is_listed_with_its_parameters(self) -> None: + names = {descriptor.name for descriptor in api.list_checks()} + for expected in ( + "project.conforms_to_schema", "provenance.every_source_pinned", "geodata.geometry_all_valid", + "validation.run_record_matches", "qgis.static_valid", "rerun.clean_execution_succeeded", + "metamorphic.relation_holds", "presentation.controls_match_pipeline", + ): + self.assertIn(expected, names) + descriptor = api.describe_check("geodata.dataset_crs_is") + by_name = {parameter.name: parameter for parameter in descriptor.parameters} + self.assertTrue(by_name["path"].required) + self.assertTrue(by_name["expected"].required) + self.assertFalse(by_name["project_dir"].required) + self.assertEqual(by_name["project_dir"].default, ".") + + def test_known_answer_checks_are_marked_and_everything_else_is_oracle_free(self) -> None: + catalogue = {descriptor.name: descriptor for descriptor in api.list_checks()} + for name in api.KNOWN_ANSWER_CHECKS: + self.assertFalse(catalogue[name].oracle_free, name) + self.assertTrue(catalogue["geodata.geometry_all_valid"].oracle_free) + self.assertEqual(catalogue["metamorphic.relation_holds"].dimension, "metamorphic_evidence") + self.assertEqual(catalogue["visual.render_substantive"].dimension, "visual_judgement") + self.assertEqual(catalogue["geodata.row_count"].dimension, "gis_correctness") + + def test_private_helpers_are_not_exposed(self) -> None: + names = {descriptor.name for descriptor in api.list_checks()} + self.assertFalse(any(".‗" in name or "._" in name for name in names)) + with self.assertRaises(api.CheckAPIError): + api.describe_check("geodata._read") + with self.assertRaises(api.CheckAPIError): + api.describe_check("spatial.connect_spatial") + + def test_eval_runner_uses_the_api_dimensions(self) -> None: + spec = importlib.util.spec_from_file_location("openmapstack_eval_runner_api_test", REPO_ROOT / "evals" / "run.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + self.assertIs(module.DIMENSIONS, api.DIMENSIONS) + + +class RunCheckTests(unittest.TestCase): + def test_result_validates_and_carries_stable_codes(self) -> None: + workspace = make_workspace() + project = minimal_project() + project["sources"]["test_source"]["version"] = {"identifier": "latest"} + write_project(workspace, project) + record = api.run_check("provenance.every_source_pinned", workspace) + self.assertEqual(validation_errors(record, _schema("check-result-v1.schema.json")), []) + self.assertEqual((record["status"], record["code"]), ("failed", "source_unpinned")) + self.assertEqual(record["dimension"], "provenance") + self.assertEqual(record["api_version"], api.CHECK_API_VERSION) + + def test_passed_results_have_no_code(self) -> None: + workspace = make_workspace() + write_project(workspace, minimal_project()) + record = api.run_check("project.parses", workspace) + self.assertEqual(record["status"], "passed") + self.assertIsNone(record["code"]) + self.assertEqual(validation_errors(record, _schema("check-result-v1.schema.json")), []) + + def test_unknown_check_and_bad_args_are_consumer_errors(self) -> None: + workspace = make_workspace() + with self.assertRaises(api.CheckAPIError): + api.run_check("geodata.nope", workspace) + with self.assertRaises(api.CheckAPIError): + api.run_check("geodata.dataset_crs_is", workspace, {"path": "x"}) # missing expected + with self.assertRaises(api.CheckAPIError): + api.run_check("project.parses", workspace, {"bogus": 1}) + + def test_a_raising_check_is_not_testable_never_passed(self) -> None: + from unittest.mock import patch + + workspace = make_workspace() + write_project(workspace, minimal_project()) + def boom(workspace, project_dir="."): + raise RuntimeError("boom") + + boom.__module__ = "openmapstack.checks.project" + with patch("openmapstack.checks.project.graph_resolves", new=boom): + record = api.run_check("project.graph_resolves", workspace) + self.assertEqual((record["status"], record["code"]), ("not_testable", "check_error")) + self.assertEqual(validation_errors(record, _schema("check-result-v1.schema.json")), []) + + +class NegotiationTests(unittest.TestCase): + def test_compatible_when_api_version_and_checks_match(self) -> None: + answer = api.negotiate(min_package_version="0.1.0", required_checks=["project.parses"]) + self.assertTrue(answer["compatible"], answer) + + def test_incompatible_answers_say_why(self) -> None: + answer = api.negotiate(required_api="openmapstack-check-api/v2", min_package_version="99.0.0", required_checks=["geodata.magic"]) + self.assertFalse(answer["compatible"]) + self.assertEqual(len(answer["problems"]), 3) + with self.assertRaises(api.CheckAPIError): + api.negotiate(min_package_version="latest") + + def test_api_info_describes_the_installation(self) -> None: + info = api.api_info() + self.assertEqual(info["schema"], api.API_INFO_SCHEMA) + self.assertEqual(info["package_version"], __version__) + self.assertEqual(info["statuses"], ["passed", "failed", "warning", "not_testable"]) + self.assertIn("metamorphic_evidence", info["dimensions"]) + + +class VerifyResultSchemaTests(unittest.TestCase): + def test_verify_json_validates_against_the_packaged_schema(self) -> None: + workspace = make_workspace() + write_project(workspace, minimal_project()) + payload = verify_project(workspace / "project.yaml").to_dict() + self.assertEqual(api.validate_verify_result(payload), []) + + +class ConsumerFixtureTests(unittest.TestCase): + """A harness that vendors nothing: CLI + JSON schemas only.""" + + def _cli(self, *argv: str) -> tuple[int, str]: + out, err = io.StringIO(), io.StringIO() + with redirect_stdout(out), redirect_stderr(err): + code = main(list(argv)) + return code, out.getvalue() + + def test_list_negotiate_run_validate(self) -> None: + code, text = self._cli("api-info", "--json", "--require-api", api.CHECK_API_VERSION, "--min-version", "0.2.0", "--require-check", "validation.run_record_matches") + self.assertEqual(code, 0, text) + info = json.loads(text) + self.assertTrue(info["negotiation"]["compatible"]) + + code, text = self._cli("checks", "--json") + self.assertEqual(code, 0) + catalogue = json.loads(text) + names = {entry["name"] for entry in catalogue["checks"]} + self.assertIn("geodata.dataset_crs_is", names) + + workspace = make_workspace() + project = minimal_project() + project["outputs"] = {"final": {"path": "data/derived/final.json", "format": "GeoJSON", "generated_by": "export"}} + write_project(workspace, project) + code, text = self._cli("check", "project.declared_files_exist", str(workspace), "--arg", 'files=["data/derived/final.json"]', "--json") + record = json.loads(text) + self.assertEqual(code, 1) + self.assertEqual(record["status"], "failed") + self.assertEqual(validation_errors(record, _schema("check-result-v1.schema.json")), []) + + code, text = self._cli("api-info", "--json", "--require-api", "openmapstack-check-api/v9") + self.assertEqual(code, 1) + self.assertFalse(json.loads(text)["negotiation"]["compatible"]) + + def test_consumer_error_is_exit_two_not_a_graded_result(self) -> None: + code, text = self._cli("check", "geodata.nope", str(make_workspace()), "--json") + self.assertEqual(code, 2) + self.assertEqual(json.loads(text)["code"], "consumer_error") + + def test_the_cli_is_reachable_as_a_subprocess(self) -> None: + completed = subprocess.run([sys.executable, "-m", "openmapstack", "api-info", "--json"], capture_output=True, text=True, cwd=REPO_ROOT, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(json.loads(completed.stdout)["check_api_version"], api.CHECK_API_VERSION) + + +if __name__ == "__main__": + unittest.main() From 5e8105cc9334d6562d79fd8bce206d6f99beef91 Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 2 Sep 2026 17:08:06 +0000 Subject: [PATCH 5/8] feat(evals): skill snapshots, complete arm provenance, paired arms, and task export openmapstack skill-snapshot --out DIR creates the controlled copy of SKILL.md, references/, and templates/ with a per-file inventory and content hash (openmapstack-skill-snapshot/v1); --inspect re-verifies one. Symlinks and escaping inventory paths are refused. The eval runner now records the same snapshot for the oms arm. A live run records one openmapstack-benchmark-arm/v1 record per arm: skill hash and commit, task-set hash, checker and check-API versions, harness commit, runtime, adapter and agent version, model, sampling, and --price-catalog-date; unknown values are null, never omitted. --arms paired runs plain and oms over identical cases, trials, and seeds and reports them side by side with task parity, per-arm quality with a Wilson interval, per-arm cost, and trajectory diagnostics; no combined score is published. --export-tasks writes vendor-neutral openmapstack-benchmark-task/v1 bundles, marking cases 070-073 as OpenMapBench-owned. Refs #13 (C2, C3). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01TMMy6A73zi7yGjK3wHBhju --- .github/workflows/eval-benchmark.yml | 14 +- AGENTS.md | 4 + README.md | 4 + evals/README.md | 54 +++- evals/run.py | 321 ++++++++++++++++++-- evals/schemas/benchmark-arm-v1.schema.json | 90 ++++++ evals/schemas/benchmark-task-v1.schema.json | 32 ++ evals/schemas/results-v2.schema.json | 1 + openmapstack/cli.py | 42 +++ openmapstack/snapshot.py | 199 ++++++++++++ tests/test_evals.py | 105 ++++++- tests/test_snapshot.py | 113 +++++++ 12 files changed, 947 insertions(+), 32 deletions(-) create mode 100644 evals/schemas/benchmark-arm-v1.schema.json create mode 100644 evals/schemas/benchmark-task-v1.schema.json create mode 100644 openmapstack/snapshot.py create mode 100644 tests/test_snapshot.py diff --git a/.github/workflows/eval-benchmark.yml b/.github/workflows/eval-benchmark.yml index 447a93f..eb5ac71 100644 --- a/.github/workflows/eval-benchmark.yml +++ b/.github/workflows/eval-benchmark.yml @@ -35,6 +35,15 @@ on: cases: description: "Space-separated case ids to run (default: every case that declares live mode)" required: false + arms: + description: "Benchmark arm(s): oms (skill injected), plain (no skill), or paired (both over identical cases, trials, and seeds)" + required: false + default: oms + type: choice + options: [oms, plain, paired] + price_catalog_date: + description: "YYYY-MM-DD of the price list behind the cost estimates (recorded in arm provenance)" + required: false schedule: - cron: "0 3 * * 1" # weekly, Monday 03:00 UTC @@ -120,6 +129,8 @@ jobs: EVAL_TIMEOUT: ${{ github.event.inputs.timeout || '1200' }} EVAL_SEED: ${{ github.event.inputs.seed || '' }} EVAL_CASES: ${{ github.event.inputs.cases || '' }} + EVAL_ARMS: ${{ github.event.inputs.arms || 'oms' }} + EVAL_PRICE_CATALOG_DATE: ${{ github.event.inputs.price_catalog_date || '' }} run: | if [[ "$EVAL_AGENT" == "claude_code" ]]; then EVAL_MODEL="$CLAUDE_MODEL" @@ -136,12 +147,13 @@ jobs: --mode live --agent "$EVAL_AGENT" --model "$EVAL_MODEL" - --skill-mode enabled + --arms "$EVAL_ARMS" --repetitions "$EVAL_REPETITIONS" --timeout "$EVAL_TIMEOUT" --json "eval-benchmark-results-$EVAL_AGENT.json" ) if [[ -n "$EVAL_SEED" ]]; then args+=(--seed "$EVAL_SEED"); fi + if [[ -n "$EVAL_PRICE_CATALOG_DATE" ]]; then args+=(--price-catalog-date "$EVAL_PRICE_CATALOG_DATE"); fi # No --case means every case that declares live mode. The list used # to be hardcoded to 001/002/004/005, which silently left the four # prompt-style cases (070-073, added by PR 8) running in no workflow diff --git a/AGENTS.md b/AGENTS.md index a688d4f..11b88c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,6 +121,10 @@ python3 evals/run.py --mode fixture openmapstack validate examples/tartu-development/project.yaml --preflight openmapstack inspect examples/tartu-development/project.yaml --json +# Check API that external harnesses consume (see docs/openmapbench-interop.md) +openmapstack api-info --json +openmapstack checks + # Coverage gate used by CI python3 -m coverage run -m unittest discover -v python3 -m coverage report diff --git a/README.md b/README.md index d3c4e9f..2c0060f 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,10 @@ openmapstack run path/to/project.yaml # Review sources, versions, overrides, ordered steps, outputs, and latest run. openmapstack inspect path/to/project.yaml +# Copy SKILL.md, references/, and templates/ into a hashed, inspectable snapshot. +openmapstack skill-snapshot --out /tmp/oms-skill --json +openmapstack skill-snapshot --inspect /tmp/oms-skill + # Read-only discovery of a warehouse source, then an approval-gated snapshot. openmapstack source discover path/to/project.yaml --source parcels openmapstack source snapshot path/to/project.yaml --source parcels \ diff --git a/evals/README.md b/evals/README.md index 4717264..1a96258 100644 --- a/evals/README.md +++ b/evals/README.md @@ -178,9 +178,45 @@ agent. `--timeout` applies to each generator or agent invocation; base seed that is incremented for each repetition. Live mode requires an explicit `--model`: a run with an unknown CLI default is -not publishable benchmark evidence. It also requires an explicit -`--skill-mode`, for the same reason — which arm ran is recorded in the result, -so inferring one mislabels the evidence rather than failing. +not publishable benchmark evidence. The arm is explicit too: `--arms oms` +(the default; the controlled skill snapshot is injected), `--arms plain` (no +skill), or `--arms paired`, which runs both arms over identical cases, trials, +and seeds. `--skill-mode enabled|disabled` remains as an alias for the single +arms. Which arm ran is recorded per trial (`results[].arm`) and per arm in +`run_config.arm_provenance`. + +### Arm provenance and paired comparison + +Every live run records one `openmapstack-benchmark-arm/v1` record per arm +(`evals/schemas/benchmark-arm-v1.schema.json`): the skill snapshot content +hash and commit, a hash over the exact task set (expectations, prompts, and +declared fixtures), the checker package and check-API versions, the harness +commit, the runtime (Python, platform, DuckDB, container image), the adapter +and the exact agent version it drove, the model id and provider, the +sampling configuration the adapter reports, and `--price-catalog-date`. A +field the harness cannot learn is `null`, never omitted. + +A paired run publishes `paired_arms`: per-arm success rate with a Wilson +interval, median cost, tokens, and duration, a `task_parity` flag proving +both arms saw the same case/trial/seed set, and trajectory diagnostics +(event counts) that are explicitly not correctness. There is no combined +"success per dollar" number; quality and cost are reported as a trade-off. +Paired bundles are retained under `evals/results////…`. + +### Exporting tasks for OpenMapBench + +```bash +python evals/run.py --export-tasks /tmp/openmapstack-tasks +python evals/run.py --export-tasks /tmp/tasks --case 070-underspecified-prompt +``` + +Each live-capable case becomes an `openmapstack-benchmark-task/v1` bundle: +prompt, declared fixtures (copied and hashed), assertion list, hard-gate +policy, and a task hash — never the reference project or the generator. +Cases 070–073 are marked `ownership: openmapbench`: they are the behavioural +prompt-style tasks whose canonical home is OpenMapBench; this repository +keeps them only as a scheduled smoke subset. See +`docs/openmapbench-interop.md` for the full contract. `--case` is repeatable, and omitting it runs every case that declares live mode (currently 001–006 and 070–073). The scheduled workflow omits it: a @@ -214,9 +250,10 @@ deterministic assertion results used by fixture mode, and `generated-project/` is copied before cleanup so every claim can be checked independently. By default, live mode copies only `SKILL.md`, `references/`, and `templates/` -into an isolated `benchmark-context/` beside the empty project, prepends an -instruction to read that controlled snapshot, and records its commit and -content hash. Eval cases and expected projects are never exposed. Use +into an isolated `benchmark-context/` beside the empty project (the same +inspectable snapshot `openmapstack skill-snapshot` produces, with a +`snapshot.json` inventory), prepends an instruction to read that controlled +snapshot, and records its commit and content hash. Eval cases and expected projects are never exposed. Use `--skill-mode disabled` for an explicit no-skill baseline; do not mix enabled and disabled trials in one published denominator. @@ -561,7 +598,10 @@ rendered snapshots as evidence. `run.py --json` reports pass/fail per assertion and rolls dimensions up within each score type: GIS correctness, project/reproducibility compliance, provenance, override handling, validation integrity, presentation contract, -and rerun success. Setup failures are reported but excluded from pass-rate +rerun success, metamorphic evidence, and visual judgement. The buckets are +owned by `openmapstack.api.DIMENSIONS` and stay separate: a metamorphic +relation that holds is self-consistency, not a correct answer, and a visual +judgement never stands in for an analytical one. Setup failures are reported but excluded from pass-rate denominators. There is intentionally no aggregate `16/16` score: a detected mutation, a conforming fixture, and a successful agent trial answer different questions. diff --git a/evals/run.py b/evals/run.py index 9058961..2d5f1a9 100755 --- a/evals/run.py +++ b/evals/run.py @@ -24,6 +24,7 @@ import argparse import importlib +import itertools import json import math import os @@ -49,6 +50,12 @@ KNOWN_MODES = {"fixture", "live", "visual"} KNOWN_AGENTS = {"claude_code", "codex", "openai_compatible"} KNOWN_CASE_TYPES = {"mutation", "positive"} +# Benchmark arms: `plain` runs the agent with no skill; `oms` injects the +# controlled skill snapshot. `paired` runs both over identical cases, trials, +# and seeds so quality and cost can be compared without a shared score. +ARM_BY_SKILL_MODE = {"disabled": "plain", "enabled": "oms"} +SKILL_MODE_BY_ARM = {arm: mode for mode, arm in ARM_BY_SKILL_MODE.items()} +KNOWN_ARM_SELECTIONS = ("oms", "plain", "paired") KNOWN_SCORE_TYPES = { "agent_benchmark", "contract_ci", @@ -59,7 +66,10 @@ sys.path.insert(0, str(REPO_ROOT)) sys.path.insert(0, str(EVALS_DIR)) +from openmapstack import __version__ as OPENMAPSTACK_VERSION # noqa: E402 +from openmapstack.api import CHECK_API_VERSION # noqa: E402 from openmapstack.checks import AssertionResult, STATUSES # noqa: E402 +from openmapstack.snapshot import create_skill_snapshot # noqa: E402 from openmapstack.rerun import ( # noqa: E402 CLEAN_RERUN_EVIDENCE, perform_clean_rerun, @@ -617,27 +627,164 @@ def _evidence_path(path: Path) -> str: def _prepare_skill_snapshot(workspace: Path) -> tuple[Path, str]: - """Copy only the distributable skill context, never eval/reference outputs.""" - import hashlib + """Copy only the distributable skill context, never eval/reference outputs. + Delegates to the shipped ``openmapstack skill-snapshot`` implementation so + the benchmark records the same inspectable snapshot a user can create. + """ destination = workspace / "benchmark-context" / "openmapstack" - destination.mkdir(parents=True) - shutil.copy2(REPO_ROOT / "SKILL.md", destination / "SKILL.md") - for directory in ("references", "templates"): - shutil.copytree( - REPO_ROOT / directory, - destination / directory, - ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), - ) + manifest = create_skill_snapshot(REPO_ROOT, destination) + return destination, manifest["content_sha256"] + + +def _task_set_hash(case_dirs: list[Path]) -> dict[str, Any]: + """Identify the exact task set: expectations, prompts, and declared fixtures.""" + import hashlib digest = hashlib.sha256() - for path in sorted(item for item in destination.rglob("*") if item.is_file()): - relative = path.relative_to(destination).as_posix() - digest.update(relative.encode("utf-8")) - digest.update(b"\0") - digest.update(path.read_bytes()) - digest.update(b"\0") - return destination, f"sha256:{digest.hexdigest()}" + ids: list[str] = [] + for case_dir in sorted(case_dirs): + case_def = _load_case(case_dir) + ids.append(case_def.get("id", case_dir.name)) + for relative in ("expected.yaml", case_def.get("live", {}).get("prompt_file", "prompt.md")): + path = case_dir / relative + if path.is_file(): + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + for fixture in (case_def.get("live") or {}).get("fixtures") or []: + source = (case_dir / fixture["source"]).resolve() + if source.is_file(): + digest.update(fixture["destination"].encode("utf-8")) + digest.update(b"\0") + digest.update(source.read_bytes()) + digest.update(b"\0") + return {"cases": ids, "sha256": f"sha256:{digest.hexdigest()}"} + + +def _arm_record( + arm: str, + *, + skill: dict[str, Any], + task_set: dict[str, Any], + revision: dict[str, Any], + agent_name: str | None, + model: str | None, + seed: int | None, + price_catalog_date: str | None, + trial_results: list[dict[str, Any]], +) -> dict[str, Any]: + """The complete provenance tuple that identifies a published arm. + + Every field is present; what the harness cannot learn is ``null`` rather + than omitted, so a reader can tell "unknown" from "not recorded". + """ + try: + import duckdb # type: ignore[import-not-found] + + duckdb_version = duckdb.__version__ + except ImportError: + duckdb_version = None + agent_runs = [result.get("agent_run") for result in trial_results if isinstance(result.get("agent_run"), dict)] + agent_version = next((run.get("version") for run in agent_runs if run.get("version")), None) + metadata = next((run.get("metadata") or {} for run in agent_runs), {}) + sampling = { + "seed": seed, + "temperature": metadata.get("temperature"), + "reasoning": metadata.get("reasoning") or metadata.get("reasoning_effort"), + } + provider = {"claude_code": "anthropic", "codex": "openai", "openai_compatible": "openai_compatible"}.get(agent_name or "") + record = { + "schema": "openmapstack-benchmark-arm/v1", + "arm": arm, + "skill": { + "mode": skill.get("mode"), + "content_sha256": skill.get("content_sha256"), + "commit": skill.get("commit"), + "dirty": revision.get("dirty"), + "entrypoint": skill.get("entrypoint"), + }, + "task_set": task_set, + "checker": {"package": "openmapstack", "package_version": OPENMAPSTACK_VERSION, "check_api_version": CHECK_API_VERSION}, + "harness": {"name": "openmapstack/evals", "commit": revision.get("commit"), "dirty": revision.get("dirty")}, + "runtime": { + "python": platform.python_version(), + "platform": platform.platform(), + "duckdb": duckdb_version, + "container_image": os.environ.get("OPENMAPSTACK_CONTAINER_IMAGE"), + }, + "tool_surface": {"adapter": agent_name, "agent_version": agent_version}, + "model": {"provider": provider, "id": model, "revision": metadata.get("model_revision")}, + "sampling": sampling, + "price_catalog_date": price_catalog_date, + } + errors = validation_errors(record, _load_eval_schema("benchmark-arm-v1.schema.json")) + if errors: + raise ValueError(f"benchmark arm record does not validate: {'; '.join(errors)}") + return record + + +def export_tasks(case_dirs: list[Path], destination: Path) -> dict[str, Any]: + """Write vendor-neutral task bundles for an external benchmark harness. + + Only live-capable cases are tasks; each bundle carries the prompt, the + declared fixtures (copied, hashed), the assertion list, and a task hash, + and never the reference project or the generator. + """ + import hashlib + + destination.mkdir(parents=True, exist_ok=True) + index: list[dict[str, Any]] = [] + for case_dir in case_dirs: + case_def = _load_case(case_dir) + if "live" not in case_def["modes"]: + continue + live = case_def["live"] + case_id = case_def.get("id", case_dir.name) + task_dir = destination / case_id + (task_dir / "fixtures").mkdir(parents=True, exist_ok=True) + prompt = (case_dir / live.get("prompt_file", "prompt.md")).read_text(encoding="utf-8") + fixtures = [] + for fixture in live.get("fixtures") or []: + source = (case_dir / fixture["source"]).resolve() + copied = task_dir / "fixtures" / Path(fixture["destination"]).name + shutil.copyfile(source, copied) + fixtures.append({ + "path": f"fixtures/{copied.name}", + "destination": fixture["destination"], + "sha256": "sha256:" + hashlib.sha256(source.read_bytes()).hexdigest(), + }) + assertions = [ + entry for entry in case_def["assertions"] + if not entry.get("modes") or "live" in entry["modes"] + ] + body = { + "id": case_id, + "prompt": prompt, + "fixtures": fixtures, + "assertions": assertions, + "hard_gate": bool(case_def.get("hard_gate", True)), + "agent_workdir": live.get("agent_workdir", case_def.get("project_dir", "project")), + "clean_rerun": "clean_rerun" in live, + } + task_hash = "sha256:" + hashlib.sha256(json.dumps(body, sort_keys=True, default=str).encode("utf-8")).hexdigest() + task = {"schema": "openmapstack-benchmark-task/v1", **body, "task_sha256": task_hash, "checker_api": CHECK_API_VERSION} + if case_id.split("-")[0] in {"070", "071", "072", "073"}: + task["ownership"] = "openmapbench" + errors = validation_errors(task, _load_eval_schema("benchmark-task-v1.schema.json")) + if errors: + raise ValueError(f"{case_id}: task bundle does not validate: {'; '.join(errors)}") + (task_dir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + index.append({"id": case_id, "task_sha256": task_hash, "ownership": task.get("ownership", "openmapstack")}) + manifest = { + "schema": "openmapstack-benchmark-task-index/v1", + "checker_api": CHECK_API_VERSION, + "package_version": OPENMAPSTACK_VERSION, + "tasks": index, + } + (destination / "index.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return manifest def _skill_augmented_prompt(prompt: str, agent_workdir: Path, skill_dir: Path) -> str: @@ -1085,6 +1232,7 @@ def finalize(payload: dict[str, Any]) -> dict[str, Any]: "live mode requires an explicit skill_mode ('enabled' or " "'disabled'); a benchmark arm must never be inferred", ) + benchmark_context["arm"] = ARM_BY_SKILL_MODE[skill_mode] if skill_mode == "enabled": skill_dir, skill_digest = _prepare_skill_snapshot(workspace) prompt = _skill_augmented_prompt(prompt, agent_workdir, skill_dir) @@ -1179,6 +1327,7 @@ def finalize(payload: dict[str, Any]) -> dict[str, Any]: "trial": trial, "case_type": case_type, "seed": seed, + "arm": benchmark_context.get("arm"), "mode": case_mode, "score_type": score_type, "supported_modes": supported_modes, @@ -1207,6 +1356,7 @@ def finalize(payload: dict[str, Any]) -> dict[str, Any]: "trial": trial, "case_type": case_type, "seed": seed, + "arm": benchmark_context.get("arm"), "mode": case_mode, "score_type": score_type, "supported_modes": supported_modes, @@ -1247,6 +1397,7 @@ def finalize(payload: dict[str, Any]) -> dict[str, Any]: "trial": trial, "case_type": case_type, "seed": seed, + "arm": benchmark_context.get("arm"), "mode": case_mode, "score_type": score_type, "supported_modes": supported_modes, @@ -1406,6 +1557,56 @@ def _agent_benchmark_summary(results: list[dict[str, Any]]) -> dict[str, Any]: } +def _paired_arms_summary(results: list[dict[str, Any]]) -> dict[str, Any] | None: + """Report `plain` and `oms` side by side, never as one number. + + Quality (success rate with interval) and cost (median USD, tokens, + duration) are separate columns of a trade-off. Trajectory measures are + diagnostics only: an agent is not penalised for reaching a correct + artifact by a different route. + """ + live = [result for result in results if result.get("mode") == "live" and result.get("status") != "skipped"] + arms = sorted({result.get("arm") for result in live if result.get("arm")}) + if len(arms) < 2: + return None + by_arm: dict[str, dict[str, Any]] = {} + signatures: dict[str, set[tuple[Any, ...]]] = {} + for arm in arms: + arm_results = [result for result in live if result.get("arm") == arm] + signatures[arm] = {(result.get("id"), result.get("trial"), result.get("seed")) for result in arm_results} + event_counts = [ + int(run.get("event_count", 0)) + for result in arm_results + if isinstance(run := result.get("agent_run"), dict) + ] + by_arm[arm] = { + "quality": _agent_benchmark_summary(arm_results), + "diagnostics": { + "median_event_count": median(event_counts) if event_counts else None, + "note": "trajectory measures are diagnostics, not correctness", + }, + } + parity = all(signatures[arm] == signatures[arms[0]] for arm in arms) + pareto = [ + { + "arm": arm, + "task_success_rate": by_arm[arm]["quality"]["task_success_rate"], + "task_success_rate_95ci": by_arm[arm]["quality"]["task_success_rate_95ci"], + "median_cost_usd": by_arm[arm]["quality"]["median_cost_usd"], + "median_tokens": by_arm[arm]["quality"]["median_tokens"], + "median_duration_s": by_arm[arm]["quality"]["median_duration_s"], + } + for arm in arms + ] + return { + "schema": "openmapstack-paired-arms/v1", + "arms": by_arm, + "task_parity": parity, + "pareto": pareto, + "note": "quality and cost are reported as a trade-off; no combined score is published", + } + + def build_summary(results: list[dict[str, Any]], run_config: dict[str, Any]) -> dict[str, Any]: ran = [r for r in results if r.get("status") != "skipped"] passed = [r for r in ran if r.get("status") == "passed"] @@ -1452,6 +1653,7 @@ def build_summary(results: list[dict[str, Any]], run_config: dict[str, Any]) -> "setup_errors": [], "score_types": score_types, "agent_benchmark": _agent_benchmark_summary(results), + "paired_arms": _paired_arms_summary(results), "mutation_score": { "total": len(mutations), "valid": valid_mutations, @@ -1513,8 +1715,24 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--skill-mode", choices=("enabled", "disabled"), - default="enabled", - help="inject the controlled OpenMapStack skill snapshot in live mode (default: enabled)", + default=None, + help="live-mode arm by skill mode: enabled = oms, disabled = plain (default: enabled; see --arms)", + ) + parser.add_argument( + "--arms", + choices=KNOWN_ARM_SELECTIONS, + default=None, + help="live-mode arm selection: oms (skill injected), plain (no skill), or paired (both, same cases/trials/seeds)", + ) + parser.add_argument( + "--price-catalog-date", + help="YYYY-MM-DD of the price list used for cost estimates; recorded in each arm's provenance", + ) + parser.add_argument( + "--export-tasks", + type=Path, + metavar="DIR", + help="write vendor-neutral openmapstack-benchmark-task/v1 bundles for the selected live cases and exit", ) parser.add_argument("--run-id", help="artifact run id (generated by default for live mode)") parser.add_argument( @@ -1533,6 +1751,16 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) revision = _git_revision() + if args.skill_mode is not None and args.arms is not None and ARM_BY_SKILL_MODE[args.skill_mode] != args.arms: + parser.error("--skill-mode and --arms disagree; pass one of them") + if args.arms is None: + args.arms = ARM_BY_SKILL_MODE[args.skill_mode or "enabled"] + arms = ["plain", "oms"] if args.arms == "paired" else [args.arms] + if args.price_catalog_date is not None: + try: + datetime.strptime(args.price_catalog_date, "%Y-%m-%d") + except ValueError: + parser.error("--price-catalog-date must be YYYY-MM-DD") try: run_id = _validate_run_id(args.run_id or _new_run_id()) if args.mode in {"live", "visual"} else None except ValueError as exc: @@ -1552,7 +1780,10 @@ def main(argv: list[str] | None = None) -> int: "artifact_root": _evidence_path(result_root) if result_root else None, "skill_commit": revision["commit"], "skill_worktree_dirty": revision["dirty"], - "skill_mode": args.skill_mode if args.mode == "live" else None, + "skill_mode": (SKILL_MODE_BY_ARM[arms[0]] if len(arms) == 1 else "paired") if args.mode == "live" else None, + "arms": arms if args.mode == "live" else None, + "price_catalog_date": args.price_catalog_date, + "checker": {"package_version": OPENMAPSTACK_VERSION, "check_api_version": CHECK_API_VERSION}, } try: @@ -1574,6 +1805,18 @@ def main(argv: list[str] | None = None) -> int: _write_summary(summary, args.json) return 2 + if args.export_tasks is not None: + try: + manifest = export_tasks(case_dirs, args.export_tasks) + except (OSError, ValueError) as exc: + print(f"Task export failed: {exc}", file=sys.stderr) + return 2 + if not manifest["tasks"]: + print("No live-capable cases selected; nothing exported", file=sys.stderr) + return 2 + print(f"Exported {len(manifest['tasks'])} task bundle(s) to {args.export_tasks}") + return 0 + if args.list: for case_dir in case_dirs: case_def = _load_case(case_dir) @@ -1611,12 +1854,14 @@ def main(argv: list[str] | None = None) -> int: return 2 results: list[dict[str, Any]] = [] + paired = len(arms) > 1 for case_dir in case_dirs: case_def = _load_case(case_dir) selected_score_type = case_def["score_types"].get(args.mode) - for trial in range(1, args.repetitions + 1): + for trial, arm in itertools.product(range(1, args.repetitions + 1), arms): trial_seed = args.seed + trial - 1 if args.seed is not None else None trial_started = time.monotonic() + arm_segment = (arm,) if paired else () try: result = run_case( case_dir, @@ -1627,7 +1872,7 @@ def main(argv: list[str] | None = None) -> int: seed=trial_seed, trial=trial, artifact_dir=( - result_root / (args.agent or case_def["live"].get("agent", "claude_code")) / case_def.get("id", case_dir.name) / str(trial) + result_root.joinpath(args.agent or case_def["live"].get("agent", "claude_code"), *arm_segment, case_def.get("id", case_dir.name), str(trial)) if args.mode == "live" and result_root is not None and not args.no_retain_artifacts and "live" in case_def else ( result_root / "visual" / case_def.get("id", case_dir.name) / str(trial) @@ -1641,7 +1886,7 @@ def main(argv: list[str] | None = None) -> int: "skill_worktree_dirty": revision["dirty"], "environment": _environment(), }, - skill_mode=args.skill_mode, + skill_mode=SKILL_MODE_BY_ARM[arm] if args.mode == "live" else None, ) except Exception as exc: # noqa: BLE001 result = { @@ -1649,6 +1894,7 @@ def main(argv: list[str] | None = None) -> int: "trial": trial, "case_type": case_def["case_type"], "seed": trial_seed, + "arm": arm if args.mode == "live" else None, "mode": args.mode, "score_type": selected_score_type, "supported_modes": case_def["modes"], @@ -1682,6 +1928,8 @@ def main(argv: list[str] | None = None) -> int: "setup_failed": "ERROR", }[result["status"]] suffix = f" trial={trial}" if args.repetitions > 1 else "" + if paired: + suffix += f" arm={arm}" print( f"{marker:5s} {result['id']:40s} {result['duration_s']:.2f}s{suffix}", end="", @@ -1694,6 +1942,25 @@ def main(argv: list[str] | None = None) -> int: else: print() + if args.mode == "live": + task_set = _task_set_hash(case_dirs) + run_config["arm_provenance"] = [ + _arm_record( + arm, + skill=next( + ((r.get("benchmark_context") or {}).get("skill") or {} for r in results if r.get("arm") == arm and (r.get("benchmark_context") or {}).get("skill")), + {"mode": SKILL_MODE_BY_ARM[arm], "content_sha256": None, "commit": revision["commit"], "entrypoint": None}, + ), + task_set=task_set, + revision=revision, + agent_name=args.agent, + model=args.model, + seed=args.seed, + price_catalog_date=args.price_catalog_date, + trial_results=[r for r in results if r.get("arm") == arm], + ) + for arm in arms + ] summary = build_summary(results, run_config) if summary["selection"]["trials_run"] == 0: summary["run_setup_failed"] = True @@ -1737,6 +2004,14 @@ def main(argv: list[str] | None = None) -> int: f"({score_text}; {mutation_score['isolated']} isolated, " f"{mutation_score['invalid']} invalid)" ) + paired_summary = summary.get("paired_arms") + if paired_summary: + parity = "task parity" if paired_summary["task_parity"] else "TASK PARITY BROKEN" + print(f"paired arms ({parity}):") + for row in paired_summary["pareto"]: + rate = f"{row['task_success_rate']:.0%}" if row["task_success_rate"] is not None else "n/a" + cost = f"${row['median_cost_usd']:.4f}" if row["median_cost_usd"] is not None else "cost n/a" + print(f" {row['arm']:6s} success {rate} {row['task_success_rate_95ci']} median {cost}, tokens {row['median_tokens']}, {row['median_duration_s']}s") skipped = summary["selection"]["case_definitions_skipped"] if skipped: print(f"Skipped case definitions: {skipped}") diff --git a/evals/schemas/benchmark-arm-v1.schema.json b/evals/schemas/benchmark-arm-v1.schema.json new file mode 100644 index 0000000..a97a771 --- /dev/null +++ b/evals/schemas/benchmark-arm-v1.schema.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openmapstack/schemas/benchmark-arm-v1.schema.json", + "title": "Complete provenance of one benchmark arm (openmapstack-benchmark-arm/v1)", + "description": "A published result identifies the whole arm. Every field must be present; a value the harness genuinely cannot learn is null, never omitted.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", "arm", "skill", "task_set", "checker", "harness", "runtime", + "tool_surface", "model", "sampling", "price_catalog_date" + ], + "properties": { + "schema": {"const": "openmapstack-benchmark-arm/v1"}, + "arm": {"enum": ["plain", "oms"]}, + "skill": { + "type": "object", + "required": ["mode", "content_sha256", "commit", "dirty", "entrypoint"], + "properties": { + "mode": {"enum": ["enabled", "disabled"]}, + "content_sha256": {"type": ["string", "null"], "pattern": "^sha256:[0-9a-f]{64}$"}, + "commit": {"type": ["string", "null"]}, + "dirty": {"type": ["boolean", "null"]}, + "entrypoint": {"type": ["string", "null"]} + } + }, + "task_set": { + "type": "object", + "required": ["cases", "sha256"], + "properties": { + "cases": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "sha256": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"} + } + }, + "checker": { + "type": "object", + "required": ["package", "package_version", "check_api_version"], + "properties": { + "package": {"const": "openmapstack"}, + "package_version": {"type": "string", "minLength": 1}, + "check_api_version": {"type": "string", "minLength": 1} + } + }, + "harness": { + "type": "object", + "required": ["name", "commit", "dirty"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "commit": {"type": ["string", "null"]}, + "dirty": {"type": ["boolean", "null"]} + } + }, + "runtime": { + "type": "object", + "required": ["python", "platform", "duckdb", "container_image"], + "properties": { + "python": {"type": "string"}, + "platform": {"type": "string"}, + "duckdb": {"type": ["string", "null"]}, + "container_image": {"type": ["string", "null"]} + } + }, + "tool_surface": { + "type": "object", + "required": ["adapter", "agent_version"], + "properties": { + "adapter": {"type": ["string", "null"]}, + "agent_version": {"type": ["string", "null"]} + } + }, + "model": { + "type": "object", + "required": ["provider", "id", "revision"], + "properties": { + "provider": {"type": ["string", "null"]}, + "id": {"type": ["string", "null"]}, + "revision": {"type": ["string", "null"]} + } + }, + "sampling": { + "type": "object", + "required": ["seed", "temperature", "reasoning"], + "properties": { + "seed": {"type": ["integer", "null"]}, + "temperature": {"type": ["number", "null"]}, + "reasoning": {"type": ["string", "object", "null"]} + } + }, + "price_catalog_date": {"type": ["string", "null"], "pattern": "^\\d{4}-\\d{2}-\\d{2}$"} + } +} diff --git a/evals/schemas/benchmark-task-v1.schema.json b/evals/schemas/benchmark-task-v1.schema.json new file mode 100644 index 0000000..d09312c --- /dev/null +++ b/evals/schemas/benchmark-task-v1.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openmapstack/schemas/benchmark-task-v1.schema.json", + "title": "Vendor-neutral benchmark task bundle (openmapstack-benchmark-task/v1)", + "type": "object", + "additionalProperties": false, + "required": ["schema", "id", "prompt", "fixtures", "assertions", "hard_gate", "agent_workdir", "clean_rerun", "task_sha256", "checker_api"], + "properties": { + "schema": {"const": "openmapstack-benchmark-task/v1"}, + "id": {"type": "string", "minLength": 1}, + "prompt": {"type": "string", "minLength": 1}, + "fixtures": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "destination", "sha256"], + "properties": { + "path": {"type": "string", "minLength": 1}, + "destination": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"} + } + } + }, + "assertions": {"type": "array", "items": {"type": "object", "required": ["assert"]}}, + "hard_gate": {"type": "boolean"}, + "agent_workdir": {"type": "string", "minLength": 1}, + "clean_rerun": {"type": "boolean"}, + "task_sha256": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "checker_api": {"type": "string", "minLength": 1}, + "ownership": {"type": "string"} + } +} diff --git a/evals/schemas/results-v2.schema.json b/evals/schemas/results-v2.schema.json index 89d413e..8182ac7 100644 --- a/evals/schemas/results-v2.schema.json +++ b/evals/schemas/results-v2.schema.json @@ -47,6 +47,7 @@ "dimensions": {"type": "object"} } }, + "paired_arms": {"type": ["object", "null"]}, "mutation_score": { "type": "object", "required": ["total", "valid", "detected", "survived", "invalid", "isolated", "score"], diff --git a/openmapstack/cli.py b/openmapstack/cli.py index 411d669..2b2f508 100644 --- a/openmapstack/cli.py +++ b/openmapstack/cli.py @@ -139,6 +139,17 @@ def build_parser() -> argparse.ArgumentParser: api_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") api_parser.set_defaults(handler=_cmd_api_info) + snapshot_parser = subparsers.add_parser( + "skill-snapshot", + help="copy SKILL.md, references/, and templates/ into a hashed, inspectable snapshot", + ) + snapshot_mode = snapshot_parser.add_mutually_exclusive_group(required=True) + snapshot_mode.add_argument("--out", type=Path, help="destination directory (must be empty or absent)") + snapshot_mode.add_argument("--inspect", type=Path, metavar="DIR", help="re-verify an existing snapshot instead of creating one") + snapshot_parser.add_argument("--source", type=Path, help="skill root holding SKILL.md (default: nearest ancestor of the current directory)") + snapshot_parser.add_argument("--json", action="store_true", help="emit the snapshot manifest or inspection as JSON") + snapshot_parser.set_defaults(handler=_cmd_skill_snapshot) + inspect_parser = subparsers.add_parser("inspect", help="summarize a project for audit and review") inspect_parser.add_argument("project", nargs="?", default="project.yaml", help="project.yaml or its directory") inspect_parser.add_argument("--json", action="store_true", help="emit the full summary as JSON") @@ -318,6 +329,37 @@ def _cmd_run(args: argparse.Namespace) -> int: return 0 if validation.ok(strict=args.strict) else 1 +def _cmd_skill_snapshot(args: argparse.Namespace) -> int: + from .snapshot import SnapshotError, create_skill_snapshot, find_skill_root, inspect_skill_snapshot + + try: + if args.inspect is not None: + report = inspect_skill_snapshot(args.inspect) + if args.json: + print(_json(report)) + else: + state = "intact" if report["intact"] else "TAMPERED" + print(f"{state}: {report['snapshot']} ({report['file_count']} files, {report['content_sha256']})") + for problem in report["problems"]: + print(f" {problem}") + return 0 if report["intact"] else 1 + source = args.source or find_skill_root() + if source is None: + raise SnapshotError("no skill root found; pass --source DIR holding SKILL.md, references/, and templates/") + manifest = create_skill_snapshot(source, args.out) + except SnapshotError as exc: + if args.json: + print(_json({"schema": "openmapstack-skill-snapshot-error/v1", "error": str(exc)})) + else: + print(f"openmapstack skill-snapshot: {exc}", file=sys.stderr) + return 2 + if args.json: + print(_json(manifest)) + else: + print(f"Wrote {args.out} ({manifest['file_count']} files, {manifest['content_sha256']})") + return 0 + + def _cmd_checks(args: argparse.Namespace) -> int: from .api import CHECK_API_VERSION, list_checks diff --git a/openmapstack/snapshot.py b/openmapstack/snapshot.py new file mode 100644 index 0000000..6ff75a8 --- /dev/null +++ b/openmapstack/snapshot.py @@ -0,0 +1,199 @@ +"""Controlled snapshots of the shipped skill (``openmapstack-skill-snapshot/v1``). + +A benchmark arm that injects the skill must say *which* skill: not a tag +that can move, not a commit whose working tree may have been dirty, but the +bytes the agent actually read. ``create_skill_snapshot`` copies exactly +``SKILL.md``, ``references/``, and ``templates/`` from a skill root, records a +per-file inventory and a content hash, and writes the manifest beside the +copy. ``inspect_skill_snapshot`` re-verifies one later. + +Safety: symlinks anywhere in the source tree are refused (a snapshot must +not quietly include files from outside the skill), and every inventory path +is confined to the snapshot root. The content hash covers the relative path +and the bytes of every file, so a renamed reference changes it. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +SNAPSHOT_SCHEMA = "openmapstack-skill-snapshot/v1" +SNAPSHOT_MANIFEST = "snapshot.json" +SKILL_ENTRYPOINT = "SKILL.md" +SKILL_DIRECTORIES = ("references", "templates") +_IGNORED = {"__pycache__", ".DS_Store"} + + +class SnapshotError(ValueError): + """The skill root or an existing snapshot is unusable.""" + + +def find_skill_root(start: Path | None = None) -> Path | None: + """Walk upwards from ``start`` to the nearest directory holding SKILL.md.""" + current = (start or Path.cwd()).resolve() + for candidate in (current, *current.parents): + if (candidate / SKILL_ENTRYPOINT).is_file() and all((candidate / name).is_dir() for name in SKILL_DIRECTORIES): + return candidate + return None + + +def _iter_skill_files(root: Path) -> list[Path]: + entrypoint = root / SKILL_ENTRYPOINT + if not entrypoint.is_file(): + raise SnapshotError(f"{SKILL_ENTRYPOINT} is missing from {root}") + files = [entrypoint] + for name in SKILL_DIRECTORIES: + directory = root / name + if not directory.is_dir(): + raise SnapshotError(f"{name}/ is missing from {root}") + for path in sorted(directory.rglob("*")): + if any(part in _IGNORED or part.endswith(".pyc") for part in path.relative_to(root).parts): + continue + if path.is_symlink(): + raise SnapshotError(f"skill tree contains a symlink, refusing to snapshot: {path.relative_to(root)}") + if path.is_file(): + files.append(path) + if entrypoint.is_symlink(): + raise SnapshotError(f"{SKILL_ENTRYPOINT} is a symlink, refusing to snapshot") + return files + + +def _content_hash(entries: list[dict[str, Any]]) -> str: + """Hash over sorted (path, bytes-hash) pairs; the same algorithm the eval + runner used before this module existed, so historical arm hashes stay + comparable.""" + digest = hashlib.sha256() + for entry in sorted(entries, key=lambda item: item["path"]): + digest.update(entry["path"].encode("utf-8")) + digest.update(b"\0") + digest.update(bytes.fromhex(entry["sha256"].removeprefix("sha256:"))) + digest.update(b"\0") + return f"sha256:{digest.hexdigest()}" + + +def _git(root: Path) -> dict[str, Any]: + revision: dict[str, Any] = {"commit": None, "dirty": None} + try: + commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, timeout=10, check=False) + if commit.returncode == 0: + revision["commit"] = commit.stdout.strip() or None + status = subprocess.run(["git", "status", "--porcelain", "--", SKILL_ENTRYPOINT, *SKILL_DIRECTORIES], cwd=root, capture_output=True, text=True, timeout=10, check=False) + if status.returncode == 0: + revision["dirty"] = bool(status.stdout.strip()) + except (OSError, subprocess.SubprocessError): + pass + return revision + + +def create_skill_snapshot(source_root: str | Path, destination: str | Path, *, now: datetime | None = None) -> dict[str, Any]: + """Copy the distributable skill into ``destination`` and return its manifest.""" + root = Path(source_root).resolve() + target = Path(destination).resolve() + if target.exists() and any(target.iterdir()): + raise SnapshotError(f"snapshot destination is not empty: {target}") + if target == root or root in target.parents: + raise SnapshotError("snapshot destination must not be inside the skill root") + files = _iter_skill_files(root) + target.mkdir(parents=True, exist_ok=True) + entries: list[dict[str, Any]] = [] + for path in files: + relative = path.relative_to(root).as_posix() + copy_to = target / relative + copy_to.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, copy_to) + data = copy_to.read_bytes() + entries.append({"path": relative, "sha256": "sha256:" + hashlib.sha256(data).hexdigest(), "bytes": len(data)}) + manifest = { + "schema": SNAPSHOT_SCHEMA, + "created_at": (now or datetime.now(timezone.utc)).isoformat().replace("+00:00", "Z"), + "source_root": root.name, + "source_git": _git(root), + "entrypoint": SKILL_ENTRYPOINT, + "files": entries, + "file_count": len(entries), + "content_sha256": _content_hash(entries), + } + (target / SNAPSHOT_MANIFEST).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return manifest + + +def hash_skill_root(source_root: str | Path) -> str: + """Content hash of a skill root without copying it.""" + root = Path(source_root).resolve() + entries = [] + for path in _iter_skill_files(root): + entries.append({"path": path.relative_to(root).as_posix(), "sha256": "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest()}) + return _content_hash(entries) + + +def inspect_skill_snapshot(snapshot_dir: str | Path) -> dict[str, Any]: + """Re-verify a snapshot against its own manifest. + + Reports missing, changed, and extra files plus any symlink or escaping + inventory path; ``intact`` is true only when the copy still matches. + """ + root = Path(snapshot_dir).resolve() + manifest_path = root / SNAPSHOT_MANIFEST + if not manifest_path.is_file(): + raise SnapshotError(f"{SNAPSHOT_MANIFEST} is missing from {root}") + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise SnapshotError(f"cannot read {SNAPSHOT_MANIFEST}: {exc}") from exc + if not isinstance(manifest, dict) or manifest.get("schema") != SNAPSHOT_SCHEMA: + raise SnapshotError(f"{SNAPSHOT_MANIFEST} is not an {SNAPSHOT_SCHEMA} document") + problems: list[str] = [] + seen: set[str] = set() + recomputed: list[dict[str, Any]] = [] + for entry in manifest.get("files") or []: + relative = entry.get("path") if isinstance(entry, dict) else None + if not isinstance(relative, str) or not relative: + problems.append("inventory entry without a path") + continue + candidate = (root / relative) + try: + candidate.resolve().relative_to(root) + except ValueError: + problems.append(f"inventory path escapes the snapshot: {relative}") + continue + if Path(relative).is_absolute() or ".." in Path(relative).parts: + problems.append(f"inventory path escapes the snapshot: {relative}") + continue + seen.add(relative) + if candidate.is_symlink(): + problems.append(f"symlink in snapshot: {relative}") + continue + if not candidate.is_file(): + problems.append(f"missing: {relative}") + continue + actual = "sha256:" + hashlib.sha256(candidate.read_bytes()).hexdigest() + recomputed.append({"path": relative, "sha256": actual}) + if actual != entry.get("sha256"): + problems.append(f"changed: {relative}") + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if relative == SNAPSHOT_MANIFEST or any(part in _IGNORED for part in path.relative_to(root).parts): + continue + if path.is_symlink(): + problems.append(f"symlink in snapshot: {relative}") + elif path.is_file() and relative not in seen: + problems.append(f"extra: {relative}") + content_hash = _content_hash(recomputed) if recomputed else None + if not problems and content_hash != manifest.get("content_sha256"): + problems.append("content_sha256 does not match the inventory") + return { + "schema": "openmapstack-skill-snapshot-inspection/v1", + "snapshot": str(root), + "intact": not problems, + "content_sha256": manifest.get("content_sha256"), + "recomputed_sha256": content_hash, + "file_count": len(seen), + "problems": problems, + "manifest": manifest, + } diff --git a/tests/test_evals.py b/tests/test_evals.py index f1a4911..c564060 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -27,7 +27,7 @@ from adapters.base import AgentRunResult # noqa: E402 -class EvalRunnerTests(unittest.TestCase): +class _RunnerHarness(unittest.TestCase): def setUp(self) -> None: self.tempdir = tempfile.TemporaryDirectory(prefix="openmapstack-eval-runner-test-") self.root = Path(self.tempdir.name) @@ -115,6 +115,8 @@ def call_main(self, argv: list[str]) -> tuple[int, str, str]: exit_code = eval_runner.main(argv) return exit_code, stdout.getvalue(), stderr.getvalue() + +class EvalRunnerTests(_RunnerHarness): def test_fixture_mode_is_default_and_passes(self) -> None: self.write_case() exit_code, stdout, _ = self.call_main([]) @@ -915,6 +917,107 @@ def test_case_flag_can_select_multiple_cases(self) -> None: self.assertEqual([result["id"] for result in payload["results"]], ["first", "second"]) +class PairedArmTests(_RunnerHarness): + """Paired plain/oms arms, arm provenance, and task export (issue #13, C2/C3).""" + + def _adapter(self, seen: list[tuple[str, int | None]]): + class Adapter: + executable = "fake-agent" + + @staticmethod + def is_available() -> bool: + return True + + @staticmethod + def run(prompt, workspace, fixture=None, timeout_s=900, model=None, seed=None): + skill_present = (workspace.parent / "benchmark-context" / "openmapstack" / "SKILL.md").is_file() + seen.append(("oms" if skill_present else "plain", seed)) + (workspace / "marker.txt").write_text("ok\n", encoding="utf-8") + return AgentRunResult( + agent="codex", model=model, workspace=workspace, duration_s=0.1, success=True, returncode=0, + command=["fake-agent"], version="fake 2", events=[{"type": "x"}] * (3 if skill_present else 5), + usage={"total_tokens": 10 if skill_present else 20}, cost_usd=0.02 if skill_present else 0.01, + permissions={}, metadata={"structured_completion": True, "temperature": 0.0}, + ) + + return Adapter() + + def test_paired_arms_share_cases_trials_and_seeds_and_are_reported_apart(self) -> None: + self.write_case("paired-live", modes=["live"]) + seen: list[tuple[str, int | None]] = [] + summary_path = self.root / "paired.json" + with patch.object(eval_runner, "_load_adapter", return_value=self._adapter(seen)): + exit_code, stdout, stderr = self.call_main([ + "--mode", "live", "--agent", "codex", "--model", "gpt-test", "--case", "paired-live", + "--arms", "paired", "--repetitions", "2", "--seed", "7", "--price-catalog-date", "2026-09-01", + "--run-id", "paired-run", "--results-dir", str(self.results_dir), "--json", str(summary_path), + ]) + self.assertEqual(exit_code, 0, (stdout, stderr)) + self.assertEqual(sorted(seen), [("oms", 7), ("oms", 8), ("plain", 7), ("plain", 8)]) + summary = json.loads(summary_path.read_text(encoding="utf-8")) + paired = summary["paired_arms"] + self.assertTrue(paired["task_parity"]) + self.assertEqual({row["arm"] for row in paired["pareto"]}, {"plain", "oms"}) + self.assertEqual(paired["arms"]["oms"]["quality"]["median_cost_usd"], 0.02) + self.assertEqual(paired["arms"]["plain"]["quality"]["median_tokens"], 20.0) + self.assertEqual(paired["arms"]["plain"]["diagnostics"]["median_event_count"], 5) + self.assertNotIn("success_per_dollar", json.dumps(paired)) + self.assertEqual(summary["run_config"]["arms"], ["plain", "oms"]) + self.assertEqual(summary["run_config"]["skill_mode"], "paired") + provenance = {record["arm"]: record for record in summary["run_config"]["arm_provenance"]} + self.assertEqual(provenance["oms"]["skill"]["mode"], "enabled") + self.assertRegex(provenance["oms"]["skill"]["content_sha256"], r"^sha256:[0-9a-f]{64}$") + self.assertIsNone(provenance["plain"]["skill"]["content_sha256"]) + self.assertEqual(provenance["oms"]["price_catalog_date"], "2026-09-01") + self.assertEqual(provenance["oms"]["tool_surface"], {"adapter": "codex", "agent_version": "fake 2"}) + self.assertEqual(provenance["oms"]["sampling"], {"seed": 7, "temperature": 0.0, "reasoning": None}) + self.assertEqual(provenance["oms"]["task_set"], provenance["plain"]["task_set"]) + self.assertEqual(provenance["oms"]["checker"]["check_api_version"], "openmapstack-check-api/v1") + for arm in ("plain", "oms"): + bundle = self.results_dir / "paired-run" / "codex" / arm / "paired-live" / "1" + self.assertTrue((bundle / "grading.json").is_file(), bundle) + self.assertIn("paired arms (task parity)", stdout) + + def test_arm_flags_must_agree_and_default_to_oms(self) -> None: + self.write_case("arm-flags", modes=["live"]) + seen: list[tuple[str, int | None]] = [] + with patch.object(eval_runner, "_load_adapter", return_value=self._adapter(seen)): + exit_code, _, _ = self.call_main(["--mode", "live", "--agent", "codex", "--model", "m", "--case", "arm-flags", "--no-retain-artifacts"]) + self.assertEqual(exit_code, 0) + self.assertEqual(seen, [("oms", None)]) + with self.assertRaises(SystemExit): + self.call_main(["--mode", "live", "--agent", "codex", "--model", "m", "--skill-mode", "disabled", "--arms", "oms"]) + with self.assertRaises(SystemExit): + self.call_main(["--mode", "live", "--agent", "codex", "--model", "m", "--price-catalog-date", "yesterday"]) + + def test_export_tasks_writes_vendor_neutral_bundles(self) -> None: + fixture = self.root / "fixture.geojson" + fixture.write_text('{"type": "FeatureCollection", "features": []}', encoding="utf-8") + case_dir = self.write_case("070-underspecified-prompt", modes=["live"], live_fixtures=[ + {"source": os.path.relpath(fixture, self.cases_dir / "070-underspecified-prompt"), "destination": "project/data/source/fixture.geojson"}, + ]) + # fixture sources must stay inside the eval tree for a live run, but export only reads them + self.write_case("fixture-only") + destination = self.root / "tasks" + exit_code, stdout, stderr = self.call_main(["--export-tasks", str(destination)]) + self.assertEqual(exit_code, 0, (stdout, stderr)) + index = json.loads((destination / "index.json").read_text(encoding="utf-8")) + self.assertEqual([task["id"] for task in index["tasks"]], ["070-underspecified-prompt"]) + task = json.loads((destination / "070-underspecified-prompt" / "task.json").read_text(encoding="utf-8")) + self.assertEqual(task["schema"], "openmapstack-benchmark-task/v1") + self.assertEqual(task["ownership"], "openmapbench") + self.assertEqual(task["prompt"], "Build the project.\n") + self.assertEqual(task["fixtures"][0]["destination"], "project/data/source/fixture.geojson") + self.assertTrue((destination / "070-underspecified-prompt" / "fixtures" / "fixture.geojson").is_file()) + self.assertFalse((destination / "070-underspecified-prompt" / "project").exists()) + self.assertEqual( + eval_runner.validation_errors(task, eval_runner._load_eval_schema("benchmark-task-v1.schema.json")), [] + ) + exit_code, _, stderr = self.call_main(["--export-tasks", str(self.root / "none"), "--case", "fixture-only"]) + self.assertEqual(exit_code, 2) + self.assertIn("No live-capable", stderr) + + class CapabilityRollupTests(unittest.TestCase): """A pass rate produced where part of the suite could not run is not the same evidence as one produced where all of it ran. The rollup makes that diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py new file mode 100644 index 0000000..7773254 --- /dev/null +++ b/tests/test_snapshot.py @@ -0,0 +1,113 @@ +"""Skill snapshots: hashed, inspectable, symlink- and escape-safe (issue #13, C2).""" + +from __future__ import annotations + +import io +import json +import os +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +from openmapstack.cli import main +from openmapstack.snapshot import ( + SnapshotError, + create_skill_snapshot, + find_skill_root, + hash_skill_root, + inspect_skill_snapshot, +) +from tests.evals.helpers import make_workspace + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _skill_root() -> Path: + root = make_workspace() / "skill" + (root / "references").mkdir(parents=True) + (root / "templates").mkdir() + (root / "SKILL.md").write_text("# skill\n", encoding="utf-8") + (root / "references/project-spec.md").write_text("spec\n", encoding="utf-8") + (root / "templates/project.yaml").write_text("schema: openmapstack-project/v1\n", encoding="utf-8") + (root / "evals").mkdir() + (root / "evals/secret-case.yaml").write_text("must not be copied\n", encoding="utf-8") + return root + + +class SnapshotTests(unittest.TestCase): + def test_snapshot_copies_only_the_distributable_skill(self) -> None: + root = _skill_root() + out = make_workspace() / "snap" + manifest = create_skill_snapshot(root, out) + self.assertEqual(manifest["schema"], "openmapstack-skill-snapshot/v1") + self.assertEqual({entry["path"] for entry in manifest["files"]}, {"SKILL.md", "references/project-spec.md", "templates/project.yaml"}) + self.assertFalse((out / "evals").exists()) + self.assertEqual(manifest["content_sha256"], hash_skill_root(root)) + self.assertTrue(inspect_skill_snapshot(out)["intact"]) + + def test_content_hash_covers_paths_and_bytes(self) -> None: + root = _skill_root() + before = hash_skill_root(root) + (root / "references/project-spec.md").write_text("spec v2\n", encoding="utf-8") + self.assertNotEqual(before, hash_skill_root(root)) + (root / "references/project-spec.md").write_text("spec\n", encoding="utf-8") + (root / "references/project-spec.md").rename(root / "references/renamed.md") + self.assertNotEqual(before, hash_skill_root(root)) + + def test_symlinks_are_refused(self) -> None: + root = _skill_root() + (root / "references/escape.md").symlink_to("/etc/hostname") + with self.assertRaises(SnapshotError): + create_skill_snapshot(root, make_workspace() / "snap") + + def test_inspection_detects_tampering_and_escapes(self) -> None: + root = _skill_root() + out = make_workspace() / "snap" + create_skill_snapshot(root, out) + (out / "SKILL.md").write_text("# edited\n", encoding="utf-8") + (out / "references/extra.md").write_text("added\n", encoding="utf-8") + report = inspect_skill_snapshot(out) + self.assertFalse(report["intact"]) + self.assertIn("changed: SKILL.md", report["problems"]) + self.assertIn("extra: references/extra.md", report["problems"]) + manifest = json.loads((out / "snapshot.json").read_text()) + manifest["files"].append({"path": "../outside.md", "sha256": "sha256:" + "0" * 64, "bytes": 1}) + (out / "snapshot.json").write_text(json.dumps(manifest)) + self.assertTrue(any("escapes" in problem for problem in inspect_skill_snapshot(out)["problems"])) + + def test_destination_must_be_empty_and_outside_the_root(self) -> None: + root = _skill_root() + with self.assertRaises(SnapshotError): + create_skill_snapshot(root, root / "snap") + out = make_workspace() / "snap" + out.mkdir() + (out / "leftover").write_text("x") + with self.assertRaises(SnapshotError): + create_skill_snapshot(root, out) + + def test_repository_skill_root_is_discoverable_and_snapshots(self) -> None: + self.assertEqual(find_skill_root(REPO_ROOT / "openmapstack"), REPO_ROOT) + out = make_workspace() / "snap" + manifest = create_skill_snapshot(REPO_ROOT, out) + self.assertIn("references/project-spec.md", {entry["path"] for entry in manifest["files"]}) + self.assertNotIn("evals/README.md", {entry["path"] for entry in manifest["files"]}) + + def test_cli_creates_and_inspects(self) -> None: + root = _skill_root() + out = make_workspace() / "snap" + buffer = io.StringIO() + with redirect_stdout(buffer): + self.assertEqual(main(["skill-snapshot", "--out", str(out), "--source", str(root), "--json"]), 0) + manifest = json.loads(buffer.getvalue()) + self.assertEqual(manifest["file_count"], 3) + with redirect_stdout(io.StringIO()): + self.assertEqual(main(["skill-snapshot", "--inspect", str(out)]), 0) + (out / "SKILL.md").write_text("# edited\n", encoding="utf-8") + with redirect_stdout(io.StringIO()): + self.assertEqual(main(["skill-snapshot", "--inspect", str(out)]), 1) + with redirect_stderr(io.StringIO()): + self.assertEqual(main(["skill-snapshot", "--out", str(make_workspace() / "x"), "--source", str(make_workspace())]), 2) + + +if __name__ == "__main__": + unittest.main() From 1f55cea52fcf6ea4e1f19a771456e20425427856 Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 2 Sep 2026 17:10:36 +0000 Subject: [PATCH 6/8] test(verify): stable reports for two projects absent from the fixtures Two synthetic projects built by pure-Python pipelines over invented geodata (trail/river crossings with a metamorphic relation and a declared parameter; facilities per district with an override, an unverified expectation, and a warning) are verified with --metamorphic, and their text and JSON reports are compared with committed goldens after path and digest normalisation. The JSON also validates against the packaged verify-result schema. Writing the first project surfaced a genuine finding: GeoJSON without a crs member reads as WGS84, so declaring EPSG:3301 for it fails geodata.dataset_crs_is correctly. The project now writes the crs member; the debugging notes record that trap and the pipeline-copy requirement for metamorphic mutations. Refs #13 (definition of done). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01TMMy6A73zi7yGjK3wHBhju --- docs/maintainers/debugging.md | 9 + tests/goldens/verify/district-facilities.json | 174 ++++++++++ tests/goldens/verify/district-facilities.txt | 28 ++ tests/goldens/verify/river-crossings.json | 185 +++++++++++ tests/goldens/verify/river-crossings.txt | 29 ++ tests/test_evals.py | 2 +- tests/test_metamorphic.py | 1 - tests/test_snapshot.py | 1 - tests/test_verify_foreign.py | 297 ++++++++++++++++++ 9 files changed, 723 insertions(+), 3 deletions(-) create mode 100644 tests/goldens/verify/district-facilities.json create mode 100644 tests/goldens/verify/district-facilities.txt create mode 100644 tests/goldens/verify/river-crossings.json create mode 100644 tests/goldens/verify/river-crossings.txt create mode 100644 tests/test_verify_foreign.py diff --git a/docs/maintainers/debugging.md b/docs/maintainers/debugging.md index d650601..6407bd8 100644 --- a/docs/maintainers/debugging.md +++ b/docs/maintainers/debugging.md @@ -113,6 +113,15 @@ When a plausible wrong project from a live/user run survives the current checks, 4. add checker/unit coverage if the defect exposes a checker bug; 5. only then add prose context here if the trap remains worth remembering. +## Metamorphic mutations must live in the pipeline copy, not only in the generator + +A metamorphic relation reruns the project's own `pipeline.py` on perturbed input and compares with the produced outputs. For a fixture mutation that means the *copied* pipeline must reproduce the defect: if only `gen.py --break=` injects it at generation time, the variant run rebuilds the healthy analysis, every relation "fails" for the wrong reason, and the mutation is not isolated. `gen.py` therefore reads the `EVAL-BREAK` warning back in pipeline mode for the pipeline-logic break modes (`order_dependent`, `distance_inverted`, `duplicate_sensitive`) and nothing else. + +Two related traps: + +- a relation's detection power depends on the data and the variant size. Widening the mini-Tartu road threshold by 1.5× cannot expose an inverted predicate because the only far parcel sits at 5450 m; the fixture declares `variant: {multiply: 3}` for that reason. When a mutation survives, check the geometry before suspecting the relation; +- a GeoJSON output without a `crs` member reads back as EPSG:4326. A pipeline that writes analysis-CRS coordinates into plain GeoJSON and declares `EPSG:3301` in the manifest fails `geodata.dataset_crs_is` correctly. Write the `crs` member (or use GeoParquet) rather than relaxing the check. + ## Generated benchmark artifacts are evidence, not source Retained live/visual evidence belongs under `evals/results//...` and CI artifacts. Do not treat generated result JSON, screenshots, event streams, or temporary projects as canonical repository state unless a fixture intentionally owns them. diff --git a/tests/goldens/verify/district-facilities.json b/tests/goldens/verify/district-facilities.json new file mode 100644 index 0000000..af86daf --- /dev/null +++ b/tests/goldens/verify/district-facilities.json @@ -0,0 +1,174 @@ +{ + "checks": [ + { + "check": "project.parses", + "message": "project.yaml parses", + "status": "passed" + }, + { + "check": "project.conforms_to_schema", + "message": "project.yaml conforms to the packaged OpenMapStack v1 JSON Schema", + "status": "passed" + }, + { + "check": "project.graph_resolves", + "message": "graph resolves: 4 steps, 4 produced symbols, 1 outputs all traced to real steps", + "status": "passed" + }, + { + "check": "project.one_canonical_pipeline", + "message": "pipeline.py is the canonical implementation; wrappers import it", + "status": "passed" + }, + { + "check": "project.assumptions_have_rationale", + "message": "all 1 assumptions have statement + rationale", + "status": "passed" + }, + { + "check": "project.status_agrees_with_validation_report", + "message": "project.status 'warning' agrees with report status 'warning'", + "status": "passed" + }, + { + "args": { + "files": [ + "data/derived/district-counts.geojson" + ] + }, + "check": "project.declared_files_exist", + "message": "all 1 declared files exist", + "status": "passed" + }, + { + "check": "provenance.every_source_has_provider_and_access", + "message": "all 2 sources declare provider + access method + retrieval timestamp", + "status": "passed" + }, + { + "check": "provenance.every_source_pinned", + "message": "all 2 sources are pinned (version_identity)", + "status": "passed" + }, + { + "check": "provenance.no_inline_credentials", + "message": "no inline credentials in 2 sources; connections are by reference", + "status": "passed" + }, + { + "check": "provenance.license_present_where_required", + "message": "license metadata present for required sources", + "status": "passed" + }, + { + "check": "provenance.rationale_present", + "message": "all 2 sources document selection rationale", + "status": "passed" + }, + { + "check": "overrides.every_override_has_provenance", + "message": "all 1 overrides carry id/action/rationale/author/timestamp", + "status": "passed" + }, + { + "check": "overrides.evidence_not_placeholder", + "message": "no placeholder evidence found", + "status": "passed" + }, + { + "check": "validation.required_all_present", + "message": "all 3 declared checks present exactly once in report", + "status": "passed" + }, + { + "check": "validation.no_implicit_pass", + "message": "every check has an explicit status", + "status": "passed" + }, + { + "check": "validation.warning_or_failed_propagates_to_status", + "message": "overall status 'warning' correctly reflects check statuses", + "status": "passed" + }, + { + "check": "validation.run_record_matches", + "message": "report run_id 'run-20260901-000001' matches a real run record with consistent hashes", + "status": "passed" + }, + { + "args": { + "check": "geodata.feature_field_equals", + "equals": 2, + "field": "facility_count", + "id": "D1", + "id_field": "district_id", + "path": "data/derived/district-counts.geojson" + }, + "check": "expectation.d1-count", + "code": "expectation_unverified", + "evidence": { + "class": "unverified", + "expected_expectation_sha256": "$DIGEST" + }, + "message": "expectation 'd1-count' is unverified; independent review must bind expectation_sha256 to sha256:$DIGEST", + "status": "warning" + }, + { + "check": "geodata.crs_not_used_for_metrics", + "message": "analysis_crs EPSG:3301 is valid for 0 metric operation(s); storage/load/reprojection steps were excluded", + "status": "passed" + }, + { + "args": { + "path": "data/derived/district-counts.geojson" + }, + "check": "geodata.geometry_all_valid", + "message": "data/derived/district-counts.geojson: all 2 features have valid geometry", + "status": "passed" + }, + { + "args": { + "path": "data/derived/district-counts.geojson" + }, + "check": "geodata.dataset_crs_is", + "code": "crs_undeclared", + "message": "output 'district_counts' declares no EPSG code in its format string, so its real CRS cannot be cross-checked", + "status": "not_testable" + }, + { + "check": "presentation.layers_use_semantic_roles", + "message": "all 2 layers declare a semantic_role", + "status": "passed" + }, + { + "check": "presentation.controls_match_pipeline", + "message": "0 filter control(s) and 0 scenario control(s) consistent", + "status": "passed" + }, + { + "check": "presentation.edit_targets_reference_real_sources", + "message": "no edit targets declared (vacuously true)", + "status": "passed" + }, + { + "check": "rerun.no_chat_dependency", + "message": "canonical project dependencies contain no chat/transcript references", + "status": "passed" + } + ], + "counts": { + "failed": 0, + "not_testable": 1, + "passed": 24, + "warning": 1 + }, + "coverage": { + "applicable": 26, + "executed": 25, + "execution_rate": 0.9615384615384616, + "not_testable": 1 + }, + "project_file": "$PROJECT/project.yaml", + "schema": "openmapstack-verify-result/v1", + "status": "warning" +} diff --git a/tests/goldens/verify/district-facilities.txt b/tests/goldens/verify/district-facilities.txt new file mode 100644 index 0000000..dd1122d --- /dev/null +++ b/tests/goldens/verify/district-facilities.txt @@ -0,0 +1,28 @@ +PASS project.parses: project.yaml parses +PASS project.conforms_to_schema: project.yaml conforms to the packaged OpenMapStack v1 JSON Schema +PASS project.graph_resolves: graph resolves: 4 steps, 4 produced symbols, 1 outputs all traced to real steps +PASS project.one_canonical_pipeline: pipeline.py is the canonical implementation; wrappers import it +PASS project.assumptions_have_rationale: all 1 assumptions have statement + rationale +PASS project.status_agrees_with_validation_report: project.status 'warning' agrees with report status 'warning' +PASS project.declared_files_exist: all 1 declared files exist +PASS provenance.every_source_has_provider_and_access: all 2 sources declare provider + access method + retrieval timestamp +PASS provenance.every_source_pinned: all 2 sources are pinned (version_identity) +PASS provenance.no_inline_credentials: no inline credentials in 2 sources; connections are by reference +PASS provenance.license_present_where_required: license metadata present for required sources +PASS provenance.rationale_present: all 2 sources document selection rationale +PASS overrides.every_override_has_provenance: all 1 overrides carry id/action/rationale/author/timestamp +PASS overrides.evidence_not_placeholder: no placeholder evidence found +PASS validation.required_all_present: all 3 declared checks present exactly once in report +PASS validation.no_implicit_pass: every check has an explicit status +PASS validation.warning_or_failed_propagates_to_status: overall status 'warning' correctly reflects check statuses +PASS validation.run_record_matches: report run_id 'run-20260901-000001' matches a real run record with consistent hashes +WARN expectation.d1-count [data/derived/district-counts.geojson]: expectation 'd1-count' is unverified; independent review must bind expectation_sha256 to sha256:10399dfde7e9808a58d5ff9447545d111f87f3ef4d3277f10ae21d4c69bacaaa +PASS geodata.crs_not_used_for_metrics: analysis_crs EPSG:3301 is valid for 0 metric operation(s); storage/load/reprojection steps were excluded +PASS geodata.geometry_all_valid [data/derived/district-counts.geojson]: data/derived/district-counts.geojson: all 2 features have valid geometry +N/A geodata.dataset_crs_is [data/derived/district-counts.geojson]: output 'district_counts' declares no EPSG code in its format string, so its real CRS cannot be cross-checked +PASS presentation.layers_use_semantic_roles: all 2 layers declare a semantic_role +PASS presentation.controls_match_pipeline: 0 filter control(s) and 0 scenario control(s) consistent +PASS presentation.edit_targets_reference_real_sources: no edit targets declared (vacuously true) +PASS rerun.no_chat_dependency: canonical project dependencies contain no chat/transcript references +WARNING: $PROJECT/project.yaml (24 passed, 1 warnings, 1 not testable, 0 failed; 25/26 applicable checks executed) + NOTE some checks could not run here; install openmapstack[geo] for geodata checks, QGIS for the .qgz checks diff --git a/tests/goldens/verify/river-crossings.json b/tests/goldens/verify/river-crossings.json new file mode 100644 index 0000000..b5f1013 --- /dev/null +++ b/tests/goldens/verify/river-crossings.json @@ -0,0 +1,185 @@ +{ + "checks": [ + { + "check": "project.parses", + "message": "project.yaml parses", + "status": "passed" + }, + { + "check": "project.conforms_to_schema", + "message": "project.yaml conforms to the packaged OpenMapStack v1 JSON Schema", + "status": "passed" + }, + { + "check": "project.graph_resolves", + "message": "graph resolves: 3 steps, 3 produced symbols, 1 outputs all traced to real steps", + "status": "passed" + }, + { + "check": "project.one_canonical_pipeline", + "message": "pipeline.py is the canonical implementation; wrappers import it", + "status": "passed" + }, + { + "check": "project.assumptions_have_rationale", + "message": "all 1 assumptions have statement + rationale", + "status": "passed" + }, + { + "check": "project.status_agrees_with_validation_report", + "message": "project.status 'validated' agrees with report status 'passed'", + "status": "passed" + }, + { + "args": { + "files": [ + "data/derived/crossing-trails.geojson" + ] + }, + "check": "project.declared_files_exist", + "message": "all 1 declared files exist", + "status": "passed" + }, + { + "check": "project.parameters_match_steps", + "message": "1 runtime parameter(s) declared; 0 bound to a processing step agree with it", + "status": "passed" + }, + { + "check": "provenance.every_source_has_provider_and_access", + "message": "all 2 sources declare provider + access method + retrieval timestamp", + "status": "passed" + }, + { + "check": "provenance.every_source_pinned", + "message": "all 2 sources are pinned (version_identity)", + "status": "passed" + }, + { + "check": "provenance.no_inline_credentials", + "message": "no inline credentials in 2 sources; connections are by reference", + "status": "passed" + }, + { + "check": "provenance.license_present_where_required", + "message": "license metadata present for required sources", + "status": "passed" + }, + { + "check": "provenance.rationale_present", + "message": "all 2 sources document selection rationale", + "status": "passed" + }, + { + "check": "overrides.every_override_has_provenance", + "message": "no overrides declared (vacuously true)", + "status": "passed" + }, + { + "check": "overrides.evidence_not_placeholder", + "message": "no placeholder evidence found", + "status": "passed" + }, + { + "check": "validation.required_all_present", + "message": "all 2 declared checks present exactly once in report", + "status": "passed" + }, + { + "check": "validation.no_implicit_pass", + "message": "every check has an explicit status", + "status": "passed" + }, + { + "check": "validation.warning_or_failed_propagates_to_status", + "message": "overall status 'passed' correctly reflects check statuses", + "status": "passed" + }, + { + "check": "validation.run_record_matches", + "message": "report run_id 'run-20260901-000000' matches a real run record with consistent hashes", + "status": "passed" + }, + { + "check": "geodata.crs_not_used_for_metrics", + "message": "analysis_crs EPSG:3301 is valid for 0 metric operation(s); storage/load/reprojection steps were excluded", + "status": "passed" + }, + { + "args": { + "path": "data/derived/crossing-trails.geojson" + }, + "check": "geodata.geometry_all_valid", + "message": "data/derived/crossing-trails.geojson: all 3 features have valid geometry", + "status": "passed" + }, + { + "args": { + "expected": "EPSG:3301", + "path": "data/derived/crossing-trails.geojson" + }, + "check": "geodata.dataset_crs_is", + "message": "data/derived/crossing-trails.geojson actual CRS metadata is EPSG:3301", + "status": "passed" + }, + { + "check": "presentation.layers_use_semantic_roles", + "message": "all 1 layers declare a semantic_role", + "status": "passed" + }, + { + "check": "presentation.controls_match_pipeline", + "message": "0 filter control(s) and 0 scenario control(s) consistent", + "status": "passed" + }, + { + "check": "presentation.edit_targets_reference_real_sources", + "message": "no edit targets declared (vacuously true)", + "status": "passed" + }, + { + "check": "metamorphic.declarations_valid", + "message": "1 metamorphic relation(s) are well-formed", + "status": "passed" + }, + { + "args": { + "forbidden_fragments": [], + "id": "trail-order" + }, + "check": "metamorphic.trail-order", + "evidence": { + "id": "trail-order", + "relation": "input_permutation_invariance", + "schema": "openmapstack-metamorphic/v1", + "variant": { + "features": 6, + "seed": 7, + "transformation": "permute_features" + } + }, + "message": "trail-order: input_permutation_invariance holds across 1 output(s)", + "status": "passed" + }, + { + "check": "rerun.no_chat_dependency", + "message": "canonical project dependencies contain no chat/transcript references", + "status": "passed" + } + ], + "counts": { + "failed": 0, + "not_testable": 0, + "passed": 28, + "warning": 0 + }, + "coverage": { + "applicable": 28, + "executed": 28, + "execution_rate": 1.0, + "not_testable": 0 + }, + "project_file": "$PROJECT/project.yaml", + "schema": "openmapstack-verify-result/v1", + "status": "passed" +} diff --git a/tests/goldens/verify/river-crossings.txt b/tests/goldens/verify/river-crossings.txt new file mode 100644 index 0000000..d53c6c8 --- /dev/null +++ b/tests/goldens/verify/river-crossings.txt @@ -0,0 +1,29 @@ +PASS project.parses: project.yaml parses +PASS project.conforms_to_schema: project.yaml conforms to the packaged OpenMapStack v1 JSON Schema +PASS project.graph_resolves: graph resolves: 3 steps, 3 produced symbols, 1 outputs all traced to real steps +PASS project.one_canonical_pipeline: pipeline.py is the canonical implementation; wrappers import it +PASS project.assumptions_have_rationale: all 1 assumptions have statement + rationale +PASS project.status_agrees_with_validation_report: project.status 'validated' agrees with report status 'passed' +PASS project.declared_files_exist: all 1 declared files exist +PASS project.parameters_match_steps: 1 runtime parameter(s) declared; 0 bound to a processing step agree with it +PASS provenance.every_source_has_provider_and_access: all 2 sources declare provider + access method + retrieval timestamp +PASS provenance.every_source_pinned: all 2 sources are pinned (version_identity) +PASS provenance.no_inline_credentials: no inline credentials in 2 sources; connections are by reference +PASS provenance.license_present_where_required: license metadata present for required sources +PASS provenance.rationale_present: all 2 sources document selection rationale +PASS overrides.every_override_has_provenance: no overrides declared (vacuously true) +PASS overrides.evidence_not_placeholder: no placeholder evidence found +PASS validation.required_all_present: all 2 declared checks present exactly once in report +PASS validation.no_implicit_pass: every check has an explicit status +PASS validation.warning_or_failed_propagates_to_status: overall status 'passed' correctly reflects check statuses +PASS validation.run_record_matches: report run_id 'run-20260901-000000' matches a real run record with consistent hashes +PASS geodata.crs_not_used_for_metrics: analysis_crs EPSG:3301 is valid for 0 metric operation(s); storage/load/reprojection steps were excluded +PASS geodata.geometry_all_valid [data/derived/crossing-trails.geojson]: data/derived/crossing-trails.geojson: all 3 features have valid geometry +PASS geodata.dataset_crs_is [data/derived/crossing-trails.geojson]: data/derived/crossing-trails.geojson actual CRS metadata is EPSG:3301 +PASS presentation.layers_use_semantic_roles: all 1 layers declare a semantic_role +PASS presentation.controls_match_pipeline: 0 filter control(s) and 0 scenario control(s) consistent +PASS presentation.edit_targets_reference_real_sources: no edit targets declared (vacuously true) +PASS metamorphic.declarations_valid: 1 metamorphic relation(s) are well-formed +PASS metamorphic.trail-order: trail-order: input_permutation_invariance holds across 1 output(s) +PASS rerun.no_chat_dependency: canonical project dependencies contain no chat/transcript references +PASSED: $PROJECT/project.yaml (28 passed, 0 warnings, 0 not testable, 0 failed; 28/28 applicable checks executed) diff --git a/tests/test_evals.py b/tests/test_evals.py index c564060..c4291e6 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -993,7 +993,7 @@ def test_arm_flags_must_agree_and_default_to_oms(self) -> None: def test_export_tasks_writes_vendor_neutral_bundles(self) -> None: fixture = self.root / "fixture.geojson" fixture.write_text('{"type": "FeatureCollection", "features": []}', encoding="utf-8") - case_dir = self.write_case("070-underspecified-prompt", modes=["live"], live_fixtures=[ + self.write_case("070-underspecified-prompt", modes=["live"], live_fixtures=[ {"source": os.path.relpath(fixture, self.cases_dir / "070-underspecified-prompt"), "destination": "project/data/source/fixture.geojson"}, ]) # fixture sources must stay inside the eval tree for a live run, but export only reads them diff --git a/tests/test_metamorphic.py b/tests/test_metamorphic.py index e72bd7e..1e08162 100644 --- a/tests/test_metamorphic.py +++ b/tests/test_metamorphic.py @@ -12,7 +12,6 @@ import io import json -import os import tempfile import textwrap import unittest diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index 7773254..675f5b9 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -4,7 +4,6 @@ import io import json -import os import unittest from contextlib import redirect_stderr, redirect_stdout from pathlib import Path diff --git a/tests/test_verify_foreign.py b/tests/test_verify_foreign.py new file mode 100644 index 0000000..aeca4de --- /dev/null +++ b/tests/test_verify_foreign.py @@ -0,0 +1,297 @@ +"""`openmapstack verify` on projects whose data this repository has never seen. + +Issue #13's definition of done asks for verify to run against at least two +projects absent from the fixtures and emit stable text and JSON reports. +The two projects here are built from scratch by pure-Python pipelines over +invented geodata (a river-crossing screening and a facility catchment +count); nothing under evals/ is read. The reports are compared against +committed goldens after path and timing normalisation, so an unintended +change in plan composition, ordering, wording, or JSON shape shows up as a +diff rather than passing unnoticed. +""" + +from __future__ import annotations + +import io +import json +import re +import subprocess +import sys +import textwrap +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +import yaml + +from openmapstack.api import validate_verify_result +from openmapstack.cli import main +from tests.evals.helpers import make_workspace + +GOLDEN_DIR = Path(__file__).resolve().parent / "goldens" / "verify" + + +def _crossings_project(root: Path) -> None: + """Project A: which planned trails cross a river (line/line predicate).""" + (root / "data/source").mkdir(parents=True) + trails = { + "type": "FeatureCollection", + "features": [ + {"type": "Feature", "properties": {"trail_id": f"T{i}", "name": f"Trail {i}"}, + "geometry": {"type": "LineString", "coordinates": [[i * 100.0, 0.0], [i * 100.0, 1000.0 if i % 2 else 400.0]]}} + for i in range(1, 7) + ], + } + river = {"type": "FeatureCollection", "features": [ + {"type": "Feature", "properties": {"river_id": "R1"}, + "geometry": {"type": "LineString", "coordinates": [[0.0, 500.0], [1000.0, 500.0]]}} + ]} + (root / "data/source/trails.geojson").write_text(json.dumps(trails), encoding="utf-8") + (root / "data/source/river.geojson").write_text(json.dumps(river), encoding="utf-8") + (root / "pipeline.py").write_text(textwrap.dedent(''' + import json, hashlib, sys + from pathlib import Path + ROOT = Path(__file__).resolve().parent + argv = sys.argv[1:] + crossing_y = float(argv[argv.index("--river-y") + 1]) if "--river-y" in argv else 500.0 + trails = json.loads((ROOT / "data/source/trails.geojson").read_text())["features"] + crossing = [] + for feature in sorted(trails, key=lambda f: f["properties"]["trail_id"]): + (x0, y0), (x1, y1) = feature["geometry"]["coordinates"] + if min(y0, y1) <= crossing_y <= max(y0, y1): + crossing.append({"type": "Feature", "properties": dict(feature["properties"]), "geometry": feature["geometry"]}) + (ROOT / "data/derived").mkdir(exist_ok=True) + out = ROOT / "data/derived/crossing-trails.geojson" + # GeoJSON without a crs member reads as WGS84; the output is in the + # analysis CRS, so say so -- verify cross-checks this against the data. + crs = {"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::3301"}} + out.write_text(json.dumps({"type": "FeatureCollection", "crs": crs, "features": crossing})) + def h(p): + return "sha256:" + hashlib.sha256(p.read_bytes()).hexdigest() + inputs = sorted(p for p in (ROOT / "data/source").rglob("*") if p.is_file()) + [ROOT / "pipeline.py"] + def agg(paths): + d = hashlib.sha256() + for p in sorted(paths, key=lambda p: p.relative_to(ROOT).as_posix()): + rel = p.relative_to(ROOT).as_posix().encode() + d.update(len(rel).to_bytes(8, "big")); d.update(rel); d.update(p.read_bytes()) + return "sha256:" + d.hexdigest() + report = {"run_id": "run-20260901-000000", "status": "passed", "checks": [ + {"id": "geometry_valid", "status": "passed", "features_checked": len(crossing)}, + {"id": "manifest_graph_resolves", "status": "passed"}, + ], "inputs_hash": agg(inputs), "outputs_hash": agg([out])} + (ROOT / "validation").mkdir(exist_ok=True) + (ROOT / "validation/latest-report.json").write_text(json.dumps(report, indent=2)) + (ROOT / "runs").mkdir(exist_ok=True) + (ROOT / "runs/run-20260901-000000.json").write_text(json.dumps({ + "run_id": "run-20260901-000000", "started_at": "2026-09-01T00:00:00Z", "completed_at": "2026-09-01T00:00:01Z", + "status": "passed", "inputs_hash": report["inputs_hash"], "outputs_hash": report["outputs_hash"], + "inputs": [{"path": p.relative_to(ROOT).as_posix(), "sha256": h(p)} for p in inputs], + "outputs": [{"path": out.relative_to(ROOT).as_posix(), "sha256": h(out)}], + }, indent=2)) + ''').lstrip(), encoding="utf-8") + subprocess.run([sys.executable, "pipeline.py"], cwd=root, check=True) + report = json.loads((root / "validation/latest-report.json").read_text()) + manifest = { + "schema": "openmapstack-project/v1", + "project": {"id": "river-crossings", "title": "Planned trails crossing the river", "question": "Which planned trails cross the river?", + "created_at": "2026-09-01T00:00:00Z", "updated_at": "2026-09-01T00:00:00Z", "status": "validated"}, + "interpretation": {"objective": "Select trails whose line intersects the river centreline.", + "assumptions": [{"id": "A1", "statement": "Crossing means the trail line intersects the river line.", "rationale": "No bridge data is available."}]}, + "sources": { + "trails": {"role": "authoritative_input", "provider": "Trail planning office", "dataset": "planned trails", "source_url": "https://example.invalid/trails", + "access": {"method": "local", "retrieved_at": "2026-09-01T00:00:00Z"}, "version": {"identifier": "plan-2026-09", "published_at": "2026-09-01"}, + "selection": {"filter": "all"}, "license": {"name": "CC BY 4.0", "url": "https://example.invalid/license"}, "rationale": "Only planning dataset available."}, + "river": {"role": "context", "provider": "Hydrology agency", "dataset": "river centreline", "source_url": "https://example.invalid/river", + "access": {"method": "local", "retrieved_at": "2026-09-01T00:00:00Z"}, "version": {"identifier": "hydro-2026", "published_at": "2026-08-01"}, + "selection": {"filter": "river_id = 'R1'"}, "license": {"name": "CC BY 4.0", "url": "https://example.invalid/license"}, "rationale": "Authoritative centreline."}, + }, + "overrides": [], + "processing": {"analysis_crs": "EPSG:3301", "storage_crs": "EPSG:3301", "steps": [ + {"id": "load_trails", "operation": "read", "source": "trails", "output": "trails_raw"}, + {"id": "load_river", "operation": "read", "source": "river", "output": "river_raw"}, + {"id": "select_crossings", "operation": "intersects_filter", "input": "trails_raw", "target": "river_raw", "crs": "EPSG:3301", "output": "crossing_trails"}, + ]}, + "outputs": {"crossing_trails": {"path": "data/derived/crossing-trails.geojson", "format": "GeoJSON (EPSG:3301)", "generated_by": "select_crossings"}}, + "validation": {"required": ["geometry_valid", "manifest_graph_resolves"], "domain_checks": [], + "metamorphic": [{"id": "trail-order", "relation": "input_permutation_invariance", "source": {"path": "data/source/trails.geojson"}, + "outputs": ["crossing_trails"], "key": "trail_id", "preconditions": {"tie_break": "keyed by trail_id; sorted output"}}]}, + "presentation": {"intent": "report", "primary_view": "report", "layout": {"type": "report"}, + "map": {"engine_preference": "maplibre", "basemap": {"id": "osm-standard", "kind": "raster-xyz", "tiles": ["https://tile.openstreetmap.org/{z}/{x}/{y}.png"], "attribution": "© OpenStreetMap contributors"}, + "layer_groups": [{"id": "analysis", "title": "Analysis"}], + "layers": [{"source": "crossing_trails", "group": "analysis", "semantic_role": "primary_result", "geometry": "line"}]}}, + "warnings": [], + "runtime": {"implementation": {"preferred_engine": "python", "pipeline": "pipeline.py", + "parameters": [{"id": "river_y", "type": "number", "canonical": 500, "binding": {"argument": "--river-y"}}]}, + "environment": {"python": "3.12"}}, + "runs": {"latest": {"id": "run-20260901-000000", "started_at": "2026-09-01T00:00:00Z", "completed_at": "2026-09-01T00:00:01Z", "status": "passed", + "inputs_hash": report["inputs_hash"], "outputs_hash": report["outputs_hash"], "validation_report": {"path": "validation/latest-report.json"}}}, + } + (root / "project.yaml").write_text(yaml.safe_dump(manifest, sort_keys=False), encoding="utf-8") + + +def _catchments_project(root: Path) -> None: + """Project B: facilities per district (point-in-polygon count), with an + unverified expectation and an override, exercising more of the plan.""" + (root / "data/source").mkdir(parents=True) + (root / "data/overrides").mkdir() + districts = {"type": "FeatureCollection", "features": [ + {"type": "Feature", "properties": {"district_id": "D1"}, "geometry": {"type": "Polygon", "coordinates": [[[0, 0], [500, 0], [500, 500], [0, 500], [0, 0]]]}}, + {"type": "Feature", "properties": {"district_id": "D2"}, "geometry": {"type": "Polygon", "coordinates": [[[500, 0], [1000, 0], [1000, 500], [500, 500], [500, 0]]]}}, + ]} + facilities = {"type": "FeatureCollection", "features": [ + {"type": "Feature", "properties": {"facility_id": f"F{i}", "status": "open"}, "geometry": {"type": "Point", "coordinates": [x, 250.0]}} + for i, x in enumerate((100.0, 200.0, 700.0, 900.0, 950.0), start=1) + ]} + (root / "data/source/districts.geojson").write_text(json.dumps(districts), encoding="utf-8") + (root / "data/source/facilities.geojson").write_text(json.dumps(facilities), encoding="utf-8") + (root / "pipeline.py").write_text(textwrap.dedent(''' + import json, hashlib + from pathlib import Path + ROOT = Path(__file__).resolve().parent + districts = json.loads((ROOT / "data/source/districts.geojson").read_text())["features"] + facilities = json.loads((ROOT / "data/source/facilities.geojson").read_text())["features"] + closed = {"F5"} # OVERRIDE-001 + counts = [] + for d in sorted(districts, key=lambda f: f["properties"]["district_id"]): + xs = [pt[0] for pt in d["geometry"]["coordinates"][0]] + n = sum(1 for f in facilities if f["properties"]["facility_id"] not in closed and min(xs) <= f["geometry"]["coordinates"][0] < max(xs)) + counts.append({"type": "Feature", "properties": {"district_id": d["properties"]["district_id"], "facility_count": n}, "geometry": d["geometry"]}) + (ROOT / "data/derived").mkdir(exist_ok=True) + out = ROOT / "data/derived/district-counts.geojson" + out.write_text(json.dumps({"type": "FeatureCollection", "features": counts})) + def h(p): + return "sha256:" + hashlib.sha256(p.read_bytes()).hexdigest() + inputs = sorted(p for d in ("data/source", "data/overrides") for p in (ROOT / d).rglob("*") if p.is_file()) + [ROOT / "pipeline.py"] + def agg(paths): + d = hashlib.sha256() + for p in sorted(paths, key=lambda p: p.relative_to(ROOT).as_posix()): + rel = p.relative_to(ROOT).as_posix().encode() + d.update(len(rel).to_bytes(8, "big")); d.update(rel); d.update(p.read_bytes()) + return "sha256:" + d.hexdigest() + report = {"run_id": "run-20260901-000001", "status": "warning", "checks": [ + {"id": "geometry_valid", "status": "passed", "features_checked": 2}, + {"id": "overrides_applied", "status": "passed", "results": [{"id": "OVERRIDE-001", "status": "applied"}]}, + {"id": "facility_completeness", "status": "warning", "reason": "no completeness baseline"}, + ], "inputs_hash": agg(inputs), "outputs_hash": agg([out])} + (ROOT / "validation").mkdir(exist_ok=True) + (ROOT / "validation/latest-report.json").write_text(json.dumps(report, indent=2)) + (ROOT / "runs").mkdir(exist_ok=True) + (ROOT / "runs/run-20260901-000001.json").write_text(json.dumps({ + "run_id": "run-20260901-000001", "started_at": "2026-09-01T00:00:00Z", "completed_at": "2026-09-01T00:00:01Z", + "status": "warning", "inputs_hash": report["inputs_hash"], "outputs_hash": report["outputs_hash"], + "inputs": [{"path": p.relative_to(ROOT).as_posix(), "sha256": h(p)} for p in inputs], + "outputs": [{"path": out.relative_to(ROOT).as_posix(), "sha256": h(out)}], + }, indent=2)) + ''').lstrip(), encoding="utf-8") + (root / "data/overrides/closures.json").write_text(json.dumps({"closed": ["F5"]}), encoding="utf-8") + subprocess.run([sys.executable, "pipeline.py"], cwd=root, check=True) + report = json.loads((root / "validation/latest-report.json").read_text()) + manifest = { + "schema": "openmapstack-project/v1", + "project": {"id": "district-facilities", "title": "Open facilities per district", "question": "How many open facilities does each district have?", + "created_at": "2026-09-01T00:00:00Z", "updated_at": "2026-09-01T00:00:00Z", "status": "warning"}, + "interpretation": {"objective": "Count open facilities inside each district polygon.", + "assumptions": [{"id": "A1", "statement": "A facility on the shared boundary belongs to the western district.", "rationale": "Half-open interval avoids double counting."}]}, + "sources": { + "districts": {"role": "authoritative_input", "provider": "City", "dataset": "districts", "source_url": "https://example.invalid/districts", + "access": {"method": "local", "retrieved_at": "2026-09-01T00:00:00Z"}, "version": {"identifier": "districts-2026", "published_at": "2026-01-01"}, + "selection": {"filter": "all"}, "license": {"name": "CC0", "url": "https://example.invalid/cc0"}, "rationale": "Official boundaries."}, + "facilities": {"role": "authoritative_input", "provider": "City", "dataset": "facilities", "source_url": "https://example.invalid/facilities", + "access": {"method": "local", "retrieved_at": "2026-09-01T00:00:00Z"}, "version": {"identifier": "facilities-2026-08", "published_at": "2026-08-01"}, + "selection": {"filter": "status = 'open'"}, "license": {"name": "CC0", "url": "https://example.invalid/cc0"}, "rationale": "Official register."}, + }, + "overrides": [{"id": "OVERRIDE-001", "action": "modify_attribute", "target": {"source": "facilities", "feature_id": "F5"}, + "change": {"field": "status", "from": "open", "to": "closed"}, "rationale": "Closed after the register was published.", + "evidence": [{"type": "url", "value": "https://example.invalid/notice/F5"}], "created_at": "2026-09-01T00:00:00Z", "created_by": "analyst"}], + "processing": {"analysis_crs": "EPSG:3301", "storage_crs": "EPSG:3301", "steps": [ + {"id": "load_districts", "operation": "read", "source": "districts", "output": "districts_raw"}, + {"id": "load_facilities", "operation": "read", "source": "facilities", "output": "facilities_raw"}, + {"id": "apply_closures", "operation": "apply_override", "input": "facilities_raw", "override": "OVERRIDE-001", "output": "facilities_effective"}, + {"id": "count_per_district", "operation": "point_in_polygon_count", "inputs": ["districts_raw", "facilities_effective"], "crs": "EPSG:3301", "output": "district_counts"}, + ]}, + "outputs": {"district_counts": {"path": "data/derived/district-counts.geojson", "format": "GeoJSON", "generated_by": "count_per_district"}}, + "validation": {"required": ["geometry_valid", "overrides_applied"], "domain_checks": [{"name": "facility_completeness", "expression": "count >= 0"}], + "expectations": [{"id": "d1-count", "check": "geodata.feature_field_equals", + "args": {"path": "data/derived/district-counts.geojson", "id_field": "district_id", "id": "D1", "field": "facility_count", "equals": 2}, + "attestation": {"status": "unverified", "reason": "awaiting register comparison"}}]}, + "presentation": {"intent": "analytical_workspace", "primary_view": "map", "layout": {"type": "map_with_sidebar"}, + "map": {"engine_preference": "maplibre", "basemap": {"id": "osm-standard", "kind": "raster-xyz", "tiles": ["https://tile.openstreetmap.org/{z}/{x}/{y}.png"], "attribution": "© OpenStreetMap contributors"}, + "layer_groups": [{"id": "analysis", "title": "Analysis"}, {"id": "user_overrides", "title": "Corrections"}], + "layers": [{"source": "district_counts", "group": "analysis", "semantic_role": "primary_result", "geometry": "polygon"}, + {"source": "facilities", "group": "user_overrides", "semantic_role": "user_override", "geometry": "point"}]}}, + "warnings": [{"id": "DATA-001", "severity": "medium", "layer": "facilities", "issue": "completeness_unknown", "statement": "The register may omit facilities.", "mitigation": "Verify before decisions."}], + "runtime": {"implementation": {"preferred_engine": "python", "pipeline": "pipeline.py"}, "environment": {"python": "3.12"}}, + "runs": {"latest": {"id": "run-20260901-000001", "started_at": "2026-09-01T00:00:00Z", "completed_at": "2026-09-01T00:00:01Z", "status": "warning", + "inputs_hash": report["inputs_hash"], "outputs_hash": report["outputs_hash"], "validation_report": {"path": "validation/latest-report.json"}}}, + } + (root / "project.yaml").write_text(yaml.safe_dump(manifest, sort_keys=False, allow_unicode=True), encoding="utf-8") + + +def _normalise_text(text: str, root: Path) -> str: + return text.replace(str(root), "$PROJECT") + + +def _normalise_json(payload: dict, root: Path) -> dict: + text = json.dumps(payload, sort_keys=True) + text = text.replace(str(root), "$PROJECT") + payload = json.loads(text) + for check in payload.get("checks", []): + evidence = check.get("evidence") + if isinstance(evidence, dict): + for volatile in ("duration_s", "removed_environment_keys", "command", "preserved_paths"): + evidence.pop(volatile, None) + for volatile in ("expected_expectation_sha256", "current_inputs_hash"): + if volatile in evidence: + evidence[volatile] = "$DIGEST" + if "message" in check: + check["message"] = re.sub(r"sha256:[0-9a-f]{64}", "sha256:$DIGEST", check["message"]) + return payload + + +class ForeignProjectVerifyTests(unittest.TestCase): + maxDiff = None + + def _run(self, name: str, builder) -> tuple[str, dict]: + root = make_workspace() / name + root.mkdir() + builder(root) + text = io.StringIO() + with redirect_stdout(text): + main(["verify", str(root / "project.yaml"), "--verbose", "--metamorphic"]) + json_out = io.StringIO() + with redirect_stdout(json_out): + main(["verify", str(root / "project.yaml"), "--json", "--metamorphic"]) + payload = json.loads(json_out.getvalue()) + self.assertEqual(validate_verify_result(payload), []) + return _normalise_text(text.getvalue(), root), _normalise_json(payload, root) + + def _assert_golden(self, name: str, text: str, payload: dict) -> None: + golden_text = GOLDEN_DIR / f"{name}.txt" + golden_json = GOLDEN_DIR / f"{name}.json" + if not golden_text.exists() or not golden_json.exists(): + GOLDEN_DIR.mkdir(parents=True, exist_ok=True) + golden_text.write_text(text, encoding="utf-8") + golden_json.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + self.fail(f"goldens for {name} were missing and have been written; re-run to compare") + self.assertEqual(text, golden_text.read_text(encoding="utf-8")) + self.assertEqual(payload, json.loads(golden_json.read_text(encoding="utf-8"))) + + def test_river_crossings_project_reports_stably(self) -> None: + text, payload = self._run("river-crossings", _crossings_project) + statuses = {check["check"]: check["status"] for check in payload["checks"]} + self.assertEqual(statuses["metamorphic.trail-order"], "passed") + self.assertEqual(statuses["project.parameters_match_steps"], "passed") + self.assertEqual(statuses["validation.run_record_matches"], "passed") + self._assert_golden("river-crossings", text, payload) + + def test_district_facilities_project_reports_stably(self) -> None: + text, payload = self._run("district-facilities", _catchments_project) + statuses = {check["check"]: check["status"] for check in payload["checks"]} + self.assertEqual(statuses["expectation.d1-count"], "warning") + self.assertEqual(statuses["validation.warning_or_failed_propagates_to_status"], "passed") + self.assertEqual(payload["status"], "warning") + self._assert_golden("district-facilities", text, payload) + + +if __name__ == "__main__": + unittest.main() From 5af4cdaa7b309378c7723ff5b3d6f25125d0daa2 Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 2 Sep 2026 17:32:57 +0000 Subject: [PATCH 7/8] fix: address review findings on snapshots, provenance, and credential references - PostGIS NUMERIC is materialised as DECIMAL with the widest precision and scale present, or as exact text beyond DECIMAL(38); a DOUBLE mapping rounded identifiers and high-precision measurements inside a snapshot that was then hashed and pinned as immutable. Live and unit tests round trip numeric(24, 9) exactly. - The arm's task-set hash covers only live-capable cases; fixture-only cases in the selection are skipped in live mode and no longer inflate the workload identity. - Arm provenance records the adapter and model that actually ran, taken from the normalized trial records, rather than the CLI override and the requested alias. - The skill snapshot content hash is the historical raw-byte algorithm (path, NUL, bytes, NUL), so an unchanged skill keeps the hash recorded in earlier benchmark arms; a test pins the algorithm. - file: connection references must be absolute and outside the project at validation time, matching the connector, so preflight cannot accept a reference every source operation refuses. Refs #13, PR #21 review. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01TMMy6A73zi7yGjK3wHBhju --- .github/workflows/eval-warehouse.yml | 8 ++-- evals/run.py | 11 ++++++ openmapstack/connectors/postgis.py | 57 ++++++++++++++++++++++++++-- openmapstack/snapshot.py | 31 +++++++-------- openmapstack/sources.py | 13 +++++-- tests/test_connectors.py | 49 +++++++++++++++++++++++- tests/test_evals.py | 14 ++++++- tests/test_snapshot.py | 14 +++++++ tests/test_sources.py | 5 ++- 9 files changed, 171 insertions(+), 31 deletions(-) diff --git a/.github/workflows/eval-warehouse.yml b/.github/workflows/eval-warehouse.yml index 552fdeb..110628d 100644 --- a/.github/workflows/eval-warehouse.yml +++ b/.github/workflows/eval-warehouse.yml @@ -54,11 +54,11 @@ jobs: sudo apt-get update -q && sudo apt-get install -y -q postgresql-client psql -h 127.0.0.1 -U postgres -d gis -v ON_ERROR_STOP=1 <<'SQL' CREATE EXTENSION IF NOT EXISTS postgis; - CREATE TABLE public.parcels (cadastral_id text primary key, land_use text, geom geometry(Polygon, 3301)); + CREATE TABLE public.parcels (cadastral_id text primary key, land_use text, area_m2 numeric(24, 9), geom geometry(Polygon, 3301)); INSERT INTO public.parcels VALUES - ('P1','ARIMAA', ST_GeomFromText('POLYGON((660100 6466500,660200 6466500,660200 6466600,660100 6466600,660100 6466500))',3301)), - ('P2','ARIMAA', ST_GeomFromText('POLYGON((660300 6466500,660400 6466500,660400 6466600,660300 6466600,660300 6466500))',3301)), - ('P3','TOOTMISMAA', ST_GeomFromText('POLYGON((660500 6466500,660600 6466500,660600 6466600,660500 6466600,660500 6466500))',3301)); + ('P1','ARIMAA', 10000.123456789, ST_GeomFromText('POLYGON((660100 6466500,660200 6466500,660200 6466600,660100 6466600,660100 6466500))',3301)), + ('P2','ARIMAA', 10000.000000001, ST_GeomFromText('POLYGON((660300 6466500,660400 6466500,660400 6466600,660300 6466600,660300 6466500))',3301)), + ('P3','TOOTMISMAA', 9999.999999999, ST_GeomFromText('POLYGON((660500 6466500,660600 6466500,660600 6466600,660500 6466600,660500 6466500))',3301)); CREATE ROLE reader LOGIN PASSWORD 'ci-only-reader'; GRANT CONNECT ON DATABASE gis TO reader; GRANT USAGE ON SCHEMA public TO reader; diff --git a/evals/run.py b/evals/run.py index 2d5f1a9..3f54541 100755 --- a/evals/run.py +++ b/evals/run.py @@ -645,6 +645,10 @@ def _task_set_hash(case_dirs: list[Path]) -> dict[str, Any]: ids: list[str] = [] for case_dir in sorted(case_dirs): case_def = _load_case(case_dir) + if "live" not in case_def["modes"]: + # A fixture-only case is skipped in live mode; it is not part of + # the workload an arm's provenance claims to identify. + continue ids.append(case_def.get("id", case_dir.name)) for relative in ("expected.yaml", case_def.get("live", {}).get("prompt_file", "prompt.md")): path = case_dir / relative @@ -689,6 +693,13 @@ def _arm_record( agent_runs = [result.get("agent_run") for result in trial_results if isinstance(result.get("agent_run"), dict)] agent_version = next((run.get("version") for run in agent_runs if run.get("version")), None) metadata = next((run.get("metadata") or {} for run in agent_runs), {}) + # Prefer what actually ran over what was requested: the adapter may be + # resolved per case when --agent is omitted, and an adapter may report + # the model it observed rather than the alias it was asked for. + observed_agents = sorted({run.get("agent") for run in agent_runs if run.get("agent")}) + observed_models = sorted({run.get("model") for run in agent_runs if run.get("model")}) + agent_name = observed_agents[0] if len(observed_agents) == 1 else (agent_name or (",".join(observed_agents) or None)) + model = observed_models[0] if len(observed_models) == 1 else (model or (",".join(observed_models) or None)) sampling = { "seed": seed, "temperature": metadata.get("temperature"), diff --git a/openmapstack/connectors/postgis.py b/openmapstack/connectors/postgis.py index 1e3afa7..b7cce94 100644 --- a/openmapstack/connectors/postgis.py +++ b/openmapstack/connectors/postgis.py @@ -192,22 +192,62 @@ def materialize(self, query: str, destination: Path, limits: ConnectorLimits) -> return len(rows) +def _decimal_type(values) -> str: + """A DuckDB type that holds every NUMERIC value exactly. + + ``numeric`` has arbitrary precision in PostgreSQL. Mapping it to DOUBLE + would round identifiers and high-precision measurements while the file + is hashed and pinned as the immutable source -- the snapshot would then + be reproducible and wrong. Infer the widest scale and precision present + and use DECIMAL; beyond DECIMAL(38) keep the exact text instead. + """ + from decimal import Decimal + + max_scale = 0 + max_integer_digits = 1 + for value in values: + if value is None: + continue + if not isinstance(value, Decimal): + value = Decimal(str(value)) + if not value.is_finite(): + return "VARCHAR" + sign, digits, exponent = value.as_tuple() + scale = max(0, -exponent) + integer_digits = max(1, len(digits) + exponent) + max_scale = max(max_scale, scale) + max_integer_digits = max(max_integer_digits, integer_digits) + precision = max_integer_digits + max_scale + if precision > 38: + return "VARCHAR" + return f"DECIMAL({precision}, {max_scale})" + + def _write_parquet(duck, destination: Path, columns, geometry_columns, srids, rows) -> None: duck_types = { "int2": "SMALLINT", "int4": "INTEGER", "int8": "BIGINT", "float4": "FLOAT", "float8": "DOUBLE", - "numeric": "DOUBLE", "bool": "BOOLEAN", "date": "DATE", "timestamp": "TIMESTAMP", "timestamptz": "TIMESTAMPTZ", + "bool": "BOOLEAN", "date": "DATE", "timestamp": "TIMESTAMP", "timestamptz": "TIMESTAMPTZ", "json": "JSON", "jsonb": "JSON", "uuid": "UUID", } definitions = [] - for column in columns: + exact_text_columns: set[int] = set() + for index, column in enumerate(columns): if column["name"] in geometry_columns: definitions.append(f'"{column["name"]}" BLOB') + elif column["type"] == "numeric": + decimal_type = _decimal_type(row[index] for row in rows) + if decimal_type == "VARCHAR": + exact_text_columns.add(index) + definitions.append(f'"{column["name"]}" {decimal_type}') else: definitions.append(f'"{column["name"]}" {duck_types.get(column["type"], "VARCHAR")}') duck.execute(f"CREATE TABLE staging ({', '.join(definitions)})") placeholders = ", ".join("?" for _ in columns) if rows: - duck.executemany(f"INSERT INTO staging VALUES ({placeholders})", [tuple(_plain(value) for value in row) for row in rows]) + duck.executemany( + f"INSERT INTO staging VALUES ({placeholders})", + [tuple(_plain(value, exact_text=index in exact_text_columns) for index, value in enumerate(row)) for row in rows], + ) selected = [] for column in columns: name = column["name"] @@ -226,9 +266,18 @@ def _write_parquet(duck, destination: Path, columns, geometry_columns, srids, ro duck.execute(f"COPY (SELECT {', '.join(selected)} FROM staging) TO '{destination.as_posix().replace(chr(39), chr(39) * 2)}' (FORMAT PARQUET)") -def _plain(value: Any) -> Any: +def _plain(value: Any, *, exact_text: bool = False) -> Any: + from decimal import Decimal + if isinstance(value, memoryview): return bytes(value) + if isinstance(value, Decimal): + # DuckDB binds Decimal exactly for DECIMAL columns. For the VARCHAR + # fallback the text must be rendered here: bound as a Decimal it would + # be cast through DOUBLE and lose the digits the fallback exists for. + if exact_text or not value.is_finite(): + return format(value, "f") if value.is_finite() else str(value) + return value if isinstance(value, (dict, list)): import json diff --git a/openmapstack/snapshot.py b/openmapstack/snapshot.py index 6ff75a8..8babef8 100644 --- a/openmapstack/snapshot.py +++ b/openmapstack/snapshot.py @@ -64,15 +64,16 @@ def _iter_skill_files(root: Path) -> list[Path]: return files -def _content_hash(entries: list[dict[str, Any]]) -> str: - """Hash over sorted (path, bytes-hash) pairs; the same algorithm the eval - runner used before this module existed, so historical arm hashes stay - comparable.""" +def _content_hash(files: list[tuple[str, bytes]]) -> str: + """Hash over sorted (relative path, raw bytes) pairs, each terminated by + a NUL: exactly the algorithm the eval runner used before this module + existed, so an unchanged skill keeps the content hash recorded in + historical benchmark arms.""" digest = hashlib.sha256() - for entry in sorted(entries, key=lambda item: item["path"]): - digest.update(entry["path"].encode("utf-8")) + for relative, data in sorted(files, key=lambda item: item[0]): + digest.update(relative.encode("utf-8")) digest.update(b"\0") - digest.update(bytes.fromhex(entry["sha256"].removeprefix("sha256:"))) + digest.update(data) digest.update(b"\0") return f"sha256:{digest.hexdigest()}" @@ -102,12 +103,14 @@ def create_skill_snapshot(source_root: str | Path, destination: str | Path, *, n files = _iter_skill_files(root) target.mkdir(parents=True, exist_ok=True) entries: list[dict[str, Any]] = [] + contents: list[tuple[str, bytes]] = [] for path in files: relative = path.relative_to(root).as_posix() copy_to = target / relative copy_to.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(path, copy_to) data = copy_to.read_bytes() + contents.append((relative, data)) entries.append({"path": relative, "sha256": "sha256:" + hashlib.sha256(data).hexdigest(), "bytes": len(data)}) manifest = { "schema": SNAPSHOT_SCHEMA, @@ -117,7 +120,7 @@ def create_skill_snapshot(source_root: str | Path, destination: str | Path, *, n "entrypoint": SKILL_ENTRYPOINT, "files": entries, "file_count": len(entries), - "content_sha256": _content_hash(entries), + "content_sha256": _content_hash(contents), } (target / SNAPSHOT_MANIFEST).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") return manifest @@ -126,10 +129,7 @@ def create_skill_snapshot(source_root: str | Path, destination: str | Path, *, n def hash_skill_root(source_root: str | Path) -> str: """Content hash of a skill root without copying it.""" root = Path(source_root).resolve() - entries = [] - for path in _iter_skill_files(root): - entries.append({"path": path.relative_to(root).as_posix(), "sha256": "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest()}) - return _content_hash(entries) + return _content_hash([(path.relative_to(root).as_posix(), path.read_bytes()) for path in _iter_skill_files(root)]) def inspect_skill_snapshot(snapshot_dir: str | Path) -> dict[str, Any]: @@ -150,7 +150,7 @@ def inspect_skill_snapshot(snapshot_dir: str | Path) -> dict[str, Any]: raise SnapshotError(f"{SNAPSHOT_MANIFEST} is not an {SNAPSHOT_SCHEMA} document") problems: list[str] = [] seen: set[str] = set() - recomputed: list[dict[str, Any]] = [] + recomputed: list[tuple[str, bytes]] = [] for entry in manifest.get("files") or []: relative = entry.get("path") if isinstance(entry, dict) else None if not isinstance(relative, str) or not relative: @@ -172,8 +172,9 @@ def inspect_skill_snapshot(snapshot_dir: str | Path) -> dict[str, Any]: if not candidate.is_file(): problems.append(f"missing: {relative}") continue - actual = "sha256:" + hashlib.sha256(candidate.read_bytes()).hexdigest() - recomputed.append({"path": relative, "sha256": actual}) + data = candidate.read_bytes() + actual = "sha256:" + hashlib.sha256(data).hexdigest() + recomputed.append((relative, data)) if actual != entry.get("sha256"): problems.append(f"changed: {relative}") for path in sorted(root.rglob("*")): diff --git a/openmapstack/sources.py b/openmapstack/sources.py index 4a3ebc3..11f6da6 100644 --- a/openmapstack/sources.py +++ b/openmapstack/sources.py @@ -225,9 +225,16 @@ def connection_reference_error(root: Path, connection: object) -> str | None: f"({', '.join(f'{item}:' for item in CONNECTION_REFERENCE_SCHEMES)}), not embed a connection string" ) if scheme == "file": - target = project_path(root, remainder.strip()) - if target is not None: - return "access.connection file references must point outside the project directory" + # The same rule the connector applies, so preflight cannot accept a + # reference every source operation will refuse. + candidate = Path(remainder.strip()).expanduser() + if not candidate.is_absolute(): + return "access.connection file references must be absolute paths outside the project" + try: + candidate.resolve().relative_to(root.resolve()) + except ValueError: + return None + return "access.connection file references must point outside the project directory" return None diff --git a/tests/test_connectors.py b/tests/test_connectors.py index 19ff869..4095bdc 100644 --- a/tests/test_connectors.py +++ b/tests/test_connectors.py @@ -189,6 +189,47 @@ def test_unsupported_backend_is_refused_not_guessed(self) -> None: self.assertEqual(caught.exception.code, "backend_unsupported") +@unittest.skipUnless(DUCKDB_AVAILABLE, "DuckDB Spatial is not available") +class NumericMaterialisationTests(unittest.TestCase): + """NUMERIC must survive materialisation exactly; a rounded snapshot that + is then hashed and pinned would be reproducible and wrong.""" + + def _round_trip(self, values): + from decimal import Decimal + + from openmapstack.connectors.postgis import _write_parquet + + duck = connect_spatial() + destination = make_workspace() / "numeric.parquet" + rows = [(f"r{i}", value) for i, value in enumerate(values)] + try: + _write_parquet(duck, destination, [{"name": "id", "type": "text"}, {"name": "amount", "type": "numeric"}], [], {}, rows) + described = duck.execute(f"DESCRIBE SELECT * FROM read_parquet('{destination.as_posix()}')").fetchall() + read = duck.execute(f"SELECT amount FROM read_parquet('{destination.as_posix()}') ORDER BY id").fetchall() + finally: + duck.close() + return {name: str(type_name) for name, type_name, *_ in described}["amount"], [row[0] for row in read] + + def test_high_precision_values_are_exact(self) -> None: + from decimal import Decimal + + values = [Decimal("12345678901234567890.123456789"), Decimal("0.000000001"), None] + type_name, read = self._round_trip(values) + self.assertTrue(type_name.startswith("DECIMAL"), type_name) + self.assertEqual(read[0], values[0]) + self.assertEqual(read[1], values[1]) + self.assertIsNone(read[2]) + self.assertNotEqual(float(values[0]), values[0]) # DOUBLE would have rounded it + + def test_beyond_decimal_38_keeps_exact_text(self) -> None: + from decimal import Decimal + + huge = Decimal("1" * 30 + "." + "2" * 20) + type_name, read = self._round_trip([huge]) + self.assertEqual(type_name, "VARCHAR") + self.assertEqual(Decimal(read[0]), huge) + + class _FakeCursor: """Answers the exact statements the PostGIS connector issues.""" @@ -314,7 +355,7 @@ def test_snapshot_round_trip(self) -> None: discovery = discover_source(project, "test_source", project_root=workspace) self.assertTrue(discovery.read_only) self.assertTrue(any(table.name == "parcels" for table in discovery.tables), discovery.to_dict()) - query = "SELECT cadastral_id, land_use, geom FROM public.parcels ORDER BY cadastral_id" + query = "SELECT cadastral_id, land_use, area_m2, geom FROM public.parcels ORDER BY cadastral_id" dry = snapshot_source(project, "test_source", query, "data/source/parcels.parquet", project_root=workspace) self.assertFalse(dry["materialized"]) record = snapshot_source(project, "test_source", query, "data/source/parcels.parquet", project_root=workspace, approve=True) @@ -325,11 +366,15 @@ def test_snapshot_round_trip(self) -> None: self.assertEqual(assess_pin(workspace, updated["sources"]["test_source"]).status, "pinned") connection = connect_spatial() try: - rows = connection.execute(f"SELECT cadastral_id, ST_GeometryType(geom) FROM read_parquet('{(workspace / 'data/source/parcels.parquet').as_posix()}') ORDER BY 1").fetchall() + rows = connection.execute(f"SELECT cadastral_id, ST_GeometryType(geom), area_m2 FROM read_parquet('{(workspace / 'data/source/parcels.parquet').as_posix()}') ORDER BY 1").fetchall() finally: connection.close() self.assertEqual([row[0] for row in rows], ["P1", "P2", "P3"]) self.assertTrue(all("POLYGON" in str(row[1]).upper() for row in rows), rows) + from decimal import Decimal + + # NUMERIC(24, 9) survives exactly; a DOUBLE mapping would have rounded it. + self.assertEqual(rows[0][2], Decimal("10000.123456789")) # A write attempt through the same reference must be refused by the policy. with self.assertRaises(ConnectorError): snapshot_source(project, "test_source", "DELETE FROM public.parcels", "data/source/x.parquet", project_root=workspace, approve=True) diff --git a/tests/test_evals.py b/tests/test_evals.py index c4291e6..0670779 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -934,7 +934,7 @@ def run(prompt, workspace, fixture=None, timeout_s=900, model=None, seed=None): seen.append(("oms" if skill_present else "plain", seed)) (workspace / "marker.txt").write_text("ok\n", encoding="utf-8") return AgentRunResult( - agent="codex", model=model, workspace=workspace, duration_s=0.1, success=True, returncode=0, + agent="codex", model="observed-model", workspace=workspace, duration_s=0.1, success=True, returncode=0, command=["fake-agent"], version="fake 2", events=[{"type": "x"}] * (3 if skill_present else 5), usage={"total_tokens": 10 if skill_present else 20}, cost_usd=0.02 if skill_present else 0.01, permissions={}, metadata={"structured_completion": True, "temperature": 0.0}, @@ -944,6 +944,9 @@ def run(prompt, workspace, fixture=None, timeout_s=900, model=None, seed=None): def test_paired_arms_share_cases_trials_and_seeds_and_are_reported_apart(self) -> None: self.write_case("paired-live", modes=["live"]) + # Present in the selection, skipped in live mode: it must not enter + # the arm's task-set identity. + self.write_case("fixture-only-neighbour") seen: list[tuple[str, int | None]] = [] summary_path = self.root / "paired.json" with patch.object(eval_runner, "_load_adapter", return_value=self._adapter(seen)): @@ -972,6 +975,7 @@ def test_paired_arms_share_cases_trials_and_seeds_and_are_reported_apart(self) - self.assertEqual(provenance["oms"]["tool_surface"], {"adapter": "codex", "agent_version": "fake 2"}) self.assertEqual(provenance["oms"]["sampling"], {"seed": 7, "temperature": 0.0, "reasoning": None}) self.assertEqual(provenance["oms"]["task_set"], provenance["plain"]["task_set"]) + self.assertEqual(provenance["oms"]["task_set"]["cases"], ["paired-live"]) self.assertEqual(provenance["oms"]["checker"]["check_api_version"], "openmapstack-check-api/v1") for arm in ("plain", "oms"): bundle = self.results_dir / "paired-run" / "codex" / arm / "paired-live" / "1" @@ -981,10 +985,16 @@ def test_paired_arms_share_cases_trials_and_seeds_and_are_reported_apart(self) - def test_arm_flags_must_agree_and_default_to_oms(self) -> None: self.write_case("arm-flags", modes=["live"]) seen: list[tuple[str, int | None]] = [] + summary_path = self.root / "arm-flags.json" with patch.object(eval_runner, "_load_adapter", return_value=self._adapter(seen)): - exit_code, _, _ = self.call_main(["--mode", "live", "--agent", "codex", "--model", "m", "--case", "arm-flags", "--no-retain-artifacts"]) + exit_code, _, _ = self.call_main(["--mode", "live", "--model", "requested-alias", "--case", "arm-flags", "--no-retain-artifacts", "--json", str(summary_path)]) self.assertEqual(exit_code, 0) self.assertEqual(seen, [("oms", None)]) + # --agent was omitted and the adapter reported what actually ran: + # provenance records the resolved adapter and observed model. + [record] = json.loads(summary_path.read_text(encoding="utf-8"))["run_config"]["arm_provenance"] + self.assertEqual(record["tool_surface"]["adapter"], "codex") + self.assertEqual(record["model"], {"provider": "openai", "id": "observed-model", "revision": None}) with self.assertRaises(SystemExit): self.call_main(["--mode", "live", "--agent", "codex", "--model", "m", "--skill-mode", "disabled", "--arms", "oms"]) with self.assertRaises(SystemExit): diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py index 675f5b9..988cade 100644 --- a/tests/test_snapshot.py +++ b/tests/test_snapshot.py @@ -53,6 +53,20 @@ def test_content_hash_covers_paths_and_bytes(self) -> None: (root / "references/project-spec.md").rename(root / "references/renamed.md") self.assertNotEqual(before, hash_skill_root(root)) + def test_content_hash_is_the_historical_raw_byte_algorithm(self) -> None: + # Benchmark arms recorded before this module hashed path, NUL, bytes, + # NUL per file in path order; an unchanged skill must keep its hash. + import hashlib + + root = _skill_root() + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file() and item.relative_to(root).parts[0] in {"SKILL.md", "references", "templates"}): + digest.update(path.relative_to(root).as_posix().encode("utf-8")) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + self.assertEqual(hash_skill_root(root), f"sha256:{digest.hexdigest()}") + def test_symlinks_are_refused(self) -> None: root = _skill_root() (root / "references/escape.md").symlink_to("/etc/hostname") diff --git a/tests/test_sources.py b/tests/test_sources.py index 1632755..af1f52f 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -159,8 +159,11 @@ def test_connection_must_be_a_reference(self) -> None: self.assertIsNotNone(connection_reference_error(workspace, "host=db dbname=gis")) self.assertIsNotNone(connection_reference_error(workspace, "postgresql://db/gis")) self.assertIsNotNone(connection_reference_error(workspace, {"ref": "env:"})) - # A secrets file inside the project directory would be committed. + # A secrets file inside the project directory would be committed, and + # a relative path is refused here exactly as the connector refuses it. self.assertIsNotNone(connection_reference_error(workspace, "file:secrets/dsn.txt")) + self.assertIsNotNone(connection_reference_error(workspace, "file:../secrets/postgis.dsn")) + self.assertIsNotNone(connection_reference_error(workspace, f"file:{workspace}/dsn.txt")) self.assertIsNone(connection_reference_error(workspace, "file:/etc/openmapstack/dsn")) def test_provenance_check_and_validate_reject_inline_credentials(self) -> None: From 8590cae64c9b3d4982ca9b98c512f973f743729f Mon Sep 17 00:00:00 2001 From: Jaak Laineste Date: Wed, 2 Sep 2026 18:06:42 +0000 Subject: [PATCH 8/8] release: prepare 0.3.0 The package gains substantial new CLI surface in this release: openmapstack source discover/snapshot, checks, check, api-info, skill-snapshot, and verify --metamorphic, plus the openmapstack-check-api/v1 interface external harnesses consume. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TMMy6A73zi7yGjK3wHBhju --- .claude-plugin/plugin.json | 2 +- openmapstack/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 0f78c50..e465392 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "open-map-stack", "displayName": "OpenMapStack", "description": "Agentic GIS / geospatial workflows: source discovery and provenance (from open data sources first), vector/raster/point-cloud pipelines, CRS and metric analysis, spatial SQL, QGIS projects, tile generation, and web maps — compiled into reproducible projects.", - "version": "0.2.0", + "version": "0.3.0", "author": { "name": "Jaak Laineste", "url": "https://github.com/jaakla" diff --git a/openmapstack/__init__.py b/openmapstack/__init__.py index f9e7313..eb422d0 100644 --- a/openmapstack/__init__.py +++ b/openmapstack/__init__.py @@ -3,4 +3,4 @@ from .validation import Check, ValidationResult, validate_project __all__ = ["Check", "ValidationResult", "validate_project"] -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/pyproject.toml b/pyproject.toml index 286e904..15ffe42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "openmapstack" -version = "0.2.0" +version = "0.3.0" description = "Validate, run, and inspect reproducible OpenMapStack projects" readme = "README.md" requires-python = ">=3.10"