diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 0f78c50..e465392 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "name": "open-map-stack", "displayName": "OpenMapStack", "description": "Agentic GIS / geospatial workflows: source discovery and provenance (from open data sources first), vector/raster/point-cloud pipelines, CRS and metric analysis, spatial SQL, QGIS projects, tile generation, and web maps — compiled into reproducible projects.", - "version": "0.2.0", + "version": "0.3.0", "author": { "name": "Jaak Laineste", "url": "https://github.com/jaakla" diff --git a/.github/workflows/eval-benchmark.yml b/.github/workflows/eval-benchmark.yml index 447a93f..eb5ac71 100644 --- a/.github/workflows/eval-benchmark.yml +++ b/.github/workflows/eval-benchmark.yml @@ -35,6 +35,15 @@ on: cases: description: "Space-separated case ids to run (default: every case that declares live mode)" required: false + arms: + description: "Benchmark arm(s): oms (skill injected), plain (no skill), or paired (both over identical cases, trials, and seeds)" + required: false + default: oms + type: choice + options: [oms, plain, paired] + price_catalog_date: + description: "YYYY-MM-DD of the price list behind the cost estimates (recorded in arm provenance)" + required: false schedule: - cron: "0 3 * * 1" # weekly, Monday 03:00 UTC @@ -120,6 +129,8 @@ jobs: EVAL_TIMEOUT: ${{ github.event.inputs.timeout || '1200' }} EVAL_SEED: ${{ github.event.inputs.seed || '' }} EVAL_CASES: ${{ github.event.inputs.cases || '' }} + EVAL_ARMS: ${{ github.event.inputs.arms || 'oms' }} + EVAL_PRICE_CATALOG_DATE: ${{ github.event.inputs.price_catalog_date || '' }} run: | if [[ "$EVAL_AGENT" == "claude_code" ]]; then EVAL_MODEL="$CLAUDE_MODEL" @@ -136,12 +147,13 @@ jobs: --mode live --agent "$EVAL_AGENT" --model "$EVAL_MODEL" - --skill-mode enabled + --arms "$EVAL_ARMS" --repetitions "$EVAL_REPETITIONS" --timeout "$EVAL_TIMEOUT" --json "eval-benchmark-results-$EVAL_AGENT.json" ) if [[ -n "$EVAL_SEED" ]]; then args+=(--seed "$EVAL_SEED"); fi + if [[ -n "$EVAL_PRICE_CATALOG_DATE" ]]; then args+=(--price-catalog-date "$EVAL_PRICE_CATALOG_DATE"); fi # No --case means every case that declares live mode. The list used # to be hardcoded to 001/002/004/005, which silently left the four # prompt-style cases (070-073, added by PR 8) running in no workflow diff --git a/.github/workflows/eval-warehouse.yml b/.github/workflows/eval-warehouse.yml new file mode 100644 index 0000000..110628d --- /dev/null +++ b/.github/workflows/eval-warehouse.yml @@ -0,0 +1,74 @@ +name: OpenMapStack warehouse connector pilot + +# The PostGIS connector is exercised end to end against a real PostGIS +# service: read-only discovery, dry-run plan, approved GeoParquet snapshot, +# and a pin the contract accepts. Scheduled/manual, and on changes to the +# connector code, so an unrelated PR is never blocked by a service container. +on: + workflow_dispatch: {} + schedule: + - cron: "37 5 * * 1" + pull_request: + paths: + - "openmapstack/connectors/**" + - "openmapstack/sources.py" + - "tests/test_connectors.py" + - ".github/workflows/eval-warehouse.yml" + +jobs: + postgis-connector: + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgis: + image: postgis/postgis:16-3.4 + env: + POSTGRES_PASSWORD: ci-only-password + POSTGRES_DB: gis + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d gis" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install the CLI with geodata and PostGIS support + run: pip install ".[geo,postgis]" + + - name: Prepare controlled DuckDB Spatial directory + run: | + echo "OPENMAPSTACK_SPATIAL_EXTENSION_DIR=${RUNNER_TEMP}/openmapstack-duckdb-extensions" >> "$GITHUB_ENV" + OPENMAPSTACK_SPATIAL_EXTENSION_DIR="${RUNNER_TEMP}/openmapstack-duckdb-extensions" python evals/prepare_spatial.py + + - name: Seed a parcels table and a read-only role + env: + PGPASSWORD: ci-only-password + run: | + sudo apt-get update -q && sudo apt-get install -y -q postgresql-client + psql -h 127.0.0.1 -U postgres -d gis -v ON_ERROR_STOP=1 <<'SQL' + CREATE EXTENSION IF NOT EXISTS postgis; + CREATE TABLE public.parcels (cadastral_id text primary key, land_use text, area_m2 numeric(24, 9), geom geometry(Polygon, 3301)); + INSERT INTO public.parcels VALUES + ('P1','ARIMAA', 10000.123456789, ST_GeomFromText('POLYGON((660100 6466500,660200 6466500,660200 6466600,660100 6466600,660100 6466500))',3301)), + ('P2','ARIMAA', 10000.000000001, ST_GeomFromText('POLYGON((660300 6466500,660400 6466500,660400 6466600,660300 6466600,660300 6466500))',3301)), + ('P3','TOOTMISMAA', 9999.999999999, ST_GeomFromText('POLYGON((660500 6466500,660600 6466500,660600 6466600,660500 6466600,660500 6466500))',3301)); + CREATE ROLE reader LOGIN PASSWORD 'ci-only-reader'; + GRANT CONNECT ON DATABASE gis TO reader; + GRANT USAGE ON SCHEMA public TO reader; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO reader; + ANALYZE public.parcels; + SQL + + - name: Run the connector suite against the live service + env: + # The credential reaches the connector only through this reference; + # the tests assert it never appears in any recorded output. + OPENMAPSTACK_TEST_POSTGIS_DSN: postgresql://reader:ci-only-reader@127.0.0.1:5432/gis + run: python -m unittest -v tests.test_connectors diff --git a/AGENTS.md b/AGENTS.md index a688d4f..11b88c6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,6 +121,10 @@ python3 evals/run.py --mode fixture openmapstack validate examples/tartu-development/project.yaml --preflight openmapstack inspect examples/tartu-development/project.yaml --json +# Check API that external harnesses consume (see docs/openmapbench-interop.md) +openmapstack api-info --json +openmapstack checks + # Coverage gate used by CI python3 -m coverage run -m unittest discover -v python3 -m coverage report diff --git a/README.md b/README.md index a083a00..2c0060f 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ It is open-first and cloud-native by default, built on shoulders of the awesome - [SKILL.md](SKILL.md) — the skill entry point: triggers, global defaults, format and compute decision matrices, anti-patterns, and a quick triage guide. - [references/data-sources.md](references/data-sources.md) - lists OSM, Overture, Sentinel/Landsat, regional portals, STAC catalogs and others. - [references/services-and-scale.md](references/services-and-scale.md) - depending on case use local installs or hosted/SaaS services for global-scale basemaps, elevation, routing, geocoding, place search, and postcodes. +- [references/user-data-sources.md](references/user-data-sources.md) - the user's own warehouse data: credentials by reference, read-only discovery, approval-gated snapshots, and the pin classes that make a warehouse table reproducible. - [references/formats-and-crs.md](references/formats-and-crs.md) - how to choose formats, conversions, projections, EPSG codes. - [references/processing.md](references/processing.md) - when and how to use GDAL/OGR, GeoPandas, xarray, DuckDB, PostGIS, PDAL and other open geo processing tools. - [references/analytics.md](references/analytics.md) — do vector/raster analytics, terrain, hydrology, network, point clouds, geocoding etc. @@ -27,6 +28,7 @@ It is open-first and cloud-native by default, built on shoulders of the awesome - [examples/tartu-development/](examples/tartu-development/) — a fully-worked reproducible project matching the acceptance scenario: source provenance + timestamps, explicit assumptions, two verified project overrides (a scenario attribute change with prior-value verification, and hypothetical scenario geometry), deterministic pipeline, machine-readable validation, and semantic presentation. - [evals/](evals/) — the eval suite grading whether an agent reaches the right analytical answer, respects the GIS-method guardrails, and reruns reproducibly, with the `openmapstack-project/v1` contract as the substrate that makes those independently checkable: `python evals/run.py --mode fixture` runs deterministic, no-LLM checks against real generated artifacts (analytical correctness against known geospatial truth, metric CRS, source immutability, schema, overrides, validation integrity, presentation contract, and clean reruns), plus adversarial cases and a pluggable live-agent benchmark (Claude Code, Codex, and any OpenAI-compatible API such as OpenRouter — URL and model via `OPENAI_COMPATIBLE_*` env, API key as a secret). - [`openmapstack/`](openmapstack/) — the installable `openmapstack validate/run/inspect` CLI for auditing and executing `openmapstack-project/v1` projects, plus [`openmapstack/checks/`](openmapstack/checks/): the reusable, semantic check library. All but five of its checks are oracle-free, so the same functions that grade the eval suite also grade a user's own project on data this repository has never seen. +- [docs/openmapbench-interop.md](docs/openmapbench-interop.md) — the narrow, versioned contract a benchmark harness such as OpenMapBench consumes: `openmapstack checks` / `check` / `api-info` (`openmapstack-check-api/v1`), the packaged result schemas, skill snapshots, arm provenance, and exported task bundles. - [`.claude-plugin/`](.claude-plugin/) — Claude Code plugin and marketplace manifests, so the repository can also be installed with `/plugin install`. Validated in CI by [`.github/workflows/plugin.yml`](.github/workflows/plugin.yml). My local Estonia-specific guidance (Maa- ja Ruumiamet, ETAK, EPSG:3301 / L-EST97) is included for convenience. But all the global sources are incuded for world-wide coverage. @@ -151,6 +153,15 @@ openmapstack run path/to/project.yaml # Review sources, versions, overrides, ordered steps, outputs, and latest run. openmapstack inspect path/to/project.yaml + +# Copy SKILL.md, references/, and templates/ into a hashed, inspectable snapshot. +openmapstack skill-snapshot --out /tmp/oms-skill --json +openmapstack skill-snapshot --inspect /tmp/oms-skill + +# Read-only discovery of a warehouse source, then an approval-gated snapshot. +openmapstack source discover path/to/project.yaml --source parcels +openmapstack source snapshot path/to/project.yaml --source parcels \ + --query "SELECT id, geom FROM cadastre.parcels" --destination data/source/parcels.parquet --approve ``` Useful automation options: @@ -176,6 +187,7 @@ PyQGIS is available. ```bash openmapstack verify path/to/project.yaml openmapstack verify path/to/project.yaml --rerun # + rebuild from source and compare +openmapstack verify path/to/project.yaml --metamorphic # + run declared no-oracle relations openmapstack verify path/to/project.yaml --json --output validation/verify-report.json openmapstack verify path/to/project.yaml --strict # warnings and not-testable also return 1 ``` @@ -217,6 +229,24 @@ inputs, or a retained local evidence file invalidates the attestation and returns it to warning status. See [the project contract](references/project-spec.md#26-validation). +Where no golden answer exists at all, `validation.metamorphic[]` declares +relations that must hold under a controlled perturbation: shuffle a source and +the result must not change, duplicate every feature and a keyed set must not +change, widen an inclusion buffer and no candidate may disappear. Each relation +states the precondition that makes it valid, is executed by +`verify --metamorphic` in an isolated copy against the project's own pipeline, +and reports `not_testable` with the reason when the precondition does not hold +on the actual data. See [the project contract](references/project-spec.md#26-validation). + +`openmapstack source` is the connector pilot for the user's own data +(DuckDB local files and PostGIS). Credentials are referenced, never stored; +discovery is read-only with a statement timeout; a snapshot is a dry run +until `--approve`, is limited by rows and bytes, lands only under +`data/source/`, and hands back the `pin` block that makes the source +reproducible. A warehouse table with only a timestamp is not pinned; an +expired backend snapshot is reported as `not_reproducible`. See +[user data sources](references/user-data-sources.md). + `validate` checks manifest structure, source retrieval/version/licensing data, CRS declarations, processing graph resolution, override provenance and files, output existence, validation-report parity/status propagation, override diff --git a/SKILL.md b/SKILL.md index 1fb2a1e..b7f599b 100644 --- a/SKILL.md +++ b/SKILL.md @@ -60,6 +60,7 @@ Hard rules for every material analysis — each is expanded in `references/proje | If the task involves... | Read | |---|---| | Finding or sourcing data (OSM, Overture, Sentinel, Landsat, building footprints, regional portals, STAC catalogs, MCP-based discovery) | `references/data-sources.md` | +| Reading the user's own warehouse or database (PostGIS, DuckDB, GeoParquet directories): credentials by reference, read-only discovery, approved snapshots, pin classes | `references/user-data-sources.md` | | Choosing local processing vs online/hosted/SaaS services for global or continental scale; basemaps, elevation, routing, geocoding, place search, postcode lookup APIs | `references/services-and-scale.md` | | Choosing a format, converting between formats, or any CRS / projection / EPSG question | `references/formats-and-crs.md` | | Compiling a reproducible GIS project artifact (`project.yaml`, pipeline, overrides, validation, presentation) | `references/project-spec.md` + `templates/` | diff --git a/docs/maintainers/architecture.md b/docs/maintainers/architecture.md index 34b0611..f6ca4be 100644 --- a/docs/maintainers/architecture.md +++ b/docs/maintainers/architecture.md @@ -36,6 +36,7 @@ The distinction matters: `SKILL.md` is the product being developed and evaluated - Project contract: `references/project-spec.md`; machine validation is also constrained by `openmapstack/schemas/project-v1.schema.json` and `openmapstack/validation.py`. - Automatic `verify` plan and applicability: `docs/verify-applicability.md` plus `openmapstack/verify.py`. - Eval semantics: `evals/README.md`, `evals/schemas/`, `evals/run.py`, case definitions, and tests. +- External check consumption: `docs/openmapbench-interop.md` plus `openmapstack/api.py` and the packaged result schemas; reporting dimensions are owned by `openmapstack.api.DIMENSIONS`. - Roadmap/current work: GitHub issues. Do not mirror their checklists here. When these disagree, resolve the inconsistency at the owning layer rather than adding another interpretation here. @@ -110,6 +111,18 @@ This environment cleanup is deliberately **not** a general sandbox or allowlist The eval harness additionally forbids reaching back into eval reference generators. That restriction is supplied by the eval caller; the shipped package intentionally does not know that `evals/` exists. +## Connectors are a trust boundary, like attestations + +`openmapstack/connectors/` reads user warehouse data on the user's behalf and is held to four rules that must survive any refactor: credentials are resolved from a reference and never recorded; sessions are read-only with a statement timeout; only a single `SELECT` reaches the backend; and nothing is materialised under `data/source/` without an explicit approval flag and within declared row/byte limits. Every message the package emits passes through `openmapstack.sources.redact`. + +The DuckDB local connector confines file access to its root (`allowed_directories` + `enable_external_access = false`) and exposes files as views so queries never spell paths. PostGIS has no durable time travel, so its pin is always a local snapshot; the transaction snapshot id is retrieval metadata, not a pin. Unverified backends are refused (`backend_unsupported`) rather than approximated. + +Pin classes live in `openmapstack/sources.py` and are shared by `validate` (`source.pin`, `source.credentials`) and `verify` (`provenance.every_source_pinned`, `provenance.no_inline_credentials`). Do not add a third interpretation. + +## Metamorphic relations execute the project's own pipeline + +`openmapstack/metamorphic.py` reuses the clean-rerun workspace preparation (`openmapstack/rerun.py`), perturbs only the copy, and compares against the produced outputs. A relation is valid only under its declared preconditions; unmet data preconditions are `not_testable`, invalid declarations fail, and unknown relation names are rejected rather than skipped. `runtime.implementation.parameters` (`openmapstack/parameters.py`) is the only sanctioned way to vary a pipeline setting from outside. Keep `metamorphic_evidence` a separate eval dimension from `gis_correctness`: a relation that holds is self-consistency, not a correct answer. + ## Integrity and path safety Project-relative paths are resolved through `openmapstack/project.py` helpers and must remain under the project root. Code that adds new file addressing should reuse the same safety model rather than joining unchecked user paths. diff --git a/docs/maintainers/debugging.md b/docs/maintainers/debugging.md index d650601..6407bd8 100644 --- a/docs/maintainers/debugging.md +++ b/docs/maintainers/debugging.md @@ -113,6 +113,15 @@ When a plausible wrong project from a live/user run survives the current checks, 4. add checker/unit coverage if the defect exposes a checker bug; 5. only then add prose context here if the trap remains worth remembering. +## Metamorphic mutations must live in the pipeline copy, not only in the generator + +A metamorphic relation reruns the project's own `pipeline.py` on perturbed input and compares with the produced outputs. For a fixture mutation that means the *copied* pipeline must reproduce the defect: if only `gen.py --break=` injects it at generation time, the variant run rebuilds the healthy analysis, every relation "fails" for the wrong reason, and the mutation is not isolated. `gen.py` therefore reads the `EVAL-BREAK` warning back in pipeline mode for the pipeline-logic break modes (`order_dependent`, `distance_inverted`, `duplicate_sensitive`) and nothing else. + +Two related traps: + +- a relation's detection power depends on the data and the variant size. Widening the mini-Tartu road threshold by 1.5× cannot expose an inverted predicate because the only far parcel sits at 5450 m; the fixture declares `variant: {multiply: 3}` for that reason. When a mutation survives, check the geometry before suspecting the relation; +- a GeoJSON output without a `crs` member reads back as EPSG:4326. A pipeline that writes analysis-CRS coordinates into plain GeoJSON and declares `EPSG:3301` in the manifest fails `geodata.dataset_crs_is` correctly. Write the `crs` member (or use GeoParquet) rather than relaxing the check. + ## Generated benchmark artifacts are evidence, not source Retained live/visual evidence belongs under `evals/results//...` and CI artifacts. Do not treat generated result JSON, screenshots, event streams, or temporary projects as canonical repository state unless a fixture intentionally owns them. diff --git a/docs/maintainers/decisions/0004-narrow-benchmark-interface.md b/docs/maintainers/decisions/0004-narrow-benchmark-interface.md new file mode 100644 index 0000000..1ae5f77 --- /dev/null +++ b/docs/maintainers/decisions/0004-narrow-benchmark-interface.md @@ -0,0 +1,37 @@ +# 0004 — A narrow versioned check API instead of an exported benchmark harness + +- Status: Accepted +- Date: 2026-09-02 +- Related: issue #13 (C1–C3); `openmapstack/api.py`; `openmapstack/snapshot.py`; `docs/openmapbench-interop.md`; `evals/run.py` + +## Context + +OpenMapBench needs to grade produced projects with the same checks this repository uses, record which skill snapshot and model configuration produced each result, and compare a plain arm with a skill arm. Two easy paths were available: let OpenMapBench vendor `openmapstack/checks/`, or export `evals/run.py` as the benchmark harness. Both couple the user-facing package to model-provider orchestration and let the two check libraries drift. + +## Decision + +Publish a small, versioned interface and nothing more: + +- `openmapstack-check-api/v1` (`api_info`, `negotiate`, `list_checks`, `run_check`, `verify --json`) with packaged JSON schemas for results; +- `openmapstack-skill-snapshot/v1` for the controlled copy of the shipped skill; +- `openmapstack-benchmark-arm/v1` as the provenance tuple a published arm must record; +- `openmapstack-benchmark-task/v1` bundles exported from the eval cases. + +Reporting dimensions are owned by `openmapstack.api.DIMENSIONS` and imported by the eval runner. Agent adapters, leaderboard policy, and run isolation remain in OpenMapBench. + +## Consequences + +- OpenMapBench upgrades by pinning a check API major and a minimum package version; `negotiate()` refuses to grade with an unknown API rather than grading wrongly. +- New checks are additive; renaming or removing one is a major bump with a migration note. +- Setup failures (`check_error`, adapter failure, missing CLI) stay outside scored denominators on both sides. +- Paired `plain`/`oms` runs in this repository exist to prove task parity and provenance recording; the public comparison is OpenMapBench's. + +## Alternatives considered + +### Vendor the check library into OpenMapBench + +Rejected: two copies of the same predicate drift, and a benchmark grading with a stale copy silently rewards a project the shipped verifier would fail. + +### Export `evals/run.py` as the benchmark harness + +Rejected: the runner's adapters, credential handling, and artifact layout are provider-specific compatibility surfaces. Making them the generic architecture would leak vendor terminology into shared interfaces and tie package releases to harness releases. diff --git a/docs/openmapbench-interop.md b/docs/openmapbench-interop.md new file mode 100644 index 0000000..d9af4fd --- /dev/null +++ b/docs/openmapbench-interop.md @@ -0,0 +1,97 @@ +# OpenMapBench interoperability contract + +OpenMapBench owns benchmark orchestration, run isolation, provider adapters, +leaderboard policy, and task governance. OpenMapStack owns the project +contract and the checks that grade a produced project. This document is the +narrow surface between them. Neither side copies the other's implementation: +OpenMapBench consumes a released `openmapstack` package through the API +below; OpenMapStack does not export a second benchmark harness. + +Owning code: `openmapstack/api.py`, `openmapstack/schemas/`, +`openmapstack/snapshot.py`, `evals/schemas/benchmark-arm-v1.schema.json`. +The consumer fixture that proves the contract without vendoring a check is +`tests/test_check_api.py::ConsumerFixtureTests`. + +## 1. Versioned check API — `openmapstack-check-api/v1` + +| Surface | Purpose | +|---|---| +| `openmapstack api-info --json` / `openmapstack.api.api_info()` | package version, check API version, project schema, result schemas, status vocabulary, dimensions | +| `openmapstack api-info --require-api … --min-version … --require-check …` / `negotiate()` | compatibility answer with every unmet requirement listed; exit 1 when incompatible | +| `openmapstack checks --json` / `list_checks()` | the catalogue: name, module, dimension, `oracle_free`, parameters with required/default | +| `openmapstack check NAME WORKSPACE --arg k=v --json` / `run_check()` | one check, one `openmapstack-check-result/v1` record | +| `openmapstack verify PROJECT --json` | the whole applicable plan, `openmapstack-verify-result/v1` | + +Additive changes (a new check, a new optional parameter, a new result +field) keep the major. Renaming or removing a check, changing a parameter's +meaning, or touching the four-state vocabulary bumps it. A consumer pins the +major and the minimum package version it was tested against. + +## 2. Result semantics a consumer may rely on + +- `status` ∈ `passed | failed | warning | not_testable`; a check that could + not establish its predicate is never `passed`. +- `code` is a stable machine identifier whenever `status` is not `passed`. + Grade on `status` and `code`; never on `detail` text. +- `dimension` names the reporting bucket (`gis_correctness`, + `reproducibility_compliance`, `provenance`, `override_handling`, + `validation_integrity`, `presentation_contract`, `rerun_success`, + `metamorphic_evidence`, `visual_judgement`). Buckets have separate + denominators and are never collapsed into one score. Deterministic + analytical correctness, metamorphic evidence, differential diagnostics, + and visual judgement stay apart. +- `oracle_free: false` marks the five known-answer checks. On arbitrary + data they are reachable only through attested `validation.expectations`; + a benchmark with a frozen expert reference may call them directly. +- A check that raises is `not_testable` with `code: check_error`. A + harness keeps such trials **outside the scored denominator** and reports + them prominently as setup failures, exactly as `evals/run.py` does + (`status: setup_failed`, exit 2). + +## 3. Reproducible arms — `openmapstack-benchmark-arm/v1` + +A published benchmark result identifies the *whole arm*, not a skill hash. +`evals/schemas/benchmark-arm-v1.schema.json` is the record OpenMapBench +must store per arm: + +| Field | Meaning | +|---|---| +| `arm` | `plain` (no skill) or `oms` (skill snapshot injected) | +| `skill` | mode, snapshot content hash, repository commit, dirty flag | +| `task_set` | case ids and a content hash over their prompts, expectations, and declared fixtures | +| `checker` | `openmapstack` package version and check API version | +| `harness` | harness repository commit and dirty flag | +| `runtime` | Python version, platform, DuckDB version, container image if any | +| `tool_surface` | adapter name and the exact agent CLI/API version it drove | +| `model` | provider, exact model id, provider revision/alias resolution when known | +| `sampling` | seed, temperature, reasoning configuration as the adapter reports them (nulls are allowed but must be present) | +| `price_catalog_date` | the date of the price list used for cost estimates | + +`openmapstack skill-snapshot --out DIR --json` produces the controlled +copy of `SKILL.md`, `references/`, and `templates/` with a per-file +inventory and content hash (`openmapstack-skill-snapshot/v1`); `--inspect` +re-verifies one. Symlinks and paths escaping the snapshot root are rejected. + +## 4. Task ownership and paired arms + +`evals/run.py --export-tasks DIR` writes the vendor-neutral task bundles +(`openmapstack-benchmark-task/v1`: prompt, declared fixtures, assertions, +hard gates, task hash) that OpenMapBench imports. Cases 070–073 (the +behavioural, prompt-style tasks) are exported as canonical OpenMapBench +tasks; this repository keeps them only as a scheduled smoke subset that +protects the integration and does not publish a competing benchmark. + +`evals/run.py --mode live --arms paired` runs `plain` and `oms` over the +same cases, trials, and seeds and reports them side by side: task parity, +per-arm success rate with a Wilson interval, per-arm median cost, tokens, +and duration, and trajectory diagnostics (event counts). It never emits a +single "success per dollar" number; quality and cost are reported as a +trade-off, and headline correctness is artifact-first. + +## 5. Evidence classes + +OpenMapBench distinguishes four evidence classes rather than one ground +truth: an authoritative answer, a frozen expert reference, metamorphic +evidence, and a differential diagnostic. Only the first two license a +known-answer check. VLM/visual review has its own denominator and judge +provenance and never turns an unverified analytical result into a pass. diff --git a/docs/verify-applicability.md b/docs/verify-applicability.md index 227cc52..31376dc 100644 --- a/docs/verify-applicability.md +++ b/docs/verify-applicability.md @@ -32,9 +32,11 @@ checker where applicable. | `project.one_canonical_pipeline` | always | one executable entrypoint is declared | missing declaration is a failure | 005 | | `project.assumptions_have_rationale` | always | assumptions are explicit and reasoned | none | 001, 070–073 | | `project.status_agrees_with_validation_report` | always | project status does not launder report state | missing report is `not_testable` | 004 | +| `project.parameters_match_steps` | `runtime.implementation.parameters` is declared | parameters are well-formed and agree with the processing steps they are bound to | none; malformed or drifting declarations fail | direct parameter-contract tests; case 015 | | `project.declared_files_exist` | at least one output path is declared | all declared outputs exist | none; missing output fails | 001, 909 | | `provenance.every_source_has_provider_and_access` | always | sources identify provider, method, and retrieval time | none | broad contract suite | -| `provenance.every_source_pinned` | always | sources carry a non-`latest` version identity | current checker does not yet recognise warehouse snapshot classes | 906 | +| `provenance.every_source_pinned` | always | every source is pinned by version identity, a hash-matched local snapshot, or an unexpired, accessible backend snapshot | none; a mutable alias is `source_unpinned`, a snapshot that cannot deliver its bytes again is `not_reproducible`, a malformed pin is `pin_invalid` | 906, 926; direct pin-class tests | +| `provenance.no_inline_credentials` | always | no source embeds a secret and `access.connection` is a reference | none | 927; direct credential-hygiene tests | | `provenance.license_present_where_required` | always | every source has declared licence metadata | none | broad contract suite | | `provenance.rationale_present` | always | source selection rationale is recorded | none | broad contract suite | | `overrides.every_override_has_provenance` | always | declared overrides carry provenance | none; zero overrides is valid | 002, 003, 012, 013, 920, 921 | @@ -44,6 +46,8 @@ checker where applicable. | `validation.warning_or_failed_propagates_to_status` | always | report aggregate reflects non-passing checks | missing report is `not_testable` | 004, 071, 072 | | `validation.run_record_matches` | always | report, manifest, inventories, hashes, and files agree | missing report is `not_testable`; missing or false records fail | 001; mutations 901–921 | | `expectation.` | once per `validation.expectations[]` entry | an independently attested known answer agrees with the produced artifact | unverified, incomplete, changed, or input-stale attestations warn without executing; allowlisted checks need DuckDB Spatial | direct attestation, staleness, path-safety, and checker-failure tests | +| `metamorphic.declarations_valid` | `validation.metamorphic` is declared | every relation parses, names an implemented relation, and addresses declared outputs | none; structural only, nothing executes | direct declaration tests; cases 015, 923–925 | +| `metamorphic.` | `--metamorphic` and the relation is declared | the declared invariant holds under the relation's controlled perturbation | executes the canonical entrypoint in an isolated copy; unmet data preconditions, unsupported source/output formats, DuckDB absent for Parquet, timeouts, and oversize sources are `not_testable`; a crashing variant or one that mutates the project's inputs fails | 015 (holds); 923 `permutation_changed_output`, 924 `monotonicity_violated`, 925 `duplicates_changed_output` | | `geodata.crs_not_used_for_metrics` | always | manifest does not declare geographic CRS for metric work | none | 001, 007, 902, 914 | | `geodata.geometry_all_valid` | once per declared readable geodata output | every geometry in the artifact is valid | DuckDB Spatial; unsupported formats and unreadable artifacts are `not_testable`, missing files fail separately | 001, 011, 918 | | `geodata.dataset_crs_is` | once per readable geodata output with declared EPSG | artifact CRS agrees with the manifest | DuckDB Spatial; absent EPSG/addressing and unreadable metadata are `not_testable` | 001, 007, 914, 919 | @@ -80,7 +84,9 @@ address them without guessing: the `verify` plan; Playwright being installed does not imply they ran; - validation evidence recomputation requires machine-readable declarations mapping report fields to artifacts and metrics; -- clean-rerun checks run only when the user supplies `--rerun`. +- clean-rerun checks run only when the user supplies `--rerun`, and declared + metamorphic relations execute only with `--metamorphic`; both rerun the + pipeline, which the static plan must never do implicitly. These omissions are reachability gaps, not implicit passes. They should enter the plan only with a versioned addressing contract and their own mutation diff --git a/evals/COVERAGE.md b/evals/COVERAGE.md index 97bf2ed..1f3ad99 100644 --- a/evals/COVERAGE.md +++ b/evals/COVERAGE.md @@ -49,6 +49,9 @@ Legend: ✅ covered · ⚠️ partially covered · ❌ not covered (tracked belo | Risk | Positive | Mutation | |---|---|---| | Pinned versions | every case | 906 `unpinned-source` | +| Pin classes (hash-matched local snapshot) | every case (all three fixture sources carry `pin: local_snapshot`) | 911 `mutated-source` (byte identity) | +| Expired / inaccessible backend snapshot is `not_reproducible` | unit tests | 926 `expired-backend-snapshot` | +| No credentials in `project.yaml`; connections by reference | every case | 927 `inline-credentials` | | Completeness reporting | 001 (mini-Tartu) | 907 `incomplete-pagination` | | Source immutability (byte identity) | every case (auto-inserted check) | 911 `mutated-source` | | Provider/access metadata | every case | — | @@ -100,13 +103,15 @@ Legend: ✅ covered · ⚠️ partially covered · ❌ not covered (tracked belo |---|---| | Reprojection invariance (4326-stored input must recover the surveyed 3301 line) | 007 (round-trip verified to sub-mm; golden distances/areas baked) | | Duplicate-input resistance | 008 (duplicated poi-z must not inflate the join) | -| Input-order permutation | ⚠️ join ordered by pair_id; an explicit permuted-input run is not yet a separate case | -| Monotonic buffer behaviour | ❌ not yet exercised | +| Input-order permutation | 015 `parcel-order` (source shuffled, outputs must be equal); mutation 923 `permutation_changed_output` | +| Monotonic buffer behaviour | 015 `road-distance-monotonic` (threshold tripled through the declared parameter, baseline keys must survive); mutation 924 `monotonicity_violated` | +| Duplicate-input resistance (declared) | 015 `parcel-duplicates` (every parcel appended once more, outputs must be equal); mutation 925 `duplicates_changed_output` | +| Invalid-precondition refusal | unit tests: count/sum semantics, missing tie-break, non-growing variant, unsupported format, source already duplicated, oversize source | ## Known gaps (tracked) 1. **PostGIS / warehouse canary** — needs a live service; candidate design is a scheduled-container case in the visual/benchmark workflow. 2. **True MultiPolygon inputs** and **nearest-neighbour tie-breaking**. -3. **Explicit input-permutation and monotonic-buffer metamorphic cases**. +3. **CRS round-trip, subset-additivity, and area-scale metamorphic relations** (the framework rejects them as undeclared rather than guessing). 4. **Raster analysis** (the suite is vector-only so far). diff --git a/evals/README.md b/evals/README.md index 4717264..1a96258 100644 --- a/evals/README.md +++ b/evals/README.md @@ -178,9 +178,45 @@ agent. `--timeout` applies to each generator or agent invocation; base seed that is incremented for each repetition. Live mode requires an explicit `--model`: a run with an unknown CLI default is -not publishable benchmark evidence. It also requires an explicit -`--skill-mode`, for the same reason — which arm ran is recorded in the result, -so inferring one mislabels the evidence rather than failing. +not publishable benchmark evidence. The arm is explicit too: `--arms oms` +(the default; the controlled skill snapshot is injected), `--arms plain` (no +skill), or `--arms paired`, which runs both arms over identical cases, trials, +and seeds. `--skill-mode enabled|disabled` remains as an alias for the single +arms. Which arm ran is recorded per trial (`results[].arm`) and per arm in +`run_config.arm_provenance`. + +### Arm provenance and paired comparison + +Every live run records one `openmapstack-benchmark-arm/v1` record per arm +(`evals/schemas/benchmark-arm-v1.schema.json`): the skill snapshot content +hash and commit, a hash over the exact task set (expectations, prompts, and +declared fixtures), the checker package and check-API versions, the harness +commit, the runtime (Python, platform, DuckDB, container image), the adapter +and the exact agent version it drove, the model id and provider, the +sampling configuration the adapter reports, and `--price-catalog-date`. A +field the harness cannot learn is `null`, never omitted. + +A paired run publishes `paired_arms`: per-arm success rate with a Wilson +interval, median cost, tokens, and duration, a `task_parity` flag proving +both arms saw the same case/trial/seed set, and trajectory diagnostics +(event counts) that are explicitly not correctness. There is no combined +"success per dollar" number; quality and cost are reported as a trade-off. +Paired bundles are retained under `evals/results////…`. + +### Exporting tasks for OpenMapBench + +```bash +python evals/run.py --export-tasks /tmp/openmapstack-tasks +python evals/run.py --export-tasks /tmp/tasks --case 070-underspecified-prompt +``` + +Each live-capable case becomes an `openmapstack-benchmark-task/v1` bundle: +prompt, declared fixtures (copied and hashed), assertion list, hard-gate +policy, and a task hash — never the reference project or the generator. +Cases 070–073 are marked `ownership: openmapbench`: they are the behavioural +prompt-style tasks whose canonical home is OpenMapBench; this repository +keeps them only as a scheduled smoke subset. See +`docs/openmapbench-interop.md` for the full contract. `--case` is repeatable, and omitting it runs every case that declares live mode (currently 001–006 and 070–073). The scheduled workflow omits it: a @@ -214,9 +250,10 @@ deterministic assertion results used by fixture mode, and `generated-project/` is copied before cleanup so every claim can be checked independently. By default, live mode copies only `SKILL.md`, `references/`, and `templates/` -into an isolated `benchmark-context/` beside the empty project, prepends an -instruction to read that controlled snapshot, and records its commit and -content hash. Eval cases and expected projects are never exposed. Use +into an isolated `benchmark-context/` beside the empty project (the same +inspectable snapshot `openmapstack skill-snapshot` produces, with a +`snapshot.json` inventory), prepends an instruction to read that controlled +snapshot, and records its commit and content hash. Eval cases and expected projects are never exposed. Use `--skill-mode disabled` for an explicit no-skill baseline; do not mix enabled and disabled trials in one published denominator. @@ -561,7 +598,10 @@ rendered snapshots as evidence. `run.py --json` reports pass/fail per assertion and rolls dimensions up within each score type: GIS correctness, project/reproducibility compliance, provenance, override handling, validation integrity, presentation contract, -and rerun success. Setup failures are reported but excluded from pass-rate +rerun success, metamorphic evidence, and visual judgement. The buckets are +owned by `openmapstack.api.DIMENSIONS` and stay separate: a metamorphic +relation that holds is self-consistency, not a correct answer, and a visual +judgement never stands in for an analytical one. Setup failures are reported but excluded from pass-rate denominators. There is intentionally no aggregate `16/16` score: a detected mutation, a conforming fixture, and a successful agent trial answer different questions. diff --git a/evals/cases/015-metamorphic-relations/expected.yaml b/evals/cases/015-metamorphic-relations/expected.yaml new file mode 100644 index 0000000..5d78fa1 --- /dev/null +++ b/evals/cases/015-metamorphic-relations/expected.yaml @@ -0,0 +1,29 @@ +id: 015-metamorphic-relations +case_type: positive +modes: [fixture] +score_types: + fixture: contract_ci +project_dir: project +hard_gate: true +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +# No golden answer is consulted here. Each relation reruns the project's own +# pipeline on a perturbed copy of its inputs (or with its declared parameter +# turned) and tests the invariant the manifest declares -- and declares the +# precondition that makes the invariant valid for this analysis. +assertions: + - assert: project.conforms_to_schema + - assert: project.parameters_match_steps + - assert: metamorphic.declarations_valid + - assert: metamorphic.relation_holds + args: { id: parcel-order } + - assert: metamorphic.relation_holds + args: { id: parcel-duplicates } + - assert: metamorphic.relation_holds + args: { id: road-distance-monotonic } + - assert: validation.run_record_matches diff --git a/evals/cases/923-order-dependent-output/expected.yaml b/evals/cases/923-order-dependent-output/expected.yaml new file mode 100644 index 0000000..261edbc --- /dev/null +++ b/evals/cases/923-order-dependent-output/expected.yaml @@ -0,0 +1,27 @@ +id: 923-order-dependent-output +case_type: mutation +modes: [fixture] +score_types: { fixture: mutation_tests } +project_dir: project +hard_gate: true +mutation: + control_generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir} --break=order_dependent" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +assertions: + - assert: project.conforms_to_schema + - assert: validation.run_record_matches + - assert: metamorphic.declarations_valid + - assert: metamorphic.relation_holds + args: { id: parcel-order } + expect: failed + expect_code: permutation_changed_output + - assert: metamorphic.relation_holds + args: { id: parcel-duplicates } + - assert: metamorphic.relation_holds + args: { id: road-distance-monotonic } diff --git a/evals/cases/924-inverted-distance-predicate/expected.yaml b/evals/cases/924-inverted-distance-predicate/expected.yaml new file mode 100644 index 0000000..22e8ebf --- /dev/null +++ b/evals/cases/924-inverted-distance-predicate/expected.yaml @@ -0,0 +1,27 @@ +id: 924-inverted-distance-predicate +case_type: mutation +modes: [fixture] +score_types: { fixture: mutation_tests } +project_dir: project +hard_gate: true +mutation: + control_generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir} --break=distance_inverted" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +assertions: + - assert: project.conforms_to_schema + - assert: validation.run_record_matches + - assert: metamorphic.declarations_valid + - assert: metamorphic.relation_holds + args: { id: parcel-order } + - assert: metamorphic.relation_holds + args: { id: parcel-duplicates } + - assert: metamorphic.relation_holds + args: { id: road-distance-monotonic } + expect: failed + expect_code: monotonicity_violated diff --git a/evals/cases/925-duplicate-sensitive-candidates/expected.yaml b/evals/cases/925-duplicate-sensitive-candidates/expected.yaml new file mode 100644 index 0000000..a51d5d6 --- /dev/null +++ b/evals/cases/925-duplicate-sensitive-candidates/expected.yaml @@ -0,0 +1,27 @@ +id: 925-duplicate-sensitive-candidates +case_type: mutation +modes: [fixture] +score_types: { fixture: mutation_tests } +project_dir: project +hard_gate: true +mutation: + control_generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir} --break=duplicate_sensitive" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +assertions: + - assert: project.conforms_to_schema + - assert: validation.run_record_matches + - assert: metamorphic.declarations_valid + - assert: metamorphic.relation_holds + args: { id: parcel-order } + - assert: metamorphic.relation_holds + args: { id: parcel-duplicates } + expect: failed + expect_code: duplicates_changed_output + - assert: metamorphic.relation_holds + args: { id: road-distance-monotonic } diff --git a/evals/cases/926-expired-backend-snapshot/expected.yaml b/evals/cases/926-expired-backend-snapshot/expected.yaml new file mode 100644 index 0000000..f01a425 --- /dev/null +++ b/evals/cases/926-expired-backend-snapshot/expected.yaml @@ -0,0 +1,25 @@ +id: 926-expired-backend-snapshot +case_type: mutation +modes: [fixture] +score_types: { fixture: mutation_tests } +project_dir: project +hard_gate: true +mutation: + control_generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir} --break=expired_snapshot" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +# A backend snapshot id plus a timestamp is not a pin once the backend may +# have dropped the snapshot. The source must read as not reproducible, not +# as pinned because a string is present. +assertions: + - assert: project.conforms_to_schema + - assert: validation.run_record_matches + - assert: provenance.no_inline_credentials + - assert: provenance.every_source_pinned + expect: failed + expect_code: not_reproducible diff --git a/evals/cases/927-inline-credentials/expected.yaml b/evals/cases/927-inline-credentials/expected.yaml new file mode 100644 index 0000000..0213d86 --- /dev/null +++ b/evals/cases/927-inline-credentials/expected.yaml @@ -0,0 +1,24 @@ +id: 927-inline-credentials +case_type: mutation +modes: [fixture] +score_types: { fixture: mutation_tests } +project_dir: project +hard_gate: true +mutation: + control_generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir}" +fixture: + generator: "{python} {evals_dir}/fixtures/reference_pipeline/gen.py {project_dir} --break=inline_credentials" + source_baseline: + - { source: ../../fixtures/mini-tartu/parcels.geojson, destination: data/source/parcels.geojson } + - { source: ../../fixtures/mini-tartu/roads.geojson, destination: data/source/roads.geojson } + - { source: ../../fixtures/mini-tartu/pois.geojson, destination: data/source/pois.geojson } + +# Secrets never enter project.yaml. A credentialed DSN under access.connection +# is rejected even when everything else about the source is in order. +assertions: + - assert: project.conforms_to_schema + - assert: validation.run_record_matches + - assert: provenance.every_source_pinned + - assert: provenance.no_inline_credentials + expect: failed + expect_code: inline_credentials diff --git a/evals/fixtures/reference_pipeline/gen.py b/evals/fixtures/reference_pipeline/gen.py index f8d2d37..c9241b9 100755 --- a/evals/fixtures/reference_pipeline/gen.py +++ b/evals/fixtures/reference_pipeline/gen.py @@ -26,6 +26,15 @@ qgis_incomplete_crs QGIS layers carry authid-only invalid CRS blocks incomplete_pagination roads source reports numberMatched > returned mutated_source pipeline rewrites its own copied "immutable" source file + expired_snapshot pois source pinned to a backend snapshot whose retention has lapsed + inline_credentials roads source embeds a credentialed connection string + order_dependent candidate rows carry their input file position (order-dependent output) + distance_inverted road-distance predicate inverted (>= instead of <=) + duplicate_sensitive duplicated source parcels produce duplicated candidates + +--road-distance-m overrides the canonical 2000 m threshold. It is the +project's one declared runtime parameter (runtime.implementation.parameters), +which is what lets a metamorphic relation vary it without editing the code. """ from __future__ import annotations @@ -99,6 +108,15 @@ def _inventory(root: Path, paths: list[Path]) -> list[dict[str, str]]: ] +ROAD_DISTANCE_CANONICAL_M = 2000 + +# Break modes that live in the pipeline's logic rather than in the generated +# bookkeeping. A generated project's copied pipeline.py reproduces these on +# rerun -- as a genuinely defective pipeline would -- so the metamorphic +# relations, which rerun the pipeline on perturbed input, can observe them. +PIPELINE_LOGIC_BREAK_MODES = {"order_dependent", "distance_inverted", "duplicate_sensitive"} + + def build( output_dir: Path, apply_override: bool, @@ -106,6 +124,7 @@ def build( with_scenario_road: bool = False, uncertain_completeness: bool = False, source_dir: Path | None = None, + road_distance_m: float = ROAD_DISTANCE_CANONICAL_M, ) -> None: output_dir.mkdir(parents=True, exist_ok=True) (output_dir / "data" / "source").mkdir(parents=True, exist_ok=True) @@ -146,9 +165,12 @@ def build( analysis_crs = "EPSG:4326" if break_mode == "wrong_crs" else "EPSG:3301" con.execute(f"CREATE TABLE parcels_raw AS SELECT * FROM ST_Read('{parcels_path}')") + # ``input_rank`` is the parcel's position in the source file. A correct + # pipeline never lets it reach the output; the two order/duplicate break + # modes below do, which is exactly what the metamorphic relations catch. con.execute( "CREATE TABLE large_parcels AS SELECT cadastral_id, land_use, municipality, geom, " - "ST_Area(geom) AS area_m2 FROM parcels_raw " + "ST_Area(geom) AS area_m2, row_number() OVER () AS input_rank FROM parcels_raw " "WHERE ST_Area(geom) >= 8000 AND land_use IN ('ARIMAA','MAATULUNDUSMAA','TOOTMISMAA')" ) con.execute(f"CREATE TABLE official_roads AS SELECT road_id, road_class, name, geom FROM ST_Read('{roads_path}')") @@ -188,12 +210,24 @@ def build( con.execute("CREATE TABLE pois_effective AS SELECT * FROM pois_raw") override_status = "applied" if apply_override else "not_testable" + distance_operator = ">=" if break_mode == "distance_inverted" else "<=" + if break_mode == "order_dependent": + # The file position leaks into the result. MIN() keeps the pipeline + # duplicate-resistant, so only the permutation relation can see it. + candidate_columns = "lp.* EXCLUDE (input_rank), MIN(lp.input_rank) AS input_rank" + group_extra = "" + elif break_mode == "duplicate_sensitive": + candidate_columns = "lp.* EXCLUDE (input_rank)" + group_extra = ", lp.input_rank" + else: + candidate_columns = "lp.* EXCLUDE (input_rank)" + group_extra = "" con.execute( "CREATE TABLE candidate_parcels AS " - "SELECT lp.*, MIN(ST_Distance(lp.geom, r.geom)) AS dist_main_road_m " + f"SELECT {candidate_columns}, MIN(ST_Distance(lp.geom, r.geom)) AS dist_main_road_m " "FROM large_parcels lp, roads r " - "GROUP BY lp.cadastral_id, lp.land_use, lp.municipality, lp.geom, lp.area_m2 " - "HAVING MIN(ST_Distance(lp.geom, r.geom)) <= 2000 " + f"GROUP BY lp.cadastral_id, lp.land_use, lp.municipality, lp.geom, lp.area_m2{group_extra} " + f"HAVING MIN(ST_Distance(lp.geom, r.geom)) {distance_operator} {float(road_distance_m)} " "ORDER BY lp.cadastral_id" ) @@ -255,7 +289,7 @@ def build( {"id": "apply_poi_override", "operation": "apply_override", "input": "pois_raw", "override": "OVERRIDE-001", "output": "pois_effective"}, {"id": "road_distance", "operation": "distance_filter", "input": "large_parcels", "target": "road_network", - "max_distance_m": 2000, "crs": analysis_crs, "output": "candidate_parcels"}, + "max_distance_m": ROAD_DISTANCE_CANONICAL_M, "crs": analysis_crs, "output": "candidate_parcels"}, ] if break_mode == "dangling_graph": steps.append({ @@ -308,6 +342,33 @@ def build( source_identifier = "latest" if break_mode == "unpinned_source" else "mini-tartu-fixture-v1" + # Every copied source is a user-approved local snapshot: its pin is the + # real content hash of the bytes in data/source/, so a swapped or edited + # file is as visible as a swapped version tag. + def local_pin(name: str) -> dict: + return { + "class": "local_snapshot", + "path": f"data/source/{name}", + "sha256": _sha256_bytes((output_dir / "data" / "source" / name).read_bytes()), + "captured_at": "2026-08-25T08:00:00Z", + } + + pois_pin = local_pin("pois.geojson") + if break_mode == "expired_snapshot": + pois_pin = { + "class": "backend_snapshot", + "identifier": "pg_export_snapshot:00000003-000001A8-1", + "captured_at": "2026-08-25T08:00:00Z", + "retention_until": "2026-08-26T08:00:00Z", + } + roads_access = {"method": "local", "retrieved_at": "2026-08-25T08:00:00Z"} + if break_mode == "inline_credentials": + roads_access = { + "method": "postgis", + "retrieved_at": "2026-08-25T08:00:00Z", + "connection": "postgresql://gis:hunter2@db.example.invalid:5432/gis", + } + project = { "schema": "openmapstack-project/v1", "project": { @@ -331,6 +392,7 @@ def build( "dataset": "mini-tartu parcels", "source_url": "file://evals/fixtures/mini-tartu/parcels.geojson", "access": {"method": "local", "retrieved_at": "2026-08-25T08:00:00Z"}, "version": {"published_at": "2026-08-25", "identifier": source_identifier}, + "pin": local_pin("parcels.geojson"), "selection": { "filter": "area_m2 >= 8000", "semantic_predicates": [{ @@ -345,8 +407,9 @@ def build( "roads": { "role": "authoritative_input", "provider": "eval-fixture", "dataset": "mini-tartu roads", "source_url": "file://evals/fixtures/mini-tartu/roads.geojson", - "access": {"method": "local", "retrieved_at": "2026-08-25T08:00:00Z"}, + "access": roads_access, "version": {"published_at": "2026-08-25", "identifier": "mini-tartu-fixture-v1"}, + "pin": local_pin("roads.geojson"), "selection": ({"completeness": {"matched": 5, "returned": 1, "page_size": 1, "pages": 1}} if break_mode == "incomplete_pagination" else {"completeness": {"matched": 1, "returned": 1}}), @@ -358,6 +421,7 @@ def build( "dataset": "mini-tartu pois", "source_url": "file://evals/fixtures/mini-tartu/pois.geojson", "access": {"method": "local", "retrieved_at": "2026-08-25T08:00:00Z"}, "version": {"published_at": "2026-08-25", "identifier": "mini-tartu-fixture-v1"}, + "pin": pois_pin, "selection": ({} if uncertain_completeness else {"completeness": {"matched": 2, "returned": 2}}), "license": {"name": "eval fixture, public domain", "url": "https://example.invalid/license"}, "rationale": "Small deterministic fixture for CI evals.", @@ -378,6 +442,25 @@ def build( "domain_checks": [ {"name": "parcel_area_range", "expression": "area_m2 > 0 AND area_m2 < 1000000"}, ], + # No-golden-answer relations the candidate set must satisfy. Each + # declares the precondition that makes it valid here: candidates + # are a keyed *set* (not a count), and the road threshold is an + # inclusion predicate, so widening it can only add parcels. + "metamorphic": [ + {"id": "parcel-order", "relation": "input_permutation_invariance", + "source": {"path": "data/source/parcels.geojson"}, + "outputs": ["candidate_parcels", "candidate_parcels_geojson"], "key": "cadastral_id", + "preconditions": {"tie_break": "candidates are keyed by cadastral_id and ordered by it; " + "no selection depends on input order"}}, + {"id": "parcel-duplicates", "relation": "duplicate_resistance", + "source": {"path": "data/source/parcels.geojson"}, + "outputs": ["candidate_parcels", "candidate_parcels_geojson"], "key": "cadastral_id", + "preconditions": {"dedup_key": "cadastral_id", "measure": "set"}}, + {"id": "road-distance-monotonic", "relation": "positive_buffer_monotonicity", + "parameter": "road_distance_m", "variant": {"multiply": 3}, + "outputs": ["candidate_parcels"], "key": "cadastral_id", + "preconditions": {"predicate": "within_distance", "expected": "superset"}}, + ], }, "presentation": { "intent": "analytical_workspace", @@ -436,7 +519,13 @@ def build( # the dashboard loads it locally, so a clean rerun must carry it or # the rebuilt project would silently lose its map library. "runtime": {"implementation": {"preferred_engine": "duckdb-spatial", "pipeline": "pipeline.py", - "dependencies": ["README.md", "vendor/maplibre-gl"]}, + "dependencies": ["README.md", "vendor/maplibre-gl"], + "parameters": [{ + "id": "road_distance_m", "type": "number", + "canonical": ROAD_DISTANCE_CANONICAL_M, + "binding": {"argument": "--road-distance-m"}, + "step": "road_distance", "field": "max_distance_m", + }]}, "environment": {"python": "3.12", "duckdb": duckdb.__version__}}, } @@ -1049,13 +1138,33 @@ def main() -> int: if isinstance(project, dict) else {} ) + # The one declared runtime parameter. The canonical run passes nothing + # and reads the manifest; a variant run binds --road-distance-m. + road_distance_m = float(ROAD_DISTANCE_CANONICAL_M) + for parameter in ((project.get("runtime") or {}).get("implementation") or {}).get("parameters") or []: + if isinstance(parameter, dict) and parameter.get("id") == "road_distance_m": + road_distance_m = float(parameter.get("canonical", road_distance_m)) + argv = sys.argv[1:] + if "--road-distance-m" in argv: + road_distance_m = float(argv[argv.index("--road-distance-m") + 1]) + break_mode = next( + ( + warning.get("issue") + for warning in (project.get("warnings") or [] if isinstance(project, dict) else []) + if isinstance(warning, dict) + and warning.get("id") == "EVAL-BREAK" + and warning.get("issue") in PIPELINE_LOGIC_BREAK_MODES + ), + None, + ) build( output_dir, apply_override=any(item.get("id") == "OVERRIDE-001" for item in overrides), - break_mode=None, + break_mode=break_mode, with_scenario_road=any(item.get("id") == "OVERRIDE-002" for item in overrides), uncertain_completeness="completeness" not in source_selection, source_dir=output_dir / "data" / "source", + road_distance_m=road_distance_m, ) print(f"rebuilt {output_dir} from local project inputs") return 0 @@ -1067,12 +1176,15 @@ def main() -> int: parser.add_argument("--scenario-road", action="store_true", help="add OVERRIDE-002 planned connector road") parser.add_argument("--uncertain-completeness", action="store_true", help="drop POI completeness counts and add a completeness warning") + parser.add_argument("--road-distance-m", type=float, default=float(ROAD_DISTANCE_CANONICAL_M), + help="road-distance threshold in metres (declared runtime parameter)") args = parser.parse_args() if args.output_dir.exists(): shutil.rmtree(args.output_dir) build(args.output_dir, apply_override=not args.no_override, break_mode=args.break_mode, - with_scenario_road=args.scenario_road, uncertain_completeness=args.uncertain_completeness) + with_scenario_road=args.scenario_road, uncertain_completeness=args.uncertain_completeness, + road_distance_m=args.road_distance_m) print(f"wrote {args.output_dir}") return 0 diff --git a/evals/run.py b/evals/run.py index 9205431..3f54541 100755 --- a/evals/run.py +++ b/evals/run.py @@ -24,6 +24,7 @@ import argparse import importlib +import itertools import json import math import os @@ -49,6 +50,12 @@ KNOWN_MODES = {"fixture", "live", "visual"} KNOWN_AGENTS = {"claude_code", "codex", "openai_compatible"} KNOWN_CASE_TYPES = {"mutation", "positive"} +# Benchmark arms: `plain` runs the agent with no skill; `oms` injects the +# controlled skill snapshot. `paired` runs both over identical cases, trials, +# and seeds so quality and cost can be compared without a shared score. +ARM_BY_SKILL_MODE = {"disabled": "plain", "enabled": "oms"} +SKILL_MODE_BY_ARM = {arm: mode for mode, arm in ARM_BY_SKILL_MODE.items()} +KNOWN_ARM_SELECTIONS = ("oms", "plain", "paired") KNOWN_SCORE_TYPES = { "agent_benchmark", "contract_ci", @@ -59,7 +66,10 @@ sys.path.insert(0, str(REPO_ROOT)) sys.path.insert(0, str(EVALS_DIR)) +from openmapstack import __version__ as OPENMAPSTACK_VERSION # noqa: E402 +from openmapstack.api import CHECK_API_VERSION # noqa: E402 from openmapstack.checks import AssertionResult, STATUSES # noqa: E402 +from openmapstack.snapshot import create_skill_snapshot # noqa: E402 from openmapstack.rerun import ( # noqa: E402 CLEAN_RERUN_EVIDENCE, perform_clean_rerun, @@ -72,16 +82,9 @@ def _load_eval_schema(name: str) -> dict[str, Any]: return json.loads((EVALS_DIR / "schemas" / name).read_text(encoding="utf-8")) -DIMENSIONS = { - "project": "reproducibility_compliance", - "overrides": "override_handling", - "provenance": "provenance", - "geodata": "gis_correctness", - "validation": "validation_integrity", - "qgis": "presentation_contract", - "presentation": "presentation_contract", - "rerun": "rerun_success", -} +# Reporting buckets are owned by the shipped API so OpenMapBench and this +# runner cannot drift apart (tests assert the import). +from openmapstack.api import DIMENSIONS # noqa: E402 @dataclass @@ -624,27 +627,175 @@ def _evidence_path(path: Path) -> str: def _prepare_skill_snapshot(workspace: Path) -> tuple[Path, str]: - """Copy only the distributable skill context, never eval/reference outputs.""" - import hashlib + """Copy only the distributable skill context, never eval/reference outputs. + Delegates to the shipped ``openmapstack skill-snapshot`` implementation so + the benchmark records the same inspectable snapshot a user can create. + """ destination = workspace / "benchmark-context" / "openmapstack" - destination.mkdir(parents=True) - shutil.copy2(REPO_ROOT / "SKILL.md", destination / "SKILL.md") - for directory in ("references", "templates"): - shutil.copytree( - REPO_ROOT / directory, - destination / directory, - ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), - ) + manifest = create_skill_snapshot(REPO_ROOT, destination) + return destination, manifest["content_sha256"] + + +def _task_set_hash(case_dirs: list[Path]) -> dict[str, Any]: + """Identify the exact task set: expectations, prompts, and declared fixtures.""" + import hashlib digest = hashlib.sha256() - for path in sorted(item for item in destination.rglob("*") if item.is_file()): - relative = path.relative_to(destination).as_posix() - digest.update(relative.encode("utf-8")) - digest.update(b"\0") - digest.update(path.read_bytes()) - digest.update(b"\0") - return destination, f"sha256:{digest.hexdigest()}" + ids: list[str] = [] + for case_dir in sorted(case_dirs): + case_def = _load_case(case_dir) + if "live" not in case_def["modes"]: + # A fixture-only case is skipped in live mode; it is not part of + # the workload an arm's provenance claims to identify. + continue + ids.append(case_def.get("id", case_dir.name)) + for relative in ("expected.yaml", case_def.get("live", {}).get("prompt_file", "prompt.md")): + path = case_dir / relative + if path.is_file(): + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + for fixture in (case_def.get("live") or {}).get("fixtures") or []: + source = (case_dir / fixture["source"]).resolve() + if source.is_file(): + digest.update(fixture["destination"].encode("utf-8")) + digest.update(b"\0") + digest.update(source.read_bytes()) + digest.update(b"\0") + return {"cases": ids, "sha256": f"sha256:{digest.hexdigest()}"} + + +def _arm_record( + arm: str, + *, + skill: dict[str, Any], + task_set: dict[str, Any], + revision: dict[str, Any], + agent_name: str | None, + model: str | None, + seed: int | None, + price_catalog_date: str | None, + trial_results: list[dict[str, Any]], +) -> dict[str, Any]: + """The complete provenance tuple that identifies a published arm. + + Every field is present; what the harness cannot learn is ``null`` rather + than omitted, so a reader can tell "unknown" from "not recorded". + """ + try: + import duckdb # type: ignore[import-not-found] + + duckdb_version = duckdb.__version__ + except ImportError: + duckdb_version = None + agent_runs = [result.get("agent_run") for result in trial_results if isinstance(result.get("agent_run"), dict)] + agent_version = next((run.get("version") for run in agent_runs if run.get("version")), None) + metadata = next((run.get("metadata") or {} for run in agent_runs), {}) + # Prefer what actually ran over what was requested: the adapter may be + # resolved per case when --agent is omitted, and an adapter may report + # the model it observed rather than the alias it was asked for. + observed_agents = sorted({run.get("agent") for run in agent_runs if run.get("agent")}) + observed_models = sorted({run.get("model") for run in agent_runs if run.get("model")}) + agent_name = observed_agents[0] if len(observed_agents) == 1 else (agent_name or (",".join(observed_agents) or None)) + model = observed_models[0] if len(observed_models) == 1 else (model or (",".join(observed_models) or None)) + sampling = { + "seed": seed, + "temperature": metadata.get("temperature"), + "reasoning": metadata.get("reasoning") or metadata.get("reasoning_effort"), + } + provider = {"claude_code": "anthropic", "codex": "openai", "openai_compatible": "openai_compatible"}.get(agent_name or "") + record = { + "schema": "openmapstack-benchmark-arm/v1", + "arm": arm, + "skill": { + "mode": skill.get("mode"), + "content_sha256": skill.get("content_sha256"), + "commit": skill.get("commit"), + "dirty": revision.get("dirty"), + "entrypoint": skill.get("entrypoint"), + }, + "task_set": task_set, + "checker": {"package": "openmapstack", "package_version": OPENMAPSTACK_VERSION, "check_api_version": CHECK_API_VERSION}, + "harness": {"name": "openmapstack/evals", "commit": revision.get("commit"), "dirty": revision.get("dirty")}, + "runtime": { + "python": platform.python_version(), + "platform": platform.platform(), + "duckdb": duckdb_version, + "container_image": os.environ.get("OPENMAPSTACK_CONTAINER_IMAGE"), + }, + "tool_surface": {"adapter": agent_name, "agent_version": agent_version}, + "model": {"provider": provider, "id": model, "revision": metadata.get("model_revision")}, + "sampling": sampling, + "price_catalog_date": price_catalog_date, + } + errors = validation_errors(record, _load_eval_schema("benchmark-arm-v1.schema.json")) + if errors: + raise ValueError(f"benchmark arm record does not validate: {'; '.join(errors)}") + return record + + +def export_tasks(case_dirs: list[Path], destination: Path) -> dict[str, Any]: + """Write vendor-neutral task bundles for an external benchmark harness. + + Only live-capable cases are tasks; each bundle carries the prompt, the + declared fixtures (copied, hashed), the assertion list, and a task hash, + and never the reference project or the generator. + """ + import hashlib + + destination.mkdir(parents=True, exist_ok=True) + index: list[dict[str, Any]] = [] + for case_dir in case_dirs: + case_def = _load_case(case_dir) + if "live" not in case_def["modes"]: + continue + live = case_def["live"] + case_id = case_def.get("id", case_dir.name) + task_dir = destination / case_id + (task_dir / "fixtures").mkdir(parents=True, exist_ok=True) + prompt = (case_dir / live.get("prompt_file", "prompt.md")).read_text(encoding="utf-8") + fixtures = [] + for fixture in live.get("fixtures") or []: + source = (case_dir / fixture["source"]).resolve() + copied = task_dir / "fixtures" / Path(fixture["destination"]).name + shutil.copyfile(source, copied) + fixtures.append({ + "path": f"fixtures/{copied.name}", + "destination": fixture["destination"], + "sha256": "sha256:" + hashlib.sha256(source.read_bytes()).hexdigest(), + }) + assertions = [ + entry for entry in case_def["assertions"] + if not entry.get("modes") or "live" in entry["modes"] + ] + body = { + "id": case_id, + "prompt": prompt, + "fixtures": fixtures, + "assertions": assertions, + "hard_gate": bool(case_def.get("hard_gate", True)), + "agent_workdir": live.get("agent_workdir", case_def.get("project_dir", "project")), + "clean_rerun": "clean_rerun" in live, + } + task_hash = "sha256:" + hashlib.sha256(json.dumps(body, sort_keys=True, default=str).encode("utf-8")).hexdigest() + task = {"schema": "openmapstack-benchmark-task/v1", **body, "task_sha256": task_hash, "checker_api": CHECK_API_VERSION} + if case_id.split("-")[0] in {"070", "071", "072", "073"}: + task["ownership"] = "openmapbench" + errors = validation_errors(task, _load_eval_schema("benchmark-task-v1.schema.json")) + if errors: + raise ValueError(f"{case_id}: task bundle does not validate: {'; '.join(errors)}") + (task_dir / "task.json").write_text(json.dumps(task, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + index.append({"id": case_id, "task_sha256": task_hash, "ownership": task.get("ownership", "openmapstack")}) + manifest = { + "schema": "openmapstack-benchmark-task-index/v1", + "checker_api": CHECK_API_VERSION, + "package_version": OPENMAPSTACK_VERSION, + "tasks": index, + } + (destination / "index.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return manifest def _skill_augmented_prompt(prompt: str, agent_workdir: Path, skill_dir: Path) -> str: @@ -815,6 +966,10 @@ def _evaluate_assertions( if args.get("hashes_before") == "$SOURCE_HASHES": args["hashes_before"] = source_hashes_before args["require_complete_tree"] = True + if assert_name == "metamorphic.relation_holds" and "forbidden_fragments" not in args: + # A variant run is a rerun: it must not reach back into the + # reference generator either. + args["forbidden_fragments"] = list(eval_forbidden_rerun_fragments()) declared_expect = entry.get("expect", "passed") mutation_role = "target" if declared_expect != "passed" else "guard" @@ -1088,6 +1243,7 @@ def finalize(payload: dict[str, Any]) -> dict[str, Any]: "live mode requires an explicit skill_mode ('enabled' or " "'disabled'); a benchmark arm must never be inferred", ) + benchmark_context["arm"] = ARM_BY_SKILL_MODE[skill_mode] if skill_mode == "enabled": skill_dir, skill_digest = _prepare_skill_snapshot(workspace) prompt = _skill_augmented_prompt(prompt, agent_workdir, skill_dir) @@ -1182,6 +1338,7 @@ def finalize(payload: dict[str, Any]) -> dict[str, Any]: "trial": trial, "case_type": case_type, "seed": seed, + "arm": benchmark_context.get("arm"), "mode": case_mode, "score_type": score_type, "supported_modes": supported_modes, @@ -1210,6 +1367,7 @@ def finalize(payload: dict[str, Any]) -> dict[str, Any]: "trial": trial, "case_type": case_type, "seed": seed, + "arm": benchmark_context.get("arm"), "mode": case_mode, "score_type": score_type, "supported_modes": supported_modes, @@ -1250,6 +1408,7 @@ def finalize(payload: dict[str, Any]) -> dict[str, Any]: "trial": trial, "case_type": case_type, "seed": seed, + "arm": benchmark_context.get("arm"), "mode": case_mode, "score_type": score_type, "supported_modes": supported_modes, @@ -1409,6 +1568,56 @@ def _agent_benchmark_summary(results: list[dict[str, Any]]) -> dict[str, Any]: } +def _paired_arms_summary(results: list[dict[str, Any]]) -> dict[str, Any] | None: + """Report `plain` and `oms` side by side, never as one number. + + Quality (success rate with interval) and cost (median USD, tokens, + duration) are separate columns of a trade-off. Trajectory measures are + diagnostics only: an agent is not penalised for reaching a correct + artifact by a different route. + """ + live = [result for result in results if result.get("mode") == "live" and result.get("status") != "skipped"] + arms = sorted({result.get("arm") for result in live if result.get("arm")}) + if len(arms) < 2: + return None + by_arm: dict[str, dict[str, Any]] = {} + signatures: dict[str, set[tuple[Any, ...]]] = {} + for arm in arms: + arm_results = [result for result in live if result.get("arm") == arm] + signatures[arm] = {(result.get("id"), result.get("trial"), result.get("seed")) for result in arm_results} + event_counts = [ + int(run.get("event_count", 0)) + for result in arm_results + if isinstance(run := result.get("agent_run"), dict) + ] + by_arm[arm] = { + "quality": _agent_benchmark_summary(arm_results), + "diagnostics": { + "median_event_count": median(event_counts) if event_counts else None, + "note": "trajectory measures are diagnostics, not correctness", + }, + } + parity = all(signatures[arm] == signatures[arms[0]] for arm in arms) + pareto = [ + { + "arm": arm, + "task_success_rate": by_arm[arm]["quality"]["task_success_rate"], + "task_success_rate_95ci": by_arm[arm]["quality"]["task_success_rate_95ci"], + "median_cost_usd": by_arm[arm]["quality"]["median_cost_usd"], + "median_tokens": by_arm[arm]["quality"]["median_tokens"], + "median_duration_s": by_arm[arm]["quality"]["median_duration_s"], + } + for arm in arms + ] + return { + "schema": "openmapstack-paired-arms/v1", + "arms": by_arm, + "task_parity": parity, + "pareto": pareto, + "note": "quality and cost are reported as a trade-off; no combined score is published", + } + + def build_summary(results: list[dict[str, Any]], run_config: dict[str, Any]) -> dict[str, Any]: ran = [r for r in results if r.get("status") != "skipped"] passed = [r for r in ran if r.get("status") == "passed"] @@ -1455,6 +1664,7 @@ def build_summary(results: list[dict[str, Any]], run_config: dict[str, Any]) -> "setup_errors": [], "score_types": score_types, "agent_benchmark": _agent_benchmark_summary(results), + "paired_arms": _paired_arms_summary(results), "mutation_score": { "total": len(mutations), "valid": valid_mutations, @@ -1516,8 +1726,24 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--skill-mode", choices=("enabled", "disabled"), - default="enabled", - help="inject the controlled OpenMapStack skill snapshot in live mode (default: enabled)", + default=None, + help="live-mode arm by skill mode: enabled = oms, disabled = plain (default: enabled; see --arms)", + ) + parser.add_argument( + "--arms", + choices=KNOWN_ARM_SELECTIONS, + default=None, + help="live-mode arm selection: oms (skill injected), plain (no skill), or paired (both, same cases/trials/seeds)", + ) + parser.add_argument( + "--price-catalog-date", + help="YYYY-MM-DD of the price list used for cost estimates; recorded in each arm's provenance", + ) + parser.add_argument( + "--export-tasks", + type=Path, + metavar="DIR", + help="write vendor-neutral openmapstack-benchmark-task/v1 bundles for the selected live cases and exit", ) parser.add_argument("--run-id", help="artifact run id (generated by default for live mode)") parser.add_argument( @@ -1536,6 +1762,16 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) revision = _git_revision() + if args.skill_mode is not None and args.arms is not None and ARM_BY_SKILL_MODE[args.skill_mode] != args.arms: + parser.error("--skill-mode and --arms disagree; pass one of them") + if args.arms is None: + args.arms = ARM_BY_SKILL_MODE[args.skill_mode or "enabled"] + arms = ["plain", "oms"] if args.arms == "paired" else [args.arms] + if args.price_catalog_date is not None: + try: + datetime.strptime(args.price_catalog_date, "%Y-%m-%d") + except ValueError: + parser.error("--price-catalog-date must be YYYY-MM-DD") try: run_id = _validate_run_id(args.run_id or _new_run_id()) if args.mode in {"live", "visual"} else None except ValueError as exc: @@ -1555,7 +1791,10 @@ def main(argv: list[str] | None = None) -> int: "artifact_root": _evidence_path(result_root) if result_root else None, "skill_commit": revision["commit"], "skill_worktree_dirty": revision["dirty"], - "skill_mode": args.skill_mode if args.mode == "live" else None, + "skill_mode": (SKILL_MODE_BY_ARM[arms[0]] if len(arms) == 1 else "paired") if args.mode == "live" else None, + "arms": arms if args.mode == "live" else None, + "price_catalog_date": args.price_catalog_date, + "checker": {"package_version": OPENMAPSTACK_VERSION, "check_api_version": CHECK_API_VERSION}, } try: @@ -1577,6 +1816,18 @@ def main(argv: list[str] | None = None) -> int: _write_summary(summary, args.json) return 2 + if args.export_tasks is not None: + try: + manifest = export_tasks(case_dirs, args.export_tasks) + except (OSError, ValueError) as exc: + print(f"Task export failed: {exc}", file=sys.stderr) + return 2 + if not manifest["tasks"]: + print("No live-capable cases selected; nothing exported", file=sys.stderr) + return 2 + print(f"Exported {len(manifest['tasks'])} task bundle(s) to {args.export_tasks}") + return 0 + if args.list: for case_dir in case_dirs: case_def = _load_case(case_dir) @@ -1614,12 +1865,14 @@ def main(argv: list[str] | None = None) -> int: return 2 results: list[dict[str, Any]] = [] + paired = len(arms) > 1 for case_dir in case_dirs: case_def = _load_case(case_dir) selected_score_type = case_def["score_types"].get(args.mode) - for trial in range(1, args.repetitions + 1): + for trial, arm in itertools.product(range(1, args.repetitions + 1), arms): trial_seed = args.seed + trial - 1 if args.seed is not None else None trial_started = time.monotonic() + arm_segment = (arm,) if paired else () try: result = run_case( case_dir, @@ -1630,7 +1883,7 @@ def main(argv: list[str] | None = None) -> int: seed=trial_seed, trial=trial, artifact_dir=( - result_root / (args.agent or case_def["live"].get("agent", "claude_code")) / case_def.get("id", case_dir.name) / str(trial) + result_root.joinpath(args.agent or case_def["live"].get("agent", "claude_code"), *arm_segment, case_def.get("id", case_dir.name), str(trial)) if args.mode == "live" and result_root is not None and not args.no_retain_artifacts and "live" in case_def else ( result_root / "visual" / case_def.get("id", case_dir.name) / str(trial) @@ -1644,7 +1897,7 @@ def main(argv: list[str] | None = None) -> int: "skill_worktree_dirty": revision["dirty"], "environment": _environment(), }, - skill_mode=args.skill_mode, + skill_mode=SKILL_MODE_BY_ARM[arm] if args.mode == "live" else None, ) except Exception as exc: # noqa: BLE001 result = { @@ -1652,6 +1905,7 @@ def main(argv: list[str] | None = None) -> int: "trial": trial, "case_type": case_def["case_type"], "seed": trial_seed, + "arm": arm if args.mode == "live" else None, "mode": args.mode, "score_type": selected_score_type, "supported_modes": case_def["modes"], @@ -1685,6 +1939,8 @@ def main(argv: list[str] | None = None) -> int: "setup_failed": "ERROR", }[result["status"]] suffix = f" trial={trial}" if args.repetitions > 1 else "" + if paired: + suffix += f" arm={arm}" print( f"{marker:5s} {result['id']:40s} {result['duration_s']:.2f}s{suffix}", end="", @@ -1697,6 +1953,25 @@ def main(argv: list[str] | None = None) -> int: else: print() + if args.mode == "live": + task_set = _task_set_hash(case_dirs) + run_config["arm_provenance"] = [ + _arm_record( + arm, + skill=next( + ((r.get("benchmark_context") or {}).get("skill") or {} for r in results if r.get("arm") == arm and (r.get("benchmark_context") or {}).get("skill")), + {"mode": SKILL_MODE_BY_ARM[arm], "content_sha256": None, "commit": revision["commit"], "entrypoint": None}, + ), + task_set=task_set, + revision=revision, + agent_name=args.agent, + model=args.model, + seed=args.seed, + price_catalog_date=args.price_catalog_date, + trial_results=[r for r in results if r.get("arm") == arm], + ) + for arm in arms + ] summary = build_summary(results, run_config) if summary["selection"]["trials_run"] == 0: summary["run_setup_failed"] = True @@ -1740,6 +2015,14 @@ def main(argv: list[str] | None = None) -> int: f"({score_text}; {mutation_score['isolated']} isolated, " f"{mutation_score['invalid']} invalid)" ) + paired_summary = summary.get("paired_arms") + if paired_summary: + parity = "task parity" if paired_summary["task_parity"] else "TASK PARITY BROKEN" + print(f"paired arms ({parity}):") + for row in paired_summary["pareto"]: + rate = f"{row['task_success_rate']:.0%}" if row["task_success_rate"] is not None else "n/a" + cost = f"${row['median_cost_usd']:.4f}" if row["median_cost_usd"] is not None else "cost n/a" + print(f" {row['arm']:6s} success {rate} {row['task_success_rate_95ci']} median {cost}, tokens {row['median_tokens']}, {row['median_duration_s']}s") skipped = summary["selection"]["case_definitions_skipped"] if skipped: print(f"Skipped case definitions: {skipped}") diff --git a/evals/schemas/benchmark-arm-v1.schema.json b/evals/schemas/benchmark-arm-v1.schema.json new file mode 100644 index 0000000..a97a771 --- /dev/null +++ b/evals/schemas/benchmark-arm-v1.schema.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openmapstack/schemas/benchmark-arm-v1.schema.json", + "title": "Complete provenance of one benchmark arm (openmapstack-benchmark-arm/v1)", + "description": "A published result identifies the whole arm. Every field must be present; a value the harness genuinely cannot learn is null, never omitted.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", "arm", "skill", "task_set", "checker", "harness", "runtime", + "tool_surface", "model", "sampling", "price_catalog_date" + ], + "properties": { + "schema": {"const": "openmapstack-benchmark-arm/v1"}, + "arm": {"enum": ["plain", "oms"]}, + "skill": { + "type": "object", + "required": ["mode", "content_sha256", "commit", "dirty", "entrypoint"], + "properties": { + "mode": {"enum": ["enabled", "disabled"]}, + "content_sha256": {"type": ["string", "null"], "pattern": "^sha256:[0-9a-f]{64}$"}, + "commit": {"type": ["string", "null"]}, + "dirty": {"type": ["boolean", "null"]}, + "entrypoint": {"type": ["string", "null"]} + } + }, + "task_set": { + "type": "object", + "required": ["cases", "sha256"], + "properties": { + "cases": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "sha256": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"} + } + }, + "checker": { + "type": "object", + "required": ["package", "package_version", "check_api_version"], + "properties": { + "package": {"const": "openmapstack"}, + "package_version": {"type": "string", "minLength": 1}, + "check_api_version": {"type": "string", "minLength": 1} + } + }, + "harness": { + "type": "object", + "required": ["name", "commit", "dirty"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "commit": {"type": ["string", "null"]}, + "dirty": {"type": ["boolean", "null"]} + } + }, + "runtime": { + "type": "object", + "required": ["python", "platform", "duckdb", "container_image"], + "properties": { + "python": {"type": "string"}, + "platform": {"type": "string"}, + "duckdb": {"type": ["string", "null"]}, + "container_image": {"type": ["string", "null"]} + } + }, + "tool_surface": { + "type": "object", + "required": ["adapter", "agent_version"], + "properties": { + "adapter": {"type": ["string", "null"]}, + "agent_version": {"type": ["string", "null"]} + } + }, + "model": { + "type": "object", + "required": ["provider", "id", "revision"], + "properties": { + "provider": {"type": ["string", "null"]}, + "id": {"type": ["string", "null"]}, + "revision": {"type": ["string", "null"]} + } + }, + "sampling": { + "type": "object", + "required": ["seed", "temperature", "reasoning"], + "properties": { + "seed": {"type": ["integer", "null"]}, + "temperature": {"type": ["number", "null"]}, + "reasoning": {"type": ["string", "object", "null"]} + } + }, + "price_catalog_date": {"type": ["string", "null"], "pattern": "^\\d{4}-\\d{2}-\\d{2}$"} + } +} diff --git a/evals/schemas/benchmark-task-v1.schema.json b/evals/schemas/benchmark-task-v1.schema.json new file mode 100644 index 0000000..d09312c --- /dev/null +++ b/evals/schemas/benchmark-task-v1.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openmapstack/schemas/benchmark-task-v1.schema.json", + "title": "Vendor-neutral benchmark task bundle (openmapstack-benchmark-task/v1)", + "type": "object", + "additionalProperties": false, + "required": ["schema", "id", "prompt", "fixtures", "assertions", "hard_gate", "agent_workdir", "clean_rerun", "task_sha256", "checker_api"], + "properties": { + "schema": {"const": "openmapstack-benchmark-task/v1"}, + "id": {"type": "string", "minLength": 1}, + "prompt": {"type": "string", "minLength": 1}, + "fixtures": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "destination", "sha256"], + "properties": { + "path": {"type": "string", "minLength": 1}, + "destination": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"} + } + } + }, + "assertions": {"type": "array", "items": {"type": "object", "required": ["assert"]}}, + "hard_gate": {"type": "boolean"}, + "agent_workdir": {"type": "string", "minLength": 1}, + "clean_rerun": {"type": "boolean"}, + "task_sha256": {"type": "string", "pattern": "^sha256:[0-9a-f]{64}$"}, + "checker_api": {"type": "string", "minLength": 1}, + "ownership": {"type": "string"} + } +} diff --git a/evals/schemas/results-v2.schema.json b/evals/schemas/results-v2.schema.json index 89d413e..8182ac7 100644 --- a/evals/schemas/results-v2.schema.json +++ b/evals/schemas/results-v2.schema.json @@ -47,6 +47,7 @@ "dimensions": {"type": "object"} } }, + "paired_arms": {"type": ["object", "null"]}, "mutation_score": { "type": "object", "required": ["total", "valid", "detected", "survived", "invalid", "isolated", "score"], diff --git a/openmapstack/__init__.py b/openmapstack/__init__.py index f9e7313..eb422d0 100644 --- a/openmapstack/__init__.py +++ b/openmapstack/__init__.py @@ -3,4 +3,4 @@ from .validation import Check, ValidationResult, validate_project __all__ = ["Check", "ValidationResult", "validate_project"] -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/openmapstack/api.py b/openmapstack/api.py new file mode 100644 index 0000000..886f479 --- /dev/null +++ b/openmapstack/api.py @@ -0,0 +1,319 @@ +"""The versioned check API that external harnesses consume. + +OpenMapBench (and any other benchmark or CI system) grades produced projects +with this package's checks. It must be able to do so without vendoring the +check implementations and without depending on module layout: the surface +it may rely on is exactly + +- ``CHECK_API_VERSION`` and ``api_info()`` for negotiation; +- ``list_checks()`` for the catalogue of check names and their parameters; +- ``run_check()`` for one check, returning a record that validates against + ``openmapstack-check-result/v1``; +- ``openmapstack verify --json``, returning ``openmapstack-verify-result/v1``; +- the JSON schemas packaged under ``openmapstack/schemas/``. + +Everything else in ``openmapstack.checks`` is implementation. + +Versioning: ``CHECK_API_VERSION`` follows ``/v``. A new check, +a new optional parameter, or a new result field is additive and does not +change the major. Renaming or removing a check, changing a parameter's +meaning, or changing the four-state status vocabulary does. A consumer +pins the major and the minimum package version it was tested against, and +``negotiate()`` answers whether the installed package satisfies both. + +Result semantics that a consumer may rely on: + +- ``status`` is one of ``passed | failed | warning | not_testable`` and a + check that could not establish its predicate is never ``passed``; +- ``code`` is a stable machine-readable identifier when the status is not + ``passed``; consumers grade on ``status`` and, for mutation-style + expectations, ``code`` -- never on ``detail`` text; +- ``dimension`` is the reporting bucket the check belongs to; buckets are + reported separately and must not be collapsed into one score; +- ``oracle_free`` is ``false`` for the checks that need a known answer. + Those transfer to arbitrary data only through attested expectations. + +Setup failures (a check that raises) are reported as ``not_testable`` with +``code: check_error`` by ``run_check`` so a broken environment cannot +produce either a pass or a graded failure; benchmark harnesses keep them +out of scored denominators. +""" + +from __future__ import annotations + +import importlib +import inspect +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from . import __version__ +from .checks import STATUSES, AssertionResult, not_testable +from .schema import validation_errors + +CHECK_API_VERSION = "openmapstack-check-api/v1" +CHECK_RESULT_SCHEMA = "openmapstack-check-result/v1" +API_INFO_SCHEMA = "openmapstack-api-info/v1" +VERIFY_RESULT_SCHEMA = "openmapstack-verify-result/v1" +PROJECT_SCHEMA = "openmapstack-project/v1" + +CHECK_MODULES = ( + "project", + "provenance", + "overrides", + "validation", + "geodata", + "presentation", + "qgis", + "visual", + "rerun", + "metamorphic", +) + +# Reporting buckets. Shared with evals/run.py, which asserts equality in +# tests so the two cannot drift apart. +DIMENSIONS = { + "project": "reproducibility_compliance", + "overrides": "override_handling", + "provenance": "provenance", + "geodata": "gis_correctness", + "validation": "validation_integrity", + "qgis": "presentation_contract", + "presentation": "presentation_contract", + "visual": "visual_judgement", + "rerun": "rerun_success", + "metamorphic": "metamorphic_evidence", +} + +# The checks that need a known answer. They are reachable on user data +# only through validation.expectations[] attestations. +KNOWN_ANSWER_CHECKS = frozenset( + { + "geodata.row_count", + "geodata.feature_present", + "geodata.feature_absent", + "geodata.feature_field_equals", + "geodata.field_range", + } +) + +_SCHEMA_DIR = Path(__file__).resolve().parent / "schemas" +_VERSION = re.compile(r"^(\d+)\.(\d+)\.(\d+)") + + +class CheckAPIError(ValueError): + """The consumer asked for something the API does not provide.""" + + +@dataclass(frozen=True) +class CheckParameter: + name: str + required: bool + default: Any = None + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = {"name": self.name, "required": self.required} + if not self.required: + payload["default"] = self.default + return payload + + +@dataclass(frozen=True) +class CheckDescriptor: + name: str + module: str + dimension: str + oracle_free: bool + summary: str + parameters: tuple[CheckParameter, ...] = field(default_factory=tuple) + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "module": self.module, + "dimension": self.dimension, + "oracle_free": self.oracle_free, + "summary": self.summary, + "parameters": [parameter.to_dict() for parameter in self.parameters], + } + + +def _load_schema(name: str) -> dict[str, Any]: + return json.loads((_SCHEMA_DIR / name).read_text(encoding="utf-8")) + + +def _describe(module_name: str, function_name: str, function: Any) -> CheckDescriptor: + signature = inspect.signature(function) + parameters: list[CheckParameter] = [] + for index, parameter in enumerate(signature.parameters.values()): + if index == 0: # workspace + continue + if parameter.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD): + continue + required = parameter.default is inspect.Parameter.empty + default = None if required else parameter.default + if isinstance(default, tuple): + default = list(default) + parameters.append(CheckParameter(parameter.name, required, default)) + doc = inspect.getdoc(function) or "" + summary = doc.strip().splitlines()[0].strip() if doc.strip() else "" + name = f"{module_name}.{function_name}" + return CheckDescriptor( + name=name, + module=module_name, + dimension=DIMENSIONS.get(module_name, "other"), + oracle_free=name not in KNOWN_ANSWER_CHECKS, + summary=summary, + parameters=tuple(parameters), + ) + + +def _is_check(function: Any) -> bool: + if not inspect.isfunction(function) or function.__name__.startswith("_"): + return False + try: + parameters = list(inspect.signature(function).parameters.values()) + except (TypeError, ValueError): + return False + return bool(parameters) and parameters[0].name == "workspace" + + +def list_checks() -> list[CheckDescriptor]: + """Every public check, discovered from the shipped modules.""" + descriptors: list[CheckDescriptor] = [] + for module_name in CHECK_MODULES: + module = importlib.import_module(f"openmapstack.checks.{module_name}") + for function_name, function in sorted(vars(module).items()): + if getattr(function, "__module__", None) != module.__name__: + continue + if _is_check(function): + descriptors.append(_describe(module_name, function_name, function)) + return descriptors + + +def describe_check(name: str) -> CheckDescriptor: + module_name, _, function_name = name.partition(".") + function = _resolve(name) + if getattr(function, "__module__", None) != f"openmapstack.checks.{module_name}": + raise CheckAPIError(f"unknown check {name!r}; see list_checks()") + return _describe(module_name, function_name, function) + + +def _resolve(name: str) -> Any: + module_name, _, function_name = name.partition(".") + if module_name not in CHECK_MODULES or not function_name: + raise CheckAPIError(f"unknown check {name!r}; see list_checks()") + module = importlib.import_module(f"openmapstack.checks.{module_name}") + function = getattr(module, function_name, None) + if function is None or not _is_check(function): + raise CheckAPIError(f"unknown check {name!r}; see list_checks()") + return function + + +def run_check(name: str, workspace: str | Path, args: dict[str, Any] | None = None) -> dict[str, Any]: + """Execute one check and return an ``openmapstack-check-result/v1`` record. + + Unknown check names and malformed arguments raise ``CheckAPIError`` + (a consumer configuration error). A check that raises while running is + reported as ``not_testable`` with ``code: check_error`` -- never as a + pass, and never as a graded failure. + """ + descriptor = describe_check(name) + function = _resolve(name) + args = dict(args or {}) + declared = {parameter.name for parameter in descriptor.parameters} + unknown = sorted(set(args) - declared) + missing = sorted(parameter.name for parameter in descriptor.parameters if parameter.required and parameter.name not in args) + if unknown or missing: + raise CheckAPIError(f"{name}: unknown args {unknown}, missing required args {missing}") + try: + result = function(Path(workspace), **args) + except Exception as exc: # noqa: BLE001 - a check must never take a harness down + result = not_testable(f"{type(exc).__name__}: {exc}", code="check_error") + if not isinstance(result, AssertionResult) or result.status not in STATUSES: + result = not_testable("check returned a malformed result", code="check_error") + data = {key: value for key, value in result.data.items() if key != "code"} + record: dict[str, Any] = { + "schema": CHECK_RESULT_SCHEMA, + "api_version": CHECK_API_VERSION, + "package_version": __version__, + "check": name, + "dimension": descriptor.dimension, + "oracle_free": descriptor.oracle_free, + "args": args, + "status": result.status, + "code": result.data.get("code"), + "detail": result.detail, + "data": json.loads(json.dumps(data, default=str)), + } + errors = validation_errors(record, _load_schema("check-result-v1.schema.json")) + if errors: # pragma: no cover - the record is built here; a failure is a bug + raise CheckAPIError(f"internal: check result does not validate: {errors}") + return record + + +def api_info() -> dict[str, Any]: + """What this installation offers, for a consumer to negotiate against.""" + checks = list_checks() + return { + "schema": API_INFO_SCHEMA, + "package": "openmapstack", + "package_version": __version__, + "check_api_version": CHECK_API_VERSION, + "project_schema": PROJECT_SCHEMA, + "result_schemas": { + "check": CHECK_RESULT_SCHEMA, + "verify": VERIFY_RESULT_SCHEMA, + }, + "statuses": list(STATUSES), + "dimensions": sorted(set(DIMENSIONS.values())), + "checks": len(checks), + "oracle_free_checks": sum(descriptor.oracle_free for descriptor in checks), + "known_answer_checks": sorted(KNOWN_ANSWER_CHECKS), + } + + +def _parse_version(text: str) -> tuple[int, int, int]: + match = _VERSION.match(text or "") + if match is None: + raise CheckAPIError(f"not a semantic version: {text!r}") + return tuple(int(part) for part in match.groups()) # type: ignore[return-value] + + +def negotiate( + *, + required_api: str = CHECK_API_VERSION, + min_package_version: str | None = None, + required_checks: list[str] | None = None, +) -> dict[str, Any]: + """Answer whether this installation satisfies a consumer's requirements. + + A consumer states the API major it was built for, the oldest package + version it was tested against, and the checks it needs. The answer + lists every unmet requirement so a harness can report *why* it is + refusing to grade rather than grading with a checker it does not + understand. + """ + problems: list[str] = [] + if required_api != CHECK_API_VERSION: + problems.append(f"check API {required_api!r} is not provided; this package offers {CHECK_API_VERSION!r}") + if min_package_version is not None and _parse_version(__version__) < _parse_version(min_package_version): + problems.append(f"package version {__version__} is older than the required {min_package_version}") + available = {descriptor.name for descriptor in list_checks()} + missing = sorted(name for name in (required_checks or []) if name not in available) + if missing: + problems.append(f"checks not provided: {missing}") + return { + "schema": "openmapstack-api-negotiation/v1", + "compatible": not problems, + "package_version": __version__, + "check_api_version": CHECK_API_VERSION, + "problems": problems, + } + + +def validate_verify_result(payload: dict[str, Any]) -> list[str]: + """Schema-validate an ``openmapstack verify --json`` document.""" + return validation_errors(payload, _load_schema("verify-result-v1.schema.json")) diff --git a/openmapstack/checks/metamorphic.py b/openmapstack/checks/metamorphic.py new file mode 100644 index 0000000..49e194a --- /dev/null +++ b/openmapstack/checks/metamorphic.py @@ -0,0 +1,74 @@ +"""Metamorphic-relation assertions. + +Thin check-library entry points over ``openmapstack.metamorphic`` so an eval +case (``assert: metamorphic.relation_holds``) and ``openmapstack verify +--metamorphic`` grade the same thing. See that module for the relations, +their preconditions, and the result vocabulary. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from . import AssertionResult, failed, load_project_yaml, not_testable, passed, project_root + + +def declarations_valid(workspace: Path, project_dir: str = ".") -> AssertionResult: + """Every ``validation.metamorphic[]`` entry parses, names a known relation, + and addresses declared outputs. Structural only: nothing is executed.""" + from ..metamorphic import DeclarationError, declared_relations, parse_declaration + + proj = load_project_yaml(workspace, project_dir) + if proj is None: + return failed("project.yaml missing", code="manifest_missing") + try: + raw_declarations = declared_relations(proj) + except DeclarationError as exc: + return failed(str(exc), code="metamorphic_declaration_invalid") + if not raw_declarations: + return not_testable("no metamorphic relations are declared", code="metamorphic_undeclared") + errors: list[str] = [] + seen: set[str] = set() + outputs = proj.get("outputs") if isinstance(proj.get("outputs"), dict) else {} + for raw in raw_declarations: + try: + declaration = parse_declaration(raw) + except DeclarationError as exc: + errors.append(str(exc)) + continue + if declaration.id in seen: + errors.append(f"{declaration.id}: duplicate relation id") + seen.add(declaration.id) + missing = [key for key in declaration.outputs if key not in outputs] + if missing: + errors.append(f"{declaration.id}: outputs {missing} are not declared outputs") + if errors: + return failed("; ".join(errors), code="metamorphic_declaration_invalid") + return passed(f"{len(raw_declarations)} metamorphic relation(s) are well-formed") + + +def relation_holds( + workspace: Path, + id: str, + project_dir: str = ".", + forbidden_fragments: list[str] | None = None, +) -> AssertionResult: + """Execute the declared relation ``id`` in an isolated variant workspace.""" + from ..metamorphic import DeclarationError, declared_relations, run_relation + + root = project_root(workspace, project_dir) + proj = load_project_yaml(workspace, project_dir) + if proj is None: + return failed("project.yaml missing", code="manifest_missing") + try: + raw_declarations = declared_relations(proj) + except DeclarationError as exc: + return failed(str(exc), code="metamorphic_declaration_invalid") + matches = [raw for raw in raw_declarations if isinstance(raw, dict) and raw.get("id") == id] + if not matches: + return failed(f"no metamorphic relation with id {id!r} is declared", code="metamorphic_relation_undeclared") + result, evidence = run_relation(root, proj, matches[0], forbidden_fragments=tuple(forbidden_fragments or ())) + data: dict[str, Any] = dict(result.data) + data["evidence"] = evidence + return AssertionResult(result.status, result.detail, data) diff --git a/openmapstack/checks/project.py b/openmapstack/checks/project.py index d13b669..26a34ed 100644 --- a/openmapstack/checks/project.py +++ b/openmapstack/checks/project.py @@ -259,3 +259,29 @@ def assumptions_have_rationale(workspace: Path, project_dir: str = ".") -> Asser code="assumption_missing_rationale", ) return passed(f"all {len(assumptions)} assumptions have statement + rationale") + + +def parameters_match_steps(workspace: Path, project_dir: str = ".") -> AssertionResult: + """``runtime.implementation.parameters`` is well-formed and each parameter + bound to a processing step agrees with that step's declared value. + + A manifest that advertises one threshold under ``parameters`` while the + step declares another is drift of the same kind as a presentation + control that disagrees with the pipeline: the whole view becomes a + confident lie. See ``openmapstack.parameters``. + """ + from ..parameters import ParameterError, declared_parameters + + proj = load_project_yaml(workspace, project_dir) + if proj is None: + return failed("project.yaml missing", code="manifest_missing") + try: + parameters = declared_parameters(proj) + except ParameterError as exc: + return failed(str(exc), code="parameters_invalid") + if not parameters: + return not_testable("no runtime parameters are declared", code="parameters_undeclared") + bound = sum(1 for parameter in parameters if parameter.step) + return passed( + f"{len(parameters)} runtime parameter(s) declared; {bound} bound to a processing step agree with it" + ) diff --git a/openmapstack/checks/provenance.py b/openmapstack/checks/provenance.py index 3e12d3b..c59eb5f 100644 --- a/openmapstack/checks/provenance.py +++ b/openmapstack/checks/provenance.py @@ -7,7 +7,7 @@ from pathlib import Path -from . import AssertionResult, failed, get_in, load_project_yaml, passed, warning +from . import AssertionResult, failed, get_in, load_project_yaml, passed, project_root, warning def every_source_has_provider_and_access(workspace: Path, project_dir: str = ".") -> AssertionResult: @@ -31,25 +31,65 @@ def every_source_has_provider_and_access(workspace: Path, project_dir: str = "." def every_source_pinned(workspace: Path, project_dir: str = ".") -> AssertionResult: - """Pinning to 'latest' is not reproducible — version.identifier/published_at - must be present and not equal to the literal string 'latest'.""" + """Every source is reproducibly pinned. + + A source without a ``pin`` block must carry a ``version.identifier`` or + ``published_at`` that is not a mutable alias such as ``latest``. A source + with a pin block is held to its pin class (``openmapstack.sources``): a + local snapshot must exist and match its hash, and a backend snapshot must + be identified, unexpired, and not known to be inaccessible. A pin that + cannot deliver the bytes again is ``not_reproducible`` -- a timestamp + string alone does not make a warehouse table pinned. + """ + from ..sources import source_pin_summary + + proj = load_project_yaml(workspace, project_dir) + if proj is None: + return failed("project.yaml missing", code="manifest_missing") + sources = proj.get("sources") or {} + if not sources: + return failed("no sources declared", code="no_sources") + assessments = source_pin_summary(project_root(workspace, project_dir), sources) + by_status: dict[str, list[str]] = {} + for key, assessment in assessments.items(): + by_status.setdefault(assessment.status, []).append(f"{key}: {assessment.reason}") + if by_status.get("invalid"): + return failed(f"sources with malformed pins: {by_status['invalid']}", code="pin_invalid") + if by_status.get("not_reproducible"): + return failed( + f"sources whose pinned snapshot cannot be obtained again: {by_status['not_reproducible']}", + code="not_reproducible", + causes={key: assessment.details.get("cause") for key, assessment in assessments.items() if assessment.status == "not_reproducible"}, + ) + if by_status.get("unpinned"): + return failed(f"sources not pinned to a version/identifier: {by_status['unpinned']}", code="source_unpinned") + classes = sorted({assessment.pin_class for assessment in assessments.values()}) + return passed(f"all {len(sources)} sources are pinned ({', '.join(classes)})", pin_classes=classes) + + +def no_inline_credentials(workspace: Path, project_dir: str = ".") -> AssertionResult: + """No source embeds a secret, and warehouse connections are by reference.""" + from ..sources import connection_reference_error, find_inline_credentials + proj = load_project_yaml(workspace, project_dir) if proj is None: return failed("project.yaml missing", code="manifest_missing") sources = proj.get("sources") or {} if not sources: return failed("no sources declared", code="no_sources") - unpinned: list[str] = [] + root = project_root(workspace, project_dir) + problems: list[str] = [] for key, src in sources.items(): - identifier = get_in(src, "version.identifier") - published_at = get_in(src, "version.published_at") - if not identifier and not published_at: - unpinned.append(key) - elif str(identifier).strip().lower() == "latest": - unpinned.append(key) - if unpinned: - return failed(f"sources not pinned to a version/identifier: {unpinned}", code="source_unpinned") - return passed(f"all {len(sources)} sources are pinned") + if not isinstance(src, dict): + continue + for finding in find_inline_credentials(src, f"sources.{key}"): + problems.append(f"{finding['path']} ({finding['pattern']})") + error = connection_reference_error(root, get_in(src, "access.connection")) + if error: + problems.append(f"sources.{key}.access.connection: {error}") + if problems: + return failed(f"credentials or connection strings embedded in the manifest: {problems}", code="inline_credentials") + return passed(f"no inline credentials in {len(sources)} sources; connections are by reference") def license_present_where_required( diff --git a/openmapstack/cli.py b/openmapstack/cli.py index a8e6b14..2b2f508 100644 --- a/openmapstack/cli.py +++ b/openmapstack/cli.py @@ -8,6 +8,9 @@ import subprocess import sys from collections.abc import Sequence + +import yaml + from pathlib import Path from typing import Any @@ -72,12 +75,81 @@ def build_parser() -> argparse.ArgumentParser: default=1800.0, help="seconds allowed for the clean rerun's canonical entrypoint (default: 1800)", ) + verify_parser.add_argument( + "--metamorphic", + action="store_true", + help="also execute every declared validation.metamorphic relation in an isolated variant workspace", + ) verify_parser.add_argument("--strict", action="store_true", help="return failure when warnings or not-testable checks exist") verify_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") verify_parser.add_argument("--output", type=Path, help="also write the JSON result to this path") verify_parser.add_argument("--verbose", action="store_true", help="show passed checks in text output") verify_parser.set_defaults(handler=_cmd_verify) + source_parser = subparsers.add_parser( + "source", + help="read-only discovery and approval-gated snapshots of a warehouse source", + ) + source_commands = source_parser.add_subparsers(dest="source_command", required=True) + discover_parser = source_commands.add_parser("discover", help="list tables/files, geometry columns, SRIDs, and row estimates") + discover_parser.add_argument("project", help="project.yaml or its directory") + discover_parser.add_argument("--source", required=True, help="key under sources.* declaring warehouse.backend") + discover_parser.add_argument("--timeout", type=float, default=60.0, help="statement timeout in seconds (default: 60)") + discover_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + discover_parser.set_defaults(handler=_cmd_source_discover) + snapshot_parser = source_commands.add_parser( + "snapshot", + help="plan a query snapshot under data/source/; materialise only with --approve", + ) + snapshot_parser.add_argument("project", help="project.yaml or its directory") + snapshot_parser.add_argument("--source", required=True, help="key under sources.* declaring warehouse.backend") + query_group = snapshot_parser.add_mutually_exclusive_group(required=True) + query_group.add_argument("--query", help="one read-only SELECT statement") + query_group.add_argument("--query-file", type=Path, help="file containing one read-only SELECT statement") + snapshot_parser.add_argument("--destination", required=True, help="project-relative .parquet path under data/source/") + snapshot_parser.add_argument("--approve", action="store_true", help="actually write the snapshot (default is a dry run)") + snapshot_parser.add_argument("--write-manifest", action="store_true", help="after a materialised snapshot, record the pin and warehouse metadata in project.yaml (rewrites the file; YAML comments are not preserved)") + snapshot_parser.add_argument("--timeout", type=float, default=60.0, help="statement timeout in seconds (default: 60)") + snapshot_parser.add_argument("--max-rows", type=int, default=100_000, help="refuse queries returning more rows (default: 100000)") + snapshot_parser.add_argument("--max-bytes", type=int, default=256 * 1024 * 1024, help="refuse snapshots larger than this (default: 256 MiB)") + snapshot_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + snapshot_parser.set_defaults(handler=_cmd_source_snapshot) + + checks_parser = subparsers.add_parser("checks", help="list the versioned check catalogue (openmapstack-check-api/v1)") + checks_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + checks_parser.set_defaults(handler=_cmd_checks) + + check_parser = subparsers.add_parser("check", help="run one named check against a project workspace") + check_parser.add_argument("name", help="check name, e.g. geodata.geometry_all_valid") + check_parser.add_argument("workspace", nargs="?", default=".", help="project directory (default: .)") + check_parser.add_argument( + "--arg", + action="append", + default=[], + metavar="KEY=VALUE", + help="check argument; VALUE is parsed as JSON when it is valid JSON, else used as a string", + ) + check_parser.add_argument("--json", action="store_true", help="emit the openmapstack-check-result/v1 record") + check_parser.set_defaults(handler=_cmd_check) + + api_parser = subparsers.add_parser("api-info", help="report API, schema, and package versions for consumers") + api_parser.add_argument("--require-api", help="check API the consumer was built for") + api_parser.add_argument("--min-version", help="oldest package version the consumer was tested against") + api_parser.add_argument("--require-check", action="append", default=[], help="a check the consumer needs; repeatable") + api_parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + api_parser.set_defaults(handler=_cmd_api_info) + + snapshot_parser = subparsers.add_parser( + "skill-snapshot", + help="copy SKILL.md, references/, and templates/ into a hashed, inspectable snapshot", + ) + snapshot_mode = snapshot_parser.add_mutually_exclusive_group(required=True) + snapshot_mode.add_argument("--out", type=Path, help="destination directory (must be empty or absent)") + snapshot_mode.add_argument("--inspect", type=Path, metavar="DIR", help="re-verify an existing snapshot instead of creating one") + snapshot_parser.add_argument("--source", type=Path, help="skill root holding SKILL.md (default: nearest ancestor of the current directory)") + snapshot_parser.add_argument("--json", action="store_true", help="emit the snapshot manifest or inspection as JSON") + snapshot_parser.set_defaults(handler=_cmd_skill_snapshot) + inspect_parser = subparsers.add_parser("inspect", help="summarize a project for audit and review") inspect_parser.add_argument("project", nargs="?", default="project.yaml", help="project.yaml or its directory") inspect_parser.add_argument("--json", action="store_true", help="emit the full summary as JSON") @@ -119,6 +191,7 @@ def _cmd_verify(args: argparse.Namespace) -> int: args.project, rerun=args.rerun, rerun_timeout_s=args.rerun_timeout, + metamorphic=args.metamorphic, ) except ProjectError as exc: print(f"openmapstack: {exc}", file=sys.stderr) @@ -256,6 +329,194 @@ def _cmd_run(args: argparse.Namespace) -> int: return 0 if validation.ok(strict=args.strict) else 1 +def _cmd_skill_snapshot(args: argparse.Namespace) -> int: + from .snapshot import SnapshotError, create_skill_snapshot, find_skill_root, inspect_skill_snapshot + + try: + if args.inspect is not None: + report = inspect_skill_snapshot(args.inspect) + if args.json: + print(_json(report)) + else: + state = "intact" if report["intact"] else "TAMPERED" + print(f"{state}: {report['snapshot']} ({report['file_count']} files, {report['content_sha256']})") + for problem in report["problems"]: + print(f" {problem}") + return 0 if report["intact"] else 1 + source = args.source or find_skill_root() + if source is None: + raise SnapshotError("no skill root found; pass --source DIR holding SKILL.md, references/, and templates/") + manifest = create_skill_snapshot(source, args.out) + except SnapshotError as exc: + if args.json: + print(_json({"schema": "openmapstack-skill-snapshot-error/v1", "error": str(exc)})) + else: + print(f"openmapstack skill-snapshot: {exc}", file=sys.stderr) + return 2 + if args.json: + print(_json(manifest)) + else: + print(f"Wrote {args.out} ({manifest['file_count']} files, {manifest['content_sha256']})") + return 0 + + +def _cmd_checks(args: argparse.Namespace) -> int: + from .api import CHECK_API_VERSION, list_checks + + descriptors = list_checks() + if args.json: + print(_json({"schema": "openmapstack-check-catalogue/v1", "api_version": CHECK_API_VERSION, + "package_version": __version__, "checks": [d.to_dict() for d in descriptors]})) + return 0 + print(f"{CHECK_API_VERSION} ({len(descriptors)} checks, openmapstack {__version__})") + for descriptor in descriptors: + required = [p.name for p in descriptor.parameters if p.required] + optional = [p.name for p in descriptor.parameters if not p.required] + oracle = "" if descriptor.oracle_free else " [known-answer]" + print(f" {descriptor.name:48s} {descriptor.dimension}{oracle}") + if required or optional: + print(f" args: required={required} optional={optional}") + return 0 + + +def _parse_check_args(pairs: Sequence[str]) -> dict[str, Any]: + parsed: dict[str, Any] = {} + for pair in pairs: + key, separator, value = pair.partition("=") + if not separator or not key.strip(): + raise ValueError(f"--arg expects KEY=VALUE, got {pair!r}") + try: + parsed[key.strip()] = json.loads(value) + except json.JSONDecodeError: + parsed[key.strip()] = value + return parsed + + +def _cmd_check(args: argparse.Namespace) -> int: + from .api import CheckAPIError, run_check + + try: + record = run_check(args.name, args.workspace, _parse_check_args(args.arg)) + except (CheckAPIError, ValueError) as exc: + if args.json: + print(_json({"schema": "openmapstack-check-result/v1", "status": "not_testable", "code": "consumer_error", "error": str(exc)})) + else: + print(f"openmapstack check: {exc}", file=sys.stderr) + return 2 + if args.json: + print(_json(record)) + else: + mark = STATUS_MARKS.get(record["status"], record["status"].upper()) + code = f" [{record['code']}]" if record.get("code") else "" + print(f"{mark} {record['check']}{code}: {record['detail']}") + return 0 if record["status"] != "failed" else 1 + + +def _cmd_api_info(args: argparse.Namespace) -> int: + from .api import CHECK_API_VERSION, CheckAPIError, api_info, negotiate + + info = api_info() + negotiation = None + if args.require_api or args.min_version or args.require_check: + try: + negotiation = negotiate( + required_api=args.require_api or CHECK_API_VERSION, + min_package_version=args.min_version, + required_checks=list(args.require_check), + ) + except CheckAPIError as exc: + print(f"openmapstack api-info: {exc}", file=sys.stderr) + return 2 + info["negotiation"] = negotiation + if args.json: + print(_json(info)) + else: + print(f"openmapstack {info['package_version']}: {info['check_api_version']}, {info['checks']} checks " + f"({info['oracle_free_checks']} oracle-free), project schema {info['project_schema']}") + if negotiation is not None: + print("compatible" if negotiation["compatible"] else "INCOMPATIBLE: " + "; ".join(negotiation["problems"])) + if negotiation is not None and not negotiation["compatible"]: + return 1 + return 0 + + +def _cmd_source_discover(args: argparse.Namespace) -> int: + from .connectors import ConnectorError, ConnectorLimits, discover_source + + try: + project_file, project = load_project(args.project) + discovery = discover_source( + project, args.source, project_root=project_file.parent, limits=ConnectorLimits(timeout_s=args.timeout) + ) + except (ProjectError, ConnectorError) as exc: + return _source_error(exc, args.json) + payload = discovery.to_dict() + if args.json: + print(_json(payload)) + return 0 + print(f"{discovery.backend} source {args.source!r}: {len(discovery.tables)} table(s)/file(s); read-only session: {discovery.read_only}") + for table in discovery.tables: + location = f"{table.schema}.{table.name}" if table.schema else table.name + srid = f"EPSG:{table.srid}" if table.srid else "srid unknown" + rows = f"~{table.row_estimate} rows" if table.row_estimate is not None else "rows unknown" + print(f" {location} [{table.kind}] geometry={table.geometry_column or '-'} {srid} {rows}") + for note in discovery.notes: + print(f" NOTE {note}") + return 0 + + +def _cmd_source_snapshot(args: argparse.Namespace) -> int: + from .connectors import ConnectorError, ConnectorLimits, apply_snapshot_to_manifest, snapshot_source + + query = args.query + if args.query_file is not None: + try: + query = args.query_file.read_text(encoding="utf-8") + except OSError as exc: + return _source_error(exc, args.json) + try: + project_file, project = load_project(args.project) + record = snapshot_source( + project, + args.source, + query, + args.destination, + project_root=project_file.parent, + approve=args.approve, + limits=ConnectorLimits(timeout_s=args.timeout, max_rows=args.max_rows, max_bytes=args.max_bytes), + ) + if args.write_manifest and record.get("materialized"): + updated = apply_snapshot_to_manifest(project, args.source, record) + project_file.write_text(yaml.safe_dump(updated, sort_keys=False, allow_unicode=True), encoding="utf-8") + record["manifest_written"] = str(project_file) + except (ProjectError, ConnectorError) as exc: + return _source_error(exc, args.json) + if args.json: + print(_json(record)) + return 0 + plan = record["plan"] + print(f"{record['backend']} source {args.source!r}: query {plan['query_sha256']} returns {plan['row_count']} row(s), {len(plan['columns'])} column(s)") + if not record["materialized"]: + print(f"DRY RUN: nothing written to {args.destination}; re-run with --approve to materialise") + return 0 + print(f"Wrote {args.destination} ({record['rows']} rows, {record['bytes']} bytes, {record['sha256']})") + if record.get("manifest_written"): + print(f"Updated {record['manifest_written']}") + else: + print("Add to project.yaml under sources.%s:" % args.source) + print(yaml.safe_dump({"pin": record["pin"], "warehouse": record["warehouse"]}, sort_keys=False).rstrip()) + return 0 + + +def _source_error(exc: Exception, as_json: bool) -> int: + code = getattr(exc, "code", type(exc).__name__) + if as_json: + print(_json({"schema": "openmapstack-source-error/v1", "status": "failed", "code": code, "error": str(exc)})) + else: + print(f"openmapstack source: {exc} [{code}]", file=sys.stderr) + return 2 + + def _cmd_inspect(args: argparse.Namespace) -> int: try: project_file, project = load_project(args.project) diff --git a/openmapstack/connectors/__init__.py b/openmapstack/connectors/__init__.py new file mode 100644 index 0000000..f2aae99 --- /dev/null +++ b/openmapstack/connectors/__init__.py @@ -0,0 +1,356 @@ +"""Safe, read-only connectors for user warehouse data (pilot). + +The connector surface is deliberately small and defensive: + +- **credentials by reference** -- ``access.connection`` names where the + credential lives (``env:NAME``, ``service:NAME``, ``file:/abs/path``); + the resolved value is used to open a session and is never recorded; +- **read-only discovery** -- a connector lists tables/files, geometry + columns, SRIDs, and row estimates through a session that cannot write; +- **explicit approval** -- ``snapshot_source`` never materialises data into + ``data/source/`` unless the caller passes ``approve=True``; without it the + call is a dry run that reports the schema and row count it *would* copy; +- **limits** -- a statement timeout, a row cap, and a byte cap apply to + every query; a query that exceeds any of them is refused, and a + half-written file is removed; +- **only SELECT** -- one statement, no DML/DDL/COPY/ATTACH keywords, no + statement separators; +- **redaction** -- every message a connector emits passes through + ``openmapstack.sources.redact``. + +Two backends are implemented as the reference pair: ``duckdb`` for local +files (GeoParquet, GeoJSON, GeoPackage, FlatGeobuf, ``.duckdb`` databases) +and ``postgis`` for PostgreSQL/PostGIS. Other warehouses are documented only +where their behaviour has been verified; they are not silently accepted. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from ..integrity import sha256_file +from ..project import get_in, project_path +from ..sources import CONNECTION_REFERENCE_SCHEMES, redact + +BACKENDS = ("duckdb", "postgis") + +_FORBIDDEN_KEYWORDS = re.compile( + r"\b(insert|update|delete|drop|alter|create|copy|grant|revoke|truncate|call|execute|" + r"attach|detach|install|load|pragma|set|reset|vacuum|merge|into|import|export|checkpoint|" + r"begin|commit|rollback|do|listen|notify|refresh|lock|security_invoker|pg_read_file|" + r"read_text|read_blob|glob)\b", + re.IGNORECASE, +) + + +class ConnectorError(Exception): + """A connector refused or failed an operation. Messages are redacted.""" + + def __init__(self, message: str, *, code: str) -> None: + super().__init__(redact(message)) + self.code = code + + +class ConnectorUnavailable(ConnectorError): + """The backend driver is not installed in this environment.""" + + def __init__(self, message: str) -> None: + super().__init__(message, code="driver_unavailable") + + +@dataclass(frozen=True) +class ConnectorLimits: + timeout_s: float = 60.0 + max_rows: int = 100_000 + max_bytes: int = 256 * 1024 * 1024 + + def validate(self) -> None: + if self.timeout_s <= 0 or self.max_rows <= 0 or self.max_bytes <= 0: + raise ConnectorError("limits must all be positive", code="limits_invalid") + + +@dataclass +class TableInfo: + schema: str | None + name: str + geometry_column: str | None + srid: int | None + geometry_type: str | None + row_estimate: int | None + kind: str = "table" # table | view | file + + def to_dict(self) -> dict[str, Any]: + return { + "schema": self.schema, + "name": self.name, + "kind": self.kind, + "geometry_column": self.geometry_column, + "srid": self.srid, + "geometry_type": self.geometry_type, + "row_estimate": self.row_estimate, + } + + +@dataclass +class Discovery: + backend: str + identity: dict[str, Any] + tables: list[TableInfo] = field(default_factory=list) + read_only: bool = True + notes: list[str] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + return { + "schema": "openmapstack-source-discovery/v1", + "backend": self.backend, + "identity": self.identity, + "read_only": self.read_only, + "tables": [table.to_dict() for table in self.tables], + "notes": self.notes, + } + + +@dataclass +class QueryPlan: + """What a snapshot would copy, established without materialising it.""" + + columns: list[dict[str, str]] + row_count: int + query_sha256: str + schema_sha256: str + backend_snapshot: dict[str, Any] | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "columns": self.columns, + "row_count": self.row_count, + "query_sha256": self.query_sha256, + "schema_sha256": self.schema_sha256, + "backend_snapshot": self.backend_snapshot, + } + + +def require_read_only_select(query: str) -> str: + """Accept exactly one SELECT/WITH statement with no side-effect keywords.""" + if not isinstance(query, str) or not query.strip(): + raise ConnectorError("query must be a non-empty SELECT statement", code="query_rejected") + text = query.strip().rstrip(";").strip() + stripped = re.sub(r"'(?:[^']|'')*'", "''", text) # ignore text inside string literals + if ";" in stripped: + raise ConnectorError("query must be a single statement", code="query_rejected") + if not re.match(r"(?is)^(select|with)\b", text): + raise ConnectorError("only SELECT (or WITH ... SELECT) queries are allowed", code="query_rejected") + match = _FORBIDDEN_KEYWORDS.search(stripped) + if match: + raise ConnectorError(f"query contains a forbidden keyword: {match.group(0).upper()}", code="query_rejected") + return text + + +def query_digest(query: str) -> str: + canonical = " ".join(query.strip().rstrip(";").split()) + return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def schema_digest(columns: list[dict[str, str]]) -> str: + canonical = json.dumps(columns, sort_keys=True, separators=(",", ":")) + return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def resolve_connection_reference(reference: object, *, environ: dict[str, str] | None = None) -> tuple[str, str]: + """Resolve ``access.connection`` to ``(scheme, secret)``. + + The secret is returned for opening a session only; callers must never + write it to a manifest, a log, or an evidence file. + """ + environ = os.environ if environ is None else environ + value = reference.get("ref") if isinstance(reference, dict) else reference + if not isinstance(value, str) or ":" not in value: + raise ConnectorError("access.connection must be a reference such as env:NAME", code="connection_reference_invalid") + scheme, _, remainder = value.partition(":") + remainder = remainder.strip() + if scheme not in CONNECTION_REFERENCE_SCHEMES or not remainder: + raise ConnectorError(f"unsupported connection reference scheme {scheme!r}", code="connection_reference_invalid") + if scheme == "env": + secret = environ.get(remainder) + if not secret: + raise ConnectorError(f"environment variable {remainder} is not set", code="connection_unresolved") + return scheme, secret + if scheme == "file": + path = Path(remainder).expanduser() + if not path.is_absolute(): + raise ConnectorError("file: connection references must be absolute paths outside the project", code="connection_reference_invalid") + try: + secret = path.read_text(encoding="utf-8").strip() + except OSError as exc: + raise ConnectorError(f"cannot read connection file: {type(exc).__name__}", code="connection_unresolved") from exc + if not secret: + raise ConnectorError("connection file is empty", code="connection_unresolved") + return scheme, secret + if scheme == "service": + return scheme, f"service={remainder}" + raise ConnectorError("keyring: references are not supported by the pilot connectors", code="connection_reference_invalid") + + +def load_connector(backend: str, connection: str, *, project_root: Path): + if backend == "duckdb": + from .duckdb_local import DuckDBLocalConnector + + return DuckDBLocalConnector(connection, project_root=project_root) + if backend == "postgis": + from .postgis import PostGISConnector + + return PostGISConnector(connection) + raise ConnectorError( + f"backend {backend!r} is not a verified connector; supported: {list(BACKENDS)}", + code="backend_unsupported", + ) + + +def _source_block(manifest: dict[str, Any], source_key: str) -> dict[str, Any]: + source = get_in(manifest, "sources", source_key) + if not isinstance(source, dict): + raise ConnectorError(f"source {source_key!r} is not declared", code="source_undeclared") + return source + + +def connector_for_source( + manifest: dict[str, Any], + source_key: str, + *, + project_root: Path, + environ: dict[str, str] | None = None, +): + source = _source_block(manifest, source_key) + backend = get_in(source, "warehouse", "backend") + if not isinstance(backend, str): + raise ConnectorError(f"source {source_key!r} declares no warehouse.backend", code="backend_undeclared") + reference = get_in(source, "access", "connection") + if backend == "duckdb" and reference is None: + # Local files need no credential: the "connection" is the project's + # own data/source directory. + return load_connector(backend, "", project_root=project_root) + _, secret = resolve_connection_reference(reference, environ=environ) + return load_connector(backend, secret, project_root=project_root) + + +def discover_source( + manifest: dict[str, Any], + source_key: str, + *, + project_root: Path, + limits: ConnectorLimits | None = None, + environ: dict[str, str] | None = None, +) -> Discovery: + limits = limits or ConnectorLimits() + limits.validate() + connector = connector_for_source(manifest, source_key, project_root=project_root, environ=environ) + return connector.discover(limits) + + +def snapshot_source( + manifest: dict[str, Any], + source_key: str, + query: str, + destination: str, + *, + project_root: Path, + approve: bool = False, + limits: ConnectorLimits | None = None, + environ: dict[str, str] | None = None, + now: datetime | None = None, +) -> dict[str, Any]: + """Plan, and with ``approve`` materialise, a query snapshot under data/source/. + + Returns a record with the plan, and when materialised the pin block and + warehouse metadata to place in the manifest. Nothing is written without + approval; a file that breaches ``limits`` is removed before returning. + """ + limits = limits or ConnectorLimits() + limits.validate() + query = require_read_only_select(query) + target = project_path(project_root, destination) + if target is None or not destination.replace("\\", "/").startswith("data/source/"): + raise ConnectorError("snapshot destination must be a project-relative path under data/source/", code="destination_invalid") + if target.suffix.lower() != ".parquet": + raise ConnectorError("snapshots are materialised as GeoParquet; use a .parquet destination", code="destination_invalid") + source = _source_block(manifest, source_key) + connector = connector_for_source(manifest, source_key, project_root=project_root, environ=environ) + plan = connector.plan(query, limits) + if plan.row_count > limits.max_rows: + raise ConnectorError( + f"query would return {plan.row_count} rows, above max_rows={limits.max_rows}", + code="row_limit_exceeded", + ) + record: dict[str, Any] = { + "schema": "openmapstack-source-snapshot/v1", + "source": source_key, + "backend": connector.backend, + "destination": destination, + "approved": bool(approve), + "materialized": False, + "plan": plan.to_dict(), + "limits": {"timeout_s": limits.timeout_s, "max_rows": limits.max_rows, "max_bytes": limits.max_bytes}, + } + if not approve: + record["note"] = "dry run: nothing was written; pass approve=True (--approve) to materialise" + return record + + if target.exists(): + raise ConnectorError( + f"{destination} already exists; sources are immutable, choose a new snapshot name", + code="destination_exists", + ) + target.parent.mkdir(parents=True, exist_ok=True) + temporary = target.with_name(target.name + ".partial") + try: + rows = connector.materialize(query, temporary, limits) + size = temporary.stat().st_size + if size > limits.max_bytes: + raise ConnectorError(f"snapshot is {size} bytes, above max_bytes={limits.max_bytes}", code="byte_limit_exceeded") + temporary.replace(target) + finally: + if temporary.exists(): + temporary.unlink() + captured_at = (now or datetime.now(timezone.utc)).isoformat().replace("+00:00", "Z") + digest = sha256_file(target) + warehouse = dict(source.get("warehouse") or {}) + warehouse.update({"query_sha256": plan.query_sha256, "schema_sha256": plan.schema_sha256}) + warehouse.update(connector.identity_for_manifest()) + record.update( + { + "materialized": True, + "rows": rows, + "bytes": target.stat().st_size, + "sha256": digest, + "pin": {"class": "local_snapshot", "path": destination, "sha256": digest, "captured_at": captured_at}, + "warehouse": warehouse, + "access": {"retrieved_at": captured_at, "downloaded_at": captured_at}, + } + ) + if plan.backend_snapshot: + record["backend_snapshot"] = plan.backend_snapshot + return record + + +def apply_snapshot_to_manifest(manifest: dict[str, Any], source_key: str, record: dict[str, Any]) -> dict[str, Any]: + """Return a copy of ``manifest`` with the snapshot's pin and metadata applied.""" + import copy + + updated = copy.deepcopy(manifest) + source = _source_block(updated, source_key) + if not record.get("materialized"): + raise ConnectorError("only a materialised snapshot can be written to the manifest", code="not_materialized") + source["pin"] = record["pin"] + source["warehouse"] = record["warehouse"] + access = source.setdefault("access", {}) + access.update(record["access"]) + file_block = access.setdefault("file", {}) + file_block.update({"name": Path(record["destination"]).name, "format": "GeoParquet", "row_count": record["rows"], "size_bytes": record["bytes"]}) + return updated diff --git a/openmapstack/connectors/duckdb_local.py b/openmapstack/connectors/duckdb_local.py new file mode 100644 index 0000000..7b7e9f6 --- /dev/null +++ b/openmapstack/connectors/duckdb_local.py @@ -0,0 +1,240 @@ +"""DuckDB connector over local files (the local half of the reference pair). + +The "connection" is a directory of geodata files, or a ``.duckdb`` database +opened read-only. Discovery uses DuckDB Spatial's metadata readers and never +loads whole files. Queries run in a fresh connection whose file access is +confined to that directory when the installed DuckDB supports +``allowed_directories``; otherwise the confinement gap is recorded in the +discovery notes rather than hidden. +""" + +from __future__ import annotations + +import threading +from pathlib import Path +from typing import Any + +from ..checks.spatial import connect_spatial +from . import ( + ConnectorError, + ConnectorLimits, + ConnectorUnavailable, + Discovery, + QueryPlan, + TableInfo, + query_digest, + schema_digest, +) + +GEO_SUFFIXES = {".parquet": "GeoParquet", ".geojson": "GeoJSON", ".json": "GeoJSON", ".gpkg": "GeoPackage", ".fgb": "FlatGeobuf", ".shp": "Shapefile"} +_MAX_DISCOVERED_FILES = 500 + + +def _escape(value: str) -> str: + return value.replace("'", "''") + + +class _Timeout: + """Interrupt a DuckDB connection after ``seconds``; DuckDB has no statement timeout.""" + + def __init__(self, connection: Any, seconds: float) -> None: + self._connection = connection + self._timer = threading.Timer(seconds, self._interrupt) + self.fired = False + + def _interrupt(self) -> None: + self.fired = True + try: + self._connection.interrupt() + except Exception: # noqa: BLE001 - best effort + pass + + def __enter__(self) -> "_Timeout": + self._timer.start() + return self + + def __exit__(self, *_: object) -> None: + self._timer.cancel() + + +class DuckDBLocalConnector: + backend = "duckdb" + + def __init__(self, connection: str, *, project_root: Path) -> None: + root = Path(connection).expanduser() if connection else project_root / "data" / "source" + if not root.is_absolute(): + root = (project_root / root) + self.root = root.resolve() + self.database: Path | None = None + if self.root.is_file() and self.root.suffix.lower() == ".duckdb": + self.database = self.root + self.root = self.root.parent + if not self.root.is_dir(): + raise ConnectorError(f"duckdb connector root is not a directory: {self.root.name}", code="connection_unresolved") + self.notes: list[str] = [] + + # -- sessions --------------------------------------------------------------- + + def _connect(self, *, allow: tuple[Path, ...] = ()): + connection = connect_spatial() + if connection is None: + raise ConnectorUnavailable("duckdb with the Spatial extension is required (pip install 'openmapstack[geo]')") + allowed = ", ".join(f"'{_escape(str(path))}'" for path in (self.root, *allow)) + try: + # Both statements must be literal SETs after the database has + # started; once external access is off it cannot be re-enabled + # for this connection, which is the point. + connection.execute(f"SET allowed_directories = [{allowed}]") + connection.execute("SET enable_external_access = false") + except Exception: # noqa: BLE001 - older DuckDB + note = "installed DuckDB does not support allowed_directories; file access is not confined to the source root" + if note not in self.notes: + self.notes.append(note) + if self.database is not None: + connection.execute(f"ATTACH '{_escape(str(self.database))}' AS warehouse (READ_ONLY)") + connection.execute("USE warehouse") + self._register_file_views(connection) + return connection + + def _candidate_files(self) -> list[Path]: + candidates = sorted(path for path in self.root.rglob("*") if path.is_file() and path.suffix.lower() in GEO_SUFFIXES) + if len(candidates) > _MAX_DISCOVERED_FILES: + note = f"only the first {_MAX_DISCOVERED_FILES} of {len(candidates)} geodata files were registered" + if note not in self.notes: + self.notes.append(note) + candidates = candidates[:_MAX_DISCOVERED_FILES] + return candidates + + def _register_file_views(self, connection) -> None: + """Expose every geodata file under the root as a view named by its + root-relative path, so a query says ``FROM "parcels.geojson"`` and + never spells a filesystem path of its own.""" + for path in self._candidate_files(): + relative = path.relative_to(self.root).as_posix() + escaped = _escape(path.as_posix()) + reader = f"read_parquet('{escaped}')" if path.suffix.lower() == ".parquet" else f"ST_Read('{escaped}')" + try: + connection.execute(f'CREATE VIEW "{relative.replace(chr(34), chr(34) * 2)}" AS SELECT * FROM {reader}') + except Exception as exc: # noqa: BLE001 - describe what can be described + note = f"{relative}: not readable ({type(exc).__name__})" + if note not in self.notes: + self.notes.append(note) + + def identity_for_manifest(self) -> dict[str, Any]: + identity: dict[str, Any] = {"backend": "duckdb"} + if self.database is not None: + identity["database"] = self.database.name + return identity + + # -- discovery -------------------------------------------------------------- + + def discover(self, limits: ConnectorLimits) -> Discovery: + connection = self._connect() + tables: list[TableInfo] = [] + try: + with _Timeout(connection, limits.timeout_s): + if self.database is not None: + tables.extend(self._discover_database(connection)) + tables.extend(self._discover_files(connection)) + finally: + connection.close() + identity = {"root": self.root.name, "database": self.database.name if self.database else None} + return Discovery("duckdb", identity, tables, read_only=True, notes=list(self.notes)) + + def _discover_database(self, connection) -> list[TableInfo]: + rows = connection.execute( + "SELECT table_schema, table_name, table_type FROM information_schema.tables " + "WHERE table_catalog = 'warehouse' ORDER BY 1, 2" + ).fetchall() + found: list[TableInfo] = [] + for schema, name, table_type in rows: + columns = connection.execute( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_catalog = 'warehouse' AND table_schema = ? AND table_name = ?", + [schema, name], + ).fetchall() + geometry = next((column for column, type_name in columns if str(type_name).upper() == "GEOMETRY"), None) + estimate = connection.execute(f'SELECT COUNT(*) FROM "{schema}"."{name}"').fetchone()[0] + found.append(TableInfo(schema, name, geometry, None, None, int(estimate), kind="view" if "VIEW" in str(table_type).upper() else "table")) + return found + + def _discover_files(self, connection) -> list[TableInfo]: + found: list[TableInfo] = [] + for path in self._candidate_files(): + relative = path.relative_to(self.root).as_posix() + try: + found.append(self._describe_file(connection, path, relative)) + except Exception as exc: # noqa: BLE001 - describe what can be described + note = f"{relative}: not readable ({type(exc).__name__})" + if note not in self.notes: + self.notes.append(note) + return found + + def _describe_file(self, connection, path: Path, relative: str) -> TableInfo: + escaped = _escape(path.as_posix()) + if path.suffix.lower() == ".parquet": + columns = connection.execute(f"DESCRIBE SELECT * FROM read_parquet('{escaped}')").fetchall() + geometry = next((str(name) for name, type_name, *_ in columns if str(type_name).upper() == "GEOMETRY"), None) + estimate = connection.execute(f"SELECT COUNT(*) FROM read_parquet('{escaped}')").fetchone()[0] + srid = None + if geometry: + crs_row = connection.execute(f'SELECT ST_CRS("{geometry}") FROM read_parquet(\'{escaped}\') LIMIT 1').fetchone() + srid = _srid_from(crs_row[0] if crs_row else None) + return TableInfo(None, relative, geometry, srid, None, int(estimate), kind="file") + meta = connection.execute(f"SELECT layers FROM ST_Read_Meta('{escaped}')").fetchone() + layer = (meta[0] or [None])[0] if meta else None + geometry = srid = geometry_type = None + estimate = None + if isinstance(layer, dict): + fields = layer.get("geometry_fields") or [] + if fields: + geometry = fields[0].get("name") or "geom" + geometry_type = fields[0].get("type") + srid = _srid_from(((fields[0].get("crs") or {}).get("auth_code"))) + estimate = layer.get("feature_count") + return TableInfo(None, relative, geometry, srid, geometry_type, int(estimate) if estimate is not None else None, kind="file") + + # -- queries ---------------------------------------------------------------- + + def plan(self, query: str, limits: ConnectorLimits) -> QueryPlan: + connection = self._connect() + try: + with _Timeout(connection, limits.timeout_s) as timeout: + try: + described = connection.execute(f"DESCRIBE SELECT * FROM ({query}) AS q").fetchall() + count = connection.execute(f"SELECT COUNT(*) FROM ({query}) AS q").fetchone()[0] + except Exception as exc: # noqa: BLE001 + if timeout.fired: + raise ConnectorError(f"query exceeded timeout_s={limits.timeout_s}", code="timeout") from exc + raise ConnectorError(f"query failed: {type(exc).__name__}: {exc}", code="query_failed") from exc + finally: + connection.close() + columns = [{"name": str(name), "type": str(type_name)} for name, type_name, *_ in described] + return QueryPlan(columns, int(count), query_digest(query), schema_digest(columns)) + + def materialize(self, query: str, destination: Path, limits: ConnectorLimits) -> int: + connection = self._connect(allow=(destination.resolve().parent,)) + try: + with _Timeout(connection, limits.timeout_s) as timeout: + try: + connection.execute( + f"COPY (SELECT * FROM ({query}) AS q LIMIT {int(limits.max_rows)}) " + f"TO '{_escape(destination.as_posix())}' (FORMAT PARQUET)" + ) + rows = connection.execute(f"SELECT COUNT(*) FROM read_parquet('{_escape(destination.as_posix())}')").fetchone()[0] + except Exception as exc: # noqa: BLE001 + if timeout.fired: + raise ConnectorError(f"query exceeded timeout_s={limits.timeout_s}", code="timeout") from exc + raise ConnectorError(f"snapshot failed: {type(exc).__name__}: {exc}", code="query_failed") from exc + finally: + connection.close() + return int(rows) + + +def _srid_from(value: object) -> int | None: + if value is None: + return None + text = str(value).strip().upper() + if text.startswith("EPSG:"): + text = text[5:] + return int(text) if text.isdigit() else None diff --git a/openmapstack/connectors/postgis.py b/openmapstack/connectors/postgis.py new file mode 100644 index 0000000..b7cce94 --- /dev/null +++ b/openmapstack/connectors/postgis.py @@ -0,0 +1,296 @@ +"""PostGIS connector (the warehouse half of the reference pair). + +Every session is opened with ``default_transaction_read_only = on`` and a +``statement_timeout``, discovery reads ``geometry_columns`` and planner +estimates, and a snapshot is materialised as GeoParquet through DuckDB from +rows fetched with geometry as WKB. + +PostgreSQL has no durable time travel: ``pg_export_snapshot()`` lives only as +long as its transaction. The pin for a PostGIS source is therefore always a +``local_snapshot``; the transaction snapshot identity and the schema digest +are recorded beside it as *retrieval* metadata, not as a pin. + +The DB-API driver (``psycopg`` 3, or ``psycopg2``) is imported lazily and +its absence is reported as ``driver_unavailable``. A ``connect`` callable +may be injected for tests. +""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from . import ( + ConnectorError, + ConnectorLimits, + ConnectorUnavailable, + Discovery, + QueryPlan, + TableInfo, + query_digest, + schema_digest, +) + +_GEOMETRY_TYPES = {"geometry", "geography"} + + +def _default_connect() -> Callable[[str], Any]: + try: + import psycopg # type: ignore[import-not-found] + + return lambda dsn: psycopg.connect(dsn) + except ImportError: + pass + try: + import psycopg2 # type: ignore[import-not-found] + + return lambda dsn: psycopg2.connect(dsn) + except ImportError as exc: + raise ConnectorUnavailable("a PostgreSQL driver (psycopg or psycopg2) is required for the postgis connector") from exc + + +class PostGISConnector: + backend = "postgis" + + def __init__(self, dsn: str, *, connect: Callable[[str], Any] | None = None) -> None: + self._dsn = dsn + self._connect = connect + self._identity: dict[str, Any] = {} + + # -- sessions --------------------------------------------------------------- + + def _session(self, limits: ConnectorLimits): + connect = self._connect or _default_connect() + try: + connection = connect(self._dsn) + except ConnectorError: + raise + except Exception as exc: # noqa: BLE001 - driver errors may carry the DSN + raise ConnectorError(f"cannot connect to PostGIS: {type(exc).__name__}", code="connection_failed") from exc + cursor = connection.cursor() + cursor.execute("SET default_transaction_read_only = on") + cursor.execute("SET transaction_read_only = on") + # SET cannot take a bound parameter; the value is an int we computed. + cursor.execute(f"SET statement_timeout = {int(limits.timeout_s * 1000)}") + return connection, cursor + + def identity_for_manifest(self) -> dict[str, Any]: + identity = {"backend": "postgis"} + for key in ("database", "server_version"): + if self._identity.get(key): + identity[key] = self._identity[key] + return identity + + # -- discovery -------------------------------------------------------------- + + def discover(self, limits: ConnectorLimits) -> Discovery: + connection, cursor = self._session(limits) + try: + cursor.execute("SELECT current_database(), current_user, version()") + database, user, version = cursor.fetchone() + self._identity = {"database": database, "user": user, "server_version": str(version).split(",")[0]} + cursor.execute( + "SELECT g.f_table_schema, g.f_table_name, g.f_geometry_column, g.srid, g.type, " + "c.reltuples::bigint, c.relkind " + "FROM geometry_columns g " + "LEFT JOIN pg_namespace n ON n.nspname = g.f_table_schema " + "LEFT JOIN pg_class c ON c.relname = g.f_table_name AND c.relnamespace = n.oid " + "ORDER BY 1, 2, 3" + ) + tables = [ + TableInfo( + str(schema), str(name), str(column), int(srid) if srid is not None else None, + str(geometry_type) if geometry_type else None, + int(estimate) if estimate is not None and estimate >= 0 else None, + kind="view" if relkind in ("v", "m") else "table", + ) + for schema, name, column, srid, geometry_type, estimate, relkind in cursor.fetchall() + ] + cursor.execute("SHOW default_transaction_read_only") + read_only = str(cursor.fetchone()[0]).lower() in {"on", "true", "1"} + except ConnectorError: + raise + except Exception as exc: # noqa: BLE001 + raise ConnectorError(f"discovery failed: {type(exc).__name__}", code="discovery_failed") from exc + finally: + _close(connection) + notes = ["row_estimate comes from the planner (pg_class.reltuples); -1 means never analysed"] + return Discovery("postgis", dict(self._identity), tables, read_only=read_only, notes=notes) + + # -- queries ---------------------------------------------------------------- + + def _columns(self, cursor, query: str) -> list[dict[str, str]]: + cursor.execute(f"SELECT * FROM ({query}) AS q LIMIT 0") + description = cursor.description or [] + oids = sorted({int(column[1]) for column in description if column[1] is not None}) + names: dict[int, str] = {} + if oids: + cursor.execute("SELECT oid, typname FROM pg_type WHERE oid = ANY(%s)", (oids,)) + names = {int(oid): str(typname) for oid, typname in cursor.fetchall()} + return [{"name": str(column[0]), "type": names.get(int(column[1]), str(column[1]))} for column in description] + + def plan(self, query: str, limits: ConnectorLimits) -> QueryPlan: + connection, cursor = self._session(limits) + try: + columns = self._columns(cursor, query) + cursor.execute(f"SELECT count(*) FROM ({query}) AS q") + count = int(cursor.fetchone()[0]) + backend_snapshot: dict[str, Any] | None = None + try: + cursor.execute("SELECT pg_current_snapshot()::text") + backend_snapshot = { + "kind": "pg_current_snapshot", + "value": str(cursor.fetchone()[0]), + "durable": False, + "note": "PostgreSQL keeps no durable time travel; the pin is the local snapshot", + } + except Exception: # noqa: BLE001 - PostgreSQL < 13 + try: + connection.rollback() + except Exception: # noqa: BLE001 + pass + except ConnectorError: + raise + except Exception as exc: # noqa: BLE001 + raise ConnectorError(f"query failed: {type(exc).__name__}", code="query_failed") from exc + finally: + _close(connection) + return QueryPlan(columns, count, query_digest(query), schema_digest(columns), backend_snapshot) + + def materialize(self, query: str, destination: Path, limits: ConnectorLimits) -> int: + from ..checks.spatial import connect_spatial + + duck = connect_spatial() + if duck is None: + raise ConnectorUnavailable("materialising GeoParquet requires duckdb with Spatial (pip install 'openmapstack[geo]')") + connection, cursor = self._session(limits) + try: + columns = self._columns(cursor, query) + geometry_columns = [column["name"] for column in columns if column["type"] in _GEOMETRY_TYPES] + selected = ", ".join( + f'ST_AsBinary("{column["name"]}") AS "{column["name"]}"' if column["name"] in geometry_columns else f'"{column["name"]}"' + for column in columns + ) + cursor.execute(f"SELECT {selected} FROM ({query}) AS q LIMIT %s", (int(limits.max_rows),)) + rows = cursor.fetchall() + srids: dict[str, int | None] = {} + for name in geometry_columns: + cursor.execute(f'SELECT ST_SRID("{name}") FROM ({query}) AS q WHERE "{name}" IS NOT NULL LIMIT 1') + row = cursor.fetchone() + srids[name] = int(row[0]) if row and row[0] else None + except ConnectorError: + raise + except Exception as exc: # noqa: BLE001 + raise ConnectorError(f"snapshot failed: {type(exc).__name__}", code="query_failed") from exc + finally: + _close(connection) + try: + _write_parquet(duck, destination, columns, geometry_columns, srids, rows) + finally: + duck.close() + return len(rows) + + +def _decimal_type(values) -> str: + """A DuckDB type that holds every NUMERIC value exactly. + + ``numeric`` has arbitrary precision in PostgreSQL. Mapping it to DOUBLE + would round identifiers and high-precision measurements while the file + is hashed and pinned as the immutable source -- the snapshot would then + be reproducible and wrong. Infer the widest scale and precision present + and use DECIMAL; beyond DECIMAL(38) keep the exact text instead. + """ + from decimal import Decimal + + max_scale = 0 + max_integer_digits = 1 + for value in values: + if value is None: + continue + if not isinstance(value, Decimal): + value = Decimal(str(value)) + if not value.is_finite(): + return "VARCHAR" + sign, digits, exponent = value.as_tuple() + scale = max(0, -exponent) + integer_digits = max(1, len(digits) + exponent) + max_scale = max(max_scale, scale) + max_integer_digits = max(max_integer_digits, integer_digits) + precision = max_integer_digits + max_scale + if precision > 38: + return "VARCHAR" + return f"DECIMAL({precision}, {max_scale})" + + +def _write_parquet(duck, destination: Path, columns, geometry_columns, srids, rows) -> None: + duck_types = { + "int2": "SMALLINT", "int4": "INTEGER", "int8": "BIGINT", "float4": "FLOAT", "float8": "DOUBLE", + "bool": "BOOLEAN", "date": "DATE", "timestamp": "TIMESTAMP", "timestamptz": "TIMESTAMPTZ", + "json": "JSON", "jsonb": "JSON", "uuid": "UUID", + } + definitions = [] + exact_text_columns: set[int] = set() + for index, column in enumerate(columns): + if column["name"] in geometry_columns: + definitions.append(f'"{column["name"]}" BLOB') + elif column["type"] == "numeric": + decimal_type = _decimal_type(row[index] for row in rows) + if decimal_type == "VARCHAR": + exact_text_columns.add(index) + definitions.append(f'"{column["name"]}" {decimal_type}') + else: + definitions.append(f'"{column["name"]}" {duck_types.get(column["type"], "VARCHAR")}') + duck.execute(f"CREATE TABLE staging ({', '.join(definitions)})") + placeholders = ", ".join("?" for _ in columns) + if rows: + duck.executemany( + f"INSERT INTO staging VALUES ({placeholders})", + [tuple(_plain(value, exact_text=index in exact_text_columns) for index, value in enumerate(row)) for row in rows], + ) + selected = [] + for column in columns: + name = column["name"] + if name in geometry_columns: + srid = srids.get(name) + geometry_expression = f'ST_GeomFromWKB("{name}")' + if srid: + try: + duck.execute(f"SELECT ST_SetSRID(ST_GeomFromWKB(NULL::BLOB), {int(srid)})") + geometry_expression = f'ST_SetSRID(ST_GeomFromWKB("{name}"), {int(srid)})' + except Exception: # noqa: BLE001 - older Spatial without CRS support + pass + selected.append(f'{geometry_expression} AS "{name}"') + else: + selected.append(f'"{name}"') + duck.execute(f"COPY (SELECT {', '.join(selected)} FROM staging) TO '{destination.as_posix().replace(chr(39), chr(39) * 2)}' (FORMAT PARQUET)") + + +def _plain(value: Any, *, exact_text: bool = False) -> Any: + from decimal import Decimal + + if isinstance(value, memoryview): + return bytes(value) + if isinstance(value, Decimal): + # DuckDB binds Decimal exactly for DECIMAL columns. For the VARCHAR + # fallback the text must be rendered here: bound as a Decimal it would + # be cast through DOUBLE and lose the digits the fallback exists for. + if exact_text or not value.is_finite(): + return format(value, "f") if value.is_finite() else str(value) + return value + if isinstance(value, (dict, list)): + import json + + return json.dumps(value) + return value + + +def _close(connection) -> None: + try: + connection.rollback() + except Exception: # noqa: BLE001 + pass + try: + connection.close() + except Exception: # noqa: BLE001 + pass diff --git a/openmapstack/metamorphic.py b/openmapstack/metamorphic.py new file mode 100644 index 0000000..ed1c0d8 --- /dev/null +++ b/openmapstack/metamorphic.py @@ -0,0 +1,661 @@ +"""Conditional metamorphic relations for projects with no golden answer. + +A metamorphic relation asks: if I perturb an input or a parameter in a +controlled way, does the output change the way the analysis semantics say it +must? It needs no frozen answer, which is what makes it usable on a user's +own data -- and it is *conditional*: each relation holds only under declared +preconditions, and a relation that cannot be addressed safely reports +``not_testable`` with the reason, never a pass. + +Relations are declared in ``validation.metamorphic[]``: + +.. code-block:: yaml + + validation: + metamorphic: + - id: parcel-order + relation: input_permutation_invariance + source: {path: data/source/parcels.geojson} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: + tie_break: "candidates are keyed by cadastral_id; no order-dependent selection" + - id: parcel-duplicates + relation: duplicate_resistance + source: {path: data/source/parcels.geojson} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: {dedup_key: cadastral_id, measure: set} + - id: road-distance-monotonic + relation: positive_buffer_monotonicity + parameter: road_distance_m + variant: {multiply: 1.5} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: {predicate: within_distance, expected: superset} + +Every relation runs the canonical entrypoint in an isolated copy prepared +exactly like a clean rerun (``openmapstack.rerun.prepare_clean_workspace``), +perturbs only that copy, compares the copy's outputs with the project's +produced outputs, and removes the copy. The project's own ``data/source`` and +``data/overrides`` are hashed before and after; a variant that reaches back +and mutates them fails outright. + +Implemented relations and their machine-checked preconditions: + +``input_permutation_invariance`` + Source features are shuffled (deterministically, from ``seed``). Every + listed output must be semantically equal. Precondition: a declared + ``tie_break`` rule and a ``key`` that is unique in the baseline output. +``duplicate_resistance`` + Every source feature is appended once more. Outputs must be equal. + Precondition: ``dedup_key`` is unique in the source, and ``measure`` is + ``set``. This relation is invalid for counts and sums, which legitimately + change when rows are duplicated; declaring another measure is rejected. +``positive_buffer_monotonicity`` + A declared numeric parameter is increased through its binding. Every + baseline output key must survive in the variant (``superset``). + Precondition: the parameter is declared, positive, and the variant is a + strict increase; the ``predicate`` names an inclusion predicate. + +Candidate relations that are *not* implemented here (CRS round trip, subset +additivity, area-scale consistency) must not be declared; the verifier +rejects unknown relation names rather than silently skipping them. +""" + +from __future__ import annotations + +import hashlib +import json +import random +import re +import shutil +import tempfile +import time +from collections.abc import Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from .checks import AssertionResult, failed, not_testable, passed +from .checks.rerun import _semantic_snapshot +from .checks.spatial import connect_spatial +from .parameters import ParameterError, declared_parameters +from .project import get_in, project_path +from .rerun import execute_canonical, prepare_clean_workspace + +METAMORPHIC_SCHEMA = "openmapstack-metamorphic/v1" +RELATIONS = ( + "input_permutation_invariance", + "duplicate_resistance", + "positive_buffer_monotonicity", +) +INCLUSION_PREDICATES = ("within_distance", "intersects_buffer", "within_buffer") +DEFAULT_TIMEOUT_S = 600.0 +DEFAULT_MAX_SOURCE_BYTES = 64 * 1024 * 1024 +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_DECLARATION_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_TRANSFORMABLE_SUFFIXES = {".geojson", ".json", ".parquet"} +_READABLE_OUTPUT_SUFFIXES = {".geojson", ".json", ".parquet"} + + +class DeclarationError(ValueError): + """A metamorphic declaration is structurally invalid.""" + + +@dataclass +class Declaration: + id: str + relation: str + outputs: list[str] + key: str + source_path: str | None = None + parameter: str | None = None + variant: dict[str, Any] = field(default_factory=dict) + preconditions: dict[str, Any] = field(default_factory=dict) + tolerance: dict[str, Any] = field(default_factory=dict) + timeout_s: float = DEFAULT_TIMEOUT_S + max_source_bytes: int = DEFAULT_MAX_SOURCE_BYTES + seed: int = 7 + + +def parse_declaration(raw: object) -> Declaration: + """Validate one ``validation.metamorphic[]`` entry structurally.""" + if not isinstance(raw, dict): + raise DeclarationError("declaration must be a mapping") + errors: list[str] = [] + declaration_id = raw.get("id") + if not isinstance(declaration_id, str) or _DECLARATION_ID.fullmatch(declaration_id) is None: + raise DeclarationError("declaration id is required") + relation = raw.get("relation") + if relation not in RELATIONS: + raise DeclarationError(f"{declaration_id}: relation must be one of {list(RELATIONS)}, got {relation!r}") + allowed = { + "id", "relation", "outputs", "key", "source", "parameter", "variant", + "preconditions", "tolerance", "limits", "seed", "description", + } + unknown = set(raw) - allowed + if unknown: + errors.append(f"unknown keys {sorted(unknown)}") + outputs = raw.get("outputs") + if not isinstance(outputs, list) or not outputs or any(not isinstance(item, str) or not item for item in outputs): + errors.append("outputs must be a non-empty list of output keys") + outputs = [] + key = raw.get("key") + if not isinstance(key, str) or _IDENTIFIER.fullmatch(key) is None: + errors.append("key must be a simple field identifier") + key = "" + preconditions = raw.get("preconditions") or {} + if not isinstance(preconditions, dict): + errors.append("preconditions must be a mapping") + preconditions = {} + tolerance = raw.get("tolerance") or {} + if not isinstance(tolerance, dict): + errors.append("tolerance must be a mapping") + tolerance = {} + limits = raw.get("limits") or {} + if not isinstance(limits, dict): + errors.append("limits must be a mapping") + limits = {} + timeout_s = limits.get("timeout_s", DEFAULT_TIMEOUT_S) + max_source_bytes = limits.get("max_source_bytes", DEFAULT_MAX_SOURCE_BYTES) + if isinstance(timeout_s, bool) or not isinstance(timeout_s, (int, float)) or timeout_s <= 0: + errors.append("limits.timeout_s must be a positive number") + timeout_s = DEFAULT_TIMEOUT_S + if isinstance(max_source_bytes, bool) or not isinstance(max_source_bytes, int) or max_source_bytes <= 0: + errors.append("limits.max_source_bytes must be a positive integer") + max_source_bytes = DEFAULT_MAX_SOURCE_BYTES + seed = raw.get("seed", 7) + if isinstance(seed, bool) or not isinstance(seed, int): + errors.append("seed must be an integer") + seed = 7 + + source_path: str | None = None + parameter: str | None = None + variant: dict[str, Any] = {} + if relation in {"input_permutation_invariance", "duplicate_resistance"}: + source = raw.get("source") + source_path = source.get("path") if isinstance(source, dict) else None + if not isinstance(source_path, str) or not source_path.strip(): + errors.append("source.path is required for this relation") + source_path = None + elif not (source_path.startswith("data/source/") or source_path.startswith("data/overrides/")): + errors.append("source.path must name a file under data/source/ or data/overrides/") + if "parameter" in raw or "variant" in raw: + errors.append("parameter/variant do not apply to this relation") + if relation == "input_permutation_invariance": + tie_break = preconditions.get("tie_break") + if not isinstance(tie_break, str) or not tie_break.strip(): + errors.append("preconditions.tie_break must state the deterministic ordering rule") + if relation == "duplicate_resistance": + dedup_key = preconditions.get("dedup_key") + if not isinstance(dedup_key, str) or _IDENTIFIER.fullmatch(dedup_key) is None: + errors.append("preconditions.dedup_key must be a simple field identifier") + measure = preconditions.get("measure", "set") + if measure != "set": + errors.append( + f"duplicate_resistance is valid only for set semantics; measure {measure!r} " + "(counts and sums change legitimately when rows are duplicated)" + ) + if relation == "positive_buffer_monotonicity": + parameter = raw.get("parameter") + if not isinstance(parameter, str) or _IDENTIFIER.fullmatch(parameter) is None: + errors.append("parameter must name a declared runtime parameter") + parameter = None + variant = raw.get("variant") or {} + if not isinstance(variant, dict) or len(variant) != 1 or not ({"multiply", "add"} & set(variant)): + errors.append("variant must declare exactly one of multiply/add") + variant = {} + else: + operation, amount = next(iter(variant.items())) + if isinstance(amount, bool) or not isinstance(amount, (int, float)): + errors.append(f"variant.{operation} must be a number") + variant = {} + elif operation == "multiply" and amount <= 1: + errors.append("variant.multiply must be > 1 so the buffer strictly grows") + variant = {} + elif operation == "add" and amount <= 0: + errors.append("variant.add must be > 0 so the buffer strictly grows") + variant = {} + predicate = preconditions.get("predicate") + if predicate not in INCLUSION_PREDICATES: + errors.append(f"preconditions.predicate must be one of {list(INCLUSION_PREDICATES)}") + if preconditions.get("expected", "superset") != "superset": + errors.append("positive_buffer_monotonicity establishes only expected: superset") + if "source" in raw: + errors.append("source does not apply to a parameter relation") + if errors: + raise DeclarationError(f"{declaration_id}: " + "; ".join(errors)) + return Declaration( + id=declaration_id, + relation=relation, + outputs=list(outputs), + key=key, + source_path=source_path, + parameter=parameter, + variant=dict(variant), + preconditions=dict(preconditions), + tolerance=dict(tolerance), + timeout_s=float(timeout_s), + max_source_bytes=int(max_source_bytes), + seed=int(seed), + ) + + +def declared_relations(manifest: dict[str, Any]) -> list[object]: + raw = get_in(manifest, "validation", "metamorphic") + if raw is None: + return [] + if not isinstance(raw, list): + raise DeclarationError("validation.metamorphic must be a list") + return list(raw) + + +# --- data access ------------------------------------------------------------ + + +def _read_features(path: Path) -> tuple[dict[str, Any], list[Any]]: + document = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(document, dict) or document.get("type") != "FeatureCollection": + raise ValueError(f"{path.name} is not a GeoJSON FeatureCollection") + features = document.get("features") + if not isinstance(features, list): + raise ValueError(f"{path.name} has no features list") + return document, features + + +def _write_features(path: Path, document: dict[str, Any], features: list[Any]) -> None: + replaced = dict(document) + replaced["features"] = features + path.write_text(json.dumps(replaced, ensure_ascii=False), encoding="utf-8") + + +def _duckdb_or_none(): + return connect_spatial() + + +def _key_values(path: Path, key: str) -> list[Any]: + """Every value of ``key`` in an output artifact, in file order.""" + suffix = path.suffix.lower() + if suffix in {".geojson", ".json"}: + _, features = _read_features(path) + values = [] + for feature in features: + properties = feature.get("properties") if isinstance(feature, dict) else None + if not isinstance(properties, dict) or key not in properties: + raise KeyError(key) + values.append(properties[key]) + return values + if suffix == ".parquet": + connection = _duckdb_or_none() + if connection is None: + raise RuntimeError("duckdb_unavailable") + try: + escaped = path.as_posix().replace("'", "''") + columns = {row[0] for row in connection.execute(f"DESCRIBE SELECT * FROM read_parquet('{escaped}')").fetchall()} + if key not in columns: + raise KeyError(key) + rows = connection.execute(f'SELECT "{key}" FROM read_parquet(\'{escaped}\')').fetchall() + finally: + connection.close() + return [row[0] for row in rows] + raise ValueError("unsupported_format") + + +def _source_key_values(path: Path, key: str) -> list[Any]: + return _key_values(path, key) + + +def _hash_tree(root: Path) -> dict[str, str]: + hashes: dict[str, str] = {} + for directory in ("data/source", "data/overrides"): + base = root / directory + if not base.is_dir(): + continue + for item in sorted(base.rglob("*")): + if item.is_file(): + hashes[item.relative_to(root).as_posix()] = hashlib.sha256(item.read_bytes()).hexdigest() + return hashes + + +# --- transformations ---------------------------------------------------------- + + +def _permute_source(path: Path, seed: int) -> dict[str, Any]: + suffix = path.suffix.lower() + if suffix in {".geojson", ".json"}: + document, features = _read_features(path) + order = list(range(len(features))) + random.Random(seed).shuffle(order) + if len(features) > 1 and order == list(range(len(features))): + order.reverse() + _write_features(path, document, [features[index] for index in order]) + return {"transformation": "permute_features", "features": len(features), "seed": seed} + if suffix == ".parquet": + connection = _duckdb_or_none() + if connection is None: + raise RuntimeError("duckdb_unavailable") + try: + escaped = path.as_posix().replace("'", "''") + count = connection.execute(f"SELECT COUNT(*) FROM read_parquet('{escaped}')").fetchone()[0] + connection.execute(f"SELECT setseed({(seed % 1000) / 1000.0})") + temp = path.with_suffix(".permuted.tmp.parquet") + connection.execute( + f"COPY (SELECT * FROM read_parquet('{escaped}') ORDER BY random()) " + f"TO '{temp.as_posix().replace(chr(39), chr(39) * 2)}' (FORMAT PARQUET)" + ) + finally: + connection.close() + temp.replace(path) + return {"transformation": "permute_rows", "rows": count, "seed": seed} + raise ValueError("unsupported_format") + + +def _duplicate_source(path: Path) -> dict[str, Any]: + suffix = path.suffix.lower() + if suffix in {".geojson", ".json"}: + document, features = _read_features(path) + _write_features(path, document, features + [json.loads(json.dumps(item)) for item in features]) + return {"transformation": "duplicate_features", "features": len(features), "duplicated": len(features)} + if suffix == ".parquet": + connection = _duckdb_or_none() + if connection is None: + raise RuntimeError("duckdb_unavailable") + try: + escaped = path.as_posix().replace("'", "''") + count = connection.execute(f"SELECT COUNT(*) FROM read_parquet('{escaped}')").fetchone()[0] + temp = path.with_suffix(".duplicated.tmp.parquet") + connection.execute( + f"COPY (SELECT * FROM read_parquet('{escaped}') UNION ALL SELECT * FROM read_parquet('{escaped}')) " + f"TO '{temp.as_posix().replace(chr(39), chr(39) * 2)}' (FORMAT PARQUET)" + ) + finally: + connection.close() + temp.replace(path) + return {"transformation": "duplicate_rows", "rows": count, "duplicated": count} + raise ValueError("unsupported_format") + + +# --- execution ---------------------------------------------------------------- + + +def _declared_output_files(manifest: dict[str, Any], keys: Sequence[str]) -> dict[str, str]: + outputs = manifest.get("outputs") if isinstance(manifest.get("outputs"), dict) else {} + resolved: dict[str, str] = {} + for key in keys: + spec = outputs.get(key) + path = spec.get("path") if isinstance(spec, dict) else None + if not isinstance(path, str) or not path.strip(): + raise DeclarationError(f"outputs[{key!r}] is not a declared output with a path") + resolved[key] = path + return resolved + + +def run_relation( + project_root: Path, + manifest: dict[str, Any], + raw_declaration: object, + *, + forbidden_fragments: Sequence[str] = (), +) -> tuple[AssertionResult, dict[str, Any]]: + """Execute one declared relation and return (result, evidence). + + The result vocabulary: + + - ``failed``: the relation was executed and does not hold, or the + declaration is invalid, or the variant reached back into the project's + immutable inputs; + - ``not_testable``: a precondition does not hold on this data or the + environment cannot run the relation (unsupported format, DuckDB absent, + timeout, oversize source); + - ``passed``: the variant ran and the declared relation holds. + """ + evidence: dict[str, Any] = {"schema": METAMORPHIC_SCHEMA} + try: + declaration = parse_declaration(raw_declaration) + except DeclarationError as exc: + evidence["class"] = "invalid" + return failed(str(exc), code="metamorphic_declaration_invalid"), evidence + evidence.update({"id": declaration.id, "relation": declaration.relation}) + root = project_root.resolve() + + try: + output_files = _declared_output_files(manifest, declaration.outputs) + except DeclarationError as exc: + evidence["class"] = "invalid" + return failed(f"{declaration.id}: {exc}", code="metamorphic_declaration_invalid"), evidence + for output_key, relative in output_files.items(): + target = project_path(root, relative) + if target is None: + evidence["class"] = "invalid" + return failed(f"{declaration.id}: output {output_key!r} path is unsafe", code="metamorphic_declaration_invalid"), evidence + if not target.is_file(): + return not_testable( + f"{declaration.id}: baseline output {relative} does not exist; run the pipeline first", + code="baseline_missing", + ), evidence + if target.suffix.lower() not in _READABLE_OUTPUT_SUFFIXES: + return not_testable( + f"{declaration.id}: output {relative} is {target.suffix or 'extensionless'}, which the relation cannot compare", + code="unsupported_format", + ), evidence + + # Baseline keys: every listed output must expose a unique key. + baseline_keys: dict[str, list[Any]] = {} + for output_key, relative in output_files.items(): + try: + values = _key_values(root / relative, declaration.key) + except KeyError: + return not_testable( + f"{declaration.id}: output {relative} has no field {declaration.key!r}", + code="precondition_unmet", + ), evidence + except RuntimeError as exc: + return not_testable(f"{declaration.id}: {exc}", code="duckdb_unavailable"), evidence + except (ValueError, OSError, json.JSONDecodeError) as exc: + return not_testable(f"{declaration.id}: cannot read {relative}: {exc}", code="unsupported_format"), evidence + if len(set(map(_hashable, values))) != len(values): + return not_testable( + f"{declaration.id}: key {declaration.key!r} is not unique in {relative}, so set semantics cannot be asserted", + code="precondition_unmet", + ), evidence + baseline_keys[output_key] = values + + # Relation-specific preconditions and the variant plan. + extra_argv: list[str] = [] + extra_env: dict[str, str] = {} + source_relative: str | None = None + if declaration.relation in {"input_permutation_invariance", "duplicate_resistance"}: + assert declaration.source_path is not None + source_relative = declaration.source_path + source_target = project_path(root, source_relative) + if source_target is None or not source_target.is_file(): + return not_testable(f"{declaration.id}: source {source_relative} does not exist", code="precondition_unmet"), evidence + if source_target.suffix.lower() not in _TRANSFORMABLE_SUFFIXES: + return not_testable( + f"{declaration.id}: source {source_relative} is {source_target.suffix or 'extensionless'}, which cannot be transformed", + code="unsupported_format", + ), evidence + size = source_target.stat().st_size + if size > declaration.max_source_bytes: + return not_testable( + f"{declaration.id}: source {source_relative} is {size} bytes, above limits.max_source_bytes={declaration.max_source_bytes}", + code="resource_limit", + ), evidence + if declaration.relation == "duplicate_resistance": + dedup_key = declaration.preconditions["dedup_key"] + try: + source_values = _source_key_values(source_target, dedup_key) + except KeyError: + return not_testable( + f"{declaration.id}: source {source_relative} has no field {dedup_key!r}", + code="precondition_unmet", + ), evidence + except RuntimeError as exc: + return not_testable(f"{declaration.id}: {exc}", code="duckdb_unavailable"), evidence + except (ValueError, OSError, json.JSONDecodeError) as exc: + return not_testable(f"{declaration.id}: cannot read {source_relative}: {exc}", code="unsupported_format"), evidence + if len(set(map(_hashable, source_values))) != len(source_values): + return not_testable( + f"{declaration.id}: source {source_relative} already has duplicate {dedup_key!r} values; " + "the analysis cannot be deduplicating on that key", + code="precondition_unmet", + ), evidence + evidence["dedup_key"] = dedup_key + else: + assert declaration.parameter is not None + try: + parameters = {parameter.id: parameter for parameter in declared_parameters(manifest)} + except ParameterError as exc: + return failed(f"{declaration.id}: {exc}", code="parameters_invalid"), evidence + parameter = parameters.get(declaration.parameter) + if parameter is None: + return failed( + f"{declaration.id}: parameter {declaration.parameter!r} is not declared under runtime.implementation.parameters", + code="metamorphic_declaration_invalid", + ), evidence + if parameter.type not in {"integer", "number"}: + return not_testable( + f"{declaration.id}: parameter {parameter.id!r} is {parameter.type}, not numeric", + code="precondition_unmet", + ), evidence + if parameter.canonical <= 0: + return not_testable( + f"{declaration.id}: parameter {parameter.id!r} canonical value {parameter.canonical!r} is not positive", + code="precondition_unmet", + ), evidence + operation, amount = next(iter(declaration.variant.items())) + variant_value = parameter.canonical * amount if operation == "multiply" else parameter.canonical + amount + if parameter.type == "integer": + variant_value = int(round(variant_value)) + if variant_value <= parameter.canonical: + return not_testable( + f"{declaration.id}: variant does not strictly increase the integer parameter", + code="precondition_unmet", + ), evidence + extra_argv, extra_env = parameter.bind(variant_value) + evidence["parameter"] = {"id": parameter.id, "canonical": parameter.canonical, "variant": variant_value} + + # Run the variant in an isolated copy; the project itself is read-only. + original_hashes = _hash_tree(root) + variant_root = Path(tempfile.mkdtemp(prefix=f"openmapstack-metamorphic-{declaration.id}-")) + started = time.monotonic() + try: + try: + command, preserved, _ = prepare_clean_workspace(root, variant_root, forbidden_fragments=forbidden_fragments) + except ValueError as exc: + return failed(f"{declaration.id}: cannot prepare variant workspace: {exc}", code="variant_preparation_failed"), evidence + evidence["preserved_paths"] = sorted(preserved) + if source_relative is not None: + variant_source = variant_root / source_relative + try: + if declaration.relation == "input_permutation_invariance": + evidence["variant"] = _permute_source(variant_source, declaration.seed) + else: + evidence["variant"] = _duplicate_source(variant_source) + except RuntimeError as exc: + return not_testable(f"{declaration.id}: {exc}", code="duckdb_unavailable"), evidence + except (ValueError, OSError, json.JSONDecodeError) as exc: + return not_testable(f"{declaration.id}: cannot transform {source_relative}: {exc}", code="unsupported_format"), evidence + else: + evidence["variant"] = {"transformation": "parameter", "argv": extra_argv, "environment": sorted(extra_env)} + + execution, removed = execute_canonical( + command, variant_root, declaration.timeout_s, extra_argv=extra_argv, extra_env=extra_env + ) + evidence["command"] = [*command, *extra_argv] + evidence["removed_environment_keys"] = removed + evidence["duration_s"] = time.monotonic() - started + if execution.get("timed_out"): + return not_testable( + f"{declaration.id}: variant run exceeded limits.timeout_s={declaration.timeout_s}", + code="variant_timeout", + ), evidence + if execution.get("returncode") != 0: + evidence["stderr_tail"] = (execution.get("stderr") or "")[-2000:] + return failed( + f"{declaration.id}: variant run exited with status {execution.get('returncode')}", + code="variant_execution_failed", + ), evidence + + if _hash_tree(root) != original_hashes: + return failed( + f"{declaration.id}: the variant run mutated the project's declared-immutable inputs", + code="original_source_mutated", + ), evidence + + # Compare. + differences: list[str] = [] + for output_key, relative in output_files.items(): + baseline_file = root / relative + variant_file = variant_root / relative + if not variant_file.is_file(): + return failed( + f"{declaration.id}: variant run did not produce output {relative}", + code="variant_output_missing", + ), evidence + if declaration.relation == "positive_buffer_monotonicity": + try: + variant_keys = set(map(_hashable, _key_values(variant_file, declaration.key))) + except RuntimeError as exc: + return not_testable(f"{declaration.id}: {exc}", code="duckdb_unavailable"), evidence + except (KeyError, ValueError, OSError, json.JSONDecodeError) as exc: + return failed( + f"{declaration.id}: variant output {relative} lost field {declaration.key!r}: {exc}", + code="monotonicity_violated", + ), evidence + lost = sorted(str(value) for value in set(map(_hashable, baseline_keys[output_key])) - variant_keys) + evidence.setdefault("counts", {})[output_key] = { + "baseline": len(baseline_keys[output_key]), + "variant": len(variant_keys), + } + if lost: + differences.append(f"{relative} lost {len(lost)} baseline feature(s) when the buffer grew: {lost[:10]}") + else: + ignored = set(declaration.tolerance.get("ignored_fields") or []) + try: + equal = _snapshot(baseline_file, ignored, declaration.tolerance) == _snapshot( + variant_file, ignored, declaration.tolerance + ) + except RuntimeError as exc: + return not_testable(f"{declaration.id}: {exc}", code="duckdb_unavailable"), evidence + except Exception as exc: # noqa: BLE001 - normalization is best effort + return not_testable(f"{declaration.id}: cannot normalize {relative}: {exc}", code="normalize_error"), evidence + if not equal: + differences.append(f"{relative} changed") + if differences: + code = { + "input_permutation_invariance": "permutation_changed_output", + "duplicate_resistance": "duplicates_changed_output", + "positive_buffer_monotonicity": "monotonicity_violated", + }[declaration.relation] + return failed(f"{declaration.id}: {'; '.join(differences)}", code=code), evidence + return passed( + f"{declaration.id}: {declaration.relation} holds across {len(output_files)} output(s)" + ), evidence + finally: + shutil.rmtree(variant_root, ignore_errors=True) + + +def _snapshot(path: Path, ignored: set[str], tolerance: dict[str, Any]) -> Any: + snapshot = _semantic_snapshot(path, ignored) + digits = tolerance.get("round_numbers") + if isinstance(digits, int) and not isinstance(digits, bool): + snapshot = _round(snapshot, digits) + return snapshot + + +def _round(value: Any, digits: int) -> Any: + if isinstance(value, float): + return round(value, digits) + if isinstance(value, list): + return [_round(item, digits) for item in value] + if isinstance(value, dict): + return {key: _round(item, digits) for key, item in value.items()} + return value + + +def _hashable(value: Any) -> Any: + if isinstance(value, (list, dict)): + return json.dumps(value, sort_keys=True, default=str) + return value diff --git a/openmapstack/parameters.py b/openmapstack/parameters.py new file mode 100644 index 0000000..5cfb715 --- /dev/null +++ b/openmapstack/parameters.py @@ -0,0 +1,190 @@ +"""Versioned parameter addressing for a project's canonical entrypoint. + +A metamorphic relation that varies a threshold, and a benchmark that wants to +run the same pipeline at a different setting, both need one thing the +manifest did not previously give them: a way to *address* a parameter of the +pipeline without editing the pipeline. ``runtime.implementation.parameters`` +is that contract. + +.. code-block:: yaml + + runtime: + implementation: + pipeline: pipeline.py + parameters: + - id: road_distance_m + type: number + canonical: 2000 + binding: {argument: "--road-distance-m"} # or {environment: OMS_ROAD_DISTANCE_M} + step: road_distance # optional: the processing step that + field: max_distance_m # consumes it, so drift is checkable + +Rules: + +- ``id`` is a simple identifier, unique within the manifest; +- ``type`` is ``integer``, ``number``, or ``string`` and ``canonical`` has that + type (booleans are not numbers); +- exactly one binding: ``argument`` (a ``--long-flag``, passed as + ``--flag value``) or ``environment`` (an ``UPPER_SNAKE`` variable); +- ``step``/``field`` are optional but come together; when present the named + processing step must exist and its field must equal ``canonical``, so a + manifest cannot advertise one threshold while the step declares another. + +The canonical run passes nothing: a pipeline must produce the accepted result +with no arguments and no variables set. Bindings exist so a *variant* run can +say "same pipeline, this one knob turned". +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Any + +from .project import get_in + +PARAMETERS_SCHEMA = "openmapstack-parameters/v1" +PARAMETER_TYPES = ("integer", "number", "string") + +_IDENTIFIER = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_ARGUMENT = re.compile(r"^--[a-z][a-z0-9]*(?:-[a-z0-9]+)*$") +_ENVIRONMENT = re.compile(r"^[A-Z][A-Z0-9_]*$") + + +class ParameterError(ValueError): + """The parameters block is malformed or drifts from the processing steps.""" + + +@dataclass(frozen=True) +class Parameter: + id: str + type: str + canonical: Any + argument: str | None = None + environment: str | None = None + step: str | None = None + field: str | None = None + + def bind(self, value: Any) -> tuple[list[str], dict[str, str]]: + """Return the argv suffix and environment additions for ``value``.""" + rendered = _render(value, self.type) + if self.argument is not None: + return [self.argument, rendered], {} + assert self.environment is not None + return [], {self.environment: rendered} + + +def _render(value: Any, type_name: str) -> str: + if type_name == "integer": + return str(int(value)) + if type_name == "number": + number = float(value) + return str(int(number)) if number.is_integer() else repr(number) + return str(value) + + +def value_has_type(value: Any, type_name: str) -> bool: + if isinstance(value, bool): + return False + if type_name == "integer": + return isinstance(value, int) + if type_name == "number": + return isinstance(value, (int, float)) + return isinstance(value, str) + + +def declared_parameters(manifest: dict[str, Any]) -> list[Parameter]: + """Parse and validate ``runtime.implementation.parameters``. + + Returns an empty list when nothing is declared. Raises ``ParameterError`` + describing every problem found, so a caller can report one failure that + names all of them. + """ + raw = get_in(manifest, "runtime", "implementation", "parameters") + if raw is None: + return [] + if not isinstance(raw, list): + raise ParameterError("runtime.implementation.parameters must be a list") + steps = get_in(manifest, "processing", "steps", default=[]) or [] + steps_by_id = { + str(step.get("id")): step for step in steps if isinstance(step, dict) and step.get("id") is not None + } + errors: list[str] = [] + seen: set[str] = set() + parameters: list[Parameter] = [] + for index, entry in enumerate(raw): + where = f"parameters[{index}]" + if not isinstance(entry, dict): + errors.append(f"{where} must be a mapping") + continue + unknown = set(entry) - {"id", "type", "canonical", "binding", "step", "field", "description"} + if unknown: + errors.append(f"{where} has unknown keys {sorted(unknown)}") + parameter_id = entry.get("id") + if not isinstance(parameter_id, str) or _IDENTIFIER.fullmatch(parameter_id) is None: + errors.append(f"{where}.id must be a simple identifier") + continue + where = f"parameters[{parameter_id}]" + if parameter_id in seen: + errors.append(f"{where} is declared more than once") + seen.add(parameter_id) + type_name = entry.get("type") + if type_name not in PARAMETER_TYPES: + errors.append(f"{where}.type must be one of {list(PARAMETER_TYPES)}") + continue + if "canonical" not in entry or not value_has_type(entry["canonical"], type_name): + errors.append(f"{where}.canonical must be a {type_name}") + continue + binding = entry.get("binding") + argument = environment = None + if not isinstance(binding, dict) or len(binding) != 1: + errors.append(f"{where}.binding must declare exactly one of argument/environment") + elif "argument" in binding: + argument = binding["argument"] + if not isinstance(argument, str) or _ARGUMENT.fullmatch(argument) is None: + errors.append(f"{where}.binding.argument must be a --long-flag") + argument = None + elif "environment" in binding: + environment = binding["environment"] + if not isinstance(environment, str) or _ENVIRONMENT.fullmatch(environment) is None: + errors.append(f"{where}.binding.environment must be an UPPER_SNAKE variable name") + environment = None + else: + errors.append(f"{where}.binding must declare exactly one of argument/environment") + step = entry.get("step") + field = entry.get("field") + if (step is None) != (field is None): + errors.append(f"{where} must declare step and field together") + elif step is not None: + if not isinstance(step, str) or step not in steps_by_id: + errors.append(f"{where}.step {step!r} is not a processing step") + elif not isinstance(field, str) or field not in steps_by_id[step]: + errors.append(f"{where}.field {field!r} is not declared on step {step!r}") + elif steps_by_id[step][field] != entry["canonical"]: + errors.append( + f"{where}.canonical {entry['canonical']!r} != processing step " + f"{step!r}.{field} = {steps_by_id[step][field]!r}" + ) + if argument is None and environment is None: + continue + parameters.append( + Parameter( + id=parameter_id, + type=type_name, + canonical=entry["canonical"], + argument=argument, + environment=environment, + step=step if isinstance(step, str) else None, + field=field if isinstance(field, str) else None, + ) + ) + if errors: + raise ParameterError("; ".join(errors)) + return parameters + + +def find_parameter(manifest: dict[str, Any], parameter_id: str) -> Parameter | None: + for parameter in declared_parameters(manifest): + if parameter.id == parameter_id: + return parameter + return None diff --git a/openmapstack/rerun.py b/openmapstack/rerun.py index 9b19175..a0b9cb6 100644 --- a/openmapstack/rerun.py +++ b/openmapstack/rerun.py @@ -235,6 +235,69 @@ def _clean_rerun_environment() -> tuple[dict[str, str], list[str]]: def _write_clean_rerun_evidence(rerun_root: Path, evidence: dict[str, Any]) -> None: (rerun_root / CLEAN_RERUN_EVIDENCE).write_text(json.dumps(evidence, indent=2, default=str), encoding="utf-8") +def prepare_clean_workspace( + project_root: Path, + rerun_root: Path, + *, + forbidden_fragments: Sequence[str] = (), +) -> tuple[list[str], set[str], dict[str, Any]]: + """Copy only the clean-rerun inputs into ``rerun_root``. + + Returns the resolved canonical command, the set of preserved + project-relative paths, and the loaded manifest. Raises ``ValueError`` + for a manifest that cannot be rerun safely. Shared by the clean rerun + and by metamorphic variant runs, which need the same isolation but then + perturb one input or parameter before executing. + """ + manifest_path = project_root / "project.yaml" + if not manifest_path.is_file(): + raise ValueError("project.yaml is missing") + try: + project = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, yaml.YAMLError) as exc: + raise ValueError(f"project.yaml cannot be loaded: {exc}") from exc + if not isinstance(project, dict): + raise ValueError("project.yaml must contain a mapping") + + preserved: set[str] = set() + command, declared_paths = canonical_rerun_command( + project_root, project, forbidden_fragments=forbidden_fragments + ) + _copy_clean_rerun_path(project_root, rerun_root, "project.yaml", "project manifest", preserved) + for conventional_path in ("data/source", "data/overrides"): + if (project_root / conventional_path).exists(): + _copy_clean_rerun_path( + project_root, + rerun_root, + conventional_path, + f"clean-rerun input {conventional_path}", + preserved, + ) + for path, field_name in declared_paths: + _copy_clean_rerun_path(project_root, rerun_root, path, field_name, preserved) + return command, preserved, project + + +def execute_canonical( + command: list[str], + rerun_root: Path, + timeout_s: int | float, + *, + extra_argv: Sequence[str] = (), + extra_env: dict[str, str] | None = None, +) -> tuple[dict[str, Any], list[str]]: + """Run the canonical entrypoint in a sanitized environment. + + Returns the execution record and the list of environment keys removed. + ``extra_argv``/``extra_env`` carry parameter bindings for variant runs. + """ + env, removed_environment = _clean_rerun_environment() + if extra_env: + env.update(extra_env) + execution = _execute_argv([*command, *extra_argv], rerun_root, timeout_s, env) + return execution, removed_environment + + def perform_clean_rerun( project_root: Path, rerun_root: Path, @@ -259,38 +322,15 @@ def perform_clean_rerun( } preserved: set[str] = set() try: - manifest_path = project_root / "project.yaml" - if not manifest_path.is_file(): - raise ValueError("project.yaml is missing") - try: - project = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) - except (OSError, UnicodeError, yaml.YAMLError) as exc: - raise ValueError(f"project.yaml cannot be loaded: {exc}") from exc - if not isinstance(project, dict): - raise ValueError("project.yaml must contain a mapping") - - command, declared_paths = canonical_rerun_command( - project_root, project, forbidden_fragments=forbidden_fragments + command, preserved, _project = prepare_clean_workspace( + project_root, rerun_root, forbidden_fragments=forbidden_fragments ) - _copy_clean_rerun_path(project_root, rerun_root, "project.yaml", "project manifest", preserved) - for conventional_path in ("data/source", "data/overrides"): - if (project_root / conventional_path).exists(): - _copy_clean_rerun_path( - project_root, - rerun_root, - conventional_path, - f"clean-rerun input {conventional_path}", - preserved, - ) - for path, field_name in declared_paths: - _copy_clean_rerun_path(project_root, rerun_root, path, field_name, preserved) evidence["preserved_paths"] = sorted(preserved) source_hashes_before = _hash_immutable_inputs(rerun_root, preserved) evidence["command"] = command - env, removed_environment = _clean_rerun_environment() + execution, removed_environment = execute_canonical(command, rerun_root, timeout_s) evidence["removed_environment_keys"] = removed_environment - execution = _execute_argv(command, rerun_root, timeout_s, env) evidence["execution"] = execution if execution.get("timed_out"): evidence["stage"] = "canonical_execution" diff --git a/openmapstack/schemas/check-result-v1.schema.json b/openmapstack/schemas/check-result-v1.schema.json new file mode 100644 index 0000000..84aedee --- /dev/null +++ b/openmapstack/schemas/check-result-v1.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openmapstack/schemas/check-result-v1.schema.json", + "title": "One executed OpenMapStack check (openmapstack-check-result/v1)", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", "api_version", "package_version", "check", "dimension", + "oracle_free", "args", "status", "code", "detail", "data" + ], + "properties": { + "schema": {"const": "openmapstack-check-result/v1"}, + "api_version": {"const": "openmapstack-check-api/v1"}, + "package_version": {"type": "string", "pattern": "^\\d+\\.\\d+\\.\\d+"}, + "check": {"type": "string", "pattern": "^[a-z_]+[.][a-z_]+$"}, + "dimension": {"type": "string", "minLength": 1}, + "oracle_free": {"type": "boolean"}, + "args": {"type": "object"}, + "status": {"enum": ["passed", "failed", "warning", "not_testable"]}, + "code": {"type": ["string", "null"]}, + "detail": {"type": "string"}, + "data": {"type": "object"} + }, + "allOf": [ + { + "if": {"properties": {"status": {"const": "passed"}}}, + "then": {"properties": {"code": {"type": "null"}}} + } + ] +} diff --git a/openmapstack/schemas/project-v1.schema.json b/openmapstack/schemas/project-v1.schema.json index 7379e90..9853486 100644 --- a/openmapstack/schemas/project-v1.schema.json +++ b/openmapstack/schemas/project-v1.schema.json @@ -24,6 +24,139 @@ "jsonScalar": { "type": ["string", "number", "boolean", "null"] }, + "identifier": { + "type": "string", + "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" + }, + "runtimeParameter": { + "type": "object", + "required": ["id", "type", "canonical", "binding"], + "additionalProperties": false, + "properties": { + "id": {"$ref": "#/$defs/identifier"}, + "type": {"enum": ["integer", "number", "string"]}, + "canonical": {"type": ["integer", "number", "string"]}, + "binding": { + "type": "object", + "minProperties": 1, + "maxProperties": 1, + "additionalProperties": false, + "properties": { + "argument": {"type": "string", "pattern": "^--[a-z][a-z0-9]*(?:-[a-z0-9]+)*$"}, + "environment": {"type": "string", "pattern": "^[A-Z][A-Z0-9_]*$"} + } + }, + "step": {"type": "string", "minLength": 1}, + "field": {"$ref": "#/$defs/identifier"}, + "description": {"type": "string"} + }, + "dependentRequired": {"step": ["field"], "field": ["step"]} + }, + "metamorphicRelation": { + "type": "object", + "required": ["id", "relation", "outputs", "key"], + "additionalProperties": false, + "properties": { + "id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"}, + "relation": { + "enum": ["input_permutation_invariance", "duplicate_resistance", "positive_buffer_monotonicity"] + }, + "outputs": {"type": "array", "minItems": 1, "items": {"type": "string", "minLength": 1}}, + "key": {"$ref": "#/$defs/identifier"}, + "source": { + "type": "object", + "required": ["path"], + "additionalProperties": false, + "properties": {"path": {"$ref": "#/$defs/safeProjectPath"}} + }, + "parameter": {"$ref": "#/$defs/identifier"}, + "variant": { + "type": "object", + "minProperties": 1, + "maxProperties": 1, + "additionalProperties": false, + "properties": { + "multiply": {"type": "number", "exclusiveMinimum": 1}, + "add": {"type": "number", "exclusiveMinimum": 0} + } + }, + "preconditions": {"type": "object"}, + "tolerance": { + "type": "object", + "additionalProperties": false, + "properties": { + "ignored_fields": {"type": "array", "items": {"type": "string", "minLength": 1}}, + "round_numbers": {"type": "integer", "minimum": 0} + } + }, + "limits": { + "type": "object", + "additionalProperties": false, + "properties": { + "timeout_s": {"type": "number", "exclusiveMinimum": 0}, + "max_source_bytes": {"type": "integer", "minimum": 1} + } + }, + "seed": {"type": "integer"}, + "description": {"type": "string"} + }, + "allOf": [ + { + "if": {"properties": {"relation": {"const": "positive_buffer_monotonicity"}}}, + "then": {"required": ["parameter", "variant", "preconditions"], "not": {"required": ["source"]}}, + "else": { + "required": ["source"], + "not": {"anyOf": [{"required": ["parameter"]}, {"required": ["variant"]}]} + } + } + ] + }, + "sourcePin": { + "type": "object", + "required": ["class", "captured_at"], + "properties": { + "class": {"enum": ["local_snapshot", "backend_snapshot"]}, + "captured_at": {"type": "string", "minLength": 1}, + "path": {"$ref": "#/$defs/safeProjectPath"}, + "sha256": {"$ref": "#/$defs/digest"}, + "identifier": {"type": "string", "minLength": 1}, + "retention_until": {"type": "string", "minLength": 1}, + "verification": { + "type": "object", + "required": ["at", "status"], + "properties": { + "at": {"type": "string", "minLength": 1}, + "status": {"enum": ["accessible", "inaccessible"]} + } + }, + "note": {"type": "string"} + }, + "allOf": [ + { + "if": {"properties": {"class": {"const": "local_snapshot"}}}, + "then": {"required": ["path", "sha256"]}, + "else": {"required": ["identifier", "retention_until"]} + } + ] + }, + "connectionReference": { + "description": "A credential reference (env:, file:, service:, keyring:). The shape is open here so that an embedded DSN is rejected by the semantic credential check with a useful message rather than by a regex.", + "type": ["string", "object"] + }, + "warehouseSource": { + "type": "object", + "required": ["backend"], + "properties": { + "backend": {"type": "string", "minLength": 1}, + "account": {"type": "string"}, + "database": {"type": "string"}, + "schema": {"type": "string"}, + "table": {"type": "string"}, + "query_sha256": {"$ref": "#/$defs/digest"}, + "schema_sha256": {"$ref": "#/$defs/digest"}, + "snapshot_capability": {"type": "string"} + } + }, "expectationAttestation": { "type": "object", "required": ["status"], @@ -183,7 +316,21 @@ "assumptions": {"type": "array"} } }, - "sources": {"type": "object", "minProperties": 1}, + "sources": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "properties": { + "pin": {"$ref": "#/$defs/sourcePin"}, + "warehouse": {"$ref": "#/$defs/warehouseSource"}, + "access": { + "type": "object", + "properties": {"connection": {"$ref": "#/$defs/connectionReference"}} + } + } + } + }, "overrides": {"type": "array"}, "processing": { "type": "object", @@ -204,6 +351,10 @@ "expectations": { "type": "array", "items": {"$ref": "#/$defs/expectation"} + }, + "metamorphic": { + "type": "array", + "items": {"$ref": "#/$defs/metamorphicRelation"} } } }, @@ -237,6 +388,12 @@ "properties": { "implementation": { "type": "object", + "properties": { + "parameters": { + "type": "array", + "items": {"$ref": "#/$defs/runtimeParameter"} + } + }, "anyOf": [ {"required": ["pipeline"]}, {"required": ["command"]} diff --git a/openmapstack/schemas/verify-result-v1.schema.json b/openmapstack/schemas/verify-result-v1.schema.json new file mode 100644 index 0000000..7191ca6 --- /dev/null +++ b/openmapstack/schemas/verify-result-v1.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openmapstack/schemas/verify-result-v1.schema.json", + "title": "openmapstack verify --json (openmapstack-verify-result/v1)", + "type": "object", + "required": ["schema", "project_file", "status", "counts", "coverage", "checks"], + "properties": { + "schema": {"const": "openmapstack-verify-result/v1"}, + "project_file": {"type": "string", "minLength": 1}, + "status": {"enum": ["passed", "failed", "warning", "not_testable"]}, + "counts": { + "type": "object", + "required": ["passed", "warning", "not_testable", "failed"], + "additionalProperties": false, + "properties": { + "passed": {"type": "integer", "minimum": 0}, + "warning": {"type": "integer", "minimum": 0}, + "not_testable": {"type": "integer", "minimum": 0}, + "failed": {"type": "integer", "minimum": 0} + } + }, + "coverage": { + "type": "object", + "required": ["applicable", "executed", "not_testable", "execution_rate"], + "additionalProperties": false, + "properties": { + "applicable": {"type": "integer", "minimum": 0}, + "executed": {"type": "integer", "minimum": 0}, + "not_testable": {"type": "integer", "minimum": 0}, + "execution_rate": {"type": ["number", "null"], "minimum": 0, "maximum": 1} + } + }, + "checks": { + "type": "array", + "items": { + "type": "object", + "required": ["check", "status", "message"], + "properties": { + "check": {"type": "string", "minLength": 1}, + "status": {"enum": ["passed", "failed", "warning", "not_testable"]}, + "message": {"type": "string"}, + "args": {"type": "object"}, + "evidence": {"type": "object"}, + "code": {"type": "string", "minLength": 1} + } + } + }, + "clean_rerun": {"type": "object"} + } +} diff --git a/openmapstack/snapshot.py b/openmapstack/snapshot.py new file mode 100644 index 0000000..8babef8 --- /dev/null +++ b/openmapstack/snapshot.py @@ -0,0 +1,200 @@ +"""Controlled snapshots of the shipped skill (``openmapstack-skill-snapshot/v1``). + +A benchmark arm that injects the skill must say *which* skill: not a tag +that can move, not a commit whose working tree may have been dirty, but the +bytes the agent actually read. ``create_skill_snapshot`` copies exactly +``SKILL.md``, ``references/``, and ``templates/`` from a skill root, records a +per-file inventory and a content hash, and writes the manifest beside the +copy. ``inspect_skill_snapshot`` re-verifies one later. + +Safety: symlinks anywhere in the source tree are refused (a snapshot must +not quietly include files from outside the skill), and every inventory path +is confined to the snapshot root. The content hash covers the relative path +and the bytes of every file, so a renamed reference changes it. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +SNAPSHOT_SCHEMA = "openmapstack-skill-snapshot/v1" +SNAPSHOT_MANIFEST = "snapshot.json" +SKILL_ENTRYPOINT = "SKILL.md" +SKILL_DIRECTORIES = ("references", "templates") +_IGNORED = {"__pycache__", ".DS_Store"} + + +class SnapshotError(ValueError): + """The skill root or an existing snapshot is unusable.""" + + +def find_skill_root(start: Path | None = None) -> Path | None: + """Walk upwards from ``start`` to the nearest directory holding SKILL.md.""" + current = (start or Path.cwd()).resolve() + for candidate in (current, *current.parents): + if (candidate / SKILL_ENTRYPOINT).is_file() and all((candidate / name).is_dir() for name in SKILL_DIRECTORIES): + return candidate + return None + + +def _iter_skill_files(root: Path) -> list[Path]: + entrypoint = root / SKILL_ENTRYPOINT + if not entrypoint.is_file(): + raise SnapshotError(f"{SKILL_ENTRYPOINT} is missing from {root}") + files = [entrypoint] + for name in SKILL_DIRECTORIES: + directory = root / name + if not directory.is_dir(): + raise SnapshotError(f"{name}/ is missing from {root}") + for path in sorted(directory.rglob("*")): + if any(part in _IGNORED or part.endswith(".pyc") for part in path.relative_to(root).parts): + continue + if path.is_symlink(): + raise SnapshotError(f"skill tree contains a symlink, refusing to snapshot: {path.relative_to(root)}") + if path.is_file(): + files.append(path) + if entrypoint.is_symlink(): + raise SnapshotError(f"{SKILL_ENTRYPOINT} is a symlink, refusing to snapshot") + return files + + +def _content_hash(files: list[tuple[str, bytes]]) -> str: + """Hash over sorted (relative path, raw bytes) pairs, each terminated by + a NUL: exactly the algorithm the eval runner used before this module + existed, so an unchanged skill keeps the content hash recorded in + historical benchmark arms.""" + digest = hashlib.sha256() + for relative, data in sorted(files, key=lambda item: item[0]): + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(data) + digest.update(b"\0") + return f"sha256:{digest.hexdigest()}" + + +def _git(root: Path) -> dict[str, Any]: + revision: dict[str, Any] = {"commit": None, "dirty": None} + try: + commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, timeout=10, check=False) + if commit.returncode == 0: + revision["commit"] = commit.stdout.strip() or None + status = subprocess.run(["git", "status", "--porcelain", "--", SKILL_ENTRYPOINT, *SKILL_DIRECTORIES], cwd=root, capture_output=True, text=True, timeout=10, check=False) + if status.returncode == 0: + revision["dirty"] = bool(status.stdout.strip()) + except (OSError, subprocess.SubprocessError): + pass + return revision + + +def create_skill_snapshot(source_root: str | Path, destination: str | Path, *, now: datetime | None = None) -> dict[str, Any]: + """Copy the distributable skill into ``destination`` and return its manifest.""" + root = Path(source_root).resolve() + target = Path(destination).resolve() + if target.exists() and any(target.iterdir()): + raise SnapshotError(f"snapshot destination is not empty: {target}") + if target == root or root in target.parents: + raise SnapshotError("snapshot destination must not be inside the skill root") + files = _iter_skill_files(root) + target.mkdir(parents=True, exist_ok=True) + entries: list[dict[str, Any]] = [] + contents: list[tuple[str, bytes]] = [] + for path in files: + relative = path.relative_to(root).as_posix() + copy_to = target / relative + copy_to.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, copy_to) + data = copy_to.read_bytes() + contents.append((relative, data)) + entries.append({"path": relative, "sha256": "sha256:" + hashlib.sha256(data).hexdigest(), "bytes": len(data)}) + manifest = { + "schema": SNAPSHOT_SCHEMA, + "created_at": (now or datetime.now(timezone.utc)).isoformat().replace("+00:00", "Z"), + "source_root": root.name, + "source_git": _git(root), + "entrypoint": SKILL_ENTRYPOINT, + "files": entries, + "file_count": len(entries), + "content_sha256": _content_hash(contents), + } + (target / SNAPSHOT_MANIFEST).write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + return manifest + + +def hash_skill_root(source_root: str | Path) -> str: + """Content hash of a skill root without copying it.""" + root = Path(source_root).resolve() + return _content_hash([(path.relative_to(root).as_posix(), path.read_bytes()) for path in _iter_skill_files(root)]) + + +def inspect_skill_snapshot(snapshot_dir: str | Path) -> dict[str, Any]: + """Re-verify a snapshot against its own manifest. + + Reports missing, changed, and extra files plus any symlink or escaping + inventory path; ``intact`` is true only when the copy still matches. + """ + root = Path(snapshot_dir).resolve() + manifest_path = root / SNAPSHOT_MANIFEST + if not manifest_path.is_file(): + raise SnapshotError(f"{SNAPSHOT_MANIFEST} is missing from {root}") + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise SnapshotError(f"cannot read {SNAPSHOT_MANIFEST}: {exc}") from exc + if not isinstance(manifest, dict) or manifest.get("schema") != SNAPSHOT_SCHEMA: + raise SnapshotError(f"{SNAPSHOT_MANIFEST} is not an {SNAPSHOT_SCHEMA} document") + problems: list[str] = [] + seen: set[str] = set() + recomputed: list[tuple[str, bytes]] = [] + for entry in manifest.get("files") or []: + relative = entry.get("path") if isinstance(entry, dict) else None + if not isinstance(relative, str) or not relative: + problems.append("inventory entry without a path") + continue + candidate = (root / relative) + try: + candidate.resolve().relative_to(root) + except ValueError: + problems.append(f"inventory path escapes the snapshot: {relative}") + continue + if Path(relative).is_absolute() or ".." in Path(relative).parts: + problems.append(f"inventory path escapes the snapshot: {relative}") + continue + seen.add(relative) + if candidate.is_symlink(): + problems.append(f"symlink in snapshot: {relative}") + continue + if not candidate.is_file(): + problems.append(f"missing: {relative}") + continue + data = candidate.read_bytes() + actual = "sha256:" + hashlib.sha256(data).hexdigest() + recomputed.append((relative, data)) + if actual != entry.get("sha256"): + problems.append(f"changed: {relative}") + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if relative == SNAPSHOT_MANIFEST or any(part in _IGNORED for part in path.relative_to(root).parts): + continue + if path.is_symlink(): + problems.append(f"symlink in snapshot: {relative}") + elif path.is_file() and relative not in seen: + problems.append(f"extra: {relative}") + content_hash = _content_hash(recomputed) if recomputed else None + if not problems and content_hash != manifest.get("content_sha256"): + problems.append("content_sha256 does not match the inventory") + return { + "schema": "openmapstack-skill-snapshot-inspection/v1", + "snapshot": str(root), + "intact": not problems, + "content_sha256": manifest.get("content_sha256"), + "recomputed_sha256": content_hash, + "file_count": len(seen), + "problems": problems, + "manifest": manifest, + } diff --git a/openmapstack/sources.py b/openmapstack/sources.py new file mode 100644 index 0000000..11f6da6 --- /dev/null +++ b/openmapstack/sources.py @@ -0,0 +1,257 @@ +"""Source pin classes and credential hygiene for ``sources.*``. + +A source is reproducible only if the bytes it contributed can be obtained +again. ``version.identifier`` answers that for a published release or a +STAC item; it does not for a warehouse table, whose "version" is a moving +target unless something freezes it. This module recognises two pin classes +that do freeze it, and reports honestly when neither holds. + +.. code-block:: yaml + + sources: + parcels: + access: + method: postgis + connection: {ref: "env:PARCELS_DSN"} # credentials by reference only + warehouse: + backend: postgis # duckdb | postgis (pilot) + account: geo-prod # host/project identity, no secrets + database: gis + schema: cadastre + table: parcels + query_sha256: "sha256:..." # digest of the exact SELECT + schema_sha256: "sha256:..." # digest of the discovered columns + pin: + class: local_snapshot # (1) user-approved local copy + path: data/source/parcels.parquet + sha256: "sha256:..." + captured_at: "2026-08-30T10:00:00Z" + # -- or -- + pin: + class: backend_snapshot # (2) backend time travel / snapshot + identifier: "pg_export_snapshot:00000003-000001A8-1" + captured_at: "2026-08-30T10:00:00Z" + retention_until: "2026-12-31T00:00:00Z" # when the backend may drop it + verification: {at: "2026-08-30T10:05:00Z", status: accessible} + +``assess_pin`` returns one of: + +- ``pinned``: the pin class validates (local bytes match, or the backend + snapshot is identified, unexpired, and not known to be inaccessible); +- ``not_reproducible``: a pin is declared but cannot deliver the bytes again + (missing or changed snapshot file, expired or inaccessible backend snapshot); +- ``unpinned``: no pin and no usable version identity, or a mutable alias + such as ``latest``; +- ``invalid``: the pin block is malformed. + +A source without a ``pin`` block keeps the original rule: a non-``latest`` +``version.identifier`` or ``published_at`` counts as pinned. That rule is +adequate for immutable published releases and is all a file download needs. + +Secrets never belong in ``project.yaml``. ``find_inline_credentials`` walks +a source block for password fragments, credentialed URLs, and well-known key +shapes, and ``connection_reference_error`` requires ``access.connection`` to +be a reference (``env:``, ``file:``, ``service:``, ``keyring:``) rather than +a DSN. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import date, datetime, timezone +from pathlib import Path +from typing import Any + +from .integrity import normalize_digest, sha256_file +from .project import get_in, project_path + +PIN_CLASSES = ("local_snapshot", "backend_snapshot") +MUTABLE_ALIASES = {"latest", "current", "head", "now", "master", "main", "live", "today"} +CONNECTION_REFERENCE_SCHEMES = ("env", "file", "service", "keyring") + +_CREDENTIAL_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("password fragment", re.compile(r"(?i)(?:^|[\s;&?,{\"'])(?:password|passwd|pwd|secret|api[_-]?key|access[_-]?key|token)\s*[=:]\s*\S")), + ("credentialed URL", re.compile(r"(?i)\b[a-z][a-z0-9+.-]*://[^/\s:@]+:[^/\s@]+@")), + ("AWS access key id", re.compile(r"\bAKIA[0-9A-Z]{16}\b")), + ("bearer token", re.compile(r"(?i)\bbearer\s+[a-z0-9._~+/-]{16,}=*")), + ("private key block", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")), +) + + +@dataclass +class PinAssessment: + status: str # pinned | not_reproducible | unpinned | invalid + pin_class: str # local_snapshot | backend_snapshot | version_identity | none + reason: str + details: dict[str, Any] = field(default_factory=dict) + + +def _parse_timestamp(value: object) -> datetime | None: + if isinstance(value, datetime): + return value if value.tzinfo else value.replace(tzinfo=timezone.utc) + if isinstance(value, date): + return datetime(value.year, value.month, value.day, tzinfo=timezone.utc) + if not isinstance(value, str) or not value.strip(): + return None + text = value.strip() + if text.endswith("Z"): + text = text[:-1] + "+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def _is_mutable_alias(value: object) -> bool: + return isinstance(value, str) and value.strip().lower() in MUTABLE_ALIASES + + +def assess_pin(root: Path, source: dict[str, Any], *, now: datetime | None = None) -> PinAssessment: + """Classify how (and whether) one source is pinned. Never raises.""" + now = now or datetime.now(timezone.utc) + pin = source.get("pin") + identifier = get_in(source, "version", "identifier") + published_at = get_in(source, "version", "published_at") + # A mutable alias is a lie about identity whatever else is declared: a + # snapshot of "latest" still cannot say *which* latest it froze. + if _is_mutable_alias(identifier) or (identifier in (None, "") and _is_mutable_alias(published_at)): + return PinAssessment("unpinned", "version_identity", f"version.identifier {identifier!r} is a mutable alias") + if pin is None: + if identifier in (None, "") and published_at in (None, ""): + return PinAssessment("unpinned", "none", "no pin block and no version.identifier/published_at") + return PinAssessment("pinned", "version_identity", "version identity is recorded and is not a mutable alias") + + if not isinstance(pin, dict): + return PinAssessment("invalid", "none", "pin must be a mapping") + pin_class = pin.get("class") + if pin_class not in PIN_CLASSES: + return PinAssessment("invalid", "none", f"pin.class must be one of {list(PIN_CLASSES)}, got {pin_class!r}") + + if pin_class == "local_snapshot": + relative = pin.get("path") + target = project_path(root, relative) + if target is None: + return PinAssessment("invalid", pin_class, "pin.path must be a safe project-relative path") + if not str(relative).replace("\\", "/").startswith("data/source/"): + return PinAssessment("invalid", pin_class, "a local snapshot must live under data/source/") + expected = normalize_digest(pin.get("sha256")) + if expected is None: + return PinAssessment("invalid", pin_class, "pin.sha256 must be a sha256 digest of the snapshot") + if _parse_timestamp(pin.get("captured_at")) is None: + return PinAssessment("invalid", pin_class, "pin.captured_at must be an ISO-8601 timestamp") + if not target.is_file(): + return PinAssessment( + "not_reproducible", pin_class, f"snapshot file is missing: {relative}", {"cause": "snapshot_missing"} + ) + actual = sha256_file(target) + if actual != expected: + return PinAssessment( + "not_reproducible", + pin_class, + f"snapshot {relative} does not match pin.sha256", + {"cause": "snapshot_hash_mismatch", "expected": expected, "actual": actual}, + ) + return PinAssessment("pinned", pin_class, f"local snapshot {relative} matches its content hash") + + identifier = pin.get("identifier") + if not isinstance(identifier, str) or not identifier.strip(): + return PinAssessment("invalid", pin_class, "pin.identifier must name the backend snapshot") + if _is_mutable_alias(identifier): + return PinAssessment("unpinned", pin_class, f"pin.identifier {identifier!r} is a mutable alias, not a snapshot") + if _parse_timestamp(pin.get("captured_at")) is None: + return PinAssessment("invalid", pin_class, "pin.captured_at must be an ISO-8601 timestamp") + retention = _parse_timestamp(pin.get("retention_until")) + if retention is None: + return PinAssessment( + "invalid", pin_class, "pin.retention_until must record when the backend may drop the snapshot" + ) + if retention <= now: + return PinAssessment( + "not_reproducible", + pin_class, + f"backend snapshot {identifier!r} retention expired at {pin.get('retention_until')}", + {"cause": "snapshot_expired", "retention_until": str(pin.get("retention_until"))}, + ) + verification = pin.get("verification") + if isinstance(verification, dict) and verification.get("status") == "inaccessible": + return PinAssessment( + "not_reproducible", + pin_class, + f"backend snapshot {identifier!r} was last verified inaccessible", + {"cause": "snapshot_inaccessible", "verified_at": verification.get("at")}, + ) + return PinAssessment("pinned", pin_class, f"backend snapshot {identifier!r} is identified and retained until {pin.get('retention_until')}") + + +def _walk_strings(value: Any, path: str): + if isinstance(value, str): + yield path, value + elif isinstance(value, dict): + for key, item in value.items(): + yield from _walk_strings(item, f"{path}.{key}" if path else str(key)) + elif isinstance(value, list): + for index, item in enumerate(value): + yield from _walk_strings(item, f"{path}[{index}]") + + +def find_inline_credentials(value: Any, path: str = "") -> list[dict[str, str]]: + """Return every string in ``value`` that looks like an embedded secret. + + Reports the manifest path and the pattern name, never the matched text, + so the finding itself cannot leak what it found. + """ + findings: list[dict[str, str]] = [] + for where, text in _walk_strings(value, path): + for name, pattern in _CREDENTIAL_PATTERNS: + if pattern.search(text): + findings.append({"path": where, "pattern": name}) + break + return findings + + +def connection_reference_error(root: Path, connection: object) -> str | None: + """``access.connection`` must be a reference, never a connection string.""" + if connection is None: + return None + reference = connection.get("ref") if isinstance(connection, dict) else connection + if not isinstance(reference, str) or not reference.strip(): + return "access.connection must be a string reference or {ref: ...}" + scheme, separator, remainder = reference.partition(":") + if not separator or scheme not in CONNECTION_REFERENCE_SCHEMES or not remainder.strip(): + return ( + "access.connection must reference credentials indirectly " + f"({', '.join(f'{item}:' for item in CONNECTION_REFERENCE_SCHEMES)}), not embed a connection string" + ) + if scheme == "file": + # The same rule the connector applies, so preflight cannot accept a + # reference every source operation will refuse. + candidate = Path(remainder.strip()).expanduser() + if not candidate.is_absolute(): + return "access.connection file references must be absolute paths outside the project" + try: + candidate.resolve().relative_to(root.resolve()) + except ValueError: + return None + return "access.connection file references must point outside the project directory" + return None + + +def source_pin_summary(root: Path, sources: dict[str, Any], *, now: datetime | None = None) -> dict[str, PinAssessment]: + return { + str(key): assess_pin(root, source, now=now) if isinstance(source, dict) else PinAssessment("invalid", "none", "source must be a mapping") + for key, source in sources.items() + } + + +def redact(text: str) -> str: + """Mask credential-like fragments in free text before it is recorded.""" + redacted = re.sub(r"(?i)\b([a-z][a-z0-9+.-]*://[^/\s:@]+):[^/\s@]+@", r"\1:***@", text) + redacted = re.sub( + r"(?i)((?:password|passwd|pwd|secret|api[_-]?key|access[_-]?key|token)\s*[=:]\s*)\S+", + r"\1***", + redacted, + ) + redacted = re.sub(r"\bAKIA[0-9A-Z]{16}\b", "AKIA****************", redacted) + return redacted diff --git a/openmapstack/validation.py b/openmapstack/validation.py index 51862f5..3b5cfda 100644 --- a/openmapstack/validation.py +++ b/openmapstack/validation.py @@ -20,6 +20,7 @@ ) from .project import ProjectError, get_in, load_json, load_project, project_path, step_outputs from .schema import project_schema_errors +from .sources import assess_pin, connection_reference_error, find_inline_credentials SCHEMA = "openmapstack-project/v1" CHECK_STATUSES = {"passed", "failed", "warning", "not_testable"} @@ -254,6 +255,24 @@ def _sources(self) -> None: else: self.add("source.provenance", "passed", "source URL, retrieval, version, selection, and rationale are pinned", path=base) + pin = assess_pin(self.root, source) + if pin.status == "pinned": + self.add("source.pin", "passed", pin.reason, path=f"{base}.pin", pin_class=pin.pin_class) + elif pin.status == "not_reproducible": + self.add("source.pin", "failed", f"not reproducible: {pin.reason}", path=f"{base}.pin", pin_class=pin.pin_class, **pin.details) + elif pin.status == "invalid": + self.add("source.pin", "failed", pin.reason, path=f"{base}.pin") + elif pin.pin_class != "none": + self.add("source.pin", "failed", pin.reason, path=f"{base}.pin") + credential_findings = [f"{item['path']} ({item['pattern']})" for item in find_inline_credentials(source, base)] + connection_error = connection_reference_error(self.root, get_in(source, "access", "connection")) + if connection_error: + credential_findings.append(f"{base}.access.connection: {connection_error}") + if credential_findings: + self.add("source.credentials", "failed", f"secrets must never enter project.yaml: {credential_findings}", path=base) + elif get_in(source, "access", "connection") is not None or source.get("warehouse") is not None: + self.add("source.credentials", "passed", "warehouse connection is referenced, not embedded", path=f"{base}.access.connection") + license_block = source.get("license") if not isinstance(license_block, dict) or not _present(license_block.get("name")) or not _present(license_block.get("url")): self.add("source.license", "failed", "license.name and license.url are required", path=f"{base}.license") diff --git a/openmapstack/verify.py b/openmapstack/verify.py index b03c2e3..5d8ecbb 100644 --- a/openmapstack/verify.py +++ b/openmapstack/verify.py @@ -29,6 +29,7 @@ from .checks import AssertionResult, not_testable from .checks import geodata as geodata_checks +from .checks import metamorphic as metamorphic_checks from .checks import overrides as overrides_checks from .checks import presentation as presentation_checks from .checks import project as project_checks @@ -37,7 +38,7 @@ from .checks import rerun as rerun_checks from .checks import validation as validation_checks from .expectations import evaluate_expectation -from .project import load_project +from .project import get_in, load_project from .rerun import perform_clean_rerun SCHEMA = "openmapstack-verify-result/v1" @@ -66,8 +67,9 @@ def to_dict(self) -> dict[str, Any]: } if self.args: payload["args"] = self.args - if self.evidence: - payload["evidence"] = self.evidence + evidence = self.evidence or (self.result.data or {}).get("evidence") + if evidence: + payload["evidence"] = evidence code = (self.result.data or {}).get("code") if code: payload["code"] = code @@ -189,9 +191,15 @@ def verify_project( *, rerun: bool = False, rerun_timeout_s: float = 1800, + metamorphic: bool = False, forbidden_fragments: Sequence[str] = (), ) -> VerifyResult: - """Run every applicable no-golden-answer check the environment supports.""" + """Run every applicable no-golden-answer check the environment supports. + + ``rerun`` and ``metamorphic`` both execute the project's canonical + entrypoint in isolated copies, so they are opt-in: the static plan must + stay cheap enough to run on every save. + """ project_file, manifest = load_project(project) root = project_file.parent result = VerifyResult(project_file=project_file) @@ -212,11 +220,14 @@ def verify_project( declared = [path for _, path, _ in _declared_output_paths(manifest)] if declared: _run(runs, "project.declared_files_exist", project_checks.declared_files_exist, root, files=declared) + if get_in(manifest, "runtime", "implementation", "parameters") is not None: + _run(runs, "project.parameters_match_steps", project_checks.parameters_match_steps, root) # -- provenance: sources are attributed, pinned, and licensed for name, fn in ( ("every_source_has_provider_and_access", provenance_checks.every_source_has_provider_and_access), ("every_source_pinned", provenance_checks.every_source_pinned), + ("no_inline_credentials", provenance_checks.no_inline_credentials), ("license_present_where_required", provenance_checks.license_present_where_required), ("rationale_present", provenance_checks.rationale_present), ): @@ -315,6 +326,24 @@ def verify_project( ): _run(runs, f"qgis.{name}", fn, root) + # -- metamorphic relations: declared invariants under controlled perturbation + relations = get_in(manifest, "validation", "metamorphic") + if relations is not None: + _run(runs, "metamorphic.declarations_valid", metamorphic_checks.declarations_valid, root) + if metamorphic and isinstance(relations, list): + for raw in relations: + relation_id = raw.get("id") if isinstance(raw, dict) else None + if not isinstance(relation_id, str) or not relation_id: + continue + _run( + runs, + f"metamorphic.{relation_id}", + metamorphic_checks.relation_holds, + root, + id=relation_id, + forbidden_fragments=list(forbidden_fragments), + ) + # -- reproducibility _run(runs, "rerun.no_chat_dependency", rerun_checks.no_chat_dependency, root) if rerun: diff --git a/pyproject.toml b/pyproject.toml index 8038f8d..15ffe42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "openmapstack" -version = "0.2.0" +version = "0.3.0" description = "Validate, run, and inspect reproducible OpenMapStack projects" readme = "README.md" requires-python = ">=3.10" @@ -21,7 +21,10 @@ geo = ["duckdb>=1.2"] # Dashboard checks in a real browser. PyQGIS has no PyPI distribution, so # the QGIS checks depend on a system install and degrade to not_testable. visual = ["playwright>=1.40"] -all = ["openmapstack[geo,visual]"] +# `openmapstack source discover/snapshot` against PostGIS. The DuckDB +# local-file connector needs only [geo]. +postgis = ["psycopg[binary]>=3.1"] +all = ["openmapstack[geo,visual,postgis]"] [project.scripts] openmapstack = "openmapstack.cli:main" diff --git a/references/project-spec.md b/references/project-spec.md index ec368c7..f39dc79 100644 --- a/references/project-spec.md +++ b/references/project-spec.md @@ -155,6 +155,61 @@ sources: - Describe the data you received in `schema` — its CRS, the key, the field roles the analysis depends on, and the columns. Earlier drafts used a bare `expected_fields` list; `schema.columns` supersedes it. - `access.retrieved_at` and `access.downloaded_at` are interchangeable to the validator, which needs one of the two. Record both when they differ (a cached extract retrieved later than it was published). +#### Pin classes — warehouse and mutable sources + +`version.identifier` pins a published release, a STAC item, or a dated +extract. It does not pin a warehouse table: "the parcels table on +2026-08-30" names a moving target unless something froze it. A source whose +bytes come from a query declares **which of two pin classes** froze them: + +```yaml +sources: + parcels: + access: + method: postgis + connection: {ref: "env:PARCELS_DSN"} # a reference, never a DSN + warehouse: + backend: postgis # duckdb | postgis (pilot backends) + account: geo-prod # host / project identity, no secrets + database: gis + schema: cadastre + table: parcels + query_sha256: "sha256:..." # digest of the exact SELECT used + schema_sha256: "sha256:..." # digest of the discovered column list + pin: + class: local_snapshot # (1) user-approved local copy + path: data/source/parcels.parquet + sha256: "sha256:..." # real content hash of that file + captured_at: "2026-08-30T10:00:00Z" + # -- or -- + pin: + class: backend_snapshot # (2) backend time travel / snapshot id + identifier: "pg_export_snapshot:00000003-000001A8-1" + captured_at: "2026-08-30T10:00:00Z" + retention_until: "2026-12-31T00:00:00Z" # when the backend may drop it + verification: {at: "2026-08-30T10:05:00Z", status: accessible} +``` + +| Pin | Reproducible when | Reported as | +|---|---|---| +| `local_snapshot` | the file exists under `data/source/` and matches `sha256` | `pinned`; a missing or edited file is `not_reproducible` | +| `backend_snapshot` | `identifier` names a real snapshot, `retention_until` is in the future, and the last `verification` did not find it inaccessible | `pinned`; expired or inaccessible is `not_reproducible` | +| none | `version.identifier` / `published_at` is present and not a mutable alias (`latest`, `current`, `head`, …) | `pinned` by version identity | + +`provenance.every_source_pinned` and `validate`'s `source.pin` check apply +this table. A backend snapshot id plus a timestamp is **not** a pin once the +retention has lapsed; the honest result is `not_reproducible`, never `pinned` +because a string is present. A mutable alias in `version.identifier` is +unpinned whatever else is declared. + +**Secrets never enter `project.yaml`.** `access.connection` is a reference — +`env:NAME`, `service:NAME` (a `pg_service.conf` entry), `keyring:NAME`, or +`file:/absolute/path/outside/the/project` — and the manifest is scanned for +password fragments, credentialed URLs, and well-known key shapes +(`source.credentials`, `provenance.no_inline_credentials`). Connector +discovery is read-only, and materialising warehouse data locally requires an +explicit approval; see `references/user-data-sources.md`. + ### 2.3 Overrides — analyst knowledge as data **Everything that is not a fact of an external dataset belongs in `overrides`.** Sources are immutable: @@ -328,6 +383,68 @@ evidence file makes the attestation stale and produces a warning. Attestation records reviewer evidence; it is not relabelled as an independently reproducible oracle in reports. +#### Metamorphic relations — invariants without a golden answer + +`validation.metamorphic[]` declares relations that must hold when an input or +parameter is perturbed in a controlled way. They need no frozen answer, so +they transfer to data nobody has an oracle for — and they are **conditional**: +each relation is valid only under preconditions the declaration must state, +and `openmapstack verify --metamorphic` reports `not_testable` with the reason +when a precondition does not hold on the actual data rather than guessing. + +```yaml +validation: + metamorphic: + - id: parcel-order + relation: input_permutation_invariance + source: {path: data/source/parcels.geojson} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: + tie_break: "candidates are keyed by cadastral_id; no selection depends on input order" + - id: parcel-duplicates + relation: duplicate_resistance + source: {path: data/source/parcels.geojson} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: {dedup_key: cadastral_id, measure: set} + - id: road-distance-monotonic + relation: positive_buffer_monotonicity + parameter: road_distance_m # declared under runtime.implementation.parameters + variant: {multiply: 3} + outputs: [candidate_parcels] + key: cadastral_id + preconditions: {predicate: within_distance, expected: superset} + limits: {timeout_s: 600, max_source_bytes: 67108864} +``` + +| Relation | Transformation | Expected | Valid only when | +|---|---|---|---| +| `input_permutation_invariance` | source features shuffled (deterministic `seed`) | outputs semantically equal | a deterministic `tie_break` rule is declared and `key` is unique in the output | +| `duplicate_resistance` | every source feature appended once more | outputs equal | the analysis deduplicates on `dedup_key` (unique in the source) and the output is a keyed *set*; counts and sums are rejected | +| `positive_buffer_monotonicity` | a declared numeric parameter increased (`multiply` > 1 or `add` > 0) | every baseline key survives (`superset`) | the parameter drives an inclusion `predicate` (`within_distance`, `intersects_buffer`, `within_buffer`) | + +Each relation reruns the canonical entrypoint in an isolated copy prepared like +a clean rerun, perturbs only that copy, compares against the produced outputs, +and deletes the copy. The project's own `data/source/` and `data/overrides/` +are hashed before and after; a variant that mutates them fails. Unknown +relation names, `source` paths outside the immutable trees, non-growing +variants, and `duplicate_resistance` declared for a count or sum are +declaration failures, not skipped checks. + +**Counterexamples — do not enable a relation mechanically:** + +- a nearest-neighbour join with ties has no order invariance until the tie + rule is fixed in the pipeline; declare `tie_break` only once it is; +- a facility *count* per parcel is not duplicate-resistant — duplicating a + facility legitimately changes the count; only a keyed set of facilities is; +- an *exclusion* buffer (parcels farther than N m) is monotonic the other way; + `positive_buffer_monotonicity` only establishes `superset` for inclusion + predicates and refuses any other `predicate`; +- a threshold that also changes a classification (`tier1` below 1 km, `tier2` + below 2 km) keeps the key set but changes attributes; the relation only + compares keys, so declare it knowing that. + ### 2.7 Presentation semantics ```yaml @@ -535,6 +652,13 @@ runtime: dependencies: # project-local files needed in a clean run - requirements.lock - config/analysis.toml + parameters: # optional; how a variant run turns one knob + - id: road_distance_m + type: number # integer | number | string + canonical: 2000 + binding: {argument: "--road-distance-m"} # or {environment: OMS_ROAD_DISTANCE_M} + step: road_distance # optional pair: the step that consumes it + field: max_distance_m # ... whose value must equal `canonical` environment: python: "3.13" duckdb: "1.2.x" @@ -566,6 +690,14 @@ warnings: # known, unresolved data-quality limits consequential decisions. ``` +`runtime.implementation.parameters` is the versioned parameter-addressing +contract (`openmapstack-parameters/v1`). The canonical run passes nothing and +must produce the accepted result; a binding exists so a metamorphic relation or +a benchmark can run *the same pipeline with one knob turned* without editing +it. When `step`/`field` are given, `openmapstack verify` fails +`project.parameters_match_steps` if the step's declared value drifts from +`canonical` — the same honesty rule as `presentation.controls`. + **Warnings** give the explicit confidence/incompleteness handling. The rendered UX surfaces them (don't imply autoconfirmed geodata is current/complete). **Runs** capture what changed between executions and let a new engineer `rerun` tomorrow. The corresponding `runs/.json` record MUST contain `inputs` and `outputs` @@ -858,8 +990,9 @@ openmapstack inspect project.yaml - `verify` derives an applicable check plan from the manifest and inspects the produced artifacts without requiring a repository-owned golden answer. It reports partial execution as warning, evaluates only allowlisted and current - attestations, and optionally performs the independent clean rerun with - `--rerun`; `--json` includes applicability coverage and evidence class. + attestations, optionally performs the independent clean rerun with + `--rerun`, and optionally executes every declared metamorphic relation with + `--metamorphic`; `--json` includes applicability coverage and evidence class. - `run` first performs preflight validation, invokes exactly the pipeline or shell-free command in `runtime.implementation`, and then performs the full artifact validation. Python pipelines use the interpreter that installed the diff --git a/references/user-data-sources.md b/references/user-data-sources.md new file mode 100644 index 0000000..4d36e87 --- /dev/null +++ b/references/user-data-sources.md @@ -0,0 +1,157 @@ +# User data sources — warehouses, credentials, snapshots, clean reruns + +Read this when an analysis must read the user's own tables: a PostGIS +database, a DuckDB file, or a directory of GeoParquet/GeoPackage files that +is not a public download. It covers what `references/data-sources.md` does +not: data that has an owner, a credential, and no published version. + +Treat warehouse access as a **connector and security problem**, not only a +documentation problem. The rules below are enforced by `openmapstack validate` +and `openmapstack verify`; the CLI implements them for the two pilot +backends, and the rest of this file says what to do by hand elsewhere. + +## The four rules + +1. **Credentials by reference.** `project.yaml` never contains a password, + a token, a key, or a DSN with a password in it. `access.connection` is a + reference: `env:NAME`, `service:NAME` (an entry in `pg_service.conf`), + `keyring:NAME`, or `file:/absolute/path/outside/the/project`. Both + `validate` (`source.credentials`) and `verify` + (`provenance.no_inline_credentials`) scan every source for embedded + secrets and fail the project when they find one. Their findings name the + manifest path and the pattern, never the secret. +2. **Discovery is read-only.** A connector session opens with + `default_transaction_read_only = on` and a statement timeout, lists + tables, geometry columns, SRIDs, and row estimates, and only ever runs + a single `SELECT`. DML, DDL, `COPY`, `ATTACH`, `INSTALL`, `SET`, and + file-reading table functions are rejected before anything reaches the + server. +3. **Materialising data locally needs explicit approval.** + `openmapstack source snapshot` is a dry run by default: it reports the + schema and row count the query would copy. Only `--approve` writes the + snapshot, and it writes under `data/source/` only, never over an + existing file, and never beyond the row and byte limits. +4. **A warehouse table is pinned only by a pin class.** A timestamp string + is not a pin. Either the bytes are frozen locally (`local_snapshot`, + hash-matched) or the backend can serve that exact state again + (`backend_snapshot` with an identifier and a retention limit). An + expired or inaccessible backend snapshot is reported as + `not_reproducible`. See `project-spec.md` section 2.2. + +## Declaring a warehouse source + +```yaml +sources: + parcels: + role: authoritative_input + provider: City GIS department + dataset: cadastral parcels (warehouse copy) + source_url: postgresql://geo-prod.internal/gis # identity only, no credentials + access: + method: postgis + connection: {ref: "env:PARCELS_DSN"} + retrieved_at: "2026-08-30T10:00:00Z" + warehouse: + backend: postgis # duckdb | postgis are the verified pilot backends + account: geo-prod + database: gis + schema: cadastre + table: parcels + query_sha256: "sha256:..." # written by `source snapshot` + schema_sha256: "sha256:..." + pin: + class: local_snapshot + path: data/source/parcels.parquet + sha256: "sha256:..." + captured_at: "2026-08-30T10:00:00Z" + version: + identifier: "cadastre.parcels @ pg_current_snapshot 1001:1001: on 2026-08-30" + published_at: 2026-08-30 + selection: + filter: "municipality = 'Tartu linn'" + license: {name: "Internal — see data owner", url: https://intranet.example/gis-data-policy} + schema: {crs: EPSG:3301, key: cadastral_id, columns: [cadastral_id, land_use, geom]} + rationale: The warehouse copy is the department's authoritative parcel layer. +``` + +## The CLI path (DuckDB local files and PostGIS) + +```bash +# 1. Read-only discovery: what is there, which column is geometry, which SRID. +openmapstack source discover project.yaml --source parcels +openmapstack source discover project.yaml --source parcels --json + +# 2. Dry run: schema, row count, and digests of the query you intend to freeze. +openmapstack source snapshot project.yaml --source parcels \ + --query "SELECT cadastral_id, land_use, geom FROM cadastre.parcels WHERE municipality = 'Tartu linn'" \ + --destination data/source/parcels.parquet + +# 3. Materialise, after the user has approved the row count and the bytes. +openmapstack source snapshot project.yaml --source parcels --query-file queries/parcels.sql \ + --destination data/source/parcels.parquet --approve --max-rows 200000 --timeout 120 + +# 4. Record the pin (printed as YAML; --write-manifest rewrites project.yaml +# and drops YAML comments, so most projects paste the block instead). +``` + +The snapshot is GeoParquet with geometry typed and, where the installed +DuckDB Spatial supports it, carrying the source SRID. The command returns the +`pin` block, the `warehouse` digests, and `access.retrieved_at` to place in +the manifest. Once placed, `openmapstack validate` reports `source.pin` as +`passed` with `pin_class: local_snapshot`. + +### DuckDB local files + +With `warehouse.backend: duckdb` and no `access.connection`, the connector +root is the project's own `data/source/`. Every geodata file under it is +exposed as a view named by its relative path, so a query reads +`FROM "parcels.geojson"` and never spells a filesystem path. File access is +confined to that root (`allowed_directories` + `enable_external_access = +false`); a query that reaches outside it fails. A `.duckdb` database is +attached read-only when the connection names one. + +### PostGIS + +`access.connection` resolves to a DSN through the reference. The session is +read-only with `statement_timeout`; discovery reads `geometry_columns` and +planner estimates; the snapshot fetches geometry as WKB and writes GeoParquet +through DuckDB (`openmapstack[geo]`), with the driver from +`openmapstack[postgis]`. + +PostgreSQL has **no durable time travel**: `pg_export_snapshot()` lives only +as long as its transaction. The pin for a PostGIS source is therefore the +local snapshot; the connector records `pg_current_snapshot()` and the schema +digest beside it as retrieval metadata (`durable: false`), never as a pin. +Declaring `pin.class: backend_snapshot` for PostGIS is honest only when an +external mechanism (a logical replica frozen for the project, a `pg_dump` +retained under a stated policy) provides the retention you record. + +## Other backends + +Only DuckDB and PostGIS are verified. `warehouse.backend` may name another +system (`bigquery`, `snowflake`, `motherduck`, `databricks`, `redshift`, +`athena`, `iceberg`, `delta`), but the CLI refuses to connect to it +(`backend_unsupported`) rather than guessing its semantics. For those: + +- pull the data with the vendor's tooling into `data/source/` and pin it as + a `local_snapshot`, or +- use the backend's own snapshot/time-travel identity (Snowflake `AT + (STATEMENT => ...)`, BigQuery `FOR SYSTEM_TIME AS OF`, Iceberg/Delta + snapshot ids) as a `backend_snapshot`, **recording the retention limit + the vendor actually guarantees** (Snowflake Time Travel defaults to one + day; BigQuery keeps seven), and expect `verify` to report + `not_reproducible` once that passes. + +Never approximate a pin by pasting the current date. The point of the pin +contract is that a reviewer can tell the difference. + +## Clean rerun with warehouse sources + +A clean rerun copies only `data/source/`, `data/overrides/`, the manifest, +and declared dependencies, and executes the pipeline with session/provider +environment variables removed. A pipeline that reads the warehouse live at +run time therefore fails the rerun unless the credential reference resolves +in the rerun environment — and if it does, the rerun proves only that the +warehouse still answers, not that it answers the same thing. Read from the +local snapshot in the pipeline; keep the query that produced it under +`warehouse.query_sha256` so the snapshot can be refreshed deliberately. diff --git a/templates/project.yaml b/templates/project.yaml index 87096ee..66890a0 100644 --- a/templates/project.yaml +++ b/templates/project.yaml @@ -57,6 +57,14 @@ sources: version: published_at: 2026-08-24 identifier: TODO # release tag, STAC item id, layer/table id + # A warehouse/query source also declares how its bytes were frozen + # (project-spec.md section 2.2, pin classes). Credentials stay out of + # this file: access.connection is a reference such as env:NAME. + # pin: + # class: local_snapshot # | backend_snapshot (identifier + retention_until) + # path: data/source/TODO.parquet + # sha256: "sha256:TODO" + # captured_at: 2026-08-25T00:00:00Z selection: bbox: [west, south, east, north] filter: TODO @@ -142,6 +150,16 @@ validation: # attestation: # status: unverified # reason: "Awaiting comparison with the authoritative register" + # Optional no-golden-answer invariants, executed by `openmapstack verify + # --metamorphic`. Declare a relation only when its precondition genuinely + # holds for this analysis (project-spec.md section 2.6 lists counterexamples). + metamorphic: [] + # - id: candidate-order + # relation: input_permutation_invariance + # source: {path: data/source/example.gpkg} + # outputs: [example_output] + # key: id + # preconditions: {tie_break: "keyed set; no order-dependent selection"} presentation: intent: analytical_workspace # analytical_workspace | discovery | report @@ -242,6 +260,15 @@ runtime: # dependencies: # - requirements.lock # - config/analysis.toml + # Declared knobs a variant run may turn (openmapstack-parameters/v1). + # The canonical run passes nothing; `step`/`field` let verify catch drift. + # parameters: + # - id: buffer_m + # type: number + # canonical: 2000 + # binding: {argument: "--buffer-m"} + # step: select_candidates + # field: max_distance_m environment: python: "3.13" duckdb: TODO diff --git a/tests/goldens/verify/district-facilities.json b/tests/goldens/verify/district-facilities.json new file mode 100644 index 0000000..af86daf --- /dev/null +++ b/tests/goldens/verify/district-facilities.json @@ -0,0 +1,174 @@ +{ + "checks": [ + { + "check": "project.parses", + "message": "project.yaml parses", + "status": "passed" + }, + { + "check": "project.conforms_to_schema", + "message": "project.yaml conforms to the packaged OpenMapStack v1 JSON Schema", + "status": "passed" + }, + { + "check": "project.graph_resolves", + "message": "graph resolves: 4 steps, 4 produced symbols, 1 outputs all traced to real steps", + "status": "passed" + }, + { + "check": "project.one_canonical_pipeline", + "message": "pipeline.py is the canonical implementation; wrappers import it", + "status": "passed" + }, + { + "check": "project.assumptions_have_rationale", + "message": "all 1 assumptions have statement + rationale", + "status": "passed" + }, + { + "check": "project.status_agrees_with_validation_report", + "message": "project.status 'warning' agrees with report status 'warning'", + "status": "passed" + }, + { + "args": { + "files": [ + "data/derived/district-counts.geojson" + ] + }, + "check": "project.declared_files_exist", + "message": "all 1 declared files exist", + "status": "passed" + }, + { + "check": "provenance.every_source_has_provider_and_access", + "message": "all 2 sources declare provider + access method + retrieval timestamp", + "status": "passed" + }, + { + "check": "provenance.every_source_pinned", + "message": "all 2 sources are pinned (version_identity)", + "status": "passed" + }, + { + "check": "provenance.no_inline_credentials", + "message": "no inline credentials in 2 sources; connections are by reference", + "status": "passed" + }, + { + "check": "provenance.license_present_where_required", + "message": "license metadata present for required sources", + "status": "passed" + }, + { + "check": "provenance.rationale_present", + "message": "all 2 sources document selection rationale", + "status": "passed" + }, + { + "check": "overrides.every_override_has_provenance", + "message": "all 1 overrides carry id/action/rationale/author/timestamp", + "status": "passed" + }, + { + "check": "overrides.evidence_not_placeholder", + "message": "no placeholder evidence found", + "status": "passed" + }, + { + "check": "validation.required_all_present", + "message": "all 3 declared checks present exactly once in report", + "status": "passed" + }, + { + "check": "validation.no_implicit_pass", + "message": "every check has an explicit status", + "status": "passed" + }, + { + "check": "validation.warning_or_failed_propagates_to_status", + "message": "overall status 'warning' correctly reflects check statuses", + "status": "passed" + }, + { + "check": "validation.run_record_matches", + "message": "report run_id 'run-20260901-000001' matches a real run record with consistent hashes", + "status": "passed" + }, + { + "args": { + "check": "geodata.feature_field_equals", + "equals": 2, + "field": "facility_count", + "id": "D1", + "id_field": "district_id", + "path": "data/derived/district-counts.geojson" + }, + "check": "expectation.d1-count", + "code": "expectation_unverified", + "evidence": { + "class": "unverified", + "expected_expectation_sha256": "$DIGEST" + }, + "message": "expectation 'd1-count' is unverified; independent review must bind expectation_sha256 to sha256:$DIGEST", + "status": "warning" + }, + { + "check": "geodata.crs_not_used_for_metrics", + "message": "analysis_crs EPSG:3301 is valid for 0 metric operation(s); storage/load/reprojection steps were excluded", + "status": "passed" + }, + { + "args": { + "path": "data/derived/district-counts.geojson" + }, + "check": "geodata.geometry_all_valid", + "message": "data/derived/district-counts.geojson: all 2 features have valid geometry", + "status": "passed" + }, + { + "args": { + "path": "data/derived/district-counts.geojson" + }, + "check": "geodata.dataset_crs_is", + "code": "crs_undeclared", + "message": "output 'district_counts' declares no EPSG code in its format string, so its real CRS cannot be cross-checked", + "status": "not_testable" + }, + { + "check": "presentation.layers_use_semantic_roles", + "message": "all 2 layers declare a semantic_role", + "status": "passed" + }, + { + "check": "presentation.controls_match_pipeline", + "message": "0 filter control(s) and 0 scenario control(s) consistent", + "status": "passed" + }, + { + "check": "presentation.edit_targets_reference_real_sources", + "message": "no edit targets declared (vacuously true)", + "status": "passed" + }, + { + "check": "rerun.no_chat_dependency", + "message": "canonical project dependencies contain no chat/transcript references", + "status": "passed" + } + ], + "counts": { + "failed": 0, + "not_testable": 1, + "passed": 24, + "warning": 1 + }, + "coverage": { + "applicable": 26, + "executed": 25, + "execution_rate": 0.9615384615384616, + "not_testable": 1 + }, + "project_file": "$PROJECT/project.yaml", + "schema": "openmapstack-verify-result/v1", + "status": "warning" +} diff --git a/tests/goldens/verify/district-facilities.txt b/tests/goldens/verify/district-facilities.txt new file mode 100644 index 0000000..dd1122d --- /dev/null +++ b/tests/goldens/verify/district-facilities.txt @@ -0,0 +1,28 @@ +PASS project.parses: project.yaml parses +PASS project.conforms_to_schema: project.yaml conforms to the packaged OpenMapStack v1 JSON Schema +PASS project.graph_resolves: graph resolves: 4 steps, 4 produced symbols, 1 outputs all traced to real steps +PASS project.one_canonical_pipeline: pipeline.py is the canonical implementation; wrappers import it +PASS project.assumptions_have_rationale: all 1 assumptions have statement + rationale +PASS project.status_agrees_with_validation_report: project.status 'warning' agrees with report status 'warning' +PASS project.declared_files_exist: all 1 declared files exist +PASS provenance.every_source_has_provider_and_access: all 2 sources declare provider + access method + retrieval timestamp +PASS provenance.every_source_pinned: all 2 sources are pinned (version_identity) +PASS provenance.no_inline_credentials: no inline credentials in 2 sources; connections are by reference +PASS provenance.license_present_where_required: license metadata present for required sources +PASS provenance.rationale_present: all 2 sources document selection rationale +PASS overrides.every_override_has_provenance: all 1 overrides carry id/action/rationale/author/timestamp +PASS overrides.evidence_not_placeholder: no placeholder evidence found +PASS validation.required_all_present: all 3 declared checks present exactly once in report +PASS validation.no_implicit_pass: every check has an explicit status +PASS validation.warning_or_failed_propagates_to_status: overall status 'warning' correctly reflects check statuses +PASS validation.run_record_matches: report run_id 'run-20260901-000001' matches a real run record with consistent hashes +WARN expectation.d1-count [data/derived/district-counts.geojson]: expectation 'd1-count' is unverified; independent review must bind expectation_sha256 to sha256:10399dfde7e9808a58d5ff9447545d111f87f3ef4d3277f10ae21d4c69bacaaa +PASS geodata.crs_not_used_for_metrics: analysis_crs EPSG:3301 is valid for 0 metric operation(s); storage/load/reprojection steps were excluded +PASS geodata.geometry_all_valid [data/derived/district-counts.geojson]: data/derived/district-counts.geojson: all 2 features have valid geometry +N/A geodata.dataset_crs_is [data/derived/district-counts.geojson]: output 'district_counts' declares no EPSG code in its format string, so its real CRS cannot be cross-checked +PASS presentation.layers_use_semantic_roles: all 2 layers declare a semantic_role +PASS presentation.controls_match_pipeline: 0 filter control(s) and 0 scenario control(s) consistent +PASS presentation.edit_targets_reference_real_sources: no edit targets declared (vacuously true) +PASS rerun.no_chat_dependency: canonical project dependencies contain no chat/transcript references +WARNING: $PROJECT/project.yaml (24 passed, 1 warnings, 1 not testable, 0 failed; 25/26 applicable checks executed) + NOTE some checks could not run here; install openmapstack[geo] for geodata checks, QGIS for the .qgz checks diff --git a/tests/goldens/verify/river-crossings.json b/tests/goldens/verify/river-crossings.json new file mode 100644 index 0000000..b5f1013 --- /dev/null +++ b/tests/goldens/verify/river-crossings.json @@ -0,0 +1,185 @@ +{ + "checks": [ + { + "check": "project.parses", + "message": "project.yaml parses", + "status": "passed" + }, + { + "check": "project.conforms_to_schema", + "message": "project.yaml conforms to the packaged OpenMapStack v1 JSON Schema", + "status": "passed" + }, + { + "check": "project.graph_resolves", + "message": "graph resolves: 3 steps, 3 produced symbols, 1 outputs all traced to real steps", + "status": "passed" + }, + { + "check": "project.one_canonical_pipeline", + "message": "pipeline.py is the canonical implementation; wrappers import it", + "status": "passed" + }, + { + "check": "project.assumptions_have_rationale", + "message": "all 1 assumptions have statement + rationale", + "status": "passed" + }, + { + "check": "project.status_agrees_with_validation_report", + "message": "project.status 'validated' agrees with report status 'passed'", + "status": "passed" + }, + { + "args": { + "files": [ + "data/derived/crossing-trails.geojson" + ] + }, + "check": "project.declared_files_exist", + "message": "all 1 declared files exist", + "status": "passed" + }, + { + "check": "project.parameters_match_steps", + "message": "1 runtime parameter(s) declared; 0 bound to a processing step agree with it", + "status": "passed" + }, + { + "check": "provenance.every_source_has_provider_and_access", + "message": "all 2 sources declare provider + access method + retrieval timestamp", + "status": "passed" + }, + { + "check": "provenance.every_source_pinned", + "message": "all 2 sources are pinned (version_identity)", + "status": "passed" + }, + { + "check": "provenance.no_inline_credentials", + "message": "no inline credentials in 2 sources; connections are by reference", + "status": "passed" + }, + { + "check": "provenance.license_present_where_required", + "message": "license metadata present for required sources", + "status": "passed" + }, + { + "check": "provenance.rationale_present", + "message": "all 2 sources document selection rationale", + "status": "passed" + }, + { + "check": "overrides.every_override_has_provenance", + "message": "no overrides declared (vacuously true)", + "status": "passed" + }, + { + "check": "overrides.evidence_not_placeholder", + "message": "no placeholder evidence found", + "status": "passed" + }, + { + "check": "validation.required_all_present", + "message": "all 2 declared checks present exactly once in report", + "status": "passed" + }, + { + "check": "validation.no_implicit_pass", + "message": "every check has an explicit status", + "status": "passed" + }, + { + "check": "validation.warning_or_failed_propagates_to_status", + "message": "overall status 'passed' correctly reflects check statuses", + "status": "passed" + }, + { + "check": "validation.run_record_matches", + "message": "report run_id 'run-20260901-000000' matches a real run record with consistent hashes", + "status": "passed" + }, + { + "check": "geodata.crs_not_used_for_metrics", + "message": "analysis_crs EPSG:3301 is valid for 0 metric operation(s); storage/load/reprojection steps were excluded", + "status": "passed" + }, + { + "args": { + "path": "data/derived/crossing-trails.geojson" + }, + "check": "geodata.geometry_all_valid", + "message": "data/derived/crossing-trails.geojson: all 3 features have valid geometry", + "status": "passed" + }, + { + "args": { + "expected": "EPSG:3301", + "path": "data/derived/crossing-trails.geojson" + }, + "check": "geodata.dataset_crs_is", + "message": "data/derived/crossing-trails.geojson actual CRS metadata is EPSG:3301", + "status": "passed" + }, + { + "check": "presentation.layers_use_semantic_roles", + "message": "all 1 layers declare a semantic_role", + "status": "passed" + }, + { + "check": "presentation.controls_match_pipeline", + "message": "0 filter control(s) and 0 scenario control(s) consistent", + "status": "passed" + }, + { + "check": "presentation.edit_targets_reference_real_sources", + "message": "no edit targets declared (vacuously true)", + "status": "passed" + }, + { + "check": "metamorphic.declarations_valid", + "message": "1 metamorphic relation(s) are well-formed", + "status": "passed" + }, + { + "args": { + "forbidden_fragments": [], + "id": "trail-order" + }, + "check": "metamorphic.trail-order", + "evidence": { + "id": "trail-order", + "relation": "input_permutation_invariance", + "schema": "openmapstack-metamorphic/v1", + "variant": { + "features": 6, + "seed": 7, + "transformation": "permute_features" + } + }, + "message": "trail-order: input_permutation_invariance holds across 1 output(s)", + "status": "passed" + }, + { + "check": "rerun.no_chat_dependency", + "message": "canonical project dependencies contain no chat/transcript references", + "status": "passed" + } + ], + "counts": { + "failed": 0, + "not_testable": 0, + "passed": 28, + "warning": 0 + }, + "coverage": { + "applicable": 28, + "executed": 28, + "execution_rate": 1.0, + "not_testable": 0 + }, + "project_file": "$PROJECT/project.yaml", + "schema": "openmapstack-verify-result/v1", + "status": "passed" +} diff --git a/tests/goldens/verify/river-crossings.txt b/tests/goldens/verify/river-crossings.txt new file mode 100644 index 0000000..d53c6c8 --- /dev/null +++ b/tests/goldens/verify/river-crossings.txt @@ -0,0 +1,29 @@ +PASS project.parses: project.yaml parses +PASS project.conforms_to_schema: project.yaml conforms to the packaged OpenMapStack v1 JSON Schema +PASS project.graph_resolves: graph resolves: 3 steps, 3 produced symbols, 1 outputs all traced to real steps +PASS project.one_canonical_pipeline: pipeline.py is the canonical implementation; wrappers import it +PASS project.assumptions_have_rationale: all 1 assumptions have statement + rationale +PASS project.status_agrees_with_validation_report: project.status 'validated' agrees with report status 'passed' +PASS project.declared_files_exist: all 1 declared files exist +PASS project.parameters_match_steps: 1 runtime parameter(s) declared; 0 bound to a processing step agree with it +PASS provenance.every_source_has_provider_and_access: all 2 sources declare provider + access method + retrieval timestamp +PASS provenance.every_source_pinned: all 2 sources are pinned (version_identity) +PASS provenance.no_inline_credentials: no inline credentials in 2 sources; connections are by reference +PASS provenance.license_present_where_required: license metadata present for required sources +PASS provenance.rationale_present: all 2 sources document selection rationale +PASS overrides.every_override_has_provenance: no overrides declared (vacuously true) +PASS overrides.evidence_not_placeholder: no placeholder evidence found +PASS validation.required_all_present: all 2 declared checks present exactly once in report +PASS validation.no_implicit_pass: every check has an explicit status +PASS validation.warning_or_failed_propagates_to_status: overall status 'passed' correctly reflects check statuses +PASS validation.run_record_matches: report run_id 'run-20260901-000000' matches a real run record with consistent hashes +PASS geodata.crs_not_used_for_metrics: analysis_crs EPSG:3301 is valid for 0 metric operation(s); storage/load/reprojection steps were excluded +PASS geodata.geometry_all_valid [data/derived/crossing-trails.geojson]: data/derived/crossing-trails.geojson: all 3 features have valid geometry +PASS geodata.dataset_crs_is [data/derived/crossing-trails.geojson]: data/derived/crossing-trails.geojson actual CRS metadata is EPSG:3301 +PASS presentation.layers_use_semantic_roles: all 1 layers declare a semantic_role +PASS presentation.controls_match_pipeline: 0 filter control(s) and 0 scenario control(s) consistent +PASS presentation.edit_targets_reference_real_sources: no edit targets declared (vacuously true) +PASS metamorphic.declarations_valid: 1 metamorphic relation(s) are well-formed +PASS metamorphic.trail-order: trail-order: input_permutation_invariance holds across 1 output(s) +PASS rerun.no_chat_dependency: canonical project dependencies contain no chat/transcript references +PASSED: $PROJECT/project.yaml (28 passed, 0 warnings, 0 not testable, 0 failed; 28/28 applicable checks executed) diff --git a/tests/test_check_api.py b/tests/test_check_api.py new file mode 100644 index 0000000..74790d1 --- /dev/null +++ b/tests/test_check_api.py @@ -0,0 +1,194 @@ +"""The versioned check API an external harness (OpenMapBench) consumes. + +``ConsumerFixtureTests`` plays the role of that harness: it uses only the +public API, the CLI, and the packaged JSON schemas -- never +``openmapstack.checks`` directly -- and proves it can list, negotiate, run, +and validate a result without vendoring a single check. +""" + +from __future__ import annotations + +import importlib.util +import io +import json +import subprocess +import sys +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +from openmapstack import __version__, api +from openmapstack.cli import main +from openmapstack.schema import validation_errors +from openmapstack.verify import verify_project +from tests.evals.helpers import make_workspace, minimal_project, write_project + +REPO_ROOT = Path(__file__).resolve().parents[1] +SCHEMAS = REPO_ROOT / "openmapstack" / "schemas" + + +def _schema(name: str) -> dict: + return json.loads((SCHEMAS / name).read_text(encoding="utf-8")) + + +class CatalogueTests(unittest.TestCase): + def test_every_public_check_is_listed_with_its_parameters(self) -> None: + names = {descriptor.name for descriptor in api.list_checks()} + for expected in ( + "project.conforms_to_schema", "provenance.every_source_pinned", "geodata.geometry_all_valid", + "validation.run_record_matches", "qgis.static_valid", "rerun.clean_execution_succeeded", + "metamorphic.relation_holds", "presentation.controls_match_pipeline", + ): + self.assertIn(expected, names) + descriptor = api.describe_check("geodata.dataset_crs_is") + by_name = {parameter.name: parameter for parameter in descriptor.parameters} + self.assertTrue(by_name["path"].required) + self.assertTrue(by_name["expected"].required) + self.assertFalse(by_name["project_dir"].required) + self.assertEqual(by_name["project_dir"].default, ".") + + def test_known_answer_checks_are_marked_and_everything_else_is_oracle_free(self) -> None: + catalogue = {descriptor.name: descriptor for descriptor in api.list_checks()} + for name in api.KNOWN_ANSWER_CHECKS: + self.assertFalse(catalogue[name].oracle_free, name) + self.assertTrue(catalogue["geodata.geometry_all_valid"].oracle_free) + self.assertEqual(catalogue["metamorphic.relation_holds"].dimension, "metamorphic_evidence") + self.assertEqual(catalogue["visual.render_substantive"].dimension, "visual_judgement") + self.assertEqual(catalogue["geodata.row_count"].dimension, "gis_correctness") + + def test_private_helpers_are_not_exposed(self) -> None: + names = {descriptor.name for descriptor in api.list_checks()} + self.assertFalse(any(".‗" in name or "._" in name for name in names)) + with self.assertRaises(api.CheckAPIError): + api.describe_check("geodata._read") + with self.assertRaises(api.CheckAPIError): + api.describe_check("spatial.connect_spatial") + + def test_eval_runner_uses_the_api_dimensions(self) -> None: + spec = importlib.util.spec_from_file_location("openmapstack_eval_runner_api_test", REPO_ROOT / "evals" / "run.py") + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + self.assertIs(module.DIMENSIONS, api.DIMENSIONS) + + +class RunCheckTests(unittest.TestCase): + def test_result_validates_and_carries_stable_codes(self) -> None: + workspace = make_workspace() + project = minimal_project() + project["sources"]["test_source"]["version"] = {"identifier": "latest"} + write_project(workspace, project) + record = api.run_check("provenance.every_source_pinned", workspace) + self.assertEqual(validation_errors(record, _schema("check-result-v1.schema.json")), []) + self.assertEqual((record["status"], record["code"]), ("failed", "source_unpinned")) + self.assertEqual(record["dimension"], "provenance") + self.assertEqual(record["api_version"], api.CHECK_API_VERSION) + + def test_passed_results_have_no_code(self) -> None: + workspace = make_workspace() + write_project(workspace, minimal_project()) + record = api.run_check("project.parses", workspace) + self.assertEqual(record["status"], "passed") + self.assertIsNone(record["code"]) + self.assertEqual(validation_errors(record, _schema("check-result-v1.schema.json")), []) + + def test_unknown_check_and_bad_args_are_consumer_errors(self) -> None: + workspace = make_workspace() + with self.assertRaises(api.CheckAPIError): + api.run_check("geodata.nope", workspace) + with self.assertRaises(api.CheckAPIError): + api.run_check("geodata.dataset_crs_is", workspace, {"path": "x"}) # missing expected + with self.assertRaises(api.CheckAPIError): + api.run_check("project.parses", workspace, {"bogus": 1}) + + def test_a_raising_check_is_not_testable_never_passed(self) -> None: + from unittest.mock import patch + + workspace = make_workspace() + write_project(workspace, minimal_project()) + def boom(workspace, project_dir="."): + raise RuntimeError("boom") + + boom.__module__ = "openmapstack.checks.project" + with patch("openmapstack.checks.project.graph_resolves", new=boom): + record = api.run_check("project.graph_resolves", workspace) + self.assertEqual((record["status"], record["code"]), ("not_testable", "check_error")) + self.assertEqual(validation_errors(record, _schema("check-result-v1.schema.json")), []) + + +class NegotiationTests(unittest.TestCase): + def test_compatible_when_api_version_and_checks_match(self) -> None: + answer = api.negotiate(min_package_version="0.1.0", required_checks=["project.parses"]) + self.assertTrue(answer["compatible"], answer) + + def test_incompatible_answers_say_why(self) -> None: + answer = api.negotiate(required_api="openmapstack-check-api/v2", min_package_version="99.0.0", required_checks=["geodata.magic"]) + self.assertFalse(answer["compatible"]) + self.assertEqual(len(answer["problems"]), 3) + with self.assertRaises(api.CheckAPIError): + api.negotiate(min_package_version="latest") + + def test_api_info_describes_the_installation(self) -> None: + info = api.api_info() + self.assertEqual(info["schema"], api.API_INFO_SCHEMA) + self.assertEqual(info["package_version"], __version__) + self.assertEqual(info["statuses"], ["passed", "failed", "warning", "not_testable"]) + self.assertIn("metamorphic_evidence", info["dimensions"]) + + +class VerifyResultSchemaTests(unittest.TestCase): + def test_verify_json_validates_against_the_packaged_schema(self) -> None: + workspace = make_workspace() + write_project(workspace, minimal_project()) + payload = verify_project(workspace / "project.yaml").to_dict() + self.assertEqual(api.validate_verify_result(payload), []) + + +class ConsumerFixtureTests(unittest.TestCase): + """A harness that vendors nothing: CLI + JSON schemas only.""" + + def _cli(self, *argv: str) -> tuple[int, str]: + out, err = io.StringIO(), io.StringIO() + with redirect_stdout(out), redirect_stderr(err): + code = main(list(argv)) + return code, out.getvalue() + + def test_list_negotiate_run_validate(self) -> None: + code, text = self._cli("api-info", "--json", "--require-api", api.CHECK_API_VERSION, "--min-version", "0.2.0", "--require-check", "validation.run_record_matches") + self.assertEqual(code, 0, text) + info = json.loads(text) + self.assertTrue(info["negotiation"]["compatible"]) + + code, text = self._cli("checks", "--json") + self.assertEqual(code, 0) + catalogue = json.loads(text) + names = {entry["name"] for entry in catalogue["checks"]} + self.assertIn("geodata.dataset_crs_is", names) + + workspace = make_workspace() + project = minimal_project() + project["outputs"] = {"final": {"path": "data/derived/final.json", "format": "GeoJSON", "generated_by": "export"}} + write_project(workspace, project) + code, text = self._cli("check", "project.declared_files_exist", str(workspace), "--arg", 'files=["data/derived/final.json"]', "--json") + record = json.loads(text) + self.assertEqual(code, 1) + self.assertEqual(record["status"], "failed") + self.assertEqual(validation_errors(record, _schema("check-result-v1.schema.json")), []) + + code, text = self._cli("api-info", "--json", "--require-api", "openmapstack-check-api/v9") + self.assertEqual(code, 1) + self.assertFalse(json.loads(text)["negotiation"]["compatible"]) + + def test_consumer_error_is_exit_two_not_a_graded_result(self) -> None: + code, text = self._cli("check", "geodata.nope", str(make_workspace()), "--json") + self.assertEqual(code, 2) + self.assertEqual(json.loads(text)["code"], "consumer_error") + + def test_the_cli_is_reachable_as_a_subprocess(self) -> None: + completed = subprocess.run([sys.executable, "-m", "openmapstack", "api-info", "--json"], capture_output=True, text=True, cwd=REPO_ROOT, check=False) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(json.loads(completed.stdout)["check_api_version"], api.CHECK_API_VERSION) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_connectors.py b/tests/test_connectors.py new file mode 100644 index 0000000..4095bdc --- /dev/null +++ b/tests/test_connectors.py @@ -0,0 +1,384 @@ +"""Connector pilot: read-only discovery, approval-gated snapshots, limits, redaction. + +The DuckDB local-file tests run wherever ``openmapstack[geo]`` is installed. +The PostGIS session/discovery/plan contract is tested against a fake DB-API +driver so it needs no server; the end-to-end snapshot path runs only when +``OPENMAPSTACK_TEST_POSTGIS_DSN`` names a reachable PostGIS database. +""" + +from __future__ import annotations + +import io +import json +import os +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +import yaml + +from openmapstack.checks.spatial import connect_spatial +from openmapstack.cli import main +from openmapstack.connectors import ( + ConnectorError, + ConnectorLimits, + ConnectorUnavailable, + apply_snapshot_to_manifest, + discover_source, + require_read_only_select, + resolve_connection_reference, + snapshot_source, +) +from openmapstack.connectors.postgis import PostGISConnector +from openmapstack.sources import assess_pin +from openmapstack.validation import validate_project +from tests.evals.helpers import make_workspace, minimal_project, write_project + +REPO_ROOT = Path(__file__).resolve().parents[1] +FIXTURES = REPO_ROOT / "evals" / "fixtures" / "mini-tartu" +DUCKDB_AVAILABLE = connect_spatial() is not None +LIVE_DSN = os.environ.get("OPENMAPSTACK_TEST_POSTGIS_DSN", "") + + +def _duckdb_project(): + workspace = make_workspace() + project = minimal_project() + project["sources"]["test_source"]["warehouse"] = {"backend": "duckdb"} + write_project(workspace, project) + (workspace / "data/source").mkdir(parents=True) + for name in ("parcels.geojson", "roads.geojson"): + (workspace / "data/source" / name).write_bytes((FIXTURES / name).read_bytes()) + return workspace, project + + +class QueryPolicyTests(unittest.TestCase): + def test_only_a_single_select_is_accepted(self) -> None: + self.assertTrue(require_read_only_select(" SELECT 1; ").startswith("SELECT")) + self.assertTrue(require_read_only_select("WITH t AS (SELECT 1) SELECT * FROM t")) + for bad in ( + "DROP TABLE parcels", + "SELECT 1; SELECT 2", + "SELECT * FROM read_text('/etc/passwd')", + "COPY (SELECT 1) TO '/tmp/x'", + "SELECT * INTO backup FROM parcels", + "SET enable_external_access = true", + "INSTALL httpfs", + "", + ): + with self.assertRaises(ConnectorError, msg=bad) as caught: + require_read_only_select(bad) + self.assertEqual(caught.exception.code, "query_rejected") + + def test_keywords_inside_string_literals_do_not_trip_the_policy(self) -> None: + self.assertTrue(require_read_only_select("SELECT 'drop' AS word, 'set;' AS other")) + + def test_connection_references_resolve_without_recording_secrets(self) -> None: + scheme, secret = resolve_connection_reference("env:OMS_TEST_DSN", environ={"OMS_TEST_DSN": "postgresql://u:pw@h/db"}) + self.assertEqual((scheme, secret), ("env", "postgresql://u:pw@h/db")) + self.assertEqual(resolve_connection_reference({"ref": "service:geo"}, environ={}), ("service", "service=geo")) + with self.assertRaises(ConnectorError) as caught: + resolve_connection_reference("env:OMS_TEST_DSN", environ={}) + self.assertEqual(caught.exception.code, "connection_unresolved") + with self.assertRaises(ConnectorError): + resolve_connection_reference("postgresql://u:pw@h/db", environ={}) + with self.assertRaises(ConnectorError): + resolve_connection_reference("file:relative/path", environ={}) + + def test_connector_errors_are_redacted(self) -> None: + error = ConnectorError("cannot open postgresql://gis:hunter2@db/gis", code="connection_failed") + self.assertNotIn("hunter2", str(error)) + self.assertIn("gis:***@db", str(error)) + + +@unittest.skipUnless(DUCKDB_AVAILABLE, "DuckDB Spatial is not available") +class DuckDBLocalConnectorTests(unittest.TestCase): + def test_discovery_describes_files_read_only(self) -> None: + workspace, project = _duckdb_project() + discovery = discover_source(project, "test_source", project_root=workspace) + self.assertEqual(discovery.backend, "duckdb") + self.assertTrue(discovery.read_only) + by_name = {table.name: table for table in discovery.tables} + self.assertEqual(by_name["parcels.geojson"].geometry_column, "geom") + self.assertEqual(by_name["parcels.geojson"].srid, 3301) + self.assertEqual(by_name["parcels.geojson"].row_estimate, 5) + self.assertEqual(discovery.notes, []) + + def test_snapshot_is_a_dry_run_unless_approved(self) -> None: + workspace, project = _duckdb_project() + query = 'SELECT cadastral_id, geom FROM "parcels.geojson" WHERE land_use = \'ARIMAA\'' + record = snapshot_source(project, "test_source", query, "data/source/arimaa.parquet", project_root=workspace) + self.assertFalse(record["materialized"]) + self.assertEqual(record["plan"]["row_count"], 2) + self.assertFalse((workspace / "data/source/arimaa.parquet").exists()) + + record = snapshot_source(project, "test_source", query, "data/source/arimaa.parquet", project_root=workspace, approve=True) + self.assertTrue(record["materialized"]) + self.assertEqual(record["rows"], 2) + self.assertEqual(record["pin"]["class"], "local_snapshot") + self.assertEqual(record["warehouse"]["query_sha256"], record["plan"]["query_sha256"]) + # The pin it hands back is one the pin contract accepts as pinned. + updated = apply_snapshot_to_manifest(project, "test_source", record) + self.assertEqual(assess_pin(workspace, updated["sources"]["test_source"]).status, "pinned") + write_project(workspace, updated) + (workspace / "pipeline.py").write_text("print('ok')\n", encoding="utf-8") + checks = {c.id: c.status for c in validate_project(workspace / "project.yaml", artifacts=False).checks if c.id in {"source.pin", "source.credentials"}} + self.assertEqual(checks, {"source.pin": "passed", "source.credentials": "passed"}) + + def test_snapshots_never_overwrite_an_existing_source(self) -> None: + workspace, project = _duckdb_project() + with self.assertRaises(ConnectorError) as caught: + snapshot_source(project, "test_source", 'SELECT * FROM "parcels.geojson"', "data/source/parcels.geojson", project_root=workspace, approve=True) + self.assertEqual(caught.exception.code, "destination_invalid") + (workspace / "data/source/taken.parquet").write_bytes(b"x") + with self.assertRaises(ConnectorError) as caught: + snapshot_source(project, "test_source", 'SELECT * FROM "parcels.geojson"', "data/source/taken.parquet", project_root=workspace, approve=True) + self.assertEqual(caught.exception.code, "destination_exists") + with self.assertRaises(ConnectorError) as caught: + snapshot_source(project, "test_source", 'SELECT * FROM "parcels.geojson"', "data/derived/out.parquet", project_root=workspace, approve=True) + self.assertEqual(caught.exception.code, "destination_invalid") + + def test_row_and_byte_limits_refuse_and_leave_no_partial_file(self) -> None: + workspace, project = _duckdb_project() + with self.assertRaises(ConnectorError) as caught: + snapshot_source(project, "test_source", 'SELECT * FROM "parcels.geojson"', "data/source/big.parquet", project_root=workspace, limits=ConnectorLimits(max_rows=2)) + self.assertEqual(caught.exception.code, "row_limit_exceeded") + with self.assertRaises(ConnectorError) as caught: + snapshot_source(project, "test_source", 'SELECT * FROM "parcels.geojson"', "data/source/tiny.parquet", project_root=workspace, approve=True, limits=ConnectorLimits(max_bytes=50)) + self.assertEqual(caught.exception.code, "byte_limit_exceeded") + self.assertEqual(sorted(p.name for p in (workspace / "data/source").iterdir()), ["parcels.geojson", "roads.geojson"]) + + def test_file_access_is_confined_to_the_source_root(self) -> None: + workspace, project = _duckdb_project() + outside = workspace / "outside.parquet" + (workspace / "data/derived").mkdir() + for target in (outside, workspace / "data/derived/secret.parquet"): + with self.assertRaises(ConnectorError, msg=target) as caught: + snapshot_source(project, "test_source", f"SELECT * FROM read_parquet('{target.as_posix()}')", "data/source/leak.parquet", project_root=workspace) + self.assertEqual(caught.exception.code, "query_failed") + + def test_cli_discover_and_snapshot(self) -> None: + workspace, _ = _duckdb_project() + out = io.StringIO() + with redirect_stdout(out): + self.assertEqual(main(["source", "discover", str(workspace), "--source", "test_source", "--json"]), 0) + self.assertEqual(json.loads(out.getvalue())["schema"], "openmapstack-source-discovery/v1") + out = io.StringIO() + with redirect_stdout(out): + code = main(["source", "snapshot", str(workspace), "--source", "test_source", "--query", 'SELECT * FROM "roads.geojson"', "--destination", "data/source/roads.parquet"]) + self.assertEqual(code, 0) + self.assertIn("DRY RUN", out.getvalue()) + out = io.StringIO() + with redirect_stdout(out): + code = main(["source", "snapshot", str(workspace), "--source", "test_source", "--query", 'SELECT * FROM "roads.geojson"', "--destination", "data/source/roads.parquet", "--approve", "--write-manifest", "--json"]) + self.assertEqual(code, 0) + record = json.loads(out.getvalue()) + self.assertTrue(record["materialized"]) + manifest = yaml.safe_load((workspace / "project.yaml").read_text(encoding="utf-8")) + self.assertEqual(manifest["sources"]["test_source"]["pin"]["path"], "data/source/roads.parquet") + err = io.StringIO() + with redirect_stderr(err), redirect_stdout(io.StringIO()): + self.assertEqual(main(["source", "snapshot", str(workspace), "--source", "test_source", "--query", "DROP TABLE x", "--destination", "data/source/x.parquet"]), 2) + self.assertIn("query_rejected", err.getvalue()) + + def test_unsupported_backend_is_refused_not_guessed(self) -> None: + workspace, project = _duckdb_project() + project["sources"]["test_source"]["warehouse"] = {"backend": "bigquery"} + project["sources"]["test_source"]["access"]["connection"] = "env:BQ" + with self.assertRaises(ConnectorError) as caught: + discover_source(project, "test_source", project_root=workspace, environ={"BQ": "x"}) + self.assertEqual(caught.exception.code, "backend_unsupported") + + +@unittest.skipUnless(DUCKDB_AVAILABLE, "DuckDB Spatial is not available") +class NumericMaterialisationTests(unittest.TestCase): + """NUMERIC must survive materialisation exactly; a rounded snapshot that + is then hashed and pinned would be reproducible and wrong.""" + + def _round_trip(self, values): + from decimal import Decimal + + from openmapstack.connectors.postgis import _write_parquet + + duck = connect_spatial() + destination = make_workspace() / "numeric.parquet" + rows = [(f"r{i}", value) for i, value in enumerate(values)] + try: + _write_parquet(duck, destination, [{"name": "id", "type": "text"}, {"name": "amount", "type": "numeric"}], [], {}, rows) + described = duck.execute(f"DESCRIBE SELECT * FROM read_parquet('{destination.as_posix()}')").fetchall() + read = duck.execute(f"SELECT amount FROM read_parquet('{destination.as_posix()}') ORDER BY id").fetchall() + finally: + duck.close() + return {name: str(type_name) for name, type_name, *_ in described}["amount"], [row[0] for row in read] + + def test_high_precision_values_are_exact(self) -> None: + from decimal import Decimal + + values = [Decimal("12345678901234567890.123456789"), Decimal("0.000000001"), None] + type_name, read = self._round_trip(values) + self.assertTrue(type_name.startswith("DECIMAL"), type_name) + self.assertEqual(read[0], values[0]) + self.assertEqual(read[1], values[1]) + self.assertIsNone(read[2]) + self.assertNotEqual(float(values[0]), values[0]) # DOUBLE would have rounded it + + def test_beyond_decimal_38_keeps_exact_text(self) -> None: + from decimal import Decimal + + huge = Decimal("1" * 30 + "." + "2" * 20) + type_name, read = self._round_trip([huge]) + self.assertEqual(type_name, "VARCHAR") + self.assertEqual(Decimal(read[0]), huge) + + +class _FakeCursor: + """Answers the exact statements the PostGIS connector issues.""" + + def __init__(self, log: list[str], *, snapshot_supported: bool = True) -> None: + self.log = log + self.description = None + self._rows: list[tuple] = [] + self._snapshot_supported = snapshot_supported + + def execute(self, statement: str, params=None) -> None: + self.log.append(statement if params is None else f"{statement} -- {params}") + text = statement.strip().lower() + self.description = None + if text.startswith("select current_database()"): + self._rows = [("gis", "reader", "PostgreSQL 16.3 (Debian), compiled by gcc")] + elif "from geometry_columns" in text: + self._rows = [("cadastre", "parcels", "geom", 3301, "MULTIPOLYGON", 79056, "r")] + elif text.startswith("show default_transaction_read_only"): + self._rows = [("on",)] + elif "limit 0" in text: + self.description = [("cadastral_id", 25, None, None, None, None, None), ("geom", 17000, None, None, None, None, None)] + self._rows = [] + elif text.startswith("select oid, typname"): + self._rows = [(25, "text"), (17000, "geometry")] + elif text.startswith("select count(*)"): + self._rows = [(3,)] + elif "pg_current_snapshot" in text: + if not self._snapshot_supported: + raise RuntimeError("function pg_current_snapshot() does not exist") + self._rows = [("1001:1001:",)] + else: + self._rows = [] + + def fetchone(self): + return self._rows[0] if self._rows else None + + def fetchall(self): + return list(self._rows) + + +class _FakeConnection: + def __init__(self, log: list[str], **kwargs) -> None: + self.log = log + self.kwargs = kwargs + self.closed = False + + def cursor(self): + return _FakeCursor(self.log, **self.kwargs) + + def rollback(self) -> None: + self.log.append("ROLLBACK") + + def close(self) -> None: + self.closed = True + + +class PostGISContractTests(unittest.TestCase): + def test_sessions_are_read_only_with_a_statement_timeout(self) -> None: + log: list[str] = [] + connector = PostGISConnector("postgresql://reader:pw@db/gis", connect=lambda dsn: _FakeConnection(log)) + discovery = connector.discover(ConnectorLimits(timeout_s=12.5)) + self.assertEqual(log[0], "SET default_transaction_read_only = on") + self.assertIn("SET statement_timeout = 12500", log) + self.assertTrue(discovery.read_only) + [table] = discovery.tables + self.assertEqual((table.schema, table.name, table.geometry_column, table.srid, table.row_estimate), ("cadastre", "parcels", "geom", 3301, 79056)) + self.assertEqual(discovery.identity["database"], "gis") + self.assertNotIn("pw", json.dumps(discovery.to_dict())) + + def test_plan_records_schema_digest_and_non_durable_backend_snapshot(self) -> None: + log: list[str] = [] + connector = PostGISConnector("dsn", connect=lambda dsn: _FakeConnection(log)) + plan = connector.plan("SELECT cadastral_id, geom FROM cadastre.parcels", ConnectorLimits()) + self.assertEqual(plan.row_count, 3) + self.assertEqual([c["type"] for c in plan.columns], ["text", "geometry"]) + self.assertEqual(plan.backend_snapshot["kind"], "pg_current_snapshot") + self.assertFalse(plan.backend_snapshot["durable"]) + self.assertTrue(plan.schema_sha256.startswith("sha256:")) + + def test_plan_survives_servers_without_pg_current_snapshot(self) -> None: + log: list[str] = [] + connector = PostGISConnector("dsn", connect=lambda dsn: _FakeConnection(log, snapshot_supported=False)) + plan = connector.plan("SELECT 1", ConnectorLimits()) + self.assertIsNone(plan.backend_snapshot) + + def test_missing_driver_is_reported_not_hidden(self) -> None: + import openmapstack.connectors.postgis as module + + original = module._default_connect + + def unavailable(): + raise ConnectorUnavailable("no driver") + + module._default_connect = unavailable + try: + with self.assertRaises(ConnectorUnavailable): + PostGISConnector("dsn").discover(ConnectorLimits()) + finally: + module._default_connect = original + + def test_driver_errors_never_leak_the_dsn(self) -> None: + def failing(dsn: str): + raise RuntimeError(f"could not connect to {dsn}") + + connector = PostGISConnector("postgresql://gis:hunter2@db/gis", connect=failing) + with self.assertRaises(ConnectorError) as caught: + connector.discover(ConnectorLimits()) + self.assertEqual(caught.exception.code, "connection_failed") + self.assertNotIn("hunter2", str(caught.exception)) + + +@unittest.skipUnless(LIVE_DSN and DUCKDB_AVAILABLE, "OPENMAPSTACK_TEST_POSTGIS_DSN is not set") +class PostGISLiveTests(unittest.TestCase): + """End-to-end against a real PostGIS: discovery, dry run, materialised + GeoParquet snapshot, and a pin the contract accepts.""" + + def test_snapshot_round_trip(self) -> None: + workspace = make_workspace() + project = minimal_project() + project["sources"]["test_source"]["warehouse"] = {"backend": "postgis", "schema": "public", "table": "parcels"} + project["sources"]["test_source"]["access"]["connection"] = "env:OPENMAPSTACK_TEST_POSTGIS_DSN" + write_project(workspace, project) + discovery = discover_source(project, "test_source", project_root=workspace) + self.assertTrue(discovery.read_only) + self.assertTrue(any(table.name == "parcels" for table in discovery.tables), discovery.to_dict()) + query = "SELECT cadastral_id, land_use, area_m2, geom FROM public.parcels ORDER BY cadastral_id" + dry = snapshot_source(project, "test_source", query, "data/source/parcels.parquet", project_root=workspace) + self.assertFalse(dry["materialized"]) + record = snapshot_source(project, "test_source", query, "data/source/parcels.parquet", project_root=workspace, approve=True) + self.assertTrue(record["materialized"]) + self.assertEqual(record["rows"], dry["plan"]["row_count"]) + self.assertNotIn("oms-test-pw", json.dumps(record)) + updated = apply_snapshot_to_manifest(project, "test_source", record) + self.assertEqual(assess_pin(workspace, updated["sources"]["test_source"]).status, "pinned") + connection = connect_spatial() + try: + rows = connection.execute(f"SELECT cadastral_id, ST_GeometryType(geom), area_m2 FROM read_parquet('{(workspace / 'data/source/parcels.parquet').as_posix()}') ORDER BY 1").fetchall() + finally: + connection.close() + self.assertEqual([row[0] for row in rows], ["P1", "P2", "P3"]) + self.assertTrue(all("POLYGON" in str(row[1]).upper() for row in rows), rows) + from decimal import Decimal + + # NUMERIC(24, 9) survives exactly; a DOUBLE mapping would have rounded it. + self.assertEqual(rows[0][2], Decimal("10000.123456789")) + # A write attempt through the same reference must be refused by the policy. + with self.assertRaises(ConnectorError): + snapshot_source(project, "test_source", "DELETE FROM public.parcels", "data/source/x.parquet", project_root=workspace, approve=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_evals.py b/tests/test_evals.py index f1a4911..0670779 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -27,7 +27,7 @@ from adapters.base import AgentRunResult # noqa: E402 -class EvalRunnerTests(unittest.TestCase): +class _RunnerHarness(unittest.TestCase): def setUp(self) -> None: self.tempdir = tempfile.TemporaryDirectory(prefix="openmapstack-eval-runner-test-") self.root = Path(self.tempdir.name) @@ -115,6 +115,8 @@ def call_main(self, argv: list[str]) -> tuple[int, str, str]: exit_code = eval_runner.main(argv) return exit_code, stdout.getvalue(), stderr.getvalue() + +class EvalRunnerTests(_RunnerHarness): def test_fixture_mode_is_default_and_passes(self) -> None: self.write_case() exit_code, stdout, _ = self.call_main([]) @@ -915,6 +917,117 @@ def test_case_flag_can_select_multiple_cases(self) -> None: self.assertEqual([result["id"] for result in payload["results"]], ["first", "second"]) +class PairedArmTests(_RunnerHarness): + """Paired plain/oms arms, arm provenance, and task export (issue #13, C2/C3).""" + + def _adapter(self, seen: list[tuple[str, int | None]]): + class Adapter: + executable = "fake-agent" + + @staticmethod + def is_available() -> bool: + return True + + @staticmethod + def run(prompt, workspace, fixture=None, timeout_s=900, model=None, seed=None): + skill_present = (workspace.parent / "benchmark-context" / "openmapstack" / "SKILL.md").is_file() + seen.append(("oms" if skill_present else "plain", seed)) + (workspace / "marker.txt").write_text("ok\n", encoding="utf-8") + return AgentRunResult( + agent="codex", model="observed-model", workspace=workspace, duration_s=0.1, success=True, returncode=0, + command=["fake-agent"], version="fake 2", events=[{"type": "x"}] * (3 if skill_present else 5), + usage={"total_tokens": 10 if skill_present else 20}, cost_usd=0.02 if skill_present else 0.01, + permissions={}, metadata={"structured_completion": True, "temperature": 0.0}, + ) + + return Adapter() + + def test_paired_arms_share_cases_trials_and_seeds_and_are_reported_apart(self) -> None: + self.write_case("paired-live", modes=["live"]) + # Present in the selection, skipped in live mode: it must not enter + # the arm's task-set identity. + self.write_case("fixture-only-neighbour") + seen: list[tuple[str, int | None]] = [] + summary_path = self.root / "paired.json" + with patch.object(eval_runner, "_load_adapter", return_value=self._adapter(seen)): + exit_code, stdout, stderr = self.call_main([ + "--mode", "live", "--agent", "codex", "--model", "gpt-test", "--case", "paired-live", + "--arms", "paired", "--repetitions", "2", "--seed", "7", "--price-catalog-date", "2026-09-01", + "--run-id", "paired-run", "--results-dir", str(self.results_dir), "--json", str(summary_path), + ]) + self.assertEqual(exit_code, 0, (stdout, stderr)) + self.assertEqual(sorted(seen), [("oms", 7), ("oms", 8), ("plain", 7), ("plain", 8)]) + summary = json.loads(summary_path.read_text(encoding="utf-8")) + paired = summary["paired_arms"] + self.assertTrue(paired["task_parity"]) + self.assertEqual({row["arm"] for row in paired["pareto"]}, {"plain", "oms"}) + self.assertEqual(paired["arms"]["oms"]["quality"]["median_cost_usd"], 0.02) + self.assertEqual(paired["arms"]["plain"]["quality"]["median_tokens"], 20.0) + self.assertEqual(paired["arms"]["plain"]["diagnostics"]["median_event_count"], 5) + self.assertNotIn("success_per_dollar", json.dumps(paired)) + self.assertEqual(summary["run_config"]["arms"], ["plain", "oms"]) + self.assertEqual(summary["run_config"]["skill_mode"], "paired") + provenance = {record["arm"]: record for record in summary["run_config"]["arm_provenance"]} + self.assertEqual(provenance["oms"]["skill"]["mode"], "enabled") + self.assertRegex(provenance["oms"]["skill"]["content_sha256"], r"^sha256:[0-9a-f]{64}$") + self.assertIsNone(provenance["plain"]["skill"]["content_sha256"]) + self.assertEqual(provenance["oms"]["price_catalog_date"], "2026-09-01") + self.assertEqual(provenance["oms"]["tool_surface"], {"adapter": "codex", "agent_version": "fake 2"}) + self.assertEqual(provenance["oms"]["sampling"], {"seed": 7, "temperature": 0.0, "reasoning": None}) + self.assertEqual(provenance["oms"]["task_set"], provenance["plain"]["task_set"]) + self.assertEqual(provenance["oms"]["task_set"]["cases"], ["paired-live"]) + self.assertEqual(provenance["oms"]["checker"]["check_api_version"], "openmapstack-check-api/v1") + for arm in ("plain", "oms"): + bundle = self.results_dir / "paired-run" / "codex" / arm / "paired-live" / "1" + self.assertTrue((bundle / "grading.json").is_file(), bundle) + self.assertIn("paired arms (task parity)", stdout) + + def test_arm_flags_must_agree_and_default_to_oms(self) -> None: + self.write_case("arm-flags", modes=["live"]) + seen: list[tuple[str, int | None]] = [] + summary_path = self.root / "arm-flags.json" + with patch.object(eval_runner, "_load_adapter", return_value=self._adapter(seen)): + exit_code, _, _ = self.call_main(["--mode", "live", "--model", "requested-alias", "--case", "arm-flags", "--no-retain-artifacts", "--json", str(summary_path)]) + self.assertEqual(exit_code, 0) + self.assertEqual(seen, [("oms", None)]) + # --agent was omitted and the adapter reported what actually ran: + # provenance records the resolved adapter and observed model. + [record] = json.loads(summary_path.read_text(encoding="utf-8"))["run_config"]["arm_provenance"] + self.assertEqual(record["tool_surface"]["adapter"], "codex") + self.assertEqual(record["model"], {"provider": "openai", "id": "observed-model", "revision": None}) + with self.assertRaises(SystemExit): + self.call_main(["--mode", "live", "--agent", "codex", "--model", "m", "--skill-mode", "disabled", "--arms", "oms"]) + with self.assertRaises(SystemExit): + self.call_main(["--mode", "live", "--agent", "codex", "--model", "m", "--price-catalog-date", "yesterday"]) + + def test_export_tasks_writes_vendor_neutral_bundles(self) -> None: + fixture = self.root / "fixture.geojson" + fixture.write_text('{"type": "FeatureCollection", "features": []}', encoding="utf-8") + self.write_case("070-underspecified-prompt", modes=["live"], live_fixtures=[ + {"source": os.path.relpath(fixture, self.cases_dir / "070-underspecified-prompt"), "destination": "project/data/source/fixture.geojson"}, + ]) + # fixture sources must stay inside the eval tree for a live run, but export only reads them + self.write_case("fixture-only") + destination = self.root / "tasks" + exit_code, stdout, stderr = self.call_main(["--export-tasks", str(destination)]) + self.assertEqual(exit_code, 0, (stdout, stderr)) + index = json.loads((destination / "index.json").read_text(encoding="utf-8")) + self.assertEqual([task["id"] for task in index["tasks"]], ["070-underspecified-prompt"]) + task = json.loads((destination / "070-underspecified-prompt" / "task.json").read_text(encoding="utf-8")) + self.assertEqual(task["schema"], "openmapstack-benchmark-task/v1") + self.assertEqual(task["ownership"], "openmapbench") + self.assertEqual(task["prompt"], "Build the project.\n") + self.assertEqual(task["fixtures"][0]["destination"], "project/data/source/fixture.geojson") + self.assertTrue((destination / "070-underspecified-prompt" / "fixtures" / "fixture.geojson").is_file()) + self.assertFalse((destination / "070-underspecified-prompt" / "project").exists()) + self.assertEqual( + eval_runner.validation_errors(task, eval_runner._load_eval_schema("benchmark-task-v1.schema.json")), [] + ) + exit_code, _, stderr = self.call_main(["--export-tasks", str(self.root / "none"), "--case", "fixture-only"]) + self.assertEqual(exit_code, 2) + self.assertIn("No live-capable", stderr) + + class CapabilityRollupTests(unittest.TestCase): """A pass rate produced where part of the suite could not run is not the same evidence as one produced where all of it ran. The rollup makes that diff --git a/tests/test_metamorphic.py b/tests/test_metamorphic.py new file mode 100644 index 0000000..1e08162 --- /dev/null +++ b/tests/test_metamorphic.py @@ -0,0 +1,451 @@ +"""Metamorphic relations: positive, deliberate-defect, and invalid-precondition paths. + +Each relation is exercised three ways, on the same discipline as the mutation +cases: it must hold on a healthy pipeline, fail on a pipeline with exactly the +defect it exists to catch, and refuse (``not_testable`` or a declaration +failure) when its preconditions do not hold -- because a relation asserted +where it is not valid is a false failure, and a relation skipped silently is +a false pass. +""" + +from __future__ import annotations + +import io +import json +import tempfile +import textwrap +import unittest +from contextlib import redirect_stdout +from copy import deepcopy +from pathlib import Path + +import yaml + +from openmapstack.checks import metamorphic as metamorphic_checks +from openmapstack.checks.project import parameters_match_steps +from openmapstack.cli import main +from openmapstack.metamorphic import DeclarationError, parse_declaration, run_relation +from openmapstack.parameters import ParameterError, declared_parameters +from openmapstack.schema import project_schema_errors +from openmapstack.verify import verify_project +from tests.evals.helpers import make_workspace, minimal_project, write_project + +# A deliberately boring pipeline: select points with x <= max_x, keyed by +# pid, deduplicated, written in pid order. ``MODE`` injects one defect. +PIPELINE = textwrap.dedent( + ''' + import json, os, sys, time + from pathlib import Path + + ROOT = Path(__file__).resolve().parent + MODE = {mode!r} + max_x = float(os.environ.get("OMS_MAX_X", "10")) + argv = sys.argv[1:] + if "--max-x" in argv: + max_x = float(argv[argv.index("--max-x") + 1]) + if MODE == "crash": + raise SystemExit(3) + if MODE == "slow": + time.sleep(5) + if MODE == "reach_back": + target = Path((ROOT / "data/source/origin.txt").read_text().strip()) + with target.open("a") as fh: + fh.write("touched\\n") + doc = json.loads((ROOT / "data/source/points.geojson").read_text()) + selected = {{}} + for index, feature in enumerate(doc["features"]): + x = feature["geometry"]["coordinates"][0] + keep = x >= max_x if MODE == "inverted" else x <= max_x + if not keep: + continue + pid = feature["properties"]["pid"] + props = dict(feature["properties"]) + if MODE == "order_dependent": + props["rank"] = index + out = {{"type": "Feature", "properties": props, "geometry": feature["geometry"]}} + if MODE == "duplicate_sensitive": + selected[(pid, index)] = out + else: + selected.setdefault(pid, out) + features = [selected[key] for key in sorted(selected, key=lambda k: str(k))] + (ROOT / "data/derived").mkdir(parents=True, exist_ok=True) + (ROOT / "data/derived/selected.geojson").write_text( + json.dumps({{"type": "FeatureCollection", "features": features}}) + ) + ''' +) + +POINTS = { + "type": "FeatureCollection", + "features": [ + {"type": "Feature", "properties": {"pid": f"p{i}"}, "geometry": {"type": "Point", "coordinates": [float(i), 0.0]}} + for i in range(1, 16) + ], +} + + +def _permutation(**extra): + declaration = { + "id": "point-order", + "relation": "input_permutation_invariance", + "source": {"path": "data/source/points.geojson"}, + "outputs": ["selected"], + "key": "pid", + "preconditions": {"tie_break": "selection is keyed by pid; output sorted by pid"}, + } + declaration.update(extra) + return declaration + + +def _duplicates(**extra): + declaration = { + "id": "point-duplicates", + "relation": "duplicate_resistance", + "source": {"path": "data/source/points.geojson"}, + "outputs": ["selected"], + "key": "pid", + "preconditions": {"dedup_key": "pid", "measure": "set"}, + } + declaration.update(extra) + return declaration + + +def _monotonic(**extra): + declaration = { + "id": "max-x-monotonic", + "relation": "positive_buffer_monotonicity", + "parameter": "max_x", + "variant": {"multiply": 1.5}, + "outputs": ["selected"], + "key": "pid", + "preconditions": {"predicate": "within_distance", "expected": "superset"}, + } + declaration.update(extra) + return declaration + + +def _parameter(**extra): + parameter = { + "id": "max_x", + "type": "number", + "canonical": 10, + "binding": {"argument": "--max-x"}, + "step": "select", + "field": "max_x", + } + parameter.update(extra) + return parameter + + +class _ProjectMixin: + def build(self, *, mode: str = "healthy", relations=None, parameters=None, run: bool = True): + workspace = make_workspace() + project = minimal_project() + project["processing"]["steps"] = [ + {"id": "load", "operation": "read", "source": "test_source", "output": "points"}, + {"id": "select", "operation": "distance_filter", "input": "points", "max_x": 10, "crs": "EPSG:3301", "output": "selected"}, + ] + project["outputs"] = {"selected": {"path": "data/derived/selected.geojson", "format": "GeoJSON", "generated_by": "select"}} + project["runtime"]["implementation"]["parameters"] = parameters if parameters is not None else [_parameter()] + project["validation"]["metamorphic"] = relations if relations is not None else [] + write_project(workspace, project) + (workspace / "pipeline.py").write_text(PIPELINE.format(mode=mode), encoding="utf-8") + (workspace / "data/source").mkdir(parents=True) + (workspace / "data/source/points.geojson").write_text(json.dumps(POINTS), encoding="utf-8") + if run: + import subprocess, sys + + subprocess.run([sys.executable, "pipeline.py"], cwd=workspace, check=True) + return workspace, project + + def relation(self, workspace, project, declaration): + return run_relation(workspace, project, declaration) + + +class PermutationInvarianceTests(_ProjectMixin, unittest.TestCase): + def test_holds_on_a_keyed_pipeline(self) -> None: + workspace, project = self.build() + result, evidence = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "passed", result.detail) + self.assertEqual(evidence["variant"]["transformation"], "permute_features") + + def test_detects_an_order_dependent_pipeline(self) -> None: + workspace, project = self.build(mode="order_dependent") + result, _ = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "permutation_changed_output") + + def test_requires_a_declared_tie_break_rule(self) -> None: + workspace, project = self.build() + result, evidence = self.relation(workspace, project, _permutation(preconditions={})) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "metamorphic_declaration_invalid") + self.assertEqual(evidence["class"], "invalid") + self.assertIn("tie_break", result.detail) + + def test_non_unique_output_key_is_not_testable(self) -> None: + # Set semantics cannot be asserted over a key that repeats. + workspace, project = self.build(mode="duplicate_sensitive") + doc = json.loads((workspace / "data/source/points.geojson").read_text()) + doc["features"].append(deepcopy(doc["features"][0])) + (workspace / "data/source/points.geojson").write_text(json.dumps(doc)) + import subprocess, sys + + subprocess.run([sys.executable, "pipeline.py"], cwd=workspace, check=True) + result, _ = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "precondition_unmet") + + +class DuplicateResistanceTests(_ProjectMixin, unittest.TestCase): + def test_holds_on_a_deduplicating_pipeline(self) -> None: + workspace, project = self.build() + result, evidence = self.relation(workspace, project, _duplicates()) + self.assertEqual(result.status, "passed", result.detail) + self.assertEqual(evidence["variant"]["duplicated"], len(POINTS["features"])) + + def test_detects_a_duplicate_sensitive_pipeline(self) -> None: + workspace, project = self.build(mode="duplicate_sensitive") + result, _ = self.relation(workspace, project, _duplicates()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "duplicates_changed_output") + + def test_rejects_count_and_sum_semantics(self) -> None: + # Counts legitimately double when rows are duplicated; declaring the + # relation for them is invalid use, not a relation that happens to fail. + workspace, project = self.build() + result, _ = self.relation( + workspace, project, _duplicates(preconditions={"dedup_key": "pid", "measure": "count"}) + ) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "metamorphic_declaration_invalid") + self.assertIn("set semantics", result.detail) + + def test_source_that_already_has_duplicates_is_not_testable(self) -> None: + workspace, project = self.build(run=False) + doc = json.loads((workspace / "data/source/points.geojson").read_text()) + doc["features"].append(deepcopy(doc["features"][0])) + (workspace / "data/source/points.geojson").write_text(json.dumps(doc)) + import subprocess, sys + + subprocess.run([sys.executable, "pipeline.py"], cwd=workspace, check=True) + result, _ = self.relation(workspace, project, _duplicates()) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "precondition_unmet") + self.assertIn("already has duplicate", result.detail) + + +class BufferMonotonicityTests(_ProjectMixin, unittest.TestCase): + def test_holds_when_a_larger_threshold_keeps_every_baseline_feature(self) -> None: + workspace, project = self.build() + result, evidence = self.relation(workspace, project, _monotonic()) + self.assertEqual(result.status, "passed", result.detail) + self.assertEqual(evidence["parameter"], {"id": "max_x", "canonical": 10, "variant": 15.0}) + self.assertEqual(evidence["command"][-2:], ["--max-x", "15"]) + self.assertEqual(evidence["counts"]["selected"], {"baseline": 10, "variant": 15}) + + def test_detects_an_inverted_predicate(self) -> None: + workspace, project = self.build(mode="inverted") + result, _ = self.relation(workspace, project, _monotonic()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "monotonicity_violated") + self.assertIn("lost", result.detail) + + def test_environment_binding_is_honoured(self) -> None: + workspace, project = self.build(parameters=[_parameter(binding={"environment": "OMS_MAX_X"})]) + result, evidence = self.relation(workspace, project, _monotonic()) + self.assertEqual(result.status, "passed", result.detail) + self.assertEqual(evidence["variant"]["environment"], ["OMS_MAX_X"]) + + def test_undeclared_parameter_is_a_declaration_failure(self) -> None: + workspace, project = self.build(parameters=[]) + result, _ = self.relation(workspace, project, _monotonic()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "metamorphic_declaration_invalid") + + def test_non_numeric_parameter_is_not_testable(self) -> None: + workspace, project = self.build( + parameters=[{"id": "max_x", "type": "string", "canonical": "ten", "binding": {"argument": "--max-x"}}] + ) + result, _ = self.relation(workspace, project, _monotonic()) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "precondition_unmet") + + def test_variant_must_strictly_grow(self) -> None: + with self.assertRaises(DeclarationError): + parse_declaration(_monotonic(variant={"multiply": 1})) + with self.assertRaises(DeclarationError): + parse_declaration(_monotonic(variant={"add": -5})) + with self.assertRaises(DeclarationError): + parse_declaration(_monotonic(preconditions={"predicate": "outside_distance"})) + + +class RelationSafetyTests(_ProjectMixin, unittest.TestCase): + def test_unknown_relation_is_rejected_not_skipped(self) -> None: + with self.assertRaises(DeclarationError): + parse_declaration(_permutation(relation="crs_round_trip_stability")) + + def test_unsupported_source_format_is_not_testable(self) -> None: + workspace, project = self.build() + (workspace / "data/source/points.gpkg").write_bytes(b"not really") + result, _ = self.relation(workspace, project, _permutation(source={"path": "data/source/points.gpkg"})) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "unsupported_format") + + def test_source_outside_immutable_trees_is_rejected(self) -> None: + with self.assertRaises(DeclarationError): + parse_declaration(_permutation(source={"path": "data/derived/selected.geojson"})) + + def test_oversize_source_is_a_resource_limit(self) -> None: + workspace, project = self.build() + result, _ = self.relation(workspace, project, _permutation(limits={"max_source_bytes": 10})) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "resource_limit") + + def test_missing_baseline_output_is_not_testable(self) -> None: + workspace, project = self.build(run=False) + result, _ = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "baseline_missing") + + def test_crashing_variant_is_a_failure(self) -> None: + workspace, project = self.build() + (workspace / "pipeline.py").write_text(PIPELINE.format(mode="crash"), encoding="utf-8") + result, evidence = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "variant_execution_failed") + self.assertIn("stderr_tail", evidence) + + def test_timeout_is_not_testable_and_the_variant_is_removed(self) -> None: + workspace, project = self.build() + (workspace / "pipeline.py").write_text(PIPELINE.format(mode="slow"), encoding="utf-8") + before = {p.name for p in Path(tempfile.gettempdir()).iterdir() if p.name.startswith("openmapstack-metamorphic-")} + result, _ = self.relation(workspace, project, _permutation(limits={"timeout_s": 0.5})) + after = {p.name for p in Path(tempfile.gettempdir()).iterdir() if p.name.startswith("openmapstack-metamorphic-")} + self.assertEqual(result.status, "not_testable") + self.assertEqual(result.data["code"], "variant_timeout") + self.assertEqual(after - before, set()) + + def test_variant_that_mutates_the_original_source_fails(self) -> None: + workspace, project = self.build() + origin = workspace / "data/source/origin.txt" + origin.write_text(str(workspace / "data/source/points.geojson")) + (workspace / "pipeline.py").write_text(PIPELINE.format(mode="reach_back"), encoding="utf-8") + result, _ = self.relation(workspace, project, _permutation()) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "original_source_mutated") + + def test_variant_workspace_is_removed_after_success(self) -> None: + workspace, project = self.build() + before = {p.name for p in Path(tempfile.gettempdir()).iterdir() if p.name.startswith("openmapstack-metamorphic-")} + self.relation(workspace, project, _permutation()) + after = {p.name for p in Path(tempfile.gettempdir()).iterdir() if p.name.startswith("openmapstack-metamorphic-")} + self.assertEqual(after - before, set()) + # And the original workspace still has exactly what it started with. + self.assertEqual(json.loads((workspace / "data/source/points.geojson").read_text()), POINTS) + + +class ParameterContractTests(_ProjectMixin, unittest.TestCase): + def test_parameters_parse_and_bind(self) -> None: + manifest = {"runtime": {"implementation": {"parameters": [_parameter()]}}, + "processing": {"steps": [{"id": "select", "max_x": 10}]}} + [parameter] = declared_parameters(manifest) + self.assertEqual(parameter.bind(15.0), (["--max-x", "15"], {})) + self.assertEqual(parameter.bind(12.5), (["--max-x", "12.5"], {})) + + def test_drift_between_canonical_and_step_is_rejected(self) -> None: + manifest = {"runtime": {"implementation": {"parameters": [_parameter(canonical=2000)]}}, + "processing": {"steps": [{"id": "select", "max_x": 10}]}} + with self.assertRaises(ParameterError) as caught: + declared_parameters(manifest) + self.assertIn("!= processing step", str(caught.exception)) + + def test_malformed_bindings_and_types_are_rejected(self) -> None: + for bad in ( + _parameter(binding={}), + _parameter(binding={"argument": "max-x"}), + _parameter(binding={"environment": "lower"}), + _parameter(binding={"argument": "--max-x", "environment": "OMS_MAX_X"}), + _parameter(type="number", canonical=True), + _parameter(type="integer", canonical=1.5), + {k: v for k, v in _parameter().items() if k != "field"}, + _parameter(step="missing", field="max_x"), + _parameter(id="not an identifier"), + ): + manifest = {"runtime": {"implementation": {"parameters": [bad]}}, + "processing": {"steps": [{"id": "select", "max_x": 10}]}} + with self.assertRaises(ParameterError, msg=bad): + declared_parameters(manifest) + + def test_project_check_reports_drift_and_absence(self) -> None: + workspace, _ = self.build(run=False) + self.assertEqual(parameters_match_steps(workspace).status, "passed") + workspace, _ = self.build(parameters=[_parameter(canonical=99)], run=False) + result = parameters_match_steps(workspace) + self.assertEqual(result.status, "failed") + self.assertEqual(result.data["code"], "parameters_invalid") + workspace, _ = self.build(parameters=[], run=False) + self.assertEqual(parameters_match_steps(workspace).status, "not_testable") + + def test_schema_accepts_the_contract_and_rejects_unknown_relations(self) -> None: + _, project = self.build(relations=[_permutation(), _duplicates(), _monotonic()], run=False) + project["runs"] = {"latest": {"id": "run-1", "started_at": "x", "completed_at": "x", "status": "passed", + "inputs_hash": "sha256:" + "0" * 64, "outputs_hash": "sha256:" + "0" * 64, + "validation_report": {"path": "validation/latest-report.json"}}} + self.assertEqual(project_schema_errors(project), []) + broken = deepcopy(project) + broken["validation"]["metamorphic"][0]["relation"] = "subset_additivity" + self.assertTrue(project_schema_errors(broken)) + broken = deepcopy(project) + broken["runtime"]["implementation"]["parameters"][0]["binding"] = {"argument": "bad flag"} + self.assertTrue(project_schema_errors(broken)) + + +class VerifyIntegrationTests(_ProjectMixin, unittest.TestCase): + def test_declarations_are_checked_statically_and_executed_only_on_request(self) -> None: + workspace, _ = self.build(relations=[_permutation(), _monotonic()]) + static = verify_project(workspace / "project.yaml") + names = [run.name for run in static.checks] + self.assertIn("metamorphic.declarations_valid", names) + self.assertIn("project.parameters_match_steps", names) + self.assertNotIn("metamorphic.point-order", names) + + executed = verify_project(workspace / "project.yaml", metamorphic=True) + by_name = {run.name: run for run in executed.checks} + self.assertEqual(by_name["metamorphic.point-order"].result.status, "passed") + self.assertEqual(by_name["metamorphic.max-x-monotonic"].result.status, "passed") + payload = executed.to_dict() + entry = next(item for item in payload["checks"] if item["check"] == "metamorphic.point-order") + self.assertEqual(entry["evidence"]["relation"], "input_permutation_invariance") + + def test_invalid_declaration_fails_the_static_plan(self) -> None: + workspace, _ = self.build(relations=[_duplicates(preconditions={"dedup_key": "pid", "measure": "count"})]) + result = verify_project(workspace / "project.yaml") + run = next(r for r in result.checks if r.name == "metamorphic.declarations_valid") + self.assertEqual(run.result.status, "failed") + self.assertEqual(result.status, "failed") + + def test_cli_flag_runs_relations(self) -> None: + workspace, _ = self.build(mode="inverted", relations=[_monotonic()]) + out = io.StringIO() + with redirect_stdout(out): + code = main(["verify", str(workspace / "project.yaml"), "--metamorphic"]) + self.assertEqual(code, 1) + self.assertIn("FAIL metamorphic.max-x-monotonic", out.getvalue()) + self.assertIn("lost 5 baseline feature(s)", out.getvalue()) + + def test_check_library_entry_matches_declared_id(self) -> None: + workspace, _ = self.build(relations=[_permutation()]) + self.assertEqual(metamorphic_checks.relation_holds(workspace, id="point-order").status, "passed") + missing = metamorphic_checks.relation_holds(workspace, id="nope") + self.assertEqual(missing.status, "failed") + self.assertEqual(missing.data["code"], "metamorphic_relation_undeclared") + self.assertEqual(metamorphic_checks.declarations_valid(workspace).status, "passed") + (workspace / "project.yaml").write_text( + yaml.safe_dump(minimal_project(), sort_keys=False), encoding="utf-8" + ) + self.assertEqual(metamorphic_checks.declarations_valid(workspace).status, "not_testable") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_snapshot.py b/tests/test_snapshot.py new file mode 100644 index 0000000..988cade --- /dev/null +++ b/tests/test_snapshot.py @@ -0,0 +1,126 @@ +"""Skill snapshots: hashed, inspectable, symlink- and escape-safe (issue #13, C2).""" + +from __future__ import annotations + +import io +import json +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +from openmapstack.cli import main +from openmapstack.snapshot import ( + SnapshotError, + create_skill_snapshot, + find_skill_root, + hash_skill_root, + inspect_skill_snapshot, +) +from tests.evals.helpers import make_workspace + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _skill_root() -> Path: + root = make_workspace() / "skill" + (root / "references").mkdir(parents=True) + (root / "templates").mkdir() + (root / "SKILL.md").write_text("# skill\n", encoding="utf-8") + (root / "references/project-spec.md").write_text("spec\n", encoding="utf-8") + (root / "templates/project.yaml").write_text("schema: openmapstack-project/v1\n", encoding="utf-8") + (root / "evals").mkdir() + (root / "evals/secret-case.yaml").write_text("must not be copied\n", encoding="utf-8") + return root + + +class SnapshotTests(unittest.TestCase): + def test_snapshot_copies_only_the_distributable_skill(self) -> None: + root = _skill_root() + out = make_workspace() / "snap" + manifest = create_skill_snapshot(root, out) + self.assertEqual(manifest["schema"], "openmapstack-skill-snapshot/v1") + self.assertEqual({entry["path"] for entry in manifest["files"]}, {"SKILL.md", "references/project-spec.md", "templates/project.yaml"}) + self.assertFalse((out / "evals").exists()) + self.assertEqual(manifest["content_sha256"], hash_skill_root(root)) + self.assertTrue(inspect_skill_snapshot(out)["intact"]) + + def test_content_hash_covers_paths_and_bytes(self) -> None: + root = _skill_root() + before = hash_skill_root(root) + (root / "references/project-spec.md").write_text("spec v2\n", encoding="utf-8") + self.assertNotEqual(before, hash_skill_root(root)) + (root / "references/project-spec.md").write_text("spec\n", encoding="utf-8") + (root / "references/project-spec.md").rename(root / "references/renamed.md") + self.assertNotEqual(before, hash_skill_root(root)) + + def test_content_hash_is_the_historical_raw_byte_algorithm(self) -> None: + # Benchmark arms recorded before this module hashed path, NUL, bytes, + # NUL per file in path order; an unchanged skill must keep its hash. + import hashlib + + root = _skill_root() + digest = hashlib.sha256() + for path in sorted(item for item in root.rglob("*") if item.is_file() and item.relative_to(root).parts[0] in {"SKILL.md", "references", "templates"}): + digest.update(path.relative_to(root).as_posix().encode("utf-8")) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + self.assertEqual(hash_skill_root(root), f"sha256:{digest.hexdigest()}") + + def test_symlinks_are_refused(self) -> None: + root = _skill_root() + (root / "references/escape.md").symlink_to("/etc/hostname") + with self.assertRaises(SnapshotError): + create_skill_snapshot(root, make_workspace() / "snap") + + def test_inspection_detects_tampering_and_escapes(self) -> None: + root = _skill_root() + out = make_workspace() / "snap" + create_skill_snapshot(root, out) + (out / "SKILL.md").write_text("# edited\n", encoding="utf-8") + (out / "references/extra.md").write_text("added\n", encoding="utf-8") + report = inspect_skill_snapshot(out) + self.assertFalse(report["intact"]) + self.assertIn("changed: SKILL.md", report["problems"]) + self.assertIn("extra: references/extra.md", report["problems"]) + manifest = json.loads((out / "snapshot.json").read_text()) + manifest["files"].append({"path": "../outside.md", "sha256": "sha256:" + "0" * 64, "bytes": 1}) + (out / "snapshot.json").write_text(json.dumps(manifest)) + self.assertTrue(any("escapes" in problem for problem in inspect_skill_snapshot(out)["problems"])) + + def test_destination_must_be_empty_and_outside_the_root(self) -> None: + root = _skill_root() + with self.assertRaises(SnapshotError): + create_skill_snapshot(root, root / "snap") + out = make_workspace() / "snap" + out.mkdir() + (out / "leftover").write_text("x") + with self.assertRaises(SnapshotError): + create_skill_snapshot(root, out) + + def test_repository_skill_root_is_discoverable_and_snapshots(self) -> None: + self.assertEqual(find_skill_root(REPO_ROOT / "openmapstack"), REPO_ROOT) + out = make_workspace() / "snap" + manifest = create_skill_snapshot(REPO_ROOT, out) + self.assertIn("references/project-spec.md", {entry["path"] for entry in manifest["files"]}) + self.assertNotIn("evals/README.md", {entry["path"] for entry in manifest["files"]}) + + def test_cli_creates_and_inspects(self) -> None: + root = _skill_root() + out = make_workspace() / "snap" + buffer = io.StringIO() + with redirect_stdout(buffer): + self.assertEqual(main(["skill-snapshot", "--out", str(out), "--source", str(root), "--json"]), 0) + manifest = json.loads(buffer.getvalue()) + self.assertEqual(manifest["file_count"], 3) + with redirect_stdout(io.StringIO()): + self.assertEqual(main(["skill-snapshot", "--inspect", str(out)]), 0) + (out / "SKILL.md").write_text("# edited\n", encoding="utf-8") + with redirect_stdout(io.StringIO()): + self.assertEqual(main(["skill-snapshot", "--inspect", str(out)]), 1) + with redirect_stderr(io.StringIO()): + self.assertEqual(main(["skill-snapshot", "--out", str(make_workspace() / "x"), "--source", str(make_workspace())]), 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sources.py b/tests/test_sources.py new file mode 100644 index 0000000..af1f52f --- /dev/null +++ b/tests/test_sources.py @@ -0,0 +1,224 @@ +"""Source pin classes and credential hygiene (issue #13, B5a).""" + +from __future__ import annotations + +import hashlib +import unittest +from copy import deepcopy +from datetime import datetime, timezone + +from openmapstack.checks.provenance import every_source_pinned, no_inline_credentials +from openmapstack.schema import project_schema_errors +from openmapstack.sources import ( + assess_pin, + connection_reference_error, + find_inline_credentials, + redact, +) +from openmapstack.validation import validate_project +from tests.evals.helpers import make_workspace, minimal_project, write_project +from tests.test_cli import valid_manifest + +NOW = datetime(2026, 9, 1, tzinfo=timezone.utc) + + +def _snapshot(workspace, name="parcels.parquet", content=b"parquet bytes"): + target = workspace / "data/source" / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + return { + "class": "local_snapshot", + "path": f"data/source/{name}", + "sha256": "sha256:" + hashlib.sha256(content).hexdigest(), + "captured_at": "2026-08-30T10:00:00Z", + } + + +def _backend(**extra): + pin = { + "class": "backend_snapshot", + "identifier": "pg_export_snapshot:00000003-000001A8-1", + "captured_at": "2026-08-30T10:00:00Z", + "retention_until": "2026-12-31T00:00:00Z", + } + pin.update(extra) + return pin + + +def _source(**extra): + source = deepcopy(minimal_project()["sources"]["test_source"]) + source.update(extra) + return source + + +class PinAssessmentTests(unittest.TestCase): + def test_version_identity_without_pin_block_is_still_accepted(self) -> None: + assessment = assess_pin(make_workspace(), _source(), now=NOW) + self.assertEqual((assessment.status, assessment.pin_class), ("pinned", "version_identity")) + + def test_mutable_alias_is_unpinned_even_with_a_valid_snapshot(self) -> None: + workspace = make_workspace() + source = _source(version={"identifier": "latest"}, pin=_snapshot(workspace)) + self.assertEqual(assess_pin(workspace, source, now=NOW).status, "unpinned") + + def test_local_snapshot_pins_when_bytes_match(self) -> None: + workspace = make_workspace() + assessment = assess_pin(workspace, _source(pin=_snapshot(workspace)), now=NOW) + self.assertEqual((assessment.status, assessment.pin_class), ("pinned", "local_snapshot")) + + def test_local_snapshot_missing_or_changed_is_not_reproducible(self) -> None: + workspace = make_workspace() + pin = _snapshot(workspace) + (workspace / pin["path"]).write_bytes(b"edited") + changed = assess_pin(workspace, _source(pin=pin), now=NOW) + self.assertEqual((changed.status, changed.details["cause"]), ("not_reproducible", "snapshot_hash_mismatch")) + (workspace / pin["path"]).unlink() + missing = assess_pin(workspace, _source(pin=pin), now=NOW) + self.assertEqual((missing.status, missing.details["cause"]), ("not_reproducible", "snapshot_missing")) + + def test_local_snapshot_must_live_under_data_source(self) -> None: + workspace = make_workspace() + pin = _snapshot(workspace) + pin["path"] = "data/derived/parcels.parquet" + self.assertEqual(assess_pin(workspace, _source(pin=pin), now=NOW).status, "invalid") + pin["path"] = "../outside.parquet" + self.assertEqual(assess_pin(workspace, _source(pin=pin), now=NOW).status, "invalid") + + def test_backend_snapshot_pins_until_retention_lapses(self) -> None: + workspace = make_workspace() + self.assertEqual(assess_pin(workspace, _source(pin=_backend()), now=NOW).status, "pinned") + expired = assess_pin(workspace, _source(pin=_backend()), now=datetime(2027, 1, 1, tzinfo=timezone.utc)) + self.assertEqual((expired.status, expired.details["cause"]), ("not_reproducible", "snapshot_expired")) + + def test_backend_snapshot_verified_inaccessible_is_not_reproducible(self) -> None: + pin = _backend(verification={"at": "2026-08-31T00:00:00Z", "status": "inaccessible"}) + assessment = assess_pin(make_workspace(), _source(pin=pin), now=NOW) + self.assertEqual((assessment.status, assessment.details["cause"]), ("not_reproducible", "snapshot_inaccessible")) + + def test_backend_snapshot_needs_identifier_and_retention(self) -> None: + workspace = make_workspace() + for broken in ( + _backend(identifier="latest"), + {k: v for k, v in _backend().items() if k != "retention_until"}, + _backend(retention_until="whenever"), + _backend(captured_at=None), + {"class": "time_travel", "captured_at": "2026-08-30T10:00:00Z"}, + "not a mapping", + ): + assessment = assess_pin(workspace, _source(pin=broken), now=NOW) + self.assertIn(assessment.status, {"invalid", "unpinned"}, broken) + + def test_provenance_check_reports_each_class_with_its_own_code(self) -> None: + workspace = make_workspace() + project = minimal_project() + project["sources"]["test_source"]["pin"] = _snapshot(workspace) + write_project(workspace, project) + self.assertEqual(every_source_pinned(workspace).status, "passed") + + project["sources"]["test_source"]["pin"] = _backend(retention_until="2020-01-01T00:00:00Z") + write_project(workspace, project) + result = every_source_pinned(workspace) + self.assertEqual((result.status, result.data["code"]), ("failed", "not_reproducible")) + self.assertEqual(result.data["causes"], {"test_source": "snapshot_expired"}) + + project["sources"]["test_source"]["pin"] = {"class": "backend_snapshot"} + write_project(workspace, project) + self.assertEqual(every_source_pinned(workspace).data["code"], "pin_invalid") + + del project["sources"]["test_source"]["pin"] + project["sources"]["test_source"]["version"] = {"identifier": "latest"} + write_project(workspace, project) + self.assertEqual(every_source_pinned(workspace).data["code"], "source_unpinned") + + +class CredentialHygieneTests(unittest.TestCase): + def test_embedded_secrets_are_found_without_being_echoed(self) -> None: + source = _source( + access={"method": "postgis", "retrieved_at": "x", "connection": "postgresql://gis:hunter2@db/gis"}, + notes="password=opensesame", + ) + findings = find_inline_credentials(source, "sources.s") + paths = {item["path"] for item in findings} + self.assertIn("sources.s.access.connection", paths) + self.assertIn("sources.s.notes", paths) + self.assertNotIn("hunter2", str(findings)) + self.assertNotIn("opensesame", str(findings)) + + def test_ordinary_urls_and_prose_are_not_flagged(self) -> None: + source = _source( + source_url="https://geoportaal.maaruum.ee/eng/spatial-data/cadastral-data-p310.html", + rationale="The token field in the source schema is a land-use token, not a credential.", + ) + self.assertEqual(find_inline_credentials(source, "sources.s"), []) + + def test_connection_must_be_a_reference(self) -> None: + workspace = make_workspace() + self.assertIsNone(connection_reference_error(workspace, "env:PARCELS_DSN")) + self.assertIsNone(connection_reference_error(workspace, {"ref": "service:geo-prod"})) + self.assertIsNone(connection_reference_error(workspace, None)) + self.assertIsNotNone(connection_reference_error(workspace, "host=db dbname=gis")) + self.assertIsNotNone(connection_reference_error(workspace, "postgresql://db/gis")) + self.assertIsNotNone(connection_reference_error(workspace, {"ref": "env:"})) + # A secrets file inside the project directory would be committed, and + # a relative path is refused here exactly as the connector refuses it. + self.assertIsNotNone(connection_reference_error(workspace, "file:secrets/dsn.txt")) + self.assertIsNotNone(connection_reference_error(workspace, "file:../secrets/postgis.dsn")) + self.assertIsNotNone(connection_reference_error(workspace, f"file:{workspace}/dsn.txt")) + self.assertIsNone(connection_reference_error(workspace, "file:/etc/openmapstack/dsn")) + + def test_provenance_check_and_validate_reject_inline_credentials(self) -> None: + workspace = make_workspace() + project = minimal_project() + project["sources"]["test_source"]["access"]["connection"] = "postgresql://gis:hunter2@db/gis" + write_project(workspace, project) + result = no_inline_credentials(workspace) + self.assertEqual((result.status, result.data["code"]), ("failed", "inline_credentials")) + + manifest = valid_manifest() + key = next(iter(manifest["sources"])) + manifest["sources"][key]["access"]["connection"] = {"ref": "env:GIS_DSN"} + write_project(workspace, manifest) + (workspace / "pipeline.py").write_text("print('ok')\n", encoding="utf-8") + checks = {c.id: c for c in validate_project(workspace / "project.yaml", artifacts=False).checks} + self.assertEqual(checks["source.credentials"].status, "passed") + manifest["sources"][key]["access"]["connection"] = "postgresql://gis:hunter2@db/gis" + write_project(workspace, manifest) + checks = {c.id: c for c in validate_project(workspace / "project.yaml", artifacts=False).checks} + self.assertEqual(checks["source.credentials"].status, "failed") + self.assertNotIn("hunter2", checks["source.credentials"].message) + + def test_redaction_masks_secrets_in_recorded_text(self) -> None: + text = "postgresql://gis:hunter2@db/gis password=abc token: xyz AKIAABCDEFGHIJKLMNOP" + masked = redact(text) + for secret in ("hunter2", "abc", "xyz", "AKIAABCDEFGHIJKLMNOP"): + self.assertNotIn(secret, masked) + self.assertIn("postgresql://gis:***@db/gis", masked) + + +class SchemaTests(unittest.TestCase): + def test_pin_and_warehouse_blocks_validate(self) -> None: + manifest = valid_manifest() + key = next(iter(manifest["sources"])) + manifest["sources"][key]["pin"] = _backend() + manifest["sources"][key]["warehouse"] = {"backend": "postgis", "database": "gis", "schema": "cadastre", "table": "parcels"} + manifest["sources"][key]["access"]["connection"] = {"ref": "env:GIS_DSN"} + self.assertEqual(project_schema_errors(manifest), []) + manifest["sources"][key]["pin"] = {"class": "local_snapshot", "captured_at": "x"} + self.assertTrue(project_schema_errors(manifest)) + manifest["sources"][key]["pin"] = {"class": "elsewhere", "captured_at": "x"} + self.assertTrue(project_schema_errors(manifest)) + + def test_validate_reports_pin_class_per_source(self) -> None: + workspace = make_workspace() + manifest = valid_manifest() + key = next(iter(manifest["sources"])) + manifest["sources"][key]["pin"] = _snapshot(workspace) + write_project(workspace, manifest) + (workspace / "pipeline.py").write_text("print('ok')\n", encoding="utf-8") + checks = [c for c in validate_project(workspace / "project.yaml", artifacts=False).checks if c.id == "source.pin"] + self.assertEqual([c.status for c in checks], ["passed"]) + self.assertEqual(checks[0].details["pin_class"], "local_snapshot") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_verify_foreign.py b/tests/test_verify_foreign.py new file mode 100644 index 0000000..aeca4de --- /dev/null +++ b/tests/test_verify_foreign.py @@ -0,0 +1,297 @@ +"""`openmapstack verify` on projects whose data this repository has never seen. + +Issue #13's definition of done asks for verify to run against at least two +projects absent from the fixtures and emit stable text and JSON reports. +The two projects here are built from scratch by pure-Python pipelines over +invented geodata (a river-crossing screening and a facility catchment +count); nothing under evals/ is read. The reports are compared against +committed goldens after path and timing normalisation, so an unintended +change in plan composition, ordering, wording, or JSON shape shows up as a +diff rather than passing unnoticed. +""" + +from __future__ import annotations + +import io +import json +import re +import subprocess +import sys +import textwrap +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +import yaml + +from openmapstack.api import validate_verify_result +from openmapstack.cli import main +from tests.evals.helpers import make_workspace + +GOLDEN_DIR = Path(__file__).resolve().parent / "goldens" / "verify" + + +def _crossings_project(root: Path) -> None: + """Project A: which planned trails cross a river (line/line predicate).""" + (root / "data/source").mkdir(parents=True) + trails = { + "type": "FeatureCollection", + "features": [ + {"type": "Feature", "properties": {"trail_id": f"T{i}", "name": f"Trail {i}"}, + "geometry": {"type": "LineString", "coordinates": [[i * 100.0, 0.0], [i * 100.0, 1000.0 if i % 2 else 400.0]]}} + for i in range(1, 7) + ], + } + river = {"type": "FeatureCollection", "features": [ + {"type": "Feature", "properties": {"river_id": "R1"}, + "geometry": {"type": "LineString", "coordinates": [[0.0, 500.0], [1000.0, 500.0]]}} + ]} + (root / "data/source/trails.geojson").write_text(json.dumps(trails), encoding="utf-8") + (root / "data/source/river.geojson").write_text(json.dumps(river), encoding="utf-8") + (root / "pipeline.py").write_text(textwrap.dedent(''' + import json, hashlib, sys + from pathlib import Path + ROOT = Path(__file__).resolve().parent + argv = sys.argv[1:] + crossing_y = float(argv[argv.index("--river-y") + 1]) if "--river-y" in argv else 500.0 + trails = json.loads((ROOT / "data/source/trails.geojson").read_text())["features"] + crossing = [] + for feature in sorted(trails, key=lambda f: f["properties"]["trail_id"]): + (x0, y0), (x1, y1) = feature["geometry"]["coordinates"] + if min(y0, y1) <= crossing_y <= max(y0, y1): + crossing.append({"type": "Feature", "properties": dict(feature["properties"]), "geometry": feature["geometry"]}) + (ROOT / "data/derived").mkdir(exist_ok=True) + out = ROOT / "data/derived/crossing-trails.geojson" + # GeoJSON without a crs member reads as WGS84; the output is in the + # analysis CRS, so say so -- verify cross-checks this against the data. + crs = {"type": "name", "properties": {"name": "urn:ogc:def:crs:EPSG::3301"}} + out.write_text(json.dumps({"type": "FeatureCollection", "crs": crs, "features": crossing})) + def h(p): + return "sha256:" + hashlib.sha256(p.read_bytes()).hexdigest() + inputs = sorted(p for p in (ROOT / "data/source").rglob("*") if p.is_file()) + [ROOT / "pipeline.py"] + def agg(paths): + d = hashlib.sha256() + for p in sorted(paths, key=lambda p: p.relative_to(ROOT).as_posix()): + rel = p.relative_to(ROOT).as_posix().encode() + d.update(len(rel).to_bytes(8, "big")); d.update(rel); d.update(p.read_bytes()) + return "sha256:" + d.hexdigest() + report = {"run_id": "run-20260901-000000", "status": "passed", "checks": [ + {"id": "geometry_valid", "status": "passed", "features_checked": len(crossing)}, + {"id": "manifest_graph_resolves", "status": "passed"}, + ], "inputs_hash": agg(inputs), "outputs_hash": agg([out])} + (ROOT / "validation").mkdir(exist_ok=True) + (ROOT / "validation/latest-report.json").write_text(json.dumps(report, indent=2)) + (ROOT / "runs").mkdir(exist_ok=True) + (ROOT / "runs/run-20260901-000000.json").write_text(json.dumps({ + "run_id": "run-20260901-000000", "started_at": "2026-09-01T00:00:00Z", "completed_at": "2026-09-01T00:00:01Z", + "status": "passed", "inputs_hash": report["inputs_hash"], "outputs_hash": report["outputs_hash"], + "inputs": [{"path": p.relative_to(ROOT).as_posix(), "sha256": h(p)} for p in inputs], + "outputs": [{"path": out.relative_to(ROOT).as_posix(), "sha256": h(out)}], + }, indent=2)) + ''').lstrip(), encoding="utf-8") + subprocess.run([sys.executable, "pipeline.py"], cwd=root, check=True) + report = json.loads((root / "validation/latest-report.json").read_text()) + manifest = { + "schema": "openmapstack-project/v1", + "project": {"id": "river-crossings", "title": "Planned trails crossing the river", "question": "Which planned trails cross the river?", + "created_at": "2026-09-01T00:00:00Z", "updated_at": "2026-09-01T00:00:00Z", "status": "validated"}, + "interpretation": {"objective": "Select trails whose line intersects the river centreline.", + "assumptions": [{"id": "A1", "statement": "Crossing means the trail line intersects the river line.", "rationale": "No bridge data is available."}]}, + "sources": { + "trails": {"role": "authoritative_input", "provider": "Trail planning office", "dataset": "planned trails", "source_url": "https://example.invalid/trails", + "access": {"method": "local", "retrieved_at": "2026-09-01T00:00:00Z"}, "version": {"identifier": "plan-2026-09", "published_at": "2026-09-01"}, + "selection": {"filter": "all"}, "license": {"name": "CC BY 4.0", "url": "https://example.invalid/license"}, "rationale": "Only planning dataset available."}, + "river": {"role": "context", "provider": "Hydrology agency", "dataset": "river centreline", "source_url": "https://example.invalid/river", + "access": {"method": "local", "retrieved_at": "2026-09-01T00:00:00Z"}, "version": {"identifier": "hydro-2026", "published_at": "2026-08-01"}, + "selection": {"filter": "river_id = 'R1'"}, "license": {"name": "CC BY 4.0", "url": "https://example.invalid/license"}, "rationale": "Authoritative centreline."}, + }, + "overrides": [], + "processing": {"analysis_crs": "EPSG:3301", "storage_crs": "EPSG:3301", "steps": [ + {"id": "load_trails", "operation": "read", "source": "trails", "output": "trails_raw"}, + {"id": "load_river", "operation": "read", "source": "river", "output": "river_raw"}, + {"id": "select_crossings", "operation": "intersects_filter", "input": "trails_raw", "target": "river_raw", "crs": "EPSG:3301", "output": "crossing_trails"}, + ]}, + "outputs": {"crossing_trails": {"path": "data/derived/crossing-trails.geojson", "format": "GeoJSON (EPSG:3301)", "generated_by": "select_crossings"}}, + "validation": {"required": ["geometry_valid", "manifest_graph_resolves"], "domain_checks": [], + "metamorphic": [{"id": "trail-order", "relation": "input_permutation_invariance", "source": {"path": "data/source/trails.geojson"}, + "outputs": ["crossing_trails"], "key": "trail_id", "preconditions": {"tie_break": "keyed by trail_id; sorted output"}}]}, + "presentation": {"intent": "report", "primary_view": "report", "layout": {"type": "report"}, + "map": {"engine_preference": "maplibre", "basemap": {"id": "osm-standard", "kind": "raster-xyz", "tiles": ["https://tile.openstreetmap.org/{z}/{x}/{y}.png"], "attribution": "© OpenStreetMap contributors"}, + "layer_groups": [{"id": "analysis", "title": "Analysis"}], + "layers": [{"source": "crossing_trails", "group": "analysis", "semantic_role": "primary_result", "geometry": "line"}]}}, + "warnings": [], + "runtime": {"implementation": {"preferred_engine": "python", "pipeline": "pipeline.py", + "parameters": [{"id": "river_y", "type": "number", "canonical": 500, "binding": {"argument": "--river-y"}}]}, + "environment": {"python": "3.12"}}, + "runs": {"latest": {"id": "run-20260901-000000", "started_at": "2026-09-01T00:00:00Z", "completed_at": "2026-09-01T00:00:01Z", "status": "passed", + "inputs_hash": report["inputs_hash"], "outputs_hash": report["outputs_hash"], "validation_report": {"path": "validation/latest-report.json"}}}, + } + (root / "project.yaml").write_text(yaml.safe_dump(manifest, sort_keys=False), encoding="utf-8") + + +def _catchments_project(root: Path) -> None: + """Project B: facilities per district (point-in-polygon count), with an + unverified expectation and an override, exercising more of the plan.""" + (root / "data/source").mkdir(parents=True) + (root / "data/overrides").mkdir() + districts = {"type": "FeatureCollection", "features": [ + {"type": "Feature", "properties": {"district_id": "D1"}, "geometry": {"type": "Polygon", "coordinates": [[[0, 0], [500, 0], [500, 500], [0, 500], [0, 0]]]}}, + {"type": "Feature", "properties": {"district_id": "D2"}, "geometry": {"type": "Polygon", "coordinates": [[[500, 0], [1000, 0], [1000, 500], [500, 500], [500, 0]]]}}, + ]} + facilities = {"type": "FeatureCollection", "features": [ + {"type": "Feature", "properties": {"facility_id": f"F{i}", "status": "open"}, "geometry": {"type": "Point", "coordinates": [x, 250.0]}} + for i, x in enumerate((100.0, 200.0, 700.0, 900.0, 950.0), start=1) + ]} + (root / "data/source/districts.geojson").write_text(json.dumps(districts), encoding="utf-8") + (root / "data/source/facilities.geojson").write_text(json.dumps(facilities), encoding="utf-8") + (root / "pipeline.py").write_text(textwrap.dedent(''' + import json, hashlib + from pathlib import Path + ROOT = Path(__file__).resolve().parent + districts = json.loads((ROOT / "data/source/districts.geojson").read_text())["features"] + facilities = json.loads((ROOT / "data/source/facilities.geojson").read_text())["features"] + closed = {"F5"} # OVERRIDE-001 + counts = [] + for d in sorted(districts, key=lambda f: f["properties"]["district_id"]): + xs = [pt[0] for pt in d["geometry"]["coordinates"][0]] + n = sum(1 for f in facilities if f["properties"]["facility_id"] not in closed and min(xs) <= f["geometry"]["coordinates"][0] < max(xs)) + counts.append({"type": "Feature", "properties": {"district_id": d["properties"]["district_id"], "facility_count": n}, "geometry": d["geometry"]}) + (ROOT / "data/derived").mkdir(exist_ok=True) + out = ROOT / "data/derived/district-counts.geojson" + out.write_text(json.dumps({"type": "FeatureCollection", "features": counts})) + def h(p): + return "sha256:" + hashlib.sha256(p.read_bytes()).hexdigest() + inputs = sorted(p for d in ("data/source", "data/overrides") for p in (ROOT / d).rglob("*") if p.is_file()) + [ROOT / "pipeline.py"] + def agg(paths): + d = hashlib.sha256() + for p in sorted(paths, key=lambda p: p.relative_to(ROOT).as_posix()): + rel = p.relative_to(ROOT).as_posix().encode() + d.update(len(rel).to_bytes(8, "big")); d.update(rel); d.update(p.read_bytes()) + return "sha256:" + d.hexdigest() + report = {"run_id": "run-20260901-000001", "status": "warning", "checks": [ + {"id": "geometry_valid", "status": "passed", "features_checked": 2}, + {"id": "overrides_applied", "status": "passed", "results": [{"id": "OVERRIDE-001", "status": "applied"}]}, + {"id": "facility_completeness", "status": "warning", "reason": "no completeness baseline"}, + ], "inputs_hash": agg(inputs), "outputs_hash": agg([out])} + (ROOT / "validation").mkdir(exist_ok=True) + (ROOT / "validation/latest-report.json").write_text(json.dumps(report, indent=2)) + (ROOT / "runs").mkdir(exist_ok=True) + (ROOT / "runs/run-20260901-000001.json").write_text(json.dumps({ + "run_id": "run-20260901-000001", "started_at": "2026-09-01T00:00:00Z", "completed_at": "2026-09-01T00:00:01Z", + "status": "warning", "inputs_hash": report["inputs_hash"], "outputs_hash": report["outputs_hash"], + "inputs": [{"path": p.relative_to(ROOT).as_posix(), "sha256": h(p)} for p in inputs], + "outputs": [{"path": out.relative_to(ROOT).as_posix(), "sha256": h(out)}], + }, indent=2)) + ''').lstrip(), encoding="utf-8") + (root / "data/overrides/closures.json").write_text(json.dumps({"closed": ["F5"]}), encoding="utf-8") + subprocess.run([sys.executable, "pipeline.py"], cwd=root, check=True) + report = json.loads((root / "validation/latest-report.json").read_text()) + manifest = { + "schema": "openmapstack-project/v1", + "project": {"id": "district-facilities", "title": "Open facilities per district", "question": "How many open facilities does each district have?", + "created_at": "2026-09-01T00:00:00Z", "updated_at": "2026-09-01T00:00:00Z", "status": "warning"}, + "interpretation": {"objective": "Count open facilities inside each district polygon.", + "assumptions": [{"id": "A1", "statement": "A facility on the shared boundary belongs to the western district.", "rationale": "Half-open interval avoids double counting."}]}, + "sources": { + "districts": {"role": "authoritative_input", "provider": "City", "dataset": "districts", "source_url": "https://example.invalid/districts", + "access": {"method": "local", "retrieved_at": "2026-09-01T00:00:00Z"}, "version": {"identifier": "districts-2026", "published_at": "2026-01-01"}, + "selection": {"filter": "all"}, "license": {"name": "CC0", "url": "https://example.invalid/cc0"}, "rationale": "Official boundaries."}, + "facilities": {"role": "authoritative_input", "provider": "City", "dataset": "facilities", "source_url": "https://example.invalid/facilities", + "access": {"method": "local", "retrieved_at": "2026-09-01T00:00:00Z"}, "version": {"identifier": "facilities-2026-08", "published_at": "2026-08-01"}, + "selection": {"filter": "status = 'open'"}, "license": {"name": "CC0", "url": "https://example.invalid/cc0"}, "rationale": "Official register."}, + }, + "overrides": [{"id": "OVERRIDE-001", "action": "modify_attribute", "target": {"source": "facilities", "feature_id": "F5"}, + "change": {"field": "status", "from": "open", "to": "closed"}, "rationale": "Closed after the register was published.", + "evidence": [{"type": "url", "value": "https://example.invalid/notice/F5"}], "created_at": "2026-09-01T00:00:00Z", "created_by": "analyst"}], + "processing": {"analysis_crs": "EPSG:3301", "storage_crs": "EPSG:3301", "steps": [ + {"id": "load_districts", "operation": "read", "source": "districts", "output": "districts_raw"}, + {"id": "load_facilities", "operation": "read", "source": "facilities", "output": "facilities_raw"}, + {"id": "apply_closures", "operation": "apply_override", "input": "facilities_raw", "override": "OVERRIDE-001", "output": "facilities_effective"}, + {"id": "count_per_district", "operation": "point_in_polygon_count", "inputs": ["districts_raw", "facilities_effective"], "crs": "EPSG:3301", "output": "district_counts"}, + ]}, + "outputs": {"district_counts": {"path": "data/derived/district-counts.geojson", "format": "GeoJSON", "generated_by": "count_per_district"}}, + "validation": {"required": ["geometry_valid", "overrides_applied"], "domain_checks": [{"name": "facility_completeness", "expression": "count >= 0"}], + "expectations": [{"id": "d1-count", "check": "geodata.feature_field_equals", + "args": {"path": "data/derived/district-counts.geojson", "id_field": "district_id", "id": "D1", "field": "facility_count", "equals": 2}, + "attestation": {"status": "unverified", "reason": "awaiting register comparison"}}]}, + "presentation": {"intent": "analytical_workspace", "primary_view": "map", "layout": {"type": "map_with_sidebar"}, + "map": {"engine_preference": "maplibre", "basemap": {"id": "osm-standard", "kind": "raster-xyz", "tiles": ["https://tile.openstreetmap.org/{z}/{x}/{y}.png"], "attribution": "© OpenStreetMap contributors"}, + "layer_groups": [{"id": "analysis", "title": "Analysis"}, {"id": "user_overrides", "title": "Corrections"}], + "layers": [{"source": "district_counts", "group": "analysis", "semantic_role": "primary_result", "geometry": "polygon"}, + {"source": "facilities", "group": "user_overrides", "semantic_role": "user_override", "geometry": "point"}]}}, + "warnings": [{"id": "DATA-001", "severity": "medium", "layer": "facilities", "issue": "completeness_unknown", "statement": "The register may omit facilities.", "mitigation": "Verify before decisions."}], + "runtime": {"implementation": {"preferred_engine": "python", "pipeline": "pipeline.py"}, "environment": {"python": "3.12"}}, + "runs": {"latest": {"id": "run-20260901-000001", "started_at": "2026-09-01T00:00:00Z", "completed_at": "2026-09-01T00:00:01Z", "status": "warning", + "inputs_hash": report["inputs_hash"], "outputs_hash": report["outputs_hash"], "validation_report": {"path": "validation/latest-report.json"}}}, + } + (root / "project.yaml").write_text(yaml.safe_dump(manifest, sort_keys=False, allow_unicode=True), encoding="utf-8") + + +def _normalise_text(text: str, root: Path) -> str: + return text.replace(str(root), "$PROJECT") + + +def _normalise_json(payload: dict, root: Path) -> dict: + text = json.dumps(payload, sort_keys=True) + text = text.replace(str(root), "$PROJECT") + payload = json.loads(text) + for check in payload.get("checks", []): + evidence = check.get("evidence") + if isinstance(evidence, dict): + for volatile in ("duration_s", "removed_environment_keys", "command", "preserved_paths"): + evidence.pop(volatile, None) + for volatile in ("expected_expectation_sha256", "current_inputs_hash"): + if volatile in evidence: + evidence[volatile] = "$DIGEST" + if "message" in check: + check["message"] = re.sub(r"sha256:[0-9a-f]{64}", "sha256:$DIGEST", check["message"]) + return payload + + +class ForeignProjectVerifyTests(unittest.TestCase): + maxDiff = None + + def _run(self, name: str, builder) -> tuple[str, dict]: + root = make_workspace() / name + root.mkdir() + builder(root) + text = io.StringIO() + with redirect_stdout(text): + main(["verify", str(root / "project.yaml"), "--verbose", "--metamorphic"]) + json_out = io.StringIO() + with redirect_stdout(json_out): + main(["verify", str(root / "project.yaml"), "--json", "--metamorphic"]) + payload = json.loads(json_out.getvalue()) + self.assertEqual(validate_verify_result(payload), []) + return _normalise_text(text.getvalue(), root), _normalise_json(payload, root) + + def _assert_golden(self, name: str, text: str, payload: dict) -> None: + golden_text = GOLDEN_DIR / f"{name}.txt" + golden_json = GOLDEN_DIR / f"{name}.json" + if not golden_text.exists() or not golden_json.exists(): + GOLDEN_DIR.mkdir(parents=True, exist_ok=True) + golden_text.write_text(text, encoding="utf-8") + golden_json.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + self.fail(f"goldens for {name} were missing and have been written; re-run to compare") + self.assertEqual(text, golden_text.read_text(encoding="utf-8")) + self.assertEqual(payload, json.loads(golden_json.read_text(encoding="utf-8"))) + + def test_river_crossings_project_reports_stably(self) -> None: + text, payload = self._run("river-crossings", _crossings_project) + statuses = {check["check"]: check["status"] for check in payload["checks"]} + self.assertEqual(statuses["metamorphic.trail-order"], "passed") + self.assertEqual(statuses["project.parameters_match_steps"], "passed") + self.assertEqual(statuses["validation.run_record_matches"], "passed") + self._assert_golden("river-crossings", text, payload) + + def test_district_facilities_project_reports_stably(self) -> None: + text, payload = self._run("district-facilities", _catchments_project) + statuses = {check["check"]: check["status"] for check in payload["checks"]} + self.assertEqual(statuses["expectation.d1-count"], "warning") + self.assertEqual(statuses["validation.warning_or_failed_propagates_to_status"], "passed") + self.assertEqual(payload["status"], "warning") + self._assert_golden("district-facilities", text, payload) + + +if __name__ == "__main__": + unittest.main()