Complete #13: metamorphic checks, source pins, connector pilot, and the OpenMapBench interface

.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "name": "open-map-stack",
   "displayName": "OpenMapStack",
   "description": "Agentic GIS / geospatial workflows: source discovery and provenance (from open data sources first), vector/raster/point-cloud pipelines, CRS and metric analysis, spatial SQL, QGIS projects, tile generation, and web maps — compiled into reproducible projects.",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "author": {
     "name": "Jaak Laineste",
     "url": "https://github.com/jaakla"

.github/workflows/eval-benchmark.yml

@@ -35,6 +35,15 @@ on:
       cases:
         description: "Space-separated case ids to run (default: every case that declares live mode)"
         required: false
+      arms:
+        description: "Benchmark arm(s): oms (skill injected), plain (no skill), or paired (both over identical cases, trials, and seeds)"
+        required: false
+        default: oms
+        type: choice
+        options: [oms, plain, paired]
+      price_catalog_date:
+        description: "YYYY-MM-DD of the price list behind the cost estimates (recorded in arm provenance)"
+        required: false
   schedule:
     - cron: "0 3 * * 1" # weekly, Monday 03:00 UTC
 
@@ -120,6 +129,8 @@ jobs:
           EVAL_TIMEOUT: ${{ github.event.inputs.timeout || '1200' }}
           EVAL_SEED: ${{ github.event.inputs.seed || '' }}
           EVAL_CASES: ${{ github.event.inputs.cases || '' }}
+          EVAL_ARMS: ${{ github.event.inputs.arms || 'oms' }}
+          EVAL_PRICE_CATALOG_DATE: ${{ github.event.inputs.price_catalog_date || '' }}
         run: |
           if [[ "$EVAL_AGENT" == "claude_code" ]]; then
             EVAL_MODEL="$CLAUDE_MODEL"
@@ -136,12 +147,13 @@ jobs:
             --mode live
             --agent "$EVAL_AGENT"
             --model "$EVAL_MODEL"
-            --skill-mode enabled
+            --arms "$EVAL_ARMS"
             --repetitions "$EVAL_REPETITIONS"
             --timeout "$EVAL_TIMEOUT"
             --json "eval-benchmark-results-$EVAL_AGENT.json"
           )
           if [[ -n "$EVAL_SEED" ]]; then args+=(--seed "$EVAL_SEED"); fi
+          if [[ -n "$EVAL_PRICE_CATALOG_DATE" ]]; then args+=(--price-catalog-date "$EVAL_PRICE_CATALOG_DATE"); fi
           # No --case means every case that declares live mode. The list used
           # to be hardcoded to 001/002/004/005, which silently left the four
           # prompt-style cases (070-073, added by PR 8) running in no workflow

.github/workflows/eval-warehouse.yml

@@ -0,0 +1,74 @@
+name: OpenMapStack warehouse connector pilot
+
+# The PostGIS connector is exercised end to end against a real PostGIS
+# service: read-only discovery, dry-run plan, approved GeoParquet snapshot,
+# and a pin the contract accepts. Scheduled/manual, and on changes to the
+# connector code, so an unrelated PR is never blocked by a service container.
+on:
+  workflow_dispatch: {}
+  schedule:
+    - cron: "37 5 * * 1"
+  pull_request:
+    paths:
+      - "openmapstack/connectors/**"
+      - "openmapstack/sources.py"
+      - "tests/test_connectors.py"
+      - ".github/workflows/eval-warehouse.yml"
+
+jobs:
+  postgis-connector:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    services:
+      postgis:
+        image: postgis/postgis:16-3.4
+        env:
+          POSTGRES_PASSWORD: ci-only-password
+          POSTGRES_DB: gis
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres -d gis"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install the CLI with geodata and PostGIS support
+        run: pip install ".[geo,postgis]"
+
+      - name: Prepare controlled DuckDB Spatial directory
+        run: |
+          echo "OPENMAPSTACK_SPATIAL_EXTENSION_DIR=${RUNNER_TEMP}/openmapstack-duckdb-extensions" >> "$GITHUB_ENV"
+          OPENMAPSTACK_SPATIAL_EXTENSION_DIR="${RUNNER_TEMP}/openmapstack-duckdb-extensions" python evals/prepare_spatial.py
+
+      - name: Seed a parcels table and a read-only role
+        env:
+          PGPASSWORD: ci-only-password
+        run: |
+          sudo apt-get update -q && sudo apt-get install -y -q postgresql-client
+          psql -h 127.0.0.1 -U postgres -d gis -v ON_ERROR_STOP=1 <<'SQL'
+          CREATE EXTENSION IF NOT EXISTS postgis;
+          CREATE TABLE public.parcels (cadastral_id text primary key, land_use text, area_m2 numeric(24, 9), geom geometry(Polygon, 3301));
+          INSERT INTO public.parcels VALUES
+            ('P1','ARIMAA', 10000.123456789, ST_GeomFromText('POLYGON((660100 6466500,660200 6466500,660200 6466600,660100 6466600,660100 6466500))',3301)),
+            ('P2','ARIMAA', 10000.000000001, ST_GeomFromText('POLYGON((660300 6466500,660400 6466500,660400 6466600,660300 6466600,660300 6466500))',3301)),
+            ('P3','TOOTMISMAA', 9999.999999999, ST_GeomFromText('POLYGON((660500 6466500,660600 6466500,660600 6466600,660500 6466600,660500 6466500))',3301));
+          CREATE ROLE reader LOGIN PASSWORD 'ci-only-reader';
+          GRANT CONNECT ON DATABASE gis TO reader;
+          GRANT USAGE ON SCHEMA public TO reader;
+          GRANT SELECT ON ALL TABLES IN SCHEMA public TO reader;
+          ANALYZE public.parcels;
+          SQL
+
+      - name: Run the connector suite against the live service
+        env:
+          # The credential reaches the connector only through this reference;
+          # the tests assert it never appears in any recorded output.
+          OPENMAPSTACK_TEST_POSTGIS_DSN: postgresql://reader:ci-only-reader@127.0.0.1:5432/gis
+        run: python -m unittest -v tests.test_connectors

AGENTS.md

@@ -121,6 +121,10 @@ python3 evals/run.py --mode fixture
 openmapstack validate examples/tartu-development/project.yaml --preflight
 openmapstack inspect examples/tartu-development/project.yaml --json
 
+# Check API that external harnesses consume (see docs/openmapbench-interop.md)
+openmapstack api-info --json
+openmapstack checks
+
 # Coverage gate used by CI
 python3 -m coverage run -m unittest discover -v
 python3 -m coverage report

README.md

@@ -16,6 +16,7 @@ It is open-first and cloud-native by default, built on shoulders of the awesome
 - [SKILL.md](SKILL.md) — the skill entry point: triggers, global defaults, format and compute decision matrices, anti-patterns, and a quick triage guide.
 - [references/data-sources.md](references/data-sources.md) - lists OSM, Overture, Sentinel/Landsat, regional portals, STAC catalogs and others.
 - [references/services-and-scale.md](references/services-and-scale.md) - depending on case use local installs or hosted/SaaS services for global-scale basemaps, elevation, routing, geocoding, place search, and postcodes.
+- [references/user-data-sources.md](references/user-data-sources.md) - the user's own warehouse data: credentials by reference, read-only discovery, approval-gated snapshots, and the pin classes that make a warehouse table reproducible.
 - [references/formats-and-crs.md](references/formats-and-crs.md) - how to choose formats, conversions, projections, EPSG codes.
 - [references/processing.md](references/processing.md) - when and how to use GDAL/OGR, GeoPandas, xarray, DuckDB, PostGIS, PDAL and other open geo processing tools.
 - [references/analytics.md](references/analytics.md) — do vector/raster analytics, terrain, hydrology, network, point clouds, geocoding etc.
@@ -27,6 +28,7 @@ It is open-first and cloud-native by default, built on shoulders of the awesome
 - [examples/tartu-development/](examples/tartu-development/) — a fully-worked reproducible project matching the acceptance scenario: source provenance + timestamps, explicit assumptions, two verified project overrides (a scenario attribute change with prior-value verification, and hypothetical scenario geometry), deterministic pipeline, machine-readable validation, and semantic presentation.
 - [evals/](evals/) — the eval suite grading whether an agent reaches the right analytical answer, respects the GIS-method guardrails, and reruns reproducibly, with the `openmapstack-project/v1` contract as the substrate that makes those independently checkable: `python evals/run.py --mode fixture` runs deterministic, no-LLM checks against real generated artifacts (analytical correctness against known geospatial truth, metric CRS, source immutability, schema, overrides, validation integrity, presentation contract, and clean reruns), plus adversarial cases and a pluggable live-agent benchmark (Claude Code, Codex, and any OpenAI-compatible API such as OpenRouter — URL and model via `OPENAI_COMPATIBLE_*` env, API key as a secret).
 - [`openmapstack/`](openmapstack/) — the installable `openmapstack validate/run/inspect` CLI for auditing and executing `openmapstack-project/v1` projects, plus [`openmapstack/checks/`](openmapstack/checks/): the reusable, semantic check library. All but five of its checks are oracle-free, so the same functions that grade the eval suite also grade a user's own project on data this repository has never seen.
+- [docs/openmapbench-interop.md](docs/openmapbench-interop.md) — the narrow, versioned contract a benchmark harness such as OpenMapBench consumes: `openmapstack checks` / `check` / `api-info` (`openmapstack-check-api/v1`), the packaged result schemas, skill snapshots, arm provenance, and exported task bundles.
 - [`.claude-plugin/`](.claude-plugin/) — Claude Code plugin and marketplace manifests, so the repository can also be installed with `/plugin install`. Validated in CI by [`.github/workflows/plugin.yml`](.github/workflows/plugin.yml).
 
 My local Estonia-specific guidance (Maa- ja Ruumiamet, ETAK, EPSG:3301 / L-EST97) is included for convenience. But all the global sources are incuded for world-wide coverage.
@@ -151,6 +153,15 @@ openmapstack run path/to/project.yaml
 
 # Review sources, versions, overrides, ordered steps, outputs, and latest run.
 openmapstack inspect path/to/project.yaml
+
+# Copy SKILL.md, references/, and templates/ into a hashed, inspectable snapshot.
+openmapstack skill-snapshot --out /tmp/oms-skill --json
+openmapstack skill-snapshot --inspect /tmp/oms-skill
+
+# Read-only discovery of a warehouse source, then an approval-gated snapshot.
+openmapstack source discover path/to/project.yaml --source parcels
+openmapstack source snapshot path/to/project.yaml --source parcels \
+  --query "SELECT id, geom FROM cadastre.parcels" --destination data/source/parcels.parquet --approve
 ```
 
 Useful automation options:
@@ -176,6 +187,7 @@ PyQGIS is available.
 ```bash
 openmapstack verify path/to/project.yaml
 openmapstack verify path/to/project.yaml --rerun     # + rebuild from source and compare
+openmapstack verify path/to/project.yaml --metamorphic   # + run declared no-oracle relations
 openmapstack verify path/to/project.yaml --json --output validation/verify-report.json
 openmapstack verify path/to/project.yaml --strict    # warnings and not-testable also return 1
 ```
@@ -217,6 +229,24 @@ inputs, or a retained local evidence file invalidates the attestation and
 returns it to warning status. See
 [the project contract](references/project-spec.md#26-validation).
 
+Where no golden answer exists at all, `validation.metamorphic[]` declares
+relations that must hold under a controlled perturbation: shuffle a source and
+the result must not change, duplicate every feature and a keyed set must not
+change, widen an inclusion buffer and no candidate may disappear. Each relation
+states the precondition that makes it valid, is executed by
+`verify --metamorphic` in an isolated copy against the project's own pipeline,
+and reports `not_testable` with the reason when the precondition does not hold
+on the actual data. See [the project contract](references/project-spec.md#26-validation).
+
+`openmapstack source` is the connector pilot for the user's own data
+(DuckDB local files and PostGIS). Credentials are referenced, never stored;
+discovery is read-only with a statement timeout; a snapshot is a dry run
+until `--approve`, is limited by rows and bytes, lands only under
+`data/source/`, and hands back the `pin` block that makes the source
+reproducible. A warehouse table with only a timestamp is not pinned; an
+expired backend snapshot is reported as `not_reproducible`. See
+[user data sources](references/user-data-sources.md).
+
 `validate` checks manifest structure, source retrieval/version/licensing data,
 CRS declarations, processing graph resolution, override provenance and files,
 output existence, validation-report parity/status propagation, override

SKILL.md

@@ -60,6 +60,7 @@ Hard rules for every material analysis — each is expanded in `references/proje
 | If the task involves... | Read |
 |---|---|
 | Finding or sourcing data (OSM, Overture, Sentinel, Landsat, building footprints, regional portals, STAC catalogs, MCP-based discovery) | `references/data-sources.md` |
+| Reading the user's own warehouse or database (PostGIS, DuckDB, GeoParquet directories): credentials by reference, read-only discovery, approved snapshots, pin classes | `references/user-data-sources.md` |
 | Choosing local processing vs online/hosted/SaaS services for global or continental scale; basemaps, elevation, routing, geocoding, place search, postcode lookup APIs | `references/services-and-scale.md` |
 | Choosing a format, converting between formats, or any CRS / projection / EPSG question | `references/formats-and-crs.md` |
 | Compiling a reproducible GIS project artifact (`project.yaml`, pipeline, overrides, validation, presentation) | `references/project-spec.md` + `templates/` |

docs/maintainers/architecture.md

@@ -36,6 +36,7 @@ The distinction matters: `SKILL.md` is the product being developed and evaluated
 - Project contract: `references/project-spec.md`; machine validation is also constrained by `openmapstack/schemas/project-v1.schema.json` and `openmapstack/validation.py`.
 - Automatic `verify` plan and applicability: `docs/verify-applicability.md` plus `openmapstack/verify.py`.
 - Eval semantics: `evals/README.md`, `evals/schemas/`, `evals/run.py`, case definitions, and tests.
+- External check consumption: `docs/openmapbench-interop.md` plus `openmapstack/api.py` and the packaged result schemas; reporting dimensions are owned by `openmapstack.api.DIMENSIONS`.
 - Roadmap/current work: GitHub issues. Do not mirror their checklists here.
 
 When these disagree, resolve the inconsistency at the owning layer rather than adding another interpretation here.
@@ -110,6 +111,18 @@ This environment cleanup is deliberately **not** a general sandbox or allowlist
 
 The eval harness additionally forbids reaching back into eval reference generators. That restriction is supplied by the eval caller; the shipped package intentionally does not know that `evals/` exists.
 
+## Connectors are a trust boundary, like attestations
+
+`openmapstack/connectors/` reads user warehouse data on the user's behalf and is held to four rules that must survive any refactor: credentials are resolved from a reference and never recorded; sessions are read-only with a statement timeout; only a single `SELECT` reaches the backend; and nothing is materialised under `data/source/` without an explicit approval flag and within declared row/byte limits. Every message the package emits passes through `openmapstack.sources.redact`.
+
+The DuckDB local connector confines file access to its root (`allowed_directories` + `enable_external_access = false`) and exposes files as views so queries never spell paths. PostGIS has no durable time travel, so its pin is always a local snapshot; the transaction snapshot id is retrieval metadata, not a pin. Unverified backends are refused (`backend_unsupported`) rather than approximated.
+
+Pin classes live in `openmapstack/sources.py` and are shared by `validate` (`source.pin`, `source.credentials`) and `verify` (`provenance.every_source_pinned`, `provenance.no_inline_credentials`). Do not add a third interpretation.
+
+## Metamorphic relations execute the project's own pipeline
+
+`openmapstack/metamorphic.py` reuses the clean-rerun workspace preparation (`openmapstack/rerun.py`), perturbs only the copy, and compares against the produced outputs. A relation is valid only under its declared preconditions; unmet data preconditions are `not_testable`, invalid declarations fail, and unknown relation names are rejected rather than skipped. `runtime.implementation.parameters` (`openmapstack/parameters.py`) is the only sanctioned way to vary a pipeline setting from outside. Keep `metamorphic_evidence` a separate eval dimension from `gis_correctness`: a relation that holds is self-consistency, not a correct answer.
+
 ## Integrity and path safety
 
 Project-relative paths are resolved through `openmapstack/project.py` helpers and must remain under the project root. Code that adds new file addressing should reuse the same safety model rather than joining unchecked user paths.

docs/maintainers/debugging.md

@@ -113,6 +113,15 @@ When a plausible wrong project from a live/user run survives the current checks,
 4. add checker/unit coverage if the defect exposes a checker bug;
 5. only then add prose context here if the trap remains worth remembering.
 
+## Metamorphic mutations must live in the pipeline copy, not only in the generator
+
+A metamorphic relation reruns the project's own `pipeline.py` on perturbed input and compares with the produced outputs. For a fixture mutation that means the *copied* pipeline must reproduce the defect: if only `gen.py --break=` injects it at generation time, the variant run rebuilds the healthy analysis, every relation "fails" for the wrong reason, and the mutation is not isolated. `gen.py` therefore reads the `EVAL-BREAK` warning back in pipeline mode for the pipeline-logic break modes (`order_dependent`, `distance_inverted`, `duplicate_sensitive`) and nothing else.
+
+Two related traps:
+
+- a relation's detection power depends on the data and the variant size. Widening the mini-Tartu road threshold by 1.5× cannot expose an inverted predicate because the only far parcel sits at 5450 m; the fixture declares `variant: {multiply: 3}` for that reason. When a mutation survives, check the geometry before suspecting the relation;
+- a GeoJSON output without a `crs` member reads back as EPSG:4326. A pipeline that writes analysis-CRS coordinates into plain GeoJSON and declares `EPSG:3301` in the manifest fails `geodata.dataset_crs_is` correctly. Write the `crs` member (or use GeoParquet) rather than relaxing the check.
+
 ## Generated benchmark artifacts are evidence, not source
 
 Retained live/visual evidence belongs under `evals/results/<run-id>/...` and CI artifacts. Do not treat generated result JSON, screenshots, event streams, or temporary projects as canonical repository state unless a fixture intentionally owns them.