From e9e6344b03905c8e7721f7275a33b36491337268 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 8 Jul 2026 16:12:29 +0200 Subject: [PATCH 1/4] generate prr collection --- deep_code/cli/main.py | 2 + deep_code/constants.py | 7 + .../utils/test_dataset_stac_generator.py | 290 ++++++++++++- deep_code/tools/new.py | 20 + deep_code/utils/dataset_stac_generator.py | 381 +++++++++++++++++- 5 files changed, 698 insertions(+), 2 deletions(-) diff --git a/deep_code/cli/main.py b/deep_code/cli/main.py index e267305..ac306f5 100644 --- a/deep_code/cli/main.py +++ b/deep_code/cli/main.py @@ -8,6 +8,7 @@ from deep_code.cli.generate_config import generate_config from deep_code.cli.lint import lint_dataset +from deep_code.cli.prr import generate_prr_collection_cmd from deep_code.cli.publish import publish @@ -20,6 +21,7 @@ def main(): main.add_command(publish) main.add_command(generate_config) main.add_command(lint_dataset) +main.add_command(generate_prr_collection_cmd) if __name__ == "__main__": main() diff --git a/deep_code/constants.py b/deep_code/constants.py index fa9e4b8..e1fee57 100644 --- a/deep_code/constants.py +++ b/deep_code/constants.py @@ -41,3 +41,10 @@ "https://stac-extensions.github.io/application/v0.1.0/schema.json" ) ZARR_MEDIA_TYPE = "application/vnd+zarr" +DATACUBE_SCHEMA_URI = "https://stac-extensions.github.io/datacube/v2.2.0/schema.json" +PROCESSING_SCHEMA_URI = ( + "https://stac-extensions.github.io/processing/v1.2.0/schema.json" +) +SCIENTIFIC_SCHEMA_URI = ( + "https://stac-extensions.github.io/scientific/v1.0.0/schema.json" +) diff --git a/deep_code/tests/utils/test_dataset_stac_generator.py b/deep_code/tests/utils/test_dataset_stac_generator.py index 0b806cd..e981df6 100644 --- a/deep_code/tests/utils/test_dataset_stac_generator.py +++ b/deep_code/tests/utils/test_dataset_stac_generator.py @@ -3,17 +3,23 @@ # Permissions are hereby granted under the terms of the MIT License: # https://opensource.org/licenses/MIT. +import json +import os +import tempfile import unittest from datetime import datetime from unittest.mock import MagicMock, patch import numpy as np -from pystac import Catalog, Item +from pystac import Catalog, Collection, Item from xarray import DataArray, Dataset from deep_code.constants import ( + DATACUBE_SCHEMA_URI, DEEPESDL_COLLECTION_SELF_HREF, + OSC_SCHEMA_URI, OSC_THEME_SCHEME, + PROCESSING_SCHEMA_URI, PRODUCT_BASE_CATALOG_SELF_HREF, VARIABLE_BASE_CATALOG_SELF_HREF, ZARR_MEDIA_TYPE, @@ -763,3 +769,285 @@ def test_update_existing_variable_catalog(self, mock_open_ds): rels = [lnk["rel"] for lnk in result["links"]] self.assertIn("child", rels) self.assertIn("related", rels) # theme link + + +class TestPRRCollection(unittest.TestCase): + """Tests for the PRR-style Collection -> Item -> Assets generation.""" + + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def setUp(self, mock_open_ds): + self.dataset = Dataset( + coords={ + "lon": ("lon", np.linspace(-20, 20, 4)), + "lat": ("lat", np.linspace(-10, 10, 3)), + "time": ( + "time", + [ + np.datetime64(datetime(2021, 1, 1), "ns"), + np.datetime64(datetime(2021, 1, 3), "ns"), + ], + ), + }, + data_vars={ + "sst": ( + ("time", "lat", "lon"), + np.random.rand(2, 3, 4), + {"units": "K", "long_name": "Sea surface temperature"}, + ), + "chl": ( + ("time", "lat", "lon"), + np.random.rand(2, 3, 4), + {"units": "mg m-3"}, + ), + # A CRS var that must be excluded from cube:variables and drive EPSG. + "spatial_ref": ((), 0, {"spatial_epsg": 3035}), + }, + attrs={"description": "PRR test cube"}, + ) + mock_open_ds.return_value = self.dataset + self.gen = OscDatasetStacGenerator( + dataset_id="test.zarr", + collection_id="prr-collection", + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + access_link="s3://bucket/test.zarr", + osc_status="ongoing", + osc_region="Global", + osc_themes=["oceans"], + osc_missions=["sentinel-3"], + documentation_link="https://example.org/doc", + ) + + # ---- helpers ---- + + def test_get_epsg_from_spatial_ref(self): + self.assertEqual(self.gen._get_epsg(), 3035) + + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def test_get_epsg_default_4326(self, mock_open_ds): + ds = Dataset( + coords={ + "lon": ("lon", np.linspace(-1, 1, 2)), + "lat": ("lat", np.linspace(-1, 1, 2)), + "time": ("time", [np.datetime64(datetime(2020, 1, 1), "ns")]), + }, + data_vars={"v": (("time", "lat", "lon"), np.random.rand(1, 2, 2))}, + ) + mock_open_ds.return_value = ds + gen = OscDatasetStacGenerator( + dataset_id="t.zarr", + collection_id="c", + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + ) + self.assertEqual(gen._get_epsg(), 4326) + + def test_get_cube_dimensions(self): + dims = self.gen._get_cube_dimensions() + self.assertEqual(set(dims), {"lon", "lat", "time"}) + self.assertEqual(dims["lon"]["type"], "spatial") + self.assertEqual(dims["lon"]["axis"], "x") + self.assertEqual(dims["lon"]["reference_system"], 3035) + self.assertEqual(dims["lon"]["extent"], [-20.0, 20.0]) + self.assertEqual(dims["lat"]["axis"], "y") + self.assertEqual(dims["lat"]["extent"], [-10.0, 10.0]) + self.assertEqual(dims["time"]["type"], "temporal") + self.assertEqual(len(dims["time"]["extent"]), 2) + + def test_get_cube_variables(self): + variables = self.gen._get_cube_variables() + # CRS variable must be excluded. + self.assertEqual(set(variables), {"sst", "chl"}) + self.assertEqual(variables["sst"]["type"], "data") + self.assertEqual(variables["sst"]["dimensions"], ["time", "lat", "lon"]) + self.assertEqual(variables["sst"]["unit"], "K") + self.assertEqual(variables["sst"]["description"], "Sea surface temperature") + # chl has a unit but no long_name/description. + self.assertEqual(variables["chl"]["unit"], "mg m-3") + self.assertNotIn("description", variables["chl"]) + + # ---- item ---- + + def test_build_prr_stac_item(self): + item = self.gen.build_prr_stac_item() + self.assertIsInstance(item, Item) + self.assertEqual(item.id, "prr-collection") + self.assertIn(DATACUBE_SCHEMA_URI, item.stac_extensions) + self.assertIn("cube:dimensions", item.properties) + self.assertIn("cube:variables", item.properties) + + # datetime is null; start/end are timezone-aware ISO strings. + self.assertIsNone(item.datetime) + self.assertTrue(item.properties["start_datetime"].endswith("+00:00")) + self.assertTrue(item.properties["end_datetime"].endswith("+00:00")) + + self.assertEqual(set(item.assets), {"zarr-data", "zarr-consolidated-metadata"}) + self.assertEqual(item.assets["zarr-data"].href, "s3://bucket/test.zarr") + self.assertEqual(item.assets["zarr-data"].media_type, ZARR_MEDIA_TYPE) + self.assertEqual( + item.assets["zarr-consolidated-metadata"].href, + "s3://bucket/test.zarr/.zmetadata", + ) + + # ---- collection ---- + + def test_build_prr_collection(self): + coll = self.gen.build_prr_collection() + self.assertIsInstance(coll, Collection) + self.assertEqual(coll.id, "prr-collection") + self.assertEqual(coll.license, "CC-BY-4.0") + + ef = coll.extra_fields + self.assertEqual(ef["osc:type"], "product") + self.assertEqual(ef["osc:status"], "ongoing") + self.assertEqual(ef["osc:region"], "Global") + self.assertCountEqual(ef["osc:variables"], ["sst", "chl"]) + self.assertEqual(ef["osc:missions"], ["sentinel-3"]) + self.assertEqual(ef["cf:parameter"], [{"name": "prr-collection"}]) + self.assertIn("processing:datetime", ef) + + # Extensions declared. + self.assertIn(OSC_SCHEMA_URI, coll.stac_extensions) + self.assertIn(PROCESSING_SCHEMA_URI, coll.stac_extensions) + + # Themes are plain dicts (not Theme objects). + themes = ef["themes"] + self.assertEqual(themes[0]["scheme"], OSC_THEME_SCHEME) + self.assertEqual(themes[0]["concepts"], [{"id": "oceans"}]) + self.assertIsInstance(themes[0], dict) + + # The single Item is attached as a child. + items = list(coll.get_items()) + self.assertEqual(len(items), 1) + self.assertEqual(items[0].id, "prr-collection") + + def test_build_prr_collection_conformant_fields(self): + """A fully configured generator emits every PRR-required field.""" + from deep_code.constants import SCIENTIFIC_SCHEMA_URI + + with patch( + "deep_code.utils.dataset_stac_generator.open_dataset", + return_value=self.dataset, + ): + gen = OscDatasetStacGenerator( + dataset_id="test.zarr", + collection_id="prr-collection", + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + access_link="s3://bucket/test.zarr", + osc_status="ongoing", + osc_region="Global", + osc_themes=["oceans"], + osc_missions=["sentinel-3"], + osc_project_description="A detailed project description.", + osc_project_website="https://project.example.org", + osc_contract_number="4000114410/15/NL/BW", + thumbnail="https://example.org/logo.jpeg", + sci_doi="10.1000/xyz123", + ) + coll = gen.build_prr_collection() + ef = coll.extra_fields + + # Scientific extension declared alongside the others. + self.assertIn(SCIENTIFIC_SCHEMA_URI, coll.stac_extensions) + # Required PRR project fields present. + self.assertEqual(ef["osc:initiative"], "earthcode") + self.assertEqual(ef["osc:project_website"], "https://project.example.org") + self.assertEqual( + ef["osc:project_description"], "A detailed project description." + ) + self.assertEqual(ef["osc:contract-number"], "4000114410/15/NL/BW") + self.assertEqual(ef["sci:doi"], "10.1000/xyz123") + # Thumbnail asset with correct role and guessed media type. + self.assertIn("thumbnail", coll.assets) + thumb = coll.assets["thumbnail"] + self.assertEqual(thumb.roles, ["thumbnail"]) + self.assertEqual(thumb.media_type, "image/jpeg") + + def test_build_prr_collection_fallbacks(self): + """project_website/description fall back to doc link / description.""" + coll = self.gen.build_prr_collection() + ef = coll.extra_fields + # Default initiative. + self.assertEqual(ef["osc:initiative"], "earthcode") + # Fallbacks: website -> documentation_link, description -> dataset description. + self.assertEqual(ef["osc:project_website"], "https://example.org/doc") + self.assertEqual(ef["osc:project_description"], "PRR test cube") + # No thumbnail / contract number configured -> absent. + self.assertNotIn("thumbnail", coll.assets) + self.assertNotIn("osc:contract-number", ef) + + def test_thumbnail_media_type_guess(self): + self.gen.thumbnail = "https://x/logo.png" + self.assertEqual(self.gen._thumbnail_media_type(), "image/png") + self.gen.thumbnail = "https://x/logo.JPG" + self.assertEqual(self.gen._thumbnail_media_type(), "image/jpeg") + self.gen.thumbnail = "https://x/logo.webp" + self.assertEqual(self.gen._thumbnail_media_type(), "image/webp") + self.gen.thumbnail_media_type = "image/tiff" + self.assertEqual(self.gen._thumbnail_media_type(), "image/tiff") + + def test_build_prr_collection_cf_params_override(self): + self.gen.cf_params = [{"name": "sst", "units": "K"}] + coll = self.gen.build_prr_collection() + self.assertEqual( + coll.extra_fields["cf:parameter"], [{"name": "sst", "units": "K"}] + ) + + @patch("deep_code.utils.dataset_stac_generator.open_dataset") + def test_build_prr_collection_no_themes(self, mock_open_ds): + mock_open_ds.return_value = self.dataset + gen = OscDatasetStacGenerator( + dataset_id="test.zarr", + collection_id="prr-collection", + workflow_id="wf", + workflow_title="WF", + license_type="CC-BY-4.0", + access_link="s3://bucket/test.zarr", + ) + coll = gen.build_prr_collection() + self.assertNotIn("themes", coll.extra_fields) + + # ---- save (local self-contained tree) ---- + + def test_save_prr_collection_writes_tree(self): + with tempfile.TemporaryDirectory() as tmp: + out = self.gen.save_prr_collection(tmp) + self.assertEqual(out, tmp) + + collection_path = os.path.join(tmp, "collection.json") + item_path = os.path.join(tmp, "prr-collection", "prr-collection.json") + self.assertTrue(os.path.isfile(collection_path)) + self.assertTrue(os.path.isfile(item_path)) + + # Files are plain-JSON serialisable (no leftover Python objects). + with open(collection_path) as f: + coll_dict = json.load(f) + with open(item_path) as f: + item_dict = json.load(f) + + self.assertEqual(coll_dict["type"], "Collection") + self.assertEqual(item_dict["type"], "Feature") + + # Structural links are relative; the Item link points at the child. + item_link = next( + lnk for lnk in coll_dict["links"] if lnk["rel"] == "item" + ) + self.assertFalse(item_link["href"].startswith("s3://")) + self.assertTrue(item_link["href"].endswith(".json")) + + # Asset hrefs stay absolute (the data lives on S3). + self.assertEqual( + item_dict["assets"]["zarr-data"]["href"], "s3://bucket/test.zarr" + ) + + def test_save_prr_collection_readable_by_pystac(self): + with tempfile.TemporaryDirectory() as tmp: + self.gen.save_prr_collection(tmp) + coll = Collection.from_file(os.path.join(tmp, "collection.json")) + items = list(coll.get_items()) + self.assertEqual(len(items), 1) + self.assertIn(DATACUBE_SCHEMA_URI, items[0].stac_extensions) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index 8fb49f3..fc53ed0 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -76,6 +76,22 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: "cf_parameter": [{"name": "[OPTIONAL: CF standard name]", "units": "[unit string]"}], } + # Fields used only by `deep-code generate-prr-collection` to build a + # PRR (Product Readiness Review) collection that conforms to + # https://eoresults.esa.int/prr_collection_specifications.html + prr = { + "prr_output_dir": "[OPTIONAL: local dir for the PRR collection tree — defaults to prr/{collection_id}]", + "osc_initiative": "[OPTIONAL: PRR initiative — 'earthcode' or 'apex' (default: earthcode)]", + "osc_contract_number": "[PRR-REQUIRED: ESA contract identifier, e.g. 4000114410/15/NL/BW]", + "osc_project_website": "[PRR-REQUIRED: project website URL — falls back to osc_project_url / documentation_link]", + "osc_project_description": "[PRR-REQUIRED: multi-line project description — falls back to description]", + "osc_missions": ["[PRR-REQUIRED: satellite mission name(s), e.g. sentinel-3]"], + "thumbnail": "[PRR-REQUIRED: URL to a collection thumbnail image (jpeg/png/webp)]", + "thumbnail_media_type": "[OPTIONAL: thumbnail MIME type — guessed from the URL suffix if omitted]", + "sci_doi": "[OPTIONAL: dataset DOI, e.g. 10.1000/xyz123 (not a DOI link)]", + "sci_citation": "[OPTIONAL: human-readable citation for the dataset]", + } + stac_catalog_comment = ( "\n# stac_catalog_s3_root: deep-code writes the following files to this S3 root:\n" "# {stac_catalog_s3_root}/catalog.json (STAC Catalog root)\n" @@ -94,4 +110,8 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: f.write(yaml.dump(required, sort_keys=False, width=1000, default_flow_style=False)) f.write("\n# --- OPTIONAL fields ---\n") f.write(yaml.dump(optional, sort_keys=False, width=1000, default_flow_style=False)) + f.write( + "\n# --- PRR fields (for `deep-code generate-prr-collection`) ---\n" + ) + f.write(yaml.dump(prr, sort_keys=False, width=1000, default_flow_style=False)) f.write(stac_catalog_comment) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 716d509..57b1ea9 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -8,12 +8,25 @@ from datetime import datetime, timezone import pandas as pd -from pystac import Catalog, Collection, Extent, Item, Asset, Link, SpatialExtent, TemporalExtent +from pystac import ( + Asset, + Catalog, + CatalogType, + Collection, + Extent, + Item, + Link, + SpatialExtent, + TemporalExtent, +) from deep_code.constants import ( CONTACTS_SCHEMA_URI, + DATACUBE_SCHEMA_URI, OSC_SCHEMA_URI, OSC_THEME_SCHEME, + PROCESSING_SCHEMA_URI, + SCIENTIFIC_SCHEMA_URI, THEMES_SCHEMA_URI, ZARR_MEDIA_TYPE, ) @@ -57,6 +70,14 @@ def __init__( osc_project_url: str | None = None, visualisation_link: str | None = None, description: str | None = None, + osc_initiative: str = "earthcode", + osc_contract_number: str | None = None, + osc_project_website: str | None = None, + osc_project_description: str | None = None, + thumbnail: str | None = None, + thumbnail_media_type: str | None = None, + sci_doi: str | None = None, + sci_citation: str | None = None, ): if " " in collection_id: raise ValueError( @@ -80,6 +101,15 @@ def __init__( self.cf_params = cf_params or {} self.visualisation_link = visualisation_link self.description = description + # PRR-specific project metadata (see PRR collection specification). + self.osc_initiative = osc_initiative or "earthcode" + self.osc_contract_number = osc_contract_number + self.osc_project_website = osc_project_website + self.osc_project_description = osc_project_description + self.thumbnail = thumbnail + self.thumbnail_media_type = thumbnail_media_type + self.sci_doi = sci_doi + self.sci_citation = sci_citation self.logger = logging.getLogger(__name__) self.dataset = open_dataset(dataset_id=dataset_id, logger=self.logger) self.variables_metadata = self.get_variables_metadata() @@ -657,6 +687,355 @@ def build_zarr_stac_catalog_file_dict( item_href: item.to_dict(transform_hrefs=False), } + # ------------------------------------------------------------------ # + # PRR (Product Readiness Review) style output # + # # + # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # + # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # + # the plain catalog.json/item.json under ``{root}/prr/``. # + # ------------------------------------------------------------------ # + + def _get_epsg(self) -> int: + """Best-effort EPSG code for the dataset, defaulting to 4326. + + Reads an ``spatial_epsg``/``epsg`` attribute from a ``crs`` or + ``spatial_ref`` variable when present; otherwise assumes geographic + WGS 84 (EPSG:4326). + """ + for var_name in ("spatial_ref", "crs"): + if var_name in self.dataset.variables: + attrs = self.dataset[var_name].attrs + for key in ("spatial_epsg", "epsg", "EPSG"): + if key in attrs: + try: + return int(attrs[key]) + except (TypeError, ValueError): + pass + return 4326 + + def _get_cube_dimensions(self) -> dict[str, dict]: + """Build the ``cube:dimensions`` object from the dataset coordinates. + + Follows the datacube STAC extension: horizontal spatial dimensions are + classified by axis (x/y), the ``time`` coordinate becomes a temporal + dimension, and any remaining index coordinate is emitted as an + additional dimension. + """ + ds = self.dataset + epsg = self._get_epsg() + x_names = {"lon", "longitude", "x"} + y_names = {"lat", "latitude", "y"} + dimensions: dict[str, dict] = {} + for name, coord in ds.coords.items(): + if name not in ds.dims: + # Skip non-dimension coordinates (e.g. scalar or auxiliary coords). + continue + lname = str(name).lower() + if lname in x_names: + dimensions[name] = { + "type": "spatial", + "axis": "x", + "extent": [float(coord.min()), float(coord.max())], + "reference_system": epsg, + } + elif lname in y_names: + dimensions[name] = { + "type": "spatial", + "axis": "y", + "extent": [float(coord.min()), float(coord.max())], + "reference_system": epsg, + } + elif lname == "time": + time_min = pd.to_datetime(coord.min().values).to_pydatetime() + time_max = pd.to_datetime(coord.max().values).to_pydatetime() + dimensions[name] = { + "type": "temporal", + "extent": [time_min.isoformat(), time_max.isoformat()], + } + else: + try: + dimensions[name] = { + "type": lname, + "extent": [float(coord.min()), float(coord.max())], + } + except (TypeError, ValueError): + dimensions[name] = { + "type": lname, + "values": [str(v) for v in coord.values.tolist()], + } + return dimensions + + def _get_cube_variables(self) -> dict[str, dict]: + """Build the ``cube:variables`` object from the dataset data variables.""" + skip = {"crs", "spatial_ref"} + variables: dict[str, dict] = {} + for name, var in self.dataset.data_vars.items(): + if name in skip: + continue + entry: dict = { + "type": "data", + "dimensions": [str(d) for d in var.dims], + } + unit = var.attrs.get("units") + if unit: + entry["unit"] = unit + description = var.attrs.get("long_name") or var.attrs.get("description") + if description: + entry["description"] = description + variables[name] = entry + return variables + + def build_prr_stac_item(self) -> Item: + """Build the single datacube Item for the PRR collection. + + One Item covers the full spatiotemporal extent of the Zarr store. It + carries the datacube extension (``cube:dimensions`` / ``cube:variables``) + and two assets pointing at the Zarr store. Structural links + (root/parent/collection/self) are left for :meth:`save_prr_collection` + to fill in via ``Collection.add_item`` + ``normalize_hrefs``. + """ + self.logger.info( + f"Building PRR STAC Item for collection '{self.collection_id}'." + ) + spatial_extent = self._get_spatial_extent() + temporal_extent = self._get_temporal_extent() + general_metadata = self._get_general_metadata() + + bbox = spatial_extent.bboxes[0] # [lon_min, lat_min, lon_max, lat_max] + lon_min, lat_min, lon_max, lat_max = bbox + geometry = { + "type": "Polygon", + "coordinates": [[ + [lon_min, lat_min], + [lon_max, lat_min], + [lon_max, lat_max], + [lon_min, lat_max], + [lon_min, lat_min], + ]], + } + + start_dt, end_dt = temporal_extent.intervals[0] + if start_dt is not None and start_dt.tzinfo is None: + start_dt = start_dt.replace(tzinfo=timezone.utc) + if end_dt is not None and end_dt.tzinfo is None: + end_dt = end_dt.replace(tzinfo=timezone.utc) + + now_iso = datetime.now(timezone.utc).isoformat() + + item = Item( + id=self.collection_id, + geometry=geometry, + bbox=bbox, + datetime=None, + properties={ + "start_datetime": start_dt.isoformat() if start_dt else None, + "end_datetime": end_dt.isoformat() if end_dt else None, + "description": general_metadata.get("description", ""), + "created": now_iso, + "updated": now_iso, + "cube:dimensions": self._get_cube_dimensions(), + "cube:variables": self._get_cube_variables(), + }, + ) + item.stac_extensions.append(DATACUBE_SCHEMA_URI) + # Asset hrefs stay absolute (the Zarr lives on S3); only the structural + # links become relative when the tree is normalised locally. + item.add_asset("zarr-data", Asset( + href=self.access_link, + media_type=ZARR_MEDIA_TYPE, + title="Zarr Data Store", + roles=["data"], + )) + item.add_asset("zarr-consolidated-metadata", Asset( + href=f"{self.access_link}/.zmetadata", + media_type="application/json", + title="Consolidated Zarr Metadata", + roles=["metadata"], + )) + self.logger.info(f"PRR STAC Item built for '{self.collection_id}'.") + return item + + def build_prr_collection(self) -> Collection: + """Build the PRR parent Collection with its single datacube Item attached. + + The Collection carries OSC extension fields (``osc:type``, ``osc:status``, + ``osc:variables``, ``osc:missions``, ``themes``), a ``cf:parameter`` list + and ``processing:datetime`` — aligning with the ESA EarthCODE PRR + endpoint (e.g. ``eoresults.esa.int``). The Item is added as a child so a + subsequent ``normalize_hrefs`` produces a self-contained tree. + """ + spatial_extent = self._get_spatial_extent() + temporal_extent = self._get_temporal_extent() + variables = self.get_variable_ids() + general_metadata = self._get_general_metadata() + + collection = Collection( + id=self.collection_id, + description=general_metadata.get("description", "No description provided."), + extent=Extent(spatial=spatial_extent, temporal=temporal_extent), + license=self.license_type, + title=self.collection_id, + ) + + osc_extension = OscExtension.add_to(collection) + osc_extension.osc_project = self.osc_project + osc_extension.osc_type = "product" + osc_extension.osc_status = self.osc_status + osc_extension.osc_region = self.osc_region + osc_extension.osc_variables = variables + osc_extension.osc_missions = self.osc_missions + osc_extension.cf_parameter = self.cf_params or [{"name": self.collection_id}] + + now_iso = datetime.now(timezone.utc).isoformat() + collection.extra_fields["created"] = now_iso + collection.extra_fields["updated"] = now_iso + + # Processing extension — the PRR reference collection declares this and + # exposes 'processing:datetime' (the ingestion/generation timestamp). + if PROCESSING_SCHEMA_URI not in collection.stac_extensions: + collection.stac_extensions.append(PROCESSING_SCHEMA_URI) + collection.extra_fields["processing:datetime"] = now_iso + + if self.osc_themes: + # Plain dicts (not Theme objects) so the collection serialises with a + # plain json encoder as well as pystac's own writer. + collection.extra_fields["themes"] = [ + { + "concepts": [{"id": theme} for theme in self.osc_themes], + "scheme": OSC_THEME_SCHEME, + } + ] + + # ---- Required PRR project-level metadata ---- + # Scientific extension is REQUIRED in the PRR profile's extension list; + # 'sci:*' values themselves remain optional. + if SCIENTIFIC_SCHEMA_URI not in collection.stac_extensions: + collection.stac_extensions.append(SCIENTIFIC_SCHEMA_URI) + + collection.extra_fields["osc:initiative"] = self.osc_initiative + project_website = ( + self.osc_project_website + or self.osc_project_url + or self.documentation_link + ) + if project_website: + collection.extra_fields["osc:project_website"] = project_website + project_description = ( + self.osc_project_description + or self.description + or general_metadata.get("description") + ) + if project_description: + collection.extra_fields["osc:project_description"] = project_description + if self.osc_contract_number: + collection.extra_fields["osc:contract-number"] = self.osc_contract_number + + # Optional scientific metadata. + if self.sci_doi: + collection.extra_fields["sci:doi"] = self.sci_doi + if self.sci_citation: + collection.extra_fields["sci:citation"] = self.sci_citation + + # Thumbnail asset (REQUIRED by the PRR spec: an asset named 'thumbnail' + # with the 'thumbnail' role). + if self.thumbnail: + collection.add_asset("thumbnail", Asset( + href=self.thumbnail, + media_type=self._thumbnail_media_type(), + title="Collection Thumbnail", + roles=["thumbnail"], + )) + + if self.documentation_link: + collection.add_link( + Link(rel="via", target=self.documentation_link, title="Documentation") + ) + if self.visualisation_link: + collection.add_link(Link( + rel="visualisation", + target=self.visualisation_link, + title="Dataset visualisation", + )) + + try: + osc_extension.validate_extension() + except ValueError as e: + raise ValueError(f"OSC Extension validation failed: {e}") + + self._warn_missing_prr_fields(collection, variables) + collection.add_item(self.build_prr_stac_item()) + return collection + + def _thumbnail_media_type(self) -> str: + """Return the thumbnail media type, guessed from the href suffix.""" + if self.thumbnail_media_type: + return self.thumbnail_media_type + href = (self.thumbnail or "").lower() + if href.endswith((".jpg", ".jpeg")): + return "image/jpeg" + if href.endswith(".webp"): + return "image/webp" + return "image/png" + + def _warn_missing_prr_fields( + self, collection: Collection, variables: list[str] + ) -> None: + """Log a warning for PRR-required fields that are absent, so the caller + knows the output is not yet spec-conformant.""" + ef = collection.extra_fields + missing = [] + if "thumbnail" not in collection.assets: + missing.append("thumbnail asset") + if not ef.get("osc:contract-number"): + missing.append("osc:contract-number") + if not ef.get("osc:project_website"): + missing.append("osc:project_website") + if not ef.get("osc:project_description"): + missing.append("osc:project_description") + if not ef.get("themes"): + missing.append("themes") + if not variables: + missing.append("osc:variables") + if not self.osc_missions: + missing.append("osc:missions") + if missing: + self.logger.warning( + "PRR collection is missing required field(s): %s. " + "The collection will not fully conform to the PRR specification " + "until these are provided in the dataset config.", + ", ".join(missing), + ) + + def save_prr_collection(self, output_dir: str) -> str: + """Write the PRR ``Collection -> Item -> Assets`` tree to a local folder. + + Produces a self-contained STAC tree with relative structural links, + ready to inspect or submit to the ESA EarthCODE PRR endpoint:: + + {output_dir}/ + ├── collection.json # STAC Collection (root) + └── {collection_id}/ + └── {collection_id}.json # datacube Item (whole Zarr) + + Zarr asset hrefs remain absolute (``s3://…``) since that is where the + data lives. + + Args: + output_dir: Local directory to write the collection tree into. + + Returns: + The ``output_dir`` that was written. + """ + self.logger.info( + f"Writing PRR STAC collection for '{self.collection_id}' to " + f"'{output_dir}'." + ) + collection = self.build_prr_collection() + collection.normalize_hrefs(output_dir) + collection.save(catalog_type=CatalogType.SELF_CONTAINED) + self.logger.info(f"PRR STAC collection written to '{output_dir}'.") + return output_dir + def build_dataset_stac_collection(self, mode: str, stac_catalog_s3_root: str | None = None) -> Collection: """Build an OSC STAC Collection for the dataset. From d9a765ddad6a7ce8ab8be1cf0c78356b60acd324 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 8 Jul 2026 16:16:28 +0200 Subject: [PATCH 2/4] updated docs --- deep_code/tools/new.py | 2 +- docs/cli.md | 37 ++++++++++++++++++++++++++ docs/configuration.md | 59 ++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 3 ++- docs/python-api.md | 23 ++++++++++++++++ 5 files changed, 122 insertions(+), 2 deletions(-) diff --git a/deep_code/tools/new.py b/deep_code/tools/new.py index fc53ed0..d910db0 100644 --- a/deep_code/tools/new.py +++ b/deep_code/tools/new.py @@ -77,7 +77,7 @@ def generate_dataset_template(output_path: Optional[str] = None) -> str: } # Fields used only by `deep-code generate-prr-collection` to build a - # PRR (Product Readiness Review) collection that conforms to + # PRR (Project Results Repository) collection that conforms to # https://eoresults.esa.int/prr_collection_specifications.html prr = { "prr_output_dir": "[OPTIONAL: local dir for the PRR collection tree — defaults to prr/{collection_id}]", diff --git a/docs/cli.md b/docs/cli.md index 734927a..a269479 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -45,3 +45,40 @@ Options: 3. Forks/clones the target metadata repo (production, staging, or testing), commits generated JSON, and opens a pull request on your behalf. The pull request description includes a "Generated with deep-code" attribution note. + +## Generate a PRR collection + +Generate a **Project Results Repository (PRR)** STAC collection as local files, ready +to submit to the [ESA EarthCODE PRR endpoint](https://eoresults.esa.int): + +```bash +deep-code generate-prr-collection dataset.yaml # writes to prr/ +deep-code generate-prr-collection dataset.yaml -o ./prr # custom output directory +``` + +This reuses the **same dataset config** as `publish`, but writes only local files and +needs no GitHub credentials or S3 write access (it only reads the Zarr store). It +produces a self-contained `Collection → Item → Assets` tree: + +``` +prr// +├── collection.json # STAC Collection (root, relative links) +└── / + └── .json # datacube Item covering the whole Zarr store +``` + +- The **Item** carries the `datacube` extension (`cube:dimensions` / `cube:variables` + extracted from the Zarr) plus `zarr-data` and `zarr-consolidated-metadata` assets. +- The **Collection** carries the OSC, Scientific, Processing, Themes and CF extensions + and the PRR-mandatory fields. + +The output conforms to the +[PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html) +when the PRR fields are set in the config. If any required field is missing, the command +still runs but logs a warning listing what is needed for full conformance. See +[PRR collection fields](configuration.md#prr-collection-fields). + +Options: + +- `--output-dir/-o`: directory to write the tree into. Defaults to `prr_output_dir` + from the config, then `prr/`. diff --git a/docs/configuration.md b/docs/configuration.md index d8d90d6..881c572 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -34,6 +34,16 @@ access_link: s3://bucket/your-dataset.zarr # defaults to s3://deep-esdl-public cf_parameter: - name: sea_surface_temperature units: kelvin + +# PRR fields (only used by `deep-code generate-prr-collection`) +osc_initiative: earthcode # earthcode | apex (default: earthcode) +osc_missions: [sentinel-3] +osc_contract_number: 4000114410/15/NL/BW +osc_project_website: https://project.example.org +osc_project_description: A detailed multi-line description of the project. +thumbnail: https://example.org/thumbnail.jpeg +sci_doi: 10.1000/xyz123 +prr_output_dir: ./prr/your-collection ``` ### Field reference @@ -54,6 +64,27 @@ cf_parameter: | `cf_parameter` | No | List of CF metadata dicts to override variable attributes (e.g. `name`, `units`). | | `stac_catalog_s3_root` | Yes | S3 root where the STAC Catalog and Item are published. Publishing fails if this field is absent. See [STAC Catalog on S3](#stac-catalog-on-s3). | +> The fields below are only read by [`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection); `publish` ignores them. "PRR-required" means the field is required by the [PRR specification](https://eoresults.esa.int/prr_collection_specifications.html), not by the command (which still runs and warns). + +### PRR collection fields + +| Field | Required | Description | +|---|---|---| +| `osc_initiative` | No | PRR initiative: `earthcode` or `apex` (default: `earthcode`). | +| `osc_missions` | PRR-required | List of satellite mission name(s), e.g. `[sentinel-3]`. | +| `osc_contract_number` | PRR-required | ESA contract identifier, e.g. `4000114410/15/NL/BW`. | +| `osc_project_website` | PRR-required | Project website URL. Falls back to `osc_project_url`, then `documentation_link`. | +| `osc_project_description` | PRR-required | Multi-line project description. Falls back to `description`. | +| `thumbnail` | PRR-required | URL to a collection thumbnail image (jpeg/png/webp). Added as an asset named `thumbnail` with role `thumbnail`. | +| `thumbnail_media_type` | No | Thumbnail MIME type. Guessed from the URL suffix when omitted. | +| `sci_doi` | No | Dataset DOI, e.g. `10.1000/xyz123` (a DOI name, not a link). | +| `sci_citation` | No | Human-readable citation for the dataset. | +| `prr_output_dir` | No | Local directory for the PRR collection tree. Defaults to `prr/{collection_id}`. | + +`themes` for a PRR collection must be drawn from the allowed set: +`atmosphere`, `cryosphere`, `land`, `magnetosphere-ionosphere`, `oceans`, `solid-earth` +(configured via `osc_themes`). See [PRR collection output](#prr-collection-output). + ### STAC Catalog on S3 `stac_catalog_s3_root` is required. deep-code writes a two-file STAC hierarchy to S3 alongside the data: @@ -79,6 +110,34 @@ S3 credentials for writing the STAC catalog are resolved in this order: then `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`, then the boto3 default chain (IAM role, `~/.aws/credentials`). +### PRR collection output + +[`deep-code generate-prr-collection`](cli.md#generate-a-prr-collection) writes a +self-contained STAC tree to a **local** directory (no S3 write, no GitHub PR): + +``` +prr/your-collection/ +├── collection.json # STAC Collection (root, relative links) +└── your-collection/ + └── your-collection.json # datacube Item covering the full Zarr store +``` + +- **Collection** — declares the OSC, Scientific, Processing, Themes and CF extensions, + and carries the PRR-mandatory fields (`osc:project`, `osc:initiative`, + `osc:contract-number`, `osc:project_website`, `osc:project_description`, + `osc:variables`, `osc:missions`, `cf:parameter`, `processing:datetime`, `themes`, + `license`) plus a required `thumbnail` asset. +- **Item** — carries the `datacube` extension (`cube:dimensions` / `cube:variables` + read from the Zarr) and the `zarr-data` / `zarr-consolidated-metadata` assets. Asset + hrefs stay absolute (`s3://…`) since that is where the data lives; structural links + are relative so the folder is portable. + +The output targets the +[PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html). +Fields that can't be defaulted (`osc_missions`, `osc_contract_number`, `thumbnail`) must +be supplied in the config — otherwise the command still generates the tree but warns that +it is not yet fully conformant. + ## Workflow config (YAML) ```yaml # Required diff --git a/docs/index.md b/docs/index.md index 843f4a5..657bb7c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,6 +8,7 @@ - Build STAC collections and catalogs for Datasets and their corresponding variables automatically from the dataset metadata. - Generate STAC catalog and item for the product (Zarr store) and publish them to S3. - Build OGC API records for Workflows and Experiments from your configs. +- Generate a Project Results Repository (PRR) STAC collection as local files for submission to the ESA EarthCODE PRR endpoint. - Flexible publishling targets i.e production/staging/testing EarthCODE metadata repositories with GitHub automation. ```mermaid @@ -15,7 +16,7 @@ flowchart LR subgraph User A["Config files
(dataset.yaml, workflow.yaml)"] - B["deep-code CLI
(generate-config, publish)"] + B["deep-code CLI
(generate-config, publish, generate-prr-collection)"] end subgraph App["deep-code internals"] diff --git a/docs/python-api.md b/docs/python-api.md index 5254f61..aa9d0ef 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -124,3 +124,26 @@ file_dict = generator.build_zarr_stac_catalog_file_dict( See [STAC Catalog on S3](configuration.md#stac-catalog-on-s3) for details on the generated structure. + +### PRR collection generation + +Build a self-contained PRR (Project Results Repository) `Collection → Item → Assets` +tree as local files. The high-level helper reads the same dataset config as the CLI: + +```python +from deep_code.tools.prr import generate_prr_collection + +out_dir = generate_prr_collection("dataset.yaml", output_dir="./prr") +# ./prr/collection.json + ./prr//.json +``` + +Or drive the generator directly: + +```python +generator.save_prr_collection("./prr") # writes a self-contained tree +``` + +The Item includes the `datacube` extension; the Collection declares the OSC, Scientific, +Processing, Themes and CF extensions and the PRR-mandatory fields. See +[PRR collection output](configuration.md#prr-collection-output) and +[Generate a PRR collection](cli.md#generate-a-prr-collection). From 3ac3d71d4904042628db55b636fc7b729b165ef8 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 8 Jul 2026 16:16:52 +0200 Subject: [PATCH 3/4] refactor --- deep_code/utils/dataset_stac_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deep_code/utils/dataset_stac_generator.py b/deep_code/utils/dataset_stac_generator.py index 57b1ea9..6deaa3c 100644 --- a/deep_code/utils/dataset_stac_generator.py +++ b/deep_code/utils/dataset_stac_generator.py @@ -688,7 +688,7 @@ def build_zarr_stac_catalog_file_dict( } # ------------------------------------------------------------------ # - # PRR (Product Readiness Review) style output # + # PRR (Project Results Repository) style output # # # # A self-contained ``Collection -> Item -> Assets`` tree that mirrors # # the ESA EarthCODE PRR tutorial. Emitted alongside (not replacing) # From ff854a6a7af808e5258bed8187256ff4e967a7a0 Mon Sep 17 00:00:00 2001 From: tejas Date: Wed, 8 Jul 2026 16:17:09 +0200 Subject: [PATCH 4/4] update CHANGES.md --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 2c2b89c..5c594e4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -90,3 +90,7 @@ - Generated project collection now includes required STAC extensions (`osc`, `themes`, `contacts`) and OSC-mandatory fields (`osc:type`, `osc:status`, `themes`, `contacts`) to pass OSC catalog validation. - Added optional `osc_project_url` field to the dataset config; used as the `via` link in the project collection. Falls back to `documentation_link` if omitted; defaults to the existing DeepESDL project collection when neither is provided. - `dataset_status` now defaults to `"ongoing"` when not specified in the dataset config. +- Added a new CLI command `generate-prr-collection` that writes a Project Results Repository (PRR) STAC collection as local files, ready for submission to the ESA EarthCODE PRR endpoint. It reuses the dataset config and needs no GitHub credentials or S3 write access. See [PRR collection specification](https://eoresults.esa.int/prr_collection_specifications.html). +- The PRR command produces a self-contained `Collection → Item → Assets` tree: the Item carries the `datacube` extension (`cube:dimensions` / `cube:variables` extracted from the Zarr) with `zarr-data` and `zarr-consolidated-metadata` assets, and the Collection declares the OSC, Scientific, Processing, Themes and CF extensions. +- Added PRR-specific dataset config fields: `osc_initiative` (default `earthcode`), `osc_missions`, `osc_contract_number`, `osc_project_website`, `osc_project_description`, `thumbnail`, `thumbnail_media_type`, `sci_doi`, `sci_citation`, and `prr_output_dir`. Missing PRR-required fields produce a warning rather than a failure. +- `generate-config` templates now document the PRR fields under a dedicated section.