Update the ozone diagnostics

changelog/886.fix.md

@@ -0,0 +1,4 @@
+Updates the ozone diagnostics to use the unified `ref/recipe_ref_ozone.yml` recipe from ESMValTool v2.15.
+This replaces the removed `recipe_ref_ozone_cmip7.yml`.
+The northern hemisphere polar cap now spans 60N to 85N, matching the upstream boundary fix.
+CMIP7 executions now evaluate the historical experiment through 2021.

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/diagnostics/ozone.py

@@ -13,7 +13,10 @@
 from climate_ref_core.esgf.obs4mips import Obs4MIPsRequest
 from climate_ref_core.metric_values.typing import FileDefinition, SeriesDefinition
 from climate_ref_core.testing import TestCase, TestDataSpecification
-from climate_ref_esmvaltool.diagnostics.base import ESMValToolDiagnostic, get_cmip_source_type
+from climate_ref_esmvaltool.diagnostics.base import (
+    ESMValToolDiagnostic,
+    get_cmip_source_type,
+)
 from climate_ref_esmvaltool.recipe import dataframe_to_recipe
 from climate_ref_esmvaltool.types import Recipe
 
@@ -81,7 +84,7 @@
                 RequireTimerange(
                     group_by=("instance_id",),
                     start=PartialDateTime(1996, 1),
-                    end=PartialDateTime(2014, 12),
+                    end=PartialDateTime(2021, 12),
                 ),
                 RequireContiguousTimerange(group_by=("instance_id",)),
                 AddSupplementaryDataset.from_defaults("areacella", SourceDatasetType.CMIP7),
@@ -92,10 +95,11 @@
             filters=(ozone_obs_filter,),
             group_by=("source_id",),
             constraints=(
+                # The recipe plots the observations through 2021, so require that coverage.
                 RequireTimerange(
                     group_by=("instance_id",),
                     start=PartialDateTime(1996, 1),
-                    end=PartialDateTime(2014, 12),
+                    end=PartialDateTime(2021, 12),
                 ),
             ),
         ),
@@ -122,7 +126,7 @@
                     slug="obs4mips",
                     facets=ozone_obs_filter.facets,
                     remove_ensembles=False,
-                    time_span=("1980", "2009"),
+                    time_span=("1996", "2015"),
                 ),
             ),
         ),
@@ -144,29 +148,54 @@
                         "region": "glb",
                     },
                     remove_ensembles=True,
-                    time_span=("1980", "2009"),
+                    time_span=("1950", "2015"),
+                    # Fabricate the CMIP7 historical series to extend to 2021-12
+                    extend_historical_to=(2021, 12),
                 ),
                 Obs4MIPsRequest(
                     slug="obs4mips",
                     facets=ozone_obs_filter.facets,
                     remove_ensembles=False,
-                    time_span=("1980", "2009"),
+                    time_span=("1996", "2021"),
                 ),
             ),
         ),
     ),
 )
 
 
+def _prepare_recipe(
+    recipe: Recipe,
+    input_files: dict[SourceDatasetType, pandas.DataFrame],
+    diagnostic: str,
+    variable: str,
+    cmip7_timerange: str | None = None,
+) -> None:
+    """
+    Prune the recipe to one diagnostic and insert the solved model dataset.
+    """
+    cmip_source = get_cmip_source_type(input_files)
+    recipe_variables = dataframe_to_recipe(input_files[cmip_source])
+
+    dataset = recipe_variables[variable]["additional_datasets"][0]
+    dataset.pop("timerange", None)
+    recipe["datasets"] = [dataset]
+    recipe["diagnostics"] = {diagnostic: recipe["diagnostics"][diagnostic]}
+
+    # The recipe includes CMIP6 time ranges
+    if cmip_source == SourceDatasetType.CMIP7 and cmip7_timerange is not None:
+        recipe["diagnostics"][diagnostic]["variables"][variable]["timerange"] = cmip7_timerange
+
+
 class O3LatTimeMapplot(ESMValToolDiagnostic):
     """
     Calculate the ozone diagnostics - zonal mean total column ozone vs. time.
     """
 
     name = "Ozone Diagnostics"
     slug = "ozone-lat-time"
-    base_recipe = "ref/recipe_ref_ozone_cmip7.yml"
-    version = 2
+    base_recipe = "ref/recipe_ref_ozone.yml"
+    version = 3
 
     data_requirements = toz_data_requirement
     facets = ()
@@ -184,14 +213,7 @@ def update_recipe(
         input_files: dict[SourceDatasetType, pandas.DataFrame],
     ) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files[get_cmip_source_type(input_files)])
-        dataset = recipe_variables["toz"]["additional_datasets"][0]
-        # set time range of model (CMIP6) dataset (should match observational period)
-        dataset["timerange"] = "1996/2014"
-        recipe["datasets"] = [dataset]
-        diagnostic = "lat_time_mapplot"
-        recipe["diagnostics"] = {diagnostic: recipe["diagnostics"][diagnostic]}
-        recipe["diagnostics"][diagnostic]["variables"]["toz"]["timerange"] = "1996/2014"
+        _prepare_recipe(recipe, input_files, "lat_time_mapplot", "toz", cmip7_timerange="1997/2021")
 
 
 class O3PolarCapTimeseriesSH(ESMValToolDiagnostic):
@@ -201,24 +223,30 @@ class O3PolarCapTimeseriesSH(ESMValToolDiagnostic):
 
     name = "Ozone Diagnostics"
     slug = "ozone-sh-oct"
-    base_recipe = "ref/recipe_ref_ozone_cmip7.yml"
-    version = 2
+    base_recipe = "ref/recipe_ref_ozone.yml"
+    version = 3
 
     data_requirements = toz_data_requirement
     facets = ()
     test_data_spec = toz_test_spec
     files = (
         FileDefinition(
             file_pattern="plots/polar_cap_time_series_SH/plot/timeseries_toz_SH_Oct.png",
-            dimensions={"variable_id": "toz", "statistic": "Southern Hemisphere October polar mean"},
+            dimensions={
+                "variable_id": "toz",
+                "statistic": "Southern Hemisphere October polar mean",
+            },
         ),
     )
     # dim0=0 is the model, dim0=1 contains the observational reference data.
     series = (
         SeriesDefinition(
             file_pattern="work/polar_cap_time_series_SH/plot/timeseries_toz_SH_Oct.nc",
             sel={"dim0": 0},
-            dimensions={"variable_id": "toz", "statistic": "Southern Hemisphere October polar mean"},
+            dimensions={
+                "variable_id": "toz",
+                "statistic": "Southern Hemisphere October polar mean",
+            },
             values_name="toz",
             index_name="time",
             attributes=[],
@@ -231,40 +259,46 @@ def update_recipe(
         input_files: dict[SourceDatasetType, pandas.DataFrame],
     ) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files[get_cmip_source_type(input_files)])
-        dataset = recipe_variables["toz"]["additional_datasets"][0]
-        # set model (CMIP6) time range to 1950...2014
-        dataset["timerange"] = "1950/2014"
-        recipe["datasets"] = [dataset]
-        diagnostic = "polar_cap_time_series_SH"
-        recipe["diagnostics"] = {diagnostic: recipe["diagnostics"][diagnostic]}
+        _prepare_recipe(
+            recipe,
+            input_files,
+            "polar_cap_time_series_SH",
+            "toz",
+            cmip7_timerange="1960/2021",
+        )
 
 
 class O3PolarCapTimeseriesNH(ESMValToolDiagnostic):
     """
-    Calculate the ozone diagnostics - March NH polar mean (60N-80N) time series.
+    Calculate the ozone diagnostics - March NH polar mean (60N-85N) time series.
     """
 
     name = "Ozone Diagnostics"
     slug = "ozone-nh-mar"
-    base_recipe = "ref/recipe_ref_ozone_cmip7.yml"
-    version = 2
+    base_recipe = "ref/recipe_ref_ozone.yml"
+    version = 3
 
     data_requirements = toz_data_requirement
     facets = ()
     test_data_spec = toz_test_spec
     files = (
         FileDefinition(
             file_pattern="plots/polar_cap_time_series_NH/plot/timeseries_toz_NH_MAR.png",
-            dimensions={"variable_id": "toz", "statistic": "Northern Hemisphere March polar mean"},
+            dimensions={
+                "variable_id": "toz",
+                "statistic": "Northern Hemisphere March polar mean",
+            },
         ),
     )
     # dim0=0 is the model, dim0=1 contains the observational reference data.
     series = (
         SeriesDefinition(
             file_pattern="work/polar_cap_time_series_NH/plot/timeseries_toz_NH_MAR.nc",
             sel={"dim0": 0},
-            dimensions={"variable_id": "toz", "statistic": "Northern Hemisphere March polar mean"},
+            dimensions={
+                "variable_id": "toz",
+                "statistic": "Northern Hemisphere March polar mean",
+            },
             values_name="toz",
             index_name="time",
             attributes=[],
@@ -277,17 +311,13 @@ def update_recipe(
         input_files: dict[SourceDatasetType, pandas.DataFrame],
     ) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files[get_cmip_source_type(input_files)])
-        dataset = recipe_variables["toz"]["additional_datasets"][0]
-        # set model (CMIP6) time range to 1950...2014
-        dataset["timerange"] = "1950/2014"
-        recipe["datasets"] = [dataset]
-        diagnostic = "polar_cap_time_series_NH"
-        # adjust plot title to reflect bug fix regarding northern boundary (see above)
-        recipe["diagnostics"][diagnostic]["scripts"]["plot"]["plots"]["timeseries"]["pyplot_kwargs"][
-            "title"
-        ] = "Total Column Ozone, 60-80N, March"
-        recipe["diagnostics"] = {diagnostic: recipe["diagnostics"][diagnostic]}
+        _prepare_recipe(
+            recipe,
+            input_files,
+            "polar_cap_time_series_NH",
+            "toz",
+            cmip7_timerange="1960/2021",
+        )
 
 
 class O3ZonalMeanProfiles(ESMValToolDiagnostic):
@@ -297,8 +327,8 @@ class O3ZonalMeanProfiles(ESMValToolDiagnostic):
 
     name = "Ozone Diagnostics"
     slug = "ozone-zonal"
-    base_recipe = "ref/recipe_ref_ozone_cmip7.yml"
-    version = 3
+    base_recipe = "ref/recipe_ref_ozone.yml"
+    version = 4
 
     data_requirements = (
         DataRequirement(
@@ -316,8 +346,8 @@ class O3ZonalMeanProfiles(ESMValToolDiagnostic):
             constraints=(
                 RequireTimerange(
                     group_by=("instance_id",),
-                    start=PartialDateTime(2005, 1),
-                    end=PartialDateTime(2014, 12),
+                    start=PartialDateTime(1990, 1),
+                    end=PartialDateTime(2000, 12),
                 ),
                 RequireContiguousTimerange(group_by=("instance_id",)),
             ),
@@ -340,7 +370,7 @@ class O3ZonalMeanProfiles(ESMValToolDiagnostic):
                             "variable_id": "o3",
                         },
                         remove_ensembles=True,
-                        time_span=("1996", "2015"),
+                        time_span=("1990", "2001"),
                     ),
                 ),
             ),
@@ -362,7 +392,7 @@ class O3ZonalMeanProfiles(ESMValToolDiagnostic):
                             "region": "glb",
                         },
                         remove_ensembles=True,
-                        time_span=("1980", "2009"),
+                        time_span=("1990", "2001"),
                     ),
                 ),
             ),
@@ -375,18 +405,7 @@ def update_recipe(
         input_files: dict[SourceDatasetType, pandas.DataFrame],
     ) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files[get_cmip_source_type(input_files)])
-        dataset = recipe_variables["o3"]["additional_datasets"][0]
-        # set model (CMIP6) time range to 2005...2014
-        dataset["timerange"] = "2005/2014"
-        recipe["datasets"] = [dataset]
-        diagnostic = "zonal_mean_profiles"
-        # adjust plot title to actual time range
-        recipe["diagnostics"][diagnostic]["scripts"]["plot"]["plots"]["zonal_mean_profile"]["pyplot_kwargs"][
-            "suptitle"
-        ] = "{long_name} (2005-2014 mean)"
-        recipe["diagnostics"] = {diagnostic: recipe["diagnostics"][diagnostic]}
-        recipe["diagnostics"][diagnostic]["variables"]["o3"]["timerange"] = "2005/2014"
+        _prepare_recipe(recipe, input_files, "zonal_mean_profiles", "o3")
 
 
 class O3LatMonthMapplot(ESMValToolDiagnostic):
@@ -396,8 +415,8 @@ class O3LatMonthMapplot(ESMValToolDiagnostic):
 
     name = "Ozone Diagnostics"
     slug = "ozone-annual-cycle"
-    base_recipe = "ref/recipe_ref_ozone_cmip7.yml"
-    version = 2
+    base_recipe = "ref/recipe_ref_ozone.yml"
+    version = 3
 
     data_requirements = toz_data_requirement
     facets = ()
@@ -415,11 +434,4 @@ def update_recipe(
         input_files: dict[SourceDatasetType, pandas.DataFrame],
     ) -> None:
         """Update the recipe."""
-        recipe_variables = dataframe_to_recipe(input_files[get_cmip_source_type(input_files)])
-        dataset = recipe_variables["toz"]["additional_datasets"][0]
-        # set model (CMIP6) time range to 2005...2014
-        dataset["timerange"] = "2005/2014"
-        recipe["datasets"] = [dataset]
-        diagnostic = "lat_month_mapplot"
-        recipe["diagnostics"] = {diagnostic: recipe["diagnostics"][diagnostic]}
-        recipe["diagnostics"][diagnostic]["variables"]["toz"]["timerange"] = "2005/2014"
+        _prepare_recipe(recipe, input_files, "lat_month_mapplot", "toz", cmip7_timerange="1997/2021")

packages/climate-ref-esmvaltool/src/climate_ref_esmvaltool/recipes.txt

@@ -1,14 +1,14 @@
 examples/recipe_python.yml                  ab3f06d269bb2c1368f4dc39da9bcb232fb2adb1fa556ba769e6c16294ffb4a3
 recipe_calculate_gwl_exceedance_stats.yml   9f28304b85f7043fb76eb29ee84da46e084941c9db80eaa2993c151dbf022472
-recipe_seaice_sensitivity.yml               07270818ebf04a58713dc989720cc9ace77f28d42b1ea189c3bdaf22f5d64577
+recipe_seaice_sensitivity.yml               c3166cb0b964c643040cb6ec61457446888734e8a606e319c34d739d1d1460a8
 recipe_zec.yml                              b0af7f789b7610ab3f29a6617124aa40c40866ead958204fc199eaf82863de51
-ref/recipe_enso_basicclimatology.yml        9ea7deb7ee668e39ac44618b96496d898bd82285c22dcee4fce4695e0c9fa82b
-ref/recipe_enso_characteristics.yml         34c2518b138068ac96d212910b979d54a8fcedee2c0089b5acd56a42c41dc3e4
+ref/recipe_enso_basicclimatology_CMIP7.yml  535dc2734838b852d4da888aa13648eecfadb562be70d81bff3ab7ef785f08ce
+ref/recipe_enso_characteristics_CMIP7.yml   c94be3273d8ba1e639f738969913ff0ebb7b8bd4a68ca4334c19f03d06a4128f
 ref/recipe_ref_annual_cycle_region.yml      88e0c2f07a76c4de4276547badcc74091516e03840eddb1f0e62469c67a0a86d
 ref/recipe_ref_cre_cmip7.yml                b15687ac9e3ace4fe45e81d099e5828888101a205a0e2ed07d8a57a90cbe3f55
 ref/recipe_ref_ecs.yml                      3121df2d8c27f42b11e56f918ecbce49b0c8dc831e6d6886493f64bc8cce3032
 ref/recipe_ref_fire.yml                     2ad82effaca4e742d8abe6a0aa07bb46e1e92ef0d2d240760f7623b0ba045926
-ref/recipe_ref_ozone_cmip7.yml              16b127ef957a1b029835b0cdd940530f439576f0d22c589cd472598d03568702
+ref/recipe_ref_ozone.yml                    09ded4946ecd762ba000916c9f8618120a2e6269930427ae5129da466165f3e5
 ref/recipe_ref_sea_ice_area_basic.yml       7d01a8527880663ca28284772f83a8356d9972fb4f022a4000e50a56ce044b09
 ref/recipe_ref_scatterplot.yml              b99d1736e16256d161847b025811d7088ad9f892d4887fb009fa99c4079135a0
 ref/recipe_ref_tcr.yml                      443dbd46f5ef13ea187a1acdb6e70ecfa7669e0ef4b21392e9a00b5d0c821ef5

packages/climate-ref-esmvaltool/tests/test-data/ozone-annual-cycle/cmip6/manifest.json

@@ -5,7 +5,7 @@
     "output.json": "978fd3b10d7e7ff386b4c39dbdc1f33ef5509fcb3afd3759132eee6524fc1b60",
     "series.json": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570"
   },
-  "diagnostic_version": 2,
+  "diagnostic_version": 3,
   "native": {
     "diagnostic.json": {
       "sha256": "e52b1e19e1ed5a2a046bb8aa01c004240305fabfe69c4ffb4e64898b1b6bbb84",
@@ -16,24 +16,24 @@
       "size": 14653
     },
     "executions/recipe/plots/lat_month_mapplot/plot/hovmoeller_anncyc_vs_lat_or_lon_toz_LatTime_Map.png": {
-      "sha256": "ee752b39ef169209ca1f42ef14afbab44e0d662d01e902d33c3f81ba073957ae",
-      "size": 78392
+      "sha256": "08a4045efe3698f22ce24822a0323938870f739948221dcd33363db2eba6c40a",
+      "size": 77763
     },
     "executions/recipe/run/lat_month_mapplot/plot/diagnostic_provenance.yml": {
-      "sha256": "6de2cf0191236dbd4dbd811bcf7fd0f3cc5f5e18070c6488055d57dc8a3f380d",
+      "sha256": "a7b06da868a1351b59aa66f895e7f818d4acb78fa68dabdb1c0230d2a86bb112",
       "size": 1979
     },
     "executions/recipe/work/lat_month_mapplot/plot/hovmoeller_anncyc_vs_lat_or_lon_toz_LatTime_Map_bottom.nc": {
-      "sha256": "a01a5275fdd301d86235adeea1168549bb4a2441337740be570577744520035a",
-      "size": 27463
+      "sha256": "468936de2c6eb55ebaf1261c10d93cfea7b51259293f63b3429c272d7fd288ac",
+      "size": 27743
     },
     "executions/recipe/work/lat_month_mapplot/plot/hovmoeller_anncyc_vs_lat_or_lon_toz_LatTime_Map_top_left.nc": {
-      "sha256": "1a1dc0e8761b49a292f326535a61eed572a955391bd5bf7a838a55a502c11289",
-      "size": 27463
+      "sha256": "aa0a5e0b2a07ba401d0ad28177e273f514368aa3db3e805e6bd6f22768f6b134",
+      "size": 27705
     },
     "executions/recipe/work/lat_month_mapplot/plot/hovmoeller_anncyc_vs_lat_or_lon_toz_LatTime_Map_top_right.nc": {
-      "sha256": "199c25186c50de2e85efd88da5a6ca01c1305450844367691864816c7640aea8",
-      "size": 24162
+      "sha256": "214b7a6a9fb1031d64dde57c1e3f232fa120dcb53c605be9910bc8e9ae7ab537",
+      "size": 24002
     },
     "output.json": {
       "sha256": "1a25bbce94af301cc8243d77eeae19bfa1813e136f4cf58c179254f0a047a88a",
@@ -45,5 +45,5 @@
     }
   },
   "schema": 2,
-  "test_case_version": 2
+  "test_case_version": 3
 }