feat(mapper): Set publication date based on resource type

cds_migrator_kit/rdm/records/transform/entities/record.py

@@ -228,6 +228,7 @@ def _verify_publication_date(self, raw_dump_entry, dojson_entry):
         if not raw_dump_entry.get("files") and not (
             dojson_entry.get("status_week_date")
             or dojson_entry.get("publication_date")
+            or dojson_entry.get("preprint_date")
         ):
             raise ManualImportRequired(
                 message="Record missing publication date",

cds_migrator_kit/rdm/records/transform/mappers/metadata.py

@@ -60,15 +60,77 @@ def map_value(self, ctx):
         return title
 
 
+def _date_precision(date_str):
+    """Return how granular a normalized date string is (year=1, month=2, day=3)."""
+    if not date_str:
+        return 0
+    return len(date_str.split("-"))
+
+
+def _is_more_accurate(candidate, current):
+    """Return True if `candidate` has finer granularity than `current`."""
+    return _date_precision(candidate) > _date_precision(current)
+
+
 class PublicationDateMapper(FieldMapper):
-    """Maps publication_date, falling back to status week or file creation date."""
+    """Maps publication_date, preferring 260 (article) or 269 (preprint).
+
+    - resource_type == "publication-article": publication_date (260)
+      always wins; preprint_date (269), if present, becomes a secondary
+      "submitted"/"preprint" entry in `dates`.
+    - any other resource_type: preprint_date (269) wins when present,
+      unless publication_date (260) is also present and at least as
+      accurate (day > month > year) - in that case publication_date wins
+      instead. Whichever one loses, if present, becomes a secondary entry
+      in `dates` ("available"/"published" for publication_date,
+      "submitted"/"preprint" for preprint_date).
+
+    Falls back to status week or file creation date when neither applies.
+    """
 
     id = "publication_date"
 
     def map_value(self, ctx):
         """Return publication_date, requiring at least one date source."""
         dojson_entry = ctx.dojson_entry
         pub_date = dojson_entry.get("publication_date")
+        # `preprint_date` is bookkeeping produced by the 269 rules (see
+        # xml_processing/rules/base.py) - drop it before it reaches the
+        # final record.
+        preprint_date = dojson_entry.pop("preprint_date", None)
+        resource_type = (dojson_entry.get("resource_type") or {}).get("id")
+
+        if resource_type == "publication-article":
+            if preprint_date:
+                dojson_entry.setdefault("dates", []).append(
+                    {
+                        "date": preprint_date,
+                        "type": {"id": "submitted"},
+                        "description": "preprint",
+                    }
+                )
+        elif preprint_date:
+            if pub_date and not _is_more_accurate(preprint_date, pub_date):
+                # publication_date is present and at least as accurate -
+                # keep it, preprint_date becomes the secondary entry.
+                dojson_entry.setdefault("dates", []).append(
+                    {
+                        "date": preprint_date,
+                        "type": {"id": "submitted"},
+                        "description": "preprint",
+                    }
+                )
+            else:
+                if pub_date:
+                    dojson_entry.setdefault("dates", []).append(
+                        {
+                            "date": pub_date,
+                            "type": {"id": "available"},
+                            "description": "published",
+                        }
+                    )
+                pub_date = preprint_date
+
         created = dojson_entry.get("status_week_date")
         files = ctx.raw_dump_entry["files"]
         if not (pub_date or created or files):

cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py

@@ -1026,7 +1026,6 @@ def imprint_info(self, key, value):
     if publication_date_str:
         try:
             publication_date = normalize(publication_date_str)
-
             return publication_date
         except (ParserError, TypeError) as e:
             raise UnexpectedValue(
@@ -1060,7 +1059,7 @@ def imprint_info(self, key, value):
         try:
             publication_date = normalize(publication_date_str)
 
-            self["publication_date"] = publication_date
+            self["preprint_date"] = publication_date
         except (ParserError, TypeError) as e:
             raise UnexpectedValue(
                 field=key,

cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py

@@ -353,7 +353,7 @@ def imprint_dates(self, key, value):
                     "type": {"id": "created"},
                 }
             )
-        self["publication_date"] = normalize(pub)
+        self["preprint_date"] = normalize(pub)
     except (ParserError, TypeError):
         raise UnexpectedValue(
             field=key,
@@ -410,7 +410,7 @@ def translated_description(self, key, value):
 def imprint_info(self, key, value):
     """Translates publication_date field."""
     if key.startswith("260"):
-        base_publication_imprint_info(self, key, value)
+        return base_publication_imprint_info(self, key, value)
     else:
         publication_date_str = value.get("a")
         if publication_date_str:

cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py

@@ -293,7 +293,7 @@ def journal(self, key, value):
             raise UnexpectedValue("Journal fields already set", field=key, value=value)
         journal_fields["pages"] = StringValue(value.get("c", "")).parse()
 
-    pub_date = self.get("publication_date")
+    pub_date = self.get("publication_date") or self.get("preprint_date")
     # if we only have 773 in the record and no other journal fields,
     # it is not journal date
     if not is_journal_year and "y" in value:

tests/cds-rdm/conftest.py

@@ -1015,6 +1015,26 @@ def date_type_v(app, date_type):
         },
     )
 
+    vocabulary_service.create(
+        system_identity,
+        {
+            "id": "submitted",
+            "props": {"datacite": "Submitted"},
+            "title": {"en": "Submitted"},
+            "type": "datetypes",
+        },
+    )
+
+    vocabulary_service.create(
+        system_identity,
+        {
+            "id": "available",
+            "props": {"datacite": "Available"},
+            "title": {"en": "Available"},
+            "type": "datetypes",
+        },
+    )
+
     return vocab
 
 

tests/cds-rdm/test_base_rules.py

@@ -13,6 +13,7 @@
 from cds_migrator_kit.errors import UnexpectedValue
 from cds_migrator_kit.rdm.records.transform.xml_processing.rules.base import (
     custom_fields_693,
+    imprint_info,
     normalize,
     note,
     recid,
@@ -397,3 +398,41 @@ def test_note_whitespace_only_ignored(self):
         record = {}
         with pytest.raises(IgnoreKey):
             note(record, "595__", {"a": "   "})
+
+
+class TestImprintInfo269:
+    """Test the 269 (preprint) imprint_info function from base.py.
+
+    It never sets `publication_date` itself - it stashes the parsed date
+    under `record["preprint_date"]`, to be reconciled with a possible 260
+    (article) date once resource_type is known, in PublicationDateMapper.
+    """
+
+    def test_imprint_info_269_full(self):
+        """Test full imprint info with place, publisher, and date."""
+        record = {"custom_fields": {}}
+        with pytest.raises(IgnoreKey):
+            imprint_info(record, "269__", {"a": "Geneva.", "b": "CERN", "c": "2021"})
+        assert record["preprint_date"] == "2021"
+        assert record["publisher"] == "CERN"
+        assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva"
+
+    def test_imprint_info_269_publisher_not_overwritten(self):
+        """Test that existing publisher is not overwritten."""
+        record = {"custom_fields": {}, "publisher": "Existing Publisher"}
+        with pytest.raises(IgnoreKey):
+            imprint_info(record, "269__", {"b": "CERN", "c": "2021"})
+        assert record["publisher"] == "Existing Publisher"
+
+    def test_imprint_info_269_no_date_ignored(self):
+        """Test that missing date raises IgnoreKey and sets no date."""
+        record = {"custom_fields": {}}
+        with pytest.raises(IgnoreKey):
+            imprint_info(record, "269__", {"a": "Geneva", "b": "CERN"})
+        assert "preprint_date" not in record
+
+    def test_imprint_info_269_invalid_date_raises_error(self):
+        """Test that invalid date raises error."""
+        record = {"custom_fields": {}}
+        with pytest.raises(UnexpectedValue):
+            imprint_info(record, "269__", {"c": "not-a-valid-date"})

tests/cds-rdm/test_full_migration.py

@@ -95,7 +95,18 @@ def suite_multi_field(record):
             },
         },
     ]
+    # resource_type is not an article), and 269 (preprint_date) is more
+    # precise than 260 (publication_date), so it wins - 260 becomes the
+    # secondary dates entry, and preprint_date never leaks into the record.
     assert dict_rec["metadata"]["publication_date"] == "2018-08-02"
+    assert dict_rec["metadata"]["dates"] == [
+        {
+            "date": "2018",
+            "type": {"id": "available", "title": {"en": "Available"}},
+            "description": "published",
+        }
+    ]
+    assert "preprint_date" not in dict_rec["metadata"]
     assert (
         dict_rec["metadata"]["title"] == "FLUKA and ActiWiz benchmark on BDF materials"
     )

tests/cds-rdm/test_it_migration.py

@@ -521,7 +521,7 @@ def test_imprint_dates_basic(self):
         record = {}
         with pytest.raises(IgnoreKey):
             imprint_dates(record, "269__", {"c": "2021"})
-        assert record["publication_date"] == "2021"
+        assert record["preprint_date"] == "2021"
 
     def test_imprint_dates_with_place(self):
         """Test imprint place is added."""
@@ -549,7 +549,7 @@ def test_imprint_dates_with_question_mark(self):
         record = {}
         with pytest.raises(IgnoreKey):
             imprint_dates(record, "269__", {"c": "2021?"})
-        assert record["publication_date"] == "2021"
+        assert record["preprint_date"] == "2021"
         assert len(record["dates"]) == 1
         assert record["dates"][0]["type"]["id"] == "created"
         assert "indeterminate" in record["dates"][0]["description"]

tests/cds-rdm/test_it_override_delegation.py

@@ -100,8 +100,8 @@ def test_imprint_dates_with_269_does_not_delegate_to_693(self):
         record = {}
         with pytest.raises(IgnoreKey):
             imprint_dates(record, "269__", {"c": "2021"})
-        # Should have publication_date but no experiments from 693
-        assert record["publication_date"] == "2021"
+        # Should have the preprint date but no experiments from 693
+        assert record["preprint_date"] == "2021"
         assert "cern:experiments" not in record.get("custom_fields", {})
 
     def test_imprint_dates_269_with_place(self):
@@ -110,7 +110,7 @@ def test_imprint_dates_269_with_place(self):
         with pytest.raises(IgnoreKey):
             imprint_dates(record, "269__", {"a": "Geneva.", "c": "2021"})
         assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva"
-        assert record["publication_date"] == "2021"
+        assert record["preprint_date"] == "2021"
 
     def test_imprint_dates_269_with_publisher(self):
         """Test that 269__ field sets publisher when not already set."""
@@ -124,7 +124,7 @@ def test_imprint_dates_933_field(self):
         record = {}
         with pytest.raises(IgnoreKey):
             imprint_dates(record, "933__", {"c": "2022"})
-        assert record["publication_date"] == "2022"
+        assert record["preprint_date"] == "2022"
 
 
 class TestConferenceTitleDelegation:
@@ -168,11 +168,12 @@ def test_imprint_info_with_260_delegates_to_base(self):
         """Test that 260__ field delegates to base_publication_imprint_info."""
         # Initialize custom_fields as base function expects it
         record = {"custom_fields": {}}
-        # Note: IT function calls base but doesn't return its value
-        # This might be a bug, but we test the actual behavior
+        # base_publication_imprint_info's return value must be propagated
+        # by the IT wrapper (previously it was silently discarded).
         result = imprint_info(
             record, "260__", {"c": "2021", "a": "Geneva", "b": "CERN"}
         )
+        assert result == "2021"
         # Check that imprint fields were set by base function
         assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva"
         assert record["publisher"] == "CERN"
@@ -479,7 +480,7 @@ def test_imprint_dates_both_693_and_269(self):
         # Both should be present
         assert "ATLAS" in record["custom_fields"]["cern:experiments"]
         assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva"
-        assert record["publication_date"] == "2020"
+        assert record["preprint_date"] == "2020"
 
     def test_conference_title_and_notes_together(self):
         """Test conference title and notes are both processed."""

tests/cds-rdm/test_json_translation_rules.py

@@ -52,7 +52,8 @@ def test_migrate_sspn_record(datadir, base_app):
             ],
             "title": "Deep Learning Methods for Particle Reconstruction in the HGCal",
             "publisher": "CERN",
-            "publication_date": "2017-06-24",
+            "publication_date": "2017",
+            "preprint_date": "2017-06-24",
             "description": "The High Granularity end-cap Calorimeter is part of the phase-2 CMS upgrade (see Figure \\ref{fig:cms})\\cite{Contardo:2020886}. It's goal it to provide measurements of high resolution in time, space and energy. Given such measurements, the purpose of this work is to discuss the use of Deep Neural Networks for the task of particle and trajectory reconstruction, identification and energy estimation, during my participation in the CERN Summer Students Program.",
             "internal_notes": [],
             "subjects": [
@@ -154,7 +155,8 @@ def test_migrate_record_all_fields(datadir, base_app):
                 }
             ],
             "publisher": "CERN",
-            "publication_date": "2018-08-02",
+            "publication_date": "2018",
+            "preprint_date": "2018-08-02",
             "description": "This note describes the FLUKA and Actiwiz benchmark with gamma spectroscopy results of various material samples, which were irradiated during the Beam Dump Facility (BDF) prototype target test in the North Area of the Super Proton Synchrotron (SPS) at CERN. The samples represent most of the materials that will be used in the construction of the BDF facility.",
             "internal_notes": [{"note": "Comments submitted after 31-08-2021 10:41"}],
             "subjects": [