From 83a3b9ca72739eed664a74d34e067a7f23656313 Mon Sep 17 00:00:00 2001 From: Saksham Date: Thu, 3 Sep 2026 17:14:37 +0200 Subject: [PATCH] feat(mapper): Set publication date based on resource type --- .../rdm/records/transform/entities/record.py | 1 + .../rdm/records/transform/mappers/metadata.py | 64 ++++++++++++++++++- .../transform/xml_processing/rules/base.py | 3 +- .../transform/xml_processing/rules/it.py | 4 +- .../xml_processing/rules/research.py | 2 +- tests/cds-rdm/conftest.py | 20 ++++++ tests/cds-rdm/test_base_rules.py | 39 +++++++++++ tests/cds-rdm/test_full_migration.py | 11 ++++ tests/cds-rdm/test_it_migration.py | 4 +- tests/cds-rdm/test_it_override_delegation.py | 15 +++-- tests/cds-rdm/test_json_translation_rules.py | 6 +- 11 files changed, 152 insertions(+), 17 deletions(-) diff --git a/cds_migrator_kit/rdm/records/transform/entities/record.py b/cds_migrator_kit/rdm/records/transform/entities/record.py index 9fc2d9af..ef2c601b 100644 --- a/cds_migrator_kit/rdm/records/transform/entities/record.py +++ b/cds_migrator_kit/rdm/records/transform/entities/record.py @@ -228,6 +228,7 @@ def _verify_publication_date(self, raw_dump_entry, dojson_entry): if not raw_dump_entry.get("files") and not ( dojson_entry.get("status_week_date") or dojson_entry.get("publication_date") + or dojson_entry.get("preprint_date") ): raise ManualImportRequired( message="Record missing publication date", diff --git a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py index a9f13362..acd6b12f 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/metadata.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/metadata.py @@ -60,8 +60,33 @@ def map_value(self, ctx): return title +def _date_precision(date_str): + """Return how granular a normalized date string is (year=1, month=2, day=3).""" + if not date_str: + return 0 + return len(date_str.split("-")) + + +def _is_more_accurate(candidate, current): + """Return True if `candidate` has finer granularity than `current`.""" + return _date_precision(candidate) > _date_precision(current) + + class PublicationDateMapper(FieldMapper): - """Maps publication_date, falling back to status week or file creation date.""" + """Maps publication_date, preferring 260 (article) or 269 (preprint). + + - resource_type == "publication-article": publication_date (260) + always wins; preprint_date (269), if present, becomes a secondary + "submitted"/"preprint" entry in `dates`. + - any other resource_type: preprint_date (269) wins when present, + unless publication_date (260) is also present and at least as + accurate (day > month > year) - in that case publication_date wins + instead. Whichever one loses, if present, becomes a secondary entry + in `dates` ("available"/"published" for publication_date, + "submitted"/"preprint" for preprint_date). + + Falls back to status week or file creation date when neither applies. + """ id = "publication_date" @@ -69,6 +94,43 @@ def map_value(self, ctx): """Return publication_date, requiring at least one date source.""" dojson_entry = ctx.dojson_entry pub_date = dojson_entry.get("publication_date") + # `preprint_date` is bookkeeping produced by the 269 rules (see + # xml_processing/rules/base.py) - drop it before it reaches the + # final record. + preprint_date = dojson_entry.pop("preprint_date", None) + resource_type = (dojson_entry.get("resource_type") or {}).get("id") + + if resource_type == "publication-article": + if preprint_date: + dojson_entry.setdefault("dates", []).append( + { + "date": preprint_date, + "type": {"id": "submitted"}, + "description": "preprint", + } + ) + elif preprint_date: + if pub_date and not _is_more_accurate(preprint_date, pub_date): + # publication_date is present and at least as accurate - + # keep it, preprint_date becomes the secondary entry. + dojson_entry.setdefault("dates", []).append( + { + "date": preprint_date, + "type": {"id": "submitted"}, + "description": "preprint", + } + ) + else: + if pub_date: + dojson_entry.setdefault("dates", []).append( + { + "date": pub_date, + "type": {"id": "available"}, + "description": "published", + } + ) + pub_date = preprint_date + created = dojson_entry.get("status_week_date") files = ctx.raw_dump_entry["files"] if not (pub_date or created or files): diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index 614403c2..fe1986f4 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -1026,7 +1026,6 @@ def imprint_info(self, key, value): if publication_date_str: try: publication_date = normalize(publication_date_str) - return publication_date except (ParserError, TypeError) as e: raise UnexpectedValue( @@ -1060,7 +1059,7 @@ def imprint_info(self, key, value): try: publication_date = normalize(publication_date_str) - self["publication_date"] = publication_date + self["preprint_date"] = publication_date except (ParserError, TypeError) as e: raise UnexpectedValue( field=key, diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py index dafc780c..a35d7946 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/it.py @@ -353,7 +353,7 @@ def imprint_dates(self, key, value): "type": {"id": "created"}, } ) - self["publication_date"] = normalize(pub) + self["preprint_date"] = normalize(pub) except (ParserError, TypeError): raise UnexpectedValue( field=key, @@ -410,7 +410,7 @@ def translated_description(self, key, value): def imprint_info(self, key, value): """Translates publication_date field.""" if key.startswith("260"): - base_publication_imprint_info(self, key, value) + return base_publication_imprint_info(self, key, value) else: publication_date_str = value.get("a") if publication_date_str: diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index 4bc21a09..c04ccee8 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -293,7 +293,7 @@ def journal(self, key, value): raise UnexpectedValue("Journal fields already set", field=key, value=value) journal_fields["pages"] = StringValue(value.get("c", "")).parse() - pub_date = self.get("publication_date") + pub_date = self.get("publication_date") or self.get("preprint_date") # if we only have 773 in the record and no other journal fields, # it is not journal date if not is_journal_year and "y" in value: diff --git a/tests/cds-rdm/conftest.py b/tests/cds-rdm/conftest.py index dd169fc3..8cf329e7 100644 --- a/tests/cds-rdm/conftest.py +++ b/tests/cds-rdm/conftest.py @@ -1015,6 +1015,26 @@ def date_type_v(app, date_type): }, ) + vocabulary_service.create( + system_identity, + { + "id": "submitted", + "props": {"datacite": "Submitted"}, + "title": {"en": "Submitted"}, + "type": "datetypes", + }, + ) + + vocabulary_service.create( + system_identity, + { + "id": "available", + "props": {"datacite": "Available"}, + "title": {"en": "Available"}, + "type": "datetypes", + }, + ) + return vocab diff --git a/tests/cds-rdm/test_base_rules.py b/tests/cds-rdm/test_base_rules.py index 91e14492..230dd154 100644 --- a/tests/cds-rdm/test_base_rules.py +++ b/tests/cds-rdm/test_base_rules.py @@ -13,6 +13,7 @@ from cds_migrator_kit.errors import UnexpectedValue from cds_migrator_kit.rdm.records.transform.xml_processing.rules.base import ( custom_fields_693, + imprint_info, normalize, note, recid, @@ -397,3 +398,41 @@ def test_note_whitespace_only_ignored(self): record = {} with pytest.raises(IgnoreKey): note(record, "595__", {"a": " "}) + + +class TestImprintInfo269: + """Test the 269 (preprint) imprint_info function from base.py. + + It never sets `publication_date` itself - it stashes the parsed date + under `record["preprint_date"]`, to be reconciled with a possible 260 + (article) date once resource_type is known, in PublicationDateMapper. + """ + + def test_imprint_info_269_full(self): + """Test full imprint info with place, publisher, and date.""" + record = {"custom_fields": {}} + with pytest.raises(IgnoreKey): + imprint_info(record, "269__", {"a": "Geneva.", "b": "CERN", "c": "2021"}) + assert record["preprint_date"] == "2021" + assert record["publisher"] == "CERN" + assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" + + def test_imprint_info_269_publisher_not_overwritten(self): + """Test that existing publisher is not overwritten.""" + record = {"custom_fields": {}, "publisher": "Existing Publisher"} + with pytest.raises(IgnoreKey): + imprint_info(record, "269__", {"b": "CERN", "c": "2021"}) + assert record["publisher"] == "Existing Publisher" + + def test_imprint_info_269_no_date_ignored(self): + """Test that missing date raises IgnoreKey and sets no date.""" + record = {"custom_fields": {}} + with pytest.raises(IgnoreKey): + imprint_info(record, "269__", {"a": "Geneva", "b": "CERN"}) + assert "preprint_date" not in record + + def test_imprint_info_269_invalid_date_raises_error(self): + """Test that invalid date raises error.""" + record = {"custom_fields": {}} + with pytest.raises(UnexpectedValue): + imprint_info(record, "269__", {"c": "not-a-valid-date"}) diff --git a/tests/cds-rdm/test_full_migration.py b/tests/cds-rdm/test_full_migration.py index 3df27f70..ee45fa23 100644 --- a/tests/cds-rdm/test_full_migration.py +++ b/tests/cds-rdm/test_full_migration.py @@ -95,7 +95,18 @@ def suite_multi_field(record): }, }, ] + # resource_type is not an article), and 269 (preprint_date) is more + # precise than 260 (publication_date), so it wins - 260 becomes the + # secondary dates entry, and preprint_date never leaks into the record. assert dict_rec["metadata"]["publication_date"] == "2018-08-02" + assert dict_rec["metadata"]["dates"] == [ + { + "date": "2018", + "type": {"id": "available", "title": {"en": "Available"}}, + "description": "published", + } + ] + assert "preprint_date" not in dict_rec["metadata"] assert ( dict_rec["metadata"]["title"] == "FLUKA and ActiWiz benchmark on BDF materials" ) diff --git a/tests/cds-rdm/test_it_migration.py b/tests/cds-rdm/test_it_migration.py index ec1b8509..54562b25 100644 --- a/tests/cds-rdm/test_it_migration.py +++ b/tests/cds-rdm/test_it_migration.py @@ -521,7 +521,7 @@ def test_imprint_dates_basic(self): record = {} with pytest.raises(IgnoreKey): imprint_dates(record, "269__", {"c": "2021"}) - assert record["publication_date"] == "2021" + assert record["preprint_date"] == "2021" def test_imprint_dates_with_place(self): """Test imprint place is added.""" @@ -549,7 +549,7 @@ def test_imprint_dates_with_question_mark(self): record = {} with pytest.raises(IgnoreKey): imprint_dates(record, "269__", {"c": "2021?"}) - assert record["publication_date"] == "2021" + assert record["preprint_date"] == "2021" assert len(record["dates"]) == 1 assert record["dates"][0]["type"]["id"] == "created" assert "indeterminate" in record["dates"][0]["description"] diff --git a/tests/cds-rdm/test_it_override_delegation.py b/tests/cds-rdm/test_it_override_delegation.py index 1ea663c9..a6dab172 100644 --- a/tests/cds-rdm/test_it_override_delegation.py +++ b/tests/cds-rdm/test_it_override_delegation.py @@ -100,8 +100,8 @@ def test_imprint_dates_with_269_does_not_delegate_to_693(self): record = {} with pytest.raises(IgnoreKey): imprint_dates(record, "269__", {"c": "2021"}) - # Should have publication_date but no experiments from 693 - assert record["publication_date"] == "2021" + # Should have the preprint date but no experiments from 693 + assert record["preprint_date"] == "2021" assert "cern:experiments" not in record.get("custom_fields", {}) def test_imprint_dates_269_with_place(self): @@ -110,7 +110,7 @@ def test_imprint_dates_269_with_place(self): with pytest.raises(IgnoreKey): imprint_dates(record, "269__", {"a": "Geneva.", "c": "2021"}) assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" - assert record["publication_date"] == "2021" + assert record["preprint_date"] == "2021" def test_imprint_dates_269_with_publisher(self): """Test that 269__ field sets publisher when not already set.""" @@ -124,7 +124,7 @@ def test_imprint_dates_933_field(self): record = {} with pytest.raises(IgnoreKey): imprint_dates(record, "933__", {"c": "2022"}) - assert record["publication_date"] == "2022" + assert record["preprint_date"] == "2022" class TestConferenceTitleDelegation: @@ -168,11 +168,12 @@ def test_imprint_info_with_260_delegates_to_base(self): """Test that 260__ field delegates to base_publication_imprint_info.""" # Initialize custom_fields as base function expects it record = {"custom_fields": {}} - # Note: IT function calls base but doesn't return its value - # This might be a bug, but we test the actual behavior + # base_publication_imprint_info's return value must be propagated + # by the IT wrapper (previously it was silently discarded). result = imprint_info( record, "260__", {"c": "2021", "a": "Geneva", "b": "CERN"} ) + assert result == "2021" # Check that imprint fields were set by base function assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" assert record["publisher"] == "CERN" @@ -479,7 +480,7 @@ def test_imprint_dates_both_693_and_269(self): # Both should be present assert "ATLAS" in record["custom_fields"]["cern:experiments"] assert record["custom_fields"]["imprint:imprint"]["place"] == "Geneva" - assert record["publication_date"] == "2020" + assert record["preprint_date"] == "2020" def test_conference_title_and_notes_together(self): """Test conference title and notes are both processed.""" diff --git a/tests/cds-rdm/test_json_translation_rules.py b/tests/cds-rdm/test_json_translation_rules.py index 38ce09f2..6f70691a 100644 --- a/tests/cds-rdm/test_json_translation_rules.py +++ b/tests/cds-rdm/test_json_translation_rules.py @@ -52,7 +52,8 @@ def test_migrate_sspn_record(datadir, base_app): ], "title": "Deep Learning Methods for Particle Reconstruction in the HGCal", "publisher": "CERN", - "publication_date": "2017-06-24", + "publication_date": "2017", + "preprint_date": "2017-06-24", "description": "The High Granularity end-cap Calorimeter is part of the phase-2 CMS upgrade (see Figure \\ref{fig:cms})\\cite{Contardo:2020886}. It's goal it to provide measurements of high resolution in time, space and energy. Given such measurements, the purpose of this work is to discuss the use of Deep Neural Networks for the task of particle and trajectory reconstruction, identification and energy estimation, during my participation in the CERN Summer Students Program.", "internal_notes": [], "subjects": [ @@ -154,7 +155,8 @@ def test_migrate_record_all_fields(datadir, base_app): } ], "publisher": "CERN", - "publication_date": "2018-08-02", + "publication_date": "2018", + "preprint_date": "2018-08-02", "description": "This note describes the FLUKA and Actiwiz benchmark with gamma spectroscopy results of various material samples, which were irradiated during the Beam Dump Facility (BDF) prototype target test in the North Area of the Super Proton Synchrotron (SPS) at CERN. The samples represent most of the materials that will be used in the construction of the BDF facility.", "internal_notes": [{"note": "Comments submitted after 31-08-2021 10:41"}], "subjects": [