From 31559c3cbcc73c679a619a1bc9d484232b7748e8 Mon Sep 17 00:00:00 2001 From: Pal Kerecsenyi Date: Thu, 27 Aug 2026 16:07:12 +0200 Subject: [PATCH 1/3] feat(north-area): add NA6* migration models --- .../rdm/records/transform/models/_config.py | 19 +++++++ .../records/transform/models/north_area.py | 39 ++++++++++++++ .../xml_processing/quality/__init__.py | 8 --- cds_migrator_kit/rdm/streams.yaml | 52 +++++++++++++++++-- setup.cfg | 5 ++ 5 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 cds_migrator_kit/rdm/records/transform/models/north_area.py delete mode 100644 cds_migrator_kit/rdm/records/transform/xml_processing/quality/__init__.py diff --git a/cds_migrator_kit/rdm/records/transform/models/_config.py b/cds_migrator_kit/rdm/records/transform/models/_config.py index 225b81fa..06a6685f 100644 --- a/cds_migrator_kit/rdm/records/transform/models/_config.py +++ b/cds_migrator_kit/rdm/records/transform/models/_config.py @@ -1,18 +1,37 @@ """Fields which we can confidently ignore in each model.""" + IGNORE_SYSTEM_KEYS = { + "0247_9", # provenance of the DOI "0248_a", "0248_p", "0248_q", "852__c", # holdings will be taken separately "852__h", + "035__h", # OAI harvest tag or timestamp + "035__d", # OAI harvest tag or timestamp + "035__m", # OAI harvest format (e.g. `marcxml`) + "035__t", # oai harvest tag + "035__u", # oai harvest tag + "035__z", # oai harvest tag "037__c", # arxiv subject "100__m", # email of contributor + "245__9", # Provenance of title + "270__m", # Contact email "300__a", # number of pages + "520__9", # Provenance of the description + "540__3", # Material of the license + "540__9", # Also material of the license + "542__3", # Also material of the license "700__m", # email of contributor + "773__t", # from SIS: can be ignored + "773__0", # from SIS: can be ignored + "773__o", # from SIS: can be ignored + "773__x", # INSPIRE publication note "8564_8", # file id "8564_s", # bibdoc id "8564_x", # icon thumbnails sizes "8564_y", # file description - done by files dump + "8564_z", # Websubmit "stamp" (migrated as file metadata) "916__y", # year, redundant value "937__c", # last modified by "937__s", # last modification date diff --git a/cds_migrator_kit/rdm/records/transform/models/north_area.py b/cds_migrator_kit/rdm/records/transform/models/north_area.py new file mode 100644 index 00000000..ca58ac86 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/models/north_area.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM North Area models (NA61-64).""" + +from cds_migrator_kit.rdm.records.transform.models._config import IGNORE_SYSTEM_KEYS +from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( + rdm_base_publication_model, +) +from cds_migrator_kit.transform.overdo import CdsOverdo + + +class NorthAreaModel(CdsOverdo): + """Translation model for North Area experiments.""" + + __query__ = """693__.e:"NA61" OR 693__.e:"SHINE NA61" OR 693__.e:"NA62" OR 693__.e:"NA63" OR 693__.e:"NA64" + -980__:THESIS -980__:DELETED -980__:HIDDEN -980__:MIGRATED -980__:DUMMY""" + + __ignore_keys__ = IGNORE_SYSTEM_KEYS | { + "270__m", # Email of contact person + "500__9", # Provenance of the note + "903__s", # 'public' + "905__m", # Submitter email address + "995__a", # "Inspire" + } + + _default_fields = { + "custom_fields": {}, + } + + +north_area_model = NorthAreaModel( + bases=(rdm_base_publication_model,), + entry_point_group="cds_migrator_kit.migrator.rules.north_area", +) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/quality/__init__.py b/cds_migrator_kit/rdm/records/transform/xml_processing/quality/__init__.py deleted file mode 100644 index 5b2e0c6d..00000000 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/quality/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2022 CERN. -# -# CDS-RDM is free software; you can redistribute it and/or modify it under -# the terms of the MIT License; see LICENSE file for more details. - -"""RDM records transform quality utilities.""" diff --git a/cds_migrator_kit/rdm/streams.yaml b/cds_migrator_kit/rdm/streams.yaml index 06be0344..98ffd313 100644 --- a/cds_migrator_kit/rdm/streams.yaml +++ b/cds_migrator_kit/rdm/streams.yaml @@ -377,7 +377,7 @@ records: data_dir: cds_migrator_kit/rdm/data/isolde plots: true create_inclusion_request: true - preferred_model: isolde # set to resolve MultipleModelsMatched as warning instead of error + preferred_model: isolde # set to resolve MultipleModelsMatched as warning instead of error extract: dirpath: cds_migrator_kit/rdm/data/isolde/dump/ transform: @@ -664,17 +664,61 @@ records: missing_users: cds_migrator_kit/rdm/data/users communities_ids: - "88a105fe-4713-493b-b555-6ab398599d21" + na61: + data_dir: cds_migrator_kit/rdm/data/north_area/na61 + plots: true + create_inclusion_request: true + extract: + dirpath: cds_migrator_kit/rdm/data/north_area/na61/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/north_area/na61/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "" + na62: + data_dir: cds_migrator_kit/rdm/data/north_area/na62 + plots: true + create_inclusion_request: true + extract: + dirpath: cds_migrator_kit/rdm/data/north_area/na62/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/north_area/na62/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "" + na63: + data_dir: cds_migrator_kit/rdm/data/north_area/na63 + plots: true + create_inclusion_request: true + extract: + dirpath: cds_migrator_kit/rdm/data/north_area/na63/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/north_area/na63/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "" + na64: + data_dir: cds_migrator_kit/rdm/data/north_area/na64 + plots: true + create_inclusion_request: true + extract: + dirpath: cds_migrator_kit/rdm/data/north_area/na64/dump/ + transform: + files_dump_dir: cds_migrator_kit/rdm/data/north_area/na64/files/ + missing_users: cds_migrator_kit/rdm/data/users + communities_ids: + - "" comments: faser-drafts: dir_path: /migration/faser-drafts/comments/ reviewers: - - {group: faser-all} + - { group: faser-all } faser: dir_path: /migration/faser/comments/ reviewers: - - {group: faser-all} + - { group: faser-all } faser-ep: dir_path: /migration/faser-ep/comments/ reviewers: - - {group: faser-all} + - { group: faser-all } diff --git a/setup.cfg b/setup.cfg index 8d5e37ec..e8d71660 100644 --- a/setup.cfg +++ b/setup.cfg @@ -99,6 +99,7 @@ cds_migrator_kit.migrator.models = technical_support = cds_migrator_kit.rdm.records.transform.models.technical_support:technical_support_model ab_dep = cds_migrator_kit.rdm.records.transform.models.ab_dep:ab_dep_model isolde = cds_migrator_kit.rdm.records.transform.models.isolde:isolde_model + north_area = cds_migrator_kit.rdm.records.transform.models.north_area:north_area_model cds_migrator_kit.migrator.rules.base = base = cds_migrator_kit.transform.xml_processing.rules.base @@ -220,6 +221,10 @@ cds_migrator_kit.migrator.rules.isolde = base = cds_migrator_kit.transform.xml_processing.rules.base base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research +cds_migrator_kit.migrator.rules.north_area = + base = cds_migrator_kit.transform.xml_processing.rules.base + base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base + publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research cds_migrator_kit.migrator.rules.people = people = cds_migrator_kit.rdm.users.transform.xml_processing.rules.people invenio_pidstore.minters = From c1423738d9e404d142347daf639dcae69ba2f2f0 Mon Sep 17 00:00:00 2001 From: Pal Kerecsenyi Date: Thu, 27 Aug 2026 16:07:27 +0200 Subject: [PATCH 2/3] feat(transform): allow matching experiments to vocabulary aliases --- .../records/transform/mappers/vocabulary.py | 8 +++++ .../rdm/records/transform/transform.py | 36 ++++++++++--------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py b/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py index ac22daed..f711d155 100644 --- a/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py +++ b/cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """YAML-backed vocabulary lookup used by custom_fields mappers.""" + from pathlib import Path import yaml @@ -48,9 +49,16 @@ def _load(filepath): for entry in entries: entry_id = entry["id"] lookup[entry_id.lower()] = entry_id + title = entry.get("title", {}).get("en", "") if title and title.lower() != entry_id.lower(): lookup[title.lower()] = entry_id + + # For now, I think this only supports one string value. If we make it support more in the future (e.g. via CSV) we + # will need to update this. + entry_aliases = entry.get("props", {}).get("aliases") + if entry_aliases: + lookup[entry_aliases.strip().lower()] = entry_id return lookup def get(self, term, vocab_type): diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index a965e4b8..415177d8 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM transform step module.""" + import logging import re from collections import OrderedDict @@ -18,8 +19,10 @@ from invenio_access.permissions import system_identity from invenio_pidstore.models import PersistentIdentifier, PIDStatus from invenio_rdm_migrator.logging import Logger -from invenio_rdm_records.proxies import current_rdm_records_service, \ - current_record_communities_service +from invenio_rdm_records.proxies import ( + current_rdm_records_service, + current_record_communities_service, +) from sqlalchemy.exc import NoResultFound from cds_migrator_kit.errors import ( @@ -33,8 +36,9 @@ from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry from cds_migrator_kit.rdm.records.transform.entities.request import RecordRequest -from cds_migrator_kit.rdm.records.transform.transform_versions import \ - RecordVersionsTransform +from cds_migrator_kit.rdm.records.transform.transform_versions import ( + RecordVersionsTransform, +) from cds_migrator_kit.transform.dumper import CDSRecordDump from cds_migrator_kit.transform.errors import LossyConversion @@ -115,7 +119,8 @@ def _transform_xml_to_json(self, raw_dump_entry): if dump.multiple_models_warning: w = dump.multiple_models_warning recid = raw_dump_entry.get("recid") or raw_dump_entry.get("record", {}).get( - "recid") + "recid" + ) matched = re.findall(r"\['(\w+)',", w.message or "") self.migration_logger.add_information( str(recid), @@ -197,12 +202,12 @@ def _transform(self, raw_dump_entry) -> Optional[MigrationEntry]: ) except ( - LossyConversion, - RestrictedFileDetected, - UnexpectedValue, - ManualImportRequired, - MissingRequiredField, - MultipleModelsMatched, + LossyConversion, + RestrictedFileDetected, + UnexpectedValue, + ManualImportRequired, + MissingRequiredField, + MultipleModelsMatched, ) as e: migration_logger.add_log(e, record=raw_dump_entry) @@ -266,9 +271,7 @@ def _existing_record_is_restricted(self, record_id): MARCXML, since access restrictions may have been changed in RDM after migration and the legacy record data would be stale. """ - record = current_rdm_records_service.read_latest( - system_identity, id_=record_id - ) + record = current_rdm_records_service.read_latest(system_identity, id_=record_id) access = record.data.get("access", {}) return access.get("record") != "public" or access.get("files") != "public" @@ -328,8 +331,9 @@ def run(self, entries): recid, { "message": "Record already migrated, skipping," - " added existing {} to communities {}".format( - recid, self.communities_ids), + " added existing {} to communities {}".format( + recid, self.communities_ids + ), "value": recid, }, ) From 88dc92e026a68b389263fed2350ac0aaed6f4a8b Mon Sep 17 00:00:00 2001 From: Pal Kerecsenyi Date: Thu, 3 Sep 2026 08:39:37 +0200 Subject: [PATCH 3/3] feat(users): only load user if it doesn't already exist --- cds_migrator_kit/users/load.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cds_migrator_kit/users/load.py b/cds_migrator_kit/users/load.py index fc7edf50..736c8a52 100644 --- a/cds_migrator_kit/users/load.py +++ b/cds_migrator_kit/users/load.py @@ -6,6 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM transform step module.""" + import csv import json import logging @@ -16,7 +17,6 @@ from invenio_accounts.models import User from invenio_db import db from invenio_rdm_migrator.load.base import Load -from sqlalchemy.exc import NoResultFound cli_logger = logging.getLogger("migrator") @@ -78,13 +78,11 @@ def _find_or_create_by_email(self, email): """Fetch or create a user account by email.""" if not email: return - try: - user = User.query.filter_by(email=email).one() + user = User.query.filter(db.func.lower(User.email) == email.lower()).first() + if user is not None: return user.id - except NoResultFound: - if not self.dry_run: - user_id = self._create_owner(email) - return user_id + if not self.dry_run: + return self._create_owner(email) def _parse_reviewer_name(self, name): """Split a "Family name, Given name" or "Given name Family name"