feat(north-area): add NA6* migration models

cds_migrator_kit/rdm/records/transform/mappers/vocabulary.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """YAML-backed vocabulary lookup used by custom_fields mappers."""
+
 from pathlib import Path
 
 import yaml
@@ -48,9 +49,16 @@ def _load(filepath):
         for entry in entries:
             entry_id = entry["id"]
             lookup[entry_id.lower()] = entry_id
+
             title = entry.get("title", {}).get("en", "")
             if title and title.lower() != entry_id.lower():
                 lookup[title.lower()] = entry_id
+
+            # For now, I think this only supports one string value. If we make it support more in the future (e.g. via CSV) we
+            # will need to update this.
+            entry_aliases = entry.get("props", {}).get("aliases")
+            if entry_aliases:
+                lookup[entry_aliases.strip().lower()] = entry_id
         return lookup
 
     def get(self, term, vocab_type):

cds_migrator_kit/rdm/records/transform/models/_config.py

@@ -1,18 +1,37 @@
 """Fields which we can confidently ignore in each model."""
+
 IGNORE_SYSTEM_KEYS = {
+    "0247_9",  # provenance of the DOI
     "0248_a",
     "0248_p",
     "0248_q",
     "852__c",  # holdings will be taken separately
     "852__h",
+    "035__h",  # OAI harvest tag or timestamp
+    "035__d",  # OAI harvest tag or timestamp
+    "035__m",  # OAI harvest format (e.g. `marcxml`)
+    "035__t",  # oai harvest tag
+    "035__u",  # oai harvest tag
+    "035__z",  # oai harvest tag
     "037__c",  # arxiv subject
     "100__m",  # email of contributor
+    "245__9",  # Provenance of title
+    "270__m",  # Contact email
     "300__a",  # number of pages
+    "520__9",  # Provenance of the description
+    "540__3",  # Material of the license
+    "540__9",  # Also material of the license
+    "542__3",  # Also material of the license
     "700__m",  # email of contributor
+    "773__t",  # from SIS: can be ignored
+    "773__0",  # from SIS: can be ignored
+    "773__o",  # from SIS: can be ignored
+    "773__x",  # INSPIRE publication note
     "8564_8",  # file id
     "8564_s",  # bibdoc id
     "8564_x",  # icon thumbnails sizes
     "8564_y",  # file description - done by files dump
+    "8564_z",  # Websubmit "stamp" (migrated as file metadata)
     "916__y",  # year, redundant value
     "937__c",  # last modified by
     "937__s",  # last modification date

cds_migrator_kit/rdm/records/transform/models/north_area.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 CERN.
+#
+# CDS-RDM is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+
+"""CDS-RDM North Area models (NA61-64)."""
+
+from cds_migrator_kit.rdm.records.transform.models._config import IGNORE_SYSTEM_KEYS
+from cds_migrator_kit.rdm.records.transform.models.base_publication_record import (
+    rdm_base_publication_model,
+)
+from cds_migrator_kit.transform.overdo import CdsOverdo
+
+
+class NorthAreaModel(CdsOverdo):
+    """Translation model for North Area experiments."""
+
+    __query__ = """693__.e:"NA61" OR 693__.e:"SHINE NA61" OR 693__.e:"NA62" OR 693__.e:"NA63" OR 693__.e:"NA64"
+    -980__:THESIS -980__:DELETED -980__:HIDDEN -980__:MIGRATED -980__:DUMMY"""
+
+    __ignore_keys__ = IGNORE_SYSTEM_KEYS | {
+        "270__m",  # Email of contact person
+        "500__9",  # Provenance of the note
+        "903__s",  # 'public'
+        "905__m",  # Submitter email address
+        "995__a",  # "Inspire"
+    }
+
+    _default_fields = {
+        "custom_fields": {},
+    }
+
+
+north_area_model = NorthAreaModel(
+    bases=(rdm_base_publication_model,),
+    entry_point_group="cds_migrator_kit.migrator.rules.north_area",
+)

cds_migrator_kit/rdm/records/transform/transform.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM transform step module."""
+
 import logging
 import re
 from collections import OrderedDict
@@ -18,8 +19,10 @@
 from invenio_access.permissions import system_identity
 from invenio_pidstore.models import PersistentIdentifier, PIDStatus
 from invenio_rdm_migrator.logging import Logger
-from invenio_rdm_records.proxies import current_rdm_records_service, \
-    current_record_communities_service
+from invenio_rdm_records.proxies import (
+    current_rdm_records_service,
+    current_record_communities_service,
+)
 from sqlalchemy.exc import NoResultFound
 
 from cds_migrator_kit.errors import (
@@ -33,8 +36,9 @@
 from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent
 from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry
 from cds_migrator_kit.rdm.records.transform.entities.request import RecordRequest
-from cds_migrator_kit.rdm.records.transform.transform_versions import \
-    RecordVersionsTransform
+from cds_migrator_kit.rdm.records.transform.transform_versions import (
+    RecordVersionsTransform,
+)
 from cds_migrator_kit.transform.dumper import CDSRecordDump
 from cds_migrator_kit.transform.errors import LossyConversion
 
@@ -115,7 +119,8 @@ def _transform_xml_to_json(self, raw_dump_entry):
         if dump.multiple_models_warning:
             w = dump.multiple_models_warning
             recid = raw_dump_entry.get("recid") or raw_dump_entry.get("record", {}).get(
-                "recid")
+                "recid"
+            )
             matched = re.findall(r"\['(\w+)',", w.message or "")
             self.migration_logger.add_information(
                 str(recid),
@@ -197,12 +202,12 @@ def _transform(self, raw_dump_entry) -> Optional[MigrationEntry]:
                 )
 
         except (
-                LossyConversion,
-                RestrictedFileDetected,
-                UnexpectedValue,
-                ManualImportRequired,
-                MissingRequiredField,
-                MultipleModelsMatched,
+            LossyConversion,
+            RestrictedFileDetected,
+            UnexpectedValue,
+            ManualImportRequired,
+            MissingRequiredField,
+            MultipleModelsMatched,
         ) as e:
             migration_logger.add_log(e, record=raw_dump_entry)
 
@@ -266,9 +271,7 @@ def _existing_record_is_restricted(self, record_id):
         MARCXML, since access restrictions may have been changed in RDM
         after migration and the legacy record data would be stale.
         """
-        record = current_rdm_records_service.read_latest(
-            system_identity, id_=record_id
-        )
+        record = current_rdm_records_service.read_latest(system_identity, id_=record_id)
         access = record.data.get("access", {})
         return access.get("record") != "public" or access.get("files") != "public"
 
@@ -328,8 +331,9 @@ def run(self, entries):
                     recid,
                     {
                         "message": "Record already migrated, skipping,"
-                                   " added existing {} to communities {}".format(
-                            recid, self.communities_ids),
+                        " added existing {} to communities {}".format(
+                            recid, self.communities_ids
+                        ),
                         "value": recid,
                     },
                 )

cds_migrator_kit/rdm/streams.yaml

@@ -377,7 +377,7 @@ records:
     data_dir: cds_migrator_kit/rdm/data/isolde
     plots: true
     create_inclusion_request: true
-    preferred_model: isolde  # set to resolve MultipleModelsMatched as warning instead of error
+    preferred_model: isolde # set to resolve MultipleModelsMatched as warning instead of error
     extract:
       dirpath: cds_migrator_kit/rdm/data/isolde/dump/
     transform:
@@ -664,17 +664,61 @@ records:
       missing_users: cds_migrator_kit/rdm/data/users
       communities_ids:
         - "88a105fe-4713-493b-b555-6ab398599d21"
+  na61:
+    data_dir: cds_migrator_kit/rdm/data/north_area/na61
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/north_area/na61/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/north_area/na61/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - ""
+  na62:
+    data_dir: cds_migrator_kit/rdm/data/north_area/na62
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/north_area/na62/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/north_area/na62/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - ""
+  na63:
+    data_dir: cds_migrator_kit/rdm/data/north_area/na63
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/north_area/na63/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/north_area/na63/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - ""
+  na64:
+    data_dir: cds_migrator_kit/rdm/data/north_area/na64
+    plots: true
+    create_inclusion_request: true
+    extract:
+      dirpath: cds_migrator_kit/rdm/data/north_area/na64/dump/
+    transform:
+      files_dump_dir: cds_migrator_kit/rdm/data/north_area/na64/files/
+      missing_users: cds_migrator_kit/rdm/data/users
+      communities_ids:
+        - ""
 comments:
   faser-drafts:
     dir_path: /migration/faser-drafts/comments/
     reviewers:
-      - {group: faser-all}
+      - { group: faser-all }
   faser:
     dir_path: /migration/faser/comments/
     reviewers:
-      - {group: faser-all}
+      - { group: faser-all }
 
   faser-ep:
     dir_path: /migration/faser-ep/comments/
     reviewers:
-      - {group: faser-all}
+      - { group: faser-all }

cds_migrator_kit/users/load.py

@@ -6,6 +6,7 @@
 # the terms of the MIT License; see LICENSE file for more details.
 
 """CDS-RDM transform step module."""
+
 import csv
 import json
 import logging
@@ -16,7 +17,6 @@
 from invenio_accounts.models import User
 from invenio_db import db
 from invenio_rdm_migrator.load.base import Load
-from sqlalchemy.exc import NoResultFound
 
 cli_logger = logging.getLogger("migrator")
 
@@ -78,13 +78,11 @@
         """Fetch or create a user account by email."""
         if not email:
             return
-        try:
-            user = User.query.filter_by(email=email).one()
+        user = User.query.filter(db.func.lower(User.email) == email.lower()).first()
+        if user is not None:
             return user.id
-        except NoResultFound:
-            if not self.dry_run:
-                user_id = self._create_owner(email)
-                return user_id
+        if not self.dry_run:
+            return self._create_owner(email)
 
     def _parse_reviewer_name(self, name):
         """Split a "Family name, Given name" or "Given name Family name"
@@ -180,7 +178,7 @@
         later on. Only fills in missing values, never overwrites an
         existing (e.g. already CERN-synced) profile.
         """
         user = User.query.get(user_id)
         if user is None:
             return
         profile = dict(user.user_profile or {})

setup.cfg

@@ -99,6 +99,7 @@ cds_migrator_kit.migrator.models =
     technical_support = cds_migrator_kit.rdm.records.transform.models.technical_support:technical_support_model
     ab_dep = cds_migrator_kit.rdm.records.transform.models.ab_dep:ab_dep_model
     isolde = cds_migrator_kit.rdm.records.transform.models.isolde:isolde_model
+    north_area = cds_migrator_kit.rdm.records.transform.models.north_area:north_area_model
 
 cds_migrator_kit.migrator.rules.base =
     base = cds_migrator_kit.transform.xml_processing.rules.base
@@ -220,6 +221,10 @@ cds_migrator_kit.migrator.rules.isolde =
     base = cds_migrator_kit.transform.xml_processing.rules.base
     base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base
     publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research
+cds_migrator_kit.migrator.rules.north_area =
+    base = cds_migrator_kit.transform.xml_processing.rules.base
+    base_records = cds_migrator_kit.rdm.records.transform.xml_processing.rules.base
+    publication = cds_migrator_kit.rdm.records.transform.xml_processing.rules.research
 cds_migrator_kit.migrator.rules.people =
     people = cds_migrator_kit.rdm.users.transform.xml_processing.rules.people
 invenio_pidstore.minters =